You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

4273 lines
93 KiB

# This file is automatically @generated by Cargo.
# It is not intended for manual editing.
version = 3
[[package]]
name = "addr2line"
version = "0.19.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a76fd60b23679b7d19bd066031410fb7e458ccc5e958eb5c325888ce4baedc97"
dependencies = [
"gimli",
externref: implement stack map-based garbage collection For host VM code, we use plain reference counting, where cloning increments the reference count, and dropping decrements it. We can avoid many of the on-stack increment/decrement operations that typically plague the performance of reference counting via Rust's ownership and borrowing system. Moving a `VMExternRef` avoids mutating its reference count, and borrowing it either avoids the reference count increment or delays it until if/when the `VMExternRef` is cloned. When passing a `VMExternRef` into compiled Wasm code, we don't want to do reference count mutations for every compiled `local.{get,set}`, nor for every function call. Therefore, we use a variation of **deferred reference counting**, where we only mutate reference counts when storing `VMExternRef`s somewhere that outlives the activation: into a global or table. Simultaneously, we over-approximate the set of `VMExternRef`s that are inside Wasm function activations. Periodically, we walk the stack at GC safe points, and use stack map information to precisely identify the set of `VMExternRef`s inside Wasm activations. Then we take the difference between this precise set and our over-approximation, and decrement the reference count for each of the `VMExternRef`s that are in our over-approximation but not in the precise set. Finally, the over-approximation is replaced with the precise set. The `VMExternRefActivationsTable` implements the over-approximized set of `VMExternRef`s referenced by Wasm activations. Calling a Wasm function and passing it a `VMExternRef` moves the `VMExternRef` into the table, and the compiled Wasm function logically "borrows" the `VMExternRef` from the table. Similarly, `global.get` and `table.get` operations clone the gotten `VMExternRef` into the `VMExternRefActivationsTable` and then "borrow" the reference out of the table. When a `VMExternRef` is returned to host code from a Wasm function, the host increments the reference count (because the reference is logically "borrowed" from the `VMExternRefActivationsTable` and the reference count from the table will be dropped at the next GC). For more general information on deferred reference counting, see *An Examination of Deferred Reference Counting and Cycle Detection* by Quinane: https://openresearch-repository.anu.edu.au/bitstream/1885/42030/2/hon-thesis.pdf cc #929 Fixes #1804
4 years ago
]
[[package]]
name = "adler"
version = "1.0.2"
externref: implement stack map-based garbage collection For host VM code, we use plain reference counting, where cloning increments the reference count, and dropping decrements it. We can avoid many of the on-stack increment/decrement operations that typically plague the performance of reference counting via Rust's ownership and borrowing system. Moving a `VMExternRef` avoids mutating its reference count, and borrowing it either avoids the reference count increment or delays it until if/when the `VMExternRef` is cloned. When passing a `VMExternRef` into compiled Wasm code, we don't want to do reference count mutations for every compiled `local.{get,set}`, nor for every function call. Therefore, we use a variation of **deferred reference counting**, where we only mutate reference counts when storing `VMExternRef`s somewhere that outlives the activation: into a global or table. Simultaneously, we over-approximate the set of `VMExternRef`s that are inside Wasm function activations. Periodically, we walk the stack at GC safe points, and use stack map information to precisely identify the set of `VMExternRef`s inside Wasm activations. Then we take the difference between this precise set and our over-approximation, and decrement the reference count for each of the `VMExternRef`s that are in our over-approximation but not in the precise set. Finally, the over-approximation is replaced with the precise set. The `VMExternRefActivationsTable` implements the over-approximized set of `VMExternRef`s referenced by Wasm activations. Calling a Wasm function and passing it a `VMExternRef` moves the `VMExternRef` into the table, and the compiled Wasm function logically "borrows" the `VMExternRef` from the table. Similarly, `global.get` and `table.get` operations clone the gotten `VMExternRef` into the `VMExternRefActivationsTable` and then "borrow" the reference out of the table. When a `VMExternRef` is returned to host code from a Wasm function, the host increments the reference count (because the reference is logically "borrowed" from the `VMExternRefActivationsTable` and the reference count from the table will be dropped at the next GC). For more general information on deferred reference counting, see *An Examination of Deferred Reference Counting and Cycle Detection* by Quinane: https://openresearch-repository.anu.edu.au/bitstream/1885/42030/2/hon-thesis.pdf cc #929 Fixes #1804
4 years ago
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe"
externref: implement stack map-based garbage collection For host VM code, we use plain reference counting, where cloning increments the reference count, and dropping decrements it. We can avoid many of the on-stack increment/decrement operations that typically plague the performance of reference counting via Rust's ownership and borrowing system. Moving a `VMExternRef` avoids mutating its reference count, and borrowing it either avoids the reference count increment or delays it until if/when the `VMExternRef` is cloned. When passing a `VMExternRef` into compiled Wasm code, we don't want to do reference count mutations for every compiled `local.{get,set}`, nor for every function call. Therefore, we use a variation of **deferred reference counting**, where we only mutate reference counts when storing `VMExternRef`s somewhere that outlives the activation: into a global or table. Simultaneously, we over-approximate the set of `VMExternRef`s that are inside Wasm function activations. Periodically, we walk the stack at GC safe points, and use stack map information to precisely identify the set of `VMExternRef`s inside Wasm activations. Then we take the difference between this precise set and our over-approximation, and decrement the reference count for each of the `VMExternRef`s that are in our over-approximation but not in the precise set. Finally, the over-approximation is replaced with the precise set. The `VMExternRefActivationsTable` implements the over-approximized set of `VMExternRef`s referenced by Wasm activations. Calling a Wasm function and passing it a `VMExternRef` moves the `VMExternRef` into the table, and the compiled Wasm function logically "borrows" the `VMExternRef` from the table. Similarly, `global.get` and `table.get` operations clone the gotten `VMExternRef` into the `VMExternRefActivationsTable` and then "borrow" the reference out of the table. When a `VMExternRef` is returned to host code from a Wasm function, the host increments the reference count (because the reference is logically "borrowed" from the `VMExternRefActivationsTable` and the reference count from the table will be dropped at the next GC). For more general information on deferred reference counting, see *An Examination of Deferred Reference Counting and Cycle Detection* by Quinane: https://openresearch-repository.anu.edu.au/bitstream/1885/42030/2/hon-thesis.pdf cc #929 Fixes #1804
4 years ago
[[package]]
name = "aead"
version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0b613b8e1e3cf911a086f53f03bf286f52fd7a7258e4fa606f0ef220d39d8877"
dependencies = [
"generic-array",
]
[[package]]
name = "aes"
version = "0.7.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9e8b47f52ea9bae42228d07ec09eb676433d7c4ed1ebdf0f1d1c29ed446f1ab8"
dependencies = [
Remove support for userfaultfd (#4040) This commit removes support for the `userfaultfd` or "uffd" syscall on Linux. This support was originally added for users migrating from Lucet to Wasmtime, but the recent developments of kernel-supported copy-on-write support for memory initialization wound up being more appropriate for these use cases than usefaultfd. The main reason for moving to copy-on-write initialization are: * The `userfaultfd` feature was never necessarily intended for this style of use case with wasm and was susceptible to subtle and rare bugs that were extremely difficult to track down. We were never 100% certain that there were kernel bugs related to userfaultfd but the suspicion never went away. * Handling faults with userfaultfd was always slow and single-threaded. Only one thread could handle faults and traveling to user-space to handle faults is inherently slower than handling them all in the kernel. The single-threaded aspect in particular presented a significant scaling bottleneck for embeddings that want to run many wasm instances in parallel. * One of the major benefits of userfaultfd was lazy initialization of wasm linear memory which is also achieved with the copy-on-write initialization support we have right now. * One of the suspected benefits of userfaultfd was less frobbing of the kernel vma structures when wasm modules are instantiated. Currently the copy-on-write support has a mitigation where we attempt to reuse the memory images where possible to avoid changing vma structures. When comparing this to userfaultfd's performance it was found that kernel modifications of vmas aren't a worrisome bottleneck so copy-on-write is suitable for this as well. Overall there are no remaining benefits that userfaultfd gives that copy-on-write doesn't, and copy-on-write solves a major downsides of userfaultfd, the scaling issue with a single faulting thread. Additionally copy-on-write support seems much more robust in terms of kernel implementation since it's only using standard memory-management syscalls which are heavily exercised. Finally copy-on-write support provides a new bonus where read-only memory in WebAssembly can be mapped directly to the same kernel cache page, even amongst many wasm instances of the same module, which was never possible with userfaultfd. In light of all this it's expected that all users of userfaultfd should migrate to the copy-on-write initialization of Wasmtime (which is enabled by default).
3 years ago
"cfg-if",
"cipher",
ISLE: guard against stale generated source in default build config. Currently, the `build.rs` script that generates Rust source from the ISLE DSL will only do this generation if the `rebuild-isle` Cargo feature is specified. By default, it is not. This is based on the principle that we (the build script) do not modify the source tree as managed by git; git-managed files are strictly a human-managed and human-edited resource. By adding the opt-in Cargo feature, a developer is requesting the build script to perform an explicit action. (In my understanding at least, this principle comes from the general philosophy of hermetic builds: the output should be a pure function of the input, and part of this is that the input is read-only. If we modify the source tree, then all bets are off.) Unfortunately, requiring the opt-in feature also creates a footgun that is easy to hit: if a developer modifies the ISLE DSL source, but forgets to specify the Cargo feature, then the compiler will silently be built successfully with stale source, and will silently exclude any changes that were made. The generated source is checked into git for a good reason: we want DSL compiler to not affect build times for the overwhelmingly common case that Cranelift is used as a dependency but the backends are not being actively developed. (This overhead comes mainly from building `islec` itself.) So, what to do? This PR implements a middle ground first described in [this conversation](https://github.com/bytecodealliance/wasmtime/pull/3506#discussion_r743113351), in which we: - Generate a hash (SHA-512) of the ISLE DSL source and produce a "manifest" of ISLE inputs alongside the generated source; and - Always read the ISLE DSL source, and see if the manifest is still valid, on builds that do not have the opt-in "rebuild" feature. This allows us to know whether the ISLE compiler output would have been the same (modulo changes to the DSL compiler itself, which are out-of-scope here), without actually building the ISLE compiler and running it. If the compiler-backend developer modifies an ISLE source file and then tries to build `cranelift-codegen` without adding the `rebuild-isle` Cargo feature, they get the following output: ```text Error: the ISLE source files that resulted in the generated Rust source * src/isa/x64/lower/isle/generated_code.rs have changed but the generated source was not rebuilt! These ISLE source files are: * src/clif.isle * src/prelude.isle * src/isa/x64/inst.isle * src/isa/x64/lower.isle Please add `--features rebuild-isle` to your `cargo build` command if you wish to rebuild the generated source, then include these changes in any git commits you make that include the changes to the ISLE. For example: $ cargo build -p cranelift-codegen --features rebuild-isle (This build script cannot do this for you by default because we cannot modify checked-into-git source without your explicit opt-in.) ``` which will tell them exactly what they need to do to fix the problem! Note that I am leaving the "Rebuild ISLE" CI job alone for now, because otherwise, we are trusting whomever submits a PR to generate the correct generated source. In other words, the manifest is a communication from the checked-in tree to the developer, but we still need to verify that the checked-in generated Rust source and the manifest are correct with respect to the checked-in ISLE source.
3 years ago
"cpufeatures",
"opaque-debug",
]
[[package]]
name = "aes-gcm"
version = "0.9.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "df5f85a83a7d8b0442b6aa7b504b8212c1733da07b98aae43d4bc21b2cb3cdf6"
dependencies = [
"aead",
"aes",
"cipher",
"ctr",
"ghash",
"subtle",
]
[[package]]
name = "ahash"
version = "0.8.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bf6ccdb167abbf410dcb915cabd428929d7f6a04980b54a11f26a39f1c7f7107"
dependencies = [
"cfg-if",
"once_cell",
"version_check",
]
[[package]]
name = "aho-corasick"
version = "0.7.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1e37cfd5e7657ada45f742d6e99ca5788580b5c529dc78faf11ece6dc702656f"
dependencies = [
"memchr",
]
[[package]]
name = "ambient-authority"
version = "0.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ec8ad6edb4840b78c5c3d88de606b22252d552b55f3a4699fbb10fc070ec3049"
[[package]]
name = "anes"
version = "0.1.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299"
[[package]]
name = "anyhow"
version = "1.0.66"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "216261ddc8289130e551ddcd5ce8a064710c0d064a4d2895c67151c92b5443f6"
[[package]]
name = "arbitrary"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c38b6b6b79f671c25e1a3e785b7b82d7562ffc9cd3efdc98627e5668a2472490"
dependencies = [
"derive_arbitrary",
]
[[package]]
name = "async-trait"
version = "0.1.53"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ed6aa3524a2dfcf9fe180c51eae2b58738348d819517ceadf95789c51fff7600"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "atty"
version = "0.2.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8"
dependencies = [
"hermit-abi 0.1.19",
"libc",
"winapi",
]
[[package]]
name = "autocfg"
version = "0.1.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0dde43e75fd43e8a1bf86103336bc699aa8d17ad1be60c76c0bdfd4828e19b78"
dependencies = [
"autocfg 1.1.0",
]
[[package]]
name = "autocfg"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"
[[package]]
name = "backtrace"
version = "0.3.67"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "233d376d6d185f2a3093e58f283f60f880315b6c60075b01f36b3b85154564ca"
dependencies = [
"addr2line",
"cc",
Remove support for userfaultfd (#4040) This commit removes support for the `userfaultfd` or "uffd" syscall on Linux. This support was originally added for users migrating from Lucet to Wasmtime, but the recent developments of kernel-supported copy-on-write support for memory initialization wound up being more appropriate for these use cases than usefaultfd. The main reason for moving to copy-on-write initialization are: * The `userfaultfd` feature was never necessarily intended for this style of use case with wasm and was susceptible to subtle and rare bugs that were extremely difficult to track down. We were never 100% certain that there were kernel bugs related to userfaultfd but the suspicion never went away. * Handling faults with userfaultfd was always slow and single-threaded. Only one thread could handle faults and traveling to user-space to handle faults is inherently slower than handling them all in the kernel. The single-threaded aspect in particular presented a significant scaling bottleneck for embeddings that want to run many wasm instances in parallel. * One of the major benefits of userfaultfd was lazy initialization of wasm linear memory which is also achieved with the copy-on-write initialization support we have right now. * One of the suspected benefits of userfaultfd was less frobbing of the kernel vma structures when wasm modules are instantiated. Currently the copy-on-write support has a mitigation where we attempt to reuse the memory images where possible to avoid changing vma structures. When comparing this to userfaultfd's performance it was found that kernel modifications of vmas aren't a worrisome bottleneck so copy-on-write is suitable for this as well. Overall there are no remaining benefits that userfaultfd gives that copy-on-write doesn't, and copy-on-write solves a major downsides of userfaultfd, the scaling issue with a single faulting thread. Additionally copy-on-write support seems much more robust in terms of kernel implementation since it's only using standard memory-management syscalls which are heavily exercised. Finally copy-on-write support provides a new bonus where read-only memory in WebAssembly can be mapped directly to the same kernel cache page, even amongst many wasm instances of the same module, which was never possible with userfaultfd. In light of all this it's expected that all users of userfaultfd should migrate to the copy-on-write initialization of Wasmtime (which is enabled by default).
3 years ago
"cfg-if",
"libc",
externref: implement stack map-based garbage collection For host VM code, we use plain reference counting, where cloning increments the reference count, and dropping decrements it. We can avoid many of the on-stack increment/decrement operations that typically plague the performance of reference counting via Rust's ownership and borrowing system. Moving a `VMExternRef` avoids mutating its reference count, and borrowing it either avoids the reference count increment or delays it until if/when the `VMExternRef` is cloned. When passing a `VMExternRef` into compiled Wasm code, we don't want to do reference count mutations for every compiled `local.{get,set}`, nor for every function call. Therefore, we use a variation of **deferred reference counting**, where we only mutate reference counts when storing `VMExternRef`s somewhere that outlives the activation: into a global or table. Simultaneously, we over-approximate the set of `VMExternRef`s that are inside Wasm function activations. Periodically, we walk the stack at GC safe points, and use stack map information to precisely identify the set of `VMExternRef`s inside Wasm activations. Then we take the difference between this precise set and our over-approximation, and decrement the reference count for each of the `VMExternRef`s that are in our over-approximation but not in the precise set. Finally, the over-approximation is replaced with the precise set. The `VMExternRefActivationsTable` implements the over-approximized set of `VMExternRef`s referenced by Wasm activations. Calling a Wasm function and passing it a `VMExternRef` moves the `VMExternRef` into the table, and the compiled Wasm function logically "borrows" the `VMExternRef` from the table. Similarly, `global.get` and `table.get` operations clone the gotten `VMExternRef` into the `VMExternRefActivationsTable` and then "borrow" the reference out of the table. When a `VMExternRef` is returned to host code from a Wasm function, the host increments the reference count (because the reference is logically "borrowed" from the `VMExternRefActivationsTable` and the reference count from the table will be dropped at the next GC). For more general information on deferred reference counting, see *An Examination of Deferred Reference Counting and Cycle Detection* by Quinane: https://openresearch-repository.anu.edu.au/bitstream/1885/42030/2/hon-thesis.pdf cc #929 Fixes #1804
4 years ago
"miniz_oxide",
"object",
"rustc-demangle",
]
[[package]]
name = "base64"
version = "0.21.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a4a4ddaa51a5bc52a6948f74c06d20aaaddb71924eab79b8c97a8c556e942d6a"
[[package]]
name = "base64ct"
version = "1.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e6b4d9b1225d28d360ec6a231d65af1fd99a2a095154c8040689617290569c5c"
[[package]]
name = "bincode"
version = "1.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad"
dependencies = [
"serde",
]
[[package]]
name = "bit-set"
version = "0.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6e11e16035ea35e4e5997b393eacbf6f63983188f7a2ad25bfb13465f5ad59de"
dependencies = [
"bit-vec",
]
[[package]]
name = "bit-vec"
4 years ago
version = "0.6.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
4 years ago
checksum = "349f9b6a179ed607305526ca489b34ad0a41aed5f7980fa90eb03160b69598fb"
[[package]]
name = "bitflags"
version = "1.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
[[package]]
name = "block-buffer"
version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4152116fd6e9dadb291ae18fc1ec3575ed6d84c29642d97890f4b4a3417297e4"
dependencies = [
"generic-array",
]
[[package]]
name = "block-buffer"
version = "0.10.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0bf7fe51849ea569fd452f37822f606a5cabb684dc918707a0193fd4664ff324"
dependencies = [
"generic-array",
]
[[package]]
name = "bstr"
version = "0.2.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ba3569f383e8f1598449f1a423e72e99569137b47740b1da11ef19af3d5c3223"
dependencies = [
"lazy_static",
"memchr",
"regex-automata",
]
[[package]]
name = "bumpalo"
version = "3.11.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "572f695136211188308f16ad2ca5c851a712c464060ae6974944458eb83880ba"
[[package]]
name = "byteorder"
version = "1.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610"
[[package]]
name = "bytes"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c4872d67bab6358e59559027aa3b9157c53d9358c51423c17554809a8858e0f8"
4 years ago
[[package]]
name = "cap-fs-ext"
version = "1.0.5"
4 years ago
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ff40fd8a96d57a204080e5debd621342612f6d6b60901201a51f518baf72691d"
4 years ago
dependencies = [
"cap-primitives",
"cap-std",
"io-lifetimes",
"windows-sys",
]
4 years ago
[[package]]
name = "cap-primitives"
version = "1.0.5"
4 years ago
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9554a7698c8db4b7777f01b2237de111c5ecea169efb1190004d9069ceb289aa"
4 years ago
dependencies = [
"ambient-authority",
4 years ago
"fs-set-times",
"io-extras",
"io-lifetimes",
4 years ago
"ipnet",
"maybe-owned",
"rustix",
"windows-sys",
4 years ago
"winx",
]
[[package]]
name = "cap-rand"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6dcd5285cc063c837f10d80010a29eda2f22fe4ce507229a03a7886f074ee6fd"
dependencies = [
"ambient-authority",
"rand 0.8.5",
]
4 years ago
[[package]]
name = "cap-std"
version = "1.0.5"
4 years ago
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a7b68a8ac703cc7bed0a46666a04b386cca214844897a69f599dcd82ea59422c"
4 years ago
dependencies = [
"cap-primitives",
"io-extras",
"io-lifetimes",
"ipnet",
"rustix",
]
[[package]]
name = "cap-tempfile"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ad935d619cca685eb3a93e31f27c5217e0d2fd90ae47977ff178039084e19c34"
dependencies = [
"cap-std",
"rand 0.8.5",
"rustix",
"uuid",
]
4 years ago
[[package]]
name = "cap-time-ext"
version = "1.0.5"
4 years ago
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "472931750f90fbf0731c886c2937521e25772942577a182e7ace5bc561d10e3b"
4 years ago
dependencies = [
"cap-primitives",
4 years ago
"once_cell",
"rustix",
"winx",
4 years ago
]
[[package]]
name = "capstone"
version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "178b3c80b4ce3939792d1dd6600575e012da806a10e81abe812dc29cbfe629ec"
dependencies = [
"capstone-sys",
"libc",
]
[[package]]
name = "capstone-sys"
version = "0.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cf2f3d97b88fa4a9a6464c7c08b8c4588aaea8c18d0651eca11f2ca15f50f1f6"
dependencies = [
"cc",
"libc",
]
[[package]]
name = "cast"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5"
[[package]]
name = "cc"
version = "1.0.73"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2fff2a6927b3bb87f9595d67196a70493f627687a71d87a0d692242c33f58c11"
dependencies = [
"jobserver",
]
[[package]]
name = "cfg-if"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
[[package]]
name = "chacha20"
version = "0.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "01b72a433d0cf2aef113ba70f62634c56fddb0f244e6377185c56a7cadbd8f91"
dependencies = [
Remove support for userfaultfd (#4040) This commit removes support for the `userfaultfd` or "uffd" syscall on Linux. This support was originally added for users migrating from Lucet to Wasmtime, but the recent developments of kernel-supported copy-on-write support for memory initialization wound up being more appropriate for these use cases than usefaultfd. The main reason for moving to copy-on-write initialization are: * The `userfaultfd` feature was never necessarily intended for this style of use case with wasm and was susceptible to subtle and rare bugs that were extremely difficult to track down. We were never 100% certain that there were kernel bugs related to userfaultfd but the suspicion never went away. * Handling faults with userfaultfd was always slow and single-threaded. Only one thread could handle faults and traveling to user-space to handle faults is inherently slower than handling them all in the kernel. The single-threaded aspect in particular presented a significant scaling bottleneck for embeddings that want to run many wasm instances in parallel. * One of the major benefits of userfaultfd was lazy initialization of wasm linear memory which is also achieved with the copy-on-write initialization support we have right now. * One of the suspected benefits of userfaultfd was less frobbing of the kernel vma structures when wasm modules are instantiated. Currently the copy-on-write support has a mitigation where we attempt to reuse the memory images where possible to avoid changing vma structures. When comparing this to userfaultfd's performance it was found that kernel modifications of vmas aren't a worrisome bottleneck so copy-on-write is suitable for this as well. Overall there are no remaining benefits that userfaultfd gives that copy-on-write doesn't, and copy-on-write solves a major downsides of userfaultfd, the scaling issue with a single faulting thread. Additionally copy-on-write support seems much more robust in terms of kernel implementation since it's only using standard memory-management syscalls which are heavily exercised. Finally copy-on-write support provides a new bonus where read-only memory in WebAssembly can be mapped directly to the same kernel cache page, even amongst many wasm instances of the same module, which was never possible with userfaultfd. In light of all this it's expected that all users of userfaultfd should migrate to the copy-on-write initialization of Wasmtime (which is enabled by default).
3 years ago
"cfg-if",
"cipher",
ISLE: guard against stale generated source in default build config. Currently, the `build.rs` script that generates Rust source from the ISLE DSL will only do this generation if the `rebuild-isle` Cargo feature is specified. By default, it is not. This is based on the principle that we (the build script) do not modify the source tree as managed by git; git-managed files are strictly a human-managed and human-edited resource. By adding the opt-in Cargo feature, a developer is requesting the build script to perform an explicit action. (In my understanding at least, this principle comes from the general philosophy of hermetic builds: the output should be a pure function of the input, and part of this is that the input is read-only. If we modify the source tree, then all bets are off.) Unfortunately, requiring the opt-in feature also creates a footgun that is easy to hit: if a developer modifies the ISLE DSL source, but forgets to specify the Cargo feature, then the compiler will silently be built successfully with stale source, and will silently exclude any changes that were made. The generated source is checked into git for a good reason: we want DSL compiler to not affect build times for the overwhelmingly common case that Cranelift is used as a dependency but the backends are not being actively developed. (This overhead comes mainly from building `islec` itself.) So, what to do? This PR implements a middle ground first described in [this conversation](https://github.com/bytecodealliance/wasmtime/pull/3506#discussion_r743113351), in which we: - Generate a hash (SHA-512) of the ISLE DSL source and produce a "manifest" of ISLE inputs alongside the generated source; and - Always read the ISLE DSL source, and see if the manifest is still valid, on builds that do not have the opt-in "rebuild" feature. This allows us to know whether the ISLE compiler output would have been the same (modulo changes to the DSL compiler itself, which are out-of-scope here), without actually building the ISLE compiler and running it. If the compiler-backend developer modifies an ISLE source file and then tries to build `cranelift-codegen` without adding the `rebuild-isle` Cargo feature, they get the following output: ```text Error: the ISLE source files that resulted in the generated Rust source * src/isa/x64/lower/isle/generated_code.rs have changed but the generated source was not rebuilt! These ISLE source files are: * src/clif.isle * src/prelude.isle * src/isa/x64/inst.isle * src/isa/x64/lower.isle Please add `--features rebuild-isle` to your `cargo build` command if you wish to rebuild the generated source, then include these changes in any git commits you make that include the changes to the ISLE. For example: $ cargo build -p cranelift-codegen --features rebuild-isle (This build script cannot do this for you by default because we cannot modify checked-into-git source without your explicit opt-in.) ``` which will tell them exactly what they need to do to fix the problem! Note that I am leaving the "Rebuild ISLE" CI job alone for now, because otherwise, we are trusting whomever submits a PR to generate the correct generated source. In other words, the manifest is a communication from the checked-in tree to the developer, but we still need to verify that the checked-in generated Rust source and the manifest are correct with respect to the checked-in ISLE source.
3 years ago
"cpufeatures",
"zeroize",
]
[[package]]
name = "chacha20poly1305"
version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3b84ed6d1d5f7aa9bdde921a5090e0ca4d934d250ea3b402a5fab3a994e28a2a"
dependencies = [
"aead",
"chacha20",
"cipher",
"poly1305",
"zeroize",
]
[[package]]
name = "ciborium"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b0c137568cc60b904a7724001b35ce2630fd00d5d84805fbb608ab89509d788f"
dependencies = [
"ciborium-io",
"ciborium-ll",
"serde",
]
[[package]]
name = "ciborium-io"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "346de753af073cc87b52b2083a506b38ac176a44cfb05497b622e27be899b369"
[[package]]
name = "ciborium-ll"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "213030a2b5a4e0c0892b6652260cf6ccac84827b83a85a534e178e3906c4cf1b"
dependencies = [
"ciborium-io",
"half",
]
[[package]]
name = "cipher"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7ee52072ec15386f770805afd189a01c8841be8696bed250fa2f13c4c0d6dfb7"
dependencies = [
"generic-array",
]
[[package]]
name = "clap"
version = "3.2.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "190814073e85d238f31ff738fcb0bf6910cedeb73376c87cd69291028966fd83"
dependencies = [
"atty",
"bitflags",
"clap_derive",
"clap_lex",
"indexmap",
"once_cell",
"strsim",
"termcolor",
"textwrap",
]
[[package]]
name = "clap_derive"
version = "3.2.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "759bf187376e1afa7b85b959e6a664a3e7a95203415dba952ad19139e798f902"
dependencies = [
"heck",
"proc-macro-error",
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "clap_lex"
version = "0.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2850f2f5a82cbf437dd5af4d49848fbdfc27c157c3d010345776f952765261c5"
dependencies = [
"os_str_bytes",
]
[[package]]
name = "codespan-reporting"
version = "0.11.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3538270d33cc669650c4b093848450d380def10c331d38c768e34cac80576e6e"
dependencies = [
"termcolor",
"unicode-width",
]
[[package]]
name = "component-fuzz-util"
version = "0.0.0"
dependencies = [
"anyhow",
"arbitrary",
"proc-macro2",
"quote",
"wasmtime-component-util",
]
[[package]]
name = "component-macro-test"
version = "0.0.0"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "component-macro-test-helpers"
version = "0.0.0"
dependencies = [
"proc-macro2",
"quote",
]
[[package]]
name = "component-test-util"
version = "0.0.0"
dependencies = [
"anyhow",
"arbitrary",
"env_logger 0.9.0",
"wasmtime",
]
[[package]]
name = "console"
version = "0.15.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a28b32d32ca44b70c3e4acd7db1babf555fa026e385fb95f18028f88848b3c31"
dependencies = [
"encode_unicode",
"libc",
"once_cell",
"regex",
"terminal_size",
"unicode-width",
"winapi",
]
[[package]]
name = "const-oid"
version = "0.6.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9d6f2aa4d0537bcc1c74df8755072bd31c1ef1a3a1b85a68e8404a8c353b7b8b"
Provide filename/line number information in `Trap` (#2452) * Provide filename/line number information in `Trap` This commit extends the `Trap` type and `Store` to retain DWARF debug information found in a wasm file unconditionally, if it's present. This then enables us to print filenames and line numbers which point back to actual source code when a trap backtrace is printed. Additionally the `FrameInfo` type has been souped up to return filename/line number information as well. The implementation here is pretty simplistic currently. The meat of all the work happens in `gimli` and `addr2line`, and otherwise wasmtime is just schlepping around bytes of dwarf debuginfo here and there! The general goal here is to assist with debugging when using wasmtime because filenames and line numbers are generally orders of magnitude better even when you already have a stack trace. Another nicety here is that backtraces will display inlined frames (learned through debug information), improving the experience in release mode as well. An example of this is that with this file: ```rust fn main() { panic!("hello"); } ``` we get this stack trace: ``` $ rustc foo.rs --target wasm32-wasi -g $ cargo run foo.wasm Finished dev [unoptimized + debuginfo] target(s) in 0.16s Running `target/debug/wasmtime foo.wasm` thread 'main' panicked at 'hello', foo.rs:2:5 note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace Error: failed to run main module `foo.wasm` Caused by: 0: failed to invoke command default 1: wasm trap: unreachable wasm backtrace: 0: 0x6c1c - panic_abort::__rust_start_panic::abort::h2d60298621b1ccbf at /rustc/7eac88abb2e57e752f3302f02be5f3ce3d7adfb4/library/panic_abort/src/lib.rs:77:17 - __rust_start_panic at /rustc/7eac88abb2e57e752f3302f02be5f3ce3d7adfb4/library/panic_abort/src/lib.rs:32:5 1: 0x68c7 - rust_panic at /rustc/7eac88abb2e57e752f3302f02be5f3ce3d7adfb4/library/std/src/panicking.rs:626:9 2: 0x65a1 - std::panicking::rust_panic_with_hook::h2345fb0909b53e12 at /rustc/7eac88abb2e57e752f3302f02be5f3ce3d7adfb4/library/std/src/panicking.rs:596:5 3: 0x1436 - std::panicking::begin_panic::{{closure}}::h106f151a6db8c8fb at /rustc/7eac88abb2e57e752f3302f02be5f3ce3d7adfb4/library/std/src/panicking.rs:506:9 4: 0xda8 - std::sys_common::backtrace::__rust_end_short_backtrace::he55aa13f22782798 at /rustc/7eac88abb2e57e752f3302f02be5f3ce3d7adfb4/library/std/src/sys_common/backtrace.rs:153:18 5: 0x1324 - std::panicking::begin_panic::h1727e7d1d719c76f at /rustc/7eac88abb2e57e752f3302f02be5f3ce3d7adfb4/library/std/src/panicking.rs:505:12 6: 0xfde - foo::main::h2db1313a64510850 at /Users/acrichton/code/wasmtime/foo.rs:2:5 7: 0x11d5 - core::ops::function::FnOnce::call_once::h20ee1cc04aeff1fc at /rustc/7eac88abb2e57e752f3302f02be5f3ce3d7adfb4/library/core/src/ops/function.rs:227:5 8: 0xddf - std::sys_common::backtrace::__rust_begin_short_backtrace::h054493e41e27e69c at /rustc/7eac88abb2e57e752f3302f02be5f3ce3d7adfb4/library/std/src/sys_common/backtrace.rs:137:18 9: 0x1d5a - std::rt::lang_start::{{closure}}::hd83784448d3fcb42 at /rustc/7eac88abb2e57e752f3302f02be5f3ce3d7adfb4/library/std/src/rt.rs:66:18 10: 0x69d8 - core::ops::function::impls::<impl core::ops::function::FnOnce<A> for &F>::call_once::h564d3dad35014917 at /rustc/7eac88abb2e57e752f3302f02be5f3ce3d7adfb4/library/core/src/ops/function.rs:259:13 - std::panicking::try::do_call::hdca4832ace5a8603 at /rustc/7eac88abb2e57e752f3302f02be5f3ce3d7adfb4/library/std/src/panicking.rs:381:40 - std::panicking::try::ha8624a1a6854b456 at /rustc/7eac88abb2e57e752f3302f02be5f3ce3d7adfb4/library/std/src/panicking.rs:345:19 - std::panic::catch_unwind::h71421f57cf2bc688 at /rustc/7eac88abb2e57e752f3302f02be5f3ce3d7adfb4/library/std/src/panic.rs:382:14 - std::rt::lang_start_internal::h260050c92cd470af at /rustc/7eac88abb2e57e752f3302f02be5f3ce3d7adfb4/library/std/src/rt.rs:51:25 11: 0x1d0c - std::rt::lang_start::h0b4bcf3c5e498224 at /rustc/7eac88abb2e57e752f3302f02be5f3ce3d7adfb4/library/std/src/rt.rs:65:5 12: 0xffc - <unknown>!__original_main 13: 0x393 - __muloti4 at /cargo/registry/src/github.com-1ecc6299db9ec823/compiler_builtins-0.1.35/src/macros.rs:269 ``` This is relatively noisy by default but there's filenames and line numbers! Additionally frame 10 can be seen to have lots of frames inlined into it. All information is always available to the embedder but we could try to handle the `__rust_begin_short_backtrace` and `__rust_end_short_backtrace` markers to trim the backtrace by default as well. The only gotcha here is that it looks like `__muloti4` is out of place. That's because the libc that Rust ships with doesn't have dwarf information, although I'm not sure why we land in that function for symbolizing it... * Add a configuration switch for debuginfo * Control debuginfo by default with `WASM_BACKTRACE_DETAILS` * Try cpp_demangle on demangling as well * Rename to WASMTIME_BACKTRACE_DETAILS
4 years ago
[[package]]
name = "cpp_demangle"
version = "0.3.5"
Provide filename/line number information in `Trap` (#2452) * Provide filename/line number information in `Trap` This commit extends the `Trap` type and `Store` to retain DWARF debug information found in a wasm file unconditionally, if it's present. This then enables us to print filenames and line numbers which point back to actual source code when a trap backtrace is printed. Additionally the `FrameInfo` type has been souped up to return filename/line number information as well. The implementation here is pretty simplistic currently. The meat of all the work happens in `gimli` and `addr2line`, and otherwise wasmtime is just schlepping around bytes of dwarf debuginfo here and there! The general goal here is to assist with debugging when using wasmtime because filenames and line numbers are generally orders of magnitude better even when you already have a stack trace. Another nicety here is that backtraces will display inlined frames (learned through debug information), improving the experience in release mode as well. An example of this is that with this file: ```rust fn main() { panic!("hello"); } ``` we get this stack trace: ``` $ rustc foo.rs --target wasm32-wasi -g $ cargo run foo.wasm Finished dev [unoptimized + debuginfo] target(s) in 0.16s Running `target/debug/wasmtime foo.wasm` thread 'main' panicked at 'hello', foo.rs:2:5 note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace Error: failed to run main module `foo.wasm` Caused by: 0: failed to invoke command default 1: wasm trap: unreachable wasm backtrace: 0: 0x6c1c - panic_abort::__rust_start_panic::abort::h2d60298621b1ccbf at /rustc/7eac88abb2e57e752f3302f02be5f3ce3d7adfb4/library/panic_abort/src/lib.rs:77:17 - __rust_start_panic at /rustc/7eac88abb2e57e752f3302f02be5f3ce3d7adfb4/library/panic_abort/src/lib.rs:32:5 1: 0x68c7 - rust_panic at /rustc/7eac88abb2e57e752f3302f02be5f3ce3d7adfb4/library/std/src/panicking.rs:626:9 2: 0x65a1 - std::panicking::rust_panic_with_hook::h2345fb0909b53e12 at /rustc/7eac88abb2e57e752f3302f02be5f3ce3d7adfb4/library/std/src/panicking.rs:596:5 3: 0x1436 - std::panicking::begin_panic::{{closure}}::h106f151a6db8c8fb at /rustc/7eac88abb2e57e752f3302f02be5f3ce3d7adfb4/library/std/src/panicking.rs:506:9 4: 0xda8 - std::sys_common::backtrace::__rust_end_short_backtrace::he55aa13f22782798 at /rustc/7eac88abb2e57e752f3302f02be5f3ce3d7adfb4/library/std/src/sys_common/backtrace.rs:153:18 5: 0x1324 - std::panicking::begin_panic::h1727e7d1d719c76f at /rustc/7eac88abb2e57e752f3302f02be5f3ce3d7adfb4/library/std/src/panicking.rs:505:12 6: 0xfde - foo::main::h2db1313a64510850 at /Users/acrichton/code/wasmtime/foo.rs:2:5 7: 0x11d5 - core::ops::function::FnOnce::call_once::h20ee1cc04aeff1fc at /rustc/7eac88abb2e57e752f3302f02be5f3ce3d7adfb4/library/core/src/ops/function.rs:227:5 8: 0xddf - std::sys_common::backtrace::__rust_begin_short_backtrace::h054493e41e27e69c at /rustc/7eac88abb2e57e752f3302f02be5f3ce3d7adfb4/library/std/src/sys_common/backtrace.rs:137:18 9: 0x1d5a - std::rt::lang_start::{{closure}}::hd83784448d3fcb42 at /rustc/7eac88abb2e57e752f3302f02be5f3ce3d7adfb4/library/std/src/rt.rs:66:18 10: 0x69d8 - core::ops::function::impls::<impl core::ops::function::FnOnce<A> for &F>::call_once::h564d3dad35014917 at /rustc/7eac88abb2e57e752f3302f02be5f3ce3d7adfb4/library/core/src/ops/function.rs:259:13 - std::panicking::try::do_call::hdca4832ace5a8603 at /rustc/7eac88abb2e57e752f3302f02be5f3ce3d7adfb4/library/std/src/panicking.rs:381:40 - std::panicking::try::ha8624a1a6854b456 at /rustc/7eac88abb2e57e752f3302f02be5f3ce3d7adfb4/library/std/src/panicking.rs:345:19 - std::panic::catch_unwind::h71421f57cf2bc688 at /rustc/7eac88abb2e57e752f3302f02be5f3ce3d7adfb4/library/std/src/panic.rs:382:14 - std::rt::lang_start_internal::h260050c92cd470af at /rustc/7eac88abb2e57e752f3302f02be5f3ce3d7adfb4/library/std/src/rt.rs:51:25 11: 0x1d0c - std::rt::lang_start::h0b4bcf3c5e498224 at /rustc/7eac88abb2e57e752f3302f02be5f3ce3d7adfb4/library/std/src/rt.rs:65:5 12: 0xffc - <unknown>!__original_main 13: 0x393 - __muloti4 at /cargo/registry/src/github.com-1ecc6299db9ec823/compiler_builtins-0.1.35/src/macros.rs:269 ``` This is relatively noisy by default but there's filenames and line numbers! Additionally frame 10 can be seen to have lots of frames inlined into it. All information is always available to the embedder but we could try to handle the `__rust_begin_short_backtrace` and `__rust_end_short_backtrace` markers to trim the backtrace by default as well. The only gotcha here is that it looks like `__muloti4` is out of place. That's because the libc that Rust ships with doesn't have dwarf information, although I'm not sure why we land in that function for symbolizing it... * Add a configuration switch for debuginfo * Control debuginfo by default with `WASM_BACKTRACE_DETAILS` * Try cpp_demangle on demangling as well * Rename to WASMTIME_BACKTRACE_DETAILS
4 years ago
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "eeaa953eaad386a53111e47172c2fedba671e5684c8dd601a5f474f4f118710f"
Provide filename/line number information in `Trap` (#2452) * Provide filename/line number information in `Trap` This commit extends the `Trap` type and `Store` to retain DWARF debug information found in a wasm file unconditionally, if it's present. This then enables us to print filenames and line numbers which point back to actual source code when a trap backtrace is printed. Additionally the `FrameInfo` type has been souped up to return filename/line number information as well. The implementation here is pretty simplistic currently. The meat of all the work happens in `gimli` and `addr2line`, and otherwise wasmtime is just schlepping around bytes of dwarf debuginfo here and there! The general goal here is to assist with debugging when using wasmtime because filenames and line numbers are generally orders of magnitude better even when you already have a stack trace. Another nicety here is that backtraces will display inlined frames (learned through debug information), improving the experience in release mode as well. An example of this is that with this file: ```rust fn main() { panic!("hello"); } ``` we get this stack trace: ``` $ rustc foo.rs --target wasm32-wasi -g $ cargo run foo.wasm Finished dev [unoptimized + debuginfo] target(s) in 0.16s Running `target/debug/wasmtime foo.wasm` thread 'main' panicked at 'hello', foo.rs:2:5 note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace Error: failed to run main module `foo.wasm` Caused by: 0: failed to invoke command default 1: wasm trap: unreachable wasm backtrace: 0: 0x6c1c - panic_abort::__rust_start_panic::abort::h2d60298621b1ccbf at /rustc/7eac88abb2e57e752f3302f02be5f3ce3d7adfb4/library/panic_abort/src/lib.rs:77:17 - __rust_start_panic at /rustc/7eac88abb2e57e752f3302f02be5f3ce3d7adfb4/library/panic_abort/src/lib.rs:32:5 1: 0x68c7 - rust_panic at /rustc/7eac88abb2e57e752f3302f02be5f3ce3d7adfb4/library/std/src/panicking.rs:626:9 2: 0x65a1 - std::panicking::rust_panic_with_hook::h2345fb0909b53e12 at /rustc/7eac88abb2e57e752f3302f02be5f3ce3d7adfb4/library/std/src/panicking.rs:596:5 3: 0x1436 - std::panicking::begin_panic::{{closure}}::h106f151a6db8c8fb at /rustc/7eac88abb2e57e752f3302f02be5f3ce3d7adfb4/library/std/src/panicking.rs:506:9 4: 0xda8 - std::sys_common::backtrace::__rust_end_short_backtrace::he55aa13f22782798 at /rustc/7eac88abb2e57e752f3302f02be5f3ce3d7adfb4/library/std/src/sys_common/backtrace.rs:153:18 5: 0x1324 - std::panicking::begin_panic::h1727e7d1d719c76f at /rustc/7eac88abb2e57e752f3302f02be5f3ce3d7adfb4/library/std/src/panicking.rs:505:12 6: 0xfde - foo::main::h2db1313a64510850 at /Users/acrichton/code/wasmtime/foo.rs:2:5 7: 0x11d5 - core::ops::function::FnOnce::call_once::h20ee1cc04aeff1fc at /rustc/7eac88abb2e57e752f3302f02be5f3ce3d7adfb4/library/core/src/ops/function.rs:227:5 8: 0xddf - std::sys_common::backtrace::__rust_begin_short_backtrace::h054493e41e27e69c at /rustc/7eac88abb2e57e752f3302f02be5f3ce3d7adfb4/library/std/src/sys_common/backtrace.rs:137:18 9: 0x1d5a - std::rt::lang_start::{{closure}}::hd83784448d3fcb42 at /rustc/7eac88abb2e57e752f3302f02be5f3ce3d7adfb4/library/std/src/rt.rs:66:18 10: 0x69d8 - core::ops::function::impls::<impl core::ops::function::FnOnce<A> for &F>::call_once::h564d3dad35014917 at /rustc/7eac88abb2e57e752f3302f02be5f3ce3d7adfb4/library/core/src/ops/function.rs:259:13 - std::panicking::try::do_call::hdca4832ace5a8603 at /rustc/7eac88abb2e57e752f3302f02be5f3ce3d7adfb4/library/std/src/panicking.rs:381:40 - std::panicking::try::ha8624a1a6854b456 at /rustc/7eac88abb2e57e752f3302f02be5f3ce3d7adfb4/library/std/src/panicking.rs:345:19 - std::panic::catch_unwind::h71421f57cf2bc688 at /rustc/7eac88abb2e57e752f3302f02be5f3ce3d7adfb4/library/std/src/panic.rs:382:14 - std::rt::lang_start_internal::h260050c92cd470af at /rustc/7eac88abb2e57e752f3302f02be5f3ce3d7adfb4/library/std/src/rt.rs:51:25 11: 0x1d0c - std::rt::lang_start::h0b4bcf3c5e498224 at /rustc/7eac88abb2e57e752f3302f02be5f3ce3d7adfb4/library/std/src/rt.rs:65:5 12: 0xffc - <unknown>!__original_main 13: 0x393 - __muloti4 at /cargo/registry/src/github.com-1ecc6299db9ec823/compiler_builtins-0.1.35/src/macros.rs:269 ``` This is relatively noisy by default but there's filenames and line numbers! Additionally frame 10 can be seen to have lots of frames inlined into it. All information is always available to the embedder but we could try to handle the `__rust_begin_short_backtrace` and `__rust_end_short_backtrace` markers to trim the backtrace by default as well. The only gotcha here is that it looks like `__muloti4` is out of place. That's because the libc that Rust ships with doesn't have dwarf information, although I'm not sure why we land in that function for symbolizing it... * Add a configuration switch for debuginfo * Control debuginfo by default with `WASM_BACKTRACE_DETAILS` * Try cpp_demangle on demangling as well * Rename to WASMTIME_BACKTRACE_DETAILS
4 years ago
dependencies = [
Remove support for userfaultfd (#4040) This commit removes support for the `userfaultfd` or "uffd" syscall on Linux. This support was originally added for users migrating from Lucet to Wasmtime, but the recent developments of kernel-supported copy-on-write support for memory initialization wound up being more appropriate for these use cases than usefaultfd. The main reason for moving to copy-on-write initialization are: * The `userfaultfd` feature was never necessarily intended for this style of use case with wasm and was susceptible to subtle and rare bugs that were extremely difficult to track down. We were never 100% certain that there were kernel bugs related to userfaultfd but the suspicion never went away. * Handling faults with userfaultfd was always slow and single-threaded. Only one thread could handle faults and traveling to user-space to handle faults is inherently slower than handling them all in the kernel. The single-threaded aspect in particular presented a significant scaling bottleneck for embeddings that want to run many wasm instances in parallel. * One of the major benefits of userfaultfd was lazy initialization of wasm linear memory which is also achieved with the copy-on-write initialization support we have right now. * One of the suspected benefits of userfaultfd was less frobbing of the kernel vma structures when wasm modules are instantiated. Currently the copy-on-write support has a mitigation where we attempt to reuse the memory images where possible to avoid changing vma structures. When comparing this to userfaultfd's performance it was found that kernel modifications of vmas aren't a worrisome bottleneck so copy-on-write is suitable for this as well. Overall there are no remaining benefits that userfaultfd gives that copy-on-write doesn't, and copy-on-write solves a major downsides of userfaultfd, the scaling issue with a single faulting thread. Additionally copy-on-write support seems much more robust in terms of kernel implementation since it's only using standard memory-management syscalls which are heavily exercised. Finally copy-on-write support provides a new bonus where read-only memory in WebAssembly can be mapped directly to the same kernel cache page, even amongst many wasm instances of the same module, which was never possible with userfaultfd. In light of all this it's expected that all users of userfaultfd should migrate to the copy-on-write initialization of Wasmtime (which is enabled by default).
3 years ago
"cfg-if",
Provide filename/line number information in `Trap` (#2452) * Provide filename/line number information in `Trap` This commit extends the `Trap` type and `Store` to retain DWARF debug information found in a wasm file unconditionally, if it's present. This then enables us to print filenames and line numbers which point back to actual source code when a trap backtrace is printed. Additionally the `FrameInfo` type has been souped up to return filename/line number information as well. The implementation here is pretty simplistic currently. The meat of all the work happens in `gimli` and `addr2line`, and otherwise wasmtime is just schlepping around bytes of dwarf debuginfo here and there! The general goal here is to assist with debugging when using wasmtime because filenames and line numbers are generally orders of magnitude better even when you already have a stack trace. Another nicety here is that backtraces will display inlined frames (learned through debug information), improving the experience in release mode as well. An example of this is that with this file: ```rust fn main() { panic!("hello"); } ``` we get this stack trace: ``` $ rustc foo.rs --target wasm32-wasi -g $ cargo run foo.wasm Finished dev [unoptimized + debuginfo] target(s) in 0.16s Running `target/debug/wasmtime foo.wasm` thread 'main' panicked at 'hello', foo.rs:2:5 note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace Error: failed to run main module `foo.wasm` Caused by: 0: failed to invoke command default 1: wasm trap: unreachable wasm backtrace: 0: 0x6c1c - panic_abort::__rust_start_panic::abort::h2d60298621b1ccbf at /rustc/7eac88abb2e57e752f3302f02be5f3ce3d7adfb4/library/panic_abort/src/lib.rs:77:17 - __rust_start_panic at /rustc/7eac88abb2e57e752f3302f02be5f3ce3d7adfb4/library/panic_abort/src/lib.rs:32:5 1: 0x68c7 - rust_panic at /rustc/7eac88abb2e57e752f3302f02be5f3ce3d7adfb4/library/std/src/panicking.rs:626:9 2: 0x65a1 - std::panicking::rust_panic_with_hook::h2345fb0909b53e12 at /rustc/7eac88abb2e57e752f3302f02be5f3ce3d7adfb4/library/std/src/panicking.rs:596:5 3: 0x1436 - std::panicking::begin_panic::{{closure}}::h106f151a6db8c8fb at /rustc/7eac88abb2e57e752f3302f02be5f3ce3d7adfb4/library/std/src/panicking.rs:506:9 4: 0xda8 - std::sys_common::backtrace::__rust_end_short_backtrace::he55aa13f22782798 at /rustc/7eac88abb2e57e752f3302f02be5f3ce3d7adfb4/library/std/src/sys_common/backtrace.rs:153:18 5: 0x1324 - std::panicking::begin_panic::h1727e7d1d719c76f at /rustc/7eac88abb2e57e752f3302f02be5f3ce3d7adfb4/library/std/src/panicking.rs:505:12 6: 0xfde - foo::main::h2db1313a64510850 at /Users/acrichton/code/wasmtime/foo.rs:2:5 7: 0x11d5 - core::ops::function::FnOnce::call_once::h20ee1cc04aeff1fc at /rustc/7eac88abb2e57e752f3302f02be5f3ce3d7adfb4/library/core/src/ops/function.rs:227:5 8: 0xddf - std::sys_common::backtrace::__rust_begin_short_backtrace::h054493e41e27e69c at /rustc/7eac88abb2e57e752f3302f02be5f3ce3d7adfb4/library/std/src/sys_common/backtrace.rs:137:18 9: 0x1d5a - std::rt::lang_start::{{closure}}::hd83784448d3fcb42 at /rustc/7eac88abb2e57e752f3302f02be5f3ce3d7adfb4/library/std/src/rt.rs:66:18 10: 0x69d8 - core::ops::function::impls::<impl core::ops::function::FnOnce<A> for &F>::call_once::h564d3dad35014917 at /rustc/7eac88abb2e57e752f3302f02be5f3ce3d7adfb4/library/core/src/ops/function.rs:259:13 - std::panicking::try::do_call::hdca4832ace5a8603 at /rustc/7eac88abb2e57e752f3302f02be5f3ce3d7adfb4/library/std/src/panicking.rs:381:40 - std::panicking::try::ha8624a1a6854b456 at /rustc/7eac88abb2e57e752f3302f02be5f3ce3d7adfb4/library/std/src/panicking.rs:345:19 - std::panic::catch_unwind::h71421f57cf2bc688 at /rustc/7eac88abb2e57e752f3302f02be5f3ce3d7adfb4/library/std/src/panic.rs:382:14 - std::rt::lang_start_internal::h260050c92cd470af at /rustc/7eac88abb2e57e752f3302f02be5f3ce3d7adfb4/library/std/src/rt.rs:51:25 11: 0x1d0c - std::rt::lang_start::h0b4bcf3c5e498224 at /rustc/7eac88abb2e57e752f3302f02be5f3ce3d7adfb4/library/std/src/rt.rs:65:5 12: 0xffc - <unknown>!__original_main 13: 0x393 - __muloti4 at /cargo/registry/src/github.com-1ecc6299db9ec823/compiler_builtins-0.1.35/src/macros.rs:269 ``` This is relatively noisy by default but there's filenames and line numbers! Additionally frame 10 can be seen to have lots of frames inlined into it. All information is always available to the embedder but we could try to handle the `__rust_begin_short_backtrace` and `__rust_end_short_backtrace` markers to trim the backtrace by default as well. The only gotcha here is that it looks like `__muloti4` is out of place. That's because the libc that Rust ships with doesn't have dwarf information, although I'm not sure why we land in that function for symbolizing it... * Add a configuration switch for debuginfo * Control debuginfo by default with `WASM_BACKTRACE_DETAILS` * Try cpp_demangle on demangling as well * Rename to WASMTIME_BACKTRACE_DETAILS
4 years ago
]
[[package]]
name = "cpufeatures"
version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "59a6001667ab124aebae2a495118e11d30984c3a653e99d86d58971708cf5e4b"
dependencies = [
"libc",
]
[[package]]
name = "cranelift"
version = "0.96.0"
dependencies = [
"cranelift-codegen",
"cranelift-frontend",
]
[[package]]
name = "cranelift-bforest"
version = "0.96.0"
dependencies = [
"cranelift-entity",
]
[[package]]
name = "cranelift-codegen"
version = "0.96.0"
dependencies = [
"anyhow",
"bincode",
"bumpalo",
"capstone",
"cranelift-bforest",
"cranelift-codegen-meta",
"cranelift-codegen-shared",
Chaos mode MVP: Skip branch optimization in MachBuffer (#6039) * fuzz: Add chaos mode control plane Co-authored-by: Falk Zwimpfer <24669719+FalkZ@users.noreply.github.com> Co-authored-by: Moritz Waser <mzrw.dev@pm.me> * fuzz: Skip branch optimization with chaos mode Co-authored-by: Falk Zwimpfer <24669719+FalkZ@users.noreply.github.com> Co-authored-by: Moritz Waser <mzrw.dev@pm.me> * fuzz: Rename chaos engine -> control plane Co-authored-by: Falk Zwimpfer <24669719+FalkZ@users.noreply.github.com> Co-authored-by: Moritz Waser <mzrw.dev@pm.me> * chaos mode: refactoring ControlPlane to be passed through the call stack by reference Co-authored-by: Falk Zwimpfer <24669719+FalkZ@users.noreply.github.com> Co-authored-by: Remo Senekowitsch <contact@remsle.dev> * fuzz: annotate chaos todos Co-authored-by: Falk Zwimpfer <24669719+FalkZ@users.noreply.github.com> Co-authored-by: Moritz Waser <mzrw.dev@pm.me> * fuzz: cleanup control plane Co-authored-by: Falk Zwimpfer <24669719+FalkZ@users.noreply.github.com> Co-authored-by: Moritz Waser <mzrw.dev@pm.me> * fuzz: remove control plane from compiler context Co-authored-by: Falk Zwimpfer <24669719+FalkZ@users.noreply.github.com> Co-authored-by: Moritz Waser <mzrw.dev@pm.me> * fuzz: move control plane into emit state Co-authored-by: Falk Zwimpfer <24669719+FalkZ@users.noreply.github.com> Co-authored-by: Moritz Waser <mzrw.dev@pm.me> * fuzz: fix remaining compiler errors Co-authored-by: Falk Zwimpfer <24669719+FalkZ@users.noreply.github.com> Co-authored-by: Moritz Waser <mzrw.dev@pm.me> * fix tests * refactor emission state ctrl plane accessors Co-authored-by: Falk Zwimpfer <24669719+FalkZ@users.noreply.github.com> Co-authored-by: Moritz Waser <mzrw.dev@pm.me> * centralize conditional compilation of chaos mode Also cleanup a few straggling dependencies on cranelift-control that aren't needed anymore. Co-authored-by: Falk Zwimpfer <24669719+FalkZ@users.noreply.github.com> Co-authored-by: Moritz Waser <mzrw.dev@pm.me> * add cranelift-control to published crates prtest:full Co-authored-by: Falk Zwimpfer <24669719+FalkZ@users.noreply.github.com> Co-authored-by: Moritz Waser <mzrw.dev@pm.me> * add cranelift-control to public crates Co-authored-by: Falk Zwimpfer <24669719+FalkZ@users.noreply.github.com> Co-authored-by: Moritz Waser <mzrw.dev@pm.me> --------- Co-authored-by: Falk Zwimpfer <24669719+FalkZ@users.noreply.github.com> Co-authored-by: Moritz Waser <mzrw.dev@pm.me> Co-authored-by: Remo Senekowitsch <contact@remsle.dev>
2 years ago
"cranelift-control",
"cranelift-entity",
"cranelift-isle",
"criterion",
"gimli",
"hashbrown 0.13.1",
"log",
"regalloc2",
"serde",
"sha2 0.10.2",
Remove heaps from core Cranelift, push them into `cranelift-wasm` (#5386) * cranelift-wasm: translate Wasm loads into lower-level CLIF operations Rather than using `heap_{load,store,addr}`. * cranelift: Remove the `heap_{addr,load,store}` instructions These are now legalized in the `cranelift-wasm` frontend. * cranelift: Remove the `ir::Heap` entity from CLIF * Port basic memory operation tests to .wat filetests * Remove test for verifying CLIF heaps * Remove `heap_addr` from replace_branching_instructions_and_cfg_predecessors.clif test * Remove `heap_addr` from readonly.clif test * Remove `heap_addr` from `table_addr.clif` test * Remove `heap_addr` from the simd-fvpromote_low.clif test * Remove `heap_addr` from simd-fvdemote.clif test * Remove `heap_addr` from the load-op-store.clif test * Remove the CLIF heap runtest * Remove `heap_addr` from the global_value.clif test * Remove `heap_addr` from fpromote.clif runtests * Remove `heap_addr` from fdemote.clif runtests * Remove `heap_addr` from memory.clif parser test * Remove `heap_addr` from reject_load_readonly.clif test * Remove `heap_addr` from reject_load_notrap.clif test * Remove `heap_addr` from load_readonly_notrap.clif test * Remove `static-heap-without-guard-pages.clif` test Will be subsumed when we port `make-heap-load-store-tests.sh` to generating `.wat` tests. * Remove `static-heap-with-guard-pages.clif` test Will be subsumed when we port `make-heap-load-store-tests.sh` over to `.wat` tests. * Remove more heap tests These will be subsumed by porting `make-heap-load-store-tests.sh` over to `.wat` tests. * Remove `heap_addr` from `simple-alias.clif` test * Remove `heap_addr` from partial-redundancy.clif test * Remove `heap_addr` from multiple-blocks.clif test * Remove `heap_addr` from fence.clif test * Remove `heap_addr` from extends.clif test * Remove runtests that rely on heaps Heaps are not a thing in CLIF or the interpreter anymore * Add generated load/store `.wat` tests * Enable memory-related wasm features in `.wat` tests * Remove CLIF heap from fcmp-mem-bug.clif test * Add a mode for compiling `.wat` all the way to assembly in filetests * Also generate WAT to assembly tests in `make-load-store-tests.sh` * cargo fmt * Reinstate `f{de,pro}mote.clif` tests without the heap bits * Remove undefined doc link * Remove outdated SVG and dot file from docs * Add docs about `None` returns for base address computation helpers * Factor out `env.heap_access_spectre_mitigation()` to a local * Expand docs for `FuncEnvironment::heaps` trait method * Restore f{de,pro}mote+load clif runtests with stack memory
2 years ago
"similar",
"smallvec",
Harvest left-hand side superoptimization candidates. Given a clif function, harvest all its integer subexpressions, so that they can be fed into [Souper](https://github.com/google/souper) as candidates for superoptimization. For some of these candidates, Souper will successfully synthesize a right-hand side that is equivalent but has lower cost than the left-hand side. Then, we can combine these left- and right-hand sides into a complete optimization, and add it to our peephole passes. To harvest the expression that produced a given value `x`, we do a post-order traversal of the dataflow graph starting from `x`. As we do this traversal, we maintain a map from clif values to their translated Souper values. We stop traversing when we reach anything that can't be translated into Souper IR: a memory load, a float-to-int conversion, a block parameter, etc. For values produced by these instructions, we create a Souper `var`, which is an input variable to the optimization. For instructions that have a direct mapping into Souper IR, we get the Souper version of each of its operands and then create the Souper version of the instruction itself. It should now be clear why we do a post-order traversal: we need an instruction's translated operands in order to translate the instruction itself. Once this instruction is translated, we update the clif-to-souper map with this new translation so that any other instruction that uses this result as an operand has access to the translated value. When the traversal is complete we return the translation of `x` as the root of left-hand side candidate.
4 years ago
"souper-ir",
"target-lexicon",
]
[[package]]
name = "cranelift-codegen-meta"
version = "0.96.0"
dependencies = [
"cranelift-codegen-shared",
]
[[package]]
name = "cranelift-codegen-shared"
version = "0.96.0"
Chaos mode MVP: Skip branch optimization in MachBuffer (#6039) * fuzz: Add chaos mode control plane Co-authored-by: Falk Zwimpfer <24669719+FalkZ@users.noreply.github.com> Co-authored-by: Moritz Waser <mzrw.dev@pm.me> * fuzz: Skip branch optimization with chaos mode Co-authored-by: Falk Zwimpfer <24669719+FalkZ@users.noreply.github.com> Co-authored-by: Moritz Waser <mzrw.dev@pm.me> * fuzz: Rename chaos engine -> control plane Co-authored-by: Falk Zwimpfer <24669719+FalkZ@users.noreply.github.com> Co-authored-by: Moritz Waser <mzrw.dev@pm.me> * chaos mode: refactoring ControlPlane to be passed through the call stack by reference Co-authored-by: Falk Zwimpfer <24669719+FalkZ@users.noreply.github.com> Co-authored-by: Remo Senekowitsch <contact@remsle.dev> * fuzz: annotate chaos todos Co-authored-by: Falk Zwimpfer <24669719+FalkZ@users.noreply.github.com> Co-authored-by: Moritz Waser <mzrw.dev@pm.me> * fuzz: cleanup control plane Co-authored-by: Falk Zwimpfer <24669719+FalkZ@users.noreply.github.com> Co-authored-by: Moritz Waser <mzrw.dev@pm.me> * fuzz: remove control plane from compiler context Co-authored-by: Falk Zwimpfer <24669719+FalkZ@users.noreply.github.com> Co-authored-by: Moritz Waser <mzrw.dev@pm.me> * fuzz: move control plane into emit state Co-authored-by: Falk Zwimpfer <24669719+FalkZ@users.noreply.github.com> Co-authored-by: Moritz Waser <mzrw.dev@pm.me> * fuzz: fix remaining compiler errors Co-authored-by: Falk Zwimpfer <24669719+FalkZ@users.noreply.github.com> Co-authored-by: Moritz Waser <mzrw.dev@pm.me> * fix tests * refactor emission state ctrl plane accessors Co-authored-by: Falk Zwimpfer <24669719+FalkZ@users.noreply.github.com> Co-authored-by: Moritz Waser <mzrw.dev@pm.me> * centralize conditional compilation of chaos mode Also cleanup a few straggling dependencies on cranelift-control that aren't needed anymore. Co-authored-by: Falk Zwimpfer <24669719+FalkZ@users.noreply.github.com> Co-authored-by: Moritz Waser <mzrw.dev@pm.me> * add cranelift-control to published crates prtest:full Co-authored-by: Falk Zwimpfer <24669719+FalkZ@users.noreply.github.com> Co-authored-by: Moritz Waser <mzrw.dev@pm.me> * add cranelift-control to public crates Co-authored-by: Falk Zwimpfer <24669719+FalkZ@users.noreply.github.com> Co-authored-by: Moritz Waser <mzrw.dev@pm.me> --------- Co-authored-by: Falk Zwimpfer <24669719+FalkZ@users.noreply.github.com> Co-authored-by: Moritz Waser <mzrw.dev@pm.me> Co-authored-by: Remo Senekowitsch <contact@remsle.dev>
2 years ago
[[package]]
name = "cranelift-control"
version = "0.96.0"
dependencies = [
"arbitrary",
]
[[package]]
name = "cranelift-entity"
version = "0.96.0"
dependencies = [
"serde",
]
[[package]]
name = "cranelift-filetests"
version = "0.0.0"
dependencies = [
"anyhow",
"cranelift",
"cranelift-codegen",
Chaos mode MVP: Skip branch optimization in MachBuffer (#6039) * fuzz: Add chaos mode control plane Co-authored-by: Falk Zwimpfer <24669719+FalkZ@users.noreply.github.com> Co-authored-by: Moritz Waser <mzrw.dev@pm.me> * fuzz: Skip branch optimization with chaos mode Co-authored-by: Falk Zwimpfer <24669719+FalkZ@users.noreply.github.com> Co-authored-by: Moritz Waser <mzrw.dev@pm.me> * fuzz: Rename chaos engine -> control plane Co-authored-by: Falk Zwimpfer <24669719+FalkZ@users.noreply.github.com> Co-authored-by: Moritz Waser <mzrw.dev@pm.me> * chaos mode: refactoring ControlPlane to be passed through the call stack by reference Co-authored-by: Falk Zwimpfer <24669719+FalkZ@users.noreply.github.com> Co-authored-by: Remo Senekowitsch <contact@remsle.dev> * fuzz: annotate chaos todos Co-authored-by: Falk Zwimpfer <24669719+FalkZ@users.noreply.github.com> Co-authored-by: Moritz Waser <mzrw.dev@pm.me> * fuzz: cleanup control plane Co-authored-by: Falk Zwimpfer <24669719+FalkZ@users.noreply.github.com> Co-authored-by: Moritz Waser <mzrw.dev@pm.me> * fuzz: remove control plane from compiler context Co-authored-by: Falk Zwimpfer <24669719+FalkZ@users.noreply.github.com> Co-authored-by: Moritz Waser <mzrw.dev@pm.me> * fuzz: move control plane into emit state Co-authored-by: Falk Zwimpfer <24669719+FalkZ@users.noreply.github.com> Co-authored-by: Moritz Waser <mzrw.dev@pm.me> * fuzz: fix remaining compiler errors Co-authored-by: Falk Zwimpfer <24669719+FalkZ@users.noreply.github.com> Co-authored-by: Moritz Waser <mzrw.dev@pm.me> * fix tests * refactor emission state ctrl plane accessors Co-authored-by: Falk Zwimpfer <24669719+FalkZ@users.noreply.github.com> Co-authored-by: Moritz Waser <mzrw.dev@pm.me> * centralize conditional compilation of chaos mode Also cleanup a few straggling dependencies on cranelift-control that aren't needed anymore. Co-authored-by: Falk Zwimpfer <24669719+FalkZ@users.noreply.github.com> Co-authored-by: Moritz Waser <mzrw.dev@pm.me> * add cranelift-control to published crates prtest:full Co-authored-by: Falk Zwimpfer <24669719+FalkZ@users.noreply.github.com> Co-authored-by: Moritz Waser <mzrw.dev@pm.me> * add cranelift-control to public crates Co-authored-by: Falk Zwimpfer <24669719+FalkZ@users.noreply.github.com> Co-authored-by: Moritz Waser <mzrw.dev@pm.me> --------- Co-authored-by: Falk Zwimpfer <24669719+FalkZ@users.noreply.github.com> Co-authored-by: Moritz Waser <mzrw.dev@pm.me> Co-authored-by: Remo Senekowitsch <contact@remsle.dev>
2 years ago
"cranelift-control",
Add ability to call CLIF functions with arbitrary arguments in filetests This resolves the work started in https://github.com/bytecodealliance/cranelift/pull/1231 and https://github.com/bytecodealliance/wasmtime/pull/1436. Cranelift filetests currently have the ability to run CLIF functions with a signature like `() -> b*` and check that the result is true under the `test run` directive. This PR adds the ability to call functions with arbitrary arguments and non-boolean returns and either print the result or check against a list of expected results: - `run` commands look like `; run: %add(2, 2) == 4` or `; run: %add(2, 2) != 5` and verify that the executed CLIF function returns the expected value - `print` commands look like `; print: %add(2, 2)` and print the result of the function to stdout To make this work, this PR compiles a single Cranelift `Function` into a `CompiledFunction` using a `SingleFunctionCompiler`. Because we will not know the signature of the function until runtime, we use a `Trampoline` to place the values in the appropriate location for the calling convention; this should look a lot like what @alexcrichton is doing with `VMTrampoline` in wasmtime (see https://github.com/bytecodealliance/wasmtime/blob/3b7cb6ee64469470fcdd68e185abca8eb2a1b20a/crates/api/src/func.rs#L510-L526, https://github.com/bytecodealliance/wasmtime/blob/3b7cb6ee64469470fcdd68e185abca8eb2a1b20a/crates/jit/src/compiler.rs#L260). To avoid re-compiling `Trampoline`s for the same function signatures, `Trampoline`s are cached in the `SingleFunctionCompiler`.
5 years ago
"cranelift-frontend",
"cranelift-interpreter",
"cranelift-jit",
"cranelift-module",
"cranelift-native",
"cranelift-reader",
"cranelift-wasm",
"file-per-thread-logger",
"filecheck",
"gimli",
"log",
"num_cpus",
"serde",
"similar",
"target-lexicon",
Add ability to call CLIF functions with arbitrary arguments in filetests This resolves the work started in https://github.com/bytecodealliance/cranelift/pull/1231 and https://github.com/bytecodealliance/wasmtime/pull/1436. Cranelift filetests currently have the ability to run CLIF functions with a signature like `() -> b*` and check that the result is true under the `test run` directive. This PR adds the ability to call functions with arbitrary arguments and non-boolean returns and either print the result or check against a list of expected results: - `run` commands look like `; run: %add(2, 2) == 4` or `; run: %add(2, 2) != 5` and verify that the executed CLIF function returns the expected value - `print` commands look like `; print: %add(2, 2)` and print the result of the function to stdout To make this work, this PR compiles a single Cranelift `Function` into a `CompiledFunction` using a `SingleFunctionCompiler`. Because we will not know the signature of the function until runtime, we use a `Trampoline` to place the values in the appropriate location for the calling convention; this should look a lot like what @alexcrichton is doing with `VMTrampoline` in wasmtime (see https://github.com/bytecodealliance/wasmtime/blob/3b7cb6ee64469470fcdd68e185abca8eb2a1b20a/crates/api/src/func.rs#L510-L526, https://github.com/bytecodealliance/wasmtime/blob/3b7cb6ee64469470fcdd68e185abca8eb2a1b20a/crates/jit/src/compiler.rs#L260). To avoid re-compiling `Trampoline`s for the same function signatures, `Trampoline`s are cached in the `SingleFunctionCompiler`.
5 years ago
"thiserror",
"toml",
"wasmparser",
"wat",
]
[[package]]
name = "cranelift-frontend"
version = "0.96.0"
dependencies = [
"cranelift-codegen",
"hashbrown 0.13.1",
"log",
"similar",
"smallvec",
"target-lexicon",
]
[[package]]
name = "cranelift-fuzzgen"
version = "0.0.0"
dependencies = [
"anyhow",
"arbitrary",
"cranelift",
"cranelift-native",
"once_cell",
"target-lexicon",
]
[[package]]
name = "cranelift-interpreter"
version = "0.96.0"
dependencies = [
"cranelift-codegen",
"cranelift-entity",
"cranelift-frontend",
"cranelift-reader",
"libm",
"log",
"smallvec",
"thiserror",
]
[[package]]
name = "cranelift-isle"
version = "0.96.0"
dependencies = [
"codespan-reporting",
"log",
"tempfile",
]
[[package]]
name = "cranelift-jit"
version = "0.96.0"
dependencies = [
"anyhow",
"cranelift",
"cranelift-codegen",
Chaos mode MVP: Skip branch optimization in MachBuffer (#6039) * fuzz: Add chaos mode control plane Co-authored-by: Falk Zwimpfer <24669719+FalkZ@users.noreply.github.com> Co-authored-by: Moritz Waser <mzrw.dev@pm.me> * fuzz: Skip branch optimization with chaos mode Co-authored-by: Falk Zwimpfer <24669719+FalkZ@users.noreply.github.com> Co-authored-by: Moritz Waser <mzrw.dev@pm.me> * fuzz: Rename chaos engine -> control plane Co-authored-by: Falk Zwimpfer <24669719+FalkZ@users.noreply.github.com> Co-authored-by: Moritz Waser <mzrw.dev@pm.me> * chaos mode: refactoring ControlPlane to be passed through the call stack by reference Co-authored-by: Falk Zwimpfer <24669719+FalkZ@users.noreply.github.com> Co-authored-by: Remo Senekowitsch <contact@remsle.dev> * fuzz: annotate chaos todos Co-authored-by: Falk Zwimpfer <24669719+FalkZ@users.noreply.github.com> Co-authored-by: Moritz Waser <mzrw.dev@pm.me> * fuzz: cleanup control plane Co-authored-by: Falk Zwimpfer <24669719+FalkZ@users.noreply.github.com> Co-authored-by: Moritz Waser <mzrw.dev@pm.me> * fuzz: remove control plane from compiler context Co-authored-by: Falk Zwimpfer <24669719+FalkZ@users.noreply.github.com> Co-authored-by: Moritz Waser <mzrw.dev@pm.me> * fuzz: move control plane into emit state Co-authored-by: Falk Zwimpfer <24669719+FalkZ@users.noreply.github.com> Co-authored-by: Moritz Waser <mzrw.dev@pm.me> * fuzz: fix remaining compiler errors Co-authored-by: Falk Zwimpfer <24669719+FalkZ@users.noreply.github.com> Co-authored-by: Moritz Waser <mzrw.dev@pm.me> * fix tests * refactor emission state ctrl plane accessors Co-authored-by: Falk Zwimpfer <24669719+FalkZ@users.noreply.github.com> Co-authored-by: Moritz Waser <mzrw.dev@pm.me> * centralize conditional compilation of chaos mode Also cleanup a few straggling dependencies on cranelift-control that aren't needed anymore. Co-authored-by: Falk Zwimpfer <24669719+FalkZ@users.noreply.github.com> Co-authored-by: Moritz Waser <mzrw.dev@pm.me> * add cranelift-control to published crates prtest:full Co-authored-by: Falk Zwimpfer <24669719+FalkZ@users.noreply.github.com> Co-authored-by: Moritz Waser <mzrw.dev@pm.me> * add cranelift-control to public crates Co-authored-by: Falk Zwimpfer <24669719+FalkZ@users.noreply.github.com> Co-authored-by: Moritz Waser <mzrw.dev@pm.me> --------- Co-authored-by: Falk Zwimpfer <24669719+FalkZ@users.noreply.github.com> Co-authored-by: Moritz Waser <mzrw.dev@pm.me> Co-authored-by: Remo Senekowitsch <contact@remsle.dev>
2 years ago
"cranelift-control",
"cranelift-entity",
"cranelift-frontend",
"cranelift-module",
"cranelift-native",
"libc",
"log",
"memmap2",
"region",
"target-lexicon",
Flush Icache on AArch64 Windows (#4997) * cranelift: Add FlushInstructionCache for AArch64 on Windows This was previously done on #3426 for linux. * wasmtime: Add FlushInstructionCache for AArch64 on Windows This was previously done on #3426 for linux. * cranelift: Add MemoryUse flag to JIT Memory Manager This allows us to keep the icache flushing code self-contained and not leak implementation details. This also changes the windows icache flushing code to only flush pages that were previously unflushed. * Add jit-icache-coherence crate * cranelift: Use `jit-icache-coherence` * wasmtime: Use `jit-icache-coherence` * jit-icache-coherence: Make rustix feature additive Mutually exclusive features cause issues. * wasmtime: Remove rustix from wasmtime-jit We now use it via jit-icache-coherence * Rename wasmtime-jit-icache-coherency crate * Use cfg-if in wasmtime-jit-icache-coherency crate * Use inline instead of inline(always) * Add unsafe marker to clear_cache * Conditionally compile all rustix operations membarrier does not exist on MacOS * Publish `wasmtime-jit-icache-coherence` * Remove explicit windows check This is implied by the target_os = "windows" above * cranelift: Remove len != 0 check This is redundant as it is done in non_protected_allocations_iter * Comment cleanups Thanks @akirilov-arm! * Make clear_cache safe * Rename pipeline_flush to pipeline_flush_mt * Revert "Make clear_cache safe" This reverts commit 21165d81c9030ed9b291a1021a367214d2942c90. * More docs! * Fix pipeline_flush reference on clear_cache * Update more docs! * Move pipeline flush after `mprotect` calls Technically the `clear_cache` operation is a lie in AArch64, so move the pipeline flush after the `mprotect` calls so that it benefits from the implicit cache cleaning done by it. * wasmtime: Remove rustix backend from icache crate * wasmtime: Use libc for macos * wasmtime: Flush icache on all arch's for windows * wasmtime: Add flags to membarrier call
2 years ago
"wasmtime-jit-icache-coherence",
"windows-sys",
]
[[package]]
name = "cranelift-module"
version = "0.96.0"
dependencies = [
"anyhow",
"cranelift-codegen",
Chaos mode MVP: Skip branch optimization in MachBuffer (#6039) * fuzz: Add chaos mode control plane Co-authored-by: Falk Zwimpfer <24669719+FalkZ@users.noreply.github.com> Co-authored-by: Moritz Waser <mzrw.dev@pm.me> * fuzz: Skip branch optimization with chaos mode Co-authored-by: Falk Zwimpfer <24669719+FalkZ@users.noreply.github.com> Co-authored-by: Moritz Waser <mzrw.dev@pm.me> * fuzz: Rename chaos engine -> control plane Co-authored-by: Falk Zwimpfer <24669719+FalkZ@users.noreply.github.com> Co-authored-by: Moritz Waser <mzrw.dev@pm.me> * chaos mode: refactoring ControlPlane to be passed through the call stack by reference Co-authored-by: Falk Zwimpfer <24669719+FalkZ@users.noreply.github.com> Co-authored-by: Remo Senekowitsch <contact@remsle.dev> * fuzz: annotate chaos todos Co-authored-by: Falk Zwimpfer <24669719+FalkZ@users.noreply.github.com> Co-authored-by: Moritz Waser <mzrw.dev@pm.me> * fuzz: cleanup control plane Co-authored-by: Falk Zwimpfer <24669719+FalkZ@users.noreply.github.com> Co-authored-by: Moritz Waser <mzrw.dev@pm.me> * fuzz: remove control plane from compiler context Co-authored-by: Falk Zwimpfer <24669719+FalkZ@users.noreply.github.com> Co-authored-by: Moritz Waser <mzrw.dev@pm.me> * fuzz: move control plane into emit state Co-authored-by: Falk Zwimpfer <24669719+FalkZ@users.noreply.github.com> Co-authored-by: Moritz Waser <mzrw.dev@pm.me> * fuzz: fix remaining compiler errors Co-authored-by: Falk Zwimpfer <24669719+FalkZ@users.noreply.github.com> Co-authored-by: Moritz Waser <mzrw.dev@pm.me> * fix tests * refactor emission state ctrl plane accessors Co-authored-by: Falk Zwimpfer <24669719+FalkZ@users.noreply.github.com> Co-authored-by: Moritz Waser <mzrw.dev@pm.me> * centralize conditional compilation of chaos mode Also cleanup a few straggling dependencies on cranelift-control that aren't needed anymore. Co-authored-by: Falk Zwimpfer <24669719+FalkZ@users.noreply.github.com> Co-authored-by: Moritz Waser <mzrw.dev@pm.me> * add cranelift-control to published crates prtest:full Co-authored-by: Falk Zwimpfer <24669719+FalkZ@users.noreply.github.com> Co-authored-by: Moritz Waser <mzrw.dev@pm.me> * add cranelift-control to public crates Co-authored-by: Falk Zwimpfer <24669719+FalkZ@users.noreply.github.com> Co-authored-by: Moritz Waser <mzrw.dev@pm.me> --------- Co-authored-by: Falk Zwimpfer <24669719+FalkZ@users.noreply.github.com> Co-authored-by: Moritz Waser <mzrw.dev@pm.me> Co-authored-by: Remo Senekowitsch <contact@remsle.dev>
2 years ago
"cranelift-control",
"hashbrown 0.13.1",
]
[[package]]
name = "cranelift-native"
version = "0.96.0"
dependencies = [
"cranelift-codegen",
"libc",
"target-lexicon",
]
[[package]]
name = "cranelift-object"
version = "0.96.0"
dependencies = [
"anyhow",
"cranelift-codegen",
Chaos mode MVP: Skip branch optimization in MachBuffer (#6039) * fuzz: Add chaos mode control plane Co-authored-by: Falk Zwimpfer <24669719+FalkZ@users.noreply.github.com> Co-authored-by: Moritz Waser <mzrw.dev@pm.me> * fuzz: Skip branch optimization with chaos mode Co-authored-by: Falk Zwimpfer <24669719+FalkZ@users.noreply.github.com> Co-authored-by: Moritz Waser <mzrw.dev@pm.me> * fuzz: Rename chaos engine -> control plane Co-authored-by: Falk Zwimpfer <24669719+FalkZ@users.noreply.github.com> Co-authored-by: Moritz Waser <mzrw.dev@pm.me> * chaos mode: refactoring ControlPlane to be passed through the call stack by reference Co-authored-by: Falk Zwimpfer <24669719+FalkZ@users.noreply.github.com> Co-authored-by: Remo Senekowitsch <contact@remsle.dev> * fuzz: annotate chaos todos Co-authored-by: Falk Zwimpfer <24669719+FalkZ@users.noreply.github.com> Co-authored-by: Moritz Waser <mzrw.dev@pm.me> * fuzz: cleanup control plane Co-authored-by: Falk Zwimpfer <24669719+FalkZ@users.noreply.github.com> Co-authored-by: Moritz Waser <mzrw.dev@pm.me> * fuzz: remove control plane from compiler context Co-authored-by: Falk Zwimpfer <24669719+FalkZ@users.noreply.github.com> Co-authored-by: Moritz Waser <mzrw.dev@pm.me> * fuzz: move control plane into emit state Co-authored-by: Falk Zwimpfer <24669719+FalkZ@users.noreply.github.com> Co-authored-by: Moritz Waser <mzrw.dev@pm.me> * fuzz: fix remaining compiler errors Co-authored-by: Falk Zwimpfer <24669719+FalkZ@users.noreply.github.com> Co-authored-by: Moritz Waser <mzrw.dev@pm.me> * fix tests * refactor emission state ctrl plane accessors Co-authored-by: Falk Zwimpfer <24669719+FalkZ@users.noreply.github.com> Co-authored-by: Moritz Waser <mzrw.dev@pm.me> * centralize conditional compilation of chaos mode Also cleanup a few straggling dependencies on cranelift-control that aren't needed anymore. Co-authored-by: Falk Zwimpfer <24669719+FalkZ@users.noreply.github.com> Co-authored-by: Moritz Waser <mzrw.dev@pm.me> * add cranelift-control to published crates prtest:full Co-authored-by: Falk Zwimpfer <24669719+FalkZ@users.noreply.github.com> Co-authored-by: Moritz Waser <mzrw.dev@pm.me> * add cranelift-control to public crates Co-authored-by: Falk Zwimpfer <24669719+FalkZ@users.noreply.github.com> Co-authored-by: Moritz Waser <mzrw.dev@pm.me> --------- Co-authored-by: Falk Zwimpfer <24669719+FalkZ@users.noreply.github.com> Co-authored-by: Moritz Waser <mzrw.dev@pm.me> Co-authored-by: Remo Senekowitsch <contact@remsle.dev>
2 years ago
"cranelift-control",
"cranelift-entity",
"cranelift-frontend",
"cranelift-module",
"log",
"object",
"target-lexicon",
]
[[package]]
name = "cranelift-reader"
version = "0.96.0"
dependencies = [
"anyhow",
"cranelift-codegen",
"smallvec",
"target-lexicon",
]
[[package]]
name = "cranelift-serde"
version = "0.96.0"
dependencies = [
"clap",
"cranelift-codegen",
"cranelift-reader",
"serde_json",
]
[[package]]
name = "cranelift-tools"
version = "0.0.0"
dependencies = [
"anyhow",
"capstone",
Remove support for userfaultfd (#4040) This commit removes support for the `userfaultfd` or "uffd" syscall on Linux. This support was originally added for users migrating from Lucet to Wasmtime, but the recent developments of kernel-supported copy-on-write support for memory initialization wound up being more appropriate for these use cases than usefaultfd. The main reason for moving to copy-on-write initialization are: * The `userfaultfd` feature was never necessarily intended for this style of use case with wasm and was susceptible to subtle and rare bugs that were extremely difficult to track down. We were never 100% certain that there were kernel bugs related to userfaultfd but the suspicion never went away. * Handling faults with userfaultfd was always slow and single-threaded. Only one thread could handle faults and traveling to user-space to handle faults is inherently slower than handling them all in the kernel. The single-threaded aspect in particular presented a significant scaling bottleneck for embeddings that want to run many wasm instances in parallel. * One of the major benefits of userfaultfd was lazy initialization of wasm linear memory which is also achieved with the copy-on-write initialization support we have right now. * One of the suspected benefits of userfaultfd was less frobbing of the kernel vma structures when wasm modules are instantiated. Currently the copy-on-write support has a mitigation where we attempt to reuse the memory images where possible to avoid changing vma structures. When comparing this to userfaultfd's performance it was found that kernel modifications of vmas aren't a worrisome bottleneck so copy-on-write is suitable for this as well. Overall there are no remaining benefits that userfaultfd gives that copy-on-write doesn't, and copy-on-write solves a major downsides of userfaultfd, the scaling issue with a single faulting thread. Additionally copy-on-write support seems much more robust in terms of kernel implementation since it's only using standard memory-management syscalls which are heavily exercised. Finally copy-on-write support provides a new bonus where read-only memory in WebAssembly can be mapped directly to the same kernel cache page, even amongst many wasm instances of the same module, which was never possible with userfaultfd. In light of all this it's expected that all users of userfaultfd should migrate to the copy-on-write initialization of Wasmtime (which is enabled by default).
3 years ago
"cfg-if",
"clap",
"cranelift",
"cranelift-codegen",
"cranelift-entity",
"cranelift-filetests",
"cranelift-frontend",
"cranelift-interpreter",
"cranelift-jit",
"cranelift-module",
"cranelift-native",
"cranelift-object",
"cranelift-reader",
"cranelift-wasm",
"filecheck",
"fxhash",
"indicatif",
"log",
"pretty_env_logger",
Harvest left-hand side superoptimization candidates. Given a clif function, harvest all its integer subexpressions, so that they can be fed into [Souper](https://github.com/google/souper) as candidates for superoptimization. For some of these candidates, Souper will successfully synthesize a right-hand side that is equivalent but has lower cost than the left-hand side. Then, we can combine these left- and right-hand sides into a complete optimization, and add it to our peephole passes. To harvest the expression that produced a given value `x`, we do a post-order traversal of the dataflow graph starting from `x`. As we do this traversal, we maintain a map from clif values to their translated Souper values. We stop traversing when we reach anything that can't be translated into Souper IR: a memory load, a float-to-int conversion, a block parameter, etc. For values produced by these instructions, we create a Souper `var`, which is an input variable to the optimization. For instructions that have a direct mapping into Souper IR, we get the Souper version of each of its operands and then create the Souper version of the instruction itself. It should now be clear why we do a post-order traversal: we need an instruction's translated operands in order to translate the instruction itself. Once this instruction is translated, we update the clif-to-souper map with this new translation so that any other instruction that uses this result as an operand has access to the translated value. When the traversal is complete we return the translation of `x` as the root of left-hand side candidate.
4 years ago
"rayon",
"serde",
"similar",
"target-lexicon",
"termcolor",
"thiserror",
"toml",
"walkdir",
"wat",
]
[[package]]
name = "cranelift-wasm"
version = "0.96.0"
dependencies = [
"cranelift-codegen",
"cranelift-entity",
"cranelift-frontend",
"hashbrown 0.13.1",
"itertools",
"log",
"serde",
"smallvec",
"target-lexicon",
"wasmparser",
Remove `wasmtime-environ`'s dependency on `cranelift-codegen` (#3199) * Move `CompiledFunction` into wasmtime-cranelift This commit moves the `wasmtime_environ::CompiledFunction` type into the `wasmtime-cranelift` crate. This type has lots of Cranelift-specific pieces of compilation and doesn't need to be generated by all Wasmtime compilers. This replaces the usage in the `Compiler` trait with a `Box<Any>` type that each compiler can select. Each compiler must still produce a `FunctionInfo`, however, which is shared information we'll deserialize for each module. The `wasmtime-debug` crate is also folded into the `wasmtime-cranelift` crate as a result of this commit. One possibility was to move the `CompiledFunction` commit into its own crate and have `wasmtime-debug` depend on that, but since `wasmtime-debug` is Cranelift-specific at this time it didn't seem like it was too too necessary to keep it separate. If `wasmtime-debug` supports other backends in the future we can recreate a new crate, perhaps with it refactored to not depend on Cranelift. * Move wasmtime_environ::reference_type This now belongs in wasmtime-cranelift and nowhere else * Remove `Type` reexport in wasmtime-environ One less dependency on `cranelift-codegen`! * Remove `types` reexport from `wasmtime-environ` Less cranelift! * Remove `SourceLoc` from wasmtime-environ Change the `srcloc`, `start_srcloc`, and `end_srcloc` fields to a custom `FilePos` type instead of `ir::SourceLoc`. These are only used in a few places so there's not much to lose from an extra abstraction for these leaf use cases outside of cranelift. * Remove wasmtime-environ's dep on cranelift's `StackMap` This commit "clones" the `StackMap` data structure in to `wasmtime-environ` to have an independent representation that that chosen by Cranelift. This allows Wasmtime to decouple this runtime dependency of stack map information and let the two evolve independently, if necessary. An alternative would be to refactor cranelift's implementation into a separate crate and have wasmtime depend on that but it seemed a bit like overkill to do so and easier to clone just a few lines for this. * Define code offsets in wasmtime-environ with `u32` Don't use Cranelift's `binemit::CodeOffset` alias to define this field type since the `wasmtime-environ` crate will be losing the `cranelift-codegen` dependency soon. * Commit to using `cranelift-entity` in Wasmtime This commit removes the reexport of `cranelift-entity` from the `wasmtime-environ` crate and instead directly depends on the `cranelift-entity` crate in all referencing crates. The original reason for the reexport was to make cranelift version bumps easier since it's less versions to change, but nowadays we have a script to do that. Otherwise this encourages crates to use whatever they want from `cranelift-entity` since we'll always depend on the whole crate. It's expected that the `cranelift-entity` crate will continue to be a lean crate in dependencies and suitable for use at both runtime and compile time. Consequently there's no need to avoid its usage in Wasmtime at runtime, since "remove Cranelift at compile time" is primarily about the `cranelift-codegen` crate. * Remove most uses of `cranelift-codegen` in `wasmtime-environ` There's only one final use remaining, which is the reexport of `TrapCode`, which will get handled later. * Limit the glob-reexport of `cranelift_wasm` This commit removes the glob reexport of `cranelift-wasm` from the `wasmtime-environ` crate. This is intended to explicitly define what we're reexporting and is a transitionary step to curtail the amount of dependencies taken on `cranelift-wasm` throughout the codebase. For example some functions used by debuginfo mapping are better imported directly from the crate since they're Cranelift-specific. Note that this is intended to be a temporary state affairs, soon this reexport will be gone entirely. Additionally this commit reduces imports from `cranelift_wasm` and also primarily imports from `crate::wasm` within `wasmtime-environ` to get a better sense of what's imported from where and what will need to be shared. * Extract types from cranelift-wasm to cranelift-wasm-types This commit creates a new crate called `cranelift-wasm-types` and extracts type definitions from the `cranelift-wasm` crate into this new crate. The purpose of this crate is to be a shared definition of wasm types that can be shared both by compilers (like Cranelift) as well as wasm runtimes (e.g. Wasmtime). This new `cranelift-wasm-types` crate doesn't depend on `cranelift-codegen` and is the final step in severing the unconditional dependency from Wasmtime to `cranelift-codegen`. The final refactoring in this commit is to then reexport this crate from `wasmtime-environ`, delete the `cranelift-codegen` dependency, and then update all `use` paths to point to these new types. The main change of substance here is that the `TrapCode` enum is mirrored from Cranelift into this `cranelift-wasm-types` crate. While this unfortunately results in three definitions (one more which is non-exhaustive in Wasmtime itself) it's hopefully not too onerous and ideally something we can patch up in the future. * Get lightbeam compiling * Remove unnecessary dependency * Fix compile with uffd * Update publish script * Fix more uffd tests * Rename cranelift-wasm-types to wasmtime-types This reflects the purpose a bit more where it's types specifically intended for Wasmtime and its support. * Fix publish script
3 years ago
"wasmtime-types",
"wat",
]
[[package]]
name = "crc32fast"
version = "1.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d"
dependencies = [
Remove support for userfaultfd (#4040) This commit removes support for the `userfaultfd` or "uffd" syscall on Linux. This support was originally added for users migrating from Lucet to Wasmtime, but the recent developments of kernel-supported copy-on-write support for memory initialization wound up being more appropriate for these use cases than usefaultfd. The main reason for moving to copy-on-write initialization are: * The `userfaultfd` feature was never necessarily intended for this style of use case with wasm and was susceptible to subtle and rare bugs that were extremely difficult to track down. We were never 100% certain that there were kernel bugs related to userfaultfd but the suspicion never went away. * Handling faults with userfaultfd was always slow and single-threaded. Only one thread could handle faults and traveling to user-space to handle faults is inherently slower than handling them all in the kernel. The single-threaded aspect in particular presented a significant scaling bottleneck for embeddings that want to run many wasm instances in parallel. * One of the major benefits of userfaultfd was lazy initialization of wasm linear memory which is also achieved with the copy-on-write initialization support we have right now. * One of the suspected benefits of userfaultfd was less frobbing of the kernel vma structures when wasm modules are instantiated. Currently the copy-on-write support has a mitigation where we attempt to reuse the memory images where possible to avoid changing vma structures. When comparing this to userfaultfd's performance it was found that kernel modifications of vmas aren't a worrisome bottleneck so copy-on-write is suitable for this as well. Overall there are no remaining benefits that userfaultfd gives that copy-on-write doesn't, and copy-on-write solves a major downsides of userfaultfd, the scaling issue with a single faulting thread. Additionally copy-on-write support seems much more robust in terms of kernel implementation since it's only using standard memory-management syscalls which are heavily exercised. Finally copy-on-write support provides a new bonus where read-only memory in WebAssembly can be mapped directly to the same kernel cache page, even amongst many wasm instances of the same module, which was never possible with userfaultfd. In light of all this it's expected that all users of userfaultfd should migrate to the copy-on-write initialization of Wasmtime (which is enabled by default).
3 years ago
"cfg-if",
]
[[package]]
name = "criterion"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e7c76e09c1aae2bc52b3d2f29e13c6572553b30c4aa1b8a49fd70de6412654cb"
dependencies = [
"anes",
"atty",
"cast",
"ciborium",
"clap",
"criterion-plot",
"itertools",
"lazy_static",
"num-traits",
"oorandom",
"plotters",
"rayon",
"regex",
"serde",
"serde_derive",
"serde_json",
"tinytemplate",
"walkdir",
]
[[package]]
name = "criterion-plot"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1"
dependencies = [
"cast",
"itertools",
]
[[package]]
name = "crossbeam-channel"
version = "0.5.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5aaa7bd5fb665c6864b5f963dd9097905c54125909c7aa94c9e18507cdbe6c53"
dependencies = [
Remove support for userfaultfd (#4040) This commit removes support for the `userfaultfd` or "uffd" syscall on Linux. This support was originally added for users migrating from Lucet to Wasmtime, but the recent developments of kernel-supported copy-on-write support for memory initialization wound up being more appropriate for these use cases than usefaultfd. The main reason for moving to copy-on-write initialization are: * The `userfaultfd` feature was never necessarily intended for this style of use case with wasm and was susceptible to subtle and rare bugs that were extremely difficult to track down. We were never 100% certain that there were kernel bugs related to userfaultfd but the suspicion never went away. * Handling faults with userfaultfd was always slow and single-threaded. Only one thread could handle faults and traveling to user-space to handle faults is inherently slower than handling them all in the kernel. The single-threaded aspect in particular presented a significant scaling bottleneck for embeddings that want to run many wasm instances in parallel. * One of the major benefits of userfaultfd was lazy initialization of wasm linear memory which is also achieved with the copy-on-write initialization support we have right now. * One of the suspected benefits of userfaultfd was less frobbing of the kernel vma structures when wasm modules are instantiated. Currently the copy-on-write support has a mitigation where we attempt to reuse the memory images where possible to avoid changing vma structures. When comparing this to userfaultfd's performance it was found that kernel modifications of vmas aren't a worrisome bottleneck so copy-on-write is suitable for this as well. Overall there are no remaining benefits that userfaultfd gives that copy-on-write doesn't, and copy-on-write solves a major downsides of userfaultfd, the scaling issue with a single faulting thread. Additionally copy-on-write support seems much more robust in terms of kernel implementation since it's only using standard memory-management syscalls which are heavily exercised. Finally copy-on-write support provides a new bonus where read-only memory in WebAssembly can be mapped directly to the same kernel cache page, even amongst many wasm instances of the same module, which was never possible with userfaultfd. In light of all this it's expected that all users of userfaultfd should migrate to the copy-on-write initialization of Wasmtime (which is enabled by default).
3 years ago
"cfg-if",
4 years ago
"crossbeam-utils",
]
[[package]]
name = "crossbeam-deque"
version = "0.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6455c0ca19f0d2fbf751b908d5c55c1f5cbc65e03c4225427254b46890bdde1e"
dependencies = [
Remove support for userfaultfd (#4040) This commit removes support for the `userfaultfd` or "uffd" syscall on Linux. This support was originally added for users migrating from Lucet to Wasmtime, but the recent developments of kernel-supported copy-on-write support for memory initialization wound up being more appropriate for these use cases than usefaultfd. The main reason for moving to copy-on-write initialization are: * The `userfaultfd` feature was never necessarily intended for this style of use case with wasm and was susceptible to subtle and rare bugs that were extremely difficult to track down. We were never 100% certain that there were kernel bugs related to userfaultfd but the suspicion never went away. * Handling faults with userfaultfd was always slow and single-threaded. Only one thread could handle faults and traveling to user-space to handle faults is inherently slower than handling them all in the kernel. The single-threaded aspect in particular presented a significant scaling bottleneck for embeddings that want to run many wasm instances in parallel. * One of the major benefits of userfaultfd was lazy initialization of wasm linear memory which is also achieved with the copy-on-write initialization support we have right now. * One of the suspected benefits of userfaultfd was less frobbing of the kernel vma structures when wasm modules are instantiated. Currently the copy-on-write support has a mitigation where we attempt to reuse the memory images where possible to avoid changing vma structures. When comparing this to userfaultfd's performance it was found that kernel modifications of vmas aren't a worrisome bottleneck so copy-on-write is suitable for this as well. Overall there are no remaining benefits that userfaultfd gives that copy-on-write doesn't, and copy-on-write solves a major downsides of userfaultfd, the scaling issue with a single faulting thread. Additionally copy-on-write support seems much more robust in terms of kernel implementation since it's only using standard memory-management syscalls which are heavily exercised. Finally copy-on-write support provides a new bonus where read-only memory in WebAssembly can be mapped directly to the same kernel cache page, even amongst many wasm instances of the same module, which was never possible with userfaultfd. In light of all this it's expected that all users of userfaultfd should migrate to the copy-on-write initialization of Wasmtime (which is enabled by default).
3 years ago
"cfg-if",
"crossbeam-epoch",
4 years ago
"crossbeam-utils",
]
[[package]]
name = "crossbeam-epoch"
version = "0.9.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "07db9d94cbd326813772c968ccd25999e5f8ae22f4f8d1b11effa37ef6ce281d"
dependencies = [
"autocfg 1.1.0",
Remove support for userfaultfd (#4040) This commit removes support for the `userfaultfd` or "uffd" syscall on Linux. This support was originally added for users migrating from Lucet to Wasmtime, but the recent developments of kernel-supported copy-on-write support for memory initialization wound up being more appropriate for these use cases than usefaultfd. The main reason for moving to copy-on-write initialization are: * The `userfaultfd` feature was never necessarily intended for this style of use case with wasm and was susceptible to subtle and rare bugs that were extremely difficult to track down. We were never 100% certain that there were kernel bugs related to userfaultfd but the suspicion never went away. * Handling faults with userfaultfd was always slow and single-threaded. Only one thread could handle faults and traveling to user-space to handle faults is inherently slower than handling them all in the kernel. The single-threaded aspect in particular presented a significant scaling bottleneck for embeddings that want to run many wasm instances in parallel. * One of the major benefits of userfaultfd was lazy initialization of wasm linear memory which is also achieved with the copy-on-write initialization support we have right now. * One of the suspected benefits of userfaultfd was less frobbing of the kernel vma structures when wasm modules are instantiated. Currently the copy-on-write support has a mitigation where we attempt to reuse the memory images where possible to avoid changing vma structures. When comparing this to userfaultfd's performance it was found that kernel modifications of vmas aren't a worrisome bottleneck so copy-on-write is suitable for this as well. Overall there are no remaining benefits that userfaultfd gives that copy-on-write doesn't, and copy-on-write solves a major downsides of userfaultfd, the scaling issue with a single faulting thread. Additionally copy-on-write support seems much more robust in terms of kernel implementation since it's only using standard memory-management syscalls which are heavily exercised. Finally copy-on-write support provides a new bonus where read-only memory in WebAssembly can be mapped directly to the same kernel cache page, even amongst many wasm instances of the same module, which was never possible with userfaultfd. In light of all this it's expected that all users of userfaultfd should migrate to the copy-on-write initialization of Wasmtime (which is enabled by default).
3 years ago
"cfg-if",
4 years ago
"crossbeam-utils",
"memoffset 0.6.5",
"once_cell",
"scopeguard",
]
[[package]]
name = "crossbeam-utils"
version = "0.8.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7d82ee10ce34d7bc12c2122495e7593a9c41347ecdd64185af4ecf72cb1a7f83"
dependencies = [
Remove support for userfaultfd (#4040) This commit removes support for the `userfaultfd` or "uffd" syscall on Linux. This support was originally added for users migrating from Lucet to Wasmtime, but the recent developments of kernel-supported copy-on-write support for memory initialization wound up being more appropriate for these use cases than usefaultfd. The main reason for moving to copy-on-write initialization are: * The `userfaultfd` feature was never necessarily intended for this style of use case with wasm and was susceptible to subtle and rare bugs that were extremely difficult to track down. We were never 100% certain that there were kernel bugs related to userfaultfd but the suspicion never went away. * Handling faults with userfaultfd was always slow and single-threaded. Only one thread could handle faults and traveling to user-space to handle faults is inherently slower than handling them all in the kernel. The single-threaded aspect in particular presented a significant scaling bottleneck for embeddings that want to run many wasm instances in parallel. * One of the major benefits of userfaultfd was lazy initialization of wasm linear memory which is also achieved with the copy-on-write initialization support we have right now. * One of the suspected benefits of userfaultfd was less frobbing of the kernel vma structures when wasm modules are instantiated. Currently the copy-on-write support has a mitigation where we attempt to reuse the memory images where possible to avoid changing vma structures. When comparing this to userfaultfd's performance it was found that kernel modifications of vmas aren't a worrisome bottleneck so copy-on-write is suitable for this as well. Overall there are no remaining benefits that userfaultfd gives that copy-on-write doesn't, and copy-on-write solves a major downsides of userfaultfd, the scaling issue with a single faulting thread. Additionally copy-on-write support seems much more robust in terms of kernel implementation since it's only using standard memory-management syscalls which are heavily exercised. Finally copy-on-write support provides a new bonus where read-only memory in WebAssembly can be mapped directly to the same kernel cache page, even amongst many wasm instances of the same module, which was never possible with userfaultfd. In light of all this it's expected that all users of userfaultfd should migrate to the copy-on-write initialization of Wasmtime (which is enabled by default).
3 years ago
"cfg-if",
"once_cell",
]
[[package]]
name = "crypto-bigint"
version = "0.2.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f83bd3bb4314701c568e340cd8cf78c975aa0ca79e03d3f6d1677d5b0c9c0c03"
dependencies = [
"generic-array",
"rand_core 0.6.3",
"subtle",
"zeroize",
]
[[package]]
name = "crypto-common"
version = "0.1.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3"
dependencies = [
"generic-array",
"typenum",
]
[[package]]
name = "crypto-mac"
version = "0.11.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b1d1a86f49236c215f271d40892d5fc950490551400b02ef360692c29815c714"
dependencies = [
"generic-array",
"subtle",
]
[[package]]
name = "ctr"
version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "049bb91fb4aaf0e3c7efa6cd5ef877dbbbd15b39dad06d9948de4ec8a75761ea"
dependencies = [
"cipher",
]
[[package]]
name = "cty"
version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b365fabc795046672053e29c954733ec3b05e4be654ab130fe8f1f94d7051f35"
[[package]]
name = "curve25519-dalek"
version = "3.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0b9fdf9972b2bd6af2d913799d9ebc165ea4d2e65878e329d9c6b372c4491b61"
dependencies = [
"byteorder",
"digest 0.9.0",
"rand_core 0.5.1",
"subtle",
"zeroize",
]
[[package]]
name = "der"
version = "0.4.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "79b71cca7d95d7681a4b3b9cdf63c8dbc3730d0584c2c74e31416d64a90493f4"
dependencies = [
"const-oid",
"crypto-bigint",
]
[[package]]
name = "derivative"
version = "2.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fcc3dd5e9e9c0b295d6e1e4d811fb6f157d5ffd784b8d202fc62eac8035a770b"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "derive_arbitrary"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "98e23c06c035dac87bd802d98f368df73a7f2cb05a66ffbd1f377e821fac4af9"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "digest"
version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066"
dependencies = [
"generic-array",
]
[[package]]
name = "digest"
version = "0.10.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f2fb860ca6fafa5552fb6d0e816a69c8e49f0908bf524e30a90d97c85892d506"
dependencies = [
"block-buffer 0.10.2",
"crypto-common",
]
[[package]]
name = "directories-next"
version = "2.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "339ee130d97a610ea5a5872d2bbb130fdf68884ff09d3028b81bec8a1ac23bbc"
dependencies = [
Remove support for userfaultfd (#4040) This commit removes support for the `userfaultfd` or "uffd" syscall on Linux. This support was originally added for users migrating from Lucet to Wasmtime, but the recent developments of kernel-supported copy-on-write support for memory initialization wound up being more appropriate for these use cases than usefaultfd. The main reason for moving to copy-on-write initialization are: * The `userfaultfd` feature was never necessarily intended for this style of use case with wasm and was susceptible to subtle and rare bugs that were extremely difficult to track down. We were never 100% certain that there were kernel bugs related to userfaultfd but the suspicion never went away. * Handling faults with userfaultfd was always slow and single-threaded. Only one thread could handle faults and traveling to user-space to handle faults is inherently slower than handling them all in the kernel. The single-threaded aspect in particular presented a significant scaling bottleneck for embeddings that want to run many wasm instances in parallel. * One of the major benefits of userfaultfd was lazy initialization of wasm linear memory which is also achieved with the copy-on-write initialization support we have right now. * One of the suspected benefits of userfaultfd was less frobbing of the kernel vma structures when wasm modules are instantiated. Currently the copy-on-write support has a mitigation where we attempt to reuse the memory images where possible to avoid changing vma structures. When comparing this to userfaultfd's performance it was found that kernel modifications of vmas aren't a worrisome bottleneck so copy-on-write is suitable for this as well. Overall there are no remaining benefits that userfaultfd gives that copy-on-write doesn't, and copy-on-write solves a major downsides of userfaultfd, the scaling issue with a single faulting thread. Additionally copy-on-write support seems much more robust in terms of kernel implementation since it's only using standard memory-management syscalls which are heavily exercised. Finally copy-on-write support provides a new bonus where read-only memory in WebAssembly can be mapped directly to the same kernel cache page, even amongst many wasm instances of the same module, which was never possible with userfaultfd. In light of all this it's expected that all users of userfaultfd should migrate to the copy-on-write initialization of Wasmtime (which is enabled by default).
3 years ago
"cfg-if",
"dirs-sys-next",
]
[[package]]
name = "dirs-next"
version = "2.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b98cf8ebf19c3d1b223e151f99a4f9f0690dca41414773390fc824184ac833e1"
dependencies = [
Remove support for userfaultfd (#4040) This commit removes support for the `userfaultfd` or "uffd" syscall on Linux. This support was originally added for users migrating from Lucet to Wasmtime, but the recent developments of kernel-supported copy-on-write support for memory initialization wound up being more appropriate for these use cases than usefaultfd. The main reason for moving to copy-on-write initialization are: * The `userfaultfd` feature was never necessarily intended for this style of use case with wasm and was susceptible to subtle and rare bugs that were extremely difficult to track down. We were never 100% certain that there were kernel bugs related to userfaultfd but the suspicion never went away. * Handling faults with userfaultfd was always slow and single-threaded. Only one thread could handle faults and traveling to user-space to handle faults is inherently slower than handling them all in the kernel. The single-threaded aspect in particular presented a significant scaling bottleneck for embeddings that want to run many wasm instances in parallel. * One of the major benefits of userfaultfd was lazy initialization of wasm linear memory which is also achieved with the copy-on-write initialization support we have right now. * One of the suspected benefits of userfaultfd was less frobbing of the kernel vma structures when wasm modules are instantiated. Currently the copy-on-write support has a mitigation where we attempt to reuse the memory images where possible to avoid changing vma structures. When comparing this to userfaultfd's performance it was found that kernel modifications of vmas aren't a worrisome bottleneck so copy-on-write is suitable for this as well. Overall there are no remaining benefits that userfaultfd gives that copy-on-write doesn't, and copy-on-write solves a major downsides of userfaultfd, the scaling issue with a single faulting thread. Additionally copy-on-write support seems much more robust in terms of kernel implementation since it's only using standard memory-management syscalls which are heavily exercised. Finally copy-on-write support provides a new bonus where read-only memory in WebAssembly can be mapped directly to the same kernel cache page, even amongst many wasm instances of the same module, which was never possible with userfaultfd. In light of all this it's expected that all users of userfaultfd should migrate to the copy-on-write initialization of Wasmtime (which is enabled by default).
3 years ago
"cfg-if",
"dirs-sys-next",
]
[[package]]
name = "dirs-sys-next"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4ebda144c4fe02d1f7ea1a7d9641b6fc6b580adcfa024ae48797ecdeb6825b4d"
dependencies = [
"libc",
"redox_users",
"winapi",
]
[[package]]
name = "downcast-rs"
version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9ea835d29036a4087793836fa931b08837ad5e957da9e23886b29586fb9b6650"
[[package]]
name = "dunce"
version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "453440c271cf5577fd2a40e4942540cb7d0d2f85e27c8d07dd0023c925a67541"
[[package]]
name = "ecdsa"
version = "0.12.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "43ee23aa5b4f68c7a092b5c3beb25f50c406adc75e2363634f242f28ab255372"
dependencies = [
"der",
"elliptic-curve",
"hmac",
"signature",
]
[[package]]
name = "ed25519"
version = "1.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3d5c4b5e5959dc2c2b89918d8e2cc40fcdd623cef026ed09d2f0ee05199dc8e4"
dependencies = [
"signature",
]
[[package]]
name = "ed25519-dalek"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c762bae6dcaf24c4c84667b8579785430908723d5c889f469d76a41d59cc7a9d"
dependencies = [
"curve25519-dalek",
"ed25519",
"rand 0.7.3",
"serde",
"sha2 0.9.9",
"zeroize",
]
[[package]]
name = "egg"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "05a6c0bbc92278f84e742f08c0ab9cb16a987376cd2bc39d228ef9c74d98d6f7"
dependencies = [
"indexmap",
"instant",
"log",
"once_cell",
"smallvec",
"symbolic_expressions",
]
[[package]]
name = "either"
version = "1.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457"
[[package]]
name = "elliptic-curve"
version = "0.10.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "beca177dcb8eb540133e7680baff45e7cc4d93bf22002676cec549f82343721b"
dependencies = [
"crypto-bigint",
"ff",
"generic-array",
"group",
"pkcs8",
"rand_core 0.6.3",
"subtle",
"zeroize",
]
[[package]]
name = "encode_unicode"
version = "0.3.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f"
Implement strings in adapter modules (#4623) * Implement strings in adapter modules This commit is a hefty addition to Wasmtime's support for the component model. This implements the final remaining type (in the current type hierarchy) unimplemented in adapter module trampolines: strings. Strings are the most complicated type to implement in adapter trampolines because they are highly structured chunks of data in memory (according to specific encodings). Additionally each lift/lower operation can choose its own encoding for strings meaning that Wasmtime, the host, may have to convert between any pairwise ordering of string encodings. The `CanonicalABI.md` in the component-model repo in general specifies all the fiddly bits of string encoding so there's not a ton of wiggle room for Wasmtime to get creative. This PR largely "just" implements that. The high-level architecture of this implementation is: * Fused adapters are first identified to determine src/dst string encodings. This statically fixes what transcoding operation is being performed. * The generated adapter will be responsible for managing calls to `realloc` and performing bounds checks. The adapter itself does not perform memory copies or validation of string contents, however. Instead each transcoding operation is modeled as an imported function into the adapter module. This means that the adapter module dynamically, during compile time, determines what string transcoders are needed. Note that an imported transcoder is not only parameterized over the transcoding operation but additionally which memory is the source and which is the destination. * The imported core wasm functions are modeled as a new `CoreDef::Transcoder` structure. These transcoders end up being small Cranelift-compiled trampolines. The Cranelift-compiled trampoline will load the actual base pointer of memory and add it to the relative pointers passed as function arguments. This trampoline then calls a transcoder "libcall" which enters Rust-defined functions for actual transcoding operations. * Each possible transcoding operation is implemented in Rust with a unique name and a unique signature depending on the needs of the transcoder. I've tried to document inline what each transcoder does. This means that the `Module::translate_string` in adapter modules is by far the largest translation method. The main reason for this is due to the management around calling the imported transcoder functions in the face of validating string pointer/lengths and performing the dance of `realloc`-vs-transcode at the right time. I've tried to ensure that each individual case in transcoding is documented well enough to understand what's going on as well. Additionally in this PR is a full implementation in the host for the `latin1+utf16` encoding which means that both lifting and lowering host strings now works with this encoding. Currently the implementation of each transcoder function is likely far from optimal. Where possible I've leaned on the standard library itself and for latin1-related things I'm leaning on the `encoding_rs` crate. I initially tried to implement everything with `encoding_rs` but was unable to uniformly do so easily. For now I settled on trying to get a known-correct (even in the face of endianness) implementation for all of these transcoders. If an when performance becomes an issue it should be possible to implement more optimized versions of each of these transcoding operations. Testing this commit has been somewhat difficult and my general plan, like with the `(list T)` type, is to rely heavily on fuzzing to cover the various cases here. In this PR though I've added a simple test that pushes some statically known strings through all the pairs of encodings between source and destination. I've attempted to pick "interesting" strings that one way or another stress the various paths in each transcoding operation to ideally get full branch coverage there. Additionally a suite of "negative" tests have also been added to ensure that validity of encoding is actually checked. * Fix a temporarily commented out case * Fix wasmtime-runtime tests * Update deny.toml configuration * Add `BSD-3-Clause` for the `encoding_rs` crate * Remove some unused licenses * Add an exemption for `encoding_rs` for now * Split up the `translate_string` method Move out all the closures and package up captured state into smaller lists of arguments. * Test out-of-bounds for zero-length strings
2 years ago
[[package]]
name = "encoding_rs"
version = "0.8.31"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9852635589dc9f9ea1b6fe9f05b50ef208c85c834a562f0c6abb1c475736ec2b"
dependencies = [
"cfg-if",
]
[[package]]
name = "env_logger"
version = "0.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "44533bbbb3bb3c1fa17d9f2e4e38bbbaf8396ba82193c4cb1b6445d711445d36"
dependencies = [
"atty",
"humantime 1.3.0",
"log",
"regex",
"termcolor",
]
[[package]]
name = "env_logger"
version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0b2cf0344971ee6c64c31be0d530793fba457d322dfec2810c453d0ef228f9c3"
dependencies = [
"atty",
"humantime 2.1.0",
"log",
"regex",
"termcolor",
]
[[package]]
name = "errno"
version = "0.2.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f639046355ee4f37944e44f60642c6f3a7efa3cf6b78c78a0d989a8ce6c396a1"
dependencies = [
"errno-dragonfly",
"libc",
"winapi",
]
[[package]]
name = "errno-dragonfly"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf"
dependencies = [
"cc",
"libc",
]
[[package]]
name = "example-fib-debug-wasm"
version = "0.0.0"
[[package]]
name = "example-tokio-wasm"
version = "0.0.0"
[[package]]
name = "example-wasi-wasm"
version = "0.0.0"
[[package]]
name = "fallible-iterator"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7"
[[package]]
name = "fastrand"
version = "1.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c3fcf0cee53519c866c09b5de1f6c56ff9d647101f81c1964fa632e148896cdf"
dependencies = [
"instant",
]
[[package]]
name = "fd-lock"
version = "3.0.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8ef1a30ae415c3a691a4f41afddc2dbcd6d70baf338368d85ebc1e8ed92cedb9"
dependencies = [
"cfg-if",
"rustix",
"windows-sys",
]
[[package]]
name = "ff"
version = "0.10.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d0f40b2dcd8bc322217a5f6559ae5f9e9d1de202a2ecee2e9eafcbece7562a4f"
dependencies = [
"rand_core 0.6.3",
"subtle",
]
[[package]]
name = "file-per-thread-logger"
version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "21e16290574b39ee41c71aeb90ae960c504ebaf1e2a1c87bd52aa56ed6e1a02f"
dependencies = [
"env_logger 0.9.0",
"log",
]
[[package]]
name = "filecheck"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2fe00b427b7c4835f8b82170eb7b9a63634376b63d73b9a9093367e82570bbaa"
dependencies = [
"regex",
"thiserror",
]
[[package]]
name = "filetime"
version = "0.2.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c0408e2626025178a6a7f7ffc05a25bc47103229f19c113755de7bf63816290c"
dependencies = [
Remove support for userfaultfd (#4040) This commit removes support for the `userfaultfd` or "uffd" syscall on Linux. This support was originally added for users migrating from Lucet to Wasmtime, but the recent developments of kernel-supported copy-on-write support for memory initialization wound up being more appropriate for these use cases than usefaultfd. The main reason for moving to copy-on-write initialization are: * The `userfaultfd` feature was never necessarily intended for this style of use case with wasm and was susceptible to subtle and rare bugs that were extremely difficult to track down. We were never 100% certain that there were kernel bugs related to userfaultfd but the suspicion never went away. * Handling faults with userfaultfd was always slow and single-threaded. Only one thread could handle faults and traveling to user-space to handle faults is inherently slower than handling them all in the kernel. The single-threaded aspect in particular presented a significant scaling bottleneck for embeddings that want to run many wasm instances in parallel. * One of the major benefits of userfaultfd was lazy initialization of wasm linear memory which is also achieved with the copy-on-write initialization support we have right now. * One of the suspected benefits of userfaultfd was less frobbing of the kernel vma structures when wasm modules are instantiated. Currently the copy-on-write support has a mitigation where we attempt to reuse the memory images where possible to avoid changing vma structures. When comparing this to userfaultfd's performance it was found that kernel modifications of vmas aren't a worrisome bottleneck so copy-on-write is suitable for this as well. Overall there are no remaining benefits that userfaultfd gives that copy-on-write doesn't, and copy-on-write solves a major downsides of userfaultfd, the scaling issue with a single faulting thread. Additionally copy-on-write support seems much more robust in terms of kernel implementation since it's only using standard memory-management syscalls which are heavily exercised. Finally copy-on-write support provides a new bonus where read-only memory in WebAssembly can be mapped directly to the same kernel cache page, even amongst many wasm instances of the same module, which was never possible with userfaultfd. In light of all this it's expected that all users of userfaultfd should migrate to the copy-on-write initialization of Wasmtime (which is enabled by default).
3 years ago
"cfg-if",
"libc",
"redox_syscall",
"winapi",
]
[[package]]
name = "flagset"
version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cda653ca797810c02f7ca4b804b40b8b95ae046eb989d356bce17919a8c25499"
[[package]]
name = "fnv"
version = "1.0.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
[[package]]
name = "form_urlencoded"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a9c384f161156f5260c24a097c56119f9be8c798586aecc13afbcbe7b7e26bf8"
dependencies = [
"percent-encoding",
]
[[package]]
name = "fs-set-times"
version = "0.18.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "857cf27edcb26c2a36d84b2954019573d335bb289876113aceacacdca47a4fd4"
dependencies = [
"io-lifetimes",
"rustix",
"windows-sys",
]
Add differential fuzzing against V8 (#3264) * Add differential fuzzing against V8 This commit adds a differential fuzzing target to Wasmtime along the lines of the wasmi and spec interpreters we already have, but with V8 instead. The intention here is that wasmi is unlikely to receive updates over time (e.g. for SIMD), and the spec interpreter is not suitable for fuzzing against in general due to its performance characteristics. The hope is that V8 is indeed appropriate to fuzz against because it's naturally receiving updates and it also is expected to have good performance. Here the `rusty_v8` crate is used which provides bindings to V8 as well as precompiled binaries by default. This matches exactly the use case we need and at least for now I think the `rusty_v8` crate will be maintained by the Deno folks as they continue to develop it. If it becomes an issue though maintaining we can evaluate other options to have differential fuzzing against. For now this commit enables the SIMD and bulk-memory feature of fuzz-target-generation which should enable them to get differentially-fuzzed with V8 in addition to the compilation fuzzing we're already getting. * Use weak linkage for GDB jit helpers This should help us deduplicate our symbol with other JIT runtimes, if any. For now this leans on some C helpers to define the weak linkage since Rust doesn't support that on stable yet. * Don't use rusty_v8 on MinGW They don't have precompiled libraries there. * Fix msvc build * Comment about execution
3 years ago
[[package]]
name = "fslock"
version = "0.1.8"
Add differential fuzzing against V8 (#3264) * Add differential fuzzing against V8 This commit adds a differential fuzzing target to Wasmtime along the lines of the wasmi and spec interpreters we already have, but with V8 instead. The intention here is that wasmi is unlikely to receive updates over time (e.g. for SIMD), and the spec interpreter is not suitable for fuzzing against in general due to its performance characteristics. The hope is that V8 is indeed appropriate to fuzz against because it's naturally receiving updates and it also is expected to have good performance. Here the `rusty_v8` crate is used which provides bindings to V8 as well as precompiled binaries by default. This matches exactly the use case we need and at least for now I think the `rusty_v8` crate will be maintained by the Deno folks as they continue to develop it. If it becomes an issue though maintaining we can evaluate other options to have differential fuzzing against. For now this commit enables the SIMD and bulk-memory feature of fuzz-target-generation which should enable them to get differentially-fuzzed with V8 in addition to the compilation fuzzing we're already getting. * Use weak linkage for GDB jit helpers This should help us deduplicate our symbol with other JIT runtimes, if any. For now this leans on some C helpers to define the weak linkage since Rust doesn't support that on stable yet. * Don't use rusty_v8 on MinGW They don't have precompiled libraries there. * Fix msvc build * Comment about execution
3 years ago
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "57eafdd0c16f57161105ae1b98a1238f97645f2f588438b2949c99a2af9616bf"
Add differential fuzzing against V8 (#3264) * Add differential fuzzing against V8 This commit adds a differential fuzzing target to Wasmtime along the lines of the wasmi and spec interpreters we already have, but with V8 instead. The intention here is that wasmi is unlikely to receive updates over time (e.g. for SIMD), and the spec interpreter is not suitable for fuzzing against in general due to its performance characteristics. The hope is that V8 is indeed appropriate to fuzz against because it's naturally receiving updates and it also is expected to have good performance. Here the `rusty_v8` crate is used which provides bindings to V8 as well as precompiled binaries by default. This matches exactly the use case we need and at least for now I think the `rusty_v8` crate will be maintained by the Deno folks as they continue to develop it. If it becomes an issue though maintaining we can evaluate other options to have differential fuzzing against. For now this commit enables the SIMD and bulk-memory feature of fuzz-target-generation which should enable them to get differentially-fuzzed with V8 in addition to the compilation fuzzing we're already getting. * Use weak linkage for GDB jit helpers This should help us deduplicate our symbol with other JIT runtimes, if any. For now this leans on some C helpers to define the weak linkage since Rust doesn't support that on stable yet. * Don't use rusty_v8 on MinGW They don't have precompiled libraries there. * Fix msvc build * Comment about execution
3 years ago
dependencies = [
"libc",
"winapi",
]
[[package]]
name = "fxhash"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c"
dependencies = [
"byteorder",
]
[[package]]
name = "generic-array"
version = "0.14.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fd48d33ec7f05fbfa152300fdad764757cbded343c1aa1cff2fbaf4134851803"
dependencies = [
"typenum",
"version_check",
]
[[package]]
name = "getrandom"
version = "0.1.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce"
dependencies = [
Remove support for userfaultfd (#4040) This commit removes support for the `userfaultfd` or "uffd" syscall on Linux. This support was originally added for users migrating from Lucet to Wasmtime, but the recent developments of kernel-supported copy-on-write support for memory initialization wound up being more appropriate for these use cases than usefaultfd. The main reason for moving to copy-on-write initialization are: * The `userfaultfd` feature was never necessarily intended for this style of use case with wasm and was susceptible to subtle and rare bugs that were extremely difficult to track down. We were never 100% certain that there were kernel bugs related to userfaultfd but the suspicion never went away. * Handling faults with userfaultfd was always slow and single-threaded. Only one thread could handle faults and traveling to user-space to handle faults is inherently slower than handling them all in the kernel. The single-threaded aspect in particular presented a significant scaling bottleneck for embeddings that want to run many wasm instances in parallel. * One of the major benefits of userfaultfd was lazy initialization of wasm linear memory which is also achieved with the copy-on-write initialization support we have right now. * One of the suspected benefits of userfaultfd was less frobbing of the kernel vma structures when wasm modules are instantiated. Currently the copy-on-write support has a mitigation where we attempt to reuse the memory images where possible to avoid changing vma structures. When comparing this to userfaultfd's performance it was found that kernel modifications of vmas aren't a worrisome bottleneck so copy-on-write is suitable for this as well. Overall there are no remaining benefits that userfaultfd gives that copy-on-write doesn't, and copy-on-write solves a major downsides of userfaultfd, the scaling issue with a single faulting thread. Additionally copy-on-write support seems much more robust in terms of kernel implementation since it's only using standard memory-management syscalls which are heavily exercised. Finally copy-on-write support provides a new bonus where read-only memory in WebAssembly can be mapped directly to the same kernel cache page, even amongst many wasm instances of the same module, which was never possible with userfaultfd. In light of all this it's expected that all users of userfaultfd should migrate to the copy-on-write initialization of Wasmtime (which is enabled by default).
3 years ago
"cfg-if",
"libc",
"wasi 0.9.0+wasi-snapshot-preview1",
]
[[package]]
name = "getrandom"
version = "0.2.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9be70c98951c83b8d2f8f60d7065fa6d5146873094452a1008da8c2f1e4205ad"
dependencies = [
Remove support for userfaultfd (#4040) This commit removes support for the `userfaultfd` or "uffd" syscall on Linux. This support was originally added for users migrating from Lucet to Wasmtime, but the recent developments of kernel-supported copy-on-write support for memory initialization wound up being more appropriate for these use cases than usefaultfd. The main reason for moving to copy-on-write initialization are: * The `userfaultfd` feature was never necessarily intended for this style of use case with wasm and was susceptible to subtle and rare bugs that were extremely difficult to track down. We were never 100% certain that there were kernel bugs related to userfaultfd but the suspicion never went away. * Handling faults with userfaultfd was always slow and single-threaded. Only one thread could handle faults and traveling to user-space to handle faults is inherently slower than handling them all in the kernel. The single-threaded aspect in particular presented a significant scaling bottleneck for embeddings that want to run many wasm instances in parallel. * One of the major benefits of userfaultfd was lazy initialization of wasm linear memory which is also achieved with the copy-on-write initialization support we have right now. * One of the suspected benefits of userfaultfd was less frobbing of the kernel vma structures when wasm modules are instantiated. Currently the copy-on-write support has a mitigation where we attempt to reuse the memory images where possible to avoid changing vma structures. When comparing this to userfaultfd's performance it was found that kernel modifications of vmas aren't a worrisome bottleneck so copy-on-write is suitable for this as well. Overall there are no remaining benefits that userfaultfd gives that copy-on-write doesn't, and copy-on-write solves a major downsides of userfaultfd, the scaling issue with a single faulting thread. Additionally copy-on-write support seems much more robust in terms of kernel implementation since it's only using standard memory-management syscalls which are heavily exercised. Finally copy-on-write support provides a new bonus where read-only memory in WebAssembly can be mapped directly to the same kernel cache page, even amongst many wasm instances of the same module, which was never possible with userfaultfd. In light of all this it's expected that all users of userfaultfd should migrate to the copy-on-write initialization of Wasmtime (which is enabled by default).
3 years ago
"cfg-if",
"libc",
"wasi 0.10.2+wasi-snapshot-preview1",
]
[[package]]
name = "ghash"
version = "0.4.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1583cc1656d7839fd3732b80cf4f38850336cdb9b8ded1cd399ca62958de3c99"
dependencies = [
"opaque-debug",
"polyval",
]
[[package]]
name = "gimli"
version = "0.27.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dec7af912d60cdbd3677c1af9352ebae6fb8394d165568a2234df0fa00f87793"
dependencies = [
"fallible-iterator",
"indexmap",
"stable_deref_trait",
]
[[package]]
name = "glob"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574"
[[package]]
name = "group"
version = "0.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1c363a5301b8f153d80747126a04b3c82073b9fe3130571a9d170cacdeaf7912"
dependencies = [
"ff",
"rand_core 0.6.3",
"subtle",
]
[[package]]
name = "half"
version = "1.8.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "eabb4a44450da02c90444cf74558da904edde8fb4e9035a9a6a4e15445af0bd7"
[[package]]
name = "hashbrown"
version = "0.12.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888"
[[package]]
name = "hashbrown"
version = "0.13.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "33ff8ae62cd3a9102e5637afc8452c55acf3844001bd5374e0b0bd7b6616c038"
dependencies = [
"ahash",
]
[[package]]
name = "heck"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2540771e65fc8cb83cd6e8a237f70c319bd5c29f78ed1084ba5d50eeac86f7f9"
[[package]]
name = "hermit-abi"
version = "0.1.19"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33"
dependencies = [
"libc",
]
[[package]]
name = "hermit-abi"
version = "0.2.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ee512640fe35acbfb4bb779db6f0d80704c2cacfa2e39b601ef3e3f47d1ae4c7"
dependencies = [
"libc",
]
[[package]]
name = "hermit-abi"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "856b5cb0902c2b6d65d5fd97dfa30f9b70c7538e770b98eab5ed52d8db923e01"
[[package]]
name = "hkdf"
version = "0.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "01706d578d5c281058480e673ae4086a9f4710d8df1ad80a5b03e39ece5f886b"
dependencies = [
"digest 0.9.0",
"hmac",
]
[[package]]
name = "hmac"
version = "0.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2a2a2320eb7ec0ebe8da8f744d7812d9fc4cb4d09344ac01898dbcb6a20ae69b"
dependencies = [
"crypto-mac",
"digest 0.9.0",
]
[[package]]
name = "humantime"
version = "1.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "df004cfca50ef23c36850aaaa59ad52cc70d0e90243c3c7737a4dd32dc7a3c4f"
dependencies = [
"quick-error 1.2.3",
]
[[package]]
name = "humantime"
version = "2.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4"
[[package]]
name = "id-arena"
version = "2.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "25a2bc672d1148e28034f176e01fffebb08b35768468cc954630da77a1449005"
[[package]]
name = "idna"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e14ddfc70884202db2244c223200c204c2bda1bc6e0998d11b5e024d657209e6"
dependencies = [
"unicode-bidi",
"unicode-normalization",
]
[[package]]
name = "indexmap"
version = "1.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "10a35a97730320ffe8e2d410b5d3b69279b98d2c14bdb8b70ea89ecf7888d41e"
dependencies = [
"autocfg 1.1.0",
"hashbrown 0.12.3",
"serde",
]
[[package]]
name = "indexmap-nostd"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8e04e2fd2b8188ea827b32ef11de88377086d690286ab35747ef7f9bf3ccb590"
[[package]]
name = "indicatif"
version = "0.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8572bccfb0665e70b7faf44ee28841b8e0823450cd4ad562a76b5a3c4bf48487"
dependencies = [
"console",
"lazy_static",
"number_prefix",
"regex",
]
[[package]]
name = "instant"
version = "0.1.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c"
dependencies = [
Remove support for userfaultfd (#4040) This commit removes support for the `userfaultfd` or "uffd" syscall on Linux. This support was originally added for users migrating from Lucet to Wasmtime, but the recent developments of kernel-supported copy-on-write support for memory initialization wound up being more appropriate for these use cases than usefaultfd. The main reason for moving to copy-on-write initialization are: * The `userfaultfd` feature was never necessarily intended for this style of use case with wasm and was susceptible to subtle and rare bugs that were extremely difficult to track down. We were never 100% certain that there were kernel bugs related to userfaultfd but the suspicion never went away. * Handling faults with userfaultfd was always slow and single-threaded. Only one thread could handle faults and traveling to user-space to handle faults is inherently slower than handling them all in the kernel. The single-threaded aspect in particular presented a significant scaling bottleneck for embeddings that want to run many wasm instances in parallel. * One of the major benefits of userfaultfd was lazy initialization of wasm linear memory which is also achieved with the copy-on-write initialization support we have right now. * One of the suspected benefits of userfaultfd was less frobbing of the kernel vma structures when wasm modules are instantiated. Currently the copy-on-write support has a mitigation where we attempt to reuse the memory images where possible to avoid changing vma structures. When comparing this to userfaultfd's performance it was found that kernel modifications of vmas aren't a worrisome bottleneck so copy-on-write is suitable for this as well. Overall there are no remaining benefits that userfaultfd gives that copy-on-write doesn't, and copy-on-write solves a major downsides of userfaultfd, the scaling issue with a single faulting thread. Additionally copy-on-write support seems much more robust in terms of kernel implementation since it's only using standard memory-management syscalls which are heavily exercised. Finally copy-on-write support provides a new bonus where read-only memory in WebAssembly can be mapped directly to the same kernel cache page, even amongst many wasm instances of the same module, which was never possible with userfaultfd. In light of all this it's expected that all users of userfaultfd should migrate to the copy-on-write initialization of Wasmtime (which is enabled by default).
3 years ago
"cfg-if",
]
[[package]]
name = "io-extras"
version = "0.17.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d79107d6e60d78351e11f0a2dc9d0eaf304a7efb592e92603783afb8479c7d97"
dependencies = [
"io-lifetimes",
"windows-sys",
]
[[package]]
name = "io-lifetimes"
version = "1.0.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1abeb7a0dd0f8181267ff8adc397075586500b81b28a73e8a0208b00fc170fb3"
dependencies = [
"libc",
"windows-sys",
]
[[package]]
name = "ipnet"
version = "2.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "879d54834c8c76457ef4293a689b2a8c59b076067ad77b15efafbb05f92a592b"
[[package]]
name = "is-terminal"
version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "22e18b0a45d56fe973d6db23972bf5bc46f988a4a2385deac9cc29572f09daef"
dependencies = [
"hermit-abi 0.3.0",
"io-lifetimes",
"rustix",
"windows-sys",
]
[[package]]
name = "isle-fuzz"
version = "0.0.0"
dependencies = [
"cranelift-isle",
"env_logger 0.9.0",
"libfuzzer-sys",
"log",
]
[[package]]
name = "islec"
version = "0.0.0"
dependencies = [
"clap",
"cranelift-isle",
"env_logger 0.9.0",
]
[[package]]
name = "itertools"
version = "0.10.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a9a9d19fa1e79b6215ff29b9d6880b706147f16e9b1dbb1e4e5947b5b02bc5e3"
dependencies = [
"either",
]
[[package]]
name = "itoa"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1aab8fc367588b89dcee83ab0fd66b72b50b72fa1904d7095045ace2b0c81c35"
[[package]]
name = "ittapi"
version = "0.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2e648c437172ce7d3ac35ca11a068755072054826fa455a916b43524fa4a62a7"
dependencies = [
"anyhow",
"ittapi-sys",
"log",
]
[[package]]
name = "ittapi-sys"
version = "0.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a9b32a4d23f72548178dde54f3c12c6b6a08598e25575c0d0fa5bd861e0dc1a5"
dependencies = [
"cc",
]
[[package]]
name = "jobserver"
version = "0.1.24"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "af25a77299a7f711a01975c35a6a424eb6862092cc2d6c72c4ed6cbc56dfc1fa"
dependencies = [
"libc",
]
[[package]]
name = "js-sys"
version = "0.3.57"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "671a26f820db17c2a2750743f1dd03bafd15b98c9f30c7c2628c024c05d73397"
dependencies = [
"wasm-bindgen",
]
[[package]]
name = "k256"
version = "0.9.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "903ae2481bcdfdb7b68e0a9baa4b7c9aff600b9ae2e8e5bb5833b8c91ab851ea"
dependencies = [
Remove support for userfaultfd (#4040) This commit removes support for the `userfaultfd` or "uffd" syscall on Linux. This support was originally added for users migrating from Lucet to Wasmtime, but the recent developments of kernel-supported copy-on-write support for memory initialization wound up being more appropriate for these use cases than usefaultfd. The main reason for moving to copy-on-write initialization are: * The `userfaultfd` feature was never necessarily intended for this style of use case with wasm and was susceptible to subtle and rare bugs that were extremely difficult to track down. We were never 100% certain that there were kernel bugs related to userfaultfd but the suspicion never went away. * Handling faults with userfaultfd was always slow and single-threaded. Only one thread could handle faults and traveling to user-space to handle faults is inherently slower than handling them all in the kernel. The single-threaded aspect in particular presented a significant scaling bottleneck for embeddings that want to run many wasm instances in parallel. * One of the major benefits of userfaultfd was lazy initialization of wasm linear memory which is also achieved with the copy-on-write initialization support we have right now. * One of the suspected benefits of userfaultfd was less frobbing of the kernel vma structures when wasm modules are instantiated. Currently the copy-on-write support has a mitigation where we attempt to reuse the memory images where possible to avoid changing vma structures. When comparing this to userfaultfd's performance it was found that kernel modifications of vmas aren't a worrisome bottleneck so copy-on-write is suitable for this as well. Overall there are no remaining benefits that userfaultfd gives that copy-on-write doesn't, and copy-on-write solves a major downsides of userfaultfd, the scaling issue with a single faulting thread. Additionally copy-on-write support seems much more robust in terms of kernel implementation since it's only using standard memory-management syscalls which are heavily exercised. Finally copy-on-write support provides a new bonus where read-only memory in WebAssembly can be mapped directly to the same kernel cache page, even amongst many wasm instances of the same module, which was never possible with userfaultfd. In light of all this it's expected that all users of userfaultfd should migrate to the copy-on-write initialization of Wasmtime (which is enabled by default).
3 years ago
"cfg-if",
"ecdsa",
"elliptic-curve",
"sha2 0.9.9",
]
[[package]]
name = "lazy_static"
version = "1.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
dependencies = [
"spin 0.5.2",
]
[[package]]
name = "leb128"
version = "0.2.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "884e2677b40cc8c339eaefcb701c32ef1fd2493d71118dc0ca4b6a736c93bd67"
[[package]]
name = "libc"
version = "0.2.133"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c0f80d65747a3e43d1596c7c5492d95d5edddaabd45a7fcdb02b95f644164966"
[[package]]
name = "libfuzzer-sys"
version = "0.4.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c8fff891139ee62800da71b7fd5b508d570b9ad95e614a53c6f453ca08366038"
dependencies = [
"arbitrary",
"cc",
"once_cell",
]
[[package]]
name = "libloading"
version = "0.7.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "efbc0f03f9a775e9f6aed295c6a1ba2253c5757a9e03d55c6caa46a681abcddd"
dependencies = [
Remove support for userfaultfd (#4040) This commit removes support for the `userfaultfd` or "uffd" syscall on Linux. This support was originally added for users migrating from Lucet to Wasmtime, but the recent developments of kernel-supported copy-on-write support for memory initialization wound up being more appropriate for these use cases than usefaultfd. The main reason for moving to copy-on-write initialization are: * The `userfaultfd` feature was never necessarily intended for this style of use case with wasm and was susceptible to subtle and rare bugs that were extremely difficult to track down. We were never 100% certain that there were kernel bugs related to userfaultfd but the suspicion never went away. * Handling faults with userfaultfd was always slow and single-threaded. Only one thread could handle faults and traveling to user-space to handle faults is inherently slower than handling them all in the kernel. The single-threaded aspect in particular presented a significant scaling bottleneck for embeddings that want to run many wasm instances in parallel. * One of the major benefits of userfaultfd was lazy initialization of wasm linear memory which is also achieved with the copy-on-write initialization support we have right now. * One of the suspected benefits of userfaultfd was less frobbing of the kernel vma structures when wasm modules are instantiated. Currently the copy-on-write support has a mitigation where we attempt to reuse the memory images where possible to avoid changing vma structures. When comparing this to userfaultfd's performance it was found that kernel modifications of vmas aren't a worrisome bottleneck so copy-on-write is suitable for this as well. Overall there are no remaining benefits that userfaultfd gives that copy-on-write doesn't, and copy-on-write solves a major downsides of userfaultfd, the scaling issue with a single faulting thread. Additionally copy-on-write support seems much more robust in terms of kernel implementation since it's only using standard memory-management syscalls which are heavily exercised. Finally copy-on-write support provides a new bonus where read-only memory in WebAssembly can be mapped directly to the same kernel cache page, even amongst many wasm instances of the same module, which was never possible with userfaultfd. In light of all this it's expected that all users of userfaultfd should migrate to the copy-on-write initialization of Wasmtime (which is enabled by default).
3 years ago
"cfg-if",
"winapi",
]
[[package]]
name = "libm"
version = "0.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c7ce35d4899fa3c0558d4f5082c98927789a01024270711cf113999b66ced65a"
[[package]]
name = "linux-raw-sys"
version = "0.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8f9f08d8963a6c613f4b1a78f4f4a4dbfadf8e6545b2d72861731e4858b8b47f"
[[package]]
name = "listenfd"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "14e4fcc00ff6731d94b70e16e71f43bda62883461f31230742e3bc6dddf12988"
dependencies = [
"libc",
"uuid",
"winapi",
]
[[package]]
name = "lock_api"
version = "0.4.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "327fa5b6a6940e4699ec49a9beae1ea4845c6bab9314e4f84ac68742139d8c53"
dependencies = [
"autocfg 1.1.0",
"scopeguard",
]
[[package]]
name = "log"
version = "0.4.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e"
dependencies = [
Remove support for userfaultfd (#4040) This commit removes support for the `userfaultfd` or "uffd" syscall on Linux. This support was originally added for users migrating from Lucet to Wasmtime, but the recent developments of kernel-supported copy-on-write support for memory initialization wound up being more appropriate for these use cases than usefaultfd. The main reason for moving to copy-on-write initialization are: * The `userfaultfd` feature was never necessarily intended for this style of use case with wasm and was susceptible to subtle and rare bugs that were extremely difficult to track down. We were never 100% certain that there were kernel bugs related to userfaultfd but the suspicion never went away. * Handling faults with userfaultfd was always slow and single-threaded. Only one thread could handle faults and traveling to user-space to handle faults is inherently slower than handling them all in the kernel. The single-threaded aspect in particular presented a significant scaling bottleneck for embeddings that want to run many wasm instances in parallel. * One of the major benefits of userfaultfd was lazy initialization of wasm linear memory which is also achieved with the copy-on-write initialization support we have right now. * One of the suspected benefits of userfaultfd was less frobbing of the kernel vma structures when wasm modules are instantiated. Currently the copy-on-write support has a mitigation where we attempt to reuse the memory images where possible to avoid changing vma structures. When comparing this to userfaultfd's performance it was found that kernel modifications of vmas aren't a worrisome bottleneck so copy-on-write is suitable for this as well. Overall there are no remaining benefits that userfaultfd gives that copy-on-write doesn't, and copy-on-write solves a major downsides of userfaultfd, the scaling issue with a single faulting thread. Additionally copy-on-write support seems much more robust in terms of kernel implementation since it's only using standard memory-management syscalls which are heavily exercised. Finally copy-on-write support provides a new bonus where read-only memory in WebAssembly can be mapped directly to the same kernel cache page, even amongst many wasm instances of the same module, which was never possible with userfaultfd. In light of all this it's expected that all users of userfaultfd should migrate to the copy-on-write initialization of Wasmtime (which is enabled by default).
3 years ago
"cfg-if",
]
[[package]]
name = "mach"
version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b823e83b2affd8f40a9ee8c29dbc56404c1e34cd2710921f2801e2cf29527afa"
dependencies = [
"libc",
]
[[package]]
name = "maybe-owned"
version = "0.3.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4facc753ae494aeb6e3c22f839b158aebd4f9270f55cd3c79906c45476c47ab4"
[[package]]
name = "memchr"
version = "2.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d"
Add a pooling allocator mode based on copy-on-write mappings of memfds. As first suggested by Jan on the Zulip here [1], a cheap and effective way to obtain copy-on-write semantics of a "backing image" for a Wasm memory is to mmap a file with `MAP_PRIVATE`. The `memfd` mechanism provided by the Linux kernel allows us to create anonymous, in-memory-only files that we can use for this mapping, so we can construct the image contents on-the-fly then effectively create a CoW overlay. Furthermore, and importantly, `madvise(MADV_DONTNEED, ...)` will discard the CoW overlay, returning the mapping to its original state. By itself this is almost enough for a very fast instantiation-termination loop of the same image over and over, without changing the address space mapping at all (which is expensive). The only missing bit is how to implement heap *growth*. But here memfds can help us again: if we create another anonymous file and map it where the extended parts of the heap would go, we can take advantage of the fact that a `mmap()` mapping can be *larger than the file itself*, with accesses beyond the end generating a `SIGBUS`, and the fact that we can cheaply resize the file with `ftruncate`, even after a mapping exists. So we can map the "heap extension" file once with the maximum memory-slot size and grow the memfd itself as `memory.grow` operations occur. The above CoW technique and heap-growth technique together allow us a fastpath of `madvise()` and `ftruncate()` only when we re-instantiate the same module over and over, as long as we can reuse the same slot. This fastpath avoids all whole-process address-space locks in the Linux kernel, which should mean it is highly scalable. It also avoids the cost of copying data on read, as the `uffd` heap backend does when servicing pagefaults; the kernel's own optimized CoW logic (same as used by all file mmaps) is used instead. [1] https://bytecodealliance.zulipchat.com/#narrow/stream/206238-general/topic/Copy.20on.20write.20based.20instance.20reuse/near/266657772
3 years ago
[[package]]
name = "memfd"
version = "0.6.2"
Add a pooling allocator mode based on copy-on-write mappings of memfds. As first suggested by Jan on the Zulip here [1], a cheap and effective way to obtain copy-on-write semantics of a "backing image" for a Wasm memory is to mmap a file with `MAP_PRIVATE`. The `memfd` mechanism provided by the Linux kernel allows us to create anonymous, in-memory-only files that we can use for this mapping, so we can construct the image contents on-the-fly then effectively create a CoW overlay. Furthermore, and importantly, `madvise(MADV_DONTNEED, ...)` will discard the CoW overlay, returning the mapping to its original state. By itself this is almost enough for a very fast instantiation-termination loop of the same image over and over, without changing the address space mapping at all (which is expensive). The only missing bit is how to implement heap *growth*. But here memfds can help us again: if we create another anonymous file and map it where the extended parts of the heap would go, we can take advantage of the fact that a `mmap()` mapping can be *larger than the file itself*, with accesses beyond the end generating a `SIGBUS`, and the fact that we can cheaply resize the file with `ftruncate`, even after a mapping exists. So we can map the "heap extension" file once with the maximum memory-slot size and grow the memfd itself as `memory.grow` operations occur. The above CoW technique and heap-growth technique together allow us a fastpath of `madvise()` and `ftruncate()` only when we re-instantiate the same module over and over, as long as we can reuse the same slot. This fastpath avoids all whole-process address-space locks in the Linux kernel, which should mean it is highly scalable. It also avoids the cost of copying data on read, as the `uffd` heap backend does when servicing pagefaults; the kernel's own optimized CoW logic (same as used by all file mmaps) is used instead. [1] https://bytecodealliance.zulipchat.com/#narrow/stream/206238-general/topic/Copy.20on.20write.20based.20instance.20reuse/near/266657772
3 years ago
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b20a59d985586e4a5aef64564ac77299f8586d8be6cf9106a5a40207e8908efb"
Add a pooling allocator mode based on copy-on-write mappings of memfds. As first suggested by Jan on the Zulip here [1], a cheap and effective way to obtain copy-on-write semantics of a "backing image" for a Wasm memory is to mmap a file with `MAP_PRIVATE`. The `memfd` mechanism provided by the Linux kernel allows us to create anonymous, in-memory-only files that we can use for this mapping, so we can construct the image contents on-the-fly then effectively create a CoW overlay. Furthermore, and importantly, `madvise(MADV_DONTNEED, ...)` will discard the CoW overlay, returning the mapping to its original state. By itself this is almost enough for a very fast instantiation-termination loop of the same image over and over, without changing the address space mapping at all (which is expensive). The only missing bit is how to implement heap *growth*. But here memfds can help us again: if we create another anonymous file and map it where the extended parts of the heap would go, we can take advantage of the fact that a `mmap()` mapping can be *larger than the file itself*, with accesses beyond the end generating a `SIGBUS`, and the fact that we can cheaply resize the file with `ftruncate`, even after a mapping exists. So we can map the "heap extension" file once with the maximum memory-slot size and grow the memfd itself as `memory.grow` operations occur. The above CoW technique and heap-growth technique together allow us a fastpath of `madvise()` and `ftruncate()` only when we re-instantiate the same module over and over, as long as we can reuse the same slot. This fastpath avoids all whole-process address-space locks in the Linux kernel, which should mean it is highly scalable. It also avoids the cost of copying data on read, as the `uffd` heap backend does when servicing pagefaults; the kernel's own optimized CoW logic (same as used by all file mmaps) is used instead. [1] https://bytecodealliance.zulipchat.com/#narrow/stream/206238-general/topic/Copy.20on.20write.20based.20instance.20reuse/near/266657772
3 years ago
dependencies = [
"rustix",
Add a pooling allocator mode based on copy-on-write mappings of memfds. As first suggested by Jan on the Zulip here [1], a cheap and effective way to obtain copy-on-write semantics of a "backing image" for a Wasm memory is to mmap a file with `MAP_PRIVATE`. The `memfd` mechanism provided by the Linux kernel allows us to create anonymous, in-memory-only files that we can use for this mapping, so we can construct the image contents on-the-fly then effectively create a CoW overlay. Furthermore, and importantly, `madvise(MADV_DONTNEED, ...)` will discard the CoW overlay, returning the mapping to its original state. By itself this is almost enough for a very fast instantiation-termination loop of the same image over and over, without changing the address space mapping at all (which is expensive). The only missing bit is how to implement heap *growth*. But here memfds can help us again: if we create another anonymous file and map it where the extended parts of the heap would go, we can take advantage of the fact that a `mmap()` mapping can be *larger than the file itself*, with accesses beyond the end generating a `SIGBUS`, and the fact that we can cheaply resize the file with `ftruncate`, even after a mapping exists. So we can map the "heap extension" file once with the maximum memory-slot size and grow the memfd itself as `memory.grow` operations occur. The above CoW technique and heap-growth technique together allow us a fastpath of `madvise()` and `ftruncate()` only when we re-instantiate the same module over and over, as long as we can reuse the same slot. This fastpath avoids all whole-process address-space locks in the Linux kernel, which should mean it is highly scalable. It also avoids the cost of copying data on read, as the `uffd` heap backend does when servicing pagefaults; the kernel's own optimized CoW logic (same as used by all file mmaps) is used instead. [1] https://bytecodealliance.zulipchat.com/#narrow/stream/206238-general/topic/Copy.20on.20write.20based.20instance.20reuse/near/266657772
3 years ago
]
[[package]]
name = "memmap2"
version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "723e3ebdcdc5c023db1df315364573789f8857c11b631a2fdfad7c00f5c046b4"
dependencies = [
"libc",
]
[[package]]
name = "memoffset"
version = "0.6.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5aa361d4faea93603064a027415f07bd8e1d5c88c9fbf68bf56a285428fd79ce"
dependencies = [
"autocfg 1.1.0",
]
[[package]]
name = "memoffset"
version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d61c719bcfbcf5d62b3a09efa6088de8c54bc0bfcd3ea7ae39fcc186108b8de1"
dependencies = [
"autocfg 1.1.0",
]
externref: implement stack map-based garbage collection For host VM code, we use plain reference counting, where cloning increments the reference count, and dropping decrements it. We can avoid many of the on-stack increment/decrement operations that typically plague the performance of reference counting via Rust's ownership and borrowing system. Moving a `VMExternRef` avoids mutating its reference count, and borrowing it either avoids the reference count increment or delays it until if/when the `VMExternRef` is cloned. When passing a `VMExternRef` into compiled Wasm code, we don't want to do reference count mutations for every compiled `local.{get,set}`, nor for every function call. Therefore, we use a variation of **deferred reference counting**, where we only mutate reference counts when storing `VMExternRef`s somewhere that outlives the activation: into a global or table. Simultaneously, we over-approximate the set of `VMExternRef`s that are inside Wasm function activations. Periodically, we walk the stack at GC safe points, and use stack map information to precisely identify the set of `VMExternRef`s inside Wasm activations. Then we take the difference between this precise set and our over-approximation, and decrement the reference count for each of the `VMExternRef`s that are in our over-approximation but not in the precise set. Finally, the over-approximation is replaced with the precise set. The `VMExternRefActivationsTable` implements the over-approximized set of `VMExternRef`s referenced by Wasm activations. Calling a Wasm function and passing it a `VMExternRef` moves the `VMExternRef` into the table, and the compiled Wasm function logically "borrows" the `VMExternRef` from the table. Similarly, `global.get` and `table.get` operations clone the gotten `VMExternRef` into the `VMExternRefActivationsTable` and then "borrow" the reference out of the table. When a `VMExternRef` is returned to host code from a Wasm function, the host increments the reference count (because the reference is logically "borrowed" from the `VMExternRefActivationsTable` and the reference count from the table will be dropped at the next GC). For more general information on deferred reference counting, see *An Examination of Deferred Reference Counting and Cycle Detection* by Quinane: https://openresearch-repository.anu.edu.au/bitstream/1885/42030/2/hon-thesis.pdf cc #929 Fixes #1804
4 years ago
[[package]]
name = "miniz_oxide"
version = "0.6.2"
externref: implement stack map-based garbage collection For host VM code, we use plain reference counting, where cloning increments the reference count, and dropping decrements it. We can avoid many of the on-stack increment/decrement operations that typically plague the performance of reference counting via Rust's ownership and borrowing system. Moving a `VMExternRef` avoids mutating its reference count, and borrowing it either avoids the reference count increment or delays it until if/when the `VMExternRef` is cloned. When passing a `VMExternRef` into compiled Wasm code, we don't want to do reference count mutations for every compiled `local.{get,set}`, nor for every function call. Therefore, we use a variation of **deferred reference counting**, where we only mutate reference counts when storing `VMExternRef`s somewhere that outlives the activation: into a global or table. Simultaneously, we over-approximate the set of `VMExternRef`s that are inside Wasm function activations. Periodically, we walk the stack at GC safe points, and use stack map information to precisely identify the set of `VMExternRef`s inside Wasm activations. Then we take the difference between this precise set and our over-approximation, and decrement the reference count for each of the `VMExternRef`s that are in our over-approximation but not in the precise set. Finally, the over-approximation is replaced with the precise set. The `VMExternRefActivationsTable` implements the over-approximized set of `VMExternRef`s referenced by Wasm activations. Calling a Wasm function and passing it a `VMExternRef` moves the `VMExternRef` into the table, and the compiled Wasm function logically "borrows" the `VMExternRef` from the table. Similarly, `global.get` and `table.get` operations clone the gotten `VMExternRef` into the `VMExternRefActivationsTable` and then "borrow" the reference out of the table. When a `VMExternRef` is returned to host code from a Wasm function, the host increments the reference count (because the reference is logically "borrowed" from the `VMExternRefActivationsTable` and the reference count from the table will be dropped at the next GC). For more general information on deferred reference counting, see *An Examination of Deferred Reference Counting and Cycle Detection* by Quinane: https://openresearch-repository.anu.edu.au/bitstream/1885/42030/2/hon-thesis.pdf cc #929 Fixes #1804
4 years ago
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b275950c28b37e794e8c55d88aeb5e139d0ce23fdbbeda68f8d7174abdf9e8fa"
externref: implement stack map-based garbage collection For host VM code, we use plain reference counting, where cloning increments the reference count, and dropping decrements it. We can avoid many of the on-stack increment/decrement operations that typically plague the performance of reference counting via Rust's ownership and borrowing system. Moving a `VMExternRef` avoids mutating its reference count, and borrowing it either avoids the reference count increment or delays it until if/when the `VMExternRef` is cloned. When passing a `VMExternRef` into compiled Wasm code, we don't want to do reference count mutations for every compiled `local.{get,set}`, nor for every function call. Therefore, we use a variation of **deferred reference counting**, where we only mutate reference counts when storing `VMExternRef`s somewhere that outlives the activation: into a global or table. Simultaneously, we over-approximate the set of `VMExternRef`s that are inside Wasm function activations. Periodically, we walk the stack at GC safe points, and use stack map information to precisely identify the set of `VMExternRef`s inside Wasm activations. Then we take the difference between this precise set and our over-approximation, and decrement the reference count for each of the `VMExternRef`s that are in our over-approximation but not in the precise set. Finally, the over-approximation is replaced with the precise set. The `VMExternRefActivationsTable` implements the over-approximized set of `VMExternRef`s referenced by Wasm activations. Calling a Wasm function and passing it a `VMExternRef` moves the `VMExternRef` into the table, and the compiled Wasm function logically "borrows" the `VMExternRef` from the table. Similarly, `global.get` and `table.get` operations clone the gotten `VMExternRef` into the `VMExternRefActivationsTable` and then "borrow" the reference out of the table. When a `VMExternRef` is returned to host code from a Wasm function, the host increments the reference count (because the reference is logically "borrowed" from the `VMExternRefActivationsTable` and the reference count from the table will be dropped at the next GC). For more general information on deferred reference counting, see *An Examination of Deferred Reference Counting and Cycle Detection* by Quinane: https://openresearch-repository.anu.edu.au/bitstream/1885/42030/2/hon-thesis.pdf cc #929 Fixes #1804
4 years ago
dependencies = [
"adler",
externref: implement stack map-based garbage collection For host VM code, we use plain reference counting, where cloning increments the reference count, and dropping decrements it. We can avoid many of the on-stack increment/decrement operations that typically plague the performance of reference counting via Rust's ownership and borrowing system. Moving a `VMExternRef` avoids mutating its reference count, and borrowing it either avoids the reference count increment or delays it until if/when the `VMExternRef` is cloned. When passing a `VMExternRef` into compiled Wasm code, we don't want to do reference count mutations for every compiled `local.{get,set}`, nor for every function call. Therefore, we use a variation of **deferred reference counting**, where we only mutate reference counts when storing `VMExternRef`s somewhere that outlives the activation: into a global or table. Simultaneously, we over-approximate the set of `VMExternRef`s that are inside Wasm function activations. Periodically, we walk the stack at GC safe points, and use stack map information to precisely identify the set of `VMExternRef`s inside Wasm activations. Then we take the difference between this precise set and our over-approximation, and decrement the reference count for each of the `VMExternRef`s that are in our over-approximation but not in the precise set. Finally, the over-approximation is replaced with the precise set. The `VMExternRefActivationsTable` implements the over-approximized set of `VMExternRef`s referenced by Wasm activations. Calling a Wasm function and passing it a `VMExternRef` moves the `VMExternRef` into the table, and the compiled Wasm function logically "borrows" the `VMExternRef` from the table. Similarly, `global.get` and `table.get` operations clone the gotten `VMExternRef` into the `VMExternRefActivationsTable` and then "borrow" the reference out of the table. When a `VMExternRef` is returned to host code from a Wasm function, the host increments the reference count (because the reference is logically "borrowed" from the `VMExternRefActivationsTable` and the reference count from the table will be dropped at the next GC). For more general information on deferred reference counting, see *An Examination of Deferred Reference Counting and Cycle Detection* by Quinane: https://openresearch-repository.anu.edu.au/bitstream/1885/42030/2/hon-thesis.pdf cc #929 Fixes #1804
4 years ago
]
4 years ago
[[package]]
name = "mio"
version = "0.8.2"
4 years ago
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "52da4364ffb0e4fe33a9841a98a3f3014fb964045ce4f7a45a398243c8d6b0c9"
4 years ago
dependencies = [
"libc",
"log",
"miow",
"ntapi",
"wasi 0.11.0+wasi-snapshot-preview1",
4 years ago
"winapi",
]
[[package]]
name = "miow"
version = "0.3.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b9f1c5b025cda876f66ef43a113f91ebc9f4ccef34843000e0adf6ebbab84e21"
dependencies = [
"winapi",
]
[[package]]
name = "ntapi"
version = "0.3.7"
4 years ago
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c28774a7fd2fbb4f0babd8237ce554b73af68021b5f695a3cebd6c59bac0980f"
4 years ago
dependencies = [
"winapi",
]
[[package]]
name = "num-bigint-dig"
version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4547ee5541c18742396ae2c895d0717d0f886d8823b8399cdaf7b07d63ad0480"
dependencies = [
"autocfg 0.1.8",
"byteorder",
"lazy_static",
"libm",
"num-integer",
"num-iter",
"num-traits",
"rand 0.8.5",
"serde",
"smallvec",
"zeroize",
]
[[package]]
name = "num-integer"
version = "0.1.45"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9"
dependencies = [
"autocfg 1.1.0",
"num-traits",
]
[[package]]
name = "num-iter"
version = "0.1.43"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7d03e6c028c5dc5cac6e2dec0efda81fc887605bb3d884578bb6d6bf7514e252"
dependencies = [
"autocfg 1.1.0",
"num-integer",
"num-traits",
]
[[package]]
name = "num-traits"
version = "0.2.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd"
dependencies = [
"autocfg 1.1.0",
"libm",
]
[[package]]
name = "num_cpus"
version = "1.15.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0fac9e2da13b5eb447a6ce3d392f23a29d8694bff781bf03a16cd9ac8697593b"
dependencies = [
"hermit-abi 0.2.6",
"libc",
]
[[package]]
name = "number_prefix"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "17b02fc0ff9a9e4b35b3342880f48e896ebf69f2967921fe8646bf5b7125956a"
[[package]]
name = "object"
version = "0.30.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ea86265d3d3dcb6a27fc51bd29a4bf387fae9d2986b823079d4986af253eb439"
dependencies = [
"crc32fast",
"hashbrown 0.13.1",
"indexmap",
"memchr",
]
[[package]]
name = "ocaml-boxroot-sys"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5186393bfbee4ce2bc5bbb82beafb77e85c1d0a557e3cfc8c8a0d63d7845fed5"
dependencies = [
"cc",
]
[[package]]
name = "ocaml-interop"
version = "0.8.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2e01e08412a7e072a90a225d2ae49a2860aeea853ce673bc63891dbf86aed063"
dependencies = [
"ocaml-boxroot-sys",
"ocaml-sys",
"static_assertions",
]
[[package]]
name = "ocaml-sys"
version = "0.22.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "73ec6ca7d41458442627435afb8f4671e83fd642e8a560171d671a1f679aa3cf"
dependencies = [
"cty",
]
Refactor and fill out wasmtime's C API (#1415) * Refactor and improve safety of C API This commit is intended to be a relatively large refactoring of the C API which is targeted at improving the safety of our C API definitions. Not all of the APIs have been updated yet but this is intended to be the start. The goal here is to make as many functions safe as we can, expressing inputs/outputs as native Rust types rather than raw pointers wherever possible. For example instead of `*const wasm_foo_t` we'd take `&wasm_foo_t`. Instead of returning `*mut wasm_foo_t` we'd return `Box<wasm_foo_t>`. No ABI/API changes are intended from this commit, it's supposed to only change how we define all these functions internally. This commit also additionally implements a few more API bindings for exposed vector types by unifying everything into one macro. Finally, this commit moves many internal caches in the C API to the `OnceCell` type which provides a safe interface for one-time initialization. * Split apart monolithic C API `lib.rs` This commit splits the monolithic `src/lib.rs` in the C API crate into lots of smaller files. The goal here is to make this a bit more readable and digestable. Each module now contains only API bindings for a particular type, roughly organized around the grouping in the wasm.h header file already. A few more extensions were added, such as filling out `*_as_*` conversions with both const and non-const versions. Additionally many APIs were made safer in the same style as the previous commit, generally preferring Rust types rather than raw pointer types. Overall no functional change is intended here, it should be mostly just code movement and minor refactorings! * Make a few wasi C bindings safer Use safe Rust types where we can and touch up a few APIs here and there. * Implement `wasm_*type_as_externtype*` APIs This commit restructures `wasm_externtype_t` to be similar to `wasm_extern_t` so type conversion between the `*_extern_*` variants to the concrete variants are all simple casts. (checked in the case of general to concrete, of course). * Consistently imlpement host info functions in the API This commit adds a small macro crate which is then used to consistently define the various host-info-related functions in the C API. The goal here is to try to mirror what the `wasm.h` header provides to provide a full implementation of the header.
5 years ago
[[package]]
name = "once_cell"
version = "1.16.0"
Refactor and fill out wasmtime's C API (#1415) * Refactor and improve safety of C API This commit is intended to be a relatively large refactoring of the C API which is targeted at improving the safety of our C API definitions. Not all of the APIs have been updated yet but this is intended to be the start. The goal here is to make as many functions safe as we can, expressing inputs/outputs as native Rust types rather than raw pointers wherever possible. For example instead of `*const wasm_foo_t` we'd take `&wasm_foo_t`. Instead of returning `*mut wasm_foo_t` we'd return `Box<wasm_foo_t>`. No ABI/API changes are intended from this commit, it's supposed to only change how we define all these functions internally. This commit also additionally implements a few more API bindings for exposed vector types by unifying everything into one macro. Finally, this commit moves many internal caches in the C API to the `OnceCell` type which provides a safe interface for one-time initialization. * Split apart monolithic C API `lib.rs` This commit splits the monolithic `src/lib.rs` in the C API crate into lots of smaller files. The goal here is to make this a bit more readable and digestable. Each module now contains only API bindings for a particular type, roughly organized around the grouping in the wasm.h header file already. A few more extensions were added, such as filling out `*_as_*` conversions with both const and non-const versions. Additionally many APIs were made safer in the same style as the previous commit, generally preferring Rust types rather than raw pointer types. Overall no functional change is intended here, it should be mostly just code movement and minor refactorings! * Make a few wasi C bindings safer Use safe Rust types where we can and touch up a few APIs here and there. * Implement `wasm_*type_as_externtype*` APIs This commit restructures `wasm_externtype_t` to be similar to `wasm_extern_t` so type conversion between the `*_extern_*` variants to the concrete variants are all simple casts. (checked in the case of general to concrete, of course). * Consistently imlpement host info functions in the API This commit adds a small macro crate which is then used to consistently define the various host-info-related functions in the C API. The goal here is to try to mirror what the `wasm.h` header provides to provide a full implementation of the header.
5 years ago
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "86f0b0d4bf799edbc74508c1e8bf170ff5f41238e5f8225603ca7caaae2b7860"
Refactor and fill out wasmtime's C API (#1415) * Refactor and improve safety of C API This commit is intended to be a relatively large refactoring of the C API which is targeted at improving the safety of our C API definitions. Not all of the APIs have been updated yet but this is intended to be the start. The goal here is to make as many functions safe as we can, expressing inputs/outputs as native Rust types rather than raw pointers wherever possible. For example instead of `*const wasm_foo_t` we'd take `&wasm_foo_t`. Instead of returning `*mut wasm_foo_t` we'd return `Box<wasm_foo_t>`. No ABI/API changes are intended from this commit, it's supposed to only change how we define all these functions internally. This commit also additionally implements a few more API bindings for exposed vector types by unifying everything into one macro. Finally, this commit moves many internal caches in the C API to the `OnceCell` type which provides a safe interface for one-time initialization. * Split apart monolithic C API `lib.rs` This commit splits the monolithic `src/lib.rs` in the C API crate into lots of smaller files. The goal here is to make this a bit more readable and digestable. Each module now contains only API bindings for a particular type, roughly organized around the grouping in the wasm.h header file already. A few more extensions were added, such as filling out `*_as_*` conversions with both const and non-const versions. Additionally many APIs were made safer in the same style as the previous commit, generally preferring Rust types rather than raw pointer types. Overall no functional change is intended here, it should be mostly just code movement and minor refactorings! * Make a few wasi C bindings safer Use safe Rust types where we can and touch up a few APIs here and there. * Implement `wasm_*type_as_externtype*` APIs This commit restructures `wasm_externtype_t` to be similar to `wasm_extern_t` so type conversion between the `*_extern_*` variants to the concrete variants are all simple casts. (checked in the case of general to concrete, of course). * Consistently imlpement host info functions in the API This commit adds a small macro crate which is then used to consistently define the various host-info-related functions in the C API. The goal here is to try to mirror what the `wasm.h` header provides to provide a full implementation of the header.
5 years ago
[[package]]
name = "oorandom"
version = "11.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575"
[[package]]
name = "opaque-debug"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5"
[[package]]
name = "openvino"
version = "0.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c7336c11cad0eb45f65436cdbf073c697397a1bfe53836cef997129d69443c77"
dependencies = [
"openvino-sys",
"thiserror",
]
wasi-nn: turn it on by default (#2859) * wasi-nn: turn it on by default This change makes the wasi-nn Cargo feature a default feature. Previously, a wasi-nn user would have to build a separate Wasmtime binary (e.g. `cargo build --features wasi-nn ...`) to use wasi-nn and the resulting binary would require OpenVINO shared libraries to be present in the environment in order to run (otherwise it would fail immediately with linking errors). With recent changes to the `openvino` crate, the wasi-nn implementation can defer the loading of the OpenVINO shared libraries until runtime (i.e., when the user Wasm program calls `wasi_ephemeral_nn::load`) and display a user-level error if anything goes wrong (e.g., the OpenVINO libraries are not present on the system). This runtime-linking addition allows the wasi-nn feature to be turned on by default and shipped with upcoming releases of Wasmtime. This change should be transparent for users who do not use wasi-nn: the `openvino` crate is small and the newly-available wasi-nn imports only affect programs in which they are used. For those interested in reviewing the runtime linking approach added to the `openvino` crate, see https://github.com/intel/openvino-rs/pull/19. * wasi-nn spec path: don't use canonicalize * Allow dependencies using the ISC license The ISC license should be [just as permissive](https://choosealicense.com/licenses/isc) as MIT, e.g., with no additional limitations. * Add a `--wasi-modules` flag This flag controls which WASI modules are made available to the Wasm program. This initial commit enables `wasi-common` by default (equivalent to `--wasi-modules=all`) and allows `wasi-nn` and `wasi-crypto` to be added in either individually (e.g., `--wasi-modules=wasi-nn`) or as a group (e.g., `--wasi-modules=all-experimental`). * wasi-crypto: fix unused dependency Co-authored-by: Pat Hickey <pat@moreproductive.org>
4 years ago
[[package]]
name = "openvino-finder"
version = "0.4.2"
wasi-nn: turn it on by default (#2859) * wasi-nn: turn it on by default This change makes the wasi-nn Cargo feature a default feature. Previously, a wasi-nn user would have to build a separate Wasmtime binary (e.g. `cargo build --features wasi-nn ...`) to use wasi-nn and the resulting binary would require OpenVINO shared libraries to be present in the environment in order to run (otherwise it would fail immediately with linking errors). With recent changes to the `openvino` crate, the wasi-nn implementation can defer the loading of the OpenVINO shared libraries until runtime (i.e., when the user Wasm program calls `wasi_ephemeral_nn::load`) and display a user-level error if anything goes wrong (e.g., the OpenVINO libraries are not present on the system). This runtime-linking addition allows the wasi-nn feature to be turned on by default and shipped with upcoming releases of Wasmtime. This change should be transparent for users who do not use wasi-nn: the `openvino` crate is small and the newly-available wasi-nn imports only affect programs in which they are used. For those interested in reviewing the runtime linking approach added to the `openvino` crate, see https://github.com/intel/openvino-rs/pull/19. * wasi-nn spec path: don't use canonicalize * Allow dependencies using the ISC license The ISC license should be [just as permissive](https://choosealicense.com/licenses/isc) as MIT, e.g., with no additional limitations. * Add a `--wasi-modules` flag This flag controls which WASI modules are made available to the Wasm program. This initial commit enables `wasi-common` by default (equivalent to `--wasi-modules=all`) and allows `wasi-nn` and `wasi-crypto` to be added in either individually (e.g., `--wasi-modules=wasi-nn`) or as a group (e.g., `--wasi-modules=all-experimental`). * wasi-crypto: fix unused dependency Co-authored-by: Pat Hickey <pat@moreproductive.org>
4 years ago
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c650edf39ea54dfbe18f0ad513858ff0bed3f6a308b677e0d5f71b330f476ccf"
dependencies = [
Remove support for userfaultfd (#4040) This commit removes support for the `userfaultfd` or "uffd" syscall on Linux. This support was originally added for users migrating from Lucet to Wasmtime, but the recent developments of kernel-supported copy-on-write support for memory initialization wound up being more appropriate for these use cases than usefaultfd. The main reason for moving to copy-on-write initialization are: * The `userfaultfd` feature was never necessarily intended for this style of use case with wasm and was susceptible to subtle and rare bugs that were extremely difficult to track down. We were never 100% certain that there were kernel bugs related to userfaultfd but the suspicion never went away. * Handling faults with userfaultfd was always slow and single-threaded. Only one thread could handle faults and traveling to user-space to handle faults is inherently slower than handling them all in the kernel. The single-threaded aspect in particular presented a significant scaling bottleneck for embeddings that want to run many wasm instances in parallel. * One of the major benefits of userfaultfd was lazy initialization of wasm linear memory which is also achieved with the copy-on-write initialization support we have right now. * One of the suspected benefits of userfaultfd was less frobbing of the kernel vma structures when wasm modules are instantiated. Currently the copy-on-write support has a mitigation where we attempt to reuse the memory images where possible to avoid changing vma structures. When comparing this to userfaultfd's performance it was found that kernel modifications of vmas aren't a worrisome bottleneck so copy-on-write is suitable for this as well. Overall there are no remaining benefits that userfaultfd gives that copy-on-write doesn't, and copy-on-write solves a major downsides of userfaultfd, the scaling issue with a single faulting thread. Additionally copy-on-write support seems much more robust in terms of kernel implementation since it's only using standard memory-management syscalls which are heavily exercised. Finally copy-on-write support provides a new bonus where read-only memory in WebAssembly can be mapped directly to the same kernel cache page, even amongst many wasm instances of the same module, which was never possible with userfaultfd. In light of all this it's expected that all users of userfaultfd should migrate to the copy-on-write initialization of Wasmtime (which is enabled by default).
3 years ago
"cfg-if",
"log",
]
wasi-nn: turn it on by default (#2859) * wasi-nn: turn it on by default This change makes the wasi-nn Cargo feature a default feature. Previously, a wasi-nn user would have to build a separate Wasmtime binary (e.g. `cargo build --features wasi-nn ...`) to use wasi-nn and the resulting binary would require OpenVINO shared libraries to be present in the environment in order to run (otherwise it would fail immediately with linking errors). With recent changes to the `openvino` crate, the wasi-nn implementation can defer the loading of the OpenVINO shared libraries until runtime (i.e., when the user Wasm program calls `wasi_ephemeral_nn::load`) and display a user-level error if anything goes wrong (e.g., the OpenVINO libraries are not present on the system). This runtime-linking addition allows the wasi-nn feature to be turned on by default and shipped with upcoming releases of Wasmtime. This change should be transparent for users who do not use wasi-nn: the `openvino` crate is small and the newly-available wasi-nn imports only affect programs in which they are used. For those interested in reviewing the runtime linking approach added to the `openvino` crate, see https://github.com/intel/openvino-rs/pull/19. * wasi-nn spec path: don't use canonicalize * Allow dependencies using the ISC license The ISC license should be [just as permissive](https://choosealicense.com/licenses/isc) as MIT, e.g., with no additional limitations. * Add a `--wasi-modules` flag This flag controls which WASI modules are made available to the Wasm program. This initial commit enables `wasi-common` by default (equivalent to `--wasi-modules=all`) and allows `wasi-nn` and `wasi-crypto` to be added in either individually (e.g., `--wasi-modules=wasi-nn`) or as a group (e.g., `--wasi-modules=all-experimental`). * wasi-crypto: fix unused dependency Co-authored-by: Pat Hickey <pat@moreproductive.org>
4 years ago
[[package]]
name = "openvino-sys"
version = "0.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6d003d61f18f7bf6dd965b4e913cbd3e7cda6a3c179115c8ee59e5c29b390f45"
dependencies = [
wasi-nn: turn it on by default (#2859) * wasi-nn: turn it on by default This change makes the wasi-nn Cargo feature a default feature. Previously, a wasi-nn user would have to build a separate Wasmtime binary (e.g. `cargo build --features wasi-nn ...`) to use wasi-nn and the resulting binary would require OpenVINO shared libraries to be present in the environment in order to run (otherwise it would fail immediately with linking errors). With recent changes to the `openvino` crate, the wasi-nn implementation can defer the loading of the OpenVINO shared libraries until runtime (i.e., when the user Wasm program calls `wasi_ephemeral_nn::load`) and display a user-level error if anything goes wrong (e.g., the OpenVINO libraries are not present on the system). This runtime-linking addition allows the wasi-nn feature to be turned on by default and shipped with upcoming releases of Wasmtime. This change should be transparent for users who do not use wasi-nn: the `openvino` crate is small and the newly-available wasi-nn imports only affect programs in which they are used. For those interested in reviewing the runtime linking approach added to the `openvino` crate, see https://github.com/intel/openvino-rs/pull/19. * wasi-nn spec path: don't use canonicalize * Allow dependencies using the ISC license The ISC license should be [just as permissive](https://choosealicense.com/licenses/isc) as MIT, e.g., with no additional limitations. * Add a `--wasi-modules` flag This flag controls which WASI modules are made available to the Wasm program. This initial commit enables `wasi-common` by default (equivalent to `--wasi-modules=all`) and allows `wasi-nn` and `wasi-crypto` to be added in either individually (e.g., `--wasi-modules=wasi-nn`) or as a group (e.g., `--wasi-modules=all-experimental`). * wasi-crypto: fix unused dependency Co-authored-by: Pat Hickey <pat@moreproductive.org>
4 years ago
"libloading",
"once_cell",
wasi-nn: turn it on by default (#2859) * wasi-nn: turn it on by default This change makes the wasi-nn Cargo feature a default feature. Previously, a wasi-nn user would have to build a separate Wasmtime binary (e.g. `cargo build --features wasi-nn ...`) to use wasi-nn and the resulting binary would require OpenVINO shared libraries to be present in the environment in order to run (otherwise it would fail immediately with linking errors). With recent changes to the `openvino` crate, the wasi-nn implementation can defer the loading of the OpenVINO shared libraries until runtime (i.e., when the user Wasm program calls `wasi_ephemeral_nn::load`) and display a user-level error if anything goes wrong (e.g., the OpenVINO libraries are not present on the system). This runtime-linking addition allows the wasi-nn feature to be turned on by default and shipped with upcoming releases of Wasmtime. This change should be transparent for users who do not use wasi-nn: the `openvino` crate is small and the newly-available wasi-nn imports only affect programs in which they are used. For those interested in reviewing the runtime linking approach added to the `openvino` crate, see https://github.com/intel/openvino-rs/pull/19. * wasi-nn spec path: don't use canonicalize * Allow dependencies using the ISC license The ISC license should be [just as permissive](https://choosealicense.com/licenses/isc) as MIT, e.g., with no additional limitations. * Add a `--wasi-modules` flag This flag controls which WASI modules are made available to the Wasm program. This initial commit enables `wasi-common` by default (equivalent to `--wasi-modules=all`) and allows `wasi-nn` and `wasi-crypto` to be added in either individually (e.g., `--wasi-modules=wasi-nn`) or as a group (e.g., `--wasi-modules=all-experimental`). * wasi-crypto: fix unused dependency Co-authored-by: Pat Hickey <pat@moreproductive.org>
4 years ago
"openvino-finder",
"pretty_env_logger",
]
[[package]]
name = "os_pipe"
version = "0.9.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fb233f06c2307e1f5ce2ecad9f8121cffbbee2c95428f44ea85222e460d0d213"
dependencies = [
"libc",
"winapi",
]
[[package]]
name = "os_str_bytes"
version = "6.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8e22443d1643a904602595ba1cd8f7d896afe56d26712531c5ff73a15b2fbf64"
[[package]]
name = "p256"
version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d053368e1bae4c8a672953397bd1bd7183dde1c72b0b7612a15719173148d186"
dependencies = [
"ecdsa",
"elliptic-curve",
"sha2 0.9.9",
]
[[package]]
name = "parking_lot"
version = "0.11.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99"
dependencies = [
"instant",
"lock_api",
"parking_lot_core",
]
[[package]]
name = "parking_lot_core"
version = "0.8.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d76e8e1493bcac0d2766c42737f34458f1c8c50c0d23bcb24ea953affb273216"
dependencies = [
Remove support for userfaultfd (#4040) This commit removes support for the `userfaultfd` or "uffd" syscall on Linux. This support was originally added for users migrating from Lucet to Wasmtime, but the recent developments of kernel-supported copy-on-write support for memory initialization wound up being more appropriate for these use cases than usefaultfd. The main reason for moving to copy-on-write initialization are: * The `userfaultfd` feature was never necessarily intended for this style of use case with wasm and was susceptible to subtle and rare bugs that were extremely difficult to track down. We were never 100% certain that there were kernel bugs related to userfaultfd but the suspicion never went away. * Handling faults with userfaultfd was always slow and single-threaded. Only one thread could handle faults and traveling to user-space to handle faults is inherently slower than handling them all in the kernel. The single-threaded aspect in particular presented a significant scaling bottleneck for embeddings that want to run many wasm instances in parallel. * One of the major benefits of userfaultfd was lazy initialization of wasm linear memory which is also achieved with the copy-on-write initialization support we have right now. * One of the suspected benefits of userfaultfd was less frobbing of the kernel vma structures when wasm modules are instantiated. Currently the copy-on-write support has a mitigation where we attempt to reuse the memory images where possible to avoid changing vma structures. When comparing this to userfaultfd's performance it was found that kernel modifications of vmas aren't a worrisome bottleneck so copy-on-write is suitable for this as well. Overall there are no remaining benefits that userfaultfd gives that copy-on-write doesn't, and copy-on-write solves a major downsides of userfaultfd, the scaling issue with a single faulting thread. Additionally copy-on-write support seems much more robust in terms of kernel implementation since it's only using standard memory-management syscalls which are heavily exercised. Finally copy-on-write support provides a new bonus where read-only memory in WebAssembly can be mapped directly to the same kernel cache page, even amongst many wasm instances of the same module, which was never possible with userfaultfd. In light of all this it's expected that all users of userfaultfd should migrate to the copy-on-write initialization of Wasmtime (which is enabled by default).
3 years ago
"cfg-if",
"instant",
"libc",
"redox_syscall",
"smallvec",
"winapi",
]
Implement support for `async` functions in Wasmtime (#2434) * Implement support for `async` functions in Wasmtime This is an implementation of [RFC 2] in Wasmtime which is to support `async`-defined host functions. At a high level support is added by executing WebAssembly code that might invoke an asynchronous host function on a separate native stack. When the host function's future is not ready we switch back to the main native stack to continue execution. There's a whole bunch of details in this commit, and it's a bit much to go over them all here in this commit message. The most important changes here are: * A new `wasmtime-fiber` crate has been written to manage the low-level details of stack-switching. Unixes use `mmap` to allocate a stack and Windows uses the native fibers implementation. We'll surely want to refactor this to move stack allocation elsewhere in the future. Fibers are intended to be relatively general with a lot of type paremters to fling values back and forth across suspension points. The whole crate is a giant wad of `unsafe` unfortunately and involves handwritten assembly with custom dwarf CFI directives to boot. Definitely deserves a close eye in review! * The `Store` type has two new methods -- `block_on` and `on_fiber` which bridge between the async and non-async worlds. Lots of unsafe fiddly bits here as we're trying to communicate context pointers between disparate portions of the code. Extra eyes and care in review is greatly appreciated. * The APIs for binding `async` functions are unfortunately pretty ugly in `Func`. This is mostly due to language limitations and compiler bugs (I believe) in Rust. Instead of `Func::wrap` we have a `Func::wrapN_async` family of methods, and we've also got a whole bunch of `Func::getN_async` methods now too. It may be worth rethinking the API of `Func` to try to make the documentation page actually grok'able. This isn't super heavily tested but the various test should suffice for engaging hopefully nearly all the infrastructure in one form or another. This is just the start though! [RFC 2]: https://github.com/bytecodealliance/rfcs/pull/2 * Add wasmtime-fiber to publish script * Save vector/float registers on ARM too. * Fix a typo * Update lock file * Implement periodically yielding with fuel consumption This commit implements APIs on `Store` to periodically yield execution of futures through the consumption of fuel. When fuel runs out a future's execution is yielded back to the caller, and then upon resumption fuel is re-injected. The goal of this is to allow cooperative multi-tasking with futures. * Fix compile without async * Save/restore the frame pointer in fiber switching Turns out this is another caller-saved register! * Simplify x86_64 fiber asm Take a leaf out of aarch64's playbook and don't have extra memory to load/store these arguments, instead leverage how `wasmtime_fiber_switch` already loads a bunch of data into registers which we can then immediately start using on a fiber's start without any extra memory accesses. * Add x86 support to wasmtime-fiber * Add ARM32 support to fiber crate * Make fiber build file probing more flexible * Use CreateFiberEx on Windows * Remove a stray no-longer-used trait declaration * Don't reach into `Caller` internals * Tweak async fuel to eventually run out. With fuel it's probably best to not provide any way to inject infinite fuel. * Fix some typos * Cleanup asm a bit * Use a shared header file to deduplicate some directives * Guarantee hidden visibility for functions * Enable gc-sections on macOS x86_64 * Add `.type` annotations for ARM * Update lock file * Fix compile error * Review comments
4 years ago
[[package]]
name = "paste"
version = "1.0.7"
Implement support for `async` functions in Wasmtime (#2434) * Implement support for `async` functions in Wasmtime This is an implementation of [RFC 2] in Wasmtime which is to support `async`-defined host functions. At a high level support is added by executing WebAssembly code that might invoke an asynchronous host function on a separate native stack. When the host function's future is not ready we switch back to the main native stack to continue execution. There's a whole bunch of details in this commit, and it's a bit much to go over them all here in this commit message. The most important changes here are: * A new `wasmtime-fiber` crate has been written to manage the low-level details of stack-switching. Unixes use `mmap` to allocate a stack and Windows uses the native fibers implementation. We'll surely want to refactor this to move stack allocation elsewhere in the future. Fibers are intended to be relatively general with a lot of type paremters to fling values back and forth across suspension points. The whole crate is a giant wad of `unsafe` unfortunately and involves handwritten assembly with custom dwarf CFI directives to boot. Definitely deserves a close eye in review! * The `Store` type has two new methods -- `block_on` and `on_fiber` which bridge between the async and non-async worlds. Lots of unsafe fiddly bits here as we're trying to communicate context pointers between disparate portions of the code. Extra eyes and care in review is greatly appreciated. * The APIs for binding `async` functions are unfortunately pretty ugly in `Func`. This is mostly due to language limitations and compiler bugs (I believe) in Rust. Instead of `Func::wrap` we have a `Func::wrapN_async` family of methods, and we've also got a whole bunch of `Func::getN_async` methods now too. It may be worth rethinking the API of `Func` to try to make the documentation page actually grok'able. This isn't super heavily tested but the various test should suffice for engaging hopefully nearly all the infrastructure in one form or another. This is just the start though! [RFC 2]: https://github.com/bytecodealliance/rfcs/pull/2 * Add wasmtime-fiber to publish script * Save vector/float registers on ARM too. * Fix a typo * Update lock file * Implement periodically yielding with fuel consumption This commit implements APIs on `Store` to periodically yield execution of futures through the consumption of fuel. When fuel runs out a future's execution is yielded back to the caller, and then upon resumption fuel is re-injected. The goal of this is to allow cooperative multi-tasking with futures. * Fix compile without async * Save/restore the frame pointer in fiber switching Turns out this is another caller-saved register! * Simplify x86_64 fiber asm Take a leaf out of aarch64's playbook and don't have extra memory to load/store these arguments, instead leverage how `wasmtime_fiber_switch` already loads a bunch of data into registers which we can then immediately start using on a fiber's start without any extra memory accesses. * Add x86 support to wasmtime-fiber * Add ARM32 support to fiber crate * Make fiber build file probing more flexible * Use CreateFiberEx on Windows * Remove a stray no-longer-used trait declaration * Don't reach into `Caller` internals * Tweak async fuel to eventually run out. With fuel it's probably best to not provide any way to inject infinite fuel. * Fix some typos * Cleanup asm a bit * Use a shared header file to deduplicate some directives * Guarantee hidden visibility for functions * Enable gc-sections on macOS x86_64 * Add `.type` annotations for ARM * Update lock file * Fix compile error * Review comments
4 years ago
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0c520e05135d6e763148b6426a837e239041653ba7becd2e538c076c738025fc"
Implement support for `async` functions in Wasmtime (#2434) * Implement support for `async` functions in Wasmtime This is an implementation of [RFC 2] in Wasmtime which is to support `async`-defined host functions. At a high level support is added by executing WebAssembly code that might invoke an asynchronous host function on a separate native stack. When the host function's future is not ready we switch back to the main native stack to continue execution. There's a whole bunch of details in this commit, and it's a bit much to go over them all here in this commit message. The most important changes here are: * A new `wasmtime-fiber` crate has been written to manage the low-level details of stack-switching. Unixes use `mmap` to allocate a stack and Windows uses the native fibers implementation. We'll surely want to refactor this to move stack allocation elsewhere in the future. Fibers are intended to be relatively general with a lot of type paremters to fling values back and forth across suspension points. The whole crate is a giant wad of `unsafe` unfortunately and involves handwritten assembly with custom dwarf CFI directives to boot. Definitely deserves a close eye in review! * The `Store` type has two new methods -- `block_on` and `on_fiber` which bridge between the async and non-async worlds. Lots of unsafe fiddly bits here as we're trying to communicate context pointers between disparate portions of the code. Extra eyes and care in review is greatly appreciated. * The APIs for binding `async` functions are unfortunately pretty ugly in `Func`. This is mostly due to language limitations and compiler bugs (I believe) in Rust. Instead of `Func::wrap` we have a `Func::wrapN_async` family of methods, and we've also got a whole bunch of `Func::getN_async` methods now too. It may be worth rethinking the API of `Func` to try to make the documentation page actually grok'able. This isn't super heavily tested but the various test should suffice for engaging hopefully nearly all the infrastructure in one form or another. This is just the start though! [RFC 2]: https://github.com/bytecodealliance/rfcs/pull/2 * Add wasmtime-fiber to publish script * Save vector/float registers on ARM too. * Fix a typo * Update lock file * Implement periodically yielding with fuel consumption This commit implements APIs on `Store` to periodically yield execution of futures through the consumption of fuel. When fuel runs out a future's execution is yielded back to the caller, and then upon resumption fuel is re-injected. The goal of this is to allow cooperative multi-tasking with futures. * Fix compile without async * Save/restore the frame pointer in fiber switching Turns out this is another caller-saved register! * Simplify x86_64 fiber asm Take a leaf out of aarch64's playbook and don't have extra memory to load/store these arguments, instead leverage how `wasmtime_fiber_switch` already loads a bunch of data into registers which we can then immediately start using on a fiber's start without any extra memory accesses. * Add x86 support to wasmtime-fiber * Add ARM32 support to fiber crate * Make fiber build file probing more flexible * Use CreateFiberEx on Windows * Remove a stray no-longer-used trait declaration * Don't reach into `Caller` internals * Tweak async fuel to eventually run out. With fuel it's probably best to not provide any way to inject infinite fuel. * Fix some typos * Cleanup asm a bit * Use a shared header file to deduplicate some directives * Guarantee hidden visibility for functions * Enable gc-sections on macOS x86_64 * Add `.type` annotations for ARM * Update lock file * Fix compile error * Review comments
4 years ago
[[package]]
name = "pem-rfc7468"
version = "0.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "84e93a3b1cc0510b03020f33f21e62acdde3dcaef432edc95bea377fbd4c2cd4"
dependencies = [
"base64ct",
]
[[package]]
name = "percent-encoding"
version = "2.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "478c572c3d73181ff3c2539045f6eb99e5491218eae919370993b890cdbdd98e"
[[package]]
name = "pin-project-lite"
version = "0.2.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116"
[[package]]
name = "pkcs1"
version = "0.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "116bee8279d783c0cf370efa1a94632f2108e5ef0bb32df31f051647810a4e2c"
dependencies = [
"der",
"pem-rfc7468",
"zeroize",
]
[[package]]
name = "pkcs8"
version = "0.7.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ee3ef9b64d26bad0536099c816c6734379e45bbd5f14798def6809e5cc350447"
dependencies = [
"der",
"pem-rfc7468",
"pkcs1",
"spki",
"zeroize",
]
[[package]]
name = "plotters"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "32a3fd9ec30b9749ce28cd91f255d569591cdf937fe280c312143e3c4bad6f2a"
dependencies = [
"num-traits",
"plotters-backend",
"plotters-svg",
"wasm-bindgen",
"web-sys",
]
[[package]]
name = "plotters-backend"
version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d88417318da0eaf0fdcdb51a0ee6c3bed624333bff8f946733049380be67ac1c"
[[package]]
name = "plotters-svg"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "521fa9638fa597e1dc53e9412a4f9cefb01187ee1f7413076f9e6749e2885ba9"
dependencies = [
"plotters-backend",
]
[[package]]
name = "poly1305"
version = "0.7.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "048aeb476be11a4b6ca432ca569e375810de9294ae78f4774e78ea98a9246ede"
dependencies = [
ISLE: guard against stale generated source in default build config. Currently, the `build.rs` script that generates Rust source from the ISLE DSL will only do this generation if the `rebuild-isle` Cargo feature is specified. By default, it is not. This is based on the principle that we (the build script) do not modify the source tree as managed by git; git-managed files are strictly a human-managed and human-edited resource. By adding the opt-in Cargo feature, a developer is requesting the build script to perform an explicit action. (In my understanding at least, this principle comes from the general philosophy of hermetic builds: the output should be a pure function of the input, and part of this is that the input is read-only. If we modify the source tree, then all bets are off.) Unfortunately, requiring the opt-in feature also creates a footgun that is easy to hit: if a developer modifies the ISLE DSL source, but forgets to specify the Cargo feature, then the compiler will silently be built successfully with stale source, and will silently exclude any changes that were made. The generated source is checked into git for a good reason: we want DSL compiler to not affect build times for the overwhelmingly common case that Cranelift is used as a dependency but the backends are not being actively developed. (This overhead comes mainly from building `islec` itself.) So, what to do? This PR implements a middle ground first described in [this conversation](https://github.com/bytecodealliance/wasmtime/pull/3506#discussion_r743113351), in which we: - Generate a hash (SHA-512) of the ISLE DSL source and produce a "manifest" of ISLE inputs alongside the generated source; and - Always read the ISLE DSL source, and see if the manifest is still valid, on builds that do not have the opt-in "rebuild" feature. This allows us to know whether the ISLE compiler output would have been the same (modulo changes to the DSL compiler itself, which are out-of-scope here), without actually building the ISLE compiler and running it. If the compiler-backend developer modifies an ISLE source file and then tries to build `cranelift-codegen` without adding the `rebuild-isle` Cargo feature, they get the following output: ```text Error: the ISLE source files that resulted in the generated Rust source * src/isa/x64/lower/isle/generated_code.rs have changed but the generated source was not rebuilt! These ISLE source files are: * src/clif.isle * src/prelude.isle * src/isa/x64/inst.isle * src/isa/x64/lower.isle Please add `--features rebuild-isle` to your `cargo build` command if you wish to rebuild the generated source, then include these changes in any git commits you make that include the changes to the ISLE. For example: $ cargo build -p cranelift-codegen --features rebuild-isle (This build script cannot do this for you by default because we cannot modify checked-into-git source without your explicit opt-in.) ``` which will tell them exactly what they need to do to fix the problem! Note that I am leaving the "Rebuild ISLE" CI job alone for now, because otherwise, we are trusting whomever submits a PR to generate the correct generated source. In other words, the manifest is a communication from the checked-in tree to the developer, but we still need to verify that the checked-in generated Rust source and the manifest are correct with respect to the checked-in ISLE source.
3 years ago
"cpufeatures",
"opaque-debug",
"universal-hash",
]
[[package]]
name = "polyval"
version = "0.5.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8419d2b623c7c0896ff2d5d96e2cb4ede590fed28fcc34934f4c33c036e620a1"
dependencies = [
Remove support for userfaultfd (#4040) This commit removes support for the `userfaultfd` or "uffd" syscall on Linux. This support was originally added for users migrating from Lucet to Wasmtime, but the recent developments of kernel-supported copy-on-write support for memory initialization wound up being more appropriate for these use cases than usefaultfd. The main reason for moving to copy-on-write initialization are: * The `userfaultfd` feature was never necessarily intended for this style of use case with wasm and was susceptible to subtle and rare bugs that were extremely difficult to track down. We were never 100% certain that there were kernel bugs related to userfaultfd but the suspicion never went away. * Handling faults with userfaultfd was always slow and single-threaded. Only one thread could handle faults and traveling to user-space to handle faults is inherently slower than handling them all in the kernel. The single-threaded aspect in particular presented a significant scaling bottleneck for embeddings that want to run many wasm instances in parallel. * One of the major benefits of userfaultfd was lazy initialization of wasm linear memory which is also achieved with the copy-on-write initialization support we have right now. * One of the suspected benefits of userfaultfd was less frobbing of the kernel vma structures when wasm modules are instantiated. Currently the copy-on-write support has a mitigation where we attempt to reuse the memory images where possible to avoid changing vma structures. When comparing this to userfaultfd's performance it was found that kernel modifications of vmas aren't a worrisome bottleneck so copy-on-write is suitable for this as well. Overall there are no remaining benefits that userfaultfd gives that copy-on-write doesn't, and copy-on-write solves a major downsides of userfaultfd, the scaling issue with a single faulting thread. Additionally copy-on-write support seems much more robust in terms of kernel implementation since it's only using standard memory-management syscalls which are heavily exercised. Finally copy-on-write support provides a new bonus where read-only memory in WebAssembly can be mapped directly to the same kernel cache page, even amongst many wasm instances of the same module, which was never possible with userfaultfd. In light of all this it's expected that all users of userfaultfd should migrate to the copy-on-write initialization of Wasmtime (which is enabled by default).
3 years ago
"cfg-if",
ISLE: guard against stale generated source in default build config. Currently, the `build.rs` script that generates Rust source from the ISLE DSL will only do this generation if the `rebuild-isle` Cargo feature is specified. By default, it is not. This is based on the principle that we (the build script) do not modify the source tree as managed by git; git-managed files are strictly a human-managed and human-edited resource. By adding the opt-in Cargo feature, a developer is requesting the build script to perform an explicit action. (In my understanding at least, this principle comes from the general philosophy of hermetic builds: the output should be a pure function of the input, and part of this is that the input is read-only. If we modify the source tree, then all bets are off.) Unfortunately, requiring the opt-in feature also creates a footgun that is easy to hit: if a developer modifies the ISLE DSL source, but forgets to specify the Cargo feature, then the compiler will silently be built successfully with stale source, and will silently exclude any changes that were made. The generated source is checked into git for a good reason: we want DSL compiler to not affect build times for the overwhelmingly common case that Cranelift is used as a dependency but the backends are not being actively developed. (This overhead comes mainly from building `islec` itself.) So, what to do? This PR implements a middle ground first described in [this conversation](https://github.com/bytecodealliance/wasmtime/pull/3506#discussion_r743113351), in which we: - Generate a hash (SHA-512) of the ISLE DSL source and produce a "manifest" of ISLE inputs alongside the generated source; and - Always read the ISLE DSL source, and see if the manifest is still valid, on builds that do not have the opt-in "rebuild" feature. This allows us to know whether the ISLE compiler output would have been the same (modulo changes to the DSL compiler itself, which are out-of-scope here), without actually building the ISLE compiler and running it. If the compiler-backend developer modifies an ISLE source file and then tries to build `cranelift-codegen` without adding the `rebuild-isle` Cargo feature, they get the following output: ```text Error: the ISLE source files that resulted in the generated Rust source * src/isa/x64/lower/isle/generated_code.rs have changed but the generated source was not rebuilt! These ISLE source files are: * src/clif.isle * src/prelude.isle * src/isa/x64/inst.isle * src/isa/x64/lower.isle Please add `--features rebuild-isle` to your `cargo build` command if you wish to rebuild the generated source, then include these changes in any git commits you make that include the changes to the ISLE. For example: $ cargo build -p cranelift-codegen --features rebuild-isle (This build script cannot do this for you by default because we cannot modify checked-into-git source without your explicit opt-in.) ``` which will tell them exactly what they need to do to fix the problem! Note that I am leaving the "Rebuild ISLE" CI job alone for now, because otherwise, we are trusting whomever submits a PR to generate the correct generated source. In other words, the manifest is a communication from the checked-in tree to the developer, but we still need to verify that the checked-in generated Rust source and the manifest are correct with respect to the checked-in ISLE source.
3 years ago
"cpufeatures",
"opaque-debug",
"universal-hash",
]
[[package]]
name = "ppv-lite86"
version = "0.2.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "eb9f9e6e233e5c4a35559a617bf40a4ec447db2e84c20b55a6f83167b7e57872"
[[package]]
name = "pqcrypto"
version = "0.14.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9da39bd0587bff4189521766c34f3203263926f7527906578a96d22a81a700d5"
dependencies = [
"pqcrypto-kyber",
"pqcrypto-traits",
]
[[package]]
name = "pqcrypto-internals"
version = "0.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0127cbc0239f585139a56effd7867921eae3425a000a72dde2b0a156062346b2"
dependencies = [
"cc",
"dunce",
"getrandom 0.2.6",
"libc",
]
[[package]]
name = "pqcrypto-kyber"
version = "0.7.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8a17989a978f7d7c1496e38806ad9ff11f36eb8e419c562eafddbbf176af4a8a"
dependencies = [
"cc",
"glob",
"libc",
"pqcrypto-internals",
"pqcrypto-traits",
]
[[package]]
name = "pqcrypto-traits"
version = "0.3.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "97e91cb6af081c6daad5fa705f8adb0634c027662052cb3174bdf2957bf07e25"
[[package]]
name = "pretty_env_logger"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "926d36b9553851b8b0005f1275891b392ee4d2d833852c417ed025477350fb9d"
dependencies = [
"env_logger 0.7.1",
"log",
]
Use structopt instead of docopt. This commit refactors the Wasmtime CLI tools to use `structopt` instead of `docopt`. The `wasmtime` tool now has the following subcommands: * `config new` - creates a new Wasmtime configuration file. * `run` - runs a WebAssembly module. * `wasm2obj` - translates a Wasm module to native object file. * `wast` - runs a test script file. If no subcommand is specified, the `run` subcommand is used. Thus, `wasmtime foo.wasm` should continue to function as expected. The `wasm2obj` and `wast` tools still exist, but delegate to the same implementation as the `wasmtime` subcommands. The standalone `wasm2obj` and `wast` tools may be removed in the future in favor of simply using `wasmtime`. Included in this commit is a breaking change to the default Wasmtime configuration file: it has been renamed from `wasmtime-cache-config.toml` to simply `config.toml`. The new name is less specific which will allow for additional (non-cache-related) settings in the future. There are some breaking changes to improve command line UX: * The `--cache-config` option has been renamed to `--config`. * The `--create-config-file` option has moved to the `config new` subcommand. As a result, the `wasm2obj` and `wast` tools cannot be used to create a new config file. * The short form of the `--optimize` option has changed from `-o` to `-O` for consistency. * The `wasm2obj` command takes the output object file as a required positional argument rather than the former required output *option* (e.g. `wasmtime wasm2obj foo.wasm foo.obj`).
5 years ago
[[package]]
name = "proc-macro-error"
version = "1.0.4"
Use structopt instead of docopt. This commit refactors the Wasmtime CLI tools to use `structopt` instead of `docopt`. The `wasmtime` tool now has the following subcommands: * `config new` - creates a new Wasmtime configuration file. * `run` - runs a WebAssembly module. * `wasm2obj` - translates a Wasm module to native object file. * `wast` - runs a test script file. If no subcommand is specified, the `run` subcommand is used. Thus, `wasmtime foo.wasm` should continue to function as expected. The `wasm2obj` and `wast` tools still exist, but delegate to the same implementation as the `wasmtime` subcommands. The standalone `wasm2obj` and `wast` tools may be removed in the future in favor of simply using `wasmtime`. Included in this commit is a breaking change to the default Wasmtime configuration file: it has been renamed from `wasmtime-cache-config.toml` to simply `config.toml`. The new name is less specific which will allow for additional (non-cache-related) settings in the future. There are some breaking changes to improve command line UX: * The `--cache-config` option has been renamed to `--config`. * The `--create-config-file` option has moved to the `config new` subcommand. As a result, the `wasm2obj` and `wast` tools cannot be used to create a new config file. * The short form of the `--optimize` option has changed from `-o` to `-O` for consistency. * The `wasm2obj` command takes the output object file as a required positional argument rather than the former required output *option* (e.g. `wasmtime wasm2obj foo.wasm foo.obj`).
5 years ago
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c"
Use structopt instead of docopt. This commit refactors the Wasmtime CLI tools to use `structopt` instead of `docopt`. The `wasmtime` tool now has the following subcommands: * `config new` - creates a new Wasmtime configuration file. * `run` - runs a WebAssembly module. * `wasm2obj` - translates a Wasm module to native object file. * `wast` - runs a test script file. If no subcommand is specified, the `run` subcommand is used. Thus, `wasmtime foo.wasm` should continue to function as expected. The `wasm2obj` and `wast` tools still exist, but delegate to the same implementation as the `wasmtime` subcommands. The standalone `wasm2obj` and `wast` tools may be removed in the future in favor of simply using `wasmtime`. Included in this commit is a breaking change to the default Wasmtime configuration file: it has been renamed from `wasmtime-cache-config.toml` to simply `config.toml`. The new name is less specific which will allow for additional (non-cache-related) settings in the future. There are some breaking changes to improve command line UX: * The `--cache-config` option has been renamed to `--config`. * The `--create-config-file` option has moved to the `config new` subcommand. As a result, the `wasm2obj` and `wast` tools cannot be used to create a new config file. * The short form of the `--optimize` option has changed from `-o` to `-O` for consistency. * The `wasm2obj` command takes the output object file as a required positional argument rather than the former required output *option* (e.g. `wasmtime wasm2obj foo.wasm foo.obj`).
5 years ago
dependencies = [
"proc-macro-error-attr",
"proc-macro2",
"quote",
"syn",
"version_check",
Use structopt instead of docopt. This commit refactors the Wasmtime CLI tools to use `structopt` instead of `docopt`. The `wasmtime` tool now has the following subcommands: * `config new` - creates a new Wasmtime configuration file. * `run` - runs a WebAssembly module. * `wasm2obj` - translates a Wasm module to native object file. * `wast` - runs a test script file. If no subcommand is specified, the `run` subcommand is used. Thus, `wasmtime foo.wasm` should continue to function as expected. The `wasm2obj` and `wast` tools still exist, but delegate to the same implementation as the `wasmtime` subcommands. The standalone `wasm2obj` and `wast` tools may be removed in the future in favor of simply using `wasmtime`. Included in this commit is a breaking change to the default Wasmtime configuration file: it has been renamed from `wasmtime-cache-config.toml` to simply `config.toml`. The new name is less specific which will allow for additional (non-cache-related) settings in the future. There are some breaking changes to improve command line UX: * The `--cache-config` option has been renamed to `--config`. * The `--create-config-file` option has moved to the `config new` subcommand. As a result, the `wasm2obj` and `wast` tools cannot be used to create a new config file. * The short form of the `--optimize` option has changed from `-o` to `-O` for consistency. * The `wasm2obj` command takes the output object file as a required positional argument rather than the former required output *option* (e.g. `wasmtime wasm2obj foo.wasm foo.obj`).
5 years ago
]
[[package]]
name = "proc-macro-error-attr"
version = "1.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869"
dependencies = [
"proc-macro2",
"quote",
"version_check",
]
[[package]]
name = "proc-macro2"
version = "1.0.37"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ec757218438d5fda206afc041538b2f6d889286160d649a86a24d37e1235afd1"
dependencies = [
"unicode-xid",
]
[[package]]
name = "proptest"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1e0d9cc07f18492d879586c92b485def06bc850da3118075cd45d50e9c95b0e5"
dependencies = [
"bit-set",
"bitflags",
"byteorder",
"lazy_static",
"num-traits",
"quick-error 2.0.1",
"rand 0.8.5",
"rand_chacha 0.3.1",
"rand_xorshift",
"regex-syntax",
"rusty-fork",
"tempfile",
]
[[package]]
name = "psm"
version = "0.1.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "871372391786ccec00d3c5d3d6608905b3d4db263639cfe075d3b60a736d115a"
dependencies = [
"cc",
]
[[package]]
name = "pulldown-cmark"
version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ffade02495f22453cd593159ea2f59827aae7f53fa8323f756799b670881dcf8"
dependencies = [
"bitflags",
"memchr",
"unicase",
]
[[package]]
name = "quick-error"
version = "1.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0"
[[package]]
name = "quick-error"
version = "2.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a993555f31e5a609f617c12db6250dedcac1b0a85076912c436e6fc9b2c8e6a3"
[[package]]
name = "quote"
version = "1.0.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a1feb54ed693b93a84e14094943b84b7c4eae204c512b7ccb95ab0c66d278ad1"
dependencies = [
"proc-macro2",
]
[[package]]
name = "rand"
Run `cargo update` on dependencies (#868) Looks like we're able to actually drop a good number of various deps, especially some large-ish ones like syn 0.14! Adding proc-macro2 v1.0.8 Adding syn v1.0.14 Removing base64 v0.10.1 Removing cloudabi v0.0.3 Removing crossbeam-utils v0.6.6 Removing failure v0.1.6 Removing failure_derive v0.1.6 Removing fuchsia-cprng v0.1.1 Removing proc-macro2 v0.4.30 Removing proc-macro2 v1.0.7 Removing quote v0.6.13 Removing rand_core v0.3.1 Removing rand_core v0.4.2 Removing rand_os v0.1.3 Removing rdrand v0.4.0 Removing syn v0.14.9 Removing syn v1.0.13 Removing synstructure v0.9.0 Removing unicode-xid v0.1.0 Removing wincolor v1.0.2 Updating arbitrary v0.3.2 -> v0.3.3 Updating arrayref v0.3.5 -> v0.3.6 Updating constant_time_eq v0.1.4 -> v0.1.5 Updating crates.io index Updating derive_arbitrary v0.3.1 -> v0.3.3 Updating indexmap v1.3.0 -> v1.3.1 Updating itoa v0.4.4 -> v0.4.5 Updating jobserver v0.1.18 -> v0.1.19 Updating memchr v2.2.1 -> v2.3.0 Updating num_cpus v1.11.1 -> v1.12.0 Updating proc-macro-error v0.4.4 -> v0.4.5 Updating proc-macro-error-attr v0.4.3 -> v0.4.5 Updating quickcheck v0.9.0 -> v0.9.2 Updating rand v0.7.2 -> v0.7.3 Updating redox_users v0.3.1 -> v0.3.4 Updating rust-argon2 v0.5.1 -> v0.7.0 Updating rustversion v1.0.1 -> v1.0.2 Updating serde_json v1.0.44 -> v1.0.45 Updating structopt v0.3.7 -> v0.3.8 Updating structopt-derive v0.4.0 -> v0.4.1 Updating termcolor v1.0.5 -> v1.1.0 Updating thread_local v1.0.0 -> v1.0.1 Updating toml v0.5.5 -> v0.5.6 Updating winapi-util v0.1.2 -> v0.1.3
5 years ago
version = "0.7.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
Run `cargo update` on dependencies (#868) Looks like we're able to actually drop a good number of various deps, especially some large-ish ones like syn 0.14! Adding proc-macro2 v1.0.8 Adding syn v1.0.14 Removing base64 v0.10.1 Removing cloudabi v0.0.3 Removing crossbeam-utils v0.6.6 Removing failure v0.1.6 Removing failure_derive v0.1.6 Removing fuchsia-cprng v0.1.1 Removing proc-macro2 v0.4.30 Removing proc-macro2 v1.0.7 Removing quote v0.6.13 Removing rand_core v0.3.1 Removing rand_core v0.4.2 Removing rand_os v0.1.3 Removing rdrand v0.4.0 Removing syn v0.14.9 Removing syn v1.0.13 Removing synstructure v0.9.0 Removing unicode-xid v0.1.0 Removing wincolor v1.0.2 Updating arbitrary v0.3.2 -> v0.3.3 Updating arrayref v0.3.5 -> v0.3.6 Updating constant_time_eq v0.1.4 -> v0.1.5 Updating crates.io index Updating derive_arbitrary v0.3.1 -> v0.3.3 Updating indexmap v1.3.0 -> v1.3.1 Updating itoa v0.4.4 -> v0.4.5 Updating jobserver v0.1.18 -> v0.1.19 Updating memchr v2.2.1 -> v2.3.0 Updating num_cpus v1.11.1 -> v1.12.0 Updating proc-macro-error v0.4.4 -> v0.4.5 Updating proc-macro-error-attr v0.4.3 -> v0.4.5 Updating quickcheck v0.9.0 -> v0.9.2 Updating rand v0.7.2 -> v0.7.3 Updating redox_users v0.3.1 -> v0.3.4 Updating rust-argon2 v0.5.1 -> v0.7.0 Updating rustversion v1.0.1 -> v1.0.2 Updating serde_json v1.0.44 -> v1.0.45 Updating structopt v0.3.7 -> v0.3.8 Updating structopt-derive v0.4.0 -> v0.4.1 Updating termcolor v1.0.5 -> v1.1.0 Updating thread_local v1.0.0 -> v1.0.1 Updating toml v0.5.5 -> v0.5.6 Updating winapi-util v0.1.2 -> v0.1.3
5 years ago
checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03"
dependencies = [
"getrandom 0.1.16",
"libc",
"rand_chacha 0.2.2",
"rand_core 0.5.1",
"rand_hc",
]
[[package]]
name = "rand"
version = "0.8.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404"
dependencies = [
"libc",
"rand_chacha 0.3.1",
"rand_core 0.6.3",
]
[[package]]
name = "rand_chacha"
version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402"
dependencies = [
"ppv-lite86",
"rand_core 0.5.1",
]
[[package]]
name = "rand_chacha"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88"
dependencies = [
"ppv-lite86",
"rand_core 0.6.3",
]
[[package]]
name = "rand_core"
version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19"
dependencies = [
"getrandom 0.1.16",
]
[[package]]
name = "rand_core"
version = "0.6.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d34f1408f55294453790c48b2f1ebbb1c5b4b7563eb1f418bcfcfdbb06ebb4e7"
dependencies = [
"getrandom 0.2.6",
]
[[package]]
name = "rand_hc"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c"
dependencies = [
"rand_core 0.5.1",
]
[[package]]
name = "rand_xorshift"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d25bf25ec5ae4a3f1b92f929810509a2f53d7dca2f50b794ff57e3face536c8f"
dependencies = [
"rand_core 0.6.3",
]
[[package]]
name = "rawbytes"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "04d0088f16afb86d12c7f239d8de4637fa68ecc99a3db227e1ab58a294713e60"
[[package]]
name = "rayon"
version = "1.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fd249e82c21598a9a426a4e00dd7adc1d640b22445ec8545feef801d1a74c221"
dependencies = [
"autocfg 1.1.0",
"crossbeam-deque",
"either",
"rayon-core",
]
[[package]]
name = "rayon-core"
version = "1.9.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9f51245e1e62e1f1629cbfec37b5793bbabcaeb90f30e94d2ba03564687353e4"
dependencies = [
"crossbeam-channel",
"crossbeam-deque",
4 years ago
"crossbeam-utils",
"num_cpus",
]
[[package]]
name = "redox_syscall"
version = "0.2.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "62f25bc4c7e55e0b0b7a1d43fb893f4fa1361d0abe38b9ce4f323c2adfe6ef42"
dependencies = [
"bitflags",
]
[[package]]
name = "redox_users"
version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b"
dependencies = [
"getrandom 0.2.6",
"redox_syscall",
"thiserror",
]
[[package]]
name = "regalloc2"
version = "0.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "80535183cae11b149d618fbd3c37e38d7cda589d82d7769e196ca9a9042d7621"
dependencies = [
"fxhash",
"log",
"serde",
"slice-group-by",
"smallvec",
]
[[package]]
name = "regex"
version = "1.5.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1a11647b6b25ff05a515cb92c365cec08801e83423a235b51e231e1808747286"
dependencies = [
"aho-corasick",
"memchr",
"regex-syntax",
]
[[package]]
name = "regex-automata"
version = "0.1.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132"
[[package]]
name = "regex-syntax"
version = "0.6.25"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f497285884f3fcff424ffc933e56d7cbca511def0c9831a7f9b5f6153e3cc89b"
[[package]]
name = "region"
version = "2.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "877e54ea2adcd70d80e9179344c97f93ef0dffd6b03e1f4529e6e83ab2fa9ae0"
dependencies = [
"bitflags",
"libc",
"mach",
"winapi",
]
[[package]]
name = "remove_dir_all"
version = "0.5.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7"
dependencies = [
"winapi",
]
[[package]]
name = "rsa"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e05c2603e2823634ab331437001b411b9ed11660fbc4066f3908c84a9439260d"
dependencies = [
"byteorder",
"digest 0.9.0",
"lazy_static",
"num-bigint-dig",
"num-integer",
"num-iter",
"num-traits",
"pkcs1",
"pkcs8",
"rand 0.8.5",
"serde",
"subtle",
"zeroize",
]
[[package]]
name = "rustc-demangle"
version = "0.1.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7ef03e0a2b150c7a90d01faf6254c9c48a41e95fb2a8c2ac1c6f0d2b9aefc342"
[[package]]
name = "rustix"
version = "0.36.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f43abb88211988493c1abb44a70efa56ff0ce98f233b7b276146f1f3f7ba9644"
dependencies = [
"bitflags",
"errno",
"io-lifetimes",
"itoa",
"libc",
"linux-raw-sys",
"once_cell",
"windows-sys",
]
[[package]]
name = "rusty-fork"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cb3dcc6e454c328bb824492db107ab7c0ae8fcffe4ad210136ef014458c1bc4f"
dependencies = [
"fnv",
"quick-error 1.2.3",
"tempfile",
"wait-timeout",
]
[[package]]
name = "ryu"
version = "1.0.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "73b4b750c782965c211b42f022f59af1fbceabdd026623714f104152f1ec149f"
[[package]]
name = "same-file"
version = "1.0.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502"
dependencies = [
"winapi-util",
]
[[package]]
name = "scopeguard"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
[[package]]
name = "serde"
version = "1.0.137"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "61ea8d54c77f8315140a05f4c7237403bf38b72704d031543aa1d16abbf517d1"
dependencies = [
"serde_derive",
]
[[package]]
name = "serde_derive"
version = "1.0.137"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1f26faba0c3959972377d3b2d306ee9f71faee9714294e41bb777f83f88578be"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "serde_json"
version = "1.0.80"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f972498cf015f7c0746cac89ebe1d6ef10c293b94175a243a2d9442c163d9944"
dependencies = [
"itoa",
"ryu",
"serde",
]
[[package]]
name = "sha2"
version = "0.9.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4d58a1e1bf39749807d89cf2d98ac2dfa0ff1cb3faa38fbb64dd88ac8013d800"
dependencies = [
"block-buffer 0.9.0",
Remove support for userfaultfd (#4040) This commit removes support for the `userfaultfd` or "uffd" syscall on Linux. This support was originally added for users migrating from Lucet to Wasmtime, but the recent developments of kernel-supported copy-on-write support for memory initialization wound up being more appropriate for these use cases than usefaultfd. The main reason for moving to copy-on-write initialization are: * The `userfaultfd` feature was never necessarily intended for this style of use case with wasm and was susceptible to subtle and rare bugs that were extremely difficult to track down. We were never 100% certain that there were kernel bugs related to userfaultfd but the suspicion never went away. * Handling faults with userfaultfd was always slow and single-threaded. Only one thread could handle faults and traveling to user-space to handle faults is inherently slower than handling them all in the kernel. The single-threaded aspect in particular presented a significant scaling bottleneck for embeddings that want to run many wasm instances in parallel. * One of the major benefits of userfaultfd was lazy initialization of wasm linear memory which is also achieved with the copy-on-write initialization support we have right now. * One of the suspected benefits of userfaultfd was less frobbing of the kernel vma structures when wasm modules are instantiated. Currently the copy-on-write support has a mitigation where we attempt to reuse the memory images where possible to avoid changing vma structures. When comparing this to userfaultfd's performance it was found that kernel modifications of vmas aren't a worrisome bottleneck so copy-on-write is suitable for this as well. Overall there are no remaining benefits that userfaultfd gives that copy-on-write doesn't, and copy-on-write solves a major downsides of userfaultfd, the scaling issue with a single faulting thread. Additionally copy-on-write support seems much more robust in terms of kernel implementation since it's only using standard memory-management syscalls which are heavily exercised. Finally copy-on-write support provides a new bonus where read-only memory in WebAssembly can be mapped directly to the same kernel cache page, even amongst many wasm instances of the same module, which was never possible with userfaultfd. In light of all this it's expected that all users of userfaultfd should migrate to the copy-on-write initialization of Wasmtime (which is enabled by default).
3 years ago
"cfg-if",
ISLE: guard against stale generated source in default build config. Currently, the `build.rs` script that generates Rust source from the ISLE DSL will only do this generation if the `rebuild-isle` Cargo feature is specified. By default, it is not. This is based on the principle that we (the build script) do not modify the source tree as managed by git; git-managed files are strictly a human-managed and human-edited resource. By adding the opt-in Cargo feature, a developer is requesting the build script to perform an explicit action. (In my understanding at least, this principle comes from the general philosophy of hermetic builds: the output should be a pure function of the input, and part of this is that the input is read-only. If we modify the source tree, then all bets are off.) Unfortunately, requiring the opt-in feature also creates a footgun that is easy to hit: if a developer modifies the ISLE DSL source, but forgets to specify the Cargo feature, then the compiler will silently be built successfully with stale source, and will silently exclude any changes that were made. The generated source is checked into git for a good reason: we want DSL compiler to not affect build times for the overwhelmingly common case that Cranelift is used as a dependency but the backends are not being actively developed. (This overhead comes mainly from building `islec` itself.) So, what to do? This PR implements a middle ground first described in [this conversation](https://github.com/bytecodealliance/wasmtime/pull/3506#discussion_r743113351), in which we: - Generate a hash (SHA-512) of the ISLE DSL source and produce a "manifest" of ISLE inputs alongside the generated source; and - Always read the ISLE DSL source, and see if the manifest is still valid, on builds that do not have the opt-in "rebuild" feature. This allows us to know whether the ISLE compiler output would have been the same (modulo changes to the DSL compiler itself, which are out-of-scope here), without actually building the ISLE compiler and running it. If the compiler-backend developer modifies an ISLE source file and then tries to build `cranelift-codegen` without adding the `rebuild-isle` Cargo feature, they get the following output: ```text Error: the ISLE source files that resulted in the generated Rust source * src/isa/x64/lower/isle/generated_code.rs have changed but the generated source was not rebuilt! These ISLE source files are: * src/clif.isle * src/prelude.isle * src/isa/x64/inst.isle * src/isa/x64/lower.isle Please add `--features rebuild-isle` to your `cargo build` command if you wish to rebuild the generated source, then include these changes in any git commits you make that include the changes to the ISLE. For example: $ cargo build -p cranelift-codegen --features rebuild-isle (This build script cannot do this for you by default because we cannot modify checked-into-git source without your explicit opt-in.) ``` which will tell them exactly what they need to do to fix the problem! Note that I am leaving the "Rebuild ISLE" CI job alone for now, because otherwise, we are trusting whomever submits a PR to generate the correct generated source. In other words, the manifest is a communication from the checked-in tree to the developer, but we still need to verify that the checked-in generated Rust source and the manifest are correct with respect to the checked-in ISLE source.
3 years ago
"cpufeatures",
"digest 0.9.0",
"opaque-debug",
]
[[package]]
name = "sha2"
version = "0.10.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "55deaec60f81eefe3cce0dc50bda92d6d8e88f2a27df7c5033b42afeb1ed2676"
dependencies = [
"cfg-if",
"cpufeatures",
"digest 0.10.3",
]
[[package]]
name = "sharded-slab"
version = "0.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "900fba806f70c630b0a382d0d825e17a0f19fcd059a2ade1ff237bcddf446b31"
dependencies = [
"lazy_static",
]
[[package]]
name = "shellexpand"
version = "2.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "83bdb7831b2d85ddf4a7b148aa19d0587eddbe8671a436b7bd1182eaad0f2829"
dependencies = [
"dirs-next",
]
[[package]]
name = "shuffling-allocator"
version = "1.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4ee9977fa98489d9006f4ab26fc5cbe2a139985baed09d2ec08dee6e506fc496"
dependencies = [
Remove support for userfaultfd (#4040) This commit removes support for the `userfaultfd` or "uffd" syscall on Linux. This support was originally added for users migrating from Lucet to Wasmtime, but the recent developments of kernel-supported copy-on-write support for memory initialization wound up being more appropriate for these use cases than usefaultfd. The main reason for moving to copy-on-write initialization are: * The `userfaultfd` feature was never necessarily intended for this style of use case with wasm and was susceptible to subtle and rare bugs that were extremely difficult to track down. We were never 100% certain that there were kernel bugs related to userfaultfd but the suspicion never went away. * Handling faults with userfaultfd was always slow and single-threaded. Only one thread could handle faults and traveling to user-space to handle faults is inherently slower than handling them all in the kernel. The single-threaded aspect in particular presented a significant scaling bottleneck for embeddings that want to run many wasm instances in parallel. * One of the major benefits of userfaultfd was lazy initialization of wasm linear memory which is also achieved with the copy-on-write initialization support we have right now. * One of the suspected benefits of userfaultfd was less frobbing of the kernel vma structures when wasm modules are instantiated. Currently the copy-on-write support has a mitigation where we attempt to reuse the memory images where possible to avoid changing vma structures. When comparing this to userfaultfd's performance it was found that kernel modifications of vmas aren't a worrisome bottleneck so copy-on-write is suitable for this as well. Overall there are no remaining benefits that userfaultfd gives that copy-on-write doesn't, and copy-on-write solves a major downsides of userfaultfd, the scaling issue with a single faulting thread. Additionally copy-on-write support seems much more robust in terms of kernel implementation since it's only using standard memory-management syscalls which are heavily exercised. Finally copy-on-write support provides a new bonus where read-only memory in WebAssembly can be mapped directly to the same kernel cache page, even amongst many wasm instances of the same module, which was never possible with userfaultfd. In light of all this it's expected that all users of userfaultfd should migrate to the copy-on-write initialization of Wasmtime (which is enabled by default).
3 years ago
"cfg-if",
"libc",
"rand 0.8.5",
"winapi",
]
[[package]]
name = "signature"
version = "1.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f2807892cfa58e081aa1f1111391c7a0649d4fa127a4ffbe34bcbfb35a1171a4"
dependencies = [
"digest 0.9.0",
"rand_core 0.6.3",
]
[[package]]
name = "similar"
version = "2.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2e24979f63a11545f5f2c60141afe249d4f19f84581ea2138065e400941d83d3"
[[package]]
name = "slice-group-by"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "03b634d87b960ab1a38c4fe143b508576f075e7c978bfad18217645ebfdfa2ec"
[[package]]
name = "smallvec"
version = "1.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f2dd574626839106c320a323308629dcb1acfc96e32a8cba364ddc61ac23ee83"
Implement an incremental compilation cache for Cranelift (#4551) This is the implementation of https://github.com/bytecodealliance/wasmtime/issues/4155, using the "inverted API" approach suggested by @cfallin (thanks!) in Cranelift, and trait object to provide a backend for an all-included experience in Wasmtime. After the suggestion of Chris, `Function` has been split into mostly two parts: - on the one hand, `FunctionStencil` contains all the fields required during compilation, and that act as a compilation cache key: if two function stencils are the same, then the result of their compilation (`CompiledCodeBase<Stencil>`) will be the same. This makes caching trivial, as the only thing to cache is the `FunctionStencil`. - on the other hand, `FunctionParameters` contain the... function parameters that are required to finalize the result of compilation into a `CompiledCode` (aka `CompiledCodeBase<Final>`) with proper final relocations etc., by applying fixups and so on. Most changes are here to accomodate those requirements, in particular that `FunctionStencil` should be `Hash`able to be used as a key in the cache: - most source locations are now relative to a base source location in the function, and as such they're encoded as `RelSourceLoc` in the `FunctionStencil`. This required changes so that there's no need to explicitly mark a `SourceLoc` as the base source location, it's automatically detected instead the first time a non-default `SourceLoc` is set. - user-defined external names in the `FunctionStencil` (aka before this patch `ExternalName::User { namespace, index }`) are now references into an external table of `UserExternalNameRef -> UserExternalName`, present in the `FunctionParameters`, and must be explicitly declared using `Function::declare_imported_user_function`. - some refactorings have been made for function names: - `ExternalName` was used as the type for a `Function`'s name; while it thus allowed `ExternalName::Libcall` in this place, this would have been quite confusing to use it there. Instead, a new enum `UserFuncName` is introduced for this name, that's either a user-defined function name (the above `UserExternalName`) or a test case name. - The future of `ExternalName` is likely to become a full reference into the `FunctionParameters`'s mapping, instead of being "either a handle for user-defined external names, or the thing itself for other variants". I'm running out of time to do this, and this is not trivial as it implies touching ISLE which I'm less familiar with. The cache computes a sha256 hash of the `FunctionStencil`, and uses this as the cache key. No equality check (using `PartialEq`) is performed in addition to the hash being the same, as we hope that this is sufficient data to avoid collisions. A basic fuzz target has been introduced that tries to do the bare minimum: - check that a function successfully compiled and cached will be also successfully reloaded from the cache, and returns the exact same function. - check that a trivial modification in the external mapping of `UserExternalNameRef -> UserExternalName` hits the cache, and that other modifications don't hit the cache. - This last check is less efficient and less likely to happen, so probably should be rethought a bit. Thanks to both @alexcrichton and @cfallin for your very useful feedback on Zulip. Some numbers show that for a large wasm module we're using internally, this is a 20% compile-time speedup, because so many `FunctionStencil`s are the same, even within a single module. For a group of modules that have a lot of code in common, we get hit rates up to 70% when they're used together. When a single function changes in a wasm module, every other function is reloaded; that's still slower than I expect (between 10% and 50% of the overall compile time), so there's likely room for improvement. Fixes #4155.
2 years ago
dependencies = [
"serde",
]
[[package]]
name = "socket2"
version = "0.4.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "66d72b759436ae32898a2af0a14218dbf55efde3feeb170eb623637db85ee1e0"
dependencies = [
"libc",
"winapi",
]
[[package]]
name = "souper-ir"
version = "2.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a50c18ce33988e1973003afbaa66e6a465ad7a614dc33f246879ccc209c2c044"
dependencies = [
"id-arena",
]
[[package]]
name = "spin"
version = "0.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d"
[[package]]
name = "spin"
version = "0.9.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7f6002a767bff9e83f8eeecf883ecb8011875a21ae8da43bffb817a57e78cc09"
[[package]]
name = "spki"
version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5c01a0c15da1b0b0e1494112e7af814a678fec9bd157881b49beac661e9b6f32"
dependencies = [
"der",
]
[[package]]
name = "stable_deref_trait"
version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3"
[[package]]
name = "static_assertions"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f"
[[package]]
name = "strsim"
version = "0.10.0"
Use structopt instead of docopt. This commit refactors the Wasmtime CLI tools to use `structopt` instead of `docopt`. The `wasmtime` tool now has the following subcommands: * `config new` - creates a new Wasmtime configuration file. * `run` - runs a WebAssembly module. * `wasm2obj` - translates a Wasm module to native object file. * `wast` - runs a test script file. If no subcommand is specified, the `run` subcommand is used. Thus, `wasmtime foo.wasm` should continue to function as expected. The `wasm2obj` and `wast` tools still exist, but delegate to the same implementation as the `wasmtime` subcommands. The standalone `wasm2obj` and `wast` tools may be removed in the future in favor of simply using `wasmtime`. Included in this commit is a breaking change to the default Wasmtime configuration file: it has been renamed from `wasmtime-cache-config.toml` to simply `config.toml`. The new name is less specific which will allow for additional (non-cache-related) settings in the future. There are some breaking changes to improve command line UX: * The `--cache-config` option has been renamed to `--config`. * The `--create-config-file` option has moved to the `config new` subcommand. As a result, the `wasm2obj` and `wast` tools cannot be used to create a new config file. * The short form of the `--optimize` option has changed from `-o` to `-O` for consistency. * The `wasm2obj` command takes the output object file as a required positional argument rather than the former required output *option* (e.g. `wasmtime wasm2obj foo.wasm foo.obj`).
5 years ago
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623"
[[package]]
name = "subtle"
version = "2.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601"
[[package]]
name = "symbolic_expressions"
version = "5.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7c68d531d83ec6c531150584c42a4290911964d5f0d79132b193b67252a23b71"
[[package]]
name = "syn"
version = "1.0.92"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7ff7c592601f11445996a06f8ad0c27f094a58857c2f89e97974ab9235b92c52"
dependencies = [
"proc-macro2",
"quote",
"unicode-xid",
]
[[package]]
name = "synstructure"
version = "0.12.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f36bdaa60a83aca3921b5259d5400cbf5e90fc51931376a9bd4a0eb79aa7210f"
dependencies = [
"proc-macro2",
"quote",
"syn",
"unicode-xid",
]
[[package]]
name = "system-interface"
version = "0.25.4"
4 years ago
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f355df185d945435f24c51fda9bf01bea6acb6c0b753e1241e5cc05413a659d4"
dependencies = [
"bitflags",
"cap-fs-ext",
"cap-std",
"fd-lock",
"io-lifetimes",
"rustix",
"windows-sys",
"winx",
]
[[package]]
name = "target-lexicon"
version = "0.12.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d7fa7e55043acb85fca6b3c01485a2eeb6b69c5d21002e273c79e465f43b7ac1"
[[package]]
name = "tempfile"
version = "3.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5cdb1ef4eaeeaddc8fbd371e5017057064af0911902ef36b39801f67cc6d79e4"
dependencies = [
Remove support for userfaultfd (#4040) This commit removes support for the `userfaultfd` or "uffd" syscall on Linux. This support was originally added for users migrating from Lucet to Wasmtime, but the recent developments of kernel-supported copy-on-write support for memory initialization wound up being more appropriate for these use cases than usefaultfd. The main reason for moving to copy-on-write initialization are: * The `userfaultfd` feature was never necessarily intended for this style of use case with wasm and was susceptible to subtle and rare bugs that were extremely difficult to track down. We were never 100% certain that there were kernel bugs related to userfaultfd but the suspicion never went away. * Handling faults with userfaultfd was always slow and single-threaded. Only one thread could handle faults and traveling to user-space to handle faults is inherently slower than handling them all in the kernel. The single-threaded aspect in particular presented a significant scaling bottleneck for embeddings that want to run many wasm instances in parallel. * One of the major benefits of userfaultfd was lazy initialization of wasm linear memory which is also achieved with the copy-on-write initialization support we have right now. * One of the suspected benefits of userfaultfd was less frobbing of the kernel vma structures when wasm modules are instantiated. Currently the copy-on-write support has a mitigation where we attempt to reuse the memory images where possible to avoid changing vma structures. When comparing this to userfaultfd's performance it was found that kernel modifications of vmas aren't a worrisome bottleneck so copy-on-write is suitable for this as well. Overall there are no remaining benefits that userfaultfd gives that copy-on-write doesn't, and copy-on-write solves a major downsides of userfaultfd, the scaling issue with a single faulting thread. Additionally copy-on-write support seems much more robust in terms of kernel implementation since it's only using standard memory-management syscalls which are heavily exercised. Finally copy-on-write support provides a new bonus where read-only memory in WebAssembly can be mapped directly to the same kernel cache page, even amongst many wasm instances of the same module, which was never possible with userfaultfd. In light of all this it's expected that all users of userfaultfd should migrate to the copy-on-write initialization of Wasmtime (which is enabled by default).
3 years ago
"cfg-if",
"fastrand",
"libc",
"redox_syscall",
"remove_dir_all",
"winapi",
]
[[package]]
name = "termcolor"
version = "1.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bab24d30b911b2376f3a13cc2cd443142f0c81dda04c118693e35b3835757755"
dependencies = [
Run `cargo update` on dependencies (#868) Looks like we're able to actually drop a good number of various deps, especially some large-ish ones like syn 0.14! Adding proc-macro2 v1.0.8 Adding syn v1.0.14 Removing base64 v0.10.1 Removing cloudabi v0.0.3 Removing crossbeam-utils v0.6.6 Removing failure v0.1.6 Removing failure_derive v0.1.6 Removing fuchsia-cprng v0.1.1 Removing proc-macro2 v0.4.30 Removing proc-macro2 v1.0.7 Removing quote v0.6.13 Removing rand_core v0.3.1 Removing rand_core v0.4.2 Removing rand_os v0.1.3 Removing rdrand v0.4.0 Removing syn v0.14.9 Removing syn v1.0.13 Removing synstructure v0.9.0 Removing unicode-xid v0.1.0 Removing wincolor v1.0.2 Updating arbitrary v0.3.2 -> v0.3.3 Updating arrayref v0.3.5 -> v0.3.6 Updating constant_time_eq v0.1.4 -> v0.1.5 Updating crates.io index Updating derive_arbitrary v0.3.1 -> v0.3.3 Updating indexmap v1.3.0 -> v1.3.1 Updating itoa v0.4.4 -> v0.4.5 Updating jobserver v0.1.18 -> v0.1.19 Updating memchr v2.2.1 -> v2.3.0 Updating num_cpus v1.11.1 -> v1.12.0 Updating proc-macro-error v0.4.4 -> v0.4.5 Updating proc-macro-error-attr v0.4.3 -> v0.4.5 Updating quickcheck v0.9.0 -> v0.9.2 Updating rand v0.7.2 -> v0.7.3 Updating redox_users v0.3.1 -> v0.3.4 Updating rust-argon2 v0.5.1 -> v0.7.0 Updating rustversion v1.0.1 -> v1.0.2 Updating serde_json v1.0.44 -> v1.0.45 Updating structopt v0.3.7 -> v0.3.8 Updating structopt-derive v0.4.0 -> v0.4.1 Updating termcolor v1.0.5 -> v1.1.0 Updating thread_local v1.0.0 -> v1.0.1 Updating toml v0.5.5 -> v0.5.6 Updating winapi-util v0.1.2 -> v0.1.3
5 years ago
"winapi-util",
]
[[package]]
name = "terminal_size"
version = "0.1.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "633c1a546cee861a1a6d0dc69ebeca693bf4296661ba7852b9d21d159e0506df"
dependencies = [
"libc",
"winapi",
]
[[package]]
name = "test-programs"
version = "0.0.0"
dependencies = [
"anyhow",
"cap-std",
Remove support for userfaultfd (#4040) This commit removes support for the `userfaultfd` or "uffd" syscall on Linux. This support was originally added for users migrating from Lucet to Wasmtime, but the recent developments of kernel-supported copy-on-write support for memory initialization wound up being more appropriate for these use cases than usefaultfd. The main reason for moving to copy-on-write initialization are: * The `userfaultfd` feature was never necessarily intended for this style of use case with wasm and was susceptible to subtle and rare bugs that were extremely difficult to track down. We were never 100% certain that there were kernel bugs related to userfaultfd but the suspicion never went away. * Handling faults with userfaultfd was always slow and single-threaded. Only one thread could handle faults and traveling to user-space to handle faults is inherently slower than handling them all in the kernel. The single-threaded aspect in particular presented a significant scaling bottleneck for embeddings that want to run many wasm instances in parallel. * One of the major benefits of userfaultfd was lazy initialization of wasm linear memory which is also achieved with the copy-on-write initialization support we have right now. * One of the suspected benefits of userfaultfd was less frobbing of the kernel vma structures when wasm modules are instantiated. Currently the copy-on-write support has a mitigation where we attempt to reuse the memory images where possible to avoid changing vma structures. When comparing this to userfaultfd's performance it was found that kernel modifications of vmas aren't a worrisome bottleneck so copy-on-write is suitable for this as well. Overall there are no remaining benefits that userfaultfd gives that copy-on-write doesn't, and copy-on-write solves a major downsides of userfaultfd, the scaling issue with a single faulting thread. Additionally copy-on-write support seems much more robust in terms of kernel implementation since it's only using standard memory-management syscalls which are heavily exercised. Finally copy-on-write support provides a new bonus where read-only memory in WebAssembly can be mapped directly to the same kernel cache page, even amongst many wasm instances of the same module, which was never possible with userfaultfd. In light of all this it's expected that all users of userfaultfd should migrate to the copy-on-write initialization of Wasmtime (which is enabled by default).
3 years ago
"cfg-if",
"os_pipe",
"target-lexicon",
"tempfile",
4 years ago
"tokio",
"tracing-subscriber",
"wasi-cap-std-sync",
"wasi-common",
"wasmtime",
"wasmtime-wasi",
"wat",
]
[[package]]
name = "textwrap"
Cranelift: do not check in generated ISLE code; regenerate on every compile. (#4143) This PR fixes #4066: it modifies the Cranelift `build.rs` workflow to invoke the ISLE DSL compiler on every compilation, rather than only when the user specifies a special "rebuild ISLE" feature. The main benefit of this change is that it vastly simplifies the mental model required of developers, and removes a bunch of failure modes we have tried to work around in other ways. There is now just one "source of truth", the ISLE source itself, in the repository, and so there is no need to understand a special "rebuild" step and how to handle merge errors. There is no special process needed to develop the compiler when modifying the DSL. And there is no "noise" in the git history produced by constantly-regenerated files. The two main downsides we discussed in #4066 are: - Compile time could increase, by adding more to the "meta" step before the main build; - It becomes less obvious where the source definitions are (everything becomes more "magic"), which makes exploration and debugging harder. This PR addresses each of these concerns: 1. To maintain reasonable compile time, it includes work to cut down the dependencies of the `cranelift-isle` crate to *nothing* (only the Rust stdlib), in the default build. It does this by putting the error-reporting bits (`miette` crate) under an optional feature, and the logging (`log` crate) under a feature-controlled macro, and manually writing an `Error` impl rather than using `thiserror`. This completely avoids proc macros and the `syn` build slowness. The user can still get nice errors out of `miette`: this is enabled by specifying a Cargo feature `--features isle-errors`. 2. To allow the user to optionally inspect the generated source, which nominally lives in a hard-to-find path inside `target/` now, this PR adds a feature `isle-in-source-tree` that, as implied by the name, moves the target for ISLE generated source into the source tree, at `cranelift/codegen/isle_generated_source/`. It seems reasonable to do this when an explicit feature (opt-in) is specified because this is how ISLE regeneration currently works as well. To prevent surprises, if the feature is *not* specified, the build fails if this directory exists.
3 years ago
version = "0.15.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
Cranelift: do not check in generated ISLE code; regenerate on every compile. (#4143) This PR fixes #4066: it modifies the Cranelift `build.rs` workflow to invoke the ISLE DSL compiler on every compilation, rather than only when the user specifies a special "rebuild ISLE" feature. The main benefit of this change is that it vastly simplifies the mental model required of developers, and removes a bunch of failure modes we have tried to work around in other ways. There is now just one "source of truth", the ISLE source itself, in the repository, and so there is no need to understand a special "rebuild" step and how to handle merge errors. There is no special process needed to develop the compiler when modifying the DSL. And there is no "noise" in the git history produced by constantly-regenerated files. The two main downsides we discussed in #4066 are: - Compile time could increase, by adding more to the "meta" step before the main build; - It becomes less obvious where the source definitions are (everything becomes more "magic"), which makes exploration and debugging harder. This PR addresses each of these concerns: 1. To maintain reasonable compile time, it includes work to cut down the dependencies of the `cranelift-isle` crate to *nothing* (only the Rust stdlib), in the default build. It does this by putting the error-reporting bits (`miette` crate) under an optional feature, and the logging (`log` crate) under a feature-controlled macro, and manually writing an `Error` impl rather than using `thiserror`. This completely avoids proc macros and the `syn` build slowness. The user can still get nice errors out of `miette`: this is enabled by specifying a Cargo feature `--features isle-errors`. 2. To allow the user to optionally inspect the generated source, which nominally lives in a hard-to-find path inside `target/` now, this PR adds a feature `isle-in-source-tree` that, as implied by the name, moves the target for ISLE generated source into the source tree, at `cranelift/codegen/isle_generated_source/`. It seems reasonable to do this when an explicit feature (opt-in) is specified because this is how ISLE regeneration currently works as well. To prevent surprises, if the feature is *not* specified, the build fails if this directory exists.
3 years ago
checksum = "b1141d4d61095b28419e22cb0bbf02755f5e54e0526f97f1e3d1d160e60885fb"
[[package]]
name = "thiserror"
version = "1.0.31"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bd829fe32373d27f76265620b5309d0340cb8550f523c1dda251d6298069069a"
dependencies = [
"thiserror-impl",
]
[[package]]
name = "thiserror-impl"
version = "1.0.31"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0396bc89e626244658bef819e22d0cc459e795a5ebe878e6ec336d1674a8d79a"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "thread_local"
version = "1.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5516c27b78311c50bf42c071425c560ac799b11c30b31f87e3081965fe5e0180"
dependencies = [
"once_cell",
]
[[package]]
name = "tinytemplate"
version = "1.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc"
dependencies = [
"serde",
"serde_json",
]
[[package]]
name = "tinyvec"
version = "1.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50"
dependencies = [
"tinyvec_macros",
]
[[package]]
name = "tinyvec_macros"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c"
[[package]]
name = "tokio"
version = "1.18.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8bfb875c82dc0a4f1f37a30e720dee181a2b3a06a428b0fc6873ea38d6407850"
dependencies = [
"bytes",
4 years ago
"libc",
"memchr",
4 years ago
"mio",
"num_cpus",
"pin-project-lite",
"socket2",
"tokio-macros",
"winapi",
]
[[package]]
name = "tokio-macros"
version = "1.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b557f72f448c511a979e2564e55d74e6c4432fc96ff4f6241bc6bded342643b7"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "toml"
version = "0.5.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8d82e1a7758622a465f8cee077614c73484dac5b836c02ff6a40d5d1010324d7"
dependencies = [
"serde",
]
[[package]]
name = "tracing"
version = "0.1.34"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5d0ecdcb44a79f0fe9844f0c4f33a342cbcbb5117de8001e6ba0dc2351327d09"
dependencies = [
Remove support for userfaultfd (#4040) This commit removes support for the `userfaultfd` or "uffd" syscall on Linux. This support was originally added for users migrating from Lucet to Wasmtime, but the recent developments of kernel-supported copy-on-write support for memory initialization wound up being more appropriate for these use cases than usefaultfd. The main reason for moving to copy-on-write initialization are: * The `userfaultfd` feature was never necessarily intended for this style of use case with wasm and was susceptible to subtle and rare bugs that were extremely difficult to track down. We were never 100% certain that there were kernel bugs related to userfaultfd but the suspicion never went away. * Handling faults with userfaultfd was always slow and single-threaded. Only one thread could handle faults and traveling to user-space to handle faults is inherently slower than handling them all in the kernel. The single-threaded aspect in particular presented a significant scaling bottleneck for embeddings that want to run many wasm instances in parallel. * One of the major benefits of userfaultfd was lazy initialization of wasm linear memory which is also achieved with the copy-on-write initialization support we have right now. * One of the suspected benefits of userfaultfd was less frobbing of the kernel vma structures when wasm modules are instantiated. Currently the copy-on-write support has a mitigation where we attempt to reuse the memory images where possible to avoid changing vma structures. When comparing this to userfaultfd's performance it was found that kernel modifications of vmas aren't a worrisome bottleneck so copy-on-write is suitable for this as well. Overall there are no remaining benefits that userfaultfd gives that copy-on-write doesn't, and copy-on-write solves a major downsides of userfaultfd, the scaling issue with a single faulting thread. Additionally copy-on-write support seems much more robust in terms of kernel implementation since it's only using standard memory-management syscalls which are heavily exercised. Finally copy-on-write support provides a new bonus where read-only memory in WebAssembly can be mapped directly to the same kernel cache page, even amongst many wasm instances of the same module, which was never possible with userfaultfd. In light of all this it's expected that all users of userfaultfd should migrate to the copy-on-write initialization of Wasmtime (which is enabled by default).
3 years ago
"cfg-if",
"log",
"pin-project-lite",
"tracing-attributes",
"tracing-core",
]
[[package]]
name = "tracing-attributes"
version = "0.1.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cc6b8ad3567499f98a1db7a752b07a7c8c7c7c34c332ec00effb2b0027974b7c"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "tracing-core"
version = "0.1.28"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7b7358be39f2f274f322d2aaed611acc57f382e8eb1e5b48cb9ae30933495ce7"
dependencies = [
"once_cell",
"valuable",
]
[[package]]
name = "tracing-subscriber"
version = "0.3.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4bc28f93baff38037f64e6f43d34cfa1605f27a49c34e8a04c5e78b0babf2596"
dependencies = [
"sharded-slab",
"thread_local",
"tracing-core",
]
[[package]]
name = "typenum"
version = "1.15.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dcf81ac59edc17cc8697ff311e8f5ef2d99fcbd9817b34cec66f90b6c3dfd987"
[[package]]
name = "unicase"
version = "2.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "50f37be617794602aabbeee0be4f259dc1778fabe05e2d67ee8f79326d5cb4f6"
dependencies = [
"version_check",
]
[[package]]
name = "unicode-bidi"
version = "0.3.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "099b7128301d285f79ddd55b9a83d5e6b9e97c92e0ea0daebee7263e932de992"
[[package]]
name = "unicode-normalization"
version = "0.1.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "854cbdc4f7bc6ae19c820d44abdc3277ac3e1b2b93db20a636825d9322fb60e6"
dependencies = [
"tinyvec",
]
[[package]]
name = "unicode-width"
version = "0.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3ed742d4ea2bd1176e236172c8429aaf54486e7ac098db29ffe6529e0ce50973"
[[package]]
name = "unicode-xid"
version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "957e51f3646910546462e67d5f7599b9e4fb8acdd304b087a6494730f9eebf04"
[[package]]
name = "universal-hash"
version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9f214e8f697e925001e66ec2c6e37a4ef93f0f78c2eed7814394e10c62025b05"
dependencies = [
"generic-array",
"subtle",
]
[[package]]
name = "url"
version = "2.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0d68c799ae75762b8c3fe375feb6600ef5602c883c5d21eb51c09f22b83c4643"
dependencies = [
"form_urlencoded",
"idna",
"percent-encoding",
]
[[package]]
name = "uuid"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8cfcd319456c4d6ea10087ed423473267e1a071f3bc0aa89f80d60997843c6f0"
dependencies = [
"getrandom 0.2.6",
]
[[package]]
name = "v8"
version = "0.44.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f3f92c29dd66c7342443280695afc5bb79d773c3aa3eb02978cf24f058ae2b3d"
dependencies = [
"bitflags",
"fslock",
"lazy_static",
"libc",
"which",
]
[[package]]
name = "valuable"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d"
[[package]]
name = "version_check"
version = "0.9.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f"
[[package]]
name = "wait-timeout"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9f200f5b12eb75f8c1ed65abd4b2db8a6e1b138a20de009dacee265a2498f3f6"
dependencies = [
"libc",
]
[[package]]
name = "walkdir"
version = "2.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "808cf2735cd4b6866113f648b791c6adc5714537bc222d9347bb203386ffda56"
dependencies = [
"same-file",
"winapi",
"winapi-util",
]
[[package]]
name = "wasi"
version = "0.9.0+wasi-snapshot-preview1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519"
[[package]]
name = "wasi"
version = "0.10.2+wasi-snapshot-preview1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6"
[[package]]
name = "wasi"
version = "0.11.0+wasi-snapshot-preview1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
[[package]]
name = "wasi-cap-std-sync"
version = "9.0.0"
dependencies = [
"anyhow",
4 years ago
"async-trait",
"cap-fs-ext",
"cap-rand",
"cap-std",
"cap-time-ext",
"fs-set-times",
"io-extras",
"io-lifetimes",
"is-terminal",
"once_cell",
"rustix",
"system-interface",
"tempfile",
"tracing",
"wasi-common",
"windows-sys",
]
[[package]]
name = "wasi-common"
version = "9.0.0"
dependencies = [
"anyhow",
"bitflags",
"cap-rand",
"cap-std",
"io-extras",
wasi-threads: an initial implementation (#5484) This commit includes a set of changes that add initial support for `wasi-threads` to Wasmtime: * feat: remove mutability from the WasiCtx Table This patch adds interior mutability to the WasiCtx Table and the Table elements. Major pain points: * `File` only needs `RwLock<cap_std::fs::File>` to implement `File::set_fdflags()` on Windows, because of [1] * Because `File` needs a `RwLock` and `RwLock*Guard` cannot be hold across an `.await`, The `async` from `async fn num_ready_bytes(&self)` had to be removed * Because `File` needs a `RwLock` and `RwLock*Guard` cannot be dereferenced in `pollable`, the signature of `fn pollable(&self) -> Option<rustix::fd::BorrowedFd>` changed to `fn pollable(&self) -> Option<Arc<dyn AsFd + '_>>` [1] https://github.com/bytecodealliance/system-interface/blob/da238e324e752033f315f09c082ad9ce35d42696/src/fs/fd_flags.rs#L210-L217 * wasi-threads: add an initial implementation This change is a first step toward implementing `wasi-threads` in Wasmtime. We may find that it has some missing pieces, but the core functionality is there: when `wasi::thread_spawn` is called by a running WebAssembly module, a function named `wasi_thread_start` is found in the module's exports and called in a new instance. The shared memory of the original instance is reused in the new instance. This new WASI proposal is in its early stages and details are still being hashed out in the [spec] and [wasi-libc] repositories. Due to its experimental state, the `wasi-threads` functionality is hidden behind both a compile-time and runtime flag: one must build with `--features wasi-threads` but also run the Wasmtime CLI with `--wasm-features threads` and `--wasi-modules experimental-wasi-threads`. One can experiment with `wasi-threads` by running: ```console $ cargo run --features wasi-threads -- \ --wasm-features threads --wasi-modules experimental-wasi-threads \ <a threads-enabled module> ``` Threads-enabled Wasm modules are not yet easy to build. Hopefully this is resolved soon, but in the meantime see the use of `THREAD_MODEL=posix` in the [wasi-libc] repository for some clues on what is necessary. Wiggle complicates things by requiring the Wasm memory to be exported with a certain name and `wasi-threads` also expects that memory to be imported; this build-time obstacle can be overcome with the `--import-memory --export-memory` flags only available in the latest Clang tree. Due to all of this, the included tests are written directly in WAT--run these with: ```console $ cargo test --features wasi-threads -p wasmtime-cli -- cli_tests ``` [spec]: https://github.com/WebAssembly/wasi-threads [wasi-libc]: https://github.com/WebAssembly/wasi-libc This change does not protect the WASI implementations themselves from concurrent access. This is already complete in previous commits or left for future commits in certain cases (e.g., wasi-nn). * wasi-threads: factor out process exit logic As is being discussed [elsewhere], either calling `proc_exit` or trapping in any thread should halt execution of all threads. The Wasmtime CLI already has logic for adapting a WebAssembly error code to a code expected in each OS. This change factors out this logic to a new function, `maybe_exit_on_error`, for use within the `wasi-threads` implementation. This will work reasonably well for CLI users of Wasmtime + `wasi-threads`, but embedders will want something better in the future: when a `wasi-threads` threads fails, they may not want their application to exit. Handling this is tricky, because it will require cancelling the threads spawned by the `wasi-threads` implementation, something that is not trivial to do in Rust. With this change, we defer that work until later in order to provide a working implementation of `wasi-threads` for experimentation. [elsewhere]: https://github.com/WebAssembly/wasi-threads/pull/17 * review: work around `fd_fdstat_set_flags` In order to make progress with wasi-threads, this change temporarily works around limitations induced by `wasi-common`'s `fd_fdstat_set_flags` to allow `&mut self` use in the implementation. Eventual resolution is tracked in https://github.com/bytecodealliance/wasmtime/issues/5643. This change makes several related helper functions (e.g., `set_fdflags`) take `&mut self` as well. * test: use `wait`/`notify` to improve `threads.wat` test Previously, the test simply executed in a loop for some hardcoded number of iterations. This changes uses `wait` and `notify` and atomic operations to keep track of when the spawned threads are done and join on the main thread appropriately. * various fixes and tweaks due to the PR review --------- Signed-off-by: Harald Hoyer <harald@profian.com> Co-authored-by: Harald Hoyer <harald@profian.com> Co-authored-by: Alex Crichton <alex@alexcrichton.com>
2 years ago
"log",
"rustix",
"thiserror",
"tracing",
"wasmtime",
It's wiggle time! (#1202) * Use wiggle in place of wig in wasi-common This is a rather massive commit that introduces `wiggle` into the picture. We still use `wig`'s macro in `old` snapshot and to generate `wasmtime-wasi` glue, but everything else is now autogenerated by `wiggle`. In summary, thanks to `wiggle`, we no longer need to worry about serialising and deserialising to and from the guest memory, and all guest (WASI) types are now proper idiomatic Rust types. While we're here, in preparation for the ephemeral snapshot, I went ahead and reorganised the internal structure of the crate. Instead of modules like `hostcalls_impl` or `hostcalls_impl::fs`, the structure now resembles that in ephemeral with modules like `path`, `fd`, etc. Now, I'm not requiring we leave it like this, but I reckon it looks cleaner this way after all. * Fix wig to use new first-class access to caller's mem * Ignore warning in proc_exit for the moment * Group unsafes together in args and environ calls * Simplify pwrite; more unsafe blocks * Simplify fd_read * Bundle up unsafes in fd_readdir * Simplify fd_write * Add comment to path_readlink re zero-len buffers * Simplify unsafes in random_get * Hide GuestPtr<str> to &str in path::get * Rewrite pread and pwrite using SeekFrom and read/write_vectored I've left the implementation of VirtualFs pretty much untouched as I don't feel that comfortable in changing the API too much. Having said that, I reckon `pread` and `pwrite` could be refactored out, and `preadv` and `pwritev` could be entirely rewritten using `seek` and `read_vectored` and `write_vectored`. * Add comment about VirtFs unsafety * Fix all mentions of FdEntry to Entry * Fix warnings on Win * Add aux struct EntryTable responsible for Fds and Entries This commit adds aux struct `EntryTable` which is private to `WasiCtx` and is basically responsible for `Fd` alloc/dealloc as well as storing matching `Entry`s. This struct is entirely private to `WasiCtx` and as such as should remain transparent to `WasiCtx` users. * Remove redundant check for empty buffer in path_readlink * Preserve and rewind file cursor in pread/pwrite * Use GuestPtr<[u8]>::copy_from_slice wherever copying bytes directly * Use GuestPtr<[u8]>::copy_from_slice in fd_readdir * Clean up unsafes around WasiCtx accessors * Fix bugs in args_get and environ_get * Fix conflicts after rebase
5 years ago
"wiggle",
"windows-sys",
]
[[package]]
name = "wasi-crypto"
version = "0.1.5"
dependencies = [
"aes-gcm",
"anyhow",
"bincode",
"byteorder",
"chacha20poly1305",
"curve25519-dalek",
"derivative",
"ed25519-dalek",
"hkdf",
"hmac",
"k256",
"p256",
"parking_lot",
"pqcrypto",
"rand_core 0.5.1",
"rand_core 0.6.3",
"rsa",
"serde",
"sha2 0.9.9",
"subtle",
"thiserror",
"xoodyak",
"zeroize",
]
[[package]]
name = "wasi-tokio"
version = "9.0.0"
dependencies = [
"anyhow",
"cap-std",
"cap-tempfile",
"io-extras",
"io-lifetimes",
"rustix",
"tempfile",
"tokio",
"wasi-cap-std-sync",
"wasi-common",
"wiggle",
]
[[package]]
name = "wasm-bindgen"
version = "0.2.80"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "27370197c907c55e3f1a9fbe26f44e937fe6451368324e009cba39e139dc08ad"
dependencies = [
Remove support for userfaultfd (#4040) This commit removes support for the `userfaultfd` or "uffd" syscall on Linux. This support was originally added for users migrating from Lucet to Wasmtime, but the recent developments of kernel-supported copy-on-write support for memory initialization wound up being more appropriate for these use cases than usefaultfd. The main reason for moving to copy-on-write initialization are: * The `userfaultfd` feature was never necessarily intended for this style of use case with wasm and was susceptible to subtle and rare bugs that were extremely difficult to track down. We were never 100% certain that there were kernel bugs related to userfaultfd but the suspicion never went away. * Handling faults with userfaultfd was always slow and single-threaded. Only one thread could handle faults and traveling to user-space to handle faults is inherently slower than handling them all in the kernel. The single-threaded aspect in particular presented a significant scaling bottleneck for embeddings that want to run many wasm instances in parallel. * One of the major benefits of userfaultfd was lazy initialization of wasm linear memory which is also achieved with the copy-on-write initialization support we have right now. * One of the suspected benefits of userfaultfd was less frobbing of the kernel vma structures when wasm modules are instantiated. Currently the copy-on-write support has a mitigation where we attempt to reuse the memory images where possible to avoid changing vma structures. When comparing this to userfaultfd's performance it was found that kernel modifications of vmas aren't a worrisome bottleneck so copy-on-write is suitable for this as well. Overall there are no remaining benefits that userfaultfd gives that copy-on-write doesn't, and copy-on-write solves a major downsides of userfaultfd, the scaling issue with a single faulting thread. Additionally copy-on-write support seems much more robust in terms of kernel implementation since it's only using standard memory-management syscalls which are heavily exercised. Finally copy-on-write support provides a new bonus where read-only memory in WebAssembly can be mapped directly to the same kernel cache page, even amongst many wasm instances of the same module, which was never possible with userfaultfd. In light of all this it's expected that all users of userfaultfd should migrate to the copy-on-write initialization of Wasmtime (which is enabled by default).
3 years ago
"cfg-if",
"wasm-bindgen-macro",
]
[[package]]
name = "wasm-bindgen-backend"
version = "0.2.80"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "53e04185bfa3a779273da532f5025e33398409573f348985af9a1cbf3774d3f4"
dependencies = [
"bumpalo",
"lazy_static",
"log",
"proc-macro2",
"quote",
"syn",
"wasm-bindgen-shared",
]
[[package]]
name = "wasm-bindgen-macro"
version = "0.2.80"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "17cae7ff784d7e83a2fe7611cfe766ecf034111b49deb850a3dc7699c08251f5"
dependencies = [
"quote",
"wasm-bindgen-macro-support",
]
[[package]]
name = "wasm-bindgen-macro-support"
version = "0.2.80"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "99ec0dc7a4756fffc231aab1b9f2f578d23cd391390ab27f952ae0c9b3ece20b"
dependencies = [
"proc-macro2",
"quote",
"syn",
"wasm-bindgen-backend",
"wasm-bindgen-shared",
]
[[package]]
name = "wasm-bindgen-shared"
version = "0.2.80"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d554b7f530dee5964d9a9468d95c1f8b8acae4f282807e7d27d4b03099a46744"
[[package]]
name = "wasm-coredump-builder"
version = "0.1.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "158180f35c9ba89a3e7763f20be93e77d5e41535c18e22c85d6dd5b5bce18108"
dependencies = [
"wasm-coredump-encoder",
"wasm-coredump-types",
"wasm-encoder 0.23.0",
]
[[package]]
name = "wasm-coredump-encoder"
version = "0.1.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2f0c99cdf3a88363570f1027e2f337de6647cac9fed5d474f86103d7c45c8700"
dependencies = [
"leb128",
"wasm-coredump-types",
]
[[package]]
name = "wasm-coredump-types"
version = "0.1.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "10e35729a021e44c20511e23ac2b215df05da243bdc4bad336fd3686552539fc"
[[package]]
name = "wasm-encoder"
version = "0.23.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1c3e4bc09095436c8e7584d86d33e6c3ee67045af8fb262cbb9cc321de553428"
dependencies = [
"leb128",
]
[[package]]
name = "wasm-encoder"
version = "0.25.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4eff853c4f09eec94d76af527eddad4e9de13b11d6286a1ef7134bc30135a2b7"
dependencies = [
"leb128",
]
[[package]]
name = "wasm-mutate"
version = "0.2.22"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "956f9fd2f785491bdb243ae608c53fd6bb9956f9e174c1fc5a77354fce396698"
dependencies = [
"egg",
"log",
"rand 0.8.5",
"thiserror",
"wasm-encoder 0.25.0",
"wasmparser",
]
[[package]]
name = "wasm-smith"
version = "0.12.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "549cb78be46f43ad6746402871336cb6a989127fb847e93eb6ba0817647485a6"
dependencies = [
"arbitrary",
"flagset",
"indexmap",
"leb128",
"wasm-encoder 0.25.0",
"wasmparser",
]
[[package]]
name = "wasm-spec-interpreter"
version = "0.0.0"
dependencies = [
"ocaml-interop",
"once_cell",
"wat",
]
[[package]]
name = "wasmi"
version = "0.20.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "01bf50edb2ea9d922aa75a7bf3c15e26a6c9e2d18c56e862b49737a582901729"
dependencies = [
"spin 0.9.4",
"wasmi_arena",
"wasmi_core",
"wasmparser-nostd",
]
[[package]]
name = "wasmi_arena"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a1ea379cbb0b41f3a9f0bf7b47036d036aae7f43383d8cc487d4deccf40dee0a"
[[package]]
name = "wasmi_core"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c5bf998ab792be85e20e771fe14182b4295571ad1d4f89d3da521c1bef5f597a"
dependencies = [
"downcast-rs",
"libm",
"num-traits",
]
[[package]]
name = "wasmparser"
version = "0.102.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "48134de3d7598219ab9eaf6b91b15d8e50d31da76b8519fe4ecfcec2cf35104b"
dependencies = [
"indexmap",
"url",
]
[[package]]
name = "wasmparser-nostd"
version = "0.91.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9c37f310b5a62bfd5ae7c0f1d8e6f98af16a5d6d84ba764e9c36439ec14e318b"
dependencies = [
"indexmap-nostd",
]
[[package]]
name = "wasmprinter"
version = "0.2.54"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2dc17ae63836d010a2bf001c26a5fedbb9a05e5f71117fb63e0ab878bfbe1ca3"
dependencies = [
"anyhow",
"wasmparser",
]
[[package]]
name = "wasmtime"
version = "9.0.0"
dependencies = [
"anyhow",
"async-trait",
"bincode",
Remove support for userfaultfd (#4040) This commit removes support for the `userfaultfd` or "uffd" syscall on Linux. This support was originally added for users migrating from Lucet to Wasmtime, but the recent developments of kernel-supported copy-on-write support for memory initialization wound up being more appropriate for these use cases than usefaultfd. The main reason for moving to copy-on-write initialization are: * The `userfaultfd` feature was never necessarily intended for this style of use case with wasm and was susceptible to subtle and rare bugs that were extremely difficult to track down. We were never 100% certain that there were kernel bugs related to userfaultfd but the suspicion never went away. * Handling faults with userfaultfd was always slow and single-threaded. Only one thread could handle faults and traveling to user-space to handle faults is inherently slower than handling them all in the kernel. The single-threaded aspect in particular presented a significant scaling bottleneck for embeddings that want to run many wasm instances in parallel. * One of the major benefits of userfaultfd was lazy initialization of wasm linear memory which is also achieved with the copy-on-write initialization support we have right now. * One of the suspected benefits of userfaultfd was less frobbing of the kernel vma structures when wasm modules are instantiated. Currently the copy-on-write support has a mitigation where we attempt to reuse the memory images where possible to avoid changing vma structures. When comparing this to userfaultfd's performance it was found that kernel modifications of vmas aren't a worrisome bottleneck so copy-on-write is suitable for this as well. Overall there are no remaining benefits that userfaultfd gives that copy-on-write doesn't, and copy-on-write solves a major downsides of userfaultfd, the scaling issue with a single faulting thread. Additionally copy-on-write support seems much more robust in terms of kernel implementation since it's only using standard memory-management syscalls which are heavily exercised. Finally copy-on-write support provides a new bonus where read-only memory in WebAssembly can be mapped directly to the same kernel cache page, even amongst many wasm instances of the same module, which was never possible with userfaultfd. In light of all this it's expected that all users of userfaultfd should migrate to the copy-on-write initialization of Wasmtime (which is enabled by default).
3 years ago
"cfg-if",
Implement strings in adapter modules (#4623) * Implement strings in adapter modules This commit is a hefty addition to Wasmtime's support for the component model. This implements the final remaining type (in the current type hierarchy) unimplemented in adapter module trampolines: strings. Strings are the most complicated type to implement in adapter trampolines because they are highly structured chunks of data in memory (according to specific encodings). Additionally each lift/lower operation can choose its own encoding for strings meaning that Wasmtime, the host, may have to convert between any pairwise ordering of string encodings. The `CanonicalABI.md` in the component-model repo in general specifies all the fiddly bits of string encoding so there's not a ton of wiggle room for Wasmtime to get creative. This PR largely "just" implements that. The high-level architecture of this implementation is: * Fused adapters are first identified to determine src/dst string encodings. This statically fixes what transcoding operation is being performed. * The generated adapter will be responsible for managing calls to `realloc` and performing bounds checks. The adapter itself does not perform memory copies or validation of string contents, however. Instead each transcoding operation is modeled as an imported function into the adapter module. This means that the adapter module dynamically, during compile time, determines what string transcoders are needed. Note that an imported transcoder is not only parameterized over the transcoding operation but additionally which memory is the source and which is the destination. * The imported core wasm functions are modeled as a new `CoreDef::Transcoder` structure. These transcoders end up being small Cranelift-compiled trampolines. The Cranelift-compiled trampoline will load the actual base pointer of memory and add it to the relative pointers passed as function arguments. This trampoline then calls a transcoder "libcall" which enters Rust-defined functions for actual transcoding operations. * Each possible transcoding operation is implemented in Rust with a unique name and a unique signature depending on the needs of the transcoder. I've tried to document inline what each transcoder does. This means that the `Module::translate_string` in adapter modules is by far the largest translation method. The main reason for this is due to the management around calling the imported transcoder functions in the face of validating string pointer/lengths and performing the dance of `realloc`-vs-transcode at the right time. I've tried to ensure that each individual case in transcoding is documented well enough to understand what's going on as well. Additionally in this PR is a full implementation in the host for the `latin1+utf16` encoding which means that both lifting and lowering host strings now works with this encoding. Currently the implementation of each transcoder function is likely far from optimal. Where possible I've leaned on the standard library itself and for latin1-related things I'm leaning on the `encoding_rs` crate. I initially tried to implement everything with `encoding_rs` but was unable to uniformly do so easily. For now I settled on trying to get a known-correct (even in the face of endianness) implementation for all of these transcoders. If an when performance becomes an issue it should be possible to implement more optimized versions of each of these transcoding operations. Testing this commit has been somewhat difficult and my general plan, like with the `(list T)` type, is to rely heavily on fuzzing to cover the various cases here. In this PR though I've added a simple test that pushes some statically known strings through all the pairs of encodings between source and destination. I've attempted to pick "interesting" strings that one way or another stress the various paths in each transcoding operation to ideally get full branch coverage there. Additionally a suite of "negative" tests have also been added to ensure that validity of encoding is actually checked. * Fix a temporarily commented out case * Fix wasmtime-runtime tests * Update deny.toml configuration * Add `BSD-3-Clause` for the `encoding_rs` crate * Remove some unused licenses * Add an exemption for `encoding_rs` for now * Split up the `translate_string` method Move out all the closures and package up captured state into smaller lists of arguments. * Test out-of-bounds for zero-length strings
2 years ago
"encoding_rs",
"indexmap",
"libc",
"log",
Use an mmap-friendly serialization format (#3257) * Use an mmap-friendly serialization format This commit reimplements the main serialization format for Wasmtime's precompiled artifacts. Previously they were generally a binary blob of `bincode`-encoded metadata prefixed with some versioning information. The downside of this format, though, is that loading a precompiled artifact required pushing all information through `bincode`. This is inefficient when some data, such as trap/address tables, are rarely accessed. The new format added in this commit is one which is designed to be `mmap`-friendly. This means that the relevant parts of the precompiled artifact are already page-aligned for updating permissions of pieces here and there. Additionally the artifact is optimized so that if data is rarely read then we can delay reading it until necessary. The new artifact format for serialized modules is an ELF file. This is not a public API guarantee, so it cannot be relied upon. In the meantime though this is quite useful for exploring precompiled modules with standard tooling like `objdump`. The ELF file is already constructed as part of module compilation, and this is the main contents of the serialized artifact. THere is some extra information, though, not encoded in each module's individual ELF file such as type information. This information continues to be `bincode`-encoded, but it's intended to be much smaller and much faster to deserialize. This extra information is appended to the end of the ELF file. This means that the original ELF file is still a valid ELF file, we just get to have extra bits at the end. More information on the new format can be found in the module docs of the serialization module of Wasmtime. Another refatoring implemented as part of this commit is to deserialize and store object files directly in `mmap`-backed storage. This avoids the need to copy bytes after the artifact is loaded into memory for each compiled module, and in a future commit it opens up the door to avoiding copying the text section into a `CodeMemory`. For now, though, the main change is that copies are not necessary when loading from a precompiled compilation artifact once the artifact is itself in mmap-based memory. To assist with managing `mmap`-based memory a new `MmapVec` type was added to `wasmtime_jit` which acts as a form of `Vec<T>` backed by a `wasmtime_runtime::Mmap`. This type notably supports `drain(..N)` to slice the buffer into disjoint regions that are all separately owned, such as having a separately owned window into one artifact for all object files contained within. Finally this commit implements a small refactoring in `wasmtime-cache` to use the standard artifact format for cache entries rather than a bincode-encoded version. This required some more hooks for serializing/deserializing but otherwise the crate still performs as before. * Review comments
3 years ago
"object",
"once_cell",
Implement support for `async` functions in Wasmtime (#2434) * Implement support for `async` functions in Wasmtime This is an implementation of [RFC 2] in Wasmtime which is to support `async`-defined host functions. At a high level support is added by executing WebAssembly code that might invoke an asynchronous host function on a separate native stack. When the host function's future is not ready we switch back to the main native stack to continue execution. There's a whole bunch of details in this commit, and it's a bit much to go over them all here in this commit message. The most important changes here are: * A new `wasmtime-fiber` crate has been written to manage the low-level details of stack-switching. Unixes use `mmap` to allocate a stack and Windows uses the native fibers implementation. We'll surely want to refactor this to move stack allocation elsewhere in the future. Fibers are intended to be relatively general with a lot of type paremters to fling values back and forth across suspension points. The whole crate is a giant wad of `unsafe` unfortunately and involves handwritten assembly with custom dwarf CFI directives to boot. Definitely deserves a close eye in review! * The `Store` type has two new methods -- `block_on` and `on_fiber` which bridge between the async and non-async worlds. Lots of unsafe fiddly bits here as we're trying to communicate context pointers between disparate portions of the code. Extra eyes and care in review is greatly appreciated. * The APIs for binding `async` functions are unfortunately pretty ugly in `Func`. This is mostly due to language limitations and compiler bugs (I believe) in Rust. Instead of `Func::wrap` we have a `Func::wrapN_async` family of methods, and we've also got a whole bunch of `Func::getN_async` methods now too. It may be worth rethinking the API of `Func` to try to make the documentation page actually grok'able. This isn't super heavily tested but the various test should suffice for engaging hopefully nearly all the infrastructure in one form or another. This is just the start though! [RFC 2]: https://github.com/bytecodealliance/rfcs/pull/2 * Add wasmtime-fiber to publish script * Save vector/float registers on ARM too. * Fix a typo * Update lock file * Implement periodically yielding with fuel consumption This commit implements APIs on `Store` to periodically yield execution of futures through the consumption of fuel. When fuel runs out a future's execution is yielded back to the caller, and then upon resumption fuel is re-injected. The goal of this is to allow cooperative multi-tasking with futures. * Fix compile without async * Save/restore the frame pointer in fiber switching Turns out this is another caller-saved register! * Simplify x86_64 fiber asm Take a leaf out of aarch64's playbook and don't have extra memory to load/store these arguments, instead leverage how `wasmtime_fiber_switch` already loads a bunch of data into registers which we can then immediately start using on a fiber's start without any extra memory accesses. * Add x86 support to wasmtime-fiber * Add ARM32 support to fiber crate * Make fiber build file probing more flexible * Use CreateFiberEx on Windows * Remove a stray no-longer-used trait declaration * Don't reach into `Caller` internals * Tweak async fuel to eventually run out. With fuel it's probably best to not provide any way to inject infinite fuel. * Fix some typos * Cleanup asm a bit * Use a shared header file to deduplicate some directives * Guarantee hidden visibility for functions * Enable gc-sections on macOS x86_64 * Add `.type` annotations for ARM * Update lock file * Fix compile error * Review comments
4 years ago
"paste",
More optimizations for calling into WebAssembly (#2759) * Combine stack-based cleanups for faster wasm calls This commit is an extension of #2757 where the goal is to optimize entry into WebAssembly. Currently wasmtime has two stack-based cleanups when entering wasm, one for the externref activation table and another for stack limits getting reset. This commit fuses these two cleanups together into one and moves some code around which enables less captures for fewer closures and such to speed up calls in to wasm a bit more. Overall this drops the execution time from 88ns to 80ns locally for me. This also updates the atomic orderings when updating the stack limit from `SeqCst` to `Relaxed`. While `SeqCst` is a reasonable starting point the usage here should be safe to use `Relaxed` since we're not using the atomics to actually protect any memory, it's simply receiving signals from other threads. * Determine whether a pc is wasm via a global map The macOS implementation of traps recently changed to using mach ports for handlers instead of signal handlers. This means that a previously relied upon invariant, each thread fixes its own trap, was broken. The macOS implementation worked around this by maintaining a global map from thread id to thread local information, however, to solve the problem. This global map is quite slow though. It involves taking a lock and updating a hash map on all calls into WebAssembly. In my local testing this accounts for >70% of the overhead of calling into WebAssembly on macOS. Naturally it'd be great to remove this! This commit fixes this issue and removes the global lock/map that is updated on all calls into WebAssembly. The fix is to maintain a global map of wasm modules and their trap addresses in the `wasmtime` crate. Doing so is relatively simple since we're already tracking this information at the `Store` level. Once we've got a global map then the macOS implementation can use this from a foreign thread and everything works out. Locally this brings the overhead, on macOS specifically, of calling into wasm from 80ns to ~20ns. * Fix compiles * Review comments
4 years ago
"psm",
Add a cranelift compile-time feature to wasmtime (#3206) * Remove unnecessary into_iter/map Forgotten from a previous refactoring, this variable was already of the right type! * Move `wasmtime_jit::Compiler` into `wasmtime` This `Compiler` struct is mostly a historical artifact at this point and wasn't necessarily pulling much weight any more. This organization also doesn't lend itself super well to compiling out `cranelift` when the `Compiler` here is used for both parallel iteration configuration settings as well as compilation. The movement into `wasmtime` is relatively small, with `Module::build_artifacts` being the main function added here which is a merging of the previous functions removed from the `wasmtime-jit` crate. * Add a `cranelift` compile-time feature to `wasmtime` This commit concludes the saga of refactoring Wasmtime and making Cranelift an optional dependency by adding a new Cargo feature to the `wasmtime` crate called `cranelift`, which is enabled by default. This feature is implemented by having a new cfg for `wasmtime` itself, `cfg(compiler)`, which is used wherever compilation is necessary. This bubbles up to disable APIs such as `Module::new`, `Func::new`, `Engine::precompile_module`, and a number of `Config` methods affecting compiler configuration. Checks are added to CI that when built in this mode Wasmtime continues to successfully build. It's hoped that although this is effectively "sprinkle `#[cfg]` until things compile" this won't be too too bad to maintain over time since it's also an use case we're interested in supporting. With `cranelift` disabled the only way to create a `Module` is with the `Module::deserialize` method, which requires some form of precompiled artifact. Two consequences of this change are: * `Module::serialize` is also disabled in this mode. The reason for this is that serialized modules contain ISA/shared flags encoded in them which were used to produce the compiled code. There's no storage for this if compilation is disabled. This could probably be re-enabled in the future if necessary, but it may not end up being all that necessary. * Deserialized modules are not checked to ensure that their ISA/shared flags are compatible with the host CPU. This is actually already the case, though, with normal modules. We'll likely want to fix this in the future using a shared implementation for both these locations. Documentation should be updated to indicate that `cranelift` can be disabled, although it's not really the most prominent documentation because this is expected to be a somewhat niche use case (albeit important, just not too common). * Always enable cranelift for the C API * Fix doc example builds * Fix check tests on GitHub Actions
3 years ago
"rayon",
"serde",
"target-lexicon",
Improve robustness of cache loading/storing (#974) * Improve robustness of cache loading/storing Today wasmtime incorrectly loads compiled compiled modules from the global cache when toggling settings such as optimizations. For example if you execute `wasmtime foo.wasm` that will cache globally an unoptimized version of the wasm module. If you then execute `wasmtime -O foo.wasm` it would then reload the unoptimized version from cache, not realizing the compilation settings were different, and use that instead. This can lead to very surprising behavior naturally! This commit updates how the cache is managed in an attempt to make it much more robust against these sorts of issues. This takes a leaf out of rustc's playbook and models the cache with a function that looks like: fn load<T: Hash>( &self, data: T, compute: fn(T) -> CacheEntry, ) -> CacheEntry; The goal here is that it guarantees that all the `data` necessary to `compute` the result of the cache entry is hashable and stored into the hash key entry. This was previously open-coded and manually managed where items were hashed explicitly, but this construction guarantees that everything reasonable `compute` could use to compile the module is stored in `data`, which is itself hashable. This refactoring then resulted in a few workarounds and a few fixes, including the original issue: * The `Module` type was split into `Module` and `ModuleLocal` where only the latter is hashed. The previous hash function for a `Module` left out items like the `start_func` and didn't hash items like the imports of the module. Omitting the `start_func` was fine since compilation didn't actually use it, but omitting imports seemed uncomfortable because while compilation didn't use the import values it did use the *number* of imports, which seems like it should then be put into the cache key. The `ModuleLocal` type now derives `Hash` to guarantee that all of its contents affect the hash key. * The `ModuleTranslationState` from `cranelift-wasm` doesn't implement `Hash` which means that we have a manual wrapper to work around that. This will be fixed with an upstream implementation, since this state affects the generated wasm code. Currently this is just a map of signatures, which is present in `Module` anyway, so we should be good for the time being. * Hashing `dyn TargetIsa` was also added, where previously it was not fully hashed. Previously only the target name was used as part of the cache key, but crucially the flags of compilation were omitted (for example the optimization flags). Unfortunately the trait object itself is not hashable so we still have to manually write a wrapper to hash it, but we likely want to add upstream some utilities to hash isa objects into cranelift itself. For now though we can continue to add hashed fields as necessary. Overall the goal here was to use the compiler to expose what we're not hashing, and then make sure we organize data and write the right code to ensure everything is hashed, and nothing more. * Update crates/environ/src/module.rs Co-Authored-By: Peter Huene <peterhuene@protonmail.com> * Fix lightbeam * Fix compilation of tests * Update the expected structure of the cache * Revert "Update the expected structure of the cache" This reverts commit 2b53fee426a4e411c313d8c1e424841ba304a9cd. * Separate the cache dir a bit * Add a test the cache is busted with opt levels * rustfmt Co-authored-by: Peter Huene <peterhuene@protonmail.com>
5 years ago
"tempfile",
"wasi-cap-std-sync",
"wasmparser",
"wasmtime-cache",
"wasmtime-component-macro",
support dynamic function calls in component model (#4442) * support dynamic function calls in component model This addresses #4310, introducing a new `component::values::Val` type for representing component values dynamically, as well as `component::types::Type` for representing the corresponding interface types. It also adds a `call` method to `component::func::Func`, which takes a slice of `Val`s as parameters and returns a `Result<Val>` representing the result. Note that I've moved `post_return` and `call_raw` from `TypedFunc` to `Func` since there was nothing specific to `TypedFunc` about them, and I wanted to reuse them. The code in both is unchanged beyond the trivial tweaks to make them fit in their new home. Signed-off-by: Joel Dice <joel.dice@fermyon.com> * order variants and match cases more consistently Signed-off-by: Joel Dice <joel.dice@fermyon.com> * implement lift for String, Box<str>, etc. This also removes the redundant `store` parameter from `Type::load`. Signed-off-by: Joel Dice <joel.dice@fermyon.com> * implement code review feedback This fixes a few issues: - Bad offset calculation when lowering - Missing variant padding - Style issues regarding `types::Handle` - Missed opportunities to reuse `Lift` and `Lower` impls It also adds forwarding `Lift` impls for `Box<[T]>`, `Vec<T>`, etc. Signed-off-by: Joel Dice <joel.dice@fermyon.com> * move `new_*` methods to specific `types` structs Per review feedback, I've moved `Type::new_record` to `Record::new_val` and added a `Type::unwrap_record` method; likewise for the other kinds of types. Signed-off-by: Joel Dice <joel.dice@fermyon.com> * make tuple, option, and expected type comparisons recursive These types should compare as equal across component boundaries as long as their type parameters are equal. Signed-off-by: Joel Dice <joel.dice@fermyon.com> * improve error diagnostic in `Type::check` We now distinguish between more failure cases to provide an informative error message. Signed-off-by: Joel Dice <joel.dice@fermyon.com> * address review feedback - Remove `WasmStr::to_str_from_memory` and `WasmList::get_from_memory` - add `try_new` methods to various `values` types - avoid using `ExactSizeIterator::len` where we can't trust it - fix over-constrained bounds on forwarded `ComponentType` impls Signed-off-by: Joel Dice <joel.dice@fermyon.com> * rearrange code per review feedback - Move functions from `types` to `values` module so we can make certain struct fields private - Rename `try_new` to just `new` Signed-off-by: Joel Dice <joel.dice@fermyon.com> * remove special-case equality test for tuples, options, and expecteds Instead, I've added a FIXME comment and will open an issue to do recursive structural equality testing. Signed-off-by: Joel Dice <joel.dice@fermyon.com>
2 years ago
"wasmtime-component-util",
Add a cranelift compile-time feature to wasmtime (#3206) * Remove unnecessary into_iter/map Forgotten from a previous refactoring, this variable was already of the right type! * Move `wasmtime_jit::Compiler` into `wasmtime` This `Compiler` struct is mostly a historical artifact at this point and wasn't necessarily pulling much weight any more. This organization also doesn't lend itself super well to compiling out `cranelift` when the `Compiler` here is used for both parallel iteration configuration settings as well as compilation. The movement into `wasmtime` is relatively small, with `Module::build_artifacts` being the main function added here which is a merging of the previous functions removed from the `wasmtime-jit` crate. * Add a `cranelift` compile-time feature to `wasmtime` This commit concludes the saga of refactoring Wasmtime and making Cranelift an optional dependency by adding a new Cargo feature to the `wasmtime` crate called `cranelift`, which is enabled by default. This feature is implemented by having a new cfg for `wasmtime` itself, `cfg(compiler)`, which is used wherever compilation is necessary. This bubbles up to disable APIs such as `Module::new`, `Func::new`, `Engine::precompile_module`, and a number of `Config` methods affecting compiler configuration. Checks are added to CI that when built in this mode Wasmtime continues to successfully build. It's hoped that although this is effectively "sprinkle `#[cfg]` until things compile" this won't be too too bad to maintain over time since it's also an use case we're interested in supporting. With `cranelift` disabled the only way to create a `Module` is with the `Module::deserialize` method, which requires some form of precompiled artifact. Two consequences of this change are: * `Module::serialize` is also disabled in this mode. The reason for this is that serialized modules contain ISA/shared flags encoded in them which were used to produce the compiled code. There's no storage for this if compilation is disabled. This could probably be re-enabled in the future if necessary, but it may not end up being all that necessary. * Deserialized modules are not checked to ensure that their ISA/shared flags are compatible with the host CPU. This is actually already the case, though, with normal modules. We'll likely want to fix this in the future using a shared implementation for both these locations. Documentation should be updated to indicate that `cranelift` can be disabled, although it's not really the most prominent documentation because this is expected to be a somewhat niche use case (albeit important, just not too common). * Always enable cranelift for the C API * Fix doc example builds * Fix check tests on GitHub Actions
3 years ago
"wasmtime-cranelift",
"wasmtime-environ",
Implement support for `async` functions in Wasmtime (#2434) * Implement support for `async` functions in Wasmtime This is an implementation of [RFC 2] in Wasmtime which is to support `async`-defined host functions. At a high level support is added by executing WebAssembly code that might invoke an asynchronous host function on a separate native stack. When the host function's future is not ready we switch back to the main native stack to continue execution. There's a whole bunch of details in this commit, and it's a bit much to go over them all here in this commit message. The most important changes here are: * A new `wasmtime-fiber` crate has been written to manage the low-level details of stack-switching. Unixes use `mmap` to allocate a stack and Windows uses the native fibers implementation. We'll surely want to refactor this to move stack allocation elsewhere in the future. Fibers are intended to be relatively general with a lot of type paremters to fling values back and forth across suspension points. The whole crate is a giant wad of `unsafe` unfortunately and involves handwritten assembly with custom dwarf CFI directives to boot. Definitely deserves a close eye in review! * The `Store` type has two new methods -- `block_on` and `on_fiber` which bridge between the async and non-async worlds. Lots of unsafe fiddly bits here as we're trying to communicate context pointers between disparate portions of the code. Extra eyes and care in review is greatly appreciated. * The APIs for binding `async` functions are unfortunately pretty ugly in `Func`. This is mostly due to language limitations and compiler bugs (I believe) in Rust. Instead of `Func::wrap` we have a `Func::wrapN_async` family of methods, and we've also got a whole bunch of `Func::getN_async` methods now too. It may be worth rethinking the API of `Func` to try to make the documentation page actually grok'able. This isn't super heavily tested but the various test should suffice for engaging hopefully nearly all the infrastructure in one form or another. This is just the start though! [RFC 2]: https://github.com/bytecodealliance/rfcs/pull/2 * Add wasmtime-fiber to publish script * Save vector/float registers on ARM too. * Fix a typo * Update lock file * Implement periodically yielding with fuel consumption This commit implements APIs on `Store` to periodically yield execution of futures through the consumption of fuel. When fuel runs out a future's execution is yielded back to the caller, and then upon resumption fuel is re-injected. The goal of this is to allow cooperative multi-tasking with futures. * Fix compile without async * Save/restore the frame pointer in fiber switching Turns out this is another caller-saved register! * Simplify x86_64 fiber asm Take a leaf out of aarch64's playbook and don't have extra memory to load/store these arguments, instead leverage how `wasmtime_fiber_switch` already loads a bunch of data into registers which we can then immediately start using on a fiber's start without any extra memory accesses. * Add x86 support to wasmtime-fiber * Add ARM32 support to fiber crate * Make fiber build file probing more flexible * Use CreateFiberEx on Windows * Remove a stray no-longer-used trait declaration * Don't reach into `Caller` internals * Tweak async fuel to eventually run out. With fuel it's probably best to not provide any way to inject infinite fuel. * Fix some typos * Cleanup asm a bit * Use a shared header file to deduplicate some directives * Guarantee hidden visibility for functions * Enable gc-sections on macOS x86_64 * Add `.type` annotations for ARM * Update lock file * Fix compile error * Review comments
4 years ago
"wasmtime-fiber",
"wasmtime-jit",
"wasmtime-runtime",
"wasmtime-wasi",
winch: Initial integration with wasmtime (#6119) * Adding in trampoline compiling method for ISA * Adding support for indirect call to memory address * Refactoring frame to externalize defined locals, so it removes WASM depedencies in trampoline case * Adding initial version of trampoline for testing * Refactoring trampoline to be re-used by other architectures * Initial wiring for winch with wasmtime * Add a Wasmtime CLI option to select `winch` This is effectively an option to select the `Strategy` enumeration. * Implement `Compiler::compile_function` for Winch Hook this into the `TargetIsa::compile_function` hook as well. Currently this doesn't take into account `Tunables`, but that's left as a TODO for later. * Filling out Winch append_code method * Adding back in changes from previous branch Most of these are a WIP. It's missing trampolines for x64, but a basic one exists for aarch64. It's missing the handling of arguments that exist on the stack. It currently imports `cranelift_wasm::WasmFuncType` since it's what's passed to the `Compiler` trait. It's a bit awkward to use in the `winch_codegen` crate since it mostly operates on `wasmparser` types. I've had to hack in a conversion to get things working. Long term, I'm not sure it's wise to rely on this type but it seems like it's easier on the Cranelift side when creating the stub IR. * Small API changes to make integration easier * Adding in new FuncEnv, only a stub for now * Removing unneeded parts of the old PoC, and refactoring trampoline code * Moving FuncEnv into a separate file * More comments for trampolines * Adding in winch integration tests for first pass * Using new addressing method to fix stack pointer error * Adding test for stack arguments * Only run tests on x86 for now, it's more complete for winch * Add in missing documentation after rebase * Updating based on feedback in draft PR * Fixing formatting on doc comment for argv register * Running formatting * Lock updates, and turning on winch feature flags during tests * Updating configuration with comments to no longer gate Strategy enum * Using the winch-environ FuncEnv, but it required changing the sig * Proper comment formatting * Removing wasmtime-winch from dev-dependencies, adding the winch feature makes this not necessary * Update doc attr to include winch check * Adding winch feature to doc generation, which seems to fix the feature error in CI * Add the `component-model` feature to the cargo doc invocation in CI To match the metadata used by the docs.rs invocation when building docs. * Add a comment clarifying the usage of `component-model` for docs.rs * Correctly order wasmtime-winch and winch-environ in the publish script * Ensure x86 test dependencies are included in cfg(target_arch) * Further constrain Winch tests to x86_64 _and_ unix --------- Co-authored-by: Alex Crichton <alex@alexcrichton.com> Co-authored-by: Saúl Cabrera <saulecabrera@gmail.com>
2 years ago
"wasmtime-winch",
"wat",
"windows-sys",
]
[[package]]
name = "wasmtime-asm-macros"
version = "9.0.0"
dependencies = [
"cfg-if",
]
[[package]]
name = "wasmtime-bench-api"
version = "9.0.0"
dependencies = [
"anyhow",
"cap-std",
"clap",
"shuffling-allocator",
"target-lexicon",
4 years ago
"wasi-cap-std-sync",
"wasmtime",
"wasmtime-cli-flags",
"wasmtime-wasi",
"wasmtime-wasi-crypto",
"wasmtime-wasi-nn",
"wat",
]
[[package]]
name = "wasmtime-c-api"
version = "9.0.0"
dependencies = [
"anyhow",
"cap-std",
"env_logger 0.9.0",
Refactor and fill out wasmtime's C API (#1415) * Refactor and improve safety of C API This commit is intended to be a relatively large refactoring of the C API which is targeted at improving the safety of our C API definitions. Not all of the APIs have been updated yet but this is intended to be the start. The goal here is to make as many functions safe as we can, expressing inputs/outputs as native Rust types rather than raw pointers wherever possible. For example instead of `*const wasm_foo_t` we'd take `&wasm_foo_t`. Instead of returning `*mut wasm_foo_t` we'd return `Box<wasm_foo_t>`. No ABI/API changes are intended from this commit, it's supposed to only change how we define all these functions internally. This commit also additionally implements a few more API bindings for exposed vector types by unifying everything into one macro. Finally, this commit moves many internal caches in the C API to the `OnceCell` type which provides a safe interface for one-time initialization. * Split apart monolithic C API `lib.rs` This commit splits the monolithic `src/lib.rs` in the C API crate into lots of smaller files. The goal here is to make this a bit more readable and digestable. Each module now contains only API bindings for a particular type, roughly organized around the grouping in the wasm.h header file already. A few more extensions were added, such as filling out `*_as_*` conversions with both const and non-const versions. Additionally many APIs were made safer in the same style as the previous commit, generally preferring Rust types rather than raw pointer types. Overall no functional change is intended here, it should be mostly just code movement and minor refactorings! * Make a few wasi C bindings safer Use safe Rust types where we can and touch up a few APIs here and there. * Implement `wasm_*type_as_externtype*` APIs This commit restructures `wasm_externtype_t` to be similar to `wasm_extern_t` so type conversion between the `*_extern_*` variants to the concrete variants are all simple casts. (checked in the case of general to concrete, of course). * Consistently imlpement host info functions in the API This commit adds a small macro crate which is then used to consistently define the various host-info-related functions in the C API. The goal here is to try to mirror what the `wasm.h` header provides to provide a full implementation of the header.
5 years ago
"once_cell",
"wasi-cap-std-sync",
"wasi-common",
"wasmtime",
Refactor and fill out wasmtime's C API (#1415) * Refactor and improve safety of C API This commit is intended to be a relatively large refactoring of the C API which is targeted at improving the safety of our C API definitions. Not all of the APIs have been updated yet but this is intended to be the start. The goal here is to make as many functions safe as we can, expressing inputs/outputs as native Rust types rather than raw pointers wherever possible. For example instead of `*const wasm_foo_t` we'd take `&wasm_foo_t`. Instead of returning `*mut wasm_foo_t` we'd return `Box<wasm_foo_t>`. No ABI/API changes are intended from this commit, it's supposed to only change how we define all these functions internally. This commit also additionally implements a few more API bindings for exposed vector types by unifying everything into one macro. Finally, this commit moves many internal caches in the C API to the `OnceCell` type which provides a safe interface for one-time initialization. * Split apart monolithic C API `lib.rs` This commit splits the monolithic `src/lib.rs` in the C API crate into lots of smaller files. The goal here is to make this a bit more readable and digestable. Each module now contains only API bindings for a particular type, roughly organized around the grouping in the wasm.h header file already. A few more extensions were added, such as filling out `*_as_*` conversions with both const and non-const versions. Additionally many APIs were made safer in the same style as the previous commit, generally preferring Rust types rather than raw pointer types. Overall no functional change is intended here, it should be mostly just code movement and minor refactorings! * Make a few wasi C bindings safer Use safe Rust types where we can and touch up a few APIs here and there. * Implement `wasm_*type_as_externtype*` APIs This commit restructures `wasm_externtype_t` to be similar to `wasm_extern_t` so type conversion between the `*_extern_*` variants to the concrete variants are all simple casts. (checked in the case of general to concrete, of course). * Consistently imlpement host info functions in the API This commit adds a small macro crate which is then used to consistently define the various host-info-related functions in the C API. The goal here is to try to mirror what the `wasm.h` header provides to provide a full implementation of the header.
5 years ago
"wasmtime-c-api-macros",
"wasmtime-wasi",
"wat",
]
Refactor and fill out wasmtime's C API (#1415) * Refactor and improve safety of C API This commit is intended to be a relatively large refactoring of the C API which is targeted at improving the safety of our C API definitions. Not all of the APIs have been updated yet but this is intended to be the start. The goal here is to make as many functions safe as we can, expressing inputs/outputs as native Rust types rather than raw pointers wherever possible. For example instead of `*const wasm_foo_t` we'd take `&wasm_foo_t`. Instead of returning `*mut wasm_foo_t` we'd return `Box<wasm_foo_t>`. No ABI/API changes are intended from this commit, it's supposed to only change how we define all these functions internally. This commit also additionally implements a few more API bindings for exposed vector types by unifying everything into one macro. Finally, this commit moves many internal caches in the C API to the `OnceCell` type which provides a safe interface for one-time initialization. * Split apart monolithic C API `lib.rs` This commit splits the monolithic `src/lib.rs` in the C API crate into lots of smaller files. The goal here is to make this a bit more readable and digestable. Each module now contains only API bindings for a particular type, roughly organized around the grouping in the wasm.h header file already. A few more extensions were added, such as filling out `*_as_*` conversions with both const and non-const versions. Additionally many APIs were made safer in the same style as the previous commit, generally preferring Rust types rather than raw pointer types. Overall no functional change is intended here, it should be mostly just code movement and minor refactorings! * Make a few wasi C bindings safer Use safe Rust types where we can and touch up a few APIs here and there. * Implement `wasm_*type_as_externtype*` APIs This commit restructures `wasm_externtype_t` to be similar to `wasm_extern_t` so type conversion between the `*_extern_*` variants to the concrete variants are all simple casts. (checked in the case of general to concrete, of course). * Consistently imlpement host info functions in the API This commit adds a small macro crate which is then used to consistently define the various host-info-related functions in the C API. The goal here is to try to mirror what the `wasm.h` header provides to provide a full implementation of the header.
5 years ago
[[package]]
name = "wasmtime-c-api-macros"
version = "0.0.0"
Refactor and fill out wasmtime's C API (#1415) * Refactor and improve safety of C API This commit is intended to be a relatively large refactoring of the C API which is targeted at improving the safety of our C API definitions. Not all of the APIs have been updated yet but this is intended to be the start. The goal here is to make as many functions safe as we can, expressing inputs/outputs as native Rust types rather than raw pointers wherever possible. For example instead of `*const wasm_foo_t` we'd take `&wasm_foo_t`. Instead of returning `*mut wasm_foo_t` we'd return `Box<wasm_foo_t>`. No ABI/API changes are intended from this commit, it's supposed to only change how we define all these functions internally. This commit also additionally implements a few more API bindings for exposed vector types by unifying everything into one macro. Finally, this commit moves many internal caches in the C API to the `OnceCell` type which provides a safe interface for one-time initialization. * Split apart monolithic C API `lib.rs` This commit splits the monolithic `src/lib.rs` in the C API crate into lots of smaller files. The goal here is to make this a bit more readable and digestable. Each module now contains only API bindings for a particular type, roughly organized around the grouping in the wasm.h header file already. A few more extensions were added, such as filling out `*_as_*` conversions with both const and non-const versions. Additionally many APIs were made safer in the same style as the previous commit, generally preferring Rust types rather than raw pointer types. Overall no functional change is intended here, it should be mostly just code movement and minor refactorings! * Make a few wasi C bindings safer Use safe Rust types where we can and touch up a few APIs here and there. * Implement `wasm_*type_as_externtype*` APIs This commit restructures `wasm_externtype_t` to be similar to `wasm_extern_t` so type conversion between the `*_extern_*` variants to the concrete variants are all simple casts. (checked in the case of general to concrete, of course). * Consistently imlpement host info functions in the API This commit adds a small macro crate which is then used to consistently define the various host-info-related functions in the C API. The goal here is to try to mirror what the `wasm.h` header provides to provide a full implementation of the header.
5 years ago
dependencies = [
"proc-macro2",
"quote",
]
[[package]]
name = "wasmtime-cache"
version = "9.0.0"
dependencies = [
"anyhow",
4 years ago
"base64",
"bincode",
"directories-next",
"file-per-thread-logger",
"filetime",
"log",
"once_cell",
"pretty_env_logger",
"rustix",
"serde",
"sha2 0.10.2",
"tempfile",
"toml",
"windows-sys",
"zstd",
]
[[package]]
name = "wasmtime-cli"
version = "9.0.0"
dependencies = [
"anyhow",
"async-trait",
"bstr",
"clap",
"component-macro-test",
"component-test-util",
"criterion",
"env_logger 0.9.0",
"filecheck",
"humantime 2.1.0",
"libc",
"listenfd",
"log",
Use an mmap-friendly serialization format (#3257) * Use an mmap-friendly serialization format This commit reimplements the main serialization format for Wasmtime's precompiled artifacts. Previously they were generally a binary blob of `bincode`-encoded metadata prefixed with some versioning information. The downside of this format, though, is that loading a precompiled artifact required pushing all information through `bincode`. This is inefficient when some data, such as trap/address tables, are rarely accessed. The new format added in this commit is one which is designed to be `mmap`-friendly. This means that the relevant parts of the precompiled artifact are already page-aligned for updating permissions of pieces here and there. Additionally the artifact is optimized so that if data is rarely read then we can delay reading it until necessary. The new artifact format for serialized modules is an ELF file. This is not a public API guarantee, so it cannot be relied upon. In the meantime though this is quite useful for exploring precompiled modules with standard tooling like `objdump`. The ELF file is already constructed as part of module compilation, and this is the main contents of the serialized artifact. THere is some extra information, though, not encoded in each module's individual ELF file such as type information. This information continues to be `bincode`-encoded, but it's intended to be much smaller and much faster to deserialize. This extra information is appended to the end of the ELF file. This means that the original ELF file is still a valid ELF file, we just get to have extra bits at the end. More information on the new format can be found in the module docs of the serialization module of Wasmtime. Another refatoring implemented as part of this commit is to deserialize and store object files directly in `mmap`-backed storage. This avoids the need to copy bytes after the artifact is loaded into memory for each compiled module, and in a future commit it opens up the door to avoiding copying the text section into a `CodeMemory`. For now, though, the main change is that copies are not necessary when loading from a precompiled compilation artifact once the artifact is itself in mmap-based memory. To assist with managing `mmap`-based memory a new `MmapVec` type was added to `wasmtime_jit` which acts as a form of `Vec<T>` backed by a `wasmtime_runtime::Mmap`. This type notably supports `drain(..N)` to slice the buffer into disjoint regions that are all separately owned, such as having a separately owned window into one artifact for all object files contained within. Finally this commit implements a small refactoring in `wasmtime-cache` to use the standard artifact format for cache entries rather than a bincode-encoded version. This required some more hooks for serializing/deserializing but otherwise the crate still performs as before. * Review comments
3 years ago
"memchr",
"num_cpus",
"once_cell",
"rayon",
"rustix",
"serde",
"serde_json",
"target-lexicon",
"tempfile",
"test-programs",
"tokio",
"wasm-coredump-builder",
"wasmparser",
"wasmtime",
"wasmtime-cache",
"wasmtime-cli-flags",
"wasmtime-component-util",
Add a cranelift compile-time feature to wasmtime (#3206) * Remove unnecessary into_iter/map Forgotten from a previous refactoring, this variable was already of the right type! * Move `wasmtime_jit::Compiler` into `wasmtime` This `Compiler` struct is mostly a historical artifact at this point and wasn't necessarily pulling much weight any more. This organization also doesn't lend itself super well to compiling out `cranelift` when the `Compiler` here is used for both parallel iteration configuration settings as well as compilation. The movement into `wasmtime` is relatively small, with `Module::build_artifacts` being the main function added here which is a merging of the previous functions removed from the `wasmtime-jit` crate. * Add a `cranelift` compile-time feature to `wasmtime` This commit concludes the saga of refactoring Wasmtime and making Cranelift an optional dependency by adding a new Cargo feature to the `wasmtime` crate called `cranelift`, which is enabled by default. This feature is implemented by having a new cfg for `wasmtime` itself, `cfg(compiler)`, which is used wherever compilation is necessary. This bubbles up to disable APIs such as `Module::new`, `Func::new`, `Engine::precompile_module`, and a number of `Config` methods affecting compiler configuration. Checks are added to CI that when built in this mode Wasmtime continues to successfully build. It's hoped that although this is effectively "sprinkle `#[cfg]` until things compile" this won't be too too bad to maintain over time since it's also an use case we're interested in supporting. With `cranelift` disabled the only way to create a `Module` is with the `Module::deserialize` method, which requires some form of precompiled artifact. Two consequences of this change are: * `Module::serialize` is also disabled in this mode. The reason for this is that serialized modules contain ISA/shared flags encoded in them which were used to produce the compiled code. There's no storage for this if compilation is disabled. This could probably be re-enabled in the future if necessary, but it may not end up being all that necessary. * Deserialized modules are not checked to ensure that their ISA/shared flags are compatible with the host CPU. This is actually already the case, though, with normal modules. We'll likely want to fix this in the future using a shared implementation for both these locations. Documentation should be updated to indicate that `cranelift` can be disabled, although it's not really the most prominent documentation because this is expected to be a somewhat niche use case (albeit important, just not too common). * Always enable cranelift for the C API * Fix doc example builds * Fix check tests on GitHub Actions
3 years ago
"wasmtime-cranelift",
"wasmtime-environ",
"wasmtime-explorer",
"wasmtime-runtime",
"wasmtime-wasi",
"wasmtime-wasi-crypto",
"wasmtime-wasi-nn",
wasi-threads: an initial implementation (#5484) This commit includes a set of changes that add initial support for `wasi-threads` to Wasmtime: * feat: remove mutability from the WasiCtx Table This patch adds interior mutability to the WasiCtx Table and the Table elements. Major pain points: * `File` only needs `RwLock<cap_std::fs::File>` to implement `File::set_fdflags()` on Windows, because of [1] * Because `File` needs a `RwLock` and `RwLock*Guard` cannot be hold across an `.await`, The `async` from `async fn num_ready_bytes(&self)` had to be removed * Because `File` needs a `RwLock` and `RwLock*Guard` cannot be dereferenced in `pollable`, the signature of `fn pollable(&self) -> Option<rustix::fd::BorrowedFd>` changed to `fn pollable(&self) -> Option<Arc<dyn AsFd + '_>>` [1] https://github.com/bytecodealliance/system-interface/blob/da238e324e752033f315f09c082ad9ce35d42696/src/fs/fd_flags.rs#L210-L217 * wasi-threads: add an initial implementation This change is a first step toward implementing `wasi-threads` in Wasmtime. We may find that it has some missing pieces, but the core functionality is there: when `wasi::thread_spawn` is called by a running WebAssembly module, a function named `wasi_thread_start` is found in the module's exports and called in a new instance. The shared memory of the original instance is reused in the new instance. This new WASI proposal is in its early stages and details are still being hashed out in the [spec] and [wasi-libc] repositories. Due to its experimental state, the `wasi-threads` functionality is hidden behind both a compile-time and runtime flag: one must build with `--features wasi-threads` but also run the Wasmtime CLI with `--wasm-features threads` and `--wasi-modules experimental-wasi-threads`. One can experiment with `wasi-threads` by running: ```console $ cargo run --features wasi-threads -- \ --wasm-features threads --wasi-modules experimental-wasi-threads \ <a threads-enabled module> ``` Threads-enabled Wasm modules are not yet easy to build. Hopefully this is resolved soon, but in the meantime see the use of `THREAD_MODEL=posix` in the [wasi-libc] repository for some clues on what is necessary. Wiggle complicates things by requiring the Wasm memory to be exported with a certain name and `wasi-threads` also expects that memory to be imported; this build-time obstacle can be overcome with the `--import-memory --export-memory` flags only available in the latest Clang tree. Due to all of this, the included tests are written directly in WAT--run these with: ```console $ cargo test --features wasi-threads -p wasmtime-cli -- cli_tests ``` [spec]: https://github.com/WebAssembly/wasi-threads [wasi-libc]: https://github.com/WebAssembly/wasi-libc This change does not protect the WASI implementations themselves from concurrent access. This is already complete in previous commits or left for future commits in certain cases (e.g., wasi-nn). * wasi-threads: factor out process exit logic As is being discussed [elsewhere], either calling `proc_exit` or trapping in any thread should halt execution of all threads. The Wasmtime CLI already has logic for adapting a WebAssembly error code to a code expected in each OS. This change factors out this logic to a new function, `maybe_exit_on_error`, for use within the `wasi-threads` implementation. This will work reasonably well for CLI users of Wasmtime + `wasi-threads`, but embedders will want something better in the future: when a `wasi-threads` threads fails, they may not want their application to exit. Handling this is tricky, because it will require cancelling the threads spawned by the `wasi-threads` implementation, something that is not trivial to do in Rust. With this change, we defer that work until later in order to provide a working implementation of `wasi-threads` for experimentation. [elsewhere]: https://github.com/WebAssembly/wasi-threads/pull/17 * review: work around `fd_fdstat_set_flags` In order to make progress with wasi-threads, this change temporarily works around limitations induced by `wasi-common`'s `fd_fdstat_set_flags` to allow `&mut self` use in the implementation. Eventual resolution is tracked in https://github.com/bytecodealliance/wasmtime/issues/5643. This change makes several related helper functions (e.g., `set_fdflags`) take `&mut self` as well. * test: use `wait`/`notify` to improve `threads.wat` test Previously, the test simply executed in a loop for some hardcoded number of iterations. This changes uses `wait` and `notify` and atomic operations to keep track of when the spawned threads are done and join on the main thread appropriately. * various fixes and tweaks due to the PR review --------- Signed-off-by: Harald Hoyer <harald@profian.com> Co-authored-by: Harald Hoyer <harald@profian.com> Co-authored-by: Alex Crichton <alex@alexcrichton.com>
2 years ago
"wasmtime-wasi-threads",
"wasmtime-wast",
"wast 55.0.0",
"wat",
"windows-sys",
]
[[package]]
name = "wasmtime-cli-flags"
version = "9.0.0"
dependencies = [
"anyhow",
"clap",
"file-per-thread-logger",
"pretty_env_logger",
"rayon",
"wasmtime",
]
[[package]]
name = "wasmtime-component-macro"
version = "9.0.0"
dependencies = [
"anyhow",
"component-macro-test-helpers",
"proc-macro2",
"quote",
"syn",
"tracing",
"wasmtime",
support dynamic function calls in component model (#4442) * support dynamic function calls in component model This addresses #4310, introducing a new `component::values::Val` type for representing component values dynamically, as well as `component::types::Type` for representing the corresponding interface types. It also adds a `call` method to `component::func::Func`, which takes a slice of `Val`s as parameters and returns a `Result<Val>` representing the result. Note that I've moved `post_return` and `call_raw` from `TypedFunc` to `Func` since there was nothing specific to `TypedFunc` about them, and I wanted to reuse them. The code in both is unchanged beyond the trivial tweaks to make them fit in their new home. Signed-off-by: Joel Dice <joel.dice@fermyon.com> * order variants and match cases more consistently Signed-off-by: Joel Dice <joel.dice@fermyon.com> * implement lift for String, Box<str>, etc. This also removes the redundant `store` parameter from `Type::load`. Signed-off-by: Joel Dice <joel.dice@fermyon.com> * implement code review feedback This fixes a few issues: - Bad offset calculation when lowering - Missing variant padding - Style issues regarding `types::Handle` - Missed opportunities to reuse `Lift` and `Lower` impls It also adds forwarding `Lift` impls for `Box<[T]>`, `Vec<T>`, etc. Signed-off-by: Joel Dice <joel.dice@fermyon.com> * move `new_*` methods to specific `types` structs Per review feedback, I've moved `Type::new_record` to `Record::new_val` and added a `Type::unwrap_record` method; likewise for the other kinds of types. Signed-off-by: Joel Dice <joel.dice@fermyon.com> * make tuple, option, and expected type comparisons recursive These types should compare as equal across component boundaries as long as their type parameters are equal. Signed-off-by: Joel Dice <joel.dice@fermyon.com> * improve error diagnostic in `Type::check` We now distinguish between more failure cases to provide an informative error message. Signed-off-by: Joel Dice <joel.dice@fermyon.com> * address review feedback - Remove `WasmStr::to_str_from_memory` and `WasmList::get_from_memory` - add `try_new` methods to various `values` types - avoid using `ExactSizeIterator::len` where we can't trust it - fix over-constrained bounds on forwarded `ComponentType` impls Signed-off-by: Joel Dice <joel.dice@fermyon.com> * rearrange code per review feedback - Move functions from `types` to `values` module so we can make certain struct fields private - Rename `try_new` to just `new` Signed-off-by: Joel Dice <joel.dice@fermyon.com> * remove special-case equality test for tuples, options, and expecteds Instead, I've added a FIXME comment and will open an issue to do recursive structural equality testing. Signed-off-by: Joel Dice <joel.dice@fermyon.com>
2 years ago
"wasmtime-component-util",
"wasmtime-wit-bindgen",
"wit-parser",
]
support dynamic function calls in component model (#4442) * support dynamic function calls in component model This addresses #4310, introducing a new `component::values::Val` type for representing component values dynamically, as well as `component::types::Type` for representing the corresponding interface types. It also adds a `call` method to `component::func::Func`, which takes a slice of `Val`s as parameters and returns a `Result<Val>` representing the result. Note that I've moved `post_return` and `call_raw` from `TypedFunc` to `Func` since there was nothing specific to `TypedFunc` about them, and I wanted to reuse them. The code in both is unchanged beyond the trivial tweaks to make them fit in their new home. Signed-off-by: Joel Dice <joel.dice@fermyon.com> * order variants and match cases more consistently Signed-off-by: Joel Dice <joel.dice@fermyon.com> * implement lift for String, Box<str>, etc. This also removes the redundant `store` parameter from `Type::load`. Signed-off-by: Joel Dice <joel.dice@fermyon.com> * implement code review feedback This fixes a few issues: - Bad offset calculation when lowering - Missing variant padding - Style issues regarding `types::Handle` - Missed opportunities to reuse `Lift` and `Lower` impls It also adds forwarding `Lift` impls for `Box<[T]>`, `Vec<T>`, etc. Signed-off-by: Joel Dice <joel.dice@fermyon.com> * move `new_*` methods to specific `types` structs Per review feedback, I've moved `Type::new_record` to `Record::new_val` and added a `Type::unwrap_record` method; likewise for the other kinds of types. Signed-off-by: Joel Dice <joel.dice@fermyon.com> * make tuple, option, and expected type comparisons recursive These types should compare as equal across component boundaries as long as their type parameters are equal. Signed-off-by: Joel Dice <joel.dice@fermyon.com> * improve error diagnostic in `Type::check` We now distinguish between more failure cases to provide an informative error message. Signed-off-by: Joel Dice <joel.dice@fermyon.com> * address review feedback - Remove `WasmStr::to_str_from_memory` and `WasmList::get_from_memory` - add `try_new` methods to various `values` types - avoid using `ExactSizeIterator::len` where we can't trust it - fix over-constrained bounds on forwarded `ComponentType` impls Signed-off-by: Joel Dice <joel.dice@fermyon.com> * rearrange code per review feedback - Move functions from `types` to `values` module so we can make certain struct fields private - Rename `try_new` to just `new` Signed-off-by: Joel Dice <joel.dice@fermyon.com> * remove special-case equality test for tuples, options, and expecteds Instead, I've added a FIXME comment and will open an issue to do recursive structural equality testing. Signed-off-by: Joel Dice <joel.dice@fermyon.com>
2 years ago
[[package]]
name = "wasmtime-component-util"
version = "9.0.0"
support dynamic function calls in component model (#4442) * support dynamic function calls in component model This addresses #4310, introducing a new `component::values::Val` type for representing component values dynamically, as well as `component::types::Type` for representing the corresponding interface types. It also adds a `call` method to `component::func::Func`, which takes a slice of `Val`s as parameters and returns a `Result<Val>` representing the result. Note that I've moved `post_return` and `call_raw` from `TypedFunc` to `Func` since there was nothing specific to `TypedFunc` about them, and I wanted to reuse them. The code in both is unchanged beyond the trivial tweaks to make them fit in their new home. Signed-off-by: Joel Dice <joel.dice@fermyon.com> * order variants and match cases more consistently Signed-off-by: Joel Dice <joel.dice@fermyon.com> * implement lift for String, Box<str>, etc. This also removes the redundant `store` parameter from `Type::load`. Signed-off-by: Joel Dice <joel.dice@fermyon.com> * implement code review feedback This fixes a few issues: - Bad offset calculation when lowering - Missing variant padding - Style issues regarding `types::Handle` - Missed opportunities to reuse `Lift` and `Lower` impls It also adds forwarding `Lift` impls for `Box<[T]>`, `Vec<T>`, etc. Signed-off-by: Joel Dice <joel.dice@fermyon.com> * move `new_*` methods to specific `types` structs Per review feedback, I've moved `Type::new_record` to `Record::new_val` and added a `Type::unwrap_record` method; likewise for the other kinds of types. Signed-off-by: Joel Dice <joel.dice@fermyon.com> * make tuple, option, and expected type comparisons recursive These types should compare as equal across component boundaries as long as their type parameters are equal. Signed-off-by: Joel Dice <joel.dice@fermyon.com> * improve error diagnostic in `Type::check` We now distinguish between more failure cases to provide an informative error message. Signed-off-by: Joel Dice <joel.dice@fermyon.com> * address review feedback - Remove `WasmStr::to_str_from_memory` and `WasmList::get_from_memory` - add `try_new` methods to various `values` types - avoid using `ExactSizeIterator::len` where we can't trust it - fix over-constrained bounds on forwarded `ComponentType` impls Signed-off-by: Joel Dice <joel.dice@fermyon.com> * rearrange code per review feedback - Move functions from `types` to `values` module so we can make certain struct fields private - Rename `try_new` to just `new` Signed-off-by: Joel Dice <joel.dice@fermyon.com> * remove special-case equality test for tuples, options, and expecteds Instead, I've added a FIXME comment and will open an issue to do recursive structural equality testing. Signed-off-by: Joel Dice <joel.dice@fermyon.com>
2 years ago
[[package]]
name = "wasmtime-cranelift"
version = "9.0.0"
dependencies = [
Remove dependency on `TargetIsa` from Wasmtime crates (#3178) This commit started off by deleting the `cranelift_codegen::settings` reexport in the `wasmtime-environ` crate and then basically played whack-a-mole until everything compiled again. The main result of this is that the `wasmtime-*` family of crates have generally less of a dependency on the `TargetIsa` trait and type from Cranelift. While the dependency isn't entirely severed yet this is at least a significant start. This commit is intended to be largely refactorings, no functional changes are intended here. The refactorings are: * A `CompilerBuilder` trait has been added to `wasmtime_environ` which server as an abstraction used to create compilers and configure them in a uniform fashion. The `wasmtime::Config` type now uses this instead of cranelift-specific settings. The `wasmtime-jit` crate exports the ability to create a compiler builder from a `CompilationStrategy`, which only works for Cranelift right now. In a cranelift-less build of Wasmtime this is expected to return a trait object that fails all requests to compile. * The `Compiler` trait in the `wasmtime_environ` crate has been souped up with a number of methods that Wasmtime and other crates needed. * The `wasmtime-debug` crate is now moved entirely behind the `wasmtime-cranelift` crate. * The `wasmtime-cranelift` crate is now only depended on by the `wasmtime-jit` crate. * Wasm types in `cranelift-wasm` no longer contain their IR type, instead they only contain the `WasmType`. This is required to get everything to align correctly but will also be required in a future refactoring where the types used by `cranelift-wasm` will be extracted to a separate crate. * I moved around a fair bit of code in `wasmtime-cranelift`. * Some gdb-specific jit-specific code has moved from `wasmtime-debug` to `wasmtime-jit`.
3 years ago
"anyhow",
"cranelift-codegen",
Chaos mode MVP: Skip branch optimization in MachBuffer (#6039) * fuzz: Add chaos mode control plane Co-authored-by: Falk Zwimpfer <24669719+FalkZ@users.noreply.github.com> Co-authored-by: Moritz Waser <mzrw.dev@pm.me> * fuzz: Skip branch optimization with chaos mode Co-authored-by: Falk Zwimpfer <24669719+FalkZ@users.noreply.github.com> Co-authored-by: Moritz Waser <mzrw.dev@pm.me> * fuzz: Rename chaos engine -> control plane Co-authored-by: Falk Zwimpfer <24669719+FalkZ@users.noreply.github.com> Co-authored-by: Moritz Waser <mzrw.dev@pm.me> * chaos mode: refactoring ControlPlane to be passed through the call stack by reference Co-authored-by: Falk Zwimpfer <24669719+FalkZ@users.noreply.github.com> Co-authored-by: Remo Senekowitsch <contact@remsle.dev> * fuzz: annotate chaos todos Co-authored-by: Falk Zwimpfer <24669719+FalkZ@users.noreply.github.com> Co-authored-by: Moritz Waser <mzrw.dev@pm.me> * fuzz: cleanup control plane Co-authored-by: Falk Zwimpfer <24669719+FalkZ@users.noreply.github.com> Co-authored-by: Moritz Waser <mzrw.dev@pm.me> * fuzz: remove control plane from compiler context Co-authored-by: Falk Zwimpfer <24669719+FalkZ@users.noreply.github.com> Co-authored-by: Moritz Waser <mzrw.dev@pm.me> * fuzz: move control plane into emit state Co-authored-by: Falk Zwimpfer <24669719+FalkZ@users.noreply.github.com> Co-authored-by: Moritz Waser <mzrw.dev@pm.me> * fuzz: fix remaining compiler errors Co-authored-by: Falk Zwimpfer <24669719+FalkZ@users.noreply.github.com> Co-authored-by: Moritz Waser <mzrw.dev@pm.me> * fix tests * refactor emission state ctrl plane accessors Co-authored-by: Falk Zwimpfer <24669719+FalkZ@users.noreply.github.com> Co-authored-by: Moritz Waser <mzrw.dev@pm.me> * centralize conditional compilation of chaos mode Also cleanup a few straggling dependencies on cranelift-control that aren't needed anymore. Co-authored-by: Falk Zwimpfer <24669719+FalkZ@users.noreply.github.com> Co-authored-by: Moritz Waser <mzrw.dev@pm.me> * add cranelift-control to published crates prtest:full Co-authored-by: Falk Zwimpfer <24669719+FalkZ@users.noreply.github.com> Co-authored-by: Moritz Waser <mzrw.dev@pm.me> * add cranelift-control to public crates Co-authored-by: Falk Zwimpfer <24669719+FalkZ@users.noreply.github.com> Co-authored-by: Moritz Waser <mzrw.dev@pm.me> --------- Co-authored-by: Falk Zwimpfer <24669719+FalkZ@users.noreply.github.com> Co-authored-by: Moritz Waser <mzrw.dev@pm.me> Co-authored-by: Remo Senekowitsch <contact@remsle.dev>
2 years ago
"cranelift-control",
"cranelift-entity",
"cranelift-frontend",
Remove dependency on `TargetIsa` from Wasmtime crates (#3178) This commit started off by deleting the `cranelift_codegen::settings` reexport in the `wasmtime-environ` crate and then basically played whack-a-mole until everything compiled again. The main result of this is that the `wasmtime-*` family of crates have generally less of a dependency on the `TargetIsa` trait and type from Cranelift. While the dependency isn't entirely severed yet this is at least a significant start. This commit is intended to be largely refactorings, no functional changes are intended here. The refactorings are: * A `CompilerBuilder` trait has been added to `wasmtime_environ` which server as an abstraction used to create compilers and configure them in a uniform fashion. The `wasmtime::Config` type now uses this instead of cranelift-specific settings. The `wasmtime-jit` crate exports the ability to create a compiler builder from a `CompilationStrategy`, which only works for Cranelift right now. In a cranelift-less build of Wasmtime this is expected to return a trait object that fails all requests to compile. * The `Compiler` trait in the `wasmtime_environ` crate has been souped up with a number of methods that Wasmtime and other crates needed. * The `wasmtime-debug` crate is now moved entirely behind the `wasmtime-cranelift` crate. * The `wasmtime-cranelift` crate is now only depended on by the `wasmtime-jit` crate. * Wasm types in `cranelift-wasm` no longer contain their IR type, instead they only contain the `WasmType`. This is required to get everything to align correctly but will also be required in a future refactoring where the types used by `cranelift-wasm` will be extracted to a separate crate. * I moved around a fair bit of code in `wasmtime-cranelift`. * Some gdb-specific jit-specific code has moved from `wasmtime-debug` to `wasmtime-jit`.
3 years ago
"cranelift-native",
"cranelift-wasm",
Remove dependency on `TargetIsa` from Wasmtime crates (#3178) This commit started off by deleting the `cranelift_codegen::settings` reexport in the `wasmtime-environ` crate and then basically played whack-a-mole until everything compiled again. The main result of this is that the `wasmtime-*` family of crates have generally less of a dependency on the `TargetIsa` trait and type from Cranelift. While the dependency isn't entirely severed yet this is at least a significant start. This commit is intended to be largely refactorings, no functional changes are intended here. The refactorings are: * A `CompilerBuilder` trait has been added to `wasmtime_environ` which server as an abstraction used to create compilers and configure them in a uniform fashion. The `wasmtime::Config` type now uses this instead of cranelift-specific settings. The `wasmtime-jit` crate exports the ability to create a compiler builder from a `CompilationStrategy`, which only works for Cranelift right now. In a cranelift-less build of Wasmtime this is expected to return a trait object that fails all requests to compile. * The `Compiler` trait in the `wasmtime_environ` crate has been souped up with a number of methods that Wasmtime and other crates needed. * The `wasmtime-debug` crate is now moved entirely behind the `wasmtime-cranelift` crate. * The `wasmtime-cranelift` crate is now only depended on by the `wasmtime-jit` crate. * Wasm types in `cranelift-wasm` no longer contain their IR type, instead they only contain the `WasmType`. This is required to get everything to align correctly but will also be required in a future refactoring where the types used by `cranelift-wasm` will be extracted to a separate crate. * I moved around a fair bit of code in `wasmtime-cranelift`. * Some gdb-specific jit-specific code has moved from `wasmtime-debug` to `wasmtime-jit`.
3 years ago
"gimli",
"log",
"object",
"target-lexicon",
"thiserror",
"wasmparser",
winch: Refactoring wasmtime compiler integration pieces to share more between Cranelift and Winch (#5944) * Enable the native target by default in winch Match cranelift-codegen's build script where if no architecture is explicitly enabled then the host architecture is implicitly enabled. * Refactor Cranelift's ISA builder to share more with Winch This commit refactors the `Builder` type to have a type parameter representing the finished ISA with Cranelift and Winch having their own typedefs for `Builder` to represent their own builders. The intention is to use this shared functionality to produce more shared code between the two codegen backends. * Moving compiler shared components to a separate crate * Restore native flag inference in compiler building This fixes an oversight from the previous commits to use `cranelift-native` to infer flags for the native host when using default settings with Wasmtime. * Move `Compiler::page_size_align` into wasmtime-environ The `cranelift-codegen` crate doesn't need this and winch wants the same implementation, so shuffle it around so everyone has access to it. * Fill out `Compiler::{flags, isa_flags}` for Winch These are easy enough to plumb through with some shared code for Wasmtime. * Plumb the `is_branch_protection_enabled` flag for Winch Just forwarding an isa-specific setting accessor. * Moving executable creation to shared compiler crate * Adding builder back in and removing from shared crate * Refactoring the shared pieces for the `CompilerBuilder` I decided to move a couple things around from Alex's initial changes. Instead of having the shared builder do everything, I went back to having each compiler have a distinct builder implementation. I refactored most of the flag setting logic into a single shared location, so we can still reduce the amount of code duplication. With them being separate, we don't need to maintain things like `LinkOpts` which Winch doesn't currently use. We also have an avenue to error when certain flags are sent to Winch if we don't support them. I'm hoping this will make things more maintainable as we build out Winch. I'm still unsure about keeping everything shared in a single crate (`cranelift_shared`). It's starting to feel like this crate is doing too much, which makes it difficult to name. There does seem to be a need for two distinct abstraction: creating the final executable and the handling of shared/ISA flags when building the compiler. I could make them into two separate crates, but there doesn't seem to be enough there yet to justify it. * Documentation updates, and renaming the finish method * Adding back in a default temporarily to pass tests, and removing some unused imports * Fixing winch tests with wrong method name * Removing unused imports from codegen shared crate * Apply documentation formatting updates Co-authored-by: Saúl Cabrera <saulecabrera@gmail.com> * Adding back in cranelift_native flag inferring * Adding new shared crate to publish list * Adding write feature to pass cargo check --------- Co-authored-by: Alex Crichton <alex@alexcrichton.com> Co-authored-by: Saúl Cabrera <saulecabrera@gmail.com>
2 years ago
"wasmtime-cranelift-shared",
"wasmtime-environ",
]
[[package]]
name = "wasmtime-cranelift-shared"
version = "9.0.0"
winch: Refactoring wasmtime compiler integration pieces to share more between Cranelift and Winch (#5944) * Enable the native target by default in winch Match cranelift-codegen's build script where if no architecture is explicitly enabled then the host architecture is implicitly enabled. * Refactor Cranelift's ISA builder to share more with Winch This commit refactors the `Builder` type to have a type parameter representing the finished ISA with Cranelift and Winch having their own typedefs for `Builder` to represent their own builders. The intention is to use this shared functionality to produce more shared code between the two codegen backends. * Moving compiler shared components to a separate crate * Restore native flag inference in compiler building This fixes an oversight from the previous commits to use `cranelift-native` to infer flags for the native host when using default settings with Wasmtime. * Move `Compiler::page_size_align` into wasmtime-environ The `cranelift-codegen` crate doesn't need this and winch wants the same implementation, so shuffle it around so everyone has access to it. * Fill out `Compiler::{flags, isa_flags}` for Winch These are easy enough to plumb through with some shared code for Wasmtime. * Plumb the `is_branch_protection_enabled` flag for Winch Just forwarding an isa-specific setting accessor. * Moving executable creation to shared compiler crate * Adding builder back in and removing from shared crate * Refactoring the shared pieces for the `CompilerBuilder` I decided to move a couple things around from Alex's initial changes. Instead of having the shared builder do everything, I went back to having each compiler have a distinct builder implementation. I refactored most of the flag setting logic into a single shared location, so we can still reduce the amount of code duplication. With them being separate, we don't need to maintain things like `LinkOpts` which Winch doesn't currently use. We also have an avenue to error when certain flags are sent to Winch if we don't support them. I'm hoping this will make things more maintainable as we build out Winch. I'm still unsure about keeping everything shared in a single crate (`cranelift_shared`). It's starting to feel like this crate is doing too much, which makes it difficult to name. There does seem to be a need for two distinct abstraction: creating the final executable and the handling of shared/ISA flags when building the compiler. I could make them into two separate crates, but there doesn't seem to be enough there yet to justify it. * Documentation updates, and renaming the finish method * Adding back in a default temporarily to pass tests, and removing some unused imports * Fixing winch tests with wrong method name * Removing unused imports from codegen shared crate * Apply documentation formatting updates Co-authored-by: Saúl Cabrera <saulecabrera@gmail.com> * Adding back in cranelift_native flag inferring * Adding new shared crate to publish list * Adding write feature to pass cargo check --------- Co-authored-by: Alex Crichton <alex@alexcrichton.com> Co-authored-by: Saúl Cabrera <saulecabrera@gmail.com>
2 years ago
dependencies = [
"anyhow",
"cranelift-codegen",
Chaos mode MVP: Skip branch optimization in MachBuffer (#6039) * fuzz: Add chaos mode control plane Co-authored-by: Falk Zwimpfer <24669719+FalkZ@users.noreply.github.com> Co-authored-by: Moritz Waser <mzrw.dev@pm.me> * fuzz: Skip branch optimization with chaos mode Co-authored-by: Falk Zwimpfer <24669719+FalkZ@users.noreply.github.com> Co-authored-by: Moritz Waser <mzrw.dev@pm.me> * fuzz: Rename chaos engine -> control plane Co-authored-by: Falk Zwimpfer <24669719+FalkZ@users.noreply.github.com> Co-authored-by: Moritz Waser <mzrw.dev@pm.me> * chaos mode: refactoring ControlPlane to be passed through the call stack by reference Co-authored-by: Falk Zwimpfer <24669719+FalkZ@users.noreply.github.com> Co-authored-by: Remo Senekowitsch <contact@remsle.dev> * fuzz: annotate chaos todos Co-authored-by: Falk Zwimpfer <24669719+FalkZ@users.noreply.github.com> Co-authored-by: Moritz Waser <mzrw.dev@pm.me> * fuzz: cleanup control plane Co-authored-by: Falk Zwimpfer <24669719+FalkZ@users.noreply.github.com> Co-authored-by: Moritz Waser <mzrw.dev@pm.me> * fuzz: remove control plane from compiler context Co-authored-by: Falk Zwimpfer <24669719+FalkZ@users.noreply.github.com> Co-authored-by: Moritz Waser <mzrw.dev@pm.me> * fuzz: move control plane into emit state Co-authored-by: Falk Zwimpfer <24669719+FalkZ@users.noreply.github.com> Co-authored-by: Moritz Waser <mzrw.dev@pm.me> * fuzz: fix remaining compiler errors Co-authored-by: Falk Zwimpfer <24669719+FalkZ@users.noreply.github.com> Co-authored-by: Moritz Waser <mzrw.dev@pm.me> * fix tests * refactor emission state ctrl plane accessors Co-authored-by: Falk Zwimpfer <24669719+FalkZ@users.noreply.github.com> Co-authored-by: Moritz Waser <mzrw.dev@pm.me> * centralize conditional compilation of chaos mode Also cleanup a few straggling dependencies on cranelift-control that aren't needed anymore. Co-authored-by: Falk Zwimpfer <24669719+FalkZ@users.noreply.github.com> Co-authored-by: Moritz Waser <mzrw.dev@pm.me> * add cranelift-control to published crates prtest:full Co-authored-by: Falk Zwimpfer <24669719+FalkZ@users.noreply.github.com> Co-authored-by: Moritz Waser <mzrw.dev@pm.me> * add cranelift-control to public crates Co-authored-by: Falk Zwimpfer <24669719+FalkZ@users.noreply.github.com> Co-authored-by: Moritz Waser <mzrw.dev@pm.me> --------- Co-authored-by: Falk Zwimpfer <24669719+FalkZ@users.noreply.github.com> Co-authored-by: Moritz Waser <mzrw.dev@pm.me> Co-authored-by: Remo Senekowitsch <contact@remsle.dev>
2 years ago
"cranelift-control",
winch: Refactoring wasmtime compiler integration pieces to share more between Cranelift and Winch (#5944) * Enable the native target by default in winch Match cranelift-codegen's build script where if no architecture is explicitly enabled then the host architecture is implicitly enabled. * Refactor Cranelift's ISA builder to share more with Winch This commit refactors the `Builder` type to have a type parameter representing the finished ISA with Cranelift and Winch having their own typedefs for `Builder` to represent their own builders. The intention is to use this shared functionality to produce more shared code between the two codegen backends. * Moving compiler shared components to a separate crate * Restore native flag inference in compiler building This fixes an oversight from the previous commits to use `cranelift-native` to infer flags for the native host when using default settings with Wasmtime. * Move `Compiler::page_size_align` into wasmtime-environ The `cranelift-codegen` crate doesn't need this and winch wants the same implementation, so shuffle it around so everyone has access to it. * Fill out `Compiler::{flags, isa_flags}` for Winch These are easy enough to plumb through with some shared code for Wasmtime. * Plumb the `is_branch_protection_enabled` flag for Winch Just forwarding an isa-specific setting accessor. * Moving executable creation to shared compiler crate * Adding builder back in and removing from shared crate * Refactoring the shared pieces for the `CompilerBuilder` I decided to move a couple things around from Alex's initial changes. Instead of having the shared builder do everything, I went back to having each compiler have a distinct builder implementation. I refactored most of the flag setting logic into a single shared location, so we can still reduce the amount of code duplication. With them being separate, we don't need to maintain things like `LinkOpts` which Winch doesn't currently use. We also have an avenue to error when certain flags are sent to Winch if we don't support them. I'm hoping this will make things more maintainable as we build out Winch. I'm still unsure about keeping everything shared in a single crate (`cranelift_shared`). It's starting to feel like this crate is doing too much, which makes it difficult to name. There does seem to be a need for two distinct abstraction: creating the final executable and the handling of shared/ISA flags when building the compiler. I could make them into two separate crates, but there doesn't seem to be enough there yet to justify it. * Documentation updates, and renaming the finish method * Adding back in a default temporarily to pass tests, and removing some unused imports * Fixing winch tests with wrong method name * Removing unused imports from codegen shared crate * Apply documentation formatting updates Co-authored-by: Saúl Cabrera <saulecabrera@gmail.com> * Adding back in cranelift_native flag inferring * Adding new shared crate to publish list * Adding write feature to pass cargo check --------- Co-authored-by: Alex Crichton <alex@alexcrichton.com> Co-authored-by: Saúl Cabrera <saulecabrera@gmail.com>
2 years ago
"cranelift-native",
"gimli",
"object",
"target-lexicon",
"wasmtime-environ",
]
[[package]]
name = "wasmtime-environ"
version = "9.0.0"
dependencies = [
Remove dependency on `TargetIsa` from Wasmtime crates (#3178) This commit started off by deleting the `cranelift_codegen::settings` reexport in the `wasmtime-environ` crate and then basically played whack-a-mole until everything compiled again. The main result of this is that the `wasmtime-*` family of crates have generally less of a dependency on the `TargetIsa` trait and type from Cranelift. While the dependency isn't entirely severed yet this is at least a significant start. This commit is intended to be largely refactorings, no functional changes are intended here. The refactorings are: * A `CompilerBuilder` trait has been added to `wasmtime_environ` which server as an abstraction used to create compilers and configure them in a uniform fashion. The `wasmtime::Config` type now uses this instead of cranelift-specific settings. The `wasmtime-jit` crate exports the ability to create a compiler builder from a `CompilationStrategy`, which only works for Cranelift right now. In a cranelift-less build of Wasmtime this is expected to return a trait object that fails all requests to compile. * The `Compiler` trait in the `wasmtime_environ` crate has been souped up with a number of methods that Wasmtime and other crates needed. * The `wasmtime-debug` crate is now moved entirely behind the `wasmtime-cranelift` crate. * The `wasmtime-cranelift` crate is now only depended on by the `wasmtime-jit` crate. * Wasm types in `cranelift-wasm` no longer contain their IR type, instead they only contain the `WasmType`. This is required to get everything to align correctly but will also be required in a future refactoring where the types used by `cranelift-wasm` will be extracted to a separate crate. * I moved around a fair bit of code in `wasmtime-cranelift`. * Some gdb-specific jit-specific code has moved from `wasmtime-debug` to `wasmtime-jit`.
3 years ago
"anyhow",
Implement fused adapters for `(list T)` types (#4558) * Implement fused adapters for `(list T)` types This commit implements one of the two remaining types for adapter fusion, lists. This implementation is particularly tricky for a number of reasons: * Lists have a number of validity checks which need to be carefully implemented. For example the byte length of the list passed to allocation in the destination module could overflow the 32-bit index space. Additionally lists in 32-bit memories need a check that their final address is in-bounds in the address space. * In the effort to go ahead and support memory64 at the lowest layers this is where much of the magic happens. Lists are naturally always stored in memory and shifting between 64/32-bit address spaces is done here. This notably required plumbing an `Options` around during flattening/size/alignment calculations due to the size/types of lists changing depending on the memory configuration. I've also added a small `factc` program in this commit which should hopefully assist in exploring and debugging adapter modules. This takes as input a component (text or binary format) and then generates an adapter module for all component function signatures found internally. This commit notably does not include tests for lists. I tried to figure out a good way to add these but I felt like there were too many cases to test and the tests would otherwise be extremely verbose. Instead I think the best testing strategy for this commit will be through #4537 which should be relatively extensible to testing adapters between modules in addition to host-based lifting/lowering. * Improve handling of lists of 0-size types * Skip overflow checks on byte sizes for 0-size types * Skip the copy loop entirely when src/dst are both 0 * Skip the increments of src/dst pointers if either is 0-size * Update semantics for zero-sized lists/strings When a list/string has a 0-byte-size the base pointer is no longer verified to be in-bounds to match the supposedly desired adapter semantics where no trap happens because no turn of the loop happens.
2 years ago
"atty",
"clap",
"cranelift-entity",
Implement fused adapters for `(list T)` types (#4558) * Implement fused adapters for `(list T)` types This commit implements one of the two remaining types for adapter fusion, lists. This implementation is particularly tricky for a number of reasons: * Lists have a number of validity checks which need to be carefully implemented. For example the byte length of the list passed to allocation in the destination module could overflow the 32-bit index space. Additionally lists in 32-bit memories need a check that their final address is in-bounds in the address space. * In the effort to go ahead and support memory64 at the lowest layers this is where much of the magic happens. Lists are naturally always stored in memory and shifting between 64/32-bit address spaces is done here. This notably required plumbing an `Options` around during flattening/size/alignment calculations due to the size/types of lists changing depending on the memory configuration. I've also added a small `factc` program in this commit which should hopefully assist in exploring and debugging adapter modules. This takes as input a component (text or binary format) and then generates an adapter module for all component function signatures found internally. This commit notably does not include tests for lists. I tried to figure out a good way to add these but I felt like there were too many cases to test and the tests would otherwise be extremely verbose. Instead I think the best testing strategy for this commit will be through #4537 which should be relatively extensible to testing adapters between modules in addition to host-based lifting/lowering. * Improve handling of lists of 0-size types * Skip overflow checks on byte sizes for 0-size types * Skip the copy loop entirely when src/dst are both 0 * Skip the increments of src/dst pointers if either is 0-size * Update semantics for zero-sized lists/strings When a list/string has a 0-byte-size the base pointer is no longer verified to be in-bounds to match the supposedly desired adapter semantics where no trap happens because no turn of the loop happens.
2 years ago
"env_logger 0.9.0",
"gimli",
"indexmap",
"log",
"object",
"serde",
Remove dependency on `TargetIsa` from Wasmtime crates (#3178) This commit started off by deleting the `cranelift_codegen::settings` reexport in the `wasmtime-environ` crate and then basically played whack-a-mole until everything compiled again. The main result of this is that the `wasmtime-*` family of crates have generally less of a dependency on the `TargetIsa` trait and type from Cranelift. While the dependency isn't entirely severed yet this is at least a significant start. This commit is intended to be largely refactorings, no functional changes are intended here. The refactorings are: * A `CompilerBuilder` trait has been added to `wasmtime_environ` which server as an abstraction used to create compilers and configure them in a uniform fashion. The `wasmtime::Config` type now uses this instead of cranelift-specific settings. The `wasmtime-jit` crate exports the ability to create a compiler builder from a `CompilationStrategy`, which only works for Cranelift right now. In a cranelift-less build of Wasmtime this is expected to return a trait object that fails all requests to compile. * The `Compiler` trait in the `wasmtime_environ` crate has been souped up with a number of methods that Wasmtime and other crates needed. * The `wasmtime-debug` crate is now moved entirely behind the `wasmtime-cranelift` crate. * The `wasmtime-cranelift` crate is now only depended on by the `wasmtime-jit` crate. * Wasm types in `cranelift-wasm` no longer contain their IR type, instead they only contain the `WasmType`. This is required to get everything to align correctly but will also be required in a future refactoring where the types used by `cranelift-wasm` will be extracted to a separate crate. * I moved around a fair bit of code in `wasmtime-cranelift`. * Some gdb-specific jit-specific code has moved from `wasmtime-debug` to `wasmtime-jit`.
3 years ago
"target-lexicon",
"thiserror",
"wasm-encoder 0.25.0",
"wasmparser",
Add initial support for fused adapter trampolines (#4501) * Add initial support for fused adapter trampolines This commit lands a significant new piece of functionality to Wasmtime's implementation of the component model in the form of the implementation of fused adapter trampolines. Internally within a component core wasm modules can communicate with each other by having their exports `canon lift`'d to get `canon lower`'d into a different component. This signifies that two components are communicating through a statically known interface via the canonical ABI at this time. Previously Wasmtime was able to identify that this communication was happening but it simply panicked with `unimplemented!` upon seeing it. This commit is the beginning of filling out this panic location with an actual implementation. The implementation route chosen here for fused adapters is to use a WebAssembly module itself for the implementation. This means that, at compile time of a component, Wasmtime is generating core WebAssembly modules which then get recursively compiled within Wasmtime as well. The choice to use WebAssembly itself as the implementation of fused adapters stems from a few motivations: * This does not represent a significant increase in the "trusted compiler base" of Wasmtime. Getting the Wasm -> CLIF translation correct once is hard enough much less for an entirely different IR to CLIF. By generating WebAssembly no new interactions with Cranelift are added which drastically reduces the possibilities for mistakes. * Using WebAssembly means that component adapters are insulated from miscompilations and mistakes. If something goes wrong it's defined well within the WebAssembly specification how it goes wrong and what happens as a result. This means that the "blast zone" for a wrong adapter is the component instance but not the entire host itself. Accesses to linear memory are guaranteed to be in-bounds and otherwise handled via well-defined traps. * A fully-finished fused adapter compiler is expected to be a significant and quite complex component of Wasmtime. Functionality along these lines is expected to be needed for Web-based polyfills of the component model and by using core WebAssembly it provides the opportunity to share code between Wasmtime and these polyfills for the component model. * Finally the runtime implementation of managing WebAssembly modules is already implemented and quite easy to integrate with, so representing fused adapters with WebAssembly results in very little extra support necessary for the runtime implementation of instantiating and managing a component. The compiler added in this commit is dubbed Wasmtime's Fused Adapter Compiler of Trampolines (FACT) because who doesn't like deriving a name from an acronym. Currently the trampoline compiler is limited in its support for interface types and only supports a few primitives. I plan on filing future PRs to flesh out the support here for all the variants of `InterfaceType`. For now this PR is primarily focused on all of the other infrastructure for the addition of a trampoline compiler. With the choice to use core WebAssembly to implement fused adapters it means that adapters need to be inserted into a module. Unfortunately adapters cannot all go into a single WebAssembly module because adapters themselves have dependencies which may be provided transitively through instances that were instantiated with other adapters. This means that a significant chunk of this PR (`adapt.rs`) is dedicated to determining precisely which adapters go into precisely which adapter modules. This partitioning process attempts to make large modules wherever it can to cut down on core wasm instantiations but is likely not optimal as it's just a simple heuristic today. With all of this added together it's now possible to start writing `*.wast` tests that internally have adapted modules communicating with one another. A `fused.wast` test suite was added as part of this PR which is the beginning of tests for the support of the fused adapter compiler added in this PR. Currently this is primarily testing some various topologies of adapters along with direct/indirect modes. This will grow many more tests over time as more types are supported. Overall I'm not 100% satisfied with the testing story of this PR. When a test fails it's very difficult to debug since everything is written in the text format of WebAssembly meaning there's no "conveniences" to print out the state of the world when things go wrong and easily debug. I think this will become even more apparent as more tests are written for more types in subsequent PRs. At this time though I know of no better alternative other than leaning pretty heavily on fuzz-testing to ensure this is all exercised. * Fix an unused field warning * Fix tests in `wasmtime-runtime` * Add some more tests for compiled trampolines * Remap exports when injecting adapters The exports of a component were accidentally left unmapped which meant that they indexed the instance indexes pre-adapter module insertion. * Fix typo * Rebase conflicts
2 years ago
"wasmprinter",
Implement variant translation in fused adapters (#4534) * Implement variant translation in fused adapters This commit implements the most general case of variants for fused adapter trampolines. Additionally a number of other primitive types are filled out here to assist with testing variants. The implementation internally was relatively straightforward given the shape of variants, but there's room for future optimization as necessary especially around converting locals to various types. This commit also introduces a "one off" fuzzer for adapters to ensure that the generated adapter is valid. I hope to extend this fuzz generator as more types are implemented to assist in various corner cases that might arise. For now the fuzzer simply tests that the output wasm module is valid, not that it actually executes correctly. I hope to integrate with a fuzzer along the lines of #4307 one day to test the run-time-correctness of the generated adapters as well, at which point this fuzzer would become obsolete. Finally this commit also fixes an issue with `u8` translation where upper bits weren't zero'd out and were passed raw across modules. Instead smaller-than-32 types now all mask out their upper bits and do sign-extension as appropriate for unsigned/signed variants. * Fuzz memory64 in the new trampoline fuzzer Currently memory64 isn't supported elsewhere in the component model implementation of Wasmtime but the trampoline compiler seems as good a place as any to ensure that it at least works in isolation. This plumbs through fuzz input into a `memory64` boolean which gets fed into compilation. Some miscellaneous bugs were fixed as a result to ensure that memory64 trampolines all validate correctly. * Tweak manifest for doc build
2 years ago
"wasmtime-component-util",
Remove `wasmtime-environ`'s dependency on `cranelift-codegen` (#3199) * Move `CompiledFunction` into wasmtime-cranelift This commit moves the `wasmtime_environ::CompiledFunction` type into the `wasmtime-cranelift` crate. This type has lots of Cranelift-specific pieces of compilation and doesn't need to be generated by all Wasmtime compilers. This replaces the usage in the `Compiler` trait with a `Box<Any>` type that each compiler can select. Each compiler must still produce a `FunctionInfo`, however, which is shared information we'll deserialize for each module. The `wasmtime-debug` crate is also folded into the `wasmtime-cranelift` crate as a result of this commit. One possibility was to move the `CompiledFunction` commit into its own crate and have `wasmtime-debug` depend on that, but since `wasmtime-debug` is Cranelift-specific at this time it didn't seem like it was too too necessary to keep it separate. If `wasmtime-debug` supports other backends in the future we can recreate a new crate, perhaps with it refactored to not depend on Cranelift. * Move wasmtime_environ::reference_type This now belongs in wasmtime-cranelift and nowhere else * Remove `Type` reexport in wasmtime-environ One less dependency on `cranelift-codegen`! * Remove `types` reexport from `wasmtime-environ` Less cranelift! * Remove `SourceLoc` from wasmtime-environ Change the `srcloc`, `start_srcloc`, and `end_srcloc` fields to a custom `FilePos` type instead of `ir::SourceLoc`. These are only used in a few places so there's not much to lose from an extra abstraction for these leaf use cases outside of cranelift. * Remove wasmtime-environ's dep on cranelift's `StackMap` This commit "clones" the `StackMap` data structure in to `wasmtime-environ` to have an independent representation that that chosen by Cranelift. This allows Wasmtime to decouple this runtime dependency of stack map information and let the two evolve independently, if necessary. An alternative would be to refactor cranelift's implementation into a separate crate and have wasmtime depend on that but it seemed a bit like overkill to do so and easier to clone just a few lines for this. * Define code offsets in wasmtime-environ with `u32` Don't use Cranelift's `binemit::CodeOffset` alias to define this field type since the `wasmtime-environ` crate will be losing the `cranelift-codegen` dependency soon. * Commit to using `cranelift-entity` in Wasmtime This commit removes the reexport of `cranelift-entity` from the `wasmtime-environ` crate and instead directly depends on the `cranelift-entity` crate in all referencing crates. The original reason for the reexport was to make cranelift version bumps easier since it's less versions to change, but nowadays we have a script to do that. Otherwise this encourages crates to use whatever they want from `cranelift-entity` since we'll always depend on the whole crate. It's expected that the `cranelift-entity` crate will continue to be a lean crate in dependencies and suitable for use at both runtime and compile time. Consequently there's no need to avoid its usage in Wasmtime at runtime, since "remove Cranelift at compile time" is primarily about the `cranelift-codegen` crate. * Remove most uses of `cranelift-codegen` in `wasmtime-environ` There's only one final use remaining, which is the reexport of `TrapCode`, which will get handled later. * Limit the glob-reexport of `cranelift_wasm` This commit removes the glob reexport of `cranelift-wasm` from the `wasmtime-environ` crate. This is intended to explicitly define what we're reexporting and is a transitionary step to curtail the amount of dependencies taken on `cranelift-wasm` throughout the codebase. For example some functions used by debuginfo mapping are better imported directly from the crate since they're Cranelift-specific. Note that this is intended to be a temporary state affairs, soon this reexport will be gone entirely. Additionally this commit reduces imports from `cranelift_wasm` and also primarily imports from `crate::wasm` within `wasmtime-environ` to get a better sense of what's imported from where and what will need to be shared. * Extract types from cranelift-wasm to cranelift-wasm-types This commit creates a new crate called `cranelift-wasm-types` and extracts type definitions from the `cranelift-wasm` crate into this new crate. The purpose of this crate is to be a shared definition of wasm types that can be shared both by compilers (like Cranelift) as well as wasm runtimes (e.g. Wasmtime). This new `cranelift-wasm-types` crate doesn't depend on `cranelift-codegen` and is the final step in severing the unconditional dependency from Wasmtime to `cranelift-codegen`. The final refactoring in this commit is to then reexport this crate from `wasmtime-environ`, delete the `cranelift-codegen` dependency, and then update all `use` paths to point to these new types. The main change of substance here is that the `TrapCode` enum is mirrored from Cranelift into this `cranelift-wasm-types` crate. While this unfortunately results in three definitions (one more which is non-exhaustive in Wasmtime itself) it's hopefully not too onerous and ideally something we can patch up in the future. * Get lightbeam compiling * Remove unnecessary dependency * Fix compile with uffd * Update publish script * Fix more uffd tests * Rename cranelift-wasm-types to wasmtime-types This reflects the purpose a bit more where it's types specifically intended for Wasmtime and its support. * Fix publish script
3 years ago
"wasmtime-types",
Implement fused adapters for `(list T)` types (#4558) * Implement fused adapters for `(list T)` types This commit implements one of the two remaining types for adapter fusion, lists. This implementation is particularly tricky for a number of reasons: * Lists have a number of validity checks which need to be carefully implemented. For example the byte length of the list passed to allocation in the destination module could overflow the 32-bit index space. Additionally lists in 32-bit memories need a check that their final address is in-bounds in the address space. * In the effort to go ahead and support memory64 at the lowest layers this is where much of the magic happens. Lists are naturally always stored in memory and shifting between 64/32-bit address spaces is done here. This notably required plumbing an `Options` around during flattening/size/alignment calculations due to the size/types of lists changing depending on the memory configuration. I've also added a small `factc` program in this commit which should hopefully assist in exploring and debugging adapter modules. This takes as input a component (text or binary format) and then generates an adapter module for all component function signatures found internally. This commit notably does not include tests for lists. I tried to figure out a good way to add these but I felt like there were too many cases to test and the tests would otherwise be extremely verbose. Instead I think the best testing strategy for this commit will be through #4537 which should be relatively extensible to testing adapters between modules in addition to host-based lifting/lowering. * Improve handling of lists of 0-size types * Skip overflow checks on byte sizes for 0-size types * Skip the copy loop entirely when src/dst are both 0 * Skip the increments of src/dst pointers if either is 0-size * Update semantics for zero-sized lists/strings When a list/string has a 0-byte-size the base pointer is no longer verified to be in-bounds to match the supposedly desired adapter semantics where no trap happens because no turn of the loop happens.
2 years ago
"wat",
]
Implement variant translation in fused adapters (#4534) * Implement variant translation in fused adapters This commit implements the most general case of variants for fused adapter trampolines. Additionally a number of other primitive types are filled out here to assist with testing variants. The implementation internally was relatively straightforward given the shape of variants, but there's room for future optimization as necessary especially around converting locals to various types. This commit also introduces a "one off" fuzzer for adapters to ensure that the generated adapter is valid. I hope to extend this fuzz generator as more types are implemented to assist in various corner cases that might arise. For now the fuzzer simply tests that the output wasm module is valid, not that it actually executes correctly. I hope to integrate with a fuzzer along the lines of #4307 one day to test the run-time-correctness of the generated adapters as well, at which point this fuzzer would become obsolete. Finally this commit also fixes an issue with `u8` translation where upper bits weren't zero'd out and were passed raw across modules. Instead smaller-than-32 types now all mask out their upper bits and do sign-extension as appropriate for unsigned/signed variants. * Fuzz memory64 in the new trampoline fuzzer Currently memory64 isn't supported elsewhere in the component model implementation of Wasmtime but the trampoline compiler seems as good a place as any to ensure that it at least works in isolation. This plumbs through fuzz input into a `memory64` boolean which gets fed into compilation. Some miscellaneous bugs were fixed as a result to ensure that memory64 trampolines all validate correctly. * Tweak manifest for doc build
2 years ago
[[package]]
name = "wasmtime-environ-fuzz"
version = "0.0.0"
dependencies = [
"arbitrary",
"component-fuzz-util",
Implement variant translation in fused adapters (#4534) * Implement variant translation in fused adapters This commit implements the most general case of variants for fused adapter trampolines. Additionally a number of other primitive types are filled out here to assist with testing variants. The implementation internally was relatively straightforward given the shape of variants, but there's room for future optimization as necessary especially around converting locals to various types. This commit also introduces a "one off" fuzzer for adapters to ensure that the generated adapter is valid. I hope to extend this fuzz generator as more types are implemented to assist in various corner cases that might arise. For now the fuzzer simply tests that the output wasm module is valid, not that it actually executes correctly. I hope to integrate with a fuzzer along the lines of #4307 one day to test the run-time-correctness of the generated adapters as well, at which point this fuzzer would become obsolete. Finally this commit also fixes an issue with `u8` translation where upper bits weren't zero'd out and were passed raw across modules. Instead smaller-than-32 types now all mask out their upper bits and do sign-extension as appropriate for unsigned/signed variants. * Fuzz memory64 in the new trampoline fuzzer Currently memory64 isn't supported elsewhere in the component model implementation of Wasmtime but the trampoline compiler seems as good a place as any to ensure that it at least works in isolation. This plumbs through fuzz input into a `memory64` boolean which gets fed into compilation. Some miscellaneous bugs were fixed as a result to ensure that memory64 trampolines all validate correctly. * Tweak manifest for doc build
2 years ago
"env_logger 0.9.0",
"libfuzzer-sys",
"wasmparser",
"wasmprinter",
"wasmtime-environ",
Deduplicate some size/align calculations (#4658) This commit is an effort to reduce the amount of complexity around managing the size/alignment calculations of types in the canonical ABI. Previously the logic for the size/alignment of a type was spread out across a number of locations. While each individual calculation is not really the most complicated thing in the world having the duplication in so many places was constantly worrying me. I've opted in this commit to centralize all of this within the runtime at least, and now there's only one "duplicate" of this information in the fuzzing infrastructure which is to some degree less important to deduplicate. This commit introduces a new `CanonicalAbiInfo` type to house all abi size/align information for both memory32 and memory64. This new type is then used pervasively throughout fused adapter compilation, dynamic `Val` management, and typed functions. This type was also able to reduce the complexity of the macro-generated code meaning that even `wasmtime-component-macro` is performing less math than it was before. One other major feature of this commit is that this ABI information is now saved within a `ComponentTypes` structure. This avoids recursive querying of size/align information frequently and instead effectively caching it. This was a worry I had for the fused adapter compiler which frequently sought out size/align information and would recursively descend each type tree each time. The `fact-valid-module` fuzzer is now nearly 10x faster in terms of iterations/s which I suspect is due to this caching.
2 years ago
"wat",
Implement variant translation in fused adapters (#4534) * Implement variant translation in fused adapters This commit implements the most general case of variants for fused adapter trampolines. Additionally a number of other primitive types are filled out here to assist with testing variants. The implementation internally was relatively straightforward given the shape of variants, but there's room for future optimization as necessary especially around converting locals to various types. This commit also introduces a "one off" fuzzer for adapters to ensure that the generated adapter is valid. I hope to extend this fuzz generator as more types are implemented to assist in various corner cases that might arise. For now the fuzzer simply tests that the output wasm module is valid, not that it actually executes correctly. I hope to integrate with a fuzzer along the lines of #4307 one day to test the run-time-correctness of the generated adapters as well, at which point this fuzzer would become obsolete. Finally this commit also fixes an issue with `u8` translation where upper bits weren't zero'd out and were passed raw across modules. Instead smaller-than-32 types now all mask out their upper bits and do sign-extension as appropriate for unsigned/signed variants. * Fuzz memory64 in the new trampoline fuzzer Currently memory64 isn't supported elsewhere in the component model implementation of Wasmtime but the trampoline compiler seems as good a place as any to ensure that it at least works in isolation. This plumbs through fuzz input into a `memory64` boolean which gets fed into compilation. Some miscellaneous bugs were fixed as a result to ensure that memory64 trampolines all validate correctly. * Tweak manifest for doc build
2 years ago
]
[[package]]
name = "wasmtime-explorer"
version = "9.0.0"
dependencies = [
"anyhow",
"capstone",
"serde",
"serde_json",
"target-lexicon",
"wasmprinter",
"wasmtime",
]
Implement support for `async` functions in Wasmtime (#2434) * Implement support for `async` functions in Wasmtime This is an implementation of [RFC 2] in Wasmtime which is to support `async`-defined host functions. At a high level support is added by executing WebAssembly code that might invoke an asynchronous host function on a separate native stack. When the host function's future is not ready we switch back to the main native stack to continue execution. There's a whole bunch of details in this commit, and it's a bit much to go over them all here in this commit message. The most important changes here are: * A new `wasmtime-fiber` crate has been written to manage the low-level details of stack-switching. Unixes use `mmap` to allocate a stack and Windows uses the native fibers implementation. We'll surely want to refactor this to move stack allocation elsewhere in the future. Fibers are intended to be relatively general with a lot of type paremters to fling values back and forth across suspension points. The whole crate is a giant wad of `unsafe` unfortunately and involves handwritten assembly with custom dwarf CFI directives to boot. Definitely deserves a close eye in review! * The `Store` type has two new methods -- `block_on` and `on_fiber` which bridge between the async and non-async worlds. Lots of unsafe fiddly bits here as we're trying to communicate context pointers between disparate portions of the code. Extra eyes and care in review is greatly appreciated. * The APIs for binding `async` functions are unfortunately pretty ugly in `Func`. This is mostly due to language limitations and compiler bugs (I believe) in Rust. Instead of `Func::wrap` we have a `Func::wrapN_async` family of methods, and we've also got a whole bunch of `Func::getN_async` methods now too. It may be worth rethinking the API of `Func` to try to make the documentation page actually grok'able. This isn't super heavily tested but the various test should suffice for engaging hopefully nearly all the infrastructure in one form or another. This is just the start though! [RFC 2]: https://github.com/bytecodealliance/rfcs/pull/2 * Add wasmtime-fiber to publish script * Save vector/float registers on ARM too. * Fix a typo * Update lock file * Implement periodically yielding with fuel consumption This commit implements APIs on `Store` to periodically yield execution of futures through the consumption of fuel. When fuel runs out a future's execution is yielded back to the caller, and then upon resumption fuel is re-injected. The goal of this is to allow cooperative multi-tasking with futures. * Fix compile without async * Save/restore the frame pointer in fiber switching Turns out this is another caller-saved register! * Simplify x86_64 fiber asm Take a leaf out of aarch64's playbook and don't have extra memory to load/store these arguments, instead leverage how `wasmtime_fiber_switch` already loads a bunch of data into registers which we can then immediately start using on a fiber's start without any extra memory accesses. * Add x86 support to wasmtime-fiber * Add ARM32 support to fiber crate * Make fiber build file probing more flexible * Use CreateFiberEx on Windows * Remove a stray no-longer-used trait declaration * Don't reach into `Caller` internals * Tweak async fuel to eventually run out. With fuel it's probably best to not provide any way to inject infinite fuel. * Fix some typos * Cleanup asm a bit * Use a shared header file to deduplicate some directives * Guarantee hidden visibility for functions * Enable gc-sections on macOS x86_64 * Add `.type` annotations for ARM * Update lock file * Fix compile error * Review comments
4 years ago
[[package]]
name = "wasmtime-fiber"
version = "9.0.0"
Implement support for `async` functions in Wasmtime (#2434) * Implement support for `async` functions in Wasmtime This is an implementation of [RFC 2] in Wasmtime which is to support `async`-defined host functions. At a high level support is added by executing WebAssembly code that might invoke an asynchronous host function on a separate native stack. When the host function's future is not ready we switch back to the main native stack to continue execution. There's a whole bunch of details in this commit, and it's a bit much to go over them all here in this commit message. The most important changes here are: * A new `wasmtime-fiber` crate has been written to manage the low-level details of stack-switching. Unixes use `mmap` to allocate a stack and Windows uses the native fibers implementation. We'll surely want to refactor this to move stack allocation elsewhere in the future. Fibers are intended to be relatively general with a lot of type paremters to fling values back and forth across suspension points. The whole crate is a giant wad of `unsafe` unfortunately and involves handwritten assembly with custom dwarf CFI directives to boot. Definitely deserves a close eye in review! * The `Store` type has two new methods -- `block_on` and `on_fiber` which bridge between the async and non-async worlds. Lots of unsafe fiddly bits here as we're trying to communicate context pointers between disparate portions of the code. Extra eyes and care in review is greatly appreciated. * The APIs for binding `async` functions are unfortunately pretty ugly in `Func`. This is mostly due to language limitations and compiler bugs (I believe) in Rust. Instead of `Func::wrap` we have a `Func::wrapN_async` family of methods, and we've also got a whole bunch of `Func::getN_async` methods now too. It may be worth rethinking the API of `Func` to try to make the documentation page actually grok'able. This isn't super heavily tested but the various test should suffice for engaging hopefully nearly all the infrastructure in one form or another. This is just the start though! [RFC 2]: https://github.com/bytecodealliance/rfcs/pull/2 * Add wasmtime-fiber to publish script * Save vector/float registers on ARM too. * Fix a typo * Update lock file * Implement periodically yielding with fuel consumption This commit implements APIs on `Store` to periodically yield execution of futures through the consumption of fuel. When fuel runs out a future's execution is yielded back to the caller, and then upon resumption fuel is re-injected. The goal of this is to allow cooperative multi-tasking with futures. * Fix compile without async * Save/restore the frame pointer in fiber switching Turns out this is another caller-saved register! * Simplify x86_64 fiber asm Take a leaf out of aarch64's playbook and don't have extra memory to load/store these arguments, instead leverage how `wasmtime_fiber_switch` already loads a bunch of data into registers which we can then immediately start using on a fiber's start without any extra memory accesses. * Add x86 support to wasmtime-fiber * Add ARM32 support to fiber crate * Make fiber build file probing more flexible * Use CreateFiberEx on Windows * Remove a stray no-longer-used trait declaration * Don't reach into `Caller` internals * Tweak async fuel to eventually run out. With fuel it's probably best to not provide any way to inject infinite fuel. * Fix some typos * Cleanup asm a bit * Use a shared header file to deduplicate some directives * Guarantee hidden visibility for functions * Enable gc-sections on macOS x86_64 * Add `.type` annotations for ARM * Update lock file * Fix compile error * Review comments
4 years ago
dependencies = [
"backtrace",
"cc",
"cfg-if",
"rustix",
"wasmtime-asm-macros",
"windows-sys",
Implement support for `async` functions in Wasmtime (#2434) * Implement support for `async` functions in Wasmtime This is an implementation of [RFC 2] in Wasmtime which is to support `async`-defined host functions. At a high level support is added by executing WebAssembly code that might invoke an asynchronous host function on a separate native stack. When the host function's future is not ready we switch back to the main native stack to continue execution. There's a whole bunch of details in this commit, and it's a bit much to go over them all here in this commit message. The most important changes here are: * A new `wasmtime-fiber` crate has been written to manage the low-level details of stack-switching. Unixes use `mmap` to allocate a stack and Windows uses the native fibers implementation. We'll surely want to refactor this to move stack allocation elsewhere in the future. Fibers are intended to be relatively general with a lot of type paremters to fling values back and forth across suspension points. The whole crate is a giant wad of `unsafe` unfortunately and involves handwritten assembly with custom dwarf CFI directives to boot. Definitely deserves a close eye in review! * The `Store` type has two new methods -- `block_on` and `on_fiber` which bridge between the async and non-async worlds. Lots of unsafe fiddly bits here as we're trying to communicate context pointers between disparate portions of the code. Extra eyes and care in review is greatly appreciated. * The APIs for binding `async` functions are unfortunately pretty ugly in `Func`. This is mostly due to language limitations and compiler bugs (I believe) in Rust. Instead of `Func::wrap` we have a `Func::wrapN_async` family of methods, and we've also got a whole bunch of `Func::getN_async` methods now too. It may be worth rethinking the API of `Func` to try to make the documentation page actually grok'able. This isn't super heavily tested but the various test should suffice for engaging hopefully nearly all the infrastructure in one form or another. This is just the start though! [RFC 2]: https://github.com/bytecodealliance/rfcs/pull/2 * Add wasmtime-fiber to publish script * Save vector/float registers on ARM too. * Fix a typo * Update lock file * Implement periodically yielding with fuel consumption This commit implements APIs on `Store` to periodically yield execution of futures through the consumption of fuel. When fuel runs out a future's execution is yielded back to the caller, and then upon resumption fuel is re-injected. The goal of this is to allow cooperative multi-tasking with futures. * Fix compile without async * Save/restore the frame pointer in fiber switching Turns out this is another caller-saved register! * Simplify x86_64 fiber asm Take a leaf out of aarch64's playbook and don't have extra memory to load/store these arguments, instead leverage how `wasmtime_fiber_switch` already loads a bunch of data into registers which we can then immediately start using on a fiber's start without any extra memory accesses. * Add x86 support to wasmtime-fiber * Add ARM32 support to fiber crate * Make fiber build file probing more flexible * Use CreateFiberEx on Windows * Remove a stray no-longer-used trait declaration * Don't reach into `Caller` internals * Tweak async fuel to eventually run out. With fuel it's probably best to not provide any way to inject infinite fuel. * Fix some typos * Cleanup asm a bit * Use a shared header file to deduplicate some directives * Guarantee hidden visibility for functions * Enable gc-sections on macOS x86_64 * Add `.type` annotations for ARM * Update lock file * Fix compile error * Review comments
4 years ago
]
[[package]]
name = "wasmtime-fuzz"
version = "0.0.0"
dependencies = [
"anyhow",
"arbitrary",
"component-fuzz-util",
"component-test-util",
"cranelift-codegen",
Chaos mode MVP: Skip branch optimization in MachBuffer (#6039) * fuzz: Add chaos mode control plane Co-authored-by: Falk Zwimpfer <24669719+FalkZ@users.noreply.github.com> Co-authored-by: Moritz Waser <mzrw.dev@pm.me> * fuzz: Skip branch optimization with chaos mode Co-authored-by: Falk Zwimpfer <24669719+FalkZ@users.noreply.github.com> Co-authored-by: Moritz Waser <mzrw.dev@pm.me> * fuzz: Rename chaos engine -> control plane Co-authored-by: Falk Zwimpfer <24669719+FalkZ@users.noreply.github.com> Co-authored-by: Moritz Waser <mzrw.dev@pm.me> * chaos mode: refactoring ControlPlane to be passed through the call stack by reference Co-authored-by: Falk Zwimpfer <24669719+FalkZ@users.noreply.github.com> Co-authored-by: Remo Senekowitsch <contact@remsle.dev> * fuzz: annotate chaos todos Co-authored-by: Falk Zwimpfer <24669719+FalkZ@users.noreply.github.com> Co-authored-by: Moritz Waser <mzrw.dev@pm.me> * fuzz: cleanup control plane Co-authored-by: Falk Zwimpfer <24669719+FalkZ@users.noreply.github.com> Co-authored-by: Moritz Waser <mzrw.dev@pm.me> * fuzz: remove control plane from compiler context Co-authored-by: Falk Zwimpfer <24669719+FalkZ@users.noreply.github.com> Co-authored-by: Moritz Waser <mzrw.dev@pm.me> * fuzz: move control plane into emit state Co-authored-by: Falk Zwimpfer <24669719+FalkZ@users.noreply.github.com> Co-authored-by: Moritz Waser <mzrw.dev@pm.me> * fuzz: fix remaining compiler errors Co-authored-by: Falk Zwimpfer <24669719+FalkZ@users.noreply.github.com> Co-authored-by: Moritz Waser <mzrw.dev@pm.me> * fix tests * refactor emission state ctrl plane accessors Co-authored-by: Falk Zwimpfer <24669719+FalkZ@users.noreply.github.com> Co-authored-by: Moritz Waser <mzrw.dev@pm.me> * centralize conditional compilation of chaos mode Also cleanup a few straggling dependencies on cranelift-control that aren't needed anymore. Co-authored-by: Falk Zwimpfer <24669719+FalkZ@users.noreply.github.com> Co-authored-by: Moritz Waser <mzrw.dev@pm.me> * add cranelift-control to published crates prtest:full Co-authored-by: Falk Zwimpfer <24669719+FalkZ@users.noreply.github.com> Co-authored-by: Moritz Waser <mzrw.dev@pm.me> * add cranelift-control to public crates Co-authored-by: Falk Zwimpfer <24669719+FalkZ@users.noreply.github.com> Co-authored-by: Moritz Waser <mzrw.dev@pm.me> --------- Co-authored-by: Falk Zwimpfer <24669719+FalkZ@users.noreply.github.com> Co-authored-by: Moritz Waser <mzrw.dev@pm.me> Co-authored-by: Remo Senekowitsch <contact@remsle.dev>
2 years ago
"cranelift-control",
"cranelift-filetests",
"cranelift-fuzzgen",
"cranelift-interpreter",
"cranelift-native",
"cranelift-reader",
"cranelift-wasm",
"libfuzzer-sys",
"once_cell",
"proc-macro2",
"quote",
"rand 0.8.5",
"smallvec",
"target-lexicon",
"wasmtime",
"wasmtime-fuzzing",
]
[[package]]
name = "wasmtime-fuzzing"
version = "0.0.0"
dependencies = [
"anyhow",
"arbitrary",
"component-fuzz-util",
"component-test-util",
"env_logger 0.9.0",
"log",
"rand 0.8.5",
"rayon",
"target-lexicon",
"tempfile",
"v8",
"wasm-encoder 0.25.0",
"wasm-mutate",
Replace binaryen -ttf based fuzzing with wasm-smith (#2336) This commit removes the binaryen support for fuzzing from wasmtime, instead switching over to `wasm-smith`. In general it's great to have what fuzzing we can, but our binaryen support suffers from a few issues: * The Rust crate, binaryen-sys, seems largely unmaintained at this point. While we could likely take ownership and/or send PRs to update the crate it seems like the maintenance is largely on us at this point. * Currently the binaryen-sys crate doesn't support fuzzing anything beyond MVP wasm, but we're interested at least in features like bulk memory and reference types. Additionally we'll also be interested in features like module-linking. New features would require either implementation work in binaryen or the binaryen-sys crate to support. * We have 4-5 fuzz-bugs right now related to timeouts simply in generating a module for wasmtime to fuzz. One investigation along these lines in the past revealed a bug in binaryen itself, and in any case these bugs would otherwise need to get investigated, reported, and possibly fixed ourselves in upstream binaryen. Overall I'm not sure at this point if maintaining binaryen fuzzing is worth it with the advent of `wasm-smith` which has similar goals for wasm module generation, but is much more readily maintainable on our end. Additonally in this commit I've added a fuzzer for wasm-smith's `SwarmConfig`-based fuzzer which should expand the coverage of tested modules. Closes #2163
4 years ago
"wasm-smith",
"wasm-spec-interpreter",
"wasmi",
"wasmparser",
"wasmprinter",
"wasmtime",
"wasmtime-wast",
"wat",
]
[[package]]
name = "wasmtime-jit"
version = "9.0.0"
dependencies = [
"addr2line",
"anyhow",
"bincode",
Remove support for userfaultfd (#4040) This commit removes support for the `userfaultfd` or "uffd" syscall on Linux. This support was originally added for users migrating from Lucet to Wasmtime, but the recent developments of kernel-supported copy-on-write support for memory initialization wound up being more appropriate for these use cases than usefaultfd. The main reason for moving to copy-on-write initialization are: * The `userfaultfd` feature was never necessarily intended for this style of use case with wasm and was susceptible to subtle and rare bugs that were extremely difficult to track down. We were never 100% certain that there were kernel bugs related to userfaultfd but the suspicion never went away. * Handling faults with userfaultfd was always slow and single-threaded. Only one thread could handle faults and traveling to user-space to handle faults is inherently slower than handling them all in the kernel. The single-threaded aspect in particular presented a significant scaling bottleneck for embeddings that want to run many wasm instances in parallel. * One of the major benefits of userfaultfd was lazy initialization of wasm linear memory which is also achieved with the copy-on-write initialization support we have right now. * One of the suspected benefits of userfaultfd was less frobbing of the kernel vma structures when wasm modules are instantiated. Currently the copy-on-write support has a mitigation where we attempt to reuse the memory images where possible to avoid changing vma structures. When comparing this to userfaultfd's performance it was found that kernel modifications of vmas aren't a worrisome bottleneck so copy-on-write is suitable for this as well. Overall there are no remaining benefits that userfaultfd gives that copy-on-write doesn't, and copy-on-write solves a major downsides of userfaultfd, the scaling issue with a single faulting thread. Additionally copy-on-write support seems much more robust in terms of kernel implementation since it's only using standard memory-management syscalls which are heavily exercised. Finally copy-on-write support provides a new bonus where read-only memory in WebAssembly can be mapped directly to the same kernel cache page, even amongst many wasm instances of the same module, which was never possible with userfaultfd. In light of all this it's expected that all users of userfaultfd should migrate to the copy-on-write initialization of Wasmtime (which is enabled by default).
3 years ago
"cfg-if",
"cpp_demangle",
"gimli",
"ittapi",
"log",
"object",
"rustc-demangle",
"serde",
"target-lexicon",
"wasmtime-environ",
"wasmtime-jit-debug",
Flush Icache on AArch64 Windows (#4997) * cranelift: Add FlushInstructionCache for AArch64 on Windows This was previously done on #3426 for linux. * wasmtime: Add FlushInstructionCache for AArch64 on Windows This was previously done on #3426 for linux. * cranelift: Add MemoryUse flag to JIT Memory Manager This allows us to keep the icache flushing code self-contained and not leak implementation details. This also changes the windows icache flushing code to only flush pages that were previously unflushed. * Add jit-icache-coherence crate * cranelift: Use `jit-icache-coherence` * wasmtime: Use `jit-icache-coherence` * jit-icache-coherence: Make rustix feature additive Mutually exclusive features cause issues. * wasmtime: Remove rustix from wasmtime-jit We now use it via jit-icache-coherence * Rename wasmtime-jit-icache-coherency crate * Use cfg-if in wasmtime-jit-icache-coherency crate * Use inline instead of inline(always) * Add unsafe marker to clear_cache * Conditionally compile all rustix operations membarrier does not exist on MacOS * Publish `wasmtime-jit-icache-coherence` * Remove explicit windows check This is implied by the target_os = "windows" above * cranelift: Remove len != 0 check This is redundant as it is done in non_protected_allocations_iter * Comment cleanups Thanks @akirilov-arm! * Make clear_cache safe * Rename pipeline_flush to pipeline_flush_mt * Revert "Make clear_cache safe" This reverts commit 21165d81c9030ed9b291a1021a367214d2942c90. * More docs! * Fix pipeline_flush reference on clear_cache * Update more docs! * Move pipeline flush after `mprotect` calls Technically the `clear_cache` operation is a lie in AArch64, so move the pipeline flush after the `mprotect` calls so that it benefits from the implicit cache cleaning done by it. * wasmtime: Remove rustix backend from icache crate * wasmtime: Use libc for macos * wasmtime: Flush icache on all arch's for windows * wasmtime: Add flags to membarrier call
2 years ago
"wasmtime-jit-icache-coherence",
"wasmtime-runtime",
"windows-sys",
]
[[package]]
name = "wasmtime-jit-debug"
version = "9.0.0"
dependencies = [
"object",
"once_cell",
"rustix",
]
Flush Icache on AArch64 Windows (#4997) * cranelift: Add FlushInstructionCache for AArch64 on Windows This was previously done on #3426 for linux. * wasmtime: Add FlushInstructionCache for AArch64 on Windows This was previously done on #3426 for linux. * cranelift: Add MemoryUse flag to JIT Memory Manager This allows us to keep the icache flushing code self-contained and not leak implementation details. This also changes the windows icache flushing code to only flush pages that were previously unflushed. * Add jit-icache-coherence crate * cranelift: Use `jit-icache-coherence` * wasmtime: Use `jit-icache-coherence` * jit-icache-coherence: Make rustix feature additive Mutually exclusive features cause issues. * wasmtime: Remove rustix from wasmtime-jit We now use it via jit-icache-coherence * Rename wasmtime-jit-icache-coherency crate * Use cfg-if in wasmtime-jit-icache-coherency crate * Use inline instead of inline(always) * Add unsafe marker to clear_cache * Conditionally compile all rustix operations membarrier does not exist on MacOS * Publish `wasmtime-jit-icache-coherence` * Remove explicit windows check This is implied by the target_os = "windows" above * cranelift: Remove len != 0 check This is redundant as it is done in non_protected_allocations_iter * Comment cleanups Thanks @akirilov-arm! * Make clear_cache safe * Rename pipeline_flush to pipeline_flush_mt * Revert "Make clear_cache safe" This reverts commit 21165d81c9030ed9b291a1021a367214d2942c90. * More docs! * Fix pipeline_flush reference on clear_cache * Update more docs! * Move pipeline flush after `mprotect` calls Technically the `clear_cache` operation is a lie in AArch64, so move the pipeline flush after the `mprotect` calls so that it benefits from the implicit cache cleaning done by it. * wasmtime: Remove rustix backend from icache crate * wasmtime: Use libc for macos * wasmtime: Flush icache on all arch's for windows * wasmtime: Add flags to membarrier call
2 years ago
[[package]]
name = "wasmtime-jit-icache-coherence"
version = "9.0.0"
Flush Icache on AArch64 Windows (#4997) * cranelift: Add FlushInstructionCache for AArch64 on Windows This was previously done on #3426 for linux. * wasmtime: Add FlushInstructionCache for AArch64 on Windows This was previously done on #3426 for linux. * cranelift: Add MemoryUse flag to JIT Memory Manager This allows us to keep the icache flushing code self-contained and not leak implementation details. This also changes the windows icache flushing code to only flush pages that were previously unflushed. * Add jit-icache-coherence crate * cranelift: Use `jit-icache-coherence` * wasmtime: Use `jit-icache-coherence` * jit-icache-coherence: Make rustix feature additive Mutually exclusive features cause issues. * wasmtime: Remove rustix from wasmtime-jit We now use it via jit-icache-coherence * Rename wasmtime-jit-icache-coherency crate * Use cfg-if in wasmtime-jit-icache-coherency crate * Use inline instead of inline(always) * Add unsafe marker to clear_cache * Conditionally compile all rustix operations membarrier does not exist on MacOS * Publish `wasmtime-jit-icache-coherence` * Remove explicit windows check This is implied by the target_os = "windows" above * cranelift: Remove len != 0 check This is redundant as it is done in non_protected_allocations_iter * Comment cleanups Thanks @akirilov-arm! * Make clear_cache safe * Rename pipeline_flush to pipeline_flush_mt * Revert "Make clear_cache safe" This reverts commit 21165d81c9030ed9b291a1021a367214d2942c90. * More docs! * Fix pipeline_flush reference on clear_cache * Update more docs! * Move pipeline flush after `mprotect` calls Technically the `clear_cache` operation is a lie in AArch64, so move the pipeline flush after the `mprotect` calls so that it benefits from the implicit cache cleaning done by it. * wasmtime: Remove rustix backend from icache crate * wasmtime: Use libc for macos * wasmtime: Flush icache on all arch's for windows * wasmtime: Add flags to membarrier call
2 years ago
dependencies = [
"cfg-if",
"libc",
"windows-sys",
]
[[package]]
name = "wasmtime-runtime"
version = "9.0.0"
dependencies = [
"anyhow",
"cc",
Remove support for userfaultfd (#4040) This commit removes support for the `userfaultfd` or "uffd" syscall on Linux. This support was originally added for users migrating from Lucet to Wasmtime, but the recent developments of kernel-supported copy-on-write support for memory initialization wound up being more appropriate for these use cases than usefaultfd. The main reason for moving to copy-on-write initialization are: * The `userfaultfd` feature was never necessarily intended for this style of use case with wasm and was susceptible to subtle and rare bugs that were extremely difficult to track down. We were never 100% certain that there were kernel bugs related to userfaultfd but the suspicion never went away. * Handling faults with userfaultfd was always slow and single-threaded. Only one thread could handle faults and traveling to user-space to handle faults is inherently slower than handling them all in the kernel. The single-threaded aspect in particular presented a significant scaling bottleneck for embeddings that want to run many wasm instances in parallel. * One of the major benefits of userfaultfd was lazy initialization of wasm linear memory which is also achieved with the copy-on-write initialization support we have right now. * One of the suspected benefits of userfaultfd was less frobbing of the kernel vma structures when wasm modules are instantiated. Currently the copy-on-write support has a mitigation where we attempt to reuse the memory images where possible to avoid changing vma structures. When comparing this to userfaultfd's performance it was found that kernel modifications of vmas aren't a worrisome bottleneck so copy-on-write is suitable for this as well. Overall there are no remaining benefits that userfaultfd gives that copy-on-write doesn't, and copy-on-write solves a major downsides of userfaultfd, the scaling issue with a single faulting thread. Additionally copy-on-write support seems much more robust in terms of kernel implementation since it's only using standard memory-management syscalls which are heavily exercised. Finally copy-on-write support provides a new bonus where read-only memory in WebAssembly can be mapped directly to the same kernel cache page, even amongst many wasm instances of the same module, which was never possible with userfaultfd. In light of all this it's expected that all users of userfaultfd should migrate to the copy-on-write initialization of Wasmtime (which is enabled by default).
3 years ago
"cfg-if",
Implement strings in adapter modules (#4623) * Implement strings in adapter modules This commit is a hefty addition to Wasmtime's support for the component model. This implements the final remaining type (in the current type hierarchy) unimplemented in adapter module trampolines: strings. Strings are the most complicated type to implement in adapter trampolines because they are highly structured chunks of data in memory (according to specific encodings). Additionally each lift/lower operation can choose its own encoding for strings meaning that Wasmtime, the host, may have to convert between any pairwise ordering of string encodings. The `CanonicalABI.md` in the component-model repo in general specifies all the fiddly bits of string encoding so there's not a ton of wiggle room for Wasmtime to get creative. This PR largely "just" implements that. The high-level architecture of this implementation is: * Fused adapters are first identified to determine src/dst string encodings. This statically fixes what transcoding operation is being performed. * The generated adapter will be responsible for managing calls to `realloc` and performing bounds checks. The adapter itself does not perform memory copies or validation of string contents, however. Instead each transcoding operation is modeled as an imported function into the adapter module. This means that the adapter module dynamically, during compile time, determines what string transcoders are needed. Note that an imported transcoder is not only parameterized over the transcoding operation but additionally which memory is the source and which is the destination. * The imported core wasm functions are modeled as a new `CoreDef::Transcoder` structure. These transcoders end up being small Cranelift-compiled trampolines. The Cranelift-compiled trampoline will load the actual base pointer of memory and add it to the relative pointers passed as function arguments. This trampoline then calls a transcoder "libcall" which enters Rust-defined functions for actual transcoding operations. * Each possible transcoding operation is implemented in Rust with a unique name and a unique signature depending on the needs of the transcoder. I've tried to document inline what each transcoder does. This means that the `Module::translate_string` in adapter modules is by far the largest translation method. The main reason for this is due to the management around calling the imported transcoder functions in the face of validating string pointer/lengths and performing the dance of `realloc`-vs-transcode at the right time. I've tried to ensure that each individual case in transcoding is documented well enough to understand what's going on as well. Additionally in this PR is a full implementation in the host for the `latin1+utf16` encoding which means that both lifting and lowering host strings now works with this encoding. Currently the implementation of each transcoder function is likely far from optimal. Where possible I've leaned on the standard library itself and for latin1-related things I'm leaning on the `encoding_rs` crate. I initially tried to implement everything with `encoding_rs` but was unable to uniformly do so easily. For now I settled on trying to get a known-correct (even in the face of endianness) implementation for all of these transcoders. If an when performance becomes an issue it should be possible to implement more optimized versions of each of these transcoding operations. Testing this commit has been somewhat difficult and my general plan, like with the `(list T)` type, is to rely heavily on fuzzing to cover the various cases here. In this PR though I've added a simple test that pushes some statically known strings through all the pairs of encodings between source and destination. I've attempted to pick "interesting" strings that one way or another stress the various paths in each transcoding operation to ideally get full branch coverage there. Additionally a suite of "negative" tests have also been added to ensure that validity of encoding is actually checked. * Fix a temporarily commented out case * Fix wasmtime-runtime tests * Update deny.toml configuration * Add `BSD-3-Clause` for the `encoding_rs` crate * Remove some unused licenses * Add an exemption for `encoding_rs` for now * Split up the `translate_string` method Move out all the closures and package up captured state into smaller lists of arguments. * Test out-of-bounds for zero-length strings
2 years ago
"encoding_rs",
"indexmap",
"libc",
externref: implement stack map-based garbage collection For host VM code, we use plain reference counting, where cloning increments the reference count, and dropping decrements it. We can avoid many of the on-stack increment/decrement operations that typically plague the performance of reference counting via Rust's ownership and borrowing system. Moving a `VMExternRef` avoids mutating its reference count, and borrowing it either avoids the reference count increment or delays it until if/when the `VMExternRef` is cloned. When passing a `VMExternRef` into compiled Wasm code, we don't want to do reference count mutations for every compiled `local.{get,set}`, nor for every function call. Therefore, we use a variation of **deferred reference counting**, where we only mutate reference counts when storing `VMExternRef`s somewhere that outlives the activation: into a global or table. Simultaneously, we over-approximate the set of `VMExternRef`s that are inside Wasm function activations. Periodically, we walk the stack at GC safe points, and use stack map information to precisely identify the set of `VMExternRef`s inside Wasm activations. Then we take the difference between this precise set and our over-approximation, and decrement the reference count for each of the `VMExternRef`s that are in our over-approximation but not in the precise set. Finally, the over-approximation is replaced with the precise set. The `VMExternRefActivationsTable` implements the over-approximized set of `VMExternRef`s referenced by Wasm activations. Calling a Wasm function and passing it a `VMExternRef` moves the `VMExternRef` into the table, and the compiled Wasm function logically "borrows" the `VMExternRef` from the table. Similarly, `global.get` and `table.get` operations clone the gotten `VMExternRef` into the `VMExternRefActivationsTable` and then "borrow" the reference out of the table. When a `VMExternRef` is returned to host code from a Wasm function, the host increments the reference count (because the reference is logically "borrowed" from the `VMExternRefActivationsTable` and the reference count from the table will be dropped at the next GC). For more general information on deferred reference counting, see *An Examination of Deferred Reference Counting and Cycle Detection* by Quinane: https://openresearch-repository.anu.edu.au/bitstream/1885/42030/2/hon-thesis.pdf cc #929 Fixes #1804
4 years ago
"log",
Mach ports continued + support aarch64-apple unwinding (#2723) * Switch macOS to using mach ports for trap handling This commit moves macOS to using mach ports instead of signals for handling traps. The motivation for this is listed in #2456, namely that once mach ports are used in a process that means traditional UNIX signal handlers won't get used. This means that if Wasmtime is integrated with Breakpad, for example, then Wasmtime's trap handler never fires and traps don't work. The `traphandlers` module is refactored as part of this commit to split the platform-specific bits into their own files (it was growing quite a lot for one inline `cfg_if!`). The `unix.rs` and `windows.rs` files remain the same as they were before with a few minor tweaks for some refactored interfaces. The `macos.rs` file is brand new and lifts almost its entire implementation from SpiderMonkey, adapted for Wasmtime though. The main gotcha with mach ports is that a separate thread is what services the exception. Some unsafe magic allows this separate thread to read non-`Send` and temporary state from other threads, but is hoped to be safe in this context. The unfortunate downside is that calling wasm on macOS now involves taking a global lock and modifying a global hash map twice-per-call. I'm not entirely sure how to get out of this cost for now, but hopefully for any embeddings on macOS it's not the end of the world. Closes #2456 * Add a sketch of arm64 apple support * store: maintain CallThreadState mapping when switching fibers * cranelift/aarch64: generate unwind directives to disable pointer auth Aarch64 post ARMv8.3 has a feature called pointer authentication, designed to fight ROP/JOP attacks: some pointers may be signed using new instructions, adding payloads to the high (previously unused) bits of the pointers. More on this here: https://lwn.net/Articles/718888/ Unwinders on aarch64 need to know if some pointers contained on the call frame contain an authentication code or not, to be able to properly authenticate them or use them directly. Since native code may have enabled it by default (as is the case on the Mac M1), and the default is that this configuration value is inherited, we need to explicitly disable it, for the only kind of supported pointers (return addresses). To do so, we set the value of a non-existing dwarf pseudo register (34) to 0, as documented in https://github.com/ARM-software/abi-aa/blob/master/aadwarf64/aadwarf64.rst#note-8. This is done at the function granularity, in the spirit of Cranelift compilation model. Alternatively, a single directive could be generated in the CIE, generating less information per module. * Make exception handling work on Mac aarch64 too * fibers: use a breakpoint instruction after the final call in wasmtime_fiber_start Co-authored-by: Alex Crichton <alex@alexcrichton.com>
4 years ago
"mach",
Add a pooling allocator mode based on copy-on-write mappings of memfds. As first suggested by Jan on the Zulip here [1], a cheap and effective way to obtain copy-on-write semantics of a "backing image" for a Wasm memory is to mmap a file with `MAP_PRIVATE`. The `memfd` mechanism provided by the Linux kernel allows us to create anonymous, in-memory-only files that we can use for this mapping, so we can construct the image contents on-the-fly then effectively create a CoW overlay. Furthermore, and importantly, `madvise(MADV_DONTNEED, ...)` will discard the CoW overlay, returning the mapping to its original state. By itself this is almost enough for a very fast instantiation-termination loop of the same image over and over, without changing the address space mapping at all (which is expensive). The only missing bit is how to implement heap *growth*. But here memfds can help us again: if we create another anonymous file and map it where the extended parts of the heap would go, we can take advantage of the fact that a `mmap()` mapping can be *larger than the file itself*, with accesses beyond the end generating a `SIGBUS`, and the fact that we can cheaply resize the file with `ftruncate`, even after a mapping exists. So we can map the "heap extension" file once with the maximum memory-slot size and grow the memfd itself as `memory.grow` operations occur. The above CoW technique and heap-growth technique together allow us a fastpath of `madvise()` and `ftruncate()` only when we re-instantiate the same module over and over, as long as we can reuse the same slot. This fastpath avoids all whole-process address-space locks in the Linux kernel, which should mean it is highly scalable. It also avoids the cost of copying data on read, as the `uffd` heap backend does when servicing pagefaults; the kernel's own optimized CoW logic (same as used by all file mmaps) is used instead. [1] https://bytecodealliance.zulipchat.com/#narrow/stream/206238-general/topic/Copy.20on.20write.20based.20instance.20reuse/near/266657772
3 years ago
"memfd",
"memoffset 0.8.0",
"once_cell",
`wasmtime`: Implement fast Wasm stack walking (#4431) * Always preserve frame pointers in Wasmtime This allows us to efficiently and simply capture Wasm stacks without maintaining and synchronizing any safety-critical side tables between the compiler and the runtime. * wasmtime: Implement fast Wasm stack walking Why do we want Wasm stack walking to be fast? Because we capture stacks whenever there is a trap and traps actually happen fairly frequently with short-lived programs and WASI's `exit`. Previously, we would rely on generating the system unwind info (e.g. `.eh_frame`) and using the system unwinder (via the `backtrace`crate) to walk the full stack and filter out any non-Wasm stack frames. This can, unfortunately, be slow for two primary reasons: 1. The system unwinder is doing `O(all-kinds-of-frames)` work rather than `O(wasm-frames)` work. 2. System unwind info and the system unwinder need to be much more general than a purpose-built stack walker for Wasm needs to be. It has to handle any kind of stack frame that any compiler might emit where as our Wasm frames are emitted by Cranelift and always have frame pointers. This translates into implementation complexity and general overhead. There can also be unnecessary-for-our-use-cases global synchronization and locks involved, further slowing down stack walking in the presence of multiple threads trying to capture stacks in parallel. This commit introduces a purpose-built stack walker for traversing just our Wasm frames. To find all the sequences of Wasm-to-Wasm stack frames, and ignore non-Wasm stack frames, we keep a linked list of `(entry stack pointer, exit frame pointer)` pairs. This linked list is maintained via Wasm-to-host and host-to-Wasm trampolines. Within a sequence of Wasm-to-Wasm calls, we can use frame pointers (which Cranelift preserves) to find the next older Wasm frame on the stack, and we keep doing this until we reach the entry stack pointer, meaning that the next older frame will be a host frame. The trampolines need to avoid a couple stumbling blocks. First, they need to be compiled ahead of time, since we may not have access to a compiler at runtime (e.g. if the `cranelift` feature is disabled) but still want to be able to call functions that have already been compiled and get stack traces for those functions. Usually this means we would compile the appropriate trampolines inside `Module::new` and the compiled module object would hold the trampolines. However, we *also* need to support calling host functions that are wrapped into `wasmtime::Func`s and there doesn't exist *any* ahead-of-time compiled module object to hold the appropriate trampolines: ```rust // Define a host function. let func_type = wasmtime::FuncType::new( vec![wasmtime::ValType::I32], vec![wasmtime::ValType::I32], ); let func = Func::new(&mut store, func_type, |_, params, results| { // ... Ok(()) }); // Call that host function. let mut results = vec![wasmtime::Val::I32(0)]; func.call(&[wasmtime::Val::I32(0)], &mut results)?; ``` Therefore, we define one host-to-Wasm trampoline and one Wasm-to-host trampoline in assembly that work for all Wasm and host function signatures. These trampolines are careful to only use volatile registers, avoid touching any register that is an argument in the calling convention ABI, and tail call to the target callee function. This allows forwarding any set of arguments and any returns to and from the callee, while also allowing us to maintain our linked list of Wasm stack and frame pointers before transferring control to the callee. These trampolines are not used in Wasm-to-Wasm calls, only when crossing the host-Wasm boundary, so they do not impose overhead on regular calls. (And if using one trampoline for all host-Wasm boundary crossing ever breaks branch prediction enough in the CPU to become any kind of bottleneck, we can do fun things like have multiple copies of the same trampoline and choose a random copy for each function, sharding the functions across branch predictor entries.) Finally, this commit also ends the use of a synthetic `Module` and allocating a stubbed out `VMContext` for host functions. Instead, we define a `VMHostFuncContext` with its own magic value, similar to `VMComponentContext`, specifically for host functions. <h2>Benchmarks</h2> <h3>Traps and Stack Traces</h3> Large improvements to taking stack traces on traps, ranging from shaving off 64% to 99.95% of the time it used to take. <details> ``` multi-threaded-traps/0 time: [2.5686 us 2.5808 us 2.5934 us] thrpt: [0.0000 elem/s 0.0000 elem/s 0.0000 elem/s] change: time: [-85.419% -85.153% -84.869%] (p = 0.00 < 0.05) thrpt: [+560.90% +573.56% +585.84%] Performance has improved. Found 8 outliers among 100 measurements (8.00%) 4 (4.00%) high mild 4 (4.00%) high severe multi-threaded-traps/1 time: [2.9021 us 2.9167 us 2.9322 us] thrpt: [341.04 Kelem/s 342.86 Kelem/s 344.58 Kelem/s] change: time: [-91.455% -91.294% -91.096%] (p = 0.00 < 0.05) thrpt: [+1023.1% +1048.6% +1070.3%] Performance has improved. Found 6 outliers among 100 measurements (6.00%) 1 (1.00%) high mild 5 (5.00%) high severe multi-threaded-traps/2 time: [2.9996 us 3.0145 us 3.0295 us] thrpt: [660.18 Kelem/s 663.47 Kelem/s 666.76 Kelem/s] change: time: [-94.040% -93.910% -93.762%] (p = 0.00 < 0.05) thrpt: [+1503.1% +1542.0% +1578.0%] Performance has improved. Found 5 outliers among 100 measurements (5.00%) 5 (5.00%) high severe multi-threaded-traps/4 time: [5.5768 us 5.6052 us 5.6364 us] thrpt: [709.68 Kelem/s 713.63 Kelem/s 717.25 Kelem/s] change: time: [-93.193% -93.121% -93.052%] (p = 0.00 < 0.05) thrpt: [+1339.2% +1353.6% +1369.1%] Performance has improved. multi-threaded-traps/8 time: [8.6408 us 9.1212 us 9.5438 us] thrpt: [838.24 Kelem/s 877.08 Kelem/s 925.84 Kelem/s] change: time: [-94.754% -94.473% -94.202%] (p = 0.00 < 0.05) thrpt: [+1624.7% +1709.2% +1806.1%] Performance has improved. multi-threaded-traps/16 time: [10.152 us 10.840 us 11.545 us] thrpt: [1.3858 Melem/s 1.4760 Melem/s 1.5761 Melem/s] change: time: [-97.042% -96.823% -96.577%] (p = 0.00 < 0.05) thrpt: [+2821.5% +3048.1% +3281.1%] Performance has improved. Found 1 outliers among 100 measurements (1.00%) 1 (1.00%) high mild many-modules-registered-traps/1 time: [2.6278 us 2.6361 us 2.6447 us] thrpt: [378.11 Kelem/s 379.35 Kelem/s 380.55 Kelem/s] change: time: [-85.311% -85.108% -84.909%] (p = 0.00 < 0.05) thrpt: [+562.65% +571.51% +580.76%] Performance has improved. Found 9 outliers among 100 measurements (9.00%) 3 (3.00%) high mild 6 (6.00%) high severe many-modules-registered-traps/8 time: [2.6294 us 2.6460 us 2.6623 us] thrpt: [3.0049 Melem/s 3.0235 Melem/s 3.0425 Melem/s] change: time: [-85.895% -85.485% -85.022%] (p = 0.00 < 0.05) thrpt: [+567.63% +588.95% +608.95%] Performance has improved. Found 8 outliers among 100 measurements (8.00%) 3 (3.00%) high mild 5 (5.00%) high severe many-modules-registered-traps/64 time: [2.6218 us 2.6329 us 2.6452 us] thrpt: [24.195 Melem/s 24.308 Melem/s 24.411 Melem/s] change: time: [-93.629% -93.551% -93.470%] (p = 0.00 < 0.05) thrpt: [+1431.4% +1450.6% +1469.5%] Performance has improved. Found 3 outliers among 100 measurements (3.00%) 3 (3.00%) high mild many-modules-registered-traps/512 time: [2.6569 us 2.6737 us 2.6923 us] thrpt: [190.17 Melem/s 191.50 Melem/s 192.71 Melem/s] change: time: [-99.277% -99.268% -99.260%] (p = 0.00 < 0.05) thrpt: [+13417% +13566% +13731%] Performance has improved. Found 4 outliers among 100 measurements (4.00%) 4 (4.00%) high mild many-modules-registered-traps/4096 time: [2.7258 us 2.7390 us 2.7535 us] thrpt: [1.4876 Gelem/s 1.4955 Gelem/s 1.5027 Gelem/s] change: time: [-99.956% -99.955% -99.955%] (p = 0.00 < 0.05) thrpt: [+221417% +223380% +224881%] Performance has improved. Found 2 outliers among 100 measurements (2.00%) 1 (1.00%) high mild 1 (1.00%) high severe many-stack-frames-traps/1 time: [1.4658 us 1.4719 us 1.4784 us] thrpt: [676.39 Kelem/s 679.38 Kelem/s 682.21 Kelem/s] change: time: [-90.368% -89.947% -89.586%] (p = 0.00 < 0.05) thrpt: [+860.23% +894.72% +938.21%] Performance has improved. Found 8 outliers among 100 measurements (8.00%) 5 (5.00%) high mild 3 (3.00%) high severe many-stack-frames-traps/8 time: [2.4772 us 2.4870 us 2.4973 us] thrpt: [3.2034 Melem/s 3.2167 Melem/s 3.2294 Melem/s] change: time: [-85.550% -85.370% -85.199%] (p = 0.00 < 0.05) thrpt: [+575.65% +583.51% +592.03%] Performance has improved. Found 8 outliers among 100 measurements (8.00%) 4 (4.00%) high mild 4 (4.00%) high severe many-stack-frames-traps/64 time: [10.109 us 10.171 us 10.236 us] thrpt: [6.2525 Melem/s 6.2925 Melem/s 6.3309 Melem/s] change: time: [-78.144% -77.797% -77.336%] (p = 0.00 < 0.05) thrpt: [+341.22% +350.38% +357.55%] Performance has improved. Found 7 outliers among 100 measurements (7.00%) 5 (5.00%) high mild 2 (2.00%) high severe many-stack-frames-traps/512 time: [126.16 us 126.54 us 126.96 us] thrpt: [4.0329 Melem/s 4.0461 Melem/s 4.0583 Melem/s] change: time: [-65.364% -64.933% -64.453%] (p = 0.00 < 0.05) thrpt: [+181.32% +185.17% +188.71%] Performance has improved. Found 4 outliers among 100 measurements (4.00%) 4 (4.00%) high severe ``` </details> <h3>Calls</h3> There is, however, a small regression in raw Wasm-to-host and host-to-Wasm call performance due the new trampolines. It seems to be on the order of about 2-10 nanoseconds per call, depending on the benchmark. I believe this regression is ultimately acceptable because 1. this overhead will be vastly dominated by whatever work a non-nop callee actually does, 2. we will need these trampolines, or something like them, when implementing the Wasm exceptions proposal to do things like translate Wasm's exceptions into Rust's `Result`s, 3. and because the performance improvements to trapping and capturing stack traces are of such a larger magnitude than this call regressions. <details> ``` sync/no-hook/host-to-wasm - typed - nop time: [28.683 ns 28.757 ns 28.844 ns] change: [+16.472% +17.183% +17.904%] (p = 0.00 < 0.05) Performance has regressed. Found 10 outliers among 100 measurements (10.00%) 1 (1.00%) low mild 4 (4.00%) high mild 5 (5.00%) high severe sync/no-hook/host-to-wasm - untyped - nop time: [42.515 ns 42.652 ns 42.841 ns] change: [+12.371% +14.614% +17.462%] (p = 0.00 < 0.05) Performance has regressed. Found 11 outliers among 100 measurements (11.00%) 1 (1.00%) high mild 10 (10.00%) high severe sync/no-hook/host-to-wasm - unchecked - nop time: [33.936 ns 34.052 ns 34.179 ns] change: [+25.478% +26.938% +28.369%] (p = 0.00 < 0.05) Performance has regressed. Found 9 outliers among 100 measurements (9.00%) 7 (7.00%) high mild 2 (2.00%) high severe sync/no-hook/host-to-wasm - typed - nop-params-and-results time: [34.290 ns 34.388 ns 34.502 ns] change: [+40.802% +42.706% +44.526%] (p = 0.00 < 0.05) Performance has regressed. Found 13 outliers among 100 measurements (13.00%) 5 (5.00%) high mild 8 (8.00%) high severe sync/no-hook/host-to-wasm - untyped - nop-params-and-results time: [62.546 ns 62.721 ns 62.919 ns] change: [+2.5014% +3.6319% +4.8078%] (p = 0.00 < 0.05) Performance has regressed. Found 12 outliers among 100 measurements (12.00%) 2 (2.00%) high mild 10 (10.00%) high severe sync/no-hook/host-to-wasm - unchecked - nop-params-and-results time: [42.609 ns 42.710 ns 42.831 ns] change: [+20.966% +22.282% +23.475%] (p = 0.00 < 0.05) Performance has regressed. Found 11 outliers among 100 measurements (11.00%) 4 (4.00%) high mild 7 (7.00%) high severe sync/hook-sync/host-to-wasm - typed - nop time: [29.546 ns 29.675 ns 29.818 ns] change: [+20.693% +21.794% +22.836%] (p = 0.00 < 0.05) Performance has regressed. Found 5 outliers among 100 measurements (5.00%) 3 (3.00%) high mild 2 (2.00%) high severe sync/hook-sync/host-to-wasm - untyped - nop time: [45.448 ns 45.699 ns 45.961 ns] change: [+17.204% +18.514% +19.590%] (p = 0.00 < 0.05) Performance has regressed. Found 14 outliers among 100 measurements (14.00%) 4 (4.00%) high mild 10 (10.00%) high severe sync/hook-sync/host-to-wasm - unchecked - nop time: [34.334 ns 34.437 ns 34.558 ns] change: [+23.225% +24.477% +25.886%] (p = 0.00 < 0.05) Performance has regressed. Found 12 outliers among 100 measurements (12.00%) 5 (5.00%) high mild 7 (7.00%) high severe sync/hook-sync/host-to-wasm - typed - nop-params-and-results time: [36.594 ns 36.763 ns 36.974 ns] change: [+41.967% +47.261% +52.086%] (p = 0.00 < 0.05) Performance has regressed. Found 12 outliers among 100 measurements (12.00%) 3 (3.00%) high mild 9 (9.00%) high severe sync/hook-sync/host-to-wasm - untyped - nop-params-and-results time: [63.541 ns 63.831 ns 64.194 ns] change: [-4.4337% -0.6855% +2.7134%] (p = 0.73 > 0.05) No change in performance detected. Found 8 outliers among 100 measurements (8.00%) 6 (6.00%) high mild 2 (2.00%) high severe sync/hook-sync/host-to-wasm - unchecked - nop-params-and-results time: [43.968 ns 44.169 ns 44.437 ns] change: [+18.772% +21.802% +24.623%] (p = 0.00 < 0.05) Performance has regressed. Found 15 outliers among 100 measurements (15.00%) 3 (3.00%) high mild 12 (12.00%) high severe async/no-hook/host-to-wasm - typed - nop time: [4.9612 us 4.9743 us 4.9889 us] change: [+9.9493% +11.911% +13.502%] (p = 0.00 < 0.05) Performance has regressed. Found 10 outliers among 100 measurements (10.00%) 6 (6.00%) high mild 4 (4.00%) high severe async/no-hook/host-to-wasm - untyped - nop time: [5.0030 us 5.0211 us 5.0439 us] change: [+10.841% +11.873% +12.977%] (p = 0.00 < 0.05) Performance has regressed. Found 10 outliers among 100 measurements (10.00%) 3 (3.00%) high mild 7 (7.00%) high severe async/no-hook/host-to-wasm - typed - nop-params-and-results time: [4.9273 us 4.9468 us 4.9700 us] change: [+4.7381% +6.8445% +8.8238%] (p = 0.00 < 0.05) Performance has regressed. Found 14 outliers among 100 measurements (14.00%) 5 (5.00%) high mild 9 (9.00%) high severe async/no-hook/host-to-wasm - untyped - nop-params-and-results time: [5.1151 us 5.1338 us 5.1555 us] change: [+9.5335% +11.290% +13.044%] (p = 0.00 < 0.05) Performance has regressed. Found 16 outliers among 100 measurements (16.00%) 3 (3.00%) high mild 13 (13.00%) high severe async/hook-sync/host-to-wasm - typed - nop time: [4.9330 us 4.9394 us 4.9467 us] change: [+10.046% +11.038% +12.035%] (p = 0.00 < 0.05) Performance has regressed. Found 12 outliers among 100 measurements (12.00%) 5 (5.00%) high mild 7 (7.00%) high severe async/hook-sync/host-to-wasm - untyped - nop time: [5.0073 us 5.0183 us 5.0310 us] change: [+9.3828% +10.565% +11.752%] (p = 0.00 < 0.05) Performance has regressed. Found 8 outliers among 100 measurements (8.00%) 3 (3.00%) high mild 5 (5.00%) high severe async/hook-sync/host-to-wasm - typed - nop-params-and-results time: [4.9610 us 4.9839 us 5.0097 us] change: [+9.0857% +11.513% +14.359%] (p = 0.00 < 0.05) Performance has regressed. Found 13 outliers among 100 measurements (13.00%) 7 (7.00%) high mild 6 (6.00%) high severe async/hook-sync/host-to-wasm - untyped - nop-params-and-results time: [5.0995 us 5.1272 us 5.1617 us] change: [+9.3600% +11.506% +13.809%] (p = 0.00 < 0.05) Performance has regressed. Found 10 outliers among 100 measurements (10.00%) 6 (6.00%) high mild 4 (4.00%) high severe async-pool/no-hook/host-to-wasm - typed - nop time: [2.4242 us 2.4316 us 2.4396 us] change: [+7.8756% +8.8803% +9.8346%] (p = 0.00 < 0.05) Performance has regressed. Found 8 outliers among 100 measurements (8.00%) 5 (5.00%) high mild 3 (3.00%) high severe async-pool/no-hook/host-to-wasm - untyped - nop time: [2.5102 us 2.5155 us 2.5210 us] change: [+12.130% +13.194% +14.270%] (p = 0.00 < 0.05) Performance has regressed. Found 12 outliers among 100 measurements (12.00%) 4 (4.00%) high mild 8 (8.00%) high severe async-pool/no-hook/host-to-wasm - typed - nop-params-and-results time: [2.4203 us 2.4310 us 2.4440 us] change: [+4.0380% +6.3623% +8.7534%] (p = 0.00 < 0.05) Performance has regressed. Found 14 outliers among 100 measurements (14.00%) 5 (5.00%) high mild 9 (9.00%) high severe async-pool/no-hook/host-to-wasm - untyped - nop-params-and-results time: [2.5501 us 2.5593 us 2.5700 us] change: [+8.8802% +10.976% +12.937%] (p = 0.00 < 0.05) Performance has regressed. Found 16 outliers among 100 measurements (16.00%) 5 (5.00%) high mild 11 (11.00%) high severe async-pool/hook-sync/host-to-wasm - typed - nop time: [2.4135 us 2.4190 us 2.4254 us] change: [+8.3640% +9.3774% +10.435%] (p = 0.00 < 0.05) Performance has regressed. Found 11 outliers among 100 measurements (11.00%) 6 (6.00%) high mild 5 (5.00%) high severe async-pool/hook-sync/host-to-wasm - untyped - nop time: [2.5172 us 2.5248 us 2.5357 us] change: [+11.543% +12.750% +13.982%] (p = 0.00 < 0.05) Performance has regressed. Found 8 outliers among 100 measurements (8.00%) 1 (1.00%) high mild 7 (7.00%) high severe async-pool/hook-sync/host-to-wasm - typed - nop-params-and-results time: [2.4214 us 2.4353 us 2.4532 us] change: [+1.5158% +5.0872% +8.6765%] (p = 0.00 < 0.05) Performance has regressed. Found 15 outliers among 100 measurements (15.00%) 2 (2.00%) high mild 13 (13.00%) high severe async-pool/hook-sync/host-to-wasm - untyped - nop-params-and-results time: [2.5499 us 2.5607 us 2.5748 us] change: [+10.146% +12.459% +14.919%] (p = 0.00 < 0.05) Performance has regressed. Found 18 outliers among 100 measurements (18.00%) 3 (3.00%) high mild 15 (15.00%) high severe sync/no-hook/wasm-to-host - nop - typed time: [6.6135 ns 6.6288 ns 6.6452 ns] change: [+37.927% +38.837% +39.869%] (p = 0.00 < 0.05) Performance has regressed. Found 7 outliers among 100 measurements (7.00%) 2 (2.00%) high mild 5 (5.00%) high severe sync/no-hook/wasm-to-host - nop-params-and-results - typed time: [15.930 ns 15.993 ns 16.067 ns] change: [+3.9583% +5.6286% +7.2430%] (p = 0.00 < 0.05) Performance has regressed. Found 12 outliers among 100 measurements (12.00%) 11 (11.00%) high mild 1 (1.00%) high severe sync/no-hook/wasm-to-host - nop - untyped time: [20.596 ns 20.640 ns 20.690 ns] change: [+4.3293% +5.2047% +6.0935%] (p = 0.00 < 0.05) Performance has regressed. Found 10 outliers among 100 measurements (10.00%) 5 (5.00%) high mild 5 (5.00%) high severe sync/no-hook/wasm-to-host - nop-params-and-results - untyped time: [42.659 ns 42.882 ns 43.159 ns] change: [-2.1466% -0.5079% +1.2554%] (p = 0.58 > 0.05) No change in performance detected. Found 15 outliers among 100 measurements (15.00%) 1 (1.00%) high mild 14 (14.00%) high severe sync/no-hook/wasm-to-host - nop - unchecked time: [10.671 ns 10.691 ns 10.713 ns] change: [+83.911% +87.620% +92.062%] (p = 0.00 < 0.05) Performance has regressed. Found 9 outliers among 100 measurements (9.00%) 2 (2.00%) high mild 7 (7.00%) high severe sync/no-hook/wasm-to-host - nop-params-and-results - unchecked time: [11.136 ns 11.190 ns 11.263 ns] change: [-29.719% -28.446% -27.029%] (p = 0.00 < 0.05) Performance has improved. Found 14 outliers among 100 measurements (14.00%) 4 (4.00%) high mild 10 (10.00%) high severe sync/hook-sync/wasm-to-host - nop - typed time: [6.7964 ns 6.8087 ns 6.8226 ns] change: [+21.531% +24.206% +27.331%] (p = 0.00 < 0.05) Performance has regressed. Found 14 outliers among 100 measurements (14.00%) 4 (4.00%) high mild 10 (10.00%) high severe sync/hook-sync/wasm-to-host - nop-params-and-results - typed time: [15.865 ns 15.921 ns 15.985 ns] change: [+4.8466% +6.3330% +7.8317%] (p = 0.00 < 0.05) Performance has regressed. Found 16 outliers among 100 measurements (16.00%) 3 (3.00%) high mild 13 (13.00%) high severe sync/hook-sync/wasm-to-host - nop - untyped time: [21.505 ns 21.587 ns 21.677 ns] change: [+8.0908% +9.1943% +10.254%] (p = 0.00 < 0.05) Performance has regressed. Found 8 outliers among 100 measurements (8.00%) 4 (4.00%) high mild 4 (4.00%) high severe sync/hook-sync/wasm-to-host - nop-params-and-results - untyped time: [44.018 ns 44.128 ns 44.261 ns] change: [-1.4671% -0.0458% +1.2443%] (p = 0.94 > 0.05) No change in performance detected. Found 14 outliers among 100 measurements (14.00%) 5 (5.00%) high mild 9 (9.00%) high severe sync/hook-sync/wasm-to-host - nop - unchecked time: [11.264 ns 11.326 ns 11.387 ns] change: [+80.225% +81.659% +83.068%] (p = 0.00 < 0.05) Performance has regressed. Found 6 outliers among 100 measurements (6.00%) 3 (3.00%) high mild 3 (3.00%) high severe sync/hook-sync/wasm-to-host - nop-params-and-results - unchecked time: [11.816 ns 11.865 ns 11.920 ns] change: [-29.152% -28.040% -26.957%] (p = 0.00 < 0.05) Performance has improved. Found 14 outliers among 100 measurements (14.00%) 8 (8.00%) high mild 6 (6.00%) high severe async/no-hook/wasm-to-host - nop - typed time: [6.6221 ns 6.6385 ns 6.6569 ns] change: [+43.618% +44.755% +45.965%] (p = 0.00 < 0.05) Performance has regressed. Found 13 outliers among 100 measurements (13.00%) 6 (6.00%) high mild 7 (7.00%) high severe async/no-hook/wasm-to-host - nop-params-and-results - typed time: [15.884 ns 15.929 ns 15.983 ns] change: [+3.5987% +5.2053% +6.7846%] (p = 0.00 < 0.05) Performance has regressed. Found 16 outliers among 100 measurements (16.00%) 3 (3.00%) high mild 13 (13.00%) high severe async/no-hook/wasm-to-host - nop - untyped time: [20.615 ns 20.702 ns 20.821 ns] change: [+6.9799% +8.1212% +9.2819%] (p = 0.00 < 0.05) Performance has regressed. Found 10 outliers among 100 measurements (10.00%) 2 (2.00%) high mild 8 (8.00%) high severe async/no-hook/wasm-to-host - nop-params-and-results - untyped time: [41.956 ns 42.207 ns 42.521 ns] change: [-4.3057% -2.7730% -1.2428%] (p = 0.00 < 0.05) Performance has improved. Found 14 outliers among 100 measurements (14.00%) 3 (3.00%) high mild 11 (11.00%) high severe async/no-hook/wasm-to-host - nop - unchecked time: [10.440 ns 10.474 ns 10.513 ns] change: [+83.959% +85.826% +87.541%] (p = 0.00 < 0.05) Performance has regressed. Found 11 outliers among 100 measurements (11.00%) 5 (5.00%) high mild 6 (6.00%) high severe async/no-hook/wasm-to-host - nop-params-and-results - unchecked time: [11.476 ns 11.512 ns 11.554 ns] change: [-29.857% -28.383% -26.978%] (p = 0.00 < 0.05) Performance has improved. Found 12 outliers among 100 measurements (12.00%) 1 (1.00%) low mild 6 (6.00%) high mild 5 (5.00%) high severe async/no-hook/wasm-to-host - nop - async-typed time: [26.427 ns 26.478 ns 26.532 ns] change: [+6.5730% +7.4676% +8.3983%] (p = 0.00 < 0.05) Performance has regressed. Found 9 outliers among 100 measurements (9.00%) 2 (2.00%) high mild 7 (7.00%) high severe async/no-hook/wasm-to-host - nop-params-and-results - async-typed time: [28.557 ns 28.693 ns 28.880 ns] change: [+1.9099% +3.7332% +5.9731%] (p = 0.00 < 0.05) Performance has regressed. Found 15 outliers among 100 measurements (15.00%) 1 (1.00%) high mild 14 (14.00%) high severe async/hook-sync/wasm-to-host - nop - typed time: [6.7488 ns 6.7630 ns 6.7784 ns] change: [+19.935% +22.080% +23.683%] (p = 0.00 < 0.05) Performance has regressed. Found 9 outliers among 100 measurements (9.00%) 4 (4.00%) high mild 5 (5.00%) high severe async/hook-sync/wasm-to-host - nop-params-and-results - typed time: [15.928 ns 16.031 ns 16.149 ns] change: [+5.5188% +6.9567% +8.3839%] (p = 0.00 < 0.05) Performance has regressed. Found 11 outliers among 100 measurements (11.00%) 9 (9.00%) high mild 2 (2.00%) high severe async/hook-sync/wasm-to-host - nop - untyped time: [21.930 ns 22.114 ns 22.296 ns] change: [+4.6674% +7.7588% +10.375%] (p = 0.00 < 0.05) Performance has regressed. Found 4 outliers among 100 measurements (4.00%) 3 (3.00%) high mild 1 (1.00%) high severe async/hook-sync/wasm-to-host - nop-params-and-results - untyped time: [42.684 ns 42.858 ns 43.081 ns] change: [-5.2957% -3.4693% -1.6217%] (p = 0.00 < 0.05) Performance has improved. Found 14 outliers among 100 measurements (14.00%) 2 (2.00%) high mild 12 (12.00%) high severe async/hook-sync/wasm-to-host - nop - unchecked time: [11.026 ns 11.053 ns 11.086 ns] change: [+70.751% +72.378% +73.961%] (p = 0.00 < 0.05) Performance has regressed. Found 10 outliers among 100 measurements (10.00%) 5 (5.00%) high mild 5 (5.00%) high severe async/hook-sync/wasm-to-host - nop-params-and-results - unchecked time: [11.840 ns 11.900 ns 11.982 ns] change: [-27.977% -26.584% -24.887%] (p = 0.00 < 0.05) Performance has improved. Found 18 outliers among 100 measurements (18.00%) 3 (3.00%) high mild 15 (15.00%) high severe async/hook-sync/wasm-to-host - nop - async-typed time: [27.601 ns 27.709 ns 27.882 ns] change: [+8.1781% +9.1102% +10.030%] (p = 0.00 < 0.05) Performance has regressed. Found 11 outliers among 100 measurements (11.00%) 2 (2.00%) low mild 3 (3.00%) high mild 6 (6.00%) high severe async/hook-sync/wasm-to-host - nop-params-and-results - async-typed time: [28.955 ns 29.174 ns 29.413 ns] change: [+1.1226% +3.0366% +5.1126%] (p = 0.00 < 0.05) Performance has regressed. Found 13 outliers among 100 measurements (13.00%) 7 (7.00%) high mild 6 (6.00%) high severe async-pool/no-hook/wasm-to-host - nop - typed time: [6.5626 ns 6.5733 ns 6.5851 ns] change: [+40.561% +42.307% +44.514%] (p = 0.00 < 0.05) Performance has regressed. Found 9 outliers among 100 measurements (9.00%) 5 (5.00%) high mild 4 (4.00%) high severe async-pool/no-hook/wasm-to-host - nop-params-and-results - typed time: [15.820 ns 15.886 ns 15.969 ns] change: [+4.1044% +5.7928% +7.7122%] (p = 0.00 < 0.05) Performance has regressed. Found 17 outliers among 100 measurements (17.00%) 4 (4.00%) high mild 13 (13.00%) high severe async-pool/no-hook/wasm-to-host - nop - untyped time: [20.481 ns 20.521 ns 20.566 ns] change: [+6.7962% +7.6950% +8.7612%] (p = 0.00 < 0.05) Performance has regressed. Found 11 outliers among 100 measurements (11.00%) 6 (6.00%) high mild 5 (5.00%) high severe async-pool/no-hook/wasm-to-host - nop-params-and-results - untyped time: [41.834 ns 41.998 ns 42.189 ns] change: [-3.8185% -2.2687% -0.7541%] (p = 0.01 < 0.05) Change within noise threshold. Found 13 outliers among 100 measurements (13.00%) 3 (3.00%) high mild 10 (10.00%) high severe async-pool/no-hook/wasm-to-host - nop - unchecked time: [10.353 ns 10.380 ns 10.414 ns] change: [+82.042% +84.591% +87.205%] (p = 0.00 < 0.05) Performance has regressed. Found 7 outliers among 100 measurements (7.00%) 4 (4.00%) high mild 3 (3.00%) high severe async-pool/no-hook/wasm-to-host - nop-params-and-results - unchecked time: [11.123 ns 11.168 ns 11.228 ns] change: [-30.813% -29.285% -27.874%] (p = 0.00 < 0.05) Performance has improved. Found 12 outliers among 100 measurements (12.00%) 11 (11.00%) high mild 1 (1.00%) high severe async-pool/no-hook/wasm-to-host - nop - async-typed time: [27.442 ns 27.528 ns 27.638 ns] change: [+7.5215% +9.9795% +12.266%] (p = 0.00 < 0.05) Performance has regressed. Found 18 outliers among 100 measurements (18.00%) 3 (3.00%) high mild 15 (15.00%) high severe async-pool/no-hook/wasm-to-host - nop-params-and-results - async-typed time: [29.014 ns 29.148 ns 29.312 ns] change: [+2.0227% +3.4722% +4.9047%] (p = 0.00 < 0.05) Performance has regressed. Found 7 outliers among 100 measurements (7.00%) 6 (6.00%) high mild 1 (1.00%) high severe async-pool/hook-sync/wasm-to-host - nop - typed time: [6.7916 ns 6.8116 ns 6.8325 ns] change: [+20.937% +22.050% +23.281%] (p = 0.00 < 0.05) Performance has regressed. Found 11 outliers among 100 measurements (11.00%) 5 (5.00%) high mild 6 (6.00%) high severe async-pool/hook-sync/wasm-to-host - nop-params-and-results - typed time: [15.917 ns 15.975 ns 16.051 ns] change: [+4.6404% +6.4217% +8.3075%] (p = 0.00 < 0.05) Performance has regressed. Found 16 outliers among 100 measurements (16.00%) 5 (5.00%) high mild 11 (11.00%) high severe async-pool/hook-sync/wasm-to-host - nop - untyped time: [21.558 ns 21.612 ns 21.679 ns] change: [+8.1158% +9.1409% +10.217%] (p = 0.00 < 0.05) Performance has regressed. Found 9 outliers among 100 measurements (9.00%) 2 (2.00%) high mild 7 (7.00%) high severe async-pool/hook-sync/wasm-to-host - nop-params-and-results - untyped time: [42.475 ns 42.614 ns 42.775 ns] change: [-6.3613% -4.4709% -2.7647%] (p = 0.00 < 0.05) Performance has improved. Found 18 outliers among 100 measurements (18.00%) 3 (3.00%) high mild 15 (15.00%) high severe async-pool/hook-sync/wasm-to-host - nop - unchecked time: [11.150 ns 11.195 ns 11.247 ns] change: [+74.424% +77.056% +79.811%] (p = 0.00 < 0.05) Performance has regressed. Found 14 outliers among 100 measurements (14.00%) 3 (3.00%) high mild 11 (11.00%) high severe async-pool/hook-sync/wasm-to-host - nop-params-and-results - unchecked time: [11.639 ns 11.695 ns 11.760 ns] change: [-30.212% -29.023% -27.954%] (p = 0.00 < 0.05) Performance has improved. Found 15 outliers among 100 measurements (15.00%) 7 (7.00%) high mild 8 (8.00%) high severe async-pool/hook-sync/wasm-to-host - nop - async-typed time: [27.480 ns 27.712 ns 27.984 ns] change: [+2.9764% +6.5061% +9.8914%] (p = 0.00 < 0.05) Performance has regressed. Found 8 outliers among 100 measurements (8.00%) 6 (6.00%) high mild 2 (2.00%) high severe async-pool/hook-sync/wasm-to-host - nop-params-and-results - async-typed time: [29.218 ns 29.380 ns 29.600 ns] change: [+5.2283% +7.7247% +10.822%] (p = 0.00 < 0.05) Performance has regressed. Found 16 outliers among 100 measurements (16.00%) 2 (2.00%) high mild 14 (14.00%) high severe ``` </details> * Add s390x support for frame pointer-based stack walking * wasmtime: Allow `Caller::get_export` to get all exports * fuzzing: Add a fuzz target to check that our stack traces are correct We generate Wasm modules that keep track of their own stack as they call and return between functions, and then we periodically check that if the host captures a backtrace, it matches what the Wasm module has recorded. * Remove VM offsets for `VMHostFuncContext` since it isn't used by JIT code * Add doc comment with stack walking implementation notes * Document the extra state that can be passed to `wasmtime_runtime::Backtrace` methods * Add extensive comments for stack walking function * Factor architecture-specific bits of stack walking out into modules * Initialize store-related fields in a vmctx to null when there is no store yet Rather than leaving them as uninitialized data. * Use `set_callee` instead of manually setting the vmctx field * Use a more informative compile error message for unsupported architectures * Document unsafety of `prepare_host_to_wasm_trampoline` * Use `bti c` instead of `hint #34` in inline aarch64 assembly * Remove outdated TODO comment * Remove setting of `last_wasm_exit_fp` in `set_jit_trap` This is no longer needed as the value is plumbed through to the backtrace code directly now. * Only set the stack limit once, in the face of re-entrancy into Wasm * Add comments for s390x-specific stack walking bits * Use the helper macro for all libcalls If we forget to use it, and then trigger a GC from the libcall, that means we could miss stack frames when walking the stack, fail to find live GC refs, and then get use after free bugs. Much less risky to always use the helper macro that takes care of all of that for us. * Use the `asm_sym!` macro in Wasm-to-libcall trampolines This macro handles the macOS-specific underscore prefix stuff for us. * wasmtime: add size and align to `externref` assertion error message * Extend the `stacks` fuzzer to have host frames in between Wasm frames This way we get one or more contiguous sequences of Wasm frames on the stack, instead of exactly one. * Add documentation for aarch64-specific backtrace helpers * Clarify that we only support little-endian aarch64 in trampoline comment * Use `.machine z13` in s390x assembly file Since apparently our CI machines have pretty old assemblers that don't have `.machine z14`. This should be fine though since these trampolines don't make use of anything that is introduced in z14. * Fix aarch64 build * Fix macOS build * Document the `asm_sym!` macro * Add windows support to the `wasmtime-asm-macros` crate * Add windows support to host<--->Wasm trampolines * Fix trap handler build on windows * Run `rustfmt` on s390x trampoline source file * Temporarily disable some assertions about a trap's backtrace in the component model tests Follow up to re-enable this and fix the associated issue: https://github.com/bytecodealliance/wasmtime/issues/4535 * Refactor libcall definitions with less macros This refactors the `libcall!` macro to use the `foreach_builtin_function!` macro to define all of the trampolines. Additionally the macro surrounding each libcall itself is no longer necessary and helps avoid too many macros. * Use `VMOpaqueContext::from_vm_host_func_context` in `VMHostFuncContext::new` * Move `backtrace` module to be submodule of `traphandlers` This avoids making some things `pub(crate)` in `traphandlers` that really shouldn't be. * Fix macOS aarch64 build * Use "i64" instead of "word" in aarch64-specific file * Save/restore entry SP and exit FP/return pointer in the face of panicking imported host functions Also clean up assertions surrounding our saved entry/exit registers. * Put "typed" vs "untyped" in the same position of call benchmark names Regardless if we are doing wasm-to-host or host-to-wasm * Fix stacks test case generator build for new `wasm-encoder` * Fix build for s390x * Expand libcalls in s390x asm * Disable more parts of component tests now that backtrace assertions are a bit tighter * Remove assertion that can maybe fail on s390x Co-authored-by: Ulrich Weigand <ulrich.weigand@de.ibm.com> Co-authored-by: Alex Crichton <alex@alexcrichton.com>
2 years ago
"paste",
"rand 0.8.5",
"rustix",
`wasmtime`: Implement fast Wasm stack walking (#4431) * Always preserve frame pointers in Wasmtime This allows us to efficiently and simply capture Wasm stacks without maintaining and synchronizing any safety-critical side tables between the compiler and the runtime. * wasmtime: Implement fast Wasm stack walking Why do we want Wasm stack walking to be fast? Because we capture stacks whenever there is a trap and traps actually happen fairly frequently with short-lived programs and WASI's `exit`. Previously, we would rely on generating the system unwind info (e.g. `.eh_frame`) and using the system unwinder (via the `backtrace`crate) to walk the full stack and filter out any non-Wasm stack frames. This can, unfortunately, be slow for two primary reasons: 1. The system unwinder is doing `O(all-kinds-of-frames)` work rather than `O(wasm-frames)` work. 2. System unwind info and the system unwinder need to be much more general than a purpose-built stack walker for Wasm needs to be. It has to handle any kind of stack frame that any compiler might emit where as our Wasm frames are emitted by Cranelift and always have frame pointers. This translates into implementation complexity and general overhead. There can also be unnecessary-for-our-use-cases global synchronization and locks involved, further slowing down stack walking in the presence of multiple threads trying to capture stacks in parallel. This commit introduces a purpose-built stack walker for traversing just our Wasm frames. To find all the sequences of Wasm-to-Wasm stack frames, and ignore non-Wasm stack frames, we keep a linked list of `(entry stack pointer, exit frame pointer)` pairs. This linked list is maintained via Wasm-to-host and host-to-Wasm trampolines. Within a sequence of Wasm-to-Wasm calls, we can use frame pointers (which Cranelift preserves) to find the next older Wasm frame on the stack, and we keep doing this until we reach the entry stack pointer, meaning that the next older frame will be a host frame. The trampolines need to avoid a couple stumbling blocks. First, they need to be compiled ahead of time, since we may not have access to a compiler at runtime (e.g. if the `cranelift` feature is disabled) but still want to be able to call functions that have already been compiled and get stack traces for those functions. Usually this means we would compile the appropriate trampolines inside `Module::new` and the compiled module object would hold the trampolines. However, we *also* need to support calling host functions that are wrapped into `wasmtime::Func`s and there doesn't exist *any* ahead-of-time compiled module object to hold the appropriate trampolines: ```rust // Define a host function. let func_type = wasmtime::FuncType::new( vec![wasmtime::ValType::I32], vec![wasmtime::ValType::I32], ); let func = Func::new(&mut store, func_type, |_, params, results| { // ... Ok(()) }); // Call that host function. let mut results = vec![wasmtime::Val::I32(0)]; func.call(&[wasmtime::Val::I32(0)], &mut results)?; ``` Therefore, we define one host-to-Wasm trampoline and one Wasm-to-host trampoline in assembly that work for all Wasm and host function signatures. These trampolines are careful to only use volatile registers, avoid touching any register that is an argument in the calling convention ABI, and tail call to the target callee function. This allows forwarding any set of arguments and any returns to and from the callee, while also allowing us to maintain our linked list of Wasm stack and frame pointers before transferring control to the callee. These trampolines are not used in Wasm-to-Wasm calls, only when crossing the host-Wasm boundary, so they do not impose overhead on regular calls. (And if using one trampoline for all host-Wasm boundary crossing ever breaks branch prediction enough in the CPU to become any kind of bottleneck, we can do fun things like have multiple copies of the same trampoline and choose a random copy for each function, sharding the functions across branch predictor entries.) Finally, this commit also ends the use of a synthetic `Module` and allocating a stubbed out `VMContext` for host functions. Instead, we define a `VMHostFuncContext` with its own magic value, similar to `VMComponentContext`, specifically for host functions. <h2>Benchmarks</h2> <h3>Traps and Stack Traces</h3> Large improvements to taking stack traces on traps, ranging from shaving off 64% to 99.95% of the time it used to take. <details> ``` multi-threaded-traps/0 time: [2.5686 us 2.5808 us 2.5934 us] thrpt: [0.0000 elem/s 0.0000 elem/s 0.0000 elem/s] change: time: [-85.419% -85.153% -84.869%] (p = 0.00 < 0.05) thrpt: [+560.90% +573.56% +585.84%] Performance has improved. Found 8 outliers among 100 measurements (8.00%) 4 (4.00%) high mild 4 (4.00%) high severe multi-threaded-traps/1 time: [2.9021 us 2.9167 us 2.9322 us] thrpt: [341.04 Kelem/s 342.86 Kelem/s 344.58 Kelem/s] change: time: [-91.455% -91.294% -91.096%] (p = 0.00 < 0.05) thrpt: [+1023.1% +1048.6% +1070.3%] Performance has improved. Found 6 outliers among 100 measurements (6.00%) 1 (1.00%) high mild 5 (5.00%) high severe multi-threaded-traps/2 time: [2.9996 us 3.0145 us 3.0295 us] thrpt: [660.18 Kelem/s 663.47 Kelem/s 666.76 Kelem/s] change: time: [-94.040% -93.910% -93.762%] (p = 0.00 < 0.05) thrpt: [+1503.1% +1542.0% +1578.0%] Performance has improved. Found 5 outliers among 100 measurements (5.00%) 5 (5.00%) high severe multi-threaded-traps/4 time: [5.5768 us 5.6052 us 5.6364 us] thrpt: [709.68 Kelem/s 713.63 Kelem/s 717.25 Kelem/s] change: time: [-93.193% -93.121% -93.052%] (p = 0.00 < 0.05) thrpt: [+1339.2% +1353.6% +1369.1%] Performance has improved. multi-threaded-traps/8 time: [8.6408 us 9.1212 us 9.5438 us] thrpt: [838.24 Kelem/s 877.08 Kelem/s 925.84 Kelem/s] change: time: [-94.754% -94.473% -94.202%] (p = 0.00 < 0.05) thrpt: [+1624.7% +1709.2% +1806.1%] Performance has improved. multi-threaded-traps/16 time: [10.152 us 10.840 us 11.545 us] thrpt: [1.3858 Melem/s 1.4760 Melem/s 1.5761 Melem/s] change: time: [-97.042% -96.823% -96.577%] (p = 0.00 < 0.05) thrpt: [+2821.5% +3048.1% +3281.1%] Performance has improved. Found 1 outliers among 100 measurements (1.00%) 1 (1.00%) high mild many-modules-registered-traps/1 time: [2.6278 us 2.6361 us 2.6447 us] thrpt: [378.11 Kelem/s 379.35 Kelem/s 380.55 Kelem/s] change: time: [-85.311% -85.108% -84.909%] (p = 0.00 < 0.05) thrpt: [+562.65% +571.51% +580.76%] Performance has improved. Found 9 outliers among 100 measurements (9.00%) 3 (3.00%) high mild 6 (6.00%) high severe many-modules-registered-traps/8 time: [2.6294 us 2.6460 us 2.6623 us] thrpt: [3.0049 Melem/s 3.0235 Melem/s 3.0425 Melem/s] change: time: [-85.895% -85.485% -85.022%] (p = 0.00 < 0.05) thrpt: [+567.63% +588.95% +608.95%] Performance has improved. Found 8 outliers among 100 measurements (8.00%) 3 (3.00%) high mild 5 (5.00%) high severe many-modules-registered-traps/64 time: [2.6218 us 2.6329 us 2.6452 us] thrpt: [24.195 Melem/s 24.308 Melem/s 24.411 Melem/s] change: time: [-93.629% -93.551% -93.470%] (p = 0.00 < 0.05) thrpt: [+1431.4% +1450.6% +1469.5%] Performance has improved. Found 3 outliers among 100 measurements (3.00%) 3 (3.00%) high mild many-modules-registered-traps/512 time: [2.6569 us 2.6737 us 2.6923 us] thrpt: [190.17 Melem/s 191.50 Melem/s 192.71 Melem/s] change: time: [-99.277% -99.268% -99.260%] (p = 0.00 < 0.05) thrpt: [+13417% +13566% +13731%] Performance has improved. Found 4 outliers among 100 measurements (4.00%) 4 (4.00%) high mild many-modules-registered-traps/4096 time: [2.7258 us 2.7390 us 2.7535 us] thrpt: [1.4876 Gelem/s 1.4955 Gelem/s 1.5027 Gelem/s] change: time: [-99.956% -99.955% -99.955%] (p = 0.00 < 0.05) thrpt: [+221417% +223380% +224881%] Performance has improved. Found 2 outliers among 100 measurements (2.00%) 1 (1.00%) high mild 1 (1.00%) high severe many-stack-frames-traps/1 time: [1.4658 us 1.4719 us 1.4784 us] thrpt: [676.39 Kelem/s 679.38 Kelem/s 682.21 Kelem/s] change: time: [-90.368% -89.947% -89.586%] (p = 0.00 < 0.05) thrpt: [+860.23% +894.72% +938.21%] Performance has improved. Found 8 outliers among 100 measurements (8.00%) 5 (5.00%) high mild 3 (3.00%) high severe many-stack-frames-traps/8 time: [2.4772 us 2.4870 us 2.4973 us] thrpt: [3.2034 Melem/s 3.2167 Melem/s 3.2294 Melem/s] change: time: [-85.550% -85.370% -85.199%] (p = 0.00 < 0.05) thrpt: [+575.65% +583.51% +592.03%] Performance has improved. Found 8 outliers among 100 measurements (8.00%) 4 (4.00%) high mild 4 (4.00%) high severe many-stack-frames-traps/64 time: [10.109 us 10.171 us 10.236 us] thrpt: [6.2525 Melem/s 6.2925 Melem/s 6.3309 Melem/s] change: time: [-78.144% -77.797% -77.336%] (p = 0.00 < 0.05) thrpt: [+341.22% +350.38% +357.55%] Performance has improved. Found 7 outliers among 100 measurements (7.00%) 5 (5.00%) high mild 2 (2.00%) high severe many-stack-frames-traps/512 time: [126.16 us 126.54 us 126.96 us] thrpt: [4.0329 Melem/s 4.0461 Melem/s 4.0583 Melem/s] change: time: [-65.364% -64.933% -64.453%] (p = 0.00 < 0.05) thrpt: [+181.32% +185.17% +188.71%] Performance has improved. Found 4 outliers among 100 measurements (4.00%) 4 (4.00%) high severe ``` </details> <h3>Calls</h3> There is, however, a small regression in raw Wasm-to-host and host-to-Wasm call performance due the new trampolines. It seems to be on the order of about 2-10 nanoseconds per call, depending on the benchmark. I believe this regression is ultimately acceptable because 1. this overhead will be vastly dominated by whatever work a non-nop callee actually does, 2. we will need these trampolines, or something like them, when implementing the Wasm exceptions proposal to do things like translate Wasm's exceptions into Rust's `Result`s, 3. and because the performance improvements to trapping and capturing stack traces are of such a larger magnitude than this call regressions. <details> ``` sync/no-hook/host-to-wasm - typed - nop time: [28.683 ns 28.757 ns 28.844 ns] change: [+16.472% +17.183% +17.904%] (p = 0.00 < 0.05) Performance has regressed. Found 10 outliers among 100 measurements (10.00%) 1 (1.00%) low mild 4 (4.00%) high mild 5 (5.00%) high severe sync/no-hook/host-to-wasm - untyped - nop time: [42.515 ns 42.652 ns 42.841 ns] change: [+12.371% +14.614% +17.462%] (p = 0.00 < 0.05) Performance has regressed. Found 11 outliers among 100 measurements (11.00%) 1 (1.00%) high mild 10 (10.00%) high severe sync/no-hook/host-to-wasm - unchecked - nop time: [33.936 ns 34.052 ns 34.179 ns] change: [+25.478% +26.938% +28.369%] (p = 0.00 < 0.05) Performance has regressed. Found 9 outliers among 100 measurements (9.00%) 7 (7.00%) high mild 2 (2.00%) high severe sync/no-hook/host-to-wasm - typed - nop-params-and-results time: [34.290 ns 34.388 ns 34.502 ns] change: [+40.802% +42.706% +44.526%] (p = 0.00 < 0.05) Performance has regressed. Found 13 outliers among 100 measurements (13.00%) 5 (5.00%) high mild 8 (8.00%) high severe sync/no-hook/host-to-wasm - untyped - nop-params-and-results time: [62.546 ns 62.721 ns 62.919 ns] change: [+2.5014% +3.6319% +4.8078%] (p = 0.00 < 0.05) Performance has regressed. Found 12 outliers among 100 measurements (12.00%) 2 (2.00%) high mild 10 (10.00%) high severe sync/no-hook/host-to-wasm - unchecked - nop-params-and-results time: [42.609 ns 42.710 ns 42.831 ns] change: [+20.966% +22.282% +23.475%] (p = 0.00 < 0.05) Performance has regressed. Found 11 outliers among 100 measurements (11.00%) 4 (4.00%) high mild 7 (7.00%) high severe sync/hook-sync/host-to-wasm - typed - nop time: [29.546 ns 29.675 ns 29.818 ns] change: [+20.693% +21.794% +22.836%] (p = 0.00 < 0.05) Performance has regressed. Found 5 outliers among 100 measurements (5.00%) 3 (3.00%) high mild 2 (2.00%) high severe sync/hook-sync/host-to-wasm - untyped - nop time: [45.448 ns 45.699 ns 45.961 ns] change: [+17.204% +18.514% +19.590%] (p = 0.00 < 0.05) Performance has regressed. Found 14 outliers among 100 measurements (14.00%) 4 (4.00%) high mild 10 (10.00%) high severe sync/hook-sync/host-to-wasm - unchecked - nop time: [34.334 ns 34.437 ns 34.558 ns] change: [+23.225% +24.477% +25.886%] (p = 0.00 < 0.05) Performance has regressed. Found 12 outliers among 100 measurements (12.00%) 5 (5.00%) high mild 7 (7.00%) high severe sync/hook-sync/host-to-wasm - typed - nop-params-and-results time: [36.594 ns 36.763 ns 36.974 ns] change: [+41.967% +47.261% +52.086%] (p = 0.00 < 0.05) Performance has regressed. Found 12 outliers among 100 measurements (12.00%) 3 (3.00%) high mild 9 (9.00%) high severe sync/hook-sync/host-to-wasm - untyped - nop-params-and-results time: [63.541 ns 63.831 ns 64.194 ns] change: [-4.4337% -0.6855% +2.7134%] (p = 0.73 > 0.05) No change in performance detected. Found 8 outliers among 100 measurements (8.00%) 6 (6.00%) high mild 2 (2.00%) high severe sync/hook-sync/host-to-wasm - unchecked - nop-params-and-results time: [43.968 ns 44.169 ns 44.437 ns] change: [+18.772% +21.802% +24.623%] (p = 0.00 < 0.05) Performance has regressed. Found 15 outliers among 100 measurements (15.00%) 3 (3.00%) high mild 12 (12.00%) high severe async/no-hook/host-to-wasm - typed - nop time: [4.9612 us 4.9743 us 4.9889 us] change: [+9.9493% +11.911% +13.502%] (p = 0.00 < 0.05) Performance has regressed. Found 10 outliers among 100 measurements (10.00%) 6 (6.00%) high mild 4 (4.00%) high severe async/no-hook/host-to-wasm - untyped - nop time: [5.0030 us 5.0211 us 5.0439 us] change: [+10.841% +11.873% +12.977%] (p = 0.00 < 0.05) Performance has regressed. Found 10 outliers among 100 measurements (10.00%) 3 (3.00%) high mild 7 (7.00%) high severe async/no-hook/host-to-wasm - typed - nop-params-and-results time: [4.9273 us 4.9468 us 4.9700 us] change: [+4.7381% +6.8445% +8.8238%] (p = 0.00 < 0.05) Performance has regressed. Found 14 outliers among 100 measurements (14.00%) 5 (5.00%) high mild 9 (9.00%) high severe async/no-hook/host-to-wasm - untyped - nop-params-and-results time: [5.1151 us 5.1338 us 5.1555 us] change: [+9.5335% +11.290% +13.044%] (p = 0.00 < 0.05) Performance has regressed. Found 16 outliers among 100 measurements (16.00%) 3 (3.00%) high mild 13 (13.00%) high severe async/hook-sync/host-to-wasm - typed - nop time: [4.9330 us 4.9394 us 4.9467 us] change: [+10.046% +11.038% +12.035%] (p = 0.00 < 0.05) Performance has regressed. Found 12 outliers among 100 measurements (12.00%) 5 (5.00%) high mild 7 (7.00%) high severe async/hook-sync/host-to-wasm - untyped - nop time: [5.0073 us 5.0183 us 5.0310 us] change: [+9.3828% +10.565% +11.752%] (p = 0.00 < 0.05) Performance has regressed. Found 8 outliers among 100 measurements (8.00%) 3 (3.00%) high mild 5 (5.00%) high severe async/hook-sync/host-to-wasm - typed - nop-params-and-results time: [4.9610 us 4.9839 us 5.0097 us] change: [+9.0857% +11.513% +14.359%] (p = 0.00 < 0.05) Performance has regressed. Found 13 outliers among 100 measurements (13.00%) 7 (7.00%) high mild 6 (6.00%) high severe async/hook-sync/host-to-wasm - untyped - nop-params-and-results time: [5.0995 us 5.1272 us 5.1617 us] change: [+9.3600% +11.506% +13.809%] (p = 0.00 < 0.05) Performance has regressed. Found 10 outliers among 100 measurements (10.00%) 6 (6.00%) high mild 4 (4.00%) high severe async-pool/no-hook/host-to-wasm - typed - nop time: [2.4242 us 2.4316 us 2.4396 us] change: [+7.8756% +8.8803% +9.8346%] (p = 0.00 < 0.05) Performance has regressed. Found 8 outliers among 100 measurements (8.00%) 5 (5.00%) high mild 3 (3.00%) high severe async-pool/no-hook/host-to-wasm - untyped - nop time: [2.5102 us 2.5155 us 2.5210 us] change: [+12.130% +13.194% +14.270%] (p = 0.00 < 0.05) Performance has regressed. Found 12 outliers among 100 measurements (12.00%) 4 (4.00%) high mild 8 (8.00%) high severe async-pool/no-hook/host-to-wasm - typed - nop-params-and-results time: [2.4203 us 2.4310 us 2.4440 us] change: [+4.0380% +6.3623% +8.7534%] (p = 0.00 < 0.05) Performance has regressed. Found 14 outliers among 100 measurements (14.00%) 5 (5.00%) high mild 9 (9.00%) high severe async-pool/no-hook/host-to-wasm - untyped - nop-params-and-results time: [2.5501 us 2.5593 us 2.5700 us] change: [+8.8802% +10.976% +12.937%] (p = 0.00 < 0.05) Performance has regressed. Found 16 outliers among 100 measurements (16.00%) 5 (5.00%) high mild 11 (11.00%) high severe async-pool/hook-sync/host-to-wasm - typed - nop time: [2.4135 us 2.4190 us 2.4254 us] change: [+8.3640% +9.3774% +10.435%] (p = 0.00 < 0.05) Performance has regressed. Found 11 outliers among 100 measurements (11.00%) 6 (6.00%) high mild 5 (5.00%) high severe async-pool/hook-sync/host-to-wasm - untyped - nop time: [2.5172 us 2.5248 us 2.5357 us] change: [+11.543% +12.750% +13.982%] (p = 0.00 < 0.05) Performance has regressed. Found 8 outliers among 100 measurements (8.00%) 1 (1.00%) high mild 7 (7.00%) high severe async-pool/hook-sync/host-to-wasm - typed - nop-params-and-results time: [2.4214 us 2.4353 us 2.4532 us] change: [+1.5158% +5.0872% +8.6765%] (p = 0.00 < 0.05) Performance has regressed. Found 15 outliers among 100 measurements (15.00%) 2 (2.00%) high mild 13 (13.00%) high severe async-pool/hook-sync/host-to-wasm - untyped - nop-params-and-results time: [2.5499 us 2.5607 us 2.5748 us] change: [+10.146% +12.459% +14.919%] (p = 0.00 < 0.05) Performance has regressed. Found 18 outliers among 100 measurements (18.00%) 3 (3.00%) high mild 15 (15.00%) high severe sync/no-hook/wasm-to-host - nop - typed time: [6.6135 ns 6.6288 ns 6.6452 ns] change: [+37.927% +38.837% +39.869%] (p = 0.00 < 0.05) Performance has regressed. Found 7 outliers among 100 measurements (7.00%) 2 (2.00%) high mild 5 (5.00%) high severe sync/no-hook/wasm-to-host - nop-params-and-results - typed time: [15.930 ns 15.993 ns 16.067 ns] change: [+3.9583% +5.6286% +7.2430%] (p = 0.00 < 0.05) Performance has regressed. Found 12 outliers among 100 measurements (12.00%) 11 (11.00%) high mild 1 (1.00%) high severe sync/no-hook/wasm-to-host - nop - untyped time: [20.596 ns 20.640 ns 20.690 ns] change: [+4.3293% +5.2047% +6.0935%] (p = 0.00 < 0.05) Performance has regressed. Found 10 outliers among 100 measurements (10.00%) 5 (5.00%) high mild 5 (5.00%) high severe sync/no-hook/wasm-to-host - nop-params-and-results - untyped time: [42.659 ns 42.882 ns 43.159 ns] change: [-2.1466% -0.5079% +1.2554%] (p = 0.58 > 0.05) No change in performance detected. Found 15 outliers among 100 measurements (15.00%) 1 (1.00%) high mild 14 (14.00%) high severe sync/no-hook/wasm-to-host - nop - unchecked time: [10.671 ns 10.691 ns 10.713 ns] change: [+83.911% +87.620% +92.062%] (p = 0.00 < 0.05) Performance has regressed. Found 9 outliers among 100 measurements (9.00%) 2 (2.00%) high mild 7 (7.00%) high severe sync/no-hook/wasm-to-host - nop-params-and-results - unchecked time: [11.136 ns 11.190 ns 11.263 ns] change: [-29.719% -28.446% -27.029%] (p = 0.00 < 0.05) Performance has improved. Found 14 outliers among 100 measurements (14.00%) 4 (4.00%) high mild 10 (10.00%) high severe sync/hook-sync/wasm-to-host - nop - typed time: [6.7964 ns 6.8087 ns 6.8226 ns] change: [+21.531% +24.206% +27.331%] (p = 0.00 < 0.05) Performance has regressed. Found 14 outliers among 100 measurements (14.00%) 4 (4.00%) high mild 10 (10.00%) high severe sync/hook-sync/wasm-to-host - nop-params-and-results - typed time: [15.865 ns 15.921 ns 15.985 ns] change: [+4.8466% +6.3330% +7.8317%] (p = 0.00 < 0.05) Performance has regressed. Found 16 outliers among 100 measurements (16.00%) 3 (3.00%) high mild 13 (13.00%) high severe sync/hook-sync/wasm-to-host - nop - untyped time: [21.505 ns 21.587 ns 21.677 ns] change: [+8.0908% +9.1943% +10.254%] (p = 0.00 < 0.05) Performance has regressed. Found 8 outliers among 100 measurements (8.00%) 4 (4.00%) high mild 4 (4.00%) high severe sync/hook-sync/wasm-to-host - nop-params-and-results - untyped time: [44.018 ns 44.128 ns 44.261 ns] change: [-1.4671% -0.0458% +1.2443%] (p = 0.94 > 0.05) No change in performance detected. Found 14 outliers among 100 measurements (14.00%) 5 (5.00%) high mild 9 (9.00%) high severe sync/hook-sync/wasm-to-host - nop - unchecked time: [11.264 ns 11.326 ns 11.387 ns] change: [+80.225% +81.659% +83.068%] (p = 0.00 < 0.05) Performance has regressed. Found 6 outliers among 100 measurements (6.00%) 3 (3.00%) high mild 3 (3.00%) high severe sync/hook-sync/wasm-to-host - nop-params-and-results - unchecked time: [11.816 ns 11.865 ns 11.920 ns] change: [-29.152% -28.040% -26.957%] (p = 0.00 < 0.05) Performance has improved. Found 14 outliers among 100 measurements (14.00%) 8 (8.00%) high mild 6 (6.00%) high severe async/no-hook/wasm-to-host - nop - typed time: [6.6221 ns 6.6385 ns 6.6569 ns] change: [+43.618% +44.755% +45.965%] (p = 0.00 < 0.05) Performance has regressed. Found 13 outliers among 100 measurements (13.00%) 6 (6.00%) high mild 7 (7.00%) high severe async/no-hook/wasm-to-host - nop-params-and-results - typed time: [15.884 ns 15.929 ns 15.983 ns] change: [+3.5987% +5.2053% +6.7846%] (p = 0.00 < 0.05) Performance has regressed. Found 16 outliers among 100 measurements (16.00%) 3 (3.00%) high mild 13 (13.00%) high severe async/no-hook/wasm-to-host - nop - untyped time: [20.615 ns 20.702 ns 20.821 ns] change: [+6.9799% +8.1212% +9.2819%] (p = 0.00 < 0.05) Performance has regressed. Found 10 outliers among 100 measurements (10.00%) 2 (2.00%) high mild 8 (8.00%) high severe async/no-hook/wasm-to-host - nop-params-and-results - untyped time: [41.956 ns 42.207 ns 42.521 ns] change: [-4.3057% -2.7730% -1.2428%] (p = 0.00 < 0.05) Performance has improved. Found 14 outliers among 100 measurements (14.00%) 3 (3.00%) high mild 11 (11.00%) high severe async/no-hook/wasm-to-host - nop - unchecked time: [10.440 ns 10.474 ns 10.513 ns] change: [+83.959% +85.826% +87.541%] (p = 0.00 < 0.05) Performance has regressed. Found 11 outliers among 100 measurements (11.00%) 5 (5.00%) high mild 6 (6.00%) high severe async/no-hook/wasm-to-host - nop-params-and-results - unchecked time: [11.476 ns 11.512 ns 11.554 ns] change: [-29.857% -28.383% -26.978%] (p = 0.00 < 0.05) Performance has improved. Found 12 outliers among 100 measurements (12.00%) 1 (1.00%) low mild 6 (6.00%) high mild 5 (5.00%) high severe async/no-hook/wasm-to-host - nop - async-typed time: [26.427 ns 26.478 ns 26.532 ns] change: [+6.5730% +7.4676% +8.3983%] (p = 0.00 < 0.05) Performance has regressed. Found 9 outliers among 100 measurements (9.00%) 2 (2.00%) high mild 7 (7.00%) high severe async/no-hook/wasm-to-host - nop-params-and-results - async-typed time: [28.557 ns 28.693 ns 28.880 ns] change: [+1.9099% +3.7332% +5.9731%] (p = 0.00 < 0.05) Performance has regressed. Found 15 outliers among 100 measurements (15.00%) 1 (1.00%) high mild 14 (14.00%) high severe async/hook-sync/wasm-to-host - nop - typed time: [6.7488 ns 6.7630 ns 6.7784 ns] change: [+19.935% +22.080% +23.683%] (p = 0.00 < 0.05) Performance has regressed. Found 9 outliers among 100 measurements (9.00%) 4 (4.00%) high mild 5 (5.00%) high severe async/hook-sync/wasm-to-host - nop-params-and-results - typed time: [15.928 ns 16.031 ns 16.149 ns] change: [+5.5188% +6.9567% +8.3839%] (p = 0.00 < 0.05) Performance has regressed. Found 11 outliers among 100 measurements (11.00%) 9 (9.00%) high mild 2 (2.00%) high severe async/hook-sync/wasm-to-host - nop - untyped time: [21.930 ns 22.114 ns 22.296 ns] change: [+4.6674% +7.7588% +10.375%] (p = 0.00 < 0.05) Performance has regressed. Found 4 outliers among 100 measurements (4.00%) 3 (3.00%) high mild 1 (1.00%) high severe async/hook-sync/wasm-to-host - nop-params-and-results - untyped time: [42.684 ns 42.858 ns 43.081 ns] change: [-5.2957% -3.4693% -1.6217%] (p = 0.00 < 0.05) Performance has improved. Found 14 outliers among 100 measurements (14.00%) 2 (2.00%) high mild 12 (12.00%) high severe async/hook-sync/wasm-to-host - nop - unchecked time: [11.026 ns 11.053 ns 11.086 ns] change: [+70.751% +72.378% +73.961%] (p = 0.00 < 0.05) Performance has regressed. Found 10 outliers among 100 measurements (10.00%) 5 (5.00%) high mild 5 (5.00%) high severe async/hook-sync/wasm-to-host - nop-params-and-results - unchecked time: [11.840 ns 11.900 ns 11.982 ns] change: [-27.977% -26.584% -24.887%] (p = 0.00 < 0.05) Performance has improved. Found 18 outliers among 100 measurements (18.00%) 3 (3.00%) high mild 15 (15.00%) high severe async/hook-sync/wasm-to-host - nop - async-typed time: [27.601 ns 27.709 ns 27.882 ns] change: [+8.1781% +9.1102% +10.030%] (p = 0.00 < 0.05) Performance has regressed. Found 11 outliers among 100 measurements (11.00%) 2 (2.00%) low mild 3 (3.00%) high mild 6 (6.00%) high severe async/hook-sync/wasm-to-host - nop-params-and-results - async-typed time: [28.955 ns 29.174 ns 29.413 ns] change: [+1.1226% +3.0366% +5.1126%] (p = 0.00 < 0.05) Performance has regressed. Found 13 outliers among 100 measurements (13.00%) 7 (7.00%) high mild 6 (6.00%) high severe async-pool/no-hook/wasm-to-host - nop - typed time: [6.5626 ns 6.5733 ns 6.5851 ns] change: [+40.561% +42.307% +44.514%] (p = 0.00 < 0.05) Performance has regressed. Found 9 outliers among 100 measurements (9.00%) 5 (5.00%) high mild 4 (4.00%) high severe async-pool/no-hook/wasm-to-host - nop-params-and-results - typed time: [15.820 ns 15.886 ns 15.969 ns] change: [+4.1044% +5.7928% +7.7122%] (p = 0.00 < 0.05) Performance has regressed. Found 17 outliers among 100 measurements (17.00%) 4 (4.00%) high mild 13 (13.00%) high severe async-pool/no-hook/wasm-to-host - nop - untyped time: [20.481 ns 20.521 ns 20.566 ns] change: [+6.7962% +7.6950% +8.7612%] (p = 0.00 < 0.05) Performance has regressed. Found 11 outliers among 100 measurements (11.00%) 6 (6.00%) high mild 5 (5.00%) high severe async-pool/no-hook/wasm-to-host - nop-params-and-results - untyped time: [41.834 ns 41.998 ns 42.189 ns] change: [-3.8185% -2.2687% -0.7541%] (p = 0.01 < 0.05) Change within noise threshold. Found 13 outliers among 100 measurements (13.00%) 3 (3.00%) high mild 10 (10.00%) high severe async-pool/no-hook/wasm-to-host - nop - unchecked time: [10.353 ns 10.380 ns 10.414 ns] change: [+82.042% +84.591% +87.205%] (p = 0.00 < 0.05) Performance has regressed. Found 7 outliers among 100 measurements (7.00%) 4 (4.00%) high mild 3 (3.00%) high severe async-pool/no-hook/wasm-to-host - nop-params-and-results - unchecked time: [11.123 ns 11.168 ns 11.228 ns] change: [-30.813% -29.285% -27.874%] (p = 0.00 < 0.05) Performance has improved. Found 12 outliers among 100 measurements (12.00%) 11 (11.00%) high mild 1 (1.00%) high severe async-pool/no-hook/wasm-to-host - nop - async-typed time: [27.442 ns 27.528 ns 27.638 ns] change: [+7.5215% +9.9795% +12.266%] (p = 0.00 < 0.05) Performance has regressed. Found 18 outliers among 100 measurements (18.00%) 3 (3.00%) high mild 15 (15.00%) high severe async-pool/no-hook/wasm-to-host - nop-params-and-results - async-typed time: [29.014 ns 29.148 ns 29.312 ns] change: [+2.0227% +3.4722% +4.9047%] (p = 0.00 < 0.05) Performance has regressed. Found 7 outliers among 100 measurements (7.00%) 6 (6.00%) high mild 1 (1.00%) high severe async-pool/hook-sync/wasm-to-host - nop - typed time: [6.7916 ns 6.8116 ns 6.8325 ns] change: [+20.937% +22.050% +23.281%] (p = 0.00 < 0.05) Performance has regressed. Found 11 outliers among 100 measurements (11.00%) 5 (5.00%) high mild 6 (6.00%) high severe async-pool/hook-sync/wasm-to-host - nop-params-and-results - typed time: [15.917 ns 15.975 ns 16.051 ns] change: [+4.6404% +6.4217% +8.3075%] (p = 0.00 < 0.05) Performance has regressed. Found 16 outliers among 100 measurements (16.00%) 5 (5.00%) high mild 11 (11.00%) high severe async-pool/hook-sync/wasm-to-host - nop - untyped time: [21.558 ns 21.612 ns 21.679 ns] change: [+8.1158% +9.1409% +10.217%] (p = 0.00 < 0.05) Performance has regressed. Found 9 outliers among 100 measurements (9.00%) 2 (2.00%) high mild 7 (7.00%) high severe async-pool/hook-sync/wasm-to-host - nop-params-and-results - untyped time: [42.475 ns 42.614 ns 42.775 ns] change: [-6.3613% -4.4709% -2.7647%] (p = 0.00 < 0.05) Performance has improved. Found 18 outliers among 100 measurements (18.00%) 3 (3.00%) high mild 15 (15.00%) high severe async-pool/hook-sync/wasm-to-host - nop - unchecked time: [11.150 ns 11.195 ns 11.247 ns] change: [+74.424% +77.056% +79.811%] (p = 0.00 < 0.05) Performance has regressed. Found 14 outliers among 100 measurements (14.00%) 3 (3.00%) high mild 11 (11.00%) high severe async-pool/hook-sync/wasm-to-host - nop-params-and-results - unchecked time: [11.639 ns 11.695 ns 11.760 ns] change: [-30.212% -29.023% -27.954%] (p = 0.00 < 0.05) Performance has improved. Found 15 outliers among 100 measurements (15.00%) 7 (7.00%) high mild 8 (8.00%) high severe async-pool/hook-sync/wasm-to-host - nop - async-typed time: [27.480 ns 27.712 ns 27.984 ns] change: [+2.9764% +6.5061% +9.8914%] (p = 0.00 < 0.05) Performance has regressed. Found 8 outliers among 100 measurements (8.00%) 6 (6.00%) high mild 2 (2.00%) high severe async-pool/hook-sync/wasm-to-host - nop-params-and-results - async-typed time: [29.218 ns 29.380 ns 29.600 ns] change: [+5.2283% +7.7247% +10.822%] (p = 0.00 < 0.05) Performance has regressed. Found 16 outliers among 100 measurements (16.00%) 2 (2.00%) high mild 14 (14.00%) high severe ``` </details> * Add s390x support for frame pointer-based stack walking * wasmtime: Allow `Caller::get_export` to get all exports * fuzzing: Add a fuzz target to check that our stack traces are correct We generate Wasm modules that keep track of their own stack as they call and return between functions, and then we periodically check that if the host captures a backtrace, it matches what the Wasm module has recorded. * Remove VM offsets for `VMHostFuncContext` since it isn't used by JIT code * Add doc comment with stack walking implementation notes * Document the extra state that can be passed to `wasmtime_runtime::Backtrace` methods * Add extensive comments for stack walking function * Factor architecture-specific bits of stack walking out into modules * Initialize store-related fields in a vmctx to null when there is no store yet Rather than leaving them as uninitialized data. * Use `set_callee` instead of manually setting the vmctx field * Use a more informative compile error message for unsupported architectures * Document unsafety of `prepare_host_to_wasm_trampoline` * Use `bti c` instead of `hint #34` in inline aarch64 assembly * Remove outdated TODO comment * Remove setting of `last_wasm_exit_fp` in `set_jit_trap` This is no longer needed as the value is plumbed through to the backtrace code directly now. * Only set the stack limit once, in the face of re-entrancy into Wasm * Add comments for s390x-specific stack walking bits * Use the helper macro for all libcalls If we forget to use it, and then trigger a GC from the libcall, that means we could miss stack frames when walking the stack, fail to find live GC refs, and then get use after free bugs. Much less risky to always use the helper macro that takes care of all of that for us. * Use the `asm_sym!` macro in Wasm-to-libcall trampolines This macro handles the macOS-specific underscore prefix stuff for us. * wasmtime: add size and align to `externref` assertion error message * Extend the `stacks` fuzzer to have host frames in between Wasm frames This way we get one or more contiguous sequences of Wasm frames on the stack, instead of exactly one. * Add documentation for aarch64-specific backtrace helpers * Clarify that we only support little-endian aarch64 in trampoline comment * Use `.machine z13` in s390x assembly file Since apparently our CI machines have pretty old assemblers that don't have `.machine z14`. This should be fine though since these trampolines don't make use of anything that is introduced in z14. * Fix aarch64 build * Fix macOS build * Document the `asm_sym!` macro * Add windows support to the `wasmtime-asm-macros` crate * Add windows support to host<--->Wasm trampolines * Fix trap handler build on windows * Run `rustfmt` on s390x trampoline source file * Temporarily disable some assertions about a trap's backtrace in the component model tests Follow up to re-enable this and fix the associated issue: https://github.com/bytecodealliance/wasmtime/issues/4535 * Refactor libcall definitions with less macros This refactors the `libcall!` macro to use the `foreach_builtin_function!` macro to define all of the trampolines. Additionally the macro surrounding each libcall itself is no longer necessary and helps avoid too many macros. * Use `VMOpaqueContext::from_vm_host_func_context` in `VMHostFuncContext::new` * Move `backtrace` module to be submodule of `traphandlers` This avoids making some things `pub(crate)` in `traphandlers` that really shouldn't be. * Fix macOS aarch64 build * Use "i64" instead of "word" in aarch64-specific file * Save/restore entry SP and exit FP/return pointer in the face of panicking imported host functions Also clean up assertions surrounding our saved entry/exit registers. * Put "typed" vs "untyped" in the same position of call benchmark names Regardless if we are doing wasm-to-host or host-to-wasm * Fix stacks test case generator build for new `wasm-encoder` * Fix build for s390x * Expand libcalls in s390x asm * Disable more parts of component tests now that backtrace assertions are a bit tighter * Remove assertion that can maybe fail on s390x Co-authored-by: Ulrich Weigand <ulrich.weigand@de.ibm.com> Co-authored-by: Alex Crichton <alex@alexcrichton.com>
2 years ago
"wasmtime-asm-macros",
"wasmtime-environ",
"wasmtime-fiber",
"wasmtime-jit-debug",
"windows-sys",
]
Remove `wasmtime-environ`'s dependency on `cranelift-codegen` (#3199) * Move `CompiledFunction` into wasmtime-cranelift This commit moves the `wasmtime_environ::CompiledFunction` type into the `wasmtime-cranelift` crate. This type has lots of Cranelift-specific pieces of compilation and doesn't need to be generated by all Wasmtime compilers. This replaces the usage in the `Compiler` trait with a `Box<Any>` type that each compiler can select. Each compiler must still produce a `FunctionInfo`, however, which is shared information we'll deserialize for each module. The `wasmtime-debug` crate is also folded into the `wasmtime-cranelift` crate as a result of this commit. One possibility was to move the `CompiledFunction` commit into its own crate and have `wasmtime-debug` depend on that, but since `wasmtime-debug` is Cranelift-specific at this time it didn't seem like it was too too necessary to keep it separate. If `wasmtime-debug` supports other backends in the future we can recreate a new crate, perhaps with it refactored to not depend on Cranelift. * Move wasmtime_environ::reference_type This now belongs in wasmtime-cranelift and nowhere else * Remove `Type` reexport in wasmtime-environ One less dependency on `cranelift-codegen`! * Remove `types` reexport from `wasmtime-environ` Less cranelift! * Remove `SourceLoc` from wasmtime-environ Change the `srcloc`, `start_srcloc`, and `end_srcloc` fields to a custom `FilePos` type instead of `ir::SourceLoc`. These are only used in a few places so there's not much to lose from an extra abstraction for these leaf use cases outside of cranelift. * Remove wasmtime-environ's dep on cranelift's `StackMap` This commit "clones" the `StackMap` data structure in to `wasmtime-environ` to have an independent representation that that chosen by Cranelift. This allows Wasmtime to decouple this runtime dependency of stack map information and let the two evolve independently, if necessary. An alternative would be to refactor cranelift's implementation into a separate crate and have wasmtime depend on that but it seemed a bit like overkill to do so and easier to clone just a few lines for this. * Define code offsets in wasmtime-environ with `u32` Don't use Cranelift's `binemit::CodeOffset` alias to define this field type since the `wasmtime-environ` crate will be losing the `cranelift-codegen` dependency soon. * Commit to using `cranelift-entity` in Wasmtime This commit removes the reexport of `cranelift-entity` from the `wasmtime-environ` crate and instead directly depends on the `cranelift-entity` crate in all referencing crates. The original reason for the reexport was to make cranelift version bumps easier since it's less versions to change, but nowadays we have a script to do that. Otherwise this encourages crates to use whatever they want from `cranelift-entity` since we'll always depend on the whole crate. It's expected that the `cranelift-entity` crate will continue to be a lean crate in dependencies and suitable for use at both runtime and compile time. Consequently there's no need to avoid its usage in Wasmtime at runtime, since "remove Cranelift at compile time" is primarily about the `cranelift-codegen` crate. * Remove most uses of `cranelift-codegen` in `wasmtime-environ` There's only one final use remaining, which is the reexport of `TrapCode`, which will get handled later. * Limit the glob-reexport of `cranelift_wasm` This commit removes the glob reexport of `cranelift-wasm` from the `wasmtime-environ` crate. This is intended to explicitly define what we're reexporting and is a transitionary step to curtail the amount of dependencies taken on `cranelift-wasm` throughout the codebase. For example some functions used by debuginfo mapping are better imported directly from the crate since they're Cranelift-specific. Note that this is intended to be a temporary state affairs, soon this reexport will be gone entirely. Additionally this commit reduces imports from `cranelift_wasm` and also primarily imports from `crate::wasm` within `wasmtime-environ` to get a better sense of what's imported from where and what will need to be shared. * Extract types from cranelift-wasm to cranelift-wasm-types This commit creates a new crate called `cranelift-wasm-types` and extracts type definitions from the `cranelift-wasm` crate into this new crate. The purpose of this crate is to be a shared definition of wasm types that can be shared both by compilers (like Cranelift) as well as wasm runtimes (e.g. Wasmtime). This new `cranelift-wasm-types` crate doesn't depend on `cranelift-codegen` and is the final step in severing the unconditional dependency from Wasmtime to `cranelift-codegen`. The final refactoring in this commit is to then reexport this crate from `wasmtime-environ`, delete the `cranelift-codegen` dependency, and then update all `use` paths to point to these new types. The main change of substance here is that the `TrapCode` enum is mirrored from Cranelift into this `cranelift-wasm-types` crate. While this unfortunately results in three definitions (one more which is non-exhaustive in Wasmtime itself) it's hopefully not too onerous and ideally something we can patch up in the future. * Get lightbeam compiling * Remove unnecessary dependency * Fix compile with uffd * Update publish script * Fix more uffd tests * Rename cranelift-wasm-types to wasmtime-types This reflects the purpose a bit more where it's types specifically intended for Wasmtime and its support. * Fix publish script
3 years ago
[[package]]
name = "wasmtime-types"
version = "9.0.0"
Remove `wasmtime-environ`'s dependency on `cranelift-codegen` (#3199) * Move `CompiledFunction` into wasmtime-cranelift This commit moves the `wasmtime_environ::CompiledFunction` type into the `wasmtime-cranelift` crate. This type has lots of Cranelift-specific pieces of compilation and doesn't need to be generated by all Wasmtime compilers. This replaces the usage in the `Compiler` trait with a `Box<Any>` type that each compiler can select. Each compiler must still produce a `FunctionInfo`, however, which is shared information we'll deserialize for each module. The `wasmtime-debug` crate is also folded into the `wasmtime-cranelift` crate as a result of this commit. One possibility was to move the `CompiledFunction` commit into its own crate and have `wasmtime-debug` depend on that, but since `wasmtime-debug` is Cranelift-specific at this time it didn't seem like it was too too necessary to keep it separate. If `wasmtime-debug` supports other backends in the future we can recreate a new crate, perhaps with it refactored to not depend on Cranelift. * Move wasmtime_environ::reference_type This now belongs in wasmtime-cranelift and nowhere else * Remove `Type` reexport in wasmtime-environ One less dependency on `cranelift-codegen`! * Remove `types` reexport from `wasmtime-environ` Less cranelift! * Remove `SourceLoc` from wasmtime-environ Change the `srcloc`, `start_srcloc`, and `end_srcloc` fields to a custom `FilePos` type instead of `ir::SourceLoc`. These are only used in a few places so there's not much to lose from an extra abstraction for these leaf use cases outside of cranelift. * Remove wasmtime-environ's dep on cranelift's `StackMap` This commit "clones" the `StackMap` data structure in to `wasmtime-environ` to have an independent representation that that chosen by Cranelift. This allows Wasmtime to decouple this runtime dependency of stack map information and let the two evolve independently, if necessary. An alternative would be to refactor cranelift's implementation into a separate crate and have wasmtime depend on that but it seemed a bit like overkill to do so and easier to clone just a few lines for this. * Define code offsets in wasmtime-environ with `u32` Don't use Cranelift's `binemit::CodeOffset` alias to define this field type since the `wasmtime-environ` crate will be losing the `cranelift-codegen` dependency soon. * Commit to using `cranelift-entity` in Wasmtime This commit removes the reexport of `cranelift-entity` from the `wasmtime-environ` crate and instead directly depends on the `cranelift-entity` crate in all referencing crates. The original reason for the reexport was to make cranelift version bumps easier since it's less versions to change, but nowadays we have a script to do that. Otherwise this encourages crates to use whatever they want from `cranelift-entity` since we'll always depend on the whole crate. It's expected that the `cranelift-entity` crate will continue to be a lean crate in dependencies and suitable for use at both runtime and compile time. Consequently there's no need to avoid its usage in Wasmtime at runtime, since "remove Cranelift at compile time" is primarily about the `cranelift-codegen` crate. * Remove most uses of `cranelift-codegen` in `wasmtime-environ` There's only one final use remaining, which is the reexport of `TrapCode`, which will get handled later. * Limit the glob-reexport of `cranelift_wasm` This commit removes the glob reexport of `cranelift-wasm` from the `wasmtime-environ` crate. This is intended to explicitly define what we're reexporting and is a transitionary step to curtail the amount of dependencies taken on `cranelift-wasm` throughout the codebase. For example some functions used by debuginfo mapping are better imported directly from the crate since they're Cranelift-specific. Note that this is intended to be a temporary state affairs, soon this reexport will be gone entirely. Additionally this commit reduces imports from `cranelift_wasm` and also primarily imports from `crate::wasm` within `wasmtime-environ` to get a better sense of what's imported from where and what will need to be shared. * Extract types from cranelift-wasm to cranelift-wasm-types This commit creates a new crate called `cranelift-wasm-types` and extracts type definitions from the `cranelift-wasm` crate into this new crate. The purpose of this crate is to be a shared definition of wasm types that can be shared both by compilers (like Cranelift) as well as wasm runtimes (e.g. Wasmtime). This new `cranelift-wasm-types` crate doesn't depend on `cranelift-codegen` and is the final step in severing the unconditional dependency from Wasmtime to `cranelift-codegen`. The final refactoring in this commit is to then reexport this crate from `wasmtime-environ`, delete the `cranelift-codegen` dependency, and then update all `use` paths to point to these new types. The main change of substance here is that the `TrapCode` enum is mirrored from Cranelift into this `cranelift-wasm-types` crate. While this unfortunately results in three definitions (one more which is non-exhaustive in Wasmtime itself) it's hopefully not too onerous and ideally something we can patch up in the future. * Get lightbeam compiling * Remove unnecessary dependency * Fix compile with uffd * Update publish script * Fix more uffd tests * Rename cranelift-wasm-types to wasmtime-types This reflects the purpose a bit more where it's types specifically intended for Wasmtime and its support. * Fix publish script
3 years ago
dependencies = [
"cranelift-entity",
"serde",
"thiserror",
"wasmparser",
]
[[package]]
name = "wasmtime-wasi"
version = "9.0.0"
dependencies = [
Reimplement `wasmtime-wasi` on top of `wasmtime` (#899) * Reimplement `wasmtime-wasi` on top of `wasmtime` This commit reimplements the `wasmtime-wasi` crate on top of the `wasmtime` API crate, instead of being placed on top of the `wasmtime-*` family of internal crates. The purpose here is to continue to exercise the API as well as avoid usage of internals wherever possible and instead use the safe API as much as possible. The `wasmtime-wasi` crate's API has been updated as part of this PR as well. The general outline of it is now: * Each module snapshot has a `WasiCtxBuilder`, `WasiCtx`, and `Wasi` type. * The `WasiCtx*` types are reexported from `wasi-common`. * The `Wasi` type is synthesized by the `wig` crate's procedural macro * The `Wasi` type exposes one constructor which takes a `Store` and a `WasiCtx`, and produces a `Wasi` * Each `Wasi` struct fields for all the exported functions in that wasi module. They're all public an they all have type `wasmtime::Func` * The `Wasi` type has a `get_export` method to fetch an struct field by name. The intention here is that we can continue to make progress on #727 by integrating WASI construction into the `Instance::new` experience, but it requires everything to be part of the same system! The main oddity required by the `wasmtime-wasi` crate is that it needs access to the caller's `memory` export, if any. This is currently done with a bit of a hack and is expected to go away once interface types are more fully baked in. * Remove now no-longer-necessary APIs from `wasmtime` * rustfmt * Rename to from_abi
5 years ago
"anyhow",
wasi-threads: an initial implementation (#5484) This commit includes a set of changes that add initial support for `wasi-threads` to Wasmtime: * feat: remove mutability from the WasiCtx Table This patch adds interior mutability to the WasiCtx Table and the Table elements. Major pain points: * `File` only needs `RwLock<cap_std::fs::File>` to implement `File::set_fdflags()` on Windows, because of [1] * Because `File` needs a `RwLock` and `RwLock*Guard` cannot be hold across an `.await`, The `async` from `async fn num_ready_bytes(&self)` had to be removed * Because `File` needs a `RwLock` and `RwLock*Guard` cannot be dereferenced in `pollable`, the signature of `fn pollable(&self) -> Option<rustix::fd::BorrowedFd>` changed to `fn pollable(&self) -> Option<Arc<dyn AsFd + '_>>` [1] https://github.com/bytecodealliance/system-interface/blob/da238e324e752033f315f09c082ad9ce35d42696/src/fs/fd_flags.rs#L210-L217 * wasi-threads: add an initial implementation This change is a first step toward implementing `wasi-threads` in Wasmtime. We may find that it has some missing pieces, but the core functionality is there: when `wasi::thread_spawn` is called by a running WebAssembly module, a function named `wasi_thread_start` is found in the module's exports and called in a new instance. The shared memory of the original instance is reused in the new instance. This new WASI proposal is in its early stages and details are still being hashed out in the [spec] and [wasi-libc] repositories. Due to its experimental state, the `wasi-threads` functionality is hidden behind both a compile-time and runtime flag: one must build with `--features wasi-threads` but also run the Wasmtime CLI with `--wasm-features threads` and `--wasi-modules experimental-wasi-threads`. One can experiment with `wasi-threads` by running: ```console $ cargo run --features wasi-threads -- \ --wasm-features threads --wasi-modules experimental-wasi-threads \ <a threads-enabled module> ``` Threads-enabled Wasm modules are not yet easy to build. Hopefully this is resolved soon, but in the meantime see the use of `THREAD_MODEL=posix` in the [wasi-libc] repository for some clues on what is necessary. Wiggle complicates things by requiring the Wasm memory to be exported with a certain name and `wasi-threads` also expects that memory to be imported; this build-time obstacle can be overcome with the `--import-memory --export-memory` flags only available in the latest Clang tree. Due to all of this, the included tests are written directly in WAT--run these with: ```console $ cargo test --features wasi-threads -p wasmtime-cli -- cli_tests ``` [spec]: https://github.com/WebAssembly/wasi-threads [wasi-libc]: https://github.com/WebAssembly/wasi-libc This change does not protect the WASI implementations themselves from concurrent access. This is already complete in previous commits or left for future commits in certain cases (e.g., wasi-nn). * wasi-threads: factor out process exit logic As is being discussed [elsewhere], either calling `proc_exit` or trapping in any thread should halt execution of all threads. The Wasmtime CLI already has logic for adapting a WebAssembly error code to a code expected in each OS. This change factors out this logic to a new function, `maybe_exit_on_error`, for use within the `wasi-threads` implementation. This will work reasonably well for CLI users of Wasmtime + `wasi-threads`, but embedders will want something better in the future: when a `wasi-threads` threads fails, they may not want their application to exit. Handling this is tricky, because it will require cancelling the threads spawned by the `wasi-threads` implementation, something that is not trivial to do in Rust. With this change, we defer that work until later in order to provide a working implementation of `wasi-threads` for experimentation. [elsewhere]: https://github.com/WebAssembly/wasi-threads/pull/17 * review: work around `fd_fdstat_set_flags` In order to make progress with wasi-threads, this change temporarily works around limitations induced by `wasi-common`'s `fd_fdstat_set_flags` to allow `&mut self` use in the implementation. Eventual resolution is tracked in https://github.com/bytecodealliance/wasmtime/issues/5643. This change makes several related helper functions (e.g., `set_fdflags`) take `&mut self` as well. * test: use `wait`/`notify` to improve `threads.wat` test Previously, the test simply executed in a loop for some hardcoded number of iterations. This changes uses `wait` and `notify` and atomic operations to keep track of when the spawned threads are done and join on the main thread appropriately. * various fixes and tweaks due to the PR review --------- Signed-off-by: Harald Hoyer <harald@profian.com> Co-authored-by: Harald Hoyer <harald@profian.com> Co-authored-by: Alex Crichton <alex@alexcrichton.com>
2 years ago
"libc",
"wasi-cap-std-sync",
"wasi-common",
"wasi-tokio",
"wasmtime",
"wiggle",
]
[[package]]
name = "wasmtime-wasi-crypto"
version = "9.0.0"
dependencies = [
"anyhow",
"wasi-crypto",
"wasmtime",
"wiggle",
]
[[package]]
name = "wasmtime-wasi-nn"
version = "9.0.0"
dependencies = [
"anyhow",
"openvino",
"thiserror",
"walkdir",
"wiggle",
]
[[package]]
wasi-threads: an initial implementation (#5484) This commit includes a set of changes that add initial support for `wasi-threads` to Wasmtime: * feat: remove mutability from the WasiCtx Table This patch adds interior mutability to the WasiCtx Table and the Table elements. Major pain points: * `File` only needs `RwLock<cap_std::fs::File>` to implement `File::set_fdflags()` on Windows, because of [1] * Because `File` needs a `RwLock` and `RwLock*Guard` cannot be hold across an `.await`, The `async` from `async fn num_ready_bytes(&self)` had to be removed * Because `File` needs a `RwLock` and `RwLock*Guard` cannot be dereferenced in `pollable`, the signature of `fn pollable(&self) -> Option<rustix::fd::BorrowedFd>` changed to `fn pollable(&self) -> Option<Arc<dyn AsFd + '_>>` [1] https://github.com/bytecodealliance/system-interface/blob/da238e324e752033f315f09c082ad9ce35d42696/src/fs/fd_flags.rs#L210-L217 * wasi-threads: add an initial implementation This change is a first step toward implementing `wasi-threads` in Wasmtime. We may find that it has some missing pieces, but the core functionality is there: when `wasi::thread_spawn` is called by a running WebAssembly module, a function named `wasi_thread_start` is found in the module's exports and called in a new instance. The shared memory of the original instance is reused in the new instance. This new WASI proposal is in its early stages and details are still being hashed out in the [spec] and [wasi-libc] repositories. Due to its experimental state, the `wasi-threads` functionality is hidden behind both a compile-time and runtime flag: one must build with `--features wasi-threads` but also run the Wasmtime CLI with `--wasm-features threads` and `--wasi-modules experimental-wasi-threads`. One can experiment with `wasi-threads` by running: ```console $ cargo run --features wasi-threads -- \ --wasm-features threads --wasi-modules experimental-wasi-threads \ <a threads-enabled module> ``` Threads-enabled Wasm modules are not yet easy to build. Hopefully this is resolved soon, but in the meantime see the use of `THREAD_MODEL=posix` in the [wasi-libc] repository for some clues on what is necessary. Wiggle complicates things by requiring the Wasm memory to be exported with a certain name and `wasi-threads` also expects that memory to be imported; this build-time obstacle can be overcome with the `--import-memory --export-memory` flags only available in the latest Clang tree. Due to all of this, the included tests are written directly in WAT--run these with: ```console $ cargo test --features wasi-threads -p wasmtime-cli -- cli_tests ``` [spec]: https://github.com/WebAssembly/wasi-threads [wasi-libc]: https://github.com/WebAssembly/wasi-libc This change does not protect the WASI implementations themselves from concurrent access. This is already complete in previous commits or left for future commits in certain cases (e.g., wasi-nn). * wasi-threads: factor out process exit logic As is being discussed [elsewhere], either calling `proc_exit` or trapping in any thread should halt execution of all threads. The Wasmtime CLI already has logic for adapting a WebAssembly error code to a code expected in each OS. This change factors out this logic to a new function, `maybe_exit_on_error`, for use within the `wasi-threads` implementation. This will work reasonably well for CLI users of Wasmtime + `wasi-threads`, but embedders will want something better in the future: when a `wasi-threads` threads fails, they may not want their application to exit. Handling this is tricky, because it will require cancelling the threads spawned by the `wasi-threads` implementation, something that is not trivial to do in Rust. With this change, we defer that work until later in order to provide a working implementation of `wasi-threads` for experimentation. [elsewhere]: https://github.com/WebAssembly/wasi-threads/pull/17 * review: work around `fd_fdstat_set_flags` In order to make progress with wasi-threads, this change temporarily works around limitations induced by `wasi-common`'s `fd_fdstat_set_flags` to allow `&mut self` use in the implementation. Eventual resolution is tracked in https://github.com/bytecodealliance/wasmtime/issues/5643. This change makes several related helper functions (e.g., `set_fdflags`) take `&mut self` as well. * test: use `wait`/`notify` to improve `threads.wat` test Previously, the test simply executed in a loop for some hardcoded number of iterations. This changes uses `wait` and `notify` and atomic operations to keep track of when the spawned threads are done and join on the main thread appropriately. * various fixes and tweaks due to the PR review --------- Signed-off-by: Harald Hoyer <harald@profian.com> Co-authored-by: Harald Hoyer <harald@profian.com> Co-authored-by: Alex Crichton <alex@alexcrichton.com>
2 years ago
name = "wasmtime-wasi-threads"
version = "9.0.0"
wasi-threads: an initial implementation (#5484) This commit includes a set of changes that add initial support for `wasi-threads` to Wasmtime: * feat: remove mutability from the WasiCtx Table This patch adds interior mutability to the WasiCtx Table and the Table elements. Major pain points: * `File` only needs `RwLock<cap_std::fs::File>` to implement `File::set_fdflags()` on Windows, because of [1] * Because `File` needs a `RwLock` and `RwLock*Guard` cannot be hold across an `.await`, The `async` from `async fn num_ready_bytes(&self)` had to be removed * Because `File` needs a `RwLock` and `RwLock*Guard` cannot be dereferenced in `pollable`, the signature of `fn pollable(&self) -> Option<rustix::fd::BorrowedFd>` changed to `fn pollable(&self) -> Option<Arc<dyn AsFd + '_>>` [1] https://github.com/bytecodealliance/system-interface/blob/da238e324e752033f315f09c082ad9ce35d42696/src/fs/fd_flags.rs#L210-L217 * wasi-threads: add an initial implementation This change is a first step toward implementing `wasi-threads` in Wasmtime. We may find that it has some missing pieces, but the core functionality is there: when `wasi::thread_spawn` is called by a running WebAssembly module, a function named `wasi_thread_start` is found in the module's exports and called in a new instance. The shared memory of the original instance is reused in the new instance. This new WASI proposal is in its early stages and details are still being hashed out in the [spec] and [wasi-libc] repositories. Due to its experimental state, the `wasi-threads` functionality is hidden behind both a compile-time and runtime flag: one must build with `--features wasi-threads` but also run the Wasmtime CLI with `--wasm-features threads` and `--wasi-modules experimental-wasi-threads`. One can experiment with `wasi-threads` by running: ```console $ cargo run --features wasi-threads -- \ --wasm-features threads --wasi-modules experimental-wasi-threads \ <a threads-enabled module> ``` Threads-enabled Wasm modules are not yet easy to build. Hopefully this is resolved soon, but in the meantime see the use of `THREAD_MODEL=posix` in the [wasi-libc] repository for some clues on what is necessary. Wiggle complicates things by requiring the Wasm memory to be exported with a certain name and `wasi-threads` also expects that memory to be imported; this build-time obstacle can be overcome with the `--import-memory --export-memory` flags only available in the latest Clang tree. Due to all of this, the included tests are written directly in WAT--run these with: ```console $ cargo test --features wasi-threads -p wasmtime-cli -- cli_tests ``` [spec]: https://github.com/WebAssembly/wasi-threads [wasi-libc]: https://github.com/WebAssembly/wasi-libc This change does not protect the WASI implementations themselves from concurrent access. This is already complete in previous commits or left for future commits in certain cases (e.g., wasi-nn). * wasi-threads: factor out process exit logic As is being discussed [elsewhere], either calling `proc_exit` or trapping in any thread should halt execution of all threads. The Wasmtime CLI already has logic for adapting a WebAssembly error code to a code expected in each OS. This change factors out this logic to a new function, `maybe_exit_on_error`, for use within the `wasi-threads` implementation. This will work reasonably well for CLI users of Wasmtime + `wasi-threads`, but embedders will want something better in the future: when a `wasi-threads` threads fails, they may not want their application to exit. Handling this is tricky, because it will require cancelling the threads spawned by the `wasi-threads` implementation, something that is not trivial to do in Rust. With this change, we defer that work until later in order to provide a working implementation of `wasi-threads` for experimentation. [elsewhere]: https://github.com/WebAssembly/wasi-threads/pull/17 * review: work around `fd_fdstat_set_flags` In order to make progress with wasi-threads, this change temporarily works around limitations induced by `wasi-common`'s `fd_fdstat_set_flags` to allow `&mut self` use in the implementation. Eventual resolution is tracked in https://github.com/bytecodealliance/wasmtime/issues/5643. This change makes several related helper functions (e.g., `set_fdflags`) take `&mut self` as well. * test: use `wait`/`notify` to improve `threads.wat` test Previously, the test simply executed in a loop for some hardcoded number of iterations. This changes uses `wait` and `notify` and atomic operations to keep track of when the spawned threads are done and join on the main thread appropriately. * various fixes and tweaks due to the PR review --------- Signed-off-by: Harald Hoyer <harald@profian.com> Co-authored-by: Harald Hoyer <harald@profian.com> Co-authored-by: Alex Crichton <alex@alexcrichton.com>
2 years ago
dependencies = [
"anyhow",
"log",
"rand 0.8.5",
"wasi-common",
"wasmtime",
"wasmtime-wasi",
]
[[package]]
name = "wasmtime-wast"
version = "9.0.0"
dependencies = [
"anyhow",
Add a dataflow-based representation of components (#4597) * Add a dataflow-based representation of components This commit updates the inlining phase of compiling a component to creating a dataflow-based representation of a component instead of creating a final `Component` with a linear list of initializers. This dataflow graph is then linearized in a final step to create the actual final `Component`. The motivation for this commit stems primarily from my work implementing strings in fused adapters. In doing this my plan is to defer most low-level transcoding to the host itself rather than implementing that in the core wasm adapter modules. This means that small cranelift-generated trampolines will be used for adapter modules to call which then call "transcoding libcalls". The cranelift-generated trampolines will get raw pointers into linear memory and pass those to the libcall which core wasm doesn't have access to when passing arguments to an import. Implementing this with the previous representation of a `Component` was becoming too tricky to bear. The initialization of a transcoder needed to happen at just the right time: before the adapter module which needed it was instantiated but after the linear memories referenced had been extracted into the `VMComponentContext`. The difficulty here is further compounded by the current adapter module injection pass already being quite complicated. Adapter modules are already renumbering the index space of runtime instances and shuffling items around in the `GlobalInitializer` list. Perhaps the worst part of this was that memories could already be referenced by host function imports or exports to the host, and if adapters referenced the same memory it shouldn't be referenced twice in the component. This meant that `ExtractMemory` initializers ideally needed to be shuffled around in the initializer list to happen as early as possible instead of wherever they happened to show up during translation. Overall I did my best to implement the transcoders but everything always came up short. I have decided to throw my hands up in the air and try a completely different approach to this, namely the dataflow-based representation in this commit. This makes it much easier to edit the component after initial translation for injection of adapters, injection of transcoders, adding dependencies on possibly-already-existing items, etc. The adapter module partitioning pass in this commit was greatly simplified to something which I believe is functionally equivalent but is probably an order of magnitude easier to understand. The biggest downside of this representation I believe is having a duplicate representation of a component. The `component::info` was largely duplicated into the `component::dfg` module in this commit. Personally though I think this is a more appropriate tradeoff than before because it's very easy to reason about "convert representation A to B" code whereas it was very difficult to reason about shuffling around `GlobalInitializer` items in optimal fashions. This may also have a cost at compile-time in terms of shuffling data around, but my hope is that we have lots of other low-hanging fruit to optimize if it ever comes to that which allows keeping this easier-to-understand representation. Finally, to reiterate, the final representation of components is not changed by this PR. To the runtime internals everything is still the same. * Fix compile of factc
2 years ago
"log",
"wasmtime",
"wast 55.0.0",
]
Initial skeleton for Winch (#4907) * Initial skeleton for Winch This commit introduces the initial skeleton for Winch, the "baseline" compiler. This skeleton contains mostly setup code for the ISA, ABI, registers, and compilation environment abstractions. It also includes the calculation of function local slots. As of this commit, the structure of these abstractions looks like the following: +------------------------+ | v +----------+ +-----+ +-----------+-----+-----------------+ | Compiler | --> | ISA | --> | Registers | ABI | Compilation Env | +----------+ +-----+ +-----------+-----+-----------------+ | ^ +------------------------------+ * Compilation environment will hold a reference to the function data * Add basic documentation to the ABI trait * Enable x86 and arm64 in cranelift-codegen * Add reg_name function for x64 * Introduce the concept of a MacroAssembler and Assembler This commit introduces the concept of a MacroAsesembler and Assembler. The MacroAssembler trait will provide a high enough interface across architectures so that each ISA implementation can use their own low-level Assembler implementation to fulfill the interface. Each Assembler will provide a 1-1 mapping to each ISA instruction. As of this commit, only a partial debug implementation is provided for the x64 Assembler. * Add a newtype over PReg Adds a newtype `Reg` over regalloc2::PReg; this ensures that Winch will operate only on the concept of `Reg`. This change is temporary until we have the necessary machinery to share a common Reg abstraction via `cranelift_asm` * Improvements to local calcuation - Add `LocalSlot::addressed_from_sp` - Use `u32` for local slot and local sizes calculation * Add helper methods to ABIArg Adds helper methods to retrieve register and type information from the argument * Make locals_size public in frame * Improve x64 register naming depending on size * Add new methods to the masm interface This commit introduces the ability for the MacroAssembler to reserve stack space, get the address of a given local and perform a stack store based on the concept of `Operand`s. There are several motivating factors to introduce the concept of an Operand: - Make the translation between Winch and Cranelift easier; - Make dispatching from the MacroAssembler to the underlying Assembler - easier by minimizing the amount of functions that we need to define - in order to satisfy the store/load combinations This commit also introduces the concept of a memory address, which essentially describes the addressing modes; as of this commit only one addressing mode is supported. We'll also need to verify that this structure will play nicely with arm64. * Blank masm implementation for arm64 * Implementation of reserve_stack, local_address, store and fp_offset for x64 * Implement function prologue and argument register spilling * Add structopt and wat * Fix debug instruction formatting * Make TargetISA trait publicly accessible * Modify the MacroAssembler finalize siganture to return a slice of strings * Introduce a simple CLI for Winch To be able to compile Wasm programs with Winch independently. Mostly meant for testing / debugging * Fix bug in x64 assembler mov_rm * Remove unused import * Move the stack slot calculation to the Frame This commit moves the calculation of the stack slots to the frame handler abstraction and also includes the calculation of the limits for the function defined locals, which will be used to zero the locals that are not associated to function arguments * Add i32 and i64 constructors to local slots * Introduce the concept of DefinedLocalsRange This commit introduces `DefinedLocalsRange` to track the stack offset at which the function-defined locals start and end; this is later used to zero-out that stack region * Add constructors for int and float registers * Add a placeholder stack implementation * Add a regset abstraction to track register availability Adds a bit set abstraction to track register availability for register allocation. The bit set has no specific knowledge about physical registers, it works on the register's hardware encoding as the source of truth. Each RegSet is expected to be created with the universe of allocatable registers per ISA when starting the compilation of a particular function. * Add an abstraction over register and immediate This is meant to be used as the source for stores. * Add a way to zero local slots and an initial skeletion of regalloc This commit introduces `zero_local_slots` to the MacroAssembler; which ensures that function defined locals are zeroed out when starting the function body. The algorithm divides the defined function locals stack range into 8 byte slots and stores a zero at each address. This process relies on register allocation if the amount of slots that need to be initialized is greater than 1. In such case, the next available register is requested to the register set and it's used to store a 0, which is then stored at every local slot * Update to wasmparser 0.92 * Correctly track if the regset has registers available * Add a result entry to the ABI signature This commuit introduces ABIResult as part of the ABISignature; this struct will track how function results are stored; initially it will consiste of a single register that will be requested to the register allocator at the end of the function; potentially causing a spill * Move zero local slots and add more granular methods to the masm This commit removes zeroing local slots from the MacroAssembler and instead adds more granular methods to it (e.g `zero`, `add`). This allows for better code sharing since most of the work done by the algorithm for zeroing slots will be the same in all targets, except for the binary emissions pieces, which is what gets delegated to the masm * Use wasmparser's visitor API and add initial support for const and add This commit adds initial support for the I32Const and I32 instructions; this involves adding a minimum for register allocation. Note that some regalloc pieces are still incomplete, since for the current set of supported instructions they are not needed. * Make the ty field public in Local * Add scratch_reg to the abi * Add a method to get a particular local from the Frame * Split the compilation environment abstraction This commit splits the compilation environment into two more concise abstractions: 1. CodeGen: the main abstraction for code generation 2. CodeGenContext: abstraction that shares the common pieces for compilation; these pieces are shared between the code generator and the register allocator * Add `push` and `load` to the MacroAssembler * Remove dead code warnings for unused paths * Map ISA features to cranelift-codegen ISA features * Apply formatting * Fix Cargo.toml after a bad rebase * Add component-compiler feature * Use clap instead of structopt * Add winch to publish.rs script * Minor formatting * Add tests to RegSet and fix two bugs when freeing and checking for register availability * Add tests to Stack * Free source register after a non-constant i32 add * Improve comments - Remove unneeded comments - And improve some of the TODO items * Update default features * Drop the ABI generic param and pass the word_size information directly To avoid dealing with dead code warnings this commit passes the word size information directly, since it's the only piece of information needed from the ABI by Codegen until now * Remove dead code This piece of code will be put back once we start integrating Winch with Wasmtime * Remove unused enum variant This variant doesn't get constructed; it should be added back once a backend is added and not enabled by default or when Winch gets integrated into Wasmtime * Fix unused code in regset tests * Update spec testsuite * Switch the visitor pattern for a simpler operator match This commit removes the usage of wasmparser's visitor pattern and instead defaults to a simpler operator matching approach. This removes the complexity of having to define all the visitor trait functions at once. * Use wasmparser's Visitor trait with a different macro strategy This commit puts back wasmparser's Visitor trait, with a sigle; simpler macro, only used for unsupported operators. * Restructure Winch This commit restuructures Winch's parts. It divides the initial approach into three main crates: `winch-codegen`,`wasmtime-winch` and `winch-tools`. `wasmtime-winch` is reponsible for the Wasmtime-Winch integration. `winch-codegen` is solely responsible for code generation. `winch-tools` is CLI tool to compile Wasm programs, mainly for testing purposes. * Refactor zero local slots This commit moves the logic of zeroing local slots from the codegen module into a method with a default implementation in the MacroAssembler trait: `zero_mem_range`. The refactored implementation is very similar to the previous implementation with the only difference that it doesn't allocates a general-purpose register; it instead uses the register allocator to retrieve the scratch register and uses this register to unroll the series of zero stores. * Tie the codegen creation to the ISA ABI This commit makes the relationship between the ISA ABI and the codegen explicit. This allows us to pass down ABI-specific bit and pieces to the codegeneration. In this case the only concrete piece that we need is the ABI word size. * Mark winch as publishable directory * Revamp winch docs This commit ensures that all the code comments in Winch are compliant with the syle used in the rest of Wasmtime's codebase. It also imptoves, generally the quality of the comments in some modules. * Panic when using multi-value when the target is aarch64 Similar to x64, this commit ensures that the abi signature of the current function doesn't use multi-value returns * Document the usage of directives * Use endianness instead of endianess in the ISA trait * Introduce a three-argument form in the MacroAssembler This commit introduces the usage of three-argument form for the MacroAssembler interface. This allows for a natural mapping for architectures like aarch64. In the case of x64, the implementation can simply restrict the implementation asserting for equality in two of the arguments of defaulting to a differnt set of instructions. As of this commit, the implementation of `add` panics if the destination and the first source arguments are not equal; internally the x64 assembler implementation will ensure that all the allowed combinations of `add` are satisfied. The reason for panicking and not emitting a `mov` followed by an `add` for example is simply because register allocation happens right before calling `add`, which ensures any register-to-register moves, if needed. This implementation will evolve in the future and this panic will be lifted if needed. * Improve the documentation for the MacroAssembler. Documents the usage of three-arg form and the intention around the high-level interface. * Format comments in remaining modules * Clean up Cargo.toml for winch pieces This commit adds missing fields to each of Winch's Cargo.toml. * Use `ModuleTranslation::get_types()` to derive the function type * Assert that start range is always word-size aligned
2 years ago
[[package]]
name = "wasmtime-winch"
version = "9.0.0"
Initial skeleton for Winch (#4907) * Initial skeleton for Winch This commit introduces the initial skeleton for Winch, the "baseline" compiler. This skeleton contains mostly setup code for the ISA, ABI, registers, and compilation environment abstractions. It also includes the calculation of function local slots. As of this commit, the structure of these abstractions looks like the following: +------------------------+ | v +----------+ +-----+ +-----------+-----+-----------------+ | Compiler | --> | ISA | --> | Registers | ABI | Compilation Env | +----------+ +-----+ +-----------+-----+-----------------+ | ^ +------------------------------+ * Compilation environment will hold a reference to the function data * Add basic documentation to the ABI trait * Enable x86 and arm64 in cranelift-codegen * Add reg_name function for x64 * Introduce the concept of a MacroAssembler and Assembler This commit introduces the concept of a MacroAsesembler and Assembler. The MacroAssembler trait will provide a high enough interface across architectures so that each ISA implementation can use their own low-level Assembler implementation to fulfill the interface. Each Assembler will provide a 1-1 mapping to each ISA instruction. As of this commit, only a partial debug implementation is provided for the x64 Assembler. * Add a newtype over PReg Adds a newtype `Reg` over regalloc2::PReg; this ensures that Winch will operate only on the concept of `Reg`. This change is temporary until we have the necessary machinery to share a common Reg abstraction via `cranelift_asm` * Improvements to local calcuation - Add `LocalSlot::addressed_from_sp` - Use `u32` for local slot and local sizes calculation * Add helper methods to ABIArg Adds helper methods to retrieve register and type information from the argument * Make locals_size public in frame * Improve x64 register naming depending on size * Add new methods to the masm interface This commit introduces the ability for the MacroAssembler to reserve stack space, get the address of a given local and perform a stack store based on the concept of `Operand`s. There are several motivating factors to introduce the concept of an Operand: - Make the translation between Winch and Cranelift easier; - Make dispatching from the MacroAssembler to the underlying Assembler - easier by minimizing the amount of functions that we need to define - in order to satisfy the store/load combinations This commit also introduces the concept of a memory address, which essentially describes the addressing modes; as of this commit only one addressing mode is supported. We'll also need to verify that this structure will play nicely with arm64. * Blank masm implementation for arm64 * Implementation of reserve_stack, local_address, store and fp_offset for x64 * Implement function prologue and argument register spilling * Add structopt and wat * Fix debug instruction formatting * Make TargetISA trait publicly accessible * Modify the MacroAssembler finalize siganture to return a slice of strings * Introduce a simple CLI for Winch To be able to compile Wasm programs with Winch independently. Mostly meant for testing / debugging * Fix bug in x64 assembler mov_rm * Remove unused import * Move the stack slot calculation to the Frame This commit moves the calculation of the stack slots to the frame handler abstraction and also includes the calculation of the limits for the function defined locals, which will be used to zero the locals that are not associated to function arguments * Add i32 and i64 constructors to local slots * Introduce the concept of DefinedLocalsRange This commit introduces `DefinedLocalsRange` to track the stack offset at which the function-defined locals start and end; this is later used to zero-out that stack region * Add constructors for int and float registers * Add a placeholder stack implementation * Add a regset abstraction to track register availability Adds a bit set abstraction to track register availability for register allocation. The bit set has no specific knowledge about physical registers, it works on the register's hardware encoding as the source of truth. Each RegSet is expected to be created with the universe of allocatable registers per ISA when starting the compilation of a particular function. * Add an abstraction over register and immediate This is meant to be used as the source for stores. * Add a way to zero local slots and an initial skeletion of regalloc This commit introduces `zero_local_slots` to the MacroAssembler; which ensures that function defined locals are zeroed out when starting the function body. The algorithm divides the defined function locals stack range into 8 byte slots and stores a zero at each address. This process relies on register allocation if the amount of slots that need to be initialized is greater than 1. In such case, the next available register is requested to the register set and it's used to store a 0, which is then stored at every local slot * Update to wasmparser 0.92 * Correctly track if the regset has registers available * Add a result entry to the ABI signature This commuit introduces ABIResult as part of the ABISignature; this struct will track how function results are stored; initially it will consiste of a single register that will be requested to the register allocator at the end of the function; potentially causing a spill * Move zero local slots and add more granular methods to the masm This commit removes zeroing local slots from the MacroAssembler and instead adds more granular methods to it (e.g `zero`, `add`). This allows for better code sharing since most of the work done by the algorithm for zeroing slots will be the same in all targets, except for the binary emissions pieces, which is what gets delegated to the masm * Use wasmparser's visitor API and add initial support for const and add This commit adds initial support for the I32Const and I32 instructions; this involves adding a minimum for register allocation. Note that some regalloc pieces are still incomplete, since for the current set of supported instructions they are not needed. * Make the ty field public in Local * Add scratch_reg to the abi * Add a method to get a particular local from the Frame * Split the compilation environment abstraction This commit splits the compilation environment into two more concise abstractions: 1. CodeGen: the main abstraction for code generation 2. CodeGenContext: abstraction that shares the common pieces for compilation; these pieces are shared between the code generator and the register allocator * Add `push` and `load` to the MacroAssembler * Remove dead code warnings for unused paths * Map ISA features to cranelift-codegen ISA features * Apply formatting * Fix Cargo.toml after a bad rebase * Add component-compiler feature * Use clap instead of structopt * Add winch to publish.rs script * Minor formatting * Add tests to RegSet and fix two bugs when freeing and checking for register availability * Add tests to Stack * Free source register after a non-constant i32 add * Improve comments - Remove unneeded comments - And improve some of the TODO items * Update default features * Drop the ABI generic param and pass the word_size information directly To avoid dealing with dead code warnings this commit passes the word size information directly, since it's the only piece of information needed from the ABI by Codegen until now * Remove dead code This piece of code will be put back once we start integrating Winch with Wasmtime * Remove unused enum variant This variant doesn't get constructed; it should be added back once a backend is added and not enabled by default or when Winch gets integrated into Wasmtime * Fix unused code in regset tests * Update spec testsuite * Switch the visitor pattern for a simpler operator match This commit removes the usage of wasmparser's visitor pattern and instead defaults to a simpler operator matching approach. This removes the complexity of having to define all the visitor trait functions at once. * Use wasmparser's Visitor trait with a different macro strategy This commit puts back wasmparser's Visitor trait, with a sigle; simpler macro, only used for unsupported operators. * Restructure Winch This commit restuructures Winch's parts. It divides the initial approach into three main crates: `winch-codegen`,`wasmtime-winch` and `winch-tools`. `wasmtime-winch` is reponsible for the Wasmtime-Winch integration. `winch-codegen` is solely responsible for code generation. `winch-tools` is CLI tool to compile Wasm programs, mainly for testing purposes. * Refactor zero local slots This commit moves the logic of zeroing local slots from the codegen module into a method with a default implementation in the MacroAssembler trait: `zero_mem_range`. The refactored implementation is very similar to the previous implementation with the only difference that it doesn't allocates a general-purpose register; it instead uses the register allocator to retrieve the scratch register and uses this register to unroll the series of zero stores. * Tie the codegen creation to the ISA ABI This commit makes the relationship between the ISA ABI and the codegen explicit. This allows us to pass down ABI-specific bit and pieces to the codegeneration. In this case the only concrete piece that we need is the ABI word size. * Mark winch as publishable directory * Revamp winch docs This commit ensures that all the code comments in Winch are compliant with the syle used in the rest of Wasmtime's codebase. It also imptoves, generally the quality of the comments in some modules. * Panic when using multi-value when the target is aarch64 Similar to x64, this commit ensures that the abi signature of the current function doesn't use multi-value returns * Document the usage of directives * Use endianness instead of endianess in the ISA trait * Introduce a three-argument form in the MacroAssembler This commit introduces the usage of three-argument form for the MacroAssembler interface. This allows for a natural mapping for architectures like aarch64. In the case of x64, the implementation can simply restrict the implementation asserting for equality in two of the arguments of defaulting to a differnt set of instructions. As of this commit, the implementation of `add` panics if the destination and the first source arguments are not equal; internally the x64 assembler implementation will ensure that all the allowed combinations of `add` are satisfied. The reason for panicking and not emitting a `mov` followed by an `add` for example is simply because register allocation happens right before calling `add`, which ensures any register-to-register moves, if needed. This implementation will evolve in the future and this panic will be lifted if needed. * Improve the documentation for the MacroAssembler. Documents the usage of three-arg form and the intention around the high-level interface. * Format comments in remaining modules * Clean up Cargo.toml for winch pieces This commit adds missing fields to each of Winch's Cargo.toml. * Use `ModuleTranslation::get_types()` to derive the function type * Assert that start range is always word-size aligned
2 years ago
dependencies = [
"anyhow",
winch: Use cranelift-codegen x64 backend for emission. (#5581) This change substitutes the string based emission mechanism with cranelift-codegen's x64 backend. This change _does not_: * Introduce new functionality in terms of supported instructions. * Change the semantics of the assembler/macroassembler in terms of the logic to emit instructions. The most notable differences between this change and the previous version are: * Handling of shared flags and ISA-specific flags, which for now are left with the default value. * Simplification of instruction emission per operand size: previously the assembler defined different methods depending on the operand size (e.g. `mov` for 64 bits, and `movl` for 32 bits). This change updates such approach so that each assembler method takes an operand size as a parameter, reducing duplication and making the code more concise and easier to integrate with the x64's `Inst` enum. * Introduction of a disassembler for testing purposes. As of this change, Winch generates the following code for the following test programs: ```wat (module (export "main" (func $main)) (func $main (result i32) (i32.const 10) (i32.const 20) i32.add )) ``` ```asm 0: 55 push rbp 1: 48 89 e5 mov rbp, rsp 4: b8 0a 00 00 00 mov eax, 0xa 9: 83 c0 14 add eax, 0x14 c: 5d pop rbp d: c3 ret ``` ```wat (module (export "main" (func $main)) (func $main (result i32) (local $foo i32) (local $bar i32) (i32.const 10) (local.set $foo) (i32.const 20) (local.set $bar) (local.get $foo) (local.get $bar) i32.add )) ``` ```asm 0: 55 push rbp 1: 48 89 e5 mov rbp, rsp 4: 48 83 ec 08 sub rsp, 8 8: 48 c7 04 24 00 00 00 00 mov qword ptr [rsp], 0 10: b8 0a 00 00 00 mov eax, 0xa 15: 89 44 24 04 mov dword ptr [rsp + 4], eax 19: b8 14 00 00 00 mov eax, 0x14 1e: 89 04 24 mov dword ptr [rsp], eax 21: 8b 04 24 mov eax, dword ptr [rsp] 24: 8b 4c 24 04 mov ecx, dword ptr [rsp + 4] 28: 01 c1 add ecx, eax 2a: 48 89 c8 mov rax, rcx 2d: 48 83 c4 08 add rsp, 8 31: 5d pop rbp 32: c3 ret ``` ```wat (module (export "main" (func $main)) (func $main (param i32) (param i32) (result i32) (local.get 0) (local.get 1) i32.add )) ``` ```asm 0: 55 push rbp 1: 48 89 e5 mov rbp, rsp 4: 48 83 ec 08 sub rsp, 8 8: 89 7c 24 04 mov dword ptr [rsp + 4], edi c: 89 34 24 mov dword ptr [rsp], esi f: 8b 04 24 mov eax, dword ptr [rsp] 12: 8b 4c 24 04 mov ecx, dword ptr [rsp + 4] 16: 01 c1 add ecx, eax 18: 48 89 c8 mov rax, rcx 1b: 48 83 c4 08 add rsp, 8 1f: 5d pop rbp 20: c3 ret ```
2 years ago
"cranelift-codegen",
winch: Refactoring wasmtime compiler integration pieces to share more between Cranelift and Winch (#5944) * Enable the native target by default in winch Match cranelift-codegen's build script where if no architecture is explicitly enabled then the host architecture is implicitly enabled. * Refactor Cranelift's ISA builder to share more with Winch This commit refactors the `Builder` type to have a type parameter representing the finished ISA with Cranelift and Winch having their own typedefs for `Builder` to represent their own builders. The intention is to use this shared functionality to produce more shared code between the two codegen backends. * Moving compiler shared components to a separate crate * Restore native flag inference in compiler building This fixes an oversight from the previous commits to use `cranelift-native` to infer flags for the native host when using default settings with Wasmtime. * Move `Compiler::page_size_align` into wasmtime-environ The `cranelift-codegen` crate doesn't need this and winch wants the same implementation, so shuffle it around so everyone has access to it. * Fill out `Compiler::{flags, isa_flags}` for Winch These are easy enough to plumb through with some shared code for Wasmtime. * Plumb the `is_branch_protection_enabled` flag for Winch Just forwarding an isa-specific setting accessor. * Moving executable creation to shared compiler crate * Adding builder back in and removing from shared crate * Refactoring the shared pieces for the `CompilerBuilder` I decided to move a couple things around from Alex's initial changes. Instead of having the shared builder do everything, I went back to having each compiler have a distinct builder implementation. I refactored most of the flag setting logic into a single shared location, so we can still reduce the amount of code duplication. With them being separate, we don't need to maintain things like `LinkOpts` which Winch doesn't currently use. We also have an avenue to error when certain flags are sent to Winch if we don't support them. I'm hoping this will make things more maintainable as we build out Winch. I'm still unsure about keeping everything shared in a single crate (`cranelift_shared`). It's starting to feel like this crate is doing too much, which makes it difficult to name. There does seem to be a need for two distinct abstraction: creating the final executable and the handling of shared/ISA flags when building the compiler. I could make them into two separate crates, but there doesn't seem to be enough there yet to justify it. * Documentation updates, and renaming the finish method * Adding back in a default temporarily to pass tests, and removing some unused imports * Fixing winch tests with wrong method name * Removing unused imports from codegen shared crate * Apply documentation formatting updates Co-authored-by: Saúl Cabrera <saulecabrera@gmail.com> * Adding back in cranelift_native flag inferring * Adding new shared crate to publish list * Adding write feature to pass cargo check --------- Co-authored-by: Alex Crichton <alex@alexcrichton.com> Co-authored-by: Saúl Cabrera <saulecabrera@gmail.com>
2 years ago
"gimli",
Implement AOT compilation for components (#5160) * Pull `Module` out of `ModuleTextBuilder` This commit is the first in what will likely be a number towards preparing for serializing a compiled component to bytes, a precompiled artifact. To that end my rough plan is to merge all of the compiled artifacts for a component into one large object file instead of having lots of separate object files and lots of separate mmaps to manage. To that end I plan on eventually using `ModuleTextBuilder` to build one large text section for all core wasm modules and trampolines, meaning that `ModuleTextBuilder` is no longer specific to one module. I've extracted out functionality such as function name calculation as well as relocation resolving (now a closure passed in) in preparation for this. For now this just keeps tests passing, and the trajectory for this should become more clear over the following commits. * Remove component-specific object emission This commit removes the `ComponentCompiler::emit_obj` function in favor of `Compiler::emit_obj`, now renamed `append_code`. This involved significantly refactoring code emission to take a flat list of functions into `append_code` and the caller is responsible for weaving together various "families" of functions and un-weaving them afterwards. * Consolidate ELF parsing in `CodeMemory` This commit moves the ELF file parsing and section iteration from `CompiledModule` into `CodeMemory` so one location keeps track of section ranges and such. This is in preparation for sharing much of this code with components which needs all the same sections to get tracked but won't be using `CompiledModule`. A small side benefit from this is that the section parsing done in `CodeMemory` and `CompiledModule` is no longer duplicated. * Remove separately tracked traps in components Previously components would generate an "always trapping" function and the metadata around which pc was allowed to trap was handled manually for components. With recent refactorings the Wasmtime-standard trap section in object files is now being generated for components as well which means that can be reused instead of custom-tracking this metadata. This commit removes the manual tracking for the `always_trap` functions and plumbs the necessary bits around to make components look more like modules. * Remove a now-unnecessary `Arc` in `Module` Not expected to have any measurable impact on performance, but complexity-wise this should make it a bit easier to understand the internals since there's no longer any need to store this somewhere else than its owner's location. * Merge compilation artifacts of components This commit is a large refactoring of the component compilation process to produce a single artifact instead of multiple binary artifacts. The core wasm compilation process is refactored as well to share as much code as necessary with the component compilation process. This method of representing a compiled component necessitated a few medium-sized changes internally within Wasmtime: * A new data structure was created, `CodeObject`, which represents metadata about a single compiled artifact. This is then stored as an `Arc` within a component and a module. For `Module` this is always uniquely owned and represents a shuffling around of data from one owner to another. For a `Component`, however, this is shared amongst all loaded modules and the top-level component. * The "module registry" which is used for symbolicating backtraces and for trap information has been updated to account for a single region of loaded code holding possibly multiple modules. This involved adding a second-level `BTreeMap` for now. This will likely slow down instantiation slightly but if it poses an issue in the future this should be able to be represented with a more clever data structure. This commit additionally solves a number of longstanding issues with components such as compiling only one host-to-wasm trampoline per signature instead of possibly once-per-module. Additionally the `SignatureCollection` registration now happens once-per-component instead of once-per-module-within-a-component. * Fix compile errors from prior commits * Support AOT-compiling components This commit adds support for AOT-compiled components in the same manner as `Module`, specifically adding: * `Engine::precompile_component` * `Component::serialize` * `Component::deserialize` * `Component::deserialize_file` Internally the support for components looks quite similar to `Module`. All the prior commits to this made adding the support here (unsurprisingly) easy. Components are represented as a single object file as are modules, and the functions for each module are all piled into the same object file next to each other (as are areas such as data sections). Support was also added here to quickly differentiate compiled components vs compiled modules via the `e_flags` field in the ELF header. * Prevent serializing exported modules on components The current representation of a module within a component means that the implementation of `Module::serialize` will not work if the module is exported from a component. The reason for this is that `serialize` doesn't actually do anything and simply returns the underlying mmap as a list of bytes. The mmap, however, has `.wasmtime.info` describing component metadata as opposed to this module's metadata. While rewriting this section could be implemented it's not so easy to do so and is otherwise seen as not super important of a feature right now anyway. * Fix windows build * Fix an unused function warning * Update crates/environ/src/compilation.rs Co-authored-by: Nick Fitzgerald <fitzgen@gmail.com> Co-authored-by: Nick Fitzgerald <fitzgen@gmail.com>
2 years ago
"object",
Initial skeleton for Winch (#4907) * Initial skeleton for Winch This commit introduces the initial skeleton for Winch, the "baseline" compiler. This skeleton contains mostly setup code for the ISA, ABI, registers, and compilation environment abstractions. It also includes the calculation of function local slots. As of this commit, the structure of these abstractions looks like the following: +------------------------+ | v +----------+ +-----+ +-----------+-----+-----------------+ | Compiler | --> | ISA | --> | Registers | ABI | Compilation Env | +----------+ +-----+ +-----------+-----+-----------------+ | ^ +------------------------------+ * Compilation environment will hold a reference to the function data * Add basic documentation to the ABI trait * Enable x86 and arm64 in cranelift-codegen * Add reg_name function for x64 * Introduce the concept of a MacroAssembler and Assembler This commit introduces the concept of a MacroAsesembler and Assembler. The MacroAssembler trait will provide a high enough interface across architectures so that each ISA implementation can use their own low-level Assembler implementation to fulfill the interface. Each Assembler will provide a 1-1 mapping to each ISA instruction. As of this commit, only a partial debug implementation is provided for the x64 Assembler. * Add a newtype over PReg Adds a newtype `Reg` over regalloc2::PReg; this ensures that Winch will operate only on the concept of `Reg`. This change is temporary until we have the necessary machinery to share a common Reg abstraction via `cranelift_asm` * Improvements to local calcuation - Add `LocalSlot::addressed_from_sp` - Use `u32` for local slot and local sizes calculation * Add helper methods to ABIArg Adds helper methods to retrieve register and type information from the argument * Make locals_size public in frame * Improve x64 register naming depending on size * Add new methods to the masm interface This commit introduces the ability for the MacroAssembler to reserve stack space, get the address of a given local and perform a stack store based on the concept of `Operand`s. There are several motivating factors to introduce the concept of an Operand: - Make the translation between Winch and Cranelift easier; - Make dispatching from the MacroAssembler to the underlying Assembler - easier by minimizing the amount of functions that we need to define - in order to satisfy the store/load combinations This commit also introduces the concept of a memory address, which essentially describes the addressing modes; as of this commit only one addressing mode is supported. We'll also need to verify that this structure will play nicely with arm64. * Blank masm implementation for arm64 * Implementation of reserve_stack, local_address, store and fp_offset for x64 * Implement function prologue and argument register spilling * Add structopt and wat * Fix debug instruction formatting * Make TargetISA trait publicly accessible * Modify the MacroAssembler finalize siganture to return a slice of strings * Introduce a simple CLI for Winch To be able to compile Wasm programs with Winch independently. Mostly meant for testing / debugging * Fix bug in x64 assembler mov_rm * Remove unused import * Move the stack slot calculation to the Frame This commit moves the calculation of the stack slots to the frame handler abstraction and also includes the calculation of the limits for the function defined locals, which will be used to zero the locals that are not associated to function arguments * Add i32 and i64 constructors to local slots * Introduce the concept of DefinedLocalsRange This commit introduces `DefinedLocalsRange` to track the stack offset at which the function-defined locals start and end; this is later used to zero-out that stack region * Add constructors for int and float registers * Add a placeholder stack implementation * Add a regset abstraction to track register availability Adds a bit set abstraction to track register availability for register allocation. The bit set has no specific knowledge about physical registers, it works on the register's hardware encoding as the source of truth. Each RegSet is expected to be created with the universe of allocatable registers per ISA when starting the compilation of a particular function. * Add an abstraction over register and immediate This is meant to be used as the source for stores. * Add a way to zero local slots and an initial skeletion of regalloc This commit introduces `zero_local_slots` to the MacroAssembler; which ensures that function defined locals are zeroed out when starting the function body. The algorithm divides the defined function locals stack range into 8 byte slots and stores a zero at each address. This process relies on register allocation if the amount of slots that need to be initialized is greater than 1. In such case, the next available register is requested to the register set and it's used to store a 0, which is then stored at every local slot * Update to wasmparser 0.92 * Correctly track if the regset has registers available * Add a result entry to the ABI signature This commuit introduces ABIResult as part of the ABISignature; this struct will track how function results are stored; initially it will consiste of a single register that will be requested to the register allocator at the end of the function; potentially causing a spill * Move zero local slots and add more granular methods to the masm This commit removes zeroing local slots from the MacroAssembler and instead adds more granular methods to it (e.g `zero`, `add`). This allows for better code sharing since most of the work done by the algorithm for zeroing slots will be the same in all targets, except for the binary emissions pieces, which is what gets delegated to the masm * Use wasmparser's visitor API and add initial support for const and add This commit adds initial support for the I32Const and I32 instructions; this involves adding a minimum for register allocation. Note that some regalloc pieces are still incomplete, since for the current set of supported instructions they are not needed. * Make the ty field public in Local * Add scratch_reg to the abi * Add a method to get a particular local from the Frame * Split the compilation environment abstraction This commit splits the compilation environment into two more concise abstractions: 1. CodeGen: the main abstraction for code generation 2. CodeGenContext: abstraction that shares the common pieces for compilation; these pieces are shared between the code generator and the register allocator * Add `push` and `load` to the MacroAssembler * Remove dead code warnings for unused paths * Map ISA features to cranelift-codegen ISA features * Apply formatting * Fix Cargo.toml after a bad rebase * Add component-compiler feature * Use clap instead of structopt * Add winch to publish.rs script * Minor formatting * Add tests to RegSet and fix two bugs when freeing and checking for register availability * Add tests to Stack * Free source register after a non-constant i32 add * Improve comments - Remove unneeded comments - And improve some of the TODO items * Update default features * Drop the ABI generic param and pass the word_size information directly To avoid dealing with dead code warnings this commit passes the word size information directly, since it's the only piece of information needed from the ABI by Codegen until now * Remove dead code This piece of code will be put back once we start integrating Winch with Wasmtime * Remove unused enum variant This variant doesn't get constructed; it should be added back once a backend is added and not enabled by default or when Winch gets integrated into Wasmtime * Fix unused code in regset tests * Update spec testsuite * Switch the visitor pattern for a simpler operator match This commit removes the usage of wasmparser's visitor pattern and instead defaults to a simpler operator matching approach. This removes the complexity of having to define all the visitor trait functions at once. * Use wasmparser's Visitor trait with a different macro strategy This commit puts back wasmparser's Visitor trait, with a sigle; simpler macro, only used for unsupported operators. * Restructure Winch This commit restuructures Winch's parts. It divides the initial approach into three main crates: `winch-codegen`,`wasmtime-winch` and `winch-tools`. `wasmtime-winch` is reponsible for the Wasmtime-Winch integration. `winch-codegen` is solely responsible for code generation. `winch-tools` is CLI tool to compile Wasm programs, mainly for testing purposes. * Refactor zero local slots This commit moves the logic of zeroing local slots from the codegen module into a method with a default implementation in the MacroAssembler trait: `zero_mem_range`. The refactored implementation is very similar to the previous implementation with the only difference that it doesn't allocates a general-purpose register; it instead uses the register allocator to retrieve the scratch register and uses this register to unroll the series of zero stores. * Tie the codegen creation to the ISA ABI This commit makes the relationship between the ISA ABI and the codegen explicit. This allows us to pass down ABI-specific bit and pieces to the codegeneration. In this case the only concrete piece that we need is the ABI word size. * Mark winch as publishable directory * Revamp winch docs This commit ensures that all the code comments in Winch are compliant with the syle used in the rest of Wasmtime's codebase. It also imptoves, generally the quality of the comments in some modules. * Panic when using multi-value when the target is aarch64 Similar to x64, this commit ensures that the abi signature of the current function doesn't use multi-value returns * Document the usage of directives * Use endianness instead of endianess in the ISA trait * Introduce a three-argument form in the MacroAssembler This commit introduces the usage of three-argument form for the MacroAssembler interface. This allows for a natural mapping for architectures like aarch64. In the case of x64, the implementation can simply restrict the implementation asserting for equality in two of the arguments of defaulting to a differnt set of instructions. As of this commit, the implementation of `add` panics if the destination and the first source arguments are not equal; internally the x64 assembler implementation will ensure that all the allowed combinations of `add` are satisfied. The reason for panicking and not emitting a `mov` followed by an `add` for example is simply because register allocation happens right before calling `add`, which ensures any register-to-register moves, if needed. This implementation will evolve in the future and this panic will be lifted if needed. * Improve the documentation for the MacroAssembler. Documents the usage of three-arg form and the intention around the high-level interface. * Format comments in remaining modules * Clean up Cargo.toml for winch pieces This commit adds missing fields to each of Winch's Cargo.toml. * Use `ModuleTranslation::get_types()` to derive the function type * Assert that start range is always word-size aligned
2 years ago
"target-lexicon",
winch: Refactoring wasmtime compiler integration pieces to share more between Cranelift and Winch (#5944) * Enable the native target by default in winch Match cranelift-codegen's build script where if no architecture is explicitly enabled then the host architecture is implicitly enabled. * Refactor Cranelift's ISA builder to share more with Winch This commit refactors the `Builder` type to have a type parameter representing the finished ISA with Cranelift and Winch having their own typedefs for `Builder` to represent their own builders. The intention is to use this shared functionality to produce more shared code between the two codegen backends. * Moving compiler shared components to a separate crate * Restore native flag inference in compiler building This fixes an oversight from the previous commits to use `cranelift-native` to infer flags for the native host when using default settings with Wasmtime. * Move `Compiler::page_size_align` into wasmtime-environ The `cranelift-codegen` crate doesn't need this and winch wants the same implementation, so shuffle it around so everyone has access to it. * Fill out `Compiler::{flags, isa_flags}` for Winch These are easy enough to plumb through with some shared code for Wasmtime. * Plumb the `is_branch_protection_enabled` flag for Winch Just forwarding an isa-specific setting accessor. * Moving executable creation to shared compiler crate * Adding builder back in and removing from shared crate * Refactoring the shared pieces for the `CompilerBuilder` I decided to move a couple things around from Alex's initial changes. Instead of having the shared builder do everything, I went back to having each compiler have a distinct builder implementation. I refactored most of the flag setting logic into a single shared location, so we can still reduce the amount of code duplication. With them being separate, we don't need to maintain things like `LinkOpts` which Winch doesn't currently use. We also have an avenue to error when certain flags are sent to Winch if we don't support them. I'm hoping this will make things more maintainable as we build out Winch. I'm still unsure about keeping everything shared in a single crate (`cranelift_shared`). It's starting to feel like this crate is doing too much, which makes it difficult to name. There does seem to be a need for two distinct abstraction: creating the final executable and the handling of shared/ISA flags when building the compiler. I could make them into two separate crates, but there doesn't seem to be enough there yet to justify it. * Documentation updates, and renaming the finish method * Adding back in a default temporarily to pass tests, and removing some unused imports * Fixing winch tests with wrong method name * Removing unused imports from codegen shared crate * Apply documentation formatting updates Co-authored-by: Saúl Cabrera <saulecabrera@gmail.com> * Adding back in cranelift_native flag inferring * Adding new shared crate to publish list * Adding write feature to pass cargo check --------- Co-authored-by: Alex Crichton <alex@alexcrichton.com> Co-authored-by: Saúl Cabrera <saulecabrera@gmail.com>
2 years ago
"wasmparser",
"wasmtime-cranelift-shared",
Initial skeleton for Winch (#4907) * Initial skeleton for Winch This commit introduces the initial skeleton for Winch, the "baseline" compiler. This skeleton contains mostly setup code for the ISA, ABI, registers, and compilation environment abstractions. It also includes the calculation of function local slots. As of this commit, the structure of these abstractions looks like the following: +------------------------+ | v +----------+ +-----+ +-----------+-----+-----------------+ | Compiler | --> | ISA | --> | Registers | ABI | Compilation Env | +----------+ +-----+ +-----------+-----+-----------------+ | ^ +------------------------------+ * Compilation environment will hold a reference to the function data * Add basic documentation to the ABI trait * Enable x86 and arm64 in cranelift-codegen * Add reg_name function for x64 * Introduce the concept of a MacroAssembler and Assembler This commit introduces the concept of a MacroAsesembler and Assembler. The MacroAssembler trait will provide a high enough interface across architectures so that each ISA implementation can use their own low-level Assembler implementation to fulfill the interface. Each Assembler will provide a 1-1 mapping to each ISA instruction. As of this commit, only a partial debug implementation is provided for the x64 Assembler. * Add a newtype over PReg Adds a newtype `Reg` over regalloc2::PReg; this ensures that Winch will operate only on the concept of `Reg`. This change is temporary until we have the necessary machinery to share a common Reg abstraction via `cranelift_asm` * Improvements to local calcuation - Add `LocalSlot::addressed_from_sp` - Use `u32` for local slot and local sizes calculation * Add helper methods to ABIArg Adds helper methods to retrieve register and type information from the argument * Make locals_size public in frame * Improve x64 register naming depending on size * Add new methods to the masm interface This commit introduces the ability for the MacroAssembler to reserve stack space, get the address of a given local and perform a stack store based on the concept of `Operand`s. There are several motivating factors to introduce the concept of an Operand: - Make the translation between Winch and Cranelift easier; - Make dispatching from the MacroAssembler to the underlying Assembler - easier by minimizing the amount of functions that we need to define - in order to satisfy the store/load combinations This commit also introduces the concept of a memory address, which essentially describes the addressing modes; as of this commit only one addressing mode is supported. We'll also need to verify that this structure will play nicely with arm64. * Blank masm implementation for arm64 * Implementation of reserve_stack, local_address, store and fp_offset for x64 * Implement function prologue and argument register spilling * Add structopt and wat * Fix debug instruction formatting * Make TargetISA trait publicly accessible * Modify the MacroAssembler finalize siganture to return a slice of strings * Introduce a simple CLI for Winch To be able to compile Wasm programs with Winch independently. Mostly meant for testing / debugging * Fix bug in x64 assembler mov_rm * Remove unused import * Move the stack slot calculation to the Frame This commit moves the calculation of the stack slots to the frame handler abstraction and also includes the calculation of the limits for the function defined locals, which will be used to zero the locals that are not associated to function arguments * Add i32 and i64 constructors to local slots * Introduce the concept of DefinedLocalsRange This commit introduces `DefinedLocalsRange` to track the stack offset at which the function-defined locals start and end; this is later used to zero-out that stack region * Add constructors for int and float registers * Add a placeholder stack implementation * Add a regset abstraction to track register availability Adds a bit set abstraction to track register availability for register allocation. The bit set has no specific knowledge about physical registers, it works on the register's hardware encoding as the source of truth. Each RegSet is expected to be created with the universe of allocatable registers per ISA when starting the compilation of a particular function. * Add an abstraction over register and immediate This is meant to be used as the source for stores. * Add a way to zero local slots and an initial skeletion of regalloc This commit introduces `zero_local_slots` to the MacroAssembler; which ensures that function defined locals are zeroed out when starting the function body. The algorithm divides the defined function locals stack range into 8 byte slots and stores a zero at each address. This process relies on register allocation if the amount of slots that need to be initialized is greater than 1. In such case, the next available register is requested to the register set and it's used to store a 0, which is then stored at every local slot * Update to wasmparser 0.92 * Correctly track if the regset has registers available * Add a result entry to the ABI signature This commuit introduces ABIResult as part of the ABISignature; this struct will track how function results are stored; initially it will consiste of a single register that will be requested to the register allocator at the end of the function; potentially causing a spill * Move zero local slots and add more granular methods to the masm This commit removes zeroing local slots from the MacroAssembler and instead adds more granular methods to it (e.g `zero`, `add`). This allows for better code sharing since most of the work done by the algorithm for zeroing slots will be the same in all targets, except for the binary emissions pieces, which is what gets delegated to the masm * Use wasmparser's visitor API and add initial support for const and add This commit adds initial support for the I32Const and I32 instructions; this involves adding a minimum for register allocation. Note that some regalloc pieces are still incomplete, since for the current set of supported instructions they are not needed. * Make the ty field public in Local * Add scratch_reg to the abi * Add a method to get a particular local from the Frame * Split the compilation environment abstraction This commit splits the compilation environment into two more concise abstractions: 1. CodeGen: the main abstraction for code generation 2. CodeGenContext: abstraction that shares the common pieces for compilation; these pieces are shared between the code generator and the register allocator * Add `push` and `load` to the MacroAssembler * Remove dead code warnings for unused paths * Map ISA features to cranelift-codegen ISA features * Apply formatting * Fix Cargo.toml after a bad rebase * Add component-compiler feature * Use clap instead of structopt * Add winch to publish.rs script * Minor formatting * Add tests to RegSet and fix two bugs when freeing and checking for register availability * Add tests to Stack * Free source register after a non-constant i32 add * Improve comments - Remove unneeded comments - And improve some of the TODO items * Update default features * Drop the ABI generic param and pass the word_size information directly To avoid dealing with dead code warnings this commit passes the word size information directly, since it's the only piece of information needed from the ABI by Codegen until now * Remove dead code This piece of code will be put back once we start integrating Winch with Wasmtime * Remove unused enum variant This variant doesn't get constructed; it should be added back once a backend is added and not enabled by default or when Winch gets integrated into Wasmtime * Fix unused code in regset tests * Update spec testsuite * Switch the visitor pattern for a simpler operator match This commit removes the usage of wasmparser's visitor pattern and instead defaults to a simpler operator matching approach. This removes the complexity of having to define all the visitor trait functions at once. * Use wasmparser's Visitor trait with a different macro strategy This commit puts back wasmparser's Visitor trait, with a sigle; simpler macro, only used for unsupported operators. * Restructure Winch This commit restuructures Winch's parts. It divides the initial approach into three main crates: `winch-codegen`,`wasmtime-winch` and `winch-tools`. `wasmtime-winch` is reponsible for the Wasmtime-Winch integration. `winch-codegen` is solely responsible for code generation. `winch-tools` is CLI tool to compile Wasm programs, mainly for testing purposes. * Refactor zero local slots This commit moves the logic of zeroing local slots from the codegen module into a method with a default implementation in the MacroAssembler trait: `zero_mem_range`. The refactored implementation is very similar to the previous implementation with the only difference that it doesn't allocates a general-purpose register; it instead uses the register allocator to retrieve the scratch register and uses this register to unroll the series of zero stores. * Tie the codegen creation to the ISA ABI This commit makes the relationship between the ISA ABI and the codegen explicit. This allows us to pass down ABI-specific bit and pieces to the codegeneration. In this case the only concrete piece that we need is the ABI word size. * Mark winch as publishable directory * Revamp winch docs This commit ensures that all the code comments in Winch are compliant with the syle used in the rest of Wasmtime's codebase. It also imptoves, generally the quality of the comments in some modules. * Panic when using multi-value when the target is aarch64 Similar to x64, this commit ensures that the abi signature of the current function doesn't use multi-value returns * Document the usage of directives * Use endianness instead of endianess in the ISA trait * Introduce a three-argument form in the MacroAssembler This commit introduces the usage of three-argument form for the MacroAssembler interface. This allows for a natural mapping for architectures like aarch64. In the case of x64, the implementation can simply restrict the implementation asserting for equality in two of the arguments of defaulting to a differnt set of instructions. As of this commit, the implementation of `add` panics if the destination and the first source arguments are not equal; internally the x64 assembler implementation will ensure that all the allowed combinations of `add` are satisfied. The reason for panicking and not emitting a `mov` followed by an `add` for example is simply because register allocation happens right before calling `add`, which ensures any register-to-register moves, if needed. This implementation will evolve in the future and this panic will be lifted if needed. * Improve the documentation for the MacroAssembler. Documents the usage of three-arg form and the intention around the high-level interface. * Format comments in remaining modules * Clean up Cargo.toml for winch pieces This commit adds missing fields to each of Winch's Cargo.toml. * Use `ModuleTranslation::get_types()` to derive the function type * Assert that start range is always word-size aligned
2 years ago
"wasmtime-environ",
"winch-codegen",
winch: Initial integration with wasmtime (#6119) * Adding in trampoline compiling method for ISA * Adding support for indirect call to memory address * Refactoring frame to externalize defined locals, so it removes WASM depedencies in trampoline case * Adding initial version of trampoline for testing * Refactoring trampoline to be re-used by other architectures * Initial wiring for winch with wasmtime * Add a Wasmtime CLI option to select `winch` This is effectively an option to select the `Strategy` enumeration. * Implement `Compiler::compile_function` for Winch Hook this into the `TargetIsa::compile_function` hook as well. Currently this doesn't take into account `Tunables`, but that's left as a TODO for later. * Filling out Winch append_code method * Adding back in changes from previous branch Most of these are a WIP. It's missing trampolines for x64, but a basic one exists for aarch64. It's missing the handling of arguments that exist on the stack. It currently imports `cranelift_wasm::WasmFuncType` since it's what's passed to the `Compiler` trait. It's a bit awkward to use in the `winch_codegen` crate since it mostly operates on `wasmparser` types. I've had to hack in a conversion to get things working. Long term, I'm not sure it's wise to rely on this type but it seems like it's easier on the Cranelift side when creating the stub IR. * Small API changes to make integration easier * Adding in new FuncEnv, only a stub for now * Removing unneeded parts of the old PoC, and refactoring trampoline code * Moving FuncEnv into a separate file * More comments for trampolines * Adding in winch integration tests for first pass * Using new addressing method to fix stack pointer error * Adding test for stack arguments * Only run tests on x86 for now, it's more complete for winch * Add in missing documentation after rebase * Updating based on feedback in draft PR * Fixing formatting on doc comment for argv register * Running formatting * Lock updates, and turning on winch feature flags during tests * Updating configuration with comments to no longer gate Strategy enum * Using the winch-environ FuncEnv, but it required changing the sig * Proper comment formatting * Removing wasmtime-winch from dev-dependencies, adding the winch feature makes this not necessary * Update doc attr to include winch check * Adding winch feature to doc generation, which seems to fix the feature error in CI * Add the `component-model` feature to the cargo doc invocation in CI To match the metadata used by the docs.rs invocation when building docs. * Add a comment clarifying the usage of `component-model` for docs.rs * Correctly order wasmtime-winch and winch-environ in the publish script * Ensure x86 test dependencies are included in cfg(target_arch) * Further constrain Winch tests to x86_64 _and_ unix --------- Co-authored-by: Alex Crichton <alex@alexcrichton.com> Co-authored-by: Saúl Cabrera <saulecabrera@gmail.com>
2 years ago
"winch-environ",
Initial skeleton for Winch (#4907) * Initial skeleton for Winch This commit introduces the initial skeleton for Winch, the "baseline" compiler. This skeleton contains mostly setup code for the ISA, ABI, registers, and compilation environment abstractions. It also includes the calculation of function local slots. As of this commit, the structure of these abstractions looks like the following: +------------------------+ | v +----------+ +-----+ +-----------+-----+-----------------+ | Compiler | --> | ISA | --> | Registers | ABI | Compilation Env | +----------+ +-----+ +-----------+-----+-----------------+ | ^ +------------------------------+ * Compilation environment will hold a reference to the function data * Add basic documentation to the ABI trait * Enable x86 and arm64 in cranelift-codegen * Add reg_name function for x64 * Introduce the concept of a MacroAssembler and Assembler This commit introduces the concept of a MacroAsesembler and Assembler. The MacroAssembler trait will provide a high enough interface across architectures so that each ISA implementation can use their own low-level Assembler implementation to fulfill the interface. Each Assembler will provide a 1-1 mapping to each ISA instruction. As of this commit, only a partial debug implementation is provided for the x64 Assembler. * Add a newtype over PReg Adds a newtype `Reg` over regalloc2::PReg; this ensures that Winch will operate only on the concept of `Reg`. This change is temporary until we have the necessary machinery to share a common Reg abstraction via `cranelift_asm` * Improvements to local calcuation - Add `LocalSlot::addressed_from_sp` - Use `u32` for local slot and local sizes calculation * Add helper methods to ABIArg Adds helper methods to retrieve register and type information from the argument * Make locals_size public in frame * Improve x64 register naming depending on size * Add new methods to the masm interface This commit introduces the ability for the MacroAssembler to reserve stack space, get the address of a given local and perform a stack store based on the concept of `Operand`s. There are several motivating factors to introduce the concept of an Operand: - Make the translation between Winch and Cranelift easier; - Make dispatching from the MacroAssembler to the underlying Assembler - easier by minimizing the amount of functions that we need to define - in order to satisfy the store/load combinations This commit also introduces the concept of a memory address, which essentially describes the addressing modes; as of this commit only one addressing mode is supported. We'll also need to verify that this structure will play nicely with arm64. * Blank masm implementation for arm64 * Implementation of reserve_stack, local_address, store and fp_offset for x64 * Implement function prologue and argument register spilling * Add structopt and wat * Fix debug instruction formatting * Make TargetISA trait publicly accessible * Modify the MacroAssembler finalize siganture to return a slice of strings * Introduce a simple CLI for Winch To be able to compile Wasm programs with Winch independently. Mostly meant for testing / debugging * Fix bug in x64 assembler mov_rm * Remove unused import * Move the stack slot calculation to the Frame This commit moves the calculation of the stack slots to the frame handler abstraction and also includes the calculation of the limits for the function defined locals, which will be used to zero the locals that are not associated to function arguments * Add i32 and i64 constructors to local slots * Introduce the concept of DefinedLocalsRange This commit introduces `DefinedLocalsRange` to track the stack offset at which the function-defined locals start and end; this is later used to zero-out that stack region * Add constructors for int and float registers * Add a placeholder stack implementation * Add a regset abstraction to track register availability Adds a bit set abstraction to track register availability for register allocation. The bit set has no specific knowledge about physical registers, it works on the register's hardware encoding as the source of truth. Each RegSet is expected to be created with the universe of allocatable registers per ISA when starting the compilation of a particular function. * Add an abstraction over register and immediate This is meant to be used as the source for stores. * Add a way to zero local slots and an initial skeletion of regalloc This commit introduces `zero_local_slots` to the MacroAssembler; which ensures that function defined locals are zeroed out when starting the function body. The algorithm divides the defined function locals stack range into 8 byte slots and stores a zero at each address. This process relies on register allocation if the amount of slots that need to be initialized is greater than 1. In such case, the next available register is requested to the register set and it's used to store a 0, which is then stored at every local slot * Update to wasmparser 0.92 * Correctly track if the regset has registers available * Add a result entry to the ABI signature This commuit introduces ABIResult as part of the ABISignature; this struct will track how function results are stored; initially it will consiste of a single register that will be requested to the register allocator at the end of the function; potentially causing a spill * Move zero local slots and add more granular methods to the masm This commit removes zeroing local slots from the MacroAssembler and instead adds more granular methods to it (e.g `zero`, `add`). This allows for better code sharing since most of the work done by the algorithm for zeroing slots will be the same in all targets, except for the binary emissions pieces, which is what gets delegated to the masm * Use wasmparser's visitor API and add initial support for const and add This commit adds initial support for the I32Const and I32 instructions; this involves adding a minimum for register allocation. Note that some regalloc pieces are still incomplete, since for the current set of supported instructions they are not needed. * Make the ty field public in Local * Add scratch_reg to the abi * Add a method to get a particular local from the Frame * Split the compilation environment abstraction This commit splits the compilation environment into two more concise abstractions: 1. CodeGen: the main abstraction for code generation 2. CodeGenContext: abstraction that shares the common pieces for compilation; these pieces are shared between the code generator and the register allocator * Add `push` and `load` to the MacroAssembler * Remove dead code warnings for unused paths * Map ISA features to cranelift-codegen ISA features * Apply formatting * Fix Cargo.toml after a bad rebase * Add component-compiler feature * Use clap instead of structopt * Add winch to publish.rs script * Minor formatting * Add tests to RegSet and fix two bugs when freeing and checking for register availability * Add tests to Stack * Free source register after a non-constant i32 add * Improve comments - Remove unneeded comments - And improve some of the TODO items * Update default features * Drop the ABI generic param and pass the word_size information directly To avoid dealing with dead code warnings this commit passes the word size information directly, since it's the only piece of information needed from the ABI by Codegen until now * Remove dead code This piece of code will be put back once we start integrating Winch with Wasmtime * Remove unused enum variant This variant doesn't get constructed; it should be added back once a backend is added and not enabled by default or when Winch gets integrated into Wasmtime * Fix unused code in regset tests * Update spec testsuite * Switch the visitor pattern for a simpler operator match This commit removes the usage of wasmparser's visitor pattern and instead defaults to a simpler operator matching approach. This removes the complexity of having to define all the visitor trait functions at once. * Use wasmparser's Visitor trait with a different macro strategy This commit puts back wasmparser's Visitor trait, with a sigle; simpler macro, only used for unsupported operators. * Restructure Winch This commit restuructures Winch's parts. It divides the initial approach into three main crates: `winch-codegen`,`wasmtime-winch` and `winch-tools`. `wasmtime-winch` is reponsible for the Wasmtime-Winch integration. `winch-codegen` is solely responsible for code generation. `winch-tools` is CLI tool to compile Wasm programs, mainly for testing purposes. * Refactor zero local slots This commit moves the logic of zeroing local slots from the codegen module into a method with a default implementation in the MacroAssembler trait: `zero_mem_range`. The refactored implementation is very similar to the previous implementation with the only difference that it doesn't allocates a general-purpose register; it instead uses the register allocator to retrieve the scratch register and uses this register to unroll the series of zero stores. * Tie the codegen creation to the ISA ABI This commit makes the relationship between the ISA ABI and the codegen explicit. This allows us to pass down ABI-specific bit and pieces to the codegeneration. In this case the only concrete piece that we need is the ABI word size. * Mark winch as publishable directory * Revamp winch docs This commit ensures that all the code comments in Winch are compliant with the syle used in the rest of Wasmtime's codebase. It also imptoves, generally the quality of the comments in some modules. * Panic when using multi-value when the target is aarch64 Similar to x64, this commit ensures that the abi signature of the current function doesn't use multi-value returns * Document the usage of directives * Use endianness instead of endianess in the ISA trait * Introduce a three-argument form in the MacroAssembler This commit introduces the usage of three-argument form for the MacroAssembler interface. This allows for a natural mapping for architectures like aarch64. In the case of x64, the implementation can simply restrict the implementation asserting for equality in two of the arguments of defaulting to a differnt set of instructions. As of this commit, the implementation of `add` panics if the destination and the first source arguments are not equal; internally the x64 assembler implementation will ensure that all the allowed combinations of `add` are satisfied. The reason for panicking and not emitting a `mov` followed by an `add` for example is simply because register allocation happens right before calling `add`, which ensures any register-to-register moves, if needed. This implementation will evolve in the future and this panic will be lifted if needed. * Improve the documentation for the MacroAssembler. Documents the usage of three-arg form and the intention around the high-level interface. * Format comments in remaining modules * Clean up Cargo.toml for winch pieces This commit adds missing fields to each of Winch's Cargo.toml. * Use `ModuleTranslation::get_types()` to derive the function type * Assert that start range is always word-size aligned
2 years ago
]
[[package]]
name = "wasmtime-wit-bindgen"
version = "9.0.0"
dependencies = [
"anyhow",
"heck",
"wit-parser",
]
[[package]]
name = "wast"
version = "35.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2ef140f1b49946586078353a453a1d28ba90adfc54dde75710bc1931de204d68"
dependencies = [
"leb128",
]
[[package]]
name = "wast"
version = "55.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4984d3e1406571f4930ba5cf79bd70f75f41d0e87e17506e0bd19b0e5d085f05"
dependencies = [
"leb128",
"memchr",
"unicode-width",
"wasm-encoder 0.25.0",
]
[[package]]
name = "wat"
version = "1.0.61"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "af2b53f4da14db05d32e70e9c617abdf6620c575bd5dd972b7400037b4df2091"
dependencies = [
"wast 55.0.0",
]
[[package]]
name = "web-sys"
version = "0.3.57"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7b17e741662c70c8bd24ac5c5b18de314a2c26c32bf8346ee1e6f53de919c283"
dependencies = [
"js-sys",
"wasm-bindgen",
]
Add differential fuzzing against V8 (#3264) * Add differential fuzzing against V8 This commit adds a differential fuzzing target to Wasmtime along the lines of the wasmi and spec interpreters we already have, but with V8 instead. The intention here is that wasmi is unlikely to receive updates over time (e.g. for SIMD), and the spec interpreter is not suitable for fuzzing against in general due to its performance characteristics. The hope is that V8 is indeed appropriate to fuzz against because it's naturally receiving updates and it also is expected to have good performance. Here the `rusty_v8` crate is used which provides bindings to V8 as well as precompiled binaries by default. This matches exactly the use case we need and at least for now I think the `rusty_v8` crate will be maintained by the Deno folks as they continue to develop it. If it becomes an issue though maintaining we can evaluate other options to have differential fuzzing against. For now this commit enables the SIMD and bulk-memory feature of fuzz-target-generation which should enable them to get differentially-fuzzed with V8 in addition to the compilation fuzzing we're already getting. * Use weak linkage for GDB jit helpers This should help us deduplicate our symbol with other JIT runtimes, if any. For now this leans on some C helpers to define the weak linkage since Rust doesn't support that on stable yet. * Don't use rusty_v8 on MinGW They don't have precompiled libraries there. * Fix msvc build * Comment about execution
3 years ago
[[package]]
name = "which"
version = "4.2.5"
Add differential fuzzing against V8 (#3264) * Add differential fuzzing against V8 This commit adds a differential fuzzing target to Wasmtime along the lines of the wasmi and spec interpreters we already have, but with V8 instead. The intention here is that wasmi is unlikely to receive updates over time (e.g. for SIMD), and the spec interpreter is not suitable for fuzzing against in general due to its performance characteristics. The hope is that V8 is indeed appropriate to fuzz against because it's naturally receiving updates and it also is expected to have good performance. Here the `rusty_v8` crate is used which provides bindings to V8 as well as precompiled binaries by default. This matches exactly the use case we need and at least for now I think the `rusty_v8` crate will be maintained by the Deno folks as they continue to develop it. If it becomes an issue though maintaining we can evaluate other options to have differential fuzzing against. For now this commit enables the SIMD and bulk-memory feature of fuzz-target-generation which should enable them to get differentially-fuzzed with V8 in addition to the compilation fuzzing we're already getting. * Use weak linkage for GDB jit helpers This should help us deduplicate our symbol with other JIT runtimes, if any. For now this leans on some C helpers to define the weak linkage since Rust doesn't support that on stable yet. * Don't use rusty_v8 on MinGW They don't have precompiled libraries there. * Fix msvc build * Comment about execution
3 years ago
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5c4fb54e6113b6a8772ee41c3404fb0301ac79604489467e0a9ce1f3e97c24ae"
Add differential fuzzing against V8 (#3264) * Add differential fuzzing against V8 This commit adds a differential fuzzing target to Wasmtime along the lines of the wasmi and spec interpreters we already have, but with V8 instead. The intention here is that wasmi is unlikely to receive updates over time (e.g. for SIMD), and the spec interpreter is not suitable for fuzzing against in general due to its performance characteristics. The hope is that V8 is indeed appropriate to fuzz against because it's naturally receiving updates and it also is expected to have good performance. Here the `rusty_v8` crate is used which provides bindings to V8 as well as precompiled binaries by default. This matches exactly the use case we need and at least for now I think the `rusty_v8` crate will be maintained by the Deno folks as they continue to develop it. If it becomes an issue though maintaining we can evaluate other options to have differential fuzzing against. For now this commit enables the SIMD and bulk-memory feature of fuzz-target-generation which should enable them to get differentially-fuzzed with V8 in addition to the compilation fuzzing we're already getting. * Use weak linkage for GDB jit helpers This should help us deduplicate our symbol with other JIT runtimes, if any. For now this leans on some C helpers to define the weak linkage since Rust doesn't support that on stable yet. * Don't use rusty_v8 on MinGW They don't have precompiled libraries there. * Fix msvc build * Comment about execution
3 years ago
dependencies = [
"either",
"lazy_static",
"libc",
]
[[package]]
name = "wiggle"
version = "9.0.0"
dependencies = [
"anyhow",
"async-trait",
"bitflags",
"proptest",
"thiserror",
"tokio",
"tracing",
"wasmtime",
"wiggle-macro",
"wiggle-test",
"witx",
]
[[package]]
name = "wiggle-generate"
version = "9.0.0"
dependencies = [
"anyhow",
"heck",
"proc-macro2",
"quote",
"shellexpand",
"syn",
"witx",
]
[[package]]
name = "wiggle-macro"
version = "9.0.0"
dependencies = [
"proc-macro2",
"quote",
"syn",
"wiggle",
"wiggle-generate",
]
[[package]]
name = "wiggle-test"
version = "0.0.0"
dependencies = [
Return `anyhow::Error` from host functions instead of `Trap`, redesign `Trap` (#5149) * Return `anyhow::Error` from host functions instead of `Trap` This commit refactors how errors are modeled when returned from host functions and additionally refactors how custom errors work with `Trap`. At a high level functions in Wasmtime that previously worked with `Result<T, Trap>` now work with `Result<T>` instead where the error is `anyhow::Error`. This includes functions such as: * Host-defined functions in a `Linker<T>` * `TypedFunc::call` * Host-related callbacks like call hooks Errors are now modeled primarily as `anyhow::Error` throughout Wasmtime. This subsequently removes the need for `Trap` to have the ability to represent all host-defined errors as it previously did. Consequently the `From` implementations for any error into a `Trap` have been removed here and the only embedder-defined way to create a `Trap` is to use `Trap::new` with a custom string. After this commit the distinction between a `Trap` and a host error is the wasm backtrace that it contains. Previously all errors in host functions would flow through a `Trap` and get a wasm backtrace attached to them, but now this only happens if a `Trap` itself is created meaning that arbitrary host-defined errors flowing from a host import to the other side won't get backtraces attached. Some internals of Wasmtime itself were updated or preserved to use `Trap::new` to capture a backtrace where it seemed useful, such as when fuel runs out. The main motivation for this commit is that it now enables hosts to thread a concrete error type from a host function all the way through to where a wasm function was invoked. Previously this could not be done since the host error was wrapped in a `Trap` that didn't provide the ability to get at the internals. A consequence of this commit is that when a host error is returned that isn't a `Trap` we'll capture a backtrace and then won't have a `Trap` to attach it to. To avoid losing the contextual information this commit uses the `Error::context` method to attach the backtrace as contextual information to ensure that the backtrace is itself not lost. This is a breaking change for likely all users of Wasmtime, but it's hoped to be a relatively minor change to workaround. Most use cases can likely change `-> Result<T, Trap>` to `-> Result<T>` and otherwise explicit creation of a `Trap` is largely no longer necessary. * Fix some doc links * add some tests and make a backtrace type public (#55) * Trap: avoid a trailing newline in the Display impl which in turn ends up with three newlines between the end of the backtrace and the `Caused by` in the anyhow Debug impl * make BacktraceContext pub, and add tests showing downcasting behavior of anyhow::Error to traps or backtraces * Remove now-unnecesary `Trap` downcasts in `Linker::module` * Fix test output expectations * Remove `Trap::i32_exit` This commit removes special-handling in the `wasmtime::Trap` type for the i32 exit code required by WASI. This is now instead modeled as a specific `I32Exit` error type in the `wasmtime-wasi` crate which is returned by the `proc_exit` hostcall. Embedders which previously tested for i32 exits now downcast to the `I32Exit` value. * Remove the `Trap::new` constructor This commit removes the ability to create a trap with an arbitrary error message. The purpose of this commit is to continue the prior trend of leaning into the `anyhow::Error` type instead of trying to recreate it with `Trap`. A subsequent simplification to `Trap` after this commit is that `Trap` will simply be an `enum` of trap codes with no extra information. This commit is doubly-motivated by the desire to always use the new `BacktraceContext` type instead of sometimes using that and sometimes using `Trap`. Most of the changes here were around updating `Trap::new` calls to `bail!` calls instead. Tests which assert particular error messages additionally often needed to use the `:?` formatter instead of the `{}` formatter because the prior formats the whole `anyhow::Error` and the latter only formats the top-most error, which now contains the backtrace. * Merge `Trap` and `TrapCode` With prior refactorings there's no more need for `Trap` to be opaque or otherwise contain a backtrace. This commit parse down `Trap` to simply an `enum` which was the old `TrapCode`. All various tests and such were updated to handle this. The main consequence of this commit is that all errors have a `BacktraceContext` context attached to them. This unfortunately means that the backtrace is printed first before the error message or trap code, but given all the prior simplifications that seems worth it at this time. * Rename `BacktraceContext` to `WasmBacktrace` This feels like a better name given how this has turned out, and additionally this commit removes having both `WasmBacktrace` and `BacktraceContext`. * Soup up documentation for errors and traps * Fix build of the C API Co-authored-by: Pat Hickey <pat@moreproductive.org>
2 years ago
"anyhow",
"env_logger 0.9.0",
"proptest",
"thiserror",
"tracing",
"tracing-subscriber",
"wiggle",
]
[[package]]
name = "winapi"
version = "0.3.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419"
dependencies = [
"winapi-i686-pc-windows-gnu",
"winapi-x86_64-pc-windows-gnu",
]
[[package]]
name = "winapi-i686-pc-windows-gnu"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
[[package]]
name = "winapi-util"
version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178"
dependencies = [
"winapi",
]
[[package]]
name = "winapi-x86_64-pc-windows-gnu"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
Initial skeleton for Winch (#4907) * Initial skeleton for Winch This commit introduces the initial skeleton for Winch, the "baseline" compiler. This skeleton contains mostly setup code for the ISA, ABI, registers, and compilation environment abstractions. It also includes the calculation of function local slots. As of this commit, the structure of these abstractions looks like the following: +------------------------+ | v +----------+ +-----+ +-----------+-----+-----------------+ | Compiler | --> | ISA | --> | Registers | ABI | Compilation Env | +----------+ +-----+ +-----------+-----+-----------------+ | ^ +------------------------------+ * Compilation environment will hold a reference to the function data * Add basic documentation to the ABI trait * Enable x86 and arm64 in cranelift-codegen * Add reg_name function for x64 * Introduce the concept of a MacroAssembler and Assembler This commit introduces the concept of a MacroAsesembler and Assembler. The MacroAssembler trait will provide a high enough interface across architectures so that each ISA implementation can use their own low-level Assembler implementation to fulfill the interface. Each Assembler will provide a 1-1 mapping to each ISA instruction. As of this commit, only a partial debug implementation is provided for the x64 Assembler. * Add a newtype over PReg Adds a newtype `Reg` over regalloc2::PReg; this ensures that Winch will operate only on the concept of `Reg`. This change is temporary until we have the necessary machinery to share a common Reg abstraction via `cranelift_asm` * Improvements to local calcuation - Add `LocalSlot::addressed_from_sp` - Use `u32` for local slot and local sizes calculation * Add helper methods to ABIArg Adds helper methods to retrieve register and type information from the argument * Make locals_size public in frame * Improve x64 register naming depending on size * Add new methods to the masm interface This commit introduces the ability for the MacroAssembler to reserve stack space, get the address of a given local and perform a stack store based on the concept of `Operand`s. There are several motivating factors to introduce the concept of an Operand: - Make the translation between Winch and Cranelift easier; - Make dispatching from the MacroAssembler to the underlying Assembler - easier by minimizing the amount of functions that we need to define - in order to satisfy the store/load combinations This commit also introduces the concept of a memory address, which essentially describes the addressing modes; as of this commit only one addressing mode is supported. We'll also need to verify that this structure will play nicely with arm64. * Blank masm implementation for arm64 * Implementation of reserve_stack, local_address, store and fp_offset for x64 * Implement function prologue and argument register spilling * Add structopt and wat * Fix debug instruction formatting * Make TargetISA trait publicly accessible * Modify the MacroAssembler finalize siganture to return a slice of strings * Introduce a simple CLI for Winch To be able to compile Wasm programs with Winch independently. Mostly meant for testing / debugging * Fix bug in x64 assembler mov_rm * Remove unused import * Move the stack slot calculation to the Frame This commit moves the calculation of the stack slots to the frame handler abstraction and also includes the calculation of the limits for the function defined locals, which will be used to zero the locals that are not associated to function arguments * Add i32 and i64 constructors to local slots * Introduce the concept of DefinedLocalsRange This commit introduces `DefinedLocalsRange` to track the stack offset at which the function-defined locals start and end; this is later used to zero-out that stack region * Add constructors for int and float registers * Add a placeholder stack implementation * Add a regset abstraction to track register availability Adds a bit set abstraction to track register availability for register allocation. The bit set has no specific knowledge about physical registers, it works on the register's hardware encoding as the source of truth. Each RegSet is expected to be created with the universe of allocatable registers per ISA when starting the compilation of a particular function. * Add an abstraction over register and immediate This is meant to be used as the source for stores. * Add a way to zero local slots and an initial skeletion of regalloc This commit introduces `zero_local_slots` to the MacroAssembler; which ensures that function defined locals are zeroed out when starting the function body. The algorithm divides the defined function locals stack range into 8 byte slots and stores a zero at each address. This process relies on register allocation if the amount of slots that need to be initialized is greater than 1. In such case, the next available register is requested to the register set and it's used to store a 0, which is then stored at every local slot * Update to wasmparser 0.92 * Correctly track if the regset has registers available * Add a result entry to the ABI signature This commuit introduces ABIResult as part of the ABISignature; this struct will track how function results are stored; initially it will consiste of a single register that will be requested to the register allocator at the end of the function; potentially causing a spill * Move zero local slots and add more granular methods to the masm This commit removes zeroing local slots from the MacroAssembler and instead adds more granular methods to it (e.g `zero`, `add`). This allows for better code sharing since most of the work done by the algorithm for zeroing slots will be the same in all targets, except for the binary emissions pieces, which is what gets delegated to the masm * Use wasmparser's visitor API and add initial support for const and add This commit adds initial support for the I32Const and I32 instructions; this involves adding a minimum for register allocation. Note that some regalloc pieces are still incomplete, since for the current set of supported instructions they are not needed. * Make the ty field public in Local * Add scratch_reg to the abi * Add a method to get a particular local from the Frame * Split the compilation environment abstraction This commit splits the compilation environment into two more concise abstractions: 1. CodeGen: the main abstraction for code generation 2. CodeGenContext: abstraction that shares the common pieces for compilation; these pieces are shared between the code generator and the register allocator * Add `push` and `load` to the MacroAssembler * Remove dead code warnings for unused paths * Map ISA features to cranelift-codegen ISA features * Apply formatting * Fix Cargo.toml after a bad rebase * Add component-compiler feature * Use clap instead of structopt * Add winch to publish.rs script * Minor formatting * Add tests to RegSet and fix two bugs when freeing and checking for register availability * Add tests to Stack * Free source register after a non-constant i32 add * Improve comments - Remove unneeded comments - And improve some of the TODO items * Update default features * Drop the ABI generic param and pass the word_size information directly To avoid dealing with dead code warnings this commit passes the word size information directly, since it's the only piece of information needed from the ABI by Codegen until now * Remove dead code This piece of code will be put back once we start integrating Winch with Wasmtime * Remove unused enum variant This variant doesn't get constructed; it should be added back once a backend is added and not enabled by default or when Winch gets integrated into Wasmtime * Fix unused code in regset tests * Update spec testsuite * Switch the visitor pattern for a simpler operator match This commit removes the usage of wasmparser's visitor pattern and instead defaults to a simpler operator matching approach. This removes the complexity of having to define all the visitor trait functions at once. * Use wasmparser's Visitor trait with a different macro strategy This commit puts back wasmparser's Visitor trait, with a sigle; simpler macro, only used for unsupported operators. * Restructure Winch This commit restuructures Winch's parts. It divides the initial approach into three main crates: `winch-codegen`,`wasmtime-winch` and `winch-tools`. `wasmtime-winch` is reponsible for the Wasmtime-Winch integration. `winch-codegen` is solely responsible for code generation. `winch-tools` is CLI tool to compile Wasm programs, mainly for testing purposes. * Refactor zero local slots This commit moves the logic of zeroing local slots from the codegen module into a method with a default implementation in the MacroAssembler trait: `zero_mem_range`. The refactored implementation is very similar to the previous implementation with the only difference that it doesn't allocates a general-purpose register; it instead uses the register allocator to retrieve the scratch register and uses this register to unroll the series of zero stores. * Tie the codegen creation to the ISA ABI This commit makes the relationship between the ISA ABI and the codegen explicit. This allows us to pass down ABI-specific bit and pieces to the codegeneration. In this case the only concrete piece that we need is the ABI word size. * Mark winch as publishable directory * Revamp winch docs This commit ensures that all the code comments in Winch are compliant with the syle used in the rest of Wasmtime's codebase. It also imptoves, generally the quality of the comments in some modules. * Panic when using multi-value when the target is aarch64 Similar to x64, this commit ensures that the abi signature of the current function doesn't use multi-value returns * Document the usage of directives * Use endianness instead of endianess in the ISA trait * Introduce a three-argument form in the MacroAssembler This commit introduces the usage of three-argument form for the MacroAssembler interface. This allows for a natural mapping for architectures like aarch64. In the case of x64, the implementation can simply restrict the implementation asserting for equality in two of the arguments of defaulting to a differnt set of instructions. As of this commit, the implementation of `add` panics if the destination and the first source arguments are not equal; internally the x64 assembler implementation will ensure that all the allowed combinations of `add` are satisfied. The reason for panicking and not emitting a `mov` followed by an `add` for example is simply because register allocation happens right before calling `add`, which ensures any register-to-register moves, if needed. This implementation will evolve in the future and this panic will be lifted if needed. * Improve the documentation for the MacroAssembler. Documents the usage of three-arg form and the intention around the high-level interface. * Format comments in remaining modules * Clean up Cargo.toml for winch pieces This commit adds missing fields to each of Winch's Cargo.toml. * Use `ModuleTranslation::get_types()` to derive the function type * Assert that start range is always word-size aligned
2 years ago
[[package]]
name = "winch-codegen"
version = "0.7.0"
Initial skeleton for Winch (#4907) * Initial skeleton for Winch This commit introduces the initial skeleton for Winch, the "baseline" compiler. This skeleton contains mostly setup code for the ISA, ABI, registers, and compilation environment abstractions. It also includes the calculation of function local slots. As of this commit, the structure of these abstractions looks like the following: +------------------------+ | v +----------+ +-----+ +-----------+-----+-----------------+ | Compiler | --> | ISA | --> | Registers | ABI | Compilation Env | +----------+ +-----+ +-----------+-----+-----------------+ | ^ +------------------------------+ * Compilation environment will hold a reference to the function data * Add basic documentation to the ABI trait * Enable x86 and arm64 in cranelift-codegen * Add reg_name function for x64 * Introduce the concept of a MacroAssembler and Assembler This commit introduces the concept of a MacroAsesembler and Assembler. The MacroAssembler trait will provide a high enough interface across architectures so that each ISA implementation can use their own low-level Assembler implementation to fulfill the interface. Each Assembler will provide a 1-1 mapping to each ISA instruction. As of this commit, only a partial debug implementation is provided for the x64 Assembler. * Add a newtype over PReg Adds a newtype `Reg` over regalloc2::PReg; this ensures that Winch will operate only on the concept of `Reg`. This change is temporary until we have the necessary machinery to share a common Reg abstraction via `cranelift_asm` * Improvements to local calcuation - Add `LocalSlot::addressed_from_sp` - Use `u32` for local slot and local sizes calculation * Add helper methods to ABIArg Adds helper methods to retrieve register and type information from the argument * Make locals_size public in frame * Improve x64 register naming depending on size * Add new methods to the masm interface This commit introduces the ability for the MacroAssembler to reserve stack space, get the address of a given local and perform a stack store based on the concept of `Operand`s. There are several motivating factors to introduce the concept of an Operand: - Make the translation between Winch and Cranelift easier; - Make dispatching from the MacroAssembler to the underlying Assembler - easier by minimizing the amount of functions that we need to define - in order to satisfy the store/load combinations This commit also introduces the concept of a memory address, which essentially describes the addressing modes; as of this commit only one addressing mode is supported. We'll also need to verify that this structure will play nicely with arm64. * Blank masm implementation for arm64 * Implementation of reserve_stack, local_address, store and fp_offset for x64 * Implement function prologue and argument register spilling * Add structopt and wat * Fix debug instruction formatting * Make TargetISA trait publicly accessible * Modify the MacroAssembler finalize siganture to return a slice of strings * Introduce a simple CLI for Winch To be able to compile Wasm programs with Winch independently. Mostly meant for testing / debugging * Fix bug in x64 assembler mov_rm * Remove unused import * Move the stack slot calculation to the Frame This commit moves the calculation of the stack slots to the frame handler abstraction and also includes the calculation of the limits for the function defined locals, which will be used to zero the locals that are not associated to function arguments * Add i32 and i64 constructors to local slots * Introduce the concept of DefinedLocalsRange This commit introduces `DefinedLocalsRange` to track the stack offset at which the function-defined locals start and end; this is later used to zero-out that stack region * Add constructors for int and float registers * Add a placeholder stack implementation * Add a regset abstraction to track register availability Adds a bit set abstraction to track register availability for register allocation. The bit set has no specific knowledge about physical registers, it works on the register's hardware encoding as the source of truth. Each RegSet is expected to be created with the universe of allocatable registers per ISA when starting the compilation of a particular function. * Add an abstraction over register and immediate This is meant to be used as the source for stores. * Add a way to zero local slots and an initial skeletion of regalloc This commit introduces `zero_local_slots` to the MacroAssembler; which ensures that function defined locals are zeroed out when starting the function body. The algorithm divides the defined function locals stack range into 8 byte slots and stores a zero at each address. This process relies on register allocation if the amount of slots that need to be initialized is greater than 1. In such case, the next available register is requested to the register set and it's used to store a 0, which is then stored at every local slot * Update to wasmparser 0.92 * Correctly track if the regset has registers available * Add a result entry to the ABI signature This commuit introduces ABIResult as part of the ABISignature; this struct will track how function results are stored; initially it will consiste of a single register that will be requested to the register allocator at the end of the function; potentially causing a spill * Move zero local slots and add more granular methods to the masm This commit removes zeroing local slots from the MacroAssembler and instead adds more granular methods to it (e.g `zero`, `add`). This allows for better code sharing since most of the work done by the algorithm for zeroing slots will be the same in all targets, except for the binary emissions pieces, which is what gets delegated to the masm * Use wasmparser's visitor API and add initial support for const and add This commit adds initial support for the I32Const and I32 instructions; this involves adding a minimum for register allocation. Note that some regalloc pieces are still incomplete, since for the current set of supported instructions they are not needed. * Make the ty field public in Local * Add scratch_reg to the abi * Add a method to get a particular local from the Frame * Split the compilation environment abstraction This commit splits the compilation environment into two more concise abstractions: 1. CodeGen: the main abstraction for code generation 2. CodeGenContext: abstraction that shares the common pieces for compilation; these pieces are shared between the code generator and the register allocator * Add `push` and `load` to the MacroAssembler * Remove dead code warnings for unused paths * Map ISA features to cranelift-codegen ISA features * Apply formatting * Fix Cargo.toml after a bad rebase * Add component-compiler feature * Use clap instead of structopt * Add winch to publish.rs script * Minor formatting * Add tests to RegSet and fix two bugs when freeing and checking for register availability * Add tests to Stack * Free source register after a non-constant i32 add * Improve comments - Remove unneeded comments - And improve some of the TODO items * Update default features * Drop the ABI generic param and pass the word_size information directly To avoid dealing with dead code warnings this commit passes the word size information directly, since it's the only piece of information needed from the ABI by Codegen until now * Remove dead code This piece of code will be put back once we start integrating Winch with Wasmtime * Remove unused enum variant This variant doesn't get constructed; it should be added back once a backend is added and not enabled by default or when Winch gets integrated into Wasmtime * Fix unused code in regset tests * Update spec testsuite * Switch the visitor pattern for a simpler operator match This commit removes the usage of wasmparser's visitor pattern and instead defaults to a simpler operator matching approach. This removes the complexity of having to define all the visitor trait functions at once. * Use wasmparser's Visitor trait with a different macro strategy This commit puts back wasmparser's Visitor trait, with a sigle; simpler macro, only used for unsupported operators. * Restructure Winch This commit restuructures Winch's parts. It divides the initial approach into three main crates: `winch-codegen`,`wasmtime-winch` and `winch-tools`. `wasmtime-winch` is reponsible for the Wasmtime-Winch integration. `winch-codegen` is solely responsible for code generation. `winch-tools` is CLI tool to compile Wasm programs, mainly for testing purposes. * Refactor zero local slots This commit moves the logic of zeroing local slots from the codegen module into a method with a default implementation in the MacroAssembler trait: `zero_mem_range`. The refactored implementation is very similar to the previous implementation with the only difference that it doesn't allocates a general-purpose register; it instead uses the register allocator to retrieve the scratch register and uses this register to unroll the series of zero stores. * Tie the codegen creation to the ISA ABI This commit makes the relationship between the ISA ABI and the codegen explicit. This allows us to pass down ABI-specific bit and pieces to the codegeneration. In this case the only concrete piece that we need is the ABI word size. * Mark winch as publishable directory * Revamp winch docs This commit ensures that all the code comments in Winch are compliant with the syle used in the rest of Wasmtime's codebase. It also imptoves, generally the quality of the comments in some modules. * Panic when using multi-value when the target is aarch64 Similar to x64, this commit ensures that the abi signature of the current function doesn't use multi-value returns * Document the usage of directives * Use endianness instead of endianess in the ISA trait * Introduce a three-argument form in the MacroAssembler This commit introduces the usage of three-argument form for the MacroAssembler interface. This allows for a natural mapping for architectures like aarch64. In the case of x64, the implementation can simply restrict the implementation asserting for equality in two of the arguments of defaulting to a differnt set of instructions. As of this commit, the implementation of `add` panics if the destination and the first source arguments are not equal; internally the x64 assembler implementation will ensure that all the allowed combinations of `add` are satisfied. The reason for panicking and not emitting a `mov` followed by an `add` for example is simply because register allocation happens right before calling `add`, which ensures any register-to-register moves, if needed. This implementation will evolve in the future and this panic will be lifted if needed. * Improve the documentation for the MacroAssembler. Documents the usage of three-arg form and the intention around the high-level interface. * Format comments in remaining modules * Clean up Cargo.toml for winch pieces This commit adds missing fields to each of Winch's Cargo.toml. * Use `ModuleTranslation::get_types()` to derive the function type * Assert that start range is always word-size aligned
2 years ago
dependencies = [
"anyhow",
"cranelift-codegen",
winch: Refactoring wasmtime compiler integration pieces to share more between Cranelift and Winch (#5944) * Enable the native target by default in winch Match cranelift-codegen's build script where if no architecture is explicitly enabled then the host architecture is implicitly enabled. * Refactor Cranelift's ISA builder to share more with Winch This commit refactors the `Builder` type to have a type parameter representing the finished ISA with Cranelift and Winch having their own typedefs for `Builder` to represent their own builders. The intention is to use this shared functionality to produce more shared code between the two codegen backends. * Moving compiler shared components to a separate crate * Restore native flag inference in compiler building This fixes an oversight from the previous commits to use `cranelift-native` to infer flags for the native host when using default settings with Wasmtime. * Move `Compiler::page_size_align` into wasmtime-environ The `cranelift-codegen` crate doesn't need this and winch wants the same implementation, so shuffle it around so everyone has access to it. * Fill out `Compiler::{flags, isa_flags}` for Winch These are easy enough to plumb through with some shared code for Wasmtime. * Plumb the `is_branch_protection_enabled` flag for Winch Just forwarding an isa-specific setting accessor. * Moving executable creation to shared compiler crate * Adding builder back in and removing from shared crate * Refactoring the shared pieces for the `CompilerBuilder` I decided to move a couple things around from Alex's initial changes. Instead of having the shared builder do everything, I went back to having each compiler have a distinct builder implementation. I refactored most of the flag setting logic into a single shared location, so we can still reduce the amount of code duplication. With them being separate, we don't need to maintain things like `LinkOpts` which Winch doesn't currently use. We also have an avenue to error when certain flags are sent to Winch if we don't support them. I'm hoping this will make things more maintainable as we build out Winch. I'm still unsure about keeping everything shared in a single crate (`cranelift_shared`). It's starting to feel like this crate is doing too much, which makes it difficult to name. There does seem to be a need for two distinct abstraction: creating the final executable and the handling of shared/ISA flags when building the compiler. I could make them into two separate crates, but there doesn't seem to be enough there yet to justify it. * Documentation updates, and renaming the finish method * Adding back in a default temporarily to pass tests, and removing some unused imports * Fixing winch tests with wrong method name * Removing unused imports from codegen shared crate * Apply documentation formatting updates Co-authored-by: Saúl Cabrera <saulecabrera@gmail.com> * Adding back in cranelift_native flag inferring * Adding new shared crate to publish list * Adding write feature to pass cargo check --------- Co-authored-by: Alex Crichton <alex@alexcrichton.com> Co-authored-by: Saúl Cabrera <saulecabrera@gmail.com>
2 years ago
"gimli",
Initial skeleton for Winch (#4907) * Initial skeleton for Winch This commit introduces the initial skeleton for Winch, the "baseline" compiler. This skeleton contains mostly setup code for the ISA, ABI, registers, and compilation environment abstractions. It also includes the calculation of function local slots. As of this commit, the structure of these abstractions looks like the following: +------------------------+ | v +----------+ +-----+ +-----------+-----+-----------------+ | Compiler | --> | ISA | --> | Registers | ABI | Compilation Env | +----------+ +-----+ +-----------+-----+-----------------+ | ^ +------------------------------+ * Compilation environment will hold a reference to the function data * Add basic documentation to the ABI trait * Enable x86 and arm64 in cranelift-codegen * Add reg_name function for x64 * Introduce the concept of a MacroAssembler and Assembler This commit introduces the concept of a MacroAsesembler and Assembler. The MacroAssembler trait will provide a high enough interface across architectures so that each ISA implementation can use their own low-level Assembler implementation to fulfill the interface. Each Assembler will provide a 1-1 mapping to each ISA instruction. As of this commit, only a partial debug implementation is provided for the x64 Assembler. * Add a newtype over PReg Adds a newtype `Reg` over regalloc2::PReg; this ensures that Winch will operate only on the concept of `Reg`. This change is temporary until we have the necessary machinery to share a common Reg abstraction via `cranelift_asm` * Improvements to local calcuation - Add `LocalSlot::addressed_from_sp` - Use `u32` for local slot and local sizes calculation * Add helper methods to ABIArg Adds helper methods to retrieve register and type information from the argument * Make locals_size public in frame * Improve x64 register naming depending on size * Add new methods to the masm interface This commit introduces the ability for the MacroAssembler to reserve stack space, get the address of a given local and perform a stack store based on the concept of `Operand`s. There are several motivating factors to introduce the concept of an Operand: - Make the translation between Winch and Cranelift easier; - Make dispatching from the MacroAssembler to the underlying Assembler - easier by minimizing the amount of functions that we need to define - in order to satisfy the store/load combinations This commit also introduces the concept of a memory address, which essentially describes the addressing modes; as of this commit only one addressing mode is supported. We'll also need to verify that this structure will play nicely with arm64. * Blank masm implementation for arm64 * Implementation of reserve_stack, local_address, store and fp_offset for x64 * Implement function prologue and argument register spilling * Add structopt and wat * Fix debug instruction formatting * Make TargetISA trait publicly accessible * Modify the MacroAssembler finalize siganture to return a slice of strings * Introduce a simple CLI for Winch To be able to compile Wasm programs with Winch independently. Mostly meant for testing / debugging * Fix bug in x64 assembler mov_rm * Remove unused import * Move the stack slot calculation to the Frame This commit moves the calculation of the stack slots to the frame handler abstraction and also includes the calculation of the limits for the function defined locals, which will be used to zero the locals that are not associated to function arguments * Add i32 and i64 constructors to local slots * Introduce the concept of DefinedLocalsRange This commit introduces `DefinedLocalsRange` to track the stack offset at which the function-defined locals start and end; this is later used to zero-out that stack region * Add constructors for int and float registers * Add a placeholder stack implementation * Add a regset abstraction to track register availability Adds a bit set abstraction to track register availability for register allocation. The bit set has no specific knowledge about physical registers, it works on the register's hardware encoding as the source of truth. Each RegSet is expected to be created with the universe of allocatable registers per ISA when starting the compilation of a particular function. * Add an abstraction over register and immediate This is meant to be used as the source for stores. * Add a way to zero local slots and an initial skeletion of regalloc This commit introduces `zero_local_slots` to the MacroAssembler; which ensures that function defined locals are zeroed out when starting the function body. The algorithm divides the defined function locals stack range into 8 byte slots and stores a zero at each address. This process relies on register allocation if the amount of slots that need to be initialized is greater than 1. In such case, the next available register is requested to the register set and it's used to store a 0, which is then stored at every local slot * Update to wasmparser 0.92 * Correctly track if the regset has registers available * Add a result entry to the ABI signature This commuit introduces ABIResult as part of the ABISignature; this struct will track how function results are stored; initially it will consiste of a single register that will be requested to the register allocator at the end of the function; potentially causing a spill * Move zero local slots and add more granular methods to the masm This commit removes zeroing local slots from the MacroAssembler and instead adds more granular methods to it (e.g `zero`, `add`). This allows for better code sharing since most of the work done by the algorithm for zeroing slots will be the same in all targets, except for the binary emissions pieces, which is what gets delegated to the masm * Use wasmparser's visitor API and add initial support for const and add This commit adds initial support for the I32Const and I32 instructions; this involves adding a minimum for register allocation. Note that some regalloc pieces are still incomplete, since for the current set of supported instructions they are not needed. * Make the ty field public in Local * Add scratch_reg to the abi * Add a method to get a particular local from the Frame * Split the compilation environment abstraction This commit splits the compilation environment into two more concise abstractions: 1. CodeGen: the main abstraction for code generation 2. CodeGenContext: abstraction that shares the common pieces for compilation; these pieces are shared between the code generator and the register allocator * Add `push` and `load` to the MacroAssembler * Remove dead code warnings for unused paths * Map ISA features to cranelift-codegen ISA features * Apply formatting * Fix Cargo.toml after a bad rebase * Add component-compiler feature * Use clap instead of structopt * Add winch to publish.rs script * Minor formatting * Add tests to RegSet and fix two bugs when freeing and checking for register availability * Add tests to Stack * Free source register after a non-constant i32 add * Improve comments - Remove unneeded comments - And improve some of the TODO items * Update default features * Drop the ABI generic param and pass the word_size information directly To avoid dealing with dead code warnings this commit passes the word size information directly, since it's the only piece of information needed from the ABI by Codegen until now * Remove dead code This piece of code will be put back once we start integrating Winch with Wasmtime * Remove unused enum variant This variant doesn't get constructed; it should be added back once a backend is added and not enabled by default or when Winch gets integrated into Wasmtime * Fix unused code in regset tests * Update spec testsuite * Switch the visitor pattern for a simpler operator match This commit removes the usage of wasmparser's visitor pattern and instead defaults to a simpler operator matching approach. This removes the complexity of having to define all the visitor trait functions at once. * Use wasmparser's Visitor trait with a different macro strategy This commit puts back wasmparser's Visitor trait, with a sigle; simpler macro, only used for unsupported operators. * Restructure Winch This commit restuructures Winch's parts. It divides the initial approach into three main crates: `winch-codegen`,`wasmtime-winch` and `winch-tools`. `wasmtime-winch` is reponsible for the Wasmtime-Winch integration. `winch-codegen` is solely responsible for code generation. `winch-tools` is CLI tool to compile Wasm programs, mainly for testing purposes. * Refactor zero local slots This commit moves the logic of zeroing local slots from the codegen module into a method with a default implementation in the MacroAssembler trait: `zero_mem_range`. The refactored implementation is very similar to the previous implementation with the only difference that it doesn't allocates a general-purpose register; it instead uses the register allocator to retrieve the scratch register and uses this register to unroll the series of zero stores. * Tie the codegen creation to the ISA ABI This commit makes the relationship between the ISA ABI and the codegen explicit. This allows us to pass down ABI-specific bit and pieces to the codegeneration. In this case the only concrete piece that we need is the ABI word size. * Mark winch as publishable directory * Revamp winch docs This commit ensures that all the code comments in Winch are compliant with the syle used in the rest of Wasmtime's codebase. It also imptoves, generally the quality of the comments in some modules. * Panic when using multi-value when the target is aarch64 Similar to x64, this commit ensures that the abi signature of the current function doesn't use multi-value returns * Document the usage of directives * Use endianness instead of endianess in the ISA trait * Introduce a three-argument form in the MacroAssembler This commit introduces the usage of three-argument form for the MacroAssembler interface. This allows for a natural mapping for architectures like aarch64. In the case of x64, the implementation can simply restrict the implementation asserting for equality in two of the arguments of defaulting to a differnt set of instructions. As of this commit, the implementation of `add` panics if the destination and the first source arguments are not equal; internally the x64 assembler implementation will ensure that all the allowed combinations of `add` are satisfied. The reason for panicking and not emitting a `mov` followed by an `add` for example is simply because register allocation happens right before calling `add`, which ensures any register-to-register moves, if needed. This implementation will evolve in the future and this panic will be lifted if needed. * Improve the documentation for the MacroAssembler. Documents the usage of three-arg form and the intention around the high-level interface. * Format comments in remaining modules * Clean up Cargo.toml for winch pieces This commit adds missing fields to each of Winch's Cargo.toml. * Use `ModuleTranslation::get_types()` to derive the function type * Assert that start range is always word-size aligned
2 years ago
"regalloc2",
"smallvec",
"target-lexicon",
"wasmparser",
]
[[package]]
name = "winch-environ"
version = "0.7.0"
dependencies = [
"wasmparser",
"wasmtime-environ",
"winch-codegen",
]
[[package]]
name = "winch-filetests"
version = "0.0.0"
dependencies = [
"anyhow",
"capstone",
"cranelift-codegen",
"serde",
"similar",
"target-lexicon",
"toml",
"wasmtime-environ",
"wat",
"winch-codegen",
"winch-environ",
"winch-test-macros",
]
[[package]]
name = "winch-test-macros"
version = "0.0.0"
dependencies = [
"glob",
"proc-macro2",
"quote",
"syn",
]
Initial skeleton for Winch (#4907) * Initial skeleton for Winch This commit introduces the initial skeleton for Winch, the "baseline" compiler. This skeleton contains mostly setup code for the ISA, ABI, registers, and compilation environment abstractions. It also includes the calculation of function local slots. As of this commit, the structure of these abstractions looks like the following: +------------------------+ | v +----------+ +-----+ +-----------+-----+-----------------+ | Compiler | --> | ISA | --> | Registers | ABI | Compilation Env | +----------+ +-----+ +-----------+-----+-----------------+ | ^ +------------------------------+ * Compilation environment will hold a reference to the function data * Add basic documentation to the ABI trait * Enable x86 and arm64 in cranelift-codegen * Add reg_name function for x64 * Introduce the concept of a MacroAssembler and Assembler This commit introduces the concept of a MacroAsesembler and Assembler. The MacroAssembler trait will provide a high enough interface across architectures so that each ISA implementation can use their own low-level Assembler implementation to fulfill the interface. Each Assembler will provide a 1-1 mapping to each ISA instruction. As of this commit, only a partial debug implementation is provided for the x64 Assembler. * Add a newtype over PReg Adds a newtype `Reg` over regalloc2::PReg; this ensures that Winch will operate only on the concept of `Reg`. This change is temporary until we have the necessary machinery to share a common Reg abstraction via `cranelift_asm` * Improvements to local calcuation - Add `LocalSlot::addressed_from_sp` - Use `u32` for local slot and local sizes calculation * Add helper methods to ABIArg Adds helper methods to retrieve register and type information from the argument * Make locals_size public in frame * Improve x64 register naming depending on size * Add new methods to the masm interface This commit introduces the ability for the MacroAssembler to reserve stack space, get the address of a given local and perform a stack store based on the concept of `Operand`s. There are several motivating factors to introduce the concept of an Operand: - Make the translation between Winch and Cranelift easier; - Make dispatching from the MacroAssembler to the underlying Assembler - easier by minimizing the amount of functions that we need to define - in order to satisfy the store/load combinations This commit also introduces the concept of a memory address, which essentially describes the addressing modes; as of this commit only one addressing mode is supported. We'll also need to verify that this structure will play nicely with arm64. * Blank masm implementation for arm64 * Implementation of reserve_stack, local_address, store and fp_offset for x64 * Implement function prologue and argument register spilling * Add structopt and wat * Fix debug instruction formatting * Make TargetISA trait publicly accessible * Modify the MacroAssembler finalize siganture to return a slice of strings * Introduce a simple CLI for Winch To be able to compile Wasm programs with Winch independently. Mostly meant for testing / debugging * Fix bug in x64 assembler mov_rm * Remove unused import * Move the stack slot calculation to the Frame This commit moves the calculation of the stack slots to the frame handler abstraction and also includes the calculation of the limits for the function defined locals, which will be used to zero the locals that are not associated to function arguments * Add i32 and i64 constructors to local slots * Introduce the concept of DefinedLocalsRange This commit introduces `DefinedLocalsRange` to track the stack offset at which the function-defined locals start and end; this is later used to zero-out that stack region * Add constructors for int and float registers * Add a placeholder stack implementation * Add a regset abstraction to track register availability Adds a bit set abstraction to track register availability for register allocation. The bit set has no specific knowledge about physical registers, it works on the register's hardware encoding as the source of truth. Each RegSet is expected to be created with the universe of allocatable registers per ISA when starting the compilation of a particular function. * Add an abstraction over register and immediate This is meant to be used as the source for stores. * Add a way to zero local slots and an initial skeletion of regalloc This commit introduces `zero_local_slots` to the MacroAssembler; which ensures that function defined locals are zeroed out when starting the function body. The algorithm divides the defined function locals stack range into 8 byte slots and stores a zero at each address. This process relies on register allocation if the amount of slots that need to be initialized is greater than 1. In such case, the next available register is requested to the register set and it's used to store a 0, which is then stored at every local slot * Update to wasmparser 0.92 * Correctly track if the regset has registers available * Add a result entry to the ABI signature This commuit introduces ABIResult as part of the ABISignature; this struct will track how function results are stored; initially it will consiste of a single register that will be requested to the register allocator at the end of the function; potentially causing a spill * Move zero local slots and add more granular methods to the masm This commit removes zeroing local slots from the MacroAssembler and instead adds more granular methods to it (e.g `zero`, `add`). This allows for better code sharing since most of the work done by the algorithm for zeroing slots will be the same in all targets, except for the binary emissions pieces, which is what gets delegated to the masm * Use wasmparser's visitor API and add initial support for const and add This commit adds initial support for the I32Const and I32 instructions; this involves adding a minimum for register allocation. Note that some regalloc pieces are still incomplete, since for the current set of supported instructions they are not needed. * Make the ty field public in Local * Add scratch_reg to the abi * Add a method to get a particular local from the Frame * Split the compilation environment abstraction This commit splits the compilation environment into two more concise abstractions: 1. CodeGen: the main abstraction for code generation 2. CodeGenContext: abstraction that shares the common pieces for compilation; these pieces are shared between the code generator and the register allocator * Add `push` and `load` to the MacroAssembler * Remove dead code warnings for unused paths * Map ISA features to cranelift-codegen ISA features * Apply formatting * Fix Cargo.toml after a bad rebase * Add component-compiler feature * Use clap instead of structopt * Add winch to publish.rs script * Minor formatting * Add tests to RegSet and fix two bugs when freeing and checking for register availability * Add tests to Stack * Free source register after a non-constant i32 add * Improve comments - Remove unneeded comments - And improve some of the TODO items * Update default features * Drop the ABI generic param and pass the word_size information directly To avoid dealing with dead code warnings this commit passes the word size information directly, since it's the only piece of information needed from the ABI by Codegen until now * Remove dead code This piece of code will be put back once we start integrating Winch with Wasmtime * Remove unused enum variant This variant doesn't get constructed; it should be added back once a backend is added and not enabled by default or when Winch gets integrated into Wasmtime * Fix unused code in regset tests * Update spec testsuite * Switch the visitor pattern for a simpler operator match This commit removes the usage of wasmparser's visitor pattern and instead defaults to a simpler operator matching approach. This removes the complexity of having to define all the visitor trait functions at once. * Use wasmparser's Visitor trait with a different macro strategy This commit puts back wasmparser's Visitor trait, with a sigle; simpler macro, only used for unsupported operators. * Restructure Winch This commit restuructures Winch's parts. It divides the initial approach into three main crates: `winch-codegen`,`wasmtime-winch` and `winch-tools`. `wasmtime-winch` is reponsible for the Wasmtime-Winch integration. `winch-codegen` is solely responsible for code generation. `winch-tools` is CLI tool to compile Wasm programs, mainly for testing purposes. * Refactor zero local slots This commit moves the logic of zeroing local slots from the codegen module into a method with a default implementation in the MacroAssembler trait: `zero_mem_range`. The refactored implementation is very similar to the previous implementation with the only difference that it doesn't allocates a general-purpose register; it instead uses the register allocator to retrieve the scratch register and uses this register to unroll the series of zero stores. * Tie the codegen creation to the ISA ABI This commit makes the relationship between the ISA ABI and the codegen explicit. This allows us to pass down ABI-specific bit and pieces to the codegeneration. In this case the only concrete piece that we need is the ABI word size. * Mark winch as publishable directory * Revamp winch docs This commit ensures that all the code comments in Winch are compliant with the syle used in the rest of Wasmtime's codebase. It also imptoves, generally the quality of the comments in some modules. * Panic when using multi-value when the target is aarch64 Similar to x64, this commit ensures that the abi signature of the current function doesn't use multi-value returns * Document the usage of directives * Use endianness instead of endianess in the ISA trait * Introduce a three-argument form in the MacroAssembler This commit introduces the usage of three-argument form for the MacroAssembler interface. This allows for a natural mapping for architectures like aarch64. In the case of x64, the implementation can simply restrict the implementation asserting for equality in two of the arguments of defaulting to a differnt set of instructions. As of this commit, the implementation of `add` panics if the destination and the first source arguments are not equal; internally the x64 assembler implementation will ensure that all the allowed combinations of `add` are satisfied. The reason for panicking and not emitting a `mov` followed by an `add` for example is simply because register allocation happens right before calling `add`, which ensures any register-to-register moves, if needed. This implementation will evolve in the future and this panic will be lifted if needed. * Improve the documentation for the MacroAssembler. Documents the usage of three-arg form and the intention around the high-level interface. * Format comments in remaining modules * Clean up Cargo.toml for winch pieces This commit adds missing fields to each of Winch's Cargo.toml. * Use `ModuleTranslation::get_types()` to derive the function type * Assert that start range is always word-size aligned
2 years ago
[[package]]
name = "winch-tools"
version = "0.0.0"
Initial skeleton for Winch (#4907) * Initial skeleton for Winch This commit introduces the initial skeleton for Winch, the "baseline" compiler. This skeleton contains mostly setup code for the ISA, ABI, registers, and compilation environment abstractions. It also includes the calculation of function local slots. As of this commit, the structure of these abstractions looks like the following: +------------------------+ | v +----------+ +-----+ +-----------+-----+-----------------+ | Compiler | --> | ISA | --> | Registers | ABI | Compilation Env | +----------+ +-----+ +-----------+-----+-----------------+ | ^ +------------------------------+ * Compilation environment will hold a reference to the function data * Add basic documentation to the ABI trait * Enable x86 and arm64 in cranelift-codegen * Add reg_name function for x64 * Introduce the concept of a MacroAssembler and Assembler This commit introduces the concept of a MacroAsesembler and Assembler. The MacroAssembler trait will provide a high enough interface across architectures so that each ISA implementation can use their own low-level Assembler implementation to fulfill the interface. Each Assembler will provide a 1-1 mapping to each ISA instruction. As of this commit, only a partial debug implementation is provided for the x64 Assembler. * Add a newtype over PReg Adds a newtype `Reg` over regalloc2::PReg; this ensures that Winch will operate only on the concept of `Reg`. This change is temporary until we have the necessary machinery to share a common Reg abstraction via `cranelift_asm` * Improvements to local calcuation - Add `LocalSlot::addressed_from_sp` - Use `u32` for local slot and local sizes calculation * Add helper methods to ABIArg Adds helper methods to retrieve register and type information from the argument * Make locals_size public in frame * Improve x64 register naming depending on size * Add new methods to the masm interface This commit introduces the ability for the MacroAssembler to reserve stack space, get the address of a given local and perform a stack store based on the concept of `Operand`s. There are several motivating factors to introduce the concept of an Operand: - Make the translation between Winch and Cranelift easier; - Make dispatching from the MacroAssembler to the underlying Assembler - easier by minimizing the amount of functions that we need to define - in order to satisfy the store/load combinations This commit also introduces the concept of a memory address, which essentially describes the addressing modes; as of this commit only one addressing mode is supported. We'll also need to verify that this structure will play nicely with arm64. * Blank masm implementation for arm64 * Implementation of reserve_stack, local_address, store and fp_offset for x64 * Implement function prologue and argument register spilling * Add structopt and wat * Fix debug instruction formatting * Make TargetISA trait publicly accessible * Modify the MacroAssembler finalize siganture to return a slice of strings * Introduce a simple CLI for Winch To be able to compile Wasm programs with Winch independently. Mostly meant for testing / debugging * Fix bug in x64 assembler mov_rm * Remove unused import * Move the stack slot calculation to the Frame This commit moves the calculation of the stack slots to the frame handler abstraction and also includes the calculation of the limits for the function defined locals, which will be used to zero the locals that are not associated to function arguments * Add i32 and i64 constructors to local slots * Introduce the concept of DefinedLocalsRange This commit introduces `DefinedLocalsRange` to track the stack offset at which the function-defined locals start and end; this is later used to zero-out that stack region * Add constructors for int and float registers * Add a placeholder stack implementation * Add a regset abstraction to track register availability Adds a bit set abstraction to track register availability for register allocation. The bit set has no specific knowledge about physical registers, it works on the register's hardware encoding as the source of truth. Each RegSet is expected to be created with the universe of allocatable registers per ISA when starting the compilation of a particular function. * Add an abstraction over register and immediate This is meant to be used as the source for stores. * Add a way to zero local slots and an initial skeletion of regalloc This commit introduces `zero_local_slots` to the MacroAssembler; which ensures that function defined locals are zeroed out when starting the function body. The algorithm divides the defined function locals stack range into 8 byte slots and stores a zero at each address. This process relies on register allocation if the amount of slots that need to be initialized is greater than 1. In such case, the next available register is requested to the register set and it's used to store a 0, which is then stored at every local slot * Update to wasmparser 0.92 * Correctly track if the regset has registers available * Add a result entry to the ABI signature This commuit introduces ABIResult as part of the ABISignature; this struct will track how function results are stored; initially it will consiste of a single register that will be requested to the register allocator at the end of the function; potentially causing a spill * Move zero local slots and add more granular methods to the masm This commit removes zeroing local slots from the MacroAssembler and instead adds more granular methods to it (e.g `zero`, `add`). This allows for better code sharing since most of the work done by the algorithm for zeroing slots will be the same in all targets, except for the binary emissions pieces, which is what gets delegated to the masm * Use wasmparser's visitor API and add initial support for const and add This commit adds initial support for the I32Const and I32 instructions; this involves adding a minimum for register allocation. Note that some regalloc pieces are still incomplete, since for the current set of supported instructions they are not needed. * Make the ty field public in Local * Add scratch_reg to the abi * Add a method to get a particular local from the Frame * Split the compilation environment abstraction This commit splits the compilation environment into two more concise abstractions: 1. CodeGen: the main abstraction for code generation 2. CodeGenContext: abstraction that shares the common pieces for compilation; these pieces are shared between the code generator and the register allocator * Add `push` and `load` to the MacroAssembler * Remove dead code warnings for unused paths * Map ISA features to cranelift-codegen ISA features * Apply formatting * Fix Cargo.toml after a bad rebase * Add component-compiler feature * Use clap instead of structopt * Add winch to publish.rs script * Minor formatting * Add tests to RegSet and fix two bugs when freeing and checking for register availability * Add tests to Stack * Free source register after a non-constant i32 add * Improve comments - Remove unneeded comments - And improve some of the TODO items * Update default features * Drop the ABI generic param and pass the word_size information directly To avoid dealing with dead code warnings this commit passes the word size information directly, since it's the only piece of information needed from the ABI by Codegen until now * Remove dead code This piece of code will be put back once we start integrating Winch with Wasmtime * Remove unused enum variant This variant doesn't get constructed; it should be added back once a backend is added and not enabled by default or when Winch gets integrated into Wasmtime * Fix unused code in regset tests * Update spec testsuite * Switch the visitor pattern for a simpler operator match This commit removes the usage of wasmparser's visitor pattern and instead defaults to a simpler operator matching approach. This removes the complexity of having to define all the visitor trait functions at once. * Use wasmparser's Visitor trait with a different macro strategy This commit puts back wasmparser's Visitor trait, with a sigle; simpler macro, only used for unsupported operators. * Restructure Winch This commit restuructures Winch's parts. It divides the initial approach into three main crates: `winch-codegen`,`wasmtime-winch` and `winch-tools`. `wasmtime-winch` is reponsible for the Wasmtime-Winch integration. `winch-codegen` is solely responsible for code generation. `winch-tools` is CLI tool to compile Wasm programs, mainly for testing purposes. * Refactor zero local slots This commit moves the logic of zeroing local slots from the codegen module into a method with a default implementation in the MacroAssembler trait: `zero_mem_range`. The refactored implementation is very similar to the previous implementation with the only difference that it doesn't allocates a general-purpose register; it instead uses the register allocator to retrieve the scratch register and uses this register to unroll the series of zero stores. * Tie the codegen creation to the ISA ABI This commit makes the relationship between the ISA ABI and the codegen explicit. This allows us to pass down ABI-specific bit and pieces to the codegeneration. In this case the only concrete piece that we need is the ABI word size. * Mark winch as publishable directory * Revamp winch docs This commit ensures that all the code comments in Winch are compliant with the syle used in the rest of Wasmtime's codebase. It also imptoves, generally the quality of the comments in some modules. * Panic when using multi-value when the target is aarch64 Similar to x64, this commit ensures that the abi signature of the current function doesn't use multi-value returns * Document the usage of directives * Use endianness instead of endianess in the ISA trait * Introduce a three-argument form in the MacroAssembler This commit introduces the usage of three-argument form for the MacroAssembler interface. This allows for a natural mapping for architectures like aarch64. In the case of x64, the implementation can simply restrict the implementation asserting for equality in two of the arguments of defaulting to a differnt set of instructions. As of this commit, the implementation of `add` panics if the destination and the first source arguments are not equal; internally the x64 assembler implementation will ensure that all the allowed combinations of `add` are satisfied. The reason for panicking and not emitting a `mov` followed by an `add` for example is simply because register allocation happens right before calling `add`, which ensures any register-to-register moves, if needed. This implementation will evolve in the future and this panic will be lifted if needed. * Improve the documentation for the MacroAssembler. Documents the usage of three-arg form and the intention around the high-level interface. * Format comments in remaining modules * Clean up Cargo.toml for winch pieces This commit adds missing fields to each of Winch's Cargo.toml. * Use `ModuleTranslation::get_types()` to derive the function type * Assert that start range is always word-size aligned
2 years ago
dependencies = [
"anyhow",
winch: Use cranelift-codegen x64 backend for emission. (#5581) This change substitutes the string based emission mechanism with cranelift-codegen's x64 backend. This change _does not_: * Introduce new functionality in terms of supported instructions. * Change the semantics of the assembler/macroassembler in terms of the logic to emit instructions. The most notable differences between this change and the previous version are: * Handling of shared flags and ISA-specific flags, which for now are left with the default value. * Simplification of instruction emission per operand size: previously the assembler defined different methods depending on the operand size (e.g. `mov` for 64 bits, and `movl` for 32 bits). This change updates such approach so that each assembler method takes an operand size as a parameter, reducing duplication and making the code more concise and easier to integrate with the x64's `Inst` enum. * Introduction of a disassembler for testing purposes. As of this change, Winch generates the following code for the following test programs: ```wat (module (export "main" (func $main)) (func $main (result i32) (i32.const 10) (i32.const 20) i32.add )) ``` ```asm 0: 55 push rbp 1: 48 89 e5 mov rbp, rsp 4: b8 0a 00 00 00 mov eax, 0xa 9: 83 c0 14 add eax, 0x14 c: 5d pop rbp d: c3 ret ``` ```wat (module (export "main" (func $main)) (func $main (result i32) (local $foo i32) (local $bar i32) (i32.const 10) (local.set $foo) (i32.const 20) (local.set $bar) (local.get $foo) (local.get $bar) i32.add )) ``` ```asm 0: 55 push rbp 1: 48 89 e5 mov rbp, rsp 4: 48 83 ec 08 sub rsp, 8 8: 48 c7 04 24 00 00 00 00 mov qword ptr [rsp], 0 10: b8 0a 00 00 00 mov eax, 0xa 15: 89 44 24 04 mov dword ptr [rsp + 4], eax 19: b8 14 00 00 00 mov eax, 0x14 1e: 89 04 24 mov dword ptr [rsp], eax 21: 8b 04 24 mov eax, dword ptr [rsp] 24: 8b 4c 24 04 mov ecx, dword ptr [rsp + 4] 28: 01 c1 add ecx, eax 2a: 48 89 c8 mov rax, rcx 2d: 48 83 c4 08 add rsp, 8 31: 5d pop rbp 32: c3 ret ``` ```wat (module (export "main" (func $main)) (func $main (param i32) (param i32) (result i32) (local.get 0) (local.get 1) i32.add )) ``` ```asm 0: 55 push rbp 1: 48 89 e5 mov rbp, rsp 4: 48 83 ec 08 sub rsp, 8 8: 89 7c 24 04 mov dword ptr [rsp + 4], edi c: 89 34 24 mov dword ptr [rsp], esi f: 8b 04 24 mov eax, dword ptr [rsp] 12: 8b 4c 24 04 mov ecx, dword ptr [rsp + 4] 16: 01 c1 add ecx, eax 18: 48 89 c8 mov rax, rcx 1b: 48 83 c4 08 add rsp, 8 1f: 5d pop rbp 20: c3 ret ```
2 years ago
"capstone",
"clap",
winch: Use cranelift-codegen x64 backend for emission. (#5581) This change substitutes the string based emission mechanism with cranelift-codegen's x64 backend. This change _does not_: * Introduce new functionality in terms of supported instructions. * Change the semantics of the assembler/macroassembler in terms of the logic to emit instructions. The most notable differences between this change and the previous version are: * Handling of shared flags and ISA-specific flags, which for now are left with the default value. * Simplification of instruction emission per operand size: previously the assembler defined different methods depending on the operand size (e.g. `mov` for 64 bits, and `movl` for 32 bits). This change updates such approach so that each assembler method takes an operand size as a parameter, reducing duplication and making the code more concise and easier to integrate with the x64's `Inst` enum. * Introduction of a disassembler for testing purposes. As of this change, Winch generates the following code for the following test programs: ```wat (module (export "main" (func $main)) (func $main (result i32) (i32.const 10) (i32.const 20) i32.add )) ``` ```asm 0: 55 push rbp 1: 48 89 e5 mov rbp, rsp 4: b8 0a 00 00 00 mov eax, 0xa 9: 83 c0 14 add eax, 0x14 c: 5d pop rbp d: c3 ret ``` ```wat (module (export "main" (func $main)) (func $main (result i32) (local $foo i32) (local $bar i32) (i32.const 10) (local.set $foo) (i32.const 20) (local.set $bar) (local.get $foo) (local.get $bar) i32.add )) ``` ```asm 0: 55 push rbp 1: 48 89 e5 mov rbp, rsp 4: 48 83 ec 08 sub rsp, 8 8: 48 c7 04 24 00 00 00 00 mov qword ptr [rsp], 0 10: b8 0a 00 00 00 mov eax, 0xa 15: 89 44 24 04 mov dword ptr [rsp + 4], eax 19: b8 14 00 00 00 mov eax, 0x14 1e: 89 04 24 mov dword ptr [rsp], eax 21: 8b 04 24 mov eax, dword ptr [rsp] 24: 8b 4c 24 04 mov ecx, dword ptr [rsp + 4] 28: 01 c1 add ecx, eax 2a: 48 89 c8 mov rax, rcx 2d: 48 83 c4 08 add rsp, 8 31: 5d pop rbp 32: c3 ret ``` ```wat (module (export "main" (func $main)) (func $main (param i32) (param i32) (result i32) (local.get 0) (local.get 1) i32.add )) ``` ```asm 0: 55 push rbp 1: 48 89 e5 mov rbp, rsp 4: 48 83 ec 08 sub rsp, 8 8: 89 7c 24 04 mov dword ptr [rsp + 4], edi c: 89 34 24 mov dword ptr [rsp], esi f: 8b 04 24 mov eax, dword ptr [rsp] 12: 8b 4c 24 04 mov ecx, dword ptr [rsp + 4] 16: 01 c1 add ecx, eax 18: 48 89 c8 mov rax, rcx 1b: 48 83 c4 08 add rsp, 8 1f: 5d pop rbp 20: c3 ret ```
2 years ago
"cranelift-codegen",
"glob",
"serde",
"similar",
Initial skeleton for Winch (#4907) * Initial skeleton for Winch This commit introduces the initial skeleton for Winch, the "baseline" compiler. This skeleton contains mostly setup code for the ISA, ABI, registers, and compilation environment abstractions. It also includes the calculation of function local slots. As of this commit, the structure of these abstractions looks like the following: +------------------------+ | v +----------+ +-----+ +-----------+-----+-----------------+ | Compiler | --> | ISA | --> | Registers | ABI | Compilation Env | +----------+ +-----+ +-----------+-----+-----------------+ | ^ +------------------------------+ * Compilation environment will hold a reference to the function data * Add basic documentation to the ABI trait * Enable x86 and arm64 in cranelift-codegen * Add reg_name function for x64 * Introduce the concept of a MacroAssembler and Assembler This commit introduces the concept of a MacroAsesembler and Assembler. The MacroAssembler trait will provide a high enough interface across architectures so that each ISA implementation can use their own low-level Assembler implementation to fulfill the interface. Each Assembler will provide a 1-1 mapping to each ISA instruction. As of this commit, only a partial debug implementation is provided for the x64 Assembler. * Add a newtype over PReg Adds a newtype `Reg` over regalloc2::PReg; this ensures that Winch will operate only on the concept of `Reg`. This change is temporary until we have the necessary machinery to share a common Reg abstraction via `cranelift_asm` * Improvements to local calcuation - Add `LocalSlot::addressed_from_sp` - Use `u32` for local slot and local sizes calculation * Add helper methods to ABIArg Adds helper methods to retrieve register and type information from the argument * Make locals_size public in frame * Improve x64 register naming depending on size * Add new methods to the masm interface This commit introduces the ability for the MacroAssembler to reserve stack space, get the address of a given local and perform a stack store based on the concept of `Operand`s. There are several motivating factors to introduce the concept of an Operand: - Make the translation between Winch and Cranelift easier; - Make dispatching from the MacroAssembler to the underlying Assembler - easier by minimizing the amount of functions that we need to define - in order to satisfy the store/load combinations This commit also introduces the concept of a memory address, which essentially describes the addressing modes; as of this commit only one addressing mode is supported. We'll also need to verify that this structure will play nicely with arm64. * Blank masm implementation for arm64 * Implementation of reserve_stack, local_address, store and fp_offset for x64 * Implement function prologue and argument register spilling * Add structopt and wat * Fix debug instruction formatting * Make TargetISA trait publicly accessible * Modify the MacroAssembler finalize siganture to return a slice of strings * Introduce a simple CLI for Winch To be able to compile Wasm programs with Winch independently. Mostly meant for testing / debugging * Fix bug in x64 assembler mov_rm * Remove unused import * Move the stack slot calculation to the Frame This commit moves the calculation of the stack slots to the frame handler abstraction and also includes the calculation of the limits for the function defined locals, which will be used to zero the locals that are not associated to function arguments * Add i32 and i64 constructors to local slots * Introduce the concept of DefinedLocalsRange This commit introduces `DefinedLocalsRange` to track the stack offset at which the function-defined locals start and end; this is later used to zero-out that stack region * Add constructors for int and float registers * Add a placeholder stack implementation * Add a regset abstraction to track register availability Adds a bit set abstraction to track register availability for register allocation. The bit set has no specific knowledge about physical registers, it works on the register's hardware encoding as the source of truth. Each RegSet is expected to be created with the universe of allocatable registers per ISA when starting the compilation of a particular function. * Add an abstraction over register and immediate This is meant to be used as the source for stores. * Add a way to zero local slots and an initial skeletion of regalloc This commit introduces `zero_local_slots` to the MacroAssembler; which ensures that function defined locals are zeroed out when starting the function body. The algorithm divides the defined function locals stack range into 8 byte slots and stores a zero at each address. This process relies on register allocation if the amount of slots that need to be initialized is greater than 1. In such case, the next available register is requested to the register set and it's used to store a 0, which is then stored at every local slot * Update to wasmparser 0.92 * Correctly track if the regset has registers available * Add a result entry to the ABI signature This commuit introduces ABIResult as part of the ABISignature; this struct will track how function results are stored; initially it will consiste of a single register that will be requested to the register allocator at the end of the function; potentially causing a spill * Move zero local slots and add more granular methods to the masm This commit removes zeroing local slots from the MacroAssembler and instead adds more granular methods to it (e.g `zero`, `add`). This allows for better code sharing since most of the work done by the algorithm for zeroing slots will be the same in all targets, except for the binary emissions pieces, which is what gets delegated to the masm * Use wasmparser's visitor API and add initial support for const and add This commit adds initial support for the I32Const and I32 instructions; this involves adding a minimum for register allocation. Note that some regalloc pieces are still incomplete, since for the current set of supported instructions they are not needed. * Make the ty field public in Local * Add scratch_reg to the abi * Add a method to get a particular local from the Frame * Split the compilation environment abstraction This commit splits the compilation environment into two more concise abstractions: 1. CodeGen: the main abstraction for code generation 2. CodeGenContext: abstraction that shares the common pieces for compilation; these pieces are shared between the code generator and the register allocator * Add `push` and `load` to the MacroAssembler * Remove dead code warnings for unused paths * Map ISA features to cranelift-codegen ISA features * Apply formatting * Fix Cargo.toml after a bad rebase * Add component-compiler feature * Use clap instead of structopt * Add winch to publish.rs script * Minor formatting * Add tests to RegSet and fix two bugs when freeing and checking for register availability * Add tests to Stack * Free source register after a non-constant i32 add * Improve comments - Remove unneeded comments - And improve some of the TODO items * Update default features * Drop the ABI generic param and pass the word_size information directly To avoid dealing with dead code warnings this commit passes the word size information directly, since it's the only piece of information needed from the ABI by Codegen until now * Remove dead code This piece of code will be put back once we start integrating Winch with Wasmtime * Remove unused enum variant This variant doesn't get constructed; it should be added back once a backend is added and not enabled by default or when Winch gets integrated into Wasmtime * Fix unused code in regset tests * Update spec testsuite * Switch the visitor pattern for a simpler operator match This commit removes the usage of wasmparser's visitor pattern and instead defaults to a simpler operator matching approach. This removes the complexity of having to define all the visitor trait functions at once. * Use wasmparser's Visitor trait with a different macro strategy This commit puts back wasmparser's Visitor trait, with a sigle; simpler macro, only used for unsupported operators. * Restructure Winch This commit restuructures Winch's parts. It divides the initial approach into three main crates: `winch-codegen`,`wasmtime-winch` and `winch-tools`. `wasmtime-winch` is reponsible for the Wasmtime-Winch integration. `winch-codegen` is solely responsible for code generation. `winch-tools` is CLI tool to compile Wasm programs, mainly for testing purposes. * Refactor zero local slots This commit moves the logic of zeroing local slots from the codegen module into a method with a default implementation in the MacroAssembler trait: `zero_mem_range`. The refactored implementation is very similar to the previous implementation with the only difference that it doesn't allocates a general-purpose register; it instead uses the register allocator to retrieve the scratch register and uses this register to unroll the series of zero stores. * Tie the codegen creation to the ISA ABI This commit makes the relationship between the ISA ABI and the codegen explicit. This allows us to pass down ABI-specific bit and pieces to the codegeneration. In this case the only concrete piece that we need is the ABI word size. * Mark winch as publishable directory * Revamp winch docs This commit ensures that all the code comments in Winch are compliant with the syle used in the rest of Wasmtime's codebase. It also imptoves, generally the quality of the comments in some modules. * Panic when using multi-value when the target is aarch64 Similar to x64, this commit ensures that the abi signature of the current function doesn't use multi-value returns * Document the usage of directives * Use endianness instead of endianess in the ISA trait * Introduce a three-argument form in the MacroAssembler This commit introduces the usage of three-argument form for the MacroAssembler interface. This allows for a natural mapping for architectures like aarch64. In the case of x64, the implementation can simply restrict the implementation asserting for equality in two of the arguments of defaulting to a differnt set of instructions. As of this commit, the implementation of `add` panics if the destination and the first source arguments are not equal; internally the x64 assembler implementation will ensure that all the allowed combinations of `add` are satisfied. The reason for panicking and not emitting a `mov` followed by an `add` for example is simply because register allocation happens right before calling `add`, which ensures any register-to-register moves, if needed. This implementation will evolve in the future and this panic will be lifted if needed. * Improve the documentation for the MacroAssembler. Documents the usage of three-arg form and the intention around the high-level interface. * Format comments in remaining modules * Clean up Cargo.toml for winch pieces This commit adds missing fields to each of Winch's Cargo.toml. * Use `ModuleTranslation::get_types()` to derive the function type * Assert that start range is always word-size aligned
2 years ago
"target-lexicon",
"toml",
Initial skeleton for Winch (#4907) * Initial skeleton for Winch This commit introduces the initial skeleton for Winch, the "baseline" compiler. This skeleton contains mostly setup code for the ISA, ABI, registers, and compilation environment abstractions. It also includes the calculation of function local slots. As of this commit, the structure of these abstractions looks like the following: +------------------------+ | v +----------+ +-----+ +-----------+-----+-----------------+ | Compiler | --> | ISA | --> | Registers | ABI | Compilation Env | +----------+ +-----+ +-----------+-----+-----------------+ | ^ +------------------------------+ * Compilation environment will hold a reference to the function data * Add basic documentation to the ABI trait * Enable x86 and arm64 in cranelift-codegen * Add reg_name function for x64 * Introduce the concept of a MacroAssembler and Assembler This commit introduces the concept of a MacroAsesembler and Assembler. The MacroAssembler trait will provide a high enough interface across architectures so that each ISA implementation can use their own low-level Assembler implementation to fulfill the interface. Each Assembler will provide a 1-1 mapping to each ISA instruction. As of this commit, only a partial debug implementation is provided for the x64 Assembler. * Add a newtype over PReg Adds a newtype `Reg` over regalloc2::PReg; this ensures that Winch will operate only on the concept of `Reg`. This change is temporary until we have the necessary machinery to share a common Reg abstraction via `cranelift_asm` * Improvements to local calcuation - Add `LocalSlot::addressed_from_sp` - Use `u32` for local slot and local sizes calculation * Add helper methods to ABIArg Adds helper methods to retrieve register and type information from the argument * Make locals_size public in frame * Improve x64 register naming depending on size * Add new methods to the masm interface This commit introduces the ability for the MacroAssembler to reserve stack space, get the address of a given local and perform a stack store based on the concept of `Operand`s. There are several motivating factors to introduce the concept of an Operand: - Make the translation between Winch and Cranelift easier; - Make dispatching from the MacroAssembler to the underlying Assembler - easier by minimizing the amount of functions that we need to define - in order to satisfy the store/load combinations This commit also introduces the concept of a memory address, which essentially describes the addressing modes; as of this commit only one addressing mode is supported. We'll also need to verify that this structure will play nicely with arm64. * Blank masm implementation for arm64 * Implementation of reserve_stack, local_address, store and fp_offset for x64 * Implement function prologue and argument register spilling * Add structopt and wat * Fix debug instruction formatting * Make TargetISA trait publicly accessible * Modify the MacroAssembler finalize siganture to return a slice of strings * Introduce a simple CLI for Winch To be able to compile Wasm programs with Winch independently. Mostly meant for testing / debugging * Fix bug in x64 assembler mov_rm * Remove unused import * Move the stack slot calculation to the Frame This commit moves the calculation of the stack slots to the frame handler abstraction and also includes the calculation of the limits for the function defined locals, which will be used to zero the locals that are not associated to function arguments * Add i32 and i64 constructors to local slots * Introduce the concept of DefinedLocalsRange This commit introduces `DefinedLocalsRange` to track the stack offset at which the function-defined locals start and end; this is later used to zero-out that stack region * Add constructors for int and float registers * Add a placeholder stack implementation * Add a regset abstraction to track register availability Adds a bit set abstraction to track register availability for register allocation. The bit set has no specific knowledge about physical registers, it works on the register's hardware encoding as the source of truth. Each RegSet is expected to be created with the universe of allocatable registers per ISA when starting the compilation of a particular function. * Add an abstraction over register and immediate This is meant to be used as the source for stores. * Add a way to zero local slots and an initial skeletion of regalloc This commit introduces `zero_local_slots` to the MacroAssembler; which ensures that function defined locals are zeroed out when starting the function body. The algorithm divides the defined function locals stack range into 8 byte slots and stores a zero at each address. This process relies on register allocation if the amount of slots that need to be initialized is greater than 1. In such case, the next available register is requested to the register set and it's used to store a 0, which is then stored at every local slot * Update to wasmparser 0.92 * Correctly track if the regset has registers available * Add a result entry to the ABI signature This commuit introduces ABIResult as part of the ABISignature; this struct will track how function results are stored; initially it will consiste of a single register that will be requested to the register allocator at the end of the function; potentially causing a spill * Move zero local slots and add more granular methods to the masm This commit removes zeroing local slots from the MacroAssembler and instead adds more granular methods to it (e.g `zero`, `add`). This allows for better code sharing since most of the work done by the algorithm for zeroing slots will be the same in all targets, except for the binary emissions pieces, which is what gets delegated to the masm * Use wasmparser's visitor API and add initial support for const and add This commit adds initial support for the I32Const and I32 instructions; this involves adding a minimum for register allocation. Note that some regalloc pieces are still incomplete, since for the current set of supported instructions they are not needed. * Make the ty field public in Local * Add scratch_reg to the abi * Add a method to get a particular local from the Frame * Split the compilation environment abstraction This commit splits the compilation environment into two more concise abstractions: 1. CodeGen: the main abstraction for code generation 2. CodeGenContext: abstraction that shares the common pieces for compilation; these pieces are shared between the code generator and the register allocator * Add `push` and `load` to the MacroAssembler * Remove dead code warnings for unused paths * Map ISA features to cranelift-codegen ISA features * Apply formatting * Fix Cargo.toml after a bad rebase * Add component-compiler feature * Use clap instead of structopt * Add winch to publish.rs script * Minor formatting * Add tests to RegSet and fix two bugs when freeing and checking for register availability * Add tests to Stack * Free source register after a non-constant i32 add * Improve comments - Remove unneeded comments - And improve some of the TODO items * Update default features * Drop the ABI generic param and pass the word_size information directly To avoid dealing with dead code warnings this commit passes the word size information directly, since it's the only piece of information needed from the ABI by Codegen until now * Remove dead code This piece of code will be put back once we start integrating Winch with Wasmtime * Remove unused enum variant This variant doesn't get constructed; it should be added back once a backend is added and not enabled by default or when Winch gets integrated into Wasmtime * Fix unused code in regset tests * Update spec testsuite * Switch the visitor pattern for a simpler operator match This commit removes the usage of wasmparser's visitor pattern and instead defaults to a simpler operator matching approach. This removes the complexity of having to define all the visitor trait functions at once. * Use wasmparser's Visitor trait with a different macro strategy This commit puts back wasmparser's Visitor trait, with a sigle; simpler macro, only used for unsupported operators. * Restructure Winch This commit restuructures Winch's parts. It divides the initial approach into three main crates: `winch-codegen`,`wasmtime-winch` and `winch-tools`. `wasmtime-winch` is reponsible for the Wasmtime-Winch integration. `winch-codegen` is solely responsible for code generation. `winch-tools` is CLI tool to compile Wasm programs, mainly for testing purposes. * Refactor zero local slots This commit moves the logic of zeroing local slots from the codegen module into a method with a default implementation in the MacroAssembler trait: `zero_mem_range`. The refactored implementation is very similar to the previous implementation with the only difference that it doesn't allocates a general-purpose register; it instead uses the register allocator to retrieve the scratch register and uses this register to unroll the series of zero stores. * Tie the codegen creation to the ISA ABI This commit makes the relationship between the ISA ABI and the codegen explicit. This allows us to pass down ABI-specific bit and pieces to the codegeneration. In this case the only concrete piece that we need is the ABI word size. * Mark winch as publishable directory * Revamp winch docs This commit ensures that all the code comments in Winch are compliant with the syle used in the rest of Wasmtime's codebase. It also imptoves, generally the quality of the comments in some modules. * Panic when using multi-value when the target is aarch64 Similar to x64, this commit ensures that the abi signature of the current function doesn't use multi-value returns * Document the usage of directives * Use endianness instead of endianess in the ISA trait * Introduce a three-argument form in the MacroAssembler This commit introduces the usage of three-argument form for the MacroAssembler interface. This allows for a natural mapping for architectures like aarch64. In the case of x64, the implementation can simply restrict the implementation asserting for equality in two of the arguments of defaulting to a differnt set of instructions. As of this commit, the implementation of `add` panics if the destination and the first source arguments are not equal; internally the x64 assembler implementation will ensure that all the allowed combinations of `add` are satisfied. The reason for panicking and not emitting a `mov` followed by an `add` for example is simply because register allocation happens right before calling `add`, which ensures any register-to-register moves, if needed. This implementation will evolve in the future and this panic will be lifted if needed. * Improve the documentation for the MacroAssembler. Documents the usage of three-arg form and the intention around the high-level interface. * Format comments in remaining modules * Clean up Cargo.toml for winch pieces This commit adds missing fields to each of Winch's Cargo.toml. * Use `ModuleTranslation::get_types()` to derive the function type * Assert that start range is always word-size aligned
2 years ago
"wasmparser",
"wasmtime-environ",
"wat",
"winch-codegen",
"winch-environ",
"winch-filetests",
"winch-test-macros",
Initial skeleton for Winch (#4907) * Initial skeleton for Winch This commit introduces the initial skeleton for Winch, the "baseline" compiler. This skeleton contains mostly setup code for the ISA, ABI, registers, and compilation environment abstractions. It also includes the calculation of function local slots. As of this commit, the structure of these abstractions looks like the following: +------------------------+ | v +----------+ +-----+ +-----------+-----+-----------------+ | Compiler | --> | ISA | --> | Registers | ABI | Compilation Env | +----------+ +-----+ +-----------+-----+-----------------+ | ^ +------------------------------+ * Compilation environment will hold a reference to the function data * Add basic documentation to the ABI trait * Enable x86 and arm64 in cranelift-codegen * Add reg_name function for x64 * Introduce the concept of a MacroAssembler and Assembler This commit introduces the concept of a MacroAsesembler and Assembler. The MacroAssembler trait will provide a high enough interface across architectures so that each ISA implementation can use their own low-level Assembler implementation to fulfill the interface. Each Assembler will provide a 1-1 mapping to each ISA instruction. As of this commit, only a partial debug implementation is provided for the x64 Assembler. * Add a newtype over PReg Adds a newtype `Reg` over regalloc2::PReg; this ensures that Winch will operate only on the concept of `Reg`. This change is temporary until we have the necessary machinery to share a common Reg abstraction via `cranelift_asm` * Improvements to local calcuation - Add `LocalSlot::addressed_from_sp` - Use `u32` for local slot and local sizes calculation * Add helper methods to ABIArg Adds helper methods to retrieve register and type information from the argument * Make locals_size public in frame * Improve x64 register naming depending on size * Add new methods to the masm interface This commit introduces the ability for the MacroAssembler to reserve stack space, get the address of a given local and perform a stack store based on the concept of `Operand`s. There are several motivating factors to introduce the concept of an Operand: - Make the translation between Winch and Cranelift easier; - Make dispatching from the MacroAssembler to the underlying Assembler - easier by minimizing the amount of functions that we need to define - in order to satisfy the store/load combinations This commit also introduces the concept of a memory address, which essentially describes the addressing modes; as of this commit only one addressing mode is supported. We'll also need to verify that this structure will play nicely with arm64. * Blank masm implementation for arm64 * Implementation of reserve_stack, local_address, store and fp_offset for x64 * Implement function prologue and argument register spilling * Add structopt and wat * Fix debug instruction formatting * Make TargetISA trait publicly accessible * Modify the MacroAssembler finalize siganture to return a slice of strings * Introduce a simple CLI for Winch To be able to compile Wasm programs with Winch independently. Mostly meant for testing / debugging * Fix bug in x64 assembler mov_rm * Remove unused import * Move the stack slot calculation to the Frame This commit moves the calculation of the stack slots to the frame handler abstraction and also includes the calculation of the limits for the function defined locals, which will be used to zero the locals that are not associated to function arguments * Add i32 and i64 constructors to local slots * Introduce the concept of DefinedLocalsRange This commit introduces `DefinedLocalsRange` to track the stack offset at which the function-defined locals start and end; this is later used to zero-out that stack region * Add constructors for int and float registers * Add a placeholder stack implementation * Add a regset abstraction to track register availability Adds a bit set abstraction to track register availability for register allocation. The bit set has no specific knowledge about physical registers, it works on the register's hardware encoding as the source of truth. Each RegSet is expected to be created with the universe of allocatable registers per ISA when starting the compilation of a particular function. * Add an abstraction over register and immediate This is meant to be used as the source for stores. * Add a way to zero local slots and an initial skeletion of regalloc This commit introduces `zero_local_slots` to the MacroAssembler; which ensures that function defined locals are zeroed out when starting the function body. The algorithm divides the defined function locals stack range into 8 byte slots and stores a zero at each address. This process relies on register allocation if the amount of slots that need to be initialized is greater than 1. In such case, the next available register is requested to the register set and it's used to store a 0, which is then stored at every local slot * Update to wasmparser 0.92 * Correctly track if the regset has registers available * Add a result entry to the ABI signature This commuit introduces ABIResult as part of the ABISignature; this struct will track how function results are stored; initially it will consiste of a single register that will be requested to the register allocator at the end of the function; potentially causing a spill * Move zero local slots and add more granular methods to the masm This commit removes zeroing local slots from the MacroAssembler and instead adds more granular methods to it (e.g `zero`, `add`). This allows for better code sharing since most of the work done by the algorithm for zeroing slots will be the same in all targets, except for the binary emissions pieces, which is what gets delegated to the masm * Use wasmparser's visitor API and add initial support for const and add This commit adds initial support for the I32Const and I32 instructions; this involves adding a minimum for register allocation. Note that some regalloc pieces are still incomplete, since for the current set of supported instructions they are not needed. * Make the ty field public in Local * Add scratch_reg to the abi * Add a method to get a particular local from the Frame * Split the compilation environment abstraction This commit splits the compilation environment into two more concise abstractions: 1. CodeGen: the main abstraction for code generation 2. CodeGenContext: abstraction that shares the common pieces for compilation; these pieces are shared between the code generator and the register allocator * Add `push` and `load` to the MacroAssembler * Remove dead code warnings for unused paths * Map ISA features to cranelift-codegen ISA features * Apply formatting * Fix Cargo.toml after a bad rebase * Add component-compiler feature * Use clap instead of structopt * Add winch to publish.rs script * Minor formatting * Add tests to RegSet and fix two bugs when freeing and checking for register availability * Add tests to Stack * Free source register after a non-constant i32 add * Improve comments - Remove unneeded comments - And improve some of the TODO items * Update default features * Drop the ABI generic param and pass the word_size information directly To avoid dealing with dead code warnings this commit passes the word size information directly, since it's the only piece of information needed from the ABI by Codegen until now * Remove dead code This piece of code will be put back once we start integrating Winch with Wasmtime * Remove unused enum variant This variant doesn't get constructed; it should be added back once a backend is added and not enabled by default or when Winch gets integrated into Wasmtime * Fix unused code in regset tests * Update spec testsuite * Switch the visitor pattern for a simpler operator match This commit removes the usage of wasmparser's visitor pattern and instead defaults to a simpler operator matching approach. This removes the complexity of having to define all the visitor trait functions at once. * Use wasmparser's Visitor trait with a different macro strategy This commit puts back wasmparser's Visitor trait, with a sigle; simpler macro, only used for unsupported operators. * Restructure Winch This commit restuructures Winch's parts. It divides the initial approach into three main crates: `winch-codegen`,`wasmtime-winch` and `winch-tools`. `wasmtime-winch` is reponsible for the Wasmtime-Winch integration. `winch-codegen` is solely responsible for code generation. `winch-tools` is CLI tool to compile Wasm programs, mainly for testing purposes. * Refactor zero local slots This commit moves the logic of zeroing local slots from the codegen module into a method with a default implementation in the MacroAssembler trait: `zero_mem_range`. The refactored implementation is very similar to the previous implementation with the only difference that it doesn't allocates a general-purpose register; it instead uses the register allocator to retrieve the scratch register and uses this register to unroll the series of zero stores. * Tie the codegen creation to the ISA ABI This commit makes the relationship between the ISA ABI and the codegen explicit. This allows us to pass down ABI-specific bit and pieces to the codegeneration. In this case the only concrete piece that we need is the ABI word size. * Mark winch as publishable directory * Revamp winch docs This commit ensures that all the code comments in Winch are compliant with the syle used in the rest of Wasmtime's codebase. It also imptoves, generally the quality of the comments in some modules. * Panic when using multi-value when the target is aarch64 Similar to x64, this commit ensures that the abi signature of the current function doesn't use multi-value returns * Document the usage of directives * Use endianness instead of endianess in the ISA trait * Introduce a three-argument form in the MacroAssembler This commit introduces the usage of three-argument form for the MacroAssembler interface. This allows for a natural mapping for architectures like aarch64. In the case of x64, the implementation can simply restrict the implementation asserting for equality in two of the arguments of defaulting to a differnt set of instructions. As of this commit, the implementation of `add` panics if the destination and the first source arguments are not equal; internally the x64 assembler implementation will ensure that all the allowed combinations of `add` are satisfied. The reason for panicking and not emitting a `mov` followed by an `add` for example is simply because register allocation happens right before calling `add`, which ensures any register-to-register moves, if needed. This implementation will evolve in the future and this panic will be lifted if needed. * Improve the documentation for the MacroAssembler. Documents the usage of three-arg form and the intention around the high-level interface. * Format comments in remaining modules * Clean up Cargo.toml for winch pieces This commit adds missing fields to each of Winch's Cargo.toml. * Use `ModuleTranslation::get_types()` to derive the function type * Assert that start range is always word-size aligned
2 years ago
]
[[package]]
name = "windows-sys"
version = "0.45.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0"
dependencies = [
"windows-targets",
]
[[package]]
name = "windows-targets"
version = "0.42.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8e2522491fbfcd58cc84d47aeb2958948c4b8982e9a2d8a2a35bbaed431390e7"
dependencies = [
"windows_aarch64_gnullvm",
"windows_aarch64_msvc",
"windows_i686_gnu",
"windows_i686_msvc",
"windows_x86_64_gnu",
"windows_x86_64_gnullvm",
"windows_x86_64_msvc",
]
[[package]]
name = "windows_aarch64_gnullvm"
version = "0.42.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8c9864e83243fdec7fc9c5444389dcbbfd258f745e7853198f365e3c4968a608"
[[package]]
name = "windows_aarch64_msvc"
version = "0.42.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4c8b1b673ffc16c47a9ff48570a9d85e25d265735c503681332589af6253c6c7"
[[package]]
name = "windows_i686_gnu"
version = "0.42.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "de3887528ad530ba7bdbb1faa8275ec7a1155a45ffa57c37993960277145d640"
[[package]]
name = "windows_i686_msvc"
version = "0.42.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bf4d1122317eddd6ff351aa852118a2418ad4214e6613a50e0191f7004372605"
[[package]]
name = "windows_x86_64_gnu"
version = "0.42.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c1040f221285e17ebccbc2591ffdc2d44ee1f9186324dd3e84e99ac68d699c45"
[[package]]
name = "windows_x86_64_gnullvm"
version = "0.42.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "628bfdf232daa22b0d64fdb62b09fcc36bb01f05a3939e20ab73aaf9470d0463"
[[package]]
name = "windows_x86_64_msvc"
version = "0.42.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "447660ad36a13288b1db4d4248e857b510e8c3a225c822ba4fb748c0aafecffd"
[[package]]
name = "winx"
version = "0.35.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "129cd8ee937d535e1a239d9d3c9c0525af0454bc0967d9211a251be062513520"
dependencies = [
"bitflags",
"io-lifetimes",
"windows-sys",
]
[[package]]
name = "wit-parser"
version = "0.6.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f887c3da527a51b321076ebe6a7513026a4757b6d4d144259946552d6fc728b3"
dependencies = [
"anyhow",
"id-arena",
"indexmap",
"log",
"pulldown-cmark",
"unicode-xid",
"url",
]
[[package]]
name = "witx"
version = "0.9.1"
dependencies = [
"anyhow",
"log",
"rayon",
"thiserror",
"wast 35.0.2",
]
[[package]]
name = "xoodyak"
version = "0.7.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "94b52dbbb9ca4862dcd5e33383af1610db2f28d081e0803e3a3b2bbdfab5eeed"
dependencies = [
"rawbytes",
"zeroize",
]
[[package]]
name = "zeroize"
version = "1.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d68d9dcec5f9b43a30d38c49f91dfedfaac384cb8f085faca366c26207dd1619"
dependencies = [
"zeroize_derive",
]
[[package]]
name = "zeroize_derive"
version = "1.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3f8f187641dad4f680d25c4bfc4225b418165984179f26ca76ec4fb6441d3a17"
dependencies = [
"proc-macro2",
"quote",
"syn",
"synstructure",
]
[[package]]
name = "zstd"
version = "0.11.1+zstd.1.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "77a16b8414fde0414e90c612eba70985577451c4c504b99885ebed24762cb81a"
dependencies = [
"zstd-safe",
]
[[package]]
name = "zstd-safe"
version = "5.0.1+zstd.1.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7c12659121420dd6365c5c3de4901f97145b79651fb1d25814020ed2ed0585ae"
dependencies = [
"libc",
"zstd-sys",
]
[[package]]
name = "zstd-sys"
version = "2.0.1+zstd.1.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9fd07cbbc53846d9145dbffdf6dd09a7a0aa52be46741825f5c97bdd4f73f12b"
dependencies = [
"cc",
"libc",
]