From dfc4358d0c7ced2f1e8f1b4913841ed6ce31ff46 Mon Sep 17 00:00:00 2001 From: Xinzhao Xu Date: Sat, 27 Jul 2024 04:59:02 +0800 Subject: [PATCH] Implement wasi-keyvalue (#8983) * Implement wasi-keyvalue * Allow preset data for In-Memory provider, rename allow_hosts to allow_redis_hosts * Add vets --------- Co-authored-by: Alex Crichton --- .github/workflows/main.yml | 43 ++ Cargo.lock | 68 ++- Cargo.toml | 3 + ci/run-tests.sh | 4 + ci/vendor-wit.sh | 2 + crates/test-programs/artifacts/build.rs | 1 + crates/test-programs/src/bin/keyvalue_main.rs | 51 ++ crates/test-programs/src/lib.rs | 8 + crates/wasi-keyvalue/Cargo.toml | 26 ++ crates/wasi-keyvalue/src/lib.rs | 442 ++++++++++++++++++ crates/wasi-keyvalue/src/provider/inmemory.rs | 94 ++++ crates/wasi-keyvalue/src/provider/mod.rs | 3 + crates/wasi-keyvalue/src/provider/redis.rs | 106 +++++ crates/wasi-keyvalue/tests/main.rs | 93 ++++ .../wit/deps/keyvalue/atomic.wit | 22 + .../wasi-keyvalue/wit/deps/keyvalue/batch.wit | 63 +++ .../wasi-keyvalue/wit/deps/keyvalue/store.wit | 122 +++++ .../wasi-keyvalue/wit/deps/keyvalue/watch.wit | 16 + .../wasi-keyvalue/wit/deps/keyvalue/world.wit | 26 ++ crates/wasi-keyvalue/wit/world.wit | 6 + scripts/publish.rs | 2 + supply-chain/audits.toml | 10 + supply-chain/config.toml | 8 + supply-chain/imports.lock | 13 + 24 files changed, 1231 insertions(+), 1 deletion(-) create mode 100644 crates/test-programs/src/bin/keyvalue_main.rs create mode 100644 crates/wasi-keyvalue/Cargo.toml create mode 100644 crates/wasi-keyvalue/src/lib.rs create mode 100644 crates/wasi-keyvalue/src/provider/inmemory.rs create mode 100644 crates/wasi-keyvalue/src/provider/mod.rs create mode 100644 crates/wasi-keyvalue/src/provider/redis.rs create mode 100644 crates/wasi-keyvalue/tests/main.rs create mode 100644 crates/wasi-keyvalue/wit/deps/keyvalue/atomic.wit create mode 100644 crates/wasi-keyvalue/wit/deps/keyvalue/batch.wit create mode 100644 crates/wasi-keyvalue/wit/deps/keyvalue/store.wit create mode 100644 crates/wasi-keyvalue/wit/deps/keyvalue/watch.wit create mode 100644 crates/wasi-keyvalue/wit/deps/keyvalue/world.wit create mode 100644 crates/wasi-keyvalue/wit/world.wit diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index ada3daffbc..c799f98f25 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -197,6 +197,7 @@ jobs: audit: ${{ steps.calculate.outputs.audit }} preview1-adapter: ${{ steps.calculate.outputs.preview1-adapter }} run-dwarf: ${{ steps.calculate.outputs.run-dwarf }} + run-wasi-keyvalue: ${{ steps.calculate.outputs.run-wasi-keyvalue }} steps: - uses: actions/checkout@v4 - id: calculate @@ -241,6 +242,9 @@ jobs: if grep -q debug names.log; then echo run-dwarf=true >> $GITHUB_OUTPUT fi + if grep -q wasi-keyvalue names.log; then + echo run-wasi-keyvalue=true >> $GITHUB_OUTPUT + fi fi matrix="$(node ./ci/build-test-matrix.js ./commits.log ./names.log $run_full)" echo "test-matrix={\"include\":$(echo $matrix)}" >> $GITHUB_OUTPUT @@ -256,6 +260,7 @@ jobs: echo audit=true >> $GITHUB_OUTPUT echo preview1-adapter=true >> $GITHUB_OUTPUT echo run-dwarf=true >> $GITHUB_OUTPUT + echo run-wasi-keyvalue=true >> $GITHUB_OUTPUT fi # Build all documentation of Wasmtime, including the C API documentation, @@ -737,6 +742,44 @@ jobs: env: GH_TOKEN: ${{ github.token }} + # Test the `wasmtime-wasi-keyvalue` crate. Split out from the main tests + # because it needs additional database service. + test_wasi_keyvalue: + name: Test wasi-keyvalue + runs-on: ubuntu-latest + needs: determine + if: needs.determine.outputs.run-wasi-keyvalue + # Setup redis server + services: + redis: + # Docker Hub image + image: redis + # Set health checks to wait until redis has started + options: >- + --health-cmd "redis-cli ping" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + # Maps port 6379 on service container to the host + - 6379:6379 + steps: + - uses: actions/checkout@v4 + with: + submodules: true + - uses: ./.github/actions/install-rust + # Install Rust targets + - run: rustup target add wasm32-wasi + - run: cargo test --all-features -p wasmtime-wasi-keyvalue + env: + RUST_BACKTRACE: 1 + + # common logic to cancel the entire run if this job fails + - run: gh run cancel ${{ github.run_id }} + if: failure() && github.event_name != 'pull_request' + env: + GH_TOKEN: ${{ github.token }} + # Test the `wasmtime-fuzzing` crate. Split out from the main tests because # `--all-features` brings in OCaml, which is a pain to get setup for all # targets. diff --git a/Cargo.lock b/Cargo.lock index e24c8de3a5..8f91d5486e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -477,6 +477,20 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" +[[package]] +name = "combine" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd" +dependencies = [ + "bytes", + "futures-core", + "memchr", + "pin-project-lite", + "tokio", + "tokio-util", +] + [[package]] name = "component-fuzz-util" version = "0.0.0" @@ -2233,6 +2247,27 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "redis" +version = "0.25.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0d7a6955c7511f60f3ba9e86c6d02b3c3f144f8c24b288d1f4e18074ab8bbec" +dependencies = [ + "async-trait", + "bytes", + "combine", + "futures-util", + "itoa", + "percent-encoding", + "pin-project-lite", + "ryu", + "sha1_smol", + "socket2 0.5.7", + "tokio", + "tokio-util", + "url", +] + [[package]] name = "redox_syscall" version = "0.2.13" @@ -2492,6 +2527,12 @@ dependencies = [ "serde", ] +[[package]] +name = "sha1_smol" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbfa15b3dddfee50a0fff136974b3e1bde555604ba463834a7eb7deb6417705d" + [[package]] name = "sha2" version = "0.10.2" @@ -2582,6 +2623,16 @@ dependencies = [ "winapi", ] +[[package]] +name = "socket2" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce305eb0b4296696835b71df73eb912e0f1ffd2556a501fcede6e0c50349191c" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + [[package]] name = "souper-ir" version = "2.1.0" @@ -2843,7 +2894,7 @@ dependencies = [ "num_cpus", "pin-project-lite", "signal-hook-registry", - "socket2", + "socket2 0.4.9", "tokio-macros", "windows-sys 0.48.0", ] @@ -3567,6 +3618,7 @@ dependencies = [ "wasmtime-test-macros", "wasmtime-wasi", "wasmtime-wasi-http", + "wasmtime-wasi-keyvalue", "wasmtime-wasi-nn", "wasmtime-wasi-runtime-config", "wasmtime-wasi-threads", @@ -3879,6 +3931,20 @@ dependencies = [ "webpki-roots", ] +[[package]] +name = "wasmtime-wasi-keyvalue" +version = "24.0.0" +dependencies = [ + "anyhow", + "async-trait", + "redis", + "test-programs-artifacts", + "tokio", + "url", + "wasmtime", + "wasmtime-wasi", +] + [[package]] name = "wasmtime-wasi-nn" version = "24.0.0" diff --git a/Cargo.toml b/Cargo.toml index ffd2b93a29..89e7f47b03 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -54,6 +54,7 @@ wasi-common = { workspace = true, default-features = true, features = ["exit"], wasmtime-wasi = { workspace = true, default-features = true, optional = true } wasmtime-wasi-nn = { workspace = true, optional = true } wasmtime-wasi-runtime-config = { workspace = true, optional = true } +wasmtime-wasi-keyvalue = { workspace = true, optional = true } wasmtime-wasi-threads = { workspace = true, optional = true } wasmtime-wasi-http = { workspace = true, optional = true } clap = { workspace = true } @@ -195,6 +196,7 @@ wasmtime-wasi = { path = "crates/wasi", version = "24.0.0", default-features = f wasmtime-wasi-http = { path = "crates/wasi-http", version = "=24.0.0", default-features = false } wasmtime-wasi-nn = { path = "crates/wasi-nn", version = "24.0.0" } wasmtime-wasi-runtime-config = { path = "crates/wasi-runtime-config", version = "24.0.0" } +wasmtime-wasi-keyvalue = { path = "crates/wasi-keyvalue", version = "24.0.0" } wasmtime-wasi-threads = { path = "crates/wasi-threads", version = "24.0.0" } wasmtime-component-util = { path = "crates/component-util", version = "=24.0.0" } wasmtime-component-macro = { path = "crates/component-macro", version = "=24.0.0" } @@ -326,6 +328,7 @@ criterion = { version = "0.5.0", default-features = false, features = ["html_rep rustc-hash = "1.1.0" libtest-mimic = "0.7.0" semver = { version = "1.0.17", default-features = false } +redis = "0.25.4" # ============================================================================= # diff --git a/ci/run-tests.sh b/ci/run-tests.sh index 274d352321..1822a3fd07 100755 --- a/ci/run-tests.sh +++ b/ci/run-tests.sh @@ -12,6 +12,9 @@ # # - wasm-spec-interpreter: brings in OCaml which is a pain to configure for all # targets, tested as part of the wastime-fuzzing CI job. +# +# - wasmtime-wasi-keyvalue: additional database service dependencies, needs its +# own CI job. cargo test \ --workspace \ @@ -20,4 +23,5 @@ cargo test \ --exclude wasmtime-wasi-nn \ --exclude wasmtime-fuzzing \ --exclude wasm-spec-interpreter \ + --exclude wasmtime-wasi-keyvalue \ $@ diff --git a/ci/vendor-wit.sh b/ci/vendor-wit.sh index 374d9c10fb..005f4d3e07 100755 --- a/ci/vendor-wit.sh +++ b/ci/vendor-wit.sh @@ -58,6 +58,8 @@ make_vendor "wasi-http" " make_vendor "wasi-runtime-config" "runtime-config@c667fe6" +make_vendor "wasi-keyvalue" "keyvalue@219ea36" + rm -rf $cache_dir # Separately (for now), vendor the `wasi-nn` WIT files since their retrieval is diff --git a/crates/test-programs/artifacts/build.rs b/crates/test-programs/artifacts/build.rs index 69fbe56b7b..018b5cce99 100644 --- a/crates/test-programs/artifacts/build.rs +++ b/crates/test-programs/artifacts/build.rs @@ -77,6 +77,7 @@ fn build_and_generate_tests() { s if s.starts_with("piped_") => "piped", s if s.starts_with("dwarf_") => "dwarf", s if s.starts_with("runtime_config_") => "runtime_config", + s if s.starts_with("keyvalue_") => "keyvalue", // If you're reading this because you hit this panic, either add it // to a test suite above or add a new "suite". The purpose of the // categorization above is to have a static assertion that tests diff --git a/crates/test-programs/src/bin/keyvalue_main.rs b/crates/test-programs/src/bin/keyvalue_main.rs new file mode 100644 index 0000000000..4fccb2c81f --- /dev/null +++ b/crates/test-programs/src/bin/keyvalue_main.rs @@ -0,0 +1,51 @@ +use test_programs::keyvalue::wasi::keyvalue::{atomics, batch, store}; + +fn main() { + let identifier = std::env::var_os("IDENTIFIER") + .unwrap() + .into_string() + .unwrap(); + let bucket = store::open(&identifier).unwrap(); + + if identifier != "" { + // for In-Memory provider, we have preset this data + assert_eq!(atomics::increment(&bucket, "atomics_key", 5).unwrap(), 5); + } + assert_eq!(atomics::increment(&bucket, "atomics_key", 1).unwrap(), 6); + + let resp = bucket.list_keys(None).unwrap(); + assert_eq!(resp.keys, vec!["atomics_key".to_string()]); + + bucket.set("hello", "world".as_bytes()).unwrap(); + + let v = bucket.get("hello").unwrap(); + assert_eq!(String::from_utf8(v.unwrap()).unwrap(), "world"); + + assert_eq!(bucket.exists("hello").unwrap(), true); + bucket.delete("hello").unwrap(); + assert_eq!(bucket.exists("hello").unwrap(), false); + + batch::set_many( + &bucket, + &[ + ("a1".to_string(), "v1".as_bytes().to_vec()), + ("b1".to_string(), "v1".as_bytes().to_vec()), + ("c1".to_string(), "v1".as_bytes().to_vec()), + ], + ) + .unwrap(); + batch::delete_many(&bucket, &["a1".to_string(), "c1".to_string()]).unwrap(); + let values = batch::get_many( + &bucket, + &["a1".to_string(), "b1".to_string(), "c1".to_string()], + ) + .unwrap(); + assert_eq!( + values, + vec![ + None, + Some(("b1".to_string(), "v1".as_bytes().to_vec())), + None + ] + ); +} diff --git a/crates/test-programs/src/lib.rs b/crates/test-programs/src/lib.rs index d4be5399d6..3586b71028 100644 --- a/crates/test-programs/src/lib.rs +++ b/crates/test-programs/src/lib.rs @@ -37,3 +37,11 @@ pub mod config { world: "wasi:config/imports", }); } + +pub mod keyvalue { + wit_bindgen::generate!({ + path: "../wasi-keyvalue/wit", + world: "wasi:keyvalue/imports", + type_section_suffix: "keyvalue", + }); +} diff --git a/crates/wasi-keyvalue/Cargo.toml b/crates/wasi-keyvalue/Cargo.toml new file mode 100644 index 0000000000..417cc126d4 --- /dev/null +++ b/crates/wasi-keyvalue/Cargo.toml @@ -0,0 +1,26 @@ +[package] +name = "wasmtime-wasi-keyvalue" +version.workspace = true +authors.workspace = true +edition.workspace = true +repository = "https://github.com/bytecodealliance/wasmtime" +license = "Apache-2.0 WITH LLVM-exception" +description = "Wasmtime implementation of the wasi-keyvalue API" + +[lints] +workspace = true + +[dependencies] +anyhow = { workspace = true } +wasmtime = { workspace = true, features = ["runtime", "async", "component-model"] } +async-trait = { workspace = true } +url = { workspace = true } +redis = { workspace = true, optional = true, features = ["tokio-comp"] } + +[dev-dependencies] +test-programs-artifacts = { workspace = true } +wasmtime-wasi = { workspace = true } +tokio = { workspace = true, features = ["macros"] } + +[features] +redis = ["dep:redis"] diff --git a/crates/wasi-keyvalue/src/lib.rs b/crates/wasi-keyvalue/src/lib.rs new file mode 100644 index 0000000000..c1149c370e --- /dev/null +++ b/crates/wasi-keyvalue/src/lib.rs @@ -0,0 +1,442 @@ +//! # Wasmtime's [wasi-keyvalue] Implementation +//! +//! This crate provides a Wasmtime host implementation of the [wasi-keyvalue] +//! API. With this crate, the runtime can run components that call APIs in +//! [wasi-keyvalue] and provide components with access to key-value storages. +//! +//! Currently supported storage backends: +//! * In-Memory (empty identifier) +//! * Redis, supported identifier format: +//! * `redis://[][:@][:port][/]` +//! * `redis+unix:///[?db=[&pass=][&user=]]` +//! +//! # Examples +//! +//! The usage of this crate is very similar to other WASI API implementations +//! such as [wasi:cli] and [wasi:http]. +//! +//! A common scenario is accessing redis in a [wasi:cli] component. +//! A standalone example of doing all this looks like: +//! +//! ``` +//! use wasmtime::{ +//! component::{Linker, ResourceTable}, +//! Config, Engine, Result, Store, +//! }; +//! use wasmtime_wasi::{WasiCtx, WasiCtxBuilder, WasiView}; +//! use wasmtime_wasi_keyvalue::{WasiKeyValue, WasiKeyValueCtx, WasiKeyValueCtxBuilder}; +//! +//! #[tokio::main] +//! async fn main() -> Result<()> { +//! let mut config = Config::new(); +//! config.async_support(true); +//! let engine = Engine::new(&config)?; +//! +//! let mut store = Store::new(&engine, Ctx { +//! table: ResourceTable::new(), +//! wasi_ctx: WasiCtxBuilder::new().build(), +//! wasi_keyvalue_ctx: WasiKeyValueCtxBuilder::new().build(), +//! }); +//! +//! let mut linker = Linker::::new(&engine); +//! wasmtime_wasi::add_to_linker_async(&mut linker)?; +//! // add `wasi-runtime-config` world's interfaces to the linker +//! wasmtime_wasi_keyvalue::add_to_linker(&mut linker, |h: &mut Ctx| { +//! WasiKeyValue::new(&h.wasi_keyvalue_ctx, &mut h.table) +//! })?; +//! +//! // ... use `linker` to instantiate within `store` ... +//! +//! Ok(()) +//! } +//! +//! struct Ctx { +//! table: ResourceTable, +//! wasi_ctx: WasiCtx, +//! wasi_keyvalue_ctx: WasiKeyValueCtx, +//! } +//! +//! impl WasiView for Ctx { +//! fn table(&mut self) -> &mut ResourceTable { &mut self.table } +//! fn ctx(&mut self) -> &mut WasiCtx { &mut self.wasi_ctx } +//! } +//! ``` +//! +//! [wasi-keyvalue]: https://github.com/WebAssembly/wasi-keyvalue +//! [wasi:cli]: https://docs.rs/wasmtime-wasi/latest +//! [wasi:http]: https://docs.rs/wasmtime-wasi-http/latest + +#![deny(missing_docs)] + +mod provider; +mod generated { + wasmtime::component::bindgen!({ + path: "wit", + world: "wasi:keyvalue/imports", + trappable_imports: true, + async: true, + with: { + "wasi:keyvalue/store/bucket": crate::Bucket, + }, + trappable_error_type: { + "wasi:keyvalue/store/error" => crate::Error, + }, + }); +} + +use self::generated::wasi::keyvalue; +use anyhow::Result; +use async_trait::async_trait; +use std::collections::HashMap; +use std::fmt::Display; +use url::Url; +use wasmtime::component::{Resource, ResourceTable, ResourceTableError}; + +#[doc(hidden)] +pub enum Error { + NoSuchStore, + AccessDenied, + Other(String), +} + +impl From for Error { + fn from(err: ResourceTableError) -> Self { + Self::Other(err.to_string()) + } +} + +pub(crate) fn to_other_error(e: impl Display) -> Error { + Error::Other(e.to_string()) +} + +#[doc(hidden)] +pub struct Bucket { + inner: Box, +} + +#[async_trait] +trait Host { + async fn get(&mut self, key: String) -> Result>, Error>; + + async fn set(&mut self, key: String, value: Vec) -> Result<(), Error>; + + async fn delete(&mut self, key: String) -> Result<(), Error>; + + async fn exists(&mut self, key: String) -> Result; + + async fn list_keys( + &mut self, + cursor: Option, + ) -> Result; + + async fn increment(&mut self, key: String, delta: u64) -> Result; + + async fn get_many( + &mut self, + keys: Vec, + ) -> Result)>>, Error>; + + async fn set_many(&mut self, key_values: Vec<(String, Vec)>) -> Result<(), Error>; + + async fn delete_many(&mut self, keys: Vec) -> Result<(), Error>; +} + +/// Builder-style structure used to create a [`WasiKeyValueCtx`]. +#[derive(Default)] +pub struct WasiKeyValueCtxBuilder { + in_memory_data: HashMap>, + #[cfg(feature = "redis")] + allowed_redis_hosts: Vec, + #[cfg(feature = "redis")] + redis_connection_timeout: Option, + #[cfg(feature = "redis")] + redis_response_timeout: Option, +} + +impl WasiKeyValueCtxBuilder { + /// Creates a builder for a new context with default parameters set. + pub fn new() -> Self { + Default::default() + } + + /// Preset data for the In-Memory provider. + pub fn in_memory_data(mut self, data: I) -> Self + where + I: IntoIterator, + K: Into, + V: Into>, + { + self.in_memory_data = data + .into_iter() + .map(|(k, v)| (k.into(), v.into())) + .collect(); + self + } + + /// Appends a list of Redis hosts to the allow-listed set each component gets + /// access to. It can be in the format `[:port]` or a unix domain + /// socket path. + /// + /// # Examples + /// + /// ``` + /// use wasmtime_wasi_keyvalue::WasiKeyValueCtxBuilder; + /// + /// # fn main() { + /// let ctx = WasiKeyValueCtxBuilder::new() + /// .allow_redis_hosts(&["localhost:1234", "/var/run/redis.sock"]) + /// .build(); + /// # } + /// ``` + #[cfg(feature = "redis")] + pub fn allow_redis_hosts(mut self, hosts: &[impl AsRef]) -> Self { + self.allowed_redis_hosts + .extend(hosts.iter().map(|h| h.as_ref().to_owned())); + self + } + + /// Sets the connection timeout parameter for the Redis provider. + #[cfg(feature = "redis")] + pub fn redis_connection_timeout(mut self, t: std::time::Duration) -> Self { + self.redis_connection_timeout = Some(t); + self + } + + /// Sets the response timeout parameter for the Redis provider. + #[cfg(feature = "redis")] + pub fn redis_response_timeout(mut self, t: std::time::Duration) -> Self { + self.redis_response_timeout = Some(t); + self + } + + /// Uses the configured context so far to construct the final [`WasiKeyValueCtx`]. + pub fn build(self) -> WasiKeyValueCtx { + WasiKeyValueCtx { + in_memory_data: self.in_memory_data, + #[cfg(feature = "redis")] + allowed_redis_hosts: self.allowed_redis_hosts, + #[cfg(feature = "redis")] + redis_connection_timeout: self.redis_connection_timeout, + #[cfg(feature = "redis")] + redis_response_timeout: self.redis_response_timeout, + } + } +} + +/// Capture the state necessary for use in the `wasi-keyvalue` API implementation. +pub struct WasiKeyValueCtx { + in_memory_data: HashMap>, + #[cfg(feature = "redis")] + allowed_redis_hosts: Vec, + #[cfg(feature = "redis")] + redis_connection_timeout: Option, + #[cfg(feature = "redis")] + redis_response_timeout: Option, +} + +impl WasiKeyValueCtx { + /// Convenience function for calling [`WasiKeyValueCtxBuilder::new`]. + pub fn builder() -> WasiKeyValueCtxBuilder { + WasiKeyValueCtxBuilder::new() + } + + #[cfg(feature = "redis")] + fn allow_redis_host(&self, u: &Url) -> bool { + let host = match u.host() { + Some(h) => match u.port() { + Some(port) => format!("{}:{}", h, port), + None => h.to_string(), + }, + // unix domain socket path + None => u.path().to_string(), + }; + self.allowed_redis_hosts.contains(&host) + } +} + +/// A wrapper capturing the needed internal `wasi-keyvalue` state. +pub struct WasiKeyValue<'a> { + ctx: &'a WasiKeyValueCtx, + table: &'a mut ResourceTable, +} + +impl<'a> WasiKeyValue<'a> { + /// Create a new view into the `wasi-keyvalue` state. + pub fn new(ctx: &'a WasiKeyValueCtx, table: &'a mut ResourceTable) -> Self { + Self { ctx, table } + } +} + +#[async_trait] +impl keyvalue::store::Host for WasiKeyValue<'_> { + async fn open(&mut self, identifier: String) -> Result, Error> { + if identifier == "" { + return Ok(self.table.push(Bucket { + inner: Box::new(provider::inmemory::InMemory::new( + self.ctx.in_memory_data.clone(), + )), + })?); + } + + let u = Url::parse(&identifier).map_err(to_other_error)?; + match u.scheme() { + "redis" | "redis+unix" => { + #[cfg(not(feature = "redis"))] + { + return Err(Error::Other( + "Cannot enable Redis support when the crate is not compiled with this feature." + .to_string(), + )); + } + #[cfg(feature = "redis")] + { + if !self.ctx.allow_redis_host(&u) { + return Err(Error::Other(format!( + "the identifier {} is not in the allowed list", + identifier + ))); + } + + let host = provider::redis::open( + identifier, + self.ctx.redis_response_timeout, + self.ctx.redis_connection_timeout, + ) + .await?; + Ok(self.table.push(Bucket { + inner: Box::new(host), + })?) + } + } + _ => Err(Error::NoSuchStore), + } + } + + fn convert_error(&mut self, err: Error) -> Result { + match err { + Error::NoSuchStore => Ok(keyvalue::store::Error::NoSuchStore), + Error::AccessDenied => Ok(keyvalue::store::Error::AccessDenied), + Error::Other(e) => Ok(keyvalue::store::Error::Other(e)), + } + } +} + +#[async_trait] +impl keyvalue::store::HostBucket for WasiKeyValue<'_> { + async fn get( + &mut self, + bucket: Resource, + key: String, + ) -> Result>, Error> { + let bucket = self.table.get_mut(&bucket)?; + bucket.inner.get(key).await + } + + async fn set( + &mut self, + bucket: Resource, + key: String, + value: Vec, + ) -> Result<(), Error> { + let bucket = self.table.get_mut(&bucket)?; + bucket.inner.set(key, value).await + } + + async fn delete(&mut self, bucket: Resource, key: String) -> Result<(), Error> { + let bucket = self.table.get_mut(&bucket)?; + bucket.inner.delete(key).await + } + + async fn exists(&mut self, bucket: Resource, key: String) -> Result { + let bucket = self.table.get_mut(&bucket)?; + bucket.inner.exists(key).await + } + + async fn list_keys( + &mut self, + bucket: Resource, + cursor: Option, + ) -> Result { + let bucket = self.table.get_mut(&bucket)?; + bucket.inner.list_keys(cursor).await + } + + fn drop(&mut self, bucket: Resource) -> Result<()> { + self.table.delete(bucket)?; + Ok(()) + } +} + +#[async_trait] +impl keyvalue::atomics::Host for WasiKeyValue<'_> { + async fn increment( + &mut self, + bucket: Resource, + key: String, + delta: u64, + ) -> Result { + let bucket = self.table.get_mut(&bucket)?; + bucket.inner.increment(key, delta).await + } +} + +#[async_trait] +impl keyvalue::batch::Host for WasiKeyValue<'_> { + async fn get_many( + &mut self, + bucket: Resource, + keys: Vec, + ) -> Result)>>, Error> { + let bucket = self.table.get_mut(&bucket)?; + bucket.inner.get_many(keys).await + } + + async fn set_many( + &mut self, + bucket: Resource, + key_values: Vec<(String, Vec)>, + ) -> Result<(), Error> { + let bucket = self.table.get_mut(&bucket)?; + bucket.inner.set_many(key_values).await + } + + async fn delete_many( + &mut self, + bucket: Resource, + keys: Vec, + ) -> Result<(), Error> { + let bucket = self.table.get_mut(&bucket)?; + bucket.inner.delete_many(keys).await + } +} + +/// Add all the `wasi-keyvalue` world's interfaces to a [`wasmtime::component::Linker`]. +pub fn add_to_linker( + l: &mut wasmtime::component::Linker, + f: impl Fn(&mut T) -> WasiKeyValue<'_> + Send + Sync + Copy + 'static, +) -> Result<()> { + keyvalue::store::add_to_linker_get_host(l, f)?; + keyvalue::atomics::add_to_linker_get_host(l, f)?; + keyvalue::batch::add_to_linker_get_host(l, f)?; + Ok(()) +} + +#[cfg(test)] +mod tests { + #[test] + #[cfg(feature = "redis")] + fn test_allow_redis_host() { + let ctx = super::WasiKeyValueCtx::builder() + .allow_redis_hosts(&["127.0.0.1:1234", "localhost", "/var/run/redis.sock"]) + .build(); + assert!(ctx.allow_redis_host(&super::Url::parse("redis://127.0.0.1:1234/db").unwrap())); + assert!(ctx.allow_redis_host(&super::Url::parse("redis://localhost").unwrap())); + assert!(!ctx.allow_redis_host(&super::Url::parse("redis://192.168.0.1").unwrap())); + assert!(ctx.allow_redis_host( + &super::Url::parse("redis+unix:///var/run/redis.sock?db=db").unwrap() + )); + assert!(!ctx.allow_redis_host( + &super::Url::parse("redis+unix:///var/local/redis.sock?db=db").unwrap() + )); + } +} diff --git a/crates/wasi-keyvalue/src/provider/inmemory.rs b/crates/wasi-keyvalue/src/provider/inmemory.rs new file mode 100644 index 0000000000..f0ba022369 --- /dev/null +++ b/crates/wasi-keyvalue/src/provider/inmemory.rs @@ -0,0 +1,94 @@ +use crate::{generated::wasi::keyvalue::store::KeyResponse, to_other_error, Error, Host}; +use async_trait::async_trait; +use std::collections::HashMap; +use std::sync::{Arc, Mutex}; + +#[derive(Default)] +pub(crate) struct InMemory { + store: Arc>>>, +} + +impl InMemory { + pub(crate) fn new(data: HashMap>) -> Self { + Self { + store: Arc::new(Mutex::new(data)), + } + } +} + +#[async_trait] +impl Host for InMemory { + async fn get(&mut self, key: String) -> Result>, Error> { + let store = self.store.lock().unwrap(); + Ok(store.get(&key).cloned()) + } + + async fn set(&mut self, key: String, value: Vec) -> Result<(), Error> { + let mut store = self.store.lock().unwrap(); + store.insert(key, value); + Ok(()) + } + + async fn delete(&mut self, key: String) -> Result<(), Error> { + let mut store = self.store.lock().unwrap(); + store.remove(&key); + Ok(()) + } + + async fn exists(&mut self, key: String) -> Result { + let store = self.store.lock().unwrap(); + Ok(store.contains_key(&key)) + } + + async fn list_keys(&mut self, cursor: Option) -> Result { + let store = self.store.lock().unwrap(); + let keys: Vec = store.keys().cloned().collect(); + let cursor = cursor.unwrap_or(0) as usize; + let keys_slice = &keys[cursor..]; + Ok(KeyResponse { + keys: keys_slice.to_vec(), + cursor: None, + }) + } + + async fn increment(&mut self, key: String, delta: u64) -> Result { + let mut store = self.store.lock().unwrap(); + let value = store + .entry(key.clone()) + .or_insert("0".to_string().into_bytes()); + let current_value = String::from_utf8(value.clone()) + .map_err(to_other_error)? + .parse::() + .map_err(to_other_error)?; + let new_value = current_value + delta; + *value = new_value.to_string().into_bytes(); + Ok(new_value) + } + + async fn get_many( + &mut self, + keys: Vec, + ) -> Result)>>, Error> { + let store = self.store.lock().unwrap(); + Ok(keys + .into_iter() + .map(|key| store.get(&key).map(|value| (key.clone(), value.clone()))) + .collect()) + } + + async fn set_many(&mut self, key_values: Vec<(String, Vec)>) -> Result<(), Error> { + let mut store = self.store.lock().unwrap(); + for (key, value) in key_values { + store.insert(key, value); + } + Ok(()) + } + + async fn delete_many(&mut self, keys: Vec) -> Result<(), Error> { + let mut store = self.store.lock().unwrap(); + for key in keys { + store.remove(&key); + } + Ok(()) + } +} diff --git a/crates/wasi-keyvalue/src/provider/mod.rs b/crates/wasi-keyvalue/src/provider/mod.rs new file mode 100644 index 0000000000..1ea5bdb90c --- /dev/null +++ b/crates/wasi-keyvalue/src/provider/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod inmemory; +#[cfg(feature = "redis")] +pub(crate) mod redis; diff --git a/crates/wasi-keyvalue/src/provider/redis.rs b/crates/wasi-keyvalue/src/provider/redis.rs new file mode 100644 index 0000000000..4020e8018f --- /dev/null +++ b/crates/wasi-keyvalue/src/provider/redis.rs @@ -0,0 +1,106 @@ +use crate::{generated::wasi::keyvalue::store::KeyResponse, Error, Host}; +use anyhow::Result; +use async_trait::async_trait; +use redis::{aio::MultiplexedConnection, AsyncCommands, RedisError}; +use std::time::Duration; + +struct Redis { + conn: MultiplexedConnection, +} + +impl From for Error { + fn from(err: RedisError) -> Self { + Self::Other(err.to_string()) + } +} + +pub(crate) async fn open( + identifier: String, + response_timeout: Option, + connection_timeout: Option, +) -> Result { + let client = redis::Client::open(identifier)?; + let conn = client + .get_multiplexed_async_connection_with_timeouts( + response_timeout.unwrap_or(Duration::MAX), + connection_timeout.unwrap_or(Duration::MAX), + ) + .await?; + Ok(Redis { conn }) +} + +#[async_trait] +impl Host for Redis { + async fn get(&mut self, key: String) -> Result>, Error> { + let v: Option> = self.conn.get(key).await?; + Ok(v) + } + + async fn set(&mut self, key: String, value: Vec) -> Result<(), Error> { + let _: () = self.conn.set(key, value).await?; + Ok(()) + } + + async fn delete(&mut self, key: String) -> Result<(), Error> { + let _: () = self.conn.del(key).await?; + Ok(()) + } + + async fn exists(&mut self, key: String) -> Result { + let exists: bool = self.conn.exists(key).await?; + Ok(exists) + } + + async fn list_keys(&mut self, cursor: Option) -> Result { + let cursor = cursor.unwrap_or(0); + let (new_cursor, keys): (u64, Vec) = redis::cmd("SCAN") + .arg(cursor) + .query_async(&mut self.conn) + .await?; + + Ok(KeyResponse { + keys, + cursor: if new_cursor == 0 { + None + } else { + Some(new_cursor) + }, + }) + } + + async fn increment(&mut self, key: String, delta: u64) -> Result { + let v: u64 = self.conn.incr(key, delta).await?; + Ok(v) + } + + async fn get_many( + &mut self, + keys: Vec, + ) -> Result)>>, Error> { + let values: Vec>> = self.conn.get(keys.clone()).await?; + + Ok(keys + .into_iter() + .zip(values.into_iter()) + .map(|(key, value)| value.map(|v| (key, v))) + .collect()) + } + + async fn set_many(&mut self, key_values: Vec<(String, Vec)>) -> Result<(), Error> { + let mut pipe = redis::pipe(); + for (key, value) in key_values { + pipe.set(key, value).ignore(); + } + pipe.query_async(&mut self.conn).await?; + Ok(()) + } + + async fn delete_many(&mut self, keys: Vec) -> Result<(), Error> { + let mut pipe = redis::pipe(); + for key in keys { + pipe.del(key).ignore(); + } + pipe.query_async(&mut self.conn).await?; + Ok(()) + } +} diff --git a/crates/wasi-keyvalue/tests/main.rs b/crates/wasi-keyvalue/tests/main.rs new file mode 100644 index 0000000000..fa103d63e6 --- /dev/null +++ b/crates/wasi-keyvalue/tests/main.rs @@ -0,0 +1,93 @@ +use anyhow::{anyhow, Result}; +use test_programs_artifacts::{foreach_keyvalue, KEYVALUE_MAIN_COMPONENT}; +use wasmtime::{ + component::{Component, Linker, ResourceTable}, + Store, +}; +use wasmtime_wasi::{bindings::Command, WasiCtx, WasiCtxBuilder, WasiView}; +use wasmtime_wasi_keyvalue::{WasiKeyValue, WasiKeyValueCtx, WasiKeyValueCtxBuilder}; + +struct Ctx { + table: ResourceTable, + wasi_ctx: WasiCtx, + wasi_keyvalue_ctx: WasiKeyValueCtx, +} + +impl WasiView for Ctx { + fn table(&mut self) -> &mut ResourceTable { + &mut self.table + } + + fn ctx(&mut self) -> &mut WasiCtx { + &mut self.wasi_ctx + } +} + +async fn run_wasi(path: &str, ctx: Ctx) -> Result<()> { + let engine = test_programs_artifacts::engine(|config| { + config.async_support(true); + }); + let mut store = Store::new(&engine, ctx); + let component = Component::from_file(&engine, path)?; + + let mut linker = Linker::new(&engine); + wasmtime_wasi::add_to_linker_async(&mut linker)?; + wasmtime_wasi_keyvalue::add_to_linker(&mut linker, |h: &mut Ctx| { + WasiKeyValue::new(&h.wasi_keyvalue_ctx, &mut h.table) + })?; + + let command = Command::instantiate_async(&mut store, &component, &linker).await?; + command + .wasi_cli_run() + .call_run(&mut store) + .await? + .map_err(|()| anyhow!("command returned with failing exit status")) +} + +macro_rules! assert_test_exists { + ($name:ident) => { + #[allow(unused_imports)] + use self::$name as _; + }; +} + +foreach_keyvalue!(assert_test_exists); + +#[tokio::test(flavor = "multi_thread")] +async fn keyvalue_main() -> Result<()> { + run_wasi( + KEYVALUE_MAIN_COMPONENT, + Ctx { + table: ResourceTable::new(), + wasi_ctx: WasiCtxBuilder::new() + .inherit_stderr() + .env("IDENTIFIER", "") + .build(), + wasi_keyvalue_ctx: WasiKeyValueCtxBuilder::new() + .in_memory_data([("atomics_key", "5")]) + .build(), + }, + ) + .await +} + +#[cfg(feature = "redis")] +#[tokio::test(flavor = "multi_thread")] +async fn keyvalue_redis() -> Result<()> { + run_wasi( + KEYVALUE_MAIN_COMPONENT, + Ctx { + table: ResourceTable::new(), + wasi_ctx: WasiCtxBuilder::new() + .inherit_stderr() + .env("IDENTIFIER", "redis://127.0.0.1/") + .build(), + wasi_keyvalue_ctx: WasiKeyValueCtxBuilder::new() + .allow_redis_hosts(&["127.0.0.1"]) + .redis_connection_timeout(std::time::Duration::from_secs(5)) + .redis_response_timeout(std::time::Duration::from_secs(5)) + .build(), + }, + ) + .await +} diff --git a/crates/wasi-keyvalue/wit/deps/keyvalue/atomic.wit b/crates/wasi-keyvalue/wit/deps/keyvalue/atomic.wit new file mode 100644 index 0000000000..059efc4889 --- /dev/null +++ b/crates/wasi-keyvalue/wit/deps/keyvalue/atomic.wit @@ -0,0 +1,22 @@ +/// A keyvalue interface that provides atomic operations. +/// +/// Atomic operations are single, indivisible operations. When a fault causes an atomic operation to +/// fail, it will appear to the invoker of the atomic operation that the action either completed +/// successfully or did nothing at all. +/// +/// Please note that this interface is bare functions that take a reference to a bucket. This is to +/// get around the current lack of a way to "extend" a resource with additional methods inside of +/// wit. Future version of the interface will instead extend these methods on the base `bucket` +/// resource. +interface atomics { + use store.{bucket, error}; + + /// Atomically increment the value associated with the key in the store by the given delta. It + /// returns the new value. + /// + /// If the key does not exist in the store, it creates a new key-value pair with the value set + /// to the given delta. + /// + /// If any other error occurs, it returns an `Err(error)`. + increment: func(bucket: borrow, key: string, delta: u64) -> result; +} \ No newline at end of file diff --git a/crates/wasi-keyvalue/wit/deps/keyvalue/batch.wit b/crates/wasi-keyvalue/wit/deps/keyvalue/batch.wit new file mode 100644 index 0000000000..70c05feb91 --- /dev/null +++ b/crates/wasi-keyvalue/wit/deps/keyvalue/batch.wit @@ -0,0 +1,63 @@ +/// A keyvalue interface that provides batch operations. +/// +/// A batch operation is an operation that operates on multiple keys at once. +/// +/// Batch operations are useful for reducing network round-trip time. For example, if you want to +/// get the values associated with 100 keys, you can either do 100 get operations or you can do 1 +/// batch get operation. The batch operation is faster because it only needs to make 1 network call +/// instead of 100. +/// +/// A batch operation does not guarantee atomicity, meaning that if the batch operation fails, some +/// of the keys may have been modified and some may not. +/// +/// This interface does has the same consistency guarantees as the `store` interface, meaning that +/// you should be able to "read your writes." +/// +/// Please note that this interface is bare functions that take a reference to a bucket. This is to +/// get around the current lack of a way to "extend" a resource with additional methods inside of +/// wit. Future version of the interface will instead extend these methods on the base `bucket` +/// resource. +interface batch { + use store.{bucket, error}; + + /// Get the key-value pairs associated with the keys in the store. It returns a list of + /// key-value pairs. + /// + /// If any of the keys do not exist in the store, it returns a `none` value for that pair in the + /// list. + /// + /// MAY show an out-of-date value if there are concurrent writes to the store. + /// + /// If any other error occurs, it returns an `Err(error)`. + get-many: func(bucket: borrow, keys: list) -> result>>>, error>; + + /// Set the values associated with the keys in the store. If the key already exists in the + /// store, it overwrites the value. + /// + /// Note that the key-value pairs are not guaranteed to be set in the order they are provided. + /// + /// If any of the keys do not exist in the store, it creates a new key-value pair. + /// + /// If any other error occurs, it returns an `Err(error)`. When an error occurs, it does not + /// rollback the key-value pairs that were already set. Thus, this batch operation does not + /// guarantee atomicity, implying that some key-value pairs could be set while others might + /// fail. + /// + /// Other concurrent operations may also be able to see the partial results. + set-many: func(bucket: borrow, key-values: list>>) -> result<_, error>; + + /// Delete the key-value pairs associated with the keys in the store. + /// + /// Note that the key-value pairs are not guaranteed to be deleted in the order they are + /// provided. + /// + /// If any of the keys do not exist in the store, it skips the key. + /// + /// If any other error occurs, it returns an `Err(error)`. When an error occurs, it does not + /// rollback the key-value pairs that were already deleted. Thus, this batch operation does not + /// guarantee atomicity, implying that some key-value pairs could be deleted while others might + /// fail. + /// + /// Other concurrent operations may also be able to see the partial results. + delete-many: func(bucket: borrow, keys: list) -> result<_, error>; +} diff --git a/crates/wasi-keyvalue/wit/deps/keyvalue/store.wit b/crates/wasi-keyvalue/wit/deps/keyvalue/store.wit new file mode 100644 index 0000000000..3354ea2f32 --- /dev/null +++ b/crates/wasi-keyvalue/wit/deps/keyvalue/store.wit @@ -0,0 +1,122 @@ +/// A keyvalue interface that provides eventually consistent key-value operations. +/// +/// Each of these operations acts on a single key-value pair. +/// +/// The value in the key-value pair is defined as a `u8` byte array and the intention is that it is +/// the common denominator for all data types defined by different key-value stores to handle data, +/// ensuring compatibility between different key-value stores. Note: the clients will be expecting +/// serialization/deserialization overhead to be handled by the key-value store. The value could be +/// a serialized object from JSON, HTML or vendor-specific data types like AWS S3 objects. +/// +/// Data consistency in a key value store refers to the guarantee that once a write operation +/// completes, all subsequent read operations will return the value that was written. +/// +/// Any implementation of this interface must have enough consistency to guarantee "reading your +/// writes." In particular, this means that the client should never get a value that is older than +/// the one it wrote, but it MAY get a newer value if one was written around the same time. These +/// guarantees only apply to the same client (which will likely be provided by the host or an +/// external capability of some kind). In this context a "client" is referring to the caller or +/// guest that is consuming this interface. Once a write request is committed by a specific client, +/// all subsequent read requests by the same client will reflect that write or any subsequent +/// writes. Another client running in a different context may or may not immediately see the result +/// due to the replication lag. As an example of all of this, if a value at a given key is A, and +/// the client writes B, then immediately reads, it should get B. If something else writes C in +/// quick succession, then the client may get C. However, a client running in a separate context may +/// still see A or B +interface store { + /// The set of errors which may be raised by functions in this package + variant error { + /// The host does not recognize the store identifier requested. + no-such-store, + + /// The requesting component does not have access to the specified store + /// (which may or may not exist). + access-denied, + + /// Some implementation-specific error has occurred (e.g. I/O) + other(string) + } + + /// A response to a `list-keys` operation. + record key-response { + /// The list of keys returned by the query. + keys: list, + /// The continuation token to use to fetch the next page of keys. If this is `null`, then + /// there are no more keys to fetch. + cursor: option + } + + /// Get the bucket with the specified identifier. + /// + /// `identifier` must refer to a bucket provided by the host. + /// + /// `error::no-such-store` will be raised if the `identifier` is not recognized. + open: func(identifier: string) -> result; + + /// A bucket is a collection of key-value pairs. Each key-value pair is stored as a entry in the + /// bucket, and the bucket itself acts as a collection of all these entries. + /// + /// It is worth noting that the exact terminology for bucket in key-value stores can very + /// depending on the specific implementation. For example: + /// + /// 1. Amazon DynamoDB calls a collection of key-value pairs a table + /// 2. Redis has hashes, sets, and sorted sets as different types of collections + /// 3. Cassandra calls a collection of key-value pairs a column family + /// 4. MongoDB calls a collection of key-value pairs a collection + /// 5. Riak calls a collection of key-value pairs a bucket + /// 6. Memcached calls a collection of key-value pairs a slab + /// 7. Azure Cosmos DB calls a collection of key-value pairs a container + /// + /// In this interface, we use the term `bucket` to refer to a collection of key-value pairs + resource bucket { + /// Get the value associated with the specified `key` + /// + /// The value is returned as an option. If the key-value pair exists in the + /// store, it returns `Ok(value)`. If the key does not exist in the + /// store, it returns `Ok(none)`. + /// + /// If any other error occurs, it returns an `Err(error)`. + get: func(key: string) -> result>, error>; + + /// Set the value associated with the key in the store. If the key already + /// exists in the store, it overwrites the value. + /// + /// If the key does not exist in the store, it creates a new key-value pair. + /// + /// If any other error occurs, it returns an `Err(error)`. + set: func(key: string, value: list) -> result<_, error>; + + /// Delete the key-value pair associated with the key in the store. + /// + /// If the key does not exist in the store, it does nothing. + /// + /// If any other error occurs, it returns an `Err(error)`. + delete: func(key: string) -> result<_, error>; + + /// Check if the key exists in the store. + /// + /// If the key exists in the store, it returns `Ok(true)`. If the key does + /// not exist in the store, it returns `Ok(false)`. + /// + /// If any other error occurs, it returns an `Err(error)`. + exists: func(key: string) -> result; + + /// Get all the keys in the store with an optional cursor (for use in pagination). It + /// returns a list of keys. Please note that for most KeyValue implementations, this is a + /// can be a very expensive operation and so it should be used judiciously. Implementations + /// can return any number of keys in a single response, but they should never attempt to + /// send more data than is reasonable (i.e. on a small edge device, this may only be a few + /// KB, while on a large machine this could be several MB). Any response should also return + /// a cursor that can be used to fetch the next page of keys. See the `key-response` record + /// for more information. + /// + /// Note that the keys are not guaranteed to be returned in any particular order. + /// + /// If the store is empty, it returns an empty list. + /// + /// MAY show an out-of-date list of keys if there are concurrent writes to the store. + /// + /// If any error occurs, it returns an `Err(error)`. + list-keys: func(cursor: option) -> result; + } +} diff --git a/crates/wasi-keyvalue/wit/deps/keyvalue/watch.wit b/crates/wasi-keyvalue/wit/deps/keyvalue/watch.wit new file mode 100644 index 0000000000..ff13f7523e --- /dev/null +++ b/crates/wasi-keyvalue/wit/deps/keyvalue/watch.wit @@ -0,0 +1,16 @@ +/// A keyvalue interface that provides watch operations. +/// +/// This interface is used to provide event-driven mechanisms to handle +/// keyvalue changes. +interface watcher { + /// A keyvalue interface that provides handle-watch operations. + use store.{bucket}; + + /// Handle the `set` event for the given bucket and key. It includes a reference to the `bucket` + /// that can be used to interact with the store. + on-set: func(bucket: bucket, key: string, value: list); + + /// Handle the `delete` event for the given bucket and key. It includes a reference to the + /// `bucket` that can be used to interact with the store. + on-delete: func(bucket: bucket, key: string); +} \ No newline at end of file diff --git a/crates/wasi-keyvalue/wit/deps/keyvalue/world.wit b/crates/wasi-keyvalue/wit/deps/keyvalue/world.wit new file mode 100644 index 0000000000..066148c1fa --- /dev/null +++ b/crates/wasi-keyvalue/wit/deps/keyvalue/world.wit @@ -0,0 +1,26 @@ +package wasi:keyvalue@0.2.0-draft; + +/// The `wasi:keyvalue/imports` world provides common APIs for interacting with key-value stores. +/// Components targeting this world will be able to do: +/// +/// 1. CRUD (create, read, update, delete) operations on key-value stores. +/// 2. Atomic `increment` and CAS (compare-and-swap) operations. +/// 3. Batch operations that can reduce the number of round trips to the network. +world imports { + /// The `store` capability allows the component to perform eventually consistent operations on + /// the key-value store. + import store; + + /// The `atomic` capability allows the component to perform atomic / `increment` and CAS + /// (compare-and-swap) operations. + import atomics; + + /// The `batch` capability allows the component to perform eventually consistent batch + /// operations that can reduce the number of round trips to the network. + import batch; +} + +world watch-service { + include imports; + export watcher; +} \ No newline at end of file diff --git a/crates/wasi-keyvalue/wit/world.wit b/crates/wasi-keyvalue/wit/world.wit new file mode 100644 index 0000000000..1d4c6ab442 --- /dev/null +++ b/crates/wasi-keyvalue/wit/world.wit @@ -0,0 +1,6 @@ +// We actually don't use this; it's just to let bindgen! find the corresponding world in wit/deps. +package wasmtime:wasi; + +world bindings { + include wasi:keyvalue/imports@0.2.0-draft; +} diff --git a/scripts/publish.rs b/scripts/publish.rs index 20322dc63b..b3fcf21541 100644 --- a/scripts/publish.rs +++ b/scripts/publish.rs @@ -69,6 +69,7 @@ const CRATES_TO_PUBLISH: &[&str] = &[ "wasmtime-wasi-http", "wasmtime-wasi-nn", "wasmtime-wasi-runtime-config", + "wasmtime-wasi-keyvalue", "wasmtime-wasi-threads", "wasmtime-wast", "wasmtime-c-api-macros", @@ -89,6 +90,7 @@ const PUBLIC_CRATES: &[&str] = &[ "wasmtime-wasi", "wasmtime-wasi-nn", "wasmtime-wasi-runtime-config", + "wasmtime-wasi-keyvalue", "wasmtime-wasi-threads", "wasmtime-cli", // all cranelift crates are considered "public" in that they can't diff --git a/supply-chain/audits.toml b/supply-chain/audits.toml index aa6c21129b..3f8c7d6d47 100644 --- a/supply-chain/audits.toml +++ b/supply-chain/audits.toml @@ -2424,6 +2424,11 @@ criteria = "safe-to-deploy" version = "1.0.17" notes = "plenty of unsafe pointer and vec tricks, but in well-structured and commented code that appears to be correct" +[[audits.sha1_smol]] +who = "Alex Crichton " +criteria = "safe-to-deploy" +version = "1.0.1" + [[audits.sha2]] who = "Benjamin Bouvier " criteria = "safe-to-deploy" @@ -2476,6 +2481,11 @@ criteria = "safe-to-deploy" delta = "0.4.7 -> 0.4.9" notes = "Minor OS compat updates but otherwise nothing major here." +[[audits.socket2]] +who = "Alex Crichton " +criteria = "safe-to-deploy" +delta = "0.4.9 -> 0.4.4" + [[audits.spin]] who = "Alex Crichton " criteria = "safe-to-run" diff --git a/supply-chain/config.toml b/supply-chain/config.toml index 290afc7a99..20fb7d75e8 100644 --- a/supply-chain/config.toml +++ b/supply-chain/config.toml @@ -218,6 +218,10 @@ criteria = "safe-to-deploy" version = "0.2.7" criteria = "safe-to-run" +[[exemptions.combine]] +version = "4.6.7" +criteria = "safe-to-deploy" + [[exemptions.console]] version = "0.15.0" criteria = "safe-to-deploy" @@ -486,6 +490,10 @@ criteria = "safe-to-deploy" version = "0.3.0" criteria = "safe-to-deploy" +[[exemptions.redis]] +version = "0.25.4" +criteria = "safe-to-deploy" + [[exemptions.redox_syscall]] version = "0.2.13" criteria = "safe-to-deploy" diff --git a/supply-chain/imports.lock b/supply-chain/imports.lock index ff04c49551..2d7845bc04 100644 --- a/supply-chain/imports.lock +++ b/supply-chain/imports.lock @@ -4008,6 +4008,13 @@ version = "0.2.9" notes = "Reviewed on https://fxrev.dev/824504" aggregated-from = "https://fuchsia.googlesource.com/fuchsia/+/refs/heads/main/third_party/rust_crates/supply-chain/audits.toml?format=TEXT" +[[audits.google.audits.socket2]] +who = "David Koloski " +criteria = "safe-to-deploy" +delta = "0.4.4 -> 0.5.5" +notes = "Reviewed at https://fxrev.dev/946307" +aggregated-from = "https://fuchsia.googlesource.com/fuchsia/+/refs/heads/main/third_party/rust_crates/supply-chain/audits.toml?format=TEXT" + [[audits.google.audits.threadpool]] who = "Dennis Kempin " criteria = "safe-to-run" @@ -4545,6 +4552,12 @@ criteria = "safe-to-deploy" delta = "0.4.6 -> 0.4.7" aggregated-from = "https://hg.mozilla.org/mozilla-central/raw-file/tip/supply-chain/audits.toml" +[[audits.mozilla.audits.socket2]] +who = "Kershaw Chang " +criteria = "safe-to-deploy" +delta = "0.5.5 -> 0.5.7" +aggregated-from = "https://hg.mozilla.org/mozilla-central/raw-file/tip/supply-chain/audits.toml" + [[audits.mozilla.audits.subtle]] who = "Simon Friedberger " criteria = "safe-to-deploy"