Browse Source

Implement wasi-runtime-config (#8950)

* Implement wasi-runtime-config

* Avoid clone the WasiRuntimeConfig every time
pull/8970/head
Xinzhao Xu 4 months ago
committed by GitHub
parent
commit
0a296b3e03
No known key found for this signature in database GPG Key ID: B5690EEEBB952194
  1. 12
      Cargo.lock
  2. 2
      Cargo.toml
  3. 80
      ci/vendor-wit.sh
  4. 1
      crates/test-programs/artifacts/build.rs
  5. 8
      crates/test-programs/src/bin/runtime_config_get.rs
  6. 7
      crates/test-programs/src/lib.rs
  7. 20
      crates/wasi-runtime-config/Cargo.toml
  8. 140
      crates/wasi-runtime-config/src/lib.rs
  9. 69
      crates/wasi-runtime-config/tests/main.rs
  10. 25
      crates/wasi-runtime-config/wit/deps/runtime-config/runtime_config.wit
  11. 6
      crates/wasi-runtime-config/wit/deps/runtime-config/world.wit
  12. 6
      crates/wasi-runtime-config/wit/world.wit
  13. 2
      scripts/publish.rs

12
Cargo.lock

@ -3557,6 +3557,7 @@ dependencies = [
"wasmtime-wasi",
"wasmtime-wasi-http",
"wasmtime-wasi-nn",
"wasmtime-wasi-runtime-config",
"wasmtime-wasi-threads",
"wasmtime-wast",
"wast 214.0.0",
@ -3886,6 +3887,17 @@ dependencies = [
"windows",
]
[[package]]
name = "wasmtime-wasi-runtime-config"
version = "24.0.0"
dependencies = [
"anyhow",
"test-programs-artifacts",
"tokio",
"wasmtime",
"wasmtime-wasi",
]
[[package]]
name = "wasmtime-wasi-threads"
version = "24.0.0"

2
Cargo.toml

@ -53,6 +53,7 @@ wasmtime-wast = { workspace = true, optional = true }
wasi-common = { workspace = true, default-features = true, features = ["exit"], optional = true }
wasmtime-wasi = { workspace = true, default-features = true, optional = true }
wasmtime-wasi-nn = { workspace = true, optional = true }
wasmtime-wasi-runtime-config = { workspace = true, optional = true }
wasmtime-wasi-threads = { workspace = true, optional = true }
wasmtime-wasi-http = { workspace = true, optional = true }
clap = { workspace = true }
@ -192,6 +193,7 @@ wasmtime-wast = { path = "crates/wast", version = "=24.0.0" }
wasmtime-wasi = { path = "crates/wasi", version = "24.0.0", default-features = false }
wasmtime-wasi-http = { path = "crates/wasi-http", version = "=24.0.0", default-features = false }
wasmtime-wasi-nn = { path = "crates/wasi-nn", version = "24.0.0" }
wasmtime-wasi-runtime-config = { path = "crates/wasi-runtime-config", version = "24.0.0" }
wasmtime-wasi-threads = { path = "crates/wasi-threads", version = "24.0.0" }
wasmtime-component-util = { path = "crates/component-util", version = "=24.0.0" }
wasmtime-component-macro = { path = "crates/component-macro", version = "=24.0.0" }

80
ci/vendor-wit.sh

@ -1,4 +1,4 @@
#!/bin/sh
#!/usr/bin/env bash
# Script to re-vendor the WIT files that Wasmtime uses as defined by a
# particular tag in upstream repositories.
@ -6,31 +6,59 @@
# This script is executed on CI to ensure that everything is up-to-date.
set -ex
# Space-separated list of wasi proposals that are vendored here along with the
# tag that they're all vendored at.
#
# This assumes that the repositories all have the pattern:
# https://github.com/WebAssembly/wasi-$repo
# and every repository has a tag `v$tag` here. That is currently done as part
# of the WASI release process.
repos="cli clocks filesystem http io random sockets"
tag=0.2.0
# First, replace the existing vendored WIT files in the `wasi` crate.
dst=crates/wasi/wit/deps
rm -rf $dst
mkdir -p $dst
for repo in $repos; do
mkdir $dst/$repo
curl -L https://github.com/WebAssembly/wasi-$repo/archive/refs/tags/v$tag.tar.gz | \
tar xzf - --strip-components=2 -C $dst/$repo wasi-$repo-$tag/wit
rm -rf $dst/$repo/deps*
done
# Also replace the `wasi-http` WIT files since they match those in the `wasi`
# crate.
rm -rf crates/wasi-http/wit/deps
cp -r $dst crates/wasi-http/wit
# The make_vendor function takes a base path (e.g., "wasi") and a list
# of packages in the format "name@tag". It constructs the full destination
# path, downloads the tarballs from GitHub, extracts the relevant files, and
# removes any unwanted directories.
make_vendor() {
local name=$1
local packages=$2
local path="crates/$name/wit/deps"
rm -rf $path
mkdir -p $path
for package in $packages; do
IFS='@' read -r repo tag <<< "$package"
mkdir -p $path/$repo
cached_extracted_dir="$cache_dir/$repo-$tag"
if [[ ! -d $cached_extracted_dir ]]; then
mkdir -p $cached_extracted_dir
curl -sL https://github.com/WebAssembly/wasi-$repo/archive/$tag.tar.gz | \
tar xzf - --strip-components=1 -C $cached_extracted_dir
rm -rf $cached_extracted_dir/wit/deps*
fi
cp -r $cached_extracted_dir/wit/* $path/$repo
done
}
cache_dir=$(mktemp -d)
make_vendor "wasi" "
cli@v0.2.0
clocks@v0.2.0
filesystem@v0.2.0
io@v0.2.0
random@v0.2.0
sockets@v0.2.0
http@v0.2.0
"
make_vendor "wasi-http" "
cli@v0.2.0
clocks@v0.2.0
filesystem@v0.2.0
io@v0.2.0
random@v0.2.0
sockets@v0.2.0
http@v0.2.0
"
make_vendor "wasi-runtime-config" "runtime-config@c667fe6"
rm -rf $cache_dir
# Separately (for now), vendor the `wasi-nn` WIT files since their retrieval is
# slightly different than above.

1
crates/test-programs/artifacts/build.rs

@ -76,6 +76,7 @@ fn build_and_generate_tests() {
s if s.starts_with("nn_") => "nn",
s if s.starts_with("piped_") => "piped",
s if s.starts_with("dwarf_") => "dwarf",
s if s.starts_with("runtime_config_") => "runtime_config",
// If you're reading this because you hit this panic, either add it
// to a test suite above or add a new "suite". The purpose of the
// categorization above is to have a static assertion that tests

8
crates/test-programs/src/bin/runtime_config_get.rs

@ -0,0 +1,8 @@
use test_programs::config::wasi::config::runtime;
fn main() {
let v = runtime::get("hello").unwrap().unwrap();
assert_eq!(v, "world");
let config = runtime::get_all().unwrap();
assert_eq!(config, [("hello".to_string(), "world".to_string())])
}

7
crates/test-programs/src/lib.rs

@ -30,3 +30,10 @@ pub mod proxy {
},
});
}
pub mod config {
wit_bindgen::generate!({
path: "../wasi-runtime-config/wit",
world: "wasi:config/imports",
});
}

20
crates/wasi-runtime-config/Cargo.toml

@ -0,0 +1,20 @@
[package]
name = "wasmtime-wasi-runtime-config"
version.workspace = true
authors.workspace = true
edition.workspace = true
repository = "https://github.com/bytecodealliance/wasmtime"
license = "Apache-2.0 WITH LLVM-exception"
description = "Wasmtime implementation of the wasi-runtime-config API"
[lints]
workspace = true
[dependencies]
anyhow = { workspace = true }
wasmtime = { workspace = true, features = ["runtime", "component-model"] }
[dev-dependencies]
test-programs-artifacts = { workspace = true }
wasmtime-wasi = { workspace = true }
tokio = { workspace = true, features = ["macros"] }

140
crates/wasi-runtime-config/src/lib.rs

@ -0,0 +1,140 @@
//! # Wasmtime's [wasi-runtime-config] Implementation
//!
//! This crate provides a Wasmtime host implementation of the [wasi-runtime-config]
//! API. With this crate, the runtime can run components that call APIs in
//! [wasi-runtime-config] and provide configuration variables for the component.
//!
//! # Examples
//!
//! The usage of this crate is very similar to other WASI API implementations
//! such as [wasi:cli] and [wasi:http].
//!
//! A common scenario is getting runtime-passed configurations in a [wasi:cli]
//! component. A standalone example of doing all this looks like:
//!
//! ```
//! use wasmtime::{
//! component::{Linker, ResourceTable},
//! Config, Engine, Result, Store,
//! };
//! use wasmtime_wasi::{WasiCtx, WasiCtxBuilder, WasiView};
//! use wasmtime_wasi_runtime_config::{WasiRuntimeConfig, WasiRuntimeConfigVariables};
//!
//! #[tokio::main]
//! async fn main() -> Result<()> {
//! let mut config = Config::new();
//! config.async_support(true);
//! let engine = Engine::new(&config)?;
//!
//! let mut store = Store::new(&engine, Ctx {
//! table: ResourceTable::new(),
//! wasi_ctx: WasiCtxBuilder::new().build(),
//! wasi_runtime_config_vars: WasiRuntimeConfigVariables::from_iter(vec![
//! ("config_key1", "value1"),
//! ("config_key2", "value2"),
//! ]),
//! });
//!
//! let mut linker = Linker::<Ctx>::new(&engine);
//! wasmtime_wasi::add_to_linker_async(&mut linker)?;
//! // add `wasi-runtime-config` world's interfaces to the linker
//! wasmtime_wasi_runtime_config::add_to_linker(&mut linker, |h: &mut Ctx| {
//! WasiRuntimeConfig::from(&h.wasi_runtime_config_vars)
//! })?;
//!
//! // ... use `linker` to instantiate within `store` ...
//!
//! Ok(())
//! }
//!
//! struct Ctx {
//! table: ResourceTable,
//! wasi_ctx: WasiCtx,
//! wasi_runtime_config_vars: WasiRuntimeConfigVariables,
//! }
//!
//! impl WasiView for Ctx {
//! fn table(&mut self) -> &mut ResourceTable { &mut self.table }
//! fn ctx(&mut self) -> &mut WasiCtx { &mut self.wasi_ctx }
//! }
//! ```
//!
//! [wasi-runtime-config]: https://github.com/WebAssembly/wasi-runtime-config
//! [wasi:cli]: https://docs.rs/wasmtime-wasi/latest
//! [wasi:http]: https://docs.rs/wasmtime-wasi-http/latest
#![deny(missing_docs)]
use anyhow::Result;
use std::collections::HashMap;
mod gen_ {
wasmtime::component::bindgen!({
path: "wit",
world: "wasi:config/imports",
trappable_imports: true,
});
}
use self::gen_::wasi::config::runtime as generated;
/// Capture the state necessary for use in the `wasi-runtime-config` API implementation.
#[derive(Default)]
pub struct WasiRuntimeConfigVariables(HashMap<String, String>);
impl<S: Into<String>> FromIterator<(S, S)> for WasiRuntimeConfigVariables {
fn from_iter<I: IntoIterator<Item = (S, S)>>(iter: I) -> Self {
Self(
iter.into_iter()
.map(|(k, v)| (k.into(), v.into()))
.collect(),
)
}
}
impl WasiRuntimeConfigVariables {
/// Create a new runtime configuration.
pub fn new() -> Self {
Default::default()
}
/// Insert a key-value pair into the configuration map.
pub fn insert(&mut self, key: impl Into<String>, value: impl Into<String>) -> &mut Self {
self.0.insert(key.into(), value.into());
self
}
}
/// A wrapper capturing the needed internal `wasi-runtime-config` state.
pub struct WasiRuntimeConfig<'a> {
vars: &'a WasiRuntimeConfigVariables,
}
impl<'a> From<&'a WasiRuntimeConfigVariables> for WasiRuntimeConfig<'a> {
fn from(vars: &'a WasiRuntimeConfigVariables) -> Self {
Self { vars }
}
}
impl generated::Host for WasiRuntimeConfig<'_> {
fn get(&mut self, key: String) -> Result<Result<Option<String>, generated::ConfigError>> {
Ok(Ok(self.vars.0.get(&key).map(|s| s.to_owned())))
}
fn get_all(&mut self) -> Result<Result<Vec<(String, String)>, generated::ConfigError>> {
Ok(Ok(self
.vars
.0
.iter()
.map(|(k, v)| (k.to_string(), v.to_string()))
.collect()))
}
}
/// Add all the `wasi-runtime-config` world's interfaces to a [`wasmtime::component::Linker`].
pub fn add_to_linker<T>(
l: &mut wasmtime::component::Linker<T>,
f: impl Fn(&mut T) -> WasiRuntimeConfig<'_> + Send + Sync + Copy + 'static,
) -> Result<()> {
generated::add_to_linker_get_host(l, f)?;
Ok(())
}

69
crates/wasi-runtime-config/tests/main.rs

@ -0,0 +1,69 @@
use anyhow::{anyhow, Result};
use test_programs_artifacts::{foreach_runtime_config, RUNTIME_CONFIG_GET_COMPONENT};
use wasmtime::{
component::{Component, Linker, ResourceTable},
Store,
};
use wasmtime_wasi::{add_to_linker_async, bindings::Command, WasiCtx, WasiCtxBuilder, WasiView};
use wasmtime_wasi_runtime_config::{WasiRuntimeConfig, WasiRuntimeConfigVariables};
struct Ctx {
table: ResourceTable,
wasi_ctx: WasiCtx,
wasi_runtime_config_vars: WasiRuntimeConfigVariables,
}
impl WasiView for Ctx {
fn table(&mut self) -> &mut ResourceTable {
&mut self.table
}
fn ctx(&mut self) -> &mut WasiCtx {
&mut self.wasi_ctx
}
}
async fn run_wasi(path: &str, ctx: Ctx) -> Result<()> {
let engine = test_programs_artifacts::engine(|config| {
config.async_support(true);
});
let mut store = Store::new(&engine, ctx);
let component = Component::from_file(&engine, path)?;
let mut linker = Linker::new(&engine);
add_to_linker_async(&mut linker)?;
wasmtime_wasi_runtime_config::add_to_linker(&mut linker, |h: &mut Ctx| {
WasiRuntimeConfig::from(&h.wasi_runtime_config_vars)
})?;
let command = Command::instantiate_async(&mut store, &component, &linker).await?;
command
.wasi_cli_run()
.call_run(&mut store)
.await?
.map_err(|()| anyhow!("command returned with failing exit status"))
}
macro_rules! assert_test_exists {
($name:ident) => {
#[allow(unused_imports)]
use self::$name as _;
};
}
foreach_runtime_config!(assert_test_exists);
#[tokio::test(flavor = "multi_thread")]
async fn runtime_config_get() -> Result<()> {
run_wasi(
RUNTIME_CONFIG_GET_COMPONENT,
Ctx {
table: ResourceTable::new(),
wasi_ctx: WasiCtxBuilder::new().build(),
wasi_runtime_config_vars: WasiRuntimeConfigVariables::from_iter(vec![(
"hello", "world",
)]),
},
)
.await
}

25
crates/wasi-runtime-config/wit/deps/runtime-config/runtime_config.wit

@ -0,0 +1,25 @@
interface runtime {
/// An error type that encapsulates the different errors that can occur fetching config
variant config-error {
/// This indicates an error from an "upstream" config source.
/// As this could be almost _anything_ (such as Vault, Kubernetes ConfigMaps, KeyValue buckets, etc),
/// the error message is a string.
upstream(string),
/// This indicates an error from an I/O operation.
/// As this could be almost _anything_ (such as a file read, network connection, etc),
/// the error message is a string.
/// Depending on how this ends up being consumed,
/// we may consider moving this to use the `wasi:io/error` type instead.
/// For simplicity right now in supporting multiple implementations, it is being left as a string.
io(string),
}
/// Gets a single opaque config value set at the given key if it exists
get: func(
/// A string key to fetch
key: string
) -> result<option<string>, config-error>;
/// Gets a list of all set config data
get-all: func() -> result<list<tuple<string, string>>, config-error>;
}

6
crates/wasi-runtime-config/wit/deps/runtime-config/world.wit

@ -0,0 +1,6 @@
package wasi:config@0.2.0-draft;
world imports {
/// The runtime interface for config
import runtime;
}

6
crates/wasi-runtime-config/wit/world.wit

@ -0,0 +1,6 @@
// We actually don't use this; it's just to let bindgen! find the corresponding world in wit/deps.
package wasmtime:wasi;
world bindings {
include wasi:config/imports@0.2.0-draft;
}

2
scripts/publish.rs

@ -67,6 +67,7 @@ const CRATES_TO_PUBLISH: &[&str] = &[
"wasmtime-wasi",
"wasmtime-wasi-http",
"wasmtime-wasi-nn",
"wasmtime-wasi-runtime-config",
"wasmtime-wasi-threads",
"wasmtime-wast",
"wasmtime-c-api-macros",
@ -86,6 +87,7 @@ const PUBLIC_CRATES: &[&str] = &[
"wasmtime",
"wasmtime-wasi",
"wasmtime-wasi-nn",
"wasmtime-wasi-runtime-config",
"wasmtime-wasi-threads",
"wasmtime-cli",
// all cranelift crates are considered "public" in that they can't

Loading…
Cancel
Save