diff --git a/.azure-pipelines.yml b/.azure-pipelines.yml deleted file mode 100644 index b913ccdd03..0000000000 --- a/.azure-pipelines.yml +++ /dev/null @@ -1,257 +0,0 @@ -name: $(Build.SourceBranch)-$(date:yyyyMMdd)$(rev:.r) -trigger: - branches: - include: - - 'master' - tags: - include: - - '*' - exclude: - - 'dev' - -jobs: -- job: rustfmt - pool: - vmImage: 'macos-10.14' - steps: - - checkout: self - submodules: true - - template: ci/azure-install-rust.yml - - script: rustup component add rustfmt - displayName: Add rustfmt - - script: cargo fmt --all -- --check - displayName: Check formatting - variables: - toolchain: stable - -# Smoke test to build docs on one builder, using OSX for now since it's the -# fastest -- job: docs - pool: - vmImage: 'macos-10.14' - steps: - - checkout: self - submodules: true - - template: ci/azure-install-rust.yml - - script: cargo doc - displayName: Build documentation - variables: - toolchain: stable - -- job: Test - strategy: - matrix: - windows-stable: - imageName: 'vs2017-win2016' - toolchain: stable - linux-stable: - imageName: 'ubuntu-16.04' - toolchain: stable - mac-stable: - imageName: 'macos-10.14' - toolchain: stable - mac-beta: - imageName: 'macos-10.14' - toolchain: beta - mac-nightly: - imageName: 'macos-10.14' - toolchain: nightly - - pool: - vmImage: $(imageName) - - steps: - - checkout: self - submodules: true - - template: ci/azure-install-rust.yml - - - script: cargo fetch - displayName: Fetch cargo dependencies - - # Build and test all features except for lightbeam - - bash: cargo test --all --exclude lightbeam --exclude wasmtime-wasi-c --exclude wasmtime-py -- --nocapture - displayName: Cargo test - env: - RUST_BACKTRACE: 1 - - # Build and test lightbeam if we're using the nightly toolchain - - bash: cargo build --package lightbeam - displayName: Cargo build lightbeam - condition: and(succeeded(), eq(variables['toolchain'], 'nightly')) - - bash: cargo test --package lightbeam -- --nocapture - displayName: Cargo test lightbeam - # Lightbeam tests fail right now, but we don't want to block on that. - continueOnError: true - condition: and(succeeded(), eq(variables['toolchain'], 'nightly')) - env: - RUST_BACKTRACE: 1 - -- job: Build - strategy: - matrix: - windows: - imageName: 'vs2017-win2016' - # Statically link against msvcrt to produce slightly more portable - # binaries on Windows by reducing our binary compatibility requirements. - RUSTFLAGS: -Ctarget-feature=+crt-static - mac: - imageName: 'macos-10.14' - # Lower the deployment target from our build image in an attempt to - # build more portable binaries that run on older releases. Note that - # 10.9 here is arbitrarily chosen and just happens to be the lowest that - # works at this time. Raising this is probably fine. - MACOSX_DEPLOYMENT_TARGET: 10.9 - variables: - toolchain: stable - pool: - vmImage: $(imageName) - steps: - - template: ci/azure-build-release.yml - -# Build the Linux release binary in an older Linux container (in this case -# Centos 6) -- job: Build_linux - variables: - toolchain: stable - container: - image: centos:6 - options: "--name ci-container -v /usr/bin/docker:/tmp/docker:ro" - steps: - - template: ci/azure-prepare-centos-6.yml - - template: ci/azure-build-release.yml - -# Build the `wasmtime-py` python extension in the same manner we build the -# binaries above, since these wheels are also native code that we're -# distributing. -# -# Note that the builds here are using a nightly compiler, not a stable compiler, -# since this is what PyO3 requires. -- job: Build_wheels - strategy: - matrix: - windows: - imageName: 'vs2017-win2016' - RUSTFLAGS: -Ctarget-feature=+crt-static - mac: - imageName: 'macos-10.14' - MACOSX_DEPLOYMENT_TARGET: 10.9 - variables: - toolchain: nightly-2019-08-15 - pool: - vmImage: $(imageName) - steps: - - template: ci/azure-build-wheels.yml -- job: Build_linux_wheels - variables: - toolchain: nightly-2019-08-15 - container: - image: centos:6 - options: "--name ci-container -v /usr/bin/docker:/tmp/docker:ro" - steps: - - template: ci/azure-prepare-centos-6.yml - - template: ci/azure-build-wheels.yml - -- job: Doc_book - displayName: "Doc - build the book" - steps: - - script: | - set -e - curl -L https://github.com/rust-lang-nursery/mdBook/releases/download/v0.3.1/mdbook-v0.3.1-x86_64-unknown-linux-gnu.tar.gz | tar xzf - - echo "##vso[task.prependpath]$PWD" - displayName: "Install mdbook" - - script: (cd docs && mdbook build) - - task: PublishPipelineArtifact@1 - inputs: - artifactName: doc_book - path: docs/book - -- job: Doc_api - displayName: "Doc - build the API documentation" - steps: - - template: ci/azure-install-rust.yml - - script: cargo doc - - task: PublishPipelineArtifact@1 - inputs: - artifactName: doc_api - path: target/doc - -- job: Publish - dependsOn: - - Build - - Build_wheels - - Build_linux - - Build_linux_wheels - - Doc_book - - Doc_api - condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI')) - steps: - # Checking out the sources is needed to be able to delete the "dev" tag, see below. - - checkout: self - persistCredentials: true - submodules: false - - task: DownloadPipelineArtifact@1 - inputs: - targetPath: $(Build.ArtifactStagingDirectory) - - script: | - set -e - mv $(Build.ArtifactStagingDirectory)/doc_book gh-pages - mv $(Build.ArtifactStagingDirectory)/doc_api gh-pages/api - displayName: Create gh-pages directory - - script: | - set -e - cd gh-pages - git init - cp ../.git/config .git - git add . - git -c user.name='ci' -c user.email='ci' commit -m init - git push -f -q https://github.com/CraneStation/wasmtime HEAD:gh-pages - displayName: Publish gh-pages directory - condition: and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/master')) - - script: | - echo "##vso[task.setvariable variable=tagName;]`echo $BUILD_SOURCEBRANCH | sed -e 's|refs/tags/||'`" - condition: startsWith(variables['Build.SourceBranch'], 'refs/tags/') - - task: GitHubRelease@0 - inputs: - gitHubConnection: 'tschneidereit-releases' - target: '$(Build.SourceVersion)' - tagSource: 'manual' - tag: '$(tagName)' - title: 'Wasmtime $(tagName)' - assets: '$(Build.ArtifactStagingDirectory)/**' - isDraft: false - isPreRelease: true - condition: and(startsWith(variables['Build.SourceBranch'], 'refs/tags/'), - ne(variables['Build.SourceBranch'], 'refs/tags/dev')) - - # GitHub doesn't support doing rolling releases for branch. - # To simulate that for dev builds, always do a release for the "dev" tag. - # While the `edit` action for the GitHubRelease task would replace any assets - # associated with the tag, it wouldn't update the tag itself. Hence, delete the - # tag if it exists, and re-create it every time. - # Also explicitly delete the GitHub release, which would otherwise turn into a draft - # and linger forever. - - task: GitHubRelease@0 - inputs: - gitHubConnection: 'tschneidereit-releases' - action: 'delete' - tag: 'dev' - # This might fail in case the target repo doesn't yet have this tag, which is fine. - continueOnError: true - condition: in(variables['Build.SourceBranch'], 'refs/heads/master', 'refs/tags/dev') - - script: | - git -c http.extraheader="AUTHORIZATION: basic ***" push origin :dev - # This might fail in case the target repo doesn't yet have this tag, which is fine. - continueOnError: true - condition: in(variables['Build.SourceBranch'], 'refs/heads/master', 'refs/tags/dev') - - task: GitHubRelease@0 - inputs: - gitHubConnection: 'tschneidereit-releases' - action: 'create' - target: '$(Build.SourceVersion)' - tag: 'dev' - tagSource: 'manual' - title: 'Latest CI build' - assets: '$(Build.ArtifactStagingDirectory)/**' - isDraft: false - isPreRelease: true - condition: in(variables['Build.SourceBranch'], 'refs/heads/master', 'refs/tags/dev') diff --git a/.github/actions/binary-compatible-builds/README.md b/.github/actions/binary-compatible-builds/README.md new file mode 100644 index 0000000000..8368fd4f1a --- /dev/null +++ b/.github/actions/binary-compatible-builds/README.md @@ -0,0 +1,9 @@ +# binary-compatible-builds + +A small (ish) action which is intended to be used and will configure builds of +Rust projects to be "more binary compatible". On Windows and macOS this +involves setting a few env vars, and on Linux this involves spinning up a CentOS +6 container which is running in the background. + +All subsequent build commands need to be wrapped in `$CENTOS` to optionally run +on `$CENTOS` on Linux to ensure builds happen inside the container. diff --git a/.github/actions/binary-compatible-builds/action.yml b/.github/actions/binary-compatible-builds/action.yml new file mode 100644 index 0000000000..de2e74ec77 --- /dev/null +++ b/.github/actions/binary-compatible-builds/action.yml @@ -0,0 +1,6 @@ +name: 'Set up a CentOS 6 container to build releases in' +description: 'Set up a CentOS 6 container to build releases in' + +runs: + using: node12 + main: 'main.js' diff --git a/.github/actions/binary-compatible-builds/main.js b/.github/actions/binary-compatible-builds/main.js new file mode 100755 index 0000000000..7b53e1733b --- /dev/null +++ b/.github/actions/binary-compatible-builds/main.js @@ -0,0 +1,68 @@ +#!/usr/bin/env node + +const child_process = require('child_process'); +const stdio = { stdio: 'inherit' }; + +// On OSX all we need to do is configure our deployment target as old as +// possible. For now 10.9 is the limit. +if (process.platform == 'darwin') { + console.log("::set-env name=MACOSX_DEPLOYMENT_TARGET::10.9"); + console.log("::set-env name=python::python3"); + return; +} + +// On Windows we build against the static CRT to reduce dll dependencies +if (process.platform == 'win32') { + console.log("::set-env name=RUSTFLAGS::-Ctarget-feature=+crt-static"); + console.log("::set-env name=python::python"); + return; +} + +// ... and on Linux we do fancy things with containers. We'll spawn an old +// CentOS container in the background with a super old glibc, and then we'll run +// commands in there with the `$CENTOS` env var. + +if (process.env.CENTOS !== undefined) { + const args = ['exec', '-w', process.cwd(), '-i', 'centos']; + for (const arg of process.argv.slice(2)) { + args.push(arg); + } + child_process.execFileSync('docker', args, stdio); + return; +} + +// Add our rust mount onto PATH, but also add some stuff to PATH from +// the packages that we install. +let path = process.env.PATH; +path = `${path}:/rust/bin`; +path = `/opt/rh/devtoolset-8/root/usr/bin:${path}`; +path = `/opt/rh/rh-python36/root/usr/bin:${path}`; + +// Spawn a container daemonized in the background which we'll connect to via +// `docker exec`. This'll have access to the current directory. +child_process.execFileSync('docker', [ + 'run', + '-di', + '--name', 'centos', + '-v', `${process.cwd()}:${process.cwd()}`, + '-v', `${child_process.execSync('rustc --print sysroot').toString().trim()}:/rust:ro`, + '--env', `PATH=${path}`, + 'centos:6', +], stdio); + +// Use ourselves to run future commands +console.log(`::set-env name=CENTOS::${__filename}`) + +// See https://edwards.sdsu.edu/research/c11-on-centos-6/ for where these +const exec = s => { + child_process.execSync(`docker exec centos ${s}`, stdio); +}; +exec('yum install -y centos-release-scl cmake xz epel-release'); +exec('yum install -y rh-python36 patchelf unzip'); +exec('yum install -y devtoolset-8-gcc devtoolset-8-binutils devtoolset-8-gcc-c++'); + +// Delete `libstdc++.so` to force gcc to link against `libstdc++.a` instead. +// This is a hack and not the right way to do this, but it ends up doing the +// right thing for now. +exec('rm -f /opt/rh/devtoolset-8/root/usr/lib/gcc/x86_64-redhat-linux/8/libstdc++.so'); +console.log("::set-env name=python::python3"); diff --git a/.github/actions/github-release/Dockerfile b/.github/actions/github-release/Dockerfile new file mode 100644 index 0000000000..5849eac7d2 --- /dev/null +++ b/.github/actions/github-release/Dockerfile @@ -0,0 +1,8 @@ +FROM node:slim + +COPY . /action +WORKDIR /action + +RUN npm install --production + +ENTRYPOINT ["node", "/action/main.js"] diff --git a/.github/actions/github-release/README.md b/.github/actions/github-release/README.md new file mode 100644 index 0000000000..c70ba8f495 --- /dev/null +++ b/.github/actions/github-release/README.md @@ -0,0 +1,18 @@ +# github-release + +An action used to publish GitHub releases for `wasmtime`. + +As of the time of this writing there's a few actions floating around which +perform github releases but they all tend to have their set of drawbacks. +Additionally nothing handles deleting releases which we need for our rolling +`dev` release. + +To handle all this this action rolls-its-own implementation using the +actions/toolkit repository and packages published there. These run in a Docker +container and take various inputs to orchestrate the release from the build. + +More comments can be found in `main.js`. + +Testing this is really hard. If you want to try though run `npm install` and +then `node main.js`. You'll have to configure a bunch of env vars though to get +anything reasonably working. diff --git a/.github/actions/github-release/action.yml b/.github/actions/github-release/action.yml new file mode 100644 index 0000000000..51a074adfa --- /dev/null +++ b/.github/actions/github-release/action.yml @@ -0,0 +1,15 @@ +name: 'wasmtime github releases' +description: 'wasmtime github releases' +inputs: + token: + description: '' + required: true + name: + description: '' + required: true + files: + description: '' + required: true +runs: + using: 'docker' + image: 'Dockerfile' diff --git a/.github/actions/github-release/main.js b/.github/actions/github-release/main.js new file mode 100644 index 0000000000..6d2f80f946 --- /dev/null +++ b/.github/actions/github-release/main.js @@ -0,0 +1,91 @@ +const core = require('@actions/core'); +const path = require("path"); +const fs = require("fs"); +const github = require('@actions/github'); +const glob = require('glob'); + +async function run() { + // Load all our inputs and env vars. Note that `getInput` reads from `INPUT_*` + const files = core.getInput('files'); + const name = core.getInput('name'); + const token = core.getInput('token'); + const slug = process.env.GITHUB_REPOSITORY; + const owner = slug.split('/')[0]; + const repo = slug.split('/')[1]; + const sha = process.env.GITHUB_SHA; + + core.info(`files: ${files}`); + core.info(`name: ${name}`); + core.info(`token: ${token}`); + + const octokit = new github.GitHub(token); + + // If this is a `dev` release then we need to actually delete the previous + // release since we can't overwrite a new one. We also need to update the + // `dev` tag while we're at it. So here you'll see: + // + // * Look for the `dev` release, then delete it if it exists + // * Update the `dev` release to our current sha, or create one if it doesn't + // exist + if (name == 'dev') { + const releases = await octokit.paginate("GET /repos/:owner/:repo/releases", { owner, repo }); + for (const release of releases) { + if (release.tag_name !== 'dev') { + continue; + } + const release_id = release.id; + core.info(`deleting release ${release_id}`); + await octokit.repos.deleteRelease({ owner, repo, release_id }); + } + + try { + core.info(`updating dev tag`); + await octokit.git.updateRef({ + owner, + repo, + ref: 'tags/dev', + sha, + force: true, + }); + } catch (e) { + console.log("ERROR: ", JSON.stringify(e, null, 2)); + core.info(`creating dev tag`); + await octokit.git.createTag({ + owner, + repo, + tag: 'dev', + message: 'dev release', + object: sha, + type: 'commit', + }); + } + } + + // Creates an official GitHub release for this `tag`, and if this is `dev` + // then we know that from the previous block this should be a fresh release. + core.info(`creating a release`); + const release = await octokit.repos.createRelease({ + owner, + repo, + tag_name: name, + prerelease: name === 'dev', + }); + + // Upload all the relevant assets for this release as just general blobs. + for (const file of glob.sync(files)) { + const size = fs.statSync(file).size; + core.info(`upload ${file}`); + await octokit.repos.uploadReleaseAsset({ + file: fs.createReadStream(file), + headers: { 'content-length': size, 'content-type': 'application/octet-stream' }, + name: path.basename(file), + url: release.data.upload_url, + }) + } +} + +run().catch(err => { + console.log("ERROR: ", JSON.stringify(err, null, 2)); + core.setFailed(err.message); + console.log(err.stack); +}); diff --git a/.github/actions/github-release/package.json b/.github/actions/github-release/package.json new file mode 100644 index 0000000000..abfc55f6ff --- /dev/null +++ b/.github/actions/github-release/package.json @@ -0,0 +1,10 @@ +{ + "name": "wasmtime-github-release", + "version": "0.0.0", + "main": "main.js", + "dependencies": { + "@actions/core": "^1.0.0", + "@actions/github": "^1.0.0", + "glob": "^7.1.5" + } +} diff --git a/.github/actions/install-rust/README.md b/.github/actions/install-rust/README.md new file mode 100644 index 0000000000..df8e94dccc --- /dev/null +++ b/.github/actions/install-rust/README.md @@ -0,0 +1,18 @@ +# install-rust + +A small github action to install `rustup` and a Rust toolchain. This is +generally expressed inline, but it was repeated enough in this repository it +seemed worthwhile to extract. + +Some gotchas: + +* Can't `--self-update` on Windows due to permission errors (a bug in Github + Actions) +* `rustup` isn't installed on macOS (a bug in Github Actions) + +When the above are fixed we should delete this action and just use this inline: + +```yml +- run: rustup update $toolchain && rustup default $toolchain + shell: bash +``` diff --git a/.github/actions/install-rust/action.yml b/.github/actions/install-rust/action.yml new file mode 100644 index 0000000000..7a19659184 --- /dev/null +++ b/.github/actions/install-rust/action.yml @@ -0,0 +1,12 @@ +name: 'Install Rust toolchain' +description: 'Install both `rustup` and a Rust toolchain' + +inputs: + toolchain: + description: 'Default toolchan to install' + required: false + default: 'stable' + +runs: + using: node12 + main: 'main.js' diff --git a/.github/actions/install-rust/main.js b/.github/actions/install-rust/main.js new file mode 100644 index 0000000000..fe01702bb7 --- /dev/null +++ b/.github/actions/install-rust/main.js @@ -0,0 +1,17 @@ +const child_process = require('child_process'); +const toolchain = process.env.INPUT_TOOLCHAIN; + +for (var i = 0, keys = Object.keys(process.env), ii = keys.length; i < ii; i++) { + console.log(keys[i] + '=' + process.env[keys[i]]); +} + +if (process.platform === 'darwin') { + child_process.execSync(`curl https://sh.rustup.rs | sh -s -- -y --default-toolchain=none --profile=minimal`); + const bindir = `${process.env.HOME}/.cargo/bin`; + console.log(`::add-path::${bindir}`); + process.env.PATH = `${process.env.PATH}:${bindir}`; + child_process.execFileSync('rustup', ['set', 'profile', 'minimal']); +} + +child_process.execFileSync('rustup', ['update', toolchain, '--no-self-update']); +child_process.execFileSync('rustup', ['default', toolchain]); diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 0000000000..3900c29149 --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,348 @@ +name: CI +on: + push: + branches: [master] + tags-ignore: [dev] + pull_request: + branches: [master] + +jobs: + # Check Code style quickly by running `rustfmt` over all code + rustfmt: + name: Rustfmt + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@master + - uses: ./.github/actions/install-rust + - run: cargo fmt --all -- --check + + # Build `mdBook` documentation for `wasmtime`, and upload it as a temporary + # build artifact + doc_book: + name: Doc - build the book + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@master + - run: | + set -e + curl -L https://github.com/rust-lang-nursery/mdBook/releases/download/v0.3.1/mdbook-v0.3.1-x86_64-unknown-linux-gnu.tar.gz | tar xzf - + echo ::add-path::`pwd` + - run: (cd docs && mdbook build) + - uses: actions/upload-artifact@v1 + with: + name: doc-book + path: docs/book + + # Build rustdoc API documentation for `wasmtime*` crates. Note that we don't + # want to document all our transitive dependencies, hence `--no-deps`. This is + # a temporary build artifact we upload to consume later. + doc_api: + name: Doc - build the API documentation + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@master + - uses: ./.github/actions/install-rust + - run: cargo doc --no-deps -p wasmtime + - run: cargo doc --no-deps -p wasmtime-api + - run: cargo doc --no-deps -p wasmtime-debug + - run: cargo doc --no-deps -p wasmtime-environ + - run: cargo doc --no-deps -p wasmtime-interface-types + - run: cargo doc --no-deps -p wasmtime-jit + - run: cargo doc --no-deps -p wasmtime-obj + - run: cargo doc --no-deps -p wasmtime-runtime + - run: cargo doc --no-deps -p wasmtime-wasi + - run: cargo doc --no-deps -p wasmtime-wast + - uses: actions/upload-artifact@v1 + with: + name: doc-api + path: target/doc + + # Perform all tests (debug mode) for `wasmtime`. This runs stable/beta/nightly + # channels of Rust as well as macOS/Linux/Windows. + test: + name: Test + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + build: [stable, beta, nightly, windows, linux] + include: + - build: stable + os: macos-latest + rust: stable + - build: beta + os: macos-latest + rust: beta + - build: nightly + os: macos-latest + rust: nightly + - build: linux + os: ubuntu-latest + rust: stable + - build: windows + os: windows-latest + rust: stable + steps: + - uses: actions/checkout@master + with: + submodules: true + - uses: ./.github/actions/install-rust + with: + toolchain: ${{ matrix.rust }} + + - run: cargo fetch + + # Build and test all features except for lightbeam + - run: cargo test --all --exclude lightbeam --exclude wasmtime-wasi-c --exclude wasmtime-py -- --nocapture + env: + RUST_BACKTRACE: 1 + + # Build and test lightbeam if we're using the nightly toolchain. Note that + # Lightbeam tests fail right now, but we don't want to block on that. + - run: cargo build --package lightbeam + if: matrix.rust == 'nightly' + - run: cargo test --package lightbeam -- --nocapture + if: matrix.rust == 'nightly' + continue-on-error: true + env: + RUST_BACKTRACE: 1 + + # Builds a Python wheel (package) for Windows/Mac/Linux. Note that we're + # careful to create binary-compatible releases here to old releases of + # Windows/Mac/Linux. This will also build wheels for Python 3.6 and 3.7. + wheels: + name: Python Wheel + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + steps: + - uses: actions/checkout@master + - uses: ./.github/actions/install-rust + with: + toolchain: nightly-2019-08-15 + - uses: ./.github/actions/binary-compatible-builds + - run: mkdir misc/wasmtime-py/wheelhouse + shell: bash + + # Install Python & dependencies needed for our `setup.py` scripts + - name: Setup Python 3.6 + uses: actions/setup-python@v1 + with: + python-version: '3.6' + architecture: x64 + - run: $CENTOS pip3 install setuptools wheel==0.31.1 setuptools-rust + shell: bash + - run: (cd misc/wasmtime-py && $CENTOS $python setup.py bdist_wheel) + shell: bash + + # Clear the build directory between building different wheels for different + # Python versions to ensure that we don't package dynamic libraries twice by + # accident. + - run: $CENTOS rm -rf misc/wasmtime-py/build + shell: bash + + # Set up Python 3.7 (and build it on Linux), reinstall dependencies, then + # rebuild our wheels + - name: Setup Python 3.7 + uses: actions/setup-python@v1 + with: + python-version: '3.7' + architecture: x64 + if: matrix.os != 'ubuntu-latest' + - name: Build Python 3.7 + run: $CENTOS sh ci/setup_centos6_python37.sh + if: matrix.os == 'ubuntu-latest' + - run: $CENTOS pip3 install setuptools wheel==0.31.1 setuptools-rust auditwheel + shell: bash + - run: (cd misc/wasmtime-py && $CENTOS $python setup.py bdist_wheel) + shell: bash + + # Move `dist/*.whl` into `wheelhouse/` so we can deploy them, but on Linux we + # need to run an `auditwheel` command as well to turn these into "manylinux" + # wheels to run across a number of distributions. + - run: cp misc/wasmtime-py/dist/*.whl misc/wasmtime-py/wheelhouse/ + shell: bash + if: matrix.os != 'ubuntu-latest' + - run: | + set -e + cd misc/wasmtime-py + for whl in dist/*.whl; do + $CENTOS auditwheel repair "$whl" -w wheelhouse/ + done + shell: bash + if: matrix.os == 'ubuntu-latest' + + # Upload this for the publishing stage of pipelines + - uses: actions/upload-artifact@v1 + with: + name: wheels-${{ matrix.os }} + path: misc/wasmtime-py/wheelhouse + + # Perform release builds of `wasmtime` and `libwasmtime_api.so`. Builds on + # Windows/Mac/Linux, and artifacts are uploaded after the build is finished. + # Note that we also run tests here to test exactly what we're deploying. + build: + name: Build wasmtime + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + steps: + - uses: actions/checkout@master + with: + submodules: true + - uses: ./.github/actions/install-rust + - uses: ./.github/actions/binary-compatible-builds + + # Build `wasmtime` and executables + - run: $CENTOS cargo build --release --bin wasmtime --bin wasm2obj + shell: bash + # Build `libwasmtime_api.so` + - run: $CENTOS cargo build --release --features wasm-c-api --manifest-path wasmtime-api/Cargo.toml + shell: bash + # Test what we just built + - run: $CENTOS cargo test --release --all --exclude lightbeam --exclude wasmtime-wasi-c --exclude wasmtime-py --exclude wasmtime-api + shell: bash + env: + RUST_BACKTRACE: 1 + + # ... and now perform some goop to move all the relevant artifacts into + # something that we'll upload from this action. + + - run: mkdir dist + shell: bash + + # Move binaries to dist folder + - run: cp target/release/{wasmtime,wasm2obj} dist + if: matrix.os != 'windows-latest' + - run: cp target/release/{wasmtime,wasm2obj}.exe dist + shell: bash + if: matrix.os == 'windows-latest' + + # Move libwasmtime_api dylib to dist folder + - run: cp target/release/libwasmtime_api.{so,a} dist + if: matrix.os == 'ubuntu-latest' + - run: cp target/release/libwasmtime_api.{dylib,a} dist + if: matrix.os == 'macos-latest' + - run: cp target/release/wasmtime_api.{dll,lib} dist + shell: bash + if: matrix.os == 'windows-latest' + + # Make a Windows MSI installer if we're on Windows + - run: | + export WT_VERSION=`cat Cargo.toml | sed -n 's/^version = "\([^"]*\)".*/\1/p'` + "$WIX/bin/candle" -arch x64 -out target/wasmtime.wixobj installer/msi/wasmtime.wxs + "$WIX/bin/light" -out dist/installer.msi target/wasmtime.wixobj -ext WixUtilExtension + rm dist/installer.wixpdb + shell: bash + if: matrix.os == 'windows-latest' + + - uses: actions/upload-artifact@v1 + with: + name: bins-${{ matrix.os }} + path: dist + + # Consumes all published artifacts from all the previous build steps, creates + # a bunch of tarballs for all of them, and then publishes the tarballs + # themselves as an artifact (for inspection) and then optionally creates + # github releases and/or tags for pushes. + publish: + name: Publish + needs: [doc_book, doc_api, wheels, build] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@master + with: + submodules: true + - run: rustup update stable && rustup default stable + + # Download all the artifacts that we'll be publishing. Should keep an eye on + # the `download-artifact` repository to see if we can ever get something + # like "download all artifacts" or "download this list of artifacts" + - name: Download book + uses: actions/download-artifact@v1 + with: + name: doc-book + - name: Download API docs + uses: actions/download-artifact@v1 + with: + name: doc-api + - name: Download macOS Wheel + uses: actions/download-artifact@v1 + with: + name: wheels-macos-latest + - name: Download macOS binaries + uses: actions/download-artifact@v1 + with: + name: bins-macos-latest + - name: Download Linux Wheel + uses: actions/download-artifact@v1 + with: + name: wheels-ubuntu-latest + - name: Download Linux binaries + uses: actions/download-artifact@v1 + with: + name: bins-ubuntu-latest + - name: Download Windows Wheel + uses: actions/download-artifact@v1 + with: + name: wheels-windows-latest + - name: Download Windows binaries + uses: actions/download-artifact@v1 + with: + name: bins-windows-latest + + - name: Assemble gh-pages + run: | + mv doc-book gh-pages + mv doc-api gh-pages/api + + # If this is a push to the master branch push to the `gh-pages` using a + # deploy key. Note that a deploy key is necessary for now because otherwise + # using the default token for github actions doesn't actually trigger a page + # rebuild. + - name: Push to gh-pages + run: curl -LsSf https://git.io/fhJ8n | rustc - && (cd gh-pages && ../rust_out) + env: + GITHUB_DEPLOY_KEY: ${{ secrets.DEPLOY_KEY }} + BUILD_REPOSITORY_ID: ${{ github.repository }} + BUILD_SOURCEVERSION: ${{ github.sha }} + if: github.event_name == 'push' && github.ref == 'refs/heads/master' + + - name: Calculate tag name + run: | + name=dev + if [[ $GITHUB_REF == refs/tags* ]]; then + name=${GITHUB_REF:10} + fi + echo ::set-output name=val::$name + echo ::set-env name=TAG::$name + id: tagname + + # Assemble all the build artifacts into tarballs and zip archives. + - name: Assemble tarballs + run: | + ./ci/build-tarballs.sh x86_64-linux ubuntu-latest + ./ci/build-tarballs.sh x86_64-windows windows-latest .exe + ./ci/build-tarballs.sh x86_64-macos macos-latest + + # Upload all assembled tarballs as an artifact of the github action run, so + # that way even PRs can inspect the output. + - uses: actions/upload-artifact@v1 + with: + name: tarballs + path: dist + + # ... and if this was an actual push (tag or `master`) then we publish a + # new release. This'll automatically publish a tag release or update `dev` + # with this `sha` + - name: Publish Release + uses: ./.github/actions/github-release + if: github.event_name == 'push' && (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags')) + with: + files: "dist/*" + name: ${{ steps.tagname.outputs.val }} + token: ${{ secrets.GITHUB_TOKEN }} diff --git a/ci/azure-build-release.yml b/ci/azure-build-release.yml deleted file mode 100644 index d716865ee4..0000000000 --- a/ci/azure-build-release.yml +++ /dev/null @@ -1,144 +0,0 @@ -steps: -- checkout: self - submodules: true - -- template: azure-install-rust.yml - -- bash: echo "##vso[task.setvariable variable=RUSTC_VERSION;]`rustc --version`" - displayName: Set rustc version string for caching - -# - bash: | -# set -e -# curl -Lfo sccache.tar.gz https://github.com/mozilla/sccache/releases/download/0.2.9/sccache-0.2.9-x86_64-apple-darwin.tar.gz -# tar -xzf sccache.tar.gz -# cp sccache-*/sccache /usr/local/bin/ -# displayName: Install sccache (OSX) -# condition: and(succeeded(), eq(variables['Agent.OS'], 'Darwin')) - -# - bash: | -# set -e -# curl -Lfo sccache.tar.gz https://github.com/mozilla/sccache/releases/download/0.2.9/sccache-0.2.9-x86_64-unknown-linux-musl.tar.gz -# tar -xzf sccache.tar.gz -# sudo cp sccache-*/sccache /usr/local/bin/ -# displayName: Install sccache (Linux) -# condition: and(succeeded(), eq(variables['Agent.OS'], 'Linux')) - -# - script: | -# curl -Lfo sccache.tar.gz https://github.com/mozilla/sccache/releases/download/0.2.9/sccache-0.2.9-x86_64-pc-windows-msvc.tar.gz -# tar -xzf sccache.tar.gz -# move sccache-* sccache -# echo "##vso[task.prependpath]%CD%\sccache" -# displayName: Install sccache (Windows) -# condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT')) - -- bash: | - cargo build --release --bin wasmtime --bin wasm2obj - cargo build --release --features wasmtime-api/wasm-c-api --package wasmtime-api - displayName: Cargo build - -# Test what we're about to release in release mode itself. This tests -# everything except lightbeam which requires nightly which happens above. -- bash: cargo test --release --all --exclude lightbeam --exclude wasmtime-wasi-c --exclude wasmtime-py --exclude wasmtime-api - displayName: Cargo test - env: - RUST_BACKTRACE: 1 - -# - script: sccache --show-stats -# displayName: post-compile sccache stats - -- bash: | - echo "##vso[task.setvariable variable=tagName;]`echo $BUILD_SOURCEBRANCH | sed -e 's|refs/tags/||'`" - displayName: Set tag name - condition: startsWith(variables['Build.SourceBranch'], 'refs/tags/') -- bash: | - echo "##vso[task.setvariable variable=tagName;]dev" - displayName: Set tag name to "dev" - condition: not(startsWith(variables['Build.SourceBranch'], 'refs/tags/')) - -- bash: echo "##vso[task.setvariable variable=basename;]wasmtime-$(tagName)-x86_64-windows" - displayName: Configure basename var - condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT')) -- bash: echo "##vso[task.setvariable variable=basename;]wasmtime-$(tagName)-x86_64-macos" - displayName: Configure basename var - condition: and(succeeded(), eq(variables['Agent.OS'], 'Darwin')) -- bash: echo "##vso[task.setvariable variable=basename;]wasmtime-$(tagName)-x86_64-linux" - displayName: Configure basename var - condition: and(succeeded(), eq( variables['Agent.OS'], 'Linux' )) - -- bash: | - set -e - mkdir -p $BUILD_BINARIESDIRECTORY/$BASENAME - if [ "$AGENT_OS" = "Windows_NT" ]; then - ext=.exe - fi - cp LICENSE README.md CACHE_CONFIGURATION.md target/release/{wasmtime,wasm2obj}$ext $BUILD_BINARIESDIRECTORY/$BASENAME - displayName: Copy binaries - -- bash: | - set -e - API_BASENAME=${BASENAME}-api - if [ "$AGENT_OS" = "Windows_NT" ]; then - shared_ext=.dll - lib_ext=.lib - elif [ "$AGENT_OS" = "Darwin" ]; then - shared_ext=.dylib - lib_ext=.a - lib_prefix=lib - else - shared_ext=.so - lib_ext=.a - lib_prefix=lib - fi - mkdir -p $BUILD_BINARIESDIRECTORY/$API_BASENAME - cp wasmtime-api/README.md wasmtime-api/LICENSE $BUILD_BINARIESDIRECTORY/$API_BASENAME - mkdir -p $BUILD_BINARIESDIRECTORY/$API_BASENAME/include - cp wasmtime-api/c-examples/wasm-c-api/include/wasm.h $BUILD_BINARIESDIRECTORY/$API_BASENAME/include - mkdir -p $BUILD_BINARIESDIRECTORY/$API_BASENAME/lib - cp target/release/${lib_prefix}wasmtime_api${shared_ext} target/release/${lib_prefix}wasmtime_api${lib_ext} $BUILD_BINARIESDIRECTORY/$API_BASENAME/lib - displayName: Copy c-api binaries - -- bash: | - set -e - export WT_VERSION=`cat Cargo.toml | sed -n 's/^version = "\([^"]*\)".*/\1/p'` - "$WIX/bin/candle" -arch x64 -out target/wasmtime.wixobj installer/msi/wasmtime.wxs - "$WIX/bin/light" -out $BUILD_ARTIFACTSTAGINGDIRECTORY/$(basename).msi target/wasmtime.wixobj -ext WixUtilExtension - rm $BUILD_ARTIFACTSTAGINGDIRECTORY/$(basename).wixpdb - displayName: Create installer (Windows) - condition: eq(variables['Agent.OS'], 'Windows_NT') - -- task: ArchiveFiles@2 - inputs: - rootFolderOrFile: $(Build.BinariesDirectory)/$(basename) - archiveType: 'zip' - archiveFile: '$(Build.ArtifactStagingDirectory)/$(basename).zip' - displayName: Archive files (Win) - condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT')) -- task: ArchiveFiles@2 - inputs: - rootFolderOrFile: $(Build.BinariesDirectory)/$(basename) - archiveType: 'tar' - tarCompression: 'xz' - archiveFile: '$(Build.ArtifactStagingDirectory)/$(basename).tar.xz' - displayName: Archive files (Unix) - condition: and(succeeded(), ne(variables['Agent.OS'], 'Windows_NT')) - -- task: ArchiveFiles@2 - inputs: - rootFolderOrFile: $(Build.BinariesDirectory)/$(basename)-api - archiveType: 'zip' - archiveFile: '$(Build.ArtifactStagingDirectory)/$(basename)-c-api.zip' - displayName: Archive c-api files (Win) - condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT')) -- task: ArchiveFiles@2 - inputs: - rootFolderOrFile: $(Build.BinariesDirectory)/$(basename)-api - archiveType: 'tar' - tarCompression: 'xz' - archiveFile: '$(Build.ArtifactStagingDirectory)/$(basename)-c-api.tar.xz' - displayName: Archive c-api files (Unix) - condition: and(succeeded(), ne(variables['Agent.OS'], 'Windows_NT')) - -- task: PublishPipelineArtifact@1 - inputs: - path: $(Build.ArtifactStagingDirectory)/ - artifactName: 'bundle-$(Agent.OS)' diff --git a/ci/azure-build-wheels.yml b/ci/azure-build-wheels.yml deleted file mode 100644 index fde5c5f2b2..0000000000 --- a/ci/azure-build-wheels.yml +++ /dev/null @@ -1,87 +0,0 @@ -steps: -- checkout: self - submodules: true - -- template: azure-install-rust.yml - -- bash: mkdir misc/wasmtime-py/wheelhouse - displayName: Pre-create wheelhouse directory - -# Note that we skip this on Linux because Python 3.6 is pre-installed in the -# CentOS container. -- task: UsePythonVersion@0 - inputs: - versionSpec: '3.6' - addToPath: true - condition: and(succeeded(), ne(variables['Agent.OS'], 'Linux')) - -# Install Python dependencies needed for our `setup.py` scripts -- bash: sudo pip3 install setuptools wheel==0.31.1 setuptools-rust auditwheel - displayName: Install Python dependencies (Linux) - condition: and(succeeded(), eq(variables['Agent.OS'], 'Linux')) -- bash: pip3 install setuptools wheel==0.31.1 setuptools-rust - displayName: Install Python dependencies (not Linux) - condition: and(succeeded(), ne(variables['Agent.OS'], 'Linux')) - -- bash: python setup.py bdist_wheel - workingDirectory: misc/wasmtime-py - displayName: Build wheels py36 - -# Clear the build directory between building different wheels for different -# Python versions to ensure that we don't package dynamic libraries twice by -# accident. -- bash: rm -rf build - workingDirectory: misc/wasmtime-py - displayName: Clear build directory - -# Note that 3.7 isn't installed on Linux so we don't do this a second time -# around. -- task: UsePythonVersion@0 - inputs: - versionSpec: '3.7' - addToPath: true - condition: and(succeeded(), ne(variables['Agent.OS'], 'Linux')) -- bash: sudo bash ci/setup_centos6_python37.sh - condition: and(succeeded(), eq(variables['Agent.OS'], 'Linux')) - -- bash: sudo pip3 install setuptools wheel==0.31.1 setuptools-rust auditwheel - displayName: Install Python dependencies (Linux) - condition: and(succeeded(), eq(variables['Agent.OS'], 'Linux')) -- bash: pip3 install setuptools wheel==0.31.1 setuptools-rust - displayName: Install Python dependencies (not Linux) - condition: and(succeeded(), ne(variables['Agent.OS'], 'Linux')) - -- bash: | - set -e - export PYTHON_SYS_EXECUTABLE=`which python3.7` - $PYTHON_SYS_EXECUTABLE setup.py bdist_wheel - workingDirectory: misc/wasmtime-py - displayName: Build wheels py37 - condition: and(succeeded(), eq(variables['Agent.OS'], 'Linux')) -- bash: python setup.py bdist_wheel - workingDirectory: misc/wasmtime-py - displayName: Build wheels py37 - condition: and(succeeded(), ne(variables['Agent.OS'], 'Linux')) - -# Move `dist/*.whl` into `wheelhouse/` so we can deploy them, but on Linux we -# need to run an `auditwheel` command as well to turn these into "manylinux" -# wheels to run across a number of distributions. -- bash: mv dist/*.whl wheelhouse/ - workingDirectory: misc/wasmtime-py - displayName: Move wheels to wheelhouse (not Linux) - condition: and(succeeded(), ne(variables['Agent.OS'], 'Linux')) -- bash: | - set -e - for whl in dist/*.whl; do - auditwheel repair "$whl" -w wheelhouse/ - done - workingDirectory: misc/wasmtime-py - displayName: Move wheels to wheelhouse (Linux) - condition: and(succeeded(), eq(variables['Agent.OS'], 'Linux')) - -# Publish our wheelhouse to azure pipelines which will later get published to -# github releases -- task: PublishPipelineArtifact@1 - inputs: - path: misc/wasmtime-py/wheelhouse - artifactName: 'wheels-$(Agent.OS)' diff --git a/ci/azure-install-rust.yml b/ci/azure-install-rust.yml deleted file mode 100644 index 64e502b775..0000000000 --- a/ci/azure-install-rust.yml +++ /dev/null @@ -1,33 +0,0 @@ -steps: - # Rustup is currently installed on Windows and Linux, but not macOS. - # It is installed in /usr/local/cargo/bin/ or C:\Program Files\Rust\.cargo\bin\ - # This steps ensures that rustup is installed, mainly for macOS, or if the - # azure image changes in the future. - - bash: | - set -ex - if [ -x "`command -v rustup`" ]; then - echo `command -v rustup` `rustup -V` already installed - rustup self update - else - if [ "$AGENT_OS" = "Windows_NT" ]; then - curl -sSf -o rustup-init.exe https://win.rustup.rs - ./rustup-init.exe -y --default-toolchain $TOOLCHAIN - echo "##vso[task.prependpath]$USERPROFILE/.cargo/bin" - else - curl https://sh.rustup.rs -sSf | sh -s -- -y --default-toolchain $TOOLCHAIN - echo "##vso[task.prependpath]$HOME/.cargo/bin" - fi - fi - displayName: Install rustup - - - bash: | - set -ex - rustup update $TOOLCHAIN - rustup default $TOOLCHAIN - displayName: Install rust - - - bash: | - set -ex - rustc -Vv - cargo -V - displayName: Query rust and cargo versions diff --git a/ci/azure-prepare-centos-6.yml b/ci/azure-prepare-centos-6.yml deleted file mode 100644 index e5f85b544a..0000000000 --- a/ci/azure-prepare-centos-6.yml +++ /dev/null @@ -1,24 +0,0 @@ -steps: -# We're executing in the container as non-root but `yum` requires root. We -# need to install `sudo` but to do that we need `sudo`. Do a bit of a weird -# hack where we use the host `docker` executable to re-execute in our own -# container with the root user to install `sudo` -- bash: /tmp/docker exec -t -u 0 ci-container sh -c "yum install -y sudo" - displayName: Configure sudo - -# See https://edwards.sdsu.edu/research/c11-on-centos-6/ for where these -# various commands came from. -- bash: | - set -e - sudo yum install -y centos-release-scl cmake xz epel-release - sudo yum install -y rh-python36 patchelf unzip - sudo yum install -y devtoolset-8-gcc devtoolset-8-binutils devtoolset-8-gcc-c++ - echo "##vso[task.prependpath]/opt/rh/devtoolset-8/root/usr/bin" - echo "##vso[task.prependpath]/opt/rh/rh-python36/root/usr/bin" - displayName: Install system dependencies - -# Delete `libstdc++.so` to force gcc to link against `libstdc++.a` instead. -# This is a hack and not the right way to do this, but it ends up doing the -# right thing for now. -- bash: sudo rm -f /opt/rh/devtoolset-8/root/usr/lib/gcc/x86_64-redhat-linux/8/libstdc++.so - displayName: Force a static libstdc++ diff --git a/ci/build-tarballs.sh b/ci/build-tarballs.sh new file mode 100755 index 0000000000..0c84db2743 --- /dev/null +++ b/ci/build-tarballs.sh @@ -0,0 +1,55 @@ +#!/bin/bash + +# A small shell script invoked from CI on the final Linux builder which actually +# assembles the release artifacts for a particular platform. This will take the +# binary artifacts of previous builders and create associated tarballs to +# publish to GitHub. +# +# The first argument of this is the "platform" name to put into the tarball, and +# the second argument is the name of the github actions platform which is where +# we source binaries from. The final third argument is ".exe" on Windows to +# handle executable extensions right. + +set -ex + +platform=$1 +src=$2 +exe=$3 + +rm -rf tmp +mkdir tmp +mkdir -p dist + +mktarball() { + dir=$1 + if [ "$exe" = "" ]; then + tar cJf dist/$dir.tar.xz -C tmp $dir + else + (cd tmp && zip -r ../dist/$dir.zip $dir) + fi +} + +# Create the main tarball of binaries +bin_pkgname=wasmtime-$TAG-$platform +mkdir tmp/$bin_pkgname +cp LICENSE README.md CACHE_CONFIGURATION.md tmp/$bin_pkgname +mv bins-$src/{wasmtime,wasm2obj}$exe tmp/$bin_pkgname +chmod +x tmp/$bin_pkgname/{wasmtime,wasm2obj}$exe +mktarball $bin_pkgname + +if [ "$exe" = ".exe" ]; then + mv bins-$src/installer.msi dist/$bin_pkgname.msi +fi + +# Create tarball of API libraries +api_pkgname=wasmtime-$TAG-$platform-c-api +mkdir tmp/$api_pkgname +mkdir tmp/$api_pkgname/lib +mkdir tmp/$api_pkgname/include +cp LICENSE README.md tmp/$api_pkgname +mv bins-$src/* tmp/$api_pkgname/lib +cp wasmtime-api/c-examples/wasm-c-api/include/wasm.h tmp/$api_pkgname/include +mktarball $api_pkgname + +# Move wheels to dist folder +mv wheels-$src/* dist