diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index ef8e04895bd..d50004f07ef 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -169,3 +169,174 @@ jobs: if: always() run: | rm -fr /stdb/* + + callgrind_benchmark: + name: run callgrind benchmarks + # DON'T run on benchmarks-runner, using docker on a self-hosted runner has + # been broken for 4 years: https://github.com/actions/runner/issues/434 . + # Fortunately, we can run on standard Github Actions infra because we don't care + # about other stuff running on the machine! + # runs-on: benchmarks-runner + runs-on: ubuntu-latest + timeout-minutes: 20 # on a successful run, runs in 8 minutes + container: + image: rust:1.72 + options: --privileged + # filter for a comment containing 'callgrind please' (chosen to not interfere with the other benchmarks) + if: ${{ github.event_name != 'issue_comment' || (github.event.issue.pull_request && contains(github.event.comment.body, 'callgrind please')) }} + env: + PR_NUMBER: ${{ github.event.inputs.pr_number || github.event.issue.number || null }} + steps: + - name: Install valgrind & iai-callgrind-runner + run: | + apt-get update + apt-get install -y valgrind protobuf-compiler bash sudo curl gh + cargo install --git https://github.com/clockworklabs/iai-callgrind.git --branch main iai-callgrind-runner + git config --global --add safe.directory /__w/SpacetimeDB/SpacetimeDB + + # can't do this off self hosted: + # - name: Enable CPU boost + # shell: bash + # run: echo "1" | sudo tee /sys/devices/system/cpu/cpufreq/boost + + - name: Check membership + if: ${{ github.event_name == 'issue_comment' }} + env: + CONTRIB_ORG: clockworklabs + COMMENT_AUTHOR: ${{ github.event.comment.user.login }} + ORG_READ_TOKEN: ${{ secrets.ORG_READ_TOKEN }} + shell: bash + run: | + curl -OL https://github.com/cli/cli/releases/download/v2.37.0/gh_2.37.0_linux_amd64.deb && sudo dpkg -i gh_2.37.0_linux_amd64.deb + if [[ $(GH_TOKEN=$ORG_READ_TOKEN gh api --paginate /orgs/{owner}/members --jq 'any(.login == env.COMMENT_AUTHOR)') != true ]]; then + gh pr comment $PR_NUMBER -b "Sorry, you don't have permission to run benchmarks." + exit 1 + fi + + - name: find PR branch + if: ${{ env.PR_NUMBER }} + shell: bash + run: echo "PR_REF=$(gh pr view $PR_NUMBER --json headRefName --jq .headRefName)" >>"$GITHUB_ENV" + + - name: Checkout sources + uses: actions/checkout@v3 + with: + ref: ${{ env.PR_REF || github.ref }} + # if we're on master we want to know what the sha of HEAD~1 is so + # that we can compare results from it to HEAD (in the "Fetch markdown + # summary PR" step). otherwise, we can use a fully shallow checkout + fetch-depth: ${{ env.PR_NUMBER && 1 || 2 }} + + - name: Unbork Github Actions state + shell: bash + run: | + echo "Letting anybody touch our git repo, in order to avoid breaking other jobs" + echo "This is necessary because we are running as root inside a docker image" + chmod -R a+rw . + + - name: Post initial comment + shell: bash + run: | + set -exo pipefail + if [[ $PR_NUMBER ]]; then + comment_parent=issues/$PR_NUMBER + comment_update=issues/comments + else + comment_parent=commits/$GITHUB_SHA + comment_update=comments + fi + comment_body="Callgrind benchmark in progress..." + comment_id=$(gh api "/repos/{owner}/{repo}/$comment_parent/comments" -f body="$comment_body" --jq .id) + echo "COMMENT_UPDATE_URL=/repos/{owner}/{repo}/$comment_update/$comment_id" >>$GITHUB_ENV + + - name: Install stable toolchain + uses: actions-rs/toolchain@v1 + with: + profile: minimal + components: clippy + toolchain: stable + target: wasm32-unknown-unknown + override: true + + - name: Build + working-directory: crates/bench/ + shell: bash + run: | + cargo build --release + + - name: Install latest wasm-opt for module optimisations + shell: bash + run: | + curl https://github.com/WebAssembly/binaryen/releases/download/version_116/binaryen-version_116-x86_64-linux.tar.gz -L | sudo tar xz -C /usr/local --strip-components=1 + + # leave CPU boost on, doesn't affect callgrind! + + - name: Branch; run bench + shell: bash + run: | + if [[ $PR_NUMBER ]]; then + BASELINE_NAME=branch + RESULTS_NAME=pr-$PR_NUMBER + BENCH_FILTER='(special|stdb_module|stdb_raw)' + echo "Running branch callgrind benchmarks" + else + BASELINE_NAME=master + RESULTS_NAME=$GITHUB_SHA + BENCH_FILTER='.*' + echo "Running master callgrind benchmarks" + fi + pushd crates/bench + cargo bench --bench callgrind -- --save-summary pretty-json + cargo run --bin summarize pack-callgrind "$BASELINE_NAME" + popd + mkdir callgrind-results + [[ ! $PR_NUMBER ]] && cp target/iai/$BASELINE_NAME.json callgrind-results/ + cp target/iai/$BASELINE_NAME.json callgrind-results/$RESULTS_NAME.json + + # this will work for both PR and master + - name: Upload callgrind results to DO spaces + uses: shallwefootball/s3-upload-action@master + with: + aws_key_id: ${{ secrets.AWS_KEY_ID }} + aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY}} + aws_bucket: "spacetimedb-ci-benchmarks" + source_dir: callgrind-results + endpoint: https://nyc3.digitaloceanspaces.com + destination_dir: callgrind-benchmarks + + - name: Fetch markdown summary PR + shell: bash + run: | + if [[ $PR_NUMBER ]]; then + OLD=master + NEW=pr-$PR_NUMBER + else + OLD=$(git rev-parse HEAD~1) + NEW=$GITHUB_SHA + fi + curl -sS https://benchmarks.spacetimedb.com/compare_callgrind/$OLD/$NEW > report.md + + - name: Post comment + shell: bash + run: | + BODY="
Benchmark results + $(cat report.md) +
" + gh api "$COMMENT_UPDATE_URL" -X PATCH -f body="$BODY" + + - name: Post failure comment + if: ${{ failure() && env.COMMENT_UPDATE_URL }} + shell: bash + run: | + BODY="Benchmarking failed. Please check [the workflow run](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}) for details." + gh api "$COMMENT_UPDATE_URL" -X PATCH -f body="$BODY" + + - name: Clean up + if: always() + shell: bash + run: | + rm -fr /stdb/* + echo "Letting anybody touch our git repo, in order to avoid breaking other jobs" + echo "This is necessary because we are running as root inside a docker image" + chmod -R a+rw . + diff --git a/.github/workflows/callgrind_benchmarks.yml b/.github/workflows/callgrind_benchmarks.yml index 761df07274d..f0f0d9f6f4a 100644 --- a/.github/workflows/callgrind_benchmarks.yml +++ b/.github/workflows/callgrind_benchmarks.yml @@ -2,7 +2,6 @@ on: push: branches: - master - - kazimuth/callgrind workflow_dispatch: inputs: @@ -27,6 +26,7 @@ jobs: # Fortunately, we can run on standard Github Actions infra because we don't care # about other stuff running on the machine! # runs-on: benchmarks-runner + runs-on: ubuntu-latest timeout-minutes: 20 # on a successful run, runs in 8 minutes container: image: rust:1.72