Use gh actions for benchmarks please

.github/workflows/benchmarks.yml

@@ -2,17 +2,16 @@ on:
   push:
     branches:
       - master
-  
+
   workflow_dispatch:
     inputs:
       pr_number:
         description: 'Pull Request Number'
         required: false
         default: ''
 
-  # note: the "benchmarks please" comments aren't dispatched here,
-  # there's a script running on one of our internal servers that reads those and then
-  # dispatches to this workflow using the workflow_dispatch there.
+  issue_comment:
+    types: [created]
 
 name: Benchmarks
 
@@ -23,32 +22,49 @@ jobs:
   benchmark:
     name: run benchmarks
     runs-on: benchmarks-runner
+    # filter for a comment containing 'benchmarks please'
+    if: ${{ github.event_name != 'issue_comment' || (github.event.issue.pull_request && contains(github.event.comment.body, 'benchmarks please')) }}
+    env:
+      PR_NUMBER: ${{ github.event.inputs.pr_number || github.event.issue.number || null }}
     steps:
       - name: Enable CPU boost
         run: echo "1" | sudo tee /sys/devices/system/cpu/cpufreq/boost
 
-      - name: Checkout sources for a PR
-        if: ${{ github.event.inputs.ref }}
-        uses: actions/checkout@v3
-        with:
-          ref: ${{ github.event.inputs.ref }}
-          fetch-depth: 0
+      - name: Check membership
+        if: ${{ env.PR_NUMBER }}
+        run: |
+          if [[ $(gh api --paginate /orgs/{owner}/members --jq 'any(.login == env.GITHUB_ACTOR)') != true ]]; then
+            gh pr comment $PR_NUMBER -b "Sorry, you don't have permission to run benchmarks."
+            exit 1
+          fi
+
+      - name: Post initial comment
+        run: |
+          if [[ $PR_NUMBER ]]; then
+            comment_parent=issues/$PR_NUMBER
+            comment_update=issues/comments
+          else
+            comment_parent=commits/$GITHUB_SHA
+            comment_update=comments
+          fi
+          comment_body="Benchmark in progress..."
+          comment_id=$(gh api "/repos/{owner}/{repo}/$comment_parent/comments" -f body="$comment_body" --jq .id)
+          echo "COMMENT_UPDATE_URL=/repos/{owner}/{repo}/$comment_update/$comment_id" >>$GITHUB_ENV
+
+      - name: find PR branch
+        if: ${{ env.PR_NUMBER }}
+        run: echo "PR_REF=$(gh pr view $PR_NUMBER --json headRefName --jq .headRefName)" >>"$GITHUB_ENV"
 
       - name: Checkout sources
-        if: github.event.inputs.ref == ''
         uses: actions/checkout@v3
         with:
-          fetch-depth: 10
-
-      - name: Set up for PR context
-        if: github.event.inputs.pr_number
-        run: |
-          echo "PR_NUMBER=${{ github.event.inputs.pr_number }}" >> $GITHUB_ENV
+          ref: ${{ env.PR_REF || github.ref }}
 
       - name: Install stable toolchain
         uses: actions-rs/toolchain@v1
         with:
           profile: minimal
+          components: clippy
           toolchain: stable
           target: wasm32-unknown-unknown
           override: true
@@ -58,40 +74,37 @@ jobs:
         run: |
           cargo build --release
 
-      - name: Install clippy for module build
-        run: |
-          rustup component add clippy
-
       - name: Install latest wasm-opt for module optimisations
         run: |
           curl https://github.com/WebAssembly/binaryen/releases/download/version_116/binaryen-version_116-x86_64-linux.tar.gz -L | sudo tar xz -C /usr/local --strip-components=1
 
       - name: Disable CPU boost
         run: echo "0" | sudo tee /sys/devices/system/cpu/cpufreq/boost
 
-      - name: Extract branch name
-        if: "! github.event.inputs.pr_number"
-        shell: bash
-        run: |
-          BRANCH_NAME=${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}
-          echo "BRANCH_NAME=$BRANCH_NAME" >> $GITHUB_ENV
-          echo "NORMALIZED_BRANCH_NAME=${BRANCH_NAME//\//-}" >> $GITHUB_ENV
-
       - name: Branch; run bench
-        if: "! github.event.inputs.pr_number"
         run: |
-          echo "Running benchmarks with sqlite"
+          if [[ $PR_NUMBER ]]; then
+            BASELINE_NAME=branch
+            RESULTS_NAME=pr-$PR_NUMBER
+            BENCH_FILTER='(special|stdb_module|stdb_raw)'
+            echo "Running benchmarks without sqlite"
+          else
+            BASELINE_NAME=master
+            RESULTS_NAME=$GITHUB_SHA
+            BENCH_FILTER='.*'
+            echo "Running benchmarks with sqlite"
+          fi
           pushd crates/bench
-          cargo bench --bench generic --bench special -- --save-baseline $NORMALIZED_BRANCH_NAME
-          cargo run --bin summarize pack $NORMALIZED_BRANCH_NAME
+          cargo bench --bench generic --bench special -- --save-baseline "$BASELINE_NAME" "$BENCH_FILTER"
+          cargo run --bin summarize pack "$BASELINE_NAME"
           popd
           mkdir criterion-results
-          cp target/criterion/$NORMALIZED_BRANCH_NAME.json criterion-results/
-          cp target/criterion/$NORMALIZED_BRANCH_NAME.json criterion-results/$GITHUB_SHA.json
+          [[ ! $PR_NUMBER ]] && cp target/criterion/$BASELINE_NAME.json criterion-results/
+          cp target/criterion/$BASELINE_NAME.json criterion-results/$RESULTS_NAME.json
 
       # TODO: can we optionally download if it only might fail?
       #- name: PR; download bench results for compare
-      #  if: github.event.inputs.pr_number
+      #  if: env.PR_NUMBER
       #  uses: actions/github-script@v6  
       #  with:
       #    github-token: ${{secrets.GITHUB_TOKEN}}
@@ -104,27 +117,11 @@ jobs:
       #        })
       #      }
 
-      - name: PR; run bench
-        if: github.event.inputs.pr_number
-        run: |
-          echo "Running benchmarks without sqlite"
-          # have to pass explicit names, otherwise it will try to run the tests and fail for some reason...
-          pushd crates/bench
-          cargo bench --bench generic --bench special -- --save-baseline branch '(special|stdb_module|stdb_raw)'
-          cargo run --bin summarize pack branch
-          popd
-          mkdir criterion-results
-          cp target/criterion/branch.json criterion-results/pr-$PR_NUMBER.json
-
       - name: PR; compare benchmarks
-        if: github.event.inputs.pr_number
+        if: ${{ env.PR_NUMBER }}
         working-directory: crates/bench/
         run: |
-          if [ -e target/criterion/$NORMALIZED_BRANCH_NAME.json ]; then
-            cargo run --bin summarize markdown-report branch.json $NORMALIZED_BRANCH_NAME.json --report-name report
-          else
-            cargo run --bin summarize markdown-report branch.json --report-name report
-          fi
+          cargo run --bin summarize markdown-report branch.json --report-name report
 
       # this will work for both PR and master
       - name: Upload criterion results to DO spaces
@@ -138,60 +135,32 @@ jobs:
           destination_dir: benchmarks
 
       - name: Fetch markdown summary PR
-        if: github.event.inputs.pr_number
         run: |
-          curl -sS https://benchmarks.spacetimedb.com/compare/master/pr-$PR_NUMBER > report.md
+          if [[ $PR_NUMBER ]]; then
+            OLD=master
+            NEW=pr-$PR_NUMBER
+          else
+            git fetch
+            OLD=$(git rev-parse HEAD~1)
+            NEW=$GITHUB_SHA
+          fi
+          curl -sS https://benchmarks.spacetimedb.com/compare/$OLD/$NEW > report.md
 
-      - name: Fetch markdown summary PR
-        if: "! github.event.inputs.pr_number"
+      - name: Post comment
         run: |
-          git fetch
-          old=$(git rev-parse HEAD~1)
-          curl -sS https://benchmarks.spacetimedb.com/compare/$old/$GITHUB_SHA > report.md
-
-      # https://stackoverflow.com/questions/58066966/commenting-a-pull-request-in-a-github-action
-      # https://github.com/boa-dev/criterion-compare-action/blob/main/main.js
-      - name: test comment
-        uses: actions/github-script@v6
-        with:
-          github-token: ${{secrets.GITHUB_TOKEN}}
-          script: |
-            let stuff = require('fs').readFileSync('report.md', 'utf8');
-            let body = `<details><summary>Benchmark results</summary>\n\n${stuff}\n\n</details>`;
-
-            try {
-              if (process.env.PR_NUMBER) {
-                let number = parseInt(process.env.PR_NUMBER);
-                core.info("context: issue number: "+number)
-                const { data: comment } = await github.rest.issues.createComment({
-                  owner: "clockworklabs",
-                  repo: "SpacetimeDB",
-                  issue_number: number,
-                  body: body,
-                });
-                core.info(
-                  `Created comment id '${comment.id}' on issue '${number}' in 'clockworklabs/SpacetimeDB'.`
-                );
-                core.setOutput("comment-id", comment.id);
-              } else {
-                const { data: comment } = github.rest.repos.createCommitComment({
-                  commit_sha: context.sha,
-                  owner: context.repo.owner,
-                  repo: context.repo.repo,
-                  body: body
-                })
-                core.info(
-                  `Created comment id '${comment.id}' on commit '${context.sha}' in 'clockworklabs/SpacetimeDB'.`
-                );
-                core.setOutput("comment-id", comment.id);
-              }
-            } catch (err) {
-              core.warning(`Failed to comment: ${err}`);
-              core.info("Commenting is not possible from forks.");
-              core.info("Logging here instead.");
-              console.log(body);
-            }
+          BODY="<details><summary>Benchmark results</summary>
 
+          $(cat report.md)
+
+          </details>"
+
+          gh api "$COMMENT_UPDATE_URL" -X PATCH -f body="$BODY"
+
+      - name: Post failure comment
+        if: ${{ failure() && env.COMMENT_UPDATE_URL }}
+        run: |
+          BODY="Benchmarking failed. Please check [the workflow run](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}) for details."
+          gh api "$COMMENT_UPDATE_URL" -X PATCH -f body="$BODY"
 
       - name: Clean up
         if: always()