Wrangle benchmarks

.github/workflows/benchmarks.yml

@@ -2,6 +2,7 @@ on:
   push:
     branches:
       - master
+      - kazimuth/benchwrangle
   workflow_dispatch:
     inputs:
       pr_number:
@@ -63,27 +64,23 @@ jobs:
         run: |
           cargo build --release
 
+      - name: Install clippy for module build
+        run: |
+          rustup component add clippy
+
       - name: Criterion compare base branch
         if: ${{ env.PR_BASE_REF }}
         uses: clockworklabs/criterion-compare-action@main
         with:
           cwd: "crates/bench"
           branchName: ${{ env.PR_BASE_REF }}
-          benchName: modules
 
       - name: Criterion compare previous commit
         if: env.PR_BASE_REF == ''
         uses: clockworklabs/criterion-compare-action@main
         with:
           cwd: "crates/bench"
           branchName: "HEAD~1"
-          benchName: modules
-
-      - name: Benchmark Vs Sqlite
-        working-directory: crates/bench/
-        run: |
-          python3 hyper_cmp.py versus > out.report
-          cat out.report >> $GITHUB_STEP_SUMMARY
 
       - name: Clean up
         if: always()

crates/bench/Cargo.toml

@@ -6,11 +6,11 @@ license-file = "LICENSE"
 description = "Bench library/utility for SpacetimeDB"
 
 [[bench]]
-name = "db"
+name = "special"
 harness = false
 
 [[bench]]
-name = "modules"
+name = "generic"
 harness = false
 
 [lib]
@@ -23,12 +23,18 @@ spacetimedb-standalone = { path = "../standalone" }
 spacetimedb-client-api = { path = "../client-api" }
 spacetimedb-testing = { path = "../testing" }
 
+log.workspace = true
 clap.workspace = true
 rusqlite.workspace = true
 criterion.workspace = true
 tempdir.workspace = true
 rand.workspace = true
 tokio.workspace = true
 serde_json.workspace = true
+serde.workspace = true
 anyhow.workspace = true
+anymap.workspace = true
 byte-unit.workspace = true
+futures.workspace = true
+tracing-subscriber.workspace = true
+lazy_static.workspace = true

crates/bench/README.md

@@ -1,4 +1,60 @@
-spacetimedb-bench# Benchmarking suite for SpacetimeDB
+# spacetimedb-bench
+Benchmarking suite for SpacetimeDB using Criterion. Provides comparisons between the underlying spacetime datastore, spacetime modules, and sqlite.
+
+Timings for spacetime modules should be understood as *latencies to call a spacetime reducer without network delays*. They include serialization and deserialization times for any arguments passed. There are also separate benchmarks that measure these times on their own.
+
+## Benchmark structure
+
+The complete structure (not yet fully implemented) is as follows:
+
+```
+[db]/[disk]/
+    empty_transaction/
+    insert_1/[schema]/[index_type]/[load]
+    insert_bulk/[schema]/[index_type]/[load]/[count]
+    insert_large_value/[count]
+    iterate/[schema]/[count]/
+    filter/[string, u64]/[index?]/[load]/[count]
+    find_unique/u32/[load]/
+    update/[load]/[count]
+    delete/[index_type]/[load]/[count]
+
+# "load" refers to the number of rows in the table at the start of the benchmark.
+
+# db: [stdb_raw, stdb_module, sqlite]
+# disk: [disk, mem]
+# schema: [person, location]
+# index_type: [unique, non_unique, multi_index]
+
+# "person" is a schema with 2 ints and a string, "location" is a schema with 3 ints.
+
+serialize/
+    bsatn/[schema]/[count]
+    json/[schema]/[count]
+    product_value/[schema]/[count]
+
+deserialize/
+    bsatn/[schema]/[count]
+    json/[schema]/[count]
+
+stdb_module/
+    print_bulk/[count]
+    large_arguments/64KiB/
+```
+
+Typically you don't want to run all benchmarks at once, there are a lot of them and it will take many minutes.
+You can pass regexes to the bench script to select what slice of benchmarks you'd like. For example,
+
+```sh
+cargo bench -- 'stdb_raw/.*/insert_bulk'
+```
+Will run all of the `insert_bulk` benchmarks against the raw spacetime backend.
+
+Similarly, 
+```sh
+cargo bench -- 'mem/.*/unique'
+```
+Will run benchmarks involving unique primary keys against all databases, without writing to disc.
 
 ## Install tools