Skip to content

Rename types in benchmarks #803

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Feb 23, 2024
Merged

Rename types in benchmarks #803

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
674c031
Rename types in benchmarks, modify benches run in action
kazimuth Feb 7, 2024
bcf6349
Fix lints
kazimuth Feb 23, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 9 additions & 18 deletions .github/workflows/benchmarks.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ on:
push:
branches:
- master
- kazimuth/bench_cleanup

workflow_dispatch:
inputs:
Expand Down Expand Up @@ -96,37 +97,25 @@ jobs:
if [[ $PR_NUMBER ]]; then
BASELINE_NAME=branch
RESULTS_NAME=pr-$PR_NUMBER
BENCH_FILTER='(special|stdb_module|stdb_raw)'
BENCH_FILTER='stdb_raw'
echo "Running benchmarks without sqlite"
else
BASELINE_NAME=master
RESULTS_NAME=$GITHUB_SHA
BENCH_FILTER='.*'
BENCH_FILTER='(stdb_raw|sqlite)'
echo "Running benchmarks with sqlite"
fi
pushd crates/bench
cargo bench --bench generic --bench special -- --save-baseline "$BASELINE_NAME" "$BENCH_FILTER"
cargo bench --bench generic -- --save-baseline "$BASELINE_NAME" "$BENCH_FILTER"
# sticker price benchmark
cargo bench --bench generic -- --save-baseline "$BASELINE_NAME" 'stdb_module/disk/update_bulk'
cargo bench --bench special -- --save-baseline "$BASELINE_NAME"
cargo run --bin summarize pack "$BASELINE_NAME"
popd
mkdir criterion-results
[[ ! $PR_NUMBER ]] && cp target/criterion/$BASELINE_NAME.json criterion-results/
cp target/criterion/$BASELINE_NAME.json criterion-results/$RESULTS_NAME.json

# TODO: can we optionally download if it only might fail?
#- name: PR; download bench results for compare
# if: env.PR_NUMBER
# uses: actions/github-script@v6
# with:
# github-token: ${{secrets.GITHUB_TOKEN}}
# script: |
# try {
# let artifact = github.rest.actions.getArtifact({
# owner: "clockwork",
# repo: "SpacetimeDB",
#
# })
# }

# this will work for both PR and master
- name: Upload criterion results to DO spaces
uses: shallwefootball/s3-upload-action@master
Expand All @@ -147,6 +136,7 @@ jobs:
OLD=$(git rev-parse HEAD~1)
NEW=$GITHUB_SHA
fi
echo "fetching https://benchmarks.spacetimedb.com/compare/$OLD/$NEW"
curl -sS https://benchmarks.spacetimedb.com/compare/$OLD/$NEW > report.md

- name: Post comment
Expand Down Expand Up @@ -314,6 +304,7 @@ jobs:
OLD=$(git rev-parse HEAD~1)
NEW=$GITHUB_SHA
fi
echo "fetching https://benchmarks.spacetimedb.com/compare/$OLD/$NEW"
curl -sS https://benchmarks.spacetimedb.com/compare_callgrind/$OLD/$NEW > report.md

- name: Post comment
Expand Down
587 changes: 112 additions & 475 deletions crates/bench/benches/callgrind.rs
View file
Open in desktop

Large diffs are not rendered by default.

129 changes: 18 additions & 111 deletions crates/bench/benches/generic.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ use lazy_static::lazy_static;
use mimalloc::MiMalloc;
use spacetimedb_bench::{
database::BenchDatabase,
schemas::{create_sequential, BenchTable, IndexStrategy, Location, Person, RandomTable, BENCH_PKEY_INDEX},
schemas::{create_sequential, u32_u64_str, u32_u64_u64, BenchTable, IndexStrategy, RandomTable},
spacetime_module, spacetime_raw, sqlite, ResultBench,
};
use spacetimedb_lib::sats::AlgebraicType;
Expand Down Expand Up @@ -40,8 +40,8 @@ fn bench_suite<DB: BenchDatabase>(c: &mut Criterion, in_memory: bool) -> ResultB

empty(&mut g, &mut db)?;

table_suite::<DB, Person>(&mut g, &mut db)?;
table_suite::<DB, Location>(&mut g, &mut db)?;
table_suite::<DB, u32_u64_str>(&mut g, &mut db)?;
table_suite::<DB, u32_u64_u64>(&mut g, &mut db)?;

Ok(())
}
Expand All @@ -55,47 +55,37 @@ fn table_suite<DB: BenchDatabase, T: BenchTable + RandomTable>(g: &mut Group, db

type TableData<TableId> = (IndexStrategy, TableId, String);
let mut prep_table = |index_strategy: IndexStrategy| -> ResultBench<TableData<DB::TableId>> {
let table_name = T::name_snake_case();
let style_name = index_strategy.snake_case();
let table_name = T::name();
let style_name = index_strategy.name();
let table_params = format!("{table_name}/{style_name}");
let table_id = db.create_table::<T>(index_strategy)?;

Ok((index_strategy, table_id, table_params))
};
let tables: [TableData<DB::TableId>; 3] = [
prep_table(IndexStrategy::Unique)?,
prep_table(IndexStrategy::NonUnique)?,
prep_table(IndexStrategy::MultiIndex)?,
let tables: [TableData<DB::TableId>; 2] = [
prep_table(IndexStrategy::Unique0)?,
//prep_table(IndexStrategy::NoIndex)?,
prep_table(IndexStrategy::BTreeEachColumn)?,
];

for (_, table_id, table_params) in &tables {
insert_1::<DB, T>(g, table_params, db, table_id, 0)?;
insert_1::<DB, T>(g, table_params, db, table_id, 1000)?;
}
for (_, table_id, table_params) in &tables {
insert_bulk::<DB, T>(g, table_params, db, table_id, 0, 100)?;
insert_bulk::<DB, T>(g, table_params, db, table_id, 1000, 100)?;
insert_bulk::<DB, T>(g, table_params, db, table_id, 2048, 256)?;
if *RUN_ONE_MILLION {
insert_bulk::<DB, T>(g, table_params, db, table_id, 0, 1_000_000)?;
}
}
for (index_strategy, table_id, table_params) in &tables {
if *index_strategy == IndexStrategy::Unique {
iterate::<DB, T>(g, table_params, db, table_id, 100)?;

if table_params.contains("person") {
// perform "find" benchmarks
find::<DB, T>(g, db, table_id, index_strategy, BENCH_PKEY_INDEX, 1000, 100)?;
}

update_bulk::<DB, T>(g, table_params, db, table_id, 1000, 10)?;
update_bulk::<DB, T>(g, table_params, db, table_id, 1000, 100)?;
if *index_strategy == IndexStrategy::Unique0 {
// Iterate is unaffected by index strategy, so only run it here
iterate::<DB, T>(g, table_params, db, table_id, 256)?;
// Update can only be performed with a unique key
update_bulk::<DB, T>(g, table_params, db, table_id, 2048, 256)?;
if *RUN_ONE_MILLION {
update_bulk::<DB, T>(g, table_params, db, table_id, 1_000_000, 1_000_000)?;
}
} else {
// perform "filter" benchmarks
filter::<DB, T>(g, db, table_id, index_strategy, 2, 1000, 100)?;
filter::<DB, T>(g, db, table_id, index_strategy, 2, 2048, 8)?;
}
}

Expand Down Expand Up @@ -156,41 +146,6 @@ fn empty<DB: BenchDatabase>(g: &mut Group, db: &mut DB) -> ResultBench<()> {
Ok(())
}

#[inline(never)]
fn insert_1<DB: BenchDatabase, T: BenchTable + RandomTable>(
g: &mut Group,
table_params: &str,
db: &mut DB,
table_id: &DB::TableId,
load: u32,
) -> ResultBench<()> {
let id = format!("insert_1/{table_params}/load={load}");
let data = create_sequential::<T>(0xdeadbeef, load + 1, 1000);

// Each iteration performs one transaction.
g.throughput(criterion::Throughput::Elements(1));

g.bench_function(&id, |b| {
bench_harness(
b,
db,
|db| {
let mut data = data.clone();
db.clear_table(table_id)?;
let row = data.pop().unwrap();
db.insert_bulk(table_id, data)?;
Ok(row)
},
|db, row| {
db.insert(table_id, row)?;
Ok(())
},
)
});
db.clear_table(table_id)?;
Ok(())
}

#[inline(never)]
fn insert_bulk<DB: BenchDatabase, T: BenchTable + RandomTable>(
g: &mut Group,
Expand Down Expand Up @@ -319,8 +274,8 @@ fn filter<DB: BenchDatabase, T: BenchTable + RandomTable>(
};
let mean_result_count = load / buckets;
let indexed = match index_strategy {
IndexStrategy::MultiIndex => "indexed",
IndexStrategy::NonUnique => "non_indexed",
IndexStrategy::BTreeEachColumn => "index",
IndexStrategy::NoIndex => "no_index",
_ => unimplemented!(),
};
let id = format!("filter/{filter_column_type}/{indexed}/load={load}/count={mean_result_count}");
Expand Down Expand Up @@ -357,53 +312,5 @@ fn filter<DB: BenchDatabase, T: BenchTable + RandomTable>(
Ok(())
}

#[inline(never)]
fn find<DB: BenchDatabase, T: BenchTable + RandomTable>(
g: &mut Group,
db: &mut DB,
table_id: &DB::TableId,
index_strategy: &IndexStrategy,
column_id: u32,
load: u32,
buckets: u32,
) -> ResultBench<()> {
assert_eq!(
*index_strategy,
IndexStrategy::Unique,
"find benchmarks require unique key"
);
let id = format!("find_unique/u32/load={load}");

let data = create_sequential::<T>(0xdeadbeef, load, buckets as u64);

db.insert_bulk(table_id, data.clone())?;

// Each iteration performs a single transaction.
g.throughput(criterion::Throughput::Elements(1));

// We loop through all buckets found in the sample data.
// This mildly increases variance on the benchmark, but makes "mean_result_count" more accurate.
// Note that all benchmarks use exactly the same sample data.
let mut i = 0;

g.bench_function(&id, |b| {
bench_harness(
b,
db,
|_| {
let value = data[i].clone().into_product_value().elements[column_id as usize].clone();
i = (i + 1) % load as usize;
Ok(value)
},
|db, value| {
db.filter::<T>(table_id, column_id, value)?;
Ok(())
},
)
});
db.clear_table(table_id)?;
Ok(())
}

criterion_group!(benches, criterion_benchmark);
criterion_main!(benches);
12 changes: 6 additions & 6 deletions crates/bench/benches/special.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ use criterion::{criterion_group, criterion_main, Criterion};
use mimalloc::MiMalloc;
use spacetimedb::db::{Config, Storage};
use spacetimedb_bench::{
schemas::{create_sequential, BenchTable, Location, Person, RandomTable},
schemas::{create_sequential, u32_u64_str, u32_u64_u64, BenchTable, RandomTable},
spacetime_module::BENCHMARKS_MODULE,
};
use spacetimedb_lib::{sats, ProductValue};
Expand All @@ -12,8 +12,8 @@ use spacetimedb_testing::modules::start_runtime;
static GLOBAL: MiMalloc = MiMalloc;

fn criterion_benchmark(c: &mut Criterion) {
serialize_benchmarks::<Person>(c);
serialize_benchmarks::<Location>(c);
serialize_benchmarks::<u32_u64_str>(c);
serialize_benchmarks::<u32_u64_u64>(c);

custom_module_benchmarks(c);
}
Expand All @@ -28,7 +28,7 @@ fn custom_module_benchmarks(c: &mut Criterion) {
let module = runtime.block_on(async { BENCHMARKS_MODULE.load_module(config, None).await });

let args = sats::product!["0".repeat(65536)];
c.bench_function("stdb_module/large_arguments/64KiB", |b| {
c.bench_function("special/stdb_module/large_arguments/64KiB", |b| {
b.iter_batched(
|| args.clone(),
|args| runtime.block_on(async { module.call_reducer_binary("fn_with_1_args", args).await.unwrap() }),
Expand All @@ -38,7 +38,7 @@ fn custom_module_benchmarks(c: &mut Criterion) {

for n in [1u32, 100, 1000] {
let args = sats::product![n];
c.bench_function(&format!("stdb_module/print_bulk/lines={n}"), |b| {
c.bench_function(&format!("special/stdb_module/print_bulk/lines={n}"), |b| {
b.iter_batched(
|| args.clone(),
|args| runtime.block_on(async { module.call_reducer_binary("print_many_things", args).await.unwrap() }),
Expand All @@ -49,7 +49,7 @@ fn custom_module_benchmarks(c: &mut Criterion) {
}

fn serialize_benchmarks<T: BenchTable + RandomTable>(c: &mut Criterion) {
let name = T::name_snake_case();
let name = T::name();
let count = 100;
let mut group = c.benchmark_group("special/serialize");
group.throughput(criterion::Throughput::Elements(count));
Expand Down
3 changes: 0 additions & 3 deletions crates/bench/src/database.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,9 +28,6 @@ pub trait BenchDatabase: Sized {
/// Perform an empty transaction.
fn empty_transaction(&mut self) -> ResultBench<()>;

/// Perform a transaction that commits a single row.
fn insert<T: BenchTable>(&mut self, table_id: &Self::TableId, row: T) -> ResultBench<()>;

/// Perform a transaction that commits many rows.
fn insert_bulk<T: BenchTable>(&mut self, table_id: &Self::TableId, rows: Vec<T>) -> ResultBench<()>;

Expand Down
32 changes: 15 additions & 17 deletions crates/bench/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ pub type ResultBench<T> = Result<T, anyhow::Error>;
mod tests {
use crate::{
database::BenchDatabase,
schemas::{create_sequential, BenchTable, IndexStrategy, Location, Person, RandomTable},
schemas::{create_sequential, u32_u64_str, u32_u64_u64, BenchTable, IndexStrategy, RandomTable},
spacetime_module::SpacetimeModule,
spacetime_raw::SpacetimeRaw,
sqlite::SQLite,
Expand Down Expand Up @@ -57,9 +57,7 @@ mod tests {

let sample_data = create_sequential::<T>(0xdeadbeef, count, 100);

for row in sample_data.clone() {
db.insert::<T>(&table_id, row)?;
}
db.insert_bulk(&table_id, sample_data.clone())?;
assert_eq!(db.count_table(&table_id)?, count, "inserted rows should be inserted");

db.clear_table(&table_id)?;
Expand All @@ -76,7 +74,7 @@ mod tests {
"bulk inserted rows should be bulk inserted"
);

if index_strategy == IndexStrategy::Unique {
if index_strategy == IndexStrategy::Unique0 {
db.update_bulk::<T>(&table_id, count)?;
assert_eq!(
db.count_table(&table_id)?,
Expand All @@ -96,28 +94,28 @@ mod tests {

#[test]
fn test_basic_invariants_sqlite() {
basic_invariants::<SQLite, Person>(IndexStrategy::Unique, true).unwrap();
basic_invariants::<SQLite, Location>(IndexStrategy::Unique, true).unwrap();
}

#[test]
fn test_basic_invariants_sqlite_multi_index() {
basic_invariants::<SQLite, Person>(IndexStrategy::MultiIndex, true).unwrap();
basic_invariants::<SQLite, Location>(IndexStrategy::MultiIndex, true).unwrap();
basic_invariants::<SQLite, u32_u64_str>(IndexStrategy::Unique0, true).unwrap();
basic_invariants::<SQLite, u32_u64_u64>(IndexStrategy::Unique0, true).unwrap();
basic_invariants::<SQLite, u32_u64_str>(IndexStrategy::BTreeEachColumn, true).unwrap();
basic_invariants::<SQLite, u32_u64_u64>(IndexStrategy::BTreeEachColumn, true).unwrap();
}

#[test]
fn test_basic_invariants_spacetime_raw() {
basic_invariants::<SpacetimeRaw, Person>(IndexStrategy::Unique, true).unwrap();
basic_invariants::<SpacetimeRaw, Location>(IndexStrategy::Unique, true).unwrap();
basic_invariants::<SpacetimeRaw, u32_u64_str>(IndexStrategy::Unique0, true).unwrap();
basic_invariants::<SpacetimeRaw, u32_u64_u64>(IndexStrategy::Unique0, true).unwrap();
basic_invariants::<SpacetimeRaw, u32_u64_str>(IndexStrategy::BTreeEachColumn, true).unwrap();
basic_invariants::<SpacetimeRaw, u32_u64_u64>(IndexStrategy::BTreeEachColumn, true).unwrap();
}

#[test]
fn test_basic_invariants_spacetime_module() {
// note: there can only be one #[test] invoking spacetime module stuff.
// #[test]s run concurrently and they fight over lockfiles.
// so, run the sub-tests here in sequence.
basic_invariants::<SpacetimeModule, Person>(IndexStrategy::Unique, true).unwrap();
basic_invariants::<SpacetimeModule, Location>(IndexStrategy::Unique, true).unwrap();
basic_invariants::<SpacetimeModule, u32_u64_str>(IndexStrategy::Unique0, true).unwrap();
basic_invariants::<SpacetimeModule, u32_u64_u64>(IndexStrategy::Unique0, true).unwrap();
basic_invariants::<SpacetimeModule, u32_u64_str>(IndexStrategy::BTreeEachColumn, true).unwrap();
basic_invariants::<SpacetimeModule, u32_u64_u64>(IndexStrategy::BTreeEachColumn, true).unwrap();
}
}
Loading