Skip to content

Commit 10c2966

Browse files
authored
Merge pull request #2325 from Kobzol/stable-benchmarks
Add stable benchmarks to the compile-time benchmark set
2 parents f8a61f3 + dc928cb commit 10c2966

File tree

3 files changed

+43
-23
lines changed

3 files changed

+43
-23
lines changed

collector/src/benchmark_set/compile_benchmarks.rs

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,20 @@
1-
//! This file contains an exhaustive list of all **non-stable** compile-time benchmarks
1+
//! This file contains an exhaustive list of all compile-time benchmarks
22
//! located in the `collector/compile-benchmarks` directory that are benchmarked in production.
33
//! If new benchmarks are added/removed, they have to also be added/removed here, and in
44
//! the [super::expand_benchmark_set] function.
55
6+
// Stable benchmarks
7+
pub(super) const CARGO: &str = "cargo";
8+
pub(super) const ENCODING: &str = "encoding";
9+
pub(super) const FUTURES: &str = "futures";
10+
pub(super) const HTML5EVER: &str = "html5ever";
11+
pub(super) const INFLATE: &str = "inflate";
12+
pub(super) const PISTON_IMAGE: &str = "piston-image";
13+
pub(super) const REGEX: &str = "regex";
14+
pub(super) const SYN: &str = "syn";
15+
pub(super) const TOKIO_WEBPUSH_SIMPLE: &str = "tokio-webpush-simple";
16+
17+
// Non-stable benchmarks
618
pub(super) const AWAIT_CALL_TREE: &str = "await-call-tree";
719
pub(super) const BITMAPS_3_2_1: &str = "bitmaps-3.2.1";
820
pub(super) const BITMAPS_3_2_1_NEW_SOLVER: &str = "bitmaps-3.2.1-new-solver";

collector/src/benchmark_set/mod.rs

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,7 @@ pub fn expand_benchmark_set(id: BenchmarkSetId) -> Vec<BenchmarkSetMember> {
4848
compile(AWAIT_CALL_TREE),
4949
compile(BITMAPS_3_2_1),
5050
compile(BITMAPS_3_2_1_NEW_SOLVER),
51+
compile(CARGO),
5152
compile(CARGO_0_87_1),
5253
compile(CLAP_DERIVE_4_5_32),
5354
compile(COERCIONS),
@@ -57,15 +58,19 @@ pub fn expand_benchmark_set(id: BenchmarkSetId) -> Vec<BenchmarkSetMember> {
5758
compile(DEEPLY_NESTED_MULTI),
5859
compile(DERIVE),
5960
compile(DIESEL_2_2_10),
61+
compile(ENCODING),
6062
compile(EXTERNS),
6163
compile(EZA_0_21_2),
64+
compile(FUTURES),
6265
compile(HELLOWORLD),
6366
compile(HELLOWORLD_TINY),
67+
compile(HTML5EVER),
6468
compile(HTML5EVER_0_31_0),
6569
compile(HTML5EVER_0_31_0_NEW_SOLVER),
6670
compile(HYPER_1_6_0),
6771
compile(IMAGE_0_25_6),
6872
compile(INCLUDE_BLOB),
73+
compile(INFLATE),
6974
compile(ISSUE_46449),
7075
compile(ISSUE_58319),
7176
compile(ISSUE_88862),
@@ -75,7 +80,9 @@ pub fn expand_benchmark_set(id: BenchmarkSetId) -> Vec<BenchmarkSetMember> {
7580
compile(MATCH_STRESS),
7681
compile(NALGEBRA_0_33_0),
7782
compile(NALGEBRA_0_33_0_NEW_SOLVER),
83+
compile(PISTON_IMAGE),
7884
compile(PROJECTION_CACHING),
85+
compile(REGEX),
7986
compile(REGEX_AUTOMATA_0_4_8),
8087
compile(REGRESSION_31157),
8188
compile(RIPGREP_14_1_1),
@@ -85,9 +92,11 @@ pub fn expand_benchmark_set(id: BenchmarkSetId) -> Vec<BenchmarkSetMember> {
8592
compile(SERDE_1_0_219_THREADS4),
8693
compile(SERDE_DERIVE_1_0_219),
8794
compile(STM32F4_0_15_1),
95+
compile(SYN),
8896
compile(SYN_2_0_101),
8997
compile(SYN_2_0_101_NEW_SOLVER),
9098
compile(TOKEN_STREAM_STRESS),
99+
compile(TOKIO_WEBPUSH_SIMPLE),
91100
compile(TT_MUNCHER),
92101
compile(TUPLE_STRESS),
93102
compile(TYPENUM_1_18_0),
@@ -168,7 +177,6 @@ mod tests {
168177
get_compile_benchmarks(Path::new(BENCHMARK_DIR), CompileBenchmarkFilter::All)
169178
.unwrap()
170179
.into_iter()
171-
.filter(|b| !b.category().is_stable())
172180
.map(|b| b.name)
173181
.collect::<Vec<BenchmarkName>>();
174182
for benchmark in &all_compile_benchmarks {

collector/src/bin/collector.rs

Lines changed: 21 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -1785,6 +1785,11 @@ async fn create_benchmark_configs(
17851785
let mut bench_runtime = false;
17861786
let mut bench_compile_benchmarks = HashSet::new();
17871787

1788+
let is_release = match artifact_id {
1789+
ArtifactId::Commit(_) => false,
1790+
ArtifactId::Tag(_) => true,
1791+
};
1792+
17881793
match job.kind() {
17891794
database::BenchmarkJobKind::Runtime => {
17901795
bench_runtime = true;
@@ -1799,32 +1804,27 @@ async fn create_benchmark_configs(
17991804
}
18001805
}
18011806
database::BenchmarkJobKind::Rustc => {
1802-
bench_rustc = true;
1807+
bench_rustc = !is_release;
18031808
}
18041809
}
18051810

1806-
let is_release = match artifact_id {
1807-
ArtifactId::Commit(_) => false,
1808-
ArtifactId::Tag(_) => true,
1809-
};
1810-
if is_release {
1811-
bench_rustc = false;
1812-
bench_compile_benchmarks.retain(|benchmark| {
1813-
all_compile_benchmarks
1814-
.iter()
1815-
.find(|b| b.name == *benchmark)
1816-
.map(|b| b.category().is_stable())
1817-
.unwrap_or(false)
1818-
});
1819-
}
1811+
let benchmarks: Vec<Benchmark> = all_compile_benchmarks
1812+
.iter()
1813+
.filter(|b| {
1814+
if !bench_compile_benchmarks.contains(&b.name) {
1815+
return false;
1816+
}
1817+
// Run stable benchmarks for releases and non-stable benchmarks for everything
1818+
// else.
1819+
let is_stable = b.category().is_stable();
1820+
is_release == is_stable
1821+
})
1822+
.cloned()
1823+
.collect();
18201824

1821-
let compile_config = if bench_rustc || !bench_compile_benchmarks.is_empty() {
1825+
let compile_config = if bench_rustc || !benchmarks.is_empty() {
18221826
Some(CompileBenchmarkConfig {
1823-
benchmarks: all_compile_benchmarks
1824-
.iter()
1825-
.filter(|b| bench_compile_benchmarks.contains(&b.name))
1826-
.cloned()
1827-
.collect(),
1827+
benchmarks,
18281828
profiles: vec![job.profile().into()],
18291829
scenarios: Scenario::all(),
18301830
backends: vec![job.backend().into()],

0 commit comments

Comments
 (0)