feat(divan): add benchmark filtering in instrumentation mode

Author: GuillaumeLagrange

Cargo.lock

@@ -1,10 +1,12 @@
 use assert_cmd::assert::OutputAssertExt;
-use predicates::str::contains;
+use predicates::{prelude::PredicateBooleanExt, str::contains};
 
 mod helpers;
 use helpers::*;
 
 const DIR: &str = "tests/simple-divan.in";
+const FIB_BENCH_NAME: &str = "fib_20";
+const BUBBLE_SORT_BENCH_NAME: &str = "bubble_sort_bench";
 
 #[test]
 fn test_divan_run_without_build() {
@@ -80,20 +82,42 @@ fn test_divan_build_and_run_filtered_by_name() {
         .arg("fib_20")
         .assert()
         .success()
+        .stdout(contains(FIB_BENCH_NAME))
+        .stdout(contains(BUBBLE_SORT_BENCH_NAME).not())
+        .stderr(contains("Finished running 2 benchmark suite(s)"));
+    cargo_codspeed(&dir)
+        .arg("run")
+        .arg("bu.*le_sort")
+        .assert()
+        .success()
+        .stdout(contains(FIB_BENCH_NAME).not())
+        .stdout(contains(BUBBLE_SORT_BENCH_NAME))
         .stderr(contains("Finished running 2 benchmark suite(s)"));
     teardown(dir);
 }
 
 #[test]
-fn test_divan_build_and_run_filtered_by_partial_name() {
+fn test_divan_build_and_run_filtered_by_name_single() {
     let dir = setup(DIR, Project::Simple);
     cargo_codspeed(&dir).arg("build").assert().success();
     cargo_codspeed(&dir)
         .arg("run")
-        .arg("bubble_sort")
+        .arg("bu.*le_sort")
+        .args(["--bench", "divan_example"])
         .assert()
         .success()
-        .stderr(contains("Finished running 2 benchmark suite(s)"));
+        .stdout(contains(FIB_BENCH_NAME).not())
+        .stdout(contains(BUBBLE_SORT_BENCH_NAME).not()) // We are filtering with a name that is not in the selected benchmark
+        .stderr(contains("Finished running 1 benchmark suite(s)"));
+    cargo_codspeed(&dir)
+        .arg("run")
+        .arg("fib")
+        .args(["--bench", "divan_example"])
+        .assert()
+        .success()
+        .stdout(contains(FIB_BENCH_NAME))
+        .stdout(contains(BUBBLE_SORT_BENCH_NAME).not())
+        .stderr(contains("Finished running 1 benchmark suite(s)"));
     teardown(dir);
 }
 

crates/cargo-codspeed/tests/simple-divan.rs

@@ -21,6 +21,8 @@ keywords = ["codspeed", "benchmark", "divan"]
 codspeed = { path = "../codspeed", version = "=4.0.0" }
 divan = { package = "codspeed-divan-compat-walltime", path = "./divan_fork", version = "=4.0.0" }
 codspeed-divan-compat-macros = { version = "=4.0.0", path = './macros' }
+regex = "1.11.3"
+clap = { version = "4", default-features = false, features = ["std", "env"] }
 
 [[bench]]
 name = "basic_example"

crates/divan_compat/Cargo.toml

@@ -0,0 +1,20 @@
+use clap::{Arg, ArgAction, Command};
+
+pub(crate) fn command() -> Command {
+    fn option(name: &'static str) -> Arg {
+        Arg::new(name).long(name)
+    }
+
+    fn flag(name: &'static str) -> Arg {
+        option(name).action(ArgAction::SetTrue)
+    }
+
+    Command::new("divan")
+        .arg(
+            Arg::new("filter")
+                .value_name("FILTER")
+                .help("Only run benchmarks whose names match this pattern")
+                .action(ArgAction::Append),
+        )
+        .arg(flag("exact").help("Filter benchmarks by exact name rather than by pattern"))
+}

crates/divan_compat/src/compat/cli.rs

@@ -0,0 +1,17 @@
+use regex::Regex;
+
+/// Filters which benchmark to run based on name.
+pub(crate) enum Filter {
+    Regex(Regex),
+    Exact(String),
+}
+
+impl Filter {
+    /// Returns `true` if a string matches this filter.
+    pub fn is_match(&self, s: &str) -> bool {
+        match self {
+            Self::Regex(r) => r.is_match(s),
+            Self::Exact(e) => e == s,
+        }
+    }
+}

crates/divan_compat/src/compat/config.rs

@@ -13,15 +13,18 @@ pub mod __private {
 }
 
 mod bench;
+mod cli;
+mod config;
 mod entry;
 mod uri;
 mod util;
 
-use std::{cell::RefCell, rc::Rc};
-
 pub use bench::*;
 use codspeed::codspeed::CodSpeed;
+use config::Filter;
 use entry::AnyBenchEntry;
+use regex::Regex;
+use std::{cell::RefCell, rc::Rc};
 
 pub fn main() {
     // Outlined steps of original divan::main and their equivalent in codspeed instrumented mode
@@ -46,8 +49,37 @@ pub fn main() {
     // codspeed URI from entry metadata directly.
 
     // 3. Filtering
-    // We do not support finer filtering that bench targets for now, do nothing here, bench
-    // filtering is managed by the `cargo-codspeed` wrappers before we reach this point.
+    let should_run_benchmark_from_filters = {
+        let mut command = cli::command();
+        let matches = command.get_matches_mut();
+        let is_exact = matches.get_flag("exact");
+
+        let parse_filter = |filter: &String| {
+            if is_exact {
+                Filter::Exact(filter.to_owned())
+            } else {
+                match Regex::new(filter) {
+                    Ok(r) => Filter::Regex(r),
+                    Err(error) => {
+                        let kind = clap::error::ErrorKind::ValueValidation;
+                        command.error(kind, error).exit();
+                    }
+                }
+            }
+        };
+
+        let filters: Option<Vec<Filter>> = matches
+            .get_many::<String>("filter")
+            .map(|arg_filters| arg_filters.map(parse_filter).collect());
+
+        move |uri: &str| {
+            if let Some(filters) = filters.as_ref() {
+                filters.iter().any(|filter| filter.is_match(uri))
+            } else {
+                true
+            }
+        }
+    };
 
     // 4. Scan the tree and execute benchmarks
     let codspeed = Rc::new(RefCell::new(CodSpeed::new()));
@@ -66,6 +98,10 @@ pub fn main() {
             entry::BenchEntryRunner::Plain(bench_fn) => {
                 let uri = uri::generate(&entry, entry.display_name());
 
+                if !should_run_benchmark_from_filters(&uri) {
+                    continue;
+                }
+
                 bench_fn(bench::Bencher::new(&codspeed, uri));
             }
             entry::BenchEntryRunner::Args(bench_runner) => {
@@ -74,6 +110,10 @@ pub fn main() {
                 for (arg_index, arg_name) in bench_runner.arg_names().iter().enumerate() {
                     let uri = uri::generate(&entry, arg_name);
 
+                    if !should_run_benchmark_from_filters(&uri) {
+                        continue;
+                    }
+
                     let bencher = bench::Bencher::new(&codspeed, uri);
 
                     bench_runner.bench(bencher, arg_index);