diff --git a/Cargo.lock b/Cargo.lock index 4e803b92..529fe3e1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -651,7 +651,7 @@ dependencies = [ [[package]] name = "ui_test" -version = "0.19.1" +version = "0.20.0" dependencies = [ "annotate-snippets", "anyhow", diff --git a/Cargo.toml b/Cargo.toml index 02216994..d499df74 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "ui_test" -version = "0.19.1" +version = "0.20.0" edition = "2021" license = "MIT OR Apache-2.0" description = "A test framework for testing rustc diagnostics output" diff --git a/src/cmd.rs b/src/cmd.rs index 63d055e2..53fa3e0f 100644 --- a/src/cmd.rs +++ b/src/cmd.rs @@ -48,7 +48,7 @@ impl CommandBuilder { } } - /// Same as [`rustc`], but with arguments for obtaining the cfgs. + /// Same as [`CommandBuilder::rustc`], but with arguments for obtaining the cfgs. pub fn cfgs() -> Self { Self { args: vec!["--print".into(), "cfg".into()], diff --git a/src/config.rs b/src/config.rs index 1ae895f9..6779bab7 100644 --- a/src/config.rs +++ b/src/config.rs @@ -5,6 +5,7 @@ pub use color_eyre; use color_eyre::eyre::Result; use std::{ ffi::OsString, + num::NonZeroUsize, path::{Path, PathBuf}, }; @@ -34,16 +35,24 @@ pub struct Config { pub program: CommandBuilder, /// The command to run to obtain the cfgs that the output is supposed to pub cfgs: CommandBuilder, + /// What to do in case the stdout/stderr output differs from the expected one. + pub output_conflict_handling: OutputConflictHandling, /// Path to a `Cargo.toml` that describes which dependencies the tests can access. pub dependencies_crate_manifest_path: Option, /// The command to run can be changed from `cargo` to any custom command to build the - /// dependencies in `dependencies_crate_manifest_path` + /// dependencies in `dependencies_crate_manifest_path`. pub dependency_builder: CommandBuilder, /// Where to dump files like the binaries compiled from tests. /// Defaults to `target/ui` in the current directory. pub out_dir: PathBuf, - /// The default edition to use on all tests + /// The default edition to use on all tests. pub edition: Option, + /// Skip test files whose names contain any of these entries. + pub skip_files: Vec, + /// Only test files whose names contain any of these entries. + pub filter_files: Vec, + /// Override the number of threads to use. + pub threads: Option, } impl Config { @@ -74,6 +83,7 @@ impl Config { }, program: CommandBuilder::rustc(), cfgs: CommandBuilder::cfgs(), + output_conflict_handling: OutputConflictHandling::Bless, dependencies_crate_manifest_path: None, dependency_builder: CommandBuilder::cargo(), out_dir: std::env::var_os("CARGO_TARGET_DIR") @@ -81,6 +91,9 @@ impl Config { .unwrap_or_else(|| std::env::current_dir().unwrap().join("target")) .join("ui"), edition: Some("2021".into()), + skip_files: Vec::new(), + filter_files: Vec::new(), + threads: None, } } @@ -98,6 +111,44 @@ impl Config { } } + /// Populate the config with the values from parsed command line arguments. + /// If neither `--bless` or `--check` are provided `default_bless` is used. + /// + /// The default output conflict handling command suggests adding `--bless` + /// to the end of the current command. + pub fn with_args(&mut self, args: &Args, default_bless: bool) { + let Args { + ref filters, + quiet: _, + check, + bless, + threads, + ref skip, + } = *args; + + self.threads = threads.or(self.threads); + + self.filter_files.extend_from_slice(filters); + self.skip_files.extend_from_slice(skip); + + let bless = match (bless, check) { + (_, true) => false, + (true, _) => true, + _ => default_bless, + }; + self.output_conflict_handling = if bless { + OutputConflictHandling::Bless + } else { + OutputConflictHandling::Error(format!( + "{} --bless", + std::env::args() + .map(|s| format!("{s:?}")) + .collect::>() + .join(" ") + )) + }; + } + /// Replace all occurrences of a path in stderr/stdout with a byte string. pub fn path_filter(&mut self, path: &Path, replacement: &'static (impl AsRef<[u8]> + ?Sized)) { self.path_stderr_filter(path, replacement); @@ -154,8 +205,8 @@ impl Config { /// Compile dependencies and return the right flags /// to find the dependencies. - pub fn build_dependencies(&self, args: &Args) -> Result> { - let dependencies = build_dependencies(args, self)?; + pub fn build_dependencies(&self) -> Result> { + let dependencies = build_dependencies(self)?; let mut args = vec![]; for (name, artifacts) in dependencies.dependencies { for dependency in artifacts { @@ -207,3 +258,15 @@ impl Config { .any(|arch| self.target.as_ref().unwrap().contains(arch)) } } + +#[derive(Debug, Clone)] +/// The different options for what to do when stdout/stderr files differ from the actual output. +pub enum OutputConflictHandling { + /// The string should be a command that can be executed to bless all tests. + Error(String), + /// Ignore mismatches in the stderr/stdout files. + Ignore, + /// Instead of erroring if the stderr/stdout differs from the expected + /// automatically replace it with the found output (after applying filters). + Bless, +} diff --git a/src/config/args.rs b/src/config/args.rs index 0d325719..2d65dc7d 100644 --- a/src/config/args.rs +++ b/src/config/args.rs @@ -3,10 +3,10 @@ use std::{borrow::Cow, num::NonZeroUsize}; -use color_eyre::eyre::{bail, ensure, eyre, Result}; +use color_eyre::eyre::{bail, ensure, Result}; /// Plain arguments if `ui_test` is used as a binary. -#[derive(Debug)] +#[derive(Debug, Default)] pub struct Args { /// Filters that will be used to match on individual tests pub filters: Vec, @@ -23,37 +23,17 @@ pub struct Args { pub bless: bool, /// The number of threads to use - pub threads: NonZeroUsize, + pub threads: Option, /// Skip tests whose names contain any of these entries. pub skip: Vec, } impl Args { - /// Dummy arguments, but with the number of threads loaded from the environment. - /// The boolearn argument decides whether to bless (`true`) or whether to error (`false`) - pub fn default(bless: bool) -> Result { - Ok(Args { - filters: vec![], - quiet: false, - bless, - check: !bless, - skip: vec![], - threads: match std::env::var_os("RUST_TEST_THREADS") { - None => std::thread::available_parallelism()?, - Some(n) => n - .to_str() - .ok_or_else(|| eyre!("could not parse RUST_TEST_THREADS env var"))? - .parse()?, - }, - }) - } - /// Parse the program arguments. /// This is meant to be used if `ui_test` is used as a `harness=false` test, called from `cargo test`. - /// The boolearn argument decides whether to bless (`true`) or whether to error (`false`) - pub fn test(bless: bool) -> Result { - Self::default(bless)?.parse_args(std::env::args().skip(1)) + pub fn test() -> Result { + Self::default().parse_args(std::env::args().skip(1)) } /// Parse arguments into an existing `Args` struct. @@ -73,7 +53,7 @@ impl Args { } else if arg == "--help" { bail!("available flags: --quiet, --check, --bless, --test-threads=n, --skip") } else if let Some(n) = parse_value("--test-threads", &arg, &mut iter)? { - self.threads = n.parse()?; + self.threads = Some(n.parse()?); } else if arg.starts_with("--") { bail!( "unknown command line flag `{arg}`: {:?}", diff --git a/src/dependencies.rs b/src/dependencies.rs index f98bd4c7..0e95e0f8 100644 --- a/src/dependencies.rs +++ b/src/dependencies.rs @@ -10,7 +10,9 @@ use std::{ sync::{Arc, OnceLock, RwLock}, }; -use crate::{build_aux, status_emitter::StatusEmitter, Args, Config, Errored, Mode}; +use crate::{ + build_aux, status_emitter::StatusEmitter, Config, Errored, Mode, OutputConflictHandling, +}; #[derive(Default, Debug)] pub struct Dependencies { @@ -43,7 +45,7 @@ fn cfgs(config: &Config) -> Result> { } /// Compiles dependencies and returns the crate names and corresponding rmeta files. -pub(crate) fn build_dependencies(args: &Args, config: &Config) -> Result { +pub(crate) fn build_dependencies(config: &Config) -> Result { let manifest_path = match &config.dependencies_crate_manifest_path { Some(path) => path.to_owned(), None => return Ok(Default::default()), @@ -57,10 +59,12 @@ pub(crate) fn build_dependencies(args: &Args, config: &Config) -> Result {} + (OutputConflictHandling::Error(_), _) => { cmd.arg("--locked"); } + _ => {} }; set_locking(&mut build); @@ -233,12 +237,7 @@ impl<'a> BuildManager<'a> { /// that need to be passed in order to build the dependencies. /// The error is only reported once, all follow up invocations of the same build will /// have a generic error about a previous build failing. - pub fn build( - &self, - what: Build, - config: &Config, - args: &Args, - ) -> Result, Errored> { + pub fn build(&self, what: Build, config: &Config) -> Result, Errored> { // Fast path without much contention. if let Some(res) = self.cache.read().unwrap().get(&what).and_then(|o| o.get()) { return res.clone().map_err(|()| Errored { @@ -276,7 +275,7 @@ impl<'a> BuildManager<'a> { .register_test(what.description().into()) .for_revision(""); let res = match &what { - Build::Dependencies => match config.build_dependencies(args) { + Build::Dependencies => match config.build_dependencies() { Ok(args) => Ok(args), Err(e) => { err = Some(Errored { @@ -288,7 +287,7 @@ impl<'a> BuildManager<'a> { Err(()) } }, - Build::Aux { aux_file } => match build_aux(aux_file, config, args, self) { + Build::Aux { aux_file } => match build_aux(aux_file, config, self) { Ok(args) => Ok(args.iter().map(Into::into).collect()), Err(e) => { err = Some(e); diff --git a/src/error.rs b/src/error.rs index 4d28fc3e..7298a699 100644 --- a/src/error.rs +++ b/src/error.rs @@ -32,6 +32,8 @@ pub enum Error { actual: Vec, /// The contents of the file. expected: Vec, + /// A command, that when run, causes the output to get blessed instead of erroring. + bless_command: String, }, /// There were errors that don't have a pattern. ErrorsWithoutPattern { diff --git a/src/lib.rs b/src/lib.rs index 6732f311..ba051be8 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -114,8 +114,8 @@ pub type Filter = Vec<(Match, &'static [u8])>; /// Run all tests as described in the config argument. /// Will additionally process command line arguments. -pub fn run_tests(config: Config) -> Result<()> { - let args = Args::test(true)?; +pub fn run_tests(mut config: Config) -> Result<()> { + let args = Args::test()?; if !args.quiet { println!("Compiler: {}", config.program.display()); } @@ -127,29 +127,35 @@ pub fn run_tests(config: Config) -> Result<()> { } else { status_emitter::Text::verbose() }; + config.with_args(&args, true); run_tests_generic( vec![config], - args, default_file_filter, default_per_file_config, (text, status_emitter::Gha:: { name }), ) } -/// The filter used by `run_tests` to only run on `.rs` files -/// and those specified in the command line args. -pub fn default_file_filter(path: &Path, args: &Args, _config: &Config) -> bool { - path.extension().is_some_and(|ext| ext == "rs") && default_filter_by_arg(path, args) +/// The filter used by `run_tests` to only run on `.rs` files that are +/// specified by [`Config::filter_files`] and [`Config::skip_files`]. +pub fn default_file_filter(path: &Path, config: &Config) -> bool { + path.extension().is_some_and(|ext| ext == "rs") && default_any_file_filter(path, config) } -/// Run on all files that are matched by the filter in the argument list -pub fn default_filter_by_arg(path: &Path, args: &Args) -> bool { - if args.filters.is_empty() { - return true; - } +/// Run on all files that are specified by [`Config::filter_files`] and +/// [`Config::skip_files`]. +/// +/// To only include rust files see [`default_file_filter`]. +pub fn default_any_file_filter(path: &Path, config: &Config) -> bool { let path = path.display().to_string(); - args.filters.iter().any(|f| path.contains(f)) + let contains_path = |files: &[String]| files.iter().any(|f| path.contains(f)); + + if contains_path(&config.skip_files) { + return false; + } + + config.filter_files.is_empty() || contains_path(&config.filter_files) } /// The default per-file config used by `run_tests`. @@ -173,7 +179,7 @@ pub fn default_per_file_config(config: &mut Config, _path: &Path, file_contents: /// Ignores various settings from `Config` that relate to finding test files. pub fn test_command(mut config: Config, path: &Path) -> Result { config.fill_host_and_target()?; - let extra_args = config.build_dependencies(&Args::default(false)?)?; + let extra_args = config.build_dependencies()?; let comments = Comments::parse_file(path)?.map_err(|errors| color_eyre::eyre::eyre!("{errors:#?}"))?; @@ -215,10 +221,11 @@ struct TestRun { } /// A version of `run_tests` that allows more fine-grained control over running tests. +/// +/// If multiple configs are provided only the first [`Config::threads`] value is used pub fn run_tests_generic( mut configs: Vec, - args: Args, - file_filter: impl Fn(&Path, &Args, &Config) -> bool + Sync, + file_filter: impl Fn(&Path, &Config) -> bool + Sync, per_file_config: impl Fn(&mut Config, &Path, &[u8]) + Sync, status_emitter: impl StatusEmitter + Send, ) -> Result<()> { @@ -230,8 +237,19 @@ pub fn run_tests_generic( let mut results = vec![]; + let num_threads = match configs.first().and_then(|config| config.threads) { + Some(threads) => threads, + None => match std::env::var_os("RUST_TEST_THREADS") { + Some(n) => n + .to_str() + .ok_or_else(|| eyre!("could not parse RUST_TEST_THREADS env var"))? + .parse()?, + None => std::thread::available_parallelism()?, + }, + }; + run_and_collect( - args.threads, + num_threads, |submit| { let mut todo = VecDeque::new(); for config in &configs { @@ -252,12 +270,7 @@ pub fn run_tests_generic( for entry in entries { todo.push_back((entry, config)); } - } else if !args - .skip - .iter() - .any(|skip| path.display().to_string().contains(skip)) - && file_filter(&path, &args, config) - { + } else if file_filter(&path, config) { let status = status_emitter.register_test(path); // Forward .rs files to the test workers. submit.send((status, config)).unwrap(); @@ -271,7 +284,7 @@ pub fn run_tests_generic( let mut config = config.clone(); per_file_config(&mut config, path, &file_contents); let result = match std::panic::catch_unwind(|| { - parse_and_test_file(&build_manager, &status, config, &args, file_contents) + parse_and_test_file(&build_manager, &status, config, file_contents) }) { Ok(Ok(res)) => res, Ok(Err(err)) => { @@ -391,7 +404,6 @@ fn parse_and_test_file( build_manager: &BuildManager<'_>, status: &dyn TestStatus, mut config: Config, - args: &Args, file_contents: Vec, ) -> Result, Errored> { let comments = parse_comments(&file_contents)?; @@ -413,7 +425,7 @@ fn parse_and_test_file( if !built_deps { status.update_status("waiting for dependencies to finish building".into()); - match build_manager.build(Build::Dependencies, &config, args) { + match build_manager.build(Build::Dependencies, &config) { Ok(extra_args) => config.program.args.extend(extra_args), Err(err) => { return TestRun { @@ -426,7 +438,7 @@ fn parse_and_test_file( built_deps = true; } - let result = status.run_test(build_manager, &config, args, &comments); + let result = status.run_test(build_manager, &config, &comments); TestRun { result, status } }) .collect()) @@ -480,7 +492,6 @@ fn build_command( fn build_aux( aux_file: &Path, config: &Config, - args: &Args, build_manager: &BuildManager<'_>, ) -> std::result::Result, Errored> { let file_contents = std::fs::read(aux_file).map_err(|err| Errored { @@ -532,7 +543,6 @@ fn build_aux( &comments, "", &config, - args, build_manager, )?; // Make sure we see our dependencies @@ -580,7 +590,6 @@ impl dyn TestStatus { &self, build_manager: &BuildManager<'_>, config: &Config, - args: &Args, comments: &Comments, ) -> TestResult { let path = self.path(); @@ -591,7 +600,6 @@ impl dyn TestStatus { comments, revision, config, - args, build_manager, )?; @@ -604,7 +612,7 @@ impl dyn TestStatus { match *mode { Mode::Run { .. } if Mode::Pass.ok(status).is_ok() => { - return run_test_binary(mode, path, revision, comments, cmd, config, args) + return run_test_binary(mode, path, revision, comments, cmd, config) } Mode::Panic | Mode::Yolo { .. } => {} Mode::Run { .. } | Mode::Pass | Mode::Fail { .. } => { @@ -623,10 +631,10 @@ impl dyn TestStatus { } } check_test_result( - cmd, *mode, path, config, args, revision, comments, status, &stdout, &stderr, + cmd, *mode, path, config, revision, comments, status, &stdout, &stderr, )?; run_rustfix( - &stderr, &stdout, path, comments, revision, config, args, *mode, extra_args, + &stderr, &stdout, path, comments, revision, config, *mode, extra_args, )?; Ok(TestOk::Ok) } @@ -650,7 +658,6 @@ fn build_aux_files( comments: &Comments, revision: &str, config: &Config, - args: &Args, build_manager: &BuildManager<'_>, ) -> Result, Errored> { let mut extra_args = vec![]; @@ -682,7 +689,6 @@ fn build_aux_files( .collect(), }, config, - args, ) .map_err( |Errored { @@ -714,7 +720,6 @@ fn run_test_binary( comments: &Comments, mut cmd: Command, config: &Config, - args: &Args, ) -> TestResult { cmd.arg("--print").arg("file-names"); let output = cmd.output().unwrap(); @@ -735,7 +740,6 @@ fn run_test_binary( &mut errors, revision, config, - args, comments, &output.stdout, &output.stderr, @@ -761,7 +765,6 @@ fn run_rustfix( comments: &Comments, revision: &str, config: &Config, - args: &Args, mode: Mode, extra_args: Vec, ) -> Result<(), Errored> { @@ -867,7 +870,6 @@ fn run_rustfix( "fixed", &Filter::default(), config, - args, &rustfix_comments, revision, ); @@ -923,7 +925,6 @@ fn check_test_result( mode: Mode, path: &Path, config: &Config, - args: &Args, revision: &str, comments: &Comments, status: ExitStatus, @@ -939,7 +940,6 @@ fn check_test_result( &mut errors, revision, config, - args, comments, stdout, &diagnostics.rendered, @@ -971,7 +971,6 @@ fn check_test_output( errors: &mut Vec, revision: &str, config: &Config, - args: &Args, comments: &Comments, stdout: &[u8], stderr: &[u8], @@ -985,7 +984,6 @@ fn check_test_output( "stderr", &config.stderr_filters, config, - args, comments, revision, ); @@ -996,7 +994,6 @@ fn check_test_output( "stdout", &config.stdout_filters, config, - args, comments, revision, ); @@ -1127,29 +1124,32 @@ fn check_output( kind: &'static str, filters: &Filter, config: &Config, - args: &Args, comments: &Comments, revision: &str, ) -> PathBuf { let target = config.target.as_ref().unwrap(); let output = normalize(path, output, filters, comments, revision, kind); let path = output_path(path, comments, revised(revision, kind), target, revision); - if args.check { - let expected_output = std::fs::read(&path).unwrap_or_default(); - if output != expected_output { - errors.push(Error::OutputDiffers { - path: path.clone(), - actual: output.clone(), - expected: expected_output, - }); + match &config.output_conflict_handling { + OutputConflictHandling::Error(bless_command) => { + let expected_output = std::fs::read(&path).unwrap_or_default(); + if output != expected_output { + errors.push(Error::OutputDiffers { + path: path.clone(), + actual: output.clone(), + expected: expected_output, + bless_command: bless_command.clone(), + }); + } } - } - if args.bless { - if output.is_empty() { - let _ = std::fs::remove_file(&path); - } else { - std::fs::write(&path, &output).unwrap(); + OutputConflictHandling::Bless => { + if output.is_empty() { + let _ = std::fs::remove_file(&path); + } else { + std::fs::write(&path, &output).unwrap(); + } } + OutputConflictHandling::Ignore => {} } path } diff --git a/src/status_emitter.rs b/src/status_emitter.rs index 2e63347b..b3223590 100644 --- a/src/status_emitter.rs +++ b/src/status_emitter.rs @@ -435,8 +435,14 @@ fn print_error(error: &Error, path: &Path) { path: output_path, actual, expected, + bless_command, } => { println!("{}", "actual output differed from expected".underline()); + println!( + "Execute `{}` to update `{}` to the actual output", + bless_command, + output_path.display() + ); println!("{}", format!("--- {}", output_path.display()).red()); println!( "{}", @@ -615,13 +621,18 @@ fn gha_error(error: &Error, test_path: &str, revision: &str) { path: output_path, actual, expected, + bless_command, } => { if expected.is_empty() { let mut err = github_actions::error( test_path, "test generated output, but there was no output file", ); - writeln!(err, "you likely need to bless the tests").unwrap(); + writeln!( + err, + "you likely need to bless the tests with `{bless_command}`" + ) + .unwrap(); return; } diff --git a/tests/integration.rs b/tests/integration.rs index a18582f5..abc9a92e 100644 --- a/tests/integration.rs +++ b/tests/integration.rs @@ -9,7 +9,8 @@ fn main() -> Result<()> { let mut config = Config { ..Config::cargo(root_dir.clone()) }; - let args = Args::test(true)?; + let args = Args::test()?; + config.with_args(&args, true); config.program.args = vec![ "test".into(), @@ -83,8 +84,7 @@ fn main() -> Result<()> { ..config }, ], - args, - |path, args, config| { + |path, config| { let fail = path .parent() .unwrap() @@ -107,7 +107,7 @@ fn main() -> Result<()> { Mode::Panic => fail, Mode::Run { .. } | Mode::Yolo { .. } | Mode::Fail { .. } => unreachable!(), } - && default_filter_by_arg(path, args) + && default_any_file_filter(path, config) }, |_, _, _| {}, ( diff --git a/tests/integrations/basic-bin/Cargo.lock b/tests/integrations/basic-bin/Cargo.lock index c9c2191a..2ae30b52 100644 --- a/tests/integrations/basic-bin/Cargo.lock +++ b/tests/integrations/basic-bin/Cargo.lock @@ -697,7 +697,7 @@ dependencies = [ [[package]] name = "ui_test" -version = "0.19.1" +version = "0.20.0" dependencies = [ "annotate-snippets", "anyhow", diff --git a/tests/integrations/basic-bin/tests/ui_tests.rs b/tests/integrations/basic-bin/tests/ui_tests.rs index cfa9581a..2a030db9 100644 --- a/tests/integrations/basic-bin/tests/ui_tests.rs +++ b/tests/integrations/basic-bin/tests/ui_tests.rs @@ -4,6 +4,11 @@ fn main() -> ui_test::color_eyre::Result<()> { let path = "../../../target"; let mut config = Config { dependencies_crate_manifest_path: Some("Cargo.toml".into()), + output_conflict_handling: if std::env::var_os("BLESS").is_some() { + OutputConflictHandling::Bless + } else { + OutputConflictHandling::Error("cargo test".to_string()) + }, ..Config::rustc("tests/actual_tests") }; config.stderr_filter("in ([0-9]m )?[0-9\\.]+s", ""); @@ -21,7 +26,6 @@ fn main() -> ui_test::color_eyre::Result<()> { run_tests_generic( vec![config], - Args::test(std::env::var_os("BLESS").is_some())?, default_file_filter, default_per_file_config, // Avoid github actions, as these would end up showing up in `Cargo.stderr` diff --git a/tests/integrations/basic-fail-mode/Cargo.lock b/tests/integrations/basic-fail-mode/Cargo.lock index 8251cf01..8542b0f8 100644 --- a/tests/integrations/basic-fail-mode/Cargo.lock +++ b/tests/integrations/basic-fail-mode/Cargo.lock @@ -697,7 +697,7 @@ dependencies = [ [[package]] name = "ui_test" -version = "0.19.1" +version = "0.20.0" dependencies = [ "annotate-snippets", "anyhow", diff --git a/tests/integrations/basic-fail-mode/tests/ui_tests.rs b/tests/integrations/basic-fail-mode/tests/ui_tests.rs index db9c91a5..30b47abc 100644 --- a/tests/integrations/basic-fail-mode/tests/ui_tests.rs +++ b/tests/integrations/basic-fail-mode/tests/ui_tests.rs @@ -8,6 +8,11 @@ fn main() -> ui_test::color_eyre::Result<()> { require_patterns: true, rustfix: RustfixMode::MachineApplicable, }, + output_conflict_handling: if std::env::var_os("BLESS").is_some() { + OutputConflictHandling::Bless + } else { + OutputConflictHandling::Error("cargo test".to_string()) + }, ..Config::rustc("tests/actual_tests") }; config.stderr_filter("in ([0-9]m )?[0-9\\.]+s", ""); @@ -17,7 +22,6 @@ fn main() -> ui_test::color_eyre::Result<()> { run_tests_generic( vec![config], - Args::test(std::env::var_os("BLESS").is_some())?, default_file_filter, default_per_file_config, // Avoid github actions, as these would end up showing up in `Cargo.stderr` diff --git a/tests/integrations/basic-fail/Cargo.lock b/tests/integrations/basic-fail/Cargo.lock index c857b0a3..42e8b316 100644 --- a/tests/integrations/basic-fail/Cargo.lock +++ b/tests/integrations/basic-fail/Cargo.lock @@ -697,7 +697,7 @@ dependencies = [ [[package]] name = "ui_test" -version = "0.19.1" +version = "0.20.0" dependencies = [ "annotate-snippets", "anyhow", diff --git a/tests/integrations/basic-fail/Cargo.stderr b/tests/integrations/basic-fail/Cargo.stderr index 7e2916c3..ddb541a3 100644 --- a/tests/integrations/basic-fail/Cargo.stderr +++ b/tests/integrations/basic-fail/Cargo.stderr @@ -9,7 +9,7 @@ Caused by: thread 'main' panicked at 'invalid mode/result combo: yolo: Err(tests failed Location: - $DIR/src/lib.rs:LL:CC)', tests/ui_tests_bless.rs:48:18 + $DIR/src/lib.rs:LL:CC)', tests/ui_tests_bless.rs:52:18 note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace error: test failed, to rerun pass `--test ui_tests_bless` Error: failed to parse rustc version info: invalid_foobarlaksdfalsdfj diff --git a/tests/integrations/basic-fail/Cargo.stdout b/tests/integrations/basic-fail/Cargo.stdout index 14b24ba9..94a15db5 100644 --- a/tests/integrations/basic-fail/Cargo.stdout +++ b/tests/integrations/basic-fail/Cargo.stdout @@ -57,6 +57,7 @@ tests/actual_tests/executable.rs FAILED: command: "$CMD" actual output differed from expected +Execute `DO NOT BLESS. These are meant to fail` to update `tests/actual_tests/executable.stdout` to the actual output --- tests/actual_tests/executable.stdout +++ -69 @@ -75,6 +76,7 @@ command: "rustc" "--error-format=json" "--extern" "basic_fail=$DIR/$DIR/../../.. run(0) test got exit status: 1, but expected 0 actual output differed from expected +Execute `DO NOT BLESS. These are meant to fail` to update `tests/actual_tests/executable_compile_err.stderr` to the actual output --- tests/actual_tests/executable_compile_err.stderr +++ +error: this file contains an unclosed delimiter @@ -145,6 +147,7 @@ tests/actual_tests/foomp.rs FAILED: command: "rustc" "--error-format=json" "--extern" "basic_fail=$DIR/$DIR/../../../target/$TMP/$TRIPLE/debug/libbasic_fail.rlib" "--extern" "basic_fail=$DIR/$DIR/../../../target/$TMP/$TRIPLE/debug/libbasic_fail-$HASH.rmeta" "-L" "$DIR/$DIR/../../../target/$TMP/$TRIPLE/debug" "-L" "$DIR/$DIR/../../../target/$TMP/$TRIPLE/debug" "--out-dir" "$TMP "tests/actual_tests/foomp.rs" "--edition" "2021" actual output differed from expected +Execute `DO NOT BLESS. These are meant to fail` to update `tests/actual_tests/foomp.stderr` to the actual output --- tests/actual_tests/foomp.stderr +++ error[E0308]: mismatched types @@ -196,6 +199,7 @@ tests/actual_tests/foomp2.rs FAILED: command: "$CMD" actual output differed from expected +Execute `DO NOT BLESS. These are meant to fail` to update `tests/actual_tests/foomp2.fixed` to the actual output --- tests/actual_tests/foomp2.fixed +++ -this is just a test file showing that diff --git a/tests/integrations/basic-fail/tests/ui_tests.rs b/tests/integrations/basic-fail/tests/ui_tests.rs index a400ffc6..b51861cd 100644 --- a/tests/integrations/basic-fail/tests/ui_tests.rs +++ b/tests/integrations/basic-fail/tests/ui_tests.rs @@ -4,6 +4,11 @@ fn main() -> ui_test::color_eyre::Result<()> { let path = "../../../target"; let mut config = Config { dependencies_crate_manifest_path: Some("Cargo.toml".into()), + // Never bless integrations-fail tests, we want to see stderr mismatches + output_conflict_handling: OutputConflictHandling::Error( + "DO NOT BLESS. These are meant to fail".into(), + ), + ..Config::rustc("tests/actual_tests") }; @@ -20,8 +25,6 @@ fn main() -> ui_test::color_eyre::Result<()> { run_tests_generic( vec![config], - // Never bless integrations-fail tests, we want to see stderr mismatches - Args::test(false)?, default_file_filter, default_per_file_config, // Avoid github actions, as these would end up showing up in `Cargo.stderr` diff --git a/tests/integrations/basic-fail/tests/ui_tests_bless.rs b/tests/integrations/basic-fail/tests/ui_tests_bless.rs index 7f62ea49..20f481ec 100644 --- a/tests/integrations/basic-fail/tests/ui_tests_bless.rs +++ b/tests/integrations/basic-fail/tests/ui_tests_bless.rs @@ -20,6 +20,11 @@ fn main() -> ui_test::color_eyre::Result<()> { let mut config = Config { dependencies_crate_manifest_path: Some("Cargo.toml".into()), + output_conflict_handling: if std::env::var_os("BLESS").is_some() { + OutputConflictHandling::Bless + } else { + OutputConflictHandling::Error("cargo test".to_string()) + }, mode, ..Config::rustc(root_dir) }; @@ -36,7 +41,6 @@ fn main() -> ui_test::color_eyre::Result<()> { config.path_stderr_filter(&std::path::Path::new(path), "$DIR"); let result = run_tests_generic( vec![config], - Args::test(std::env::var_os("BLESS").is_some())?, default_file_filter, default_per_file_config, // Avoid github actions, as these would end up showing up in `Cargo.stderr` diff --git a/tests/integrations/basic-fail/tests/ui_tests_invalid_program.rs b/tests/integrations/basic-fail/tests/ui_tests_invalid_program.rs index 951f8b0f..4c2493a5 100644 --- a/tests/integrations/basic-fail/tests/ui_tests_invalid_program.rs +++ b/tests/integrations/basic-fail/tests/ui_tests_invalid_program.rs @@ -3,13 +3,15 @@ use ui_test::*; fn main() -> ui_test::color_eyre::Result<()> { let config = Config { program: CommandBuilder::cmd("invalid_foobarlaksdfalsdfj"), + // Never bless integrations-fail tests, we want to see stderr mismatches + output_conflict_handling: OutputConflictHandling::Error( + "DO NOT BLESS. These are meant to fail".into(), + ), ..Config::rustc("tests/actual_tests") }; run_tests_generic( vec![config], - // Never bless integrations-fail tests, we want to see stderr mismatches - Args::test(false)?, default_file_filter, default_per_file_config, // Avoid github actions, as these would end up showing up in `Cargo.stderr` diff --git a/tests/integrations/basic-fail/tests/ui_tests_invalid_program2.rs b/tests/integrations/basic-fail/tests/ui_tests_invalid_program2.rs index 941bf659..97a3101e 100644 --- a/tests/integrations/basic-fail/tests/ui_tests_invalid_program2.rs +++ b/tests/integrations/basic-fail/tests/ui_tests_invalid_program2.rs @@ -3,14 +3,16 @@ use ui_test::*; fn main() -> ui_test::color_eyre::Result<()> { let config = Config { program: CommandBuilder::cmd("invalid_foobarlaksdfalsdfj"), + // Never bless integrations-fail tests, we want to see stderr mismatches + output_conflict_handling: OutputConflictHandling::Error( + "DO NOT BLESS. These are meant to fail".into(), + ), host: Some("foo".into()), ..Config::rustc("tests/actual_tests") }; run_tests_generic( vec![config], - // Never bless integrations-fail tests, we want to see stderr mismatches - Args::test(false)?, default_file_filter, default_per_file_config, // Avoid github actions, as these would end up showing up in `Cargo.stderr` diff --git a/tests/integrations/basic/Cargo.lock b/tests/integrations/basic/Cargo.lock index fe20730c..9699eb27 100644 --- a/tests/integrations/basic/Cargo.lock +++ b/tests/integrations/basic/Cargo.lock @@ -697,7 +697,7 @@ dependencies = [ [[package]] name = "ui_test" -version = "0.19.1" +version = "0.20.0" dependencies = [ "annotate-snippets", "anyhow", diff --git a/tests/integrations/basic/tests/ui_tests.rs b/tests/integrations/basic/tests/ui_tests.rs index ac38a121..2d972ae0 100644 --- a/tests/integrations/basic/tests/ui_tests.rs +++ b/tests/integrations/basic/tests/ui_tests.rs @@ -4,6 +4,11 @@ fn main() -> ui_test::color_eyre::Result<()> { let path = "../../../target"; let mut config = Config { dependencies_crate_manifest_path: Some("Cargo.toml".into()), + output_conflict_handling: if std::env::var_os("BLESS").is_some() { + OutputConflictHandling::Bless + } else { + OutputConflictHandling::Error("cargo test".to_string()) + }, ..Config::rustc("tests/actual_tests") }; config.stderr_filter("in ([0-9]m )?[0-9\\.]+s", ""); @@ -19,7 +24,6 @@ fn main() -> ui_test::color_eyre::Result<()> { run_tests_generic( vec![config], - Args::test(std::env::var_os("BLESS").is_some())?, default_file_filter, default_per_file_config, // Avoid github actions, as these would end up showing up in `Cargo.stderr`