Add the directive compare-output-by-lines

Author: ywxt

src/doc/rustc-dev-guide/src/tests/directives.md

@@ -111,6 +111,7 @@ for more details.
 | `forbid-output`                   | A pattern which must not appear in stderr/`cfail` output                                                                 | `ui`, `incremental`                          | Regex pattern                                                                           |
 | `run-flags`                       | Flags passed to the test executable                                                                                      | `ui`                                         | Arbitrary flags                                                                         |
 | `known-bug`                       | No error annotation needed due to known bug                                                                              | `ui`, `crashes`, `incremental`               | Issue number `#123456`                                                                  |
+| `compare-output-by-lines`         | Compare the output by lines, rather than as a single string                                                              | All                                          | N/A                                                                                     |
 
 [^check_stdout]: presently <!-- date-check: Oct 2024 --> this has a weird quirk
     where the test binary's stdout and stderr gets concatenated and then

src/doc/rustc-dev-guide/src/tests/ui.md

@@ -95,6 +95,7 @@ will check for output files:
   [Normalization](#normalization)).
 - `dont-check-compiler-stderr` — Ignores stderr from the compiler.
 - `dont-check-compiler-stdout` — Ignores stdout from the compiler.
+- `compare-output-by-lines` — Some tests have non-deterministic orders of output, so we need to compare by lines.
 
 UI tests run with `-Zdeduplicate-diagnostics=no` flag which disables rustc's
 built-in diagnostic deduplication mechanism. This means you may see some

src/tools/compiletest/src/directives.rs

@@ -205,6 +205,8 @@ pub struct TestProps {
     pub dont_require_annotations: HashSet<ErrorKind>,
     /// Whether pretty printers should be disabled in gdb.
     pub disable_gdb_pretty_printers: bool,
+    /// Compare the output by lines, rather than as a single string.
+    pub compare_output_by_lines: bool,
 }
 
 mod directives {
@@ -254,6 +256,7 @@ mod directives {
     // This isn't a real directive, just one that is probably mistyped often
     pub const INCORRECT_COMPILER_FLAGS: &'static str = "compiler-flags";
     pub const DISABLE_GDB_PRETTY_PRINTERS: &'static str = "disable-gdb-pretty-printers";
+    pub const COMPARE_OUTPUT_BY_LINES: &'static str = "compare-output-by-lines";
 }
 
 impl TestProps {
@@ -310,6 +313,7 @@ impl TestProps {
             add_core_stubs: false,
             dont_require_annotations: Default::default(),
             disable_gdb_pretty_printers: false,
+            compare_output_by_lines: false,
         }
     }
 
@@ -664,6 +668,11 @@ impl TestProps {
                         DISABLE_GDB_PRETTY_PRINTERS,
                         &mut self.disable_gdb_pretty_printers,
                     );
+                    config.set_name_directive(
+                        ln,
+                        COMPARE_OUTPUT_BY_LINES,
+                        &mut self.compare_output_by_lines,
+                    );
                 },
             );
 

src/tools/compiletest/src/directives/directive_names.rs

@@ -17,6 +17,7 @@ pub(crate) const KNOWN_DIRECTIVE_NAMES: &[&str] = &[
     "check-run-results",
     "check-stdout",
     "check-test-line-numbers-match",
+    "compare-output-by-lines",
     "compile-flags",
     "disable-gdb-pretty-printers",
     "doc-flags",

src/tools/compiletest/src/runtest.rs

@@ -2749,7 +2749,11 @@ impl<'test> TestCx<'test> {
         // Wrapper tools set by `runner` might provide extra output on failure,
         // for example a WebAssembly runtime might print the stack trace of an
         // `unreachable` instruction by default.
-        let compare_output_by_lines = self.config.runner.is_some();
+        //
+        // Also, some tests like `ui/parallel-rustc` have non-deterministic
+        // orders of output, so we need to compare by lines.
+        let compare_output_by_lines =
+            self.props.compare_output_by_lines || self.config.runner.is_some();
 
         let tmp;
         let (expected, actual): (&str, &str) = if compare_output_by_lines {

tests/ui/parallel-rustc/cache-after-waiting-issue-111528.rs

@@ -1,16 +1,18 @@
+// Test for #111528, the ice issue cause waiting on a query that panicked
+//
 //@ compile-flags: -Z threads=16
 //@ build-fail
+//@ compare-output-by-lines
 
-#![crate_type="rlib"]
+#![crate_type = "rlib"]
 #![allow(warnings)]
 
-#[export_name="fail"]
-pub fn a() {
-}
+#[export_name = "fail"]
+pub fn a() {}
 
-#[export_name="fail"]
+#[export_name = "fail"]
 pub fn b() {
-//~^ ERROR symbol `fail` is already defined
+    //~^ ERROR symbol `fail` is already defined
 }
 
 fn main() {}

tests/ui/parallel-rustc/cache-after-waiting-issue-111528.stderr

@@ -1,5 +1,5 @@
 error: symbol `fail` is already defined
-  --> $DIR/cache-after-waiting-issue-111528.rs:12:1
+  --> $DIR/cache-after-waiting-issue-111528.rs:14:1
    |
 LL | pub fn b() {
    | ^^^^^^^^^^

tests/ui/parallel-rustc/cycle_crash-issue-135870.rs

@@ -0,0 +1,8 @@
+// Test for #135870, which causes a deadlock bug
+//
+//@ compile-flags: -Z threads=2
+//@ compare-output-by-lines
+
+const FOO: usize = FOO; //~ ERROR cycle detected when simplifying constant for the type system `FOO`
+
+fn main() {}

tests/ui/parallel-rustc/cycle_crash.stderr renamed to tests/ui/parallel-rustc/cycle_crash-issue-135870.stderr

@@ -1,11 +1,9 @@
 error[E0391]: cycle detected when simplifying constant for the type system `FOO`
-  --> $DIR/cycle_crash.rs:3:1
    |
 LL | const FOO: usize = FOO;
    | ^^^^^^^^^^^^^^^^
    |
 note: ...which requires const-evaluating + checking `FOO`...
-  --> $DIR/cycle_crash.rs:3:20
    |
 LL | const FOO: usize = FOO;
    |                    ^^^