[Utils] Adds support for diff based tests to lit's --update-tests

llvm/utils/lit/lit/DiffUpdater.py

@@ -0,0 +1,55 @@
+import shutil
+
+"""
+This file provides the `diff_test_updater` function, which is invoked on failed RUN lines when lit is executed with --update-tests.
+It checks whether the failed command is `diff` and, if so, uses heuristics to determine which file is the checked-in reference file and which file is output from the test case.
+The heuristics are currently as follows:
+    - if exactly one file ends with ".expected" (common pattern in LLVM), that file is the reference file and the other is the output file
+    - if exactly one file path contains ".tmp" (e.g. because it contains the expansion of "%t"), that file is the reference file and the other is the output file
+If the command matches one of these patterns the output file content is copied to the reference file to make the test pass.
+Otherwise the test is ignored.
+
+Possible improvements:
+    - Support stdin patterns like "my_binary %s | diff expected.txt"
+    - Scan RUN lines to see if a file is the source of output from a previous command.
+      If it is then it is not a reference file that can be copied to, regardless of name, since the test will overwrite it anyways.
+    - Only update the parts that need updating (based on the diff output). Could help avoid noisy updates when e.g. whitespace changes are ignored.
+"""
+
+
+def get_source_and_target(a, b):
+    """
+    Try to figure out which file is the test output and which is the reference.
+    """
+    expected_suffix = ".expected"
+    if a.endswith(expected_suffix) and not b.endswith(expected_suffix):
+        return b, a
+    if b.endswith(expected_suffix) and not a.endswith(expected_suffix):
+        return a, b
+
+    tmp_substr = ".tmp"
+    if tmp_substr in a and not tmp_substr in b:
+        return a, b
+    if tmp_substr in b and not tmp_substr in a:
+        return b, a
+
+    return None
+
+
+def filter_flags(args):
+    return [arg for arg in args if not arg.startswith("-")]
+
+
+def diff_test_updater(result, test):
+    args = filter_flags(result.command.args)
+    if len(args) != 3:
+        return None
+    [cmd, a, b] = args
+    if cmd != "diff":
+        return None
+    res = get_source_and_target(a, b)
+    if not res:
+        return f"update-diff-test: could not deduce source and target from {a} and {b}"
+    source, target = res
+    shutil.copy(source, target)
+    return f"update-diff-test: copied {source} to {target}"

llvm/utils/lit/lit/LitConfig.py

@@ -8,6 +8,7 @@
 import lit.formats
 import lit.TestingConfig
 import lit.util
+from lit.DiffUpdater import diff_test_updater
 
 # LitConfig must be a new style class for properties to work
 class LitConfig(object):
@@ -93,7 +94,7 @@ def __init__(
         self.per_test_coverage = per_test_coverage
         self.gtest_sharding = bool(gtest_sharding)
         self.update_tests = update_tests
-        self.test_updaters = []
+        self.test_updaters = [diff_test_updater]
 
     @property
     def maxIndividualTestTime(self):

llvm/utils/lit/tests/Inputs/diff-test-update/.gitignore

@@ -0,0 +1 @@
+*.txt

llvm/utils/lit/tests/Inputs/diff-test-update/1.in

@@ -0,0 +1 @@
+FOO

llvm/utils/lit/tests/Inputs/diff-test-update/2.in

@@ -0,0 +1 @@
+BAR

llvm/utils/lit/tests/Inputs/diff-test-update/diff-bail.test

@@ -0,0 +1,3 @@
+# There is no indication here of which file is the reference file to update
+# RUN: diff %S/1.in %S/2.in
+

llvm/utils/lit/tests/Inputs/diff-test-update/diff-bail2.test

@@ -0,0 +1,7 @@
+# RUN: mkdir %t
+# RUN: cp %S/1.in %t/1.txt
+# RUN: cp %S/2.in %t/2.txt
+
+# There is no indication here of which file is the reference file to update
+# RUN: diff %t/1.txt %t/2.txt
+

llvm/utils/lit/tests/Inputs/diff-test-update/diff-expected.test

@@ -0,0 +1,5 @@
+# RUN: mkdir %t
+# RUN: cp %S/1.in %t/my-file.expected
+# RUN: cp %S/2.in %t/my-file.txt
+# RUN: diff %t/my-file.expected %t/my-file.txt
+

llvm/utils/lit/tests/Inputs/diff-test-update/diff-tmp-dir.test

@@ -0,0 +1,6 @@
+# RUN: mkdir %t
+# RUN: touch %S/empty.txt
+# RUN: cp %S/1.in %t/1.txt
+
+# RUN: diff %t/1.txt %S/empty.txt
+

llvm/utils/lit/tests/Inputs/diff-test-update/diff-tmp.test

@@ -0,0 +1,3 @@
+# RUN: cp %S/1.in %t.txt
+# RUN: cp %S/2.in %S/diff-t-out.txt
+# RUN: diff %t.txt %S/diff-t-out.txt

llvm/utils/lit/tests/Inputs/diff-test-update/lit.cfg

@@ -0,0 +1,8 @@
+import lit.formats
+
+config.name = "diff-test-update"
+config.suffixes = [".test"]
+config.test_format = lit.formats.ShTest()
+config.test_source_root = None
+config.test_exec_root = None
+

llvm/utils/lit/tests/diff-test-update.py

@@ -0,0 +1,10 @@
+# RUN: not %{lit} --update-tests -v %S/Inputs/diff-test-update | FileCheck %s
+
+# CHECK: # update-diff-test: could not deduce source and target from {{.*}}/Inputs/diff-test-update/1.in and {{.*}}/Inputs/diff-test-update/2.in
+# CHECK: # update-diff-test: could not deduce source and target from {{.*}}/diff-test-update/Output/diff-bail2.test.tmp/1.txt and {{.*}}/diff-test-update/Output/diff-bail2.test.tmp/2.txt
+# CHECK: # update-diff-test: copied {{.*}}/Output/diff-expected.test.tmp/my-file.txt to {{.*}}/Output/diff-expected.test.tmp/my-file.expected
+# CHECK: # update-diff-test: copied {{.*}}/Output/diff-tmp-dir.test.tmp/1.txt to {{.*}}/Inputs/diff-test-update/empty.txt
+# CHECK: # update-diff-test: copied {{.*}}/Inputs/diff-test-update/Output/diff-tmp.test.tmp.txt to {{.*}}/Inputs/diff-test-update/diff-t-out.txt
+
+
+# CHECK: Failed: 5 (100.00%)