CI: improve the content of results.json (#57632)

Author: IanButterworth

test/buildkitetestjson.jl

@@ -33,19 +33,20 @@ function json_repr(io::IO, val::AbstractVector; indent::Int=0)
     print(io, '[')
     for i in eachindex(val)
         print(io, '\n', ' '^(indent + 2))
-        json_repr(io, val[i]; indent=indent+2)
+        json_repr(io, val[i]; indent=indent + 2)
         i == lastindex(val) || print(io, ',')
     end
     print(io, '\n', ' '^indent, ']')
 end
 function json_repr(io::IO, val::Dict; indent::Int=0)
     print(io, '{')
+    len = length(val)
     for (i, (k, v)) in enumerate(pairs(val))
         print(io, '\n', ' '^(indent + 2))
         json_repr(io, string(k))
         print(io, ": ")
-        json_repr(io, v; indent=indent+2)
-        i === length(val) || print(io, ',')
+        json_repr(io, v; indent=indent + 2)
+        i == len || print(io, ',')
     end
     print(io, '\n', ' '^indent, '}')
 end
@@ -54,81 +55,105 @@ json_repr(io::IO, val::Any; indent::Int=0) = json_repr(io, string(val))
 # Test result processing
 
 function result_dict(testset::Test.DefaultTestSet, prefix::String="")
-    Dict{String, Any}(
+    scope = if isempty(prefix)
+        testset.description == "Overall" ? "" : testset.description
+    else
+        join((prefix, testset.description), '/')
+    end
+    data = Dict{String,Any}(
         "id" => Base.UUID(rand(UInt128)),
-        "scope" => join((prefix, testset.description), '/'),
+        "scope" => scope,
+        "tags" => Dict{String,String}(
+            "job_label" => get(ENV, "BUILDKITE_LABEL", "unknown"),
+            "job_id" => get(ENV, "BUILDKITE_JOB_ID", "unknown"),
+            "job_group" => get(ENV, "BUILDKITE_GROUP_LABEL", "unknown"),
+            "os" => string(Sys.KERNEL),
+            "arch" => string(Sys.ARCH),
+            "julia_version" => string(VERSION),
+            "testset" => testset.description,
+        ),
         "history" => if !isnothing(testset.time_end)
-            Dict{String, Any}(
+            Dict{String,Any}(
                 "start_at" => testset.time_start,
                 "end_at" => testset.time_end,
                 "duration" => testset.time_end - testset.time_start)
         else
-            Dict{String, Any}("start_at" => testset.time_start, "duration" => 0.0)
+            Dict{String,Any}("start_at" => testset.time_start, "duration" => 0.0)
         end)
+    return data
 end
 
-function result_dict(result::Test.Result)
-    file, line = if !hasproperty(result, :source) || isnothing(result.source)
-        "unknown", 0
-    else
-        something(result.source.file, "unknown"), result.source.line
-    end
-    status = if result isa Test.Pass && result.test_type === :skipped
+# Test paths on runners are often in deep directories, so just make them contain enough information
+# to be able to identify the file. Also convert Windows-style paths to Unix-style paths so tests can
+# be grouped by file.
+function generalize_file_paths(path::AbstractString)
+    pathsep = Sys.iswindows() ? '\\' : '/'
+    path = replace(path,
+        string(Sys.STDLIB, pathsep) => "",
+        string(normpath(Sys.BUILD_ROOT_PATH), pathsep) => "",
+        string(dirname(Sys.BINDIR), pathsep) => ""
+    )
+    return Sys.iswindows() ? replace(path, "\\" => "/") : path
+end
+
+# passed, failed, skipped, or unknown
+function get_status(result)
+    if result isa Test.Pass && result.test_type === :skipped
         "skipped"
+    elseif result isa Test.Broken
+        "skipped" # buildkite don't have a "broken" status
     elseif result isa Test.Pass
         "passed"
     elseif result isa Test.Fail || result isa Test.Error
         "failed"
     else
         "unknown"
     end
-    data = Dict{String, Any}(
-        "name" => "$(result.test_type): $(result.orig_expr)",
+end
+
+function result_dict(result::Test.Result)
+    file, line = if !hasproperty(result, :source) || isnothing(result.source)
+        "unknown", 0
+    else
+        something(result.source.file, "unknown"), result.source.line
+    end
+    file = generalize_file_paths(string(file))
+
+    status = get_status(result)
+
+    result_show = sprint(show, result; context=:color => false)
+    firstline = split(result_show, '\n')[1]
+    primary_reason = split(firstline, " at ")[1]
+
+    data = Dict{String,Any}(
+        "name" => "$(primary_reason). Expression: $(result.orig_expr)",
         "location" => string(file, ':', line),
         "file_name" => file,
         "result" => status)
-    add_failure_info!(data, result)
-end
 
-function add_failure_info!(data::Dict{String, Any}, result::Test.Result)
-    if result isa Test.Fail
-        data["failure_reason"] = if result.test_type === :test && !isnothing(result.data)
-            "Evaluated: $(result.data)"
-        elseif result.test_type === :test_throws_nothing
-            "No exception thrown"
-        elseif result.test_type === :test_throws_wrong
-            "Wrong exception type thrown"
-        else
-            "unknown"
-        end
-    elseif result isa Test.Error
-        data["failure_reason"] = if result.test_type === :test_error
-            if occursin("\nStacktrace:\n", result.backtrace)
-                err, trace = split(result.backtrace, "\nStacktrace:\n", limit=2)
-                data["failure_expanded"] =
-                    [Dict{String,Any}("expanded" => split(err, '\n'),
-                                      "backtrace" => split(trace, '\n'))]
-            end
-            "Exception (unexpectedly) thrown during test"
-        elseif result.test_type === :test_nonbool
-            "Expected the expression to evaluate to a Bool, not a $(typeof(result.data))"
-        elseif result.test_type === :test_unbroken
-            "Expected this test to be broken, but it passed"
+    job_label = replace(get(ENV, "BUILDKITE_LABEL", "job label not found"), r":\w+:\s*" => "")
+    if result isa Test.Fail || result isa Test.Error
+        data["failure_reason"] = generalize_file_paths(firstline) * " | $job_label"
+        err_trace = split(result_show, "\nStacktrace:\n", limit=2)
+        if length(err_trace) == 2
+            err, trace = err_trace
+            data["failure_expanded"] = [Dict{String,Any}("expanded" => split(err, '\n'), "backtrace" => split(trace, '\n'))]
         else
-            "unknown"
+            data["failure_expanded"] = [Dict{String,Any}("expanded" => split(result_show, '\n'), "backtrace" => [])]
         end
     end
-    data
+    return data
 end
 
-function collect_results!(results::Vector{Dict{String, Any}}, testset::Test.DefaultTestSet, prefix::String="")
+function collect_results!(results::Vector{Dict{String,Any}}, testset::Test.DefaultTestSet, prefix::String="")
     common_data = result_dict(testset, prefix)
     result_offset = length(results) + 1
-    result_counts = Dict{Tuple{String, String}, Int}()
+    result_counts = Dict{Tuple{String,String},Int}()
+    get_rid(rdata) = (rdata["location"], rdata["result"])
     for (i, result) in enumerate(testset.results)
         if result isa Test.Result
             rdata = result_dict(result)
-            rid = (rdata["location"], rdata["result"])
+            rid = get_rid(rdata)
             if haskey(result_counts, rid)
                 result_counts[rid] += 1
             else
@@ -142,17 +167,17 @@ function collect_results!(results::Vector{Dict{String, Any}}, testset::Test.Defa
     # Modify names to hold `result_counts`
     for i in result_offset:length(results)
         result = results[i]
-        rid = (result["location"], result["result"])
+        rid = get_rid(result)
         if get(result_counts, rid, 0) > 1
             result["name"] = replace(result["name"], r"^([^:]):" =>
                 SubstitutionString("\\1 (x$(result_counts[rid])):"))
         end
     end
-    results
+    return results
 end
 
 function write_testset_json_files(dir::String, testset::Test.DefaultTestSet)
-    data = Dict{String, Any}[]
+    data = Dict{String,Any}[]
     collect_results!(data, testset)
     files = String[]
     # Buildkite is limited to 5000 results per file https://buildkite.com/docs/test-analytics/importing-json