From 875494586eaf715a9fc2763fd75c95fb07ace9e8 Mon Sep 17 00:00:00 2001 From: Tomas Date: Mon, 28 Mar 2022 20:12:13 +0200 Subject: [PATCH 01/17] Move copying of merged wrapper native components to copynative step --- src/tests/Directory.Build.targets | 30 ++++++++++++++---------------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/src/tests/Directory.Build.targets b/src/tests/Directory.Build.targets index a11b397670dab2..4e21dbc31b875a 100644 --- a/src/tests/Directory.Build.targets +++ b/src/tests/Directory.Build.targets @@ -114,21 +114,6 @@ true - - - - - - - - - @@ -227,7 +212,20 @@ - + + + + + + + + + From 28d41717572cb72fa3631311dbebc7faf890e7a2 Mon Sep 17 00:00:00 2001 From: Tomas Date: Mon, 28 Mar 2022 20:49:41 +0200 Subject: [PATCH 02/17] Move export of test exclusion file before ILLink changing CORE_ROOT --- src/tests/Common/CLRTest.Execute.Bash.targets | 3 +-- src/tests/Common/CLRTest.Execute.Batch.targets | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/src/tests/Common/CLRTest.Execute.Bash.targets b/src/tests/Common/CLRTest.Execute.Bash.targets index fe56a45f80f244..961f10eb0e566b 100644 --- a/src/tests/Common/CLRTest.Execute.Bash.targets +++ b/src/tests/Common/CLRTest.Execute.Bash.targets @@ -306,8 +306,6 @@ $__Command msbuild $CORE_ROOT/wasm-test-runner/WasmTestRunner.proj /p:NetCoreApp Date: Mon, 28 Mar 2022 23:50:49 +0200 Subject: [PATCH 03/17] Fix file mode for test execution bash scripts --- src/tests/Common/CLRTest.Execute.Bash.targets | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/tests/Common/CLRTest.Execute.Bash.targets b/src/tests/Common/CLRTest.Execute.Bash.targets index 961f10eb0e566b..d2d04b567885d5 100644 --- a/src/tests/Common/CLRTest.Execute.Bash.targets +++ b/src/tests/Common/CLRTest.Execute.Bash.targets @@ -560,15 +560,23 @@ $(BashCLRTestExitCodeCheck) + + $(OutputPath)/$(MSBuildProjectName).sh + + + endings before running the scripts on Unix platforms. In our current lab + infra it shouldn't really matter as the execution scripts are regenerated + in the 'test run' phase before sending the items to Helix i.o.w. at + the point where we already know the exact targeting platform. --> + From 72d1ab8ab06f736bcde0df105f852f80c1dc04a6 Mon Sep 17 00:00:00 2001 From: Tomas Date: Tue, 29 Mar 2022 01:53:59 +0200 Subject: [PATCH 04/17] Add out-of-proc test folders to Helix work item payloads --- src/tests/Common/helixpublishwitharcade.proj | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/tests/Common/helixpublishwitharcade.proj b/src/tests/Common/helixpublishwitharcade.proj index c1363a55d87fe8..fc8318de8d84bc 100644 --- a/src/tests/Common/helixpublishwitharcade.proj +++ b/src/tests/Common/helixpublishwitharcade.proj @@ -298,10 +298,15 @@ <_MergedWrapperName>%(_MergedWrapperRunScript.FileName) <_MergedWrapperRunScriptRelative Condition="'%(_MergedWrapperRunScript.Identity)' != ''">$([System.IO.Path]::GetRelativePath($(TestBinDir), %(_MergedWrapperRunScript.FullPath))) + + <_MergedWrapperOutOfProcessTestMarkers Include="$(_MergedWrapperDirectory)/**/*.OutOfProcessTest" /> + <_MergedWrapperOutOfProcessTestFiles Include="%(_MergedWrapperOutOfProcessTestMarkers.RootDir)%(_MergedWrapperOutOfProcessTestMarkers.Directory)/**/*" /> + <_MergedPayloadGroups Include="$(_MergedWrapperName)" /> <_MergedPayloadFiles Include="$(_MergedWrapperDirectory)**" /> + <_MergedPayloadFiles Include="@(_MergedWrapperOutOfProcessTestFiles)" /> <_MergedPayloadFiles Update="@(_MergedPayloadFiles)"> From db296bbaa7e6122ea819eca5c20c22bb5c9247e9 Mon Sep 17 00:00:00 2001 From: Tomas Date: Tue, 29 Mar 2022 16:09:16 +0200 Subject: [PATCH 05/17] Fix Helix publishing in the presence of out-of-process tests --- src/tests/Common/helixpublishwitharcade.proj | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/tests/Common/helixpublishwitharcade.proj b/src/tests/Common/helixpublishwitharcade.proj index fc8318de8d84bc..64c54d22f840b2 100644 --- a/src/tests/Common/helixpublishwitharcade.proj +++ b/src/tests/Common/helixpublishwitharcade.proj @@ -294,18 +294,21 @@ <_MergedWrapperRunScript Include="$([System.IO.Path]::ChangeExtension('%(_MergedWrapperMarker.Identity)', '.$(TestScriptExtension)'))" /> - <_MergedWrapperDirectory>%(_MergedWrapperRunScript.RootDir)%(Directory) + <_MergedWrapperDirectory>$([System.IO.Path]::GetDirectoryName('%(_MergedWrapperRunScript.Identity)')) + <_MergedWrapperParentDirectory>$([System.IO.Path]::GetDirectoryName('$(_MergedWrapperDirectory)')) <_MergedWrapperName>%(_MergedWrapperRunScript.FileName) <_MergedWrapperRunScriptRelative Condition="'%(_MergedWrapperRunScript.Identity)' != ''">$([System.IO.Path]::GetRelativePath($(TestBinDir), %(_MergedWrapperRunScript.FullPath))) - <_MergedWrapperOutOfProcessTestMarkers Include="$(_MergedWrapperDirectory)/**/*.OutOfProcessTest" /> - <_MergedWrapperOutOfProcessTestFiles Include="%(_MergedWrapperOutOfProcessTestMarkers.RootDir)%(_MergedWrapperOutOfProcessTestMarkers.Directory)/**/*" /> + <_MergedWrapperOutOfProcessTestMarkers Include="$(_MergedWrapperParentDirectory)/**/*.OutOfProcessTest" /> + <_MergedWrapperOutOfProcessTestFiles + Include="%(_MergedWrapperOutOfProcessTestMarkers.RootDir)%(_MergedWrapperOutOfProcessTestMarkers.Directory)/**" + Condition="'@(_MergedWrapperOutOfProcessTestMarkers)' != ''" /> <_MergedPayloadGroups Include="$(_MergedWrapperName)" /> - <_MergedPayloadFiles Include="$(_MergedWrapperDirectory)**" /> + <_MergedPayloadFiles Include="$(_MergedWrapperDirectory)/**" /> <_MergedPayloadFiles Include="@(_MergedWrapperOutOfProcessTestFiles)" /> <_MergedPayloadFiles Update="@(_MergedPayloadFiles)"> - + From b5428f5aadcc4eb8f5894b6078b213e96de75b41 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Tue, 29 Mar 2022 12:07:20 -0700 Subject: [PATCH 06/17] Use a marker file triggered off of a project property to determine which files to skip running the Mono AOT compiler on (some tests fail at AOT time, so we need this to work around that limitation) --- src/tests/Directory.Build.targets | 28 ++++++++++++++++--- .../ForeignThreadExceptions.csproj | 1 + ...ling_Disabled_NativeAssemblyEnabled.csproj | 1 + .../SuppressGCTransitionTest.csproj | 1 + .../UnmanagedCallersOnlyTest.csproj | 1 + .../Directed/callconv/Directory.Build.props | 7 +++++ .../General/Vector256_1/Vector256_1_r.csproj | 1 + .../ELEMENT_TYPE_IU/u_conv_il_r.ilproj | 2 +- .../deadoponerrorinfunclet_il_d.ilproj | 2 +- .../deadoponerrorinfunclet_il_r.ilproj | 2 +- .../flowgraph/dev10_bug679008/helper.ilproj | 1 + .../CodeQuality/Roslyn/CscBench.csproj | 1 + .../jit64/localloc/ehverify/eh07_large.ilproj | 1 + .../bigvtbl/Directory.Build.props | 7 +++++ .../UnitTest/UnitTestMultiModule.ilproj | 1 + .../classloader/RefFields/Validate.csproj | 1 + .../NestedStructs/case03.csproj | 1 + .../NestedStructs/case04.csproj | 1 + .../NestedStructs/case05.csproj | 1 + .../Directory.Build.props | 7 +++++ .../generics/Variance/IL/Unbox002.csproj | 1 + .../generics/Variance/IL/Unbox003.csproj | 1 + .../generics/Variance/IL/vsw543506.csproj | 1 + .../typeequivalence/simple/Simple.csproj | 1 + src/tests/build.proj | 8 ++++-- 25 files changed, 71 insertions(+), 9 deletions(-) create mode 100644 src/tests/JIT/Directed/callconv/Directory.Build.props create mode 100644 src/tests/JIT/opt/virtualstubdispatch/bigvtbl/Directory.Build.props create mode 100644 src/tests/Loader/classloader/explicitlayout/objrefandnonobjrefoverlap/Directory.Build.props diff --git a/src/tests/Directory.Build.targets b/src/tests/Directory.Build.targets index 4e21dbc31b875a..86f3956a5e9376 100644 --- a/src/tests/Directory.Build.targets +++ b/src/tests/Directory.Build.targets @@ -257,20 +257,40 @@ - + + + + + + + + + File="$(IntermediateOutputPath)\$(MSBuildProjectName).MergedTestAssembly" + Lines="MergedTestAssembly" + Overwrite="true" + WriteOnlyWhenDifferent="true" /> + + + + + Lines="OutOfProcessTest" + Overwrite="true" + WriteOnlyWhenDifferent="true" /> diff --git a/src/tests/Exceptions/ForeignThread/ForeignThreadExceptions.csproj b/src/tests/Exceptions/ForeignThread/ForeignThreadExceptions.csproj index a4e1b468a2615c..d282ce1b519196 100644 --- a/src/tests/Exceptions/ForeignThread/ForeignThreadExceptions.csproj +++ b/src/tests/Exceptions/ForeignThread/ForeignThreadExceptions.csproj @@ -1,6 +1,7 @@ Exe + true diff --git a/src/tests/Interop/DisabledRuntimeMarshalling/DisabledRuntimeMarshalling_Disabled_NativeAssemblyEnabled.csproj b/src/tests/Interop/DisabledRuntimeMarshalling/DisabledRuntimeMarshalling_Disabled_NativeAssemblyEnabled.csproj index b8914d641b733b..23d37e11f3adf6 100644 --- a/src/tests/Interop/DisabledRuntimeMarshalling/DisabledRuntimeMarshalling_Disabled_NativeAssemblyEnabled.csproj +++ b/src/tests/Interop/DisabledRuntimeMarshalling/DisabledRuntimeMarshalling_Disabled_NativeAssemblyEnabled.csproj @@ -1,6 +1,7 @@ true + true diff --git a/src/tests/Interop/SuppressGCTransition/SuppressGCTransitionTest.csproj b/src/tests/Interop/SuppressGCTransition/SuppressGCTransitionTest.csproj index abb1fec49b7226..c08644530f62f5 100644 --- a/src/tests/Interop/SuppressGCTransition/SuppressGCTransitionTest.csproj +++ b/src/tests/Interop/SuppressGCTransition/SuppressGCTransitionTest.csproj @@ -2,6 +2,7 @@ Exe True + true diff --git a/src/tests/Interop/UnmanagedCallersOnly/UnmanagedCallersOnlyTest.csproj b/src/tests/Interop/UnmanagedCallersOnly/UnmanagedCallersOnlyTest.csproj index d7246c35374fcc..5a5439c4127a50 100644 --- a/src/tests/Interop/UnmanagedCallersOnly/UnmanagedCallersOnlyTest.csproj +++ b/src/tests/Interop/UnmanagedCallersOnly/UnmanagedCallersOnlyTest.csproj @@ -2,6 +2,7 @@ Exe true + true diff --git a/src/tests/JIT/Directed/callconv/Directory.Build.props b/src/tests/JIT/Directed/callconv/Directory.Build.props new file mode 100644 index 00000000000000..49b99b7a101933 --- /dev/null +++ b/src/tests/JIT/Directed/callconv/Directory.Build.props @@ -0,0 +1,7 @@ + + + + + true + + \ No newline at end of file diff --git a/src/tests/JIT/HardwareIntrinsics/General/Vector256_1/Vector256_1_r.csproj b/src/tests/JIT/HardwareIntrinsics/General/Vector256_1/Vector256_1_r.csproj index 2e1ffa2dcc1c7d..e1a746da5ca68b 100644 --- a/src/tests/JIT/HardwareIntrinsics/General/Vector256_1/Vector256_1_r.csproj +++ b/src/tests/JIT/HardwareIntrinsics/General/Vector256_1/Vector256_1_r.csproj @@ -2,6 +2,7 @@ Exe true + true Embedded diff --git a/src/tests/JIT/Methodical/ELEMENT_TYPE_IU/u_conv_il_r.ilproj b/src/tests/JIT/Methodical/ELEMENT_TYPE_IU/u_conv_il_r.ilproj index 6dbbc645a2d318..85643608c575f3 100644 --- a/src/tests/JIT/Methodical/ELEMENT_TYPE_IU/u_conv_il_r.ilproj +++ b/src/tests/JIT/Methodical/ELEMENT_TYPE_IU/u_conv_il_r.ilproj @@ -1,6 +1,6 @@ - Exe + true PdbOnly diff --git a/src/tests/JIT/Methodical/eh/deadcode/deadoponerrorinfunclet_il_d.ilproj b/src/tests/JIT/Methodical/eh/deadcode/deadoponerrorinfunclet_il_d.ilproj index 68ec24f2a5d6bc..672ad9b544dd29 100644 --- a/src/tests/JIT/Methodical/eh/deadcode/deadoponerrorinfunclet_il_d.ilproj +++ b/src/tests/JIT/Methodical/eh/deadcode/deadoponerrorinfunclet_il_d.ilproj @@ -1,6 +1,6 @@ - Exe + true Full diff --git a/src/tests/JIT/Methodical/eh/deadcode/deadoponerrorinfunclet_il_r.ilproj b/src/tests/JIT/Methodical/eh/deadcode/deadoponerrorinfunclet_il_r.ilproj index b0ef594900714b..85b1aa395cdcd7 100644 --- a/src/tests/JIT/Methodical/eh/deadcode/deadoponerrorinfunclet_il_r.ilproj +++ b/src/tests/JIT/Methodical/eh/deadcode/deadoponerrorinfunclet_il_r.ilproj @@ -1,6 +1,6 @@ - Exe + true PdbOnly diff --git a/src/tests/JIT/Methodical/flowgraph/dev10_bug679008/helper.ilproj b/src/tests/JIT/Methodical/flowgraph/dev10_bug679008/helper.ilproj index 790327597861e5..fe39c536a46d78 100644 --- a/src/tests/JIT/Methodical/flowgraph/dev10_bug679008/helper.ilproj +++ b/src/tests/JIT/Methodical/flowgraph/dev10_bug679008/helper.ilproj @@ -3,6 +3,7 @@ Library BuildOnly false + true Full diff --git a/src/tests/JIT/Performance/CodeQuality/Roslyn/CscBench.csproj b/src/tests/JIT/Performance/CodeQuality/Roslyn/CscBench.csproj index f9c3ccf6806776..33d8c00a50546c 100644 --- a/src/tests/JIT/Performance/CodeQuality/Roslyn/CscBench.csproj +++ b/src/tests/JIT/Performance/CodeQuality/Roslyn/CscBench.csproj @@ -3,6 +3,7 @@ Exe true $(NoWarn);xUnit1013 + true pdbonly diff --git a/src/tests/JIT/jit64/localloc/ehverify/eh07_large.ilproj b/src/tests/JIT/jit64/localloc/ehverify/eh07_large.ilproj index 3339b7ce992bec..d8e88df212c2d4 100644 --- a/src/tests/JIT/jit64/localloc/ehverify/eh07_large.ilproj +++ b/src/tests/JIT/jit64/localloc/ehverify/eh07_large.ilproj @@ -2,6 +2,7 @@ Exe true + true PdbOnly diff --git a/src/tests/JIT/opt/virtualstubdispatch/bigvtbl/Directory.Build.props b/src/tests/JIT/opt/virtualstubdispatch/bigvtbl/Directory.Build.props new file mode 100644 index 00000000000000..fb357e9d5c42ed --- /dev/null +++ b/src/tests/JIT/opt/virtualstubdispatch/bigvtbl/Directory.Build.props @@ -0,0 +1,7 @@ + + + + + true + + \ No newline at end of file diff --git a/src/tests/Loader/classloader/MethodImpl/CovariantReturns/UnitTest/UnitTestMultiModule.ilproj b/src/tests/Loader/classloader/MethodImpl/CovariantReturns/UnitTest/UnitTestMultiModule.ilproj index 5fca8b3905421c..661a9dbdf808f2 100644 --- a/src/tests/Loader/classloader/MethodImpl/CovariantReturns/UnitTest/UnitTestMultiModule.ilproj +++ b/src/tests/Loader/classloader/MethodImpl/CovariantReturns/UnitTest/UnitTestMultiModule.ilproj @@ -1,6 +1,7 @@ Exe + true diff --git a/src/tests/Loader/classloader/RefFields/Validate.csproj b/src/tests/Loader/classloader/RefFields/Validate.csproj index 89e9243940878a..d25be953ff91f4 100644 --- a/src/tests/Loader/classloader/RefFields/Validate.csproj +++ b/src/tests/Loader/classloader/RefFields/Validate.csproj @@ -2,6 +2,7 @@ true Exe + true diff --git a/src/tests/Loader/classloader/explicitlayout/NestedStructs/case03.csproj b/src/tests/Loader/classloader/explicitlayout/NestedStructs/case03.csproj index 7308df24c65b72..09a937da38315d 100644 --- a/src/tests/Loader/classloader/explicitlayout/NestedStructs/case03.csproj +++ b/src/tests/Loader/classloader/explicitlayout/NestedStructs/case03.csproj @@ -2,6 +2,7 @@ Exe enable + true diff --git a/src/tests/Loader/classloader/explicitlayout/NestedStructs/case04.csproj b/src/tests/Loader/classloader/explicitlayout/NestedStructs/case04.csproj index 6b6ec60a598c27..f9048994f863ee 100644 --- a/src/tests/Loader/classloader/explicitlayout/NestedStructs/case04.csproj +++ b/src/tests/Loader/classloader/explicitlayout/NestedStructs/case04.csproj @@ -2,6 +2,7 @@ Exe enable + true diff --git a/src/tests/Loader/classloader/explicitlayout/NestedStructs/case05.csproj b/src/tests/Loader/classloader/explicitlayout/NestedStructs/case05.csproj index e84314924ecab0..1540275888e063 100644 --- a/src/tests/Loader/classloader/explicitlayout/NestedStructs/case05.csproj +++ b/src/tests/Loader/classloader/explicitlayout/NestedStructs/case05.csproj @@ -2,6 +2,7 @@ Exe enable + true diff --git a/src/tests/Loader/classloader/explicitlayout/objrefandnonobjrefoverlap/Directory.Build.props b/src/tests/Loader/classloader/explicitlayout/objrefandnonobjrefoverlap/Directory.Build.props new file mode 100644 index 00000000000000..fb357e9d5c42ed --- /dev/null +++ b/src/tests/Loader/classloader/explicitlayout/objrefandnonobjrefoverlap/Directory.Build.props @@ -0,0 +1,7 @@ + + + + + true + + \ No newline at end of file diff --git a/src/tests/Loader/classloader/generics/Variance/IL/Unbox002.csproj b/src/tests/Loader/classloader/generics/Variance/IL/Unbox002.csproj index d28c31d472d2c8..4f0c61ddca2933 100644 --- a/src/tests/Loader/classloader/generics/Variance/IL/Unbox002.csproj +++ b/src/tests/Loader/classloader/generics/Variance/IL/Unbox002.csproj @@ -2,6 +2,7 @@ true Exe + true diff --git a/src/tests/Loader/classloader/generics/Variance/IL/Unbox003.csproj b/src/tests/Loader/classloader/generics/Variance/IL/Unbox003.csproj index efa4c2dd1a2c79..572143f323f037 100644 --- a/src/tests/Loader/classloader/generics/Variance/IL/Unbox003.csproj +++ b/src/tests/Loader/classloader/generics/Variance/IL/Unbox003.csproj @@ -2,6 +2,7 @@ true Exe + true diff --git a/src/tests/Loader/classloader/generics/Variance/IL/vsw543506.csproj b/src/tests/Loader/classloader/generics/Variance/IL/vsw543506.csproj index bb8b2efa6467d9..9003f0a82135d5 100644 --- a/src/tests/Loader/classloader/generics/Variance/IL/vsw543506.csproj +++ b/src/tests/Loader/classloader/generics/Variance/IL/vsw543506.csproj @@ -2,6 +2,7 @@ true Exe + true diff --git a/src/tests/baseservices/typeequivalence/simple/Simple.csproj b/src/tests/baseservices/typeequivalence/simple/Simple.csproj index e8bb27e9d97873..37e57397b2c53a 100644 --- a/src/tests/baseservices/typeequivalence/simple/Simple.csproj +++ b/src/tests/baseservices/typeequivalence/simple/Simple.csproj @@ -2,6 +2,7 @@ Exe True + true diff --git a/src/tests/build.proj b/src/tests/build.proj index 593b4556e45c4e..a226aa1e44aafe 100644 --- a/src/tests/build.proj +++ b/src/tests/build.proj @@ -109,7 +109,9 @@ - + + + @@ -121,7 +123,7 @@ - + @@ -393,6 +395,8 @@ + + From c27b8bf120bd8c1a96db9feaa54375c719a4d1ea Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Tue, 29 Mar 2022 15:08:27 -0700 Subject: [PATCH 07/17] Pass runtimeVariant to the Core_Root construction for the test exclusion list creation --- eng/pipelines/common/templates/runtimes/run-test-job.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/common/templates/runtimes/run-test-job.yml b/eng/pipelines/common/templates/runtimes/run-test-job.yml index 544e6c9ed29c2a..03288003d85f02 100644 --- a/eng/pipelines/common/templates/runtimes/run-test-job.yml +++ b/eng/pipelines/common/templates/runtimes/run-test-job.yml @@ -300,7 +300,7 @@ jobs: # Compose the Core_Root folder containing all artifacts needed for running # CoreCLR tests. This step also compiles the framework using Crossgen / Crossgen2 # in ReadyToRun jobs. - - script: $(Build.SourcesDirectory)/src/tests/build$(scriptExt) generatelayoutonly $(logRootNameArg)Layout $(runtimeFlavorArgs) $(crossgenArg) $(buildConfig) $(archType) $(crossArg) $(priorityArg) $(librariesOverrideArg) + - script: $(Build.SourcesDirectory)/src/tests/build$(scriptExt) generatelayoutonly $(logRootNameArg)Layout $(runtimeFlavorArgs) $(crossgenArg) $(buildConfig) $(archType) $(crossArg) $(priorityArg) $(librariesOverrideArg) $(runtimeVariantArg) displayName: Generate CORE_ROOT # Build a Mono LLVM AOT cross-compiler for non-amd64 targets (in this case, just arm64) From 36fb2f3da0de408375006d1acad94dd10761b106 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Tue, 29 Mar 2022 15:27:06 -0700 Subject: [PATCH 08/17] Don't write the output element when there's no test output. This causes issues with Helix test uploading. --- src/tests/Common/XUnitWrapperLibrary/TestSummary.cs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/tests/Common/XUnitWrapperLibrary/TestSummary.cs b/src/tests/Common/XUnitWrapperLibrary/TestSummary.cs index a2f23bad9bbf8f..0de2af1ec838bf 100644 --- a/src/tests/Common/XUnitWrapperLibrary/TestSummary.cs +++ b/src/tests/Common/XUnitWrapperLibrary/TestSummary.cs @@ -70,9 +70,10 @@ public string GetTestResultOutput(string assemblyName) foreach (var test in _testResults) { resultsFile.Append($@"" : string.Empty; if (test.Exception is not null) { - resultsFile.AppendLine($@"result=""Fail"">"); + resultsFile.AppendLine($@"result=""Fail"">{outputElement}"); } else if (test.SkipReason is not null) { @@ -80,7 +81,7 @@ public string GetTestResultOutput(string assemblyName) } else { - resultsFile.AppendLine($@" result=""Pass"">"); + resultsFile.AppendLine($@" result=""Pass"">{outputElement}"); } } From c0aeb0ed24265b2583888325f29bc32cf236d9cd Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Tue, 29 Mar 2022 15:47:14 -0700 Subject: [PATCH 09/17] Write xunit results the way xharness expects to read them for wasm so we correctly report failures and don't report a test harness failure. --- .../GeneratedTestRunner.cs | 18 ++++++++++++++++-- .../XHarnessRunnerLibrary/RunnerEntryPoint.cs | 12 +++++++++--- 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/src/tests/Common/XHarnessRunnerLibrary/GeneratedTestRunner.cs b/src/tests/Common/XHarnessRunnerLibrary/GeneratedTestRunner.cs index 70f9b7a1405878..f47b0b8cad795b 100644 --- a/src/tests/Common/XHarnessRunnerLibrary/GeneratedTestRunner.cs +++ b/src/tests/Common/XHarnessRunnerLibrary/GeneratedTestRunner.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.Diagnostics; using System.IO; +using System.Text; using System.Threading.Tasks; using Microsoft.DotNet.XHarness.Common; using Microsoft.DotNet.XHarness.TestRunners.Common; @@ -15,16 +16,20 @@ public sealed class GeneratedTestRunner : TestRunner TestFilter.ISearchClause? _filter; Func _runTestsCallback; HashSet _testExclusionList; + private readonly Boolean _writeBase64TestResults; + public GeneratedTestRunner( LogWriter logger, Func runTestsCallback, string assemblyName, - HashSet testExclusionList) + HashSet testExclusionList, + bool writeBase64TestResults) :base(logger) { _assemblyName = assemblyName; _runTestsCallback = runTestsCallback; _testExclusionList = testExclusionList; + _writeBase64TestResults = writeBase64TestResults; ResultsFileName = $"{_assemblyName}.testResults.xml"; } @@ -53,7 +58,16 @@ public override string WriteResultsToFile(XmlResultJargon xmlResultJargon) public override void WriteResultsToFile(TextWriter writer, XmlResultJargon jargon) { Debug.Assert(jargon == XmlResultJargon.xUnit); - writer.WriteLine(LastTestRun.GetTestResultOutput(_assemblyName)); + string lastTestResults = LastTestRun.GetTestResultOutput(_assemblyName); + if (_writeBase64TestResults) + { + byte[] base64Results = Convert.ToBase64String(lastTestResults); + writer.WriteLine($"STARTRESULTXML {Encoding.UTF16.GetByteCount(lastTestResults)} {base64Results} ENDRESULTXML"); + } + else + { + writer.WriteLine(lastTestResults); + } } public override void SkipTests(IEnumerable tests) diff --git a/src/tests/Common/XHarnessRunnerLibrary/RunnerEntryPoint.cs b/src/tests/Common/XHarnessRunnerLibrary/RunnerEntryPoint.cs index bf1e624e5b8357..b1b1f4e44e72ef 100644 --- a/src/tests/Common/XHarnessRunnerLibrary/RunnerEntryPoint.cs +++ b/src/tests/Common/XHarnessRunnerLibrary/RunnerEntryPoint.cs @@ -36,6 +36,12 @@ public static async Task RunTests( bool anyFailedTests = false; entryPoint.TestsCompleted += (o, e) => anyFailedTests = e.FailedTests > 0; await entryPoint.RunAsync(); + + if (OperatingSystem.IsBrowser()) + { + // Browser expects all xharness processes to exit with 0, even in case of failure + return 0; + } return anyFailedTests ? 1 : 0; } @@ -65,7 +71,7 @@ public AppleEntryPoint( protected override bool IsXunit => true; protected override TestRunner GetTestRunner(LogWriter logWriter) { - var runner = new GeneratedTestRunner(logWriter, _runTestsCallback, _assemblyName, _testExclusionList); + var runner = new GeneratedTestRunner(logWriter, _runTestsCallback, _assemblyName, _testExclusionList, writeBase64TestResults: true); if (_methodNameToRun is not null) { runner.SkipMethod(_methodNameToRun, isExcluded: false); @@ -103,7 +109,7 @@ public AndroidEntryPoint( protected override bool IsXunit => true; protected override TestRunner GetTestRunner(LogWriter logWriter) { - var runner = new GeneratedTestRunner(logWriter, _runTestsCallback, _assemblyName, _testExclusionList); + var runner = new GeneratedTestRunner(logWriter, _runTestsCallback, _assemblyName, _testExclusionList, writeBase64TestResults: false); if (_methodNameToRun is not null) { runner.SkipMethod(_methodNameToRun, isExcluded: false); @@ -151,7 +157,7 @@ public WasmEntryPoint( protected override bool IsXunit => true; protected override TestRunner GetTestRunner(LogWriter logWriter) { - var runner = new GeneratedTestRunner(logWriter, _runTestsCallback, _assemblyName, _testExclusionList); + var runner = new GeneratedTestRunner(logWriter, _runTestsCallback, _assemblyName, _testExclusionList, writeBase64TestResults: true); if (_methodNameToRun is not null) { runner.SkipMethod(_methodNameToRun, isExcluded: false); From 7b9f40b78bdd591459818b498df905d13bc109f2 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Tue, 29 Mar 2022 18:00:25 -0700 Subject: [PATCH 10/17] Fix GeneratedTestRunner to build. --- .../Common/XHarnessRunnerLibrary/GeneratedTestRunner.cs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/tests/Common/XHarnessRunnerLibrary/GeneratedTestRunner.cs b/src/tests/Common/XHarnessRunnerLibrary/GeneratedTestRunner.cs index f47b0b8cad795b..50b21605a74319 100644 --- a/src/tests/Common/XHarnessRunnerLibrary/GeneratedTestRunner.cs +++ b/src/tests/Common/XHarnessRunnerLibrary/GeneratedTestRunner.cs @@ -61,8 +61,9 @@ public override void WriteResultsToFile(TextWriter writer, XmlResultJargon jargo string lastTestResults = LastTestRun.GetTestResultOutput(_assemblyName); if (_writeBase64TestResults) { - byte[] base64Results = Convert.ToBase64String(lastTestResults); - writer.WriteLine($"STARTRESULTXML {Encoding.UTF16.GetByteCount(lastTestResults)} {base64Results} ENDRESULTXML"); + byte[] encodedBytes = Encoding.Unicode.GetBytes(lastTestResults); + string base64Results = Convert.ToBase64String(encodedBytes); + writer.WriteLine($"STARTRESULTXML {Encoding.UTF16.GetByteCount(encodedBytes.Length)} {base64Results} ENDRESULTXML"); } else { From 1805466bc7393d93067f8948da2b3879c277d933 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Wed, 30 Mar 2022 09:23:58 -0700 Subject: [PATCH 11/17] Fix typo --- src/tests/Common/XHarnessRunnerLibrary/GeneratedTestRunner.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tests/Common/XHarnessRunnerLibrary/GeneratedTestRunner.cs b/src/tests/Common/XHarnessRunnerLibrary/GeneratedTestRunner.cs index 50b21605a74319..c30c99f7e33061 100644 --- a/src/tests/Common/XHarnessRunnerLibrary/GeneratedTestRunner.cs +++ b/src/tests/Common/XHarnessRunnerLibrary/GeneratedTestRunner.cs @@ -63,7 +63,7 @@ public override void WriteResultsToFile(TextWriter writer, XmlResultJargon jargo { byte[] encodedBytes = Encoding.Unicode.GetBytes(lastTestResults); string base64Results = Convert.ToBase64String(encodedBytes); - writer.WriteLine($"STARTRESULTXML {Encoding.UTF16.GetByteCount(encodedBytes.Length)} {base64Results} ENDRESULTXML"); + writer.WriteLine($"STARTRESULTXML {encodedBytes.Length} {base64Results} ENDRESULTXML"); } else { From 164a39cac7e90f75b688bdc12a5ae468c0135d58 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Wed, 30 Mar 2022 11:35:27 -0700 Subject: [PATCH 12/17] Fix copying native wrappers for merged test runners. Pre-emptively mark another test as RequiresProcessIsolation as I stumbled upon it while fixing this --- src/tests/Common/testgrouping.proj | 36 ------------------ src/tests/Directory.Build.targets | 16 +++++++- .../pinning/object-pin/CMakeLists.txt | 8 ---- src/tests/JIT/Directed/pinvoke/CMakeLists.txt | 8 ---- .../structs/systemvbringup/CMakeLists.txt | 8 ---- .../V1.2-M02/b108129/CMakeLists.txt | 7 ---- .../JIT/jit64/hfa/main/dll/CMakeLists.txt | 15 -------- .../JIT/jit64/mcc/interop/CMakeLists.txt | 38 ------------------- .../Loader/NativeLibs/FromNativePaths.csproj | 1 + 9 files changed, 16 insertions(+), 121 deletions(-) diff --git a/src/tests/Common/testgrouping.proj b/src/tests/Common/testgrouping.proj index 6c32c7f7e61953..c45c5985364a8b 100644 --- a/src/tests/Common/testgrouping.proj +++ b/src/tests/Common/testgrouping.proj @@ -180,42 +180,6 @@ $(TestBinDir)JIT\HardwareIntrinsics\JIT.HardwareIntrinsics.XUnitWrapper.dll - - JIT.Methodical.a-dA-D - $(TestBinDir)JIT\Methodical\JIT.Methodical.XUnitWrapper.dll - - - - JIT.Methodical.eh - $(TestBinDir)JIT\Methodical\JIT.Methodical.XUnitWrapper.dll - - - - JIT.Methodical.eE - $(TestBinDir)JIT\Methodical\JIT.Methodical.XUnitWrapper.dll - - - - JIT.Methodical.f-iF-I - $(TestBinDir)JIT\Methodical\JIT.Methodical.XUnitWrapper.dll - - diff --git a/src/tests/Directory.Build.targets b/src/tests/Directory.Build.targets index 86f3956a5e9376..3960392d5bb228 100644 --- a/src/tests/Directory.Build.targets +++ b/src/tests/Directory.Build.targets @@ -114,6 +114,21 @@ true + + + + + + + + + @@ -209,7 +224,6 @@ - true 1 + true From e2fe4ec16a27a97d3d47772bda002380ab2c2763 Mon Sep 17 00:00:00 2001 From: Tomas Date: Wed, 30 Mar 2022 23:25:56 +0200 Subject: [PATCH 13/17] Fix copying of native test components to merged wrapper outputs Due to Pri0 / Pri1 test grouping we may end up building the wrapper in a different group than its components. Make sure that we populate all native components of merged wrapper dependencies before we copy them over to the merged wrapper output folder. Thanks Tomas --- src/tests/Common/Directory.Build.targets | 3 +++ src/tests/Directory.Build.targets | 20 +++++--------------- 2 files changed, 8 insertions(+), 15 deletions(-) diff --git a/src/tests/Common/Directory.Build.targets b/src/tests/Common/Directory.Build.targets index 4f31cd9fc7646c..79d1b9253c57b8 100644 --- a/src/tests/Common/Directory.Build.targets +++ b/src/tests/Common/Directory.Build.targets @@ -191,4 +191,7 @@ + + + diff --git a/src/tests/Directory.Build.targets b/src/tests/Directory.Build.targets index 3960392d5bb228..96d65ab0eabf50 100644 --- a/src/tests/Directory.Build.targets +++ b/src/tests/Directory.Build.targets @@ -114,21 +114,6 @@ true - - - - - - - - - @@ -224,6 +209,11 @@ + + Date: Thu, 31 Mar 2022 19:52:29 +0200 Subject: [PATCH 14/17] Revert changes to testgrouping.proj per Jeremy's PR feedback --- src/tests/Common/testgrouping.proj | 36 ++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/src/tests/Common/testgrouping.proj b/src/tests/Common/testgrouping.proj index c45c5985364a8b..6c32c7f7e61953 100644 --- a/src/tests/Common/testgrouping.proj +++ b/src/tests/Common/testgrouping.proj @@ -180,6 +180,42 @@ $(TestBinDir)JIT\HardwareIntrinsics\JIT.HardwareIntrinsics.XUnitWrapper.dll + + JIT.Methodical.a-dA-D + $(TestBinDir)JIT\Methodical\JIT.Methodical.XUnitWrapper.dll + + + + JIT.Methodical.eh + $(TestBinDir)JIT\Methodical\JIT.Methodical.XUnitWrapper.dll + + + + JIT.Methodical.eE + $(TestBinDir)JIT\Methodical\JIT.Methodical.XUnitWrapper.dll + + + + JIT.Methodical.f-iF-I + $(TestBinDir)JIT\Methodical\JIT.Methodical.XUnitWrapper.dll + + From df883b36bc42ca95d8626c2013a9bbb2b8e6be3a Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Wed, 30 Mar 2022 15:05:09 -0700 Subject: [PATCH 15/17] Exclude wasm support files. --- src/tests/Common/helixpublishwitharcade.proj | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/tests/Common/helixpublishwitharcade.proj b/src/tests/Common/helixpublishwitharcade.proj index 64c54d22f840b2..4bf0fbd0604682 100644 --- a/src/tests/Common/helixpublishwitharcade.proj +++ b/src/tests/Common/helixpublishwitharcade.proj @@ -282,7 +282,8 @@ - <_MergedWrapperMarker Include="$(TestBinDir)**\*.MergedTestAssembly" /> + + <_MergedWrapperMarker Include="$(TestBinDir)**\*.MergedTestAssembly" Exclude="$(TestBinDir)**\supportFiles\*.MergedTestAssembly" /> From e572666bf02300cda00e5464c639b79453b33ea3 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Fri, 1 Apr 2022 10:06:00 -0700 Subject: [PATCH 16/17] Update src/tests/Loader/classloader/MethodImpl/CovariantReturns/UnitTest/UnitTestMultiModule.ilproj --- .../CovariantReturns/UnitTest/UnitTestMultiModule.ilproj | 1 - 1 file changed, 1 deletion(-) diff --git a/src/tests/Loader/classloader/MethodImpl/CovariantReturns/UnitTest/UnitTestMultiModule.ilproj b/src/tests/Loader/classloader/MethodImpl/CovariantReturns/UnitTest/UnitTestMultiModule.ilproj index 661a9dbdf808f2..5fca8b3905421c 100644 --- a/src/tests/Loader/classloader/MethodImpl/CovariantReturns/UnitTest/UnitTestMultiModule.ilproj +++ b/src/tests/Loader/classloader/MethodImpl/CovariantReturns/UnitTest/UnitTestMultiModule.ilproj @@ -1,7 +1,6 @@ Exe - true From a8a45ba624bd89a07ab9aab508337fe3023cae71 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Fri, 1 Apr 2022 13:45:16 -0700 Subject: [PATCH 17/17] Apply suggestions from code review --- .../Loader/classloader/generics/Variance/IL/Unbox002.csproj | 1 - .../Loader/classloader/generics/Variance/IL/Unbox003.csproj | 1 - .../Loader/classloader/generics/Variance/IL/vsw543506.csproj | 1 - 3 files changed, 3 deletions(-) diff --git a/src/tests/Loader/classloader/generics/Variance/IL/Unbox002.csproj b/src/tests/Loader/classloader/generics/Variance/IL/Unbox002.csproj index 4f0c61ddca2933..d28c31d472d2c8 100644 --- a/src/tests/Loader/classloader/generics/Variance/IL/Unbox002.csproj +++ b/src/tests/Loader/classloader/generics/Variance/IL/Unbox002.csproj @@ -2,7 +2,6 @@ true Exe - true diff --git a/src/tests/Loader/classloader/generics/Variance/IL/Unbox003.csproj b/src/tests/Loader/classloader/generics/Variance/IL/Unbox003.csproj index 572143f323f037..efa4c2dd1a2c79 100644 --- a/src/tests/Loader/classloader/generics/Variance/IL/Unbox003.csproj +++ b/src/tests/Loader/classloader/generics/Variance/IL/Unbox003.csproj @@ -2,7 +2,6 @@ true Exe - true diff --git a/src/tests/Loader/classloader/generics/Variance/IL/vsw543506.csproj b/src/tests/Loader/classloader/generics/Variance/IL/vsw543506.csproj index 9003f0a82135d5..bb8b2efa6467d9 100644 --- a/src/tests/Loader/classloader/generics/Variance/IL/vsw543506.csproj +++ b/src/tests/Loader/classloader/generics/Variance/IL/vsw543506.csproj @@ -2,7 +2,6 @@ true Exe - true