add a 'Run Flaky Tests' step that does not fail the build (#1239)

Author: analogrelay

Directory.Build.props

@@ -17,9 +17,11 @@
     <OutDirName Condition="'$(IsReferenceAssemblyProject)' == 'true'">$(MSBuildProjectName)-ref</OutDirName>
   </PropertyGroup>
 
+  <Import Project="eng\FlakyTests.BeforeArcade.props" />
   <Import Project="eng\Workarounds.BeforeArcade.props" />
   <Import Project="Sdk.props" Sdk="Microsoft.DotNet.Arcade.Sdk" />
   <Import Project="eng\Workarounds.AfterArcade.props" />
+  <Import Project="eng\FlakyTests.AfterArcade.props" />
 
   <PropertyGroup Condition="'$(CopyrightMicrosoft)' != ''">
     <Copyright>$(CopyrightMicrosoft)</Copyright>

Directory.Build.targets

@@ -47,8 +47,5 @@
   <Import Project="Sdk.targets" Sdk="Microsoft.DotNet.Arcade.Sdk" />
   <Import Project="eng\Workarounds.AfterArcade.targets" />
 
-  <!-- Only needed on AzDO: Flaky Test handling -->
-  <Import Project="eng\FlakyTests.targets" Condition="'$(BUILD_BUILDNUMBER)' != ''" />
-
   <Import Project="eng\targets\Npm.Common.targets"  Condition="'$(MSBuildProjectExtension)' == '.npmproj'" />
 </Project>

azure-pipelines.yml

@@ -91,6 +91,9 @@ jobs:
           -prepareMachine
           $(_BuildArgs)
         displayName: Build and Publish
+      - script: eng\scripts\ci-flaky-tests.cmd
+        displayName: Run Flaky Tests
+        continueOnError: true
       - powershell: eng\common\msbuild.ps1 eng/repo.targets /t:TagCiBuilds '/p:IsFinalBuild=$(IsFinalBuild)' '/p:CI=true'
         displayName: Set CI Tags
         condition: eq(variables['_BuildConfig'], 'Release')
@@ -101,6 +104,14 @@ jobs:
       - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0
         displayName: Detect components
         condition: and(succeeded(), eq(variables['system.pullrequest.isfork'], false), eq(variables['_BuildConfig'], 'Release'))
+      - task: PublishTestResults@2
+        displayName: Publish Flaky Test Results
+        inputs:
+          testResultsFormat: 'xUnit'
+          testResultsFiles: '*.xml' 
+          searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)/Flaky'
+        continueOnError: true
+        condition: always()
       - task: PublishBuildArtifacts@1
         displayName: Upload package artifacts
         condition: and(eq(variables['system.pullrequest.isfork'], false), eq(variables['_BuildConfig'], 'Release'))
@@ -133,9 +144,20 @@ jobs:
           --configuration $(_BuildConfig)
           --prepareMachine
         displayName: Build
+      - script: eng/scripts/ci-flaky-tests.sh
+        displayName: Run Flaky Tests
+        continueOnError: true
       - script: eng/scripts/KillProcesses.sh
         displayName: Kill processes
         condition: always()
+      - task: PublishTestResults@2
+        displayName: Publish Flaky Test Results
+        inputs:
+          testResultsFormat: 'xUnit'
+          testResultsFiles: '*.xml' 
+          searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)/Flaky'
+        continueOnError: true
+        condition: always()
 
     - job: OSX_10_13
       displayName: 'OSX'
@@ -160,6 +182,18 @@ jobs:
           --configuration $(_BuildConfig)
           --prepareMachine
         displayName: Build
+      - script: eng/scripts/ci-flaky-tests.sh
+        displayName: Run Flaky Tests
+        continueOnError: true
       - script: eng/scripts/KillProcesses.sh
         displayName: Kill processes
         condition: always()
+      - task: PublishTestResults@2
+        displayName: Publish Flaky Test Results
+        inputs:
+          testResultsFormat: 'xUnit'
+          testResultsFiles: '*.xml' 
+          searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)/Flaky'
+        continueOnError: true
+        condition: always()
+

eng/FlakyTests.AfterArcade.props

@@ -0,0 +1,7 @@
+<Project>
+  <!-- Override where xUnit logs and results go if we're doing the flaky run -->
+  <PropertyGroup Condition="'$(RunFlakyTests)' == 'true'">
+    <ArtifactsLogDir>$(ArtifactsDir)log\$(Configuration)\Flaky\</ArtifactsLogDir>
+    <ArtifactsTestResultsDir>$(ArtifactsDir)TestResults\$(Configuration)\Flaky\</ArtifactsTestResultsDir>
+  </PropertyGroup>
+</Project>

eng/FlakyTests.BeforeArcade.props

@@ -0,0 +1,18 @@
+<Project>
+  <!-- Local Dev Flakiness -->
+  <PropertyGroup>
+    <_FlakyRunAdditionalArgs>-trait "Flaky:All=true"</_FlakyRunAdditionalArgs>
+    <_NonFlakyRunAdditionalArgs>-notrait "Flaky:All=true"</_NonFlakyRunAdditionalArgs>
+  </PropertyGroup>
+
+  <!-- Azure Pipelines Flakiness -->
+  <PropertyGroup Condition="'$(AGENT_OS)' != ''">
+    <_FlakyRunAdditionalArgs>$(_FlakyRunAdditionalArgs) -trait "Flaky:AzP:All=true" -trait "Flaky:AzP:OS:$(AGENT_OS)=true"</_FlakyRunAdditionalArgs>
+    <_NonFlakyRunAdditionalArgs>$(_NonFlakyRunAdditionalArgs) -notrait "Flaky:AzP:All=true" -notrait "Flaky:AzP:OS:$(AGENT_OS)=true"</_NonFlakyRunAdditionalArgs>
+  </PropertyGroup>
+
+  <PropertyGroup>
+    <TestRunnerAdditionalArguments Condition="'$(RunFlakyTests)' == ''">$(_NonFlakyRunAdditionalArgs)</TestRunnerAdditionalArguments>
+    <TestRunnerAdditionalArguments Condition="'$(RunFlakyTests)' == 'true'">$(_FlakyRunAdditionalArgs)</TestRunnerAdditionalArguments>
+  </PropertyGroup>
+</Project>

eng/FlakyTests.targets

@@ -0,0 +1,3 @@
+@echo off
+powershell -ExecutionPolicy ByPass -NoProfile -command "& """%~dp0..\common\Build.ps1""" -test -ci /p:RunFlakyTests=true %*"
+exit /b %ErrorLevel%

eng/scripts/ci-flaky-tests.cmd

@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+
+source="${BASH_SOURCE[0]}"
+
+# resolve $SOURCE until the file is no longer a symlink
+while [[ -h $source ]]; do
+  scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
+  source="$(readlink "$source")"
+
+  # if $source was a relative symlink, we need to resolve it relative to the path where 
+  # the symlink file was located
+  [[ $source != /* ]] && source="$scriptroot/$source"
+done
+scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
+
+. "$scriptroot/../common/build.sh" --ci --test -p:RunFlakyTests=true $@