Skip to content

Commit 42e9a7d

Browse files
analogrelayanalogrelay
authored
add FlakyAttribute to mark flaky tests (#1222)
part of dotnet/aspnetcore#8237
1 parent 11727db commit 42e9a7d

File tree

8 files changed

+297
-0
lines changed

8 files changed

+297
-0
lines changed

‎Directory.Build.targets‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,5 +47,8 @@
4747
<Import Project="Sdk.targets" Sdk="Microsoft.DotNet.Arcade.Sdk" />
4848
<Import Project="eng\Workarounds.AfterArcade.targets" />
4949

50+
<!-- Only needed on AzDO: Flaky Test handling -->
51+
<Import Project="eng\FlakyTests.targets" Condition="'$(BUILD_BUILDNUMBER)' != ''" />
52+
5053
<Import Project="eng\targets\Npm.Common.targets" Condition="'$(MSBuildProjectExtension)' == '.npmproj'" />
5154
</Project>

‎eng/FlakyTests.targets‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
<Project>
2+
<Target Name="_GenerateTestPasses" DependsOnTargets="$(_GetTestsToRunTarget)">
3+
<PropertyGroup>
4+
<_FlakyRunAdditionalArgs>-trait "Flaky:All=true" -trait "Flaky:AzP:All=true" -trait "Flaky:AzP:OS:$(AGENT_OS)=true"</_FlakyRunAdditionalArgs>
5+
<_NonFlakyRunAdditionalArgs>-notrait "Flaky:All=true" -notrait "Flaky:AzP:All=true" -notrait "Flaky:AzP:OS:$(AGENT_OS)=true"</_NonFlakyRunAdditionalArgs>
6+
</PropertyGroup>
7+
<ItemGroup>
8+
<!-- Take the TestToRun values and update them to only run non-flaky tests -->
9+
<NonFlakyRuns Include="@(TestToRun)">
10+
<TestRunnerAdditionalArguments>%(TestRunnerAdditionalArguments) $(_NonFlakyRunAdditionalArgs)</TestRunnerAdditionalArguments>
11+
</NonFlakyRuns>
12+
13+
<!-- Add new runs that do run flaky tests but ignore the exit code -->
14+
<FlakyRuns Include="@(TestToRun)">
15+
<TestRunnerAdditionalArguments>%(TestRunnerAdditionalArguments) $(_FlakyRunAdditionalArgs)</TestRunnerAdditionalArguments>
16+
<TestRunnerIgnoreExitCode>true</TestRunnerIgnoreExitCode>
17+
<ResultsHtmlPath>$([System.IO.Path]::ChangeExtension(%(ResultsHtmlPath), '.flaky.html'))</ResultsHtmlPath>
18+
<ResultsStdOutPath>$([System.IO.Path]::ChangeExtension(%(ResultsStdOutPath), '.flaky.log'))</ResultsStdOutPath>
19+
<ResultsXmlPath>$([System.IO.Path]::ChangeExtension(%(ResultsXmlPath), '.flaky.xml'))</ResultsXmlPath>
20+
</FlakyRuns>
21+
22+
<!-- Replace the previous runs -->
23+
<TestToRun Remove="@(TestToRun)" />
24+
25+
<!-- TODO: Replace with below when https://github.com/dotnet/arcade/issues/2182 is in -->
26+
<!-- <TestToRun Include="@(NonFlakyRuns);@(FlakyRuns)" /> -->
27+
<TestToRun Include="@(NonFlakyRuns)" />
28+
</ItemGroup>
29+
</Target>
30+
31+
<!-- Replace Arcade's Test target with a custom one -->
32+
<Target Name="Test" DependsOnTargets="_GenerateTestPasses;RunTests" Condition="'$(IsUnitTestProject)' == 'true' or '$(IsPerformanceTestProject)' == 'true'" />
33+
</Project>

‎src/Logging/Logging.Testing/src/LoggedTest/LoggedTestBase.cs‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -91,6 +91,14 @@ public virtual void Initialize(MethodInfo methodInfo, object[] testMethodArgumen
9191

9292
public virtual void Dispose()
9393
{
94+
if(_testLog == null)
95+
{
96+
// It seems like sometimes the MSBuild goop that adds the test framework can end up in a bad state and not actually add it
97+
// Not sure yet why that happens but the exception isn't clear so I'm adding this error so we can detect it better.
98+
// -anurse
99+
throw new InvalidOperationException("LoggedTest base class was used but nothing initialized it! The test framework may not be enabled. Try cleaning your 'obj' directory.");
100+
}
101+
94102
_initializationException?.Throw();
95103
_testLog.Dispose();
96104
}

‎src/TestingUtils/Microsoft.AspNetCore.Testing/src/AzurePipelines.cs‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
using System;
2+
using System.Collections.Generic;
3+
using System.Reflection;
4+
5+
namespace Microsoft.AspNetCore.Testing
6+
{
7+
public static class AzurePipelines
8+
{
9+
public const string All = Prefix + "All";
10+
public const string Windows = OsPrefix + "Windows_NT";
11+
public const string macOS = OsPrefix + "Darwin";
12+
public const string Linux = OsPrefix + "Linux";
13+
14+
private const string Prefix = "AzP:";
15+
private const string OsPrefix = Prefix + "OS:";
16+
}
17+
}

‎src/TestingUtils/Microsoft.AspNetCore.Testing/src/HelixQueues.cs‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
using System;
2+
using System.Collections.Generic;
3+
using System.Reflection;
4+
5+
namespace Microsoft.AspNetCore.Testing
6+
{
7+
public static class HelixQueues
8+
{
9+
public const string All = Prefix + "All";
10+
11+
public const string Fedora28Amd64 = QueuePrefix + "Fedora.28." + Amd64Suffix;
12+
public const string Fedora27Amd64 = QueuePrefix + "Fedora.27." + Amd64Suffix;
13+
public const string Redhat7Amd64 = QueuePrefix + "Redhat.7." + Amd64Suffix;
14+
public const string Debian9Amd64 = QueuePrefix + "Debian.9." + Amd64Suffix;
15+
public const string Debian8Amd64 = QueuePrefix + "Debian.8." + Amd64Suffix;
16+
public const string Centos7Amd64 = QueuePrefix + "Centos.7." + Amd64Suffix;
17+
public const string Ubuntu1604Amd64 = QueuePrefix + "Ubuntu.1604." + Amd64Suffix;
18+
public const string Ubuntu1810Amd64 = QueuePrefix + "Ubuntu.1810." + Amd64Suffix;
19+
public const string macOS1012Amd64 = QueuePrefix + "OSX.1012." + Amd64Suffix;
20+
public const string Windows10Amd64 = QueuePrefix + "Windows.10.Amd64.ClientRS4.VS2017.Open"; // Doesn't have the default suffix!
21+
22+
private const string Prefix = "Helix:";
23+
private const string QueuePrefix = Prefix + "Queue:";
24+
private const string Amd64Suffix = "Amd64.Open";
25+
}
26+
}

‎src/TestingUtils/Microsoft.AspNetCore.Testing/src/xunit/FlakyAttribute.cs‎

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
using System;
2+
using System.Collections.Generic;
3+
using Xunit.Sdk;
4+
5+
namespace Microsoft.AspNetCore.Testing.xunit
6+
{
7+
/// <summary>
8+
/// Marks a test as "Flaky" so that the build will sequester it and ignore failures.
9+
/// </summary>
10+
/// <remarks>
11+
/// <para>
12+
/// This attribute works by applying xUnit.net "Traits" based on the criteria specified in the attribute
13+
/// properties. Once these traits are applied, build scripts can include/exclude tests based on them.
14+
/// </para>
15+
/// <para>
16+
/// All flakiness-related traits start with <code>Flaky:</code> and are grouped first by the process running the tests: Azure Pipelines (AzP) or Helix.
17+
/// Then there is a segment specifying the "selector" which indicates where the test is flaky. Finally a segment specifying the value of that selector.
18+
/// The value of these traits is always either "true" or the trait is not present. We encode the entire selector in the name of the trait because xUnit.net only
19+
/// provides "==" and "!=" operators for traits, there is no way to check if a trait "contains" or "does not contain" a value. VSTest does support "contains" checks
20+
/// but does not appear to support "does not contain" checks. Using this pattern means we can use simple "==" and "!=" checks to either only run flaky tests, or exclude
21+
/// flaky tests.
22+
/// </para>
23+
/// </remarks>
24+
/// <example>
25+
/// <code>
26+
/// [Fact]
27+
/// [Flaky("...", HelixQueues.Fedora28Amd64, AzurePipelines.macOS)]
28+
/// public void FlakyTest()
29+
/// {
30+
/// // Flakiness
31+
/// }
32+
/// </code>
33+
///
34+
/// <para>
35+
/// The above example generates the following facets:
36+
/// </para>
37+
///
38+
/// <list type="bullet">
39+
/// <item>
40+
/// <description><c>Flaky:Helix:Queue:Fedora.28.Amd64.Open</c> = <c>true</c></description>
41+
/// </item>
42+
/// <item>
43+
/// <description><c>Flaky:AzP:OS:Darwin</c> = <c>true</c></description>
44+
/// </item>
45+
/// </list>
46+
///
47+
/// <para>
48+
/// Given the above attribute, the Azure Pipelines macOS run can easily filter this test out by passing <c>-notrait "Flaky:AzP:OS:all=true" -notrait "Flaky:AzP:OS:Darwin=true"</c>
49+
/// to <c>xunit.console.exe</c>. Similarly, it can run only flaky tests using <c>-trait "Flaky:AzP:OS:all=true" -trait "Flaky:AzP:OS:Darwin=true"</c>
50+
/// </para>
51+
/// </example>
52+
[TraitDiscoverer("Microsoft.AspNetCore.Testing.xunit.FlakyTestDiscoverer", "Microsoft.AspNetCore.Testing")]
53+
[AttributeUsage(AttributeTargets.Method)]
54+
public sealed class FlakyAttribute : Attribute, ITraitAttribute
55+
{
56+
/// <summary>
57+
/// Gets a URL to a GitHub issue tracking this flaky test.
58+
/// </summary>
59+
public string GitHubIssueUrl { get; }
60+
61+
public IReadOnlyList<string> Filters { get; }
62+
63+
/// <summary>
64+
/// Initializes a new instance of the <see cref="FlakyAttribute"/> class with the specified <see cref="GitHubIssueUrl"/> and a list of <see cref="Filters"/>. If no
65+
/// filters are provided, the test is considered flaky in all environments.
66+
/// </summary>
67+
/// <param name="gitHubIssueUrl">The URL to a GitHub issue tracking this flaky test.</param>
68+
/// <param name="filters">A list of filters that define where this test is flaky. Use values in <see cref="AzurePipelines"/> and <see cref="HelixQueues"/>.</param>
69+
public FlakyAttribute(string gitHubIssueUrl, params string[] filters)
70+
{
71+
GitHubIssueUrl = gitHubIssueUrl;
72+
Filters = new List<string>(filters);
73+
}
74+
}
75+
}

‎src/TestingUtils/Microsoft.AspNetCore.Testing/src/xunit/FlakyTestDiscoverer.cs‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
using System;
2+
using System.Collections.Generic;
3+
using System.Linq;
4+
using Xunit.Abstractions;
5+
using Xunit.Sdk;
6+
7+
namespace Microsoft.AspNetCore.Testing.xunit
8+
{
9+
public class FlakyTestDiscoverer : ITraitDiscoverer
10+
{
11+
public IEnumerable<KeyValuePair<string, string>> GetTraits(IAttributeInfo traitAttribute)
12+
{
13+
if (traitAttribute is ReflectionAttributeInfo attribute && attribute.Attribute is FlakyAttribute flakyAttribute)
14+
{
15+
return GetTraitsCore(flakyAttribute);
16+
}
17+
else
18+
{
19+
throw new InvalidOperationException("The 'Flaky' attribute is only supported via reflection.");
20+
}
21+
}
22+
23+
private IEnumerable<KeyValuePair<string, string>> GetTraitsCore(FlakyAttribute attribute)
24+
{
25+
if (attribute.Filters.Count > 0)
26+
{
27+
foreach (var filter in attribute.Filters)
28+
{
29+
yield return new KeyValuePair<string, string>($"Flaky:{filter}", "true");
30+
}
31+
}
32+
else
33+
{
34+
yield return new KeyValuePair<string, string>($"Flaky:All", "true");
35+
}
36+
}
37+
}
38+
}

‎src/TestingUtils/Microsoft.AspNetCore.Testing/test/FlakyAttributeTest.cs‎

Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,97 @@
1+
using Microsoft.AspNetCore.Testing.xunit;
2+
using System;
3+
using System.Collections.Generic;
4+
using Xunit;
5+
6+
namespace Microsoft.AspNetCore.Testing.Tests
7+
{
8+
public class FlakyAttributeTest
9+
{
10+
[Fact]
11+
[Flaky("http://example.com")]
12+
public void AlwaysFlaky()
13+
{
14+
if (!string.IsNullOrEmpty(Environment.GetEnvironmentVariable("HELIX")) || !string.IsNullOrEmpty(Environment.GetEnvironmentVariable("AGENT_OS")))
15+
{
16+
throw new Exception("Flaky!");
17+
}
18+
}
19+
20+
[Fact]
21+
[Flaky("http://example.com", HelixQueues.All)]
22+
public void FlakyInHelixOnly()
23+
{
24+
if (!string.IsNullOrEmpty(Environment.GetEnvironmentVariable("HELIX")))
25+
{
26+
throw new Exception("Flaky on Helix!");
27+
}
28+
}
29+
30+
[Fact]
31+
[Flaky("http://example.com", HelixQueues.macOS1012Amd64, HelixQueues.Fedora28Amd64)]
32+
public void FlakyInSpecificHelixQueue()
33+
{
34+
// Today we don't run Extensions tests on Helix, but this test should light up when we do.
35+
var queueName = Environment.GetEnvironmentVariable("HELIX");
36+
if (!string.IsNullOrEmpty(queueName))
37+
{
38+
var failingQueues = new HashSet<string>(StringComparer.OrdinalIgnoreCase) { HelixQueues.macOS1012Amd64, HelixQueues.Fedora28Amd64 };
39+
if (failingQueues.Contains(queueName))
40+
{
41+
throw new Exception($"Flaky on Helix Queue '{queueName}' !");
42+
}
43+
}
44+
}
45+
46+
[Fact]
47+
[Flaky("http://example.com", AzurePipelines.All)]
48+
public void FlakyInAzPOnly()
49+
{
50+
if (!string.IsNullOrEmpty(Environment.GetEnvironmentVariable("AGENT_OS")))
51+
{
52+
throw new Exception("Flaky on AzP!");
53+
}
54+
}
55+
56+
[Fact]
57+
[Flaky("http://example.com", AzurePipelines.Windows)]
58+
public void FlakyInAzPWindowsOnly()
59+
{
60+
if (string.Equals(Environment.GetEnvironmentVariable("AGENT_OS"), AzurePipelines.Windows))
61+
{
62+
throw new Exception("Flaky on AzP Windows!");
63+
}
64+
}
65+
66+
[Fact]
67+
[Flaky("http://example.com", AzurePipelines.macOS)]
68+
public void FlakyInAzPmacOSOnly()
69+
{
70+
if (string.Equals(Environment.GetEnvironmentVariable("AGENT_OS"), AzurePipelines.macOS))
71+
{
72+
throw new Exception("Flaky on AzP macOS!");
73+
}
74+
}
75+
76+
[Fact]
77+
[Flaky("http://example.com", AzurePipelines.Linux)]
78+
public void FlakyInAzPLinuxOnly()
79+
{
80+
if (string.Equals(Environment.GetEnvironmentVariable("AGENT_OS"), AzurePipelines.Linux))
81+
{
82+
throw new Exception("Flaky on AzP Linux!");
83+
}
84+
}
85+
86+
[Fact]
87+
[Flaky("http://example.com", AzurePipelines.Linux, AzurePipelines.macOS)]
88+
public void FlakyInAzPNonWindowsOnly()
89+
{
90+
var agentOs = Environment.GetEnvironmentVariable("AGENT_OS");
91+
if (string.Equals(agentOs, "Linux") || string.Equals(agentOs, AzurePipelines.macOS))
92+
{
93+
throw new Exception("Flaky on AzP non-Windows!");
94+
}
95+
}
96+
}
97+
}

0 commit comments

Comments
 (0)