diff --git a/Directory.Build.targets b/Directory.Build.targets index 8a08d03d871..3cf8fe094dc 100644 --- a/Directory.Build.targets +++ b/Directory.Build.targets @@ -47,5 +47,8 @@ + + + diff --git a/eng/FlakyTests.targets b/eng/FlakyTests.targets new file mode 100644 index 00000000000..289ccc79880 --- /dev/null +++ b/eng/FlakyTests.targets @@ -0,0 +1,33 @@ + + + + <_FlakyRunAdditionalArgs>-trait "Flaky:All=true" -trait "Flaky:AzP:All=true" -trait "Flaky:AzP:OS:$(AGENT_OS)=true" + <_NonFlakyRunAdditionalArgs>-notrait "Flaky:All=true" -notrait "Flaky:AzP:All=true" -notrait "Flaky:AzP:OS:$(AGENT_OS)=true" + + + + + %(TestRunnerAdditionalArguments) $(_NonFlakyRunAdditionalArgs) + + + + + %(TestRunnerAdditionalArguments) $(_FlakyRunAdditionalArgs) + true + $([System.IO.Path]::ChangeExtension(%(ResultsHtmlPath), '.flaky.html')) + $([System.IO.Path]::ChangeExtension(%(ResultsStdOutPath), '.flaky.log')) + $([System.IO.Path]::ChangeExtension(%(ResultsXmlPath), '.flaky.xml')) + + + + + + + + + + + + + + diff --git a/src/Logging/Logging.Testing/src/LoggedTest/LoggedTestBase.cs b/src/Logging/Logging.Testing/src/LoggedTest/LoggedTestBase.cs index 72de6a87c35..492de61cb60 100644 --- a/src/Logging/Logging.Testing/src/LoggedTest/LoggedTestBase.cs +++ b/src/Logging/Logging.Testing/src/LoggedTest/LoggedTestBase.cs @@ -91,6 +91,14 @@ public virtual void Initialize(MethodInfo methodInfo, object[] testMethodArgumen public virtual void Dispose() { + if(_testLog == null) + { + // It seems like sometimes the MSBuild goop that adds the test framework can end up in a bad state and not actually add it + // Not sure yet why that happens but the exception isn't clear so I'm adding this error so we can detect it better. + // -anurse + throw new InvalidOperationException("LoggedTest base class was used but nothing initialized it! The test framework may not be enabled. Try cleaning your 'obj' directory."); + } + _initializationException?.Throw(); _testLog.Dispose(); } diff --git a/src/TestingUtils/Microsoft.AspNetCore.Testing/src/AzurePipelines.cs b/src/TestingUtils/Microsoft.AspNetCore.Testing/src/AzurePipelines.cs new file mode 100644 index 00000000000..ae1eac3b905 --- /dev/null +++ b/src/TestingUtils/Microsoft.AspNetCore.Testing/src/AzurePipelines.cs @@ -0,0 +1,17 @@ +using System; +using System.Collections.Generic; +using System.Reflection; + +namespace Microsoft.AspNetCore.Testing +{ + public static class AzurePipelines + { + public const string All = Prefix + "All"; + public const string Windows = OsPrefix + "Windows_NT"; + public const string macOS = OsPrefix + "Darwin"; + public const string Linux = OsPrefix + "Linux"; + + private const string Prefix = "AzP:"; + private const string OsPrefix = Prefix + "OS:"; + } +} diff --git a/src/TestingUtils/Microsoft.AspNetCore.Testing/src/HelixQueues.cs b/src/TestingUtils/Microsoft.AspNetCore.Testing/src/HelixQueues.cs new file mode 100644 index 00000000000..84828b6b83e --- /dev/null +++ b/src/TestingUtils/Microsoft.AspNetCore.Testing/src/HelixQueues.cs @@ -0,0 +1,26 @@ +using System; +using System.Collections.Generic; +using System.Reflection; + +namespace Microsoft.AspNetCore.Testing +{ + public static class HelixQueues + { + public const string All = Prefix + "All"; + + public const string Fedora28Amd64 = QueuePrefix + "Fedora.28." + Amd64Suffix; + public const string Fedora27Amd64 = QueuePrefix + "Fedora.27." + Amd64Suffix; + public const string Redhat7Amd64 = QueuePrefix + "Redhat.7." + Amd64Suffix; + public const string Debian9Amd64 = QueuePrefix + "Debian.9." + Amd64Suffix; + public const string Debian8Amd64 = QueuePrefix + "Debian.8." + Amd64Suffix; + public const string Centos7Amd64 = QueuePrefix + "Centos.7." + Amd64Suffix; + public const string Ubuntu1604Amd64 = QueuePrefix + "Ubuntu.1604." + Amd64Suffix; + public const string Ubuntu1810Amd64 = QueuePrefix + "Ubuntu.1810." + Amd64Suffix; + public const string macOS1012Amd64 = QueuePrefix + "OSX.1012." + Amd64Suffix; + public const string Windows10Amd64 = QueuePrefix + "Windows.10.Amd64.ClientRS4.VS2017.Open"; // Doesn't have the default suffix! + + private const string Prefix = "Helix:"; + private const string QueuePrefix = Prefix + "Queue:"; + private const string Amd64Suffix = "Amd64.Open"; + } +} diff --git a/src/TestingUtils/Microsoft.AspNetCore.Testing/src/xunit/FlakyAttribute.cs b/src/TestingUtils/Microsoft.AspNetCore.Testing/src/xunit/FlakyAttribute.cs new file mode 100644 index 00000000000..b613a9bf4dc --- /dev/null +++ b/src/TestingUtils/Microsoft.AspNetCore.Testing/src/xunit/FlakyAttribute.cs @@ -0,0 +1,75 @@ +using System; +using System.Collections.Generic; +using Xunit.Sdk; + +namespace Microsoft.AspNetCore.Testing.xunit +{ + /// + /// Marks a test as "Flaky" so that the build will sequester it and ignore failures. + /// + /// + /// + /// This attribute works by applying xUnit.net "Traits" based on the criteria specified in the attribute + /// properties. Once these traits are applied, build scripts can include/exclude tests based on them. + /// + /// + /// All flakiness-related traits start with Flaky: and are grouped first by the process running the tests: Azure Pipelines (AzP) or Helix. + /// Then there is a segment specifying the "selector" which indicates where the test is flaky. Finally a segment specifying the value of that selector. + /// The value of these traits is always either "true" or the trait is not present. We encode the entire selector in the name of the trait because xUnit.net only + /// provides "==" and "!=" operators for traits, there is no way to check if a trait "contains" or "does not contain" a value. VSTest does support "contains" checks + /// but does not appear to support "does not contain" checks. Using this pattern means we can use simple "==" and "!=" checks to either only run flaky tests, or exclude + /// flaky tests. + /// + /// + /// + /// + /// [Fact] + /// [Flaky("...", HelixQueues.Fedora28Amd64, AzurePipelines.macOS)] + /// public void FlakyTest() + /// { + /// // Flakiness + /// } + /// + /// + /// + /// The above example generates the following facets: + /// + /// + /// + /// + /// Flaky:Helix:Queue:Fedora.28.Amd64.Open = true + /// + /// + /// Flaky:AzP:OS:Darwin = true + /// + /// + /// + /// + /// Given the above attribute, the Azure Pipelines macOS run can easily filter this test out by passing -notrait "Flaky:AzP:OS:all=true" -notrait "Flaky:AzP:OS:Darwin=true" + /// to xunit.console.exe. Similarly, it can run only flaky tests using -trait "Flaky:AzP:OS:all=true" -trait "Flaky:AzP:OS:Darwin=true" + /// + /// + [TraitDiscoverer("Microsoft.AspNetCore.Testing.xunit.FlakyTestDiscoverer", "Microsoft.AspNetCore.Testing")] + [AttributeUsage(AttributeTargets.Method)] + public sealed class FlakyAttribute : Attribute, ITraitAttribute + { + /// + /// Gets a URL to a GitHub issue tracking this flaky test. + /// + public string GitHubIssueUrl { get; } + + public IReadOnlyList Filters { get; } + + /// + /// Initializes a new instance of the class with the specified and a list of . If no + /// filters are provided, the test is considered flaky in all environments. + /// + /// The URL to a GitHub issue tracking this flaky test. + /// A list of filters that define where this test is flaky. Use values in and . + public FlakyAttribute(string gitHubIssueUrl, params string[] filters) + { + GitHubIssueUrl = gitHubIssueUrl; + Filters = new List(filters); + } + } +} diff --git a/src/TestingUtils/Microsoft.AspNetCore.Testing/src/xunit/FlakyTestDiscoverer.cs b/src/TestingUtils/Microsoft.AspNetCore.Testing/src/xunit/FlakyTestDiscoverer.cs new file mode 100644 index 00000000000..344b9b23780 --- /dev/null +++ b/src/TestingUtils/Microsoft.AspNetCore.Testing/src/xunit/FlakyTestDiscoverer.cs @@ -0,0 +1,38 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using Xunit.Abstractions; +using Xunit.Sdk; + +namespace Microsoft.AspNetCore.Testing.xunit +{ + public class FlakyTestDiscoverer : ITraitDiscoverer + { + public IEnumerable> GetTraits(IAttributeInfo traitAttribute) + { + if (traitAttribute is ReflectionAttributeInfo attribute && attribute.Attribute is FlakyAttribute flakyAttribute) + { + return GetTraitsCore(flakyAttribute); + } + else + { + throw new InvalidOperationException("The 'Flaky' attribute is only supported via reflection."); + } + } + + private IEnumerable> GetTraitsCore(FlakyAttribute attribute) + { + if (attribute.Filters.Count > 0) + { + foreach (var filter in attribute.Filters) + { + yield return new KeyValuePair($"Flaky:{filter}", "true"); + } + } + else + { + yield return new KeyValuePair($"Flaky:All", "true"); + } + } + } +} diff --git a/src/TestingUtils/Microsoft.AspNetCore.Testing/test/FlakyAttributeTest.cs b/src/TestingUtils/Microsoft.AspNetCore.Testing/test/FlakyAttributeTest.cs new file mode 100644 index 00000000000..e9accf62741 --- /dev/null +++ b/src/TestingUtils/Microsoft.AspNetCore.Testing/test/FlakyAttributeTest.cs @@ -0,0 +1,97 @@ +using Microsoft.AspNetCore.Testing.xunit; +using System; +using System.Collections.Generic; +using Xunit; + +namespace Microsoft.AspNetCore.Testing.Tests +{ + public class FlakyAttributeTest + { + [Fact] + [Flaky("http://example.com")] + public void AlwaysFlaky() + { + if (!string.IsNullOrEmpty(Environment.GetEnvironmentVariable("HELIX")) || !string.IsNullOrEmpty(Environment.GetEnvironmentVariable("AGENT_OS"))) + { + throw new Exception("Flaky!"); + } + } + + [Fact] + [Flaky("http://example.com", HelixQueues.All)] + public void FlakyInHelixOnly() + { + if (!string.IsNullOrEmpty(Environment.GetEnvironmentVariable("HELIX"))) + { + throw new Exception("Flaky on Helix!"); + } + } + + [Fact] + [Flaky("http://example.com", HelixQueues.macOS1012Amd64, HelixQueues.Fedora28Amd64)] + public void FlakyInSpecificHelixQueue() + { + // Today we don't run Extensions tests on Helix, but this test should light up when we do. + var queueName = Environment.GetEnvironmentVariable("HELIX"); + if (!string.IsNullOrEmpty(queueName)) + { + var failingQueues = new HashSet(StringComparer.OrdinalIgnoreCase) { HelixQueues.macOS1012Amd64, HelixQueues.Fedora28Amd64 }; + if (failingQueues.Contains(queueName)) + { + throw new Exception($"Flaky on Helix Queue '{queueName}' !"); + } + } + } + + [Fact] + [Flaky("http://example.com", AzurePipelines.All)] + public void FlakyInAzPOnly() + { + if (!string.IsNullOrEmpty(Environment.GetEnvironmentVariable("AGENT_OS"))) + { + throw new Exception("Flaky on AzP!"); + } + } + + [Fact] + [Flaky("http://example.com", AzurePipelines.Windows)] + public void FlakyInAzPWindowsOnly() + { + if (string.Equals(Environment.GetEnvironmentVariable("AGENT_OS"), AzurePipelines.Windows)) + { + throw new Exception("Flaky on AzP Windows!"); + } + } + + [Fact] + [Flaky("http://example.com", AzurePipelines.macOS)] + public void FlakyInAzPmacOSOnly() + { + if (string.Equals(Environment.GetEnvironmentVariable("AGENT_OS"), AzurePipelines.macOS)) + { + throw new Exception("Flaky on AzP macOS!"); + } + } + + [Fact] + [Flaky("http://example.com", AzurePipelines.Linux)] + public void FlakyInAzPLinuxOnly() + { + if (string.Equals(Environment.GetEnvironmentVariable("AGENT_OS"), AzurePipelines.Linux)) + { + throw new Exception("Flaky on AzP Linux!"); + } + } + + [Fact] + [Flaky("http://example.com", AzurePipelines.Linux, AzurePipelines.macOS)] + public void FlakyInAzPNonWindowsOnly() + { + var agentOs = Environment.GetEnvironmentVariable("AGENT_OS"); + if (string.Equals(agentOs, "Linux") || string.Equals(agentOs, AzurePipelines.macOS)) + { + throw new Exception("Flaky on AzP non-Windows!"); + } + } + } +}