add FlakyAttribute to mark flaky tests

src/TestingUtils/Microsoft.AspNetCore.Testing/src/HelixQueues.cs

@@ -0,0 +1,28 @@
+using System;
+using System.Collections.Generic;
+using System.Reflection;
+
+namespace Microsoft.AspNetCore.Testing
+{
+    public static class HelixQueues
+    {
+        public const string All = "all";
+        public const string None = "none";
+
+        // Queue names end in ';' because it makes it easier to concat these into a list using a constant expression:
+        //  HelixQueues.Fedora28 + HelixQueues.Centos7
+
+        public const string Fedora28 = "Fedora.28." + HelixSuffix + ";";
+        public const string Fedora27 = "Fedora.27." + HelixSuffix + ";";
+        public const string Redhat7 = "Redhat.7." + HelixSuffix + ";";
+        public const string Debian9 = "Debian.9." + HelixSuffix + ";";
+        public const string Debian8 = "Debian.8." + HelixSuffix + ";";
+        public const string Centos7 = "Centos.7." + HelixSuffix + ";";
+        public const string Ubuntu1604 = "Ubuntu.1604." + HelixSuffix + ";";
+        public const string Ubuntu1810 = "Ubuntu.1810." + HelixSuffix + ";";
+        public const string macOS1012 = "OSX.1012." + HelixSuffix + ";";
+        public const string Windows10 = "Windows.10.Amd64.ClientRS4.VS2017.Open;"; // Doesn't have the default suffix!
+
+        private const string HelixSuffix = "Amd64.Open";
+    }
+}

src/TestingUtils/Microsoft.AspNetCore.Testing/src/xunit/FlakyAttribute.cs

@@ -0,0 +1,43 @@
+using System;
+using System.Collections.Generic;
+using Xunit.Sdk;
+
+namespace Microsoft.AspNetCore.Testing.xunit
+{
+    [TraitDiscoverer("Microsoft.AspNetCore.Testing.xunit.FlakyTestDiscoverer", "Microsoft.AspNetCore.Testing")]
+    [AttributeUsage(AttributeTargets.Method)]
+    public sealed class FlakyAttribute : Attribute, ITraitAttribute
+    {
+        private List<string> _queues = new List<string>() { "all" };
+
+        /// <summary>
+        /// Gets a URL to a GitHub issue tracking this flaky test.
+        /// </summary>
+        public string GitHubIssueUrl { get; }
+
+        /// <summary>
+        /// Gets or sets a list of helix queues on which this test is flaky. Defaults to <see cref="HelixQueues.All"/> indicating it is flaky on all Helix queues. See
+        /// <see cref="HelixQueues"/> for a list of valid values.
+        /// </summary>
+        public string OnHelixQueues
+        {
+            get => string.Join(";", _queues);
+            set => _queues = new List<string>(value.Split(';'));
+        }
+
+        /// <summary>
+        /// Gets or sets a boolean indicating if this test is flaky on Azure DevOps Pipelines. Defaults to <see langword="true" />.
+        /// </summary>
+        public bool OnAzDO { get; set; } = true;
+
+        /// <summary>
+        /// Gets a list of Helix queues on which the test is flaky (including <see cref="HelixQueues.All"/> if specified).
+        /// </summary>
+        public IReadOnlyList<string> FlakyQueues => _queues;
+
+        public FlakyAttribute(string gitHubIssueUrl)
+        {
+            GitHubIssueUrl = gitHubIssueUrl;
+        }
+    }
+}

src/TestingUtils/Microsoft.AspNetCore.Testing/src/xunit/FlakyTestDiscoverer.cs

@@ -0,0 +1,35 @@
+using System;
+using System.Collections.Generic;
+using Xunit.Abstractions;
+using Xunit.Sdk;
+
+namespace Microsoft.AspNetCore.Testing.xunit
+{
+    public class FlakyTestDiscoverer : ITraitDiscoverer
+    {
+        public IEnumerable<KeyValuePair<string, string>> GetTraits(IAttributeInfo traitAttribute)
+        {
+            if (traitAttribute is ReflectionAttributeInfo attribute && attribute.Attribute is FlakyAttribute flakyAttribute)
+            {
+                return GetTraitsCore(flakyAttribute);
+            }
+            else
+            {
+                throw new InvalidOperationException("The 'Flaky' attribute is only supported via reflection.");
+            }
+        }
+
+        private IEnumerable<KeyValuePair<string, string>> GetTraitsCore(FlakyAttribute attribute)
+        {
+            if(attribute.OnAzDO)
+            {
+                yield return new KeyValuePair<string, string>("Flaky:AzDO", "true");
+            }
+
+            foreach(var queue in attribute.FlakyQueues)
+            {
+                yield return new KeyValuePair<string, string>($"Flaky:Helix:{queue}", "true");
+            }
+        }
+    }
+}

src/TestingUtils/Microsoft.AspNetCore.Testing/test/FlakyAttributeTest.cs

@@ -0,0 +1,59 @@
+using Microsoft.AspNetCore.Testing.xunit;
+using System;
+using System.Collections.Generic;
+using Xunit;
+
+namespace Microsoft.AspNetCore.Testing.Tests
+{
+    public class FlakyAttributeTest
+    {
+        [Fact]
+        [Flaky("http://example.com")]
+        public void AlwaysFlaky()
+        {
+            throw new Exception("Flakey!");
+        }
+
+        [Fact]
+        [Flaky("http://example.com", OnAzDO = false)]
+        public void FlakyInHelixOnly()
+        {
+            if (!string.IsNullOrEmpty(Environment.GetEnvironmentVariable("HELIX")))
+            {
+                throw new Exception("Flaky on Helix!");
+            }
+        }
+
+        [Fact]
+        [Flaky("http://example.com", OnAzDO = false, OnHelixQueues = HelixQueues.macOS1012 + HelixQueues.Fedora28)]
+        public void FlakyInSpecificHelixQueue()
+        {
+            var queueName = Environment.GetEnvironmentVariable("HELIX");
+            if (!string.IsNullOrEmpty(queueName))
+            {
+
+                // Normalize the queue name to have a trailing ';' (this is only for testing anyway)
+                if (!queueName.EndsWith(";"))
+                {
+                    queueName = $"{queueName};";
+                }
+
+                var failingQueues = new HashSet<string>(StringComparer.OrdinalIgnoreCase) { HelixQueues.macOS1012, HelixQueues.Fedora28 };
+                if (failingQueues.Contains(queueName))
+                {
+                    throw new Exception("Flaky on Helix!");
+                }
+            }
+        }
+
+        [Fact]
+        [Flaky("http://example.com", OnHelixQueues = HelixQueues.None)]
+        public void FlakyInAzDoOnly()
+        {
+            if (!string.IsNullOrEmpty(Environment.GetEnvironmentVariable("BUILD_BUILDNUMBER")))
+            {
+                throw new Exception("Flaky on AzDO!");
+            }
+        }
+    }
+}