Adds test collections filtering

build.bat

@@ -26,6 +26,7 @@ SET APIKEY=
 SET APIKEYPROVIDED="<empty>"
 SET FEED="elasticsearch-net"
 SET NEST_INTEGRATION_CLUSTER=
+SET NEST_TEST_FILTER=
 
 IF /I "%1"=="skiptests" (
 	set SKIPTESTS="1"
@@ -48,6 +49,7 @@ IF /I "%1"=="release" (
 IF /I "%1%"=="integrate" (
 	IF NOT [%2]==[] (set ESVERSIONS="%2")
 	IF NOT [%3]==[] (set NEST_INTEGRATION_CLUSTER="%3")
+	IF NOT [%4]==[] (set NEST_TEST_FILTER="%4")
 )
 
 IF /I "%1%"=="canary" (
@@ -61,5 +63,5 @@ IF /I "%1%"=="canary" (
 	IF /I "%2"=="skiptests" (set SKIPTESTS=1)
 )
 
-ECHO starting build using target=%TARGET% version=%VERSION% esversions=%ESVERSIONS% skiptests=%SKIPTESTS% apiKey=%APIKEYPROVIDED% feed=%FEED% escluster=%NEST_INTEGRATION_CLUSTER%
-"packages\build\FAKE\tools\Fake.exe" "build\\scripts\\Targets.fsx" "target=%TARGET%" "version=%VERSION%" "esversions=%ESVERSIONS%" "skiptests=%SKIPTESTS%" "apiKey=%APIKEY%" "feed=%FEED%" "escluster=%NEST_INTEGRATION_CLUSTER%"
+ECHO starting build using target=%TARGET% version=%VERSION% esversions=%ESVERSIONS% skiptests=%SKIPTESTS% apiKey=%APIKEYPROVIDED% feed=%FEED% escluster=%NEST_INTEGRATION_CLUSTER% testfilter=%NEST_TEST_FILTER%
+"packages\build\FAKE\tools\Fake.exe" "build\\scripts\\Targets.fsx" "target=%TARGET%" "version=%VERSION%" "esversions=%ESVERSIONS%" "skiptests=%SKIPTESTS%" "apiKey=%APIKEY%" "feed=%FEED%" "escluster=%NEST_INTEGRATION_CLUSTER%" "testfilter=%NEST_TEST_FILTER%"

build/scripts/Building.fsx

@@ -38,16 +38,13 @@ type Build() =
             link DotNetFramework.NetStandard1_3
         )
 
-    static let compile target = 
-        compileDesktop target
+    static member Compile() = 
+        //let target = if runningRelease then "Rebuild" else "Build"
+        compileDesktop "Rebuild"
         //we only need this output when doing a release otherwise depend on test to validate the build
         if runningRelease then compileCore()
         if not isMono && runningRelease then gitLink()
 
-    static member QuickCompile() = compile "Build"
-
-    static member Compile() = compile "Rebuild"
-
     static member Clean() =
         CleanDir Paths.BuildOutput
         DotNetProject.All |> Seq.iter(fun p -> CleanDir(Paths.BinFolder p.Name))

build/scripts/Targets.fsx

@@ -32,17 +32,13 @@ Target "Test"  <| fun _ -> Tests.RunUnitTests()
 Target "InheritDoc"  <| fun _ -> InheritDoc.patchInheritDocs()
 
 Target "TestForever"  <| fun _ -> Tests.RunUnitTestsForever()
-
-Target "QuickTest"  <| fun _ -> Tests.RunUnitTests()
 
-Target "Integrate"  <| fun _ -> Tests.RunIntegrationTests() (getBuildParamOrDefault "esversions" "") (getBuildParamOrDefault "escluster" "")
+Target "Integrate"  <| fun _ -> Tests.RunIntegrationTests (getBuildParamOrDefault "esversions" "") (getBuildParamOrDefault "escluster" "") (getBuildParamOrDefault "testfilter" "")
 
 Target "Profile" <| fun _ -> Profiler.Run()
 
 Target "Benchmark" <| fun _ -> Benchmarker.Run()
 
-Target "QuickCompile"  <| fun _ -> Build.QuickCompile()
-
 Target "Version" <| fun _ -> 
     Versioning.PatchAssemblyInfos()
     Versioning.PatchProjectJsons()
@@ -82,9 +78,6 @@ Target "Canary" <| fun _ ->
   ==> "Release"
   ==> "Canary"
 
-"QuickCompile"
-  ==> "QuickTest"
-
 "BuildApp"
   ==> "Integrate"
 

build/scripts/Testing.fsx

@@ -37,7 +37,7 @@ module Tests =
         testProjectJson "all"
 
 
-    let RunIntegrationTests() commaSeparatedEsVersions clusterFilter =
+    let RunIntegrationTests commaSeparatedEsVersions clusterFilter testFilter =
         let esVersions = 
             match commaSeparatedEsVersions with
             | "" -> failwith "when running integrate you have to pass a comma separated list of elasticsearch versions to test"
@@ -46,4 +46,5 @@ module Tests =
         for esVersion in esVersions do
             setProcessEnvironVar "NEST_INTEGRATION_CLUSTER" clusterFilter
             setProcessEnvironVar "NEST_INTEGRATION_VERSION" esVersion
+            setProcessEnvironVar "NEST_TEST_FILTER" testFilter
             testDesktopClr "all"

src/Tests/Framework/Configuration/EnvironmentConfiguration.cs

@@ -12,18 +12,20 @@ public class EnvironmentConfiguration : TestConfigurationBase
 		public override bool TestAgainstAlreadyRunningElasticsearch { get; protected set; } = false;
 		public override bool ForceReseed { get; protected set; } = true;
 		public override ElasticsearchVersion ElasticsearchVersion { get; protected set; } = new ElasticsearchVersion("5.0.0");
-		public override TestMode Mode { get; protected set; } = TestMode.Unit;
-		public override string ClusterFilter { get; protected set; }
-
+		public override TestMode Mode { get; protected set; } = TestMode.Unit;
+		public override string ClusterFilter { get; protected set; }
+		public override string TestFilter { get; protected set; }
+
 		public EnvironmentConfiguration()
 		{
 			//if env var NEST_INTEGRATION_VERSION is set assume integration mode
 			//used by the build script FAKE
 			var version = Environment.GetEnvironmentVariable("NEST_INTEGRATION_VERSION");
-			if (!string.IsNullOrEmpty(version)) Mode = TestMode.Integration;
+			if (!string.IsNullOrEmpty(version)) Mode = TestMode.Integration;
 
 			this.ElasticsearchVersion = new ElasticsearchVersion(string.IsNullOrWhiteSpace(version) ? "5.0.0" : version);
 			this.ClusterFilter = Environment.GetEnvironmentVariable("NEST_INTEGRATION_CLUSTER");
+			this.TestFilter = Environment.GetEnvironmentVariable("NEST_TEST_FILTER");
 		}
 	}
 }

src/Tests/Framework/Configuration/ITestConfiguration.cs

@@ -10,8 +10,9 @@ namespace Tests.Framework.Configuration
 	public interface ITestConfiguration
 	{
 		TestMode Mode { get; }
-		ElasticsearchVersion ElasticsearchVersion { get; }
+		ElasticsearchVersion ElasticsearchVersion { get; }
 		string ClusterFilter { get; }
+		string TestFilter { get; }
 		bool ForceReseed { get; }
 		bool TestAgainstAlreadyRunningElasticsearch { get; }
 

src/Tests/Framework/Configuration/NodeConfiguration.cs

@@ -14,6 +14,7 @@ public class NodeConfiguration : ITestConfiguration
 		public bool RunIntegrationTests { get; }
 		public bool RunUnitTests { get; }
 		public string ClusterFilter { get; }
+		public string TestFilter { get; }
 
 		public string TypeOfCluster { get; set; }
 		public ElasticsearchPlugin[] RequiredPlugins { get; set; } = { };
@@ -29,6 +30,7 @@ public NodeConfiguration(ITestConfiguration configuration)
 			this.RunIntegrationTests = configuration.RunIntegrationTests;
 			this.RunUnitTests = configuration.RunUnitTests;
 			this.ClusterFilter = configuration.ClusterFilter;
+			this.TestFilter = configuration.TestFilter;
 		}
 	}
 }

src/Tests/Framework/Configuration/TestConfigurationBase.cs

@@ -12,8 +12,9 @@ public abstract class TestConfigurationBase : ITestConfiguration
 		public abstract bool TestAgainstAlreadyRunningElasticsearch { get; protected set; }
 		public abstract ElasticsearchVersion ElasticsearchVersion { get; protected set; }
 		public abstract bool ForceReseed { get; protected set; }
-		public abstract TestMode Mode { get; protected set; }
+		public abstract TestMode Mode { get; protected set; }
 		public abstract string ClusterFilter { get; protected set; }
+		public abstract string TestFilter { get; protected set; }
 
 		public virtual bool RunIntegrationTests => Mode == TestMode.Mixed || Mode == TestMode.Integration;
 		public virtual bool RunUnitTests => Mode == TestMode.Mixed || Mode == TestMode.Unit;

src/Tests/Framework/Configuration/YamlConfiguration.cs

@@ -12,20 +12,22 @@ public class YamlConfiguration : TestConfigurationBase
 		public override bool ForceReseed { get; protected set; } = true;
 		public override TestMode Mode { get; protected set; } = TestMode.Unit;
 		public override string ClusterFilter { get; protected set; }
+		public override string TestFilter { get; protected set; }
 
 		public YamlConfiguration(string configurationFile)
 		{
 			if (!File.Exists(configurationFile)) return;
 
 			var config = File.ReadAllLines(configurationFile)
-				.Where(l=>!l.Trim().StartsWith("#"))
+				.Where(l=>!l.Trim().StartsWith("#") && !string.IsNullOrWhiteSpace(l))
 				.ToDictionary(ConfigName, ConfigValue);
 
 			this.Mode = GetTestMode(config["mode"]);
 			this.ElasticsearchVersion = new ElasticsearchVersion(config["elasticsearch_version"]);
 			this.ForceReseed = bool.Parse(config["force_reseed"]);
 			this.TestAgainstAlreadyRunningElasticsearch = bool.Parse(config["test_against_already_running_elasticsearch"]);
 			this.ClusterFilter = config.ContainsKey("cluster_filter") ? config["cluster_filter"] : null;
+			this.TestFilter = config.ContainsKey("test_filter") ? config["test_filter"] : null;
 		}
 
 		private static string ConfigName(string configLine) => Parse(configLine, 0);

src/Tests/Framework/XUnitPlumbing/IntegrationTestDiscoverer.cs

@@ -1,6 +1,7 @@
 using System;
 using System.Linq;
 using System.Reflection;
+using Xunit;
 using Xunit.Abstractions;
 
 namespace Tests.Framework
@@ -15,8 +16,11 @@ public IntegrationTestDiscoverer(IMessageSink diagnosticMessageSink)
 		protected override bool SkipMethod(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo factAttribute)
 		{
 			var classOfMethod = Type.GetType(testMethod.TestClass.Class.Name, true, true);
-			var method = classOfMethod.GetMethod(testMethod.Method.Name);
-			var collectionType = testMethod.TestClass?.TestCollection?.CollectionDefinition?.Name;
+			var method = classOfMethod.GetMethod(testMethod.Method.Name, BindingFlags.FlattenHierarchy | BindingFlags.NonPublic | BindingFlags.Public)
+				?? testMethod.Method.ToRuntimeMethod();
+
+			var collectionType = TestAssemblyRunner.GetClusterForCollection(testMethod.TestClass?.TestCollection);
+
 			return TypeSkipVersionAttributeSatisfies(classOfMethod) ||
 				   MethodSkipVersionAttributeSatisfies(method) ||
 				   RequiresPluginButRunningAgainstSnapshot(classOfMethod, collectionType);

src/Tests/Framework/XUnitPlumbing/NestTestDiscoverer.cs

@@ -25,21 +25,20 @@ public IEnumerable<IXunitTestCase> Discover(ITestFrameworkDiscoveryOptions disco
 				? Enumerable.Empty<IXunitTestCase>()
 				: new[] { new XunitTestCase(DiagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), testMethod) };
 
-		protected static bool RequiresPluginButRunningAgainstSnapshot(Type classOfMethod, string collectionClass)
+		protected static bool RequiresPluginButRunningAgainstSnapshot(Type classOfMethod, Type collectionType)
 		{
 			Attribute[] classAttributes, collectionAttributes = { };
 #if !DOTNETCORE
 			classAttributes = Attribute.GetCustomAttributes(classOfMethod, typeof(RequiresPluginAttribute), true);
 #else
 			classAttributes =  classOfMethod.GetTypeInfo().GetCustomAttributes(typeof(RequiresPluginAttribute), true).ToArray();
 #endif
-			if (collectionClass != null)
+			if (collectionType != null)
 			{
-				var classOfCollection = Type.GetType(collectionClass, true, true);
 #if !DOTNETCORE
-				collectionAttributes = Attribute.GetCustomAttributes(classOfCollection, typeof(RequiresPluginAttribute), true);
+				collectionAttributes = Attribute.GetCustomAttributes(collectionType, typeof(RequiresPluginAttribute), true);
 #else
-				collectionAttributes =  classOfCollection.GetTypeInfo().GetCustomAttributes(typeof(RequiresPluginAttribute), true).ToArray();
+				collectionAttributes =  collectionType.GetTypeInfo().GetCustomAttributes(typeof(RequiresPluginAttribute), true).ToArray();
 #endif
 			}
 			if (!classAttributes.Concat(collectionAttributes).Any()) return false;

src/Tests/Framework/XUnitPlumbing/UnitTestDiscoverer.cs

@@ -1,4 +1,5 @@
 using System;
+using Xunit;
 using Xunit.Abstractions;
 
 namespace Tests.Framework
@@ -11,9 +12,9 @@ public UnitTestDiscoverer(IMessageSink diagnosticMessageSink)
 		protected override bool SkipMethod(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo factAttribute)
 		{
 			var classOfMethod = Type.GetType(testMethod.TestClass.Class.Name, true, true);
-			var collectionType = testMethod.TestClass?.TestCollection?.CollectionDefinition?.Name;
 			//in mixed mode we do not want to run any api tests for plugins when running against a snapshot
 			//because the client is "hot"
+			var collectionType = TestAssemblyRunner.GetClusterForCollection(testMethod.TestClass?.TestCollection);
 			return TestClient.Configuration.RunIntegrationTests && RequiresPluginButRunningAgainstSnapshot(classOfMethod, collectionType);
 		}
 	}

src/Tests/Framework/Xunit/TestAssemblyRunner.cs

@@ -15,7 +15,6 @@ namespace Xunit
 {
 	class TestAssemblyRunner : XunitTestAssemblyRunner
 	{
-
 		readonly Dictionary<Type, object> assemblyFixtureMappings = new Dictionary<Type, object>();
 
 		public TestAssemblyRunner(ITestAssembly testAssembly,
@@ -27,9 +26,11 @@ public TestAssemblyRunner(ITestAssembly testAssembly,
 		{
 		}
 
-		protected override Task<RunSummary> RunTestCollectionAsync(IMessageBus bus, ITestCollection col, IEnumerable<IXunitTestCase> testCases, CancellationTokenSource cts) =>
+		protected override Task<RunSummary> RunTestCollectionAsync(IMessageBus bus, ITestCollection col,
+				IEnumerable<IXunitTestCase> testCases, CancellationTokenSource cts) =>
 			new TestCollectionRunner(
-				assemblyFixtureMappings, col, testCases, DiagnosticMessageSink, bus, TestCaseOrderer, new ExceptionAggregator(Aggregator), cts
+				assemblyFixtureMappings, col, testCases, DiagnosticMessageSink, bus, TestCaseOrderer,
+				new ExceptionAggregator(Aggregator), cts
 			).RunAsync();
 
 		private class GroupedByCluster
@@ -39,16 +40,17 @@ private class GroupedByCluster
 			public List<IXunitTestCase> TestCases { get; set; }
 		}
 
-		protected override async Task<RunSummary> RunTestCollectionsAsync(IMessageBus messageBus, CancellationTokenSource cancellationTokenSource)
+		protected override async Task<RunSummary> RunTestCollectionsAsync(IMessageBus messageBus,
+			CancellationTokenSource cancellationTokenSource)
 		{
-
 			//bit side effecty, sets up assemblyFixtureMapping before possibly letting xunit do its regular concurrency thing
 			var grouped = (from c in OrderTestCollections()
-						   let cluster = ClusterFixture(c.Item1)
-						   let testcase = new GroupedByCluster { Collection = c.Item1, TestCases = c.Item2, Cluster = cluster }
-						   group testcase by testcase.Cluster into g
-						   orderby g.Count() descending
-						   select g).ToList();
+				let cluster = ClusterFixture(c.Item1)
+				let testcase = new GroupedByCluster {Collection = c.Item1, TestCases = c.Item2, Cluster = cluster}
+				group testcase by testcase.Cluster
+				into g
+				orderby g.Count() descending
+				select g).ToList();
 
 			//If we are not running any integration tests we do not care about only keeping a single IClusterFixture
 			//active at a time, so let xunit do what it does best.
@@ -65,12 +67,14 @@ orderby g.Count() descending
 			var summaries = new ConcurrentBag<RunSummary>();
 			var clusterTotals = new Dictionary<string, Stopwatch>();
 			var clusterFilter = TestClient.Configuration.ClusterFilter;
+			var testFilter = TestClient.Configuration.TestFilter;
 			foreach (var group in grouped)
 			{
 				var type = group.Key?.GetType();
 				var clusterName = type?.Name.Replace("Cluster", "") ?? "UNKNOWN";
 
-				if (!string.IsNullOrWhiteSpace(clusterFilter) && !string.Equals(clusterName, clusterFilter, StringComparison.OrdinalIgnoreCase))
+				if (!string.IsNullOrWhiteSpace(clusterFilter) &&
+				    !string.Equals(clusterName, clusterFilter, StringComparison.OrdinalIgnoreCase))
 					continue;
 
 				var dop = group.Key != null && group.Key.MaxConcurrency > 0
@@ -86,20 +90,31 @@ orderby g.Count() descending
 					group.Key?.Start();
 					await group.ForEachAsync(dop, async g =>
 					{
+						var test = g.Collection.DisplayName.Replace("Test collection for", "");
+						if (!string.IsNullOrWhiteSpace(testFilter) &&
+						    test.IndexOf(testFilter, StringComparison.OrdinalIgnoreCase) < 0)
+							return;
+
+						//display tests we execute when we filter so we get confirmation on the command line we run the tests we expect
+						if (!string.IsNullOrWhiteSpace(testFilter))
+							Console.WriteLine(" -> " + test);
+
 						try
 						{
 							var summary = await RunTestCollectionAsync(messageBus, g.Collection, g.TestCases, cancellationTokenSource);
 							summaries.Add(summary);
 						}
-						catch (TaskCanceledException) { }
-					});
+						catch (TaskCanceledException)
+						{
+						}
+				});
 				}
 				clusterTotals[clusterName].Stop();
 			}
 
 			Console.WriteLine("--------");
 			Console.WriteLine("Individual cluster running times");
-			foreach(var kv in clusterTotals)
+			foreach (var kv in clusterTotals)
 				Console.WriteLine($"- {kv.Key}: {kv.Value.Elapsed.ToString()}");
 			Console.WriteLine("--------");
 
@@ -114,12 +129,7 @@ await group.ForEachAsync(dop, async g =>
 
 		protected ClusterBase ClusterFixture(ITestCollection testCollection)
 		{
-			var collectionTypeName = testCollection.DisplayName.Split(' ').Last();
-			var collectionType = Type.GetType(collectionTypeName);
-			var clusterType = collectionType.GetTypeInfo().ImplementedInterfaces
-						.Where(i => i.GetTypeInfo().IsGenericType && i.GetGenericTypeDefinition() == typeof(IClusterFixture<>))
-						.Select(i => i.GetTypeInfo().GenericTypeArguments.Single())
-						.FirstOrDefault();
+			var clusterType = GetClusterForCollection(testCollection);
 
 			ClusterBase cluster = null;
 			if (clusterType == null) return null;
@@ -133,5 +143,16 @@ protected ClusterBase ClusterFixture(ITestCollection testCollection)
 			});
 			return cluster;
 		}
+
+		public static Type GetClusterForCollection(ITestCollection testCollection)
+		{
+			var collectionTypeName = testCollection.DisplayName.Split(' ').Last();
+			var collectionType = Type.GetType(collectionTypeName);
+			var clusterType = collectionType.GetTypeInfo().ImplementedInterfaces
+				.Where(i => i.GetTypeInfo().IsGenericType && i.GetGenericTypeDefinition() == typeof(IClusterFixture<>))
+				.Select(i => i.GetTypeInfo().GenericTypeArguments.Single())
+				.FirstOrDefault();
+			return clusterType;
+		}
 	}
-}
+}

src/Tests/Framework/Xunit/TestFramework.cs

@@ -1,7 +1,7 @@
 using System;
 using System.Reflection;
 using Tests.Framework;
-using Tests.Framework.Configuration;
+using Tests.Framework.Configuration;
 using Xunit.Abstractions;
 using Xunit.Sdk;
 
@@ -20,9 +20,13 @@ protected override ITestFrameworkExecutor CreateExecutor(AssemblyName assemblyNa
 			Console.WriteLine($" - {nameof(config.TestAgainstAlreadyRunningElasticsearch)}: {config.TestAgainstAlreadyRunningElasticsearch}");
 			Console.WriteLine($" - {nameof(config.ElasticsearchVersion)}: {config.ElasticsearchVersion}");
 			Console.WriteLine($" - {nameof(config.ForceReseed)}: {config.ForceReseed}");
-			Console.WriteLine($" - {nameof(config.Mode)}: {config.Mode.ToString()}");
-			if (config.Mode == TestMode.Integration)
+			Console.WriteLine($" - {nameof(config.Mode)}: {config.Mode.ToString()}");
+			if (config.Mode == TestMode.Integration)
+			{
 				Console.WriteLine($" - {nameof(config.ClusterFilter)}: {config.ClusterFilter}");
+				Console.WriteLine($" - {nameof(config.TestFilter)}: {config.TestFilter}");
+
+			}
 			Console.WriteLine($" - {nameof(config.RunIntegrationTests)}: {config.RunIntegrationTests}");
 			Console.WriteLine($" - {nameof(config.RunUnitTests)}: {config.RunUnitTests}");