Include similarity in index settings (#2924)

Similarity settings are nested under settings>index
Rename DefaultSimilarity to ClassicSimilarity
Add DFISimilarity
Add similarity settings to create index integration tests
Add XML comments for each similarity
Refactor SimilarityJsonConverter to not look up the NEST ISimilarity type with Type.GetType() but to infer it from the type property value directly.
Add Similarity to the updatable index settings

Closes #2890

Author: russcam

src/Nest/Aggregations/AggregationContainer.cs

@@ -45,7 +45,7 @@ public static implicit operator AggregationDictionary(AggregationBase aggregator
 
 		public void Add(string key, AggregationContainer value) => this.BackingDictionary.Add(ValidateKey(key), value);
 
-		public override string ValidateKey(string key)
+		protected override string ValidateKey(string key)
 		{
 			if (AggregateJsonConverter.AllReservedAggregationNames.Contains(key))
 				throw new ArgumentException(

src/Nest/CommonAbstractions/DictionaryLike/IsADictionaryBase.cs

@@ -10,7 +10,7 @@ public abstract class IsADictionaryBase<TKey, TValue> : IIsADictionary<TKey, TVa
 		protected Dictionary<TKey, TValue> BackingDictionary { get; }
 		private ICollection<KeyValuePair<TKey, TValue>> Self => BackingDictionary;
 
-		protected IsADictionaryBase() { this.BackingDictionary = new Dictionary<TKey, TValue>(); }
+		protected IsADictionaryBase() => this.BackingDictionary = new Dictionary<TKey, TValue>();
 
 		protected IsADictionaryBase(IDictionary<TKey, TValue> backingDictionary)
 		{
@@ -48,18 +48,18 @@ void ICollection<KeyValuePair<TKey, TValue>>.Add(KeyValuePair<TKey, TValue> item
 		bool IDictionary<TKey, TValue>.Remove(TKey key) => this.BackingDictionary.Remove(key);
 		bool IDictionary<TKey, TValue>.TryGetValue(TKey key, out TValue value) => this.BackingDictionary.TryGetValue(key, out value);
 
-		public virtual TKey ValidateKey(TKey key) => key;
+		protected virtual TKey ValidateKey(TKey key) => key;
 
 		TValue IDictionary<TKey, TValue>.this[TKey key]
 		{
-			get { return this.BackingDictionary[key]; }
-			set { this.BackingDictionary[ValidateKey(key)] = value; }
+			get => this.BackingDictionary[key];
+			set => this.BackingDictionary[ValidateKey(key)] = value;
 		}
 
 		public TValue this[TKey key]
 		{
-			get { return this.BackingDictionary[key]; }
-			set { this.BackingDictionary[ValidateKey(key)] = value; }
+			get => this.BackingDictionary[key];
+			set => this.BackingDictionary[ValidateKey(key)] = value;
 		}
 	}
 }

src/Nest/IndexModules/IndexSettings/IndexState.cs

@@ -13,9 +13,6 @@ public interface IIndexState
 
 		[JsonProperty("mappings")]
 		IMappings Mappings { get; set; }
-
-		[JsonProperty("similarity")]
-		ISimilarities Similarity { get; set; }
 	}
 
 	public class IndexState : IIndexState
@@ -25,7 +22,5 @@ public class IndexState : IIndexState
 		public IMappings Mappings { get; set; }
 
 		public IAliases Aliases { get; set; }
-
-		public ISimilarities Similarity { get; set; }
 	}
 }

src/Nest/IndexModules/IndexSettings/Settings/DynamicIndexSettings.cs

@@ -91,6 +91,11 @@ public interface IDynamicIndexSettings : IIsADictionary<string, object>
 		/// Configure analysis
 		/// </summary>
 		IAnalysis Analysis { get; set; }
+
+		/// <summary>
+		/// Configure similarity
+		/// </summary>
+		ISimilarities Similarity { get; set; }
 	}
 
 	public class DynamicIndexSettings : IsADictionaryBase<string, object>, IDynamicIndexSettings
@@ -147,6 +152,9 @@ public DynamicIndexSettings(IDictionary<string, object> container) : base(contai
 		/// <inheritdoc/>
 		public IAnalysis Analysis { get; set; }
 
+		/// <inheritdoc/>
+		public ISimilarities Similarity { get; set; }
+
 		/// <summary>
 		/// Add any setting to the index
 		/// </summary>
@@ -227,7 +235,12 @@ public TDescriptor Translog(Func<TranslogSettingsDescriptor, ITranslogSettings>
 		public TDescriptor UnassignedNodeLeftDelayedTimeout(Time time) =>
 			Assign(a => a.UnassignedNodeLeftDelayedTimeout = time);
 
+		/// <inheritdoc/>
 		public TDescriptor Analysis(Func<AnalysisDescriptor, IAnalysis> selector) =>
 			Assign(a => a.Analysis = selector?.Invoke(new AnalysisDescriptor()));
+
+		/// <inheritdoc/>
+		public TDescriptor Similarity(Func<SimilaritiesDescriptor, IPromise<ISimilarities>> selector) =>
+			Assign(a => a.Similarity = selector?.Invoke(new SimilaritiesDescriptor())?.Value);
 	}
 }

src/Nest/IndexModules/IndexSettings/Settings/IndexSettingsConverter.cs

@@ -75,6 +75,7 @@ public override void WriteJson(JsonWriter writer, object value, JsonSerializer s
 			d[UpdatableIndexSettings.SlowlogIndexingSource] = indexing?.Source;
 
 			d[UpdatableIndexSettings.Analysis] = ds.Analysis;
+			d[UpdatableIndexSettings.Similarity] = ds.Similarity;
 
 			var indexSettings = value as IIndexSettings;
 
@@ -94,7 +95,17 @@ public override void WriteJson(JsonWriter writer, object value, JsonSerializer s
 			base.WriteJson(writer, d, serializer);
 		}
 
-		private object AsArrayOrSingleItem<T>(IEnumerable<T> items)
+		public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer)
+		{
+			var s = new IndexSettings();
+			SetKnownIndexSettings(reader, serializer, s);
+			if (!typeof (IUpdateIndexSettingsRequest).IsAssignableFrom(objectType)) return s;
+
+			var request = new UpdateIndexSettingsRequest() { IndexSettings =  s};
+			return request;
+		}
+
+		private static object AsArrayOrSingleItem<T>(IEnumerable<T> items)
 		{
 			if (items == null || !items.Any())
 				return null;
@@ -105,29 +116,21 @@ private object AsArrayOrSingleItem<T>(IEnumerable<T> items)
 			return items;
 		}
 
-		public JObject Flatten(JObject original, string prefix = "", JObject newObject = null)
+		private static JObject Flatten(JObject original, string prefix = "", JObject newObject = null)
 		{
 			newObject = newObject ?? new JObject();
 			foreach (var property in original.Properties())
 			{
-				if (property.Value is JObject && property.Name != UpdatableIndexSettings.Analysis)
+				if (property.Value is JObject &&
+				    property.Name != UpdatableIndexSettings.Analysis &&
+				    property.Name != UpdatableIndexSettings.Similarity)
 					Flatten(property.Value.Value<JObject>(), prefix + property.Name + ".", newObject);
 				else newObject.Add(prefix + property.Name, property.Value);
 			}
 			return newObject;
 		}
 
-		public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer)
-		{
-			var s = new IndexSettings();
-			SetKnownIndexSettings(reader, serializer, s);
-			if (!typeof (IUpdateIndexSettingsRequest).IsAssignableFrom(objectType)) return s;
-
-			var request = new UpdateIndexSettingsRequest() { IndexSettings =  s};
-			return request;
-		}
-
-		private void SetKnownIndexSettings(JsonReader reader, JsonSerializer serializer, IIndexSettings s)
+		private static void SetKnownIndexSettings(JsonReader reader, JsonSerializer serializer, IIndexSettings s)
 		{
 			var settings = Flatten(JObject.Load(reader)).Properties().ToDictionary(kv => kv.Name);
 
@@ -219,6 +222,8 @@ private void SetKnownIndexSettings(JsonReader reader, JsonSerializer serializer,
 				var setting = kv.Value;
 				if (kv.Key == UpdatableIndexSettings.Analysis || kv.Key == "index.analysis")
 					s.Analysis = setting.Value.Value<JObject>().ToObject<Analysis>(serializer);
+				if (kv.Key == UpdatableIndexSettings.Similarity || kv.Key == "index.similarity")
+					s.Similarity = setting.Value.Value<JObject>().ToObject<Similarities>(serializer);
 				else
 				{
 					dict?.Add(kv.Key, serializer.Deserialize(kv.Value.Value.CreateReader()));

src/Nest/IndexModules/IndexSettings/Settings/UpdatableIndexSettings.cs

@@ -50,6 +50,8 @@ public static class UpdatableIndexSettings
 		public const string MergeSchedulerMaxThreadCount = "index.merge.scheduler.max_thread_count";
 		public const string MergeSchedulerAutoThrottle = "index.merge.scheduler.auto_throttle";
 
+		public const string Similarity = "similarity";
+
 		public const string SlowlogSearchThresholdQueryWarn = "index.search.slowlog.threshold.query.warn";
 		public const string SlowlogSearchThresholdQueryInfo = "index.search.slowlog.threshold.query.info";
 		public const string SlowlogSearchThresholdQueryDebug = "index.search.slowlog.threshold.query.debug";

src/Nest/IndexModules/Similarity/BM25Similarity.cs

@@ -3,8 +3,8 @@
 namespace Nest
 {
 	/// <summary>
-	/// BM25 Similarity. Introduced in Stephen E. Robertson, Steve Walker, Susan Jones, Micheline Hancock-Beaulieu, 
-	/// and Mike Gatford. Okapi at TREC-3. In Proceedings of the Third Text REtrieval Conference (TREC 1994). Gaithersburg, USA, November 1994.
+	/// BM25 Similarity. Introduced in Stephen E. Robertson, Steve Walker, Susan Jones, Micheline Hancock-Beaulieu,
+	/// and Mike Gatford. Okapi at TREC-3. In Proceedings of the Third Text Retrieval Conference (TREC 1994). Gaithersburg, USA, November 1994.
 	/// </summary>
 	public interface IBM25Similarity : ISimilarity
 	{
@@ -41,7 +41,7 @@ public class BM25Similarity : IBM25Similarity
 		public bool? DiscountOverlaps { get; set; }
 	}
 	/// <inheritdoc/>
-	public class BM25SimilarityDescriptor 
+	public class BM25SimilarityDescriptor
 		: DescriptorBase<BM25SimilarityDescriptor, IBM25Similarity>, IBM25Similarity
 	{
 		string ISimilarity.Type => "BM25";
@@ -50,11 +50,11 @@ public class BM25SimilarityDescriptor
 		double? IBM25Similarity.B { get; set; }
 
 		/// <inheritdoc/>
-		public BM25SimilarityDescriptor DiscountOverlaps(bool? discount = true) => Assign(a => a.DiscountOverlaps = discount);
+		public BM25SimilarityDescriptor DiscountOverlaps(bool discount = true) => Assign(a => a.DiscountOverlaps = discount);
 		/// <inheritdoc/>
-		public BM25SimilarityDescriptor K1(double? k1) => Assign(a => a.K1 = k1);
+		public BM25SimilarityDescriptor K1(double k1) => Assign(a => a.K1 = k1);
 		/// <inheritdoc/>
-		public BM25SimilarityDescriptor B(double? b) => Assign(a => a.B = b);
+		public BM25SimilarityDescriptor B(double b) => Assign(a => a.B = b);
 	}
 
 }

src/Nest/IndexModules/Similarity/ClassicSimilarity.cs

@@ -0,0 +1,37 @@
+using Newtonsoft.Json;
+
+namespace Nest
+{
+	/// <summary>
+	/// The classic similarity that is based on the TF/IDF model.
+	/// </summary>
+	public interface IClassicSimilarity : ISimilarity
+	{
+		/// <summary>
+		/// Determines whether overlap tokens (tokens with 0 position increment) are ignored when computing norm.
+		/// By default this is <c>true</c>, meaning overlap tokens do not count when computing norms.
+		/// </summary>
+		[JsonProperty("discount_overlaps")]
+		bool? DiscountOverlaps { get; set; }
+	}
+
+	/// <inheritdoc />
+	public class ClassicSimilarity : IClassicSimilarity
+	{
+		public string Type => "classic";
+
+		/// <inheritdoc />
+		public bool? DiscountOverlaps { get; set; }
+	}
+
+	/// <inheritdoc />
+	public class ClassicSimilarityDescriptor
+		: DescriptorBase<ClassicSimilarityDescriptor, IClassicSimilarity>, IClassicSimilarity
+	{
+		string ISimilarity.Type => "classic";
+		bool? IClassicSimilarity.DiscountOverlaps { get; set; }
+
+		/// <inheritdoc />
+		public ClassicSimilarityDescriptor DiscountOverlaps(bool discount = true) => Assign(a => a.DiscountOverlaps = discount);
+	}
+}

src/Nest/IndexModules/Similarity/CustomSimilarity.cs

@@ -4,33 +4,41 @@
 
 namespace Nest
 {
+	/// <summary>
+	/// A custom similarity
+	/// </summary>
 	public interface ICustomSimilarity : ISimilarity, IIsADictionary<string, object> { }
 
 	/// <inheritdoc/>
 	public class CustomSimilarity : IsADictionaryBase<string, object>, ICustomSimilarity
 	{
-		public string Type { get { return this["type"] as string; } set { this.Add("type", value); } }
+		public string Type
+		{
+			get => this["type"] as string;
+			set => this.Add("type", value);
+		}
 
 		public CustomSimilarity(string type)
 		{
 			if (!string.IsNullOrEmpty(type)) this.Type = type;
 		}
 
 		internal CustomSimilarity(IDictionary<string, object> container) : base(container) { }
+
 		internal CustomSimilarity(Dictionary<string, object> container)
-			: base(container.Select(kv => kv).ToDictionary(kv => kv.Key, kv => kv.Value))
-		{}
+			: base(container.Select(kv => kv).ToDictionary(kv => kv.Key, kv => kv.Value)) { }
 
 		public void Add(string key, object value) => BackingDictionary.Add(key, value);
 	}
+
 	/// <inheritdoc/>
 	public class CustomSimilarityDescriptor
 		: IsADictionaryDescriptorBase<CustomSimilarityDescriptor, ICustomSimilarity, string, object>
 	{
-
 		public CustomSimilarityDescriptor() : base(new CustomSimilarity(string.Empty)) { }
 
 		internal CustomSimilarityDescriptor Type(string type) => Assign("type", type);
+
 		public CustomSimilarityDescriptor Add(string key, object value) => Assign(key, value);
 	}
 

src/Nest/IndexModules/Similarity/DFI/DFIIndependenceMeasure.cs

@@ -0,0 +1,22 @@
+using System.Runtime.Serialization;
+using Newtonsoft.Json;
+using Newtonsoft.Json.Converters;
+
+namespace Nest
+{
+	/// <summary>
+	/// <see cref="IDFISimilarity"/> independence measure
+	/// </summary>
+	[JsonConverter(typeof(StringEnumConverter))]
+	public enum DFIIndependenceMeasure
+	{
+		[EnumMember(Value = "standardized")]
+		Standardized,
+
+		[EnumMember(Value = "saturated")]
+		Saturated,
+
+		[EnumMember(Value = "chisquared")]
+		ChiSquared
+	}
+}