Skip to content

Commit c7e51cb

Browse files
Mpdreamzrusscam
Mpdreamz
authored and
russcam
committed
Add support for nori plugin (#3430)
* add analysis-nori plugin to writable cluster and added nori_tokenizer * add nori_part_of_speech token filter * add nori analyzer * add Nori() to AnalyzeTokenDescriptor, CodeStandard tests caught this
1 parent 2130ddf commit c7e51cb

File tree

13 files changed

+248
-3
lines changed

13 files changed

+248
-3
lines changed

‎src/Nest/Analysis/Analyzers/AnalyzerJsonConverter.cs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ public override object ReadJson(JsonReader reader, Type objectType, object exist
2929
case "simple": return o.ToObject<SimpleAnalyzer>(ElasticContractResolver.Empty);
3030
case "fingerprint": return o.ToObject<FingerprintAnalyzer>(ElasticContractResolver.Empty);
3131
case "kuromoji": return o.ToObject<KuromojiAnalyzer>(ElasticContractResolver.Empty);
32+
case "nori": return o.ToObject<NoriAnalyzer>(ElasticContractResolver.Empty);
3233
default:
3334
if (o.Property("tokenizer") != null)
3435
return o.ToObject<CustomAnalyzer>(ElasticContractResolver.Empty);

‎src/Nest/Analysis/Analyzers/Analyzers.cs‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -97,5 +97,9 @@ public AnalyzersDescriptor Fingerprint(string name, Func<FingerprintAnalyzerDesc
9797
/// </summary>
9898
public AnalyzersDescriptor Kuromoji(string name, Func<KuromojiAnalyzerDescriptor, IKuromojiAnalyzer> selector = null) =>
9999
Assign(name, selector.InvokeOrDefault(new KuromojiAnalyzerDescriptor()));
100+
101+
/// <inheritdoc cref="INoriAnalyzer"/>
102+
public AnalyzersDescriptor Nori(string name, Func<NoriAnalyzerDescriptor, INoriAnalyzer> selector) =>
103+
Assign(name, selector?.Invoke(new NoriAnalyzerDescriptor()));
100104
}
101105
}

‎src/Nest/Analysis/Analyzers/NoriAnalyzer.cs‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
using System.Collections.Generic;
2+
using Newtonsoft.Json;
3+
4+
namespace Nest
5+
{
6+
/// <summary>
7+
///The nori analyzer consists of the following tokenizer and token filters:
8+
///<para> - nori_tokenizer</para>
9+
///<para> - nori_part_of_speech token filter</para>
10+
///<para> - nori_readingform token filter</para>
11+
///<para> - lowercase token filter</para>
12+
/// </summary>
13+
public interface INoriAnalyzer : IAnalyzer
14+
{
15+
/// <inheritdoc cref="INoriTokenizer.DecompoundMode"/>
16+
[JsonProperty("decompound_mode")]
17+
NoriDecompoundMode? DecompoundMode { get; set; }
18+
19+
/// <inheritdoc cref="INoriTokenizer.UserDictionary"/>
20+
[JsonProperty("user_dictionary")]
21+
string UserDictionary { get; set; }
22+
23+
/// <inheritdoc cref="INoriPartOfSpeechTokenFilter.StopTags"/>
24+
[JsonProperty("stoptags")]
25+
IEnumerable<string> StopTags { get; set; }
26+
}
27+
28+
/// <inheritdoc cref="INoriAnalyzer"/>
29+
public class NoriAnalyzer : AnalyzerBase, INoriAnalyzer
30+
{
31+
public NoriAnalyzer() : base("nori") {}
32+
33+
/// <inheritdoc cref="INoriTokenizer.DecompoundMode"/>
34+
public NoriDecompoundMode? DecompoundMode { get; set; }
35+
36+
/// <inheritdoc cref="INoriTokenizer.UserDictionary"/>
37+
public string UserDictionary { get; set; }
38+
39+
///<inheritdoc cref="INoriPartOfSpeechTokenFilter.StopTags" />
40+
public IEnumerable<string> StopTags { get; set; }
41+
}
42+
43+
/// <inheritdoc cref="INoriAnalyzer"/>
44+
public class NoriAnalyzerDescriptor : AnalyzerDescriptorBase<NoriAnalyzerDescriptor, INoriAnalyzer>, INoriAnalyzer
45+
{
46+
protected override string Type => "nori";
47+
48+
NoriDecompoundMode? INoriAnalyzer.DecompoundMode { get; set; }
49+
string INoriAnalyzer.UserDictionary { get; set; }
50+
IEnumerable<string> INoriAnalyzer.StopTags { get; set; }
51+
52+
/// <inheritdoc cref="INoriTokenizer.DecompoundMode"/>
53+
public NoriAnalyzerDescriptor DecompoundMode(NoriDecompoundMode? mode) => Assign(a => a.DecompoundMode = mode);
54+
55+
/// <inheritdoc cref="INoriTokenizer.UserDictionary"/>
56+
public NoriAnalyzerDescriptor UserDictionary(string path) => Assign(a => a.UserDictionary = path);
57+
58+
///<inheritdoc cref="INoriPartOfSpeechTokenFilter.StopTags" />
59+
public NoriAnalyzerDescriptor StopTags(IEnumerable<string> stopTags) => Assign(a => a.StopTags = stopTags);
60+
61+
///<inheritdoc cref="INoriPartOfSpeechTokenFilter.StopTags" />
62+
public NoriAnalyzerDescriptor StopTags(params string[] stopTags) => Assign(a => a.StopTags = stopTags);
63+
64+
}
65+
}

‎src/Nest/Analysis/TokenFilters/NoriPartOfSpeechTokenFilter.cs‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
using System.Collections.Generic;
2+
using Newtonsoft.Json;
3+
4+
namespace Nest
5+
{
6+
/// <summary> The nori_part_of_speech token filter removes tokens that match a set of part-of-speech tags. </summary>
7+
public interface INoriPartOfSpeechTokenFilter : ITokenFilter
8+
{
9+
/// <summary> An array of part-of-speech tags that should be removed. </summary>
10+
[JsonProperty("stoptags")]
11+
IEnumerable<string> StopTags { get; set; }
12+
}
13+
14+
/// <inheritdoc cref="INoriPartOfSpeechTokenFilter"/>
15+
public class NoriPartOfSpeechTokenFilter : TokenFilterBase, INoriPartOfSpeechTokenFilter
16+
{
17+
public NoriPartOfSpeechTokenFilter() : base("nori_part_of_speech") { }
18+
19+
///<inheritdoc cref="INoriPartOfSpeechTokenFilter.StopTags" />
20+
public IEnumerable<string> StopTags { get; set; }
21+
}
22+
/// <inheritdoc cref="INoriPartOfSpeechTokenFilter"/>
23+
public class NoriPartOfSpeechTokenFilterDescriptor
24+
: TokenFilterDescriptorBase<NoriPartOfSpeechTokenFilterDescriptor, INoriPartOfSpeechTokenFilter>, INoriPartOfSpeechTokenFilter
25+
{
26+
protected override string Type => "nori_part_of_speech";
27+
28+
IEnumerable<string> INoriPartOfSpeechTokenFilter.StopTags { get; set; }
29+
30+
///<inheritdoc cref="INoriPartOfSpeechTokenFilter.StopTags" />
31+
public NoriPartOfSpeechTokenFilterDescriptor StopTags(IEnumerable<string> stopTags) => Assign(a => a.StopTags = stopTags);
32+
33+
///<inheritdoc cref="INoriPartOfSpeechTokenFilter.StopTags" />
34+
public NoriPartOfSpeechTokenFilterDescriptor StopTags(params string[] stopTags) => Assign(a => a.StopTags = stopTags);
35+
36+
}
37+
38+
}

‎src/Nest/Analysis/TokenFilters/TokenFilters.cs‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -318,6 +318,10 @@ public TokenFiltersDescriptor IcuNormalization(string name, Func<IcuNormalizatio
318318
public TokenFiltersDescriptor IcuTransform(string name, Func<IcuTransformTokenFilterDescriptor, IIcuTransformTokenFilter> selector) =>
319319
Assign(name, selector.Invoke(new IcuTransformTokenFilterDescriptor()));
320320

321+
/// <inheritdoc cref="INoriPartOfSpeechTokenFilter"/>
322+
public TokenFiltersDescriptor NoriPartOfSpeech(string name, Func<NoriPartOfSpeechTokenFilterDescriptor, INoriPartOfSpeechTokenFilter> selector) =>
323+
Assign(name, selector.Invoke(new NoriPartOfSpeechTokenFilterDescriptor()));
324+
321325
/// <summary>
322326
/// A token filter of type multiplexer will emit multiple tokens at the same position, each version of the token
323327
/// having been run through a different filter. Identical output tokens at the same position will be removed.

‎src/Nest/Analysis/Tokenizers/NoriTokenizer.cs‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
using System.Runtime.Serialization;
2+
using Newtonsoft.Json;
3+
using Newtonsoft.Json.Converters;
4+
5+
namespace Nest
6+
{
7+
/// <summary> The decompound mode determines how the tokenizer handles compound tokens. </summary>
8+
[JsonConverter(typeof(StringEnumConverter))]
9+
public enum NoriDecompoundMode
10+
{
11+
/// <summary> Decomposes compounds and discards the original form (default). </summary>
12+
[EnumMember(Value="discard")]
13+
Discard,
14+
/// <summary> No decomposition for compounds </summary>
15+
[EnumMember(Value="none")]
16+
None,
17+
/// <summary> Decomposes compounds and keeps the original form </summary>
18+
[EnumMember(Value="mixed")]
19+
Mixed
20+
}
21+
22+
/// <summary> Tokenizer that ships with the analysis-nori plugin</summary>
23+
public interface INoriTokenizer : ITokenizer
24+
{
25+
/// <summary>
26+
/// The regular expression pattern, defaults to \W+.
27+
/// </summary>
28+
[JsonProperty("decompound_mode")]
29+
NoriDecompoundMode? DecompoundMode { get; set; }
30+
31+
/// <summary>
32+
/// The Nori tokenizer uses the mecab-ko-dic dictionary by default. A user_dictionary with custom nouns (NNG) may be appended to
33+
/// the default dictionary. This property allows you to specify this file on disk
34+
/// </summary>
35+
[JsonProperty("user_dictionary")]
36+
string UserDictionary { get; set; }
37+
}
38+
39+
/// <inheritdoc cref="INoriTokenizer"/>
40+
public class NoriTokenizer : TokenizerBase, INoriTokenizer
41+
{
42+
public NoriTokenizer() => this.Type = "nori_tokenizer";
43+
/// <inheritdoc cref="INoriTokenizer.DecompoundMode"/>
44+
public NoriDecompoundMode? DecompoundMode { get; set; }
45+
/// <inheritdoc cref="INoriTokenizer.UserDictionary"/>
46+
public string UserDictionary { get; set; }
47+
}
48+
/// <inheritdoc cref="INoriTokenizer"/>
49+
public class NoriTokenizerDescriptor
50+
: TokenizerDescriptorBase<NoriTokenizerDescriptor, INoriTokenizer>, INoriTokenizer
51+
{
52+
protected override string Type => "nori_tokenizer";
53+
54+
NoriDecompoundMode? INoriTokenizer.DecompoundMode { get; set; }
55+
string INoriTokenizer.UserDictionary { get; set; }
56+
57+
/// <inheritdoc cref="INoriTokenizer.DecompoundMode"/>
58+
public NoriTokenizerDescriptor DecompoundMode(NoriDecompoundMode? mode) => Assign(a => a.DecompoundMode = mode);
59+
60+
/// <inheritdoc cref="INoriTokenizer.UserDictionary"/>
61+
public NoriTokenizerDescriptor UserDictionary(string path) => Assign(a => a.UserDictionary = path);
62+
}
63+
}

‎src/Nest/Analysis/Tokenizers/Tokenizers.cs‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -113,6 +113,10 @@ public TokenizersDescriptor Kuromoji(string name, Func<KuromojiTokenizerDescript
113113
public TokenizersDescriptor Icu(string name, Func<IcuTokenizerDescriptor, IIcuTokenizer> selector) =>
114114
Assign(name, selector?.Invoke(new IcuTokenizerDescriptor()));
115115

116+
/// <inheritdoc cref="INoriTokenizer"/>
117+
public TokenizersDescriptor Nori(string name, Func<NoriTokenizerDescriptor, INoriTokenizer> selector) =>
118+
Assign(name, selector?.Invoke(new NoriTokenizerDescriptor()));
119+
116120
/// <inheritdoc cref="ICharGroupTokenizer.TokenizeOnCharacters"/>>
117121
public TokenizersDescriptor CharGroup(string name, Func<CharGroupTokenizerDescriptor, ICharGroupTokenizer> selector) =>
118122
Assign(name, selector?.Invoke(new CharGroupTokenizerDescriptor()));

‎src/Nest/Indices/Analyze/AnalyzeTokenFilters.cs‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -324,6 +324,10 @@ public AnalyzeTokenFiltersDescriptor IcuNormalization(Func<IcuNormalizationToken
324324
public AnalyzeTokenFiltersDescriptor IcuTransform(Func<IcuTransformTokenFilterDescriptor, IIcuTransformTokenFilter> selector) =>
325325
AssignIfNotNull(selector.Invoke(new IcuTransformTokenFilterDescriptor()));
326326

327+
/// <inheritdoc cref="INoriPartOfSpeechTokenFilter"/>
328+
public AnalyzeTokenFiltersDescriptor NoriPartOfSpeech(Func<NoriPartOfSpeechTokenFilterDescriptor, INoriPartOfSpeechTokenFilter> selector) =>
329+
AssignIfNotNull(selector.Invoke(new NoriPartOfSpeechTokenFilterDescriptor()));
330+
327331
///<inheritdoc cref="IMultiplexerTokenFilter"/>
328332
public AnalyzeTokenFiltersDescriptor Multiplexer(Func<MultiplexerTokenFilterDescriptor, IMultiplexerTokenFilter> selector) =>
329333
AssignIfNotNull(selector.Invoke(new MultiplexerTokenFilterDescriptor()));

‎src/Nest/Indices/Analyze/AnalyzeTokenizersDescriptor.cs‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -92,6 +92,10 @@ public ITokenizer Kuromoji(Func<KuromojiTokenizerDescriptor, IKuromojiTokenizer>
9292
public ITokenizer Icu(Func<IcuTokenizerDescriptor, IIcuTokenizer> selector) =>
9393
(selector?.Invoke(new IcuTokenizerDescriptor()));
9494

95+
/// <inheritdoc cref="INoriTokenizer"/>
96+
public ITokenizer Nori(Func<NoriTokenizerDescriptor, INoriTokenizer> selector) =>
97+
selector.Invoke(new NoriTokenizerDescriptor());
98+
9599
/// <inheritdoc cref="ICharGroupTokenizer.TokenizeOnCharacters"/>>
96100
public ITokenizer CharGroup(Func<CharGroupTokenizerDescriptor, ICharGroupTokenizer> selector) => selector?.Invoke(new CharGroupTokenizerDescriptor());
97101
}

‎src/Tests/Tests.Core/ManagedElasticsearch/Clusters/WritableCluster.cs‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
1-
using Tests.Core.ManagedElasticsearch.NodeSeeders;
1+
using Elastic.Managed.Ephemeral.Plugins;
2+
using Tests.Core.ManagedElasticsearch.NodeSeeders;
23
using static Elastic.Managed.Ephemeral.Plugins.ElasticsearchPlugin;
34

45
namespace Tests.Core.ManagedElasticsearch.Clusters
@@ -12,7 +13,9 @@ public WritableCluster() : base(new ClientTestClusterConfiguration(
1213
AnalysisKuromoji,
1314
AnalysisIcu,
1415
AnalysisPhonetic,
15-
MapperMurmur3
16+
MapperMurmur3,
17+
//TODO move this to elasticsearch-net abstractions
18+
new ElasticsearchPlugin("analysis-nori", v => v >= "6.4.0")
1619
)
1720
{
1821
MaxConcurrency = 4

0 commit comments

Comments
 (0)