Skip to content

Commit bb013da

Browse files
rishabhmauryarishabhmaurya
authored
[DerivedField] object type support in mappings and new settings for derived field (#13717)
 --------- Signed-off-by: Rishabh Maurya <[email protected]>
1 parent 6c1896b commit bb013da

File tree

3 files changed

+172
-4
lines changed

3 files changed

+172
-4
lines changed

‎server/src/main/java/org/opensearch/index/mapper/DerivedField.java‎

Lines changed: 77 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88

99
package org.opensearch.index.mapper;
1010

11+
import org.opensearch.Version;
1112
import org.opensearch.common.annotation.PublicApi;
1213
import org.opensearch.core.common.io.stream.StreamInput;
1314
import org.opensearch.core.common.io.stream.StreamOutput;
@@ -18,17 +19,21 @@
1819
import org.opensearch.script.Script;
1920

2021
import java.io.IOException;
22+
import java.util.Map;
2123
import java.util.Objects;
2224

2325
/**
2426
* DerivedField representation: expects a name, type and script.
2527
*/
2628
@PublicApi(since = "2.14.0")
2729
public class DerivedField implements Writeable, ToXContentFragment {
28-
2930
private final String name;
3031
private final String type;
3132
private final Script script;
33+
private String sourceIndexedField;
34+
private Map<String, Object> properties;
35+
private Boolean ignoreMalformed;
36+
private String format;
3237

3338
public DerivedField(String name, String type, Script script) {
3439
this.name = name;
@@ -40,20 +45,51 @@ public DerivedField(StreamInput in) throws IOException {
4045
name = in.readString();
4146
type = in.readString();
4247
script = new Script(in);
48+
if (in.getVersion().onOrAfter(Version.V_2_15_0)) {
49+
if (in.readBoolean()) {
50+
properties = in.readMap();
51+
}
52+
sourceIndexedField = in.readOptionalString();
53+
format = in.readOptionalString();
54+
ignoreMalformed = in.readOptionalBoolean();
55+
}
4356
}
4457

4558
@Override
4659
public void writeTo(StreamOutput out) throws IOException {
4760
out.writeString(name);
4861
out.writeString(type);
4962
script.writeTo(out);
63+
if (out.getVersion().onOrAfter(Version.V_2_15_0)) {
64+
if (properties == null) {
65+
out.writeBoolean(false);
66+
} else {
67+
out.writeBoolean(true);
68+
out.writeMap(properties);
69+
}
70+
out.writeOptionalString(sourceIndexedField);
71+
out.writeOptionalString(format);
72+
out.writeOptionalBoolean(ignoreMalformed);
73+
}
5074
}
5175

5276
@Override
5377
public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException {
5478
builder.startObject(name);
5579
builder.field("type", type);
5680
builder.field("script", script);
81+
if (properties != null) {
82+
builder.field("properties", properties);
83+
}
84+
if (sourceIndexedField != null) {
85+
builder.field("source_indexed_field", sourceIndexedField);
86+
}
87+
if (format != null) {
88+
builder.field("format", format);
89+
}
90+
if (ignoreMalformed != null) {
91+
builder.field("ignore_malformed", ignoreMalformed);
92+
}
5793
builder.endObject();
5894
return builder;
5995
}
@@ -70,9 +106,41 @@ public Script getScript() {
70106
return script;
71107
}
72108

109+
public Map<String, Object> getProperties() {
110+
return properties;
111+
}
112+
113+
public String getSourceIndexedField() {
114+
return sourceIndexedField;
115+
}
116+
117+
public String getFormat() {
118+
return format;
119+
}
120+
121+
public boolean getIgnoreMalformed() {
122+
return Boolean.TRUE.equals(ignoreMalformed);
123+
}
124+
125+
public void setProperties(Map<String, Object> properties) {
126+
this.properties = properties;
127+
}
128+
129+
public void setSourceIndexedField(String sourceIndexedField) {
130+
this.sourceIndexedField = sourceIndexedField;
131+
}
132+
133+
public void setFormat(String format) {
134+
this.format = format;
135+
}
136+
137+
public void setIgnoreMalformed(boolean ignoreMalformed) {
138+
this.ignoreMalformed = ignoreMalformed;
139+
}
140+
73141
@Override
74142
public int hashCode() {
75-
return Objects.hash(name, type, script);
143+
return Objects.hash(name, type, script, sourceIndexedField, properties, ignoreMalformed, format);
76144
}
77145

78146
@Override
@@ -84,7 +152,12 @@ public boolean equals(Object obj) {
84152
return false;
85153
}
86154
DerivedField other = (DerivedField) obj;
87-
return Objects.equals(name, other.name) && Objects.equals(type, other.type) && Objects.equals(script, other.script);
155+
return Objects.equals(name, other.name)
156+
&& Objects.equals(type, other.type)
157+
&& Objects.equals(script, other.script)
158+
&& Objects.equals(sourceIndexedField, other.sourceIndexedField)
159+
&& Objects.equals(properties, other.properties)
160+
&& Objects.equals(ignoreMalformed, other.ignoreMalformed)
161+
&& Objects.equals(format, other.format);
88162
}
89-
90163
}

‎server/src/main/java/org/opensearch/search/builder/SearchSourceBuilder.java‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1004,6 +1004,37 @@ public SearchSourceBuilder derivedField(String name, String type, Script script)
10041004
return this;
10051005
}
10061006

1007+
/**
1008+
* Adds a derived field with the given name with provided type, script and other parameters
1009+
* @param name name of the derived field
1010+
* @param type type of the derived field
1011+
* @param script script associated with derived field
1012+
* @param properties map of field name and type of field for nested fields within object derived field
1013+
* @param sourceIndexedField source text field which is indexed to filter documents for better performance
1014+
* @param format date format
1015+
* @param ignoreMalformed ignores malformed fields instead of failing search request
1016+
*/
1017+
public SearchSourceBuilder derivedField(
1018+
String name,
1019+
String type,
1020+
Script script,
1021+
Map<String, Object> properties,
1022+
String sourceIndexedField,
1023+
String format,
1024+
Boolean ignoreMalformed
1025+
) {
1026+
if (derivedFields == null) {
1027+
derivedFields = new ArrayList<>();
1028+
}
1029+
DerivedField derivedField = new DerivedField(name, type, script);
1030+
derivedField.setProperties(properties);
1031+
derivedField.setSourceIndexedField(sourceIndexedField);
1032+
derivedField.setFormat(format);
1033+
derivedField.setIgnoreMalformed(ignoreMalformed);
1034+
derivedFields.add(derivedField);
1035+
return this;
1036+
}
1037+
10071038
/**
10081039
* Sets the boost a specific index or alias will receive when the query is executed
10091040
* against it.

‎server/src/test/java/org/opensearch/search/builder/SearchSourceBuilderTests.java‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -357,6 +357,70 @@ public void testDerivedFieldsParsingAndSerialization() throws IOException {
357357

358358
}
359359

360+
public void testDerivedFieldsParsingAndSerializationObjectType() throws IOException {
361+
{
362+
String restContent = "{\n"
363+
+ " \"derived\": {\n"
364+
+ " \"duration\": {\n"
365+
+ " \"type\": \"long\",\n"
366+
+ " \"script\": \"emit(doc['test'])\"\n"
367+
+ " },\n"
368+
+ " \"ip_from_message\": {\n"
369+
+ " \"type\": \"keyword\",\n"
370+
+ " \"script\": \"emit(doc['message'])\"\n"
371+
+ " },\n"
372+
+ " \"object\": {\n"
373+
+ " \"type\": \"object\",\n"
374+
+ " \"script\": \"emit(doc['test'])\",\n"
375+
+ " \"format\": \"dd-MM-yyyy\",\n"
376+
+ " \"source_indexed_field\": \"test\",\n"
377+
+ " \"ignore_malformed\": true,\n"
378+
+ " \"properties\": {\n"
379+
+ " \"sub_field\": \"text\"\n"
380+
+ " }\n"
381+
+ " }\n"
382+
+ " },\n"
383+
+ " \"query\" : {\n"
384+
+ " \"match\": { \"content\": { \"query\": \"foo bar\" }}\n"
385+
+ " }\n"
386+
+ "}";
387+
388+
String expectedContent =
389+
"{\"query\":{\"match\":{\"content\":{\"query\":\"foo bar\",\"operator\":\"OR\",\"prefix_length\":0,\"max_expansions\":50,\"fuzzy_transpositions\":true,\"lenient\":false,\"zero_terms_query\":\"NONE\",\"auto_generate_synonyms_phrase_query\":true,\"boost\":1.0}}},\"derived\":{\"duration\":{\"type\":\"long\",\"script\":\"emit(doc['test'])\"},\"ip_from_message\":{\"type\":\"keyword\",\"script\":\"emit(doc['message'])\"},\"object\":{\"format\":\"dd-MM-yyyy\",\"source_indexed_field\":\"test\",\"ignore_malformed\":true,\"type\":\"object\",\"script\":\"emit(doc['test'])\",\"properties\":{\"sub_field\":\"text\"}},\"derived_field\":{\"type\":\"object\",\"script\":{\"source\":\"emit(doc['message']\",\"lang\":\"painless\"},\"properties\":{\"sub_field_2\":\"keyword\"},\"source_indexed_field\":\"message\",\"format\":\"dd-MM-yyyy\",\"ignore_malformed\":true}}}";
390+
391+
try (XContentParser parser = createParser(JsonXContent.jsonXContent, restContent)) {
392+
SearchSourceBuilder searchSourceBuilder = SearchSourceBuilder.fromXContent(parser);
393+
searchSourceBuilder.derivedField(
394+
"derived_field",
395+
"object",
396+
new Script("emit(doc['message']"),
397+
Map.of("sub_field_2", "keyword"),
398+
"message",
399+
"dd-MM-yyyy",
400+
true
401+
);
402+
searchSourceBuilder = rewrite(searchSourceBuilder);
403+
assertEquals(3, searchSourceBuilder.getDerivedFieldsObject().size());
404+
assertEquals(1, searchSourceBuilder.getDerivedFields().size());
405+
assertEquals(1, searchSourceBuilder.getDerivedFields().get(0).getProperties().size());
406+
assertEquals("message", searchSourceBuilder.getDerivedFields().get(0).getSourceIndexedField());
407+
assertEquals("dd-MM-yyyy", searchSourceBuilder.getDerivedFields().get(0).getFormat());
408+
assertTrue(searchSourceBuilder.getDerivedFields().get(0).getIgnoreMalformed());
409+
410+
try (BytesStreamOutput output = new BytesStreamOutput()) {
411+
searchSourceBuilder.writeTo(output);
412+
try (StreamInput in = new NamedWriteableAwareStreamInput(output.bytes().streamInput(), namedWriteableRegistry)) {
413+
SearchSourceBuilder deserializedBuilder = new SearchSourceBuilder(in);
414+
String actualContent = deserializedBuilder.toString();
415+
assertEquals(expectedContent, actualContent);
416+
assertEquals(searchSourceBuilder.hashCode(), deserializedBuilder.hashCode());
417+
assertNotSame(searchSourceBuilder, deserializedBuilder);
418+
}
419+
}
420+
}
421+
}
422+
}
423+
360424
public void testAggsParsing() throws IOException {
361425
{
362426
String restContent = "{\n"

0 commit comments

Comments
 (0)