Adding support for derive source feature and implementing it for various type of field mappers (opensearch-project#17759)

* Adding support for derive source feature and implementing it for some of the field types

Signed-off-by: Tanik Pansuriya <[email protected]>Signed-off-by: TJ Neuenfeldt <[email protected]>

Author: tanik98

CHANGELOG.md

@@ -12,6 +12,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Update API of Message in index to add the timestamp for lag calculation in ingestion polling ([#17977](https://github.com/opensearch-project/OpenSearch/pull/17977/))
 - Add composite directory factory ([#17988](https://github.com/opensearch-project/OpenSearch/pull/17988))
 - Add pull-based ingestion error metrics and make internal queue size configurable ([#18088](https://github.com/opensearch-project/OpenSearch/pull/18088))
+- Adding support for derive source feature and implementing it for various type of field mappers ([#17759](https://github.com/opensearch-project/OpenSearch/pull/17759))
 - [Security Manager Replacement] Enhance Java Agent to intercept newByteChannel ([#17989](https://github.com/opensearch-project/OpenSearch/pull/17989))
 - Enabled Async Shard Batch Fetch by default ([#18139](https://github.com/opensearch-project/OpenSearch/pull/18139))
 - Allow to get the search request from the QueryCoordinatorContext ([#17818](https://github.com/opensearch-project/OpenSearch/pull/17818))

modules/mapper-extras/src/main/java/org/opensearch/index/mapper/ScaledFloatFieldMapper.java

@@ -500,6 +500,29 @@ private static double objectToDouble(Object value) {
         return doubleValue;
     }
 
+    @Override
+    protected void canDeriveSourceInternal() {
+        checkStoredAndDocValuesForDerivedSource();
+    }
+
+    /**
+     * 1. If it has doc values, build source using doc values
+     * 2. If doc_values is disabled in field mapping, then build source using stored field
+     * <p>
+     * Considerations:
+     *    1. When using doc values, for multi value field, result would be in sorted order
+     *    2. There might be precision loss as values are stored as long after multiplying it with "scaling_factor" for
+     *       both doc values and stored field
+     */
+    @Override
+    protected DerivedFieldGenerator derivedFieldGenerator() {
+        return new DerivedFieldGenerator(
+            mappedFieldType,
+            new SortedNumericDocValuesFetcher(mappedFieldType, simpleName()),
+            new StoredFieldFetcher(mappedFieldType, simpleName())
+        );
+    }
+
     private static class ScaledFloatIndexFieldData extends IndexNumericFieldData {
 
         private final IndexNumericFieldData scaledFieldData;

modules/mapper-extras/src/test/java/org/opensearch/index/mapper/ScaledFloatFieldMapperTests.java

@@ -32,8 +32,15 @@
 
 package org.opensearch.index.mapper;
 
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.SortedNumericDocValuesField;
+import org.apache.lucene.document.StoredField;
+import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.DocValuesType;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
 import org.apache.lucene.index.IndexableField;
+import org.apache.lucene.store.Directory;
 import org.opensearch.cluster.metadata.IndexMetadata;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.xcontent.XContentFactory;
@@ -54,6 +61,8 @@
 
 public class ScaledFloatFieldMapperTests extends MapperTestCase {
 
+    private static final String FIELD_NAME = "field";
+
     @Override
     protected Collection<? extends Plugin> getPlugins() {
         return singletonList(new MapperExtrasModulePlugin());
@@ -383,6 +392,78 @@ public void testNullValue() throws IOException {
         assertFalse(dvField.fieldType().stored());
     }
 
+    public void testPossibleToDeriveSource_WhenDocValuesAndStoredDisabled() throws IOException {
+        ScaledFloatFieldMapper mapper = getMapper(FieldMapper.CopyTo.empty(), false, false);
+        assertThrows(UnsupportedOperationException.class, mapper::canDeriveSource);
+    }
+
+    public void testPossibleToDeriveSource_WhenCopyToPresent() throws IOException {
+        FieldMapper.CopyTo copyTo = new FieldMapper.CopyTo.Builder().add("copy_to_field").build();
+        ScaledFloatFieldMapper mapper = getMapper(copyTo, true, true);
+        assertThrows(UnsupportedOperationException.class, mapper::canDeriveSource);
+    }
+
+    public void testDerivedValueFetching_DocValues() throws IOException {
+        try (Directory directory = newDirectory()) {
+            ScaledFloatFieldMapper mapper = getMapper(FieldMapper.CopyTo.empty(), true, false);
+            float value = 11.523f;
+            try (IndexWriter iw = new IndexWriter(directory, new IndexWriterConfig())) {
+                iw.addDocument(createDocument(value, true));
+            }
+
+            try (DirectoryReader reader = DirectoryReader.open(directory)) {
+                XContentBuilder builder = XContentFactory.jsonBuilder().startObject();
+                mapper.deriveSource(builder, reader.leaves().get(0).reader(), 0);
+                builder.endObject();
+                String source = builder.toString();
+                assertEquals("{\"" + FIELD_NAME + "\":" + 11.52 + "}", source);
+            }
+        }
+    }
+
+    public void testDerivedValueFetching_StoredField() throws IOException {
+        try (Directory directory = newDirectory()) {
+            ScaledFloatFieldMapper mapper = getMapper(FieldMapper.CopyTo.empty(), false, true);
+            float value = 11.523f;
+            try (IndexWriter iw = new IndexWriter(directory, new IndexWriterConfig())) {
+                iw.addDocument(createDocument(value, false));
+            }
+
+            try (DirectoryReader reader = DirectoryReader.open(directory)) {
+                XContentBuilder builder = XContentFactory.jsonBuilder().startObject();
+                mapper.deriveSource(builder, reader.leaves().get(0).reader(), 0);
+                builder.endObject();
+                String source = builder.toString();
+                assertEquals("{\"" + FIELD_NAME + "\":" + 11.52 + "}", source);
+            }
+        }
+    }
+
+    private ScaledFloatFieldMapper getMapper(FieldMapper.CopyTo copyTo, boolean hasDocValues, boolean isStored) throws IOException {
+        MapperService mapperService = createMapperService(
+            fieldMapping(
+                b -> b.field("type", "scaled_float").field("store", isStored).field("doc_values", hasDocValues).field("scaling_factor", 100)
+            )
+        );
+        ScaledFloatFieldMapper mapper = (ScaledFloatFieldMapper) mapperService.documentMapper().mappers().getMapper(FIELD_NAME);
+        mapper.copyTo = copyTo;
+        return mapper;
+    }
+
+    /**
+     * Helper method to create a document with both doc values and stored fields
+     */
+    private Document createDocument(double value, boolean hasDocValues) {
+        long scaledValue = Math.round(value * 100);
+        Document doc = new Document();
+        if (hasDocValues) {
+            doc.add(new SortedNumericDocValuesField(FIELD_NAME, scaledValue));
+        } else {
+            doc.add(new StoredField(FIELD_NAME, scaledValue));
+        }
+        return doc;
+    }
+
     /**
      * `index_options` was deprecated and is rejected as of 7.0
      */

server/src/main/java/org/opensearch/index/mapper/BooleanFieldMapper.java

@@ -412,4 +412,34 @@ protected String contentType() {
         return CONTENT_TYPE;
     }
 
+    @Override
+    protected void canDeriveSourceInternal() {
+        checkStoredAndDocValuesForDerivedSource();
+    }
+
+    /**
+     * 1. If it has doc values, build source using doc values
+     * 2. If doc_values is disabled in field mapping, then build source using stored field
+     *
+     * <p>
+     * Considerations:
+     *    1. Result will be in boolean type and not in the provided string value type at time of ingestion,
+     *       i.e. [false, "false", ""] will become boolean false
+     *    2. When using doc values, for multi value field, result will be in sorted order, i.e. at start there will
+     *       be 0 or more false and at end there will be 0 or more true
+     *    2. When using stored field, for multi value field order would be preserved
+     */
+    @Override
+    protected DerivedFieldGenerator derivedFieldGenerator() {
+        return new DerivedFieldGenerator(mappedFieldType, new SortedNumericDocValuesFetcher(mappedFieldType, simpleName()) {
+            @Override
+            public Object convert(Object value) {
+                Long val = (Long) value;
+                if (val == null) {
+                    return null;
+                }
+                return val == 1;
+            }
+        }, new StoredFieldFetcher(mappedFieldType, simpleName()));
+    }
 }

server/src/main/java/org/opensearch/index/mapper/ConstantKeywordFieldMapper.java

@@ -8,6 +8,7 @@
 
 package org.opensearch.index.mapper;
 
+import org.apache.lucene.index.LeafReader;
 import org.apache.lucene.search.MatchAllDocsQuery;
 import org.apache.lucene.search.MatchNoDocsQuery;
 import org.apache.lucene.search.MultiTermQuery;
@@ -25,6 +26,7 @@
 import org.opensearch.common.lucene.BytesRefs;
 import org.opensearch.common.regex.Regex;
 import org.opensearch.common.time.DateMathParser;
+import org.opensearch.core.xcontent.XContentBuilder;
 import org.opensearch.index.fielddata.IndexFieldData;
 import org.opensearch.index.fielddata.plain.ConstantIndexFieldData;
 import org.opensearch.index.query.QueryShardContext;
@@ -74,6 +76,26 @@ private static ConstantKeywordFieldMapper toType(FieldMapper in) {
         return (ConstantKeywordFieldMapper) in;
     }
 
+    @Override
+    public void canDeriveSource() {
+        if (this.copyTo() != null && !this.copyTo().copyToFields().isEmpty()) {
+            throw new UnsupportedOperationException("Unable to derive source for fields with copy_to parameter set");
+        }
+    }
+
+    /**
+     * For each doc, it will return constant value defined in field mapping
+     * <p>
+     * Note: Doc for which, field in absent, deriveSource will still consider the still to be present, and it will
+     *       return the same.
+     */
+    @Override
+    public void deriveSource(XContentBuilder builder, LeafReader leafReader, int docId) throws IOException {
+        if (value != null) {
+            builder.field(name(), value);
+        }
+    }
+
     /**
      * Builder for the binary field mapper
      *

server/src/main/java/org/opensearch/index/mapper/DateFieldMapper.java

@@ -226,6 +226,38 @@ private static DateFieldMapper toType(FieldMapper in) {
         return (DateFieldMapper) in;
     }
 
+    @Override
+    protected void canDeriveSourceInternal() {
+        checkStoredAndDocValuesForDerivedSource();
+    }
+
+    /**
+     * 1. If it has doc values, build source using doc values
+     * 2. If doc_values is disabled in field mapping, then build source using stored field
+     * <p>
+     * Considerations:
+     *     1. When building source using doc_values, for multi-value field, it will result values in sorted order
+     * <p>
+     * Date format:
+     *     1. If "print_format" specified in field mapping, then derived source will have date in this format
+     *     2. If multiple date formats are specified in field mapping and "print_format" is not specified then
+     *        derived source will contain date in first date format from "||" separated list of format defined in
+     *        "format"
+     */
+    @Override
+    protected DerivedFieldGenerator derivedFieldGenerator() {
+        return new DerivedFieldGenerator(mappedFieldType, new SortedNumericDocValuesFetcher(mappedFieldType, simpleName()) {
+            @Override
+            public Object convert(Object value) {
+                Long val = (Long) value;
+                if (val == null) {
+                    return null;
+                }
+                return fieldType().dateTimeFormatter().format(resolution.toInstant(val).atZone(ZoneOffset.UTC));
+            }
+        }, new StoredFieldFetcher(mappedFieldType, simpleName()));
+    }
+
     /**
      * Builder for the date field mapper
      *

server/src/main/java/org/opensearch/index/mapper/DerivedFieldGenerator.java

@@ -0,0 +1,61 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.mapper;
+
+import org.apache.lucene.index.LeafReader;
+import org.opensearch.core.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.Objects;
+
+/**
+ * DerivedSourceGenerator is used to generate derived source field based on field mapping and how
+ * it is stored in lucene
+ */
+public class DerivedFieldGenerator {
+
+    private final MappedFieldType mappedFieldType;
+    private final FieldValueFetcher fieldValueFetcher;
+
+    public DerivedFieldGenerator(
+        MappedFieldType mappedFieldType,
+        FieldValueFetcher docValuesFetcher,
+        FieldValueFetcher storedFieldFetcher
+    ) {
+        this.mappedFieldType = mappedFieldType;
+        if (Objects.requireNonNull(getDerivedFieldPreference()) == FieldValueType.DOC_VALUES) {
+            assert docValuesFetcher != null;
+            this.fieldValueFetcher = docValuesFetcher;
+        } else {
+            assert storedFieldFetcher != null;
+            this.fieldValueFetcher = storedFieldFetcher;
+        }
+    }
+
+    /**
+     * Get the preference of the derived field based on field mapping, should be overridden at a FieldMapper to
+     * alter the preference of derived field
+     */
+    public FieldValueType getDerivedFieldPreference() {
+        if (mappedFieldType.hasDocValues()) {
+            return FieldValueType.DOC_VALUES;
+        }
+        return FieldValueType.STORED;
+    }
+
+    /**
+     * Generate the derived field value based on the preference of derived field and field value type
+     * @param builder - builder to store the derived source filed
+     * @param reader - leafReader to read data from
+     * @param docId - docId for which we want to generate the source
+     */
+    public void generate(XContentBuilder builder, LeafReader reader, int docId) throws IOException {
+        fieldValueFetcher.write(builder, fieldValueFetcher.fetch(reader, docId));
+    }
+}