Adding Id, Ignored and Routing ParquetFields for Metadata

Signed-off-by: Sagar Darji <[email protected]>

Author: darjisagar7

modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/fields/core/metadata/DocCountParquetField.java

@@ -56,6 +56,6 @@ public ArrowType getArrowType() {
 
     @Override
     public FieldType getFieldType() {
-        return FieldType.notNullable(getArrowType());
+        return FieldType.nullable(getArrowType());
     }
 }

modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/fields/core/metadata/IdParquetField.java

@@ -0,0 +1,61 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package com.parquet.parquetdataformat.fields.core.metadata;
+
+import com.parquet.parquetdataformat.fields.ParquetField;
+import com.parquet.parquetdataformat.vsr.ManagedVSR;
+import org.apache.arrow.vector.VarCharVector;
+import org.apache.arrow.vector.types.pojo.ArrowType;
+import org.apache.arrow.vector.types.pojo.FieldType;
+import org.apache.lucene.util.BytesRef;
+import org.opensearch.index.mapper.MappedFieldType;
+
+/**
+ * Parquet field implementation for handling document ID metadata in OpenSearch documents.
+ *
+ * <p>This class provides the conversion logic between OpenSearch document ID fields and Apache Arrow
+ * UTF-8 string vectors for columnar storage in Parquet format. Document ID values are stored
+ * using Apache Arrow's {@link VarCharVector}, which provides efficient variable-length string storage.</p>
+ *
+ * <p>This field type corresponds to OpenSearch's {@code _id} metadata field and
+ * supports unique document identifiers. The ID values are processed from {@link BytesRef} objects
+ * and stored directly in the Arrow vector with proper offset and length handling.</p>
+ *
+ * <p><strong>Usage Example:</strong></p>
+ * <pre>{@code
+ * IdParquetField idField = new IdParquetField();
+ * ArrowType arrowType = idField.getArrowType(); // Returns UTF-8 string type
+ * FieldType fieldType = idField.getFieldType(); // Returns nullable UTF-8 field type
+ * }</pre>
+ *
+ * @see ParquetField
+ * @see VarCharVector
+ * @see ArrowType.Utf8
+ * @since 1.0
+ */
+public class IdParquetField extends ParquetField {
+
+    @Override
+    protected void addToGroup(MappedFieldType mappedFieldType, ManagedVSR managedVSR, Object parseValue) {
+        VarCharVector idVector = (VarCharVector) managedVSR.getVector(mappedFieldType.name());
+        int rowIndex = managedVSR.getRowCount();
+        BytesRef bytesRef = (BytesRef) parseValue;
+        idVector.setSafe(rowIndex, bytesRef.bytes, bytesRef.offset, bytesRef.length);
+    }
+
+    @Override
+    public ArrowType getArrowType() {
+        return new ArrowType.Utf8();
+    }
+
+    @Override
+    public FieldType getFieldType() {
+        return FieldType.nullable(getArrowType());
+    }
+}

modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/fields/core/metadata/IgnoredParquetField.java

@@ -0,0 +1,61 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package com.parquet.parquetdataformat.fields.core.metadata;
+
+import com.parquet.parquetdataformat.fields.ParquetField;
+import com.parquet.parquetdataformat.vsr.ManagedVSR;
+import org.apache.arrow.vector.VarCharVector;
+import org.apache.arrow.vector.types.pojo.ArrowType;
+import org.apache.arrow.vector.types.pojo.FieldType;
+import org.opensearch.index.mapper.MappedFieldType;
+
+import java.nio.charset.StandardCharsets;
+
+/**
+ * Parquet field implementation for handling ignored field data types in OpenSearch documents.
+ *
+ * <p>This class provides the conversion logic between OpenSearch ignored fields and Apache Arrow
+ * UTF-8 string vectors for columnar storage in Parquet format. Ignored field values are stored
+ * using Apache Arrow's {@link VarCharVector}, which provides efficient variable-length string storage.</p>
+ *
+ * <p>This field type corresponds to OpenSearch's {@code ignored} field mapping and
+ * supports fields that are indexed but not stored in the document source. The field values
+ * are converted to UTF-8 string representation before storage in the Arrow vector.</p>
+ *
+ * <p><strong>Usage Example:</strong></p>
+ * <pre>{@code
+ * IgnoredParquetField ignoredField = new IgnoredParquetField();
+ * ArrowType arrowType = ignoredField.getArrowType(); // Returns UTF-8 string type
+ * FieldType fieldType = ignoredField.getFieldType(); // Returns nullable UTF-8 field type
+ * }</pre>
+ *
+ * @see ParquetField
+ * @see VarCharVector
+ * @see ArrowType.Utf8
+ * @since 1.0
+ */
+public class IgnoredParquetField extends ParquetField {
+
+    @Override
+    protected void addToGroup(MappedFieldType mappedFieldType, ManagedVSR managedVSR, Object parseValue) {
+        VarCharVector varCharVector = (VarCharVector) managedVSR.getVector(mappedFieldType.name());
+        int rowIndex = managedVSR.getRowCount();
+        varCharVector.setSafe(rowIndex, parseValue.toString().getBytes(StandardCharsets.UTF_8));
+    }
+
+    @Override
+    public ArrowType getArrowType() {
+        return new ArrowType.Utf8();
+    }
+
+    @Override
+    public FieldType getFieldType() {
+        return FieldType.nullable(getArrowType());
+    }
+}

modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/fields/core/metadata/RoutingParquetField.java

@@ -0,0 +1,61 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package com.parquet.parquetdataformat.fields.core.metadata;
+
+import com.parquet.parquetdataformat.fields.ParquetField;
+import com.parquet.parquetdataformat.vsr.ManagedVSR;
+import org.apache.arrow.vector.VarCharVector;
+import org.apache.arrow.vector.types.pojo.ArrowType;
+import org.apache.arrow.vector.types.pojo.FieldType;
+import org.opensearch.index.mapper.MappedFieldType;
+
+import java.nio.charset.StandardCharsets;
+
+/**
+ * Parquet field implementation for handling routing metadata in OpenSearch documents.
+ *
+ * <p>This class provides the conversion logic between OpenSearch routing fields and Apache Arrow
+ * UTF-8 string vectors for columnar storage in Parquet format. Routing values are stored
+ * using Apache Arrow's {@link VarCharVector}, which provides efficient variable-length string storage.</p>
+ *
+ * <p>This field type corresponds to OpenSearch's {@code _routing} metadata field and
+ * supports custom routing values that determine which shard a document is stored on. The routing
+ * value is converted to UTF-8 bytes before storage in the Arrow vector.</p>
+ *
+ * <p><strong>Usage Example:</strong></p>
+ * <pre>{@code
+ * RoutingParquetField routingField = new RoutingParquetField();
+ * ArrowType arrowType = routingField.getArrowType(); // Returns UTF-8 string type
+ * FieldType fieldType = routingField.getFieldType(); // Returns nullable UTF-8 field type
+ * }</pre>
+ *
+ * @see ParquetField
+ * @see VarCharVector
+ * @see ArrowType.Utf8
+ * @since 1.0
+ */
+public class RoutingParquetField extends ParquetField {
+
+    @Override
+    protected void addToGroup(MappedFieldType mappedFieldType, ManagedVSR managedVSR, Object parseValue) {
+        VarCharVector routingVector = (VarCharVector) managedVSR.getVector(mappedFieldType.name());
+        int rowIndex = managedVSR.getRowCount();
+        routingVector.setSafe(rowIndex, parseValue.toString().getBytes(StandardCharsets.UTF_8));
+    }
+
+    @Override
+    public ArrowType getArrowType() {
+        return new ArrowType.Utf8();
+    }
+
+    @Override
+    public FieldType getFieldType() {
+        return FieldType.nullable(getArrowType());
+    }
+}

modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/plugins/fields/MetadataFieldPlugin.java

@@ -10,8 +10,14 @@
 
 import com.parquet.parquetdataformat.fields.ParquetField;
 import com.parquet.parquetdataformat.fields.core.metadata.DocCountParquetField;
+import com.parquet.parquetdataformat.fields.core.metadata.IdParquetField;
+import com.parquet.parquetdataformat.fields.core.metadata.IgnoredParquetField;
+import com.parquet.parquetdataformat.fields.core.metadata.RoutingParquetField;
 import com.parquet.parquetdataformat.fields.core.metadata.SizeParquetField;
 import org.opensearch.index.mapper.DocCountFieldMapper;
+import org.opensearch.index.mapper.IdFieldMapper;
+import org.opensearch.index.mapper.IgnoredFieldMapper;
+import org.opensearch.index.mapper.RoutingFieldMapper;
 
 import java.util.HashMap;
 import java.util.Map;
@@ -36,5 +42,8 @@ public Map<String, ParquetField> getParquetFields() {
     private static void registerMetadataFields(final Map<String, ParquetField> fieldMap) {
         fieldMap.put(DocCountFieldMapper.CONTENT_TYPE, new DocCountParquetField());
         fieldMap.put("_size", new SizeParquetField());
+        fieldMap.put(RoutingFieldMapper.CONTENT_TYPE, new RoutingParquetField());
+        fieldMap.put(IgnoredFieldMapper.CONTENT_TYPE, new IgnoredParquetField());
+        fieldMap.put(IdFieldMapper.CONTENT_TYPE, new IdParquetField());
     }
 }

server/src/main/java/org/opensearch/index/mapper/IdFieldMapper.java

@@ -298,7 +298,11 @@ private IdFieldMapper(Supplier<Boolean> fieldDataEnabled) {
     @Override
     public void preParse(ParseContext context) {
         BytesRef id = Uid.encodeId(context.sourceToParse().id());
-        context.doc().add(new Field(NAME, id, Defaults.FIELD_TYPE));
+        if (isPluggableDataFormatFeatureEnabled()) {
+            context.compositeDocumentInput().addField(fieldType(), id);
+        } else {
+            context.doc().add(new Field(NAME, id, Defaults.FIELD_TYPE));
+        }
     }
 
     @Override

server/src/main/java/org/opensearch/index/mapper/IgnoredFieldMapper.java

@@ -114,7 +114,11 @@ private IgnoredFieldMapper() {
     @Override
     public void postParse(ParseContext context) {
         for (String field : context.getIgnoredFields()) {
-            context.doc().add(new Field(NAME, field, Defaults.FIELD_TYPE));
+            if (isPluggableDataFormatFeatureEnabled()) {
+                context.compositeDocumentInput().addField(fieldType(), field);
+            } else {
+                context.doc().add(new Field(NAME, field, Defaults.FIELD_TYPE));
+            }
         }
     }
 

server/src/main/java/org/opensearch/index/mapper/RoutingFieldMapper.java

@@ -148,8 +148,12 @@ public boolean required() {
     public void preParse(ParseContext context) {
         String routing = context.sourceToParse().routing();
         if (routing != null) {
-            context.doc().add(new Field(fieldType().name(), routing, Defaults.FIELD_TYPE));
-            createFieldNamesField(context);
+            if (isPluggableDataFormatFeatureEnabled()) {
+                context.compositeDocumentInput().addField(fieldType(), routing);
+            } else {
+                context.doc().add(new Field(fieldType().name(), routing, Defaults.FIELD_TYPE));
+                createFieldNamesField(context);
+            }
         }
     }