Add Flush Flow and Metric Tests

Author: Shailesh Singh

server/src/main/java/org/opensearch/index/compositeindex/datacube/ComparisonType.java

@@ -0,0 +1,22 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.compositeindex.datacube;
+
+import org.opensearch.common.annotation.ExperimentalApi;
+
+/**
+ * Represents the type of comparison to be performed on a dimension.
+ *
+ * @opensearch.experimental
+ */
+@ExperimentalApi
+public enum ComparisonType {
+    SIGNED,
+    UNSIGNED
+}

server/src/main/java/org/opensearch/index/compositeindex/datacube/Dimension.java

@@ -12,6 +12,7 @@
 import org.opensearch.common.annotation.ExperimentalApi;
 import org.opensearch.core.xcontent.ToXContent;
 
+import java.util.Comparator;
 import java.util.List;
 import java.util.function.Consumer;
 
@@ -45,4 +46,26 @@ public interface Dimension extends ToXContent {
     List<String> getSubDimensionNames();
 
     DocValuesType getDocValuesType();
+
+    default ComparisonType getComparisonType() {
+        return ComparisonType.SIGNED;
+    }
+
+    default Comparator<Long> comparator() {
+        return (a, b) -> {
+            if (a == null && b == null) {
+                return 0;
+            }
+            if (b == null) {
+                return -1;
+            }
+            if (a == null) {
+                return 1;
+            }
+            return getComparisonType() == ComparisonType.UNSIGNED
+                ? Long.compareUnsigned(a, b)
+                : Long.compare(a, b);
+        };
+    }
+
 }

server/src/main/java/org/opensearch/index/compositeindex/datacube/DimensionFactory.java

@@ -24,7 +24,6 @@
 import java.util.stream.Collectors;
 
 import static org.opensearch.index.compositeindex.datacube.DateDimension.CALENDAR_INTERVALS;
-import static org.opensearch.index.compositeindex.datacube.KeywordDimension.KEYWORD;
 
 /**
  * Dimension factory class mainly used to parse and create dimension from the mappings
@@ -36,16 +35,17 @@ public class DimensionFactory {
     public static Dimension parseAndCreateDimension(
         String name,
         String type,
-        Boolean isUnsignedLong,
         Map<String, Object> dimensionMap,
         Mapper.TypeParser.ParserContext c
     ) {
         switch (type) {
             case DateDimension.DATE:
                 return parseAndCreateDateDimension(name, dimensionMap, c);
             case NumericDimension.NUMERIC:
-                return new NumericDimension(name, isUnsignedLong);
-            case KEYWORD:
+                return new NumericDimension(name);
+            case UnsignedLongDimension.UNSIGNED_LONG:
+                return new UnsignedLongDimension(name);
+            case KeywordDimension.KEYWORD:
                 return new KeywordDimension(name);
             default:
                 throw new IllegalArgumentException(
@@ -54,15 +54,6 @@ public static Dimension parseAndCreateDimension(
         }
     }
 
-    public static Dimension parseAndCreateDimension(
-        String name,
-        String type,
-        Map<String, Object> dimensionMap,
-        Mapper.TypeParser.ParserContext c
-    ) {
-        return parseAndCreateDimension(name, type, false, dimensionMap, c);
-    }
-
     public static Dimension parseAndCreateDimension(
         String name,
         Mapper.Builder builder,
@@ -78,7 +69,9 @@ public static Dimension parseAndCreateDimension(
             case DATE:
                 return parseAndCreateDateDimension(name, dimensionMap, c);
             case NUMERIC:
-                return new NumericDimension(name, builder.isUnsignedLong());
+                return new NumericDimension(name);
+            case UNSIGNED_LONG:
+                return new UnsignedLongDimension(name);
             case KEYWORD:
                 return new KeywordDimension(name);
             default:

server/src/main/java/org/opensearch/index/compositeindex/datacube/DimensionType.java

@@ -23,6 +23,12 @@ public enum DimensionType {
      */
     NUMERIC,
 
+    /**
+     * Represents an unsigned long dimension type.
+     * This is used for dimensions that contain numerical values of type unsigned long.
+     */
+    UNSIGNED_LONG,
+
     /**
      * Represents a date dimension type.
      * This is used for dimensions that contain date or timestamp values.

server/src/main/java/org/opensearch/index/compositeindex/datacube/NumericDimension.java

@@ -26,19 +26,11 @@
 @ExperimentalApi
 public class NumericDimension implements Dimension {
     public static final String NUMERIC = "numeric";
-    public static final String IS_UNSIGNED_LONG_FIELD = "isUnsignedLong";
 
-    private final String field;
-    private final Boolean isUnsignedLong;
+    protected final String field;
 
     public NumericDimension(String field) {
         this.field = field;
-        isUnsignedLong = false;
-    }
-
-    public NumericDimension(String field, Boolean isUnsignedLong) {
-        this.field = field;
-        this.isUnsignedLong = isUnsignedLong;
     }
 
     public String getField() {
@@ -70,7 +62,6 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         builder.startObject();
         builder.field(CompositeDataCubeFieldType.NAME, field);
         builder.field(CompositeDataCubeFieldType.TYPE, NUMERIC);
-        builder.field(IS_UNSIGNED_LONG_FIELD, isUnsignedLong);
         builder.endObject();
         return builder;
     }
@@ -88,7 +79,4 @@ public int hashCode() {
         return Objects.hash(field);
     }
 
-    public Boolean isUnsignedLong() {
-        return isUnsignedLong;
-    }
 }

server/src/main/java/org/opensearch/index/compositeindex/datacube/ReadDimension.java

@@ -26,15 +26,24 @@ public class ReadDimension implements Dimension {
     public static final String READ = "read";
     private final String field;
     private final DocValuesType docValuesType;
+    private final ComparisonType comparisonType;
 
     public ReadDimension(String field) {
         this.field = field;
         this.docValuesType = DocValuesType.SORTED_NUMERIC;
+        this.comparisonType = ComparisonType.SIGNED;
     }
 
     public ReadDimension(String field, DocValuesType docValuesType) {
         this.field = field;
         this.docValuesType = docValuesType;
+        this.comparisonType = ComparisonType.SIGNED;
+    }
+
+    public ReadDimension(String field, ComparisonType comparisonType){
+        this.field = field;
+        this.docValuesType = DocValuesType.SORTED_NUMERIC;
+        this.comparisonType = comparisonType;
     }
 
     public String getField() {
@@ -82,4 +91,10 @@ public boolean equals(Object o) {
     public int hashCode() {
         return Objects.hash(field);
     }
+
+    @Override
+    public ComparisonType getComparisonType() {
+        return comparisonType;
+    }
+
 }

server/src/main/java/org/opensearch/index/compositeindex/datacube/UnsignedLongDimension.java

@@ -0,0 +1,38 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.compositeindex.datacube;
+
+import org.opensearch.core.xcontent.XContentBuilder;
+import org.opensearch.index.mapper.CompositeDataCubeFieldType;
+
+import java.io.IOException;
+import java.util.Comparator;
+
+public class UnsignedLongDimension extends NumericDimension {
+    public static final String UNSIGNED_LONG = "unsigned_long";
+
+    public UnsignedLongDimension(String field) {
+        super(field);
+    }
+
+    @Override
+    public ComparisonType getComparisonType() {
+        return ComparisonType.UNSIGNED;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        builder.field(CompositeDataCubeFieldType.NAME, field);
+        builder.field(CompositeDataCubeFieldType.TYPE, UNSIGNED_LONG);
+        builder.endObject();
+        return builder;
+    }
+
+}

server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilder.java

@@ -13,20 +13,16 @@
 import org.apache.lucene.store.IndexOutput;
 import org.apache.lucene.util.LongValues;
 import org.opensearch.common.annotation.ExperimentalApi;
+import org.opensearch.index.compositeindex.datacube.Dimension;
+import org.opensearch.index.compositeindex.datacube.UnsignedLongDimension;
 import org.opensearch.index.compositeindex.datacube.startree.StarTreeDocument;
 import org.opensearch.index.compositeindex.datacube.startree.StarTreeField;
 import org.opensearch.index.compositeindex.datacube.startree.index.StarTreeValues;
 import org.opensearch.index.compositeindex.datacube.startree.utils.SequentialDocValuesIterator;
 import org.opensearch.index.mapper.MapperService;
 
 import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Iterator;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
+import java.util.*;
 import java.util.concurrent.atomic.AtomicInteger;
 
 /**
@@ -271,18 +267,16 @@ public StarTreeDocument next() {
      */
     private void sortStarTreeDocumentsFromDimensionId(StarTreeDocument[] starTreeDocuments, int dimensionId) {
         Arrays.sort(starTreeDocuments, (o1, o2) -> {
+            List<Dimension> dimensionsOrder = starTreeField.getDimensionsOrder();
             for (int i = dimensionId; i < numDimensions; i++) {
                 if (!Objects.equals(o1.dimensions[i], o2.dimensions[i])) {
-                    if (o1.dimensions[i] == null && o2.dimensions[i] == null) {
-                        return 0;
+                    Dimension dimension = dimensionsOrder.get(i);
+                    if (dimension instanceof UnsignedLongDimension) {
+                        UnsignedLongDimension unsignedLongDimension = (UnsignedLongDimension) dimension;
+                        return unsignedLongDimension.comparator().compare(o1.dimensions[i], o2.dimensions[i]);
                     }
-                    if (o1.dimensions[i] == null) {
-                        return 1;
-                    }
-                    if (o2.dimensions[i] == null) {
-                        return -1;
-                    }
-                    return Long.compare(o1.dimensions[i], o2.dimensions[i]);
+                    Comparator<Long> comparator = dimension.comparator();
+                    return comparator.compare(o1.dimensions[i], o2.dimensions[i]);
                 }
             }
             return 0;

server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/meta/StarTreeMetadata.java

@@ -14,6 +14,7 @@
 import org.apache.lucene.store.IndexInput;
 import org.opensearch.common.annotation.ExperimentalApi;
 import org.opensearch.index.compositeindex.CompositeIndexMetadata;
+import org.opensearch.index.compositeindex.datacube.Dimension;
 import org.opensearch.index.compositeindex.datacube.Metric;
 import org.opensearch.index.compositeindex.datacube.MetricStat;
 import org.opensearch.index.compositeindex.datacube.startree.StarTreeFieldConfiguration;

server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/node/FixedLengthStarTreeNode.java

@@ -8,11 +8,12 @@
 package org.opensearch.index.compositeindex.datacube.startree.fileformats.node;
 
 import org.apache.lucene.store.RandomAccessInput;
+import org.opensearch.index.compositeindex.datacube.Dimension;
 import org.opensearch.index.compositeindex.datacube.startree.node.StarTreeNode;
 import org.opensearch.index.compositeindex.datacube.startree.node.StarTreeNodeType;
-
 import java.io.IOException;
 import java.io.UncheckedIOException;
+import java.util.Comparator;
 import java.util.Iterator;
 
 /**
@@ -192,15 +193,15 @@ public StarTreeNode getChildStarNode() throws IOException {
     }
 
     @Override
-    public StarTreeNode getChildForDimensionValue(Long dimensionValue) throws IOException {
+    public StarTreeNode getChildForDimensionValue(Long dimensionValue, Dimension dimension) throws IOException {
         // there will be no children for leaf nodes
         if (isLeaf()) {
             return null;
         }
 
         StarTreeNode resultStarTreeNode = null;
         if (null != dimensionValue) {
-            resultStarTreeNode = binarySearchChild(dimensionValue);
+            resultStarTreeNode = binarySearchChild(dimensionValue, dimension);
         }
         return resultStarTreeNode;
     }
@@ -240,7 +241,7 @@ private static FixedLengthStarTreeNode matchStarTreeNodeTypeOrNull(FixedLengthSt
      * @return The child node if found, null otherwise
      * @throws IOException If there's an error reading from the input
      */
-    private FixedLengthStarTreeNode binarySearchChild(long dimensionValue) throws IOException {
+    private FixedLengthStarTreeNode binarySearchChild(long dimensionValue, Dimension dimension) throws IOException {
 
         int low = firstChildId;
 
@@ -255,14 +256,15 @@ private FixedLengthStarTreeNode binarySearchChild(long dimensionValue) throws IO
             high--;
         }
 
+        Comparator<Long> comparator = dimension.comparator();
         while (low <= high) {
             int mid = low + (high - low) / 2;
             FixedLengthStarTreeNode midNode = new FixedLengthStarTreeNode(in, mid);
             long midDimensionValue = midNode.getDimensionValue();
-
-            if (midDimensionValue == dimensionValue) {
+            int compare = comparator.compare(midDimensionValue, dimensionValue);
+            if (compare == 0) {
                 return midNode;
-            } else if (midDimensionValue < dimensionValue) {
+            } else if (compare < 0) {
                 low = mid + 1;
             } else {
                 high = mid - 1;