Skip to content

Commit 5c9fee9

Browse files
author
Shailesh Singh
committed
add query changes to support unsigned-long in star tree
Signed-off-by: Shailesh Singh <[email protected]>
1 parent b823d1f commit 5c9fee9

File tree

10 files changed

+254
-47
lines changed

10 files changed

+254
-47
lines changed

‎CHANGELOG.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
4141
- Propagate the sourceIncludes and excludes fields from fetchSourceContext to FieldsVisitor. ([#17080](https://github.com/opensearch-project/OpenSearch/pull/17080))
4242
- [Star Tree] [Search] Resolving Date histogram with metric aggregation using star-tree ([#16674](https://github.com/opensearch-project/OpenSearch/pull/16674))
4343
- [Star Tree] [Search] Extensible design to support different query and field types ([#17137](https://github.com/opensearch-project/OpenSearch/pull/17137))
44+
- [Star Tree] [Search] Add query changes to support unsigned-long in star tree ([#17275](https://github.com/opensearch-project/OpenSearch/pull/17275))
4445

4546
### Dependencies
4647
- Bump `com.google.cloud:google-cloud-core-http` from 2.23.0 to 2.47.0 ([#16504](https://github.com/opensearch-project/OpenSearch/pull/16504))

‎server/src/main/java/org/opensearch/index/compositeindex/datacube/DimensionDataType.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@
1919
public enum DimensionDataType {
2020
LONG {
2121
@Override
22-
int compare(Long a, Long b) {
22+
public int compare(Long a, Long b) {
2323
if (a == null && b == null) {
2424
return 0;
2525
}
@@ -34,7 +34,7 @@ int compare(Long a, Long b) {
3434
},
3535
UNSIGNED_LONG {
3636
@Override
37-
int compare(Long a, Long b) {
37+
public int compare(Long a, Long b) {
3838
if (a == null && b == null) {
3939
return 0;
4040
}
@@ -48,5 +48,5 @@ int compare(Long a, Long b) {
4848
}
4949
};
5050

51-
abstract int compare(Long a, Long b);
51+
public abstract int compare(Long a, Long b);
5252
}

‎server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/node/FixedLengthStarTreeNode.java‎

Lines changed: 30 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414

1515
import java.io.IOException;
1616
import java.io.UncheckedIOException;
17+
import java.util.Comparator;
1718
import java.util.Iterator;
1819

1920
/**
@@ -193,15 +194,16 @@ public StarTreeNode getChildStarNode() throws IOException {
193194
}
194195

195196
@Override
196-
public StarTreeNode getChildForDimensionValue(Long dimensionValue, StarTreeNode lastMatchedChild) throws IOException {
197+
public StarTreeNode getChildForDimensionValue(Long dimensionValue, StarTreeNode lastMatchedChild, Comparator<Long> comparator)
198+
throws IOException {
197199
// there will be no children for leaf nodes
198200
if (isLeaf()) {
199201
return null;
200202
}
201203

202204
StarTreeNode resultStarTreeNode = null;
203205
if (null != dimensionValue) {
204-
resultStarTreeNode = binarySearchChild(dimensionValue, lastMatchedChild);
206+
resultStarTreeNode = binarySearchChild(dimensionValue, lastMatchedChild, comparator);
205207
}
206208
return resultStarTreeNode;
207209
}
@@ -238,11 +240,13 @@ private static FixedLengthStarTreeNode matchStarTreeNodeTypeOrNull(FixedLengthSt
238240
* Performs a binary search to find a child node with the given dimension value.
239241
*
240242
* @param dimensionValue The dimension value to search for
243+
* @param lastMatchedNode : If not null, we begin the binary search from the node after this.
244+
* @param comparator : Comparator (LONG or UNSIGNED_LONG) to compare the dimension values
241245
* @return The child node if found, null otherwise
242246
* @throws IOException If there's an error reading from the input
243247
*/
244-
private FixedLengthStarTreeNode binarySearchChild(long dimensionValue, StarTreeNode lastMatchedNode) throws IOException {
245-
248+
private FixedLengthStarTreeNode binarySearchChild(long dimensionValue, StarTreeNode lastMatchedNode, Comparator<Long> comparator)
249+
throws IOException {
246250
int low = firstChildId;
247251

248252
int high = getInt(LAST_CHILD_ID_OFFSET);
@@ -268,10 +272,10 @@ private FixedLengthStarTreeNode binarySearchChild(long dimensionValue, StarTreeN
268272
int mid = low + (high - low) / 2;
269273
FixedLengthStarTreeNode midNode = new FixedLengthStarTreeNode(in, mid);
270274
long midDimensionValue = midNode.getDimensionValue();
271-
272-
if (midDimensionValue == dimensionValue) {
275+
int compare = comparator.compare(midDimensionValue, dimensionValue);
276+
if (compare == 0) {
273277
return midNode;
274-
} else if (midDimensionValue < dimensionValue) {
278+
} else if (compare < 0) {
275279
low = mid + 1;
276280
} else {
277281
high = mid - 1;
@@ -281,16 +285,18 @@ private FixedLengthStarTreeNode binarySearchChild(long dimensionValue, StarTreeN
281285
}
282286

283287
@Override
284-
public void collectChildrenInRange(long low, long high, StarTreeNodeCollector collector) throws IOException {
285-
if (low <= high) {
286-
FixedLengthStarTreeNode lowStarTreeNode = binarySearchChild(low, true, null);
288+
public void collectChildrenInRange(long low, long high, StarTreeNodeCollector collector, Comparator<Long> comparator)
289+
throws IOException {
290+
if (comparator.compare(low, high) <= 0) {
291+
FixedLengthStarTreeNode lowStarTreeNode = binarySearchChild(low, true, null, comparator);
287292
if (lowStarTreeNode != null) {
288-
FixedLengthStarTreeNode highStarTreeNode = binarySearchChild(high, false, lowStarTreeNode);
293+
FixedLengthStarTreeNode highStarTreeNode = binarySearchChild(high, false, lowStarTreeNode, comparator);
289294
if (highStarTreeNode != null) {
290295
for (int lowNodeId = lowStarTreeNode.nodeId(); lowNodeId <= highStarTreeNode.nodeId(); ++lowNodeId) {
291296
collector.collectStarTreeNode(new FixedLengthStarTreeNode(in, lowNodeId));
292297
}
293-
} else if (lowStarTreeNode.getDimensionValue() <= high) { // Low StarTreeNode is the last default node for that dimension.
298+
} else if (comparator.compare(lowStarTreeNode.getDimensionValue(), high) <= 0) { // Low StarTreeNode is the last default//
299+
// node for that dimension.
294300
collector.collectStarTreeNode(lowStarTreeNode);
295301
}
296302
}
@@ -302,11 +308,16 @@ public void collectChildrenInRange(long low, long high, StarTreeNodeCollector co
302308
* @param dimensionValue : The dimension to match.
303309
* @param matchNextHighest : If true then we try to return @dimensionValue or the next Highest. Else, we return @dimensionValue or the next Lowest.
304310
* @param lastMatchedNode : If not null, we begin the binary search from the node after this.
311+
* @param comparator : Comparator (LONG or UNSIGNED_LONG) to compare the dimension values
305312
* @return : Matched node or null.
306313
* @throws IOException :
307314
*/
308-
private FixedLengthStarTreeNode binarySearchChild(long dimensionValue, boolean matchNextHighest, StarTreeNode lastMatchedNode)
309-
throws IOException {
315+
private FixedLengthStarTreeNode binarySearchChild(
316+
long dimensionValue,
317+
boolean matchNextHighest,
318+
StarTreeNode lastMatchedNode,
319+
Comparator<Long> comparator
320+
) throws IOException {
310321

311322
int low = firstChildId;
312323
int tempLow = low;
@@ -342,17 +353,18 @@ private FixedLengthStarTreeNode binarySearchChild(long dimensionValue, boolean m
342353
FixedLengthStarTreeNode midNode = new FixedLengthStarTreeNode(in, mid);
343354
long midDimensionValue = midNode.getDimensionValue();
344355

345-
if (midDimensionValue == dimensionValue) {
356+
int compare = comparator.compare(midDimensionValue, dimensionValue);
357+
if (compare == 0) {
346358
return midNode;
347359
} else {
348-
if (midDimensionValue < dimensionValue) { // Going to the right from mid to search next
360+
if (compare < 0) { // Going to the right from mid to search next
349361
tempLow = mid + 1;
350362
// We are going out of bounds for this dimension on the right side.
351363
if (tempLow > high || tempLow == nullNodeId) {
352364
return matchNextHighest ? null : midNode;
353365
} else {
354366
FixedLengthStarTreeNode nodeGreaterThanMid = new FixedLengthStarTreeNode(in, tempLow);
355-
if (nodeGreaterThanMid.getDimensionValue() > dimensionValue) {
367+
if (comparator.compare(nodeGreaterThanMid.getDimensionValue(), dimensionValue) > 0) {
356368
return matchNextHighest ? nodeGreaterThanMid : midNode;
357369
}
358370
}
@@ -363,7 +375,7 @@ private FixedLengthStarTreeNode binarySearchChild(long dimensionValue, boolean m
363375
return matchNextHighest ? midNode : null;
364376
} else {
365377
FixedLengthStarTreeNode nodeLessThanMid = new FixedLengthStarTreeNode(in, tempHigh);
366-
if (nodeLessThanMid.getDimensionValue() < dimensionValue) {
378+
if (comparator.compare(nodeLessThanMid.getDimensionValue(), dimensionValue) < 0) {
367379
return matchNextHighest ? midNode : nodeLessThanMid;
368380
}
369381
}

‎server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/node/StarTreeNode.java‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -9,9 +9,11 @@
99
package org.opensearch.index.compositeindex.datacube.startree.node;
1010

1111
import org.opensearch.common.annotation.ExperimentalApi;
12+
import org.opensearch.index.compositeindex.datacube.DimensionDataType;
1213
import org.opensearch.search.startree.StarTreeNodeCollector;
1314

1415
import java.io.IOException;
16+
import java.util.Comparator;
1517
import java.util.Iterator;
1618

1719
/**
@@ -109,26 +111,29 @@ public interface StarTreeNode {
109111
* @throws IOException if an I/O error occurs while retrieving the child node
110112
*/
111113
default StarTreeNode getChildForDimensionValue(Long dimensionValue) throws IOException {
112-
return getChildForDimensionValue(dimensionValue, null);
114+
return getChildForDimensionValue(dimensionValue, null, DimensionDataType.LONG::compare);
113115
}
114116

115117
/**
116118
* Matches the given @dimensionValue amongst the child default nodes for this node.
117119
* @param dimensionValue : Value to match
118120
* @param lastMatchedChild : If not null, binary search will use this as the start/low
121+
* @param comparator : Comparator (LONG or UNSIGNED_LONG) to compare the dimension values
119122
* @return : Matched StarTreeNode or null if not found
120123
* @throws IOException : Any exception in reading the node data from index.
121124
*/
122-
StarTreeNode getChildForDimensionValue(Long dimensionValue, StarTreeNode lastMatchedChild) throws IOException;
125+
StarTreeNode getChildForDimensionValue(Long dimensionValue, StarTreeNode lastMatchedChild, Comparator<Long> comparator)
126+
throws IOException;
123127

124128
/**
125129
* Collects all matching child nodes whose dimension values lie within the range of low and high, both inclusive.
126130
* @param low : Starting of the range ( inclusive )
127131
* @param high : End of the range ( inclusive )
128132
* @param collector : Collector to collect the matched child StarTreeNode's
133+
* @param comparator : Comparator (LONG or UNSIGNED_LONG) to compare the dimension values
129134
* @throws IOException : Any exception in reading the node data from index.
130135
*/
131-
void collectChildrenInRange(long low, long high, StarTreeNodeCollector collector) throws IOException;
136+
void collectChildrenInRange(long low, long high, StarTreeNodeCollector collector, Comparator<Long> comparator) throws IOException;
132137

133138
/**
134139
* Returns the child star node for a node in the star-tree.

‎server/src/main/java/org/opensearch/search/startree/filter/ExactMatchDimFilter.java‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
import org.opensearch.search.startree.filter.provider.DimensionFilterMapper;
1919

2020
import java.io.IOException;
21+
import java.util.Comparator;
2122
import java.util.List;
2223
import java.util.Optional;
2324
import java.util.TreeSet;
@@ -35,6 +36,8 @@ public class ExactMatchDimFilter implements DimensionFilter {
3536
// Order is essential for successive binary search
3637
private TreeSet<Long> convertedOrdinals;
3738

39+
Comparator<Long> comparator;
40+
3841
public ExactMatchDimFilter(String dimensionName, List<Object> valuesToMatch) {
3942
this.dimensionName = dimensionName;
4043
this.rawValues = valuesToMatch;
@@ -50,6 +53,7 @@ public void initialiseForSegment(StarTreeValues starTreeValues, SearchContext se
5053
DimensionFilterMapper dimensionFilterMapper = DimensionFilterMapper.Factory.fromMappedFieldType(
5154
searchContext.mapperService().fieldType(dimensionName)
5255
);
56+
this.comparator = dimensionFilterMapper.comparator();
5357
for (Object rawValue : rawValues) {
5458
Optional<Long> ordinal = dimensionFilterMapper.getMatchingOrdinal(
5559
matchedDim.getField(),
@@ -69,7 +73,7 @@ public void matchStarTreeNodes(StarTreeNode parentNode, StarTreeValues starTreeV
6973
if (parentNode != null) {
7074
StarTreeNode lastMatchedNode = null;
7175
for (long ordinal : convertedOrdinals) {
72-
lastMatchedNode = parentNode.getChildForDimensionValue(ordinal, lastMatchedNode);
76+
lastMatchedNode = parentNode.getChildForDimensionValue(ordinal, lastMatchedNode, comparator);
7377
if (lastMatchedNode != null) {
7478
collector.collectStarTreeNode(lastMatchedNode);
7579
}

‎server/src/main/java/org/opensearch/search/startree/filter/RangeMatchDimFilter.java‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
import org.opensearch.search.startree.filter.provider.DimensionFilterMapper;
1717

1818
import java.io.IOException;
19+
import java.util.Comparator;
1920
import java.util.Optional;
2021

2122
/**
@@ -37,6 +38,8 @@ public class RangeMatchDimFilter implements DimensionFilter {
3738

3839
private boolean skipRangeCollection = false;
3940

41+
private Comparator<Long> comparator;
42+
4043
public RangeMatchDimFilter(String dimensionName, Object low, Object high, boolean includeLow, boolean includeHigh) {
4144
this.dimensionName = dimensionName;
4245
this.low = low;
@@ -51,6 +54,8 @@ public void initialiseForSegment(StarTreeValues starTreeValues, SearchContext se
5154
DimensionFilterMapper dimensionFilterMapper = DimensionFilterMapper.Factory.fromMappedFieldType(
5255
searchContext.mapperService().fieldType(dimensionName)
5356
);
57+
this.comparator = dimensionFilterMapper.comparator();
58+
5459
lowOrdinal = 0L;
5560
if (low != null) {
5661
MatchType lowMatchType = includeLow ? MatchType.GTE : MatchType.GT;
@@ -77,13 +82,13 @@ public void initialiseForSegment(StarTreeValues starTreeValues, SearchContext se
7782
public void matchStarTreeNodes(StarTreeNode parentNode, StarTreeValues starTreeValues, StarTreeNodeCollector collector)
7883
throws IOException {
7984
if (parentNode != null && !skipRangeCollection) {
80-
parentNode.collectChildrenInRange(lowOrdinal, highOrdinal, collector);
85+
parentNode.collectChildrenInRange(lowOrdinal, highOrdinal, collector, comparator);
8186
}
8287
}
8388

8489
@Override
8590
public boolean matchDimValue(long ordinal, StarTreeValues starTreeValues) {
86-
return lowOrdinal <= ordinal && ordinal <= highOrdinal;
91+
return comparator.compare(lowOrdinal, ordinal) <= 0 && comparator.compare(ordinal, highOrdinal) <= 0;
8792
}
8893

8994
}

‎server/src/main/java/org/opensearch/search/startree/filter/provider/DimensionFilterMapper.java‎

Lines changed: 42 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -14,9 +14,11 @@
1414
import org.apache.lucene.sandbox.document.HalfFloatPoint;
1515
import org.apache.lucene.util.BytesRef;
1616
import org.apache.lucene.util.NumericUtils;
17+
import org.opensearch.common.Numbers;
1718
import org.opensearch.common.annotation.ExperimentalApi;
1819
import org.opensearch.common.lucene.BytesRefs;
1920
import org.opensearch.common.lucene.Lucene;
21+
import org.opensearch.index.compositeindex.datacube.DimensionDataType;
2022
import org.opensearch.index.compositeindex.datacube.startree.index.StarTreeValues;
2123
import org.opensearch.index.compositeindex.datacube.startree.utils.iterator.SortedSetStarTreeValuesIterator;
2224
import org.opensearch.index.mapper.KeywordFieldMapper.KeywordFieldType;
@@ -29,6 +31,7 @@
2931

3032
import java.io.IOException;
3133
import java.util.ArrayList;
34+
import java.util.Comparator;
3235
import java.util.List;
3336
import java.util.Map;
3437
import java.util.Optional;
@@ -40,6 +43,7 @@
4043
import static org.opensearch.index.mapper.NumberFieldMapper.NumberType.INTEGER;
4144
import static org.opensearch.index.mapper.NumberFieldMapper.NumberType.LONG;
4245
import static org.opensearch.index.mapper.NumberFieldMapper.NumberType.SHORT;
46+
import static org.opensearch.index.mapper.NumberFieldMapper.NumberType.UNSIGNED_LONG;
4347
import static org.opensearch.index.mapper.NumberFieldMapper.NumberType.hasDecimalPart;
4448
import static org.opensearch.index.mapper.NumberFieldMapper.NumberType.signum;
4549

@@ -88,6 +92,20 @@ Optional<Long> getMatchingOrdinal(
8892
DimensionFilter.MatchType matchType
8993
);
9094

95+
/**
96+
* Returns the dimensionDataType used for comparing dimension values. <br>
97+
* This determines how numeric values are compared: <br>
98+
* - DimensionDataType.UNSIGNED_LONG for unsigned long values <br>
99+
* - DimensionDataType.LONG for all other numeric types (DEFAULT)
100+
*/
101+
default DimensionDataType getDimensionDataType() {
102+
return DimensionDataType.LONG;
103+
}
104+
105+
default Comparator<Long> comparator() {
106+
return (a, b) -> getDimensionDataType().compare(a, b);
107+
}
108+
91109
/**
92110
* Singleton Factory for @{@link DimensionFilterMapper}
93111
*/
@@ -109,7 +127,9 @@ class Factory {
109127
DOUBLE.typeName(),
110128
new DoubleFieldMapperNumeric(),
111129
org.opensearch.index.mapper.KeywordFieldMapper.CONTENT_TYPE,
112-
new KeywordFieldMapper()
130+
new KeywordFieldMapper(),
131+
UNSIGNED_LONG.typeName(),
132+
new UnsignedLongFieldMapperNumeric()
113133
);
114134

115135
public static DimensionFilterMapper fromMappedFieldType(MappedFieldType mappedFieldType) {
@@ -161,14 +181,14 @@ public DimensionFilter getRangeMatchFilter(
161181
Long parsedLow = rawLow == null ? defaultMinimum() : numberFieldType.numberType().parse(rawLow, true).longValue();
162182
Long parsedHigh = rawHigh == null ? defaultMaximum() : numberFieldType.numberType().parse(rawHigh, true).longValue();
163183

164-
boolean lowerTermHasDecimalPart = hasDecimalPart(parsedLow);
184+
boolean lowerTermHasDecimalPart = hasDecimalPart(rawLow);
165185
if ((lowerTermHasDecimalPart == false && includeLow == false) || (lowerTermHasDecimalPart && signum(parsedLow) > 0)) {
166186
if (parsedLow.equals(defaultMaximum())) {
167187
return new MatchNoneFilter();
168188
}
169189
++parsedLow;
170190
}
171-
boolean upperTermHasDecimalPart = hasDecimalPart(parsedHigh);
191+
boolean upperTermHasDecimalPart = hasDecimalPart(rawHigh);
172192
if ((upperTermHasDecimalPart == false && includeHigh == false) || (upperTermHasDecimalPart && signum(parsedHigh) < 0)) {
173193
if (parsedHigh.equals(defaultMinimum())) {
174194
return new MatchNoneFilter();
@@ -208,6 +228,25 @@ Long defaultMaximum() {
208228
}
209229
}
210230

231+
class UnsignedLongFieldMapperNumeric extends NumericNonDecimalMapper {
232+
233+
@Override
234+
Long defaultMinimum() {
235+
return Numbers.MIN_UNSIGNED_LONG_VALUE_AS_LONG;
236+
}
237+
238+
@Override
239+
Long defaultMaximum() {
240+
return Numbers.MAX_UNSIGNED_LONG_VALUE_AS_LONG;
241+
}
242+
243+
@Override
244+
public DimensionDataType getDimensionDataType() {
245+
return DimensionDataType.UNSIGNED_LONG;
246+
}
247+
248+
}
249+
211250
abstract class NumericDecimalFieldMapper extends NumericMapper {
212251

213252
@Override

0 commit comments

Comments
 (0)