Skip to content

Commit f0e5003

Browse files
sandeshkr419sandeshkr419
authored
[Star Tree] IP field search support (#18671)
* ip field changes Signed-off-by: Sandesh Kumar <[email protected]> * increase coverage Signed-off-by: Sandesh Kumar <[email protected]> --------- Signed-off-by: Sandesh Kumar <[email protected]>
1 parent 3a5da00 commit f0e5003

File tree

5 files changed

+134
-19
lines changed

5 files changed

+134
-19
lines changed

‎CHANGELOG.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
4848
- APIs for stream transport and new stream-based search api action ([#18722](https://github.com/opensearch-project/OpenSearch/pull/18722))
4949
- Added the core process for warming merged segments in remote-store enabled domains ([#18683](https://github.com/opensearch-project/OpenSearch/pull/18683))
5050
- Optimize Composite Aggregations by removing unnecessary object allocations ([#18531](https://github.com/opensearch-project/OpenSearch/pull/18531))
51+
- [Star-Tree] Add search support for ip field type ([#18671](https://github.com/opensearch-project/OpenSearch/pull/18671))
5152

5253
### Changed
5354
- Update Subject interface to use CheckedRunnable ([#18570](https://github.com/opensearch-project/OpenSearch/issues/18570))

‎server/src/main/java/org/opensearch/index/mapper/IpFieldMapper.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -237,7 +237,7 @@ public String typeName() {
237237
return CONTENT_TYPE;
238238
}
239239

240-
private static InetAddress parse(Object value) {
240+
public static InetAddress parse(Object value) {
241241
if (value instanceof InetAddress) {
242242
return (InetAddress) value;
243243
} else {

‎server/src/main/java/org/opensearch/search/startree/filter/provider/DimensionFilterMapper.java‎

Lines changed: 68 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010

1111
import org.apache.lucene.document.DoublePoint;
1212
import org.apache.lucene.document.FloatPoint;
13+
import org.apache.lucene.document.InetAddressPoint;
1314
import org.apache.lucene.index.TermsEnum;
1415
import org.apache.lucene.sandbox.document.HalfFloatPoint;
1516
import org.apache.lucene.util.BytesRef;
@@ -32,6 +33,7 @@
3233
import org.opensearch.search.startree.filter.RangeMatchDimFilter;
3334

3435
import java.io.IOException;
36+
import java.net.InetAddress;
3537
import java.util.ArrayList;
3638
import java.util.Comparator;
3739
import java.util.List;
@@ -161,7 +163,9 @@ class Factory {
161163
org.opensearch.index.mapper.KeywordFieldMapper.CONTENT_TYPE,
162164
new KeywordFieldMapper(),
163165
UNSIGNED_LONG.typeName(),
164-
new UnsignedLongFieldMapperNumeric()
166+
new UnsignedLongFieldMapperNumeric(),
167+
org.opensearch.index.mapper.IpFieldMapper.CONTENT_TYPE,
168+
new IpFieldMapper()
165169
);
166170

167171
public static DimensionFilterMapper fromMappedFieldType(MappedFieldType mappedFieldType, SearchContext searchContext) {
@@ -406,25 +410,25 @@ Number getNextHigh(Number parsedValue) {
406410
}
407411
}
408412

409-
class KeywordFieldMapper implements DimensionFilterMapper {
413+
abstract class OrdinalFieldMapper implements DimensionFilterMapper {
414+
415+
abstract Object parseRawField(String field, Object rawValue, MappedFieldType mappedFieldType) throws IllegalArgumentException;
410416

411417
@Override
412418
public DimensionFilter getExactMatchFilter(MappedFieldType mappedFieldType, List<Object> rawValues) {
413-
KeywordFieldType keywordFieldType = (KeywordFieldType) mappedFieldType;
414419
List<Object> convertedValues = new ArrayList<>(rawValues.size());
415420
for (Object rawValue : rawValues) {
416-
convertedValues.add(parseRawKeyword(mappedFieldType.name(), rawValue, keywordFieldType));
421+
convertedValues.add(parseRawField(mappedFieldType.name(), rawValue, mappedFieldType));
417422
}
418423
return new ExactMatchDimFilter(mappedFieldType.name(), convertedValues);
419424
}
420425

421426
@Override
422427
public DimensionFilter getRangeMatchFilter(MappedFieldType mappedFieldType, StarTreeRangeQuery rangeQuery) {
423-
KeywordFieldType keywordFieldType = (KeywordFieldType) mappedFieldType;
424428
return new RangeMatchDimFilter(
425429
mappedFieldType.name(),
426-
parseRawKeyword(mappedFieldType.name(), rangeQuery.from(), keywordFieldType),
427-
parseRawKeyword(mappedFieldType.name(), rangeQuery.to(), keywordFieldType),
430+
parseRawField(mappedFieldType.name(), rangeQuery.from(), mappedFieldType),
431+
parseRawField(mappedFieldType.name(), rangeQuery.to(), mappedFieldType),
428432
rangeQuery.includeLower(),
429433
rangeQuery.includeUpper()
430434
);
@@ -484,8 +488,20 @@ public Optional<Long> getMatchingOrdinal(
484488
}
485489
}
486490

491+
@Override
492+
public int compareValues(Object v1, Object v2) {
493+
if (!(v1 instanceof BytesRef) || !(v2 instanceof BytesRef)) {
494+
throw new IllegalArgumentException("Expected BytesRef values for comparison");
495+
}
496+
return ((BytesRef) v1).compareTo((BytesRef) v2);
497+
}
498+
}
499+
500+
class KeywordFieldMapper extends OrdinalFieldMapper {
501+
487502
// TODO : Think around making TermBasedFT#indexedValueForSearch() accessor public for reuse here.
488-
private Object parseRawKeyword(String field, Object rawValue, KeywordFieldType keywordFieldType) {
503+
Object parseRawField(String field, Object rawValue, MappedFieldType mappedFieldType) {
504+
KeywordFieldType keywordFieldType = (KeywordFieldType) mappedFieldType;
489505
Object parsedValue = null;
490506
if (rawValue != null) {
491507
if (keywordFieldType.getTextSearchInfo().getSearchAnalyzer() == Lucene.KEYWORD_ANALYZER) {
@@ -499,12 +515,51 @@ private Object parseRawKeyword(String field, Object rawValue, KeywordFieldType k
499515
}
500516
return parsedValue;
501517
}
518+
}
502519

503-
@Override
504-
public int compareValues(Object v1, Object v2) {
505-
if (!(v1 instanceof BytesRef) || !(v2 instanceof BytesRef)) {
506-
throw new IllegalArgumentException("Expected BytesRef values for keyword comparison");
520+
/**
521+
* This class provides functionality to map IP address values for exact and range-based
522+
* filtering within a Star-Tree index. It handles the conversion of IP address
523+
* objects into sortable {@link BytesRef}.
524+
*/
525+
class IpFieldMapper extends OrdinalFieldMapper {
526+
527+
/**
528+
* Parses a raw IP address value into a sortable {@link BytesRef}.
529+
*
530+
* This method handles various input types, including {@link InetAddress}, {@link BytesRef},
531+
* and {@link String}, converting them into a binary representation using
532+
* {@link InetAddressPoint#encode(InetAddress)}.
533+
*
534+
* @param field The name of the field being processed.
535+
* @param rawValue The raw IP address value.
536+
* @return A {@link BytesRef} representation of the IP address, or null if the input is null.
537+
*/
538+
Object parseRawField(String field, Object rawValue, MappedFieldType mappedFieldType) throws IllegalArgumentException {
539+
Object parsedValue = null;
540+
if (rawValue != null) {
541+
try {
542+
switch (rawValue) {
543+
case InetAddress inetAddress -> {
544+
parsedValue = new BytesRef(InetAddressPoint.encode(inetAddress));
545+
}
546+
case BytesRef bytesRef -> {
547+
return bytesRef;
548+
}
549+
case String s -> {
550+
InetAddress addr = InetAddress.getByName(s);
551+
parsedValue = new BytesRef(InetAddressPoint.encode(addr));
552+
}
553+
default -> {
554+
throw new IllegalArgumentException(
555+
"Unsupported value type for IP field [" + field + "]: " + rawValue.getClass().getName()
556+
);
557+
}
558+
}
559+
} catch (Exception e) {
560+
throw new IllegalArgumentException("Failed to parse IP value for field [" + field + "]", e);
561+
}
507562
}
508-
return ((BytesRef) v1).compareTo((BytesRef) v2);
563+
return parsedValue;
509564
}
510565
}

‎server/src/test/java/org/opensearch/search/aggregations/startree/DimensionFilterAndMapperTests.java‎

Lines changed: 32 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88

99
package org.opensearch.search.aggregations.startree;
1010

11+
import org.apache.lucene.document.InetAddressPoint;
1112
import org.apache.lucene.index.TermsEnum;
1213
import org.apache.lucene.util.BytesRef;
1314
import org.opensearch.index.compositeindex.datacube.Metric;
@@ -18,7 +19,9 @@
1819
import org.opensearch.index.compositeindex.datacube.startree.index.StarTreeValues;
1920
import org.opensearch.index.compositeindex.datacube.startree.utils.iterator.SortedSetStarTreeValuesIterator;
2021
import org.opensearch.index.mapper.CompositeDataCubeFieldType;
22+
import org.opensearch.index.mapper.IpFieldMapper;
2123
import org.opensearch.index.mapper.KeywordFieldMapper;
24+
import org.opensearch.index.mapper.MappedFieldType;
2225
import org.opensearch.index.mapper.MapperService;
2326
import org.opensearch.index.mapper.NumberFieldMapper;
2427
import org.opensearch.index.mapper.StarTreeMapper;
@@ -38,6 +41,8 @@
3841
import org.opensearch.test.OpenSearchTestCase;
3942

4043
import java.io.IOException;
44+
import java.net.InetAddress;
45+
import java.net.UnknownHostException;
4146
import java.util.Collections;
4247
import java.util.List;
4348
import java.util.Optional;
@@ -47,12 +52,35 @@
4752

4853
public class DimensionFilterAndMapperTests extends OpenSearchTestCase {
4954

50-
public void testKeywordOrdinalMapping() throws IOException {
55+
public void testIpMapping() throws Exception {
56+
MappedFieldType mappedFieldType = new IpFieldMapper.IpFieldType("ip_field");
57+
BytesRef ipAsBytes = new BytesRef(InetAddressPoint.encode(InetAddress.getByName("192.168.1.1")));
58+
testOrdinalMapping(mappedFieldType, ipAsBytes);
59+
}
60+
61+
public void testRawValuesIpParsing() throws UnknownHostException {
5162
SearchContext searchContext = mock(SearchContext.class);
52-
DimensionFilterMapper dimensionFilterMapper = DimensionFilterMapper.Factory.fromMappedFieldType(
53-
new KeywordFieldMapper.KeywordFieldType("keyword"),
54-
searchContext
63+
MappedFieldType mappedFieldType = new IpFieldMapper.IpFieldType("ip_field");
64+
DimensionFilterMapper dimensionFilterMapper = DimensionFilterMapper.Factory.fromMappedFieldType(mappedFieldType, searchContext);
65+
66+
assertThrows(IllegalArgumentException.class, () -> dimensionFilterMapper.getExactMatchFilter(mappedFieldType, List.of(1.0f)));
67+
assertThrows(
68+
IllegalArgumentException.class,
69+
() -> dimensionFilterMapper.getExactMatchFilter(mappedFieldType, List.of("not.a.valid.ip"))
5570
);
71+
DimensionFilter df = dimensionFilterMapper.getExactMatchFilter(mappedFieldType, List.of(InetAddress.getByName("192.168.1.1")));
72+
assertEquals("ip_field", df.getMatchingDimension());
73+
}
74+
75+
public void testKeywordOrdinalMapping() throws IOException {
76+
MappedFieldType mappedFieldType = new KeywordFieldMapper.KeywordFieldType("keyword");
77+
BytesRef bytesRef = new BytesRef(new byte[] { 17, 29 });
78+
testOrdinalMapping(mappedFieldType, bytesRef);
79+
}
80+
81+
private void testOrdinalMapping(final MappedFieldType mappedFieldType, final BytesRef bytesRef) throws IOException {
82+
SearchContext searchContext = mock(SearchContext.class);
83+
DimensionFilterMapper dimensionFilterMapper = DimensionFilterMapper.Factory.fromMappedFieldType(mappedFieldType, searchContext);
5684
StarTreeValues starTreeValues = mock(StarTreeValues.class);
5785
SortedSetStarTreeValuesIterator sortedSetStarTreeValuesIterator = mock(SortedSetStarTreeValuesIterator.class);
5886
TermsEnum termsEnum = mock(TermsEnum.class);
@@ -61,7 +89,6 @@ public void testKeywordOrdinalMapping() throws IOException {
6189
Optional<Long> matchingOrdinal;
6290

6391
// Case Exact Match and found
64-
BytesRef bytesRef = new BytesRef(new byte[] { 17, 29 });
6592
when(sortedSetStarTreeValuesIterator.lookupTerm(bytesRef)).thenReturn(1L);
6693
matchingOrdinal = dimensionFilterMapper.getMatchingOrdinal("field", bytesRef, starTreeValues, MatchType.EXACT);
6794
assertTrue(matchingOrdinal.isPresent());

‎server/src/test/java/org/opensearch/search/aggregations/startree/MetricAggregatorTests.java‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,10 +18,12 @@
1818
import org.apache.lucene.document.DoubleField;
1919
import org.apache.lucene.document.Field;
2020
import org.apache.lucene.document.FloatField;
21+
import org.apache.lucene.document.InetAddressPoint;
2122
import org.apache.lucene.document.IntField;
2223
import org.apache.lucene.document.KeywordField;
2324
import org.apache.lucene.document.LongField;
2425
import org.apache.lucene.document.SortedNumericDocValuesField;
26+
import org.apache.lucene.document.SortedSetDocValuesField;
2527
import org.apache.lucene.index.DirectoryReader;
2628
import org.apache.lucene.index.IndexWriterConfig;
2729
import org.apache.lucene.index.IndexableField;
@@ -32,6 +34,7 @@
3234
import org.apache.lucene.search.Query;
3335
import org.apache.lucene.store.Directory;
3436
import org.apache.lucene.tests.index.RandomIndexWriter;
37+
import org.apache.lucene.util.BytesRef;
3538
import org.opensearch.common.lucene.Lucene;
3639
import org.opensearch.common.settings.Settings;
3740
import org.opensearch.common.util.MockBigArrays;
@@ -47,6 +50,7 @@
4750
import org.opensearch.index.compositeindex.datacube.MetricStat;
4851
import org.opensearch.index.compositeindex.datacube.NumericDimension;
4952
import org.opensearch.index.compositeindex.datacube.OrdinalDimension;
53+
import org.opensearch.index.mapper.IpFieldMapper;
5054
import org.opensearch.index.mapper.KeywordFieldMapper;
5155
import org.opensearch.index.mapper.MappedFieldType;
5256
import org.opensearch.index.mapper.MapperService;
@@ -77,6 +81,7 @@
7781

7882
import java.io.IOException;
7983
import java.math.BigInteger;
84+
import java.net.InetAddress;
8085
import java.util.ArrayList;
8186
import java.util.Collections;
8287
import java.util.LinkedHashMap;
@@ -136,6 +141,7 @@ public void testStarTreeDocValues() throws IOException {
136141
new DimensionFieldData("half_float_field", () -> random().nextFloat(50), DimensionTypes.HALF_FLOAT),
137142
new DimensionFieldData("float_field", () -> random().nextFloat(50), DimensionTypes.FLOAT),
138143
new DimensionFieldData("double_field", () -> random().nextDouble(50), DimensionTypes.DOUBLE),
144+
new DimensionFieldData("ip_field", this::randomIp, DimensionTypes.IP),
139145
new DimensionFieldData("unsigned_long_field", () -> {
140146
long queryValue = randomBoolean()
141147
? 9223372036854775807L - random().nextInt(100000)
@@ -658,6 +664,28 @@ NumberFieldMapper.NumberType numberType() {
658664
public IndexableField getField(String fieldName, Supplier<Object> valueSupplier) {
659665
return new BigIntegerField(fieldName, (BigInteger) valueSupplier.get(), Field.Store.YES);
660666
}
667+
}),
668+
IP(new DimensionFieldDataSupplier() {
669+
@Override
670+
public IndexableField getField(String fieldName, Supplier<Object> valueSupplier) {
671+
try {
672+
InetAddress address = InetAddress.getByName((String) valueSupplier.get());
673+
return new SortedSetDocValuesField(fieldName, new BytesRef(InetAddressPoint.encode(address)));
674+
675+
} catch (Exception e) {
676+
throw new RuntimeException(e);
677+
}
678+
}
679+
680+
@Override
681+
public MappedFieldType getMappedField(String fieldName) {
682+
return new IpFieldMapper.IpFieldType(fieldName);
683+
}
684+
685+
@Override
686+
public Dimension getDimension(String fieldName) {
687+
return new OrdinalDimension(fieldName);
688+
}
661689
});
662690

663691
private final DimensionFieldDataSupplier dimensionFieldDataSupplier;
@@ -680,4 +708,8 @@ private String asUnsignedDecimalString(long l) {
680708
return b.toString();
681709
}
682710

711+
private String randomIp() {
712+
Random r = random();
713+
return r.nextInt(256) + "." + r.nextInt(256) + "." + r.nextInt(256) + "." + r.nextInt(256);
714+
}
683715
}

0 commit comments

Comments
 (0)