Upgrade lucene to version 10.2.1

CHANGELOG.md

@@ -30,6 +30,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 ### Changed
 
 ### Dependencies
+- Update Apache Lucene from 10.1.0 to 10.2.1 ([#17961](https://github.com/opensearch-project/OpenSearch/pull/17961))
 - Bump `com.google.code.gson:gson` from 2.12.1 to 2.13.1 ([#17923](https://github.com/opensearch-project/OpenSearch/pull/17923), [#18266](https://github.com/opensearch-project/OpenSearch/pull/18266))
 - Bump `com.github.spotbugs:spotbugs-annotations` from 4.9.0 to 4.9.3 ([#17922](https://github.com/opensearch-project/OpenSearch/pull/17922))
 - Bump `com.microsoft.azure:msal4j` from 1.18.0 to 1.20.0 ([#17925](https://github.com/opensearch-project/OpenSearch/pull/17925))

gradle/libs.versions.toml

@@ -1,6 +1,6 @@
 [versions]
 opensearch        = "3.1.0"
-lucene            = "10.1.0"
+lucene            = "10.2.1"
 
 bundled_jdk_vendor = "adoptium"
 bundled_jdk = "21.0.7+6"
@@ -11,7 +11,7 @@ jts               = "1.15.0"
 jackson           = "2.18.2"
 jackson_databind  = "2.18.2"
 snakeyaml         = "2.1"
-icu4j             = "75.1"
+icu4j             = "77.1"
 supercsv          = "2.4.0"
 log4j             = "2.21.0"
 slf4j             = "1.7.36"

libs/core/licenses/lucene-core-10.2.1.jar.sha1

@@ -0,0 +1 @@
+f7bee3425ff6bff5e3ba50abfa1a0fdab2b2e46f

libs/core/src/main/java/org/opensearch/Version.java

@@ -118,7 +118,7 @@ public class Version implements Comparable<Version>, ToXContentFragment {
     public static final Version V_2_19_2 = new Version(2190299, org.apache.lucene.util.Version.LUCENE_9_12_1);
     public static final Version V_2_19_3 = new Version(2190399, org.apache.lucene.util.Version.LUCENE_9_12_1);
     public static final Version V_3_0_0 = new Version(3000099, org.apache.lucene.util.Version.LUCENE_10_1_0);
-    public static final Version V_3_1_0 = new Version(3010099, org.apache.lucene.util.Version.LUCENE_10_1_0);
+    public static final Version V_3_1_0 = new Version(3010099, org.apache.lucene.util.Version.LUCENE_10_2_1);
     public static final Version CURRENT = V_3_1_0;
 
     public static Version fromId(int id) {

modules/lang-expression/licenses/lucene-expressions-10.2.1.jar.sha1

@@ -0,0 +1 @@
+7704566e72d10bd73bea5ee0a7cfcdc98b9f50cf

plugins/analysis-icu/licenses/icu4j-77.1.jar.sha1

@@ -0,0 +1 @@
+38693cf0b1d7362a8b726af74dc06026a7c23809

plugins/analysis-icu/licenses/lucene-analysis-icu-10.2.1.jar.sha1

@@ -0,0 +1 @@
+4ffa9bc8accae6d5a4c988653d1fd12d6384bc63

plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-10.2.1.jar.sha1

@@ -0,0 +1 @@
+3c37f5e3b1177a55ec8953ec343af5c578a02f40

plugins/analysis-nori/licenses/lucene-analysis-nori-10.2.1.jar.sha1

@@ -0,0 +1 @@
+df44aacdf3a1ef743f42e53dca65590b95bbcbdf

plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-10.2.1.jar.sha1

@@ -0,0 +1 @@
+026d48b44b17dff97ae5afd9007b124410ede1d1

plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-10.2.1.jar.sha1

@@ -0,0 +1 @@
+e9c9447fd1729bdfa4b284fdfe52202300a0be8c

plugins/analysis-stempel/licenses/lucene-analysis-stempel-10.2.1.jar.sha1

@@ -0,0 +1 @@
+80a5c751308add50f6d1d33acbfafd58d27b8b55

plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-10.2.1.jar.sha1

@@ -0,0 +1 @@
+68df86cb2c1dddc3937824ea2cbe7803bf543ca1

server/licenses/lucene-analysis-common-10.2.1.jar.sha1

@@ -0,0 +1 @@
+82578c7a8804490f1f375ccf63e080afb8c0736b

server/licenses/lucene-backward-codecs-10.2.1.jar.sha1

@@ -0,0 +1 @@
+32eef677c300ac9989bda1045904adf8e925b9eb

server/licenses/lucene-core-10.2.1.jar.sha1

@@ -0,0 +1 @@
+f7bee3425ff6bff5e3ba50abfa1a0fdab2b2e46f

server/licenses/lucene-grouping-10.2.1.jar.sha1

@@ -0,0 +1 @@
+5d5069134efb0c6b482c2dcfecf8a90acf1ee03a

server/licenses/lucene-highlighter-10.2.1.jar.sha1

@@ -0,0 +1 @@
+4166f6ddb748bfab3355ef29253377c60ef5c440

server/licenses/lucene-join-10.2.1.jar.sha1

@@ -0,0 +1 @@
+ef2fe517b73f4b144b9ada326ae69c8f3e996042

server/licenses/lucene-memory-10.2.1.jar.sha1

@@ -0,0 +1 @@
+7e595a1dff0859818dea02729123c3e1b2520e7f

server/licenses/lucene-misc-10.2.1.jar.sha1

@@ -0,0 +1 @@
+d4244b7b5f406e40d3f9805a5cb056e351242e86

server/licenses/lucene-queries-10.2.1.jar.sha1

@@ -0,0 +1 @@
+ddb958b144351a7e693d8866d086aed443792ff6

server/licenses/lucene-queryparser-10.2.1.jar.sha1

@@ -0,0 +1 @@
+f9d7cb8ec0f9fc838326e499c4bd0cb044af75e2

server/licenses/lucene-sandbox-10.2.1.jar.sha1

@@ -0,0 +1 @@
+ec539453eb49dd9e02f8ff4b14dc36a08f91436f

server/licenses/lucene-spatial-extras-10.2.1.jar.sha1

@@ -0,0 +1 @@
+d89b7dc7e2c86b5e8184b60732211f762c074979

server/licenses/lucene-spatial3d-10.2.1.jar.sha1

@@ -0,0 +1 @@
+9935b848d1e92eb8938b0cc5ce1b3ee565b0e231

server/licenses/lucene-suggest-10.2.1.jar.sha1

@@ -0,0 +1 @@
+f798a0f8f3686a75bd3830610156d3b087299340

server/src/main/java/org/opensearch/search/aggregations/bucket/composite/PointsSortedDocsProducer.java

@@ -86,7 +86,7 @@ DocIdSet processLeaf(Query query, CompositeValuesCollectorQueue queue, LeafReade
             upperBucket = (Long) upperValue;
         }
 
-        DocIdSetBuilder builder = fillDocIdSet ? new DocIdSetBuilder(context.reader().maxDoc(), values, field) : null;
+        DocIdSetBuilder builder = fillDocIdSet ? new DocIdSetBuilder(context.reader().maxDoc(), values) : null;
         Visitor visitor = new Visitor(context, queue, builder, values.getBytesPerDimension(), lowerBucket, upperBucket);
         try {
             values.intersect(visitor);

server/src/main/java/org/opensearch/search/aggregations/bucket/composite/UnsignedLongPointsSortedDocsProducer.java

@@ -61,7 +61,7 @@ DocIdSet processLeaf(Query query, CompositeValuesCollectorQueue queue, LeafReade
             }
             upperBucket = (BigInteger) upperValue;
         }
-        DocIdSetBuilder builder = fillDocIdSet ? new DocIdSetBuilder(context.reader().maxDoc(), values, field) : null;
+        DocIdSetBuilder builder = fillDocIdSet ? new DocIdSetBuilder(context.reader().maxDoc(), values) : null;
         Visitor visitor = new Visitor(
             context,
             queue,

server/src/main/java/org/opensearch/search/aggregations/metrics/CardinalityAggregator.java

@@ -358,7 +358,7 @@
                 postingMap.put(term, scorer);
             }
 
-            this.queue = new DisiPriorityQueue(postingMap.size());
+            this.queue = DisiPriorityQueue.ofMaxSize(postingMap.size());
             for (Scorer scorer : postingMap.values()) {
                 queue.add(new DisiWrapper(scorer, false));
             }
@@ -452,15 +452,13 @@
         }
 
         @Override
-        public int nextDoc() {
-            // don't expect this to be called based on its usage in DefaultBulkScorer
-            throw new UnsupportedOperationException();
+        public int nextDoc() throws IOException {
+            return advance(slowDocId + 1);
         }
 
         @Override
         public long cost() {
-            // don't expect this to be called based on its usage in DefaultBulkScorer
-            throw new UnsupportedOperationException();
+            return queue.top() == null ? 0 : queue.top().approximation.cost();
         }
     }
 

server/src/main/java/org/opensearch/search/approximate/ApproximatePointRangeQuery.java

@@ -367,7 +367,7 @@
                     if (sortOrder == null || sortOrder.equals(SortOrder.ASC)) {
                         return new ScorerSupplier() {
 
-                            final DocIdSetBuilder result = new DocIdSetBuilder(reader.maxDoc(), values, pointRangeQuery.getField());
+                            final DocIdSetBuilder result = new DocIdSetBuilder(reader.maxDoc(), values);
                             final PointValues.IntersectVisitor visitor = getIntersectVisitor(result, docCount);
                             long cost = -1;
 
@@ -395,7 +395,7 @@
                         size += deletedDocs;
                         return new ScorerSupplier() {
 
-                            final DocIdSetBuilder result = new DocIdSetBuilder(reader.maxDoc(), values, pointRangeQuery.getField());
+                            final DocIdSetBuilder result = new DocIdSetBuilder(reader.maxDoc(), values);
                             final PointValues.IntersectVisitor visitor = getIntersectVisitor(result, docCount);
                             long cost = -1;
 

server/src/main/java/org/opensearch/search/query/BitmapIndexQuery.java

@@ -112,7 +112,7 @@ public ScorerSupplier scorerSupplier(LeafReaderContext context) throws IOExcepti
                 return new ScorerSupplier() {
                     long cost = -1;
 
-                    final DocIdSetBuilder result = new DocIdSetBuilder(reader.maxDoc(), values, field);
+                    final DocIdSetBuilder result = new DocIdSetBuilder(reader.maxDoc(), values);
                     final MergePointVisitor visitor = new MergePointVisitor(result);
 
                     @Override

server/src/test/java/org/opensearch/search/query/QueryPhaseTests.java

@@ -988,7 +988,7 @@ public void testMinScore() throws Exception {
         context.trackTotalHitsUpTo(5);
 
         QueryPhase.executeInternal(context.withCleanQueryResult(), queryPhaseSearcher);
-        assertEquals(10, context.queryResult().topDocs().topDocs.totalHits.value());
+        assertTrue(context.queryResult().topDocs().topDocs.totalHits.value() >= 5);
 
         reader.close();
         dir.close();

server/src/test/java/org/opensearch/search/query/QueryProfilePhaseTests.java

@@ -1233,18 +1233,18 @@ public void testMinScore() throws Exception {
         context.trackTotalHitsUpTo(5);
 
         QueryPhase.executeInternal(context.withCleanQueryResult().withProfilers(), queryPhaseSearcher);
-        assertEquals(10, context.queryResult().topDocs().topDocs.totalHits.value());
+        assertTrue(context.queryResult().topDocs().topDocs.totalHits.value() >= 5);
         assertProfileData(context, "BooleanQuery", query -> {
             assertThat(query.getTimeBreakdown().keySet(), not(empty()));
             assertThat(query.getTimeBreakdown().get("score"), greaterThan(0L));
-            assertThat(query.getTimeBreakdown().get("score_count"), equalTo(10L));
+            assertThat(query.getTimeBreakdown().get("score_count"), greaterThanOrEqualTo(5L));
             if (executor != null) {
                 assertThat(query.getTimeBreakdown().get("max_score"), greaterThan(0L));
                 assertThat(query.getTimeBreakdown().get("min_score"), greaterThan(0L));
                 assertThat(query.getTimeBreakdown().get("avg_score"), greaterThan(0L));
-                assertThat(query.getTimeBreakdown().get("max_score_count"), equalTo(10L));
-                assertThat(query.getTimeBreakdown().get("min_score_count"), equalTo(10L));
-                assertThat(query.getTimeBreakdown().get("avg_score_count"), equalTo(10L));
+                assertThat(query.getTimeBreakdown().get("max_score_count"), greaterThanOrEqualTo(5L));
+                assertThat(query.getTimeBreakdown().get("min_score_count"), greaterThanOrEqualTo(5L));
+                assertThat(query.getTimeBreakdown().get("avg_score_count"), greaterThanOrEqualTo(5L));
             }
             assertThat(query.getTimeBreakdown().get("create_weight"), greaterThan(0L));
             assertThat(query.getTimeBreakdown().get("create_weight_count"), equalTo(1L));

test/framework/licenses/lucene-codecs-10.2.1.jar.sha1

@@ -0,0 +1 @@
+7fad476daca759c657ead0e83e46a343fd09d38d

test/framework/licenses/lucene-test-framework-10.2.1.jar.sha1

@@ -0,0 +1 @@
+f01401e66a71eca1a70d19cf8269f47d191b1a37