Merge branch 'main' into extend-classpath-plugins

Author: cwperks

CHANGELOG.md

@@ -15,6 +15,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - [Security Manager Replacement] Enhance Java Agent to intercept newByteChannel ([#17989](https://github.com/opensearch-project/OpenSearch/pull/17989))
 - Enabled Async Shard Batch Fetch by default ([#18139](https://github.com/opensearch-project/OpenSearch/pull/18139))
 - Allow to get the search request from the QueryCoordinatorContext ([#17818](https://github.com/opensearch-project/OpenSearch/pull/17818))
+- Improve sort-query performance by retaining the default `totalHitsThreshold` for approximated `match_all` queries ([#18189](https://github.com/opensearch-project/OpenSearch/pull/18189))
 - Enable testing for ExtensiblePlugins using classpath plugins ([#16908](https://github.com/opensearch-project/OpenSearch/pull/16908))
 
 ### Changed
@@ -26,6 +27,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Update Apache HttpClient5 and HttpCore5 (CVE-2025-27820) ([#18152](https://github.com/opensearch-project/OpenSearch/pull/18152))
 - Bump `org.apache.commons:commons-collections4` from 4.4 to 4.5.0 ([#18101](https://github.com/opensearch-project/OpenSearch/pull/18101))
 - Bump `netty` from 4.1.118.Final to 4.1.121.Final ([#18192](https://github.com/opensearch-project/OpenSearch/pull/18192))
+- Bump `org.apache.commons:commons-configuration2` from 2.11.0 to 2.12.0 ([#18103](https://github.com/opensearch-project/OpenSearch/pull/18103))
 
 ### Deprecated
 
@@ -36,8 +38,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Remove package org.opensearch.transport.grpc and replace with org.opensearch.plugin.transport.grpc ([#18031](https://github.com/opensearch-project/OpenSearch/pull/18031))
 - Fix the native plugin installation error cause by the pgp public key change ([#18147](https://github.com/opensearch-project/OpenSearch/pull/18147))
 - Fix object field exists query ([#17843](https://github.com/opensearch-project/OpenSearch/pull/17843))
-- Use Bad Request status for InputCoercionEcception ([#18161](https://github.com/opensearch-project/OpenSearch/pull/18161))
+- Use Bad Request status for InputCoercionException ([#18161](https://github.com/opensearch-project/OpenSearch/pull/18161))
 - Null check field names in QueryStringQueryBuilder ([#18194](https://github.com/opensearch-project/OpenSearch/pull/18194))
+- Avoid NPE if on SnapshotInfo if 'shallow' boolean not present ([#18187](https://github.com/opensearch-project/OpenSearch/issues/18187))
 
 ### Security
 

plugins/repository-hdfs/build.gradle

@@ -74,7 +74,7 @@ dependencies {
   api "commons-codec:commons-codec:${versions.commonscodec}"
   api 'commons-collections:commons-collections:3.2.2'
   api "org.apache.commons:commons-compress:${versions.commonscompress}"
-  api 'org.apache.commons:commons-configuration2:2.11.0'
+  api 'org.apache.commons:commons-configuration2:2.12.0'
   api "commons-io:commons-io:${versions.commonsio}"
   api 'org.apache.commons:commons-lang3:3.17.0'
   implementation 'com.google.re2j:re2j:1.8'

plugins/repository-hdfs/licenses/commons-configuration2-2.11.0.jar.sha1

@@ -0,0 +1 @@
+0cab81350c85ca35db8c47621a19dcddf128883f

plugins/repository-hdfs/licenses/commons-configuration2-2.12.0.jar.sha1

@@ -1145,7 +1145,7 @@ public void testSortMissingNumbersMinMax() throws Exception {
             .get();
         assertNoFailures(searchResponse);
 
-        assertThat(searchResponse.getHits().getTotalHits().value(), equalTo(2L));
+        assertThat(searchResponse.getHits().getTotalHits().value(), equalTo(3L));
         assertThat(searchResponse.getHits().getAt(0).getId(), equalTo("1"));
         // The order here could be unstable (depends on document order) since missing == field value
         assertThat(searchResponse.getHits().getAt(1).getId(), is(oneOf("3", "2")));

server/src/internalClusterTest/java/org/opensearch/search/sort/FieldSortIT.java

@@ -4568,7 +4568,7 @@ public IndexShardSnapshot loadShardSnapshot(BlobContainer shardContainer, Snapsh
             SnapshotId snapshotId = snapshotInfo.snapshotId();
             if (snapshotInfo.getPinnedTimestamp() != 0) {
                 return () -> IndexShardSnapshotStatus.newDone(0L, 0L, 0, 0, 0, 0, "1");
-            } else if (snapshotInfo.isRemoteStoreIndexShallowCopyEnabled()) {
+            } else if (Boolean.TRUE.equals(snapshotInfo.isRemoteStoreIndexShallowCopyEnabled())) {
                 if (shardContainer.blobExists(REMOTE_STORE_SHARD_SHALLOW_COPY_SNAPSHOT_FORMAT.blobName(snapshotId.getUUID()))) {
                     return REMOTE_STORE_SHARD_SHALLOW_COPY_SNAPSHOT_FORMAT.read(
                         shardContainer,

server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java

@@ -449,7 +449,7 @@ public boolean canApproximate(SearchContext context) {
         if (context.from() + context.size() == 0) {
             this.setSize(SearchContext.DEFAULT_TRACK_TOTAL_HITS_UP_TO);
         } else {
-            this.setSize(Math.max(context.from() + context.size(), context.trackTotalHitsUpTo() + 1));
+            this.setSize(Math.max(context.from() + context.size(), context.trackTotalHitsUpTo()));
         }
         if (context.request() != null && context.request().source() != null) {
             FieldSortBuilder primarySortField = FieldSortBuilder.getPrimaryFieldSortOrNull(context.request().source());

server/src/main/java/org/opensearch/search/approximate/ApproximatePointRangeQuery.java

@@ -12,6 +12,7 @@
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.QueryVisitor;
+import org.apache.lucene.search.ScoreMode;
 import org.apache.lucene.search.Weight;
 import org.opensearch.search.internal.SearchContext;
 
@@ -47,7 +48,11 @@ public Query rewrite(IndexSearcher indexSearcher) throws IOException {
             // Default to the original query. This suggests that we were not called from ContextIndexSearcher.
             return originalQuery.rewrite(indexSearcher);
         }
-        return resolvedQuery.rewrite(indexSearcher);
+        Query rewritten = resolvedQuery.rewrite(indexSearcher);
+        if (rewritten != resolvedQuery) {
+            resolvedQuery = rewritten;
+        }
+        return this;
     }
 
     public void setContext(SearchContext context) {
@@ -78,6 +83,15 @@ public boolean equals(Object o) {
         return true;
     }
 
+    @Override
+    public Weight createWeight(IndexSearcher indexSearcher, ScoreMode scoreMode, float boost) throws IOException {
+        if (resolvedQuery == null) {
+            // Default to the original query.
+            return originalQuery.createWeight(indexSearcher, scoreMode, boost);
+        }
+        return resolvedQuery.createWeight(indexSearcher, scoreMode, boost);
+    }
+
     @Override
     public int hashCode() {
         int h = classHash();

server/src/main/java/org/opensearch/search/approximate/ApproximateScoreQuery.java

@@ -74,6 +74,7 @@
 import org.opensearch.common.util.CachedSupplier;
 import org.opensearch.index.search.OpenSearchToParentBlockJoinQuery;
 import org.opensearch.search.DocValueFormat;
+import org.opensearch.search.approximate.ApproximateScoreQuery;
 import org.opensearch.search.collapse.CollapseContext;
 import org.opensearch.search.internal.ScrollContext;
 import org.opensearch.search.internal.SearchContext;
@@ -724,6 +725,8 @@ static int shortcutTotalHitCount(IndexReader reader, Query query) throws IOExcep
                 query = ((ConstantScoreQuery) query).getQuery();
             } else if (query instanceof BoostQuery) {
                 query = ((BoostQuery) query).getQuery();
+            } else if (query instanceof ApproximateScoreQuery) {
+                query = ((ApproximateScoreQuery) query).getOriginalQuery();
             } else {
                 break;
             }

server/src/main/java/org/opensearch/search/query/TopDocsCollectorContext.java

@@ -287,13 +287,20 @@ public void testRangeQuery() throws IOException {
         String date2 = "2016-04-28T11:33:52";
         long instant1 = DateFormatters.from(DateFieldMapper.getDefaultDateTimeFormatter().parse(date1)).toInstant().toEpochMilli();
         long instant2 = DateFormatters.from(DateFieldMapper.getDefaultDateTimeFormatter().parse(date2)).toInstant().toEpochMilli() + 999;
-        Query expected = new ApproximatePointRangeQuery(
+        ApproximatePointRangeQuery approximatePointRangeQuery = new ApproximatePointRangeQuery(
             "field",
             pack(new long[] { instant1 }).bytes,
             pack(new long[] { instant2 }).bytes,
             new long[] { instant1 }.length,
             ApproximatePointRangeQuery.LONG_FORMAT
         );
+        Query expected = new ApproximateScoreQuery(
+            new IndexOrDocValuesQuery(
+                LongPoint.newRangeQuery("field", instant1, instant2),
+                SortedNumericDocValuesField.newSlowRangeQuery("field", instant1, instant2)
+            ),
+            approximatePointRangeQuery
+        );
         Query rangeQuery = ft.rangeQuery(date1, date2, true, true, null, null, null, context);
         assertTrue(rangeQuery instanceof ApproximateScoreQuery);
         ((ApproximateScoreQuery) rangeQuery).setContext(new TestSearchContext(context));