Adding Cluster Endpoints for search failures and refactoring document status counters

CHANGELOG.md

@@ -26,6 +26,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Add async periodic flush task support for pull-based ingestion ([#19878](https://github.com/opensearch-project/OpenSearch/pull/19878))
 - Add support for context aware segments ([#19098](https://github.com/opensearch-project/OpenSearch/pull/19098))
 - Implement GRPC FunctionScoreQuery ([#19888](https://github.com/opensearch-project/OpenSearch/pull/19888))
+- Add search API tracker ([#18601](https://github.com/opensearch-project/OpenSearch/pull/18601))
 
 ### Changed
 - Faster `terms` query creation for `keyword` field with index and docValues enabled ([#19350](https://github.com/opensearch-project/OpenSearch/pull/19350))

libs/core/src/main/java/org/opensearch/core/rest/RestStatus.java

@@ -536,6 +536,17 @@ public int getStatusFamilyCode() {
         return status / 100;
     }
 
+    public String getStatusType() {
+        int family = getStatusFamilyCode();
+        if (family <= 3) {
+            return StatusType.SUCCESS.toString();
+        } else if (family == 4) {
+            return StatusType.USER_ERROR.toString();
+        } else {
+            return StatusType.SYSTEM_FAILURE.toString();
+        }
+    }
+
     public static RestStatus readFrom(StreamInput in) throws IOException {
         return RestStatus.valueOf(in.readString());
     }

libs/core/src/main/java/org/opensearch/core/rest/StatusType.java

@@ -0,0 +1,27 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.core.rest;
+
+import java.util.Locale;
+
+/**
+ * Enum for status type which keeps track of potential response families in {@link RestStatus}
+ *
+ * @opensearch.api
+ */
+public enum StatusType {
+    SUCCESS,
+    USER_ERROR,
+    SYSTEM_FAILURE;
+
+    @Override
+    public String toString() {
+        return name().toLowerCase(Locale.ROOT);
+    }
+}

rest-api-spec/src/main/resources/rest-api-spec/test/nodes.stats/11_indices_metrics.yml

@@ -1,7 +1,9 @@
 ---
 "Metric - blank":
   - skip:
+      version: " - 3.2.99"
       features: [arbitrary_key]
+      reason: "Status Counter was introduced in v3.3.0"
   - do:
       nodes.info: {}
   - set:
@@ -25,11 +27,14 @@
   - is_true:  nodes.$node_id.indices.segments
   - is_true:  nodes.$node_id.indices.translog
   - is_true:  nodes.$node_id.indices.recovery
+  - is_true:  nodes.$node_id.indices.status_counter
 
 ---
 "Metric - _all":
   - skip:
       features: [arbitrary_key]
+      version: " - 3.2.99"
+      reason: "Status Counter was introduced in v3.3.0"
   - do:
       nodes.info: {}
   - set:
@@ -53,11 +58,14 @@
   - is_true:  nodes.$node_id.indices.segments
   - is_true:  nodes.$node_id.indices.translog
   - is_true:  nodes.$node_id.indices.recovery
+  - is_true:  nodes.$node_id.indices.status_counter
 
 ---
 "Metric - indices _all":
   - skip:
       features: [arbitrary_key]
+      version: " - 3.2.99"
+      reason: "Status Counter was introduced in v3.3.0"
   - do:
       nodes.info: {}
   - set:
@@ -81,11 +89,14 @@
   - is_true:  nodes.$node_id.indices.segments
   - is_true:  nodes.$node_id.indices.translog
   - is_true:  nodes.$node_id.indices.recovery
+  - is_true:  nodes.$node_id.indices.status_counter
 
 ---
 "Metric - one":
   - skip:
       features: [arbitrary_key]
+      version: " - 3.2.99"
+      reason: "Status Counter was introduced in v3.3.0"
   - do:
       nodes.info: {}
   - set:
@@ -109,11 +120,14 @@
   - is_false:  nodes.$node_id.indices.segments
   - is_false:  nodes.$node_id.indices.translog
   - is_false:  nodes.$node_id.indices.recovery
+  - is_true:   nodes.$node_id.indices.status_counter
 
 ---
 "Metric - multi":
   - skip:
       features: [arbitrary_key]
+      version: " - 3.2.99"
+      reason: "Status Counter was introduced in v3.3.0"
   - do:
       nodes.info: {}
   - set:
@@ -137,12 +151,13 @@
   - is_false:  nodes.$node_id.indices.segments
   - is_false:  nodes.$node_id.indices.translog
   - is_false:  nodes.$node_id.indices.recovery
+  - is_true:   nodes.$node_id.indices.status_counter
 
 ---
 "Metric - indexing doc_status":
   - skip:
-      version: " - 2.10.99"
-      reason: "Doc Status Stats were introduced in v2.11.0"
+      version: " - 3.2.99"
+      reason: "Doc Status Stats were introduced in v2.11.0 and moved to Status Counter in v3.3.0"
   - do:
       nodes.info: {}
   - set:
@@ -154,7 +169,6 @@
   - is_false: nodes.$node_id.indices.docs
   - is_false: nodes.$node_id.indices.store
   - is_true: nodes.$node_id.indices.indexing
-  - is_true: nodes.$node_id.indices.indexing.doc_status
   - is_false: nodes.$node_id.indices.get
   - is_false: nodes.$node_id.indices.search
   - is_false: nodes.$node_id.indices.merges
@@ -167,6 +181,39 @@
   - is_false: nodes.$node_id.indices.segments
   - is_false: nodes.$node_id.indices.translog
   - is_false: nodes.$node_id.indices.recovery
+  - is_true:  nodes.$node_id.indices.status_counter
+  - is_true:  nodes.$node_id.indices.status_counter.doc_status
+
+---
+"Metric - indexing search_response_status":
+  - skip:
+      version: " - 3.2.99"
+      reason: "Search Response Status Stats were introduced in v3.3.0"
+  - do:
+      nodes.info: {}
+  - set:
+      nodes._arbitrary_key_: node_id
+
+  - do:
+      nodes.stats: { metric: indices, index_metric: indexing }
+
+  - is_false: nodes.$node_id.indices.docs
+  - is_false: nodes.$node_id.indices.store
+  - is_true:  nodes.$node_id.indices.indexing
+  - is_false: nodes.$node_id.indices.get
+  - is_false: nodes.$node_id.indices.search
+  - is_false: nodes.$node_id.indices.merges
+  - is_false: nodes.$node_id.indices.refresh
+  - is_false: nodes.$node_id.indices.flush
+  - is_false: nodes.$node_id.indices.warmer
+  - is_false: nodes.$node_id.indices.query_cache
+  - is_false: nodes.$node_id.indices.fielddata
+  - is_false: nodes.$node_id.indices.completion
+  - is_false: nodes.$node_id.indices.segments
+  - is_false: nodes.$node_id.indices.translog
+  - is_false: nodes.$node_id.indices.recovery
+  - is_true:  nodes.$node_id.indices.status_counter
+  - is_true:  nodes.$node_id.indices.status_counter.search_response_status
 
 ---
 "Metric - recovery":

server/src/internalClusterTest/java/org/opensearch/nodestats/NodeStatsIT.java

@@ -13,6 +13,7 @@
 import org.opensearch.action.admin.cluster.node.stats.NodeStats;
 import org.opensearch.action.admin.cluster.node.stats.NodesStatsResponse;
 import org.opensearch.action.admin.indices.stats.CommonStatsFlags;
+import org.opensearch.action.admin.indices.stats.DocStatusStats;
 import org.opensearch.action.bulk.BulkItemResponse;
 import org.opensearch.action.bulk.BulkRequest;
 import org.opensearch.action.bulk.BulkResponse;
@@ -32,7 +33,6 @@
 import org.opensearch.index.IndexNotFoundException;
 import org.opensearch.index.engine.DocumentMissingException;
 import org.opensearch.index.engine.VersionConflictEngineException;
-import org.opensearch.index.shard.IndexingStats.Stats.DocStatusStats;
 import org.opensearch.indices.NodeIndicesStats;
 import org.opensearch.test.OpenSearchIntegTestCase;
 import org.opensearch.test.OpenSearchIntegTestCase.ClusterScope;
@@ -47,7 +47,7 @@
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.LongAdder;
 
 import static java.util.Collections.singletonMap;
 import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked;
@@ -281,15 +281,14 @@ public void testNodeIndicesStatsDocStatsWithAggregations() {
                 .getNodes()
                 .get(0)
                 .getIndices()
-                .getIndexing()
-                .getTotal()
+                .getStatusCounterStats()
                 .getDocStatusStats();
 
             assertTrue(
                 Arrays.equals(
                     docStatusStats.getDocStatusCounter(),
                     expectedDocStatusStats.getDocStatusCounter(),
-                    Comparator.comparingLong(AtomicLong::longValue)
+                    Comparator.comparingLong(LongAdder::longValue)
                 )
             );
         }
@@ -543,15 +542,14 @@ private void assertDocStatusStats() {
             .getNodes()
             .get(0)
             .getIndices()
-            .getIndexing()
-            .getTotal()
+            .getStatusCounterStats()
             .getDocStatusStats();
 
         assertTrue(
             Arrays.equals(
                 docStatusStats.getDocStatusCounter(),
                 expectedDocStatusStats.getDocStatusCounter(),
-                Comparator.comparingLong(AtomicLong::longValue)
+                Comparator.comparingLong(LongAdder::longValue)
             )
         );
     }

server/src/internalClusterTest/java/org/opensearch/search/msearch/MultiSearchStatsIT.java

@@ -0,0 +1,87 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.search.msearch;
+
+import org.opensearch.ExceptionsHelper;
+import org.opensearch.action.admin.indices.stats.SearchResponseStatusStats;
+import org.opensearch.action.search.MultiSearchRequestBuilder;
+import org.opensearch.action.search.MultiSearchResponse;
+import org.opensearch.action.search.SearchResponse;
+import org.opensearch.index.query.QueryBuilders;
+import org.opensearch.test.OpenSearchIntegTestCase;
+import org.opensearch.test.OpenSearchIntegTestCase.ClusterScope;
+import org.opensearch.test.OpenSearchIntegTestCase.Scope;
+
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.concurrent.atomic.LongAdder;
+
+@ClusterScope(scope = Scope.TEST, numDataNodes = 1, numClientNodes = 0, supportsDedicatedMasters = false)
+public class MultiSearchStatsIT extends OpenSearchIntegTestCase {
+    private final SearchResponseStatusStats expectedSearchResponseStatusStats = new SearchResponseStatusStats();
+
+    public void testNodeIndicesStatsDocStatusStatsIndexBulk() {
+        createIndex("test");
+        ensureGreen();
+        client().prepareIndex("test").setId("1").setSource("field", "xxx").get();
+        client().prepareIndex("test").setId("2").setSource("field", "yyy").get();
+        refresh();
+
+        int failureCount = randomIntBetween(0, 50);
+        int successCount = randomIntBetween(0, 50);
+
+        MultiSearchRequestBuilder multiSearchRequestBuilder = client().prepareMultiSearch();
+
+        for (int i = 0; i < failureCount; i++) {
+            multiSearchRequestBuilder.add(client().prepareSearch("noIndex").setQuery(QueryBuilders.termQuery("field", "yyy")));
+        }
+
+        for (int i = 0; i < successCount; i++) {
+            multiSearchRequestBuilder.add(client().prepareSearch("test").setQuery(QueryBuilders.matchAllQuery()));
+        }
+
+        MultiSearchResponse response = multiSearchRequestBuilder.get();
+
+        for (MultiSearchResponse.Item item : response) {
+            if (item.isFailure()) {
+                updateExpectedDocStatusCounter(expectedSearchResponseStatusStats, item.getFailure());
+            } else {
+                updateExpectedDocStatusCounter(expectedSearchResponseStatusStats, item.getResponse());
+            }
+        }
+        assertSearchResponseStatusStats();
+    }
+
+    private void assertSearchResponseStatusStats() {
+        SearchResponseStatusStats searchResponseStatusStats = client().admin()
+            .cluster()
+            .prepareNodesStats()
+            .execute()
+            .actionGet()
+            .getNodes()
+            .get(0)
+            .getIndices()
+            .getStatusCounterStats()
+            .getSearchResponseStatusStats();
+
+        Arrays.equals(
+            searchResponseStatusStats.getSearchResponseStatusCounter(),
+            expectedSearchResponseStatusStats.getSearchResponseStatusCounter(),
+            Comparator.comparingLong(LongAdder::longValue)
+        );
+    }
+
+    private void updateExpectedDocStatusCounter(SearchResponseStatusStats expectedSearchResponseStatusStats, SearchResponse r) {
+        expectedSearchResponseStatusStats.inc(r.status());
+    }
+
+    private void updateExpectedDocStatusCounter(SearchResponseStatusStats expectedSearchResponseStatusStats, Exception e) {
+        expectedSearchResponseStatusStats.inc(ExceptionsHelper.status(e));
+    }
+}