Fix flaky TSC stats tests

Signed-off-by: Peter Alfonsi <[email protected]>

Author: Peter Alfonsi

modules/cache-common/src/internalClusterTest/java/org/opensearch/cache/common/tier/TieredSpilloverCacheStatsIT.java

@@ -20,6 +20,7 @@
 import org.opensearch.common.cache.stats.ImmutableCacheStatsHolder;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.unit.TimeValue;
+import org.opensearch.index.IndexSettings;
 import org.opensearch.index.cache.request.RequestCacheStats;
 import org.opensearch.index.query.QueryBuilders;
 import org.opensearch.indices.IndicesRequestCache;
@@ -102,8 +103,8 @@ public void testIndicesLevelAggregation() throws Exception {
         );
 
         for (ImmutableCacheStatsHolder statsHolder : List.of(allLevelsStatsHolder, indicesOnlyStatsHolder)) {
-            assertEquals(index1ExpectedStats, statsHolder.getStatsForDimensionValues(List.of(index1Name)));
-            assertEquals(index2ExpectedStats, statsHolder.getStatsForDimensionValues(List.of(index2Name)));
+            assertStatsEqual(index1ExpectedStats, statsHolder.getStatsForDimensionValues(List.of(index1Name)));
+            assertStatsEqual(index2ExpectedStats, statsHolder.getStatsForDimensionValues(List.of(index2Name)));
         }
     }
 
@@ -139,7 +140,7 @@ public void testIndicesAndTierLevelAggregation() throws Exception {
                 values.get("itemsOnHeapIndex1AfterTest")
             )
         );
-        assertEquals(
+        assertStatsEqual(
             index1HeapExpectedStats,
             allLevelsStatsHolder.getStatsForDimensionValues(List.of(index1Name, TIER_DIMENSION_VALUE_ON_HEAP))
         );
@@ -153,7 +154,7 @@ public void testIndicesAndTierLevelAggregation() throws Exception {
                 values.get("itemsOnHeapIndex2AfterTest")
             )
         );
-        assertEquals(
+        assertStatsEqual(
             index2HeapExpectedStats,
             allLevelsStatsHolder.getStatsForDimensionValues(List.of(index2Name, TIER_DIMENSION_VALUE_ON_HEAP))
         );
@@ -167,7 +168,7 @@ public void testIndicesAndTierLevelAggregation() throws Exception {
                 values.get("itemsOnDiskIndex1AfterTest")
             )
         );
-        assertEquals(
+        assertStatsEqual(
             index1DiskExpectedStats,
             allLevelsStatsHolder.getStatsForDimensionValues(List.of(index1Name, TIER_DIMENSION_VALUE_DISK))
         );
@@ -181,7 +182,7 @@ public void testIndicesAndTierLevelAggregation() throws Exception {
                 values.get("itemsOnDiskIndex2AfterTest")
             )
         );
-        assertEquals(
+        assertStatsEqual(
             index2DiskExpectedStats,
             allLevelsStatsHolder.getStatsForDimensionValues(List.of(index2Name, TIER_DIMENSION_VALUE_DISK))
         );
@@ -218,7 +219,7 @@ public void testTierLevelAggregation() throws Exception {
             )
         );
         ImmutableCacheStats heapStats = tiersOnlyStatsHolder.getStatsForDimensionValues(List.of(TIER_DIMENSION_VALUE_ON_HEAP));
-        assertEquals(totalHeapExpectedStats, heapStats);
+        assertStatsEqual(totalHeapExpectedStats, heapStats);
         ImmutableCacheStats totalDiskExpectedStats = returnNullIfAllZero(
             new ImmutableCacheStats(
                 values.get("hitsOnDiskIndex1") + values.get("hitsOnDiskIndex2"),
@@ -229,7 +230,7 @@ public void testTierLevelAggregation() throws Exception {
             )
         );
         ImmutableCacheStats diskStats = tiersOnlyStatsHolder.getStatsForDimensionValues(List.of(TIER_DIMENSION_VALUE_DISK));
-        assertEquals(totalDiskExpectedStats, diskStats);
+        assertStatsEqual(totalDiskExpectedStats, diskStats);
     }
 
     public void testInvalidLevelsAreIgnored() throws Exception {
@@ -322,7 +323,7 @@ public void testStatsMatchOldApi() throws Exception {
         ImmutableCacheStats totalStats = getNodeCacheStatsResult(client, List.of()).getTotalStats();
 
         // Check the new stats API values are as expected
-        assertEquals(
+        assertStatsEqual(
             new ImmutableCacheStats(expectedHits, expectedMisses, 0, expectedEntries * singleSearchSize, expectedEntries),
             totalStats
         );
@@ -351,6 +352,8 @@ private void startIndex(Client client, String indexName) throws InterruptedExcep
                         .put(IndicesRequestCache.INDEX_CACHE_REQUEST_ENABLED_SETTING.getKey(), true)
                         .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
                         .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
+                        // Disable index refreshing to avoid cache being invalidated mid-test
+                        .put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), TimeValue.timeValueMillis(-1))
                         .build()
                 )
                 .get()
@@ -498,4 +501,13 @@ private static ImmutableCacheStatsHolder getNodeCacheStatsResult(Client client,
         NodeCacheStats ncs = nodeStatsResponse.getNodes().get(0).getNodeCacheStats();
         return ncs.getStatsByCache(CacheType.INDICES_REQUEST_CACHE);
     }
+
+    // Check each stat separately for more transparency if there's a failure
+    private void assertStatsEqual(ImmutableCacheStats expected, ImmutableCacheStats actual) {
+        assertEquals(expected.getHits(), actual.getHits());
+        assertEquals(expected.getMisses(), actual.getMisses());
+        assertEquals(expected.getEvictions(), actual.getEvictions());
+        assertEquals(expected.getSizeInBytes(), actual.getSizeInBytes());
+        assertEquals(expected.getItems(), actual.getItems());
+    }
 }

server/src/internalClusterTest/java/org/opensearch/indices/CacheStatsAPIIndicesRequestCacheIT.java

@@ -23,12 +23,14 @@
 import org.opensearch.common.cache.stats.ImmutableCacheStats;
 import org.opensearch.common.cache.stats.ImmutableCacheStatsHolder;
 import org.opensearch.common.settings.Settings;
+import org.opensearch.common.unit.TimeValue;
 import org.opensearch.common.util.FeatureFlags;
 import org.opensearch.common.xcontent.XContentFactory;
 import org.opensearch.common.xcontent.XContentHelper;
 import org.opensearch.core.xcontent.MediaTypeRegistry;
 import org.opensearch.core.xcontent.ToXContent;
 import org.opensearch.core.xcontent.XContentBuilder;
+import org.opensearch.index.IndexSettings;
 import org.opensearch.index.cache.request.RequestCacheStats;
 import org.opensearch.index.query.QueryBuilders;
 import org.opensearch.test.OpenSearchIntegTestCase;
@@ -266,6 +268,8 @@ private void startIndex(Client client, String indexName) throws InterruptedExcep
                         .put(IndicesRequestCache.INDEX_CACHE_REQUEST_ENABLED_SETTING.getKey(), true)
                         .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
                         .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
+                        // Disable index refreshing to avoid cache being invalidated mid-test
+                        .put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), TimeValue.timeValueMillis(-1))
                 )
                 .get()
         );