Skip to content

Commit 8e07111

Browse files
github-actions[bot]sachinpkale
github-actions[bot]
authored and
sachinpkale
committed
[SnapshotV2] Add timestamp of last successful fetch of pinned timestamps in node stats (#15611)
--------- Signed-off-by: Lakshya Taragi <[email protected]> (cherry picked from commit be9f942) Signed-off-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
1 parent afd1cbc commit 8e07111

File tree

13 files changed

+187
-7
lines changed

13 files changed

+187
-7
lines changed

‎CHANGELOG.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
1414
- Add support for msearch API to pass search pipeline name - ([#15923](https://github.com/opensearch-project/OpenSearch/pull/15923))
1515
- Add success and failure metrics for async shard fetch ([#15976](https://github.com/opensearch-project/OpenSearch/pull/15976))
1616
- [S3 Repository] Change default retry mechanism of s3 clients to Standard Mode ([#15978](https://github.com/opensearch-project/OpenSearch/pull/15978))
17+
- Add new metric REMOTE_STORE to NodeStats API response ([#15611](https://github.com/opensearch-project/OpenSearch/pull/15611))
1718

1819
### Dependencies
1920
- Bump `org.apache.logging.log4j:log4j-core` from 2.23.1 to 2.24.0 ([#15858](https://github.com/opensearch-project/OpenSearch/pull/15858))

‎server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStorePinnedTimestampsIT.java‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,8 @@
99
package org.opensearch.remotestore;
1010

1111
import org.opensearch.action.LatchedActionListener;
12+
import org.opensearch.action.admin.cluster.node.stats.NodeStats;
13+
import org.opensearch.action.admin.cluster.node.stats.NodesStatsResponse;
1214
import org.opensearch.common.collect.Tuple;
1315
import org.opensearch.common.settings.Settings;
1416
import org.opensearch.common.unit.TimeValue;
@@ -20,6 +22,8 @@
2022
import java.util.Set;
2123
import java.util.concurrent.CountDownLatch;
2224

25+
import static org.opensearch.action.admin.cluster.node.stats.NodesStatsRequest.Metric.REMOTE_STORE;
26+
2327
@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0)
2428
public class RemoteStorePinnedTimestampsIT extends RemoteStoreBaseIntegTestCase {
2529
static final String INDEX_NAME = "remote-store-test-idx-1";
@@ -180,4 +184,41 @@ public void onFailure(Exception e) {
180184
assertBusy(() -> assertEquals(Set.of(timestamp2, timestamp3), RemoteStorePinnedTimestampService.getPinnedTimestamps().v2()));
181185
remoteStorePinnedTimestampService.rescheduleAsyncUpdatePinnedTimestampTask(TimeValue.timeValueMinutes(3));
182186
}
187+
188+
public void testLastSuccessfulFetchOfPinnedTimestampsPresentInNodeStats() throws Exception {
189+
logger.info("Starting up cluster manager");
190+
logger.info("cluster.remote_store.pinned_timestamps.enabled set to true");
191+
logger.info("cluster.remote_store.pinned_timestamps.scheduler_interval set to minimum value of 1minute");
192+
Settings pinnedTimestampEnabledSettings = Settings.builder()
193+
.put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_PINNED_TIMESTAMP_ENABLED.getKey(), true)
194+
.put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_PINNED_TIMESTAMP_SCHEDULER_INTERVAL.getKey(), "1m")
195+
.build();
196+
internalCluster().startClusterManagerOnlyNode(pinnedTimestampEnabledSettings);
197+
String remoteNodeName = internalCluster().startDataOnlyNodes(1, pinnedTimestampEnabledSettings).get(0);
198+
ensureStableCluster(2);
199+
RemoteStorePinnedTimestampService remoteStorePinnedTimestampService = internalCluster().getInstance(
200+
RemoteStorePinnedTimestampService.class,
201+
remoteNodeName
202+
);
203+
204+
remoteStorePinnedTimestampService.rescheduleAsyncUpdatePinnedTimestampTask(TimeValue.timeValueSeconds(1));
205+
206+
assertBusy(() -> {
207+
long lastSuccessfulFetchOfPinnedTimestamps = RemoteStorePinnedTimestampService.getPinnedTimestamps().v1();
208+
assertTrue(lastSuccessfulFetchOfPinnedTimestamps > 0L);
209+
NodesStatsResponse nodesStatsResponse = internalCluster().client()
210+
.admin()
211+
.cluster()
212+
.prepareNodesStats()
213+
.addMetric(REMOTE_STORE.metricName())
214+
.execute()
215+
.actionGet();
216+
for (NodeStats nodeStats : nodesStatsResponse.getNodes()) {
217+
long lastRecordedFetch = nodeStats.getRemoteStoreNodeStats().getLastSuccessfulFetchOfPinnedTimestamps();
218+
assertTrue(lastRecordedFetch >= lastSuccessfulFetchOfPinnedTimestamps);
219+
}
220+
});
221+
222+
remoteStorePinnedTimestampService.rescheduleAsyncUpdatePinnedTimestampTask(TimeValue.timeValueMinutes(3));
223+
}
183224
}

‎server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodeStats.java‎

Lines changed: 25 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,7 @@
6060
import org.opensearch.monitor.process.ProcessStats;
6161
import org.opensearch.node.AdaptiveSelectionStats;
6262
import org.opensearch.node.NodesResourceUsageStats;
63+
import org.opensearch.node.remotestore.RemoteStoreNodeStats;
6364
import org.opensearch.ratelimitting.admissioncontrol.stats.AdmissionControlStats;
6465
import org.opensearch.repositories.RepositoriesStats;
6566
import org.opensearch.script.ScriptCacheStats;
@@ -163,6 +164,9 @@ public class NodeStats extends BaseNodeResponse implements ToXContentFragment {
163164
@Nullable
164165
private NodeCacheStats nodeCacheStats;
165166

167+
@Nullable
168+
private RemoteStoreNodeStats remoteStoreNodeStats;
169+
166170
public NodeStats(StreamInput in) throws IOException {
167171
super(in);
168172
timestamp = in.readVLong();
@@ -258,6 +262,12 @@ public NodeStats(StreamInput in) throws IOException {
258262
} else {
259263
nodeCacheStats = null;
260264
}
265+
// TODO: change version to V_2_18_0
266+
if (in.getVersion().onOrAfter(Version.CURRENT)) {
267+
remoteStoreNodeStats = in.readOptionalWriteable(RemoteStoreNodeStats::new);
268+
} else {
269+
remoteStoreNodeStats = null;
270+
}
261271
}
262272

263273
public NodeStats(
@@ -289,7 +299,8 @@ public NodeStats(
289299
@Nullable SegmentReplicationRejectionStats segmentReplicationRejectionStats,
290300
@Nullable RepositoriesStats repositoriesStats,
291301
@Nullable AdmissionControlStats admissionControlStats,
292-
@Nullable NodeCacheStats nodeCacheStats
302+
@Nullable NodeCacheStats nodeCacheStats,
303+
@Nullable RemoteStoreNodeStats remoteStoreNodeStats
293304
) {
294305
super(node);
295306
this.timestamp = timestamp;
@@ -320,6 +331,7 @@ public NodeStats(
320331
this.repositoriesStats = repositoriesStats;
321332
this.admissionControlStats = admissionControlStats;
322333
this.nodeCacheStats = nodeCacheStats;
334+
this.remoteStoreNodeStats = remoteStoreNodeStats;
323335
}
324336

325337
public long getTimestamp() {
@@ -482,6 +494,11 @@ public NodeCacheStats getNodeCacheStats() {
482494
return nodeCacheStats;
483495
}
484496

497+
@Nullable
498+
public RemoteStoreNodeStats getRemoteStoreNodeStats() {
499+
return remoteStoreNodeStats;
500+
}
501+
485502
@Override
486503
public void writeTo(StreamOutput out) throws IOException {
487504
super.writeTo(out);
@@ -547,6 +564,10 @@ public void writeTo(StreamOutput out) throws IOException {
547564
if (out.getVersion().onOrAfter(Version.V_2_14_0)) {
548565
out.writeOptionalWriteable(nodeCacheStats);
549566
}
567+
// TODO: change version to V_2_18_0
568+
if (out.getVersion().onOrAfter(Version.CURRENT)) {
569+
out.writeOptionalWriteable(remoteStoreNodeStats);
570+
}
550571
}
551572

552573
@Override
@@ -652,6 +673,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
652673
if (getNodeCacheStats() != null) {
653674
getNodeCacheStats().toXContent(builder, params);
654675
}
676+
if (getRemoteStoreNodeStats() != null) {
677+
getRemoteStoreNodeStats().toXContent(builder, params);
678+
}
655679
return builder;
656680
}
657681
}

‎server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodesStatsRequest.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -251,7 +251,8 @@ public enum Metric {
251251
SEGMENT_REPLICATION_BACKPRESSURE("segment_replication_backpressure"),
252252
REPOSITORIES("repositories"),
253253
ADMISSION_CONTROL("admission_control"),
254-
CACHE_STATS("caches");
254+
CACHE_STATS("caches"),
255+
REMOTE_STORE("remote_store");
255256

256257
private String metricName;
257258

‎server/src/main/java/org/opensearch/action/admin/cluster/node/stats/TransportNodesStatsAction.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -129,7 +129,8 @@ protected NodeStats nodeOperation(NodeStatsRequest nodeStatsRequest) {
129129
NodesStatsRequest.Metric.SEGMENT_REPLICATION_BACKPRESSURE.containedIn(metrics),
130130
NodesStatsRequest.Metric.REPOSITORIES.containedIn(metrics),
131131
NodesStatsRequest.Metric.ADMISSION_CONTROL.containedIn(metrics),
132-
NodesStatsRequest.Metric.CACHE_STATS.containedIn(metrics)
132+
NodesStatsRequest.Metric.CACHE_STATS.containedIn(metrics),
133+
NodesStatsRequest.Metric.REMOTE_STORE.containedIn(metrics)
133134
);
134135
}
135136

‎server/src/main/java/org/opensearch/action/admin/cluster/stats/TransportClusterStatsAction.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -174,6 +174,7 @@ protected ClusterStatsNodeResponse nodeOperation(ClusterStatsNodeRequest nodeReq
174174
false,
175175
false,
176176
false,
177+
false,
177178
false
178179
);
179180
List<ShardStats> shardsStats = new ArrayList<>();

‎server/src/main/java/org/opensearch/node/NodeService.java‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,7 @@
5454
import org.opensearch.indices.IndicesService;
5555
import org.opensearch.ingest.IngestService;
5656
import org.opensearch.monitor.MonitorService;
57+
import org.opensearch.node.remotestore.RemoteStoreNodeStats;
5758
import org.opensearch.plugins.PluginsService;
5859
import org.opensearch.ratelimitting.admissioncontrol.AdmissionControlService;
5960
import org.opensearch.repositories.RepositoriesService;
@@ -241,7 +242,8 @@ public NodeStats stats(
241242
boolean segmentReplicationTrackerStats,
242243
boolean repositoriesStats,
243244
boolean admissionControl,
244-
boolean cacheService
245+
boolean cacheService,
246+
boolean remoteStoreNodeStats
245247
) {
246248
// for indices stats we want to include previous allocated shards stats as well (it will
247249
// only be applied to the sensible ones to use, like refresh/merge/flush/indexing stats)
@@ -274,7 +276,8 @@ public NodeStats stats(
274276
segmentReplicationTrackerStats ? this.segmentReplicationStatsTracker.getTotalRejectionStats() : null,
275277
repositoriesStats ? this.repositoriesService.getRepositoriesStats() : null,
276278
admissionControl ? this.admissionControlService.stats() : null,
277-
cacheService ? this.cacheService.stats(indices) : null
279+
cacheService ? this.cacheService.stats(indices) : null,
280+
remoteStoreNodeStats ? new RemoteStoreNodeStats() : null
278281
);
279282
}
280283

‎server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeStats.java‎

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
/*
2+
* SPDX-License-Identifier: Apache-2.0
3+
*
4+
* The OpenSearch Contributors require contributions made to
5+
* this file be licensed under the Apache-2.0 license or a
6+
* compatible open source license.
7+
*/
8+
9+
package org.opensearch.node.remotestore;
10+
11+
import org.opensearch.core.common.io.stream.StreamInput;
12+
import org.opensearch.core.common.io.stream.StreamOutput;
13+
import org.opensearch.core.common.io.stream.Writeable;
14+
import org.opensearch.core.xcontent.ToXContentFragment;
15+
import org.opensearch.core.xcontent.XContentBuilder;
16+
17+
import java.io.IOException;
18+
import java.util.Objects;
19+
20+
/**
21+
* Node level remote store stats
22+
* @opensearch.internal
23+
*/
24+
public class RemoteStoreNodeStats implements Writeable, ToXContentFragment {
25+
26+
public static final String STATS_NAME = "remote_store";
27+
public static final String LAST_SUCCESSFUL_FETCH_OF_PINNED_TIMESTAMPS = "last_successful_fetch_of_pinned_timestamps";
28+
29+
/**
30+
* Time stamp for the last successful fetch of pinned timestamps by the {@linkplain RemoteStorePinnedTimestampService}
31+
*/
32+
private final long lastSuccessfulFetchOfPinnedTimestamps;
33+
34+
public RemoteStoreNodeStats() {
35+
this.lastSuccessfulFetchOfPinnedTimestamps = RemoteStorePinnedTimestampService.getPinnedTimestamps().v1();
36+
}
37+
38+
public long getLastSuccessfulFetchOfPinnedTimestamps() {
39+
return this.lastSuccessfulFetchOfPinnedTimestamps;
40+
}
41+
42+
public RemoteStoreNodeStats(StreamInput in) throws IOException {
43+
this.lastSuccessfulFetchOfPinnedTimestamps = in.readLong();
44+
}
45+
46+
@Override
47+
public void writeTo(StreamOutput out) throws IOException {
48+
out.writeLong(this.lastSuccessfulFetchOfPinnedTimestamps);
49+
}
50+
51+
@Override
52+
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
53+
builder.startObject(STATS_NAME);
54+
builder.field(LAST_SUCCESSFUL_FETCH_OF_PINNED_TIMESTAMPS, this.lastSuccessfulFetchOfPinnedTimestamps);
55+
return builder.endObject();
56+
}
57+
58+
@Override
59+
public String toString() {
60+
return "RemoteStoreNodeStats{ lastSuccessfulFetchOfPinnedTimestamps=" + lastSuccessfulFetchOfPinnedTimestamps + "}";
61+
}
62+
63+
@Override
64+
public boolean equals(Object o) {
65+
if (o == null) {
66+
return false;
67+
}
68+
if (o.getClass() != RemoteStoreNodeStats.class) {
69+
return false;
70+
}
71+
RemoteStoreNodeStats other = (RemoteStoreNodeStats) o;
72+
return this.lastSuccessfulFetchOfPinnedTimestamps == other.lastSuccessfulFetchOfPinnedTimestamps;
73+
}
74+
75+
@Override
76+
public int hashCode() {
77+
return Objects.hash(lastSuccessfulFetchOfPinnedTimestamps);
78+
}
79+
}

‎server/src/test/java/org/opensearch/action/admin/cluster/node/stats/NodeStatsTests.java‎

Lines changed: 21 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -95,6 +95,7 @@
9595
import org.opensearch.node.NodeResourceUsageStats;
9696
import org.opensearch.node.NodesResourceUsageStats;
9797
import org.opensearch.node.ResponseCollectorService;
98+
import org.opensearch.node.remotestore.RemoteStoreNodeStats;
9899
import org.opensearch.ratelimitting.admissioncontrol.controllers.AdmissionController;
99100
import org.opensearch.ratelimitting.admissioncontrol.controllers.CpuBasedAdmissionController;
100101
import org.opensearch.ratelimitting.admissioncontrol.enums.AdmissionControlActionType;
@@ -614,6 +615,14 @@ public void testSerialization() throws IOException {
614615
} else {
615616
assertEquals(nodeCacheStats, deserializedNodeCacheStats);
616617
}
618+
619+
RemoteStoreNodeStats remoteStoreNodeStats = nodeStats.getRemoteStoreNodeStats();
620+
RemoteStoreNodeStats deserializedRemoteStoreNodeStats = deserializedNodeStats.getRemoteStoreNodeStats();
621+
if (remoteStoreNodeStats == null) {
622+
assertNull(deserializedRemoteStoreNodeStats);
623+
} else {
624+
assertEquals(remoteStoreNodeStats, deserializedRemoteStoreNodeStats);
625+
}
617626
}
618627
}
619628
}
@@ -995,6 +1004,16 @@ public void apply(String action, AdmissionControlActionType admissionControlActi
9951004
nodeCacheStats = new NodeCacheStats(cacheStatsMap, flags);
9961005
}
9971006

1007+
RemoteStoreNodeStats remoteStoreNodeStats = null;
1008+
if (frequently()) {
1009+
remoteStoreNodeStats = new RemoteStoreNodeStats() {
1010+
@Override
1011+
public long getLastSuccessfulFetchOfPinnedTimestamps() {
1012+
return 123456L;
1013+
}
1014+
};
1015+
}
1016+
9981017
// TODO: Only remote_store based aspects of NodeIndicesStats are being tested here.
9991018
// It is possible to test other metrics in NodeIndicesStats as well since it extends Writeable now
10001019
return new NodeStats(
@@ -1026,7 +1045,8 @@ public void apply(String action, AdmissionControlActionType admissionControlActi
10261045
segmentReplicationRejectionStats,
10271046
null,
10281047
admissionControlStats,
1029-
nodeCacheStats
1048+
nodeCacheStats,
1049+
remoteStoreNodeStats
10301050
);
10311051
}
10321052

‎server/src/test/java/org/opensearch/action/admin/cluster/stats/ClusterStatsNodesTests.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -349,6 +349,7 @@ private ClusterStatsNodeResponse createClusterStatsNodeResponse(
349349
null,
350350
null,
351351
null,
352+
null,
352353
null
353354
);
354355
if (defaultBehavior) {

0 commit comments

Comments
 (0)