Skip to content

Add Snapshot and Remote Restore ITs for Warm Index #18281

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 17 commits into from
May 27, 2025
Merged

Add Snapshot and Remote Restore ITs for Warm Index #18281

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
17 commits
Select commit Hold shift + click to select a range
ae4d9d5
Snapshot IT's with Warm Index setup
May 13, 2025
2f3c89d
Clone and Concurrent Snapshot V2 IT's with Warm Index setup
May 13, 2025
f8e8a37
Small fix with Spotless check
May 14, 2025
b9b5359
Modified DeleteSnapshotV2IT for Warm Index
May 15, 2025
bb85535
Removed FS Leak Suppresion
May 22, 2025
0ca93b7
spotless fix
May 22, 2025
fbd05f8
IT test fix
May 22, 2025
a7f10ea
Add Changelog
May 22, 2025
82b7012
Fix gradle check issue
May 22, 2025
d759388
Removed Changelog
Gagan6164 May 23, 2025
cf2c194
Add support for warm index in RemoteRestoreSnapshotIT test
May 23, 2025
ee8a9ba
Resolved comment
May 26, 2025
1a8b6d5
small comment change
May 26, 2025
4124fbd
Resolve comments
May 26, 2025
603e5d9
Small fix
May 26, 2025
5ad8cea
Changing the fileCache.prune() to fileCache.clear()
May 27, 2025
abb55c1
test case change
May 27, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
86 changes: 58 additions & 28 deletions server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteRestoreSnapshotIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,9 @@

package org.opensearch.remotestore;

import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;

import org.opensearch.action.DocWriteResponse;
import org.opensearch.action.LatchedActionListener;
import org.opensearch.action.admin.cluster.remotestore.restore.RestoreRemoteStoreRequest;
Expand All @@ -30,17 +33,20 @@
import org.opensearch.common.util.io.IOUtils;
import org.opensearch.core.action.ActionListener;
import org.opensearch.core.common.unit.ByteSizeUnit;
import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.core.index.Index;
import org.opensearch.core.rest.RestStatus;
import org.opensearch.index.IndexService;
import org.opensearch.index.IndexSettings;
import org.opensearch.index.remote.RemoteStoreEnums.PathHashAlgorithm;
import org.opensearch.index.remote.RemoteStoreEnums.PathType;
import org.opensearch.index.shard.IndexShard;
import org.opensearch.index.store.remote.file.CleanerDaemonThreadLeakFilter;
import org.opensearch.indices.IndicesService;
import org.opensearch.indices.RemoteStoreSettings;
import org.opensearch.indices.recovery.RecoveryState;
import org.opensearch.indices.replication.common.ReplicationType;
import org.opensearch.node.Node;
import org.opensearch.node.remotestore.RemoteStorePinnedTimestampService;
import org.opensearch.repositories.RepositoriesService;
import org.opensearch.repositories.Repository;
Expand All @@ -60,6 +66,7 @@
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.Map;
Expand All @@ -77,6 +84,7 @@
import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY;
import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_STORE_ENABLED;
import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY;
import static org.opensearch.common.util.FeatureFlags.WRITABLE_WARM_INDEX_SETTING;
import static org.opensearch.index.remote.RemoteStoreEnums.DataCategory.SEGMENTS;
import static org.opensearch.index.remote.RemoteStoreEnums.DataCategory.TRANSLOG;
import static org.opensearch.index.remote.RemoteStoreEnums.DataType.DATA;
Expand All @@ -89,9 +97,31 @@
import static org.hamcrest.Matchers.greaterThanOrEqualTo;
import static org.hamcrest.Matchers.lessThanOrEqualTo;

@ThreadLeakFilters(filters = CleanerDaemonThreadLeakFilter.class)
@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0)
public class RemoteRestoreSnapshotIT extends RemoteSnapshotIT {

public RemoteRestoreSnapshotIT(Settings nodeSettings) {
super(nodeSettings);
}

@ParametersFactory
public static Collection<Object[]> parameters() {
return Arrays.asList(
new Object[] { Settings.builder().put(WRITABLE_WARM_INDEX_SETTING.getKey(), false).build() },
new Object[] { Settings.builder().put(WRITABLE_WARM_INDEX_SETTING.getKey(), true).build() }
);
}

@Override
protected Settings nodeSettings(int nodeOrdinal) {
ByteSizeValue cacheSize = new ByteSizeValue(16, ByteSizeUnit.GB);
return Settings.builder()
.put(super.nodeSettings(nodeOrdinal))
.put(Node.NODE_SEARCH_CACHE_SIZE_SETTING.getKey(), cacheSize.toString())
.build();
}

private void assertDocsPresentInIndex(Client client, String indexName, int numOfDocs) {
for (int i = 0; i < numOfDocs; i++) {
String id = Integer.toString(i);
Expand All @@ -102,7 +132,7 @@ private void assertDocsPresentInIndex(Client client, String indexName, int numOf

public void testRestoreOperationsShallowCopyEnabled() throws Exception {
String clusterManagerNode = internalCluster().startClusterManagerOnlyNode();
String primary = internalCluster().startDataOnlyNode();
String primary = internalCluster().startDataAndWarmNodes(1).get(0);
String indexName1 = "testindex1";
String indexName2 = "testindex2";
String snapshotRepoName = "test-restore-snapshot-repo";
Expand All @@ -128,7 +158,7 @@ public void testRestoreOperationsShallowCopyEnabled() throws Exception {
indexDocuments(client, indexName2, numDocsInIndex2);
ensureGreen(indexName1, indexName2);

internalCluster().startDataOnlyNode();
internalCluster().startDataAndWarmNodes(1);
logger.info("--> snapshot");

SnapshotInfo snapshotInfo = createSnapshot(snapshotRepoName, snapshotName1, new ArrayList<>(Arrays.asList(indexName1, indexName2)));
Expand Down Expand Up @@ -200,7 +230,7 @@ public void testRestoreOperationsShallowCopyEnabled() throws Exception {
*/
public void testRemoteStoreCustomDataOnIndexCreationAndRestore() {
String clusterManagerNode = internalCluster().startClusterManagerOnlyNode();
internalCluster().startDataOnlyNode();
internalCluster().startDataAndWarmNodes(1);
String indexName1 = "testindex1";
String indexName2 = "testindex2";
String snapshotRepoName = "test-restore-snapshot-repo";
Expand Down Expand Up @@ -313,7 +343,7 @@ private void validatePathType(String index, PathType pathType, @Nullable PathHas

public void testRestoreInSameRemoteStoreEnabledIndex() throws IOException {
String clusterManagerNode = internalCluster().startClusterManagerOnlyNode();
String primary = internalCluster().startDataOnlyNode();
String primary = internalCluster().startDataAndWarmNodes(1).get(0);
String indexName1 = "testindex1";
String indexName2 = "testindex2";
String snapshotRepoName = "test-restore-snapshot-repo";
Expand All @@ -339,7 +369,7 @@ public void testRestoreInSameRemoteStoreEnabledIndex() throws IOException {
indexDocuments(client, indexName2, numDocsInIndex2);
ensureGreen(indexName1, indexName2);

internalCluster().startDataOnlyNode();
internalCluster().startDataAndWarmNodes(1);
logger.info("--> snapshot");
SnapshotInfo snapshotInfo1 = createSnapshot(
snapshotRepoName,
Expand Down Expand Up @@ -441,7 +471,7 @@ void assertRemoteSegmentsAndTranslogUploaded(String idx) throws IOException {

public void testRemoteRestoreIndexRestoredFromSnapshot() throws IOException, ExecutionException, InterruptedException {
internalCluster().startClusterManagerOnlyNode();
internalCluster().startDataOnlyNodes(2);
internalCluster().startDataAndWarmNodes(2);

String indexName1 = "testindex1";
String snapshotRepoName = "test-restore-snapshot-repo";
Expand Down Expand Up @@ -500,7 +530,7 @@ public void testRemoteRestoreIndexRestoredFromSnapshot() throws IOException, Exe

public void testSuccessfulIndexRestoredFromSnapshotWithUpdatedSetting() throws IOException, ExecutionException, InterruptedException {
internalCluster().startClusterManagerOnlyNode();
internalCluster().startDataOnlyNodes(2);
internalCluster().startDataAndWarmNodes(2);

String indexName1 = "testindex1";
String snapshotRepoName = "test-restore-snapshot-repo";
Expand Down Expand Up @@ -716,7 +746,7 @@ public void testRestoreShallowSnapshotIndexAfterSnapshot() throws ExecutionExcep

public void testInvalidRestoreRequestScenarios() throws Exception {
internalCluster().startClusterManagerOnlyNode();
internalCluster().startDataOnlyNode();
internalCluster().startDataAndWarmNodes(1);
String index = "test-index";
String snapshotRepo = "test-restore-snapshot-repo";
String newRemoteStoreRepo = "test-new-rs-repo";
Expand All @@ -736,7 +766,7 @@ public void testInvalidRestoreRequestScenarios() throws Exception {
indexDocuments(client, index, numDocsInIndex);
ensureGreen(index);

internalCluster().startDataOnlyNode();
internalCluster().startDataAndWarmNodes(1);
logger.info("--> snapshot");

SnapshotInfo snapshotInfo = createSnapshot(snapshotRepo, snapshotName1, new ArrayList<>(List.of(index)));
Expand Down Expand Up @@ -896,8 +926,8 @@ public void testInvalidRestoreRequestScenarios() throws Exception {

public void testCreateSnapshotV2_Orphan_Timestamp_Cleanup() throws Exception {
internalCluster().startClusterManagerOnlyNode(pinnedTimestampSettings());
internalCluster().startDataOnlyNode(pinnedTimestampSettings());
internalCluster().startDataOnlyNode(pinnedTimestampSettings());
internalCluster().startDataAndWarmNodes(1, pinnedTimestampSettings());
internalCluster().startDataAndWarmNodes(1, pinnedTimestampSettings());
String indexName1 = "testindex1";
String indexName2 = "testindex2";
String snapshotRepoName = "test-create-snapshot-repo";
Expand Down Expand Up @@ -962,8 +992,8 @@ public void onFailure(Exception e) {}
public void testMixedSnapshotCreationWithV2RepositorySetting() throws Exception {

internalCluster().startClusterManagerOnlyNode(pinnedTimestampSettings());
internalCluster().startDataOnlyNode(pinnedTimestampSettings());
internalCluster().startDataOnlyNode(pinnedTimestampSettings());
internalCluster().startDataAndWarmNodes(1, pinnedTimestampSettings());
internalCluster().startDataAndWarmNodes(1, pinnedTimestampSettings());
String indexName1 = "testindex1";
String indexName2 = "testindex2";
String indexName3 = "testindex3";
Expand Down Expand Up @@ -1034,8 +1064,8 @@ public void testMixedSnapshotCreationWithV2RepositorySetting() throws Exception

public void testConcurrentSnapshotV2CreateOperation() throws InterruptedException, ExecutionException {
internalCluster().startClusterManagerOnlyNode(pinnedTimestampSettings());
internalCluster().startDataOnlyNode(pinnedTimestampSettings());
internalCluster().startDataOnlyNode(pinnedTimestampSettings());
internalCluster().startDataAndWarmNodes(1, pinnedTimestampSettings());
internalCluster().startDataAndWarmNodes(1, pinnedTimestampSettings());
String indexName1 = "testindex1";
String indexName2 = "testindex2";
String snapshotRepoName = "test-create-snapshot-repo";
Expand Down Expand Up @@ -1108,8 +1138,8 @@ public void testConcurrentSnapshotV2CreateOperation_MasterChange() throws Except
internalCluster().startClusterManagerOnlyNode(pinnedTimestampSettings());
internalCluster().startClusterManagerOnlyNode(pinnedTimestampSettings());
internalCluster().startClusterManagerOnlyNode(pinnedTimestampSettings());
internalCluster().startDataOnlyNode(pinnedTimestampSettings());
internalCluster().startDataOnlyNode(pinnedTimestampSettings());
internalCluster().startDataAndWarmNodes(1, pinnedTimestampSettings());
internalCluster().startDataAndWarmNodes(1, pinnedTimestampSettings());
String indexName1 = "testindex1";
String indexName2 = "testindex2";
String snapshotRepoName = "test-create-snapshot-repo";
Expand Down Expand Up @@ -1184,8 +1214,8 @@ public void testConcurrentSnapshotV2CreateOperation_MasterChange() throws Except

public void testCreateSnapshotV2() throws Exception {
internalCluster().startClusterManagerOnlyNode(pinnedTimestampSettings());
internalCluster().startDataOnlyNode(pinnedTimestampSettings());
internalCluster().startDataOnlyNode(pinnedTimestampSettings());
internalCluster().startDataAndWarmNodes(1, pinnedTimestampSettings());
internalCluster().startDataAndWarmNodes(1, pinnedTimestampSettings());
String indexName1 = "testindex1";
String indexName2 = "testindex2";
String indexName3 = "testindex3";
Expand Down Expand Up @@ -1254,8 +1284,8 @@ public void forceSyncPinnedTimestamps() {

public void testCreateSnapshotV2WithRedIndex() throws Exception {
internalCluster().startClusterManagerOnlyNode(pinnedTimestampSettings());
internalCluster().startDataOnlyNode(pinnedTimestampSettings());
internalCluster().startDataOnlyNode(pinnedTimestampSettings());
internalCluster().startDataAndWarmNodes(1, pinnedTimestampSettings());
internalCluster().startDataAndWarmNodes(1, pinnedTimestampSettings());
String indexName1 = "testindex1";
String indexName2 = "testindex2";
String snapshotRepoName = "test-create-snapshot-repo";
Expand Down Expand Up @@ -1302,8 +1332,8 @@ public void testCreateSnapshotV2WithRedIndex() throws Exception {

public void testCreateSnapshotV2WithIndexingLoad() throws Exception {
internalCluster().startClusterManagerOnlyNode(pinnedTimestampSettings());
internalCluster().startDataOnlyNode(pinnedTimestampSettings());
internalCluster().startDataOnlyNode(pinnedTimestampSettings());
internalCluster().startDataAndWarmNodes(1, pinnedTimestampSettings());
internalCluster().startDataAndWarmNodes(1, pinnedTimestampSettings());
String indexName1 = "testindex1";
String indexName2 = "testindex2";
String snapshotRepoName = "test-create-snapshot-repo";
Expand Down Expand Up @@ -1370,8 +1400,8 @@ public void testCreateSnapshotV2WithIndexingLoad() throws Exception {

public void testCreateSnapshotV2WithShallowCopySettingDisabled() throws Exception {
internalCluster().startClusterManagerOnlyNode(pinnedTimestampSettings());
internalCluster().startDataOnlyNode(pinnedTimestampSettings());
internalCluster().startDataOnlyNode(pinnedTimestampSettings());
internalCluster().startDataAndWarmNodes(1, pinnedTimestampSettings());
internalCluster().startDataAndWarmNodes(1, pinnedTimestampSettings());
String indexName1 = "testindex1";
String indexName2 = "testindex2";
String snapshotRepoName = "test-create-snapshot-repo";
Expand Down Expand Up @@ -1419,7 +1449,7 @@ public void testCreateSnapshotV2WithShallowCopySettingDisabled() throws Exceptio
public void testClusterManagerFailoverDuringSnapshotCreation() throws Exception {

internalCluster().startClusterManagerOnlyNodes(3, pinnedTimestampSettings());
internalCluster().startDataOnlyNode(pinnedTimestampSettings());
internalCluster().startDataAndWarmNodes(1, pinnedTimestampSettings());
String indexName1 = "testindex1";
String indexName2 = "testindex2";
String snapshotRepoName = "test-create-snapshot-repo";
Expand Down Expand Up @@ -1493,8 +1523,8 @@ public void testClusterManagerFailoverDuringSnapshotCreation() throws Exception

public void testConcurrentV1SnapshotAndV2RepoSettingUpdate() throws Exception {
internalCluster().startClusterManagerOnlyNode(pinnedTimestampSettings());
internalCluster().startDataOnlyNode(pinnedTimestampSettings());
internalCluster().startDataOnlyNode(pinnedTimestampSettings());
internalCluster().startDataAndWarmNodes(1, pinnedTimestampSettings());
internalCluster().startDataAndWarmNodes(1, pinnedTimestampSettings());
String snapshotRepoName = "test-create-snapshot-repo";
String snapshotName1 = "test-create-snapshot-v1";
Path absolutePath1 = randomRepoPath().toAbsolutePath();
Expand Down
26 changes: 26 additions & 0 deletions server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteSnapshotIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,10 @@
import org.opensearch.cluster.metadata.IndexMetadata;
import org.opensearch.cluster.metadata.RepositoryMetadata;
import org.opensearch.common.settings.Settings;
import org.opensearch.index.IndexModule;
import org.opensearch.index.IndexSettings;
import org.opensearch.indices.RemoteStoreSettings;
import org.opensearch.node.Node;
import org.opensearch.repositories.fs.ReloadableFsRepository;
import org.opensearch.snapshots.AbstractSnapshotIntegTestCase;
import org.opensearch.transport.client.Client;
Expand All @@ -24,9 +26,20 @@
import java.nio.file.Path;
import java.util.concurrent.ExecutionException;

import static org.opensearch.common.util.FeatureFlags.WRITABLE_WARM_INDEX_SETTING;
import static org.opensearch.repositories.fs.ReloadableFsRepository.REPOSITORIES_FAILRATE_SETTING;
import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked;

public abstract class RemoteSnapshotIT extends AbstractSnapshotIntegTestCase {

public RemoteSnapshotIT(Settings nodeSettings) {
super(nodeSettings);
}

public RemoteSnapshotIT() {
super();
}

protected static final String BASE_REMOTE_REPO = "test-rs-repo" + TEST_REMOTE_STORE_REPO_SUFFIX;
protected Path remoteRepoPath;

Expand All @@ -38,6 +51,16 @@ public void setup() {
@After
public void teardown() {
clusterAdmin().prepareCleanupRepository(BASE_REMOTE_REPO).get();
if (WRITABLE_WARM_INDEX_SETTING.get(settings)) {
assertAcked(client().admin().indices().prepareDelete("_all").get());
var nodes = internalCluster().getDataNodeInstances(Node.class);
for (var node : nodes) {
var fileCache = node.fileCache();
if (fileCache != null) {
fileCache.clear();
}
}
}
}

@Override
Expand All @@ -61,6 +84,9 @@ protected Settings.Builder getIndexSettings(int numOfShards, int numOfReplicas)
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, numOfShards)
.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, numOfReplicas)
.put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), "300s");
if (WRITABLE_WARM_INDEX_SETTING.get(settings)) {
settingsBuilder.put(IndexModule.IS_WARM_INDEX_SETTING.getKey(), true);
}
return settingsBuilder;
}

Expand Down
Loading
Loading