Index Level Encryption plugin

CHANGELOG.md

@@ -19,7 +19,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Remote reindex: Add support for configurable retry mechanism ([#12561](https://github.com/opensearch-project/OpenSearch/pull/12561))
 - [Admission Control] Integrate IO Usage Tracker to the Resource Usage Collector Service and Emit IO Usage Stats ([#11880](https://github.com/opensearch-project/OpenSearch/pull/11880))
 - Tracing for deep search path ([#12103](https://github.com/opensearch-project/OpenSearch/pull/12103))
-- Add explicit dependency to validatePom and generatePom tasks ([#12103](https://github.com/opensearch-project/OpenSearch/pull/12807))
+- Add explicit dependency to validatePom and generatePom tasks ([#12807](https://github.com/opensearch-project/OpenSearch/pull/12807))
+- Replace configureEach with all for publication iteration ([#12876](https://github.com/opensearch-project/OpenSearch/pull/12876))
+- Index level encryption features ([#12902](https://github.com/opensearch-project/OpenSearch/pull/12902))
 
 ### Dependencies
 - Bump `log4j-core` from 2.18.0 to 2.19.0

sandbox/plugins/cryptodirectory/build.gradle

@@ -0,0 +1,25 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ *
+ * Modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+apply plugin: 'opensearch.build'
+apply plugin: 'opensearch.publish'
+apply plugin: 'opensearch.yaml-rest-test'
+apply plugin: 'opensearch.internal-cluster-test'
+
+opensearchplugin {
+  description 'Encrypts and decrypts index data at rest.'
+  classname 'org.opensearch.index.store.CryptoDirectoryPlugin'
+}
+
+dependencies {
+        testImplementation project(path: ':modules:reindex')
+        testImplementation "commons-io:commons-io:2.13.0"
+}

sandbox/plugins/cryptodirectory/src/internalClusterTest/java/org/opensearch/index/store/CryptoDirectoryIntegTestCase.java

@@ -0,0 +1,127 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+package org.opensearch.index.store;
+
+import org.opensearch.action.admin.indices.delete.DeleteIndexRequest;
+import org.opensearch.action.search.SearchResponse;
+import org.opensearch.common.settings.Settings;
+import org.opensearch.index.IndexNotFoundException;
+import org.opensearch.index.reindex.ReindexAction;
+import org.opensearch.index.reindex.ReindexModulePlugin;
+import org.opensearch.index.reindex.ReindexRequestBuilder;
+import org.opensearch.plugins.Plugin;
+import org.opensearch.test.OpenSearchIntegTestCase;
+
+import java.util.Arrays;
+import java.util.Collection;
+
+import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS;
+import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_SHARDS;
+import static org.hamcrest.Matchers.is;
+
+public class CryptoDirectoryIntegTestCase extends OpenSearchIntegTestCase {
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        return Arrays.asList(
+            CryptoDirectoryPlugin.class,
+            MockCryptoKeyProviderPlugin.class,
+            MockCryptoPlugin.class,
+            ReindexModulePlugin.class
+        );
+    }
+
+    @Override
+    public Settings indexSettings() {
+        return Settings.builder()
+            .put(super.indexSettings())
+            .put("index.store.type", "cryptofs")
+            .put("index.store.kms.type", "dummy")
+            .build();
+    }
+
+    public void testEmptyStoreTypeSettings() {
+        Settings settings = Settings.builder()
+            .put(super.indexSettings())
+            .put("index.store.type", "cryptofs")
+            .put(SETTING_NUMBER_OF_SHARDS, 1)
+            .put(SETTING_NUMBER_OF_REPLICAS, 0)
+            .build();
+
+        // Create an index and index some documents
+        createIndex("test", settings);
+        long nbDocs = randomIntBetween(10, 1000);
+        final Exception e = expectThrows(Exception.class, () -> {
+            for (long i = 0; i < nbDocs; i++) {
+                index("test", "doc", "" + i, "foo", "bar");
+            }
+        });
+        assertTrue(e instanceof Exception);
+    }
+
+    public void testUnavailableStoreType() {
+        Settings settings = Settings.builder()
+            .put(super.indexSettings())
+            .put("index.store.type", "cryptofs")
+            .put("index.store.kms.type", "unavailable")
+            .put(SETTING_NUMBER_OF_SHARDS, 1)
+            .put(SETTING_NUMBER_OF_REPLICAS, 0)
+            .build();
+
+        // Create an index and index some documents
+        createIndex("test", settings);
+        long nbDocs = randomIntBetween(10, 1000);
+        final Exception e = expectThrows(Exception.class, () -> {
+            for (long i = 0; i < nbDocs; i++) {
+                index("test", "doc", "" + i, "foo", "bar");
+            }
+        });
+        assertTrue(e instanceof Exception);
+    }
+
+    public void testReindex() {
+        // Create an index and index some documents
+        createIndex("test");
+        createIndex("test_copy");
+        long nbDocs = randomIntBetween(10, 1000);
+        for (long i = 0; i < nbDocs; i++) {
+            index("test", "doc", "" + i, "foo", "bar");
+        }
+        refresh();
+        SearchResponse response = client().prepareSearch("test").get();
+        assertThat(response.getHits().getTotalHits().value, is(nbDocs));
+
+        // Reindex
+        reindex().source("test").destination("test_copy").refresh(true).get();
+        SearchResponse copy_response = client().prepareSearch("test_copy").get();
+        assertThat(copy_response.getHits().getTotalHits().value, is(nbDocs));
+
+    }
+
+    public void testDelete() {
+        // Create an index and index some documents
+        createIndex("todelete");
+        long nbDocs = randomIntBetween(10, 1000);
+        for (long i = 0; i < nbDocs; i++) {
+            index("todelete", "doc", "" + i, "foo", "bar");
+        }
+        refresh();
+        SearchResponse response = client().prepareSearch("todelete").get();
+        assertThat(response.getHits().getTotalHits().value, is(nbDocs));
+
+        // Deleteindex
+        DeleteIndexRequest deleteIndexRequest = new DeleteIndexRequest("todelete");
+        client().admin().indices().delete(deleteIndexRequest).actionGet();
+
+        final IndexNotFoundException e = expectThrows(IndexNotFoundException.class, () -> client().prepareSearch("todelete").get());
+        assertTrue(e.getMessage().contains("no such index"));
+    }
+
+    ReindexRequestBuilder reindex() {
+        return new ReindexRequestBuilder(client(), ReindexAction.INSTANCE);
+    }
+}