Merge branch 'main' into change_wildcard_default_value

Signed-off-by: Michael Froh <[email protected]>

Author: msfroh

CHANGELOG.md

@@ -11,6 +11,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Use Lucene `pack` method for `half_float` and `usigned_long` when using `ApproximatePointRangeQuery`.
 - Add a mapper for context aware segments grouping criteria ([#19233](https://github.com/opensearch-project/OpenSearch/pull/19233))
 - Return full error for GRPC error response ([#19568](https://github.com/opensearch-project/OpenSearch/pull/19568))
+- Add support for repository with Server side encryption enabled and client side encryption as well based on a flag. ([#19630)](https://github.com/opensearch-project/OpenSearch/pull/19630))
 - Add pluggable gRPC interceptors with explicit ordering([#19005](https://github.com/opensearch-project/OpenSearch/pull/19005))
 - Add BindableServices extension point to transport-grpc-spi ([#19304](https://github.com/opensearch-project/OpenSearch/pull/19304))
 - Add metrics for the merged segment warmer feature ([#18929](https://github.com/opensearch-project/OpenSearch/pull/18929))
@@ -32,6 +33,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Refactor the IndexingStats.Stats class to use the Builder pattern instead of constructors ([#19306](https://github.com/opensearch-project/OpenSearch/pull/19306))
 - Remove FeatureFlag.MERGED_SEGMENT_WARMER_EXPERIMENTAL_FLAG. ([#19715](https://github.com/opensearch-project/OpenSearch/pull/19715))
 - Change the default value of doc_values in WildcardFieldMapper to true. ([#19796](https://github.com/opensearch-project/OpenSearch/pull/19796))
+- Make Engine#loadHistoryUUID() protected and Origin#isFromTranslog() public ([#19753](https://github.com/opensearch-project/OpenSearch/pull/19752))
 
 ### Fixed
 - Fix Allocation and Rebalance Constraints of WeightFunction are incorrectly reset ([#19012](https://github.com/opensearch-project/OpenSearch/pull/19012))

distribution/tools/fips-demo-installer-cli/src/test/java/org/opensearch/tools/cli/fips/truststore/CreateFipsTrustStoreTests.java

@@ -8,52 +8,39 @@
 
 package org.opensearch.tools.cli.fips.truststore;
 
+import org.opensearch.cli.SuppressForbidden;
 import org.opensearch.test.OpenSearchTestCase;
-import org.junit.AfterClass;
 import org.junit.Before;
 import org.junit.BeforeClass;
+import org.junit.ClassRule;
+import org.junit.rules.TemporaryFolder;
 
 import java.io.IOException;
 import java.io.PrintWriter;
 import java.io.Writer;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.security.KeyStore;
-import java.util.Comparator;
 import java.util.function.Consumer;
-import java.util.stream.Stream;
 
 import picocli.CommandLine;
 
 public class CreateFipsTrustStoreTests extends OpenSearchTestCase {
+    @ClassRule
+    public static TemporaryFolder tempFolder = new TemporaryFolder();
 
     private static final Path JAVA_HOME = Path.of(System.getProperty("java.home"));
-    private static Path sharedTempDir;
+    private static Path confDir;
 
     private CommandLine.Model.CommandSpec spec;
 
     @BeforeClass
+    @SuppressForbidden(reason = "the java.io.File is exposed by TemporaryFolder")
     public static void setUpClass() throws IOException {
-        sharedTempDir = Files.createTempDirectory(Path.of(System.getProperty("java.io.tmpdir")), "fips-test-");
-        Path confDir = sharedTempDir.resolve("config");
+        confDir = tempFolder.newFolder().toPath().resolve("config");
         Files.createDirectories(confDir);
     }
 
-    @AfterClass
-    public static void tearDownClass() throws IOException {
-        if (sharedTempDir != null && Files.exists(sharedTempDir)) {
-            try (Stream<Path> walk = Files.walk(sharedTempDir)) {
-                walk.sorted(Comparator.reverseOrder()).forEach(path -> {
-                    try {
-                        Files.delete(path);
-                    } catch (Exception e) {
-                        // Ignore
-                    }
-                });
-            }
-        }
-    }
-
     @Before
     public void setUp() throws Exception {
         super.setUp();
@@ -67,7 +54,6 @@ class DummyCommand {}
         spec = commandLine.getCommandSpec();
 
         // Clean up any existing truststore file from previous tests
-        Path confDir = sharedTempDir.resolve("config");
         Path trustStorePath = confDir.resolve("opensearch-fips-truststore.bcfks");
         if (Files.exists(trustStorePath)) {
             Files.delete(trustStorePath);
@@ -150,10 +136,9 @@ public void testConvertToBCFKS() throws Exception {
         CommonOptions options = new CommonOptions();
         options.force = false;
         String password = "testPassword123";
-        Path confPath = sharedTempDir.resolve("config");
 
         // when
-        Path result = CreateFipsTrustStore.convertToBCFKS(spec, sourceKeyStore, options, password, confPath);
+        Path result = CreateFipsTrustStore.convertToBCFKS(spec, sourceKeyStore, options, password, confDir);
 
         // then
         assertNotNull(result);
@@ -181,16 +166,15 @@ public void testConvertToBCFKSFileExistsWithoutForce() throws Exception {
         String password = "testPassword123";
 
         // Create file first to simulate existing truststore
-        Path confPath = sharedTempDir.resolve("config");
-        Path trustStorePath = confPath.resolve("opensearch-fips-truststore.bcfks");
+        Path trustStorePath = confDir.resolve("opensearch-fips-truststore.bcfks");
         Files.createFile(trustStorePath);
 
         assertTrue("Test setup: file should exist", Files.exists(trustStorePath));
 
         // when/then
         RuntimeException exception = expectThrows(
             RuntimeException.class,
-            () -> CreateFipsTrustStore.convertToBCFKS(spec, sourceKeyStore, options, password, confPath)
+            () -> CreateFipsTrustStore.convertToBCFKS(spec, sourceKeyStore, options, password, confDir)
         );
         assertEquals("Operation cancelled. Trust store file already exists.", exception.getMessage());
     }
@@ -207,14 +191,13 @@ public void testConvertToBCFKSFileExistsWithForce() throws Exception {
         String password = "testPassword123";
 
         // Create file first
-        Path confPath = sharedTempDir.resolve("config");
-        Path trustStorePath = confPath.resolve("opensearch-fips-truststore.bcfks");
+        Path trustStorePath = confDir.resolve("opensearch-fips-truststore.bcfks");
         Files.createFile(trustStorePath);
 
         assertTrue(Files.exists(trustStorePath));
 
         // when
-        Path result = CreateFipsTrustStore.convertToBCFKS(spec, sourceKeyStore, options, password, confPath);
+        Path result = CreateFipsTrustStore.convertToBCFKS(spec, sourceKeyStore, options, password, confDir);
 
         // then
         assertNotNull(result);

distribution/tools/fips-demo-installer-cli/src/test/java/org/opensearch/tools/cli/fips/truststore/FipsTrustStoreCommandTestCase.java

@@ -13,6 +13,8 @@
 import org.junit.AfterClass;
 import org.junit.Before;
 import org.junit.BeforeClass;
+import org.junit.ClassRule;
+import org.junit.rules.TemporaryFolder;
 
 import java.io.PrintWriter;
 import java.io.StringWriter;
@@ -24,32 +26,24 @@
 import picocli.CommandLine;
 
 public abstract class FipsTrustStoreCommandTestCase extends OpenSearchTestCase {
+    @ClassRule
+    public static TemporaryFolder tempFolder = new TemporaryFolder();
 
     protected StringWriter outputCapture;
     protected StringWriter errorCapture;
     protected CommandLine commandLine;
     protected static Path sharedTempDir;
 
     @BeforeClass
+    @SuppressForbidden(reason = "the java.io.File is exposed by TemporaryFolder")
     static void setUpClass() throws Exception {
-        sharedTempDir = Files.createTempDirectory(Path.of(System.getProperty("java.io.tmpdir")), "system-command-test-");
+        sharedTempDir = tempFolder.newFolder().toPath();
         setProperties();
     }
 
     @AfterClass
     static void tearDownClass() throws Exception {
         clearProperties();
-        if (sharedTempDir != null && Files.exists(sharedTempDir)) {
-            try (var walk = Files.walk(sharedTempDir)) {
-                walk.sorted(java.util.Comparator.reverseOrder()).forEach(path -> {
-                    try {
-                        Files.delete(path);
-                    } catch (Exception e) {
-                        // Ignore
-                    }
-                });
-            }
-        }
     }
 
     @SuppressForbidden(reason = "set system properties as part of test setup")

distribution/tools/fips-demo-installer-cli/src/test/java/org/opensearch/tools/cli/fips/truststore/TrustStoreServiceTests.java

@@ -8,9 +8,10 @@
 
 package org.opensearch.tools.cli.fips.truststore;
 
+import org.opensearch.cli.SuppressForbidden;
 import org.opensearch.test.OpenSearchTestCase;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
+import org.junit.ClassRule;
+import org.junit.rules.TemporaryFolder;
 
 import java.io.ByteArrayInputStream;
 import java.io.PrintWriter;
@@ -27,34 +28,15 @@
 import static org.opensearch.tools.cli.fips.truststore.ConfigureSystemTrustStore.findPKCS11ProviderService;
 
 public class TrustStoreServiceTests extends OpenSearchTestCase {
-
-    private static Path sharedTempDir;
+    @ClassRule
+    public static TemporaryFolder tempFolder = new TemporaryFolder();
 
     private CommandLine.Model.CommandSpec spec;
     private StringWriter outputCapture;
     private Path confPath;
 
-    @BeforeClass
-    public static void setUpClass() throws Exception {
-        sharedTempDir = Files.createTempDirectory(Path.of(System.getProperty("java.io.tmpdir")), "truststore-test-");
-    }
-
-    @AfterClass
-    public static void tearDownClass() throws Exception {
-        if (sharedTempDir != null && Files.exists(sharedTempDir)) {
-            try (var walk = Files.walk(sharedTempDir)) {
-                walk.sorted(java.util.Comparator.reverseOrder()).forEach(path -> {
-                    try {
-                        Files.delete(path);
-                    } catch (Exception e) {
-                        // Ignore
-                    }
-                });
-            }
-        }
-    }
-
     @Override
+    @SuppressForbidden(reason = "the java.io.File is exposed by TemporaryFolder")
     public void setUp() throws Exception {
         super.setUp();
         outputCapture = new StringWriter();
@@ -66,7 +48,7 @@ class TestCommand {}
         commandLine.setOut(new PrintWriter(outputCapture, true));
         spec = commandLine.getCommandSpec();
 
-        confPath = Files.createTempDirectory(sharedTempDir, "conf-");
+        confPath = Files.createTempDirectory(tempFolder.newFolder().toPath(), "conf-");
     }
 
     public void testUseSystemTrustStoreUserCancels() {

plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3BlobStore.java

@@ -230,7 +230,7 @@ public boolean serverSideEncryptionBucketKey() {
      * null as the S3 client ignores null header values
      */
     public String serverSideEncryptionEncryptionContext() {
-        return serverSideEncryptionEncryptionContext.isEmpty()
+        return serverSideEncryptionEncryptionContext == null || serverSideEncryptionEncryptionContext.isEmpty()
             ? null
             : Base64.getEncoder().encodeToString(serverSideEncryptionEncryptionContext.getBytes(StandardCharsets.UTF_8));
     }
@@ -239,7 +239,7 @@ public String serverSideEncryptionEncryptionContext() {
      * Returns the expected bucket owner if set, else null as the S3 client ignores null header values
      */
     public String expectedBucketOwner() {
-        return expectedBucketOwner.isEmpty() ? null : expectedBucketOwner;
+        return expectedBucketOwner == null || expectedBucketOwner.isEmpty() ? null : expectedBucketOwner;
     }
 
     public long bufferSizeInBytes() {

plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3Repository.java

@@ -683,4 +683,10 @@ protected void doClose() {
         }
         super.doClose();
     }
+
+    @Override
+    public boolean isSeverSideEncryptionEnabled() {
+        // s3 is always server side encrypted.
+        return true;
+    }
 }

plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3RepositoryTests.java

@@ -33,6 +33,7 @@
 package org.opensearch.repositories.s3;
 
 import software.amazon.awssdk.services.s3.S3Client;
+import software.amazon.awssdk.services.s3.model.ServerSideEncryption;
 
 import org.opensearch.cluster.metadata.RepositoryMetadata;
 import org.opensearch.common.blobstore.BlobStoreException;
@@ -175,6 +176,18 @@ public void testValidateHttpLClientType_Invalid_Values() {
         }
     }
 
+    public void testIsSeverSideEncryptionEnabled_When_AWSKMS_Type() {
+        Settings settings = Settings.builder()
+            .put(S3Repository.SERVER_SIDE_ENCRYPTION_TYPE_SETTING.getKey(), ServerSideEncryption.AWS_KMS.toString())
+            .build();
+        final RepositoryMetadata metadata = new RepositoryMetadata("dummy-repo", "mock", settings);
+        try (S3Repository s3Repo = createS3Repo(metadata)) {
+
+            // Don't expect any Exception
+            assertTrue(s3Repo.isSeverSideEncryptionEnabled());
+        }
+    }
+
     private S3Repository createS3Repo(RepositoryMetadata metadata) {
         return new S3Repository(
             metadata,

release-notes/opensearch.release-notes-3.3.1.md

@@ -1,8 +1,8 @@
 ## Version 3.3.1 Release Notes
 
-Compatible with OpenSearch and OpenSearch Dashboards version 3.3.1
+Compatible with OpenSearch 3.3.1 and OpenSearch Dashboards 3.3.0
 
 ### Fixed
 * Fix issue with updating core with a patch number other than 0 ([#19377](https://github.com/opensearch-project/OpenSearch/pull/19377))
 * [Star Tree] Fix sub-aggregator casting for search with profile=true ([#19652](https://github.com/opensearch-project/OpenSearch/pull/19652))
-* Fix bwc @timestamp upgrade issue by adding a version check on skip_list param ([#19671](https://github.com/opensearch-project/OpenSearch/pull/19671))
+* Fix bwc @timestamp upgrade issue by adding a version check on skip_list param ([#19671](https://github.com/opensearch-project/OpenSearch/pull/19671))

release-notes/opensearch.release-notes-3.3.2.md

@@ -0,0 +1,14 @@
+## Version 3.3.2 Release Notes
+
+Compatible with OpenSearch 3.3.2 and OpenSearch Dashboards 3.3.0
+
+### Fixed
+* [Star Tree] Fix sub-aggregator casting for search with profile=true ([#19652](https://github.com/opensearch-project/OpenSearch/pull/19652))
+* [Java Agent] Allow JRT protocol URLs in protection domain extraction ([#19683](https://github.com/opensearch-project/OpenSearch/pull/19683))
+* Fix bwc @timestamp upgrade issue by adding a version check on skip_list param ([#19671](https://github.com/opensearch-project/OpenSearch/pull/19671))
+* Fix issue with updating core with a patch number other than 0 ([#19377](https://github.com/opensearch-project/OpenSearch/pull/19377))
+* Fix IndexOutOfBoundsException when running include/exclude on non-existent prefix in terms aggregations ([#19637](https://github.com/opensearch-project/OpenSearch/pull/19637))
+* Add S3Repository.LEGACY_MD5_CHECKSUM_CALCULATION to list of repository-s3 settings ([#19789](https://github.com/opensearch-project/OpenSearch/pull/19789))
+
+### Dependencies
+* Bump ch.qos.logback modules from 1.5.18 to 1.5.20 in HDFS test fixture ([#19764](https://github.com/opensearch-project/OpenSearch/pull/19764))

server/src/main/java/org/opensearch/action/admin/cluster/remotestore/metadata/TransportRemoteStoreMetadataAction.java

@@ -198,7 +198,9 @@ private Map<String, Map<String, Object>> getSegmentMetadata(
             IndexMetadata.INDEX_REMOTE_SEGMENT_STORE_REPOSITORY_SETTING.get(indexMetadata.getSettings()),
             index.getUUID(),
             shardId,
-            indexSettings.getRemoteStorePathStrategy()
+            indexSettings.getRemoteStorePathStrategy(),
+            null,
+            RemoteStoreUtils.isServerSideEncryptionEnabledIndex(indexSettings.getIndexMetadata())
         );
 
         Map<String, RemoteSegmentMetadata> segmentMetadataMapWithFilenames = remoteDirectory.readLatestNMetadataFiles(5);
@@ -257,7 +259,8 @@ private Map<String, Map<String, Object>> getTranslogMetadataFiles(
             tracker,
             indexSettings.getRemoteStorePathStrategy(),
             new RemoteStoreSettings(clusterService.getSettings(), clusterService.getClusterSettings()),
-            RemoteStoreUtils.determineTranslogMetadataEnabled(indexMetadata)
+            RemoteStoreUtils.determineTranslogMetadataEnabled(indexMetadata),
+            RemoteStoreUtils.isServerSideEncryptionEnabledIndex(indexSettings.getIndexMetadata())
         );
 
         Map<String, TranslogTransferMetadata> metadataMap = manager.readLatestNMetadataFiles(5);