From ced7e5385c5d926baef79c0a9e7bc074eb62b9c1 Mon Sep 17 00:00:00 2001
From: Mohit Godwani <mgodwan@amazon.com>
Date: Mon, 19 Aug 2024 09:39:40 +0530
Subject: [PATCH 01/12] Create Index using context

Signed-off-by: Mohit Godwani <mgodwan@amazon.com>
---
 .../opensearch/OpenSearchServerException.java |  8 ++
 .../CreateIndexClusterStateUpdateRequest.java | 14 ++++
 .../indices/create/CreateIndexRequest.java    | 42 +++++++++++
 .../create/TransportCreateIndexAction.java    |  1 +
 .../metadata/MetadataCreateIndexService.java  | 75 +++++++++++++++++--
 .../MetadataIndexTemplateService.java         |  2 +-
 .../indices/InvalidIndexContextException.java | 32 ++++++++
 7 files changed, 166 insertions(+), 8 deletions(-)
 create mode 100644 server/src/main/java/org/opensearch/indices/InvalidIndexContextException.java

diff --git a/server/src/main/java/org/opensearch/OpenSearchServerException.java b/server/src/main/java/org/opensearch/OpenSearchServerException.java
index c5a5ce12b238c..7b9aded2cb740 100644
--- a/server/src/main/java/org/opensearch/OpenSearchServerException.java
+++ b/server/src/main/java/org/opensearch/OpenSearchServerException.java
@@ -1201,6 +1201,14 @@ public static void registerExceptions() {
                 V_2_13_0
             )
         );
+        registerExceptionHandle(
+            new OpenSearchExceptionHandle(
+                org.opensearch.indices.InvalidIndexContextException.class,
+                org.opensearch.indices.InvalidIndexContextException::new,
+                174,
+                V_3_0_0
+            )
+        );
         registerExceptionHandle(
             new OpenSearchExceptionHandle(
                 org.opensearch.cluster.block.IndexCreateBlockException.class,
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/create/CreateIndexClusterStateUpdateRequest.java b/server/src/main/java/org/opensearch/action/admin/indices/create/CreateIndexClusterStateUpdateRequest.java
index ad45e5346f9fa..d7e86744ad528 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/create/CreateIndexClusterStateUpdateRequest.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/create/CreateIndexClusterStateUpdateRequest.java
@@ -37,6 +37,7 @@
 import org.opensearch.action.support.ActiveShardCount;
 import org.opensearch.cluster.ack.ClusterStateUpdateRequest;
 import org.opensearch.cluster.block.ClusterBlock;
+import org.opensearch.cluster.metadata.Context;
 import org.opensearch.cluster.metadata.IndexMetadata;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.core.index.Index;
@@ -65,6 +66,8 @@ public class CreateIndexClusterStateUpdateRequest extends ClusterStateUpdateRequ
 
     private final Set<Alias> aliases = new HashSet<>();
 
+    private Context context;
+
     private final Set<ClusterBlock> blocks = new HashSet<>();
 
     private ActiveShardCount waitForActiveShards = ActiveShardCount.DEFAULT;
@@ -90,6 +93,11 @@ public CreateIndexClusterStateUpdateRequest aliases(Set<Alias> aliases) {
         return this;
     }
 
+    public CreateIndexClusterStateUpdateRequest context(Context context) {
+        this.context = context;
+        return this;
+    }
+
     public CreateIndexClusterStateUpdateRequest recoverFrom(Index recoverFrom) {
         this.recoverFrom = recoverFrom;
         return this;
@@ -130,6 +138,10 @@ public Set<Alias> aliases() {
         return aliases;
     }
 
+    public Context context() {
+        return context;
+    }
+
     public Set<ClusterBlock> blocks() {
         return blocks;
     }
@@ -199,6 +211,8 @@ public String toString() {
             + settings
             + ", aliases="
             + aliases
+            + ", context="
+            + context
             + ", blocks="
             + blocks
             + ", waitForActiveShards="
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/create/CreateIndexRequest.java b/server/src/main/java/org/opensearch/action/admin/indices/create/CreateIndexRequest.java
index 01b4cd779c261..b14cb8cc51b6f 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/create/CreateIndexRequest.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/create/CreateIndexRequest.java
@@ -42,6 +42,7 @@
 import org.opensearch.action.support.ActiveShardCount;
 import org.opensearch.action.support.IndicesOptions;
 import org.opensearch.action.support.master.AcknowledgedRequest;
+import org.opensearch.cluster.metadata.Context;
 import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.xcontent.LoggingDeprecationHandler;
@@ -89,6 +90,7 @@ public class CreateIndexRequest extends AcknowledgedRequest<CreateIndexRequest>
     public static final ParseField MAPPINGS = new ParseField("mappings");
     public static final ParseField SETTINGS = new ParseField("settings");
     public static final ParseField ALIASES = new ParseField("aliases");
+    public static final ParseField CONTEXT = new ParseField("context");
 
     private String cause = "";
 
@@ -100,6 +102,8 @@ public class CreateIndexRequest extends AcknowledgedRequest<CreateIndexRequest>
 
     private final Set<Alias> aliases = new HashSet<>();
 
+    private Context context;
+
     private ActiveShardCount waitForActiveShards = ActiveShardCount.DEFAULT;
 
     public CreateIndexRequest(StreamInput in) throws IOException {
@@ -128,6 +132,9 @@ public CreateIndexRequest(StreamInput in) throws IOException {
             aliases.add(new Alias(in));
         }
         waitForActiveShards = ActiveShardCount.readFrom(in);
+        if (in.getVersion().onOrAfter(Version.V_3_0_0)) {
+            context = in.readOptionalWriteable(Context::new);
+        }
     }
 
     public CreateIndexRequest() {}
@@ -524,6 +531,8 @@ public CreateIndexRequest source(Map<String, ?> source, DeprecationHandler depre
                 }
             } else if (ALIASES.match(name, deprecationHandler)) {
                 aliases((Map<String, Object>) entry.getValue());
+            } else if (CONTEXT.match(name, deprecationHandler)) {
+                context((Map<String, Object>) entry.getValue());
             } else {
                 throw new OpenSearchParseException("unknown key [{}] for create index", name);
             }
@@ -571,6 +580,36 @@ public CreateIndexRequest waitForActiveShards(final int waitForActiveShards) {
         return waitForActiveShards(ActiveShardCount.from(waitForActiveShards));
     }
 
+    public CreateIndexRequest context(Map<String, ?> source) {
+        try {
+            XContentBuilder builder = XContentFactory.jsonBuilder();
+            builder.map(source);
+            return context(BytesReference.bytes(builder));
+        } catch (IOException e) {
+            throw new OpenSearchGenerationException("Failed to generate [" + source + "]", e);
+        }
+    }
+
+    public CreateIndexRequest context(BytesReference source) throws IOException {
+        // EMPTY is safe here because we never call namedObject
+        try (XContentParser parser = XContentHelper.createParser(NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, source)) {
+            // move to the first alias
+            context(Context.fromXContent(parser));
+            return this;
+        } catch (IOException e) {
+            throw new OpenSearchParseException("Failed to parse aliases", e);
+        }
+    }
+
+    public CreateIndexRequest context(Context context) {
+        this.context = context;
+        return this;
+    }
+
+    public Context context() {
+        return context;
+    }
+
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
@@ -593,6 +632,9 @@ public void writeTo(StreamOutput out) throws IOException {
             alias.writeTo(out);
         }
         waitForActiveShards.writeTo(out);
+        if (out.getVersion().onOrAfter(Version.V_3_0_0)) {
+            out.writeOptionalWriteable(context);
+        }
     }
 
     @Override
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/create/TransportCreateIndexAction.java b/server/src/main/java/org/opensearch/action/admin/indices/create/TransportCreateIndexAction.java
index b5f822bd45b7e..250693c130c85 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/create/TransportCreateIndexAction.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/create/TransportCreateIndexAction.java
@@ -121,6 +121,7 @@ protected void clusterManagerOperation(
             .settings(request.settings())
             .mappings(request.mappings())
             .aliases(request.aliases())
+            .context(request.context())
             .waitForActiveShards(request.waitForActiveShards());
 
         createIndexService.createIndex(
diff --git a/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java b/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java
index 50d25b11ef810..f2998789c50a4 100644
--- a/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java
+++ b/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java
@@ -49,6 +49,7 @@
 import org.opensearch.cluster.ClusterState;
 import org.opensearch.cluster.ack.ClusterStateUpdateResponse;
 import org.opensearch.cluster.ack.CreateIndexClusterStateUpdateResponse;
+import org.opensearch.cluster.applicationtemplates.SystemTemplatesService;
 import org.opensearch.cluster.block.ClusterBlock;
 import org.opensearch.cluster.block.ClusterBlockLevel;
 import org.opensearch.cluster.block.ClusterBlocks;
@@ -75,6 +76,7 @@
 import org.opensearch.common.settings.Setting;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.unit.TimeValue;
+import org.opensearch.common.util.FeatureFlags;
 import org.opensearch.common.xcontent.XContentHelper;
 import org.opensearch.core.action.ActionListener;
 import org.opensearch.core.common.Strings;
@@ -89,6 +91,7 @@
 import org.opensearch.index.mapper.DocumentMapper;
 import org.opensearch.index.mapper.MapperService;
 import org.opensearch.index.mapper.MapperService.MergeReason;
+import org.opensearch.index.mapper.Mapping;
 import org.opensearch.index.query.QueryShardContext;
 import org.opensearch.index.remote.RemoteStoreCustomMetadataResolver;
 import org.opensearch.index.remote.RemoteStoreEnums.PathHashAlgorithm;
@@ -98,7 +101,10 @@
 import org.opensearch.index.translog.Translog;
 import org.opensearch.indices.IndexCreationException;
 import org.opensearch.indices.IndicesService;
+import org.opensearch.indices.InvalidAliasNameException;
+import org.opensearch.indices.InvalidIndexContextException;
 import org.opensearch.indices.InvalidIndexNameException;
+import org.opensearch.indices.InvalidIndexTemplateException;
 import org.opensearch.indices.RemoteStoreSettings;
 import org.opensearch.indices.ShardLimitValidator;
 import org.opensearch.indices.SystemIndices;
@@ -145,6 +151,7 @@
 import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY;
 import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REPLICATION_TYPE;
 import static org.opensearch.cluster.metadata.Metadata.DEFAULT_REPLICA_COUNT_SETTING;
+import static org.opensearch.cluster.metadata.MetadataIndexTemplateService.findContextTemplate;
 import static org.opensearch.index.IndexModule.INDEX_STORE_TYPE_SETTING;
 import static org.opensearch.indices.IndicesService.CLUSTER_REPLICATION_TYPE_SETTING;
 import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.isRemoteDataAttributePresent;
@@ -421,6 +428,7 @@ public ClusterState applyCreateIndexRequest(
         if (sourceMetadata != null) {
             // If source metadata was provided, it means we're recovering from an existing index,
             // in which case templates don't apply, so create the index from the source metadata
+            // QUESTION: Should we apply context in this case?
             return applyCreateIndexRequestWithExistingMetadata(currentState, request, silent, sourceMetadata, metadataTransformer);
         } else {
             // Hidden indices apply templates slightly differently (ignoring wildcard '*'
@@ -501,6 +509,7 @@ private ClusterState applyCreateIndexWithTemporaryService(
         // create the index here (on the master) to validate it can be created, as well as adding the mapping
         return indicesService.<ClusterState, Exception>withTempIndexService(temporaryIndexMeta, indexService -> {
             try {
+
                 updateIndexMappingsAndBuildSortOrder(indexService, request, mappings, sourceMetadata);
             } catch (Exception e) {
                 logger.log(silent ? Level.DEBUG : Level.INFO, "failed on parsing mappings on index creation [{}]", request.index(), e);
@@ -619,17 +628,22 @@ private ClusterState applyCreateIndexRequestWithV1Templates(
             templates.stream().map(IndexTemplateMetadata::name).collect(Collectors.toList())
         );
 
-        final Map<String, Object> mappings = Collections.unmodifiableMap(
+        final List<Map<String, Object>> mappings = List.of(Collections.unmodifiableMap(
             parseV1Mappings(
                 request.mappings(),
                 templates.stream().map(IndexTemplateMetadata::getMappings).collect(toList()),
                 xContentRegistry
             )
-        );
+        ));
+        Optional<Template> context = applyContext(request, currentState, mappings);
+        Settings aggregateSettings = Settings.builder().put(MetadataIndexTemplateService.resolveSettings(templates))
+            .put(context.isPresent() ? context.get().settings() : Settings.EMPTY)
+            .build();
+
         final Settings aggregatedIndexSettings = aggregateIndexSettings(
             currentState,
             request,
-            MetadataIndexTemplateService.resolveSettings(templates),
+            aggregateSettings,
             null,
             settings,
             indexScopedSettings,
@@ -640,17 +654,21 @@ private ClusterState applyCreateIndexRequestWithV1Templates(
         int routingNumShards = getIndexNumberOfRoutingShards(aggregatedIndexSettings, null);
         IndexMetadata tmpImd = buildAndValidateTemporaryIndexMetadata(aggregatedIndexSettings, request, routingNumShards);
 
+        List<Map<String, AliasMetadata>> aliasMeta = new ArrayList<>();
+        aliasMeta.addAll(MetadataIndexTemplateService.resolveAliases(templates));
+        context.map(Template::aliases).ifPresent(aliasMeta::add);
+
         return applyCreateIndexWithTemporaryService(
             currentState,
             request,
             silent,
             null,
             tmpImd,
-            Collections.singletonList(mappings),
+            mappings,
             indexService -> resolveAndValidateAliases(
                 request.index(),
                 request.aliases(),
-                MetadataIndexTemplateService.resolveAliases(templates),
+                aliasMeta,
                 currentState.metadata(),
                 aliasValidator,
                 // the context is only used for validation so it's fine to pass fake values for the
@@ -691,10 +709,16 @@ private ClusterState applyCreateIndexRequestWithV2Template(
             xContentRegistry,
             request.index()
         );
+
+        Optional<Template> context = applyContext(request, currentState, mappings);
+        Settings aggregateSettings = Settings.builder().put(MetadataIndexTemplateService.resolveSettings(currentState.metadata(), templateName))
+            .put(context.isPresent() ? context.get().settings() : Settings.EMPTY)
+            .build();
+
         final Settings aggregatedIndexSettings = aggregateIndexSettings(
             currentState,
             request,
-            MetadataIndexTemplateService.resolveSettings(currentState.metadata(), templateName),
+            aggregateSettings,
             null,
             settings,
             indexScopedSettings,
@@ -705,6 +729,9 @@ private ClusterState applyCreateIndexRequestWithV2Template(
         int routingNumShards = getIndexNumberOfRoutingShards(aggregatedIndexSettings, null);
         IndexMetadata tmpImd = buildAndValidateTemporaryIndexMetadata(aggregatedIndexSettings, request, routingNumShards);
 
+        List<Map<String, AliasMetadata>> aliasMeta = MetadataIndexTemplateService.resolveAliases(currentState.metadata(), templateName);
+        context.map(Template::aliases).ifPresent(aliasMeta::add);
+
         return applyCreateIndexWithTemporaryService(
             currentState,
             request,
@@ -715,7 +742,7 @@ private ClusterState applyCreateIndexRequestWithV2Template(
             indexService -> resolveAndValidateAliases(
                 request.index(),
                 request.aliases(),
-                MetadataIndexTemplateService.resolveAliases(currentState.metadata(), templateName),
+                aliasMeta,
                 currentState.metadata(),
                 aliasValidator,
                 // the context is only used for validation so it's fine to pass fake values for the
@@ -728,6 +755,17 @@ private ClusterState applyCreateIndexRequestWithV2Template(
         );
     }
 
+    private Optional<Template> applyContext(CreateIndexClusterStateUpdateRequest request, ClusterState currentState,
+                              List<Map<String, Object>> mappings) throws IOException {
+        if (request.context() != null) {
+            String contextTemplate = MetadataIndexTemplateService.findContextTemplate(currentState.metadata(), request.context());
+            ComponentTemplate componentTemplate = currentState.metadata().componentTemplates().get(contextTemplate);
+            mappings.add(MapperService.parseMapping(xContentRegistry, componentTemplate.template().mappings().toString()));
+            return Optional.of(componentTemplate.template());
+        }
+        return Optional.empty();
+    }
+
     public static List<Map<String, Object>> collectV2Mappings(
         final String requestMappings,
         final ClusterState currentState,
@@ -1354,6 +1392,7 @@ private static void validateActiveShardCount(ActiveShardCount waitForActiveShard
     private void validate(CreateIndexClusterStateUpdateRequest request, ClusterState state) {
         validateIndexName(request.index(), state);
         validateIndexSettings(request.index(), request.settings(), forbidPrivateIndexSettings);
+        validateContext(request, state);
     }
 
     public void validateIndexSettings(String indexName, final Settings settings, final boolean forbidPrivateIndexSettings)
@@ -1694,4 +1733,26 @@ static void validateTranslogDurabilitySettings(Settings requestSettings, Cluster
         }
 
     }
+
+
+    static void validateContext(CreateIndexClusterStateUpdateRequest request, ClusterState clusterState) {
+        final boolean isContextAllowed = FeatureFlags.isEnabled(FeatureFlags.APPLICATION_BASED_CONFIGURATION_TEMPLATES);
+
+        if (request.context() != null && !isContextAllowed) {
+            throw new InvalidIndexContextException(
+                request.context().name(),
+                request.index(),
+                "index specifies a context which cannot be used without enabling: "
+                    + SystemTemplatesService.SETTING_APPLICATION_BASED_CONFIGURATION_TEMPLATES_ENABLED.getKey()
+            );
+        }
+
+        if (request.context() != null && findContextTemplate(clusterState.metadata(), request.context()) == null) {
+            throw new InvalidIndexContextException(
+                request.context().name(),
+                request.index(),
+                "index specifies a context which is not loaded on the cluster."
+            );
+        }
+    }
 }
diff --git a/server/src/main/java/org/opensearch/cluster/metadata/MetadataIndexTemplateService.java b/server/src/main/java/org/opensearch/cluster/metadata/MetadataIndexTemplateService.java
index 6b638c9920c27..0d4ef8a7f6c74 100644
--- a/server/src/main/java/org/opensearch/cluster/metadata/MetadataIndexTemplateService.java
+++ b/server/src/main/java/org/opensearch/cluster/metadata/MetadataIndexTemplateService.java
@@ -585,7 +585,7 @@ private void validateComponentTemplateRequest(ComponentTemplate componentTemplat
         }
     }
 
-    private static String findContextTemplate(Metadata metadata, Context context) {
+    static String findContextTemplate(Metadata metadata, Context context) {
         if (context == null) {
             return null;
         }
diff --git a/server/src/main/java/org/opensearch/indices/InvalidIndexContextException.java b/server/src/main/java/org/opensearch/indices/InvalidIndexContextException.java
new file mode 100644
index 0000000000000..d750a38fa612b
--- /dev/null
+++ b/server/src/main/java/org/opensearch/indices/InvalidIndexContextException.java
@@ -0,0 +1,32 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.indices;
+
+import org.opensearch.OpenSearchException;
+import org.opensearch.core.common.io.stream.StreamInput;
+import org.opensearch.core.common.io.stream.StreamOutput;
+import org.opensearch.core.rest.RestStatus;
+
+import java.io.IOException;
+
+public class InvalidIndexContextException extends OpenSearchException {
+
+    public InvalidIndexContextException(String indexName, String name, String description) {
+        super("Invalid context name [{}] provide for index: {}, [{}]", name, indexName, description);
+    }
+
+    public InvalidIndexContextException(StreamInput in) throws IOException {
+        super(in);
+    }
+
+    @Override
+    public RestStatus status() {
+        return RestStatus.BAD_REQUEST;
+    }
+}

From 0b439c2d71a5cdf1cc9fc9e04f422c733f47a066 Mon Sep 17 00:00:00 2001
From: Mohit Godwani <mgodwan@amazon.com>
Date: Tue, 20 Aug 2024 14:23:17 +0530
Subject: [PATCH 02/12] Add changelog

Signed-off-by: Mohit Godwani <mgodwan@amazon.com>
---
 CHANGELOG.md                                  |  1 +
 .../metadata/MetadataCreateIndexService.java  | 31 ++++++++++---------
 .../indices/InvalidIndexContextException.java | 10 +++++-
 3 files changed, 27 insertions(+), 15 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index deea2778dedd2..2be8b40911572 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -26,6 +26,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Add allowlist setting for ingest-geoip and ingest-useragent ([#15325](https://github.com/opensearch-project/OpenSearch/pull/15325))
 - Adding access to noSubMatches and noOverlappingMatches in Hyphenation ([#13895](https://github.com/opensearch-project/OpenSearch/pull/13895))
 - Add support for index level max slice count setting for concurrent segment search ([#15336](https://github.com/opensearch-project/OpenSearch/pull/15336))
+- Add index creation using the context field ([#15290](https://github.com/opensearch-project/OpenSearch/pull/15290))
 
 ### Dependencies
 - Bump `netty` from 4.1.111.Final to 4.1.112.Final ([#15081](https://github.com/opensearch-project/OpenSearch/pull/15081))
diff --git a/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java b/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java
index f2998789c50a4..6b0d9bfb91d75 100644
--- a/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java
+++ b/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java
@@ -91,7 +91,6 @@
 import org.opensearch.index.mapper.DocumentMapper;
 import org.opensearch.index.mapper.MapperService;
 import org.opensearch.index.mapper.MapperService.MergeReason;
-import org.opensearch.index.mapper.Mapping;
 import org.opensearch.index.query.QueryShardContext;
 import org.opensearch.index.remote.RemoteStoreCustomMetadataResolver;
 import org.opensearch.index.remote.RemoteStoreEnums.PathHashAlgorithm;
@@ -101,10 +100,8 @@
 import org.opensearch.index.translog.Translog;
 import org.opensearch.indices.IndexCreationException;
 import org.opensearch.indices.IndicesService;
-import org.opensearch.indices.InvalidAliasNameException;
 import org.opensearch.indices.InvalidIndexContextException;
 import org.opensearch.indices.InvalidIndexNameException;
-import org.opensearch.indices.InvalidIndexTemplateException;
 import org.opensearch.indices.RemoteStoreSettings;
 import org.opensearch.indices.ShardLimitValidator;
 import org.opensearch.indices.SystemIndices;
@@ -628,15 +625,18 @@ private ClusterState applyCreateIndexRequestWithV1Templates(
             templates.stream().map(IndexTemplateMetadata::name).collect(Collectors.toList())
         );
 
-        final List<Map<String, Object>> mappings = List.of(Collections.unmodifiableMap(
-            parseV1Mappings(
-                request.mappings(),
-                templates.stream().map(IndexTemplateMetadata::getMappings).collect(toList()),
-                xContentRegistry
+        final List<Map<String, Object>> mappings = List.of(
+            Collections.unmodifiableMap(
+                parseV1Mappings(
+                    request.mappings(),
+                    templates.stream().map(IndexTemplateMetadata::getMappings).collect(toList()),
+                    xContentRegistry
+                )
             )
-        ));
+        );
         Optional<Template> context = applyContext(request, currentState, mappings);
-        Settings aggregateSettings = Settings.builder().put(MetadataIndexTemplateService.resolveSettings(templates))
+        Settings aggregateSettings = Settings.builder()
+            .put(MetadataIndexTemplateService.resolveSettings(templates))
             .put(context.isPresent() ? context.get().settings() : Settings.EMPTY)
             .build();
 
@@ -711,7 +711,8 @@ private ClusterState applyCreateIndexRequestWithV2Template(
         );
 
         Optional<Template> context = applyContext(request, currentState, mappings);
-        Settings aggregateSettings = Settings.builder().put(MetadataIndexTemplateService.resolveSettings(currentState.metadata(), templateName))
+        Settings aggregateSettings = Settings.builder()
+            .put(MetadataIndexTemplateService.resolveSettings(currentState.metadata(), templateName))
             .put(context.isPresent() ? context.get().settings() : Settings.EMPTY)
             .build();
 
@@ -755,8 +756,11 @@ private ClusterState applyCreateIndexRequestWithV2Template(
         );
     }
 
-    private Optional<Template> applyContext(CreateIndexClusterStateUpdateRequest request, ClusterState currentState,
-                              List<Map<String, Object>> mappings) throws IOException {
+    private Optional<Template> applyContext(
+        CreateIndexClusterStateUpdateRequest request,
+        ClusterState currentState,
+        List<Map<String, Object>> mappings
+    ) throws IOException {
         if (request.context() != null) {
             String contextTemplate = MetadataIndexTemplateService.findContextTemplate(currentState.metadata(), request.context());
             ComponentTemplate componentTemplate = currentState.metadata().componentTemplates().get(contextTemplate);
@@ -1734,7 +1738,6 @@ static void validateTranslogDurabilitySettings(Settings requestSettings, Cluster
 
     }
 
-
     static void validateContext(CreateIndexClusterStateUpdateRequest request, ClusterState clusterState) {
         final boolean isContextAllowed = FeatureFlags.isEnabled(FeatureFlags.APPLICATION_BASED_CONFIGURATION_TEMPLATES);
 
diff --git a/server/src/main/java/org/opensearch/indices/InvalidIndexContextException.java b/server/src/main/java/org/opensearch/indices/InvalidIndexContextException.java
index d750a38fa612b..40e9d25bf95c3 100644
--- a/server/src/main/java/org/opensearch/indices/InvalidIndexContextException.java
+++ b/server/src/main/java/org/opensearch/indices/InvalidIndexContextException.java
@@ -10,13 +10,21 @@
 
 import org.opensearch.OpenSearchException;
 import org.opensearch.core.common.io.stream.StreamInput;
-import org.opensearch.core.common.io.stream.StreamOutput;
 import org.opensearch.core.rest.RestStatus;
 
 import java.io.IOException;
 
+/**
+ * Exception when the context provided in the creation of an index is invalid.
+ */
 public class InvalidIndexContextException extends OpenSearchException {
 
+    /**
+     *
+     * @param indexName name of the index
+     * @param name context name provided
+     * @param description error message
+     */
     public InvalidIndexContextException(String indexName, String name, String description) {
         super("Invalid context name [{}] provide for index: {}, [{}]", name, indexName, description);
     }

From 6c353b45f8219d49ac34dbf13e9b03e10a1f7d97 Mon Sep 17 00:00:00 2001
From: Mohit Godwani <mgodwan@amazon.com>
Date: Tue, 20 Aug 2024 18:29:23 +0530
Subject: [PATCH 03/12] Fix index creation bugs using context

Signed-off-by: Mohit Godwani <mgodwan@amazon.com>
---
 .../opensearch/cluster/metadata/Context.java  |   6 +-
 .../cluster/metadata/IndexMetadata.java       |  49 +++++++-
 .../metadata/MetadataCreateIndexService.java  | 114 +++++++++---------
 .../common/settings/IndexScopedSettings.java  |   2 +
 .../org/opensearch/index/IndexSettings.java   |   8 ++
 5 files changed, 118 insertions(+), 61 deletions(-)

diff --git a/server/src/main/java/org/opensearch/cluster/metadata/Context.java b/server/src/main/java/org/opensearch/cluster/metadata/Context.java
index 4bd6134e8a318..3e834369020d8 100644
--- a/server/src/main/java/org/opensearch/cluster/metadata/Context.java
+++ b/server/src/main/java/org/opensearch/cluster/metadata/Context.java
@@ -26,7 +26,7 @@
  * Class encapsulating the context metadata associated with an index template/index.
  */
 @ExperimentalApi
-public class Context extends AbstractDiffable<ComposableIndexTemplate> implements ToXContentObject {
+public class Context extends AbstractDiffable<Context> implements ToXContentObject {
 
     private static final ParseField NAME = new ParseField("name");
     private static final ParseField VERSION = new ParseField("version");
@@ -103,9 +103,9 @@ public void writeTo(StreamOutput out) throws IOException {
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject();
         builder.field(NAME.getPreferredName(), this.name);
-        builder.field("version", this.version);
+        builder.field(VERSION.getPreferredName(), this.version);
         if (params != null) {
-            builder.field("params", this.params);
+            builder.field(PARAMS.getPreferredName(), this.params);
         }
         builder.endObject();
         return builder;
diff --git a/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java b/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java
index df0d2609ad83d..dc85eb28e9f0f 100644
--- a/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java
+++ b/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java
@@ -639,6 +639,7 @@ public static APIBlock readFrom(StreamInput input) throws IOException {
     public static final String KEY_PRIMARY_TERMS = "primary_terms";
     public static final String REMOTE_STORE_CUSTOM_KEY = "remote_store";
     public static final String TRANSLOG_METADATA_KEY = "translog_metadata";
+    public static final String CONTEXT_KEY = "context";
 
     public static final String INDEX_STATE_FILE_PREFIX = "state-";
 
@@ -689,6 +690,8 @@ public static APIBlock readFrom(StreamInput input) throws IOException {
 
     private final int indexTotalShardsPerNodeLimit;
 
+    private final Context context;
+
     private IndexMetadata(
         final Index index,
         final long version,
@@ -715,7 +718,8 @@ private IndexMetadata(
         final ActiveShardCount waitForActiveShards,
         final Map<String, RolloverInfo> rolloverInfos,
         final boolean isSystem,
-        final int indexTotalShardsPerNodeLimit
+        final int indexTotalShardsPerNodeLimit,
+        final Context context
     ) {
 
         this.index = index;
@@ -751,6 +755,7 @@ private IndexMetadata(
         this.isSystem = isSystem;
         this.isRemoteSnapshot = IndexModule.Type.REMOTE_SNAPSHOT.match(this.settings);
         this.indexTotalShardsPerNodeLimit = indexTotalShardsPerNodeLimit;
+        this.context = context;
         assert numberOfShards * routingFactor == routingNumShards : routingNumShards + " must be a multiple of " + numberOfShards;
     }
 
@@ -979,6 +984,9 @@ public boolean equals(Object o) {
         if (isSystem != that.isSystem) {
             return false;
         }
+        if (!Objects.equals(context, that.context)) {
+            return false;
+        }
         return true;
     }
 
@@ -997,6 +1005,7 @@ public int hashCode() {
         result = 31 * result + inSyncAllocationIds.hashCode();
         result = 31 * result + rolloverInfos.hashCode();
         result = 31 * result + Boolean.hashCode(isSystem);
+        result = 31 * result + Objects.hashCode(context);
         return result;
     }
 
@@ -1041,6 +1050,7 @@ private static class IndexMetadataDiff implements Diff<IndexMetadata> {
         private final Diff<Map<Integer, Set<String>>> inSyncAllocationIds;
         private final Diff<Map<String, RolloverInfo>> rolloverInfos;
         private final boolean isSystem;
+        private final Context context;
 
         IndexMetadataDiff(IndexMetadata before, IndexMetadata after) {
             index = after.index.getName();
@@ -1063,6 +1073,7 @@ private static class IndexMetadataDiff implements Diff<IndexMetadata> {
             );
             rolloverInfos = DiffableUtils.diff(before.rolloverInfos, after.rolloverInfos, DiffableUtils.getStringKeySerializer());
             isSystem = after.isSystem;
+            context = after.context;
         }
 
         private static final DiffableUtils.DiffableValueReader<String, AliasMetadata> ALIAS_METADATA_DIFF_VALUE_READER =
@@ -1094,6 +1105,9 @@ private static class IndexMetadataDiff implements Diff<IndexMetadata> {
             );
             rolloverInfos = DiffableUtils.readJdkMapDiff(in, DiffableUtils.getStringKeySerializer(), ROLLOVER_INFO_DIFF_VALUE_READER);
             isSystem = in.readBoolean();
+            if (in.getVersion().onOrAfter(Version.V_3_0_0)) {
+                context = in.readOptionalWriteable(Context::new);
+            }
         }
 
         @Override
@@ -1113,6 +1127,9 @@ public void writeTo(StreamOutput out) throws IOException {
             inSyncAllocationIds.writeTo(out);
             rolloverInfos.writeTo(out);
             out.writeBoolean(isSystem);
+            if (out.getVersion().onOrAfter(Version.V_3_0_0)) {
+                out.writeOptionalWriteable(context);
+            }
         }
 
         @Override
@@ -1132,6 +1149,7 @@ public IndexMetadata apply(IndexMetadata part) {
             builder.inSyncAllocationIds.putAll(inSyncAllocationIds.apply(part.inSyncAllocationIds));
             builder.rolloverInfos.putAll(rolloverInfos.apply(part.rolloverInfos));
             builder.system(part.isSystem);
+            builder.context(context);
             return builder.build();
         }
     }
@@ -1173,6 +1191,10 @@ public static IndexMetadata readFrom(StreamInput in) throws IOException {
             builder.putRolloverInfo(new RolloverInfo(in));
         }
         builder.system(in.readBoolean());
+
+        if (in.getVersion().onOrAfter(Version.V_3_0_0)) {
+            builder.context(in.readOptionalWriteable(Context::new));
+        }
         return builder.build();
     }
 
@@ -1210,6 +1232,10 @@ public void writeTo(StreamOutput out) throws IOException {
             cursor.writeTo(out);
         }
         out.writeBoolean(isSystem);
+
+        if (out.getVersion().onOrAfter(Version.V_3_0_0)) {
+            out.writeOptionalWriteable(context);
+        }
     }
 
     public boolean isSystem() {
@@ -1251,6 +1277,7 @@ public static class Builder {
         private final Map<String, RolloverInfo> rolloverInfos;
         private Integer routingNumShards;
         private boolean isSystem;
+        private Context context;
 
         public Builder(String index) {
             this.index = index;
@@ -1278,6 +1305,7 @@ public Builder(IndexMetadata indexMetadata) {
             this.inSyncAllocationIds = new HashMap<>(indexMetadata.inSyncAllocationIds);
             this.rolloverInfos = new HashMap<>(indexMetadata.rolloverInfos);
             this.isSystem = indexMetadata.isSystem;
+            this.context = indexMetadata.context;
         }
 
         public Builder index(String index) {
@@ -1494,6 +1522,15 @@ public boolean isSystem() {
             return isSystem;
         }
 
+        public Builder context(Context context) {
+            this.context = context;
+            return this;
+        }
+
+        public Context context() {
+            return context;
+        }
+
         public IndexMetadata build() {
             final Map<String, AliasMetadata> tmpAliases = aliases;
             Settings tmpSettings = settings;
@@ -1622,7 +1659,8 @@ public IndexMetadata build() {
                 waitForActiveShards,
                 rolloverInfos,
                 isSystem,
-                indexTotalShardsPerNodeLimit
+                indexTotalShardsPerNodeLimit,
+                context
             );
         }
 
@@ -1725,6 +1763,11 @@ public static void toXContent(IndexMetadata indexMetadata, XContentBuilder build
             builder.endObject();
             builder.field(KEY_SYSTEM, indexMetadata.isSystem);
 
+            if (indexMetadata.context != null) {
+                builder.field(CONTEXT_KEY);
+                indexMetadata.context.toXContent(builder, params);
+            }
+
             builder.endObject();
         }
 
@@ -1806,6 +1849,8 @@ public static IndexMetadata fromXContent(XContentParser parser) throws IOExcepti
                         // simply ignored when upgrading from 2.x
                         assert Version.CURRENT.major <= 5;
                         parser.skipChildren();
+                    } else if (CONTEXT_KEY.equals(currentFieldName)) {
+                        builder.context(Context.fromXContent(parser));
                     } else {
                         // assume it's custom index metadata
                         builder.putCustom(currentFieldName, parser.mapStrings());
diff --git a/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java b/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java
index 6b0d9bfb91d75..e3004d9449383 100644
--- a/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java
+++ b/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java
@@ -128,11 +128,11 @@
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.function.BiConsumer;
 import java.util.function.BiFunction;
-import java.util.function.Function;
 import java.util.function.Predicate;
 import java.util.function.Supplier;
 import java.util.stream.Collectors;
 import java.util.stream.IntStream;
+import java.util.stream.Stream;
 
 import static java.util.stream.Collectors.toList;
 import static org.opensearch.cluster.metadata.IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING;
@@ -425,7 +425,6 @@ public ClusterState applyCreateIndexRequest(
         if (sourceMetadata != null) {
             // If source metadata was provided, it means we're recovering from an existing index,
             // in which case templates don't apply, so create the index from the source metadata
-            // QUESTION: Should we apply context in this case?
             return applyCreateIndexRequestWithExistingMetadata(currentState, request, silent, sourceMetadata, metadataTransformer);
         } else {
             // Hidden indices apply templates slightly differently (ignoring wildcard '*'
@@ -499,21 +498,30 @@ private ClusterState applyCreateIndexWithTemporaryService(
         final IndexMetadata sourceMetadata,
         final IndexMetadata temporaryIndexMeta,
         final List<Map<String, Object>> mappings,
-        final Function<IndexService, List<AliasMetadata>> aliasSupplier,
+        final BiFunction<IndexService, Map<String, AliasMetadata>, List<AliasMetadata>> aliasSupplier,
         final List<String> templatesApplied,
         final BiConsumer<Metadata.Builder, IndexMetadata> metadataTransformer
     ) throws Exception {
         // create the index here (on the master) to validate it can be created, as well as adding the mapping
         return indicesService.<ClusterState, Exception>withTempIndexService(temporaryIndexMeta, indexService -> {
-            try {
+            Settings.Builder tmpSettingsBuilder = Settings.builder().put(temporaryIndexMeta.getSettings());
+
+            List<Map<String, Object>> updatedMappings = new ArrayList<>();
+            updatedMappings.addAll(mappings);
+
+            Template contextTemplate = applyContext(request, currentState, updatedMappings, tmpSettingsBuilder);
 
-                updateIndexMappingsAndBuildSortOrder(indexService, request, mappings, sourceMetadata);
+            try {
+                updateIndexMappingsAndBuildSortOrder(indexService, request, updatedMappings, sourceMetadata);
             } catch (Exception e) {
                 logger.log(silent ? Level.DEBUG : Level.INFO, "failed on parsing mappings on index creation [{}]", request.index(), e);
                 throw e;
             }
 
-            final List<AliasMetadata> aliases = aliasSupplier.apply(indexService);
+            final List<AliasMetadata> aliases = aliasSupplier.apply(
+                indexService,
+                Optional.ofNullable(contextTemplate).map(Template::aliases).orElse(Map.of())
+            );
 
             final IndexMetadata indexMetadata;
             try {
@@ -521,7 +529,7 @@ private ClusterState applyCreateIndexWithTemporaryService(
                     request.index(),
                     aliases,
                     indexService.mapperService()::documentMapper,
-                    temporaryIndexMeta.getSettings(),
+                    tmpSettingsBuilder.build(),
                     temporaryIndexMeta.getRoutingNumShards(),
                     sourceMetadata,
                     temporaryIndexMeta.isSystem(),
@@ -547,6 +555,33 @@ private ClusterState applyCreateIndexWithTemporaryService(
         });
     }
 
+    private Template applyContext(
+        CreateIndexClusterStateUpdateRequest request,
+        ClusterState currentState,
+        List<Map<String, Object>> mappings,
+        Settings.Builder settingsBuilder
+    ) throws IOException {
+        if (request.context() != null) {
+            String contextTemplate = MetadataIndexTemplateService.findContextTemplate(currentState.metadata(), request.context());
+            ComponentTemplate componentTemplate = currentState.metadata().componentTemplates().get(contextTemplate);
+
+            if (componentTemplate.template().mappings() != null) {
+                // Mappings added at last (priority to mappings provided)
+                mappings.add(MapperService.parseMapping(xContentRegistry, componentTemplate.template().mappings().toString()));
+            }
+
+            if (componentTemplate.template().settings() != null) {
+                // Settings applied at last
+                settingsBuilder.put(componentTemplate.template().settings());
+            }
+
+            settingsBuilder.put(IndexSettings.INDEX_CONTEXT_CREATED_VERSION.getKey(), componentTemplate.version());
+
+            return componentTemplate.template();
+        }
+        return null;
+    }
+
     /**
      * Given a state and index settings calculated after applying templates, validate metadata for
      * the new index, returning an {@link IndexMetadata} for the new index.
@@ -625,25 +660,17 @@ private ClusterState applyCreateIndexRequestWithV1Templates(
             templates.stream().map(IndexTemplateMetadata::name).collect(Collectors.toList())
         );
 
-        final List<Map<String, Object>> mappings = List.of(
-            Collections.unmodifiableMap(
-                parseV1Mappings(
-                    request.mappings(),
-                    templates.stream().map(IndexTemplateMetadata::getMappings).collect(toList()),
-                    xContentRegistry
-                )
+        final Map<String, Object> mappings = Collections.unmodifiableMap(
+            parseV1Mappings(
+                request.mappings(),
+                templates.stream().map(IndexTemplateMetadata::getMappings).collect(toList()),
+                xContentRegistry
             )
         );
-        Optional<Template> context = applyContext(request, currentState, mappings);
-        Settings aggregateSettings = Settings.builder()
-            .put(MetadataIndexTemplateService.resolveSettings(templates))
-            .put(context.isPresent() ? context.get().settings() : Settings.EMPTY)
-            .build();
-
         final Settings aggregatedIndexSettings = aggregateIndexSettings(
             currentState,
             request,
-            aggregateSettings,
+            MetadataIndexTemplateService.resolveSettings(templates),
             null,
             settings,
             indexScopedSettings,
@@ -654,21 +681,17 @@ private ClusterState applyCreateIndexRequestWithV1Templates(
         int routingNumShards = getIndexNumberOfRoutingShards(aggregatedIndexSettings, null);
         IndexMetadata tmpImd = buildAndValidateTemporaryIndexMetadata(aggregatedIndexSettings, request, routingNumShards);
 
-        List<Map<String, AliasMetadata>> aliasMeta = new ArrayList<>();
-        aliasMeta.addAll(MetadataIndexTemplateService.resolveAliases(templates));
-        context.map(Template::aliases).ifPresent(aliasMeta::add);
-
         return applyCreateIndexWithTemporaryService(
             currentState,
             request,
             silent,
             null,
             tmpImd,
-            mappings,
-            indexService -> resolveAndValidateAliases(
+            Collections.singletonList(mappings),
+            (indexService, contextAlias) -> resolveAndValidateAliases(
                 request.index(),
                 request.aliases(),
-                aliasMeta,
+                Stream.concat(Stream.of(contextAlias), MetadataIndexTemplateService.resolveAliases(templates).stream()).collect(toList()),
                 currentState.metadata(),
                 aliasValidator,
                 // the context is only used for validation so it's fine to pass fake values for the
@@ -709,17 +732,10 @@ private ClusterState applyCreateIndexRequestWithV2Template(
             xContentRegistry,
             request.index()
         );
-
-        Optional<Template> context = applyContext(request, currentState, mappings);
-        Settings aggregateSettings = Settings.builder()
-            .put(MetadataIndexTemplateService.resolveSettings(currentState.metadata(), templateName))
-            .put(context.isPresent() ? context.get().settings() : Settings.EMPTY)
-            .build();
-
         final Settings aggregatedIndexSettings = aggregateIndexSettings(
             currentState,
             request,
-            aggregateSettings,
+            MetadataIndexTemplateService.resolveSettings(currentState.metadata(), templateName),
             null,
             settings,
             indexScopedSettings,
@@ -730,9 +746,6 @@ private ClusterState applyCreateIndexRequestWithV2Template(
         int routingNumShards = getIndexNumberOfRoutingShards(aggregatedIndexSettings, null);
         IndexMetadata tmpImd = buildAndValidateTemporaryIndexMetadata(aggregatedIndexSettings, request, routingNumShards);
 
-        List<Map<String, AliasMetadata>> aliasMeta = MetadataIndexTemplateService.resolveAliases(currentState.metadata(), templateName);
-        context.map(Template::aliases).ifPresent(aliasMeta::add);
-
         return applyCreateIndexWithTemporaryService(
             currentState,
             request,
@@ -740,10 +753,13 @@ private ClusterState applyCreateIndexRequestWithV2Template(
             null,
             tmpImd,
             mappings,
-            indexService -> resolveAndValidateAliases(
+            (indexService, contextAlias) -> resolveAndValidateAliases(
                 request.index(),
                 request.aliases(),
-                aliasMeta,
+                Stream.concat(
+                    Stream.of(contextAlias),
+                    MetadataIndexTemplateService.resolveAliases(currentState.metadata(), templateName).stream()
+                ).collect(toList()),
                 currentState.metadata(),
                 aliasValidator,
                 // the context is only used for validation so it's fine to pass fake values for the
@@ -756,20 +772,6 @@ private ClusterState applyCreateIndexRequestWithV2Template(
         );
     }
 
-    private Optional<Template> applyContext(
-        CreateIndexClusterStateUpdateRequest request,
-        ClusterState currentState,
-        List<Map<String, Object>> mappings
-    ) throws IOException {
-        if (request.context() != null) {
-            String contextTemplate = MetadataIndexTemplateService.findContextTemplate(currentState.metadata(), request.context());
-            ComponentTemplate componentTemplate = currentState.metadata().componentTemplates().get(contextTemplate);
-            mappings.add(MapperService.parseMapping(xContentRegistry, componentTemplate.template().mappings().toString()));
-            return Optional.of(componentTemplate.template());
-        }
-        return Optional.empty();
-    }
-
     public static List<Map<String, Object>> collectV2Mappings(
         final String requestMappings,
         final ClusterState currentState,
@@ -835,7 +837,7 @@ private ClusterState applyCreateIndexRequestWithExistingMetadata(
             sourceMetadata,
             tmpImd,
             Collections.singletonList(mappings),
-            indexService -> resolveAndValidateAliases(
+            (indexService, contextTemplate) -> resolveAndValidateAliases(
                 request.index(),
                 request.aliases(),
                 Collections.emptyList(),
diff --git a/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java b/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java
index 284eb43aa5509..69587f91fa15f 100644
--- a/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java
+++ b/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java
@@ -249,6 +249,8 @@ public final class IndexScopedSettings extends AbstractScopedSettings {
                 StarTreeIndexSettings.DEFAULT_DATE_INTERVALS,
                 StarTreeIndexSettings.STAR_TREE_MAX_DATE_INTERVALS_SETTING,
 
+                IndexSettings.INDEX_CONTEXT_CREATED_VERSION,
+
                 // validate that built-in similarities don't get redefined
                 Setting.groupSetting("index.similarity.", (s) -> {
                     Map<String, Settings> groups = s.getAsGroups();
diff --git a/server/src/main/java/org/opensearch/index/IndexSettings.java b/server/src/main/java/org/opensearch/index/IndexSettings.java
index 9cab68d646b6e..2a76c7d5b4354 100644
--- a/server/src/main/java/org/opensearch/index/IndexSettings.java
+++ b/server/src/main/java/org/opensearch/index/IndexSettings.java
@@ -734,6 +734,14 @@ public static IndexMergePolicy fromString(String text) {
         Property.IndexScope
     );
 
+    public static final Setting<Long> INDEX_CONTEXT_CREATED_VERSION = Setting.longSetting(
+        "index.context.created_version",
+        0,
+        0,
+        Property.PrivateIndex,
+        Property.IndexScope
+    );
+
     private final Index index;
     private final Version version;
     private final Logger logger;

From 37ac54bb179269cb62d563b1c7419aae2d61e929 Mon Sep 17 00:00:00 2001
From: Mohit Godwani <mgodwan@amazon.com>
Date: Wed, 21 Aug 2024 02:04:38 +0530
Subject: [PATCH 04/12] Fix node bootup, add context to response

Signed-off-by: Mohit Godwani <mgodwan@amazon.com>
---
 .../client/indices/GetIndexResponse.java      | 15 ++++++-
 .../client/indices/GetIndexResponseTests.java |  3 +-
 .../admin/indices/get/GetIndexRequest.java    |  5 ++-
 .../admin/indices/get/GetIndexResponse.java   | 40 +++++++++++++++++--
 .../indices/get/TransportGetIndexAction.java  | 20 +++++++++-
 .../opensearch/cluster/metadata/Context.java  |  2 +-
 .../cluster/metadata/IndexMetadata.java       |  6 +++
 .../metadata/MetadataCreateIndexService.java  | 12 +++++-
 8 files changed, 90 insertions(+), 13 deletions(-)

diff --git a/client/rest-high-level/src/main/java/org/opensearch/client/indices/GetIndexResponse.java b/client/rest-high-level/src/main/java/org/opensearch/client/indices/GetIndexResponse.java
index 6ec1c312c9ba9..36168980f0e16 100644
--- a/client/rest-high-level/src/main/java/org/opensearch/client/indices/GetIndexResponse.java
+++ b/client/rest-high-level/src/main/java/org/opensearch/client/indices/GetIndexResponse.java
@@ -34,6 +34,7 @@
 
 import org.apache.lucene.util.CollectionUtil;
 import org.opensearch.cluster.metadata.AliasMetadata;
+import org.opensearch.cluster.metadata.Context;
 import org.opensearch.cluster.metadata.MappingMetadata;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.core.xcontent.XContentParser;
@@ -167,6 +168,7 @@ private static IndexEntry parseIndexEntry(XContentParser parser) throws IOExcept
         Settings indexSettings = null;
         Settings indexDefaultSettings = null;
         String dataStream = null;
+        Context context = null;
         // We start at START_OBJECT since fromXContent ensures that
         while (parser.nextToken() != Token.END_OBJECT) {
             ensureExpectedToken(Token.FIELD_NAME, parser.currentToken(), parser);
@@ -185,6 +187,8 @@ private static IndexEntry parseIndexEntry(XContentParser parser) throws IOExcept
                     case "defaults":
                         indexDefaultSettings = Settings.fromXContent(parser);
                         break;
+                    case "context":
+                        context = Context.fromXContent(parser);
                     default:
                         parser.skipChildren();
                 }
@@ -197,7 +201,7 @@ private static IndexEntry parseIndexEntry(XContentParser parser) throws IOExcept
                 parser.skipChildren();
             }
         }
-        return new IndexEntry(indexAliases, indexMappings, indexSettings, indexDefaultSettings, dataStream);
+        return new IndexEntry(indexAliases, indexMappings, indexSettings, indexDefaultSettings, dataStream, context);
     }
 
     // This is just an internal container to make stuff easier for returning
@@ -207,19 +211,22 @@ private static class IndexEntry {
         Settings indexSettings = Settings.EMPTY;
         Settings indexDefaultSettings = Settings.EMPTY;
         String dataStream;
+        Context context;
 
         IndexEntry(
             List<AliasMetadata> indexAliases,
             MappingMetadata indexMappings,
             Settings indexSettings,
             Settings indexDefaultSettings,
-            String dataStream
+            String dataStream,
+            Context context
         ) {
             if (indexAliases != null) this.indexAliases = indexAliases;
             if (indexMappings != null) this.indexMappings = indexMappings;
             if (indexSettings != null) this.indexSettings = indexSettings;
             if (indexDefaultSettings != null) this.indexDefaultSettings = indexDefaultSettings;
             if (dataStream != null) this.dataStream = dataStream;
+            if (context != null) this.context = context;
         }
     }
 
@@ -229,6 +236,7 @@ public static GetIndexResponse fromXContent(XContentParser parser) throws IOExce
         Map<String, Settings> settings = new HashMap<>();
         Map<String, Settings> defaultSettings = new HashMap<>();
         Map<String, String> dataStreams = new HashMap<>();
+        Map<String, Context> contexts = new HashMap<>();
         List<String> indices = new ArrayList<>();
 
         if (parser.currentToken() == null) {
@@ -254,6 +262,9 @@ public static GetIndexResponse fromXContent(XContentParser parser) throws IOExce
                 if (indexEntry.dataStream != null) {
                     dataStreams.put(indexName, indexEntry.dataStream);
                 }
+                if (indexEntry.context != null) {
+                    contexts.put(indexName, indexEntry.context);
+                }
             } else if (parser.currentToken() == Token.START_ARRAY) {
                 parser.skipChildren();
             } else {
diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/indices/GetIndexResponseTests.java b/client/rest-high-level/src/test/java/org/opensearch/client/indices/GetIndexResponseTests.java
index a00f0487116dc..33a6e79abe238 100644
--- a/client/rest-high-level/src/test/java/org/opensearch/client/indices/GetIndexResponseTests.java
+++ b/client/rest-high-level/src/test/java/org/opensearch/client/indices/GetIndexResponseTests.java
@@ -97,7 +97,8 @@ protected org.opensearch.action.admin.indices.get.GetIndexResponse createServerT
             aliases,
             settings,
             defaultSettings,
-            dataStreams
+            dataStreams,
+            null
         );
     }
 
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/get/GetIndexRequest.java b/server/src/main/java/org/opensearch/action/admin/indices/get/GetIndexRequest.java
index 47c59791edf04..174e1cac1805e 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/get/GetIndexRequest.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/get/GetIndexRequest.java
@@ -57,7 +57,8 @@ public class GetIndexRequest extends ClusterInfoRequest<GetIndexRequest> {
     public enum Feature {
         ALIASES((byte) 0),
         MAPPINGS((byte) 1),
-        SETTINGS((byte) 2);
+        SETTINGS((byte) 2),
+        CONTEXT((byte) 3);
 
         private static final Feature[] FEATURES = new Feature[Feature.values().length];
 
@@ -86,7 +87,7 @@ public static Feature fromId(byte id) {
         }
     }
 
-    private static final Feature[] DEFAULT_FEATURES = new Feature[] { Feature.ALIASES, Feature.MAPPINGS, Feature.SETTINGS };
+    private static final Feature[] DEFAULT_FEATURES = new Feature[] { Feature.ALIASES, Feature.MAPPINGS, Feature.SETTINGS, Feature.CONTEXT };
     private Feature[] features = DEFAULT_FEATURES;
     private boolean humanReadable = false;
     private transient boolean includeDefaults = false;
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/get/GetIndexResponse.java b/server/src/main/java/org/opensearch/action/admin/indices/get/GetIndexResponse.java
index 5a237b8d3470f..6c6b43aa36546 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/get/GetIndexResponse.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/get/GetIndexResponse.java
@@ -34,6 +34,7 @@
 
 import org.opensearch.Version;
 import org.opensearch.cluster.metadata.AliasMetadata;
+import org.opensearch.cluster.metadata.Context;
 import org.opensearch.cluster.metadata.MappingMetadata;
 import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.settings.Settings;
@@ -68,6 +69,7 @@ public class GetIndexResponse extends ActionResponse implements ToXContentObject
     private Map<String, Settings> settings = Map.of();
     private Map<String, Settings> defaultSettings = Map.of();
     private Map<String, String> dataStreams = Map.of();
+    private Map<String, Context> contexts = Map.of();
     private final String[] indices;
 
     public GetIndexResponse(
@@ -76,7 +78,8 @@ public GetIndexResponse(
         final Map<String, List<AliasMetadata>> aliases,
         final Map<String, Settings> settings,
         final Map<String, Settings> defaultSettings,
-        final Map<String, String> dataStreams
+        final Map<String, String> dataStreams,
+        final Map<String, Context> contexts
     ) {
         this.indices = indices;
         // to have deterministic order
@@ -96,6 +99,9 @@ public GetIndexResponse(
         if (dataStreams != null) {
             this.dataStreams = Collections.unmodifiableMap(dataStreams);
         }
+        if (contexts != null) {
+            this.contexts = Collections.unmodifiableMap(contexts);
+        }
     }
 
     GetIndexResponse(StreamInput in) throws IOException {
@@ -160,6 +166,15 @@ public GetIndexResponse(
             dataStreamsMapBuilder.put(in.readString(), in.readOptionalString());
         }
         dataStreams = Collections.unmodifiableMap(dataStreamsMapBuilder);
+
+        if (in.getVersion().onOrAfter(Version.V_3_0_0)) {
+            final Map<String, Context> contextMapBuilder = new HashMap<>();
+            int contextSize = in.readVInt();
+            for (int i = 0; i < contextSize; i++) {
+                contextMapBuilder.put(in.readString(), in.readOptionalWriteable(Context::new));
+            }
+            contexts = Collections.unmodifiableMap(contextMapBuilder);
+        }
     }
 
     public String[] indices() {
@@ -214,6 +229,11 @@ public Map<String, Settings> getSettings() {
         return settings();
     }
 
+
+    public Map<String, Context> contexts() {
+        return contexts;
+    }
+
     /**
      * Returns the string value for the specified index and setting.  If the includeDefaults flag was not set or set to
      * false on the {@link GetIndexRequest}, this method will only return a value where the setting was explicitly set
@@ -277,6 +297,14 @@ public void writeTo(StreamOutput out) throws IOException {
             out.writeString(indexEntry.getKey());
             out.writeOptionalString(indexEntry.getValue());
         }
+
+        if (out.getVersion().before(Version.V_3_0_0)) {
+            out.writeVInt(contexts.size());
+            for (final Map.Entry<String, Context> indexEntry : contexts.entrySet()) {
+                out.writeString(indexEntry.getKey());
+                out.writeOptionalWriteable(indexEntry.getValue());
+            }
+        }
     }
 
     @Override
@@ -320,6 +348,11 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
                     if (dataStream != null) {
                         builder.field("data_stream", dataStream);
                     }
+
+                    Context context = contexts.get(index);
+                    if (context != null) {
+                        builder.field("context", context);
+                    }
                 }
                 builder.endObject();
             }
@@ -343,11 +376,12 @@ public boolean equals(Object o) {
             && Objects.equals(mappings, that.mappings)
             && Objects.equals(settings, that.settings)
             && Objects.equals(defaultSettings, that.defaultSettings)
-            && Objects.equals(dataStreams, that.dataStreams);
+            && Objects.equals(dataStreams, that.dataStreams)
+            && Objects.equals(contexts, that.contexts);
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(Arrays.hashCode(indices), aliases, mappings, settings, defaultSettings, dataStreams);
+        return Objects.hash(Arrays.hashCode(indices), aliases, mappings, settings, defaultSettings, dataStreams, contexts);
     }
 }
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/get/TransportGetIndexAction.java b/server/src/main/java/org/opensearch/action/admin/indices/get/TransportGetIndexAction.java
index 755119401c6b5..130430008a6a2 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/get/TransportGetIndexAction.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/get/TransportGetIndexAction.java
@@ -33,9 +33,11 @@
 package org.opensearch.action.admin.indices.get;
 
 import org.opensearch.action.support.ActionFilters;
+import org.opensearch.action.support.TransportAction;
 import org.opensearch.action.support.clustermanager.info.TransportClusterInfoAction;
 import org.opensearch.cluster.ClusterState;
 import org.opensearch.cluster.metadata.AliasMetadata;
+import org.opensearch.cluster.metadata.Context;
 import org.opensearch.cluster.metadata.IndexMetadata;
 import org.opensearch.cluster.metadata.IndexNameExpressionResolver;
 import org.opensearch.cluster.metadata.MappingMetadata;
@@ -110,6 +112,7 @@ protected void doClusterManagerOperation(
         Map<String, List<AliasMetadata>> aliasesResult = Map.of();
         Map<String, Settings> settings = Map.of();
         Map<String, Settings> defaultSettings = Map.of();
+        Map<String, Context> contexts = Map.of();
         final Map<String, String> dataStreams = new HashMap<>(
             StreamSupport.stream(Spliterators.spliterator(state.metadata().findDataStreams(concreteIndices).entrySet(), 0), false)
                 .collect(Collectors.toMap(k -> k.getKey(), v -> v.getValue().getName()))
@@ -118,6 +121,7 @@ protected void doClusterManagerOperation(
         boolean doneAliases = false;
         boolean doneMappings = false;
         boolean doneSettings = false;
+        boolean doneContext = false;
         for (GetIndexRequest.Feature feature : features) {
             switch (feature) {
                 case MAPPINGS:
@@ -159,11 +163,23 @@ protected void doClusterManagerOperation(
                         doneSettings = true;
                     }
                     break;
-
+                case CONTEXT:
+                    if (!doneContext) {
+                        final Map<String, Context> contextBuilder = new HashMap<>();
+                        for (String index : concreteIndices) {
+                            Context indexContext = state.metadata().index(index).context();
+                            if (indexContext != null) {
+                                contextBuilder.put(index, indexContext);
+                            }
+                        }
+                        contexts = contextBuilder;
+                        doneContext = true;
+                    }
+                    break;
                 default:
                     throw new IllegalStateException("feature [" + feature + "] is not valid");
             }
         }
-        listener.onResponse(new GetIndexResponse(concreteIndices, mappingsResult, aliasesResult, settings, defaultSettings, dataStreams));
+        listener.onResponse(new GetIndexResponse(concreteIndices, mappingsResult, aliasesResult, settings, defaultSettings, dataStreams, contexts));
     }
 }
diff --git a/server/src/main/java/org/opensearch/cluster/metadata/Context.java b/server/src/main/java/org/opensearch/cluster/metadata/Context.java
index 3e834369020d8..3b0cfb23ae6b4 100644
--- a/server/src/main/java/org/opensearch/cluster/metadata/Context.java
+++ b/server/src/main/java/org/opensearch/cluster/metadata/Context.java
@@ -104,7 +104,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         builder.startObject();
         builder.field(NAME.getPreferredName(), this.name);
         builder.field(VERSION.getPreferredName(), this.version);
-        if (params != null) {
+        if (this.params != null) {
             builder.field(PARAMS.getPreferredName(), this.params);
         }
         builder.endObject();
diff --git a/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java b/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java
index dc85eb28e9f0f..da4dafc257afe 100644
--- a/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java
+++ b/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java
@@ -1107,6 +1107,8 @@ private static class IndexMetadataDiff implements Diff<IndexMetadata> {
             isSystem = in.readBoolean();
             if (in.getVersion().onOrAfter(Version.V_3_0_0)) {
                 context = in.readOptionalWriteable(Context::new);
+            } else {
+                context = null;
             }
         }
 
@@ -1242,6 +1244,10 @@ public boolean isSystem() {
         return isSystem;
     }
 
+    public Context context() {
+        return context;
+    }
+
     public boolean isRemoteSnapshot() {
         return isRemoteSnapshot;
     }
diff --git a/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java b/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java
index e3004d9449383..c50b180996e8d 100644
--- a/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java
+++ b/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java
@@ -533,7 +533,8 @@ private ClusterState applyCreateIndexWithTemporaryService(
                     temporaryIndexMeta.getRoutingNumShards(),
                     sourceMetadata,
                     temporaryIndexMeta.isSystem(),
-                    temporaryIndexMeta.getCustomData()
+                    temporaryIndexMeta.getCustomData(),
+                    temporaryIndexMeta.context()
                 );
             } catch (Exception e) {
                 logger.info("failed to build index metadata [{}]", request.index());
@@ -608,6 +609,10 @@ IndexMetadata buildAndValidateTemporaryIndexMetadata(
         tmpImdBuilder.system(isSystem);
         addRemoteStoreCustomMetadata(tmpImdBuilder, true);
 
+        if (request.context() != null) {
+            tmpImdBuilder.context(request.context());
+        }
+
         // Set up everything, now locally create the index to see that things are ok, and apply
         IndexMetadata tempMetadata = tmpImdBuilder.build();
         validateActiveShardCount(request.waitForActiveShards(), tempMetadata);
@@ -1280,7 +1285,8 @@ static IndexMetadata buildIndexMetadata(
         int routingNumShards,
         @Nullable IndexMetadata sourceMetadata,
         boolean isSystem,
-        Map<String, DiffableStringMap> customData
+        Map<String, DiffableStringMap> customData,
+        Context context
     ) {
         IndexMetadata.Builder indexMetadataBuilder = createIndexMetadataBuilder(indexName, sourceMetadata, indexSettings, routingNumShards);
         indexMetadataBuilder.system(isSystem);
@@ -1305,6 +1311,8 @@ static IndexMetadata buildIndexMetadata(
             indexMetadataBuilder.putCustom(entry.getKey(), entry.getValue());
         }
 
+        indexMetadataBuilder.context(context);
+
         indexMetadataBuilder.state(IndexMetadata.State.OPEN);
         return indexMetadataBuilder.build();
     }

From 3e0f7ec41346200e81c50c1cca59fc04766caffc Mon Sep 17 00:00:00 2001
From: Mohit Godwani <mgodwan@amazon.com>
Date: Wed, 21 Aug 2024 02:14:53 +0530
Subject: [PATCH 05/12] Fix node bootup, add context to response

Signed-off-by: Mohit Godwani <mgodwan@amazon.com>
---
 .../opensearch/client/indices/GetIndexResponse.java    | 10 ++++++++--
 .../action/admin/indices/get/GetIndexRequest.java      |  6 +++++-
 .../action/admin/indices/get/GetIndexResponse.java     |  1 -
 .../admin/indices/get/TransportGetIndexAction.java     |  5 +++--
 4 files changed, 16 insertions(+), 6 deletions(-)

diff --git a/client/rest-high-level/src/main/java/org/opensearch/client/indices/GetIndexResponse.java b/client/rest-high-level/src/main/java/org/opensearch/client/indices/GetIndexResponse.java
index 36168980f0e16..227322c58cc47 100644
--- a/client/rest-high-level/src/main/java/org/opensearch/client/indices/GetIndexResponse.java
+++ b/client/rest-high-level/src/main/java/org/opensearch/client/indices/GetIndexResponse.java
@@ -62,6 +62,7 @@ public class GetIndexResponse {
     private Map<String, Settings> settings;
     private Map<String, Settings> defaultSettings;
     private Map<String, String> dataStreams;
+    private Map<String, Context> contexts;
     private String[] indices;
 
     GetIndexResponse(
@@ -70,7 +71,8 @@ public class GetIndexResponse {
         Map<String, List<AliasMetadata>> aliases,
         Map<String, Settings> settings,
         Map<String, Settings> defaultSettings,
-        Map<String, String> dataStreams
+        Map<String, String> dataStreams,
+        Map<String, Context> contexts
     ) {
         this.indices = indices;
         // to have deterministic order
@@ -90,6 +92,9 @@ public class GetIndexResponse {
         if (dataStreams != null) {
             this.dataStreams = dataStreams;
         }
+        if (contexts != null) {
+            this.contexts = contexts;
+        }
     }
 
     public String[] getIndices() {
@@ -189,6 +194,7 @@ private static IndexEntry parseIndexEntry(XContentParser parser) throws IOExcept
                         break;
                     case "context":
                         context = Context.fromXContent(parser);
+                        break;
                     default:
                         parser.skipChildren();
                 }
@@ -271,6 +277,6 @@ public static GetIndexResponse fromXContent(XContentParser parser) throws IOExce
                 parser.nextToken();
             }
         }
-        return new GetIndexResponse(indices.toArray(new String[0]), mappings, aliases, settings, defaultSettings, dataStreams);
+        return new GetIndexResponse(indices.toArray(new String[0]), mappings, aliases, settings, defaultSettings, dataStreams, contexts);
     }
 }
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/get/GetIndexRequest.java b/server/src/main/java/org/opensearch/action/admin/indices/get/GetIndexRequest.java
index 174e1cac1805e..601b53f88baa3 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/get/GetIndexRequest.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/get/GetIndexRequest.java
@@ -87,7 +87,11 @@ public static Feature fromId(byte id) {
         }
     }
 
-    private static final Feature[] DEFAULT_FEATURES = new Feature[] { Feature.ALIASES, Feature.MAPPINGS, Feature.SETTINGS, Feature.CONTEXT };
+    private static final Feature[] DEFAULT_FEATURES = new Feature[] {
+        Feature.ALIASES,
+        Feature.MAPPINGS,
+        Feature.SETTINGS,
+        Feature.CONTEXT };
     private Feature[] features = DEFAULT_FEATURES;
     private boolean humanReadable = false;
     private transient boolean includeDefaults = false;
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/get/GetIndexResponse.java b/server/src/main/java/org/opensearch/action/admin/indices/get/GetIndexResponse.java
index 6c6b43aa36546..4c7f50407476b 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/get/GetIndexResponse.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/get/GetIndexResponse.java
@@ -229,7 +229,6 @@ public Map<String, Settings> getSettings() {
         return settings();
     }
 
-
     public Map<String, Context> contexts() {
         return contexts;
     }
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/get/TransportGetIndexAction.java b/server/src/main/java/org/opensearch/action/admin/indices/get/TransportGetIndexAction.java
index 130430008a6a2..c6f4a8cd49ae9 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/get/TransportGetIndexAction.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/get/TransportGetIndexAction.java
@@ -33,7 +33,6 @@
 package org.opensearch.action.admin.indices.get;
 
 import org.opensearch.action.support.ActionFilters;
-import org.opensearch.action.support.TransportAction;
 import org.opensearch.action.support.clustermanager.info.TransportClusterInfoAction;
 import org.opensearch.cluster.ClusterState;
 import org.opensearch.cluster.metadata.AliasMetadata;
@@ -180,6 +179,8 @@ protected void doClusterManagerOperation(
                     throw new IllegalStateException("feature [" + feature + "] is not valid");
             }
         }
-        listener.onResponse(new GetIndexResponse(concreteIndices, mappingsResult, aliasesResult, settings, defaultSettings, dataStreams, contexts));
+        listener.onResponse(
+            new GetIndexResponse(concreteIndices, mappingsResult, aliasesResult, settings, defaultSettings, dataStreams, contexts)
+        );
     }
 }

From 1080866d2c0256655e54accac247e10cb480a2c6 Mon Sep 17 00:00:00 2001
From: Mohit Godwani <mgodwan@amazon.com>
Date: Wed, 21 Aug 2024 11:55:30 +0530
Subject: [PATCH 06/12] Add settings validation

Signed-off-by: Mohit Godwani <mgodwan@amazon.com>
---
 .../metadata/MetadataCreateIndexService.java  | 26 ++++++++++++++++---
 .../MetadataIndexTemplateService.java         | 17 +++++++-----
 .../MetadataUpdateSettingsService.java        | 18 +++++++++++++
 .../common/settings/IndexScopedSettings.java  |  1 +
 .../org/opensearch/index/IndexSettings.java   |  9 +++++++
 5 files changed, 62 insertions(+), 9 deletions(-)

diff --git a/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java b/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java
index c50b180996e8d..5299602986793 100644
--- a/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java
+++ b/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java
@@ -77,6 +77,7 @@
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.unit.TimeValue;
 import org.opensearch.common.util.FeatureFlags;
+import org.opensearch.common.util.set.Sets;
 import org.opensearch.common.xcontent.XContentHelper;
 import org.opensearch.core.action.ActionListener;
 import org.opensearch.core.common.Strings;
@@ -148,7 +149,7 @@
 import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY;
 import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REPLICATION_TYPE;
 import static org.opensearch.cluster.metadata.Metadata.DEFAULT_REPLICA_COUNT_SETTING;
-import static org.opensearch.cluster.metadata.MetadataIndexTemplateService.findContextTemplate;
+import static org.opensearch.cluster.metadata.MetadataIndexTemplateService.findContextTemplateName;
 import static org.opensearch.index.IndexModule.INDEX_STORE_TYPE_SETTING;
 import static org.opensearch.indices.IndicesService.CLUSTER_REPLICATION_TYPE_SETTING;
 import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.isRemoteDataAttributePresent;
@@ -563,7 +564,7 @@ private Template applyContext(
         Settings.Builder settingsBuilder
     ) throws IOException {
         if (request.context() != null) {
-            String contextTemplate = MetadataIndexTemplateService.findContextTemplate(currentState.metadata(), request.context());
+            String contextTemplate = MetadataIndexTemplateService.findContextTemplateName(currentState.metadata(), request.context());
             ComponentTemplate componentTemplate = currentState.metadata().componentTemplates().get(contextTemplate);
 
             if (componentTemplate.template().mappings() != null) {
@@ -572,17 +573,36 @@ private Template applyContext(
             }
 
             if (componentTemplate.template().settings() != null) {
+                validateOverlap(settingsBuilder.keys(), componentTemplate.template().settings(), request.index()).ifPresent(message -> {
+                    ValidationException validationException = new ValidationException();
+                    validationException.addValidationError(message);
+                    throw validationException;
+                });
                 // Settings applied at last
                 settingsBuilder.put(componentTemplate.template().settings());
             }
 
             settingsBuilder.put(IndexSettings.INDEX_CONTEXT_CREATED_VERSION.getKey(), componentTemplate.version());
+            settingsBuilder.put(IndexSettings.INDEX_CONTEXT_CURRENT_VERSION.getKey(), componentTemplate.version());
 
             return componentTemplate.template();
         }
         return null;
     }
 
+    static Optional<String> validateOverlap(Set<String> requestSettings, Settings contextTemplateSettings, String indexName) {
+        if (requestSettings.stream().anyMatch(contextTemplateSettings::hasValue)) {
+            return Optional.of(
+                "Cannot apply context template as user provide settings have overlap with the included context template."
+                    + "Please remove the settings ["
+                    + Sets.intersection(requestSettings, contextTemplateSettings.keySet())
+                    + "] to continue using the context for index: "
+                    + indexName
+            );
+        }
+        return Optional.empty();
+    }
+
     /**
      * Given a state and index settings calculated after applying templates, validate metadata for
      * the new index, returning an {@link IndexMetadata} for the new index.
@@ -1760,7 +1780,7 @@ static void validateContext(CreateIndexClusterStateUpdateRequest request, Cluste
             );
         }
 
-        if (request.context() != null && findContextTemplate(clusterState.metadata(), request.context()) == null) {
+        if (request.context() != null && findContextTemplateName(clusterState.metadata(), request.context()) == null) {
             throw new InvalidIndexContextException(
                 request.context().name(),
                 request.index(),
diff --git a/server/src/main/java/org/opensearch/cluster/metadata/MetadataIndexTemplateService.java b/server/src/main/java/org/opensearch/cluster/metadata/MetadataIndexTemplateService.java
index 0d4ef8a7f6c74..43d1551831950 100644
--- a/server/src/main/java/org/opensearch/cluster/metadata/MetadataIndexTemplateService.java
+++ b/server/src/main/java/org/opensearch/cluster/metadata/MetadataIndexTemplateService.java
@@ -452,7 +452,7 @@ static void validateNotInUse(Metadata metadata, String templateNameOrWildcard) {
         final Set<String> componentsBeingUsed = new HashSet<>();
         final List<String> templatesStillUsing = metadata.templatesV2().entrySet().stream().filter(e -> {
             Set<String> referredComponentTemplates = new HashSet<>(e.getValue().composedOf());
-            String systemTemplateUsed = findContextTemplate(metadata, e.getValue().context());
+            String systemTemplateUsed = findContextTemplateName(metadata, e.getValue().context());
             if (systemTemplateUsed != null) {
                 referredComponentTemplates.add(systemTemplateUsed);
             }
@@ -568,7 +568,7 @@ public static void validateV2TemplateRequest(
             );
         }
 
-        if (template.context() != null && findContextTemplate(metadata, template.context()) == null) {
+        if (template.context() != null && findContextTemplateName(metadata, template.context()) == null) {
             throw new InvalidIndexTemplateException(
                 name,
                 "index template [" + name + "] specifies a context which is not loaded on the cluster."
@@ -585,7 +585,12 @@ private void validateComponentTemplateRequest(ComponentTemplate componentTemplat
         }
     }
 
-    static String findContextTemplate(Metadata metadata, Context context) {
+    static ComponentTemplate findComponentTemplate(Metadata metadata, Context context) {
+        String contextTemplateName = findContextTemplateName(metadata, context);
+        return metadata.componentTemplates().getOrDefault(contextTemplateName, null);
+    }
+
+    static String findContextTemplateName(Metadata metadata, Context context) {
         if (context == null) {
             return null;
         }
@@ -1246,7 +1251,7 @@ public static List<CompressedXContent> collectMappings(final ClusterState state,
 
         // Now use context mappings which take the highest precedence
         Optional.ofNullable(template.context())
-            .map(ctx -> findContextTemplate(state.metadata(), ctx))
+            .map(ctx -> findContextTemplateName(state.metadata(), ctx))
             .map(name -> state.metadata().componentTemplates().get(name))
             .map(ComponentTemplate::template)
             .map(Template::mappings)
@@ -1317,7 +1322,7 @@ private static Settings resolveSettings(Metadata metadata, ComposableIndexTempla
         Optional.ofNullable(template.template()).map(Template::settings).ifPresent(templateSettings::put);
 
         // Add the template referred by context since it will take the highest precedence.
-        final String systemTemplate = findContextTemplate(metadata, template.context());
+        final String systemTemplate = findContextTemplateName(metadata, template.context());
         final ComponentTemplate componentTemplate = metadata.componentTemplates().get(systemTemplate);
         Optional.ofNullable(componentTemplate).map(ComponentTemplate::template).map(Template::settings).ifPresent(templateSettings::put);
 
@@ -1367,7 +1372,7 @@ public static List<Map<String, AliasMetadata>> resolveAliases(final Metadata met
 
         // Now use context referenced template's aliases which take the highest precedence
         if (template.context() != null) {
-            final String systemTemplate = findContextTemplate(metadata, template.context());
+            final String systemTemplate = findContextTemplateName(metadata, template.context());
             final ComponentTemplate componentTemplate = metadata.componentTemplates().get(systemTemplate);
             Optional.ofNullable(componentTemplate.template()).map(Template::aliases).ifPresent(aliases::add);
         }
diff --git a/server/src/main/java/org/opensearch/cluster/metadata/MetadataUpdateSettingsService.java b/server/src/main/java/org/opensearch/cluster/metadata/MetadataUpdateSettingsService.java
index 7d4c3512ed757..5eaae5ce60c76 100644
--- a/server/src/main/java/org/opensearch/cluster/metadata/MetadataUpdateSettingsService.java
+++ b/server/src/main/java/org/opensearch/cluster/metadata/MetadataUpdateSettingsService.java
@@ -65,16 +65,20 @@
 import org.opensearch.threadpool.ThreadPool;
 
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.HashSet;
+import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
 
 import static org.opensearch.action.support.ContextPreservingActionListener.wrapPreservingContext;
+import static org.opensearch.cluster.metadata.MetadataCreateIndexService.validateOverlap;
 import static org.opensearch.cluster.metadata.MetadataCreateIndexService.validateRefreshIntervalSettings;
 import static org.opensearch.cluster.metadata.MetadataCreateIndexService.validateTranslogDurabilitySettings;
+import static org.opensearch.cluster.metadata.MetadataIndexTemplateService.findComponentTemplate;
 import static org.opensearch.common.settings.AbstractScopedSettings.ARCHIVED_SETTINGS_PREFIX;
 import static org.opensearch.index.IndexSettings.same;
 
@@ -196,6 +200,7 @@ public ClusterState execute(ClusterState currentState) {
                     Set<Index> openIndices = new HashSet<>();
                     Set<Index> closeIndices = new HashSet<>();
                     final String[] actualIndices = new String[request.indices().length];
+                    final List<String> validationErrors = new ArrayList<>();
                     for (int i = 0; i < request.indices().length; i++) {
                         Index index = request.indices()[i];
                         actualIndices[i] = index.getName();
@@ -205,6 +210,19 @@ public ClusterState execute(ClusterState currentState) {
                         } else {
                             closeIndices.add(index);
                         }
+                        if (metadata.context() != null) {
+                            validateOverlap(
+                                normalizedSettings.keySet(),
+                                findComponentTemplate(currentState.metadata(), metadata.context()).template().settings(),
+                                index.getName()
+                            ).ifPresent(validationErrors::add);
+                        }
+                    }
+
+                    if (validationErrors.size() > 0) {
+                        ValidationException exception = new ValidationException();
+                        exception.addValidationErrors(validationErrors);
+                        throw exception;
                     }
 
                     if (!skippedSettings.isEmpty() && !openIndices.isEmpty()) {
diff --git a/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java b/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java
index 69587f91fa15f..3edf56625beab 100644
--- a/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java
+++ b/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java
@@ -250,6 +250,7 @@ public final class IndexScopedSettings extends AbstractScopedSettings {
                 StarTreeIndexSettings.STAR_TREE_MAX_DATE_INTERVALS_SETTING,
 
                 IndexSettings.INDEX_CONTEXT_CREATED_VERSION,
+                IndexSettings.INDEX_CONTEXT_CURRENT_VERSION,
 
                 // validate that built-in similarities don't get redefined
                 Setting.groupSetting("index.similarity.", (s) -> {
diff --git a/server/src/main/java/org/opensearch/index/IndexSettings.java b/server/src/main/java/org/opensearch/index/IndexSettings.java
index 2a76c7d5b4354..4c104e154ed71 100644
--- a/server/src/main/java/org/opensearch/index/IndexSettings.java
+++ b/server/src/main/java/org/opensearch/index/IndexSettings.java
@@ -742,6 +742,15 @@ public static IndexMergePolicy fromString(String text) {
         Property.IndexScope
     );
 
+    public static final Setting<Long> INDEX_CONTEXT_CURRENT_VERSION = Setting.longSetting(
+        "index.context.current_version",
+        0,
+        0,
+        Property.PrivateIndex,
+        Property.Dynamic,
+        Property.IndexScope
+    );
+
     private final Index index;
     private final Version version;
     private final Logger logger;

From 88246c999dedd58321163ddb6ee382597e4cdd7e Mon Sep 17 00:00:00 2001
From: Mohit Godwani <mgodwan@amazon.com>
Date: Fri, 23 Aug 2024 21:12:16 +0530
Subject: [PATCH 07/12] Build tests

Signed-off-by: Mohit Godwani <mgodwan@amazon.com>
---
 .../action/admin/indices/get/GetIndexResponse.java         | 2 +-
 .../java/org/opensearch/ExceptionSerializationTests.java   | 2 ++
 .../action/admin/indices/get/GetIndexResponseTests.java    | 7 ++++++-
 .../cluster/metadata/MetadataCreateIndexServiceTests.java  | 3 ++-
 4 files changed, 11 insertions(+), 3 deletions(-)

diff --git a/server/src/main/java/org/opensearch/action/admin/indices/get/GetIndexResponse.java b/server/src/main/java/org/opensearch/action/admin/indices/get/GetIndexResponse.java
index 4c7f50407476b..cb4f466df40c8 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/get/GetIndexResponse.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/get/GetIndexResponse.java
@@ -297,7 +297,7 @@ public void writeTo(StreamOutput out) throws IOException {
             out.writeOptionalString(indexEntry.getValue());
         }
 
-        if (out.getVersion().before(Version.V_3_0_0)) {
+        if (out.getVersion().onOrAfter(Version.V_3_0_0)) {
             out.writeVInt(contexts.size());
             for (final Map.Entry<String, Context> indexEntry : contexts.entrySet()) {
                 out.writeString(indexEntry.getKey());
diff --git a/server/src/test/java/org/opensearch/ExceptionSerializationTests.java b/server/src/test/java/org/opensearch/ExceptionSerializationTests.java
index d7026159d9ec0..2e4a2d7bdd59c 100644
--- a/server/src/test/java/org/opensearch/ExceptionSerializationTests.java
+++ b/server/src/test/java/org/opensearch/ExceptionSerializationTests.java
@@ -99,6 +99,7 @@
 import org.opensearch.index.shard.PrimaryShardClosedException;
 import org.opensearch.index.shard.ShardNotInPrimaryModeException;
 import org.opensearch.indices.IndexTemplateMissingException;
+import org.opensearch.indices.InvalidIndexContextException;
 import org.opensearch.indices.InvalidIndexTemplateException;
 import org.opensearch.indices.recovery.PeerRecoveryNotFound;
 import org.opensearch.indices.recovery.RecoverFilesRecoveryException;
@@ -896,6 +897,7 @@ public void testIds() {
         ids.put(171, CryptoRegistryException.class);
         ids.put(172, ViewNotFoundException.class);
         ids.put(173, ViewAlreadyExistsException.class);
+        ids.put(174, InvalidIndexContextException.class);
         ids.put(10001, IndexCreateBlockException.class);
 
         Map<Class<? extends OpenSearchException>, Integer> reverse = new HashMap<>();
diff --git a/server/src/test/java/org/opensearch/action/admin/indices/get/GetIndexResponseTests.java b/server/src/test/java/org/opensearch/action/admin/indices/get/GetIndexResponseTests.java
index 89d47328a08ed..c9b7858ed24ca 100644
--- a/server/src/test/java/org/opensearch/action/admin/indices/get/GetIndexResponseTests.java
+++ b/server/src/test/java/org/opensearch/action/admin/indices/get/GetIndexResponseTests.java
@@ -36,6 +36,7 @@
 import org.opensearch.action.admin.indices.alias.get.GetAliasesResponseTests;
 import org.opensearch.action.admin.indices.mapping.get.GetMappingsResponseTests;
 import org.opensearch.cluster.metadata.AliasMetadata;
+import org.opensearch.cluster.metadata.Context;
 import org.opensearch.cluster.metadata.MappingMetadata;
 import org.opensearch.common.settings.IndexScopedSettings;
 import org.opensearch.common.settings.Settings;
@@ -66,6 +67,7 @@ protected GetIndexResponse createTestInstance() {
         Map<String, Settings> settings = new HashMap<>();
         Map<String, Settings> defaultSettings = new HashMap<>();
         Map<String, String> dataStreams = new HashMap<>();
+        Map<String, Context> contexts = new HashMap<>();
         IndexScopedSettings indexScopedSettings = IndexScopedSettings.DEFAULT_SCOPED_SETTINGS;
         boolean includeDefaults = randomBoolean();
         for (String index : indices) {
@@ -90,7 +92,10 @@ protected GetIndexResponse createTestInstance() {
             if (randomBoolean()) {
                 dataStreams.put(index, randomAlphaOfLength(5).toLowerCase(Locale.ROOT));
             }
+            if (randomBoolean()) {
+                contexts.put(index, new Context(randomAlphaOfLength(5).toLowerCase(Locale.ROOT)));
+            }
         }
-        return new GetIndexResponse(indices, mappings, aliases, settings, defaultSettings, dataStreams);
+        return new GetIndexResponse(indices, mappings, aliases, settings, defaultSettings, dataStreams, contexts);
     }
 }
diff --git a/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java b/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java
index 86ca8b3ad6319..81f0ede543746 100644
--- a/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java
+++ b/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java
@@ -1704,7 +1704,8 @@ public void testBuildIndexMetadata() {
             4,
             sourceIndexMetadata,
             false,
-            new HashMap<>()
+            new HashMap<>(),
+            null
         );
 
         assertThat(indexMetadata.getAliases().size(), is(1));

From 2e0ab7c5fdc67824840af4840893de0ce498995d Mon Sep 17 00:00:00 2001
From: Mohit Godwani <mgodwan@amazon.com>
Date: Sun, 25 Aug 2024 22:59:49 +0530
Subject: [PATCH 08/12] Add more tests

Signed-off-by: Mohit Godwani <mgodwan@amazon.com>
---
 .../client/indices/GetIndexResponse.java      |  4 +
 .../client/indices/GetIndexResponseTests.java |  7 ++
 .../admin/indices/create/CreateIndexIT.java   | 60 +++++++++++++++
 .../create/CreateIndexRequestBuilder.java     | 21 ++++++
 .../metadata/MetadataCreateIndexService.java  | 12 +--
 .../MetadataIndexTemplateService.java         |  6 +-
 .../indices/get/GetIndexActionTests.java      | 23 +++++-
 .../MetadataCreateIndexServiceTests.java      | 74 +++++++++++++++++++
 .../ClusterStateCreationUtils.java            | 39 ++++++++++
 9 files changed, 235 insertions(+), 11 deletions(-)

diff --git a/client/rest-high-level/src/main/java/org/opensearch/client/indices/GetIndexResponse.java b/client/rest-high-level/src/main/java/org/opensearch/client/indices/GetIndexResponse.java
index 227322c58cc47..1ceaeab6c0064 100644
--- a/client/rest-high-level/src/main/java/org/opensearch/client/indices/GetIndexResponse.java
+++ b/client/rest-high-level/src/main/java/org/opensearch/client/indices/GetIndexResponse.java
@@ -129,6 +129,10 @@ public Map<String, String> getDataStreams() {
         return dataStreams;
     }
 
+    public Map<String, Context> contexts() {
+        return contexts;
+    }
+
     /**
      * Returns the string value for the specified index and setting. If the includeDefaults flag was not set or set to
      * false on the {@link GetIndexRequest}, this method will only return a value where the setting was explicitly set
diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/indices/GetIndexResponseTests.java b/client/rest-high-level/src/test/java/org/opensearch/client/indices/GetIndexResponseTests.java
index 33a6e79abe238..fa313e68f8a35 100644
--- a/client/rest-high-level/src/test/java/org/opensearch/client/indices/GetIndexResponseTests.java
+++ b/client/rest-high-level/src/test/java/org/opensearch/client/indices/GetIndexResponseTests.java
@@ -36,6 +36,7 @@
 import org.opensearch.client.AbstractResponseTestCase;
 import org.opensearch.client.GetAliasesResponseTests;
 import org.opensearch.cluster.metadata.AliasMetadata;
+import org.opensearch.cluster.metadata.Context;
 import org.opensearch.cluster.metadata.MappingMetadata;
 import org.opensearch.common.settings.IndexScopedSettings;
 import org.opensearch.common.settings.Settings;
@@ -66,6 +67,7 @@ protected org.opensearch.action.admin.indices.get.GetIndexResponse createServerT
         final Map<String, Settings> settings = new HashMap<>();
         final Map<String, Settings> defaultSettings = new HashMap<>();
         final Map<String, String> dataStreams = new HashMap<>();
+        final Map<String, Context> contexts = new HashMap<>();
         IndexScopedSettings indexScopedSettings = IndexScopedSettings.DEFAULT_SCOPED_SETTINGS;
         boolean includeDefaults = randomBoolean();
         for (String index : indices) {
@@ -90,6 +92,10 @@ protected org.opensearch.action.admin.indices.get.GetIndexResponse createServerT
             if (randomBoolean()) {
                 dataStreams.put(index, randomAlphaOfLength(5).toLowerCase(Locale.ROOT));
             }
+
+            if (randomBoolean()) {
+                contexts.put(index, new Context(randomAlphaOfLength(5).toLowerCase(Locale.ROOT)));
+            }
         }
         return new org.opensearch.action.admin.indices.get.GetIndexResponse(
             indices,
@@ -117,6 +123,7 @@ protected void assertInstances(
         assertEquals(serverTestInstance.getSettings(), clientInstance.getSettings());
         assertEquals(serverTestInstance.defaultSettings(), clientInstance.getDefaultSettings());
         assertEquals(serverTestInstance.getAliases(), clientInstance.getAliases());
+        assertEquals(serverTestInstance.contexts(), clientInstance.contexts());
     }
 
     private static MappingMetadata createMappingsForIndex() {
diff --git a/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/create/CreateIndexIT.java b/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/create/CreateIndexIT.java
index fbe713d9e22c4..bd3c9e1456074 100644
--- a/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/create/CreateIndexIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/create/CreateIndexIT.java
@@ -41,16 +41,24 @@
 import org.opensearch.action.support.IndicesOptions;
 import org.opensearch.action.support.master.AcknowledgedResponse;
 import org.opensearch.cluster.ClusterState;
+import org.opensearch.cluster.applicationtemplates.ClusterStateSystemTemplateLoader;
+import org.opensearch.cluster.applicationtemplates.SystemTemplate;
+import org.opensearch.cluster.applicationtemplates.SystemTemplateMetadata;
+import org.opensearch.cluster.applicationtemplates.TemplateRepositoryMetadata;
+import org.opensearch.cluster.metadata.Context;
 import org.opensearch.cluster.metadata.IndexMetadata;
 import org.opensearch.cluster.metadata.MappingMetadata;
 import org.opensearch.cluster.metadata.Metadata;
+import org.opensearch.cluster.service.ClusterService;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.settings.SettingsException;
 import org.opensearch.common.unit.TimeValue;
 import org.opensearch.common.xcontent.XContentFactory;
 import org.opensearch.core.action.ActionListener;
+import org.opensearch.core.common.bytes.BytesReference;
 import org.opensearch.index.IndexNotFoundException;
 import org.opensearch.index.IndexService;
+import org.opensearch.index.IndexSettings;
 import org.opensearch.index.mapper.MapperParsingException;
 import org.opensearch.index.mapper.MapperService;
 import org.opensearch.index.query.RangeQueryBuilder;
@@ -59,7 +67,10 @@
 import org.opensearch.test.OpenSearchIntegTestCase.ClusterScope;
 import org.opensearch.test.OpenSearchIntegTestCase.Scope;
 
+import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
 import java.util.Map;
+import java.util.UUID;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.function.BiFunction;
@@ -430,4 +441,53 @@ public void testCreateIndexWithNullReplicaCountPickUpClusterReplica() {
             );
         }
     }
+
+    public void testCreateIndexWithContextSettingsAndTemplate() throws Exception {
+        int numReplicas = 1;
+        String indexName = "test-idx-1";
+        Settings settings = Settings.builder()
+            .put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 1)
+            .put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), (String) null)
+            .build();
+        Context context = new Context("test");
+
+        String templateContent = "{\n"
+            + "  \"template\": {\n"
+            + "    \"settings\": {\n"
+            + "      \"merge.policy\": \"log_byte_size\"\n"
+            + "    }\n"
+            + "  },\n"
+            + "  \"_meta\": {\n"
+            + "    \"_type\": \"@abc_template\",\n"
+            + "    \"_version\": 1\n"
+            + "  },\n"
+            + "  \"version\": 1\n"
+            + "}\n";
+
+        ClusterStateSystemTemplateLoader loader = new ClusterStateSystemTemplateLoader(
+            internalCluster().clusterManagerClient(),
+            () -> internalCluster().getInstance(ClusterService.class).state()
+        );
+        loader.loadTemplate(
+            new SystemTemplate(
+                BytesReference.fromByteBuffer(ByteBuffer.wrap(templateContent.getBytes(StandardCharsets.UTF_8))),
+                SystemTemplateMetadata.fromComponentTemplateInfo("test", 1L),
+                new TemplateRepositoryMetadata(UUID.randomUUID().toString(), 1L)
+            )
+        );
+
+        assertAcked(client().admin().indices().prepareCreate(indexName).setSettings(settings).setContext(context).get());
+
+        IndicesService indicesService = internalCluster().getInstance(IndicesService.class, internalCluster().getClusterManagerName());
+
+        for (IndexService indexService : indicesService) {
+            assertEquals(indexName, indexService.index().getName());
+            assertEquals(
+                numReplicas,
+                (int) indexService.getIndexSettings().getSettings().getAsInt(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, null)
+            );
+            assertEquals(context, indexService.getMetadata().context());
+            assertEquals("log_byte_size", indexService.getMetadata().getSettings().get(IndexSettings.INDEX_MERGE_POLICY.getKey()));
+        }
+    }
 }
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/create/CreateIndexRequestBuilder.java b/server/src/main/java/org/opensearch/action/admin/indices/create/CreateIndexRequestBuilder.java
index b233f45422967..27a580434333a 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/create/CreateIndexRequestBuilder.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/create/CreateIndexRequestBuilder.java
@@ -36,6 +36,7 @@
 import org.opensearch.action.support.ActiveShardCount;
 import org.opensearch.action.support.master.AcknowledgedRequestBuilder;
 import org.opensearch.client.OpenSearchClient;
+import org.opensearch.cluster.metadata.Context;
 import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.xcontent.LoggingDeprecationHandler;
@@ -275,4 +276,24 @@ public CreateIndexRequestBuilder setWaitForActiveShards(ActiveShardCount waitFor
     public CreateIndexRequestBuilder setWaitForActiveShards(final int waitForActiveShards) {
         return setWaitForActiveShards(ActiveShardCount.from(waitForActiveShards));
     }
+
+    /**
+     * Adds context that will be added when the index gets created.
+     *
+     * @param source The mapping source
+     */
+    public CreateIndexRequestBuilder setContext(Map<String, ?> source) {
+        request.context(source);
+        return this;
+    }
+
+    /**
+     * Adds context that will be added when the index gets created.
+     *
+     * @param source The context source
+     */
+    public CreateIndexRequestBuilder setContext(Context source) {
+        request.context(source);
+        return this;
+    }
 }
diff --git a/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java b/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java
index 5299602986793..488f1e07b988e 100644
--- a/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java
+++ b/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java
@@ -564,8 +564,10 @@ private Template applyContext(
         Settings.Builder settingsBuilder
     ) throws IOException {
         if (request.context() != null) {
-            String contextTemplate = MetadataIndexTemplateService.findContextTemplateName(currentState.metadata(), request.context());
-            ComponentTemplate componentTemplate = currentState.metadata().componentTemplates().get(contextTemplate);
+            ComponentTemplate componentTemplate = MetadataIndexTemplateService.findComponentTemplate(
+                currentState.metadata(),
+                request.context()
+            );
 
             if (componentTemplate.template().mappings() != null) {
                 // Mappings added at last (priority to mappings provided)
@@ -1426,7 +1428,7 @@ private static void validateActiveShardCount(ActiveShardCount waitForActiveShard
     private void validate(CreateIndexClusterStateUpdateRequest request, ClusterState state) {
         validateIndexName(request.index(), state);
         validateIndexSettings(request.index(), request.settings(), forbidPrivateIndexSettings);
-        validateContext(request, state);
+        validateContext(request);
     }
 
     public void validateIndexSettings(String indexName, final Settings settings, final boolean forbidPrivateIndexSettings)
@@ -1768,7 +1770,7 @@ static void validateTranslogDurabilitySettings(Settings requestSettings, Cluster
 
     }
 
-    static void validateContext(CreateIndexClusterStateUpdateRequest request, ClusterState clusterState) {
+    void validateContext(CreateIndexClusterStateUpdateRequest request) {
         final boolean isContextAllowed = FeatureFlags.isEnabled(FeatureFlags.APPLICATION_BASED_CONFIGURATION_TEMPLATES);
 
         if (request.context() != null && !isContextAllowed) {
@@ -1780,7 +1782,7 @@ static void validateContext(CreateIndexClusterStateUpdateRequest request, Cluste
             );
         }
 
-        if (request.context() != null && findContextTemplateName(clusterState.metadata(), request.context()) == null) {
+        if (request.context() != null && findContextTemplateName(clusterService.state().metadata(), request.context()) == null) {
             throw new InvalidIndexContextException(
                 request.context().name(),
                 request.index(),
diff --git a/server/src/main/java/org/opensearch/cluster/metadata/MetadataIndexTemplateService.java b/server/src/main/java/org/opensearch/cluster/metadata/MetadataIndexTemplateService.java
index 43d1551831950..c81e1c6093cc7 100644
--- a/server/src/main/java/org/opensearch/cluster/metadata/MetadataIndexTemplateService.java
+++ b/server/src/main/java/org/opensearch/cluster/metadata/MetadataIndexTemplateService.java
@@ -1322,8 +1322,7 @@ private static Settings resolveSettings(Metadata metadata, ComposableIndexTempla
         Optional.ofNullable(template.template()).map(Template::settings).ifPresent(templateSettings::put);
 
         // Add the template referred by context since it will take the highest precedence.
-        final String systemTemplate = findContextTemplateName(metadata, template.context());
-        final ComponentTemplate componentTemplate = metadata.componentTemplates().get(systemTemplate);
+        final ComponentTemplate componentTemplate = findComponentTemplate(metadata, template.context());
         Optional.ofNullable(componentTemplate).map(ComponentTemplate::template).map(Template::settings).ifPresent(templateSettings::put);
 
         return templateSettings.build();
@@ -1372,8 +1371,7 @@ public static List<Map<String, AliasMetadata>> resolveAliases(final Metadata met
 
         // Now use context referenced template's aliases which take the highest precedence
         if (template.context() != null) {
-            final String systemTemplate = findContextTemplateName(metadata, template.context());
-            final ComponentTemplate componentTemplate = metadata.componentTemplates().get(systemTemplate);
+            final ComponentTemplate componentTemplate = findComponentTemplate(metadata, template.context());
             Optional.ofNullable(componentTemplate.template()).map(Template::aliases).ifPresent(aliases::add);
         }
 
diff --git a/server/src/test/java/org/opensearch/action/admin/indices/get/GetIndexActionTests.java b/server/src/test/java/org/opensearch/action/admin/indices/get/GetIndexActionTests.java
index 2d9ec2b6d3c02..67d2163affd28 100644
--- a/server/src/test/java/org/opensearch/action/admin/indices/get/GetIndexActionTests.java
+++ b/server/src/test/java/org/opensearch/action/admin/indices/get/GetIndexActionTests.java
@@ -36,6 +36,7 @@
 import org.opensearch.action.support.ActionFilters;
 import org.opensearch.action.support.replication.ClusterStateCreationUtils;
 import org.opensearch.cluster.ClusterState;
+import org.opensearch.cluster.metadata.Context;
 import org.opensearch.cluster.metadata.IndexNameExpressionResolver;
 import org.opensearch.cluster.service.ClusterService;
 import org.opensearch.common.settings.IndexScopedSettings;
@@ -68,7 +69,7 @@ public class GetIndexActionTests extends OpenSearchSingleNodeTestCase {
     private ThreadPool threadPool;
     private SettingsFilter settingsFilter;
     private final String indexName = "test_index";
-
+    private Context context;
     private TestTransportGetIndexAction getIndexAction;
 
     @Before
@@ -91,6 +92,7 @@ public void setUp() throws Exception {
         );
         transportService.start();
         transportService.acceptIncomingRequests();
+        context = new Context(randomAlphaOfLength(5));
         getIndexAction = new GetIndexActionTests.TestTransportGetIndexAction();
     }
 
@@ -135,6 +137,23 @@ public void testDoNotIncludeDefaults() {
         );
     }
 
+    public void testContextInResponse() {
+        GetIndexRequest contextIndexRequest = new GetIndexRequest().indices(indexName);
+        getIndexAction.execute(
+            null,
+            contextIndexRequest,
+            ActionListener.wrap(
+                resp -> assertTrue(
+                    "index context should be present as it was set",
+                    resp.contexts().get(indexName) != null && resp.contexts().get(indexName).equals(context)
+                ),
+                exception -> {
+                    throw new AssertionError(exception);
+                }
+            )
+        );
+    }
+
     class TestTransportGetIndexAction extends TransportGetIndexAction {
 
         TestTransportGetIndexAction() {
@@ -157,7 +176,7 @@ protected void doClusterManagerOperation(
             ClusterState state,
             ActionListener<GetIndexResponse> listener
         ) {
-            ClusterState stateWithIndex = ClusterStateCreationUtils.state(indexName, 1, 1);
+            ClusterState stateWithIndex = ClusterStateCreationUtils.stateWithContext(indexName, 1, 1, context);
             super.doClusterManagerOperation(request, concreteIndices, stateWithIndex, listener);
         }
     }
diff --git a/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java b/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java
index 81f0ede543746..466ee24d88e10 100644
--- a/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java
+++ b/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java
@@ -81,6 +81,7 @@
 import org.opensearch.indices.IndexCreationException;
 import org.opensearch.indices.IndicesService;
 import org.opensearch.indices.InvalidAliasNameException;
+import org.opensearch.indices.InvalidIndexContextException;
 import org.opensearch.indices.InvalidIndexNameException;
 import org.opensearch.indices.RemoteStoreSettings;
 import org.opensearch.indices.ShardLimitValidator;
@@ -115,6 +116,7 @@
 import java.util.Optional;
 import java.util.Set;
 import java.util.UUID;
+import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.function.BiConsumer;
 import java.util.function.BiFunction;
@@ -2232,6 +2234,78 @@ public void testIndexCreationWithIndexStoreTypeRemoteStoreThrowsException() {
         );
     }
 
+    public void testCreateIndexWithContextDisabled() throws Exception {
+        request = new CreateIndexClusterStateUpdateRequest("create index", "test", "test").context(new Context(randomAlphaOfLength(5)));
+        withTemporaryClusterService((clusterService, threadPool) -> {
+            MetadataCreateIndexService checkerService = new MetadataCreateIndexService(
+                Settings.EMPTY,
+                clusterService,
+                indicesServices,
+                null,
+                null,
+                createTestShardLimitService(randomIntBetween(1, 1000), false, clusterService),
+                mock(Environment.class),
+                IndexScopedSettings.DEFAULT_SCOPED_SETTINGS,
+                threadPool,
+                null,
+                new SystemIndices(Collections.emptyMap()),
+                false,
+                new AwarenessReplicaBalance(Settings.EMPTY, clusterService.getClusterSettings()),
+                DefaultRemoteStoreSettings.INSTANCE,
+                repositoriesServiceSupplier
+            );
+            CountDownLatch counter = new CountDownLatch(1);
+            InvalidIndexContextException exception = expectThrows(
+                InvalidIndexContextException.class,
+                () -> checkerService.validateContext(request)
+            );
+            assertTrue(
+                "Invalid exception message." + exception.getMessage(),
+                exception.getMessage().contains("index specifies a context which cannot be used without enabling")
+            );
+        });
+    }
+
+    public void testCreateIndexWithContextAbsent() throws Exception {
+        FeatureFlags.initializeFeatureFlags(Settings.builder().put(FeatureFlags.APPLICATION_BASED_CONFIGURATION_TEMPLATES, true).build());
+        try {
+            request = new CreateIndexClusterStateUpdateRequest("create index", "test", "test").context(new Context(randomAlphaOfLength(5)));
+            withTemporaryClusterService((clusterService, threadPool) -> {
+                MetadataCreateIndexService checkerService = new MetadataCreateIndexService(
+                    Settings.EMPTY,
+                    clusterService,
+                    indicesServices,
+                    null,
+                    null,
+                    createTestShardLimitService(randomIntBetween(1, 1000), false, clusterService),
+                    mock(Environment.class),
+                    IndexScopedSettings.DEFAULT_SCOPED_SETTINGS,
+                    threadPool,
+                    null,
+                    new SystemIndices(Collections.emptyMap()),
+                    false,
+                    new AwarenessReplicaBalance(Settings.EMPTY, clusterService.getClusterSettings()),
+                    DefaultRemoteStoreSettings.INSTANCE,
+                    repositoriesServiceSupplier
+                );
+                CountDownLatch counter = new CountDownLatch(1);
+                InvalidIndexContextException exception = expectThrows(
+                    InvalidIndexContextException.class,
+                    () -> checkerService.validateContext(request)
+                );
+                assertTrue(
+                    "Invalid exception message." + exception.getMessage(),
+                    exception.getMessage().contains("index specifies a context which is not loaded on the cluster.")
+                );
+            });
+        } finally {
+            // Disable so that other tests which are not dependent on this are not impacted.
+            FeatureFlags.initializeFeatureFlags(
+                Settings.builder().put(FeatureFlags.APPLICATION_BASED_CONFIGURATION_TEMPLATES, false).build()
+            );
+        }
+    }
+
     private IndexTemplateMetadata addMatchingTemplate(Consumer<IndexTemplateMetadata.Builder> configurator) {
         IndexTemplateMetadata.Builder builder = templateMetadataBuilder("template1", "te*");
         configurator.accept(builder);
diff --git a/test/framework/src/main/java/org/opensearch/action/support/replication/ClusterStateCreationUtils.java b/test/framework/src/main/java/org/opensearch/action/support/replication/ClusterStateCreationUtils.java
index 182b2c9288a3d..8650500df8e95 100644
--- a/test/framework/src/main/java/org/opensearch/action/support/replication/ClusterStateCreationUtils.java
+++ b/test/framework/src/main/java/org/opensearch/action/support/replication/ClusterStateCreationUtils.java
@@ -35,6 +35,7 @@
 import org.opensearch.Version;
 import org.opensearch.cluster.ClusterName;
 import org.opensearch.cluster.ClusterState;
+import org.opensearch.cluster.metadata.Context;
 import org.opensearch.cluster.metadata.IndexMetadata;
 import org.opensearch.cluster.metadata.Metadata;
 import org.opensearch.cluster.node.DiscoveryNode;
@@ -371,6 +372,44 @@ public static ClusterState stateWithAssignedPrimariesAndReplicas(String[] indice
         return state.build();
     }
 
+    public static ClusterState stateWithContext(String index, final int numberOfNodes, final int numberOfPrimaries, Context context) {
+        DiscoveryNodes.Builder discoBuilder = DiscoveryNodes.builder();
+        Set<String> nodes = new HashSet<>();
+        for (int i = 0; i < numberOfNodes; i++) {
+            final DiscoveryNode node = newNode(i);
+            discoBuilder = discoBuilder.add(node);
+            nodes.add(node.getId());
+        }
+        discoBuilder.localNodeId(newNode(0).getId());
+        discoBuilder.clusterManagerNodeId(randomFrom(nodes));
+        IndexMetadata indexMetadata = IndexMetadata.builder(index)
+            .settings(
+                Settings.builder()
+                    .put(SETTING_VERSION_CREATED, Version.CURRENT)
+                    .put(SETTING_NUMBER_OF_SHARDS, numberOfPrimaries)
+                    .put(SETTING_NUMBER_OF_REPLICAS, 0)
+                    .put(SETTING_CREATION_DATE, System.currentTimeMillis())
+            )
+            .context(context)
+            .build();
+
+        IndexRoutingTable.Builder indexRoutingTable = IndexRoutingTable.builder(indexMetadata.getIndex());
+        for (int i = 0; i < numberOfPrimaries; i++) {
+            ShardId shardId = new ShardId(indexMetadata.getIndex(), i);
+            IndexShardRoutingTable.Builder indexShardRoutingBuilder = new IndexShardRoutingTable.Builder(shardId);
+            indexShardRoutingBuilder.addShard(
+                TestShardRouting.newShardRouting(shardId, randomFrom(nodes), true, ShardRoutingState.STARTED)
+            );
+            indexRoutingTable.addIndexShard(indexShardRoutingBuilder.build());
+        }
+
+        ClusterState.Builder state = ClusterState.builder(new ClusterName("test"));
+        state.nodes(discoBuilder);
+        state.metadata(Metadata.builder().put(indexMetadata, false).generateClusterUuidIfNeeded());
+        state.routingTable(RoutingTable.builder().add(indexRoutingTable).build());
+        return state.build();
+    }
+
     /**
      * Creates cluster state with and index that has one shard and as many replicas as numberOfReplicas.
      * Primary will be STARTED in cluster state but replicas will be one of UNASSIGNED, INITIALIZING, STARTED or RELOCATING.

From d14f3ceb7012282bef269d4613a50dda67668f01 Mon Sep 17 00:00:00 2001
From: Mohit Godwani <mgodwan@amazon.com>
Date: Mon, 26 Aug 2024 11:04:01 +0530
Subject: [PATCH 09/12] Add more tests

Signed-off-by: Mohit Godwani <mgodwan@amazon.com>
---
 .../metadata/MetadataCreateIndexService.java  |   2 +-
 .../cluster/metadata/IndexMetadataTests.java  |   3 +
 .../MetadataCreateIndexServiceTests.java      | 192 ++++++++++++++++++
 3 files changed, 196 insertions(+), 1 deletion(-)

diff --git a/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java b/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java
index 488f1e07b988e..aa0a194fcef57 100644
--- a/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java
+++ b/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java
@@ -557,7 +557,7 @@ private ClusterState applyCreateIndexWithTemporaryService(
         });
     }
 
-    private Template applyContext(
+    Template applyContext(
         CreateIndexClusterStateUpdateRequest request,
         ClusterState currentState,
         List<Map<String, Object>> mappings,
diff --git a/server/src/test/java/org/opensearch/cluster/metadata/IndexMetadataTests.java b/server/src/test/java/org/opensearch/cluster/metadata/IndexMetadataTests.java
index 393a652952771..3b0a2fed52c2d 100644
--- a/server/src/test/java/org/opensearch/cluster/metadata/IndexMetadataTests.java
+++ b/server/src/test/java/org/opensearch/cluster/metadata/IndexMetadataTests.java
@@ -118,6 +118,7 @@ public void testIndexMetadataSerialization() throws IOException {
                     randomNonNegativeLong()
                 )
             )
+            .context(new Context(randomAlphaOfLength(5)))
             .build();
         assertEquals(system, metadata.isSystem());
 
@@ -145,6 +146,7 @@ public void testIndexMetadataSerialization() throws IOException {
         assertEquals(metadata.getRoutingFactor(), fromXContentMeta.getRoutingFactor());
         assertEquals(metadata.primaryTerm(0), fromXContentMeta.primaryTerm(0));
         assertEquals(metadata.isSystem(), fromXContentMeta.isSystem());
+        assertEquals(metadata.context(), fromXContentMeta.context());
         final Map<String, DiffableStringMap> expectedCustom = Map.of("my_custom", new DiffableStringMap(customMap));
         assertEquals(metadata.getCustomData(), expectedCustom);
         assertEquals(metadata.getCustomData(), fromXContentMeta.getCustomData());
@@ -167,6 +169,7 @@ public void testIndexMetadataSerialization() throws IOException {
             assertEquals(deserialized.getCustomData(), expectedCustom);
             assertEquals(metadata.getCustomData(), deserialized.getCustomData());
             assertEquals(metadata.isSystem(), deserialized.isSystem());
+            assertEquals(metadata.context(), deserialized.context());
         }
     }
 
diff --git a/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java b/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java
index 466ee24d88e10..8d708990d9e38 100644
--- a/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java
+++ b/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java
@@ -56,6 +56,7 @@
 import org.opensearch.cluster.routing.allocation.decider.MaxRetryAllocationDecider;
 import org.opensearch.cluster.service.ClusterService;
 import org.opensearch.common.UUIDs;
+import org.opensearch.common.ValidationException;
 import org.opensearch.common.blobstore.BlobStore;
 import org.opensearch.common.compress.CompressedXContent;
 import org.opensearch.common.settings.ClusterSettings;
@@ -66,12 +67,16 @@
 import org.opensearch.common.util.BigArrays;
 import org.opensearch.common.util.FeatureFlags;
 import org.opensearch.common.xcontent.XContentFactory;
+import org.opensearch.common.xcontent.json.JsonXContent;
 import org.opensearch.core.index.Index;
+import org.opensearch.core.xcontent.DeprecationHandler;
 import org.opensearch.core.xcontent.NamedXContentRegistry;
+import org.opensearch.core.xcontent.XContentParser;
 import org.opensearch.env.Environment;
 import org.opensearch.index.IndexModule;
 import org.opensearch.index.IndexNotFoundException;
 import org.opensearch.index.IndexSettings;
+import org.opensearch.index.engine.EngineConfig;
 import org.opensearch.index.mapper.MapperService;
 import org.opensearch.index.query.QueryShardContext;
 import org.opensearch.index.remote.RemoteStoreEnums.PathHashAlgorithm;
@@ -115,9 +120,11 @@
 import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
+import java.util.TreeMap;
 import java.util.UUID;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.BiConsumer;
 import java.util.function.BiFunction;
 import java.util.function.Consumer;
@@ -149,6 +156,7 @@
 import static org.opensearch.cluster.metadata.MetadataCreateIndexService.resolveAndValidateAliases;
 import static org.opensearch.common.util.FeatureFlags.REMOTE_STORE_MIGRATION_EXPERIMENTAL;
 import static org.opensearch.index.IndexModule.INDEX_STORE_TYPE_SETTING;
+import static org.opensearch.index.IndexSettings.INDEX_MERGE_POLICY;
 import static org.opensearch.index.IndexSettings.INDEX_REFRESH_INTERVAL_SETTING;
 import static org.opensearch.index.IndexSettings.INDEX_REMOTE_TRANSLOG_BUFFER_INTERVAL_SETTING;
 import static org.opensearch.index.IndexSettings.INDEX_SOFT_DELETES_SETTING;
@@ -2306,6 +2314,190 @@ public void testCreateIndexWithContextAbsent() throws Exception {
         }
     }
 
+    public void testApplyContext() throws IOException {
+        FeatureFlags.initializeFeatureFlags(Settings.builder().put(FeatureFlags.APPLICATION_BASED_CONFIGURATION_TEMPLATES, true).build());
+        request = new CreateIndexClusterStateUpdateRequest("create index", "test", "test").context(new Context(randomAlphaOfLength(5)));
+
+        final Map<String, Object> mappings = new HashMap<>();
+        mappings.put("_doc", "\"properties\": { \"field1\": {\"type\": \"text\"}}");
+        List<Map<String, Object>> allMappings = new ArrayList<>();
+        allMappings.add(mappings);
+
+        Settings.Builder settingsBuilder = Settings.builder().put(INDEX_SOFT_DELETES_SETTING.getKey(), "false");
+
+        String templateContent = "{\n"
+            + "  \"template\": {\n"
+            + "    \"settings\": {\n"
+            + "      \"index.codec\": \"best_compression\",\n"
+            + "      \"index.merge.policy\": \"log_byte_size\",\n"
+            + "      \"index.refresh_interval\": \"60s\"\n"
+            + "    },\n"
+            + "    \"mappings\": {\n"
+            + "      \"properties\": {\n"
+            + "        \"field1\": {\n"
+            + "          \"type\": \"integer\"\n"
+            + "        }\n"
+            + "      }\n"
+            + "    }\n"
+            + "  },\n"
+            + "  \"_meta\": {\n"
+            + "    \"_type\": \"@abc_template\",\n"
+            + "    \"_version\": 1\n"
+            + "  },\n"
+            + "  \"version\": 1\n"
+            + "}\n";
+
+        AtomicReference<ComponentTemplate> componentTemplate = new AtomicReference<>();
+        try (
+            XContentParser contentParser = JsonXContent.jsonXContent.createParser(
+                NamedXContentRegistry.EMPTY,
+                DeprecationHandler.IGNORE_DEPRECATIONS,
+                templateContent
+            )
+        ) {
+            componentTemplate.set(ComponentTemplate.parse(contentParser));
+        }
+
+        String contextName = randomAlphaOfLength(5);
+        try {
+            request = new CreateIndexClusterStateUpdateRequest("create index", "test", "test").context(new Context(contextName));
+            withTemporaryClusterService((clusterService, threadPool) -> {
+                MetadataCreateIndexService checkerService = new MetadataCreateIndexService(
+                    Settings.EMPTY,
+                    clusterService,
+                    indicesServices,
+                    null,
+                    null,
+                    createTestShardLimitService(randomIntBetween(1, 1000), false, clusterService),
+                    mock(Environment.class),
+                    IndexScopedSettings.DEFAULT_SCOPED_SETTINGS,
+                    threadPool,
+                    null,
+                    new SystemIndices(Collections.emptyMap()),
+                    false,
+                    new AwarenessReplicaBalance(Settings.EMPTY, clusterService.getClusterSettings()),
+                    DefaultRemoteStoreSettings.INSTANCE,
+                    repositoriesServiceSupplier
+                );
+
+                ClusterState mockState = mock(ClusterState.class);
+                Metadata metadata = mock(Metadata.class);
+
+                when(mockState.metadata()).thenReturn(metadata);
+                when(metadata.systemTemplatesLookup()).thenReturn(Map.of(contextName, new TreeMap<>() {
+                    {
+                        put(1L, contextName);
+                    }
+                }));
+                when(metadata.componentTemplates()).thenReturn(Map.of(contextName, componentTemplate.get()));
+
+                try {
+                    Template template = checkerService.applyContext(request, mockState, allMappings, settingsBuilder);
+                    assertEquals(componentTemplate.get().template(), template);
+
+                    assertEquals(2, allMappings.size());
+                    assertEquals(mappings, allMappings.get(0));
+                    assertEquals(
+                        MapperService.parseMapping(NamedXContentRegistry.EMPTY, componentTemplate.get().template().mappings().toString()),
+                        allMappings.get(1)
+                    );
+
+                    assertEquals("60s", settingsBuilder.get(INDEX_REFRESH_INTERVAL_SETTING.getKey()));
+                    assertEquals("log_byte_size", settingsBuilder.get(INDEX_MERGE_POLICY.getKey()));
+                    assertEquals("best_compression", settingsBuilder.get(EngineConfig.INDEX_CODEC_SETTING.getKey()));
+                    assertEquals("false", settingsBuilder.get(INDEX_SOFT_DELETES_SETTING.getKey()));
+                } catch (IOException ex) {
+                    throw new AssertionError(ex);
+                }
+            });
+        } finally {
+            // Disable so that other tests which are not dependent on this are not impacted.
+            FeatureFlags.initializeFeatureFlags(
+                Settings.builder().put(FeatureFlags.APPLICATION_BASED_CONFIGURATION_TEMPLATES, false).build()
+            );
+        }
+    }
+
+    public void testApplyContextWithSettingsOverlap() throws IOException {
+        FeatureFlags.initializeFeatureFlags(Settings.builder().put(FeatureFlags.APPLICATION_BASED_CONFIGURATION_TEMPLATES, true).build());
+        request = new CreateIndexClusterStateUpdateRequest("create index", "test", "test").context(new Context(randomAlphaOfLength(5)));
+        Settings.Builder settingsBuilder = Settings.builder().put(INDEX_REFRESH_INTERVAL_SETTING.getKey(), "30s");
+        String templateContent = "{\n"
+            + "  \"template\": {\n"
+            + "    \"settings\": {\n"
+            + "      \"index.refresh_interval\": \"60s\"\n"
+            + "    }\n"
+            + "   },\n"
+            + "  \"_meta\": {\n"
+            + "    \"_type\": \"@abc_template\",\n"
+            + "    \"_version\": 1\n"
+            + "  },\n"
+            + "  \"version\": 1\n"
+            + "}\n";
+
+        AtomicReference<ComponentTemplate> componentTemplate = new AtomicReference<>();
+        try (
+            XContentParser contentParser = JsonXContent.jsonXContent.createParser(
+                NamedXContentRegistry.EMPTY,
+                DeprecationHandler.IGNORE_DEPRECATIONS,
+                templateContent
+            )
+        ) {
+            componentTemplate.set(ComponentTemplate.parse(contentParser));
+        }
+
+        String contextName = randomAlphaOfLength(5);
+        try {
+            request = new CreateIndexClusterStateUpdateRequest("create index", "test", "test").context(new Context(contextName));
+            withTemporaryClusterService((clusterService, threadPool) -> {
+                MetadataCreateIndexService checkerService = new MetadataCreateIndexService(
+                    Settings.EMPTY,
+                    clusterService,
+                    indicesServices,
+                    null,
+                    null,
+                    createTestShardLimitService(randomIntBetween(1, 1000), false, clusterService),
+                    mock(Environment.class),
+                    IndexScopedSettings.DEFAULT_SCOPED_SETTINGS,
+                    threadPool,
+                    null,
+                    new SystemIndices(Collections.emptyMap()),
+                    false,
+                    new AwarenessReplicaBalance(Settings.EMPTY, clusterService.getClusterSettings()),
+                    DefaultRemoteStoreSettings.INSTANCE,
+                    repositoriesServiceSupplier
+                );
+
+                ClusterState mockState = mock(ClusterState.class);
+                Metadata metadata = mock(Metadata.class);
+
+                when(mockState.metadata()).thenReturn(metadata);
+                when(metadata.systemTemplatesLookup()).thenReturn(Map.of(contextName, new TreeMap<>() {
+                    {
+                        put(1L, contextName);
+                    }
+                }));
+                when(metadata.componentTemplates()).thenReturn(Map.of(contextName, componentTemplate.get()));
+
+                ValidationException validationException = expectThrows(
+                    ValidationException.class,
+                    () -> checkerService.applyContext(request, mockState, List.of(), settingsBuilder)
+                );
+                assertEquals(1, validationException.validationErrors().size());
+                assertTrue(
+                    "Invalid exception message: " + validationException.getMessage(),
+                    validationException.getMessage()
+                        .contains("Cannot apply context template as user provide settings have overlap with the included context template")
+                );
+            });
+        } finally {
+            // Disable so that other tests which are not dependent on this are not impacted.
+            FeatureFlags.initializeFeatureFlags(
+                Settings.builder().put(FeatureFlags.APPLICATION_BASED_CONFIGURATION_TEMPLATES, false).build()
+            );
+        }
+    }
+
     private IndexTemplateMetadata addMatchingTemplate(Consumer<IndexTemplateMetadata.Builder> configurator) {
         IndexTemplateMetadata.Builder builder = templateMetadataBuilder("template1", "te*");
         configurator.accept(builder);

From 6d3b1cb010050c52d5ed97c16400970a7ceb247c Mon Sep 17 00:00:00 2001
From: Mohit Godwani <mgodwan@amazon.com>
Date: Mon, 26 Aug 2024 13:40:04 +0530
Subject: [PATCH 10/12] Add more tests

Signed-off-by: Mohit Godwani <mgodwan@amazon.com>
---
 .../indices/settings/UpdateSettingsIT.java    | 62 +++++++++++++++++++
 .../indices/create/CreateIndexRequest.java    |  2 +-
 .../create/CreateIndexRequestTests.java       | 21 +++++++
 .../test/OpenSearchIntegTestCase.java         |  8 +++
 4 files changed, 92 insertions(+), 1 deletion(-)

diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/settings/UpdateSettingsIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/settings/UpdateSettingsIT.java
index beb0ea797bbec..475d0a154a98b 100644
--- a/server/src/internalClusterTest/java/org/opensearch/indices/settings/UpdateSettingsIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/indices/settings/UpdateSettingsIT.java
@@ -35,6 +35,11 @@
 import org.opensearch.action.admin.cluster.health.ClusterHealthResponse;
 import org.opensearch.action.admin.indices.settings.get.GetSettingsResponse;
 import org.opensearch.cluster.ClusterState;
+import org.opensearch.cluster.applicationtemplates.ClusterStateSystemTemplateLoader;
+import org.opensearch.cluster.applicationtemplates.SystemTemplate;
+import org.opensearch.cluster.applicationtemplates.SystemTemplateMetadata;
+import org.opensearch.cluster.applicationtemplates.TemplateRepositoryMetadata;
+import org.opensearch.cluster.metadata.Context;
 import org.opensearch.cluster.metadata.IndexMetadata;
 import org.opensearch.cluster.service.ClusterService;
 import org.opensearch.common.Priority;
@@ -42,6 +47,7 @@
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.settings.SettingsException;
 import org.opensearch.common.unit.TimeValue;
+import org.opensearch.core.common.bytes.BytesReference;
 import org.opensearch.index.IndexModule;
 import org.opensearch.index.IndexService;
 import org.opensearch.index.VersionType;
@@ -51,10 +57,14 @@
 import org.opensearch.test.OpenSearchIntegTestCase;
 import org.opensearch.threadpool.ThreadPool;
 
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
+import java.util.UUID;
 
 import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_BLOCKS_METADATA;
 import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_BLOCKS_READ;
@@ -99,6 +109,58 @@ public void testInvalidDynamicUpdate() {
         assertNotEquals(indexMetadata.getSettings().get("index.dummy"), "invalid dynamic value");
     }
 
+    public void testDynamicUpdateWithContextSettingOverlap() throws IOException {
+        String templateContent = "{\n"
+            + "  \"template\": {\n"
+            + "    \"settings\": {\n"
+            + "      \"index.merge.policy\": \"log_byte_size\"\n"
+            + "    }\n"
+            + "  },\n"
+            + "  \"_meta\": {\n"
+            + "    \"_type\": \"@abc_template\",\n"
+            + "    \"_version\": 1\n"
+            + "  },\n"
+            + "  \"version\": 1\n"
+            + "}\n";
+
+        ClusterStateSystemTemplateLoader loader = new ClusterStateSystemTemplateLoader(
+            internalCluster().clusterManagerClient(),
+            () -> internalCluster().getInstance(ClusterService.class).state()
+        );
+        loader.loadTemplate(
+            new SystemTemplate(
+                BytesReference.fromByteBuffer(ByteBuffer.wrap(templateContent.getBytes(StandardCharsets.UTF_8))),
+                SystemTemplateMetadata.fromComponentTemplateInfo("testcontext", 1L),
+                new TemplateRepositoryMetadata(UUID.randomUUID().toString(), 1L)
+            )
+        );
+
+        createIndex("test", new Context("testcontext"));
+
+        IllegalArgumentException validationException = expectThrows(
+            IllegalArgumentException.class,
+            () -> client().admin()
+                .indices()
+                .prepareUpdateSettings("test")
+                .setSettings(Settings.builder().put("index.merge.policy", "tiered"))
+                .execute()
+                .actionGet()
+        );
+        assertTrue(
+            validationException.getMessage()
+                .contains("Cannot apply context template as user provide settings have overlap with the included context template")
+        );
+
+        assertAcked(
+            client().admin()
+                .indices()
+                .prepareUpdateSettings("test")
+                .setSettings(Settings.builder().put("index.refresh_interval", "60s"))
+                .execute()
+                .actionGet()
+        );
+    }
+
     @Override
     protected Collection<Class<? extends Plugin>> nodePlugins() {
         return Arrays.asList(DummySettingPlugin.class, FinalSettingPlugin.class);
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/create/CreateIndexRequest.java b/server/src/main/java/org/opensearch/action/admin/indices/create/CreateIndexRequest.java
index b14cb8cc51b6f..c0b413fb9fe95 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/create/CreateIndexRequest.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/create/CreateIndexRequest.java
@@ -590,7 +590,7 @@ public CreateIndexRequest context(Map<String, ?> source) {
         }
     }
 
-    public CreateIndexRequest context(BytesReference source) throws IOException {
+    public CreateIndexRequest context(BytesReference source) {
         // EMPTY is safe here because we never call namedObject
         try (XContentParser parser = XContentHelper.createParser(NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, source)) {
             // move to the first alias
diff --git a/server/src/test/java/org/opensearch/action/admin/indices/create/CreateIndexRequestTests.java b/server/src/test/java/org/opensearch/action/admin/indices/create/CreateIndexRequestTests.java
index 89e072d783747..ee150c7b2bb71 100644
--- a/server/src/test/java/org/opensearch/action/admin/indices/create/CreateIndexRequestTests.java
+++ b/server/src/test/java/org/opensearch/action/admin/indices/create/CreateIndexRequestTests.java
@@ -165,6 +165,27 @@ public void testToString() throws IOException {
         assertThat(request.toString(), containsString("mappings='{\"_doc\":{}}'"));
     }
 
+    public void testContext() throws IOException {
+        String contextName = "Test";
+        String contextVersion = "1";
+        Map<String, Object> paramsMap = Map.of("foo", "bar");
+        try (XContentBuilder builder = MediaTypeRegistry.contentBuilder(randomFrom(XContentType.values()))) {
+            builder.startObject()
+                .startObject("context")
+                .field("name", contextName)
+                .field("version", contextVersion)
+                .field("params", paramsMap)
+                .endObject()
+                .endObject();
+
+            CreateIndexRequest parsedCreateIndexRequest = new CreateIndexRequest();
+            parsedCreateIndexRequest.source(builder);
+            assertEquals(contextName, parsedCreateIndexRequest.context().name());
+            assertEquals(contextVersion, parsedCreateIndexRequest.context().version());
+            assertEquals(paramsMap, parsedCreateIndexRequest.context().params());
+        }
+    }
+
     public static void assertMappingsEqual(Map<String, String> expected, Map<String, String> actual) throws IOException {
         assertEquals(expected.keySet(), actual.keySet());
 
diff --git a/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java b/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java
index b86cce682c68e..41c2e65dc0561 100644
--- a/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java
+++ b/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java
@@ -79,6 +79,7 @@
 import org.opensearch.cluster.ClusterState;
 import org.opensearch.cluster.coordination.OpenSearchNodeCommand;
 import org.opensearch.cluster.health.ClusterHealthStatus;
+import org.opensearch.cluster.metadata.Context;
 import org.opensearch.cluster.metadata.IndexMetadata;
 import org.opensearch.cluster.metadata.Metadata;
 import org.opensearch.cluster.routing.IndexRoutingTable;
@@ -746,6 +747,13 @@ public final void createIndex(String name, Settings indexSettings) {
         assertAcked(prepareCreate(name).setSettings(indexSettings));
     }
 
+    /**
+     * creates an index with the given setting
+     */
+    public final void createIndex(String name, Context context) {
+        assertAcked(prepareCreate(name).setContext(context));
+    }
+
     /**
      * Creates a new {@link CreateIndexRequestBuilder} with the settings obtained from {@link #indexSettings()}.
      */

From c860f7d7517b71ed660cbe8a00d9b225e347a278 Mon Sep 17 00:00:00 2001
From: Mohit Godwani <mgodwan@amazon.com>
Date: Thu, 29 Aug 2024 15:48:03 +0530
Subject: [PATCH 11/12] Address PR comments

Signed-off-by: Mohit Godwani <mgodwan@amazon.com>
---
 .../action/admin/indices/create/CreateIndexRequest.java  | 5 ++++-
 .../java/org/opensearch/cluster/metadata/Context.java    | 9 +++++++++
 .../main/java/org/opensearch/index/IndexSettings.java    | 1 -
 3 files changed, 13 insertions(+), 2 deletions(-)

diff --git a/server/src/main/java/org/opensearch/action/admin/indices/create/CreateIndexRequest.java b/server/src/main/java/org/opensearch/action/admin/indices/create/CreateIndexRequest.java
index c0b413fb9fe95..1634ebbad227b 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/create/CreateIndexRequest.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/create/CreateIndexRequest.java
@@ -597,7 +597,7 @@ public CreateIndexRequest context(BytesReference source) {
             context(Context.fromXContent(parser));
             return this;
         } catch (IOException e) {
-            throw new OpenSearchParseException("Failed to parse aliases", e);
+            throw new OpenSearchParseException("Failed to parse context", e);
         }
     }
 
@@ -653,6 +653,9 @@ public String toString() {
             + '\''
             + ", aliases="
             + aliases
+            + '\''
+            + ", context="
+            + context
             + ", waitForActiveShards="
             + waitForActiveShards
             + '}';
diff --git a/server/src/main/java/org/opensearch/cluster/metadata/Context.java b/server/src/main/java/org/opensearch/cluster/metadata/Context.java
index 3b0cfb23ae6b4..4a6c5a457591d 100644
--- a/server/src/main/java/org/opensearch/cluster/metadata/Context.java
+++ b/server/src/main/java/org/opensearch/cluster/metadata/Context.java
@@ -127,4 +127,13 @@ public boolean equals(Object o) {
     public int hashCode() {
         return Objects.hash(name, version, params);
     }
+
+    @Override
+    public String toString() {
+        return "Context{" +
+            "name='" + name + '\'' +
+            ", version='" + version + '\'' +
+            ", params=" + params +
+            '}';
+    }
 }
diff --git a/server/src/main/java/org/opensearch/index/IndexSettings.java b/server/src/main/java/org/opensearch/index/IndexSettings.java
index 4c104e154ed71..0811de770d65a 100644
--- a/server/src/main/java/org/opensearch/index/IndexSettings.java
+++ b/server/src/main/java/org/opensearch/index/IndexSettings.java
@@ -747,7 +747,6 @@ public static IndexMergePolicy fromString(String text) {
         0,
         0,
         Property.PrivateIndex,
-        Property.Dynamic,
         Property.IndexScope
     );
 

From 247e2408bbdfc856ab55ceaa4ec7288b6a88ab3f Mon Sep 17 00:00:00 2001
From: Mohit Godwani <mgodwan@amazon.com>
Date: Thu, 29 Aug 2024 16:59:14 +0530
Subject: [PATCH 12/12] Apply spotless

Signed-off-by: Mohit Godwani <mgodwan@amazon.com>
---
 .../main/java/org/opensearch/cluster/metadata/Context.java  | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/server/src/main/java/org/opensearch/cluster/metadata/Context.java b/server/src/main/java/org/opensearch/cluster/metadata/Context.java
index 4a6c5a457591d..ceaef4dbc8d14 100644
--- a/server/src/main/java/org/opensearch/cluster/metadata/Context.java
+++ b/server/src/main/java/org/opensearch/cluster/metadata/Context.java
@@ -130,10 +130,6 @@ public int hashCode() {
 
     @Override
     public String toString() {
-        return "Context{" +
-            "name='" + name + '\'' +
-            ", version='" + version + '\'' +
-            ", params=" + params +
-            '}';
+        return "Context{" + "name='" + name + '\'' + ", version='" + version + '\'' + ", params=" + params + '}';
     }
 }