Skip to content

Commit 3a3e93f

Browse files
dakroneelasticmachinemartijnvg
authored
Add rolling upgrade tests for component and composable templates (#58969)
Backport of #58867 to 7.x branch. This adds rolling upgrade tests that component and composable templates can be read from older versions of the cluster. Relates to #58643 * Added skip versions * Make `tests.upgrade_from_version` available to `UpgradeClusterClientYamlTestSuiteIT` and use that to skip tests based on versions. * added jdocs Co-authored-by: Elastic Machine <[email protected]> Co-authored-by: Martijn van Groningen <[email protected]>
1 parent e0ee210 commit 3a3e93f

File tree

6 files changed

+187
-24
lines changed

6 files changed

+187
-24
lines changed

‎qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/UpgradeClusterClientYamlTestSuiteIT.java‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
1212
import com.carrotsearch.randomizedtesting.annotations.TimeoutSuite;
1313
import org.apache.lucene.util.TimeUnits;
14+
import org.elasticsearch.Version;
1415
import org.elasticsearch.common.settings.Settings;
1516
import org.elasticsearch.test.rest.ESRestTestCase;
1617
import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
@@ -19,6 +20,13 @@
1920
@TimeoutSuite(millis = 5 * TimeUnits.MINUTE) // to account for slow as hell VMs
2021
public class UpgradeClusterClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
2122

23+
protected static final Version UPGRADE_FROM_VERSION = Version.fromString(System.getProperty("tests.upgrade_from_version"));
24+
25+
@Override
26+
protected Version overwriteEsVersion(Version esVersionFromApi) {
27+
return UPGRADE_FROM_VERSION;
28+
}
29+
2230
@Override
2331
protected boolean preserveIndicesUponCompletion() {
2432
return true;

‎qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/10_basic.yml‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -80,3 +80,32 @@
8080
index: queries
8181
include_type_name: false
8282
- match: { queries.mappings.properties.id.type: "keyword" }
83+
84+
---
85+
"Component and composable templates can be retrieved":
86+
- skip:
87+
version: " - 7.12.99"
88+
reason: "#58969 only available from 7.13.0"
89+
- do:
90+
cluster.get_component_template:
91+
name: my-ct
92+
93+
- match: {component_templates.0.name: my-ct}
94+
- match: {component_templates.0.component_template.version: 2}
95+
- match: {component_templates.0.component_template._meta: {foo: bar, baz: {eggplant: true}}}
96+
- match: {component_templates.0.component_template.template.settings: {index: {number_of_shards: '1', number_of_replicas: '0'}}}
97+
- is_true: component_templates.0.component_template.template.mappings
98+
- match: {component_templates.0.component_template.template.aliases: {aliasname: {}}}
99+
100+
- do:
101+
indices.get_index_template:
102+
name: my-it
103+
104+
- match: {index_templates.0.index_template.index_patterns: ["test-*"]}
105+
- match: {index_templates.0.index_template.template.settings.index: {number_of_shards: '1', number_of_replicas: '0'}}
106+
- is_true: index_templates.0.index_template.template.mappings
107+
- length: {index_templates.0.index_template.template.aliases: 3}
108+
- is_true: index_templates.0.index_template.template.aliases.test_alias
109+
- match: {index_templates.0.index_template.template.aliases.test_blias.index_routing: "b" }
110+
- match: {index_templates.0.index_template.template.aliases.test_blias.search_routing: "b" }
111+
- match: {index_templates.0.index_template.template.aliases.test_clias.filter.term.user: "kimchy" }

‎qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/10_basic.yml‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -204,3 +204,53 @@
204204
tasks.get:
205205
wait_for_completion: true
206206
task_id: $task
207+
208+
---
209+
"Component and composable template validation":
210+
- skip:
211+
version: " - 7.12.99"
212+
reason: "#58969 only available from 7.13.0"
213+
- do:
214+
cluster.put_component_template:
215+
name: my-ct
216+
body:
217+
template:
218+
settings:
219+
number_of_shards: 1
220+
number_of_replicas: 0
221+
mappings:
222+
dynamic: false
223+
aliases:
224+
aliasname: {}
225+
version: 2
226+
_meta:
227+
foo: bar
228+
baz:
229+
eggplant: true
230+
231+
- do:
232+
indices.put_index_template:
233+
name: my-it
234+
body:
235+
index_patterns: [test-*]
236+
composed_of: ["my-ct"]
237+
template:
238+
settings:
239+
number_of_shards: 1
240+
number_of_replicas: 0
241+
mappings:
242+
properties:
243+
field:
244+
type: text
245+
aliases:
246+
test_alias: {}
247+
test_blias: { routing: b }
248+
test_clias: { filter: { term: { user: kimchy }}}
249+
250+
- do:
251+
cluster.get_component_template:
252+
name: my-ct
253+
254+
- do:
255+
indices.get_index_template:
256+
name: my-it

‎qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/10_basic.yml‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -133,3 +133,60 @@
133133
wait_for_completion: true
134134
task_id: $task_id
135135
- match: { task.headers.X-Opaque-Id: "Reindexing Again" }
136+
137+
---
138+
"Component and composable templates can be retrieved and updated":
139+
- skip:
140+
version: " - 7.12.99"
141+
reason: "#58969 only available from 7.13.0"
142+
- do:
143+
cluster.get_component_template:
144+
name: my-ct
145+
146+
- match: {component_templates.0.name: my-ct}
147+
- match: {component_templates.0.component_template.version: 2}
148+
- match: {component_templates.0.component_template._meta: {foo: bar, baz: {eggplant: true}}}
149+
- match: {component_templates.0.component_template.template.settings: {index: {number_of_shards: '1', number_of_replicas: '0'}}}
150+
- match: {component_templates.0.component_template.template.mappings: {dynamic: false}}
151+
- match: {component_templates.0.component_template.template.aliases: {aliasname: {}}}
152+
153+
- do:
154+
indices.get_index_template:
155+
name: my-it
156+
157+
- match: {index_templates.0.index_template.index_patterns: ["test-*"]}
158+
- match: {index_templates.0.index_template.template.settings.index: {number_of_shards: '1', number_of_replicas: '0'}}
159+
- is_true: index_templates.0.index_template.template.mappings
160+
- length: {index_templates.0.index_template.template.aliases: 3}
161+
- is_true: index_templates.0.index_template.template.aliases.test_alias
162+
- match: {index_templates.0.index_template.template.aliases.test_blias.index_routing: "b" }
163+
- match: {index_templates.0.index_template.template.aliases.test_blias.search_routing: "b" }
164+
- match: {index_templates.0.index_template.template.aliases.test_clias.filter.term.user: "kimchy" }
165+
- do:
166+
cluster.put_component_template:
167+
name: my-ct
168+
body:
169+
template:
170+
settings:
171+
number_of_shards: 1
172+
number_of_replicas: 0
173+
mappings:
174+
dynamic: true
175+
aliases:
176+
aliasname: {}
177+
version: 2
178+
_meta:
179+
foo: bar
180+
baz:
181+
eggplant: true
182+
183+
- do:
184+
cluster.get_component_template:
185+
name: my-ct
186+
187+
- match: {component_templates.0.name: my-ct}
188+
- match: {component_templates.0.component_template.version: 2}
189+
- match: {component_templates.0.component_template._meta: {foo: bar, baz: {eggplant: true}}}
190+
- match: {component_templates.0.component_template.template.settings: {index: {number_of_shards: '1', number_of_replicas: '0'}}}
191+
- is_true: component_templates.0.component_template.template.mappings
192+
- match: {component_templates.0.component_template.template.aliases: {aliasname: {}}}

‎server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java‎

Lines changed: 31 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -189,7 +189,7 @@ ClusterState addComponentTemplate(final ClusterState currentState, final boolean
189189
}
190190

191191
CompressedXContent mappings = template.template().mappings();
192-
String stringMappings = mappings == null ? null : mappings.string();
192+
String stringMappings = wrapMappingsIfNecessary(mappings == null ? null : mappings.string(), xContentRegistry);
193193

194194
// We may need to normalize index settings, so do that also
195195
Settings finalSettings = template.template().settings();
@@ -228,17 +228,6 @@ ClusterState addComponentTemplate(final ClusterState currentState, final boolean
228228
}
229229
}
230230

231-
// Mappings in component templates don't include _doc, so update the mappings to include this single type
232-
if (stringMappings != null) {
233-
Map<String, Object> parsedMappings = MapperService.parseMapping(xContentRegistry, stringMappings);
234-
if (parsedMappings.size() > 0) {
235-
stringMappings = Strings.toString(XContentFactory.jsonBuilder()
236-
.startObject()
237-
.field(MapperService.SINGLE_MAPPING_NAME, parsedMappings)
238-
.endObject());
239-
}
240-
}
241-
242231
final Template finalTemplate = new Template(finalSettings,
243232
stringMappings == null ? null : new CompressedXContent(stringMappings), template.template().aliases());
244233
final ComponentTemplate finalComponentTemplate = new ComponentTemplate(finalTemplate, template.version(), template.metadata());
@@ -282,6 +271,35 @@ ClusterState addComponentTemplate(final ClusterState currentState, final boolean
282271
.build();
283272
}
284273

274+
@Nullable
275+
private static String wrapMappingsIfNecessary(@Nullable String mappings, NamedXContentRegistry xContentRegistry) throws Exception {
276+
// Mappings in templates don't have to include _doc, so update
277+
// the mappings to include this single type if necessary
278+
279+
String stringMappings = mappings;
280+
if (stringMappings != null) {
281+
Map<String, Object> parsedMappings = MapperService.parseMapping(xContentRegistry, stringMappings);
282+
if (parsedMappings.size() > 0) {
283+
if (parsedMappings.size() == 1) {
284+
final String keyName = parsedMappings.keySet().iterator().next();
285+
// Check if it's already wrapped in `_doc`, only rewrap if needed
286+
if (MapperService.SINGLE_MAPPING_NAME.equals(keyName) == false) {
287+
stringMappings = Strings.toString(XContentFactory.jsonBuilder()
288+
.startObject()
289+
.field(MapperService.SINGLE_MAPPING_NAME, parsedMappings)
290+
.endObject());
291+
}
292+
} else {
293+
stringMappings = Strings.toString(XContentFactory.jsonBuilder()
294+
.startObject()
295+
.field(MapperService.SINGLE_MAPPING_NAME, parsedMappings)
296+
.endObject());
297+
}
298+
}
299+
}
300+
return stringMappings;
301+
}
302+
285303
/**
286304
* Remove the given component template from the cluster state. The component template name
287305
* supports simple regex wildcards for removing multiple component templates at a time.
@@ -486,18 +504,8 @@ public ClusterState addIndexTemplateV2(final ClusterState currentState, final bo
486504
// If an inner template was specified, its mappings may need to be
487505
// adjusted (to add _doc) and it should be validated
488506
CompressedXContent mappings = innerTemplate.mappings();
489-
String stringMappings = mappings == null ? null : mappings.string();
507+
String stringMappings = wrapMappingsIfNecessary(mappings == null ? null : mappings.string(), xContentRegistry);
490508

491-
// Mappings in index templates don't include _doc, so update the mappings to include this single type
492-
if (stringMappings != null) {
493-
Map<String, Object> parsedMappings = MapperService.parseMapping(xContentRegistry, stringMappings);
494-
if (parsedMappings.size() > 0) {
495-
stringMappings = Strings.toString(XContentFactory.jsonBuilder()
496-
.startObject()
497-
.field(MapperService.SINGLE_MAPPING_NAME, parsedMappings)
498-
.endObject());
499-
}
500-
}
501509
final Template finalTemplate = new Template(finalSettings,
502510
stringMappings == null ? null : new CompressedXContent(stringMappings), innerTemplate.aliases());
503511
finalIndexTemplate = new ComposableIndexTemplate(template.indexPatterns(), finalTemplate, template.composedOf(),

‎test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCase.java‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -123,7 +123,7 @@ public void initAndResetContext() throws Exception {
123123
validateSpec(restSpec);
124124
final List<HttpHost> hosts = getClusterHosts();
125125
Tuple<Version, Version> versionVersionTuple = readVersionsFromCatNodes(adminClient());
126-
final Version esVersion = versionVersionTuple.v1();
126+
final Version esVersion = overwriteEsVersion(versionVersionTuple.v1());
127127
final Version masterVersion = versionVersionTuple.v2();
128128
final String os = readOsFromNodesInfo(adminClient());
129129

@@ -157,6 +157,17 @@ public void initAndResetContext() throws Exception {
157157
restTestExecutionContext.clear();
158158
}
159159

160+
/**
161+
* Allows test suites to return another version then the lowest version returned from the cat node api.
162+
*
163+
* For example, in rolling upgrade bwc tests, the version of the old cluster should be used, and if
164+
* the cluster is fully upgraded then the cat api doesn't return the version of the old cluster. These
165+
* tests provide the es version via system property.
166+
*/
167+
protected Version overwriteEsVersion(Version esVersionFromApi) {
168+
return esVersionFromApi;
169+
}
170+
160171
protected ClientYamlTestClient initClientYamlTestClient(
161172
final ClientYamlSuiteRestSpec restSpec,
162173
final RestClient restClient,

0 commit comments

Comments
 (0)