diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java index 3087c16fcf9b8..f9094c14a9d84 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java @@ -1007,3 +1007,4 @@ private static String encodePart(String pathPart) { } } } + diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/RollupClient.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/RollupClient.java index 148dbee223e07..3a9759e47a20e 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/RollupClient.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/RollupClient.java @@ -24,6 +24,8 @@ import org.elasticsearch.client.rollup.DeleteRollupJobResponse; import org.elasticsearch.client.rollup.GetRollupJobRequest; import org.elasticsearch.client.rollup.GetRollupJobResponse; +import org.elasticsearch.client.rollup.GetRollupCapsRequest; +import org.elasticsearch.client.rollup.GetRollupCapsResponse; import org.elasticsearch.client.rollup.PutRollupJobRequest; import org.elasticsearch.client.rollup.PutRollupJobResponse; @@ -137,6 +139,8 @@ public GetRollupJobResponse getRollupJob(GetRollupJobRequest request, RequestOpt * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized * @param listener the listener to be notified upon request completion */ + + public void getRollupJobAsync(GetRollupJobRequest request, RequestOptions options, ActionListener listener) { restHighLevelClient.performRequestAsyncAndParseEntity(request, RollupRequestConverters::getJob, @@ -144,4 +148,39 @@ public void getRollupJobAsync(GetRollupJobRequest request, RequestOptions option GetRollupJobResponse::fromXContent, listener, Collections.emptySet()); } + + /** + * Get the Rollup Capabilities of a target (non-rollup) index or pattern + * See + * the docs for more. + * @param request the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @return the response + * @throws IOException in case there is a problem sending the request or parsing back the response + */ + public GetRollupCapsResponse getRollupCapabilities(GetRollupCapsRequest request, RequestOptions options) throws IOException { + return restHighLevelClient.performRequestAndParseEntity(request, + RollupRequestConverters::getRollupCaps, + options, + GetRollupCapsResponse::fromXContent, + Collections.emptySet()); + } + + /** + * Asynchronously Get the Rollup Capabilities of a target (non-rollup) index or pattern + * See + * the docs for more. + * @param request the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @param listener the listener to be notified upon request completion + */ + public void getRollupCapabilitiesAsync(GetRollupCapsRequest request, RequestOptions options, + ActionListener listener) { + restHighLevelClient.performRequestAsyncAndParseEntity(request, + RollupRequestConverters::getRollupCaps, + options, + GetRollupCapsResponse::fromXContent, + listener, + Collections.emptySet()); + } } diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/RollupRequestConverters.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/RollupRequestConverters.java index 2c2025837cdb1..6b6a05ed03c65 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/RollupRequestConverters.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/RollupRequestConverters.java @@ -23,6 +23,7 @@ import org.apache.http.client.methods.HttpPut; import org.elasticsearch.client.rollup.DeleteRollupJobRequest; import org.elasticsearch.client.rollup.GetRollupJobRequest; +import org.elasticsearch.client.rollup.GetRollupCapsRequest; import org.elasticsearch.client.rollup.PutRollupJobRequest; import java.io.IOException; @@ -68,4 +69,16 @@ static Request deleteJob(final DeleteRollupJobRequest deleteRollupJobRequest) th request.setEntity(createEntity(deleteRollupJobRequest, REQUEST_BODY_CONTENT_TYPE)); return request; } + + static Request getRollupCaps(final GetRollupCapsRequest getRollupCapsRequest) throws IOException { + String endpoint = new RequestConverters.EndpointBuilder() + .addPathPartAsIs("_xpack") + .addPathPartAsIs("rollup") + .addPathPartAsIs("data") + .addPathPart(getRollupCapsRequest.getIndexPattern()) + .build(); + Request request = new Request(HttpGet.METHOD_NAME, endpoint); + request.setEntity(createEntity(getRollupCapsRequest, REQUEST_BODY_CONTENT_TYPE)); + return request; + } } diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/rollup/GetRollupCapsRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/rollup/GetRollupCapsRequest.java new file mode 100644 index 0000000000000..a5e215a71b255 --- /dev/null +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/rollup/GetRollupCapsRequest.java @@ -0,0 +1,70 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.client.rollup; + +import org.elasticsearch.client.Validatable; +import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.xcontent.ToXContentObject; +import org.elasticsearch.common.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.Objects; + +public class GetRollupCapsRequest implements Validatable, ToXContentObject { + private static final String ID = "id"; + private final String indexPattern; + + public GetRollupCapsRequest(final String indexPattern) { + if (Strings.isNullOrEmpty(indexPattern) || indexPattern.equals("*")) { + this.indexPattern = MetaData.ALL; + } else { + this.indexPattern = indexPattern; + } + } + + public String getIndexPattern() { + return indexPattern; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(ID, indexPattern); + builder.endObject(); + return builder; + } + + @Override + public int hashCode() { + return Objects.hash(indexPattern); + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + GetRollupCapsRequest other = (GetRollupCapsRequest) obj; + return Objects.equals(indexPattern, other.indexPattern); + } +} diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/rollup/GetRollupCapsResponse.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/rollup/GetRollupCapsResponse.java new file mode 100644 index 0000000000000..872dc7440a092 --- /dev/null +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/rollup/GetRollupCapsResponse.java @@ -0,0 +1,92 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.client.rollup; + +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.ToXContentObject; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; + +import java.io.IOException; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; + +public class GetRollupCapsResponse implements ToXContentObject { + + private final Map jobs; + + public GetRollupCapsResponse(final Map jobs) { + this.jobs = Collections.unmodifiableMap(Objects.requireNonNull(jobs)); + } + + public Map getJobs() { + return jobs; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException { + builder.startObject(); + for (Map.Entry entry : jobs.entrySet()) { + entry.getValue().toXContent(builder, params); + } + builder.endObject(); + return builder; + } + + public static GetRollupCapsResponse fromXContent(final XContentParser parser) throws IOException { + Map jobs = new HashMap<>(); + XContentParser.Token token = parser.nextToken(); + if (token.equals(XContentParser.Token.START_OBJECT)) { + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + if (token.equals(XContentParser.Token.FIELD_NAME)) { + String pattern = parser.currentName(); + + RollableIndexCaps cap = RollableIndexCaps.PARSER.apply(pattern).apply(parser, null); + jobs.put(pattern, cap); + } + } + } + return new GetRollupCapsResponse(jobs); + } + + @Override + public int hashCode() { + return Objects.hash(jobs); + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + GetRollupCapsResponse other = (GetRollupCapsResponse) obj; + return Objects.equals(jobs, other.jobs); + } + + @Override + public final String toString() { + return Strings.toString(this); + } +} diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/rollup/RollableIndexCaps.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/rollup/RollableIndexCaps.java new file mode 100644 index 0000000000000..cf849e38dd0b4 --- /dev/null +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/rollup/RollableIndexCaps.java @@ -0,0 +1,103 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.client.rollup; + +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.ToXContentFragment; +import org.elasticsearch.common.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; +import java.util.Objects; +import java.util.function.Function; +import java.util.stream.Collectors; + +/** + * Represents the rollup capabilities of a non-rollup index. E.g. what values/aggregations + * were rolled up for this index, in what rollup jobs that data is stored and where those + * concrete rollup indices exist + * + * The index name can either be a single index, or an index pattern (logstash-*) + */ +public class RollableIndexCaps implements ToXContentFragment { + private static final ParseField ROLLUP_JOBS = new ParseField("rollup_jobs"); + + public static final Function> PARSER = indexName -> { + @SuppressWarnings("unchecked") + ConstructingObjectParser p + = new ConstructingObjectParser<>(indexName, + a -> new RollableIndexCaps(indexName, (List) a[0])); + + p.declareObjectArray(ConstructingObjectParser.constructorArg(), RollupJobCaps.PARSER::apply, + ROLLUP_JOBS); + return p; + }; + + private final String indexName; + private final List jobCaps; + + RollableIndexCaps(final String indexName, final List caps) { + this.indexName = indexName; + this.jobCaps = Collections.unmodifiableList(Objects.requireNonNull(caps) + .stream() + .sorted(Comparator.comparing(RollupJobCaps::getJobID)) + .collect(Collectors.toList())); + } + + public String getIndexName() { + return indexName; + } + + public List getJobCaps() { + return jobCaps; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(indexName); + { + builder.field(ROLLUP_JOBS.getPreferredName(), jobCaps); + } + builder.endObject(); + return builder; + } + + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } + + if (other == null || getClass() != other.getClass()) { + return false; + } + + RollableIndexCaps that = (RollableIndexCaps) other; + return Objects.equals(this.jobCaps, that.jobCaps) + && Objects.equals(this.indexName, that.indexName); + } + + @Override + public int hashCode() { + return Objects.hash(jobCaps, indexName); + } +} diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/rollup/RollupJobCaps.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/rollup/RollupJobCaps.java new file mode 100644 index 0000000000000..7ba1aaa4d7c2b --- /dev/null +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/rollup/RollupJobCaps.java @@ -0,0 +1,199 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.client.rollup; + +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.collect.Tuple; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.ToXContentFragment; +import org.elasticsearch.common.xcontent.ToXContentObject; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.function.Function; + +/** + * Represents the Rollup capabilities for a specific job on a single rollup index + */ +public class RollupJobCaps implements ToXContentObject { + private static final ParseField JOB_ID = new ParseField("job_id"); + private static final ParseField ROLLUP_INDEX = new ParseField("rollup_index"); + private static final ParseField INDEX_PATTERN = new ParseField("index_pattern"); + private static final ParseField FIELDS = new ParseField("fields"); + private static final String NAME = "rollup_job_caps"; + + public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>(NAME, + a -> { + @SuppressWarnings("unchecked") + List> caps = (List>) a[3]; + if (caps.isEmpty()) { + return new RollupJobCaps((String) a[0], (String) a[1], (String) a[2], Collections.emptyMap()); + } + Map mapCaps = new HashMap<>(caps.size()); + caps.forEach(c -> mapCaps.put(c.v1(), c.v2())); + return new RollupJobCaps((String) a[0], (String) a[1], (String) a[2], mapCaps); + }); + + static { + PARSER.declareString(ConstructingObjectParser.constructorArg(), JOB_ID); + PARSER.declareString(ConstructingObjectParser.constructorArg(), ROLLUP_INDEX); + PARSER.declareString(ConstructingObjectParser.constructorArg(), INDEX_PATTERN); + PARSER.declareNamedObjects(ConstructingObjectParser.constructorArg(), + (p, c, name) -> new Tuple<>(name, RollupFieldCaps.fromXContent(p)), FIELDS); + } + + private final String jobID; + private final String rollupIndex; + private final String indexPattern; + private final Map fieldCapLookup; + + RollupJobCaps(final String jobID, final String rollupIndex, + final String indexPattern, final Map fieldCapLookup) { + this.jobID = jobID; + this.rollupIndex = rollupIndex; + this.indexPattern = indexPattern; + this.fieldCapLookup = Collections.unmodifiableMap(Objects.requireNonNull(fieldCapLookup)); + } + + public Map getFieldCaps() { + return fieldCapLookup; + } + + public String getRollupIndex() { + return rollupIndex; + } + + public String getIndexPattern() { + return indexPattern; + } + + public String getJobID() { + return jobID; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + { + builder.field(JOB_ID.getPreferredName(), jobID); + builder.field(ROLLUP_INDEX.getPreferredName(), rollupIndex); + builder.field(INDEX_PATTERN.getPreferredName(), indexPattern); + builder.startObject(FIELDS.getPreferredName()); + { + for (Map.Entry fieldCap : fieldCapLookup.entrySet()) { + builder.array(fieldCap.getKey(), fieldCap.getValue()); + } + } + builder.endObject(); + } + builder.endObject(); + return builder; + } + + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } + + if (other == null || getClass() != other.getClass()) { + return false; + } + + RollupJobCaps that = (RollupJobCaps) other; + + return Objects.equals(this.jobID, that.jobID) + && Objects.equals(this.indexPattern, that.indexPattern) + && Objects.equals(this.rollupIndex, that.rollupIndex) + && Objects.equals(this.fieldCapLookup, that.fieldCapLookup); + } + + @Override + public int hashCode() { + return Objects.hash(jobID, rollupIndex, fieldCapLookup, indexPattern); + } + + public static class RollupFieldCaps implements ToXContentFragment { + private static final String NAME = "rollup_field_caps"; + private final List> aggs; + + public static final Function> PARSER = fieldName -> { + @SuppressWarnings("unchecked") + ConstructingObjectParser parser + = new ConstructingObjectParser<>(NAME, a -> new RollupFieldCaps((List>) a[0])); + + parser.declareObjectArray(ConstructingObjectParser.constructorArg(), + (p, c) -> p.map(), new ParseField(fieldName)); + return parser; + }; + + RollupFieldCaps(final List> aggs) { + this.aggs = Collections.unmodifiableList(Objects.requireNonNull(aggs)); + } + + public List> getAggs() { + return aggs; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + for (Map agg : aggs) { + builder.map(agg); + } + return builder; + } + + public static RollupFieldCaps fromXContent(XContentParser parser) throws IOException { + List> aggs = new ArrayList<>(); + if (parser.nextToken().equals(XContentParser.Token.START_ARRAY)) { + while (parser.nextToken().equals(XContentParser.Token.START_OBJECT)) { + aggs.add(Collections.unmodifiableMap(parser.map())); + } + } + return new RollupFieldCaps(Collections.unmodifiableList(aggs)); + } + + + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } + + if (other == null || getClass() != other.getClass()) { + return false; + } + + RollupFieldCaps that = (RollupFieldCaps) other; + return Objects.equals(this.aggs, that.aggs); + } + + @Override + public int hashCode() { + return Objects.hash(aggs); + } + } +} diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/RollupIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/RollupIT.java index ce693b7e7c407..7a5f873d45cc7 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/RollupIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/RollupIT.java @@ -19,6 +19,7 @@ package org.elasticsearch.client; import org.elasticsearch.ElasticsearchStatusException; +import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest; import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; import org.elasticsearch.action.admin.indices.refresh.RefreshResponse; import org.elasticsearch.action.bulk.BulkItemResponse; @@ -28,14 +29,18 @@ import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.support.WriteRequest; +import org.elasticsearch.client.rollup.DeleteRollupJobRequest; +import org.elasticsearch.client.rollup.DeleteRollupJobResponse; +import org.elasticsearch.client.rollup.GetRollupCapsRequest; +import org.elasticsearch.client.rollup.GetRollupCapsResponse; import org.elasticsearch.client.rollup.GetRollupJobRequest; import org.elasticsearch.client.rollup.GetRollupJobResponse; import org.elasticsearch.client.rollup.GetRollupJobResponse.IndexerState; import org.elasticsearch.client.rollup.GetRollupJobResponse.JobWrapper; -import org.elasticsearch.client.rollup.DeleteRollupJobRequest; -import org.elasticsearch.client.rollup.DeleteRollupJobResponse; import org.elasticsearch.client.rollup.PutRollupJobRequest; import org.elasticsearch.client.rollup.PutRollupJobResponse; +import org.elasticsearch.client.rollup.RollableIndexCaps; +import org.elasticsearch.client.rollup.RollupJobCaps; import org.elasticsearch.client.rollup.job.config.DateHistogramGroupConfig; import org.elasticsearch.client.rollup.job.config.GroupConfig; import org.elasticsearch.client.rollup.job.config.MetricConfig; @@ -53,17 +58,19 @@ import java.util.Arrays; import java.util.Collections; +import java.util.HashSet; import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.Set; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.hamcrest.Matchers.either; +import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.hasKey; import static org.hamcrest.Matchers.hasSize; -import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.lessThan; @@ -229,4 +236,115 @@ public void testGetMissingRollupJob() throws Exception { assertThat(getResponse.getJobs(), empty()); } + public void testGetRollupCaps() throws Exception { + final Set values = new HashSet<>(); + double sum = 0.0d; + int max = Integer.MIN_VALUE; + int min = Integer.MAX_VALUE; + + final BulkRequest bulkRequest = new BulkRequest(); + bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + for (int minute = 0; minute < 60; minute++) { + for (int second = 0; second < 60; second = second + 10) { + final int value = randomIntBetween(0, 100); + + final IndexRequest indexRequest = new IndexRequest("docs", "doc"); + indexRequest.source(jsonBuilder() + .startObject() + .field("value", value) + .field("date", String.format(Locale.ROOT, "2018-01-01T00:%02d:%02dZ", minute, second)) + .endObject()); + bulkRequest.add(indexRequest); + + values.add(value); + sum += value; + if (value > max) { + max = value; + } + if (value < min) { + min = value; + } + } + } + + final int numDocs = bulkRequest.numberOfActions(); + + BulkResponse bulkResponse = highLevelClient().bulk(bulkRequest, RequestOptions.DEFAULT); + assertEquals(RestStatus.OK, bulkResponse.status()); + if (bulkResponse.hasFailures()) { + for (BulkItemResponse itemResponse : bulkResponse.getItems()) { + if (itemResponse.isFailed()) { + logger.fatal(itemResponse.getFailureMessage()); + } + } + } + assertFalse(bulkResponse.hasFailures()); + + RefreshResponse refreshResponse = highLevelClient().indices().refresh(new RefreshRequest("docs"), RequestOptions.DEFAULT); + assertEquals(0, refreshResponse.getFailedShards()); + + final String id = randomAlphaOfLength(10); + final String indexPattern = randomFrom("docs", "d*", "doc*"); + final String rollupIndex = randomFrom("rollup", "test"); + final String cron = "*/1 * * * * ?"; + final int pageSize = randomIntBetween(numDocs, numDocs * 10); + // TODO expand this to also test with histogram and terms? + final GroupConfig groups = new GroupConfig(new DateHistogramGroupConfig("date", DateHistogramInterval.DAY)); + final List metrics = Collections.singletonList(new MetricConfig("value", SUPPORTED_METRICS)); + final TimeValue timeout = TimeValue.timeValueSeconds(randomIntBetween(30, 600)); + + PutRollupJobRequest putRollupJobRequest = + new PutRollupJobRequest(new RollupJobConfig(id, indexPattern, rollupIndex, cron, pageSize, groups, metrics, timeout)); + + final RollupClient rollupClient = highLevelClient().rollup(); + PutRollupJobResponse response = execute(putRollupJobRequest, rollupClient::putRollupJob, rollupClient::putRollupJobAsync); + assertTrue(response.isAcknowledged()); + + // wait for the PutJob api to create the index w/ metadata + highLevelClient().cluster().health(new ClusterHealthRequest(rollupIndex).waitForYellowStatus(), RequestOptions.DEFAULT); + + GetRollupCapsRequest getRollupCapsRequest = new GetRollupCapsRequest(indexPattern); + GetRollupCapsResponse capsResponse = highLevelClient().rollup() + .getRollupCapabilities(getRollupCapsRequest, RequestOptions.DEFAULT); + + assertNotNull(capsResponse); + Map rolledPatterns = capsResponse.getJobs(); + assertThat(rolledPatterns.size(), equalTo(1)); + + RollableIndexCaps docsPattern = rolledPatterns.get(indexPattern); + assertThat(docsPattern.getIndexName(), equalTo(indexPattern)); + + List rollupJobs = docsPattern.getJobCaps(); + assertThat(rollupJobs.size(), equalTo(1)); + + RollupJobCaps jobCaps = rollupJobs.get(0); + assertThat(jobCaps.getJobID(), equalTo(id)); + assertThat(jobCaps.getRollupIndex(), equalTo(rollupIndex)); + assertThat(jobCaps.getIndexPattern(), equalTo(indexPattern)); + + Map fieldCaps = jobCaps.getFieldCaps(); + + List> timestampCaps = fieldCaps.get("date").getAggs(); + for (Map.Entry entry : timestampCaps.get(0).entrySet()) { + switch (entry.getKey()) { + case "agg": + assertThat(entry.getValue(), equalTo("date_histogram")); + break; + case "delay": + assertThat(entry.getValue(), equalTo("foo")); + break; + case "interval": + assertThat(entry.getValue(), equalTo("1d")); + break; + case "time_zone": + assertThat(entry.getValue(), equalTo("UTC")); + break; + default: + fail("Unknown field cap: [" + entry.getKey() + "]"); + } + } + + List> valueCaps = fieldCaps.get("value").getAggs(); + assertThat(valueCaps.size(), equalTo(SUPPORTED_METRICS.size())); + } } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/RollupDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/RollupDocumentationIT.java index 0a5a688c5fb41..86562297306b2 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/RollupDocumentationIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/RollupDocumentationIT.java @@ -20,6 +20,8 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.LatchedActionListener; +import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest; +import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; import org.elasticsearch.action.admin.indices.refresh.RefreshResponse; import org.elasticsearch.action.bulk.BulkRequest; @@ -27,38 +29,53 @@ import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.support.WriteRequest; import org.elasticsearch.client.ESRestHighLevelClientTestCase; +import org.elasticsearch.client.Request; import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.client.Response; import org.elasticsearch.client.RestHighLevelClient; +import org.elasticsearch.client.rollup.DeleteRollupJobRequest; +import org.elasticsearch.client.rollup.DeleteRollupJobResponse; +import org.elasticsearch.client.rollup.GetRollupCapsRequest; +import org.elasticsearch.client.rollup.GetRollupCapsResponse; import org.elasticsearch.client.rollup.GetRollupJobRequest; import org.elasticsearch.client.rollup.GetRollupJobResponse; import org.elasticsearch.client.rollup.GetRollupJobResponse.JobWrapper; import org.elasticsearch.client.rollup.GetRollupJobResponse.RollupIndexerJobStats; import org.elasticsearch.client.rollup.GetRollupJobResponse.RollupJobStatus; -import org.elasticsearch.client.rollup.DeleteRollupJobRequest; -import org.elasticsearch.client.rollup.DeleteRollupJobResponse; import org.elasticsearch.client.rollup.PutRollupJobRequest; import org.elasticsearch.client.rollup.PutRollupJobResponse; +import org.elasticsearch.client.rollup.RollableIndexCaps; +import org.elasticsearch.client.rollup.RollupJobCaps; import org.elasticsearch.client.rollup.job.config.DateHistogramGroupConfig; import org.elasticsearch.client.rollup.job.config.GroupConfig; import org.elasticsearch.client.rollup.job.config.HistogramGroupConfig; import org.elasticsearch.client.rollup.job.config.MetricConfig; import org.elasticsearch.client.rollup.job.config.RollupJobConfig; import org.elasticsearch.client.rollup.job.config.TermsGroupConfig; +import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval; +import org.junit.After; import org.junit.Before; +import java.io.BufferedReader; import java.io.IOException; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.Locale; +import java.util.Map; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.isOneOf; public class RollupDocumentationIT extends ESRestHighLevelClientTestCase { @@ -219,6 +236,176 @@ public void onFailure(Exception e) { assertTrue(latch.await(30L, TimeUnit.SECONDS)); } + public void testGetRollupCaps() throws Exception { + RestHighLevelClient client = highLevelClient(); + + DateHistogramGroupConfig dateHistogram = + new DateHistogramGroupConfig("timestamp", DateHistogramInterval.HOUR, new DateHistogramInterval("7d"), "UTC"); // <1> + TermsGroupConfig terms = new TermsGroupConfig("hostname", "datacenter"); + HistogramGroupConfig histogram = new HistogramGroupConfig(5L, "load", "net_in", "net_out"); + GroupConfig groups = new GroupConfig(dateHistogram, histogram, terms); + List metrics = new ArrayList<>(); // <1> + metrics.add(new MetricConfig("temperature", Arrays.asList("min", "max", "sum"))); + metrics.add(new MetricConfig("voltage", Arrays.asList("avg", "value_count"))); + + //tag::x-pack-rollup-get-rollup-caps-setup + final String indexPattern = "docs"; + final String rollupIndexName = "rollup"; + final String cron = "*/1 * * * * ?"; + final int pageSize = 100; + final TimeValue timeout = null; + + String id = "job_1"; + RollupJobConfig config = new RollupJobConfig(id, indexPattern, rollupIndexName, cron, + pageSize, groups, metrics, timeout); + + PutRollupJobRequest request = new PutRollupJobRequest(config); + PutRollupJobResponse response = client.rollup().putRollupJob(request, RequestOptions.DEFAULT); + + boolean acknowledged = response.isAcknowledged(); + //end::x-pack-rollup-get-rollup-caps-setup + assertTrue(acknowledged); + + ClusterHealthRequest healthRequest = new ClusterHealthRequest(config.getRollupIndex()).waitForYellowStatus(); + ClusterHealthResponse healthResponse = client.cluster().health(healthRequest, RequestOptions.DEFAULT); + assertFalse(healthResponse.isTimedOut()); + assertThat(healthResponse.getStatus(), isOneOf(ClusterHealthStatus.YELLOW, ClusterHealthStatus.GREEN)); + + // Now that the job is created, we should have a rollup index with metadata. + // We can test out the caps API now. + + //tag::x-pack-rollup-get-rollup-caps-request + GetRollupCapsRequest getRollupCapsRequest = new GetRollupCapsRequest("docs"); + //end::x-pack-rollup-get-rollup-caps-request + + //tag::x-pack-rollup-get-rollup-caps-execute + GetRollupCapsResponse capsResponse = client.rollup().getRollupCapabilities(getRollupCapsRequest, RequestOptions.DEFAULT); + //end::x-pack-rollup-get-rollup-caps-execute + + //tag::x-pack-rollup-get-rollup-caps-response + Map rolledPatterns = capsResponse.getJobs(); + + RollableIndexCaps docsPattern = rolledPatterns.get("docs"); + + // indexName will be "docs" in this case... the index pattern that we rolled up + String indexName = docsPattern.getIndexName(); + + // Each index pattern can have multiple jobs that rolled it up, so `getJobCaps()` + // returns a list of jobs that rolled up the pattern + List rollupJobs = docsPattern.getJobCaps(); + RollupJobCaps jobCaps = rollupJobs.get(0); + + // jobID is the identifier we used when we created the job (e.g. `job1`) + String jobID = jobCaps.getJobID(); + + // rollupIndex is the location that the job stored it's rollup docs (e.g. `rollup`) + String rollupIndex = jobCaps.getRollupIndex(); + + // indexPattern is the same as the indexName that we retrieved earlier, redundant info + assert jobCaps.getIndexPattern().equals(indexName); + + // Finally, fieldCaps are the capabilities of individual fields in the config + // The key is the field name, and the value is a RollupFieldCaps object which + // provides more info. + Map fieldCaps = jobCaps.getFieldCaps(); + + // If we retrieve the "timestamp" field, it returns a list of maps. Each list + // item represents a different aggregation that can be run against the "timestamp" + // field, and any additional details specific to that agg (interval, etc) + List> timestampCaps = fieldCaps.get("timestamp").getAggs(); + assert timestampCaps.get(0).toString().equals("{agg=date_histogram, delay=7d, interval=1h, time_zone=UTC}"); + + // In contrast to the timestamp field, the temperature field has multiple aggs configured + List> temperatureCaps = fieldCaps.get("temperature").getAggs(); + assert temperatureCaps.toString().equals("[{agg=min}, {agg=max}, {agg=sum}]"); + //end::x-pack-rollup-get-rollup-caps-response + + assertThat(indexName, equalTo("docs")); + assertThat(jobID, equalTo("job_1")); + assertThat(rollupIndex, equalTo("rollup")); + assertThat(fieldCaps.size(), equalTo(8)); + + // tag::x-pack-rollup-get-rollup-caps-execute-listener + ActionListener listener = new ActionListener() { + @Override + public void onResponse(GetRollupCapsResponse response) { + // <1> + } + + @Override + public void onFailure(Exception e) { + // <2> + } + }; + // end::x-pack-rollup-get-rollup-caps-execute-listener + + // Replace the empty listener by a blocking listener in test + final CountDownLatch latch = new CountDownLatch(1); + listener = new LatchedActionListener<>(listener, latch); + + // tag::x-pack-rollup-get-rollup-caps-execute-async + client.rollup().getRollupCapabilitiesAsync(getRollupCapsRequest, RequestOptions.DEFAULT, listener); // <1> + // end::x-pack-rollup-get-rollup-caps-execute-async + + assertTrue(latch.await(30L, TimeUnit.SECONDS)); + } + + @After + public void wipeRollup() throws Exception { + // TODO move this to ESRestTestCase + deleteRollupJobs(); + waitForPendingRollupTasks(); + } + + private void deleteRollupJobs() throws Exception { + Response response = adminClient().performRequest(new Request("GET", "/_xpack/rollup/job/_all")); + Map jobs = entityAsMap(response); + @SuppressWarnings("unchecked") + List> jobConfigs = + (List>) XContentMapValues.extractValue("jobs", jobs); + + if (jobConfigs == null) { + return; + } + + for (Map jobConfig : jobConfigs) { + @SuppressWarnings("unchecked") + String jobId = (String) ((Map) jobConfig.get("config")).get("id"); + Request request = new Request("DELETE", "/_xpack/rollup/job/" + jobId); + request.addParameter("ignore", "404"); // Ignore 404s because they imply someone was racing us to delete this + adminClient().performRequest(request); + } + } + + private void waitForPendingRollupTasks() throws Exception { + assertBusy(() -> { + try { + Request request = new Request("GET", "/_cat/tasks"); + request.addParameter("detailed", "true"); + Response response = adminClient().performRequest(request); + + try (BufferedReader responseReader = new BufferedReader( + new InputStreamReader(response.getEntity().getContent(), StandardCharsets.UTF_8))) { + int activeTasks = 0; + String line; + StringBuilder tasksListString = new StringBuilder(); + while ((line = responseReader.readLine()) != null) { + + // We only care about Rollup jobs, otherwise this fails too easily due to unrelated tasks + if (line.startsWith("xpack/rollup/job") == true) { + activeTasks++; + tasksListString.append(line).append('\n'); + } + } + assertEquals(activeTasks + " active tasks found:\n" + tasksListString, 0, activeTasks); + } + } catch (IOException e) { + // Throw an assertion error so we retry + throw new AssertionError("Error getting active tasks list", e); + } + }); + } + public void testDeleteRollupJob() throws Exception { RestHighLevelClient client = highLevelClient(); diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/rollup/GetRollupCapsRequestTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/rollup/GetRollupCapsRequestTests.java new file mode 100644 index 0000000000000..bca08f7bfc505 --- /dev/null +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/rollup/GetRollupCapsRequestTests.java @@ -0,0 +1,33 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.client.rollup; + +import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.test.ESTestCase; + +import static org.hamcrest.Matchers.equalTo; + +public class GetRollupCapsRequestTests extends ESTestCase { + + public void testImplicitIndexPattern() { + String pattern = randomFrom("", "*", MetaData.ALL, null); + GetRollupCapsRequest request = new GetRollupCapsRequest(pattern); + assertThat(request.getIndexPattern(), equalTo(MetaData.ALL)); + } +} diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/rollup/GetRollupCapsResponseTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/rollup/GetRollupCapsResponseTests.java new file mode 100644 index 0000000000000..af595b210e9d4 --- /dev/null +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/rollup/GetRollupCapsResponseTests.java @@ -0,0 +1,152 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.client.rollup; + +import org.elasticsearch.client.rollup.job.config.DateHistogramGroupConfig; +import org.elasticsearch.client.rollup.job.config.GroupConfig; +import org.elasticsearch.client.rollup.job.config.HistogramGroupConfig; +import org.elasticsearch.client.rollup.job.config.MetricConfig; +import org.elasticsearch.client.rollup.job.config.RollupJobConfig; +import org.elasticsearch.client.rollup.job.config.RollupJobConfigTests; +import org.elasticsearch.client.rollup.job.config.TermsGroupConfig; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder; +import org.elasticsearch.search.aggregations.bucket.histogram.HistogramAggregationBuilder; +import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder; +import org.elasticsearch.test.AbstractXContentTestCase; +import org.junit.Before; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import static java.util.Collections.singletonMap; + +public class GetRollupCapsResponseTests extends AbstractXContentTestCase { + + private Map indices; + + @Before + private void setupIndices() throws IOException { + int numIndices = randomIntBetween(1,5); + indices = new HashMap<>(numIndices); + for (int i = 0; i < numIndices; i++) { + String indexName = "index_" + randomAlphaOfLength(10); + int numJobs = randomIntBetween(1,5); + List jobs = new ArrayList<>(numJobs); + for (int j = 0; j < numJobs; j++) { + RollupJobConfig config = RollupJobConfigTests.randomRollupJobConfig(randomAlphaOfLength(10)); + jobs.add(new RollupJobCaps(config.getId(), config.getIndexPattern(), + config.getRollupIndex(), createRollupFieldCaps(config))); + } + RollableIndexCaps cap = new RollableIndexCaps(indexName, jobs); + indices.put(indexName, cap); + } + } + + @Override + protected GetRollupCapsResponse createTestInstance() { + return new GetRollupCapsResponse(indices); + } + + @Override + protected boolean supportsUnknownFields() { + return false; + } + + @Override + protected GetRollupCapsResponse doParseInstance(final XContentParser parser) throws IOException { + return GetRollupCapsResponse.fromXContent(parser); + } + + /** + * Lifted from core's RollupJobCaps, so that we can test without having to include this actual logic in the request + */ + private static Map createRollupFieldCaps(final RollupJobConfig rollupJobConfig) { + final Map>> tempFieldCaps = new HashMap<>(); + + final GroupConfig groupConfig = rollupJobConfig.getGroupConfig(); + if (groupConfig != null) { + // Create RollupFieldCaps for the date histogram + final DateHistogramGroupConfig dateHistogram = groupConfig.getDateHistogram(); + final Map dateHistogramAggCap = new HashMap<>(); + dateHistogramAggCap.put("agg", DateHistogramAggregationBuilder.NAME); + dateHistogramAggCap.put("interval", dateHistogram.getInterval().toString()); + if (dateHistogram.getDelay() != null) { + dateHistogramAggCap.put("delay", dateHistogram.getDelay().toString()); + } + dateHistogramAggCap.put("time_zone", dateHistogram.getTimeZone()); + + List> dateAggCaps = tempFieldCaps.getOrDefault(dateHistogram.getField(), new ArrayList<>()); + dateAggCaps.add(dateHistogramAggCap); + tempFieldCaps.put(dateHistogram.getField(), dateAggCaps); + + // Create RollupFieldCaps for the histogram + final HistogramGroupConfig histogram = groupConfig.getHistogram(); + if (histogram != null) { + final Map histogramAggCap = new HashMap<>(); + histogramAggCap.put("agg", HistogramAggregationBuilder.NAME); + histogramAggCap.put("interval", histogram.getInterval()); + Arrays.stream(rollupJobConfig.getGroupConfig().getHistogram().getFields()).forEach(field -> { + List> caps = tempFieldCaps.getOrDefault(field, new ArrayList<>()); + caps.add(histogramAggCap); + tempFieldCaps.put(field, caps); + }); + } + + // Create RollupFieldCaps for the term + final TermsGroupConfig terms = groupConfig.getTerms(); + if (terms != null) { + final Map termsAggCap = singletonMap("agg", TermsAggregationBuilder.NAME); + Arrays.stream(rollupJobConfig.getGroupConfig().getTerms().getFields()).forEach(field -> { + List> caps = tempFieldCaps.getOrDefault(field, new ArrayList<>()); + caps.add(termsAggCap); + tempFieldCaps.put(field, caps); + }); + } + } + + // Create RollupFieldCaps for the metrics + final List metricsConfig = rollupJobConfig.getMetricsConfig(); + if (metricsConfig.size() > 0) { + rollupJobConfig.getMetricsConfig().forEach(metricConfig -> { + final List> metrics = metricConfig.getMetrics().stream() + .map(metric -> singletonMap("agg", (Object) metric)) + .collect(Collectors.toList()); + metrics.forEach(m -> { + List> caps = tempFieldCaps + .getOrDefault(metricConfig.getField(), new ArrayList<>()); + caps.add(m); + tempFieldCaps.put(metricConfig.getField(), caps); + }); + }); + } + + return Collections.unmodifiableMap(tempFieldCaps.entrySet() + .stream() + .collect(Collectors.toMap(Map.Entry::getKey, + e -> new RollupJobCaps.RollupFieldCaps(e.getValue())))); + } + +} diff --git a/docs/java-rest/high-level/rollup/get_rollup_caps.asciidoc b/docs/java-rest/high-level/rollup/get_rollup_caps.asciidoc new file mode 100644 index 0000000000000..c11f5d231b09d --- /dev/null +++ b/docs/java-rest/high-level/rollup/get_rollup_caps.asciidoc @@ -0,0 +1,83 @@ +-- +:api: rollup-get-rollup-caps +:request: GetRollupCapsRequest +:response: GetRollupCapsResponse +-- + +[id="{upid}-x-pack-{api}"] +=== Get Rollup Capabilities API + +The Get Rollup Capabilities API allows the user to query a target index pattern (`logstash-*`, etc) +and determine if there are any rollup jobs that are/were configured to rollup that pattern. +The API accepts a `GetRollupCapsRequest` object as a request and returns a `GetRollupCapsResponse`. + +[id="{upid}-x-pack-{api}-request"] +==== Get Rollup Capabilities Request + +A +{request}+ requires a single parameter: the target index or index pattern (e.g. `logstash-*`): + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests-file}[x-pack-{api}-request] +-------------------------------------------------- + +[id="{upid}-x-pack-{api}-execution"] +==== Execution + +The Get Rollup Capabilities API can be executed through a `RollupClient` +instance. Such instance can be retrieved from a `RestHighLevelClient` +using the `rollup()` method: + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests-file}[x-pack-{api}-execute] +-------------------------------------------------- + +[id="{upid}-x-pack-{api}-response"] +==== Response + +The returned +{response}+ holds lists and maps of values which correspond to the capabilities +of the target index/index pattern (what jobs were configured for the pattern, where the data is stored, what +aggregations are available, etc). It provides essentially the same data as the original job configuration, +just presented in a different manner. + +For example, if we had created a job with the following config: + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests-file}[x-pack-{api}-setup] +-------------------------------------------------- + +The +{response}+ object would contain the same information, laid out in a slightly different manner: + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests-file}[x-pack-{api}-response] +-------------------------------------------------- + +[id="{upid}-x-pack-{api}-async"] +==== Asynchronous Execution + +This request can be executed asynchronously: + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests-file}[x-pack-{api}-execute-async] +-------------------------------------------------- +<1> The +{request}+ to execute and the `ActionListener` to use when +the execution completes + +The asynchronous method does not block and returns immediately. Once it is +completed the `ActionListener` is called back using the `onResponse` method +if the execution successfully completed or using the `onFailure` method if +it failed. + +A typical listener for +{response}+ looks like: + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests-file}[x-pack-{api}-execute-listener] +-------------------------------------------------- +<1> Called when the execution is successfully completed. The response is +provided as an argument +<2> Called in case of failure. The raised exception is provided as an argument diff --git a/docs/java-rest/high-level/supported-apis.asciidoc b/docs/java-rest/high-level/supported-apis.asciidoc index 5190d6ec13c4a..36c0e3578c1a6 100644 --- a/docs/java-rest/high-level/supported-apis.asciidoc +++ b/docs/java-rest/high-level/supported-apis.asciidoc @@ -301,10 +301,12 @@ The Java High Level REST Client supports the following Rollup APIs: * <> * <<{upid}-rollup-delete-job>> * <> +* <<{upid}-x-pack-rollup-get-rollup-caps>> include::rollup/put_job.asciidoc[] include::rollup/delete_job.asciidoc[] include::rollup/get_job.asciidoc[] +include::rollup/get_rollup_caps.asciidoc[] == Security APIs diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/RollupField.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/RollupField.java index d0de0e02038c2..a6a1feaf133f5 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/RollupField.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/RollupField.java @@ -70,7 +70,7 @@ public class RollupField { * @param extra The type of value this field is (VALUE, INTERVAL, etc) * @return formatted field name */ - public static String formatFieldName(ValuesSourceAggregationBuilder source, String extra) { + public static String formatFieldName(ValuesSourceAggregationBuilder source, String extra) { return source.field() + "." + source.getType() + "." + extra; } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/GetRollupCapsAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/GetRollupCapsAction.java index 128874a6c8c87..d28d14a0ac02d 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/GetRollupCapsAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/GetRollupCapsAction.java @@ -5,7 +5,6 @@ */ package org.elasticsearch.xpack.core.rollup.action; - import org.elasticsearch.action.Action; import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionRequestBuilder; @@ -118,11 +117,11 @@ public Response() { } public Response(Map jobs) { - this.jobs = Objects.requireNonNull(jobs); + this.jobs = Collections.unmodifiableMap(Objects.requireNonNull(jobs)); } Response(StreamInput in) throws IOException { - jobs = in.readMap(StreamInput::readString, RollableIndexCaps::new); + jobs = Collections.unmodifiableMap(in.readMap(StreamInput::readString, RollableIndexCaps::new)); } public Map getJobs() { @@ -138,8 +137,10 @@ public void writeTo(StreamOutput out) throws IOException { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); - for (Map.Entry entry : jobs.entrySet()) { - entry.getValue().toXContent(builder, params); + { + for (Map.Entry entry : jobs.entrySet()) { + entry.getValue().toXContent(builder, params); + } } builder.endObject(); return builder; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/RollableIndexCaps.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/RollableIndexCaps.java index 91f581f1c09e0..d844717167335 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/RollableIndexCaps.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/RollableIndexCaps.java @@ -9,14 +9,15 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; -import org.elasticsearch.common.xcontent.ToXContentFragment; +import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; import java.io.IOException; -import java.util.ArrayList; +import java.util.Collections; import java.util.Comparator; import java.util.List; import java.util.Objects; +import java.util.stream.Collectors; /** * Represents the rollup capabilities of a non-rollup index. E.g. what values/aggregations @@ -25,15 +26,18 @@ * * The index name can either be a single index, or an index pattern (logstash-*) */ -public class RollableIndexCaps implements Writeable, ToXContentFragment { - static ParseField ROLLUP_JOBS = new ParseField("rollup_jobs"); +public class RollableIndexCaps implements Writeable, ToXContentObject { + private static final ParseField ROLLUP_JOBS = new ParseField("rollup_jobs"); - private String indexName; - private List jobCaps; + private final String indexName; + private final List jobCaps; - public RollableIndexCaps(String indexName) { + public RollableIndexCaps(String indexName, List caps) { this.indexName = indexName; - this.jobCaps = new ArrayList<>(); + this.jobCaps = Collections.unmodifiableList(Objects.requireNonNull(caps) + .stream() + .sorted(Comparator.comparing(RollupJobCaps::getJobID)) + .collect(Collectors.toList())); } public RollableIndexCaps(StreamInput in) throws IOException { @@ -41,10 +45,6 @@ public RollableIndexCaps(StreamInput in) throws IOException { this.jobCaps = in.readList(RollupJobCaps::new); } - public void addJobCap(RollupJobCaps jobCap) { - jobCaps.add(jobCap); - } - public String getIndexName() { return indexName; } @@ -62,8 +62,9 @@ public void writeTo(StreamOutput out) throws IOException { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(indexName); - jobCaps.sort(Comparator.comparing(RollupJobCaps::getJobID)); - builder.field(ROLLUP_JOBS.getPreferredName(), jobCaps); + { + builder.field(ROLLUP_JOBS.getPreferredName(), jobCaps); + } builder.endObject(); return builder; } @@ -81,7 +82,7 @@ public boolean equals(Object other) { RollableIndexCaps that = (RollableIndexCaps) other; return Objects.equals(this.jobCaps, that.jobCaps) - && Objects.equals(this.indexName, that.indexName); + && Objects.equals(this.indexName, that.indexName); } @Override diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/RollupJobCaps.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/RollupJobCaps.java index 054d08df999f3..93cf0cbeeb30c 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/RollupJobCaps.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/RollupJobCaps.java @@ -9,6 +9,7 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ToXContentFragment; import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder; @@ -23,6 +24,7 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -41,10 +43,10 @@ public class RollupJobCaps implements Writeable, ToXContentObject { private static ParseField INDEX_PATTERN = new ParseField("index_pattern"); private static ParseField FIELDS = new ParseField("fields"); - private String jobID; - private String rollupIndex; - private String indexPattern; - private Map fieldCapLookup = new HashMap<>(); + private final String jobID; + private final String rollupIndex; + private final String indexPattern; + private final Map fieldCapLookup; // TODO now that these rollup caps are being used more widely (e.g. search), perhaps we should // store the RollupJob and translate into FieldCaps on demand for json output. Would make working with @@ -56,6 +58,13 @@ public RollupJobCaps(RollupJobConfig job) { fieldCapLookup = createRollupFieldCaps(job); } + public RollupJobCaps(String jobID, String rollupIndex, String indexPattern, Map fieldCapLookup) { + this.jobID = jobID; + this.rollupIndex = rollupIndex; + this.indexPattern = indexPattern; + this.fieldCapLookup = Collections.unmodifiableMap(Objects.requireNonNull(fieldCapLookup)); + } + public RollupJobCaps(StreamInput in) throws IOException { this.jobID = in.readString(); this.rollupIndex = in.readString(); @@ -90,15 +99,19 @@ public void writeTo(StreamOutput out) throws IOException { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); - builder.field(JOB_ID.getPreferredName(), jobID); - builder.field(ROLLUP_INDEX.getPreferredName(), rollupIndex); - builder.field(INDEX_PATTERN.getPreferredName(), indexPattern); - builder.startObject(FIELDS.getPreferredName()); - for (Map.Entry fieldCap : fieldCapLookup.entrySet()) { - builder.array(fieldCap.getKey(), fieldCap.getValue()); + { + builder.field(JOB_ID.getPreferredName(), jobID); + builder.field(ROLLUP_INDEX.getPreferredName(), rollupIndex); + builder.field(INDEX_PATTERN.getPreferredName(), indexPattern); + builder.startObject(FIELDS.getPreferredName()); + { + for (Map.Entry fieldCap : fieldCapLookup.entrySet()) { + builder.array(fieldCap.getKey(), fieldCap.getValue()); + } + } + builder.endObject(); } builder.endObject(); - builder.endObject(); return builder; } @@ -121,11 +134,11 @@ public boolean equals(Object other) { @Override public int hashCode() { - return Objects.hash(jobID, rollupIndex, fieldCapLookup); + return Objects.hash(jobID, rollupIndex, fieldCapLookup, indexPattern); } - static Map createRollupFieldCaps(final RollupJobConfig rollupJobConfig) { - final Map fieldCapLookup = new HashMap<>(); + private static Map createRollupFieldCaps(final RollupJobConfig rollupJobConfig) { + final Map>> tempFieldCaps = new HashMap<>(); final GroupConfig groupConfig = rollupJobConfig.getGroupConfig(); if (groupConfig != null) { @@ -139,9 +152,9 @@ static Map createRollupFieldCaps(final RollupJobConfig } dateHistogramAggCap.put(DateHistogramGroupConfig.TIME_ZONE, dateHistogram.getTimeZone()); - final RollupFieldCaps dateHistogramFieldCaps = new RollupFieldCaps(); - dateHistogramFieldCaps.addAgg(dateHistogramAggCap); - fieldCapLookup.put(dateHistogram.getField(), dateHistogramFieldCaps); + List> dateAggCaps = tempFieldCaps.getOrDefault(dateHistogram.getField(), new ArrayList<>()); + dateAggCaps.add(dateHistogramAggCap); + tempFieldCaps.put(dateHistogram.getField(), dateAggCaps); // Create RollupFieldCaps for the histogram final HistogramGroupConfig histogram = groupConfig.getHistogram(); @@ -149,67 +162,61 @@ static Map createRollupFieldCaps(final RollupJobConfig final Map histogramAggCap = new HashMap<>(); histogramAggCap.put("agg", HistogramAggregationBuilder.NAME); histogramAggCap.put(HistogramGroupConfig.INTERVAL, histogram.getInterval()); - for (String field : histogram.getFields()) { - RollupFieldCaps caps = fieldCapLookup.get(field); - if (caps == null) { - caps = new RollupFieldCaps(); - } - caps.addAgg(histogramAggCap); - fieldCapLookup.put(field, caps); - } + Arrays.stream(rollupJobConfig.getGroupConfig().getHistogram().getFields()).forEach(field -> { + List> caps = tempFieldCaps.getOrDefault(field, new ArrayList<>()); + caps.add(histogramAggCap); + tempFieldCaps.put(field, caps); + }); } // Create RollupFieldCaps for the term final TermsGroupConfig terms = groupConfig.getTerms(); if (terms != null) { final Map termsAggCap = singletonMap("agg", TermsAggregationBuilder.NAME); - for (String field : terms.getFields()) { - RollupFieldCaps caps = fieldCapLookup.get(field); - if (caps == null) { - caps = new RollupFieldCaps(); - } - caps.addAgg(termsAggCap); - fieldCapLookup.put(field, caps); - } + Arrays.stream(rollupJobConfig.getGroupConfig().getTerms().getFields()).forEach(field -> { + List> caps = tempFieldCaps.getOrDefault(field, new ArrayList<>()); + caps.add(termsAggCap); + tempFieldCaps.put(field, caps); + }); } } // Create RollupFieldCaps for the metrics final List metricsConfig = rollupJobConfig.getMetricsConfig(); if (metricsConfig.size() > 0) { - metricsConfig.forEach(metricConfig -> { - final List> metrics = metricConfig.getMetrics().stream() - .map(metric -> singletonMap("agg", (Object) metric)) - .collect(Collectors.toList()); - - metrics.forEach(m -> { - RollupFieldCaps caps = fieldCapLookup.get(metricConfig.getField()); - if (caps == null) { - caps = new RollupFieldCaps(); - } - caps.addAgg(m); - fieldCapLookup.put(metricConfig.getField(), caps); + rollupJobConfig.getMetricsConfig().forEach(metricConfig -> { + final List> metrics = metricConfig.getMetrics().stream() + .map(metric -> singletonMap("agg", (Object) metric)) + .collect(Collectors.toList()); + metrics.forEach(m -> { + List> caps = tempFieldCaps + .getOrDefault(metricConfig.getField(), new ArrayList<>()); + caps.add(m); + tempFieldCaps.put(metricConfig.getField(), caps); + }); }); - }); } - return Collections.unmodifiableMap(fieldCapLookup); - } - public static class RollupFieldCaps implements Writeable, ToXContentObject { - private List> aggs = new ArrayList<>(); + return Collections.unmodifiableMap(tempFieldCaps.entrySet() + .stream() + .collect(Collectors.toMap(Map.Entry::getKey, + e -> new RollupFieldCaps(e.getValue())))); + } - RollupFieldCaps() { } + public static class RollupFieldCaps implements Writeable, ToXContentFragment { + private final List> aggs; RollupFieldCaps(StreamInput in) throws IOException { int size = in.readInt(); - aggs = new ArrayList<>(size); + List> inAggs = new ArrayList<>(size); for (int i = 0; i < size; i++) { - aggs.add(in.readMap()); + inAggs.add(in.readMap()); } + this.aggs = Collections.unmodifiableList(inAggs); } - void addAgg(Map agg) { - aggs.add(agg); + RollupFieldCaps(List> aggs) { + this.aggs = Collections.unmodifiableList(Objects.requireNonNull(aggs)); } public List> getAggs() { @@ -243,7 +250,6 @@ public boolean equals(Object other) { } RollupFieldCaps that = (RollupFieldCaps) other; - return Objects.equals(this.aggs, that.aggs); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/GroupConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/GroupConfig.java index f7685f4e6143b..b7c69ecda0ee2 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/GroupConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/GroupConfig.java @@ -75,7 +75,7 @@ public GroupConfig(final DateHistogramGroupConfig dateHistogram, this.terms = terms; } - GroupConfig(final StreamInput in) throws IOException { + public GroupConfig(final StreamInput in) throws IOException { dateHistogram = new DateHistogramGroupConfig(in); histogram = in.readOptionalWriteable(HistogramGroupConfig::new); terms = in.readOptionalWriteable(TermsGroupConfig::new); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/RollupJobConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/RollupJobConfig.java index 68e2dc50d9f5c..51a4736e3adee 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/RollupJobConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/RollupJobConfig.java @@ -20,7 +20,6 @@ import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.xpack.core.scheduler.Cron; import java.io.IOException; import java.util.Collections; @@ -116,10 +115,6 @@ public RollupJobConfig(final String id, if (pageSize <= 0) { throw new IllegalArgumentException("Page size is mandatory and must be a positive long"); } - // Cron doesn't have a parse helper method to see if the cron is valid, - // so just construct a temporary cron object and if the cron is bad, it'll - // throw an exception - Cron testCron = new Cron(cron); if (groupConfig == null && (metricsConfig == null || metricsConfig.isEmpty())) { throw new IllegalArgumentException("At least one grouping or metric must be configured"); } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/RollupJobConfigTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/RollupJobConfigTests.java index a5a82bc2bb090..09d00e11fef92 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/RollupJobConfigTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/RollupJobConfigTests.java @@ -87,15 +87,6 @@ public void testEmptyID() { assertThat(e.getMessage(), equalTo("Id must be a non-null, non-empty string")); } - public void testBadCron() { - final RollupJobConfig sample = randomRollupJobConfig(random()); - - Exception e = expectThrows(IllegalArgumentException.class, () -> - new RollupJobConfig(sample.getId(), sample.getIndexPattern(), sample.getRollupIndex(), "0 * * *", sample.getPageSize(), - sample.getGroupConfig(), sample.getMetricsConfig(), sample.getTimeout())); - assertThat(e.getMessage(), equalTo("invalid cron expression [0 * * *]")); - } - public void testMatchAllIndexPattern() { final RollupJobConfig sample = randomRollupJobConfig(random()); diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/TermsGroupConfigSerializingTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/TermsGroupConfigSerializingTests.java index b0e33579eb353..f817f83574070 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/TermsGroupConfigSerializingTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/TermsGroupConfigSerializingTests.java @@ -19,6 +19,7 @@ import static org.elasticsearch.xpack.core.rollup.ConfigTestHelpers.randomTermsGroupConfig; import static org.hamcrest.Matchers.equalTo; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; public class TermsGroupConfigSerializingTests extends AbstractSerializingTestCase { @@ -74,4 +75,59 @@ public void testValidateFieldWrongType() { assertThat(e.validationErrors().get(0), equalTo("The field referenced by a terms group must be a [numeric] or " + "[keyword/text] type, but found [geo_point] for field [my_field]")); } + + public void testValidateFieldMatchingNotAggregatable() { + ActionRequestValidationException e = new ActionRequestValidationException(); + Map> responseMap = new HashMap<>(); + + // Have to mock fieldcaps because the ctor's aren't public... + FieldCapabilities fieldCaps = mock(FieldCapabilities.class); + when(fieldCaps.isAggregatable()).thenReturn(false); + responseMap.put("my_field", Collections.singletonMap(getRandomType(), fieldCaps)); + + TermsGroupConfig config = new TermsGroupConfig("my_field"); + config.validateMappings(responseMap, e); + assertThat(e.validationErrors().get(0), equalTo("The field [my_field] must be aggregatable across all indices, but is not.")); + } + + public void testValidateMatchingField() { + ActionRequestValidationException e = new ActionRequestValidationException(); + Map> responseMap = new HashMap<>(); + String type = getRandomType(); + + // Have to mock fieldcaps because the ctor's aren't public... + FieldCapabilities fieldCaps = mock(FieldCapabilities.class); + when(fieldCaps.isAggregatable()).thenReturn(true); + responseMap.put("my_field", Collections.singletonMap(type, fieldCaps)); + + TermsGroupConfig config = new TermsGroupConfig("my_field"); + config.validateMappings(responseMap, e); + if (e.validationErrors().size() != 0) { + fail(e.getMessage()); + } + } + + private String getRandomType() { + int n = randomIntBetween(0,8); + if (n == 0) { + return "keyword"; + } else if (n == 1) { + return "text"; + } else if (n == 2) { + return "long"; + } else if (n == 3) { + return "integer"; + } else if (n == 4) { + return "short"; + } else if (n == 5) { + return "float"; + } else if (n == 6) { + return "double"; + } else if (n == 7) { + return "scaled_float"; + } else if (n == 8) { + return "half_float"; + } + return "long"; + } } diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/RollupResponseTranslator.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/RollupResponseTranslator.java index 0c1ca89f32d77..41edf220e667e 100644 --- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/RollupResponseTranslator.java +++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/RollupResponseTranslator.java @@ -27,10 +27,10 @@ import org.elasticsearch.search.aggregations.bucket.histogram.InternalHistogram; import org.elasticsearch.search.aggregations.bucket.terms.LongTerms; import org.elasticsearch.search.aggregations.bucket.terms.StringTerms; -import org.elasticsearch.search.aggregations.metrics.InternalNumericMetricsAggregation.SingleValue; import org.elasticsearch.search.aggregations.metrics.InternalAvg; import org.elasticsearch.search.aggregations.metrics.InternalMax; import org.elasticsearch.search.aggregations.metrics.InternalMin; +import org.elasticsearch.search.aggregations.metrics.InternalNumericMetricsAggregation.SingleValue; import org.elasticsearch.search.aggregations.metrics.InternalSum; import org.elasticsearch.search.aggregations.metrics.SumAggregationBuilder; import org.elasticsearch.search.internal.InternalSearchResponse; diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportGetRollupCapsAction.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportGetRollupCapsAction.java index 6d565e43b8644..af36d7bc6718c 100644 --- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportGetRollupCapsAction.java +++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportGetRollupCapsAction.java @@ -24,11 +24,13 @@ import org.elasticsearch.xpack.core.rollup.action.RollableIndexCaps; import org.elasticsearch.xpack.core.rollup.action.RollupJobCaps; +import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.TreeMap; import java.util.function.Supplier; +import java.util.stream.Collectors; public class TransportGetRollupCapsAction extends HandledTransportAction { @@ -49,7 +51,7 @@ protected void doExecute(Task task, GetRollupCapsAction.Request request, ActionL } static Map getCaps(String indexPattern, ImmutableOpenMap indices) { - Map allCaps = new TreeMap<>(); + Map > allCaps = new TreeMap<>(); for (ObjectObjectCursor entry : indices) { // Does this index have rollup metadata? @@ -69,16 +71,21 @@ static Map getCaps(String indexPattern, ImmutableOpen ? jobCap.getIndexPattern() : indexPattern; // Do we already have an entry for this index pattern? - RollableIndexCaps indexCaps = allCaps.get(pattern); + List indexCaps = allCaps.get(pattern); if (indexCaps == null) { - indexCaps = new RollableIndexCaps(pattern); + indexCaps = new ArrayList<>(); } - indexCaps.addJobCap(jobCap); + indexCaps.add(jobCap); allCaps.put(pattern, indexCaps); }); }); } - return allCaps; + + // Convert the mutable lists into the RollableIndexCaps + return allCaps.entrySet() + .stream() + .collect(Collectors.toMap(Map.Entry::getKey, + e -> new RollableIndexCaps(e.getKey(), e.getValue()))); } static Optional findRollupIndexCaps(String indexName, IndexMetaData indexMetaData) { diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportGetRollupIndexCapsAction.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportGetRollupIndexCapsAction.java index 5d81f4046ebd8..518fe7ec29f0f 100644 --- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportGetRollupIndexCapsAction.java +++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportGetRollupIndexCapsAction.java @@ -18,12 +18,15 @@ import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.rollup.action.GetRollupIndexCapsAction; import org.elasticsearch.xpack.core.rollup.action.RollableIndexCaps; +import org.elasticsearch.xpack.core.rollup.action.RollupJobCaps; +import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.TreeMap; import java.util.function.Supplier; +import java.util.stream.Collectors; import java.util.stream.StreamSupport; public class TransportGetRollupIndexCapsAction extends HandledTransportAction getCapsByRollupIndex(List resolvedIndexNames, ImmutableOpenMap indices) { - Map allCaps = new TreeMap<>(); + Map > allCaps = new TreeMap<>(); StreamSupport.stream(indices.spliterator(), false) .filter(entry -> resolvedIndexNames.contains(entry.key)) @@ -63,17 +66,20 @@ static Map getCapsByRollupIndex(List resolved .ifPresent(cap -> { cap.getJobCaps().forEach(jobCap -> { // Do we already have an entry for this index? - RollableIndexCaps indexCaps = allCaps.get(jobCap.getRollupIndex()); + List indexCaps = allCaps.get(jobCap.getRollupIndex()); if (indexCaps == null) { - indexCaps = new RollableIndexCaps(jobCap.getRollupIndex()); + indexCaps = new ArrayList<>(); } - indexCaps.addJobCap(jobCap); + indexCaps.add(jobCap); allCaps.put(jobCap.getRollupIndex(), indexCaps); }); }); }); - - return allCaps; + // Convert the mutable lists into the RollableIndexCaps + return allCaps.entrySet() + .stream() + .collect(Collectors.toMap(Map.Entry::getKey, + e -> new RollableIndexCaps(e.getKey(), e.getValue()))); } } diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportGetRollupJobAction.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportGetRollupJobAction.java index a72dbfbe6b94f..0f684de9ea268 100644 --- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportGetRollupJobAction.java +++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportGetRollupJobAction.java @@ -124,4 +124,4 @@ protected GetRollupJobsAction.Response newResponse(GetRollupJobsAction.Request r protected GetRollupJobsAction.Response readTaskResponse(StreamInput in) throws IOException { return new GetRollupJobsAction.Response(in); } -} \ No newline at end of file +} diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/rest/RestGetRollupCapsAction.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/rest/RestGetRollupCapsAction.java index dfda4d0e9e15b..0619666ebd1bd 100644 --- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/rest/RestGetRollupCapsAction.java +++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/rest/RestGetRollupCapsAction.java @@ -12,8 +12,8 @@ import org.elasticsearch.rest.RestController; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.action.RestToXContentListener; -import org.elasticsearch.xpack.rollup.Rollup; import org.elasticsearch.xpack.core.rollup.action.GetRollupCapsAction; +import org.elasticsearch.xpack.rollup.Rollup; import java.io.IOException; diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/rest/RestStartRollupJobAction.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/rest/RestStartRollupJobAction.java index dfa5c977f2a5f..b64d5a719d1b1 100644 --- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/rest/RestStartRollupJobAction.java +++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/rest/RestStartRollupJobAction.java @@ -12,8 +12,8 @@ import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.action.RestToXContentListener; import org.elasticsearch.xpack.core.rollup.RollupField; -import org.elasticsearch.xpack.rollup.Rollup; import org.elasticsearch.xpack.core.rollup.action.StartRollupJobAction; +import org.elasticsearch.xpack.rollup.Rollup; import java.io.IOException; diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/rest/RestStopRollupJobAction.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/rest/RestStopRollupJobAction.java index f51952b7f6143..0f03cdd1ebe2c 100644 --- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/rest/RestStopRollupJobAction.java +++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/rest/RestStopRollupJobAction.java @@ -12,8 +12,8 @@ import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.action.RestToXContentListener; import org.elasticsearch.xpack.core.rollup.RollupField; -import org.elasticsearch.xpack.rollup.Rollup; import org.elasticsearch.xpack.core.rollup.action.StopRollupJobAction; +import org.elasticsearch.xpack.rollup.Rollup; import java.io.IOException; diff --git a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/RollupResponseTranslationTests.java b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/RollupResponseTranslationTests.java index 0a133cc8e0754..576f37d78440e 100644 --- a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/RollupResponseTranslationTests.java +++ b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/RollupResponseTranslationTests.java @@ -56,13 +56,13 @@ import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder; import org.elasticsearch.search.aggregations.metrics.Avg; import org.elasticsearch.search.aggregations.metrics.AvgAggregationBuilder; -import org.elasticsearch.search.aggregations.metrics.InternalAvg; import org.elasticsearch.search.aggregations.metrics.CardinalityAggregationBuilder; import org.elasticsearch.search.aggregations.metrics.GeoBoundsAggregationBuilder; +import org.elasticsearch.search.aggregations.metrics.InternalAvg; import org.elasticsearch.search.aggregations.metrics.InternalMax; +import org.elasticsearch.search.aggregations.metrics.InternalSum; import org.elasticsearch.search.aggregations.metrics.MaxAggregationBuilder; import org.elasticsearch.search.aggregations.metrics.MinAggregationBuilder; -import org.elasticsearch.search.aggregations.metrics.InternalSum; import org.elasticsearch.search.aggregations.metrics.SumAggregationBuilder; import org.elasticsearch.search.aggregations.support.ValueType; import org.elasticsearch.search.internal.InternalSearchResponse; @@ -76,7 +76,6 @@ import java.util.Map; import static java.util.Collections.singleton; -import static org.elasticsearch.xpack.core.rollup.RollupField.COUNT_FIELD; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.not; import static org.hamcrest.core.IsEqual.equalTo; @@ -144,7 +143,7 @@ public void testMissingLiveIndex() { List subaggs = new ArrayList<>(2); Map metadata = new HashMap<>(1); - metadata.put(RollupField.ROLLUP_META + "." + COUNT_FIELD, "foo." + COUNT_FIELD); + metadata.put(RollupField.ROLLUP_META + "." + RollupField.COUNT_FIELD, "foo." + RollupField.COUNT_FIELD); InternalSum sum = mock(InternalSum.class); when(sum.getValue()).thenReturn(10.0); when(sum.value()).thenReturn(10.0); @@ -243,7 +242,7 @@ public void testTranslateRollup() { List subaggs = new ArrayList<>(2); Map metadata = new HashMap<>(1); - metadata.put(RollupField.ROLLUP_META + "." + COUNT_FIELD, "foo." + COUNT_FIELD); + metadata.put(RollupField.ROLLUP_META + "." + RollupField.COUNT_FIELD, "foo." + RollupField.COUNT_FIELD); InternalSum sum = mock(InternalSum.class); when(sum.getValue()).thenReturn(10.0); when(sum.value()).thenReturn(10.0); @@ -368,7 +367,7 @@ public void testSimpleReduction() { List subaggs = new ArrayList<>(2); Map metadata = new HashMap<>(1); - metadata.put(RollupField.ROLLUP_META + "." + COUNT_FIELD, "foo." + COUNT_FIELD); + metadata.put(RollupField.ROLLUP_META + "." + RollupField.COUNT_FIELD, "foo." + RollupField.COUNT_FIELD); InternalSum sum = mock(InternalSum.class); when(sum.getValue()).thenReturn(10.0); when(sum.value()).thenReturn(10.0); @@ -1050,7 +1049,7 @@ public void testStringTerms() throws IOException { TermsAggregationBuilder rollupTerms = new TermsAggregationBuilder("terms", ValueType.STRING) .field("stringfield.terms." + RollupField.VALUE) - .subAggregation(new SumAggregationBuilder("terms." + COUNT_FIELD) + .subAggregation(new SumAggregationBuilder("terms." + RollupField.COUNT_FIELD) .field("stringfield.terms." + RollupField.COUNT_FIELD)); KeywordFieldMapper.Builder nrBuilder = new KeywordFieldMapper.Builder("terms"); @@ -1091,7 +1090,7 @@ public void testStringTermsNullValue() throws IOException { TermsAggregationBuilder rollupTerms = new TermsAggregationBuilder("terms", ValueType.STRING) .field("stringfield.terms." + RollupField.VALUE) - .subAggregation(new SumAggregationBuilder("terms." + COUNT_FIELD) + .subAggregation(new SumAggregationBuilder("terms." + RollupField.COUNT_FIELD) .field("stringfield.terms." + RollupField.COUNT_FIELD)); KeywordFieldMapper.Builder nrBuilder = new KeywordFieldMapper.Builder("terms"); @@ -1139,7 +1138,7 @@ public void testLongTerms() throws IOException { TermsAggregationBuilder rollupTerms = new TermsAggregationBuilder("terms", ValueType.LONG) .field("longfield.terms." + RollupField.VALUE) - .subAggregation(new SumAggregationBuilder("terms." + COUNT_FIELD) + .subAggregation(new SumAggregationBuilder("terms." + RollupField.COUNT_FIELD) .field("longfield.terms." + RollupField.COUNT_FIELD)); NumberFieldMapper.Builder nrBuilder = new NumberFieldMapper.Builder("terms", NumberFieldMapper.NumberType.LONG); diff --git a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/GetRollupCapsActionRequestTests.java b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/GetRollupCapsActionRequestTests.java index 9068bcfce36a4..fd0021513e814 100644 --- a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/GetRollupCapsActionRequestTests.java +++ b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/GetRollupCapsActionRequestTests.java @@ -75,8 +75,8 @@ public void testMissingJob() throws IOException { String indexPattern = randomBoolean() ? randomAlphaOfLength(10) : randomAlphaOfLength(10) + "-*"; MappingMetaData mappingMeta = new MappingMetaData(RollupField.NAME, Collections.singletonMap(RollupField.NAME, - Collections.singletonMap("_meta", - Collections.emptyMap()))); + Collections.singletonMap("_meta", + Collections.emptyMap()))); ImmutableOpenMap.Builder mappings = ImmutableOpenMap.builder(1); mappings.put(RollupField.NAME, mappingMeta); @@ -92,10 +92,10 @@ public void testOneJob() throws IOException { RollupJobConfig job = ConfigTestHelpers.randomRollupJobConfig(random(), jobName); MappingMetaData mappingMeta = new MappingMetaData(RollupField.TYPE_NAME, - Collections.singletonMap(RollupField.TYPE_NAME, - Collections.singletonMap("_meta", - Collections.singletonMap(RollupField.ROLLUP_META, - Collections.singletonMap(jobName, job))))); + Collections.singletonMap(RollupField.TYPE_NAME, + Collections.singletonMap("_meta", + Collections.singletonMap(RollupField.ROLLUP_META, + Collections.singletonMap(jobName, job))))); ImmutableOpenMap.Builder mappings = ImmutableOpenMap.builder(1); mappings.put(RollupField.TYPE_NAME, mappingMeta); @@ -117,9 +117,9 @@ public void testMultipleJobs() throws IOException { } MappingMetaData mappingMeta = new MappingMetaData(RollupField.TYPE_NAME, - Collections.singletonMap(RollupField.TYPE_NAME, - Collections.singletonMap("_meta", - Collections.singletonMap(RollupField.ROLLUP_META, jobs)))); + Collections.singletonMap(RollupField.TYPE_NAME, + Collections.singletonMap("_meta", + Collections.singletonMap(RollupField.ROLLUP_META, jobs)))); ImmutableOpenMap.Builder mappings = ImmutableOpenMap.builder(1); mappings.put(RollupField.TYPE_NAME, mappingMeta); @@ -151,9 +151,9 @@ public void testAllIndices() throws IOException { } MappingMetaData mappingMeta = new MappingMetaData(RollupField.TYPE_NAME, - Collections.singletonMap(RollupField.TYPE_NAME, - Collections.singletonMap("_meta", - Collections.singletonMap(RollupField.ROLLUP_META, jobs)))); + Collections.singletonMap(RollupField.TYPE_NAME, + Collections.singletonMap("_meta", + Collections.singletonMap(RollupField.ROLLUP_META, jobs)))); ImmutableOpenMap.Builder mappings = ImmutableOpenMap.builder(1); mappings.put(RollupField.TYPE_NAME, mappingMeta); @@ -183,9 +183,9 @@ public void testOneIndex() throws IOException { } MappingMetaData mappingMeta = new MappingMetaData(RollupField.TYPE_NAME, - Collections.singletonMap(RollupField.TYPE_NAME, - Collections.singletonMap("_meta", - Collections.singletonMap(RollupField.ROLLUP_META, jobs)))); + Collections.singletonMap(RollupField.TYPE_NAME, + Collections.singletonMap("_meta", + Collections.singletonMap(RollupField.ROLLUP_META, jobs)))); ImmutableOpenMap.Builder mappings = ImmutableOpenMap.builder(1); mappings.put(RollupField.TYPE_NAME, mappingMeta); diff --git a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/PutJobActionRequestTests.java b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/PutJobActionRequestTests.java index 848bd5f13dd9e..2765c9ca8c1d2 100644 --- a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/PutJobActionRequestTests.java +++ b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/PutJobActionRequestTests.java @@ -8,8 +8,8 @@ import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.test.AbstractStreamableXContentTestCase; -import org.elasticsearch.xpack.core.rollup.action.PutRollupJobAction.Request; import org.elasticsearch.xpack.core.rollup.ConfigTestHelpers; +import org.elasticsearch.xpack.core.rollup.action.PutRollupJobAction.Request; import org.junit.Before; import java.io.IOException; @@ -39,8 +39,7 @@ protected Request createBlankInstance() { } @Override - protected Request doParseInstance(final XContentParser parser) throws IOException { + protected Request doParseInstance(XContentParser parser) throws IOException { return Request.fromXContent(parser, jobId); } - } diff --git a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/SearchActionTests.java b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/SearchActionTests.java index 5f9bd9fa01d6f..c2c5096fcd7de 100644 --- a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/SearchActionTests.java +++ b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/SearchActionTests.java @@ -71,7 +71,6 @@ import static java.util.Collections.emptyList; import static java.util.Collections.singleton; import static org.elasticsearch.xpack.core.rollup.ConfigTestHelpers.randomHistogramGroupConfig; -import static org.elasticsearch.xpack.core.rollup.RollupField.COUNT_FIELD; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; import static org.mockito.Mockito.mock; @@ -630,7 +629,7 @@ public void testRollupOnly() throws IOException { List subaggs = new ArrayList<>(2); Map metadata = new HashMap<>(1); - metadata.put(RollupField.ROLLUP_META + "." + COUNT_FIELD, "foo." + COUNT_FIELD); + metadata.put(RollupField.ROLLUP_META + "." + RollupField.COUNT_FIELD, "foo." + RollupField.COUNT_FIELD); InternalSum sum = mock(InternalSum.class); when(sum.getValue()).thenReturn(10.0); when(sum.value()).thenReturn(10.0); @@ -747,7 +746,7 @@ public void testBoth() throws IOException { List subaggs = new ArrayList<>(2); Map metadata = new HashMap<>(1); - metadata.put(RollupField.ROLLUP_META + "." + COUNT_FIELD, "foo." + COUNT_FIELD); + metadata.put(RollupField.ROLLUP_META + "." + RollupField.COUNT_FIELD, "foo." + RollupField.COUNT_FIELD); InternalSum sum = mock(InternalSum.class); when(sum.getValue()).thenReturn(10.0); when(sum.value()).thenReturn(10.0);