From 32fa6603f5c5ed22abd209ae8d7fc3c832b28d17 Mon Sep 17 00:00:00 2001 From: Benjamin Trent Date: Thu, 14 Feb 2019 07:59:44 -0600 Subject: [PATCH 1/8] [DATA-FRAME] add preview endpoint --- .../integration/DataFramePivotRestIT.java | 35 ++++ .../xpack/dataframe/DataFrame.java | 9 +- .../PreviewDataFrameTransformAction.java | 163 ++++++++++++++++++ ...nsportPreviewDataFrameTransformAction.java | 56 ++++++ .../RestPreviewDataFrameTransformAction.java | 42 +++++ .../transforms/DataFrameIndexer.java | 2 +- .../transforms/DataFramePreviewer.java | 52 ++++++ 7 files changed, 356 insertions(+), 3 deletions(-) create mode 100644 x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/action/PreviewDataFrameTransformAction.java create mode 100644 x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/action/TransportPreviewDataFrameTransformAction.java create mode 100644 x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/rest/action/RestPreviewDataFrameTransformAction.java create mode 100644 x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/transforms/DataFramePreviewer.java diff --git a/x-pack/plugin/data-frame/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFramePivotRestIT.java b/x-pack/plugin/data-frame/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFramePivotRestIT.java index eb8203e1dd2e2..6245561d1060c 100644 --- a/x-pack/plugin/data-frame/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFramePivotRestIT.java +++ b/x-pack/plugin/data-frame/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFramePivotRestIT.java @@ -10,10 +10,14 @@ import org.elasticsearch.client.Response; import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.junit.Before; +import org.junit.Ignore; import java.io.IOException; +import java.util.Arrays; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.TimeUnit; import static org.hamcrest.Matchers.equalTo; @@ -224,6 +228,37 @@ public void testDateHistogramPivot() throws Exception { assertOnePivotValue(dataFrameIndex + "/_search?q=by_day:2017-01-15", 3.82); } + @SuppressWarnings("unchecked") + public void testPreviewTransform() throws Exception { + final Request createPreviewRequest = new Request("POST", DATAFRAME_ENDPOINT + "/_preview"); + + String config = "{" + + " \"source\": \"reviews\"," + + " \"id\": \"doesnot-matter\"," + + " \"dest\": \"doesnot-matter\","; + + config += " \"pivot\": {" + + " \"group_by\": [ {" + + " {\"reviewer\": {\"terms\": { \"field\": \"user_id\" }}}," + + " {\"by_day\": {\"date_histogram\": " + + " {\"interval\": \"1d\",\"field\":\"timestamp\",\"format\":\"yyyy-MM-DD\"} } } ]," + + " \"aggregations\": {" + + " \"avg_rating\": {" + + " \"avg\": {" + + " \"field\": \"stars\"" + + " } } } }" + + "}"; + createPreviewRequest.setJsonEntity(config); + Map previewDataframeResponse = entityAsMap(client().performRequest(createPreviewRequest)); + List> preview = (List>)previewDataframeResponse.get("preview"); + assertThat(preview.size(), equalTo(10)); + Set expectedFields = new HashSet<>(Arrays.asList("reviewer", "by_day", "avg_rating")); + preview.forEach(p -> { + Set keys = p.keySet(); + assertThat(keys, equalTo(expectedFields)); + }); + } + private void startAndWaitForTransform(String transformId, String dataFrameIndex) throws IOException, Exception { // start the transform final Request startTransformRequest = new Request("POST", DATAFRAME_ENDPOINT + transformId + "/_start"); diff --git a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/DataFrame.java b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/DataFrame.java index 4ef39d630f06c..18e8f01055af9 100644 --- a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/DataFrame.java +++ b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/DataFrame.java @@ -50,12 +50,14 @@ import org.elasticsearch.xpack.dataframe.action.DeleteDataFrameTransformAction; import org.elasticsearch.xpack.dataframe.action.GetDataFrameTransformsAction; import org.elasticsearch.xpack.dataframe.action.GetDataFrameTransformsStatsAction; +import org.elasticsearch.xpack.dataframe.action.PreviewDataFrameTransformAction; import org.elasticsearch.xpack.dataframe.action.PutDataFrameTransformAction; import org.elasticsearch.xpack.dataframe.action.StartDataFrameTransformAction; import org.elasticsearch.xpack.dataframe.action.StopDataFrameTransformAction; import org.elasticsearch.xpack.dataframe.action.TransportDeleteDataFrameTransformAction; import org.elasticsearch.xpack.dataframe.action.TransportGetDataFrameTransformsAction; import org.elasticsearch.xpack.dataframe.action.TransportGetDataFrameTransformsStatsAction; +import org.elasticsearch.xpack.dataframe.action.TransportPreviewDataFrameTransformAction; import org.elasticsearch.xpack.dataframe.action.TransportPutDataFrameTransformAction; import org.elasticsearch.xpack.dataframe.action.TransportStartDataFrameTransformAction; import org.elasticsearch.xpack.dataframe.action.TransportStopDataFrameTransformAction; @@ -64,6 +66,7 @@ import org.elasticsearch.xpack.dataframe.rest.action.RestDeleteDataFrameTransformAction; import org.elasticsearch.xpack.dataframe.rest.action.RestGetDataFrameTransformsAction; import org.elasticsearch.xpack.dataframe.rest.action.RestGetDataFrameTransformsStatsAction; +import org.elasticsearch.xpack.dataframe.rest.action.RestPreviewDataFrameTransformAction; import org.elasticsearch.xpack.dataframe.rest.action.RestPutDataFrameTransformAction; import org.elasticsearch.xpack.dataframe.rest.action.RestStartDataFrameTransformAction; import org.elasticsearch.xpack.dataframe.rest.action.RestStopDataFrameTransformAction; @@ -137,7 +140,8 @@ public List getRestHandlers(final Settings settings, final RestCont new RestStopDataFrameTransformAction(settings, restController), new RestDeleteDataFrameTransformAction(settings, restController), new RestGetDataFrameTransformsAction(settings, restController), - new RestGetDataFrameTransformsStatsAction(settings, restController) + new RestGetDataFrameTransformsStatsAction(settings, restController), + new RestPreviewDataFrameTransformAction(settings, restController) ); } @@ -153,7 +157,8 @@ public List getRestHandlers(final Settings settings, final RestCont new ActionHandler<>(StopDataFrameTransformAction.INSTANCE, TransportStopDataFrameTransformAction.class), new ActionHandler<>(DeleteDataFrameTransformAction.INSTANCE, TransportDeleteDataFrameTransformAction.class), new ActionHandler<>(GetDataFrameTransformsAction.INSTANCE, TransportGetDataFrameTransformsAction.class), - new ActionHandler<>(GetDataFrameTransformsStatsAction.INSTANCE, TransportGetDataFrameTransformsStatsAction.class) + new ActionHandler<>(GetDataFrameTransformsStatsAction.INSTANCE, TransportGetDataFrameTransformsStatsAction.class), + new ActionHandler<>(PreviewDataFrameTransformAction.INSTANCE, TransportPreviewDataFrameTransformAction.class) ); } diff --git a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/action/PreviewDataFrameTransformAction.java b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/action/PreviewDataFrameTransformAction.java new file mode 100644 index 0000000000000..e6c6844633be8 --- /dev/null +++ b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/action/PreviewDataFrameTransformAction.java @@ -0,0 +1,163 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.dataframe.action; + +import org.elasticsearch.action.Action; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.action.support.master.AcknowledgedRequest; +import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.action.support.master.MasterNodeOperationRequestBuilder; +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.common.collect.HppcMaps; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.ToXContentObject; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.xpack.dataframe.transforms.DataFrameTransformConfig; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +public class PreviewDataFrameTransformAction extends Action { + + public static final PreviewDataFrameTransformAction INSTANCE = new PreviewDataFrameTransformAction(); + public static final String NAME = "cluster:admin/data_frame/preview"; + + private PreviewDataFrameTransformAction() { + super(NAME); + } + + @Override + public Response newResponse() { + return new Response(); + } + + public static class Request extends AcknowledgedRequest implements ToXContentObject { + + private DataFrameTransformConfig config; + + public Request(DataFrameTransformConfig config) { + this.setConfig(config); + } + + public Request() { + + } + + public static Request fromXContent(final XContentParser parser) throws IOException { + return new Request(DataFrameTransformConfig.fromXContent(parser, null, false)); + } + + @Override + public ActionRequestValidationException validate() { + return null; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + return this.config.toXContent(builder, params); + } + + public DataFrameTransformConfig getConfig() { + return config; + } + + public void setConfig(DataFrameTransformConfig config) { + this.config = config; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + this.config = new DataFrameTransformConfig(in); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + this.config.writeTo(out); + } + + @Override + public int hashCode() { + return Objects.hash(config); + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + Request other = (Request) obj; + return Objects.equals(config, other.config); + } + } + + public static class RequestBuilder extends MasterNodeOperationRequestBuilder { + + protected RequestBuilder(ElasticsearchClient client, PreviewDataFrameTransformAction action) { + super(client, action, new Request()); + } + } + + public static class Response extends ActionResponse implements ToXContentObject { + + private List> docs; + + public Response() {} + + public Response(StreamInput in) throws IOException { + int size = in.readInt(); + this.docs = new ArrayList<>(size); + for (int i = 0; i < size; i++) { + this.docs.add(in.readMap()); + } + } + + public Response(List> docs) { + this.docs = new ArrayList<>(docs); + } + + public void setDocs(List> docs) { + this.docs = new ArrayList<>(docs); + } + + @Override + public void readFrom(StreamInput in) throws IOException { + int size = in.readInt(); + this.docs = new ArrayList<>(size); + for (int i = 0; i < size; i++) { + this.docs.add(in.readMap()); + } + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeInt(docs.size()); + for (Map doc : docs) { + out.writeMap(doc); + } + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field("data_frame_preview", docs); + builder.endObject(); + return builder; + } + } +} diff --git a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/action/TransportPreviewDataFrameTransformAction.java b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/action/TransportPreviewDataFrameTransformAction.java new file mode 100644 index 0000000000000..5ff995e54eabc --- /dev/null +++ b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/action/TransportPreviewDataFrameTransformAction.java @@ -0,0 +1,56 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.dataframe.action; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.HandledTransportAction; +import org.elasticsearch.client.Client; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.license.LicenseUtils; +import org.elasticsearch.license.XPackLicenseState; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.core.XPackField; +import org.elasticsearch.xpack.dataframe.transforms.DataFramePreviewer; + +import java.util.function.Supplier; + +public class TransportPreviewDataFrameTransformAction extends + HandledTransportAction { + + private final XPackLicenseState licenseState; + private final Client client; + private final ThreadPool threadPool; + + @Inject + public TransportPreviewDataFrameTransformAction(TransportService transportService, ActionFilters actionFilters, + Client client, ThreadPool threadPool, XPackLicenseState licenseState) { + super(PreviewDataFrameTransformAction.NAME,transportService, actionFilters, + (Supplier) PreviewDataFrameTransformAction.Request::new); + this.licenseState = licenseState; + this.client = client; + this.threadPool = threadPool; + } + + @Override + protected void doExecute(Task task, + PreviewDataFrameTransformAction.Request request, + ActionListener listener) { + if (!licenseState.isDataFrameAllowed()) { + listener.onFailure(LicenseUtils.newComplianceException(XPackField.DATA_FRAME)); + return; + } + + DataFramePreviewer previewer = new DataFramePreviewer(request.getConfig(), threadPool.getThreadContext().getHeaders()); + previewer.getPreview(client, ActionListener.wrap( + previewResponse -> listener.onResponse(new PreviewDataFrameTransformAction.Response(previewResponse)), + listener::onFailure + )); + } +} diff --git a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/rest/action/RestPreviewDataFrameTransformAction.java b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/rest/action/RestPreviewDataFrameTransformAction.java new file mode 100644 index 0000000000000..088d675f882db --- /dev/null +++ b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/rest/action/RestPreviewDataFrameTransformAction.java @@ -0,0 +1,42 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.dataframe.rest.action; + +import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestController; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.action.RestToXContentListener; +import org.elasticsearch.xpack.core.dataframe.DataFrameField; +import org.elasticsearch.xpack.dataframe.action.PreviewDataFrameTransformAction; +import org.elasticsearch.xpack.dataframe.action.PutDataFrameTransformAction; +import org.elasticsearch.xpack.dataframe.transforms.DataFrameTransformConfig; + +import java.io.IOException; + +public class RestPreviewDataFrameTransformAction extends BaseRestHandler { + + public RestPreviewDataFrameTransformAction(Settings settings, RestController controller) { + super(settings); + controller.registerHandler(RestRequest.Method.POST, DataFrameField.REST_BASE_PATH + "transforms/_preview", this); + } + + @Override + public String getName() { + return "data_frame_preview_transform_action"; + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException { + XContentParser parser = restRequest.contentParser(); + + PreviewDataFrameTransformAction.Request request = PreviewDataFrameTransformAction.Request.fromXContent(parser); + return channel -> client.execute(PreviewDataFrameTransformAction.INSTANCE, request, new RestToXContentListener<>(channel)); + } +} diff --git a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/transforms/DataFrameIndexer.java b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/transforms/DataFrameIndexer.java index 771e513f05047..0aa1e4cb41336 100644 --- a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/transforms/DataFrameIndexer.java +++ b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/transforms/DataFrameIndexer.java @@ -32,7 +32,7 @@ public abstract class DataFrameIndexer extends AsyncTwoPhaseIndexer, DataFrameIndexerTransformStats> { - private static final String COMPOSITE_AGGREGATION_NAME = "_data_frame"; + static final String COMPOSITE_AGGREGATION_NAME = "_data_frame"; private static final Logger logger = LogManager.getLogger(DataFrameIndexer.class); private Pivot pivot; diff --git a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/transforms/DataFramePreviewer.java b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/transforms/DataFramePreviewer.java new file mode 100644 index 0000000000000..c19d683e56afe --- /dev/null +++ b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/transforms/DataFramePreviewer.java @@ -0,0 +1,52 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.dataframe.transforms; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.search.SearchAction; +import org.elasticsearch.client.Client; +import org.elasticsearch.search.aggregations.bucket.composite.CompositeAggregation; +import org.elasticsearch.xpack.core.ClientHelper; +import org.elasticsearch.xpack.core.dataframe.transform.DataFrameIndexerTransformStats; +import org.elasticsearch.xpack.dataframe.transforms.pivot.Pivot; + +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import static org.elasticsearch.xpack.dataframe.transforms.DataFrameIndexer.COMPOSITE_AGGREGATION_NAME; + +/** + * Class to enable previewing the documents that would result from the provided config + */ +public class DataFramePreviewer { + + private final Map headers; + private final Pivot pivot; + + public DataFramePreviewer(DataFrameTransformConfig config, + Map headers) { + this.headers = headers; + this.pivot = new Pivot(config.getSource(), config.getQueryConfig().getQuery(), config.getPivotConfig()); + } + + public void getPreview(Client client, ActionListener>> listener) { + ClientHelper.executeWithHeadersAsync(headers, + ClientHelper.DATA_FRAME_ORIGIN, + client, + SearchAction.INSTANCE, + pivot.buildSearchRequest(null), + ActionListener.wrap( + r -> { + final CompositeAggregation agg = r.getAggregations().get(COMPOSITE_AGGREGATION_NAME); + DataFrameIndexerTransformStats stats = new DataFrameIndexerTransformStats(); + listener.onResponse(pivot.extractResults(agg, stats).collect(Collectors.toList())); + }, + listener::onFailure + )); + } +} From e6ff6e0e0c2801e1a8447561f35bb069a54c4ed8 Mon Sep 17 00:00:00 2001 From: Benjamin Trent Date: Thu, 14 Feb 2019 14:18:46 -0600 Subject: [PATCH 2/8] adjusting preview tests and fixing parser --- .../integration/DataFramePivotRestIT.java | 11 ++- .../PreviewDataFrameTransformAction.java | 41 ++++++++--- .../RestPreviewDataFrameTransformAction.java | 2 - ...wDataFrameTransformActionRequestTests.java | 68 +++++++++++++++++++ ...ataFrameTransformsActionResponseTests.java | 50 ++++++++++++++ 5 files changed, 156 insertions(+), 16 deletions(-) create mode 100644 x-pack/plugin/data-frame/src/test/java/org/elasticsearch/xpack/dataframe/action/PreviewDataFrameTransformActionRequestTests.java create mode 100644 x-pack/plugin/data-frame/src/test/java/org/elasticsearch/xpack/dataframe/action/PreviewDataFrameTransformsActionResponseTests.java diff --git a/x-pack/plugin/data-frame/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFramePivotRestIT.java b/x-pack/plugin/data-frame/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFramePivotRestIT.java index 6245561d1060c..6c08c098661ab 100644 --- a/x-pack/plugin/data-frame/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFramePivotRestIT.java +++ b/x-pack/plugin/data-frame/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFramePivotRestIT.java @@ -230,7 +230,7 @@ public void testDateHistogramPivot() throws Exception { @SuppressWarnings("unchecked") public void testPreviewTransform() throws Exception { - final Request createPreviewRequest = new Request("POST", DATAFRAME_ENDPOINT + "/_preview"); + final Request createPreviewRequest = new Request("POST", DATAFRAME_ENDPOINT + "_preview"); String config = "{" + " \"source\": \"reviews\"," @@ -238,10 +238,9 @@ public void testPreviewTransform() throws Exception { + " \"dest\": \"doesnot-matter\","; config += " \"pivot\": {" - + " \"group_by\": [ {" + + " \"group_by\": [" + " {\"reviewer\": {\"terms\": { \"field\": \"user_id\" }}}," - + " {\"by_day\": {\"date_histogram\": " - + " {\"interval\": \"1d\",\"field\":\"timestamp\",\"format\":\"yyyy-MM-DD\"} } } ]," + + " {\"by_day\": {\"date_histogram\": {\"interval\": \"1d\",\"field\":\"timestamp\",\"format\":\"yyyy-MM-DD\"}}}]," + " \"aggregations\": {" + " \"avg_rating\": {" + " \"avg\": {" @@ -250,8 +249,8 @@ public void testPreviewTransform() throws Exception { + "}"; createPreviewRequest.setJsonEntity(config); Map previewDataframeResponse = entityAsMap(client().performRequest(createPreviewRequest)); - List> preview = (List>)previewDataframeResponse.get("preview"); - assertThat(preview.size(), equalTo(10)); + List> preview = (List>)previewDataframeResponse.get("data_frame_preview"); + assertThat(preview.size(), equalTo(393)); Set expectedFields = new HashSet<>(Arrays.asList("reviewer", "by_day", "avg_rating")); preview.forEach(p -> { Set keys = p.keySet(); diff --git a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/action/PreviewDataFrameTransformAction.java b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/action/PreviewDataFrameTransformAction.java index e6c6844633be8..b4c31bca4a3a9 100644 --- a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/action/PreviewDataFrameTransformAction.java +++ b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/action/PreviewDataFrameTransformAction.java @@ -10,12 +10,12 @@ import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.ActionResponse; import org.elasticsearch.action.support.master.AcknowledgedRequest; -import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.action.support.master.MasterNodeOperationRequestBuilder; import org.elasticsearch.client.ElasticsearchClient; -import org.elasticsearch.common.collect.HppcMaps; +import org.elasticsearch.common.ParseField; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.ObjectParser; import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; @@ -23,7 +23,6 @@ import java.io.IOException; import java.util.ArrayList; -import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Objects; @@ -50,9 +49,7 @@ public Request(DataFrameTransformConfig config) { this.setConfig(config); } - public Request() { - - } + public Request() { } public static Request fromXContent(final XContentParser parser) throws IOException { return new Request(DataFrameTransformConfig.fromXContent(parser, null, false)); @@ -116,7 +113,12 @@ protected RequestBuilder(ElasticsearchClient client, PreviewDataFrameTransformAc public static class Response extends ActionResponse implements ToXContentObject { private List> docs; + public static ParseField DATA_FRAME_PREVIEW = new ParseField("data_frame_preview"); + static ObjectParser PARSER = new ObjectParser<>("data_frame_transform_preview", Response::new); + static { + PARSER.declareObjectArray(Response::setDocs, (p, c) -> p.mapOrdered(), DATA_FRAME_PREVIEW); + } public Response() {} public Response(StreamInput in) throws IOException { @@ -148,16 +150,39 @@ public void readFrom(StreamInput in) throws IOException { public void writeTo(StreamOutput out) throws IOException { out.writeInt(docs.size()); for (Map doc : docs) { - out.writeMap(doc); + out.writeMapWithConsistentOrder(doc); } } @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); - builder.field("data_frame_preview", docs); + builder.field(DATA_FRAME_PREVIEW.getPreferredName(), docs); builder.endObject(); return builder; } + + @Override + public boolean equals(Object obj) { + if (obj == this) { + return true; + } + + if (obj == null || obj.getClass() == getClass() == false) { + return false; + } + + Response other = (Response) obj; + return Objects.equals(other.docs, docs); + } + + @Override + public int hashCode() { + return Objects.hashCode(docs); + } + + public static Response fromXContent(final XContentParser parser) throws IOException { + return PARSER.parse(parser, null); + } } } diff --git a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/rest/action/RestPreviewDataFrameTransformAction.java b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/rest/action/RestPreviewDataFrameTransformAction.java index 088d675f882db..7fdf7b4106a2f 100644 --- a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/rest/action/RestPreviewDataFrameTransformAction.java +++ b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/rest/action/RestPreviewDataFrameTransformAction.java @@ -15,8 +15,6 @@ import org.elasticsearch.rest.action.RestToXContentListener; import org.elasticsearch.xpack.core.dataframe.DataFrameField; import org.elasticsearch.xpack.dataframe.action.PreviewDataFrameTransformAction; -import org.elasticsearch.xpack.dataframe.action.PutDataFrameTransformAction; -import org.elasticsearch.xpack.dataframe.transforms.DataFrameTransformConfig; import java.io.IOException; diff --git a/x-pack/plugin/data-frame/src/test/java/org/elasticsearch/xpack/dataframe/action/PreviewDataFrameTransformActionRequestTests.java b/x-pack/plugin/data-frame/src/test/java/org/elasticsearch/xpack/dataframe/action/PreviewDataFrameTransformActionRequestTests.java new file mode 100644 index 0000000000000..72ba6d90024b7 --- /dev/null +++ b/x-pack/plugin/data-frame/src/test/java/org/elasticsearch/xpack/dataframe/action/PreviewDataFrameTransformActionRequestTests.java @@ -0,0 +1,68 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.dataframe.action; + +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.search.SearchModule; +import org.elasticsearch.test.AbstractStreamableXContentTestCase; +import org.elasticsearch.xpack.dataframe.action.PreviewDataFrameTransformAction.Request; +import org.elasticsearch.xpack.dataframe.transforms.DataFrameTransformConfig; +import org.elasticsearch.xpack.dataframe.transforms.DataFrameTransformConfigTests; +import org.junit.Before; + +import java.io.IOException; + +import static java.util.Collections.emptyList; + +public class PreviewDataFrameTransformActionRequestTests extends AbstractStreamableXContentTestCase { + + private NamedWriteableRegistry namedWriteableRegistry; + private NamedXContentRegistry namedXContentRegistry; + + @Before + public void registerAggregationNamedObjects() throws Exception { + // register aggregations as NamedWriteable + SearchModule searchModule = new SearchModule(Settings.EMPTY, false, emptyList()); + namedWriteableRegistry = new NamedWriteableRegistry(searchModule.getNamedWriteables()); + namedXContentRegistry = new NamedXContentRegistry(searchModule.getNamedXContents()); + } + + @Override + protected NamedWriteableRegistry getNamedWriteableRegistry() { + return namedWriteableRegistry; + } + + @Override + protected NamedXContentRegistry xContentRegistry() { + return namedXContentRegistry; + } + + @Override + protected Request doParseInstance(XContentParser parser) throws IOException { + return Request.fromXContent(parser); + } + + @Override + protected Request createBlankInstance() { + return new Request(); + } + + @Override + protected boolean supportsUnknownFields() { + return false; + } + + @Override + protected Request createTestInstance() { + DataFrameTransformConfig config = DataFrameTransformConfigTests.randomDataFrameTransformConfig(); + return new Request(config); + } + +} diff --git a/x-pack/plugin/data-frame/src/test/java/org/elasticsearch/xpack/dataframe/action/PreviewDataFrameTransformsActionResponseTests.java b/x-pack/plugin/data-frame/src/test/java/org/elasticsearch/xpack/dataframe/action/PreviewDataFrameTransformsActionResponseTests.java new file mode 100644 index 0000000000000..6b253abb25899 --- /dev/null +++ b/x-pack/plugin/data-frame/src/test/java/org/elasticsearch/xpack/dataframe/action/PreviewDataFrameTransformsActionResponseTests.java @@ -0,0 +1,50 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.dataframe.action; + +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.test.AbstractStreamableXContentTestCase; +import org.elasticsearch.xpack.dataframe.action.PreviewDataFrameTransformAction.Response; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class PreviewDataFrameTransformsActionResponseTests extends AbstractStreamableXContentTestCase { + + + @Override + protected Response doParseInstance(XContentParser parser) throws IOException { + return Response.fromXContent(parser); + } + + @Override + protected Response createBlankInstance() { + return new Response(); + } + + @Override + protected Response createTestInstance() { + int size = randomIntBetween(0, 10); + List> data = new ArrayList<>(size); + for (int i = 0; i < size; i++) { + Map datum = new HashMap<>(); + Map entry = new HashMap<>(); + entry.put("value1", randomIntBetween(1, 100)); + datum.put(randomAlphaOfLength(10), entry); + data.add(datum); + } + return new Response(data); + } + + @Override + protected boolean supportsUnknownFields() { + return false; + } +} From b3fa363fd8aac61cdc6adc2b70207a41843ae478 Mon Sep 17 00:00:00 2001 From: Benjamin Trent Date: Thu, 14 Feb 2019 14:29:48 -0600 Subject: [PATCH 3/8] adjusing preview transport --- .../PreviewDataFrameTransformAction.java | 8 +-- ...nsportPreviewDataFrameTransformAction.java | 34 ++++++++++-- .../transforms/DataFrameIndexer.java | 2 +- .../transforms/DataFramePreviewer.java | 52 ------------------- 4 files changed, 36 insertions(+), 60 deletions(-) delete mode 100644 x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/transforms/DataFramePreviewer.java diff --git a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/action/PreviewDataFrameTransformAction.java b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/action/PreviewDataFrameTransformAction.java index b4c31bca4a3a9..53f7941619af7 100644 --- a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/action/PreviewDataFrameTransformAction.java +++ b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/action/PreviewDataFrameTransformAction.java @@ -92,10 +92,10 @@ public int hashCode() { @Override public boolean equals(Object obj) { - if (obj == null) { - return false; + if (obj == this) { + return true; } - if (getClass() != obj.getClass()) { + if (obj == null || getClass() != obj.getClass()) { return false; } Request other = (Request) obj; @@ -168,7 +168,7 @@ public boolean equals(Object obj) { return true; } - if (obj == null || obj.getClass() == getClass() == false) { + if (obj == null || obj.getClass() != getClass()) { return false; } diff --git a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/action/TransportPreviewDataFrameTransformAction.java b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/action/TransportPreviewDataFrameTransformAction.java index 5ff995e54eabc..9aa63c9908f92 100644 --- a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/action/TransportPreviewDataFrameTransformAction.java +++ b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/action/TransportPreviewDataFrameTransformAction.java @@ -7,19 +7,28 @@ package org.elasticsearch.xpack.dataframe.action; import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.search.SearchAction; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.HandledTransportAction; import org.elasticsearch.client.Client; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.license.LicenseUtils; import org.elasticsearch.license.XPackLicenseState; +import org.elasticsearch.search.aggregations.bucket.composite.CompositeAggregation; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.core.ClientHelper; import org.elasticsearch.xpack.core.XPackField; -import org.elasticsearch.xpack.dataframe.transforms.DataFramePreviewer; +import org.elasticsearch.xpack.core.dataframe.transform.DataFrameIndexerTransformStats; +import org.elasticsearch.xpack.dataframe.transforms.pivot.Pivot; +import java.util.List; +import java.util.Map; import java.util.function.Supplier; +import java.util.stream.Collectors; + +import static org.elasticsearch.xpack.dataframe.transforms.DataFrameIndexer.COMPOSITE_AGGREGATION_NAME; public class TransportPreviewDataFrameTransformAction extends HandledTransportAction { @@ -47,10 +56,29 @@ protected void doExecute(Task task, return; } - DataFramePreviewer previewer = new DataFramePreviewer(request.getConfig(), threadPool.getThreadContext().getHeaders()); - previewer.getPreview(client, ActionListener.wrap( + Pivot pivot = new Pivot(request.getConfig().getSource(), + request.getConfig().getQueryConfig().getQuery(), + request.getConfig().getPivotConfig()); + + getPreview(pivot, ActionListener.wrap( previewResponse -> listener.onResponse(new PreviewDataFrameTransformAction.Response(previewResponse)), listener::onFailure )); } + + private void getPreview(Pivot pivot, ActionListener>> listener) { + ClientHelper.executeWithHeadersAsync(threadPool.getThreadContext().getHeaders(), + ClientHelper.DATA_FRAME_ORIGIN, + client, + SearchAction.INSTANCE, + pivot.buildSearchRequest(null), + ActionListener.wrap( + r -> { + final CompositeAggregation agg = r.getAggregations().get(COMPOSITE_AGGREGATION_NAME); + DataFrameIndexerTransformStats stats = new DataFrameIndexerTransformStats(); + listener.onResponse(pivot.extractResults(agg, stats).collect(Collectors.toList())); + }, + listener::onFailure + )); + } } diff --git a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/transforms/DataFrameIndexer.java b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/transforms/DataFrameIndexer.java index 0aa1e4cb41336..11b90955664e9 100644 --- a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/transforms/DataFrameIndexer.java +++ b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/transforms/DataFrameIndexer.java @@ -32,7 +32,7 @@ public abstract class DataFrameIndexer extends AsyncTwoPhaseIndexer, DataFrameIndexerTransformStats> { - static final String COMPOSITE_AGGREGATION_NAME = "_data_frame"; + public static final String COMPOSITE_AGGREGATION_NAME = "_data_frame"; private static final Logger logger = LogManager.getLogger(DataFrameIndexer.class); private Pivot pivot; diff --git a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/transforms/DataFramePreviewer.java b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/transforms/DataFramePreviewer.java deleted file mode 100644 index c19d683e56afe..0000000000000 --- a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/transforms/DataFramePreviewer.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -package org.elasticsearch.xpack.dataframe.transforms; - -import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.search.SearchAction; -import org.elasticsearch.client.Client; -import org.elasticsearch.search.aggregations.bucket.composite.CompositeAggregation; -import org.elasticsearch.xpack.core.ClientHelper; -import org.elasticsearch.xpack.core.dataframe.transform.DataFrameIndexerTransformStats; -import org.elasticsearch.xpack.dataframe.transforms.pivot.Pivot; - -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -import static org.elasticsearch.xpack.dataframe.transforms.DataFrameIndexer.COMPOSITE_AGGREGATION_NAME; - -/** - * Class to enable previewing the documents that would result from the provided config - */ -public class DataFramePreviewer { - - private final Map headers; - private final Pivot pivot; - - public DataFramePreviewer(DataFrameTransformConfig config, - Map headers) { - this.headers = headers; - this.pivot = new Pivot(config.getSource(), config.getQueryConfig().getQuery(), config.getPivotConfig()); - } - - public void getPreview(Client client, ActionListener>> listener) { - ClientHelper.executeWithHeadersAsync(headers, - ClientHelper.DATA_FRAME_ORIGIN, - client, - SearchAction.INSTANCE, - pivot.buildSearchRequest(null), - ActionListener.wrap( - r -> { - final CompositeAggregation agg = r.getAggregations().get(COMPOSITE_AGGREGATION_NAME); - DataFrameIndexerTransformStats stats = new DataFrameIndexerTransformStats(); - listener.onResponse(pivot.extractResults(agg, stats).collect(Collectors.toList())); - }, - listener::onFailure - )); - } -} From 35996da24cb8fa43d3978f4103b4c26a10bfb6cc Mon Sep 17 00:00:00 2001 From: Benjamin Trent Date: Thu, 14 Feb 2019 14:33:06 -0600 Subject: [PATCH 4/8] remove unused import --- .../xpack/dataframe/integration/DataFramePivotRestIT.java | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/plugin/data-frame/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFramePivotRestIT.java b/x-pack/plugin/data-frame/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFramePivotRestIT.java index 6c08c098661ab..73a466e844f79 100644 --- a/x-pack/plugin/data-frame/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFramePivotRestIT.java +++ b/x-pack/plugin/data-frame/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFramePivotRestIT.java @@ -10,7 +10,6 @@ import org.elasticsearch.client.Response; import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.junit.Before; -import org.junit.Ignore; import java.io.IOException; import java.util.Arrays; From d63cd99032f9407335d6d093e9715358b6ed0ce5 Mon Sep 17 00:00:00 2001 From: Benjamin Trent Date: Tue, 19 Feb 2019 07:53:19 -0600 Subject: [PATCH 5/8] adjusting test --- .../xpack/dataframe/integration/DataFramePivotRestIT.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/x-pack/plugin/data-frame/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFramePivotRestIT.java b/x-pack/plugin/data-frame/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFramePivotRestIT.java index 73a466e844f79..9a8b954e96c71 100644 --- a/x-pack/plugin/data-frame/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFramePivotRestIT.java +++ b/x-pack/plugin/data-frame/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFramePivotRestIT.java @@ -237,9 +237,9 @@ public void testPreviewTransform() throws Exception { + " \"dest\": \"doesnot-matter\","; config += " \"pivot\": {" - + " \"group_by\": [" - + " {\"reviewer\": {\"terms\": { \"field\": \"user_id\" }}}," - + " {\"by_day\": {\"date_histogram\": {\"interval\": \"1d\",\"field\":\"timestamp\",\"format\":\"yyyy-MM-DD\"}}}]," + + " \"group_by\": {" + + " \"reviewer\": {\"terms\": { \"field\": \"user_id\" }}," + + " \"by_day\": {\"date_histogram\": {\"interval\": \"1d\",\"field\":\"timestamp\",\"format\":\"yyyy-MM-DD\"}}}," + " \"aggregations\": {" + " \"avg_rating\": {" + " \"avg\": {" From f36e552bd52c64558382b5a521850c524c4a3103 Mon Sep 17 00:00:00 2001 From: Benjamin Trent Date: Wed, 20 Feb 2019 16:44:25 -0600 Subject: [PATCH 6/8] Addressing PR comments --- .../integration/DataFramePivotRestIT.java | 4 +-- .../PreviewDataFrameTransformAction.java | 33 ++++++++++++++++--- .../transforms/DataFrameTransformConfig.java | 10 +++--- 3 files changed, 35 insertions(+), 12 deletions(-) diff --git a/x-pack/plugin/data-frame/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFramePivotRestIT.java b/x-pack/plugin/data-frame/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFramePivotRestIT.java index 9a8b954e96c71..97410b55fd5f3 100644 --- a/x-pack/plugin/data-frame/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFramePivotRestIT.java +++ b/x-pack/plugin/data-frame/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFramePivotRestIT.java @@ -232,9 +232,7 @@ public void testPreviewTransform() throws Exception { final Request createPreviewRequest = new Request("POST", DATAFRAME_ENDPOINT + "_preview"); String config = "{" - + " \"source\": \"reviews\"," - + " \"id\": \"doesnot-matter\"," - + " \"dest\": \"doesnot-matter\","; + + " \"source\": \"reviews\","; config += " \"pivot\": {" + " \"group_by\": {" diff --git a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/action/PreviewDataFrameTransformAction.java b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/action/PreviewDataFrameTransformAction.java index 53f7941619af7..1ad0fc7605685 100644 --- a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/action/PreviewDataFrameTransformAction.java +++ b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/action/PreviewDataFrameTransformAction.java @@ -13,16 +13,24 @@ import org.elasticsearch.action.support.master.MasterNodeOperationRequestBuilder; import org.elasticsearch.client.ElasticsearchClient; import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.ObjectParser; import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.search.SearchModule; import org.elasticsearch.xpack.dataframe.transforms.DataFrameTransformConfig; import java.io.IOException; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Objects; @@ -32,6 +40,13 @@ public class PreviewDataFrameTransformAction extends Action content = parser.map(); + // Destination and ID are not required for Preview, so we just supply our own + content.put(DataFrameTransformConfig.DESTINATION.getPreferredName(), "unused-transform-preview-index"); + try(XContentBuilder xContentBuilder = XContentFactory.jsonBuilder().map(content); + XContentParser newParser = XContentType.JSON + .xContent() + .createParser(searchRegistry, + LoggingDeprecationHandler.INSTANCE, + BytesReference.bytes(xContentBuilder).streamInput())) { + return new Request(DataFrameTransformConfig.fromXContent(newParser, "transform-preview", true)); + } } @Override @@ -113,11 +138,11 @@ protected RequestBuilder(ElasticsearchClient client, PreviewDataFrameTransformAc public static class Response extends ActionResponse implements ToXContentObject { private List> docs; - public static ParseField DATA_FRAME_PREVIEW = new ParseField("data_frame_preview"); + public static ParseField PREVIEW = new ParseField("preview"); static ObjectParser PARSER = new ObjectParser<>("data_frame_transform_preview", Response::new); static { - PARSER.declareObjectArray(Response::setDocs, (p, c) -> p.mapOrdered(), DATA_FRAME_PREVIEW); + PARSER.declareObjectArray(Response::setDocs, (p, c) -> p.mapOrdered(), PREVIEW); } public Response() {} @@ -157,7 +182,7 @@ public void writeTo(StreamOutput out) throws IOException { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); - builder.field(DATA_FRAME_PREVIEW.getPreferredName(), docs); + builder.field(PREVIEW.getPreferredName(), docs); builder.endObject(); return builder; } diff --git a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/transforms/DataFrameTransformConfig.java b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/transforms/DataFrameTransformConfig.java index b5bd22f3a5e62..13403e51dcd5a 100644 --- a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/transforms/DataFrameTransformConfig.java +++ b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/transforms/DataFrameTransformConfig.java @@ -35,13 +35,13 @@ */ public class DataFrameTransformConfig extends AbstractDiffable implements Writeable, ToXContentObject { - private static final String NAME = "data_frame_transforms"; - private static final ParseField SOURCE = new ParseField("source"); - private static final ParseField DESTINATION = new ParseField("dest"); - private static final ParseField QUERY = new ParseField("query"); + public static final String NAME = "data_frame_transforms"; + public static final ParseField SOURCE = new ParseField("source"); + public static final ParseField DESTINATION = new ParseField("dest"); + public static final ParseField QUERY = new ParseField("query"); // types of transforms - private static final ParseField PIVOT_TRANSFORM = new ParseField("pivot"); + public static final ParseField PIVOT_TRANSFORM = new ParseField("pivot"); private static final ConstructingObjectParser STRICT_PARSER = createParser(false); private static final ConstructingObjectParser LENIENT_PARSER = createParser(true); From 0a39f647dda893ed826151852225b55d8aab2487 Mon Sep 17 00:00:00 2001 From: Benjamin Trent Date: Thu, 21 Feb 2019 15:03:16 -0600 Subject: [PATCH 7/8] Fixing failing test and adjusting for pr comments --- .../action/PreviewDataFrameTransformAction.java | 13 +------------ ...PreviewDataFrameTransformActionRequestTests.java | 6 ++++-- 2 files changed, 5 insertions(+), 14 deletions(-) diff --git a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/action/PreviewDataFrameTransformAction.java b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/action/PreviewDataFrameTransformAction.java index 1ad0fc7605685..72589650679ff 100644 --- a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/action/PreviewDataFrameTransformAction.java +++ b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/action/PreviewDataFrameTransformAction.java @@ -16,21 +16,17 @@ import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; -import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.ObjectParser; import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; -import org.elasticsearch.search.SearchModule; import org.elasticsearch.xpack.dataframe.transforms.DataFrameTransformConfig; import java.io.IOException; import java.util.ArrayList; -import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Objects; @@ -40,13 +36,6 @@ public class PreviewDataFrameTransformAction extends Action Date: Thu, 21 Feb 2019 16:00:06 -0600 Subject: [PATCH 8/8] fixing integration test --- .../xpack/dataframe/integration/DataFramePivotRestIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/data-frame/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFramePivotRestIT.java b/x-pack/plugin/data-frame/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFramePivotRestIT.java index 97410b55fd5f3..70e90c60d9e5d 100644 --- a/x-pack/plugin/data-frame/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFramePivotRestIT.java +++ b/x-pack/plugin/data-frame/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFramePivotRestIT.java @@ -246,7 +246,7 @@ public void testPreviewTransform() throws Exception { + "}"; createPreviewRequest.setJsonEntity(config); Map previewDataframeResponse = entityAsMap(client().performRequest(createPreviewRequest)); - List> preview = (List>)previewDataframeResponse.get("data_frame_preview"); + List> preview = (List>)previewDataframeResponse.get("preview"); assertThat(preview.size(), equalTo(393)); Set expectedFields = new HashSet<>(Arrays.asList("reviewer", "by_day", "avg_rating")); preview.forEach(p -> {