-
Notifications
You must be signed in to change notification settings - Fork 25.6k
[ML-DataFrame] Add _preview endpoint #38924
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Merged
Merged
Changes from 5 commits
Commits
Show all changes
8 commits
Select commit
Hold shift + click to select a range
32fa660
[DATA-FRAME] add preview endpoint
benwtrent e6ff6e0
adjusting preview tests and fixing parser
benwtrent b3fa363
adjusing preview transport
benwtrent 35996da
remove unused import
benwtrent d63cd99
adjusting test
benwtrent f36e552
Addressing PR comments
benwtrent 0a39f64
Fixing failing test and adjusting for pr comments
benwtrent f66a765
fixing integration test
benwtrent File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
188 changes: 188 additions & 0 deletions
188
...c/main/java/org/elasticsearch/xpack/dataframe/action/PreviewDataFrameTransformAction.java
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,188 @@ | ||
| /* | ||
| * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one | ||
| * or more contributor license agreements. Licensed under the Elastic License; | ||
| * you may not use this file except in compliance with the Elastic License. | ||
| */ | ||
|
|
||
| package org.elasticsearch.xpack.dataframe.action; | ||
|
|
||
| import org.elasticsearch.action.Action; | ||
| import org.elasticsearch.action.ActionRequestValidationException; | ||
| import org.elasticsearch.action.ActionResponse; | ||
| import org.elasticsearch.action.support.master.AcknowledgedRequest; | ||
| import org.elasticsearch.action.support.master.MasterNodeOperationRequestBuilder; | ||
| import org.elasticsearch.client.ElasticsearchClient; | ||
| import org.elasticsearch.common.ParseField; | ||
| import org.elasticsearch.common.io.stream.StreamInput; | ||
| import org.elasticsearch.common.io.stream.StreamOutput; | ||
| import org.elasticsearch.common.xcontent.ObjectParser; | ||
| import org.elasticsearch.common.xcontent.ToXContentObject; | ||
| import org.elasticsearch.common.xcontent.XContentBuilder; | ||
| import org.elasticsearch.common.xcontent.XContentParser; | ||
| import org.elasticsearch.xpack.dataframe.transforms.DataFrameTransformConfig; | ||
|
|
||
| import java.io.IOException; | ||
| import java.util.ArrayList; | ||
| import java.util.List; | ||
| import java.util.Map; | ||
| import java.util.Objects; | ||
|
|
||
| public class PreviewDataFrameTransformAction extends Action<PreviewDataFrameTransformAction.Response> { | ||
|
|
||
| public static final PreviewDataFrameTransformAction INSTANCE = new PreviewDataFrameTransformAction(); | ||
| public static final String NAME = "cluster:admin/data_frame/preview"; | ||
|
|
||
| private PreviewDataFrameTransformAction() { | ||
| super(NAME); | ||
| } | ||
|
|
||
| @Override | ||
| public Response newResponse() { | ||
| return new Response(); | ||
| } | ||
|
|
||
| public static class Request extends AcknowledgedRequest<Request> implements ToXContentObject { | ||
|
|
||
| private DataFrameTransformConfig config; | ||
|
|
||
| public Request(DataFrameTransformConfig config) { | ||
| this.setConfig(config); | ||
| } | ||
|
|
||
| public Request() { } | ||
|
|
||
| public static Request fromXContent(final XContentParser parser) throws IOException { | ||
| return new Request(DataFrameTransformConfig.fromXContent(parser, null, false)); | ||
| } | ||
|
|
||
| @Override | ||
| public ActionRequestValidationException validate() { | ||
| return null; | ||
| } | ||
|
|
||
| @Override | ||
| public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { | ||
| return this.config.toXContent(builder, params); | ||
| } | ||
|
|
||
| public DataFrameTransformConfig getConfig() { | ||
| return config; | ||
| } | ||
|
|
||
| public void setConfig(DataFrameTransformConfig config) { | ||
| this.config = config; | ||
| } | ||
|
|
||
| @Override | ||
| public void readFrom(StreamInput in) throws IOException { | ||
| super.readFrom(in); | ||
| this.config = new DataFrameTransformConfig(in); | ||
| } | ||
|
|
||
| @Override | ||
| public void writeTo(StreamOutput out) throws IOException { | ||
| super.writeTo(out); | ||
| this.config.writeTo(out); | ||
| } | ||
|
|
||
| @Override | ||
| public int hashCode() { | ||
| return Objects.hash(config); | ||
| } | ||
|
|
||
| @Override | ||
| public boolean equals(Object obj) { | ||
| if (obj == this) { | ||
| return true; | ||
| } | ||
| if (obj == null || getClass() != obj.getClass()) { | ||
| return false; | ||
| } | ||
| Request other = (Request) obj; | ||
| return Objects.equals(config, other.config); | ||
| } | ||
| } | ||
|
|
||
| public static class RequestBuilder extends MasterNodeOperationRequestBuilder<Request, Response, RequestBuilder> { | ||
|
|
||
| protected RequestBuilder(ElasticsearchClient client, PreviewDataFrameTransformAction action) { | ||
| super(client, action, new Request()); | ||
| } | ||
| } | ||
|
|
||
| public static class Response extends ActionResponse implements ToXContentObject { | ||
|
|
||
| private List<Map<String, Object>> docs; | ||
| public static ParseField DATA_FRAME_PREVIEW = new ParseField("data_frame_preview"); | ||
benwtrent marked this conversation as resolved.
Outdated
Show resolved
Hide resolved
|
||
|
|
||
| static ObjectParser<Response, Void> PARSER = new ObjectParser<>("data_frame_transform_preview", Response::new); | ||
| static { | ||
| PARSER.declareObjectArray(Response::setDocs, (p, c) -> p.mapOrdered(), DATA_FRAME_PREVIEW); | ||
| } | ||
| public Response() {} | ||
|
|
||
| public Response(StreamInput in) throws IOException { | ||
| int size = in.readInt(); | ||
| this.docs = new ArrayList<>(size); | ||
| for (int i = 0; i < size; i++) { | ||
| this.docs.add(in.readMap()); | ||
| } | ||
| } | ||
|
|
||
| public Response(List<Map<String, Object>> docs) { | ||
| this.docs = new ArrayList<>(docs); | ||
| } | ||
|
|
||
| public void setDocs(List<Map<String, Object>> docs) { | ||
| this.docs = new ArrayList<>(docs); | ||
| } | ||
|
|
||
| @Override | ||
| public void readFrom(StreamInput in) throws IOException { | ||
| int size = in.readInt(); | ||
| this.docs = new ArrayList<>(size); | ||
| for (int i = 0; i < size; i++) { | ||
| this.docs.add(in.readMap()); | ||
| } | ||
| } | ||
|
|
||
| @Override | ||
| public void writeTo(StreamOutput out) throws IOException { | ||
| out.writeInt(docs.size()); | ||
| for (Map<String, Object> doc : docs) { | ||
| out.writeMapWithConsistentOrder(doc); | ||
| } | ||
| } | ||
|
|
||
| @Override | ||
| public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { | ||
| builder.startObject(); | ||
| builder.field(DATA_FRAME_PREVIEW.getPreferredName(), docs); | ||
| builder.endObject(); | ||
| return builder; | ||
| } | ||
|
|
||
| @Override | ||
| public boolean equals(Object obj) { | ||
| if (obj == this) { | ||
| return true; | ||
| } | ||
|
|
||
| if (obj == null || obj.getClass() != getClass()) { | ||
| return false; | ||
| } | ||
|
|
||
| Response other = (Response) obj; | ||
| return Objects.equals(other.docs, docs); | ||
| } | ||
|
|
||
| @Override | ||
| public int hashCode() { | ||
| return Objects.hashCode(docs); | ||
| } | ||
|
|
||
| public static Response fromXContent(final XContentParser parser) throws IOException { | ||
| return PARSER.parse(parser, null); | ||
| } | ||
| } | ||
| } | ||
84 changes: 84 additions & 0 deletions
84
...va/org/elasticsearch/xpack/dataframe/action/TransportPreviewDataFrameTransformAction.java
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,84 @@ | ||
| /* | ||
| * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one | ||
| * or more contributor license agreements. Licensed under the Elastic License; | ||
| * you may not use this file except in compliance with the Elastic License. | ||
| */ | ||
|
|
||
| package org.elasticsearch.xpack.dataframe.action; | ||
|
|
||
| import org.elasticsearch.action.ActionListener; | ||
| import org.elasticsearch.action.search.SearchAction; | ||
| import org.elasticsearch.action.support.ActionFilters; | ||
| import org.elasticsearch.action.support.HandledTransportAction; | ||
| import org.elasticsearch.client.Client; | ||
| import org.elasticsearch.common.inject.Inject; | ||
| import org.elasticsearch.license.LicenseUtils; | ||
| import org.elasticsearch.license.XPackLicenseState; | ||
| import org.elasticsearch.search.aggregations.bucket.composite.CompositeAggregation; | ||
| import org.elasticsearch.tasks.Task; | ||
| import org.elasticsearch.threadpool.ThreadPool; | ||
| import org.elasticsearch.transport.TransportService; | ||
| import org.elasticsearch.xpack.core.ClientHelper; | ||
| import org.elasticsearch.xpack.core.XPackField; | ||
| import org.elasticsearch.xpack.core.dataframe.transform.DataFrameIndexerTransformStats; | ||
| import org.elasticsearch.xpack.dataframe.transforms.pivot.Pivot; | ||
|
|
||
| import java.util.List; | ||
| import java.util.Map; | ||
| import java.util.function.Supplier; | ||
| import java.util.stream.Collectors; | ||
|
|
||
| import static org.elasticsearch.xpack.dataframe.transforms.DataFrameIndexer.COMPOSITE_AGGREGATION_NAME; | ||
|
|
||
| public class TransportPreviewDataFrameTransformAction extends | ||
| HandledTransportAction<PreviewDataFrameTransformAction.Request, PreviewDataFrameTransformAction.Response> { | ||
|
|
||
| private final XPackLicenseState licenseState; | ||
| private final Client client; | ||
| private final ThreadPool threadPool; | ||
|
|
||
| @Inject | ||
| public TransportPreviewDataFrameTransformAction(TransportService transportService, ActionFilters actionFilters, | ||
| Client client, ThreadPool threadPool, XPackLicenseState licenseState) { | ||
| super(PreviewDataFrameTransformAction.NAME,transportService, actionFilters, | ||
| (Supplier<PreviewDataFrameTransformAction.Request>) PreviewDataFrameTransformAction.Request::new); | ||
| this.licenseState = licenseState; | ||
| this.client = client; | ||
| this.threadPool = threadPool; | ||
| } | ||
|
|
||
| @Override | ||
| protected void doExecute(Task task, | ||
| PreviewDataFrameTransformAction.Request request, | ||
| ActionListener<PreviewDataFrameTransformAction.Response> listener) { | ||
| if (!licenseState.isDataFrameAllowed()) { | ||
| listener.onFailure(LicenseUtils.newComplianceException(XPackField.DATA_FRAME)); | ||
| return; | ||
| } | ||
|
|
||
| Pivot pivot = new Pivot(request.getConfig().getSource(), | ||
| request.getConfig().getQueryConfig().getQuery(), | ||
| request.getConfig().getPivotConfig()); | ||
|
|
||
| getPreview(pivot, ActionListener.wrap( | ||
| previewResponse -> listener.onResponse(new PreviewDataFrameTransformAction.Response(previewResponse)), | ||
| listener::onFailure | ||
| )); | ||
| } | ||
|
|
||
| private void getPreview(Pivot pivot, ActionListener<List<Map<String, Object>>> listener) { | ||
| ClientHelper.executeWithHeadersAsync(threadPool.getThreadContext().getHeaders(), | ||
| ClientHelper.DATA_FRAME_ORIGIN, | ||
| client, | ||
| SearchAction.INSTANCE, | ||
| pivot.buildSearchRequest(null), | ||
| ActionListener.wrap( | ||
| r -> { | ||
| final CompositeAggregation agg = r.getAggregations().get(COMPOSITE_AGGREGATION_NAME); | ||
| DataFrameIndexerTransformStats stats = new DataFrameIndexerTransformStats(); | ||
| listener.onResponse(pivot.extractResults(agg, stats).collect(Collectors.toList())); | ||
| }, | ||
| listener::onFailure | ||
| )); | ||
| } | ||
| } |
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.