Skip to content

Commit 738c0fe

Browse files
jasontedorjasontedor
committed
Merge branch 'master' of github.com:elastic/elasticsearch into ccr-docs
* 'master' of github.com:elastic/elasticsearch: Fix line length for org.elasticsearch.common.* files (elastic#34888) [ML] Extract common native process base class (elastic#34856) Refactor children aggregator into a generic ParentJoinAggregator (elastic#34845) [Style] Fix line lengths in action.admin.indices (elastic#34890) HLRC - add support for source exists API (elastic#34519) [CCR] Retry when no index shard stats can be found (elastic#34852) [Docs] audit logfile structured format (elastic#34584) [Test] Fix FullClusterRestartIT.testShrink() with copy_settings param (elastic#34853) Fix LineLength Check Suppressions: index.fielddata (elastic#34891) TEST: Stablize Minio Free Port Search (elastic#34894) Delete flaky SettingsBasedHostProviderIT test (elastic#34813) [ML] Include message in field_stats for text log files (elastic#34861) [TEST] HLRC: Expand failure messages in API checks (elastic#34838) Lowercase static final DeprecationLogger instance names (elastic#34887)
2 parents 6337e6e + af28d1f commit 738c0fe

File tree

221 files changed

+1955
-1596
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

221 files changed

+1955
-1596
lines changed

‎buildSrc/src/main/resources/checkstyle_suppressions.xml‎

Lines changed: 0 additions & 111 deletions
Large diffs are not rendered by default.

‎client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -261,6 +261,18 @@ private static Request getStyleRequest(String method, GetRequest getRequest) {
261261

262262
return request;
263263
}
264+
265+
static Request sourceExists(GetRequest getRequest) {
266+
Request request = new Request(HttpHead.METHOD_NAME, endpoint(getRequest.index(), getRequest.type(), getRequest.id(), "_source"));
267+
268+
Params parameters = new Params(request);
269+
parameters.withPreference(getRequest.preference());
270+
parameters.withRouting(getRequest.routing());
271+
parameters.withRefresh(getRequest.refresh());
272+
parameters.withRealtime(getRequest.realtime());
273+
// Version params are not currently supported by the source exists API so are not passed
274+
return request;
275+
}
264276

265277
static Request multiGet(MultiGetRequest multiGetRequest) throws IOException {
266278
Request request = new Request(HttpPost.METHOD_NAME, "/_mget");

‎client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -727,6 +727,32 @@ public final void existsAsync(GetRequest getRequest, RequestOptions options, Act
727727
emptySet());
728728
}
729729

730+
/**
731+
* Checks for the existence of a document with a "_source" field. Returns true if it exists, false otherwise.
732+
* See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-get.html#_source">Source exists API
733+
* on elastic.co</a>
734+
* @param getRequest the request
735+
* @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
736+
* @return <code>true</code> if the document and _source field exists, <code>false</code> otherwise
737+
* @throws IOException in case there is a problem sending the request
738+
*/
739+
public boolean existsSource(GetRequest getRequest, RequestOptions options) throws IOException {
740+
return performRequest(getRequest, RequestConverters::sourceExists, options, RestHighLevelClient::convertExistsResponse, emptySet());
741+
}
742+
743+
/**
744+
* Asynchronously checks for the existence of a document with a "_source" field. Returns true if it exists, false otherwise.
745+
* See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-get.html#_source">Source exists API
746+
* on elastic.co</a>
747+
* @param getRequest the request
748+
* @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
749+
* @param listener the listener to be notified upon request completion
750+
*/
751+
public final void existsSourceAsync(GetRequest getRequest, RequestOptions options, ActionListener<Boolean> listener) {
752+
performRequestAsync(getRequest, RequestConverters::sourceExists, options, RestHighLevelClient::convertExistsResponse, listener,
753+
emptySet());
754+
}
755+
730756
/**
731757
* Index a document using the Index API.
732758
* See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-index_.html">Index API on elastic.co</a>

‎client/rest-high-level/src/test/java/org/elasticsearch/client/CrudIT.java‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -194,6 +194,61 @@ public void testExists() throws IOException {
194194
assertFalse(execute(getRequest, highLevelClient()::exists, highLevelClient()::existsAsync));
195195
}
196196
}
197+
198+
public void testSourceExists() throws IOException {
199+
{
200+
GetRequest getRequest = new GetRequest("index", "type", "id");
201+
assertFalse(execute(getRequest, highLevelClient()::existsSource, highLevelClient()::existsSourceAsync));
202+
}
203+
IndexRequest index = new IndexRequest("index", "type", "id");
204+
index.source("{\"field1\":\"value1\",\"field2\":\"value2\"}", XContentType.JSON);
205+
index.setRefreshPolicy(RefreshPolicy.IMMEDIATE);
206+
highLevelClient().index(index, RequestOptions.DEFAULT);
207+
{
208+
GetRequest getRequest = new GetRequest("index", "type", "id");
209+
assertTrue(execute(getRequest, highLevelClient()::existsSource, highLevelClient()::existsSourceAsync));
210+
}
211+
{
212+
GetRequest getRequest = new GetRequest("index", "type", "does_not_exist");
213+
assertFalse(execute(getRequest, highLevelClient()::existsSource, highLevelClient()::existsSourceAsync));
214+
}
215+
{
216+
GetRequest getRequest = new GetRequest("index", "type", "does_not_exist").version(1);
217+
assertFalse(execute(getRequest, highLevelClient()::existsSource, highLevelClient()::existsSourceAsync));
218+
}
219+
}
220+
221+
public void testSourceDoesNotExist() throws IOException {
222+
final String noSourceIndex = "no_source";
223+
{
224+
// Prepare
225+
Settings settings = Settings.builder()
226+
.put("number_of_shards", 1)
227+
.put("number_of_replicas", 0)
228+
.build();
229+
String mapping = "\"_doc\": { \"_source\": {\n" +
230+
" \"enabled\": false\n" +
231+
" } }";
232+
createIndex(noSourceIndex, settings, mapping);
233+
assertEquals(
234+
RestStatus.OK,
235+
highLevelClient().bulk(
236+
new BulkRequest()
237+
.add(new IndexRequest(noSourceIndex, "_doc", "1")
238+
.source(Collections.singletonMap("foo", 1), XContentType.JSON))
239+
.add(new IndexRequest(noSourceIndex, "_doc", "2")
240+
.source(Collections.singletonMap("foo", 2), XContentType.JSON))
241+
.setRefreshPolicy(RefreshPolicy.IMMEDIATE),
242+
RequestOptions.DEFAULT
243+
).status()
244+
);
245+
}
246+
{
247+
GetRequest getRequest = new GetRequest(noSourceIndex, "_doc", "1");
248+
assertTrue(execute(getRequest, highLevelClient()::exists, highLevelClient()::existsAsync));
249+
assertFalse(execute(getRequest, highLevelClient()::existsSource, highLevelClient()::existsSourceAsync));
250+
}
251+
}
197252

198253
public void testGet() throws IOException {
199254
{

‎client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java‎

Lines changed: 23 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -650,7 +650,6 @@ public void testApiNamingConventions() throws Exception {
650650
"cluster.remote_info",
651651
"count",
652652
"create",
653-
"exists_source",
654653
"get_source",
655654
"indices.delete_alias",
656655
"indices.delete_template",
@@ -712,41 +711,49 @@ public void testApiNamingConventions() throws Exception {
712711

713712
assertTrue("method [" + apiName + "] is not final",
714713
Modifier.isFinal(method.getClass().getModifiers()) || Modifier.isFinal(method.getModifiers()));
715-
assertTrue(Modifier.isPublic(method.getModifiers()));
714+
assertTrue("method [" + method + "] should be public", Modifier.isPublic(method.getModifiers()));
716715

717716
//we convert all the method names to snake case, hence we need to look for the '_async' suffix rather than 'Async'
718717
if (apiName.endsWith("_async")) {
719718
assertTrue("async method [" + method.getName() + "] doesn't have corresponding sync method",
720719
methods.containsKey(apiName.substring(0, apiName.length() - 6)));
721-
assertThat(method.getReturnType(), equalTo(Void.TYPE));
722-
assertEquals(0, method.getExceptionTypes().length);
720+
assertThat("async method [" + method + "] should return void", method.getReturnType(), equalTo(Void.TYPE));
721+
assertEquals("async method [" + method + "] should not throw any exceptions", 0, method.getExceptionTypes().length);
723722
if (apiName.equals("security.get_ssl_certificates_async")) {
724723
assertEquals(2, method.getParameterTypes().length);
725724
assertThat(method.getParameterTypes()[0], equalTo(RequestOptions.class));
726725
assertThat(method.getParameterTypes()[1], equalTo(ActionListener.class));
727726
} else {
728-
assertEquals(3, method.getParameterTypes().length);
729-
assertThat(method.getParameterTypes()[0].getSimpleName(), endsWith("Request"));
730-
assertThat(method.getParameterTypes()[1], equalTo(RequestOptions.class));
731-
assertThat(method.getParameterTypes()[2], equalTo(ActionListener.class));
727+
assertEquals("async method [" + method + "] has the wrong number of arguments", 3, method.getParameterTypes().length);
728+
assertThat("the first parameter to async method [" + method + "] should be a request type",
729+
method.getParameterTypes()[0].getSimpleName(), endsWith("Request"));
730+
assertThat("the second parameter to async method [" + method + "] is the wrong type",
731+
method.getParameterTypes()[1], equalTo(RequestOptions.class));
732+
assertThat("the third parameter to async method [" + method + "] is the wrong type",
733+
method.getParameterTypes()[2], equalTo(ActionListener.class));
732734
}
733735
} else {
734736
//A few methods return a boolean rather than a response object
735737
if (apiName.equals("ping") || apiName.contains("exist")) {
736-
assertThat(method.getReturnType().getSimpleName(), equalTo("boolean"));
738+
assertThat("the return type for method [" + method + "] is incorrect",
739+
method.getReturnType().getSimpleName(), equalTo("boolean"));
737740
} else {
738-
assertThat(method.getReturnType().getSimpleName(), endsWith("Response"));
741+
assertThat("the return type for method [" + method + "] is incorrect",
742+
method.getReturnType().getSimpleName(), endsWith("Response"));
739743
}
740744

741-
assertEquals(1, method.getExceptionTypes().length);
745+
assertEquals("incorrect number of exceptions for method [" + method + "]", 1, method.getExceptionTypes().length);
742746
//a few methods don't accept a request object as argument
743747
if (apiName.equals("ping") || apiName.equals("info") || apiName.equals("security.get_ssl_certificates")) {
744-
assertEquals(1, method.getParameterTypes().length);
745-
assertThat(method.getParameterTypes()[0], equalTo(RequestOptions.class));
748+
assertEquals("incorrect number of arguments for method [" + method + "]", 1, method.getParameterTypes().length);
749+
assertThat("the parameter to method [" + method + "] is the wrong type",
750+
method.getParameterTypes()[0], equalTo(RequestOptions.class));
746751
} else {
747-
assertEquals(apiName, 2, method.getParameterTypes().length);
748-
assertThat(method.getParameterTypes()[0].getSimpleName(), endsWith("Request"));
749-
assertThat(method.getParameterTypes()[1], equalTo(RequestOptions.class));
752+
assertEquals("incorrect number of arguments for method [" + method + "]", 2, method.getParameterTypes().length);
753+
assertThat("the first parameter to method [" + method + "] is the wrong type",
754+
method.getParameterTypes()[0].getSimpleName(), endsWith("Request"));
755+
assertThat("the second parameter to method [" + method + "] is the wrong type",
756+
method.getParameterTypes()[1], equalTo(RequestOptions.class));
750757
}
751758

752759
boolean remove = apiSpec.remove(apiName);

‎docs/java-rest/high-level/document/exists.asciidoc‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,3 +29,10 @@ include-tagged::{doc-tests-file}[{api}-request]
2929
<5> Disable fetching stored fields.
3030

3131
include::../execution.asciidoc[]
32+
33+
34+
==== Source exists request
35+
A variant of the exists request is `existsSource` method which has the additional check
36+
that the document in question has stored the `source`. If the mapping for the index has opted
37+
to remove support for storing JSON source in documents then this method will return false
38+
for documents in this index.

‎docs/reference/settings/audit-settings.asciidoc‎

Lines changed: 33 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -18,10 +18,18 @@ Set to `true` to enable auditing on the node. The default value is `false`.
1818
`xpack.security.audit.outputs`::
1919
Specifies where audit logs are output. For example: `[ index, logfile ]`. The
2020
default value is `logfile`, which puts the auditing events in a dedicated
21-
`<clustername>_access.log` file on the node. You can also specify `index`, which
22-
puts the auditing events in an {es} index that is prefixed with
23-
`.security_audit_log`. The index can reside on the same cluster or a separate
24-
cluster.
21+
file named `<clustername>_audit.log` on each node.
22+
You can also specify `index`, which puts the auditing events in an {es} index
23+
that is prefixed with `.security_audit_log`. The index can reside on the same
24+
cluster or a separate cluster.
25+
26+
For backwards compatibility reasons, if you use the logfile output type, a
27+
`<clustername>_access.log` file is also created. It contains the same
28+
information, but it uses the older (pre-6.5.0) formatting style.
29+
If the backwards compatible format is not required, it should be disabled.
30+
To do that, change its logger level to `off` in the `log4j2.properties` file.
31+
For more information, see <<configuring-logging-levels>>.
32+
2533
+
2634
--
2735
TIP: If the index is unavailable, it is possible for auditing events to
@@ -57,17 +65,27 @@ audited in plain text when including the request body in audit events.
5765
[[node-audit-settings]]
5866
==== Local Node Info Settings
5967

60-
`xpack.security.audit.logfile.prefix.emit_node_name`::
61-
Specifies whether to include the node's name in the local node info. The
62-
default value is `true`.
63-
64-
`xpack.security.audit.logfile.prefix.emit_node_host_address`::
65-
Specifies whether to include the node's IP address in the local node info. The
66-
default value is `false`.
67-
68-
`xpack.security.audit.logfile.prefix.emit_node_host_name`::
69-
Specifies whether to include the node's host name in the local node info. The
70-
default value is `false`.
68+
`xpack.security.audit.logfile.emit_node_name`::
69+
Specifies whether to include the <<node.name,node name>> as a field in
70+
each audit event.
71+
The default value is `true`.
72+
73+
`xpack.security.audit.logfile.emit_node_host_address`::
74+
Specifies whether to include the node's IP address as a field in each audit event.
75+
The default value is `false`.
76+
77+
`xpack.security.audit.logfile.emit_node_host_name`::
78+
Specifies whether to include the node's host name as a field in each audit event.
79+
The default value is `false`.
80+
81+
`xpack.security.audit.logfile.emit_node_id`::
82+
Specifies whether to include the node id as a field in each audit event.
83+
This is available for the new format only. That is to say, this information
84+
does not exist in the `<clustername>_access.log` file.
85+
Unlike <<node.name,node name>>, whose value might change if the administrator
86+
changes the setting in the config file, the node id will persist across cluster
87+
restarts and the administrator cannot change it.
88+
The default value is `true`.
7189

7290
[[index-audit-settings]]
7391
==== Audit Log Indexing Configuration Settings

‎modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CommonAnalysisPlugin.java‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919

2020
package org.elasticsearch.analysis.common;
2121

22+
import org.apache.logging.log4j.LogManager;
2223
import org.apache.lucene.analysis.Analyzer;
2324
import org.apache.lucene.analysis.CharArraySet;
2425
import org.apache.lucene.analysis.LowerCaseFilter;
@@ -115,7 +116,6 @@
115116
import org.elasticsearch.cluster.service.ClusterService;
116117
import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
117118
import org.elasticsearch.common.logging.DeprecationLogger;
118-
import org.elasticsearch.common.logging.Loggers;
119119
import org.elasticsearch.common.regex.Regex;
120120
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
121121
import org.elasticsearch.env.Environment;
@@ -151,7 +151,7 @@
151151

152152
public class CommonAnalysisPlugin extends Plugin implements AnalysisPlugin, ScriptPlugin {
153153

154-
private static final DeprecationLogger DEPRECATION_LOGGER = new DeprecationLogger(Loggers.getLogger(CommonAnalysisPlugin.class));
154+
private static final DeprecationLogger deprecationLogger = new DeprecationLogger(LogManager.getLogger(CommonAnalysisPlugin.class));
155155

156156
private final SetOnce<ScriptService> scriptService = new SetOnce<>();
157157

@@ -376,7 +376,7 @@ public List<PreConfiguredCharFilter> getPreConfiguredCharFilters() {
376376
filters.add(PreConfiguredCharFilter.singleton("html_strip", false, HTMLStripCharFilter::new));
377377
filters.add(PreConfiguredCharFilter.singletonWithVersion("htmlStrip", false, (reader, version) -> {
378378
if (version.onOrAfter(org.elasticsearch.Version.V_6_3_0)) {
379-
DEPRECATION_LOGGER.deprecatedAndMaybeLog("htmlStrip_deprecation",
379+
deprecationLogger.deprecatedAndMaybeLog("htmlStrip_deprecation",
380380
"The [htmpStrip] char filter name is deprecated and will be removed in a future version. "
381381
+ "Please change the filter name to [html_strip] instead.");
382382
}
@@ -414,7 +414,7 @@ public List<PreConfiguredTokenFilter> getPreConfiguredTokenFilters() {
414414
new EdgeNGramTokenFilter(input, 1)));
415415
filters.add(PreConfiguredTokenFilter.singletonWithVersion("edgeNGram", false, (reader, version) -> {
416416
if (version.onOrAfter(org.elasticsearch.Version.V_6_4_0)) {
417-
DEPRECATION_LOGGER.deprecatedAndMaybeLog("edgeNGram_deprecation",
417+
deprecationLogger.deprecatedAndMaybeLog("edgeNGram_deprecation",
418418
"The [edgeNGram] token filter name is deprecated and will be removed in a future version. "
419419
+ "Please change the filter name to [edge_ngram] instead.");
420420
}
@@ -438,7 +438,7 @@ public List<PreConfiguredTokenFilter> getPreConfiguredTokenFilters() {
438438
filters.add(PreConfiguredTokenFilter.singleton("ngram", false, reader -> new NGramTokenFilter(reader, 1, 2, false)));
439439
filters.add(PreConfiguredTokenFilter.singletonWithVersion("nGram", false, (reader, version) -> {
440440
if (version.onOrAfter(org.elasticsearch.Version.V_6_4_0)) {
441-
DEPRECATION_LOGGER.deprecatedAndMaybeLog("nGram_deprecation",
441+
deprecationLogger.deprecatedAndMaybeLog("nGram_deprecation",
442442
"The [nGram] token filter name is deprecated and will be removed in a future version. "
443443
+ "Please change the filter name to [ngram] instead.");
444444
}

‎modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/LegacyDelimitedPayloadTokenFilterFactory.java‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -19,16 +19,16 @@
1919

2020
package org.elasticsearch.analysis.common;
2121

22+
import org.apache.logging.log4j.LogManager;
2223
import org.elasticsearch.Version;
2324
import org.elasticsearch.common.logging.DeprecationLogger;
24-
import org.elasticsearch.common.logging.Loggers;
2525
import org.elasticsearch.common.settings.Settings;
2626
import org.elasticsearch.env.Environment;
2727
import org.elasticsearch.index.IndexSettings;
2828

2929
public class LegacyDelimitedPayloadTokenFilterFactory extends DelimitedPayloadTokenFilterFactory {
30-
private static final DeprecationLogger DEPRECATION_LOGGER =
31-
new DeprecationLogger(Loggers.getLogger(LegacyDelimitedPayloadTokenFilterFactory.class));
30+
private static final DeprecationLogger deprecationLogger =
31+
new DeprecationLogger(LogManager.getLogger(LegacyDelimitedPayloadTokenFilterFactory.class));
3232

3333
LegacyDelimitedPayloadTokenFilterFactory(IndexSettings indexSettings, Environment env, String name, Settings settings) {
3434
super(indexSettings, env, name, settings);
@@ -37,7 +37,7 @@ public class LegacyDelimitedPayloadTokenFilterFactory extends DelimitedPayloadTo
3737
"[delimited_payload_filter] is not supported for new indices, use [delimited_payload] instead");
3838
}
3939
if (indexSettings.getIndexVersionCreated().onOrAfter(Version.V_6_2_0)) {
40-
DEPRECATION_LOGGER.deprecated("Deprecated [delimited_payload_filter] used, replaced by [delimited_payload]");
40+
deprecationLogger.deprecated("Deprecated [delimited_payload_filter] used, replaced by [delimited_payload]");
4141
}
4242
}
4343
}

0 commit comments

Comments
 (0)