diff options
Diffstat (limited to 'core/src/main/java/org/elasticsearch')
85 files changed, 1890 insertions, 315 deletions
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/repositories/put/PutRepositoryRequest.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/repositories/put/PutRepositoryRequest.java index a06175a598..f0f8d50b4c 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/cluster/repositories/put/PutRepositoryRequest.java +++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/repositories/put/PutRepositoryRequest.java @@ -142,9 +142,11 @@ public class PutRepositoryRequest extends AcknowledgedRequest<PutRepositoryReque /** * Sets the repository settings. * - * @param source repository settings in json, yaml or properties format + * @param source repository settings in json or yaml format * @return this request + * @deprecated use {@link #settings(String, XContentType)} to avoid content type auto-detection */ + @Deprecated public PutRepositoryRequest settings(String source) { this.settings = Settings.builder().loadFromSource(source).build(); return this; @@ -153,6 +155,18 @@ public class PutRepositoryRequest extends AcknowledgedRequest<PutRepositoryReque /** * Sets the repository settings. * + * @param source repository settings in json or yaml format + * @param xContentType the content type of the source + * @return this request + */ + public PutRepositoryRequest settings(String source, XContentType xContentType) { + this.settings = Settings.builder().loadFromSource(source, xContentType).build(); + return this; + } + + /** + * Sets the repository settings. + * * @param source repository settings * @return this request */ @@ -160,7 +174,7 @@ public class PutRepositoryRequest extends AcknowledgedRequest<PutRepositoryReque try { XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); builder.map(source); - settings(builder.string()); + settings(builder.string(), builder.contentType()); } catch (IOException e) { throw new ElasticsearchGenerationException("Failed to generate [" + source + "]", e); } diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/repositories/put/PutRepositoryRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/repositories/put/PutRepositoryRequestBuilder.java index 39cfa6af7f..aed09daff2 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/cluster/repositories/put/PutRepositoryRequestBuilder.java +++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/repositories/put/PutRepositoryRequestBuilder.java @@ -22,6 +22,7 @@ package org.elasticsearch.action.admin.cluster.repositories.put; import org.elasticsearch.action.support.master.AcknowledgedRequestBuilder; import org.elasticsearch.client.ElasticsearchClient; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentType; import java.util.Map; @@ -89,17 +90,31 @@ public class PutRepositoryRequestBuilder extends AcknowledgedRequestBuilder<PutR } /** - * Sets the repository settings in Json, Yaml or properties format + * Sets the repository settings in Json or Yaml format * * @param source repository settings * @return this builder + * @deprecated use {@link #setSettings(String, XContentType)} instead to avoid content type auto detection */ + @Deprecated public PutRepositoryRequestBuilder setSettings(String source) { request.settings(source); return this; } /** + * Sets the repository settings in Json or Yaml format + * + * @param source repository settings + * @param xContentType the contenty type of the source + * @return this builder + */ + public PutRepositoryRequestBuilder setSettings(String source, XContentType xContentType) { + request.settings(source, xContentType); + return this; + } + + /** * Sets the repository settings * * @param source repository settings diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/settings/ClusterUpdateSettingsRequest.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/settings/ClusterUpdateSettingsRequest.java index e5f5bdb7fb..bd0110e644 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/cluster/settings/ClusterUpdateSettingsRequest.java +++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/settings/ClusterUpdateSettingsRequest.java @@ -83,13 +83,23 @@ public class ClusterUpdateSettingsRequest extends AcknowledgedRequest<ClusterUpd /** * Sets the source containing the transient settings to be updated. They will not survive a full cluster restart + * @deprecated use {@link #transientSettings(String, XContentType)} to avoid content type detection */ + @Deprecated public ClusterUpdateSettingsRequest transientSettings(String source) { this.transientSettings = Settings.builder().loadFromSource(source).build(); return this; } /** + * Sets the source containing the transient settings to be updated. They will not survive a full cluster restart + */ + public ClusterUpdateSettingsRequest transientSettings(String source, XContentType xContentType) { + this.transientSettings = Settings.builder().loadFromSource(source, xContentType).build(); + return this; + } + + /** * Sets the transient settings to be updated. They will not survive a full cluster restart */ @SuppressWarnings("unchecked") @@ -97,7 +107,7 @@ public class ClusterUpdateSettingsRequest extends AcknowledgedRequest<ClusterUpd try { XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); builder.map(source); - transientSettings(builder.string()); + transientSettings(builder.string(), builder.contentType()); } catch (IOException e) { throw new ElasticsearchGenerationException("Failed to generate [" + source + "]", e); } @@ -122,13 +132,23 @@ public class ClusterUpdateSettingsRequest extends AcknowledgedRequest<ClusterUpd /** * Sets the source containing the persistent settings to be updated. They will get applied cross restarts + * @deprecated use {@link #persistentSettings(String, XContentType)} to avoid content type detection */ + @Deprecated public ClusterUpdateSettingsRequest persistentSettings(String source) { this.persistentSettings = Settings.builder().loadFromSource(source).build(); return this; } /** + * Sets the source containing the persistent settings to be updated. They will get applied cross restarts + */ + public ClusterUpdateSettingsRequest persistentSettings(String source, XContentType xContentType) { + this.persistentSettings = Settings.builder().loadFromSource(source, xContentType).build(); + return this; + } + + /** * Sets the persistent settings to be updated. They will get applied cross restarts */ @SuppressWarnings("unchecked") @@ -136,7 +156,7 @@ public class ClusterUpdateSettingsRequest extends AcknowledgedRequest<ClusterUpd try { XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); builder.map(source); - persistentSettings(builder.string()); + persistentSettings(builder.string(), builder.contentType()); } catch (IOException e) { throw new ElasticsearchGenerationException("Failed to generate [" + source + "]", e); } diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/settings/ClusterUpdateSettingsRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/settings/ClusterUpdateSettingsRequestBuilder.java index f0492edfeb..906b1867b1 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/cluster/settings/ClusterUpdateSettingsRequestBuilder.java +++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/settings/ClusterUpdateSettingsRequestBuilder.java @@ -22,6 +22,7 @@ package org.elasticsearch.action.admin.cluster.settings; import org.elasticsearch.action.support.master.AcknowledgedRequestBuilder; import org.elasticsearch.client.ElasticsearchClient; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentType; import java.util.Map; @@ -52,13 +53,23 @@ public class ClusterUpdateSettingsRequestBuilder extends AcknowledgedRequestBuil /** * Sets the source containing the transient settings to be updated. They will not survive a full cluster restart + * @deprecated use {@link #setTransientSettings(String, XContentType)} to avoid content type detection */ + @Deprecated public ClusterUpdateSettingsRequestBuilder setTransientSettings(String settings) { request.transientSettings(settings); return this; } /** + * Sets the source containing the transient settings to be updated. They will not survive a full cluster restart + */ + public ClusterUpdateSettingsRequestBuilder setTransientSettings(String settings, XContentType xContentType) { + request.transientSettings(settings, xContentType); + return this; + } + + /** * Sets the transient settings to be updated. They will not survive a full cluster restart */ public ClusterUpdateSettingsRequestBuilder setTransientSettings(Map settings) { @@ -84,13 +95,23 @@ public class ClusterUpdateSettingsRequestBuilder extends AcknowledgedRequestBuil /** * Sets the source containing the persistent settings to be updated. They will get applied cross restarts + * @deprecated use {@link #setPersistentSettings(String, XContentType)} to avoid content type detection */ + @Deprecated public ClusterUpdateSettingsRequestBuilder setPersistentSettings(String settings) { request.persistentSettings(settings); return this; } /** + * Sets the source containing the persistent settings to be updated. They will get applied cross restarts + */ + public ClusterUpdateSettingsRequestBuilder setPersistentSettings(String settings, XContentType xContentType) { + request.persistentSettings(settings, xContentType); + return this; + } + + /** * Sets the persistent settings to be updated. They will get applied cross restarts */ public ClusterUpdateSettingsRequestBuilder setPersistentSettings(Map settings) { diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequest.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequest.java index ae715050e8..3267b6d9c9 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequest.java +++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequest.java @@ -288,19 +288,35 @@ public class CreateSnapshotRequest extends MasterNodeRequest<CreateSnapshotReque } /** - * Sets repository-specific snapshot settings in JSON, YAML or properties format + * Sets repository-specific snapshot settings in JSON or YAML format * <p> * See repository documentation for more information. * * @param source repository-specific snapshot settings * @return this request + * @deprecated use {@link #settings(String, XContentType)} to avoid content type detection */ + @Deprecated public CreateSnapshotRequest settings(String source) { this.settings = Settings.builder().loadFromSource(source).build(); return this; } /** + * Sets repository-specific snapshot settings in JSON or YAML format + * <p> + * See repository documentation for more information. + * + * @param source repository-specific snapshot settings + * @param xContentType the content type of the source + * @return this request + */ + public CreateSnapshotRequest settings(String source, XContentType xContentType) { + this.settings = Settings.builder().loadFromSource(source, xContentType).build(); + return this; + } + + /** * Sets repository-specific snapshot settings. * <p> * See repository documentation for more information. @@ -312,7 +328,7 @@ public class CreateSnapshotRequest extends MasterNodeRequest<CreateSnapshotReque try { XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); builder.map(source); - settings(builder.string()); + settings(builder.string(), builder.contentType()); } catch (IOException e) { throw new ElasticsearchGenerationException("Failed to generate [" + source + "]", e); } diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequestBuilder.java index ebdd206b5c..d3b5e12351 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequestBuilder.java +++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequestBuilder.java @@ -23,6 +23,7 @@ import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.action.support.master.MasterNodeOperationRequestBuilder; import org.elasticsearch.client.ElasticsearchClient; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentType; import java.util.Map; @@ -147,13 +148,29 @@ public class CreateSnapshotRequestBuilder extends MasterNodeOperationRequestBuil * * @param source repository-specific snapshot settings * @return this builder + * @deprecated use {@link #setSettings(String, XContentType)} to avoid content type detection */ + @Deprecated public CreateSnapshotRequestBuilder setSettings(String source) { request.settings(source); return this; } /** + * Sets repository-specific snapshot settings in YAML or JSON format + * <p> + * See repository documentation for more information. + * + * @param source repository-specific snapshot settings + * @param xContentType the content type of the source + * @return this builder + */ + public CreateSnapshotRequestBuilder setSettings(String source, XContentType xContentType) { + request.settings(source, xContentType); + return this; + } + + /** * Sets repository-specific snapshot settings. * <p> * See repository documentation for more information. diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java index a7bbd02ee5..9d8ed49aaa 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java +++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java @@ -313,19 +313,35 @@ public class RestoreSnapshotRequest extends MasterNodeRequest<RestoreSnapshotReq } /** - * Sets repository-specific restore settings in JSON, YAML or properties format + * Sets repository-specific restore settings in JSON or YAML format * <p> * See repository documentation for more information. * * @param source repository-specific snapshot settings * @return this request + * @deprecated use {@link #settings(String, XContentType)} to avoid content type detection */ + @Deprecated public RestoreSnapshotRequest settings(String source) { this.settings = Settings.builder().loadFromSource(source).build(); return this; } /** + * Sets repository-specific restore settings in JSON or YAML format + * <p> + * See repository documentation for more information. + * + * @param source repository-specific snapshot settings + * @param xContentType the content type of the source + * @return this request + */ + public RestoreSnapshotRequest settings(String source, XContentType xContentType) { + this.settings = Settings.builder().loadFromSource(source, xContentType).build(); + return this; + } + + /** * Sets repository-specific restore settings * <p> * See repository documentation for more information. @@ -337,7 +353,7 @@ public class RestoreSnapshotRequest extends MasterNodeRequest<RestoreSnapshotReq try { XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); builder.map(source); - settings(builder.string()); + settings(builder.string(), builder.contentType()); } catch (IOException e) { throw new ElasticsearchGenerationException("Failed to generate [" + source + "]", e); } @@ -436,7 +452,9 @@ public class RestoreSnapshotRequest extends MasterNodeRequest<RestoreSnapshotReq /** * Sets settings that should be added/changed in all restored indices + * @deprecated use {@link #indexSettings(String, XContentType)} to avoid content type detection */ + @Deprecated public RestoreSnapshotRequest indexSettings(String source) { this.indexSettings = Settings.builder().loadFromSource(source).build(); return this; @@ -445,11 +463,19 @@ public class RestoreSnapshotRequest extends MasterNodeRequest<RestoreSnapshotReq /** * Sets settings that should be added/changed in all restored indices */ + public RestoreSnapshotRequest indexSettings(String source, XContentType xContentType) { + this.indexSettings = Settings.builder().loadFromSource(source, xContentType).build(); + return this; + } + + /** + * Sets settings that should be added/changed in all restored indices + */ public RestoreSnapshotRequest indexSettings(Map<String, Object> source) { try { XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); builder.map(source); - indexSettings(builder.string()); + indexSettings(builder.string(), builder.contentType()); } catch (IOException e) { throw new ElasticsearchGenerationException("Failed to generate [" + source + "]", e); } diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequestBuilder.java index 661a1a1d01..807e238724 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequestBuilder.java +++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequestBuilder.java @@ -23,6 +23,7 @@ import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.action.support.master.MasterNodeOperationRequestBuilder; import org.elasticsearch.client.ElasticsearchClient; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentType; import java.util.List; import java.util.Map; @@ -153,19 +154,35 @@ public class RestoreSnapshotRequestBuilder extends MasterNodeOperationRequestBui } /** - * Sets repository-specific restore settings in JSON, YAML or properties format + * Sets repository-specific restore settings in JSON or YAML format * <p> * See repository documentation for more information. * * @param source repository-specific snapshot settings * @return this builder + * @deprecated use {@link #setSettings(String, XContentType)} to avoid content type detection */ + @Deprecated public RestoreSnapshotRequestBuilder setSettings(String source) { request.settings(source); return this; } /** + * Sets repository-specific restore settings in JSON or YAML format + * <p> + * See repository documentation for more information. + * + * @param source repository-specific snapshot settings + * @param xContentType the content type of the source + * @return this builder + */ + public RestoreSnapshotRequestBuilder setSettings(String source, XContentType xContentType) { + request.settings(source, xContentType); + return this; + } + + /** * Sets repository-specific restore settings * <p> * See repository documentation for more information. @@ -251,7 +268,9 @@ public class RestoreSnapshotRequestBuilder extends MasterNodeOperationRequestBui * * @param source index settings * @return this builder + * @deprecated use {@link #setIndexSettings(String, XContentType)} to avoid content type detection */ + @Deprecated public RestoreSnapshotRequestBuilder setIndexSettings(String source) { request.indexSettings(source); return this; @@ -261,6 +280,18 @@ public class RestoreSnapshotRequestBuilder extends MasterNodeOperationRequestBui * Sets index settings that should be added or replaced during restore * * @param source index settings + * @param xContentType the content type of the source + * @return this builder + */ + public RestoreSnapshotRequestBuilder setIndexSettings(String source, XContentType xContentType) { + request.indexSettings(source, xContentType); + return this; + } + + /** + * Sets index settings that should be added or replaced during restore + * + * @param source index settings * @return this builder */ public RestoreSnapshotRequestBuilder setIndexSettings(Map<String, Object> source) { diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/PutStoredScriptRequest.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/PutStoredScriptRequest.java index 0dcfe22514..28c70dc45b 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/PutStoredScriptRequest.java +++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/PutStoredScriptRequest.java @@ -19,14 +19,18 @@ package org.elasticsearch.action.admin.cluster.storedscripts; +import org.elasticsearch.Version; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.support.master.AcknowledgedRequest; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.common.xcontent.XContentType; import java.io.IOException; +import java.util.Objects; import static org.elasticsearch.action.ValidateActions.addValidationError; @@ -35,17 +39,23 @@ public class PutStoredScriptRequest extends AcknowledgedRequest<PutStoredScriptR private String id; private String lang; private BytesReference content; + private XContentType xContentType; public PutStoredScriptRequest() { super(); } + @Deprecated public PutStoredScriptRequest(String id, String lang, BytesReference content) { - super(); + this(id, lang, content, XContentFactory.xContentType(content)); + } + public PutStoredScriptRequest(String id, String lang, BytesReference content, XContentType xContentType) { + super(); this.id = id; this.lang = lang; this.content = content; + this.xContentType = Objects.requireNonNull(xContentType); } @Override @@ -93,9 +103,25 @@ public class PutStoredScriptRequest extends AcknowledgedRequest<PutStoredScriptR return content; } + public XContentType xContentType() { + return xContentType; + } + + /** + * Set the script source using bytes. + * @deprecated this method is deprecated as it relies on content type detection. Use {@link #content(BytesReference, XContentType)} + */ + @Deprecated public PutStoredScriptRequest content(BytesReference content) { - this.content = content; + return content(content, XContentFactory.xContentType(content)); + } + /** + * Set the script source and the content type of the bytes. + */ + public PutStoredScriptRequest content(BytesReference content, XContentType xContentType) { + this.content = content; + this.xContentType = Objects.requireNonNull(xContentType); return this; } @@ -111,6 +137,11 @@ public class PutStoredScriptRequest extends AcknowledgedRequest<PutStoredScriptR id = in.readOptionalString(); content = in.readBytesReference(); + if (in.getVersion().after(Version.V_5_3_0_UNRELEASED)) { // TODO update to onOrAfter after backporting + xContentType = XContentType.readFrom(in); + } else { + xContentType = XContentFactory.xContentType(content); + } } @Override @@ -120,6 +151,9 @@ public class PutStoredScriptRequest extends AcknowledgedRequest<PutStoredScriptR out.writeString(lang == null ? "" : lang); out.writeOptionalString(id); out.writeBytesReference(content); + if (out.getVersion().after(Version.V_5_3_0_UNRELEASED)) { // TODO update to onOrAfter after backporting + xContentType.writeTo(out); + } } @Override @@ -127,8 +161,8 @@ public class PutStoredScriptRequest extends AcknowledgedRequest<PutStoredScriptR String source = "_na_"; try { - source = XContentHelper.convertToJson(content, false); - } catch (Exception exception) { + source = XContentHelper.convertToJson(content, false, xContentType); + } catch (Exception e) { // ignore } diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/PutStoredScriptRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/PutStoredScriptRequestBuilder.java index b701745e47..f8223d6919 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/PutStoredScriptRequestBuilder.java +++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/PutStoredScriptRequestBuilder.java @@ -22,6 +22,7 @@ package org.elasticsearch.action.admin.cluster.storedscripts; import org.elasticsearch.action.support.master.AcknowledgedRequestBuilder; import org.elasticsearch.client.ElasticsearchClient; import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.xcontent.XContentType; public class PutStoredScriptRequestBuilder extends AcknowledgedRequestBuilder<PutStoredScriptRequest, PutStoredScriptResponse, PutStoredScriptRequestBuilder> { @@ -32,19 +33,29 @@ public class PutStoredScriptRequestBuilder extends AcknowledgedRequestBuilder<Pu public PutStoredScriptRequestBuilder setId(String id) { request.id(id); - return this; } - public PutStoredScriptRequestBuilder setLang(String lang) { - request.lang(lang); - + /** + * Set the source of the script. + * @deprecated this method requires content type detection. Use {@link #setContent(BytesReference, XContentType)} instead + */ + @Deprecated + public PutStoredScriptRequestBuilder setContent(BytesReference content) { + request.content(content); return this; } - public PutStoredScriptRequestBuilder setContent(BytesReference content) { - request.content(content); + /** + * Set the source of the script along with the content type of the source + */ + public PutStoredScriptRequestBuilder setContent(BytesReference source, XContentType xContentType) { + request.content(source, xContentType); + return this; + } + public PutStoredScriptRequestBuilder setLang(String lang) { + request.lang(lang); return this; } } diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeRequest.java b/core/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeRequest.java index 6d0824eeb3..08f220e019 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeRequest.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeRequest.java @@ -75,7 +75,7 @@ public class AnalyzeRequest extends SingleShardRequest<AnalyzeRequest> { try { XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); builder.map(definition); - this.definition = Settings.builder().loadFromSource(builder.string()).build(); + this.definition = Settings.builder().loadFromSource(builder.string(), builder.contentType()).build(); } catch (IOException e) { throw new IllegalArgumentException("Failed to parse [" + definition + "]", e); } diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequest.java b/core/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequest.java index 203483d89b..798c08bd18 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequest.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequest.java @@ -21,6 +21,7 @@ package org.elasticsearch.action.admin.indices.create; import org.elasticsearch.ElasticsearchGenerationException; import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.Version; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.IndicesRequest; import org.elasticsearch.action.admin.indices.alias.Alias; @@ -43,10 +44,11 @@ import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; import java.io.IOException; -import java.nio.charset.StandardCharsets; +import java.io.UncheckedIOException; import java.util.HashMap; import java.util.HashSet; import java.util.Map; +import java.util.Objects; import java.util.Set; import static org.elasticsearch.action.ValidateActions.addValidationError; @@ -169,19 +171,29 @@ public class CreateIndexRequest extends AcknowledgedRequest<CreateIndexRequest> } /** - * The settings to create the index with (either json/yaml/properties format) + * The settings to create the index with (either json or yaml format) + * @deprecated use {@link #source(String, XContentType)} instead to avoid content type detection */ + @Deprecated public CreateIndexRequest settings(String source) { this.settings = Settings.builder().loadFromSource(source).build(); return this; } /** + * The settings to create the index with (either json or yaml format) + */ + public CreateIndexRequest settings(String source, XContentType xContentType) { + this.settings = Settings.builder().loadFromSource(source, xContentType).build(); + return this; + } + + /** * Allows to set the settings using a json builder. */ public CreateIndexRequest settings(XContentBuilder builder) { try { - settings(builder.string()); + settings(builder.string(), builder.contentType()); } catch (IOException e) { throw new ElasticsearchGenerationException("Failed to generate json settings from builder", e); } @@ -196,7 +208,7 @@ public class CreateIndexRequest extends AcknowledgedRequest<CreateIndexRequest> try { XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); builder.map(source); - settings(builder.string()); + settings(builder.string(), XContentType.JSON); } catch (IOException e) { throw new ElasticsearchGenerationException("Failed to generate [" + source + "]", e); } @@ -208,13 +220,42 @@ public class CreateIndexRequest extends AcknowledgedRequest<CreateIndexRequest> * * @param type The mapping type * @param source The mapping source + * @deprecated use {@link #mapping(String, String, XContentType)} to avoid content type detection */ + @Deprecated public CreateIndexRequest mapping(String type, String source) { + return mapping(type, new BytesArray(source), XContentFactory.xContentType(source)); + } + + /** + * Adds mapping that will be added when the index gets created. + * + * @param type The mapping type + * @param source The mapping source + * @param xContentType The content type of the source + */ + public CreateIndexRequest mapping(String type, String source, XContentType xContentType) { + return mapping(type, new BytesArray(source), xContentType); + } + + /** + * Adds mapping that will be added when the index gets created. + * + * @param type The mapping type + * @param source The mapping source + * @param xContentType the content type of the mapping source + */ + private CreateIndexRequest mapping(String type, BytesReference source, XContentType xContentType) { if (mappings.containsKey(type)) { throw new IllegalStateException("mappings for type \"" + type + "\" were already defined"); } - mappings.put(type, source); - return this; + Objects.requireNonNull(xContentType); + try { + mappings.put(type, XContentHelper.convertToJson(source, false, false, xContentType)); + return this; + } catch (IOException e) { + throw new UncheckedIOException("failed to convert to json", e); + } } /** @@ -232,15 +273,7 @@ public class CreateIndexRequest extends AcknowledgedRequest<CreateIndexRequest> * @param source The mapping source */ public CreateIndexRequest mapping(String type, XContentBuilder source) { - if (mappings.containsKey(type)) { - throw new IllegalStateException("mappings for type \"" + type + "\" were already defined"); - } - try { - mappings.put(type, source.string()); - } catch (IOException e) { - throw new IllegalArgumentException("Failed to build json for mapping request", e); - } - return this; + return mapping(type, source.bytes(), source.contentType()); } /** @@ -261,7 +294,7 @@ public class CreateIndexRequest extends AcknowledgedRequest<CreateIndexRequest> try { XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); builder.map(source); - return mapping(type, builder.string()); + return mapping(type, builder); } catch (IOException e) { throw new ElasticsearchGenerationException("Failed to generate [" + source + "]", e); } @@ -331,9 +364,18 @@ public class CreateIndexRequest extends AcknowledgedRequest<CreateIndexRequest> /** * Sets the settings and mappings as a single source. + * @deprecated use {@link #source(String, XContentType)} */ + @Deprecated public CreateIndexRequest source(String source) { - return source(source.getBytes(StandardCharsets.UTF_8)); + return source(new BytesArray(source)); + } + + /** + * Sets the settings and mappings as a single source. + */ + public CreateIndexRequest source(String source, XContentType xContentType) { + return source(new BytesArray(source), xContentType); } /** @@ -345,7 +387,9 @@ public class CreateIndexRequest extends AcknowledgedRequest<CreateIndexRequest> /** * Sets the settings and mappings as a single source. + * @deprecated use {@link #source(byte[], XContentType)} */ + @Deprecated public CreateIndexRequest source(byte[] source) { return source(source, 0, source.length); } @@ -353,6 +397,15 @@ public class CreateIndexRequest extends AcknowledgedRequest<CreateIndexRequest> /** * Sets the settings and mappings as a single source. */ + public CreateIndexRequest source(byte[] source, XContentType xContentType) { + return source(source, 0, source.length, xContentType); + } + + /** + * Sets the settings and mappings as a single source. + * @deprecated use {@link #source(byte[], int, int, XContentType)} + */ + @Deprecated public CreateIndexRequest source(byte[] source, int offset, int length) { return source(new BytesArray(source, offset, length)); } @@ -360,13 +413,27 @@ public class CreateIndexRequest extends AcknowledgedRequest<CreateIndexRequest> /** * Sets the settings and mappings as a single source. */ + public CreateIndexRequest source(byte[] source, int offset, int length, XContentType xContentType) { + return source(new BytesArray(source, offset, length), xContentType); + } + + /** + * Sets the settings and mappings as a single source. + * @deprecated use {@link #source(BytesReference, XContentType)} + */ + @Deprecated public CreateIndexRequest source(BytesReference source) { XContentType xContentType = XContentFactory.xContentType(source); - if (xContentType != null) { - source(XContentHelper.convertToMap(source, false).v2()); - } else { - settings(source.utf8ToString()); - } + source(source, xContentType); + return this; + } + + /** + * Sets the settings and mappings as a single source. + */ + public CreateIndexRequest source(BytesReference source, XContentType xContentType) { + Objects.requireNonNull(xContentType); + source(XContentHelper.convertToMap(source, false, xContentType).v2()); return this; } @@ -483,7 +550,13 @@ public class CreateIndexRequest extends AcknowledgedRequest<CreateIndexRequest> readTimeout(in); int size = in.readVInt(); for (int i = 0; i < size; i++) { - mappings.put(in.readString(), in.readString()); + final String type = in.readString(); + String source = in.readString(); + if (in.getVersion().before(Version.V_6_0_0_alpha1_UNRELEASED)) { // TODO change to 5.3.0 after backport + // we do not know the content type that comes from earlier versions so we autodetect and convert + source = XContentHelper.convertToJson(new BytesArray(source), false, false, XContentFactory.xContentType(source)); + } + mappings.put(type, source); } int customSize = in.readVInt(); for (int i = 0; i < customSize; i++) { diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequestBuilder.java index eaae4d53b7..237c88244b 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequestBuilder.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequestBuilder.java @@ -27,6 +27,7 @@ import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentType; import java.util.Map; @@ -76,14 +77,24 @@ public class CreateIndexRequestBuilder extends AcknowledgedRequestBuilder<Create } /** - * The settings to create the index with (either json/yaml/properties format) + * The settings to create the index with (either json or yaml format) + * @deprecated use {@link #setSettings(String, XContentType)} to avoid content type detection */ + @Deprecated public CreateIndexRequestBuilder setSettings(String source) { request.settings(source); return this; } /** + * The settings to create the index with (either json or yaml format) + */ + public CreateIndexRequestBuilder setSettings(String source, XContentType xContentType) { + request.settings(source, xContentType); + return this; + } + + /** * A simplified version of settings that takes key value pairs settings. */ public CreateIndexRequestBuilder setSettings(Object... settings) { @@ -104,13 +115,27 @@ public class CreateIndexRequestBuilder extends AcknowledgedRequestBuilder<Create * * @param type The mapping type * @param source The mapping source + * @deprecated use {@link #addMapping(String, String, XContentType)} to avoid content type auto-detection */ + @Deprecated public CreateIndexRequestBuilder addMapping(String type, String source) { request.mapping(type, source); return this; } /** + * Adds mapping that will be added when the index gets created. + * + * @param type The mapping type + * @param source The mapping source + * @param xContentType The content type of the source + */ + public CreateIndexRequestBuilder addMapping(String type, String source, XContentType xContentType) { + request.mapping(type, source, xContentType); + return this; + } + + /** * The cause for this index creation. */ public CreateIndexRequestBuilder setCause(String cause) { @@ -191,7 +216,9 @@ public class CreateIndexRequestBuilder extends AcknowledgedRequestBuilder<Create /** * Sets the settings and mappings as a single source. + * @deprecated use {@link #setSource(String, XContentType)} */ + @Deprecated public CreateIndexRequestBuilder setSource(String source) { request.source(source); return this; @@ -200,6 +227,16 @@ public class CreateIndexRequestBuilder extends AcknowledgedRequestBuilder<Create /** * Sets the settings and mappings as a single source. */ + public CreateIndexRequestBuilder setSource(String source, XContentType xContentType) { + request.source(source, xContentType); + return this; + } + + /** + * Sets the settings and mappings as a single source. + * @deprecated use {@link #setSource(BytesReference, XContentType)} + */ + @Deprecated public CreateIndexRequestBuilder setSource(BytesReference source) { request.source(source); return this; @@ -208,6 +245,16 @@ public class CreateIndexRequestBuilder extends AcknowledgedRequestBuilder<Create /** * Sets the settings and mappings as a single source. */ + public CreateIndexRequestBuilder setSource(BytesReference source, XContentType xContentType) { + request.source(source, xContentType); + return this; + } + + /** + * Sets the settings and mappings as a single source. + * @deprecated use {@link #setSource(byte[], XContentType)} + */ + @Deprecated public CreateIndexRequestBuilder setSource(byte[] source) { request.source(source); return this; @@ -216,6 +263,16 @@ public class CreateIndexRequestBuilder extends AcknowledgedRequestBuilder<Create /** * Sets the settings and mappings as a single source. */ + public CreateIndexRequestBuilder setSource(byte[] source, XContentType xContentType) { + request.source(source, xContentType); + return this; + } + + /** + * Sets the settings and mappings as a single source. + * @deprecated use {@link #setSource(byte[], int, int, XContentType)} + */ + @Deprecated public CreateIndexRequestBuilder setSource(byte[] source, int offset, int length) { request.source(source, offset, length); return this; @@ -224,6 +281,14 @@ public class CreateIndexRequestBuilder extends AcknowledgedRequestBuilder<Create /** * Sets the settings and mappings as a single source. */ + public CreateIndexRequestBuilder setSource(byte[] source, int offset, int length, XContentType xContentType) { + request.source(source, offset, length, xContentType); + return this; + } + + /** + * Sets the settings and mappings as a single source. + */ public CreateIndexRequestBuilder setSource(Map<String, ?> source) { request.source(source); return this; diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponse.java b/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponse.java index e0cedcf841..3f4ddaf08d 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponse.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponse.java @@ -27,6 +27,7 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.mapper.Mapper; import java.io.IOException; @@ -108,20 +109,25 @@ public class GetFieldMappingsResponse extends ActionResponse implements ToXConte /** Returns the mappings as a map. Note that the returned map has a single key which is always the field's {@link Mapper#name}. */ public Map<String, Object> sourceAsMap() { - return XContentHelper.convertToMap(source, true).v2(); + return XContentHelper.convertToMap(source, true, XContentType.JSON).v2(); } public boolean isNull() { return NULL.fullName().equals(fullName) && NULL.source.length() == source.length(); } + //pkg-private for testing + BytesReference getSource() { + return source; + } + @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.field("full_name", fullName); if (params.paramAsBoolean("pretty", false)) { builder.field("mapping", sourceAsMap()); } else { - builder.rawField("mapping", source); + builder.rawField("mapping", source, XContentType.JSON); } return builder; } diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/TransportGetFieldMappingsIndexAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/TransportGetFieldMappingsIndexAction.java index 864c6703c4..ec84ce4e34 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/TransportGetFieldMappingsIndexAction.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/TransportGetFieldMappingsIndexAction.java @@ -220,4 +220,4 @@ public class TransportGetFieldMappingsIndexAction extends TransportSingleShardAc } } -}
\ No newline at end of file +} diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequest.java b/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequest.java index 152bc51654..400701f91c 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequest.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequest.java @@ -22,19 +22,24 @@ package org.elasticsearch.action.admin.indices.mapping.put; import com.carrotsearch.hppc.ObjectHashSet; import org.elasticsearch.ElasticsearchGenerationException; +import org.elasticsearch.Version; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.IndicesRequest; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.action.support.master.AcknowledgedRequest; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.Index; import java.io.IOException; +import java.io.UncheckedIOException; import java.util.Arrays; import java.util.Map; import java.util.Objects; @@ -245,7 +250,7 @@ public class PutMappingRequest extends AcknowledgedRequest<PutMappingRequest> im */ public PutMappingRequest source(XContentBuilder mappingBuilder) { try { - return source(mappingBuilder.string()); + return source(mappingBuilder.string(), mappingBuilder.contentType()); } catch (IOException e) { throw new IllegalArgumentException("Failed to build json for mapping request", e); } @@ -259,7 +264,7 @@ public class PutMappingRequest extends AcknowledgedRequest<PutMappingRequest> im try { XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); builder.map(mappingSource); - return source(builder.string()); + return source(builder.string(), XContentType.JSON); } catch (IOException e) { throw new ElasticsearchGenerationException("Failed to generate [" + mappingSource + "]", e); } @@ -267,10 +272,31 @@ public class PutMappingRequest extends AcknowledgedRequest<PutMappingRequest> im /** * The mapping source definition. + * @deprecated use {@link #source(String, XContentType)} */ + @Deprecated public PutMappingRequest source(String mappingSource) { - this.source = mappingSource; - return this; + return source(mappingSource, XContentFactory.xContentType(mappingSource)); + } + + /** + * The mapping source definition. + */ + public PutMappingRequest source(String mappingSource, XContentType xContentType) { + return source(new BytesArray(mappingSource), xContentType); + } + + /** + * The mapping source definition. + */ + public PutMappingRequest source(BytesReference mappingSource, XContentType xContentType) { + Objects.requireNonNull(xContentType); + try { + this.source = XContentHelper.convertToJson(mappingSource, false, false, xContentType); + return this; + } catch (IOException e) { + throw new UncheckedIOException("failed to convert source to json", e); + } } /** True if all fields that span multiple types should be updated, false otherwise */ @@ -291,6 +317,10 @@ public class PutMappingRequest extends AcknowledgedRequest<PutMappingRequest> im indicesOptions = IndicesOptions.readIndicesOptions(in); type = in.readOptionalString(); source = in.readString(); + if (in.getVersion().before(Version.V_6_0_0_alpha1_UNRELEASED)) { // TODO change to V_5_3 once backported + // we do not know the format from earlier versions so convert if necessary + source = XContentHelper.convertToJson(new BytesArray(source), false, false, XContentFactory.xContentType(source)); + } updateAllTypes = in.readBoolean(); readTimeout(in); concreteIndex = in.readOptionalWriteable(Index::new); diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequestBuilder.java index c21c40cf04..012a593ebc 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequestBuilder.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequestBuilder.java @@ -23,6 +23,7 @@ import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.action.support.master.AcknowledgedRequestBuilder; import org.elasticsearch.client.ElasticsearchClient; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.Index; import java.util.Map; @@ -82,13 +83,23 @@ public class PutMappingRequestBuilder extends AcknowledgedRequestBuilder<PutMapp /** * The mapping source definition. + * @deprecated use {@link #setSource(String, XContentType)} */ + @Deprecated public PutMappingRequestBuilder setSource(String mappingSource) { request.source(mappingSource); return this; } /** + * The mapping source definition. + */ + public PutMappingRequestBuilder setSource(String mappingSource, XContentType xContentType) { + request.source(mappingSource, xContentType); + return this; + } + + /** * A specialized simplified mapping source method, takes the form of simple properties definition: * ("field1", "type=string,store=true"). */ diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/settings/put/UpdateSettingsRequest.java b/core/src/main/java/org/elasticsearch/action/admin/indices/settings/put/UpdateSettingsRequest.java index e912c6ec73..80f3fb1a07 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/settings/put/UpdateSettingsRequest.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/settings/put/UpdateSettingsRequest.java @@ -121,14 +121,24 @@ public class UpdateSettingsRequest extends AcknowledgedRequest<UpdateSettingsReq } /** - * Sets the settings to be updated (either json/yaml/properties format) + * Sets the settings to be updated (either json or yaml format) + * @deprecated use {@link #settings(String, XContentType)} to avoid content type detection */ + @Deprecated public UpdateSettingsRequest settings(String source) { this.settings = Settings.builder().loadFromSource(source).build(); return this; } /** + * Sets the settings to be updated (either json or yaml format) + */ + public UpdateSettingsRequest settings(String source, XContentType xContentType) { + this.settings = Settings.builder().loadFromSource(source, xContentType).build(); + return this; + } + + /** * Returns <code>true</code> iff the settings update should only add but not update settings. If the setting already exists * it should not be overwritten by this update. The default is <code>false</code> */ @@ -146,14 +156,14 @@ public class UpdateSettingsRequest extends AcknowledgedRequest<UpdateSettingsReq } /** - * Sets the settings to be updated (either json/yaml/properties format) + * Sets the settings to be updated (either json or yaml format) */ @SuppressWarnings("unchecked") public UpdateSettingsRequest settings(Map source) { try { XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); builder.map(source); - settings(builder.string()); + settings(builder.string(), builder.contentType()); } catch (IOException e) { throw new ElasticsearchGenerationException("Failed to generate [" + source + "]", e); } diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/settings/put/UpdateSettingsRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/admin/indices/settings/put/UpdateSettingsRequestBuilder.java index 36dfbf3b2d..a9cecbfc5a 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/settings/put/UpdateSettingsRequestBuilder.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/settings/put/UpdateSettingsRequestBuilder.java @@ -23,6 +23,7 @@ import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.action.support.master.AcknowledgedRequestBuilder; import org.elasticsearch.client.ElasticsearchClient; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentType; import java.util.Map; @@ -70,15 +71,25 @@ public class UpdateSettingsRequestBuilder extends AcknowledgedRequestBuilder<Upd } /** - * Sets the settings to be updated (either json/yaml/properties format) + * Sets the settings to be updated (either json or yaml format) + * @deprecated use {@link #setSettings(String, XContentType)} to avoid content type detection */ + @Deprecated public UpdateSettingsRequestBuilder setSettings(String source) { request.settings(source); return this; } /** - * Sets the settings to be updated (either json/yaml/properties format) + * Sets the settings to be updated (either json or yaml format) + */ + public UpdateSettingsRequestBuilder setSettings(String source, XContentType xContentType) { + request.settings(source, xContentType); + return this; + } + + /** + * Sets the settings to be updated */ public UpdateSettingsRequestBuilder setSettings(Map<String, Object> source) { request.settings(source); diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequest.java b/core/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequest.java index a7d6241d31..b0c13540df 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequest.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequest.java @@ -45,11 +45,13 @@ import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.common.xcontent.support.XContentMapValues; import java.io.IOException; +import java.io.UncheckedIOException; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; @@ -179,21 +181,31 @@ public class PutIndexTemplateRequest extends MasterNodeRequest<PutIndexTemplateR } /** - * The settings to create the index template with (either json/yaml/properties format). + * The settings to create the index template with (either json/yaml format). + * @deprecated use {@link #settings(String, XContentType)} */ + @Deprecated public PutIndexTemplateRequest settings(String source) { this.settings = Settings.builder().loadFromSource(source).build(); return this; } /** - * The settings to crete the index template with (either json/yaml/properties format). + * The settings to create the index template with (either json/yaml format). + */ + public PutIndexTemplateRequest settings(String source, XContentType xContentType) { + this.settings = Settings.builder().loadFromSource(source, xContentType).build(); + return this; + } + + /** + * The settings to create the index template with (either json or yaml format). */ public PutIndexTemplateRequest settings(Map<String, Object> source) { try { XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); builder.map(source); - settings(builder.string()); + settings(builder.string(), XContentType.JSON); } catch (IOException e) { throw new ElasticsearchGenerationException("Failed to generate [" + source + "]", e); } @@ -209,10 +221,23 @@ public class PutIndexTemplateRequest extends MasterNodeRequest<PutIndexTemplateR * * @param type The mapping type * @param source The mapping source + * @deprecated use {@link #mapping(String, String, XContentType)} */ + @Deprecated public PutIndexTemplateRequest mapping(String type, String source) { - mappings.put(type, source); - return this; + XContentType xContentType = XContentFactory.xContentType(source); + return mapping(type, source, xContentType); + } + + /** + * Adds mapping that will be added when the index gets created. + * + * @param type The mapping type + * @param source The mapping source + * @param xContentType The type of content contained within the source + */ + public PutIndexTemplateRequest mapping(String type, String source, XContentType xContentType) { + return mapping(type, new BytesArray(source), xContentType); } /** @@ -234,12 +259,24 @@ public class PutIndexTemplateRequest extends MasterNodeRequest<PutIndexTemplateR * @param source The mapping source */ public PutIndexTemplateRequest mapping(String type, XContentBuilder source) { + return mapping(type, source.bytes(), source.contentType()); + } + + /** + * Adds mapping that will be added when the index gets created. + * + * @param type The mapping type + * @param source The mapping source + * @param xContentType the source content type + */ + public PutIndexTemplateRequest mapping(String type, BytesReference source, XContentType xContentType) { + Objects.requireNonNull(xContentType); try { - mappings.put(type, source.string()); + mappings.put(type, XContentHelper.convertToJson(source, false, false, xContentType)); + return this; } catch (IOException e) { - throw new IllegalArgumentException("Failed to build json for mapping request", e); + throw new UncheckedIOException("failed to convert source to json", e); } - return this; } /** @@ -256,7 +293,7 @@ public class PutIndexTemplateRequest extends MasterNodeRequest<PutIndexTemplateR try { XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); builder.map(source); - return mapping(type, builder.string()); + return mapping(type, builder); } catch (IOException e) { throw new ElasticsearchGenerationException("Failed to generate [" + source + "]", e); } @@ -280,7 +317,7 @@ public class PutIndexTemplateRequest extends MasterNodeRequest<PutIndexTemplateR */ public PutIndexTemplateRequest source(XContentBuilder templateBuilder) { try { - return source(templateBuilder.bytes()); + return source(templateBuilder.bytes(), templateBuilder.contentType()); } catch (Exception e) { throw new IllegalArgumentException("Failed to build json for template request", e); } @@ -350,7 +387,9 @@ public class PutIndexTemplateRequest extends MasterNodeRequest<PutIndexTemplateR /** * The template source definition. + * @deprecated use {@link #source(String, XContentType)} */ + @Deprecated public PutIndexTemplateRequest source(String templateSource) { return source(XContentHelper.convertToMap(XContentFactory.xContent(templateSource), templateSource, true)); } @@ -358,6 +397,15 @@ public class PutIndexTemplateRequest extends MasterNodeRequest<PutIndexTemplateR /** * The template source definition. */ + public PutIndexTemplateRequest source(String templateSource, XContentType xContentType) { + return source(XContentHelper.convertToMap(xContentType.xContent(), templateSource, true)); + } + + /** + * The template source definition. + * @deprecated use {@link #source(byte[], XContentType)} + */ + @Deprecated public PutIndexTemplateRequest source(byte[] source) { return source(source, 0, source.length); } @@ -365,6 +413,15 @@ public class PutIndexTemplateRequest extends MasterNodeRequest<PutIndexTemplateR /** * The template source definition. */ + public PutIndexTemplateRequest source(byte[] source, XContentType xContentType) { + return source(source, 0, source.length, xContentType); + } + + /** + * The template source definition. + * @deprecated use {@link #source(byte[], int, int, XContentType)} + */ + @Deprecated public PutIndexTemplateRequest source(byte[] source, int offset, int length) { return source(new BytesArray(source, offset, length)); } @@ -372,10 +429,26 @@ public class PutIndexTemplateRequest extends MasterNodeRequest<PutIndexTemplateR /** * The template source definition. */ + public PutIndexTemplateRequest source(byte[] source, int offset, int length, XContentType xContentType) { + return source(new BytesArray(source, offset, length), xContentType); + } + + /** + * The template source definition. + * @deprecated use {@link #source(BytesReference, XContentType)} + */ + @Deprecated public PutIndexTemplateRequest source(BytesReference source) { return source(XContentHelper.convertToMap(source, true).v2()); } + /** + * The template source definition. + */ + public PutIndexTemplateRequest source(BytesReference source, XContentType xContentType) { + return source(XContentHelper.convertToMap(source, true, xContentType).v2()); + } + public PutIndexTemplateRequest custom(IndexMetaData.Custom custom) { customs.put(custom.type(), custom); return this; @@ -471,7 +544,14 @@ public class PutIndexTemplateRequest extends MasterNodeRequest<PutIndexTemplateR settings = readSettingsFromStream(in); int size = in.readVInt(); for (int i = 0; i < size; i++) { - mappings.put(in.readString(), in.readString()); + final String type = in.readString(); + String mappingSource = in.readString(); + if (in.getVersion().before(Version.V_6_0_0_alpha1_UNRELEASED)) { // TODO change to V_5_3_0 once backported + // we do not know the incoming type so convert it if needed + mappingSource = + XContentHelper.convertToJson(new BytesArray(mappingSource), false, false, XContentFactory.xContentType(mappingSource)); + } + mappings.put(type, mappingSource); } int customSize = in.readVInt(); for (int i = 0; i < customSize; i++) { diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequestBuilder.java index c1db96ae7c..e7ca25e511 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequestBuilder.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequestBuilder.java @@ -24,6 +24,7 @@ import org.elasticsearch.client.ElasticsearchClient; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentType; import java.util.Collections; import java.util.List; @@ -100,15 +101,25 @@ public class PutIndexTemplateRequestBuilder } /** - * The settings to crete the index template with (either json/yaml/properties format) + * The settings to crete the index template with (either json or yaml format) + * @deprecated use {@link #setSettings(String, XContentType)} */ + @Deprecated public PutIndexTemplateRequestBuilder setSettings(String source) { request.settings(source); return this; } /** - * The settings to crete the index template with (either json/yaml/properties format) + * The settings to crete the index template with (either json or yaml format) + */ + public PutIndexTemplateRequestBuilder setSettings(String source, XContentType xContentType) { + request.settings(source, xContentType); + return this; + } + + /** + * The settings to crete the index template with (either json or yaml format) */ public PutIndexTemplateRequestBuilder setSettings(Map<String, Object> source) { request.settings(source); @@ -120,13 +131,27 @@ public class PutIndexTemplateRequestBuilder * * @param type The mapping type * @param source The mapping source + * @deprecated use {@link #addMapping(String, String, XContentType)} */ + @Deprecated public PutIndexTemplateRequestBuilder addMapping(String type, String source) { request.mapping(type, source); return this; } /** + * Adds mapping that will be added when the index template gets created. + * + * @param type The mapping type + * @param source The mapping source + * @param xContentType The type/format of the source + */ + public PutIndexTemplateRequestBuilder addMapping(String type, String source, XContentType xContentType) { + request.mapping(type, source, xContentType); + return this; + } + + /** * A specialized simplified mapping source method, takes the form of simple properties definition: * ("field1", "type=string,store=true"). */ @@ -226,7 +251,9 @@ public class PutIndexTemplateRequestBuilder /** * The template source definition. + * @deprecated use {@link #setSource(BytesReference, XContentType)} */ + @Deprecated public PutIndexTemplateRequestBuilder setSource(String templateSource) { request.source(templateSource); return this; @@ -235,6 +262,16 @@ public class PutIndexTemplateRequestBuilder /** * The template source definition. */ + public PutIndexTemplateRequestBuilder setSource(BytesReference templateSource, XContentType xContentType) { + request.source(templateSource, xContentType); + return this; + } + + /** + * The template source definition. + * @deprecated use {@link #setSource(BytesReference, XContentType)} + */ + @Deprecated public PutIndexTemplateRequestBuilder setSource(BytesReference templateSource) { request.source(templateSource); return this; @@ -242,7 +279,9 @@ public class PutIndexTemplateRequestBuilder /** * The template source definition. + * @deprecated use {@link #setSource(byte[], XContentType)} */ + @Deprecated public PutIndexTemplateRequestBuilder setSource(byte[] templateSource) { request.source(templateSource); return this; @@ -251,8 +290,26 @@ public class PutIndexTemplateRequestBuilder /** * The template source definition. */ + public PutIndexTemplateRequestBuilder setSource(byte[] templateSource, XContentType xContentType) { + request.source(templateSource, xContentType); + return this; + } + + /** + * The template source definition. + * @deprecated use {@link #setSource(byte[], int, int, XContentType)} + */ + @Deprecated public PutIndexTemplateRequestBuilder setSource(byte[] templateSource, int offset, int length) { request.source(templateSource, offset, length); return this; } + + /** + * The template source definition. + */ + public PutIndexTemplateRequestBuilder setSource(byte[] templateSource, int offset, int length, XContentType xContentType) { + request.source(templateSource, offset, length, xContentType); + return this; + } } diff --git a/core/src/main/java/org/elasticsearch/action/bulk/BulkRequest.java b/core/src/main/java/org/elasticsearch/action/bulk/BulkRequest.java index 20d5e64f49..371659586f 100644 --- a/core/src/main/java/org/elasticsearch/action/bulk/BulkRequest.java +++ b/core/src/main/java/org/elasticsearch/action/bulk/BulkRequest.java @@ -43,6 +43,7 @@ import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.XContent; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.VersionType; import org.elasticsearch.search.fetch.subphase.FetchSourceContext; @@ -244,7 +245,9 @@ public class BulkRequest extends ActionRequest implements CompositeIndicesReques /** * Adds a framed data in binary format + * @deprecated use {@link #add(byte[], int, int, XContentType)} */ + @Deprecated public BulkRequest add(byte[] data, int from, int length) throws IOException { return add(data, from, length, null, null); } @@ -252,6 +255,15 @@ public class BulkRequest extends ActionRequest implements CompositeIndicesReques /** * Adds a framed data in binary format */ + public BulkRequest add(byte[] data, int from, int length, XContentType xContentType) throws IOException { + return add(data, from, length, null, null, xContentType); + } + + /** + * Adds a framed data in binary format + * @deprecated use {@link #add(byte[], int, int, String, String, XContentType)} + */ + @Deprecated public BulkRequest add(byte[] data, int from, int length, @Nullable String defaultIndex, @Nullable String defaultType) throws IOException { return add(new BytesArray(data, from, length), defaultIndex, defaultType); } @@ -259,6 +271,17 @@ public class BulkRequest extends ActionRequest implements CompositeIndicesReques /** * Adds a framed data in binary format */ + public BulkRequest add(byte[] data, int from, int length, @Nullable String defaultIndex, @Nullable String defaultType, + XContentType xContentType) throws IOException { + return add(new BytesArray(data, from, length), defaultIndex, defaultType, xContentType); + } + + /** + * Adds a framed data in binary format + * + * @deprecated use {@link #add(BytesReference, String, String, XContentType)} + */ + @Deprecated public BulkRequest add(BytesReference data, @Nullable String defaultIndex, @Nullable String defaultType) throws IOException { return add(data, defaultIndex, defaultType, null, null, null, null, null, true); } @@ -266,12 +289,40 @@ public class BulkRequest extends ActionRequest implements CompositeIndicesReques /** * Adds a framed data in binary format */ + public BulkRequest add(BytesReference data, @Nullable String defaultIndex, @Nullable String defaultType, + XContentType xContentType) throws IOException { + return add(data, defaultIndex, defaultType, null, null, null, null, null, true, xContentType); + } + + /** + * Adds a framed data in binary format + * + * @deprecated use {@link #add(BytesReference, String, String, boolean, XContentType)} + */ + @Deprecated public BulkRequest add(BytesReference data, @Nullable String defaultIndex, @Nullable String defaultType, boolean allowExplicitIndex) throws IOException { return add(data, defaultIndex, defaultType, null, null, null, null, null, allowExplicitIndex); } + /** + * Adds a framed data in binary format + */ + public BulkRequest add(BytesReference data, @Nullable String defaultIndex, @Nullable String defaultType, boolean allowExplicitIndex, + XContentType xContentType) throws IOException { + return add(data, defaultIndex, defaultType, null, null, null, null, null, allowExplicitIndex, xContentType); + } + + @Deprecated public BulkRequest add(BytesReference data, @Nullable String defaultIndex, @Nullable String defaultType, @Nullable String defaultRouting, @Nullable String[] defaultFields, @Nullable FetchSourceContext defaultFetchSourceContext, @Nullable String defaultPipeline, @Nullable Object payload, boolean allowExplicitIndex) throws IOException { - XContent xContent = XContentFactory.xContent(data); + XContentType xContentType = XContentFactory.xContentType(data); + return add(data, defaultIndex, defaultType, defaultRouting, defaultFields, defaultFetchSourceContext, defaultPipeline, payload, + allowExplicitIndex, xContentType); + } + + public BulkRequest add(BytesReference data, @Nullable String defaultIndex, @Nullable String defaultType, @Nullable String + defaultRouting, @Nullable String[] defaultFields, @Nullable FetchSourceContext defaultFetchSourceContext, @Nullable String + defaultPipeline, @Nullable Object payload, boolean allowExplicitIndex, XContentType xContentType) throws IOException { + XContent xContent = xContentType.xContent(); int line = 0; int from = 0; int length = data.length(); @@ -387,16 +438,16 @@ public class BulkRequest extends ActionRequest implements CompositeIndicesReques if ("index".equals(action)) { if (opType == null) { internalAdd(new IndexRequest(index, type, id).routing(routing).parent(parent).version(version).versionType(versionType) - .setPipeline(pipeline).source(data.slice(from, nextMarker - from)), payload); + .setPipeline(pipeline).source(data.slice(from, nextMarker - from), xContentType), payload); } else { internalAdd(new IndexRequest(index, type, id).routing(routing).parent(parent).version(version).versionType(versionType) .create("create".equals(opType)).setPipeline(pipeline) - .source(data.slice(from, nextMarker - from)), payload); + .source(data.slice(from, nextMarker - from), xContentType), payload); } } else if ("create".equals(action)) { internalAdd(new IndexRequest(index, type, id).routing(routing).parent(parent).version(version).versionType(versionType) .create(true).setPipeline(pipeline) - .source(data.slice(from, nextMarker - from)), payload); + .source(data.slice(from, nextMarker - from), xContentType), payload); } else if ("update".equals(action)) { UpdateRequest updateRequest = new UpdateRequest(index, type, id).routing(routing).parent(parent).retryOnConflict(retryOnConflict) .version(version).versionType(versionType) diff --git a/core/src/main/java/org/elasticsearch/action/bulk/BulkRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/bulk/BulkRequestBuilder.java index c48a8f507b..8f634fa28a 100644 --- a/core/src/main/java/org/elasticsearch/action/bulk/BulkRequestBuilder.java +++ b/core/src/main/java/org/elasticsearch/action/bulk/BulkRequestBuilder.java @@ -32,6 +32,7 @@ import org.elasticsearch.action.update.UpdateRequestBuilder; import org.elasticsearch.client.ElasticsearchClient; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.xcontent.XContentType; /** * A bulk request holds an ordered {@link IndexRequest}s and {@link DeleteRequest}s and allows to executes @@ -97,7 +98,9 @@ public class BulkRequestBuilder extends ActionRequestBuilder<BulkRequest, BulkRe /** * Adds a framed data in binary format + * @deprecated use {@link #add(byte[], int, int, XContentType)} */ + @Deprecated public BulkRequestBuilder add(byte[] data, int from, int length) throws Exception { request.add(data, from, length, null, null); return this; @@ -106,12 +109,31 @@ public class BulkRequestBuilder extends ActionRequestBuilder<BulkRequest, BulkRe /** * Adds a framed data in binary format */ + public BulkRequestBuilder add(byte[] data, int from, int length, XContentType xContentType) throws Exception { + request.add(data, from, length, null, null, xContentType); + return this; + } + + /** + * Adds a framed data in binary format + * @deprecated use {@link #add(byte[], int, int, String, String, XContentType)} + */ + @Deprecated public BulkRequestBuilder add(byte[] data, int from, int length, @Nullable String defaultIndex, @Nullable String defaultType) throws Exception { request.add(data, from, length, defaultIndex, defaultType); return this; } /** + * Adds a framed data in binary format + */ + public BulkRequestBuilder add(byte[] data, int from, int length, @Nullable String defaultIndex, @Nullable String defaultType, + XContentType xContentType) throws Exception { + request.add(data, from, length, defaultIndex, defaultType, xContentType); + return this; + } + + /** * Sets the number of shard copies that must be active before proceeding with the write. * See {@link ReplicationRequest#waitForActiveShards(ActiveShardCount)} for details. */ diff --git a/core/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java b/core/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java index 12981664c2..f5faa18403 100644 --- a/core/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java +++ b/core/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java @@ -328,7 +328,7 @@ public class TransportShardBulkAction extends TransportWriteAction<BulkShardRequ if ((updateRequest.fetchSource() != null && updateRequest.fetchSource().fetchSource()) || (updateRequest.fields() != null && updateRequest.fields().length > 0)) { Tuple<XContentType, Map<String, Object>> sourceAndContent = - XContentHelper.convertToMap(indexSourceAsBytes, true); + XContentHelper.convertToMap(indexSourceAsBytes, true, updateIndexRequest.getContentType()); updateResponse.setGetResult(updateHelper.extractGetResult(updateRequest, request.index(), indexResponse.getVersion(), sourceAndContent.v2(), sourceAndContent.v1(), indexSourceAsBytes)); } @@ -427,8 +427,9 @@ public class TransportShardBulkAction extends TransportWriteAction<BulkShardRequ */ public static Engine.IndexResult executeIndexRequestOnReplica(IndexRequest request, IndexShard replica) throws IOException { final ShardId shardId = replica.shardId(); - SourceToParse sourceToParse = SourceToParse.source(SourceToParse.Origin.REPLICA, shardId.getIndexName(), request.type(), request.id(), request.source()) - .routing(request.routing()).parent(request.parent()); + SourceToParse sourceToParse = + SourceToParse.source(SourceToParse.Origin.REPLICA, shardId.getIndexName(), request.type(), request.id(), request.source(), + request.getContentType()).routing(request.routing()).parent(request.parent()); final Engine.Index operation; try { @@ -445,8 +446,9 @@ public class TransportShardBulkAction extends TransportWriteAction<BulkShardRequ /** Utility method to prepare an index operation on primary shards */ static Engine.Index prepareIndexOperationOnPrimary(IndexRequest request, IndexShard primary) { - SourceToParse sourceToParse = SourceToParse.source(SourceToParse.Origin.PRIMARY, request.index(), request.type(), request.id(), request.source()) - .routing(request.routing()).parent(request.parent()); + SourceToParse sourceToParse = + SourceToParse.source(SourceToParse.Origin.PRIMARY, request.index(), request.type(), request.id(), request.source(), + request.getContentType()).routing(request.routing()).parent(request.parent()); return primary.prepareIndexOnPrimary(sourceToParse, request.version(), request.versionType(), request.getAutoGeneratedTimestamp(), request.isRetry()); } diff --git a/core/src/main/java/org/elasticsearch/action/bulk/byscroll/ClientScrollableHitSource.java b/core/src/main/java/org/elasticsearch/action/bulk/byscroll/ClientScrollableHitSource.java index 8831b2f636..4f2aefc101 100644 --- a/core/src/main/java/org/elasticsearch/action/bulk/byscroll/ClientScrollableHitSource.java +++ b/core/src/main/java/org/elasticsearch/action/bulk/byscroll/ClientScrollableHitSource.java @@ -37,6 +37,8 @@ import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.concurrent.AbstractRunnable; import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.mapper.ParentFieldMapper; import org.elasticsearch.index.mapper.RoutingFieldMapper; import org.elasticsearch.search.SearchHit; @@ -233,6 +235,10 @@ public class ClientScrollableHitSource extends ScrollableHitSource { } @Override + public XContentType getXContentType() { + return XContentFactory.xContentType(source); + } + @Override public long getVersion() { return delegate.getVersion(); } diff --git a/core/src/main/java/org/elasticsearch/action/bulk/byscroll/ScrollableHitSource.java b/core/src/main/java/org/elasticsearch/action/bulk/byscroll/ScrollableHitSource.java index 8b8a350ffd..73aa653698 100644 --- a/core/src/main/java/org/elasticsearch/action/bulk/byscroll/ScrollableHitSource.java +++ b/core/src/main/java/org/elasticsearch/action/bulk/byscroll/ScrollableHitSource.java @@ -32,6 +32,7 @@ import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.search.SearchHit; import org.elasticsearch.threadpool.ThreadPool; @@ -80,7 +81,7 @@ public abstract class ScrollableHitSource implements Closeable { }); } protected abstract void doStartNextScroll(String scrollId, TimeValue extraKeepAlive, Consumer<? super Response> onResponse); - + @Override public final void close() { String scrollId = this.scrollId.get(); @@ -190,6 +191,10 @@ public abstract class ScrollableHitSource implements Closeable { */ @Nullable BytesReference getSource(); /** + * The content type of the hit source. Returns null if the source didn't come back from the search. + */ + @Nullable XContentType getXContentType(); + /** * The document id of the parent of the hit if there is a parent or null if there isn't. */ @Nullable String getParent(); @@ -209,6 +214,7 @@ public abstract class ScrollableHitSource implements Closeable { private final long version; private BytesReference source; + private XContentType xContentType; private String parent; private String routing; @@ -244,8 +250,14 @@ public abstract class ScrollableHitSource implements Closeable { return source; } - public BasicHit setSource(BytesReference source) { + @Override + public XContentType getXContentType() { + return xContentType; + } + + public BasicHit setSource(BytesReference source, XContentType xContentType) { this.source = source; + this.xContentType = xContentType; return this; } diff --git a/core/src/main/java/org/elasticsearch/action/index/IndexRequest.java b/core/src/main/java/org/elasticsearch/action/index/IndexRequest.java index d1a9927c67..7e3f14368d 100644 --- a/core/src/main/java/org/elasticsearch/action/index/IndexRequest.java +++ b/core/src/main/java/org/elasticsearch/action/index/IndexRequest.java @@ -47,6 +47,7 @@ import java.io.IOException; import java.nio.charset.StandardCharsets; import java.util.Locale; import java.util.Map; +import java.util.Objects; import static org.elasticsearch.action.ValidateActions.addValidationError; @@ -55,10 +56,10 @@ import static org.elasticsearch.action.ValidateActions.addValidationError; * created using {@link org.elasticsearch.client.Requests#indexRequest(String)}. * * The index requires the {@link #index()}, {@link #type(String)}, {@link #id(String)} and - * {@link #source(byte[])} to be set. + * {@link #source(byte[], XContentType)} to be set. * - * The source (content to index) can be set in its bytes form using ({@link #source(byte[])}), - * its string form ({@link #source(String)}) or using a {@link org.elasticsearch.common.xcontent.XContentBuilder} + * The source (content to index) can be set in its bytes form using ({@link #source(byte[], XContentType)}), + * its string form ({@link #source(String, XContentType)}) or using a {@link org.elasticsearch.common.xcontent.XContentBuilder} * ({@link #source(org.elasticsearch.common.xcontent.XContentBuilder)}). * * If the {@link #id(String)} is not set, it will be automatically generated. @@ -83,7 +84,7 @@ public class IndexRequest extends ReplicatedWriteRequest<IndexRequest> implement private long version = Versions.MATCH_ANY; private VersionType versionType = VersionType.INTERNAL; - private XContentType contentType = Requests.INDEX_CONTENT_TYPE; + private XContentType contentType; private String pipeline; @@ -103,7 +104,7 @@ public class IndexRequest extends ReplicatedWriteRequest<IndexRequest> implement /** * Constructs a new index request against the specific index. The {@link #type(String)} - * {@link #source(byte[])} must be set. + * {@link #source(byte[], XContentType)} must be set. */ public IndexRequest(String index) { this.index = index; @@ -140,7 +141,9 @@ public class IndexRequest extends ReplicatedWriteRequest<IndexRequest> implement if (source == null) { validationException = addValidationError("source is missing", validationException); } - + if (contentType == null) { + validationException = addValidationError("content type is missing", validationException); + } final long resolvedVersion = resolveVersionDefaults(); if (opType() == OpType.CREATE) { if (versionType != VersionType.INTERNAL) { @@ -179,21 +182,14 @@ public class IndexRequest extends ReplicatedWriteRequest<IndexRequest> implement } /** - * The content type that will be used when generating a document from user provided objects like Maps. + * The content type. This will be used when generating a document from user provided objects like Maps and when parsing the + * source at index time */ public XContentType getContentType() { return contentType; } /** - * Sets the content type that will be used when generating a document from user provided objects (like Map). - */ - public IndexRequest contentType(XContentType contentType) { - this.contentType = contentType; - return this; - } - - /** * The type of the indexed document. */ @Override @@ -284,16 +280,16 @@ public class IndexRequest extends ReplicatedWriteRequest<IndexRequest> implement } public Map<String, Object> sourceAsMap() { - return XContentHelper.convertToMap(source, false).v2(); + return XContentHelper.convertToMap(source, false, contentType).v2(); } /** - * Index the Map as a {@link org.elasticsearch.client.Requests#INDEX_CONTENT_TYPE}. + * Index the Map in {@link Requests#INDEX_CONTENT_TYPE} format * * @param source The map to index */ public IndexRequest source(Map source) throws ElasticsearchGenerationException { - return source(source, contentType); + return source(source, Requests.INDEX_CONTENT_TYPE); } /** @@ -314,24 +310,32 @@ public class IndexRequest extends ReplicatedWriteRequest<IndexRequest> implement /** * Sets the document source to index. * - * Note, its preferable to either set it using {@link #source(org.elasticsearch.common.xcontent.XContentBuilder)} - * or using the {@link #source(byte[])}. + * @deprecated use {@link #source(String, XContentType)} */ + @Deprecated public IndexRequest source(String source) { - this.source = new BytesArray(source.getBytes(StandardCharsets.UTF_8)); - return this; + return source(new BytesArray(source), XContentFactory.xContentType(source)); + } + + /** + * Sets the document source to index. + * + * Note, its preferable to either set it using {@link #source(org.elasticsearch.common.xcontent.XContentBuilder)} + * or using the {@link #source(byte[], XContentType)}. + */ + public IndexRequest source(String source, XContentType xContentType) { + return source(new BytesArray(source), xContentType); } /** * Sets the content source to index. */ public IndexRequest source(XContentBuilder sourceBuilder) { - source = sourceBuilder.bytes(); - return this; + return source(sourceBuilder.bytes(), sourceBuilder.contentType()); } /** - * Sets the content source to index. + * Sets the content source to index using the default content type ({@link Requests#INDEX_CONTENT_TYPE}) * <p> * <b>Note: the number of objects passed to this method must be an even * number. Also the first argument in each pair (the field name) must have a @@ -339,6 +343,18 @@ public class IndexRequest extends ReplicatedWriteRequest<IndexRequest> implement * </p> */ public IndexRequest source(Object... source) { + return source(Requests.INDEX_CONTENT_TYPE, source); + } + + /** + * Sets the content source to index. + * <p> + * <b>Note: the number of objects passed to this method as varargs must be an even + * number. Also the first argument in each pair (the field name) must have a + * valid String representation.</b> + * </p> + */ + public IndexRequest source(XContentType xContentType, Object... source) { if (source.length % 2 != 0) { throw new IllegalArgumentException("The number of object passed must be even but was [" + source.length + "]"); } @@ -346,7 +362,7 @@ public class IndexRequest extends ReplicatedWriteRequest<IndexRequest> implement throw new IllegalArgumentException("you are using the removed method for source with bytes and unsafe flag, the unsafe flag was removed, please just use source(BytesReference)"); } try { - XContentBuilder builder = XContentFactory.contentBuilder(contentType); + XContentBuilder builder = XContentFactory.contentBuilder(xContentType); builder.startObject(); for (int i = 0; i < source.length; i++) { builder.field(source[i++].toString(), source[i]); @@ -360,30 +376,63 @@ public class IndexRequest extends ReplicatedWriteRequest<IndexRequest> implement /** * Sets the document to index in bytes form. + * @deprecated use {@link #source(BytesReference, XContentType)} */ + @Deprecated public IndexRequest source(BytesReference source) { - this.source = source; + return source(source, XContentFactory.xContentType(source)); + + } + + /** + * Sets the document to index in bytes form. + */ + public IndexRequest source(BytesReference source, XContentType xContentType) { + this.source = Objects.requireNonNull(source); + this.contentType = Objects.requireNonNull(xContentType); return this; } /** * Sets the document to index in bytes form. + * @deprecated use {@link #source(byte[], XContentType)} */ + @Deprecated public IndexRequest source(byte[] source) { return source(source, 0, source.length); } /** + * Sets the document to index in bytes form. + */ + public IndexRequest source(byte[] source, XContentType xContentType) { + return source(source, 0, source.length, xContentType); + } + + /** * Sets the document to index in bytes form (assumed to be safe to be used from different * threads). * * @param source The source to index * @param offset The offset in the byte array * @param length The length of the data + * @deprecated use {@link #source(byte[], int, int, XContentType)} */ + @Deprecated public IndexRequest source(byte[] source, int offset, int length) { - this.source = new BytesArray(source, offset, length); - return this; + return source(new BytesArray(source, offset, length), XContentFactory.xContentType(source)); + } + + /** + * Sets the document to index in bytes form (assumed to be safe to be used from different + * threads). + * + * @param source The source to index + * @param offset The offset in the byte array + * @param length The length of the data + */ + public IndexRequest source(byte[] source, int offset, int length, XContentType xContentType) { + return source(new BytesArray(source, offset, length), xContentType); } /** @@ -515,6 +564,11 @@ public class IndexRequest extends ReplicatedWriteRequest<IndexRequest> implement pipeline = in.readOptionalString(); isRetry = in.readBoolean(); autoGeneratedTimestamp = in.readLong(); + if (in.getVersion().after(Version.V_5_3_0_UNRELEASED)) { // TODO update to onOrAfter after backporting + contentType = in.readOptionalWriteable(XContentType::readFrom); + } else { + contentType = XContentFactory.xContentType(source); + } } @Override @@ -543,6 +597,9 @@ public class IndexRequest extends ReplicatedWriteRequest<IndexRequest> implement out.writeOptionalString(pipeline); out.writeBoolean(isRetry); out.writeLong(autoGeneratedTimestamp); + if (out.getVersion().after(Version.V_5_3_0_UNRELEASED)) { // TODO update to onOrAfter after backporting + out.writeOptionalWriteable(contentType); + } } @Override diff --git a/core/src/main/java/org/elasticsearch/action/index/IndexRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/index/IndexRequestBuilder.java index f7df8bffce..7af43ec35e 100644 --- a/core/src/main/java/org/elasticsearch/action/index/IndexRequestBuilder.java +++ b/core/src/main/java/org/elasticsearch/action/index/IndexRequestBuilder.java @@ -82,13 +82,23 @@ public class IndexRequestBuilder extends ReplicationRequestBuilder<IndexRequest, /** * Sets the source. + * @deprecated use {@link #setSource(BytesReference, XContentType)} */ + @Deprecated public IndexRequestBuilder setSource(BytesReference source) { request.source(source); return this; } /** + * Sets the source. + */ + public IndexRequestBuilder setSource(BytesReference source, XContentType xContentType) { + request.source(source, xContentType); + return this; + } + + /** * Index the Map as a JSON. * * @param source The map to index @@ -112,14 +122,27 @@ public class IndexRequestBuilder extends ReplicationRequestBuilder<IndexRequest, * Sets the document source to index. * <p> * Note, its preferable to either set it using {@link #setSource(org.elasticsearch.common.xcontent.XContentBuilder)} - * or using the {@link #setSource(byte[])}. + * or using the {@link #setSource(byte[], XContentType)}. + * @deprecated use {@link #setSource(String, XContentType)} */ + @Deprecated public IndexRequestBuilder setSource(String source) { request.source(source); return this; } /** + * Sets the document source to index. + * <p> + * Note, its preferable to either set it using {@link #setSource(org.elasticsearch.common.xcontent.XContentBuilder)} + * or using the {@link #setSource(byte[], XContentType)}. + */ + public IndexRequestBuilder setSource(String source, XContentType xContentType) { + request.source(source, xContentType); + return this; + } + + /** * Sets the content source to index. */ public IndexRequestBuilder setSource(XContentBuilder sourceBuilder) { @@ -129,26 +152,52 @@ public class IndexRequestBuilder extends ReplicationRequestBuilder<IndexRequest, /** * Sets the document to index in bytes form. + * @deprecated use {@link #setSource(byte[], XContentType)} */ + @Deprecated public IndexRequestBuilder setSource(byte[] source) { request.source(source); return this; } /** + * Sets the document to index in bytes form. + */ + public IndexRequestBuilder setSource(byte[] source, XContentType xContentType) { + request.source(source, xContentType); + return this; + } + + /** * Sets the document to index in bytes form (assumed to be safe to be used from different * threads). * * @param source The source to index * @param offset The offset in the byte array * @param length The length of the data + * @deprecated use {@link #setSource(byte[], int, int, XContentType)} */ + @Deprecated public IndexRequestBuilder setSource(byte[] source, int offset, int length) { request.source(source, offset, length); return this; } /** + * Sets the document to index in bytes form (assumed to be safe to be used from different + * threads). + * + * @param source The source to index + * @param offset The offset in the byte array + * @param length The length of the data + * @param xContentType The type/format of the source + */ + public IndexRequestBuilder setSource(byte[] source, int offset, int length, XContentType xContentType) { + request.source(source, offset, length, xContentType); + return this; + } + + /** * Constructs a simple document with a field name and value pairs. * <p> * <b>Note: the number of objects passed to this method must be an even @@ -162,10 +211,15 @@ public class IndexRequestBuilder extends ReplicationRequestBuilder<IndexRequest, } /** - * The content type that will be used to generate a document from user provided objects (like Map). + * Constructs a simple document with a field name and value pairs. + * <p> + * <b>Note: the number of objects passed as varargs to this method must be an even + * number. Also the first argument in each pair (the field name) must have a + * valid String representation.</b> + * </p> */ - public IndexRequestBuilder setContentType(XContentType contentType) { - request.contentType(contentType); + public IndexRequestBuilder setSource(XContentType xContentType, Object... source) { + request.source(xContentType, source); return this; } diff --git a/core/src/main/java/org/elasticsearch/action/ingest/PutPipelineRequest.java b/core/src/main/java/org/elasticsearch/action/ingest/PutPipelineRequest.java index 10416146ba..a28950b24c 100644 --- a/core/src/main/java/org/elasticsearch/action/ingest/PutPipelineRequest.java +++ b/core/src/main/java/org/elasticsearch/action/ingest/PutPipelineRequest.java @@ -19,32 +19,40 @@ package org.elasticsearch.action.ingest; +import org.elasticsearch.Version; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.support.master.AcknowledgedRequest; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentType; import java.io.IOException; import java.util.Objects; -import static org.elasticsearch.action.ValidateActions.addValidationError; - public class PutPipelineRequest extends AcknowledgedRequest<PutPipelineRequest> { private String id; private BytesReference source; + private XContentType xContentType; + /** + * Create a new pipeline request + * @deprecated use {@link #PutPipelineRequest(String, BytesReference, XContentType)} to avoid content type auto-detection + */ + @Deprecated public PutPipelineRequest(String id, BytesReference source) { - if (id == null) { - throw new IllegalArgumentException("id is missing"); - } - if (source == null) { - throw new IllegalArgumentException("source is missing"); - } + this(id, source, XContentFactory.xContentType(source)); + } - this.id = id; - this.source = source; + /** + * Create a new pipeline request with the id and source along with the content type of the source + */ + public PutPipelineRequest(String id, BytesReference source, XContentType xContentType) { + this.id = Objects.requireNonNull(id); + this.source = Objects.requireNonNull(source); + this.xContentType = Objects.requireNonNull(xContentType); } PutPipelineRequest() { @@ -63,11 +71,20 @@ public class PutPipelineRequest extends AcknowledgedRequest<PutPipelineRequest> return source; } + public XContentType getXContentType() { + return xContentType; + } + @Override public void readFrom(StreamInput in) throws IOException { super.readFrom(in); id = in.readString(); source = in.readBytesReference(); + if (in.getVersion().after(Version.V_5_3_0_UNRELEASED)) { // TODO update to onOrAfter after backporting + xContentType = XContentType.readFrom(in); + } else { + xContentType = XContentFactory.xContentType(source); + } } @Override @@ -75,5 +92,8 @@ public class PutPipelineRequest extends AcknowledgedRequest<PutPipelineRequest> super.writeTo(out); out.writeString(id); out.writeBytesReference(source); + if (out.getVersion().after(Version.V_5_3_0_UNRELEASED)) { // TODO update to onOrAfter after backporting + xContentType.writeTo(out); + } } } diff --git a/core/src/main/java/org/elasticsearch/action/ingest/PutPipelineRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/ingest/PutPipelineRequestBuilder.java index bd927115fb..c03b3b84f8 100644 --- a/core/src/main/java/org/elasticsearch/action/ingest/PutPipelineRequestBuilder.java +++ b/core/src/main/java/org/elasticsearch/action/ingest/PutPipelineRequestBuilder.java @@ -22,6 +22,7 @@ package org.elasticsearch.action.ingest; import org.elasticsearch.action.ActionRequestBuilder; import org.elasticsearch.client.ElasticsearchClient; import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.xcontent.XContentType; public class PutPipelineRequestBuilder extends ActionRequestBuilder<PutPipelineRequest, WritePipelineResponse, PutPipelineRequestBuilder> { @@ -29,8 +30,13 @@ public class PutPipelineRequestBuilder extends ActionRequestBuilder<PutPipelineR super(client, action, new PutPipelineRequest()); } + @Deprecated public PutPipelineRequestBuilder(ElasticsearchClient client, PutPipelineAction action, String id, BytesReference source) { super(client, action, new PutPipelineRequest(id, source)); } + public PutPipelineRequestBuilder(ElasticsearchClient client, PutPipelineAction action, String id, BytesReference source, + XContentType xContentType) { + super(client, action, new PutPipelineRequest(id, source, xContentType)); + } } diff --git a/core/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequest.java b/core/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequest.java index ef7b5e3d5b..362d2e7ab9 100644 --- a/core/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequest.java +++ b/core/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequest.java @@ -19,11 +19,14 @@ package org.elasticsearch.action.ingest; +import org.elasticsearch.Version; import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.ingest.ConfigurationUtils; import org.elasticsearch.ingest.IngestDocument; import org.elasticsearch.ingest.Pipeline; @@ -34,6 +37,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Objects; import static org.elasticsearch.ingest.IngestDocument.MetaData; @@ -42,12 +46,23 @@ public class SimulatePipelineRequest extends ActionRequest { private String id; private boolean verbose; private BytesReference source; + private XContentType xContentType; + /** + * Create a new request + * @deprecated use {@link #SimulatePipelineRequest(BytesReference, XContentType)} that does not attempt content autodetection + */ + @Deprecated public SimulatePipelineRequest(BytesReference source) { - if (source == null) { - throw new IllegalArgumentException("source is missing"); - } - this.source = source; + this(source, XContentFactory.xContentType(source)); + } + + /** + * Creates a new request with the given source and its content type + */ + public SimulatePipelineRequest(BytesReference source, XContentType xContentType) { + this.source = Objects.requireNonNull(source); + this.xContentType = Objects.requireNonNull(xContentType); } SimulatePipelineRequest() { @@ -78,12 +93,21 @@ public class SimulatePipelineRequest extends ActionRequest { return source; } + public XContentType getXContentType() { + return xContentType; + } + @Override public void readFrom(StreamInput in) throws IOException { super.readFrom(in); id = in.readOptionalString(); verbose = in.readBoolean(); source = in.readBytesReference(); + if (in.getVersion().after(Version.V_5_3_0_UNRELEASED)) { // TODO update to onOrAfter after backporting + xContentType = XContentType.readFrom(in); + } else { + xContentType = XContentFactory.xContentType(source); + } } @Override @@ -92,6 +116,9 @@ public class SimulatePipelineRequest extends ActionRequest { out.writeOptionalString(id); out.writeBoolean(verbose); out.writeBytesReference(source); + if (out.getVersion().after(Version.V_5_3_0_UNRELEASED)) { // TODO update to onOrAfter after backporting + xContentType.writeTo(out); + } } public static final class Fields { diff --git a/core/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequestBuilder.java index 4a13fa111e..bb5d0e4e40 100644 --- a/core/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequestBuilder.java +++ b/core/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequestBuilder.java @@ -22,22 +22,46 @@ package org.elasticsearch.action.ingest; import org.elasticsearch.action.ActionRequestBuilder; import org.elasticsearch.client.ElasticsearchClient; import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.xcontent.XContentType; public class SimulatePipelineRequestBuilder extends ActionRequestBuilder<SimulatePipelineRequest, SimulatePipelineResponse, SimulatePipelineRequestBuilder> { + /** + * Create a new builder for {@link SimulatePipelineRequest}s + */ public SimulatePipelineRequestBuilder(ElasticsearchClient client, SimulatePipelineAction action) { super(client, action, new SimulatePipelineRequest()); } + /** + * Create a new builder for {@link SimulatePipelineRequest}s + * @deprecated use {@link #SimulatePipelineRequestBuilder(ElasticsearchClient, SimulatePipelineAction, BytesReference, XContentType)} to + * avoid content type auto-detection on the source bytes + */ + @Deprecated public SimulatePipelineRequestBuilder(ElasticsearchClient client, SimulatePipelineAction action, BytesReference source) { super(client, action, new SimulatePipelineRequest(source)); } + /** + * Create a new builder for {@link SimulatePipelineRequest}s + */ + public SimulatePipelineRequestBuilder(ElasticsearchClient client, SimulatePipelineAction action, BytesReference source, + XContentType xContentType) { + super(client, action, new SimulatePipelineRequest(source, xContentType)); + } + + /** + * Set the id for the pipeline to simulate + */ public SimulatePipelineRequestBuilder setId(String id) { request.setId(id); return this; } + /** + * Enable or disable verbose mode + */ public SimulatePipelineRequestBuilder setVerbose(boolean verbose) { request.setVerbose(verbose); return this; diff --git a/core/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineTransportAction.java b/core/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineTransportAction.java index 61fd400a1d..3f67007df6 100644 --- a/core/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineTransportAction.java +++ b/core/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineTransportAction.java @@ -47,7 +47,7 @@ public class SimulatePipelineTransportAction extends HandledTransportAction<Simu @Override protected void doExecute(SimulatePipelineRequest request, ActionListener<SimulatePipelineResponse> listener) { - final Map<String, Object> source = XContentHelper.convertToMap(request.getSource(), false).v2(); + final Map<String, Object> source = XContentHelper.convertToMap(request.getSource(), false, request.getXContentType()).v2(); final SimulatePipelineRequest.Parsed simulateRequest; try { diff --git a/core/src/main/java/org/elasticsearch/action/termvectors/TermVectorsRequest.java b/core/src/main/java/org/elasticsearch/action/termvectors/TermVectorsRequest.java index b83713e3a6..544b14523b 100644 --- a/core/src/main/java/org/elasticsearch/action/termvectors/TermVectorsRequest.java +++ b/core/src/main/java/org/elasticsearch/action/termvectors/TermVectorsRequest.java @@ -20,6 +20,7 @@ package org.elasticsearch.action.termvectors; import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.Version; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.RealtimeRequest; import org.elasticsearch.action.ValidateActions; @@ -33,7 +34,9 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.lucene.uid.Versions; import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.VersionType; import java.io.IOException; @@ -63,6 +66,8 @@ public class TermVectorsRequest extends SingleShardRequest<TermVectorsRequest> i private BytesReference doc; + private XContentType xContentType; + private String routing; private String parent; @@ -156,8 +161,9 @@ public class TermVectorsRequest extends SingleShardRequest<TermVectorsRequest> i super(other.index()); this.id = other.id(); this.type = other.type(); - if (this.doc != null) { + if (other.doc != null) { this.doc = new BytesArray(other.doc().toBytesRef(), true); + this.xContentType = other.xContentType; } this.flagsEnum = other.getFlags().clone(); this.preference = other.preference(); @@ -225,22 +231,36 @@ public class TermVectorsRequest extends SingleShardRequest<TermVectorsRequest> i return doc; } + public XContentType xContentType() { + return xContentType; + } + /** * Sets an artificial document from which term vectors are requested for. */ public TermVectorsRequest doc(XContentBuilder documentBuilder) { - return this.doc(documentBuilder.bytes(), true); + return this.doc(documentBuilder.bytes(), true, documentBuilder.contentType()); } /** * Sets an artificial document from which term vectors are requested for. + * @deprecated use {@link #doc(BytesReference, boolean, XContentType)} to avoid content auto detection */ + @Deprecated public TermVectorsRequest doc(BytesReference doc, boolean generateRandomId) { + return this.doc(doc, generateRandomId, XContentFactory.xContentType(doc)); + } + + /** + * Sets an artificial document from which term vectors are requested for. + */ + public TermVectorsRequest doc(BytesReference doc, boolean generateRandomId, XContentType xContentType) { // assign a random id to this artificial document, for routing if (generateRandomId) { this.id(String.valueOf(randomInt.getAndAdd(1))); } this.doc = doc; + this.xContentType = xContentType; return this; } @@ -479,6 +499,11 @@ public class TermVectorsRequest extends SingleShardRequest<TermVectorsRequest> i if (in.readBoolean()) { doc = in.readBytesReference(); + if (in.getVersion().after(Version.V_5_3_0_UNRELEASED)) { // TODO update to onOrAfter after backporting + xContentType = XContentType.readFrom(in); + } else { + xContentType = XContentFactory.xContentType(doc); + } } routing = in.readOptionalString(); parent = in.readOptionalString(); @@ -519,6 +544,9 @@ public class TermVectorsRequest extends SingleShardRequest<TermVectorsRequest> i out.writeBoolean(doc != null); if (doc != null) { out.writeBytesReference(doc); + if (out.getVersion().after(Version.V_5_3_0_UNRELEASED)) { // TODO update to onOrAfter after backporting + xContentType.writeTo(out); + } } out.writeOptionalString(routing); out.writeOptionalString(parent); diff --git a/core/src/main/java/org/elasticsearch/action/update/TransportUpdateAction.java b/core/src/main/java/org/elasticsearch/action/update/TransportUpdateAction.java index 45e4aa8afd..4e2a7b466b 100644 --- a/core/src/main/java/org/elasticsearch/action/update/TransportUpdateAction.java +++ b/core/src/main/java/org/elasticsearch/action/update/TransportUpdateAction.java @@ -186,7 +186,8 @@ public class TransportUpdateAction extends TransportInstanceSingleOperationActio UpdateResponse update = new UpdateResponse(response.getShardInfo(), response.getShardId(), response.getType(), response.getId(), response.getSeqNo(), response.getVersion(), response.getResult()); if ((request.fetchSource() != null && request.fetchSource().fetchSource()) || (request.fields() != null && request.fields().length > 0)) { - Tuple<XContentType, Map<String, Object>> sourceAndContent = XContentHelper.convertToMap(upsertSourceBytes, true); + Tuple<XContentType, Map<String, Object>> sourceAndContent = + XContentHelper.convertToMap(upsertSourceBytes, true, upsertRequest.getContentType()); update.setGetResult(updateHelper.extractGetResult(request, request.concreteIndex(), response.getVersion(), sourceAndContent.v2(), sourceAndContent.v1(), upsertSourceBytes)); } else { update.setGetResult(null); diff --git a/core/src/main/java/org/elasticsearch/action/update/UpdateRequest.java b/core/src/main/java/org/elasticsearch/action/update/UpdateRequest.java index 0d2801e44a..504a297627 100644 --- a/core/src/main/java/org/elasticsearch/action/update/UpdateRequest.java +++ b/core/src/main/java/org/elasticsearch/action/update/UpdateRequest.java @@ -29,8 +29,6 @@ import org.elasticsearch.action.support.single.instance.InstanceShardOperationRe import org.elasticsearch.common.Nullable; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.logging.DeprecationLogger; -import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.common.lucene.uid.Versions; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; @@ -52,8 +50,6 @@ import static org.elasticsearch.action.ValidateActions.addValidationError; public class UpdateRequest extends InstanceShardOperationRequest<UpdateRequest> implements DocWriteRequest<UpdateRequest>, WriteRequest<UpdateRequest> { - private static final DeprecationLogger DEPRECATION_LOGGER = - new DeprecationLogger(Loggers.getLogger(UpdateRequest.class)); private String type; private String id; @@ -559,7 +555,9 @@ public class UpdateRequest extends InstanceShardOperationRequest<UpdateRequest> /** * Sets the doc to use for updates when a script is not specified. + * @deprecated use {@link #doc(String, XContentType)} */ + @Deprecated public UpdateRequest doc(String source) { safeDoc().source(source); return this; @@ -568,6 +566,16 @@ public class UpdateRequest extends InstanceShardOperationRequest<UpdateRequest> /** * Sets the doc to use for updates when a script is not specified. */ + public UpdateRequest doc(String source, XContentType xContentType) { + safeDoc().source(source, xContentType); + return this; + } + + /** + * Sets the doc to use for updates when a script is not specified. + * @deprecated use {@link #doc(byte[], XContentType)} + */ + @Deprecated public UpdateRequest doc(byte[] source) { safeDoc().source(source); return this; @@ -576,12 +584,30 @@ public class UpdateRequest extends InstanceShardOperationRequest<UpdateRequest> /** * Sets the doc to use for updates when a script is not specified. */ + public UpdateRequest doc(byte[] source, XContentType xContentType) { + safeDoc().source(source, xContentType); + return this; + } + + /** + * Sets the doc to use for updates when a script is not specified. + * @deprecated use {@link #doc(byte[], int, int, XContentType)} + */ + @Deprecated public UpdateRequest doc(byte[] source, int offset, int length) { safeDoc().source(source, offset, length); return this; } /** + * Sets the doc to use for updates when a script is not specified. + */ + public UpdateRequest doc(byte[] source, int offset, int length, XContentType xContentType) { + safeDoc().source(source, offset, length, xContentType); + return this; + } + + /** * Sets the doc to use for updates when a script is not specified, the doc provided * is a field and value pairs. */ @@ -591,10 +617,11 @@ public class UpdateRequest extends InstanceShardOperationRequest<UpdateRequest> } /** - * Sets the doc to use for updates when a script is not specified. + * Sets the doc to use for updates when a script is not specified, the doc provided + * is a field and value pairs. */ - public UpdateRequest doc(String field, Object value) { - safeDoc().source(field, value); + public UpdateRequest doc(XContentType xContentType, Object... source) { + safeDoc().source(xContentType, source); return this; } @@ -644,7 +671,9 @@ public class UpdateRequest extends InstanceShardOperationRequest<UpdateRequest> /** * Sets the doc source of the update request to be used when the document does not exists. + * @deprecated use {@link #upsert(String, XContentType)} */ + @Deprecated public UpdateRequest upsert(String source) { safeUpsertRequest().source(source); return this; @@ -653,6 +682,16 @@ public class UpdateRequest extends InstanceShardOperationRequest<UpdateRequest> /** * Sets the doc source of the update request to be used when the document does not exists. */ + public UpdateRequest upsert(String source, XContentType xContentType) { + safeUpsertRequest().source(source, xContentType); + return this; + } + + /** + * Sets the doc source of the update request to be used when the document does not exists. + * @deprecated use {@link #upsert(byte[], XContentType)} + */ + @Deprecated public UpdateRequest upsert(byte[] source) { safeUpsertRequest().source(source); return this; @@ -661,12 +700,30 @@ public class UpdateRequest extends InstanceShardOperationRequest<UpdateRequest> /** * Sets the doc source of the update request to be used when the document does not exists. */ + public UpdateRequest upsert(byte[] source, XContentType xContentType) { + safeUpsertRequest().source(source, xContentType); + return this; + } + + /** + * Sets the doc source of the update request to be used when the document does not exists. + * @deprecated use {@link #upsert(byte[], int, int, XContentType)} + */ + @Deprecated public UpdateRequest upsert(byte[] source, int offset, int length) { safeUpsertRequest().source(source, offset, length); return this; } /** + * Sets the doc source of the update request to be used when the document does not exists. + */ + public UpdateRequest upsert(byte[] source, int offset, int length, XContentType xContentType) { + safeUpsertRequest().source(source, offset, length, xContentType); + return this; + } + + /** * Sets the doc source of the update request to be used when the document does not exists. The doc * includes field and value pairs. */ @@ -675,6 +732,15 @@ public class UpdateRequest extends InstanceShardOperationRequest<UpdateRequest> return this; } + /** + * Sets the doc source of the update request to be used when the document does not exists. The doc + * includes field and value pairs. + */ + public UpdateRequest upsert(XContentType xContentType, Object... source) { + safeUpsertRequest().source(xContentType, source); + return this; + } + public IndexRequest upsertRequest() { return this.upsertRequest; } diff --git a/core/src/main/java/org/elasticsearch/action/update/UpdateRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/update/UpdateRequestBuilder.java index 50d84a2412..1a4d4077b1 100644 --- a/core/src/main/java/org/elasticsearch/action/update/UpdateRequestBuilder.java +++ b/core/src/main/java/org/elasticsearch/action/update/UpdateRequestBuilder.java @@ -223,7 +223,9 @@ public class UpdateRequestBuilder extends InstanceShardOperationRequestBuilder<U /** * Sets the doc to use for updates when a script is not specified. + * @deprecated use {@link #setDoc(String, XContentType)} */ + @Deprecated public UpdateRequestBuilder setDoc(String source) { request.doc(source); return this; @@ -232,6 +234,16 @@ public class UpdateRequestBuilder extends InstanceShardOperationRequestBuilder<U /** * Sets the doc to use for updates when a script is not specified. */ + public UpdateRequestBuilder setDoc(String source, XContentType xContentType) { + request.doc(source, xContentType); + return this; + } + + /** + * Sets the doc to use for updates when a script is not specified. + * @deprecated use {@link #setDoc(byte[], XContentType)} + */ + @Deprecated public UpdateRequestBuilder setDoc(byte[] source) { request.doc(source); return this; @@ -240,6 +252,16 @@ public class UpdateRequestBuilder extends InstanceShardOperationRequestBuilder<U /** * Sets the doc to use for updates when a script is not specified. */ + public UpdateRequestBuilder setDoc(byte[] source, XContentType xContentType) { + request.doc(source, xContentType); + return this; + } + + /** + * Sets the doc to use for updates when a script is not specified. + * @deprecated use {@link #setDoc(byte[], int, int, XContentType)} + */ + @Deprecated public UpdateRequestBuilder setDoc(byte[] source, int offset, int length) { request.doc(source, offset, length); return this; @@ -248,8 +270,8 @@ public class UpdateRequestBuilder extends InstanceShardOperationRequestBuilder<U /** * Sets the doc to use for updates when a script is not specified. */ - public UpdateRequestBuilder setDoc(String field, Object value) { - request.doc(field, value); + public UpdateRequestBuilder setDoc(byte[] source, int offset, int length, XContentType xContentType) { + request.doc(source, offset, length, xContentType); return this; } @@ -263,6 +285,15 @@ public class UpdateRequestBuilder extends InstanceShardOperationRequestBuilder<U } /** + * Sets the doc to use for updates when a script is not specified, the doc provided + * is a field and value pairs. + */ + public UpdateRequestBuilder setDoc(XContentType xContentType, Object... source) { + request.doc(xContentType, source); + return this; + } + + /** * Sets the index request to be used if the document does not exists. Otherwise, a {@link org.elasticsearch.index.engine.DocumentMissingException} * is thrown. */ @@ -297,7 +328,9 @@ public class UpdateRequestBuilder extends InstanceShardOperationRequestBuilder<U /** * Sets the doc source of the update request to be used when the document does not exists. + * @deprecated use {@link #setUpsert(String, XContentType)} */ + @Deprecated public UpdateRequestBuilder setUpsert(String source) { request.upsert(source); return this; @@ -306,6 +339,16 @@ public class UpdateRequestBuilder extends InstanceShardOperationRequestBuilder<U /** * Sets the doc source of the update request to be used when the document does not exists. */ + public UpdateRequestBuilder setUpsert(String source, XContentType xContentType) { + request.upsert(source, xContentType); + return this; + } + + /** + * Sets the doc source of the update request to be used when the document does not exists. + * @deprecated use {@link #setDoc(byte[], XContentType)} + */ + @Deprecated public UpdateRequestBuilder setUpsert(byte[] source) { request.upsert(source); return this; @@ -314,12 +357,30 @@ public class UpdateRequestBuilder extends InstanceShardOperationRequestBuilder<U /** * Sets the doc source of the update request to be used when the document does not exists. */ + public UpdateRequestBuilder setUpsert(byte[] source, XContentType xContentType) { + request.upsert(source, xContentType); + return this; + } + + /** + * Sets the doc source of the update request to be used when the document does not exists. + * @deprecated use {@link #setUpsert(byte[], int, int, XContentType)} + */ + @Deprecated public UpdateRequestBuilder setUpsert(byte[] source, int offset, int length) { request.upsert(source, offset, length); return this; } /** + * Sets the doc source of the update request to be used when the document does not exists. + */ + public UpdateRequestBuilder setUpsert(byte[] source, int offset, int length, XContentType xContentType) { + request.upsert(source, offset, length, xContentType); + return this; + } + + /** * Sets the doc source of the update request to be used when the document does not exists. The doc * includes field and value pairs. */ @@ -329,6 +390,15 @@ public class UpdateRequestBuilder extends InstanceShardOperationRequestBuilder<U } /** + * Sets the doc source of the update request to be used when the document does not exists. The doc + * includes field and value pairs. + */ + public UpdateRequestBuilder setUpsert(XContentType xContentType, Object... source) { + request.upsert(xContentType, source); + return this; + } + + /** * Sets whether the specified doc parameter should be used as upsert document. */ public UpdateRequestBuilder setDocAsUpsert(boolean shouldUpsertDoc) { diff --git a/core/src/main/java/org/elasticsearch/client/ClusterAdminClient.java b/core/src/main/java/org/elasticsearch/client/ClusterAdminClient.java index 14abc77513..3f705a215e 100644 --- a/core/src/main/java/org/elasticsearch/client/ClusterAdminClient.java +++ b/core/src/main/java/org/elasticsearch/client/ClusterAdminClient.java @@ -112,6 +112,7 @@ import org.elasticsearch.action.ingest.SimulatePipelineResponse; import org.elasticsearch.action.ingest.WritePipelineResponse; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.tasks.TaskId; /** @@ -545,10 +546,17 @@ public interface ClusterAdminClient extends ElasticsearchClient { /** * Stores an ingest pipeline + * @deprecated use {@link #preparePutPipeline(String, BytesReference, XContentType)} */ + @Deprecated PutPipelineRequestBuilder preparePutPipeline(String id, BytesReference source); /** + * Stores an ingest pipeline + */ + PutPipelineRequestBuilder preparePutPipeline(String id, BytesReference source, XContentType xContentType); + + /** * Deletes a stored ingest pipeline */ void deletePipeline(DeletePipelineRequest request, ActionListener<WritePipelineResponse> listener); @@ -596,9 +604,15 @@ public interface ClusterAdminClient extends ElasticsearchClient { /** * Simulates an ingest pipeline */ + @Deprecated SimulatePipelineRequestBuilder prepareSimulatePipeline(BytesReference source); /** + * Simulates an ingest pipeline + */ + SimulatePipelineRequestBuilder prepareSimulatePipeline(BytesReference source, XContentType xContentType); + + /** * Explain the allocation of a shard */ void allocationExplain(ClusterAllocationExplainRequest request, ActionListener<ClusterAllocationExplainResponse> listener); diff --git a/core/src/main/java/org/elasticsearch/client/support/AbstractClient.java b/core/src/main/java/org/elasticsearch/client/support/AbstractClient.java index b93c00ca9f..5a789af3ee 100644 --- a/core/src/main/java/org/elasticsearch/client/support/AbstractClient.java +++ b/core/src/main/java/org/elasticsearch/client/support/AbstractClient.java @@ -343,6 +343,7 @@ import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.component.AbstractComponent; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.tasks.TaskId; import org.elasticsearch.threadpool.ThreadPool; @@ -1082,6 +1083,11 @@ public abstract class AbstractClient extends AbstractComponent implements Client } @Override + public PutPipelineRequestBuilder preparePutPipeline(String id, BytesReference source, XContentType xContentType) { + return new PutPipelineRequestBuilder(this, PutPipelineAction.INSTANCE, id, source, xContentType); + } + + @Override public void deletePipeline(DeletePipelineRequest request, ActionListener<WritePipelineResponse> listener) { execute(DeletePipelineAction.INSTANCE, request, listener); } @@ -1132,6 +1138,11 @@ public abstract class AbstractClient extends AbstractComponent implements Client } @Override + public SimulatePipelineRequestBuilder prepareSimulatePipeline(BytesReference source, XContentType xContentType) { + return new SimulatePipelineRequestBuilder(this, SimulatePipelineAction.INSTANCE, source, xContentType); + } + + @Override public void allocationExplain(ClusterAllocationExplainRequest request, ActionListener<ClusterAllocationExplainResponse> listener) { execute(ClusterAllocationExplainAction.INSTANCE, request, listener); } diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaData.java b/core/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaData.java index 4ba244e107..5bba34904d 100644 --- a/core/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaData.java +++ b/core/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaData.java @@ -26,6 +26,7 @@ import org.elasticsearch.cluster.AbstractDiffable; import org.elasticsearch.cluster.Diff; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.collect.MapBuilder; import org.elasticsearch.common.compress.CompressedXContent; diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java index 0efb640c5a..0bde4a23b0 100644 --- a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java +++ b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java @@ -265,11 +265,13 @@ public class MetaDataCreateIndexService extends AbstractComponent { for (IndexTemplateMetaData template : templates) { templateNames.add(template.getName()); for (ObjectObjectCursor<String, CompressedXContent> cursor : template.mappings()) { + String mappingString = cursor.value.string(); if (mappings.containsKey(cursor.key)) { XContentHelper.mergeDefaults(mappings.get(cursor.key), - MapperService.parseMapping(xContentRegistry, cursor.value.string())); + MapperService.parseMapping(xContentRegistry, mappingString)); } else { - mappings.put(cursor.key, MapperService.parseMapping(xContentRegistry, cursor.value.string())); + mappings.put(cursor.key, + MapperService.parseMapping(xContentRegistry, mappingString)); } } // handle custom diff --git a/core/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java b/core/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java index 279be58e01..69adc5878f 100644 --- a/core/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java +++ b/core/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java @@ -237,6 +237,7 @@ public final class ClusterSettings extends AbstractScopedSettings { HttpTransportSettings.SETTING_CORS_ALLOW_METHODS, HttpTransportSettings.SETTING_CORS_ALLOW_HEADERS, HttpTransportSettings.SETTING_HTTP_DETAILED_ERRORS_ENABLED, + HttpTransportSettings.SETTING_HTTP_CONTENT_TYPE_REQUIRED, HttpTransportSettings.SETTING_HTTP_MAX_CONTENT_LENGTH, HttpTransportSettings.SETTING_HTTP_MAX_CHUNK_SIZE, HttpTransportSettings.SETTING_HTTP_MAX_HEADER_SIZE, diff --git a/core/src/main/java/org/elasticsearch/common/settings/Settings.java b/core/src/main/java/org/elasticsearch/common/settings/Settings.java index 883f1e7c82..579be7ce31 100644 --- a/core/src/main/java/org/elasticsearch/common/settings/Settings.java +++ b/core/src/main/java/org/elasticsearch/common/settings/Settings.java @@ -37,6 +37,7 @@ import org.elasticsearch.common.unit.SizeValue; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentType; import java.io.IOException; import java.io.InputStream; @@ -960,7 +961,9 @@ public final class Settings implements ToXContent { /** * Loads settings from the actual string content that represents them using the * {@link SettingsLoaderFactory#loaderFromSource(String)}. + * @deprecated use {@link #loadFromSource(String, XContentType)} to avoid content type detection */ + @Deprecated public Builder loadFromSource(String source) { SettingsLoader settingsLoader = SettingsLoaderFactory.loaderFromSource(source); try { @@ -973,8 +976,23 @@ public final class Settings implements ToXContent { } /** + * Loads settings from the actual string content that represents them using the + * {@link SettingsLoaderFactory#loaderFromXContentType(XContentType)} method to obtain a loader + */ + public Builder loadFromSource(String source, XContentType xContentType) { + SettingsLoader settingsLoader = SettingsLoaderFactory.loaderFromXContentType(xContentType); + try { + Map<String, String> loadedSettings = settingsLoader.load(source); + put(loadedSettings); + } catch (Exception e) { + throw new SettingsException("Failed to load settings from [" + source + "]", e); + } + return this; + } + + /** * Loads settings from a url that represents them using the - * {@link SettingsLoaderFactory#loaderFromSource(String)}. + * {@link SettingsLoaderFactory#loaderFromResource(String)}. */ public Builder loadFromPath(Path path) throws IOException { // NOTE: loadFromStream will close the input stream @@ -983,7 +1001,7 @@ public final class Settings implements ToXContent { /** * Loads settings from a stream that represents them using the - * {@link SettingsLoaderFactory#loaderFromSource(String)}. + * {@link SettingsLoaderFactory#loaderFromResource(String)}. */ public Builder loadFromStream(String resourceName, InputStream is) throws IOException { SettingsLoader settingsLoader = SettingsLoaderFactory.loaderFromResource(resourceName); diff --git a/core/src/main/java/org/elasticsearch/common/settings/loader/SettingsLoaderFactory.java b/core/src/main/java/org/elasticsearch/common/settings/loader/SettingsLoaderFactory.java index 5f2da22c5f..5d8cb4918b 100644 --- a/core/src/main/java/org/elasticsearch/common/settings/loader/SettingsLoaderFactory.java +++ b/core/src/main/java/org/elasticsearch/common/settings/loader/SettingsLoaderFactory.java @@ -19,6 +19,8 @@ package org.elasticsearch.common.settings.loader; +import org.elasticsearch.common.xcontent.XContentType; + /** * A class holding factory methods for settings loaders that attempts * to infer the type of the underlying settings content. @@ -33,9 +35,7 @@ public final class SettingsLoaderFactory { * name. This factory method assumes that if the resource name ends * with ".json" then the content should be parsed as JSON, else if * the resource name ends with ".yml" or ".yaml" then the content - * should be parsed as YAML, else if the resource name ends with - * ".properties" then the content should be parsed as properties, - * otherwise default to attempting to parse as JSON. Note that the + * should be parsed as YAML, otherwise throws an exception. Note that the * parsers returned by this method will not accept null-valued * keys. * @@ -59,13 +59,15 @@ public final class SettingsLoaderFactory { * contains an opening and closing brace ('{' and '}') then the * content should be parsed as JSON, else if the underlying content * fails this condition but contains a ':' then the content should - * be parsed as YAML, and otherwise should be parsed as properties. + * be parsed as YAML, and otherwise throws an exception. * Note that the JSON and YAML parsers returned by this method will * accept null-valued keys. * * @param source The underlying settings content. * @return A settings loader. + * @deprecated use {@link #loaderFromXContentType(XContentType)} instead */ + @Deprecated public static SettingsLoader loaderFromSource(String source) { if (source.indexOf('{') != -1 && source.indexOf('}') != -1) { return new JsonSettingsLoader(true); @@ -76,4 +78,20 @@ public final class SettingsLoaderFactory { } } + /** + * Returns a {@link SettingsLoader} based on the {@link XContentType}. Note only {@link XContentType#JSON} and + * {@link XContentType#YAML} are supported + * + * @param xContentType The content type + * @return A settings loader. + */ + public static SettingsLoader loaderFromXContentType(XContentType xContentType) { + if (xContentType == XContentType.JSON) { + return new JsonSettingsLoader(true); + } else if (xContentType == XContentType.YAML) { + return new YamlSettingsLoader(true); + } else { + throw new IllegalArgumentException("unsupported content type [" + xContentType + "]"); + } + } } diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java b/core/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java index df34ec726f..189e9d3c8d 100644 --- a/core/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java +++ b/core/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java @@ -964,21 +964,60 @@ public final class XContentBuilder implements BytesStream, Releasable, Flushable // Raw fields ////////////////////////////////// + /** + * Writes a raw field with the value taken from the bytes in the stream + * @deprecated use {@link #rawField(String, InputStream, XContentType)} to avoid content type auto-detection + */ + @Deprecated public XContentBuilder rawField(String name, InputStream value) throws IOException { generator.writeRawField(name, value); return this; } + /** + * Writes a raw field with the value taken from the bytes in the stream + */ + public XContentBuilder rawField(String name, InputStream value, XContentType contentType) throws IOException { + generator.writeRawField(name, value, contentType); + return this; + } + + /** + * Writes a raw field with the given bytes as the value + * @deprecated use {@link #rawField(String name, BytesReference, XContentType)} to avoid content type auto-detection + */ + @Deprecated public XContentBuilder rawField(String name, BytesReference value) throws IOException { generator.writeRawField(name, value); return this; } + /** + * Writes a raw field with the given bytes as the value + */ + public XContentBuilder rawField(String name, BytesReference value, XContentType contentType) throws IOException { + generator.writeRawField(name, value, contentType); + return this; + } + + /** + * Writes a value with the source coming directly from the bytes + * @deprecated use {@link #rawValue(BytesReference, XContentType)} to avoid content type auto-detection + */ + @Deprecated public XContentBuilder rawValue(BytesReference value) throws IOException { generator.writeRawValue(value); return this; } + /** + * Writes a value with the source coming directly from the bytes + */ + public XContentBuilder rawValue(BytesReference value, XContentType contentType) throws IOException { + generator.writeRawValue(value, contentType); + return this; + } + public XContentBuilder copyCurrentStructure(XContentParser parser) throws IOException { generator.copyCurrentStructure(parser); return this; diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/XContentGenerator.java b/core/src/main/java/org/elasticsearch/common/xcontent/XContentGenerator.java index 478f3a8a08..60a188ca6c 100644 --- a/core/src/main/java/org/elasticsearch/common/xcontent/XContentGenerator.java +++ b/core/src/main/java/org/elasticsearch/common/xcontent/XContentGenerator.java @@ -86,12 +86,42 @@ public interface XContentGenerator extends Closeable, Flushable { void writeBinary(byte[] value, int offset, int length) throws IOException; + /** + * Writes a raw field with the value taken from the bytes in the stream + * @deprecated use {@link #writeRawField(String, InputStream, XContentType)} to avoid content type auto-detection + */ + @Deprecated void writeRawField(String name, InputStream value) throws IOException; + /** + * Writes a raw field with the value taken from the bytes in the stream + */ + void writeRawField(String name, InputStream value, XContentType xContentType) throws IOException; + + /** + * Writes a raw field with the given bytes as the value + * @deprecated use {@link #writeRawField(String, BytesReference, XContentType)} to avoid content type auto-detection + */ + @Deprecated void writeRawField(String name, BytesReference value) throws IOException; + /** + * Writes a raw field with the given bytes as the value + */ + void writeRawField(String name, BytesReference value, XContentType xContentType) throws IOException; + + /** + * Writes a value with the source coming directly from the bytes + * @deprecated use {@link #writeRawValue(BytesReference, XContentType)} to avoid content type auto-detection + */ + @Deprecated void writeRawValue(BytesReference value) throws IOException; + /** + * Writes a value with the source coming directly from the bytes + */ + void writeRawValue(BytesReference value, XContentType xContentType) throws IOException; + void copyCurrentStructure(XContentParser parser) throws IOException; /** diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java b/core/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java index 6cf8d12c21..dd7508280d 100644 --- a/core/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java +++ b/core/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java @@ -41,6 +41,11 @@ import static org.elasticsearch.common.xcontent.ToXContent.EMPTY_PARAMS; @SuppressWarnings("unchecked") public class XContentHelper { + /** + * Creates a parser based on the bytes provided + * @deprecated use {@link #createParser(NamedXContentRegistry, BytesReference, XContentType)} to avoid content type auto-detection + */ + @Deprecated public static XContentParser createParser(NamedXContentRegistry xContentRegistry, BytesReference bytes) throws IOException { Compressor compressor = CompressorFactory.compressor(bytes); if (compressor != null) { @@ -48,17 +53,49 @@ public class XContentHelper { if (compressedInput.markSupported() == false) { compressedInput = new BufferedInputStream(compressedInput); } - XContentType contentType = XContentFactory.xContentType(compressedInput); + final XContentType contentType = XContentFactory.xContentType(compressedInput); return XContentFactory.xContent(contentType).createParser(xContentRegistry, compressedInput); } else { return XContentFactory.xContent(bytes).createParser(xContentRegistry, bytes.streamInput()); } } + /** + * Creates a parser for the bytes using the supplied content-type + */ + public static XContentParser createParser(NamedXContentRegistry xContentRegistry, BytesReference bytes, + XContentType xContentType) throws IOException { + Objects.requireNonNull(xContentType); + Compressor compressor = CompressorFactory.compressor(bytes); + if (compressor != null) { + InputStream compressedInput = compressor.streamInput(bytes.streamInput()); + if (compressedInput.markSupported() == false) { + compressedInput = new BufferedInputStream(compressedInput); + } + return XContentFactory.xContent(xContentType).createParser(xContentRegistry, compressedInput); + } else { + return xContentType.xContent().createParser(xContentRegistry, bytes.streamInput()); + } + } + + /** + * Converts the given bytes into a map that is optionally ordered. + * @deprecated this method relies on auto-detection of content type. Use {@link #convertToMap(BytesReference, boolean, XContentType)} + * instead with the proper {@link XContentType} + */ + @Deprecated public static Tuple<XContentType, Map<String, Object>> convertToMap(BytesReference bytes, boolean ordered) throws ElasticsearchParseException { + return convertToMap(bytes, ordered, null); + } + + /** + * Converts the given bytes into a map that is optionally ordered. The provided {@link XContentType} must be non-null. + */ + public static Tuple<XContentType, Map<String, Object>> convertToMap(BytesReference bytes, boolean ordered, XContentType xContentType) + throws ElasticsearchParseException { try { - XContentType contentType; + final XContentType contentType; InputStream input; Compressor compressor = CompressorFactory.compressor(bytes); if (compressor != null) { @@ -66,13 +103,12 @@ public class XContentHelper { if (compressedStreamInput.markSupported() == false) { compressedStreamInput = new BufferedInputStream(compressedStreamInput); } - contentType = XContentFactory.xContentType(compressedStreamInput); input = compressedStreamInput; } else { - contentType = XContentFactory.xContentType(bytes); input = bytes.streamInput(); } - return new Tuple<>(contentType, convertToMap(XContentFactory.xContent(contentType), input, ordered)); + contentType = xContentType != null ? xContentType : XContentFactory.xContentType(input); + return new Tuple<>(Objects.requireNonNull(contentType), convertToMap(XContentFactory.xContent(contentType), input, ordered)); } catch (IOException e) { throw new ElasticsearchParseException("Failed to parse content to map", e); } @@ -105,15 +141,27 @@ public class XContentHelper { } } + @Deprecated public static String convertToJson(BytesReference bytes, boolean reformatJson) throws IOException { return convertToJson(bytes, reformatJson, false); } + @Deprecated public static String convertToJson(BytesReference bytes, boolean reformatJson, boolean prettyPrint) throws IOException { - XContentType xContentType = XContentFactory.xContentType(bytes); + return convertToJson(bytes, reformatJson, prettyPrint, XContentFactory.xContentType(bytes)); + } + + public static String convertToJson(BytesReference bytes, boolean reformatJson, XContentType xContentType) throws IOException { + return convertToJson(bytes, reformatJson, false, xContentType); + } + + public static String convertToJson(BytesReference bytes, boolean reformatJson, boolean prettyPrint, XContentType xContentType) + throws IOException { + Objects.requireNonNull(xContentType); if (xContentType == XContentType.JSON && !reformatJson) { return bytes.utf8ToString(); } + // It is safe to use EMPTY here because this never uses namedObject try (XContentParser parser = XContentFactory.xContent(xContentType).createParser(NamedXContentRegistry.EMPTY, bytes.streamInput())) { @@ -365,7 +413,10 @@ public class XContentHelper { /** * Writes a "raw" (bytes) field, handling cases where the bytes are compressed, and tries to optimize writing using * {@link XContentBuilder#rawField(String, org.elasticsearch.common.bytes.BytesReference)}. + * @deprecated use {@link #writeRawField(String, BytesReference, XContentType, XContentBuilder, Params)} to avoid content type + * auto-detection */ + @Deprecated public static void writeRawField(String field, BytesReference source, XContentBuilder builder, ToXContent.Params params) throws IOException { Compressor compressor = CompressorFactory.compressor(source); if (compressor != null) { @@ -377,6 +428,22 @@ public class XContentHelper { } /** + * Writes a "raw" (bytes) field, handling cases where the bytes are compressed, and tries to optimize writing using + * {@link XContentBuilder#rawField(String, org.elasticsearch.common.bytes.BytesReference, XContentType)}. + */ + public static void writeRawField(String field, BytesReference source, XContentType xContentType, XContentBuilder builder, + ToXContent.Params params) throws IOException { + Objects.requireNonNull(xContentType); + Compressor compressor = CompressorFactory.compressor(source); + if (compressor != null) { + InputStream compressedStreamInput = compressor.streamInput(source.streamInput()); + builder.rawField(field, compressedStreamInput, xContentType); + } else { + builder.rawField(field, source, xContentType); + } + } + + /** * Returns the bytes that represent the XContent output of the provided {@link ToXContent} object, using the provided * {@link XContentType}. Wraps the output into a new anonymous object. */ diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/XContentType.java b/core/src/main/java/org/elasticsearch/common/xcontent/XContentType.java index ddd736e0d0..8e3c298270 100644 --- a/core/src/main/java/org/elasticsearch/common/xcontent/XContentType.java +++ b/core/src/main/java/org/elasticsearch/common/xcontent/XContentType.java @@ -29,6 +29,7 @@ import org.elasticsearch.common.xcontent.yaml.YamlXContent; import java.io.IOException; import java.util.Locale; +import java.util.Objects; /** * The content type of {@link org.elasticsearch.common.xcontent.XContent}. @@ -40,7 +41,7 @@ public enum XContentType implements Writeable { */ JSON(0) { @Override - protected String mediaTypeWithoutParameters() { + public String mediaTypeWithoutParameters() { return "application/json"; } @@ -64,7 +65,7 @@ public enum XContentType implements Writeable { */ SMILE(1) { @Override - protected String mediaTypeWithoutParameters() { + public String mediaTypeWithoutParameters() { return "application/smile"; } @@ -83,7 +84,7 @@ public enum XContentType implements Writeable { */ YAML(2) { @Override - protected String mediaTypeWithoutParameters() { + public String mediaTypeWithoutParameters() { return "application/yaml"; } @@ -102,7 +103,7 @@ public enum XContentType implements Writeable { */ CBOR(3) { @Override - protected String mediaTypeWithoutParameters() { + public String mediaTypeWithoutParameters() { return "application/cbor"; } @@ -117,12 +118,18 @@ public enum XContentType implements Writeable { } }; + /** + * Accepts either a format string, which is equivalent to {@link XContentType#shortName()} or a media type that optionally has + * parameters and attempts to match the value to an {@link XContentType}. The comparisons are done in lower case format and this method + * also supports a wildcard accept for {@code application/*}. This method can be used to parse the {@code Accept} HTTP header or a + * format query string parameter. This method will return {@code null} if no match is found + */ public static XContentType fromMediaTypeOrFormat(String mediaType) { if (mediaType == null) { return null; } for (XContentType type : values()) { - if (isSameMediaTypeAs(mediaType, type)) { + if (isSameMediaTypeOrFormatAs(mediaType, type)) { return type; } } @@ -133,7 +140,22 @@ public enum XContentType implements Writeable { return null; } - private static boolean isSameMediaTypeAs(String stringType, XContentType type) { + /** + * Attempts to match the given media type with the known {@link XContentType} values. This match is done in a case-insensitive manner. + * The provided media type should not include any parameters. This method is suitable for parsing part of the {@code Content-Type} + * HTTP header. This method will return {@code null} if no match is found + */ + public static XContentType fromMediaType(String mediaType) { + final String lowercaseMediaType = Objects.requireNonNull(mediaType, "mediaType cannot be null").toLowerCase(Locale.ROOT); + for (XContentType type : values()) { + if (type.mediaTypeWithoutParameters().equals(lowercaseMediaType)) { + return type; + } + } + return null; + } + + private static boolean isSameMediaTypeOrFormatAs(String stringType, XContentType type) { return type.mediaTypeWithoutParameters().equalsIgnoreCase(stringType) || stringType.toLowerCase(Locale.ROOT).startsWith(type.mediaTypeWithoutParameters().toLowerCase(Locale.ROOT) + ";") || type.shortName().equalsIgnoreCase(stringType); @@ -157,7 +179,7 @@ public enum XContentType implements Writeable { public abstract XContent xContent(); - protected abstract String mediaTypeWithoutParameters(); + public abstract String mediaTypeWithoutParameters(); public static XContentType readFrom(StreamInput in) throws IOException { int index = in.readVInt(); diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContentGenerator.java b/core/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContentGenerator.java index 07ae16b96c..1e09f8334f 100644 --- a/core/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContentGenerator.java +++ b/core/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContentGenerator.java @@ -307,6 +307,11 @@ public class JsonXContentGenerator implements XContentGenerator { if (contentType == null) { throw new IllegalArgumentException("Can't write raw bytes whose xcontent-type can't be guessed"); } + writeRawField(name, content, contentType); + } + + @Override + public void writeRawField(String name, InputStream content, XContentType contentType) throws IOException { if (mayWriteRawData(contentType) == false) { // EMPTY is safe here because we never call namedObject when writing raw data try (XContentParser parser = XContentFactory.xContent(contentType).createParser(NamedXContentRegistry.EMPTY, content)) { @@ -328,6 +333,11 @@ public class JsonXContentGenerator implements XContentGenerator { if (contentType == null) { throw new IllegalArgumentException("Can't write raw bytes whose xcontent-type can't be guessed"); } + writeRawField(name, content, contentType); + } + + @Override + public final void writeRawField(String name, BytesReference content, XContentType contentType) throws IOException { if (mayWriteRawData(contentType) == false) { writeFieldName(name); copyRawValue(content, contentType.xContent()); @@ -345,6 +355,11 @@ public class JsonXContentGenerator implements XContentGenerator { if (contentType == null) { throw new IllegalArgumentException("Can't write raw bytes whose xcontent-type can't be guessed"); } + writeRawValue(content, contentType); + } + + @Override + public final void writeRawValue(BytesReference content, XContentType contentType) throws IOException { if (mayWriteRawData(contentType) == false) { copyRawValue(content, contentType.xContent()); } else { diff --git a/core/src/main/java/org/elasticsearch/http/HttpTransportSettings.java b/core/src/main/java/org/elasticsearch/http/HttpTransportSettings.java index 60bc3449d0..b5e254aa4c 100644 --- a/core/src/main/java/org/elasticsearch/http/HttpTransportSettings.java +++ b/core/src/main/java/org/elasticsearch/http/HttpTransportSettings.java @@ -68,6 +68,8 @@ public final class HttpTransportSettings { Setting.intSetting("http.publish_port", -1, -1, Property.NodeScope); public static final Setting<Boolean> SETTING_HTTP_DETAILED_ERRORS_ENABLED = Setting.boolSetting("http.detailed_errors.enabled", true, Property.NodeScope); + public static final Setting<Boolean> SETTING_HTTP_CONTENT_TYPE_REQUIRED = + Setting.boolSetting("http.content_type.required", false, Property.NodeScope); public static final Setting<ByteSizeValue> SETTING_HTTP_MAX_CONTENT_LENGTH = Setting.byteSizeSetting("http.max_content_length", new ByteSizeValue(100, ByteSizeUnit.MB), Property.NodeScope); public static final Setting<ByteSizeValue> SETTING_HTTP_MAX_CHUNK_SIZE = diff --git a/core/src/main/java/org/elasticsearch/index/IndexingSlowLog.java b/core/src/main/java/org/elasticsearch/index/IndexingSlowLog.java index 8672bbc835..b1d8ac1886 100644 --- a/core/src/main/java/org/elasticsearch/index/IndexingSlowLog.java +++ b/core/src/main/java/org/elasticsearch/index/IndexingSlowLog.java @@ -190,7 +190,7 @@ public final class IndexingSlowLog implements IndexingOperationListener { return sb.toString(); } try { - String source = XContentHelper.convertToJson(doc.source(), reformat); + String source = XContentHelper.convertToJson(doc.source(), reformat, doc.getXContentType()); sb.append(", source[").append(Strings.cleanTruncate(source, maxSourceCharsToLog)).append("]"); } catch (IOException e) { sb.append(", source[_failed_to_convert_]"); diff --git a/core/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java index fdc45530b9..a8c74101bf 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java @@ -30,6 +30,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.text.Text; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.analysis.IndexAnalyzers; @@ -256,8 +257,9 @@ public class DocumentMapper implements ToXContent { return this.objectMappers; } + // TODO this method looks like it is only used in tests... public ParsedDocument parse(String index, String type, String id, BytesReference source) throws MapperParsingException { - return parse(SourceToParse.source(index, type, id, source)); + return parse(SourceToParse.source(index, type, id, source, XContentType.JSON)); } public ParsedDocument parse(SourceToParse source) throws MapperParsingException { diff --git a/core/src/main/java/org/elasticsearch/index/mapper/DocumentMapperParser.java b/core/src/main/java/org/elasticsearch/index/mapper/DocumentMapperParser.java index 031f139075..be25775c13 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/DocumentMapperParser.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/DocumentMapperParser.java @@ -24,9 +24,9 @@ import org.elasticsearch.common.Nullable; import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.xcontent.NamedXContentRegistry; -import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.analysis.IndexAnalyzers; import org.elasticsearch.index.query.QueryShardContext; @@ -80,7 +80,7 @@ public class DocumentMapperParser { public DocumentMapper parse(@Nullable String type, CompressedXContent source, String defaultSource) throws MapperParsingException { Map<String, Object> mapping = null; if (source != null) { - Map<String, Object> root = XContentHelper.convertToMap(source.compressedReference(), true).v2(); + Map<String, Object> root = XContentHelper.convertToMap(source.compressedReference(), true, XContentType.JSON).v2(); Tuple<String, Map<String, Object>> t = extractMapping(type, root); type = t.v1(); mapping = t.v2(); @@ -162,7 +162,7 @@ public class DocumentMapperParser { private Tuple<String, Map<String, Object>> extractMapping(String type, String source) throws MapperParsingException { Map<String, Object> root; - try (XContentParser parser = XContentFactory.xContent(source).createParser(xContentRegistry, source)) { + try (XContentParser parser = XContentType.JSON.xContent().createParser(xContentRegistry, source)) { root = parser.mapOrdered(); } catch (Exception e) { throw new MapperParsingException("failed to parse mapping definition", e); diff --git a/core/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java b/core/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java index 014ff45200..97891b28aa 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java @@ -154,6 +154,7 @@ final class DocumentParser { source.routing(), context.docs(), context.sourceToParse().source(), + context.sourceToParse().getXContentType(), update ).parent(source.parent()); } diff --git a/core/src/main/java/org/elasticsearch/index/mapper/MapperService.java b/core/src/main/java/org/elasticsearch/index/mapper/MapperService.java index ee4b5fea15..41b29f3c67 100755 --- a/core/src/main/java/org/elasticsearch/index/mapper/MapperService.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/MapperService.java @@ -30,6 +30,7 @@ import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.MappingMetaData; import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.regex.Regex; import org.elasticsearch.common.settings.Setting; @@ -37,6 +38,7 @@ import org.elasticsearch.common.settings.Setting.Property; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.AbstractIndexComponent; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.analysis.IndexAnalyzers; @@ -189,8 +191,11 @@ public class MapperService extends AbstractIndexComponent implements Closeable { return this.documentParser; } + /** + * Parses the mappings (formatted as JSON) into a map + */ public static Map<String, Object> parseMapping(NamedXContentRegistry xContentRegistry, String mappingSource) throws Exception { - try (XContentParser parser = XContentFactory.xContent(mappingSource).createParser(xContentRegistry, mappingSource)) { + try (XContentParser parser = XContentType.JSON.xContent().createParser(xContentRegistry, mappingSource)) { return parser.map(); } } @@ -318,7 +323,8 @@ public class MapperService extends AbstractIndexComponent implements Closeable { && mappers.containsKey(type) == false; try { - DocumentMapper documentMapper = documentParser.parse(type, entry.getValue(), applyDefault ? defaultMappingSourceOrLastStored : null); + DocumentMapper documentMapper = + documentParser.parse(type, entry.getValue(), applyDefault ? defaultMappingSourceOrLastStored : null); documentMappers.add(documentMapper); } catch (Exception e) { throw new MapperParsingException("Failed to parse mapping [{}]: {}", e, entry.getKey(), e.getMessage()); diff --git a/core/src/main/java/org/elasticsearch/index/mapper/ParsedDocument.java b/core/src/main/java/org/elasticsearch/index/mapper/ParsedDocument.java index db8bdf9df7..f7d5804be0 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/ParsedDocument.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/ParsedDocument.java @@ -22,8 +22,8 @@ package org.elasticsearch.index.mapper; import org.apache.lucene.document.Field; import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.mapper.ParseContext.Document; -import org.elasticsearch.index.mapper.SeqNoFieldMapper; import java.util.List; @@ -43,6 +43,7 @@ public class ParsedDocument { private final List<Document> documents; private BytesReference source; + private XContentType xContentType; private Mapping dynamicMappingsUpdate; @@ -55,6 +56,7 @@ public class ParsedDocument { String routing, List<Document> documents, BytesReference source, + XContentType xContentType, Mapping dynamicMappingsUpdate) { this.version = version; this.seqID = seqID; @@ -65,6 +67,7 @@ public class ParsedDocument { this.documents = documents; this.source = source; this.dynamicMappingsUpdate = dynamicMappingsUpdate; + this.xContentType = xContentType; } public BytesRef uid() { @@ -105,8 +108,13 @@ public class ParsedDocument { return this.source; } - public void setSource(BytesReference source) { + public XContentType getXContentType() { + return this.xContentType; + } + + public void setSource(BytesReference source, XContentType xContentType) { this.source = source; + this.xContentType = xContentType; } public ParsedDocument parent(String parent) { diff --git a/core/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java index e06ec80a47..2eb2fdeaa0 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java @@ -241,7 +241,8 @@ public class SourceFieldMapper extends MetadataFieldMapper { if (filter != null) { // we don't update the context source if we filter, we want to keep it as is... - Tuple<XContentType, Map<String, Object>> mapTuple = XContentHelper.convertToMap(source, true); + Tuple<XContentType, Map<String, Object>> mapTuple = + XContentHelper.convertToMap(source, true, context.sourceToParse().getXContentType()); Map<String, Object> filteredSource = filter.apply(mapTuple.v2()); BytesStreamOutput bStream = new BytesStreamOutput(); XContentType contentType = mapTuple.v1(); diff --git a/core/src/main/java/org/elasticsearch/index/mapper/SourceToParse.java b/core/src/main/java/org/elasticsearch/index/mapper/SourceToParse.java index 0cafc50bbe..a8a983ecde 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/SourceToParse.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/SourceToParse.java @@ -23,15 +23,18 @@ import java.util.Objects; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentType; public class SourceToParse { - public static SourceToParse source(String index, String type, String id, BytesReference source) { - return source(Origin.PRIMARY, index, type, id, source); + public static SourceToParse source(String index, String type, String id, BytesReference source, XContentType contentType) { + return source(Origin.PRIMARY, index, type, id, source, contentType); } - public static SourceToParse source(Origin origin, String index, String type, String id, BytesReference source) { - return new SourceToParse(origin, index, type, id, source); + public static SourceToParse source(Origin origin, String index, String type, String id, BytesReference source, + XContentType contentType) { + return new SourceToParse(origin, index, type, id, source, contentType); } private final Origin origin; @@ -48,14 +51,17 @@ public class SourceToParse { private String parentId; - private SourceToParse(Origin origin, String index, String type, String id, BytesReference source) { + private XContentType xContentType; + + private SourceToParse(Origin origin, String index, String type, String id, BytesReference source, XContentType xContentType) { this.origin = Objects.requireNonNull(origin); this.index = Objects.requireNonNull(index); this.type = Objects.requireNonNull(type); this.id = Objects.requireNonNull(id); // we always convert back to byte array, since we store it and Field only supports bytes.. // so, we might as well do it here, and improve the performance of working with direct byte arrays - this.source = new BytesArray(source.toBytesRef()); + this.source = new BytesArray(Objects.requireNonNull(source).toBytesRef()); + this.xContentType = Objects.requireNonNull(xContentType); } public Origin origin() { @@ -91,6 +97,10 @@ public class SourceToParse { return this.routing; } + public XContentType getXContentType() { + return this.xContentType; + } + public SourceToParse routing(String routing) { this.routing = routing; return this; diff --git a/core/src/main/java/org/elasticsearch/index/query/MoreLikeThisQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/MoreLikeThisQueryBuilder.java index bfb579e0c3..ee0eff2a45 100644 --- a/core/src/main/java/org/elasticsearch/index/query/MoreLikeThisQueryBuilder.java +++ b/core/src/main/java/org/elasticsearch/index/query/MoreLikeThisQueryBuilder.java @@ -28,6 +28,7 @@ import org.apache.lucene.search.Query; import org.apache.lucene.util.BytesRef; import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.ExceptionsHelper; +import org.elasticsearch.Version; import org.elasticsearch.action.termvectors.MultiTermVectorsItemResponse; import org.elasticsearch.action.termvectors.MultiTermVectorsRequest; import org.elasticsearch.action.termvectors.MultiTermVectorsResponse; @@ -161,6 +162,7 @@ public class MoreLikeThisQueryBuilder extends AbstractQueryBuilder<MoreLikeThisQ private String type; private String id; private BytesReference doc; + private XContentType xContentType; private String[] fields; private Map<String, String> perFieldAnalyzer; private String routing; @@ -178,6 +180,7 @@ public class MoreLikeThisQueryBuilder extends AbstractQueryBuilder<MoreLikeThisQ this.type = copy.type; this.id = copy.id; this.doc = copy.doc; + this.xContentType = copy.xContentType; this.fields = copy.fields; this.perFieldAnalyzer = copy.perFieldAnalyzer; this.version = copy.version; @@ -214,6 +217,7 @@ public class MoreLikeThisQueryBuilder extends AbstractQueryBuilder<MoreLikeThisQ this.index = index; this.type = type; this.doc = doc.bytes(); + this.xContentType = doc.contentType(); } /** @@ -224,6 +228,11 @@ public class MoreLikeThisQueryBuilder extends AbstractQueryBuilder<MoreLikeThisQ type = in.readOptionalString(); if (in.readBoolean()) { doc = (BytesReference) in.readGenericValue(); + if (in.getVersion().after(Version.V_5_3_0_UNRELEASED)) { // TODO update to onOrAfter after backporting + xContentType = XContentType.readFrom(in); + } else { + xContentType = XContentFactory.xContentType(doc); + } } else { id = in.readString(); } @@ -241,6 +250,9 @@ public class MoreLikeThisQueryBuilder extends AbstractQueryBuilder<MoreLikeThisQ out.writeBoolean(doc != null); if (doc != null) { out.writeGenericValue(doc); + if (out.getVersion().after(Version.V_5_3_0_UNRELEASED)) { // TODO update to onOrAfter after backporting + xContentType.writeTo(out); + } } else { out.writeString(id); } @@ -325,6 +337,10 @@ public class MoreLikeThisQueryBuilder extends AbstractQueryBuilder<MoreLikeThisQ return this; } + XContentType xContentType() { + return xContentType; + } + /** * Convert this to a {@link TermVectorsRequest} for fetching the terms of the document. */ @@ -342,7 +358,7 @@ public class MoreLikeThisQueryBuilder extends AbstractQueryBuilder<MoreLikeThisQ .termStatistics(false); // for artificial docs to make sure that the id has changed in the item too if (doc != null) { - termVectorsRequest.doc(doc, true); + termVectorsRequest.doc(doc, true, xContentType); this.id = termVectorsRequest.id(); } return termVectorsRequest; @@ -366,6 +382,7 @@ public class MoreLikeThisQueryBuilder extends AbstractQueryBuilder<MoreLikeThisQ item.id = parser.text(); } else if (Field.DOC.match(currentFieldName)) { item.doc = jsonBuilder().copyCurrentStructure(parser).bytes(); + item.xContentType = XContentType.JSON; } else if (Field.FIELDS.match(currentFieldName)) { if (token == XContentParser.Token.START_ARRAY) { List<String> fields = new ArrayList<>(); @@ -416,12 +433,7 @@ public class MoreLikeThisQueryBuilder extends AbstractQueryBuilder<MoreLikeThisQ builder.field(Field.ID.getPreferredName(), this.id); } if (this.doc != null) { - XContentType contentType = XContentFactory.xContentType(this.doc); - if (contentType == builder.contentType()) { - builder.rawField(Field.DOC.getPreferredName(), this.doc); - } else { - builder.rawField(Field.DOC.getPreferredName(), doc); - } + builder.rawField(Field.DOC.getPreferredName(), this.doc, xContentType); } if (this.fields != null) { builder.array(Field.FIELDS.getPreferredName(), this.fields); diff --git a/core/src/main/java/org/elasticsearch/index/shard/TranslogRecoveryPerformer.java b/core/src/main/java/org/elasticsearch/index/shard/TranslogRecoveryPerformer.java index 6f4ae66a12..ad00e39097 100644 --- a/core/src/main/java/org/elasticsearch/index/shard/TranslogRecoveryPerformer.java +++ b/core/src/main/java/org/elasticsearch/index/shard/TranslogRecoveryPerformer.java @@ -22,6 +22,7 @@ import org.apache.logging.log4j.Logger; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.index.VersionType; import org.elasticsearch.index.engine.Engine; import org.elasticsearch.index.engine.IgnoreOnRecoveryEngineException; @@ -157,7 +158,8 @@ public class TranslogRecoveryPerformer { Translog.Index index = (Translog.Index) operation; // we set canHaveDuplicates to true all the time such that we de-optimze the translog case and ensure that all // autoGeneratedID docs that are coming from the primary are updated correctly. - Engine.Index engineIndex = IndexShard.prepareIndex(docMapper(index.type()), source(shardId.getIndexName(), index.type(), index.id(), index.source()) + Engine.Index engineIndex = IndexShard.prepareIndex(docMapper(index.type()), + source(shardId.getIndexName(), index.type(), index.id(), index.source(), XContentFactory.xContentType(index.source())) .routing(index.routing()).parent(index.parent()), index.seqNo(), index.primaryTerm(), index.version(), index.versionType().versionTypeForReplicationAndRecovery(), origin, index.getAutoGeneratedIdTimestamp(), true); maybeAddMappingUpdate(engineIndex.type(), engineIndex.parsedDoc().dynamicMappingsUpdate(), engineIndex.id(), allowMappingUpdates); diff --git a/core/src/main/java/org/elasticsearch/index/termvectors/TermVectorsService.java b/core/src/main/java/org/elasticsearch/index/termvectors/TermVectorsService.java index 520cb13390..7981c89124 100644 --- a/core/src/main/java/org/elasticsearch/index/termvectors/TermVectorsService.java +++ b/core/src/main/java/org/elasticsearch/index/termvectors/TermVectorsService.java @@ -36,6 +36,7 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.lucene.uid.Versions; import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.index.engine.Engine; import org.elasticsearch.index.get.GetField; @@ -270,7 +271,8 @@ public class TermVectorsService { private static Fields generateTermVectorsFromDoc(IndexShard indexShard, TermVectorsRequest request) throws IOException { // parse the document, at the moment we do update the mapping, just like percolate - ParsedDocument parsedDocument = parseDocument(indexShard, indexShard.shardId().getIndexName(), request.type(), request.doc()); + ParsedDocument parsedDocument = + parseDocument(indexShard, indexShard.shardId().getIndexName(), request.type(), request.doc(), request.xContentType()); // select the right fields and generate term vectors ParseContext.Document doc = parsedDocument.rootDoc(); @@ -293,13 +295,15 @@ public class TermVectorsService { String[] values = doc.getValues(field.name()); getFields.add(new GetField(field.name(), Arrays.asList((Object[]) values))); } - return generateTermVectors(indexShard, XContentHelper.convertToMap(parsedDocument.source(), true).v2(), getFields, request.offsets(), request.perFieldAnalyzer(), seenFields); + return generateTermVectors(indexShard, XContentHelper.convertToMap(parsedDocument.source(), true, request.xContentType()).v2(), + getFields, request.offsets(), request.perFieldAnalyzer(), seenFields); } - private static ParsedDocument parseDocument(IndexShard indexShard, String index, String type, BytesReference doc) { + private static ParsedDocument parseDocument(IndexShard indexShard, String index, String type, BytesReference doc, + XContentType xContentType) { MapperService mapperService = indexShard.mapperService(); DocumentMapperForType docMapper = mapperService.documentMapperWithAutoCreate(type); - ParsedDocument parsedDocument = docMapper.getDocumentMapper().parse(source(index, type, "_id_for_tv_api", doc)); + ParsedDocument parsedDocument = docMapper.getDocumentMapper().parse(source(index, type, "_id_for_tv_api", doc, xContentType)); if (docMapper.getMapping() != null) { parsedDocument.addDynamicMappingsUpdate(docMapper.getMapping()); } diff --git a/core/src/main/java/org/elasticsearch/ingest/PipelineConfiguration.java b/core/src/main/java/org/elasticsearch/ingest/PipelineConfiguration.java index 5cb9f6111f..3e874a0fd1 100644 --- a/core/src/main/java/org/elasticsearch/ingest/PipelineConfiguration.java +++ b/core/src/main/java/org/elasticsearch/ingest/PipelineConfiguration.java @@ -19,6 +19,7 @@ package org.elasticsearch.ingest; +import org.elasticsearch.Version; import org.elasticsearch.cluster.AbstractDiffable; import org.elasticsearch.cluster.Diff; import org.elasticsearch.common.ParseField; @@ -29,10 +30,13 @@ import org.elasticsearch.common.xcontent.ContextParser; import org.elasticsearch.common.xcontent.ObjectParser; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.common.xcontent.XContentType; import java.io.IOException; import java.util.Map; +import java.util.Objects; /** * Encapsulates a pipeline's id and configuration as a blob @@ -45,8 +49,9 @@ public final class PipelineConfiguration extends AbstractDiffable<PipelineConfig PARSER.declareField((parser, builder, aVoid) -> { XContentBuilder contentBuilder = XContentBuilder.builder(parser.contentType().xContent()); XContentHelper.copyCurrentStructure(contentBuilder.generator(), parser); - builder.setConfig(contentBuilder.bytes()); + builder.setConfig(contentBuilder.bytes(), contentBuilder.contentType()); }, new ParseField("config"), ObjectParser.ValueType.OBJECT); + } public static ContextParser<Void, PipelineConfiguration> getParser() { @@ -56,17 +61,19 @@ public final class PipelineConfiguration extends AbstractDiffable<PipelineConfig private String id; private BytesReference config; + private XContentType xContentType; void setId(String id) { this.id = id; } - void setConfig(BytesReference config) { + void setConfig(BytesReference config, XContentType xContentType) { this.config = config; + this.xContentType = xContentType; } PipelineConfiguration build() { - return new PipelineConfiguration(id, config); + return new PipelineConfiguration(id, config, xContentType); } } @@ -75,10 +82,12 @@ public final class PipelineConfiguration extends AbstractDiffable<PipelineConfig // and the way the map of maps config is read requires a deep copy (it removes instead of gets entries to check for unused options) // also the get pipeline api just directly returns this to the caller private final BytesReference config; + private final XContentType xContentType; - public PipelineConfiguration(String id, BytesReference config) { - this.id = id; - this.config = config; + public PipelineConfiguration(String id, BytesReference config, XContentType xContentType) { + this.id = Objects.requireNonNull(id); + this.config = Objects.requireNonNull(config); + this.xContentType = Objects.requireNonNull(xContentType); } public String getId() { @@ -86,7 +95,17 @@ public final class PipelineConfiguration extends AbstractDiffable<PipelineConfig } public Map<String, Object> getConfigAsMap() { - return XContentHelper.convertToMap(config, true).v2(); + return XContentHelper.convertToMap(config, true, xContentType).v2(); + } + + // pkg-private for tests + XContentType getXContentType() { + return xContentType; + } + + // pkg-private for tests + BytesReference getConfig() { + return config; } @Override @@ -99,7 +118,13 @@ public final class PipelineConfiguration extends AbstractDiffable<PipelineConfig } public static PipelineConfiguration readFrom(StreamInput in) throws IOException { - return new PipelineConfiguration(in.readString(), in.readBytesReference()); + if (in.getVersion().after(Version.V_5_3_0_UNRELEASED)) { // TODO update to onOrAfter after backporting + return new PipelineConfiguration(in.readString(), in.readBytesReference(), XContentType.readFrom(in)); + } else { + final String id = in.readString(); + final BytesReference config = in.readBytesReference(); + return new PipelineConfiguration(id, config, XContentFactory.xContentType(config)); + } } public static Diff<PipelineConfiguration> readDiffFrom(StreamInput in) throws IOException { @@ -110,6 +135,9 @@ public final class PipelineConfiguration extends AbstractDiffable<PipelineConfig public void writeTo(StreamOutput out) throws IOException { out.writeString(id); out.writeBytesReference(config); + if (out.getVersion().after(Version.V_5_3_0_UNRELEASED)) { // TODO update to onOrAfter after backporting + xContentType.writeTo(out); + } } @Override diff --git a/core/src/main/java/org/elasticsearch/ingest/PipelineStore.java b/core/src/main/java/org/elasticsearch/ingest/PipelineStore.java index 1171865a00..d476d7c2bd 100644 --- a/core/src/main/java/org/elasticsearch/ingest/PipelineStore.java +++ b/core/src/main/java/org/elasticsearch/ingest/PipelineStore.java @@ -162,7 +162,7 @@ public class PipelineStore extends AbstractComponent implements ClusterStateAppl throw new IllegalStateException("Ingest info is empty"); } - Map<String, Object> pipelineConfig = XContentHelper.convertToMap(request.getSource(), false).v2(); + Map<String, Object> pipelineConfig = XContentHelper.convertToMap(request.getSource(), false, request.getXContentType()).v2(); Pipeline pipeline = factory.create(request.getId(), pipelineConfig, processorFactories); List<IllegalArgumentException> exceptions = new ArrayList<>(); for (Processor processor : pipeline.flattenAllProcessors()) { @@ -185,7 +185,7 @@ public class PipelineStore extends AbstractComponent implements ClusterStateAppl pipelines = new HashMap<>(); } - pipelines.put(request.getId(), new PipelineConfiguration(request.getId(), request.getSource())); + pipelines.put(request.getId(), new PipelineConfiguration(request.getId(), request.getSource(), request.getXContentType())); ClusterState.Builder newState = ClusterState.builder(currentState); newState.metaData(MetaData.builder(currentState.getMetaData()) .putCustom(IngestMetadata.TYPE, new IngestMetadata(pipelines)) diff --git a/core/src/main/java/org/elasticsearch/rest/AbstractRestChannel.java b/core/src/main/java/org/elasticsearch/rest/AbstractRestChannel.java index 4b3505e97e..bdc78c82dd 100644 --- a/core/src/main/java/org/elasticsearch/rest/AbstractRestChannel.java +++ b/core/src/main/java/org/elasticsearch/rest/AbstractRestChannel.java @@ -20,7 +20,6 @@ package org.elasticsearch.rest; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Strings; -import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; @@ -58,28 +57,37 @@ public abstract class AbstractRestChannel implements RestChannel { @Override public XContentBuilder newBuilder() throws IOException { - return newBuilder(request.hasContent() ? request.content() : null, true); + return newBuilder(request.getXContentType(), true); } @Override public XContentBuilder newErrorBuilder() throws IOException { // Disable filtering when building error responses - return newBuilder(request.hasContent() ? request.content() : null, false); + return newBuilder(request.getXContentType(), false); } + /** + * Creates a new {@link XContentBuilder} for a response to be sent using this channel. The builder's type is determined by the following + * logic. If the request has a format parameter that will be used to attempt to map to an {@link XContentType}. If there is no format + * parameter, the HTTP Accept header is checked to see if it can be matched to a {@link XContentType}. If this first attempt to map + * fails, the request content type will be used if the value is not {@code null}; if the value is {@code null} the output format falls + * back to JSON. + */ @Override - public XContentBuilder newBuilder(@Nullable BytesReference autoDetectSource, boolean useFiltering) throws IOException { - XContentType contentType = XContentType.fromMediaTypeOrFormat(format); - if (contentType == null) { - // try and guess it from the auto detect source - if (autoDetectSource != null) { - contentType = XContentFactory.xContentType(autoDetectSource); + public XContentBuilder newBuilder(@Nullable XContentType requestContentType, boolean useFiltering) throws IOException { + // try to determine the response content type from the media type or the format query string parameter, with the format parameter + // taking precedence over the Accept header + XContentType responseContentType = XContentType.fromMediaTypeOrFormat(format); + if (responseContentType == null) { + if (requestContentType != null) { + // if there was a parsed content-type for the incoming request use that since no format was specified using the query + // string parameter or the HTTP Accept header + responseContentType = requestContentType; + } else { + // default to JSON output when all else fails + responseContentType = XContentType.JSON; } } - if (contentType == null) { - // default to JSON - contentType = XContentType.JSON; - } Set<String> includes = Collections.emptySet(); Set<String> excludes = Collections.emptySet(); @@ -89,7 +97,7 @@ public abstract class AbstractRestChannel implements RestChannel { excludes = filters.stream().filter(EXCLUDE_FILTER).map(f -> f.substring(1)).collect(toSet()); } - XContentBuilder builder = new XContentBuilder(XContentFactory.xContent(contentType), bytesOutput(), includes, excludes); + XContentBuilder builder = new XContentBuilder(XContentFactory.xContent(responseContentType), bytesOutput(), includes, excludes); if (pretty) { builder.prettyPrint().lfAtEnd(); } @@ -125,5 +133,4 @@ public abstract class AbstractRestChannel implements RestChannel { public boolean detailedErrorsEnabled() { return detailedErrorsEnabled; } - } diff --git a/core/src/main/java/org/elasticsearch/rest/BytesRestResponse.java b/core/src/main/java/org/elasticsearch/rest/BytesRestResponse.java index 2d45bc1765..a7b709625c 100644 --- a/core/src/main/java/org/elasticsearch/rest/BytesRestResponse.java +++ b/core/src/main/java/org/elasticsearch/rest/BytesRestResponse.java @@ -30,6 +30,7 @@ import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.logging.ESLoggerFactory; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.common.xcontent.XContentParser; import java.io.IOException; @@ -146,6 +147,13 @@ public class BytesRestResponse extends RestResponse { return builder; } + static BytesRestResponse createSimpleErrorResponse(RestStatus status, String errorMessage) throws IOException { + return new BytesRestResponse(status, JsonXContent.contentBuilder().startObject() + .field("error", errorMessage) + .field("status", status.getStatus()) + .endObject()); + } + public static ElasticsearchStatusException errorFromXContent(XContentParser parser) throws IOException { XContentParser.Token token = parser.nextToken(); ensureExpectedToken(XContentParser.Token.START_OBJECT, token, parser::getTokenLocation); diff --git a/core/src/main/java/org/elasticsearch/rest/RestChannel.java b/core/src/main/java/org/elasticsearch/rest/RestChannel.java index 2a56313fd8..8c8346f0ef 100644 --- a/core/src/main/java/org/elasticsearch/rest/RestChannel.java +++ b/core/src/main/java/org/elasticsearch/rest/RestChannel.java @@ -20,9 +20,9 @@ package org.elasticsearch.rest; import org.elasticsearch.common.Nullable; -import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentType; import java.io.IOException; @@ -35,7 +35,7 @@ public interface RestChannel { XContentBuilder newErrorBuilder() throws IOException; - XContentBuilder newBuilder(@Nullable BytesReference autoDetectSource, boolean useFiltering) throws IOException; + XContentBuilder newBuilder(@Nullable XContentType xContentType, boolean useFiltering) throws IOException; BytesStreamOutput bytesOutput(); @@ -47,5 +47,4 @@ public interface RestChannel { boolean detailedErrorsEnabled(); void sendResponse(RestResponse response); - } diff --git a/core/src/main/java/org/elasticsearch/rest/RestController.java b/core/src/main/java/org/elasticsearch/rest/RestController.java index 5ac82b7e45..fd3587df2a 100644 --- a/core/src/main/java/org/elasticsearch/rest/RestController.java +++ b/core/src/main/java/org/elasticsearch/rest/RestController.java @@ -22,18 +22,19 @@ package org.elasticsearch.rest; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; +import java.util.List; import java.util.Objects; import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.Supplier; import java.util.function.UnaryOperator; import org.apache.logging.log4j.message.ParameterizedMessage; -import org.apache.logging.log4j.util.Supplier; import org.elasticsearch.client.node.NodeClient; import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.breaker.CircuitBreaker; import org.elasticsearch.common.bytes.BytesArray; -import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.component.AbstractComponent; import org.elasticsearch.common.io.Streams; import org.elasticsearch.common.io.stream.BytesStreamOutput; @@ -42,12 +43,15 @@ import org.elasticsearch.common.path.PathTrie; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.http.HttpServerTransport; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.http.HttpTransportSettings; import org.elasticsearch.indices.breaker.CircuitBreakerService; +import org.elasticsearch.common.xcontent.XContentType; import static org.elasticsearch.rest.RestStatus.BAD_REQUEST; import static org.elasticsearch.rest.RestStatus.FORBIDDEN; import static org.elasticsearch.rest.RestStatus.INTERNAL_SERVER_ERROR; +import static org.elasticsearch.rest.RestStatus.NOT_ACCEPTABLE; import static org.elasticsearch.rest.RestStatus.OK; public class RestController extends AbstractComponent { @@ -67,6 +71,10 @@ public class RestController extends AbstractComponent { /** Rest headers that are copied to internal requests made during a rest request. */ private final Set<String> headersToCopy; + private final boolean isContentTypeRequired; + + private final DeprecationLogger deprecationLogger; + public RestController(Settings settings, Set<String> headersToCopy, UnaryOperator<RestHandler> handlerWrapper, NodeClient client, CircuitBreakerService circuitBreakerService) { super(settings); @@ -77,10 +85,10 @@ public class RestController extends AbstractComponent { this.handlerWrapper = handlerWrapper; this.client = client; this.circuitBreakerService = circuitBreakerService; + this.isContentTypeRequired = HttpTransportSettings.SETTING_HTTP_CONTENT_TYPE_REQUIRED.get(settings); + this.deprecationLogger = new DeprecationLogger(logger); } - - /** * Registers a REST handler to be executed when the provided {@code method} and {@code path} match the request. * @@ -165,15 +173,22 @@ public class RestController extends AbstractComponent { } RestChannel responseChannel = channel; try { - int contentLength = request.content().length(); - if (canTripCircuitBreaker(request)) { - inFlightRequestsBreaker(circuitBreakerService).addEstimateBytesAndMaybeBreak(contentLength, "<http_request>"); + final int contentLength = request.hasContent() ? request.content().length() : 0; + assert contentLength >= 0 : "content length was negative, how is that possible?"; + final RestHandler handler = getHandler(request); + + if (contentLength > 0 && hasContentTypeOrCanAutoDetect(request, handler) == false) { + sendContentTypeErrorMessage(request, responseChannel); } else { - inFlightRequestsBreaker(circuitBreakerService).addWithoutBreaking(contentLength); + if (canTripCircuitBreaker(request)) { + inFlightRequestsBreaker(circuitBreakerService).addEstimateBytesAndMaybeBreak(contentLength, "<http_request>"); + } else { + inFlightRequestsBreaker(circuitBreakerService).addWithoutBreaking(contentLength); + } + // iff we could reserve bytes for the request we need to send the response also over this channel + responseChannel = new ResourceHandlingHttpChannel(channel, circuitBreakerService, contentLength); + dispatchRequest(request, responseChannel, client, threadContext, handler); } - // iff we could reserve bytes for the request we need to send the response also over this channel - responseChannel = new ResourceHandlingHttpChannel(channel, circuitBreakerService, contentLength); - dispatchRequest(request, responseChannel, client, threadContext); } catch (Exception e) { try { responseChannel.sendResponse(new BytesRestResponse(channel, e)); @@ -185,33 +200,75 @@ public class RestController extends AbstractComponent { } } - void dispatchRequest(final RestRequest request, final RestChannel channel, final NodeClient client, ThreadContext threadContext) throws Exception { - if (!checkRequestParameters(request, channel)) { - return; - } - try (ThreadContext.StoredContext ignored = threadContext.stashContext()) { - for (String key : headersToCopy) { - String httpHeader = request.header(key); - if (httpHeader != null) { - threadContext.putHeader(key, httpHeader); + void dispatchRequest(final RestRequest request, final RestChannel channel, final NodeClient client, ThreadContext threadContext, + final RestHandler handler) throws Exception { + if (checkRequestParameters(request, channel) == false) { + channel.sendResponse(BytesRestResponse.createSimpleErrorResponse(BAD_REQUEST, "error traces in responses are disabled.")); + } else { + try (ThreadContext.StoredContext ignored = threadContext.stashContext()) { + for (String key : headersToCopy) { + String httpHeader = request.header(key); + if (httpHeader != null) { + threadContext.putHeader(key, httpHeader); + } } - } - final RestHandler handler = getHandler(request); + if (handler == null) { + if (request.method() == RestRequest.Method.OPTIONS) { + // when we have OPTIONS request, simply send OK by default (with the Access Control Origin header which gets automatically added) - if (handler == null) { - if (request.method() == RestRequest.Method.OPTIONS) { - // when we have OPTIONS request, simply send OK by default (with the Access Control Origin header which gets automatically added) - channel.sendResponse(new BytesRestResponse(OK, BytesRestResponse.TEXT_CONTENT_TYPE, BytesArray.EMPTY)); + channel.sendResponse(new BytesRestResponse(OK, BytesRestResponse.TEXT_CONTENT_TYPE, BytesArray.EMPTY)); + } else { + final String msg = "No handler found for uri [" + request.uri() + "] and method [" + request.method() + "]"; + channel.sendResponse(new BytesRestResponse(BAD_REQUEST, msg)); + } } else { - final String msg = "No handler found for uri [" + request.uri() + "] and method [" + request.method() + "]"; - channel.sendResponse(new BytesRestResponse(BAD_REQUEST, msg)); + final RestHandler wrappedHandler = Objects.requireNonNull(handlerWrapper.apply(handler)); + wrappedHandler.handleRequest(request, channel, client); } + } + } + } + + /** + * If a request contains content, this method will return {@code true} if the {@code Content-Type} header is present, matches an + * {@link XContentType} or the request is plain text, and content type is required. If content type is not required then this method + * returns true unless a content type could not be inferred from the body and the rest handler does not support plain text + */ + private boolean hasContentTypeOrCanAutoDetect(final RestRequest restRequest, final RestHandler restHandler) { + if (restRequest.getXContentType() == null) { + if (restHandler != null && restHandler.supportsPlainText()) { + // content type of null with a handler that supports plain text gets through for now. Once we remove plain text this can + // be removed! + deprecationLogger.deprecated("Plain text request bodies are deprecated. Use request parameters or body " + + "in a supported format."); + } else if (isContentTypeRequired) { + return false; } else { - final RestHandler wrappedHandler = Objects.requireNonNull(handlerWrapper.apply(handler)); - wrappedHandler.handleRequest(request, channel, client); + deprecationLogger.deprecated("Content type detection for rest requests is deprecated. Specify the content type using " + + "the [Content-Type] header."); + XContentType xContentType = XContentFactory.xContentType(restRequest.content()); + if (xContentType == null) { + return false; + } else { + restRequest.setXContentType(xContentType); + } } } + return true; + } + + private void sendContentTypeErrorMessage(RestRequest restRequest, RestChannel channel) throws IOException { + final List<String> contentTypeHeader = restRequest.getAllHeaderValues("Content-Type"); + final String errorMessage; + if (contentTypeHeader == null) { + errorMessage = "Content-Type header is missing"; + } else { + errorMessage = "Content-Type header [" + + Strings.collectionToCommaDelimitedString(restRequest.getAllHeaderValues("Content-Type")) + "] is not supported"; + } + + channel.sendResponse(BytesRestResponse.createSimpleErrorResponse(NOT_ACCEPTABLE, errorMessage)); } /** @@ -222,15 +279,6 @@ public class RestController extends AbstractComponent { // error_trace cannot be used when we disable detailed errors // we consume the error_trace parameter first to ensure that it is always consumed if (request.paramAsBoolean("error_trace", false) && channel.detailedErrorsEnabled() == false) { - try { - XContentBuilder builder = channel.newErrorBuilder(); - builder.startObject().field("error", "error traces in responses are disabled.").endObject().string(); - RestResponse response = new BytesRestResponse(BAD_REQUEST, builder); - response.addHeader("Content-Type", "application/json"); - channel.sendResponse(response); - } catch (IOException e) { - logger.warn("Failed to send response", e); - } return false; } @@ -312,8 +360,8 @@ public class RestController extends AbstractComponent { } @Override - public XContentBuilder newBuilder(@Nullable BytesReference autoDetectSource, boolean useFiltering) throws IOException { - return delegate.newBuilder(autoDetectSource, useFiltering); + public XContentBuilder newBuilder(@Nullable XContentType xContentType, boolean useFiltering) throws IOException { + return delegate.newBuilder(xContentType, useFiltering); } @Override diff --git a/core/src/main/java/org/elasticsearch/rest/RestHandler.java b/core/src/main/java/org/elasticsearch/rest/RestHandler.java index 393e425baf..740b98ae73 100644 --- a/core/src/main/java/org/elasticsearch/rest/RestHandler.java +++ b/core/src/main/java/org/elasticsearch/rest/RestHandler.java @@ -28,7 +28,6 @@ public interface RestHandler { /** * Handles a rest request. - * * @param request The request to handle * @param channel The channel to write the request response to * @param client A client to use to make internal requests on behalf of the original request @@ -38,4 +37,13 @@ public interface RestHandler { default boolean canTripCircuitBreaker() { return true; } + + /** + * Indicates if a RestHandler supports plain text bodies + * @deprecated use request parameters or bodies that can be parsed with XContent! + */ + @Deprecated + default boolean supportsPlainText() { + return false; + } } diff --git a/core/src/main/java/org/elasticsearch/rest/RestRequest.java b/core/src/main/java/org/elasticsearch/rest/RestRequest.java index b4b0614f8a..7d41faf12e 100644 --- a/core/src/main/java/org/elasticsearch/rest/RestRequest.java +++ b/core/src/main/java/org/elasticsearch/rest/RestRequest.java @@ -19,6 +19,7 @@ package org.elasticsearch.rest; +import org.apache.lucene.util.SetOnce; import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.common.Booleans; import org.elasticsearch.common.CheckedConsumer; @@ -26,20 +27,26 @@ import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.collect.Tuple; +import org.elasticsearch.common.logging.DeprecationLogger; +import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; import java.io.IOException; import java.net.SocketAddress; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.regex.Pattern; import java.util.stream.Collectors; import static org.elasticsearch.common.unit.ByteSizeValue.parseBytesSizeValue; @@ -47,12 +54,25 @@ import static org.elasticsearch.common.unit.TimeValue.parseTimeValue; public abstract class RestRequest implements ToXContent.Params { + private static final DeprecationLogger DEPRECATION_LOGGER = new DeprecationLogger(Loggers.getLogger(RestRequest.class)); + // tchar pattern as defined by RFC7230 section 3.2.6 + private static final Pattern TCHAR_PATTERN = Pattern.compile("[a-zA-z0-9!#$%&'*+\\-.\\^_`|~]+"); + private final NamedXContentRegistry xContentRegistry; private final Map<String, String> params; + private final Map<String, List<String>> headers; private final String rawPath; private final Set<String> consumedParams = new HashSet<>(); + private final SetOnce<XContentType> xContentType = new SetOnce<>(); - public RestRequest(NamedXContentRegistry xContentRegistry, String uri) { + /** + * Creates a new RestRequest + * @param xContentRegistry the xContentRegistry to use when parsing XContent + * @param uri the URI of the request that potentially contains request parameters + * @param headers a map of the headers. This map should implement a Case-Insensitive hashing for keys as HTTP header names are case + * insensitive + */ + public RestRequest(NamedXContentRegistry xContentRegistry, String uri, Map<String, List<String>> headers) { this.xContentRegistry = xContentRegistry; final Map<String, String> params = new HashMap<>(); int pathEndPos = uri.indexOf('?'); @@ -63,12 +83,32 @@ public abstract class RestRequest implements ToXContent.Params { RestUtils.decodeQueryString(uri, pathEndPos + 1, params); } this.params = params; + this.headers = Collections.unmodifiableMap(headers); + final List<String> contentType = getAllHeaderValues("Content-Type"); + final XContentType xContentType = parseContentType(contentType); + if (xContentType != null) { + this.xContentType.set(xContentType); + } } - public RestRequest(NamedXContentRegistry xContentRegistry, Map<String, String> params, String path) { + /** + * Creates a new RestRequest + * @param xContentRegistry the xContentRegistry to use when parsing XContent + * @param params the parameters of the request + * @param path the path of the request. This should not contain request parameters + * @param headers a map of the headers. This map should implement a Case-Insensitive hashing for keys as HTTP header names are case + * insensitive + */ + public RestRequest(NamedXContentRegistry xContentRegistry, Map<String, String> params, String path, Map<String, List<String>> headers) { this.xContentRegistry = xContentRegistry; this.params = params; this.rawPath = path; + this.headers = Collections.unmodifiableMap(headers); + final List<String> contentType = getAllHeaderValues("Content-Type"); + final XContentType xContentType = parseContentType(contentType); + if (xContentType != null) { + this.xContentType.set(xContentType); + } } public enum Method { @@ -100,9 +140,53 @@ public abstract class RestRequest implements ToXContent.Params { public abstract BytesReference content(); - public abstract String header(String name); + /** + * Get the value of the header or {@code null} if not found. This method only retrieves the first header value if multiple values are + * sent. Use of {@link #getAllHeaderValues(String)} should be preferred + */ + public final String header(String name) { + List<String> values = headers.get(name); + if (values != null && values.isEmpty() == false) { + return values.get(0); + } + return null; + } + + /** + * Get all values for the header or {@code null} if the header was not found + */ + public final List<String> getAllHeaderValues(String name) { + List<String> values = headers.get(name); + if (values != null) { + return Collections.unmodifiableList(values); + } + return null; + } + + /** + * Get all of the headers and values associated with the headers. Modifications of this map are not supported. + */ + public final Map<String, List<String>> getHeaders() { + return headers; + } - public abstract Iterable<Map.Entry<String, String>> headers(); + /** + * The {@link XContentType} that was parsed from the {@code Content-Type} header. This value will be {@code null} in the case of + * a request without a valid {@code Content-Type} header, a request without content ({@link #hasContent()}, or a plain text request + */ + @Nullable + public final XContentType getXContentType() { + return xContentType.get(); + } + + /** + * Sets the {@link XContentType} + * @deprecated this is only used to allow BWC with content-type detection + */ + @Deprecated + final void setXContentType(XContentType xContentType) { + this.xContentType.set(xContentType); + } @Nullable public SocketAddress getRemoteAddress() { @@ -254,8 +338,10 @@ public abstract class RestRequest implements ToXContent.Params { BytesReference content = content(); if (content.length() == 0) { throw new ElasticsearchParseException("Body required"); + } else if (xContentType.get() == null) { + throw new IllegalStateException("unknown content type"); } - return XContentFactory.xContent(content).createParser(xContentRegistry, content); + return xContentType.get().xContent().createParser(xContentRegistry, content); } /** @@ -283,11 +369,12 @@ public abstract class RestRequest implements ToXContent.Params { * if you need to handle the absence request content gracefully. */ public final XContentParser contentOrSourceParamParser() throws IOException { - BytesReference content = contentOrSourceParam(); + Tuple<XContentType, BytesReference> tuple = contentOrSourceParam(); + BytesReference content = tuple.v2(); if (content.length() == 0) { throw new ElasticsearchParseException("Body required"); } - return XContentFactory.xContent(content).createParser(xContentRegistry, content); + return tuple.v1().xContent().createParser(xContentRegistry, content); } /** @@ -296,9 +383,11 @@ public abstract class RestRequest implements ToXContent.Params { * back to the user when there isn't request content. */ public final void withContentOrSourceParamParserOrNull(CheckedConsumer<XContentParser, IOException> withParser) throws IOException { - BytesReference content = contentOrSourceParam(); + Tuple<XContentType, BytesReference> tuple = contentOrSourceParam(); + BytesReference content = tuple.v2(); + XContentType xContentType = tuple.v1(); if (content.length() > 0) { - try (XContentParser parser = XContentFactory.xContent(content).createParser(xContentRegistry, content)) { + try (XContentParser parser = xContentType.xContent().createParser(xContentRegistry, content)) { withParser.accept(parser); } } else { @@ -310,7 +399,66 @@ public abstract class RestRequest implements ToXContent.Params { * Get the content of the request or the contents of the {@code source} param. Prefer {@link #contentOrSourceParamParser()} or * {@link #withContentOrSourceParamParserOrNull(CheckedConsumer)} if you need a parser. */ - public final BytesReference contentOrSourceParam() { + public final Tuple<XContentType, BytesReference> contentOrSourceParam() { + if (hasContent()) { + if (xContentType.get() == null) { + throw new IllegalStateException("unknown content type"); + } + return new Tuple<>(xContentType.get(), content()); + } + + String source = param("source"); + String typeParam = param("source_content_type"); + if (source != null) { + BytesArray bytes = new BytesArray(source); + final XContentType xContentType; + if (typeParam != null) { + xContentType = parseContentType(Collections.singletonList(typeParam)); + } else { + DEPRECATION_LOGGER.deprecated("Deprecated use of the [source] parameter without the [source_content_type] parameter. Use " + + "the [source_content_type] parameter to specify the content type of the source such as [application/json]"); + xContentType = XContentFactory.xContentType(bytes); + } + + if (xContentType == null) { + throw new IllegalStateException("could not determine source content type"); + } + return new Tuple<>(xContentType, bytes); + } + return new Tuple<>(XContentType.JSON, BytesArray.EMPTY); + } + + /** + * Call a consumer with the parser for the contents of this request if it has contents, otherwise with a parser for the {@code source} + * parameter if there is one, otherwise with {@code null}. Use {@link #contentOrSourceParamParser()} if you should throw an exception + * back to the user when there isn't request content. This version allows for plain text content + */ + @Deprecated + public final void withContentOrSourceParamParserOrNullLenient(CheckedConsumer<XContentParser, IOException> withParser) + throws IOException { + if (hasContent() && xContentType.get() == null) { + withParser.accept(null); + } + + Tuple<XContentType, BytesReference> tuple = contentOrSourceParam(); + BytesReference content = tuple.v2(); + XContentType xContentType = tuple.v1(); + if (content.length() > 0) { + try (XContentParser parser = xContentType.xContent().createParser(xContentRegistry, content)) { + withParser.accept(parser); + } + } else { + withParser.accept(null); + } + } + + /** + * Get the content of the request or the contents of the {@code source} param without the xcontent type. This is useful the request can + * accept non xcontent values. + * @deprecated we should only take xcontent + */ + @Deprecated + public final BytesReference getContentOrSourceParamOnly() { if (hasContent()) { return content(); } @@ -320,4 +468,29 @@ public abstract class RestRequest implements ToXContent.Params { } return BytesArray.EMPTY; } + + /** + * Parses the given content type string for the media type. This method currently ignores parameters. + */ + // TODO stop ignoring parameters such as charset... + private static XContentType parseContentType(List<String> header) { + if (header == null || header.isEmpty()) { + return null; + } else if (header.size() > 1) { + throw new IllegalArgumentException("only one Content-Type header should be provided"); + } + + String rawContentType = header.get(0); + final String[] elements = rawContentType.split("[ \t]*;"); + if (elements.length > 0) { + final String[] splitMediaType = elements[0].split("/"); + if (splitMediaType.length == 2 && TCHAR_PATTERN.matcher(splitMediaType[0]).matches() + && TCHAR_PATTERN.matcher(splitMediaType[1].trim()).matches()) { + return XContentType.fromMediaType(elements[0]); + } else { + throw new IllegalArgumentException("invalid Content-Type header [" + rawContentType + "]"); + } + } + throw new IllegalArgumentException("empty Content-Type header"); + } } diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestPutStoredScriptAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestPutStoredScriptAction.java index cee46eba04..358c3656ce 100644 --- a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestPutStoredScriptAction.java +++ b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestPutStoredScriptAction.java @@ -65,7 +65,7 @@ public class RestPutStoredScriptAction extends BaseRestHandler { "specifying lang [" + lang + "] as part of the url path is deprecated, use request content instead"); } - PutStoredScriptRequest putRequest = new PutStoredScriptRequest(id, lang, content); + PutStoredScriptRequest putRequest = new PutStoredScriptRequest(id, lang, content, request.getXContentType()); return channel -> client.admin().cluster().putStoredScript(putRequest, new AcknowledgedRestListener<>(channel)); } } diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestCreateIndexAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestCreateIndexAction.java index ea0cf7bf0c..8c314efee6 100644 --- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestCreateIndexAction.java +++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestCreateIndexAction.java @@ -42,7 +42,7 @@ public class RestCreateIndexAction extends BaseRestHandler { public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { CreateIndexRequest createIndexRequest = new CreateIndexRequest(request.param("index")); if (request.hasContent()) { - createIndexRequest.source(request.content()); + createIndexRequest.source(request.content(), request.getXContentType()); } createIndexRequest.updateAllTypes(request.paramAsBoolean("update_all_types", false)); createIndexRequest.timeout(request.paramAsTime("timeout", createIndexRequest.timeout())); diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutIndexTemplateAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutIndexTemplateAction.java index c48ed3716d..b376f3dab3 100644 --- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutIndexTemplateAction.java +++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutIndexTemplateAction.java @@ -57,7 +57,7 @@ public class RestPutIndexTemplateAction extends BaseRestHandler { putRequest.masterNodeTimeout(request.paramAsTime("master_timeout", putRequest.masterNodeTimeout())); putRequest.create(request.paramAsBoolean("create", false)); putRequest.cause(request.param("cause", "")); - putRequest.source(request.content()); + putRequest.source(request.content(), request.getXContentType()); return channel -> client.admin().indices().putTemplate(putRequest, new AcknowledgedRestListener<>(channel)); } diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutMappingAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutMappingAction.java index 60c434c6b0..06fe679264 100644 --- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutMappingAction.java +++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutMappingAction.java @@ -20,7 +20,6 @@ package org.elasticsearch.rest.action.admin.indices; import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest; -import org.elasticsearch.action.admin.indices.mapping.put.PutMappingResponse; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.client.node.NodeClient; import org.elasticsearch.common.Strings; @@ -65,11 +64,11 @@ public class RestPutMappingAction extends BaseRestHandler { public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { PutMappingRequest putMappingRequest = putMappingRequest(Strings.splitStringByCommaToArray(request.param("index"))); putMappingRequest.type(request.param("type")); - putMappingRequest.source(request.content().utf8ToString()); + putMappingRequest.source(request.content(), request.getXContentType()); putMappingRequest.updateAllTypes(request.paramAsBoolean("update_all_types", false)); putMappingRequest.timeout(request.paramAsTime("timeout", putMappingRequest.timeout())); putMappingRequest.masterNodeTimeout(request.paramAsTime("master_timeout", putMappingRequest.masterNodeTimeout())); putMappingRequest.indicesOptions(IndicesOptions.fromRequest(request, putMappingRequest.indicesOptions())); - return channel -> client.admin().indices().putMapping(putMappingRequest, new AcknowledgedRestListener<PutMappingResponse>(channel)); + return channel -> client.admin().indices().putMapping(putMappingRequest, new AcknowledgedRestListener<>(channel)); } } diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestUpdateSettingsAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestUpdateSettingsAction.java index 9e38fb5e07..47037460e2 100644 --- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestUpdateSettingsAction.java +++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestUpdateSettingsAction.java @@ -66,7 +66,9 @@ public class RestUpdateSettingsAction extends BaseRestHandler { Settings.Builder updateSettings = Settings.builder(); String bodySettingsStr = request.content().utf8ToString(); if (Strings.hasText(bodySettingsStr)) { - Settings buildSettings = Settings.builder().loadFromSource(bodySettingsStr).build(); + Settings buildSettings = Settings.builder() + .loadFromSource(bodySettingsStr, request.getXContentType()) + .build(); for (Map.Entry<String, String> entry : buildSettings.getAsMap().entrySet()) { String key = entry.getKey(); String value = entry.getValue(); diff --git a/core/src/main/java/org/elasticsearch/rest/action/document/RestBulkAction.java b/core/src/main/java/org/elasticsearch/rest/action/document/RestBulkAction.java index af0faac553..8fdf2792a4 100644 --- a/core/src/main/java/org/elasticsearch/rest/action/document/RestBulkAction.java +++ b/core/src/main/java/org/elasticsearch/rest/action/document/RestBulkAction.java @@ -93,7 +93,7 @@ public class RestBulkAction extends BaseRestHandler { bulkRequest.timeout(request.paramAsTime("timeout", BulkShardRequest.DEFAULT_TIMEOUT)); bulkRequest.setRefreshPolicy(request.param("refresh")); bulkRequest.add(request.content(), defaultIndex, defaultType, defaultRouting, defaultFields, - defaultFetchSourceContext, defaultPipeline, null, allowExplicitIndex); + defaultFetchSourceContext, defaultPipeline, null, allowExplicitIndex, request.getXContentType()); return channel -> client.bulk(bulkRequest, new RestBuilderListener<BulkResponse>(channel) { @Override diff --git a/core/src/main/java/org/elasticsearch/rest/action/document/RestGetSourceAction.java b/core/src/main/java/org/elasticsearch/rest/action/document/RestGetSourceAction.java index 728e1ff599..83d424ed74 100644 --- a/core/src/main/java/org/elasticsearch/rest/action/document/RestGetSourceAction.java +++ b/core/src/main/java/org/elasticsearch/rest/action/document/RestGetSourceAction.java @@ -66,7 +66,7 @@ public class RestGetSourceAction extends BaseRestHandler { client.get(getRequest, new RestResponseListener<GetResponse>(channel) { @Override public RestResponse buildResponse(GetResponse response) throws Exception { - XContentBuilder builder = channel.newBuilder(response.getSourceInternal(), false); + XContentBuilder builder = channel.newBuilder(request.getXContentType(), false); if (response.isSourceEmpty()) { // check if doc source (or doc itself) is missing return new BytesRestResponse(NOT_FOUND, builder); } else { diff --git a/core/src/main/java/org/elasticsearch/rest/action/document/RestIndexAction.java b/core/src/main/java/org/elasticsearch/rest/action/document/RestIndexAction.java index 042b0d57b5..ddaf226875 100644 --- a/core/src/main/java/org/elasticsearch/rest/action/document/RestIndexAction.java +++ b/core/src/main/java/org/elasticsearch/rest/action/document/RestIndexAction.java @@ -65,7 +65,7 @@ public class RestIndexAction extends BaseRestHandler { indexRequest.routing(request.param("routing")); indexRequest.parent(request.param("parent")); // order is important, set it after routing, so it will set the routing indexRequest.setPipeline(request.param("pipeline")); - indexRequest.source(request.content()); + indexRequest.source(request.content(), request.getXContentType()); indexRequest.timeout(request.paramAsTime("timeout", IndexRequest.DEFAULT_TIMEOUT)); indexRequest.setRefreshPolicy(request.param("refresh")); indexRequest.version(RestActions.parseVersion(request)); diff --git a/core/src/main/java/org/elasticsearch/rest/action/ingest/RestPutPipelineAction.java b/core/src/main/java/org/elasticsearch/rest/action/ingest/RestPutPipelineAction.java index b7a8e6366b..2496c9b4a2 100644 --- a/core/src/main/java/org/elasticsearch/rest/action/ingest/RestPutPipelineAction.java +++ b/core/src/main/java/org/elasticsearch/rest/action/ingest/RestPutPipelineAction.java @@ -21,7 +21,10 @@ package org.elasticsearch.rest.action.ingest; import org.elasticsearch.action.ingest.PutPipelineRequest; import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestController; import org.elasticsearch.rest.RestRequest; @@ -38,7 +41,8 @@ public class RestPutPipelineAction extends BaseRestHandler { @Override public RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException { - PutPipelineRequest request = new PutPipelineRequest(restRequest.param("id"), restRequest.contentOrSourceParam()); + Tuple<XContentType, BytesReference> sourceTuple = restRequest.contentOrSourceParam(); + PutPipelineRequest request = new PutPipelineRequest(restRequest.param("id"), sourceTuple.v2(), sourceTuple.v1()); request.masterNodeTimeout(restRequest.paramAsTime("master_timeout", request.masterNodeTimeout())); request.timeout(restRequest.paramAsTime("timeout", request.timeout())); return channel -> client.admin().cluster().putPipeline(request, new AcknowledgedRestListener<>(channel)); diff --git a/core/src/main/java/org/elasticsearch/rest/action/ingest/RestSimulatePipelineAction.java b/core/src/main/java/org/elasticsearch/rest/action/ingest/RestSimulatePipelineAction.java index 5885dc9df6..9dbe1808a8 100644 --- a/core/src/main/java/org/elasticsearch/rest/action/ingest/RestSimulatePipelineAction.java +++ b/core/src/main/java/org/elasticsearch/rest/action/ingest/RestSimulatePipelineAction.java @@ -21,7 +21,10 @@ package org.elasticsearch.rest.action.ingest; import org.elasticsearch.action.ingest.SimulatePipelineRequest; import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestController; import org.elasticsearch.rest.RestRequest; @@ -40,7 +43,8 @@ public class RestSimulatePipelineAction extends BaseRestHandler { @Override public RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException { - SimulatePipelineRequest request = new SimulatePipelineRequest(restRequest.contentOrSourceParam()); + Tuple<XContentType, BytesReference> sourceTuple = restRequest.contentOrSourceParam(); + SimulatePipelineRequest request = new SimulatePipelineRequest(sourceTuple.v2(), sourceTuple.v1()); request.setId(restRequest.param("id")); request.setVerbose(restRequest.paramAsBoolean("verbose", false)); return channel -> client.admin().cluster().simulatePipeline(request, new RestToXContentListener<>(channel)); diff --git a/core/src/main/java/org/elasticsearch/rest/action/search/RestClearScrollAction.java b/core/src/main/java/org/elasticsearch/rest/action/search/RestClearScrollAction.java index e7ee7c7397..47252f5a10 100644 --- a/core/src/main/java/org/elasticsearch/rest/action/search/RestClearScrollAction.java +++ b/core/src/main/java/org/elasticsearch/rest/action/search/RestClearScrollAction.java @@ -20,12 +20,10 @@ package org.elasticsearch.rest.action.search; import org.elasticsearch.action.search.ClearScrollRequest; -import org.elasticsearch.action.search.ClearScrollResponse; import org.elasticsearch.client.node.NodeClient; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestController; @@ -50,26 +48,34 @@ public class RestClearScrollAction extends BaseRestHandler { String scrollIds = request.param("scroll_id"); ClearScrollRequest clearRequest = new ClearScrollRequest(); clearRequest.setScrollIds(Arrays.asList(splitScrollIds(scrollIds))); - BytesReference body = request.contentOrSourceParam(); - if (body.length() > 0) { - if (XContentFactory.xContentType(body) == null) { - scrollIds = body.utf8ToString(); - clearRequest.setScrollIds(Arrays.asList(splitScrollIds(scrollIds))); + request.withContentOrSourceParamParserOrNullLenient((xContentParser -> { + if (xContentParser == null) { + if (request.hasContent()) { + // TODO: why do we accept this plain text value? maybe we can just use the scroll params? + BytesReference body = request.content(); + String bodyScrollIds = body.utf8ToString(); + clearRequest.setScrollIds(Arrays.asList(splitScrollIds(bodyScrollIds))); + } } else { // NOTE: if rest request with xcontent body has request parameters, these parameters does not override xcontent value clearRequest.setScrollIds(null); - try (XContentParser parser = request.contentOrSourceParamParser()) { - buildFromContent(parser, clearRequest); + try { + buildFromContent(xContentParser, clearRequest); } catch (IOException e) { throw new IllegalArgumentException("Failed to parse request body", e); } } - } + })); - return channel -> client.clearScroll(clearRequest, new RestStatusToXContentListener<ClearScrollResponse>(channel)); + return channel -> client.clearScroll(clearRequest, new RestStatusToXContentListener<>(channel)); + } + + @Override + public boolean supportsPlainText() { + return true; } - public static String[] splitScrollIds(String scrollIds) { + private static String[] splitScrollIds(String scrollIds) { if (scrollIds == null) { return Strings.EMPTY_ARRAY; } diff --git a/core/src/main/java/org/elasticsearch/rest/action/search/RestMultiSearchAction.java b/core/src/main/java/org/elasticsearch/rest/action/search/RestMultiSearchAction.java index 567823a6e2..a803958618 100644 --- a/core/src/main/java/org/elasticsearch/rest/action/search/RestMultiSearchAction.java +++ b/core/src/main/java/org/elasticsearch/rest/action/search/RestMultiSearchAction.java @@ -26,10 +26,11 @@ import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.client.node.NodeClient; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContent; -import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.query.QueryParseContext; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestController; @@ -93,7 +94,7 @@ public class RestMultiSearchAction extends BaseRestHandler { } /** - * Parses a multi-line {@link RestRequest} body, instanciating a {@link SearchRequest} for each line and applying the given consumer. + * Parses a multi-line {@link RestRequest} body, instantiating a {@link SearchRequest} for each line and applying the given consumer. */ public static void parseMultiLineRequest(RestRequest request, IndicesOptions indicesOptions, boolean allowExplicitIndex, BiConsumer<SearchRequest, XContentParser> consumer) throws IOException { @@ -103,9 +104,10 @@ public class RestMultiSearchAction extends BaseRestHandler { String searchType = request.param("search_type"); String routing = request.param("routing"); - final BytesReference data = request.contentOrSourceParam(); + final Tuple<XContentType, BytesReference> sourceTuple = request.contentOrSourceParam(); + final XContent xContent = sourceTuple.v1().xContent(); + final BytesReference data = sourceTuple.v2(); - XContent xContent = XContentFactory.xContent(data); int from = 0; int length = data.length(); byte marker = xContent.streamSeparator(); @@ -176,7 +178,7 @@ public class RestMultiSearchAction extends BaseRestHandler { break; } BytesReference bytes = data.slice(from, nextMarker - from); - try (XContentParser parser = XContentFactory.xContent(bytes).createParser(request.getXContentRegistry(), bytes)) { + try (XContentParser parser = xContent.createParser(request.getXContentRegistry(), bytes)) { consumer.accept(searchRequest, parser); } // move pointers diff --git a/core/src/main/java/org/elasticsearch/rest/action/search/RestSearchScrollAction.java b/core/src/main/java/org/elasticsearch/rest/action/search/RestSearchScrollAction.java index 87a6f464cc..2a60fc6317 100644 --- a/core/src/main/java/org/elasticsearch/rest/action/search/RestSearchScrollAction.java +++ b/core/src/main/java/org/elasticsearch/rest/action/search/RestSearchScrollAction.java @@ -24,7 +24,6 @@ import org.elasticsearch.client.node.NodeClient; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestController; @@ -58,25 +57,33 @@ public class RestSearchScrollAction extends BaseRestHandler { searchScrollRequest.scroll(new Scroll(parseTimeValue(scroll, null, "scroll"))); } - BytesReference body = request.contentOrSourceParam(); - if (body.length() > 0) { - if (XContentFactory.xContentType(body) == null) { - if (scrollId == null) { - scrollId = body.utf8ToString(); - searchScrollRequest.scrollId(scrollId); + request.withContentOrSourceParamParserOrNull(xContentParser -> { + if (xContentParser == null) { + if (request.hasContent()) { + // TODO: why do we accept this plain text value? maybe we can just use the scroll params? + BytesReference body = request.getContentOrSourceParamOnly(); + if (scrollId == null) { + String bodyScrollId = body.utf8ToString(); + searchScrollRequest.scrollId(bodyScrollId); + } } } else { // NOTE: if rest request with xcontent body has request parameters, these parameters override xcontent values - try (XContentParser parser = request.contentOrSourceParamParser()) { - buildFromContent(parser, searchScrollRequest); + try { + buildFromContent(xContentParser, searchScrollRequest); } catch (IOException e) { throw new IllegalArgumentException("Failed to parse request body", e); } } - } + }); return channel -> client.searchScroll(searchScrollRequest, new RestStatusToXContentListener<>(channel)); } + @Override + public boolean supportsPlainText() { + return true; + } + public static void buildFromContent(XContentParser parser, SearchScrollRequest searchScrollRequest) throws IOException { if (parser.nextToken() != XContentParser.Token.START_OBJECT) { throw new IllegalArgumentException("Malformed content, must start with an object"); diff --git a/core/src/main/java/org/elasticsearch/script/ScriptMetaData.java b/core/src/main/java/org/elasticsearch/script/ScriptMetaData.java index e9fb6e126f..87afc21a8c 100644 --- a/core/src/main/java/org/elasticsearch/script/ScriptMetaData.java +++ b/core/src/main/java/org/elasticsearch/script/ScriptMetaData.java @@ -276,7 +276,6 @@ public final class ScriptMetaData implements MetaData.Custom, Writeable, ToXCont } else { source = new StoredScriptSource(id.substring(0, split), parser.text(), Collections.emptyMap()); } - scripts.put(id, source); id = null; diff --git a/core/src/main/java/org/elasticsearch/script/ScriptService.java b/core/src/main/java/org/elasticsearch/script/ScriptService.java index f9bcc77ab4..692e081a7b 100644 --- a/core/src/main/java/org/elasticsearch/script/ScriptService.java +++ b/core/src/main/java/org/elasticsearch/script/ScriptService.java @@ -389,7 +389,7 @@ public class ScriptService extends AbstractComponent implements Closeable, Clust request.content().length() + "] for script [" + request.id() + "]"); } - StoredScriptSource source = StoredScriptSource.parse(request.lang(), request.content()); + StoredScriptSource source = StoredScriptSource.parse(request.lang(), request.content(), request.xContentType()); if (isLangSupported(source.getLang()) == false) { throw new IllegalArgumentException("unable to put stored script with unsupported lang [" + source.getLang() + "]"); diff --git a/core/src/main/java/org/elasticsearch/script/StoredScriptSource.java b/core/src/main/java/org/elasticsearch/script/StoredScriptSource.java index 69fc1ed34c..e6c5b09362 100644 --- a/core/src/main/java/org/elasticsearch/script/StoredScriptSource.java +++ b/core/src/main/java/org/elasticsearch/script/StoredScriptSource.java @@ -40,6 +40,7 @@ import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentParser.Token; +import org.elasticsearch.common.xcontent.XContentType; import java.io.IOException; import java.io.UncheckedIOException; @@ -231,8 +232,8 @@ public class StoredScriptSource extends AbstractDiffable<StoredScriptSource> imp * @param content The content from the request to be parsed as described above. * @return The parsed {@link StoredScriptSource}. */ - public static StoredScriptSource parse(String lang, BytesReference content) { - try (XContentParser parser = XContentHelper.createParser(NamedXContentRegistry.EMPTY, content)) { + public static StoredScriptSource parse(String lang, BytesReference content, XContentType xContentType) { + try (XContentParser parser = xContentType.xContent().createParser(NamedXContentRegistry.EMPTY, content)) { Token token = parser.nextToken(); if (token != Token.START_OBJECT) { |