diff options
4 files changed, 249 insertions, 0 deletions
diff --git a/docs/reference/ingest/ingest-node.asciidoc b/docs/reference/ingest/ingest-node.asciidoc index f871703dd5..1e949d4df6 100644 --- a/docs/reference/ingest/ingest-node.asciidoc +++ b/docs/reference/ingest/ingest-node.asciidoc @@ -1272,6 +1272,28 @@ Throws an error when the field is not an array. } -------------------------------------------------- +[[json-processor]] +=== JSON Processor +Converts a JSON string into a structured JSON object. + +[[json-options]] +.Json Options +[options="header"] +|====== +| Name | Required | Default | Description +| `field` | yes | - | The field to be parsed +| `target_field` | no | `field` | The field to insert the converted structured object into +|====== + +[source,js] +-------------------------------------------------- +{ + "json": { + "field": "{\"foo\": 2000}" + } +} +-------------------------------------------------- + [[lowercase-processor]] === Lowercase Processor Converts a string to its lowercase equivalent. diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/JsonProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/JsonProcessor.java new file mode 100644 index 0000000000..024c3aef94 --- /dev/null +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/JsonProcessor.java @@ -0,0 +1,82 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.ingest.common; + +import com.fasterxml.jackson.core.JsonParseException; +import org.elasticsearch.common.xcontent.json.JsonXContent; +import org.elasticsearch.ingest.AbstractProcessor; +import org.elasticsearch.ingest.ConfigurationUtils; +import org.elasticsearch.ingest.IngestDocument; +import org.elasticsearch.ingest.Processor; + +import java.util.Map; + +/** + * Processor that serializes a string-valued field into a + * map of maps. + */ +public final class JsonProcessor extends AbstractProcessor { + + public static final String TYPE = "json"; + + private final String field; + private final String targetField; + + JsonProcessor(String tag, String field, String targetField) { + super(tag); + this.field = field; + this.targetField = targetField; + } + + public String getField() { + return field; + } + + public String getTargetField() { + return targetField; + } + + @Override + public void execute(IngestDocument document) throws Exception { + String stringValue = document.getFieldValue(field, String.class); + try { + Map<String, Object> mapValue = JsonXContent.jsonXContent.createParser(stringValue).map(); + document.setFieldValue(targetField, mapValue); + } catch (JsonParseException e) { + throw new IllegalArgumentException(e); + } + } + + @Override + public String getType() { + return TYPE; + } + + public static final class Factory implements Processor.Factory { + @Override + public JsonProcessor create(Map<String, Processor.Factory> registry, String processorTag, + Map<String, Object> config) throws Exception { + String field = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field"); + String targetField = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "target_field", field); + return new JsonProcessor(processorTag, field, targetField); + } + } +} + diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/JsonProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/JsonProcessorFactoryTests.java new file mode 100644 index 0000000000..6b935b8795 --- /dev/null +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/JsonProcessorFactoryTests.java @@ -0,0 +1,69 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.ingest.common; + +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.ingest.TestTemplateService; +import org.elasticsearch.test.ESTestCase; +import org.junit.Before; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +import static org.hamcrest.CoreMatchers.equalTo; + +public class JsonProcessorFactoryTests extends ESTestCase { + + private static final JsonProcessor.Factory FACTORY = new JsonProcessor.Factory(); + + public void testCreate() throws Exception { + String processorTag = randomAsciiOfLength(10); + String randomField = randomAsciiOfLength(10); + String randomTargetField = randomAsciiOfLength(5); + Map<String, Object> config = new HashMap<>(); + config.put("field", randomField); + config.put("target_field", randomTargetField); + JsonProcessor jsonProcessor = FACTORY.create(null, processorTag, config); + assertThat(jsonProcessor.getTag(), equalTo(processorTag)); + assertThat(jsonProcessor.getField(), equalTo(randomField)); + assertThat(jsonProcessor.getTargetField(), equalTo(randomTargetField)); + } + + public void testCreateWithDefaultTarget() throws Exception { + String processorTag = randomAsciiOfLength(10); + String randomField = randomAsciiOfLength(10); + Map<String, Object> config = new HashMap<>(); + config.put("field", randomField); + JsonProcessor jsonProcessor = FACTORY.create(null, processorTag, config); + assertThat(jsonProcessor.getTag(), equalTo(processorTag)); + assertThat(jsonProcessor.getField(), equalTo(randomField)); + assertThat(jsonProcessor.getTargetField(), equalTo(randomField)); + } + + public void testCreateWithMissingField() throws Exception { + Map<String, Object> config = new HashMap<>(); + String processorTag = randomAsciiOfLength(10); + ElasticsearchException exception = expectThrows(ElasticsearchParseException.class, + () -> FACTORY.create(null, processorTag, config)); + assertThat(exception.getMessage(), equalTo("[field] required property is missing")); + } +} diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/JsonProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/JsonProcessorTests.java new file mode 100644 index 0000000000..c62ebbb12a --- /dev/null +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/JsonProcessorTests.java @@ -0,0 +1,76 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.ingest.common; + +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.common.xcontent.json.JsonXContent; +import org.elasticsearch.ingest.IngestDocument; +import org.elasticsearch.ingest.RandomDocumentPicks; +import org.elasticsearch.test.ESTestCase; + +import java.util.HashMap; +import java.util.Map; + +import static org.elasticsearch.ingest.IngestDocumentMatcher.assertIngestDocument; +import static org.hamcrest.Matchers.equalTo; + +public class JsonProcessorTests extends ESTestCase { + + @SuppressWarnings("unchecked") + public void testExecute() throws Exception { + String processorTag = randomAsciiOfLength(3); + String randomField = randomAsciiOfLength(3); + String randomTargetField = randomAsciiOfLength(2); + JsonProcessor jsonProcessor = new JsonProcessor(processorTag, randomField, randomTargetField); + Map<String, Object> document = new HashMap<>(); + + Map<String, Object> randomJsonMap = RandomDocumentPicks.randomSource(random()); + XContentBuilder builder = JsonXContent.contentBuilder().map(randomJsonMap); + String randomJson = XContentHelper.convertToJson(builder.bytes(), false); + document.put(randomField, randomJson); + + IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document); + jsonProcessor.execute(ingestDocument); + Map<String, Object> jsonified = ingestDocument.getFieldValue(randomTargetField, Map.class); + assertIngestDocument(ingestDocument.getFieldValue(randomTargetField, Object.class), jsonified); + } + + public void testInvalidJson() { + JsonProcessor jsonProcessor = new JsonProcessor("tag", "field", "target_field"); + Map<String, Object> document = new HashMap<>(); + document.put("field", "invalid json"); + IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document); + + Exception exception = expectThrows(IllegalArgumentException.class, () -> jsonProcessor.execute(ingestDocument)); + assertThat(exception.getMessage(), equalTo("com.fasterxml.jackson.core.JsonParseException: Unrecognized token" + + " 'invalid': was expecting ('true', 'false' or 'null')\n" + + " at [Source: invalid json; line: 1, column: 8]")); + } + + public void testFieldMissing() { + JsonProcessor jsonProcessor = new JsonProcessor("tag", "field", "target_field"); + Map<String, Object> document = new HashMap<>(); + IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document); + + Exception exception = expectThrows(IllegalArgumentException.class, () -> jsonProcessor.execute(ingestDocument)); + assertThat(exception.getMessage(), equalTo("field [field] not present as part of path [field]")); + } +} |