diff options
author | Martijn van Groningen <martijn.v.groningen@gmail.com> | 2016-04-20 18:00:11 +0200 |
---|---|---|
committer | Martijn van Groningen <martijn.v.groningen@gmail.com> | 2016-04-21 13:40:43 +0200 |
commit | dd2184ab259fb544659ec28ce97467729e2974fe (patch) | |
tree | 4d0d7d9f41aa85356a96cb4323ceff75d37d7d5c /plugins/ingest-attachment | |
parent | 9eb242a5fedfe10af8dc208e63f952a4917c5af9 (diff) |
ingest: Streamline option naming for several processors:
* `rename` processor, renamed `to` to `target_field`
* `date` processor, renamed `match_field` to `field` and renamed `match_formats` to `formats`
* `geoip` processor, renamed `source_field` to `field` and renamed `fields` to `properties`
* `attachment` processor, renamed `source_field` to `field` and renamed `fields` to `properties`
Closes #17835
Diffstat (limited to 'plugins/ingest-attachment')
5 files changed, 80 insertions, 80 deletions
diff --git a/plugins/ingest-attachment/src/main/java/org/elasticsearch/ingest/attachment/AttachmentProcessor.java b/plugins/ingest-attachment/src/main/java/org/elasticsearch/ingest/attachment/AttachmentProcessor.java index 8b3b313356..d47e03dcbf 100644 --- a/plugins/ingest-attachment/src/main/java/org/elasticsearch/ingest/attachment/AttachmentProcessor.java +++ b/plugins/ingest-attachment/src/main/java/org/elasticsearch/ingest/attachment/AttachmentProcessor.java @@ -48,17 +48,17 @@ public final class AttachmentProcessor extends AbstractProcessor { private static final int NUMBER_OF_CHARS_INDEXED = 100000; - private final String sourceField; + private final String field; private final String targetField; - private final Set<Field> fields; + private final Set<Property> properties; private final int indexedChars; - AttachmentProcessor(String tag, String sourceField, String targetField, Set<Field> fields, + AttachmentProcessor(String tag, String field, String targetField, Set<Property> properties, int indexedChars) throws IOException { super(tag); - this.sourceField = sourceField; + this.field = field; this.targetField = targetField; - this.fields = fields; + this.properties = properties; this.indexedChars = indexedChars; } @@ -68,62 +68,62 @@ public final class AttachmentProcessor extends AbstractProcessor { try { Metadata metadata = new Metadata(); - byte[] input = ingestDocument.getFieldValueAsBytes(sourceField); + byte[] input = ingestDocument.getFieldValueAsBytes(field); String parsedContent = TikaImpl.parse(input, metadata, indexedChars); - if (fields.contains(Field.CONTENT) && Strings.hasLength(parsedContent)) { + if (properties.contains(Property.CONTENT) && Strings.hasLength(parsedContent)) { // somehow tika seems to append a newline at the end automatically, lets remove that again - additionalFields.put(Field.CONTENT.toLowerCase(), parsedContent.trim()); + additionalFields.put(Property.CONTENT.toLowerCase(), parsedContent.trim()); } - if (fields.contains(Field.LANGUAGE) && Strings.hasLength(parsedContent)) { + if (properties.contains(Property.LANGUAGE) && Strings.hasLength(parsedContent)) { LanguageIdentifier identifier = new LanguageIdentifier(parsedContent); String language = identifier.getLanguage(); - additionalFields.put(Field.LANGUAGE.toLowerCase(), language); + additionalFields.put(Property.LANGUAGE.toLowerCase(), language); } - if (fields.contains(Field.DATE)) { + if (properties.contains(Property.DATE)) { String createdDate = metadata.get(TikaCoreProperties.CREATED); if (createdDate != null) { - additionalFields.put(Field.DATE.toLowerCase(), createdDate); + additionalFields.put(Property.DATE.toLowerCase(), createdDate); } } - if (fields.contains(Field.TITLE)) { + if (properties.contains(Property.TITLE)) { String title = metadata.get(TikaCoreProperties.TITLE); if (Strings.hasLength(title)) { - additionalFields.put(Field.TITLE.toLowerCase(), title); + additionalFields.put(Property.TITLE.toLowerCase(), title); } } - if (fields.contains(Field.AUTHOR)) { + if (properties.contains(Property.AUTHOR)) { String author = metadata.get("Author"); if (Strings.hasLength(author)) { - additionalFields.put(Field.AUTHOR.toLowerCase(), author); + additionalFields.put(Property.AUTHOR.toLowerCase(), author); } } - if (fields.contains(Field.KEYWORDS)) { + if (properties.contains(Property.KEYWORDS)) { String keywords = metadata.get("Keywords"); if (Strings.hasLength(keywords)) { - additionalFields.put(Field.KEYWORDS.toLowerCase(), keywords); + additionalFields.put(Property.KEYWORDS.toLowerCase(), keywords); } } - if (fields.contains(Field.CONTENT_TYPE)) { + if (properties.contains(Property.CONTENT_TYPE)) { String contentType = metadata.get(Metadata.CONTENT_TYPE); if (Strings.hasLength(contentType)) { - additionalFields.put(Field.CONTENT_TYPE.toLowerCase(), contentType); + additionalFields.put(Property.CONTENT_TYPE.toLowerCase(), contentType); } } - if (fields.contains(Field.CONTENT_LENGTH)) { + if (properties.contains(Property.CONTENT_LENGTH)) { String contentLength = metadata.get(Metadata.CONTENT_LENGTH); String length = Strings.hasLength(contentLength) ? contentLength : String.valueOf(parsedContent.length()); - additionalFields.put(Field.CONTENT_LENGTH.toLowerCase(), length); + additionalFields.put(Property.CONTENT_LENGTH.toLowerCase(), length); } } catch (Throwable e) { - throw new ElasticsearchParseException("Error parsing document in field [{}]", e, sourceField); + throw new ElasticsearchParseException("Error parsing document in field [{}]", e, field); } ingestDocument.setFieldValue(targetField, additionalFields); @@ -134,16 +134,16 @@ public final class AttachmentProcessor extends AbstractProcessor { return TYPE; } - String getSourceField() { - return sourceField; + String getField() { + return field; } String getTargetField() { return targetField; } - Set<Field> getFields() { - return fields; + Set<Property> getProperties() { + return properties; } int getIndexedChars() { @@ -152,35 +152,35 @@ public final class AttachmentProcessor extends AbstractProcessor { public static final class Factory extends AbstractProcessorFactory<AttachmentProcessor> { - static final Set<Field> DEFAULT_FIELDS = EnumSet.allOf(Field.class); + static final Set<Property> DEFAULT_PROPERTIES = EnumSet.allOf(Property.class); @Override public AttachmentProcessor doCreate(String processorTag, Map<String, Object> config) throws Exception { - String sourceField = readStringProperty(TYPE, processorTag, config, "source_field"); + String field = readStringProperty(TYPE, processorTag, config, "field"); String targetField = readStringProperty(TYPE, processorTag, config, "target_field", "attachment"); - List<String> fieldNames = readOptionalList(TYPE, processorTag, config, "fields"); + List<String> properyNames = readOptionalList(TYPE, processorTag, config, "properties"); int indexedChars = readIntProperty(TYPE, processorTag, config, "indexed_chars", NUMBER_OF_CHARS_INDEXED); - final Set<Field> fields; - if (fieldNames != null) { - fields = EnumSet.noneOf(Field.class); - for (String fieldName : fieldNames) { + final Set<Property> properties; + if (properyNames != null) { + properties = EnumSet.noneOf(Property.class); + for (String fieldName : properyNames) { try { - fields.add(Field.parse(fieldName)); + properties.add(Property.parse(fieldName)); } catch (Exception e) { - throw newConfigurationException(TYPE, processorTag, "fields", "illegal field option [" + - fieldName + "]. valid values are " + Arrays.toString(Field.values())); + throw newConfigurationException(TYPE, processorTag, "properties", "illegal field option [" + + fieldName + "]. valid values are " + Arrays.toString(Property.values())); } } } else { - fields = DEFAULT_FIELDS; + properties = DEFAULT_PROPERTIES; } - return new AttachmentProcessor(processorTag, sourceField, targetField, fields, indexedChars); + return new AttachmentProcessor(processorTag, field, targetField, properties, indexedChars); } } - public enum Field { + enum Property { CONTENT, TITLE, @@ -191,7 +191,7 @@ public final class AttachmentProcessor extends AbstractProcessor { CONTENT_LENGTH, LANGUAGE; - public static Field parse(String value) { + public static Property parse(String value) { return valueOf(value.toUpperCase(Locale.ROOT)); } diff --git a/plugins/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/AttachmentProcessorFactoryTests.java b/plugins/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/AttachmentProcessorFactoryTests.java index 7d48b217ab..09fb4e3951 100644 --- a/plugins/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/AttachmentProcessorFactoryTests.java +++ b/plugins/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/AttachmentProcessorFactoryTests.java @@ -43,22 +43,22 @@ public class AttachmentProcessorFactoryTests extends ESTestCase { public void testBuildDefaults() throws Exception { Map<String, Object> config = new HashMap<>(); - config.put("source_field", "_field"); + config.put("field", "_field"); String processorTag = randomAsciiOfLength(10); config.put(AbstractProcessorFactory.TAG_KEY, processorTag); AttachmentProcessor processor = factory.create(config); assertThat(processor.getTag(), equalTo(processorTag)); - assertThat(processor.getSourceField(), equalTo("_field")); + assertThat(processor.getField(), equalTo("_field")); assertThat(processor.getTargetField(), equalTo("attachment")); - assertThat(processor.getFields(), sameInstance(AttachmentProcessor.Factory.DEFAULT_FIELDS)); + assertThat(processor.getProperties(), sameInstance(AttachmentProcessor.Factory.DEFAULT_PROPERTIES)); } public void testConfigureIndexedChars() throws Exception { int indexedChars = randomIntBetween(1, 100000); Map<String, Object> config = new HashMap<>(); - config.put("source_field", "_field"); + config.put("field", "_field"); config.put("indexed_chars", indexedChars); String processorTag = randomAsciiOfLength(10); @@ -70,53 +70,53 @@ public class AttachmentProcessorFactoryTests extends ESTestCase { public void testBuildTargetField() throws Exception { Map<String, Object> config = new HashMap<>(); - config.put("source_field", "_field"); + config.put("field", "_field"); config.put("target_field", "_field"); AttachmentProcessor processor = factory.create(config); - assertThat(processor.getSourceField(), equalTo("_field")); + assertThat(processor.getField(), equalTo("_field")); assertThat(processor.getTargetField(), equalTo("_field")); } public void testBuildFields() throws Exception { - Set<AttachmentProcessor.Field> fields = EnumSet.noneOf(AttachmentProcessor.Field.class); + Set<AttachmentProcessor.Property> properties = EnumSet.noneOf(AttachmentProcessor.Property.class); List<String> fieldNames = new ArrayList<>(); - int numFields = scaledRandomIntBetween(1, AttachmentProcessor.Field.values().length); + int numFields = scaledRandomIntBetween(1, AttachmentProcessor.Property.values().length); for (int i = 0; i < numFields; i++) { - AttachmentProcessor.Field field = AttachmentProcessor.Field.values()[i]; - fields.add(field); - fieldNames.add(field.name().toLowerCase(Locale.ROOT)); + AttachmentProcessor.Property property = AttachmentProcessor.Property.values()[i]; + properties.add(property); + fieldNames.add(property.name().toLowerCase(Locale.ROOT)); } Map<String, Object> config = new HashMap<>(); - config.put("source_field", "_field"); - config.put("fields", fieldNames); + config.put("field", "_field"); + config.put("properties", fieldNames); AttachmentProcessor processor = factory.create(config); - assertThat(processor.getSourceField(), equalTo("_field")); - assertThat(processor.getFields(), equalTo(fields)); + assertThat(processor.getField(), equalTo("_field")); + assertThat(processor.getProperties(), equalTo(properties)); } public void testBuildIllegalFieldOption() throws Exception { Map<String, Object> config = new HashMap<>(); - config.put("source_field", "_field"); - config.put("fields", Collections.singletonList("invalid")); + config.put("field", "_field"); + config.put("properties", Collections.singletonList("invalid")); try { factory.create(config); fail("exception expected"); } catch (ElasticsearchParseException e) { - assertThat(e.getMessage(), containsString("[fields] illegal field option [invalid]")); + assertThat(e.getMessage(), containsString("[properties] illegal field option [invalid]")); // ensure allowed fields are mentioned - for (AttachmentProcessor.Field field : AttachmentProcessor.Field.values()) { - assertThat(e.getMessage(), containsString(field.name())); + for (AttachmentProcessor.Property property : AttachmentProcessor.Property.values()) { + assertThat(e.getMessage(), containsString(property.name())); } } config = new HashMap<>(); - config.put("source_field", "_field"); - config.put("fields", "invalid"); + config.put("field", "_field"); + config.put("properties", "invalid"); try { factory.create(config); fail("exception expected"); } catch (ElasticsearchParseException e) { - assertThat(e.getMessage(), equalTo("[fields] property isn't a list, but of type [java.lang.String]")); + assertThat(e.getMessage(), equalTo("[properties] property isn't a list, but of type [java.lang.String]")); } } } diff --git a/plugins/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/AttachmentProcessorTests.java b/plugins/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/AttachmentProcessorTests.java index 021971c707..945c5d3479 100644 --- a/plugins/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/AttachmentProcessorTests.java +++ b/plugins/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/AttachmentProcessorTests.java @@ -51,7 +51,7 @@ public class AttachmentProcessorTests extends ESTestCase { @Before public void createStandardProcessor() throws IOException { processor = new AttachmentProcessor(randomAsciiOfLength(10), "source_field", - "target_field", EnumSet.allOf(AttachmentProcessor.Field.class), 10000); + "target_field", EnumSet.allOf(AttachmentProcessor.Property.class), 10000); } public void testEnglishTextDocument() throws Exception { @@ -66,25 +66,25 @@ public class AttachmentProcessorTests extends ESTestCase { public void testHtmlDocumentWithRandomFields() throws Exception { //date is not present in the html doc - ArrayList<AttachmentProcessor.Field> fieldsList = new ArrayList<>(EnumSet.complementOf(EnumSet.of - (AttachmentProcessor.Field.DATE))); - Set<AttachmentProcessor.Field> selectedFields = new HashSet<>(); + ArrayList<AttachmentProcessor.Property> fieldsList = new ArrayList<>(EnumSet.complementOf(EnumSet.of + (AttachmentProcessor.Property.DATE))); + Set<AttachmentProcessor.Property> selectedProperties = new HashSet<>(); int numFields = randomIntBetween(1, fieldsList.size()); String[] selectedFieldNames = new String[numFields]; for (int i = 0; i < numFields; i++) { - AttachmentProcessor.Field field; + AttachmentProcessor.Property property; do { - field = randomFrom(fieldsList); - } while (selectedFields.add(field) == false); + property = randomFrom(fieldsList); + } while (selectedProperties.add(property) == false); - selectedFieldNames[i] = field.toLowerCase(); + selectedFieldNames[i] = property.toLowerCase(); } if (randomBoolean()) { - selectedFields.add(AttachmentProcessor.Field.DATE); + selectedProperties.add(AttachmentProcessor.Property.DATE); } processor = new AttachmentProcessor(randomAsciiOfLength(10), "source_field", - "target_field", selectedFields, 10000); + "target_field", selectedProperties, 10000); Map<String, Object> attachmentData = parseDocument("htmlWithEmptyDateMeta.html", processor); assertThat(attachmentData.keySet(), hasSize(selectedFieldNames.length)); diff --git a/plugins/ingest-attachment/src/test/resources/rest-api-spec/test/ingest_attachment/20_attachment_processor.yaml b/plugins/ingest-attachment/src/test/resources/rest-api-spec/test/ingest_attachment/20_attachment_processor.yaml index 85c2f0d245..9c46610f10 100644 --- a/plugins/ingest-attachment/src/test/resources/rest-api-spec/test/ingest_attachment/20_attachment_processor.yaml +++ b/plugins/ingest-attachment/src/test/resources/rest-api-spec/test/ingest_attachment/20_attachment_processor.yaml @@ -9,7 +9,7 @@ "processors": [ { "attachment" : { - "source_field" : "field1" + "field" : "field1" } } ] @@ -51,8 +51,8 @@ "processors": [ { "attachment" : { - "source_field" : "field1", - "fields" : ["language"] + "field" : "field1", + "properties" : ["language"] } } ] @@ -87,7 +87,7 @@ "processors": [ { "attachment" : { - "source_field" : "field1", + "field" : "field1", "indexed_chars": 30 } } diff --git a/plugins/ingest-attachment/src/test/resources/rest-api-spec/test/ingest_attachment/30_files_supported.yaml b/plugins/ingest-attachment/src/test/resources/rest-api-spec/test/ingest_attachment/30_files_supported.yaml index 4f56603a67..3b9183e930 100644 --- a/plugins/ingest-attachment/src/test/resources/rest-api-spec/test/ingest_attachment/30_files_supported.yaml +++ b/plugins/ingest-attachment/src/test/resources/rest-api-spec/test/ingest_attachment/30_files_supported.yaml @@ -9,7 +9,7 @@ "processors": [ { "attachment" : { - "source_field" : "field1" + "field" : "field1" } } ] @@ -49,7 +49,7 @@ "processors": [ { "attachment" : { - "source_field" : "field1" + "field" : "field1" } } ] |