summaryrefslogtreecommitdiff
path: root/plugins/ingest-attachment
diff options
context:
space:
mode:
authorMartijn van Groningen <martijn.v.groningen@gmail.com>2016-04-20 18:00:11 +0200
committerMartijn van Groningen <martijn.v.groningen@gmail.com>2016-04-21 13:40:43 +0200
commitdd2184ab259fb544659ec28ce97467729e2974fe (patch)
tree4d0d7d9f41aa85356a96cb4323ceff75d37d7d5c /plugins/ingest-attachment
parent9eb242a5fedfe10af8dc208e63f952a4917c5af9 (diff)
ingest: Streamline option naming for several processors:
* `rename` processor, renamed `to` to `target_field` * `date` processor, renamed `match_field` to `field` and renamed `match_formats` to `formats` * `geoip` processor, renamed `source_field` to `field` and renamed `fields` to `properties` * `attachment` processor, renamed `source_field` to `field` and renamed `fields` to `properties` Closes #17835
Diffstat (limited to 'plugins/ingest-attachment')
-rw-r--r--plugins/ingest-attachment/src/main/java/org/elasticsearch/ingest/attachment/AttachmentProcessor.java82
-rw-r--r--plugins/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/AttachmentProcessorFactoryTests.java46
-rw-r--r--plugins/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/AttachmentProcessorTests.java20
-rw-r--r--plugins/ingest-attachment/src/test/resources/rest-api-spec/test/ingest_attachment/20_attachment_processor.yaml8
-rw-r--r--plugins/ingest-attachment/src/test/resources/rest-api-spec/test/ingest_attachment/30_files_supported.yaml4
5 files changed, 80 insertions, 80 deletions
diff --git a/plugins/ingest-attachment/src/main/java/org/elasticsearch/ingest/attachment/AttachmentProcessor.java b/plugins/ingest-attachment/src/main/java/org/elasticsearch/ingest/attachment/AttachmentProcessor.java
index 8b3b313356..d47e03dcbf 100644
--- a/plugins/ingest-attachment/src/main/java/org/elasticsearch/ingest/attachment/AttachmentProcessor.java
+++ b/plugins/ingest-attachment/src/main/java/org/elasticsearch/ingest/attachment/AttachmentProcessor.java
@@ -48,17 +48,17 @@ public final class AttachmentProcessor extends AbstractProcessor {
private static final int NUMBER_OF_CHARS_INDEXED = 100000;
- private final String sourceField;
+ private final String field;
private final String targetField;
- private final Set<Field> fields;
+ private final Set<Property> properties;
private final int indexedChars;
- AttachmentProcessor(String tag, String sourceField, String targetField, Set<Field> fields,
+ AttachmentProcessor(String tag, String field, String targetField, Set<Property> properties,
int indexedChars) throws IOException {
super(tag);
- this.sourceField = sourceField;
+ this.field = field;
this.targetField = targetField;
- this.fields = fields;
+ this.properties = properties;
this.indexedChars = indexedChars;
}
@@ -68,62 +68,62 @@ public final class AttachmentProcessor extends AbstractProcessor {
try {
Metadata metadata = new Metadata();
- byte[] input = ingestDocument.getFieldValueAsBytes(sourceField);
+ byte[] input = ingestDocument.getFieldValueAsBytes(field);
String parsedContent = TikaImpl.parse(input, metadata, indexedChars);
- if (fields.contains(Field.CONTENT) && Strings.hasLength(parsedContent)) {
+ if (properties.contains(Property.CONTENT) && Strings.hasLength(parsedContent)) {
// somehow tika seems to append a newline at the end automatically, lets remove that again
- additionalFields.put(Field.CONTENT.toLowerCase(), parsedContent.trim());
+ additionalFields.put(Property.CONTENT.toLowerCase(), parsedContent.trim());
}
- if (fields.contains(Field.LANGUAGE) && Strings.hasLength(parsedContent)) {
+ if (properties.contains(Property.LANGUAGE) && Strings.hasLength(parsedContent)) {
LanguageIdentifier identifier = new LanguageIdentifier(parsedContent);
String language = identifier.getLanguage();
- additionalFields.put(Field.LANGUAGE.toLowerCase(), language);
+ additionalFields.put(Property.LANGUAGE.toLowerCase(), language);
}
- if (fields.contains(Field.DATE)) {
+ if (properties.contains(Property.DATE)) {
String createdDate = metadata.get(TikaCoreProperties.CREATED);
if (createdDate != null) {
- additionalFields.put(Field.DATE.toLowerCase(), createdDate);
+ additionalFields.put(Property.DATE.toLowerCase(), createdDate);
}
}
- if (fields.contains(Field.TITLE)) {
+ if (properties.contains(Property.TITLE)) {
String title = metadata.get(TikaCoreProperties.TITLE);
if (Strings.hasLength(title)) {
- additionalFields.put(Field.TITLE.toLowerCase(), title);
+ additionalFields.put(Property.TITLE.toLowerCase(), title);
}
}
- if (fields.contains(Field.AUTHOR)) {
+ if (properties.contains(Property.AUTHOR)) {
String author = metadata.get("Author");
if (Strings.hasLength(author)) {
- additionalFields.put(Field.AUTHOR.toLowerCase(), author);
+ additionalFields.put(Property.AUTHOR.toLowerCase(), author);
}
}
- if (fields.contains(Field.KEYWORDS)) {
+ if (properties.contains(Property.KEYWORDS)) {
String keywords = metadata.get("Keywords");
if (Strings.hasLength(keywords)) {
- additionalFields.put(Field.KEYWORDS.toLowerCase(), keywords);
+ additionalFields.put(Property.KEYWORDS.toLowerCase(), keywords);
}
}
- if (fields.contains(Field.CONTENT_TYPE)) {
+ if (properties.contains(Property.CONTENT_TYPE)) {
String contentType = metadata.get(Metadata.CONTENT_TYPE);
if (Strings.hasLength(contentType)) {
- additionalFields.put(Field.CONTENT_TYPE.toLowerCase(), contentType);
+ additionalFields.put(Property.CONTENT_TYPE.toLowerCase(), contentType);
}
}
- if (fields.contains(Field.CONTENT_LENGTH)) {
+ if (properties.contains(Property.CONTENT_LENGTH)) {
String contentLength = metadata.get(Metadata.CONTENT_LENGTH);
String length = Strings.hasLength(contentLength) ? contentLength : String.valueOf(parsedContent.length());
- additionalFields.put(Field.CONTENT_LENGTH.toLowerCase(), length);
+ additionalFields.put(Property.CONTENT_LENGTH.toLowerCase(), length);
}
} catch (Throwable e) {
- throw new ElasticsearchParseException("Error parsing document in field [{}]", e, sourceField);
+ throw new ElasticsearchParseException("Error parsing document in field [{}]", e, field);
}
ingestDocument.setFieldValue(targetField, additionalFields);
@@ -134,16 +134,16 @@ public final class AttachmentProcessor extends AbstractProcessor {
return TYPE;
}
- String getSourceField() {
- return sourceField;
+ String getField() {
+ return field;
}
String getTargetField() {
return targetField;
}
- Set<Field> getFields() {
- return fields;
+ Set<Property> getProperties() {
+ return properties;
}
int getIndexedChars() {
@@ -152,35 +152,35 @@ public final class AttachmentProcessor extends AbstractProcessor {
public static final class Factory extends AbstractProcessorFactory<AttachmentProcessor> {
- static final Set<Field> DEFAULT_FIELDS = EnumSet.allOf(Field.class);
+ static final Set<Property> DEFAULT_PROPERTIES = EnumSet.allOf(Property.class);
@Override
public AttachmentProcessor doCreate(String processorTag, Map<String, Object> config) throws Exception {
- String sourceField = readStringProperty(TYPE, processorTag, config, "source_field");
+ String field = readStringProperty(TYPE, processorTag, config, "field");
String targetField = readStringProperty(TYPE, processorTag, config, "target_field", "attachment");
- List<String> fieldNames = readOptionalList(TYPE, processorTag, config, "fields");
+ List<String> properyNames = readOptionalList(TYPE, processorTag, config, "properties");
int indexedChars = readIntProperty(TYPE, processorTag, config, "indexed_chars", NUMBER_OF_CHARS_INDEXED);
- final Set<Field> fields;
- if (fieldNames != null) {
- fields = EnumSet.noneOf(Field.class);
- for (String fieldName : fieldNames) {
+ final Set<Property> properties;
+ if (properyNames != null) {
+ properties = EnumSet.noneOf(Property.class);
+ for (String fieldName : properyNames) {
try {
- fields.add(Field.parse(fieldName));
+ properties.add(Property.parse(fieldName));
} catch (Exception e) {
- throw newConfigurationException(TYPE, processorTag, "fields", "illegal field option [" +
- fieldName + "]. valid values are " + Arrays.toString(Field.values()));
+ throw newConfigurationException(TYPE, processorTag, "properties", "illegal field option [" +
+ fieldName + "]. valid values are " + Arrays.toString(Property.values()));
}
}
} else {
- fields = DEFAULT_FIELDS;
+ properties = DEFAULT_PROPERTIES;
}
- return new AttachmentProcessor(processorTag, sourceField, targetField, fields, indexedChars);
+ return new AttachmentProcessor(processorTag, field, targetField, properties, indexedChars);
}
}
- public enum Field {
+ enum Property {
CONTENT,
TITLE,
@@ -191,7 +191,7 @@ public final class AttachmentProcessor extends AbstractProcessor {
CONTENT_LENGTH,
LANGUAGE;
- public static Field parse(String value) {
+ public static Property parse(String value) {
return valueOf(value.toUpperCase(Locale.ROOT));
}
diff --git a/plugins/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/AttachmentProcessorFactoryTests.java b/plugins/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/AttachmentProcessorFactoryTests.java
index 7d48b217ab..09fb4e3951 100644
--- a/plugins/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/AttachmentProcessorFactoryTests.java
+++ b/plugins/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/AttachmentProcessorFactoryTests.java
@@ -43,22 +43,22 @@ public class AttachmentProcessorFactoryTests extends ESTestCase {
public void testBuildDefaults() throws Exception {
Map<String, Object> config = new HashMap<>();
- config.put("source_field", "_field");
+ config.put("field", "_field");
String processorTag = randomAsciiOfLength(10);
config.put(AbstractProcessorFactory.TAG_KEY, processorTag);
AttachmentProcessor processor = factory.create(config);
assertThat(processor.getTag(), equalTo(processorTag));
- assertThat(processor.getSourceField(), equalTo("_field"));
+ assertThat(processor.getField(), equalTo("_field"));
assertThat(processor.getTargetField(), equalTo("attachment"));
- assertThat(processor.getFields(), sameInstance(AttachmentProcessor.Factory.DEFAULT_FIELDS));
+ assertThat(processor.getProperties(), sameInstance(AttachmentProcessor.Factory.DEFAULT_PROPERTIES));
}
public void testConfigureIndexedChars() throws Exception {
int indexedChars = randomIntBetween(1, 100000);
Map<String, Object> config = new HashMap<>();
- config.put("source_field", "_field");
+ config.put("field", "_field");
config.put("indexed_chars", indexedChars);
String processorTag = randomAsciiOfLength(10);
@@ -70,53 +70,53 @@ public class AttachmentProcessorFactoryTests extends ESTestCase {
public void testBuildTargetField() throws Exception {
Map<String, Object> config = new HashMap<>();
- config.put("source_field", "_field");
+ config.put("field", "_field");
config.put("target_field", "_field");
AttachmentProcessor processor = factory.create(config);
- assertThat(processor.getSourceField(), equalTo("_field"));
+ assertThat(processor.getField(), equalTo("_field"));
assertThat(processor.getTargetField(), equalTo("_field"));
}
public void testBuildFields() throws Exception {
- Set<AttachmentProcessor.Field> fields = EnumSet.noneOf(AttachmentProcessor.Field.class);
+ Set<AttachmentProcessor.Property> properties = EnumSet.noneOf(AttachmentProcessor.Property.class);
List<String> fieldNames = new ArrayList<>();
- int numFields = scaledRandomIntBetween(1, AttachmentProcessor.Field.values().length);
+ int numFields = scaledRandomIntBetween(1, AttachmentProcessor.Property.values().length);
for (int i = 0; i < numFields; i++) {
- AttachmentProcessor.Field field = AttachmentProcessor.Field.values()[i];
- fields.add(field);
- fieldNames.add(field.name().toLowerCase(Locale.ROOT));
+ AttachmentProcessor.Property property = AttachmentProcessor.Property.values()[i];
+ properties.add(property);
+ fieldNames.add(property.name().toLowerCase(Locale.ROOT));
}
Map<String, Object> config = new HashMap<>();
- config.put("source_field", "_field");
- config.put("fields", fieldNames);
+ config.put("field", "_field");
+ config.put("properties", fieldNames);
AttachmentProcessor processor = factory.create(config);
- assertThat(processor.getSourceField(), equalTo("_field"));
- assertThat(processor.getFields(), equalTo(fields));
+ assertThat(processor.getField(), equalTo("_field"));
+ assertThat(processor.getProperties(), equalTo(properties));
}
public void testBuildIllegalFieldOption() throws Exception {
Map<String, Object> config = new HashMap<>();
- config.put("source_field", "_field");
- config.put("fields", Collections.singletonList("invalid"));
+ config.put("field", "_field");
+ config.put("properties", Collections.singletonList("invalid"));
try {
factory.create(config);
fail("exception expected");
} catch (ElasticsearchParseException e) {
- assertThat(e.getMessage(), containsString("[fields] illegal field option [invalid]"));
+ assertThat(e.getMessage(), containsString("[properties] illegal field option [invalid]"));
// ensure allowed fields are mentioned
- for (AttachmentProcessor.Field field : AttachmentProcessor.Field.values()) {
- assertThat(e.getMessage(), containsString(field.name()));
+ for (AttachmentProcessor.Property property : AttachmentProcessor.Property.values()) {
+ assertThat(e.getMessage(), containsString(property.name()));
}
}
config = new HashMap<>();
- config.put("source_field", "_field");
- config.put("fields", "invalid");
+ config.put("field", "_field");
+ config.put("properties", "invalid");
try {
factory.create(config);
fail("exception expected");
} catch (ElasticsearchParseException e) {
- assertThat(e.getMessage(), equalTo("[fields] property isn't a list, but of type [java.lang.String]"));
+ assertThat(e.getMessage(), equalTo("[properties] property isn't a list, but of type [java.lang.String]"));
}
}
}
diff --git a/plugins/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/AttachmentProcessorTests.java b/plugins/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/AttachmentProcessorTests.java
index 021971c707..945c5d3479 100644
--- a/plugins/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/AttachmentProcessorTests.java
+++ b/plugins/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/AttachmentProcessorTests.java
@@ -51,7 +51,7 @@ public class AttachmentProcessorTests extends ESTestCase {
@Before
public void createStandardProcessor() throws IOException {
processor = new AttachmentProcessor(randomAsciiOfLength(10), "source_field",
- "target_field", EnumSet.allOf(AttachmentProcessor.Field.class), 10000);
+ "target_field", EnumSet.allOf(AttachmentProcessor.Property.class), 10000);
}
public void testEnglishTextDocument() throws Exception {
@@ -66,25 +66,25 @@ public class AttachmentProcessorTests extends ESTestCase {
public void testHtmlDocumentWithRandomFields() throws Exception {
//date is not present in the html doc
- ArrayList<AttachmentProcessor.Field> fieldsList = new ArrayList<>(EnumSet.complementOf(EnumSet.of
- (AttachmentProcessor.Field.DATE)));
- Set<AttachmentProcessor.Field> selectedFields = new HashSet<>();
+ ArrayList<AttachmentProcessor.Property> fieldsList = new ArrayList<>(EnumSet.complementOf(EnumSet.of
+ (AttachmentProcessor.Property.DATE)));
+ Set<AttachmentProcessor.Property> selectedProperties = new HashSet<>();
int numFields = randomIntBetween(1, fieldsList.size());
String[] selectedFieldNames = new String[numFields];
for (int i = 0; i < numFields; i++) {
- AttachmentProcessor.Field field;
+ AttachmentProcessor.Property property;
do {
- field = randomFrom(fieldsList);
- } while (selectedFields.add(field) == false);
+ property = randomFrom(fieldsList);
+ } while (selectedProperties.add(property) == false);
- selectedFieldNames[i] = field.toLowerCase();
+ selectedFieldNames[i] = property.toLowerCase();
}
if (randomBoolean()) {
- selectedFields.add(AttachmentProcessor.Field.DATE);
+ selectedProperties.add(AttachmentProcessor.Property.DATE);
}
processor = new AttachmentProcessor(randomAsciiOfLength(10), "source_field",
- "target_field", selectedFields, 10000);
+ "target_field", selectedProperties, 10000);
Map<String, Object> attachmentData = parseDocument("htmlWithEmptyDateMeta.html", processor);
assertThat(attachmentData.keySet(), hasSize(selectedFieldNames.length));
diff --git a/plugins/ingest-attachment/src/test/resources/rest-api-spec/test/ingest_attachment/20_attachment_processor.yaml b/plugins/ingest-attachment/src/test/resources/rest-api-spec/test/ingest_attachment/20_attachment_processor.yaml
index 85c2f0d245..9c46610f10 100644
--- a/plugins/ingest-attachment/src/test/resources/rest-api-spec/test/ingest_attachment/20_attachment_processor.yaml
+++ b/plugins/ingest-attachment/src/test/resources/rest-api-spec/test/ingest_attachment/20_attachment_processor.yaml
@@ -9,7 +9,7 @@
"processors": [
{
"attachment" : {
- "source_field" : "field1"
+ "field" : "field1"
}
}
]
@@ -51,8 +51,8 @@
"processors": [
{
"attachment" : {
- "source_field" : "field1",
- "fields" : ["language"]
+ "field" : "field1",
+ "properties" : ["language"]
}
}
]
@@ -87,7 +87,7 @@
"processors": [
{
"attachment" : {
- "source_field" : "field1",
+ "field" : "field1",
"indexed_chars": 30
}
}
diff --git a/plugins/ingest-attachment/src/test/resources/rest-api-spec/test/ingest_attachment/30_files_supported.yaml b/plugins/ingest-attachment/src/test/resources/rest-api-spec/test/ingest_attachment/30_files_supported.yaml
index 4f56603a67..3b9183e930 100644
--- a/plugins/ingest-attachment/src/test/resources/rest-api-spec/test/ingest_attachment/30_files_supported.yaml
+++ b/plugins/ingest-attachment/src/test/resources/rest-api-spec/test/ingest_attachment/30_files_supported.yaml
@@ -9,7 +9,7 @@
"processors": [
{
"attachment" : {
- "source_field" : "field1"
+ "field" : "field1"
}
}
]
@@ -49,7 +49,7 @@
"processors": [
{
"attachment" : {
- "source_field" : "field1"
+ "field" : "field1"
}
}
]