1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
|
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.analysis;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.core.Lucene43StopFilter;
import org.apache.lucene.analysis.core.StopFilter;
import org.apache.lucene.analysis.core.WhitespaceTokenizer;
import org.apache.lucene.search.suggest.analyzing.SuggestStopFilter;
import org.apache.lucene.util.Version;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.settings.Settings.Builder;
import org.elasticsearch.env.Environment;
import org.elasticsearch.test.ESTokenStreamTestCase;
import java.io.IOException;
import java.io.StringReader;
import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.instanceOf;
public class StopTokenFilterTests extends ESTokenStreamTestCase {
public void testPositionIncrementSetting() throws IOException {
Builder builder = Settings.settingsBuilder().put("index.analysis.filter.my_stop.type", "stop")
.put("index.analysis.filter.my_stop.enable_position_increments", false);
if (random().nextBoolean()) {
builder.put("index.analysis.filter.my_stop.version", "5.0");
}
builder.put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString());
Settings settings = builder.build();
try {
AnalysisTestsHelper.createAnalysisServiceFromSettings(settings);
fail("Expected IllegalArgumentException");
} catch (IllegalArgumentException e) {
assertThat(e.getMessage(), containsString("enable_position_increments is not supported anymore"));
}
}
public void testCorrectPositionIncrementSetting() throws IOException {
Builder builder = Settings.settingsBuilder().put("index.analysis.filter.my_stop.type", "stop");
int thingToDo = random().nextInt(3);
if (thingToDo == 0) {
builder.put("index.analysis.filter.my_stop.version", Version.LATEST);
} else if (thingToDo == 1) {
builder.put("index.analysis.filter.my_stop.version", Version.LUCENE_4_0);
if (random().nextBoolean()) {
builder.put("index.analysis.filter.my_stop.enable_position_increments", true);
}
} else {
// don't specify
}
builder.put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString());
AnalysisService analysisService = AnalysisTestsHelper.createAnalysisServiceFromSettings(builder.build());
TokenFilterFactory tokenFilter = analysisService.tokenFilter("my_stop");
assertThat(tokenFilter, instanceOf(StopTokenFilterFactory.class));
Tokenizer tokenizer = new WhitespaceTokenizer();
tokenizer.setReader(new StringReader("foo bar"));
TokenStream create = tokenFilter.create(tokenizer);
if (thingToDo == 1) {
assertThat(create, instanceOf(Lucene43StopFilter.class));
} else {
assertThat(create, instanceOf(StopFilter.class));
}
}
public void testDeprecatedPositionIncrementSettingWithVersions() throws IOException {
Settings settings = Settings.settingsBuilder()
.put("index.analysis.filter.my_stop.type", "stop")
.put("index.analysis.filter.my_stop.enable_position_increments", false)
.put("index.analysis.filter.my_stop.version", "4.3")
.put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
.build();
AnalysisService analysisService = AnalysisTestsHelper.createAnalysisServiceFromSettings(settings);
TokenFilterFactory tokenFilter = analysisService.tokenFilter("my_stop");
assertThat(tokenFilter, instanceOf(StopTokenFilterFactory.class));
Tokenizer tokenizer = new WhitespaceTokenizer();
tokenizer.setReader(new StringReader("foo bar"));
TokenStream create = tokenFilter.create(tokenizer);
assertThat(create, instanceOf(Lucene43StopFilter.class));
}
public void testThatSuggestStopFilterWorks() throws Exception {
Settings settings = Settings.settingsBuilder()
.put("index.analysis.filter.my_stop.type", "stop")
.put("index.analysis.filter.my_stop.remove_trailing", false)
.put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
.build();
AnalysisService analysisService = AnalysisTestsHelper.createAnalysisServiceFromSettings(settings);
TokenFilterFactory tokenFilter = analysisService.tokenFilter("my_stop");
assertThat(tokenFilter, instanceOf(StopTokenFilterFactory.class));
Tokenizer tokenizer = new WhitespaceTokenizer();
tokenizer.setReader(new StringReader("foo an"));
TokenStream create = tokenFilter.create(tokenizer);
assertThat(create, instanceOf(SuggestStopFilter.class));
}
}
|
