diff options
42 files changed, 1135 insertions, 658 deletions
diff --git a/core/src/main/java/org/elasticsearch/action/suggest/TransportSuggestAction.java b/core/src/main/java/org/elasticsearch/action/suggest/TransportSuggestAction.java index 0ed9857855..616dbf9493 100644 --- a/core/src/main/java/org/elasticsearch/action/suggest/TransportSuggestAction.java +++ b/core/src/main/java/org/elasticsearch/action/suggest/TransportSuggestAction.java @@ -142,8 +142,7 @@ public class TransportSuggestAction extends TransportBroadcastAction<SuggestRequ if (parser.nextToken() != XContentParser.Token.START_OBJECT) { throw new IllegalArgumentException("suggest content missing"); } - final SuggestionSearchContext context = suggestPhase.parseElement().parseInternal(parser, indexService.mapperService(), - indexService.fieldData(), request.shardId().getIndexName(), request.shardId().id()); + final SuggestionSearchContext context = suggestPhase.parseElement().parseInternal(parser, indexService.newQueryShardContext()); final Suggest result = suggestPhase.execute(context, searcher.searcher()); return new ShardSuggestResponse(request.shardId(), result); } diff --git a/core/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java b/core/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java index d84328d68e..8c2dc444d2 100644 --- a/core/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java +++ b/core/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java @@ -39,7 +39,7 @@ import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.functionscore.ScoreFunctionBuilder; import org.elasticsearch.search.rescore.RescoreBuilder; import org.elasticsearch.search.suggest.SuggestionBuilder; -import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder.SmoothingModel; +import org.elasticsearch.search.suggest.phrase.SmoothingModel; import org.elasticsearch.tasks.Task; import org.joda.time.DateTime; import org.joda.time.DateTimeZone; diff --git a/core/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java b/core/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java index a7f1480623..d14d6e77ff 100644 --- a/core/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java +++ b/core/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java @@ -38,7 +38,7 @@ import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.functionscore.ScoreFunctionBuilder; import org.elasticsearch.search.rescore.RescoreBuilder; import org.elasticsearch.search.suggest.SuggestionBuilder; -import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder.SmoothingModel; +import org.elasticsearch.search.suggest.phrase.SmoothingModel; import org.elasticsearch.tasks.Task; import org.joda.time.ReadableInstant; diff --git a/core/src/main/java/org/elasticsearch/indices/query/IndicesQueriesRegistry.java b/core/src/main/java/org/elasticsearch/indices/query/IndicesQueriesRegistry.java index a9e90884a6..b0b212d2ab 100644 --- a/core/src/main/java/org/elasticsearch/indices/query/IndicesQueriesRegistry.java +++ b/core/src/main/java/org/elasticsearch/indices/query/IndicesQueriesRegistry.java @@ -19,12 +19,12 @@ package org.elasticsearch.indices.query; -import java.util.Map; - import org.elasticsearch.common.component.AbstractComponent; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.query.QueryParser; +import java.util.Map; + public class IndicesQueriesRegistry extends AbstractComponent { private Map<String, QueryParser<?>> queryParsers; diff --git a/core/src/main/java/org/elasticsearch/search/SearchService.java b/core/src/main/java/org/elasticsearch/search/SearchService.java index cf9c0cebce..ff6d8897d5 100644 --- a/core/src/main/java/org/elasticsearch/search/SearchService.java +++ b/core/src/main/java/org/elasticsearch/search/SearchService.java @@ -751,7 +751,7 @@ public class SearchService extends AbstractLifecycleComponent<SearchService> imp if (source.rescores() != null) { try { for (RescoreBuilder<?> rescore : source.rescores()) { - context.addRescore(rescore.build(context.getQueryShardContext())); + context.addRescore(rescore.build(queryShardContext)); } } catch (IOException e) { throw new SearchContextException(context, "failed to create RescoreSearchContext", e); @@ -776,7 +776,7 @@ public class SearchService extends AbstractLifecycleComponent<SearchService> imp if (source.highlighter() != null) { HighlightBuilder highlightBuilder = source.highlighter(); try { - context.highlight(highlightBuilder.build(context.getQueryShardContext())); + context.highlight(highlightBuilder.build(queryShardContext)); } catch (IOException e) { throw new SearchContextException(context, "failed to create SearchContextHighlighter", e); } diff --git a/core/src/main/java/org/elasticsearch/search/suggest/SuggestBuilder.java b/core/src/main/java/org/elasticsearch/search/suggest/SuggestBuilder.java index d16e8e1d84..2852204bb6 100644 --- a/core/src/main/java/org/elasticsearch/search/suggest/SuggestBuilder.java +++ b/core/src/main/java/org/elasticsearch/search/suggest/SuggestBuilder.java @@ -26,9 +26,12 @@ import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.lucene.BytesRefs; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.query.QueryParseContext; +import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.search.suggest.SuggestionSearchContext.SuggestionContext; import java.io.IOException; import java.util.ArrayList; @@ -137,6 +140,21 @@ public class SuggestBuilder extends ToXContentToBytes implements Writeable<Sugge return suggestBuilder; } + public SuggestionSearchContext build(QueryShardContext context) throws IOException { + SuggestionSearchContext suggestionSearchContext = new SuggestionSearchContext(); + for (SuggestionBuilder<?> suggestionBuilder : suggestions) { + SuggestionContext suggestionContext = suggestionBuilder.build(context); + if (suggestionContext.getText() == null) { + if (globalText == null) { + throw new IllegalArgumentException("The required text option is missing"); + } + suggestionContext.setText(BytesRefs.toBytesRef(globalText)); + } + suggestionSearchContext.addSuggestion(suggestionBuilder.name(), suggestionContext); + } + return suggestionSearchContext; + } + @Override public SuggestBuilder readFrom(StreamInput in) throws IOException { final SuggestBuilder builder = new SuggestBuilder(); diff --git a/core/src/main/java/org/elasticsearch/search/suggest/SuggestContextParser.java b/core/src/main/java/org/elasticsearch/search/suggest/SuggestContextParser.java index a7aa3fd60b..53d510bf53 100644 --- a/core/src/main/java/org/elasticsearch/search/suggest/SuggestContextParser.java +++ b/core/src/main/java/org/elasticsearch/search/suggest/SuggestContextParser.java @@ -19,12 +19,11 @@ package org.elasticsearch.search.suggest; import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.index.fielddata.IndexFieldDataService; -import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.index.query.QueryShardContext; import java.io.IOException; public interface SuggestContextParser { - SuggestionSearchContext.SuggestionContext parse(XContentParser parser, MapperService mapperService, IndexFieldDataService indexFieldDataService) throws IOException; + SuggestionSearchContext.SuggestionContext parse(XContentParser parser, QueryShardContext shardContext) throws IOException; } diff --git a/core/src/main/java/org/elasticsearch/search/suggest/SuggestParseElement.java b/core/src/main/java/org/elasticsearch/search/suggest/SuggestParseElement.java index cf6b391ec6..b9454dc264 100644 --- a/core/src/main/java/org/elasticsearch/search/suggest/SuggestParseElement.java +++ b/core/src/main/java/org/elasticsearch/search/suggest/SuggestParseElement.java @@ -21,8 +21,8 @@ package org.elasticsearch.search.suggest; import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.index.fielddata.IndexFieldDataService; import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.search.SearchParseElement; import org.elasticsearch.search.internal.SearchContext; import org.elasticsearch.search.suggest.SuggestionSearchContext.SuggestionContext; @@ -44,14 +44,13 @@ public final class SuggestParseElement implements SearchParseElement { @Override public void parse(XContentParser parser, SearchContext context) throws Exception { - SuggestionSearchContext suggestionSearchContext = parseInternal(parser, context.mapperService(), context.fieldData(), - context.shardTarget().index(), context.shardTarget().shardId()); + SuggestionSearchContext suggestionSearchContext = parseInternal(parser, context.getQueryShardContext()); context.suggest(suggestionSearchContext); } - public SuggestionSearchContext parseInternal(XContentParser parser, MapperService mapperService, IndexFieldDataService fieldDataService, - String index, int shardId) throws IOException { + public SuggestionSearchContext parseInternal(XContentParser parser, QueryShardContext shardContext) throws IOException { SuggestionSearchContext suggestionSearchContext = new SuggestionSearchContext(); + MapperService mapperService = shardContext.getMapperService(); BytesRef globalText = null; String fieldName = null; @@ -95,10 +94,20 @@ public final class SuggestParseElement implements SearchParseElement { throw new IllegalArgumentException("Suggester[" + fieldName + "] not supported"); } final SuggestContextParser contextParser = suggesters.get(fieldName).getContextParser(); - suggestionContext = contextParser.parse(parser, mapperService, fieldDataService); + suggestionContext = contextParser.parse(parser, shardContext); } } if (suggestionContext != null) { + if (suggestText != null) { + suggestionContext.setText(suggestText); + } + if (prefix != null) { + suggestionContext.setPrefix(prefix); + } + if (regex != null) { + suggestionContext.setRegex(regex); + } + if (suggestText != null && prefix == null) { suggestionContext.setPrefix(suggestText); suggestionContext.setText(suggestText); @@ -110,6 +119,8 @@ public final class SuggestParseElement implements SearchParseElement { suggestionContext.setText(regex); } suggestionContexts.put(suggestionName, suggestionContext); + } else { + throw new IllegalArgumentException("suggestion context could not be parsed correctly"); } } } @@ -117,9 +128,6 @@ public final class SuggestParseElement implements SearchParseElement { for (Map.Entry<String, SuggestionContext> entry : suggestionContexts.entrySet()) { String suggestionName = entry.getKey(); SuggestionContext suggestionContext = entry.getValue(); - - suggestionContext.setShard(shardId); - suggestionContext.setIndex(index); SuggestUtils.verifySuggestion(mapperService, globalText, suggestionContext); suggestionSearchContext.addSuggestion(suggestionName, suggestionContext); } diff --git a/core/src/main/java/org/elasticsearch/search/suggest/SuggestUtils.java b/core/src/main/java/org/elasticsearch/search/suggest/SuggestUtils.java index 989546d50b..03fb785b91 100644 --- a/core/src/main/java/org/elasticsearch/search/suggest/SuggestUtils.java +++ b/core/src/main/java/org/elasticsearch/search/suggest/SuggestUtils.java @@ -271,10 +271,10 @@ public final class SuggestUtils { return false; } return true; - } + public static void verifySuggestion(MapperService mapperService, BytesRef globalText, SuggestionContext suggestion) { // Verify options and set defaults if (suggestion.getField() == null) { @@ -294,7 +294,6 @@ public final class SuggestUtils { } } - public static ShingleTokenFilterFactory.Factory getShingleFilterFactory(Analyzer analyzer) { if (analyzer instanceof NamedAnalyzer) { analyzer = ((NamedAnalyzer)analyzer).analyzer(); diff --git a/core/src/main/java/org/elasticsearch/search/suggest/Suggesters.java b/core/src/main/java/org/elasticsearch/search/suggest/Suggesters.java index c26649f638..9857a06da6 100644 --- a/core/src/main/java/org/elasticsearch/search/suggest/Suggesters.java +++ b/core/src/main/java/org/elasticsearch/search/suggest/Suggesters.java @@ -20,8 +20,6 @@ package org.elasticsearch.search.suggest; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.util.ExtensionPoint; -import org.elasticsearch.indices.IndicesService; -import org.elasticsearch.script.ScriptService; import org.elasticsearch.search.suggest.completion.CompletionSuggester; import org.elasticsearch.search.suggest.phrase.PhraseSuggester; import org.elasticsearch.search.suggest.term.TermSuggester; @@ -42,21 +40,17 @@ public final class Suggesters extends ExtensionPoint.ClassMap<Suggester> { this(Collections.emptyMap()); } + @Inject public Suggesters(Map<String, Suggester> suggesters) { super("suggester", Suggester.class, new HashSet<>(Arrays.asList("phrase", "term", "completion")), Suggesters.class, SuggestParseElement.class, SuggestPhase.class); - this.parsers = Collections.unmodifiableMap(suggesters); - } - - @Inject - public Suggesters(Map<String, Suggester> suggesters, ScriptService scriptService, IndicesService indexServices) { - this(addBuildIns(suggesters, scriptService, indexServices)); + this.parsers = Collections.unmodifiableMap(addBuildIns(suggesters)); } - private static Map<String, Suggester> addBuildIns(Map<String, Suggester> suggesters, ScriptService scriptService, IndicesService indexServices) { + private static Map<String, Suggester> addBuildIns(Map<String, Suggester> suggesters) { final Map<String, Suggester> map = new HashMap<>(); - map.put("phrase", new PhraseSuggester(scriptService, indexServices)); - map.put("term", new TermSuggester()); - map.put("completion", new CompletionSuggester()); + map.put("phrase", PhraseSuggester.PROTOTYPE); + map.put("term", TermSuggester.PROTOTYPE); + map.put("completion", CompletionSuggester.PROTOTYPE); map.putAll(suggesters); return map; } diff --git a/core/src/main/java/org/elasticsearch/search/suggest/SuggestionBuilder.java b/core/src/main/java/org/elasticsearch/search/suggest/SuggestionBuilder.java index 1fdb38df88..70f3d061b4 100644 --- a/core/src/main/java/org/elasticsearch/search/suggest/SuggestionBuilder.java +++ b/core/src/main/java/org/elasticsearch/search/suggest/SuggestionBuilder.java @@ -19,15 +19,20 @@ package org.elasticsearch.search.suggest; +import org.apache.lucene.analysis.Analyzer; import org.elasticsearch.action.support.ToXContentToBytes; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.ParseFieldMatcher; import org.elasticsearch.common.io.stream.NamedWriteable; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.lucene.BytesRefs; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.query.QueryParseContext; +import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.search.suggest.SuggestionSearchContext.SuggestionContext; import java.io.IOException; import java.util.Objects; @@ -192,6 +197,56 @@ public abstract class SuggestionBuilder<T extends SuggestionBuilder<T>> extends protected abstract SuggestionBuilder<T> innerFromXContent(QueryParseContext parseContext, String name) throws IOException; + protected abstract SuggestionContext build(QueryShardContext context) throws IOException; + + /** + * Transfers the text, prefix, regex, analyzer, fieldname, size and shard size settings from the + * original {@link SuggestionBuilder} to the target {@link SuggestionContext} + */ + protected void populateCommonFields(MapperService mapperService, + SuggestionSearchContext.SuggestionContext suggestionContext) throws IOException { + + if (analyzer != null) { + Analyzer luceneAnalyzer = mapperService.analysisService().analyzer(analyzer); + if (luceneAnalyzer == null) { + throw new IllegalArgumentException("Analyzer [" + luceneAnalyzer + "] doesn't exists"); + } + suggestionContext.setAnalyzer(luceneAnalyzer); + } + + if (fieldname != null) { + suggestionContext.setField(fieldname); + } + + if (size != null) { + suggestionContext.setSize(size); + } + + if (shardSize != null) { + suggestionContext.setShardSize(shardSize); + } else { + // if no shard size is set in builder, use size (or at least 5) + suggestionContext.setShardSize(Math.max(suggestionContext.getSize(), 5)); + } + + if (text != null) { + suggestionContext.setText(BytesRefs.toBytesRef(text)); + } + if (prefix != null) { + suggestionContext.setPrefix(BytesRefs.toBytesRef(prefix)); + } + if (regex != null) { + suggestionContext.setRegex(BytesRefs.toBytesRef(regex)); + } + if (text != null && prefix == null) { + suggestionContext.setPrefix(BytesRefs.toBytesRef(text)); + } else if (text == null && prefix != null) { + suggestionContext.setText(BytesRefs.toBytesRef(prefix)); + } else if (text == null && regex != null) { + suggestionContext.setText(BytesRefs.toBytesRef(regex)); + } + } + private String getSuggesterName() { //default impl returns the same as writeable name, but we keep the distinction between the two just to make sure return getWriteableName(); diff --git a/core/src/main/java/org/elasticsearch/search/suggest/SuggestionSearchContext.java b/core/src/main/java/org/elasticsearch/search/suggest/SuggestionSearchContext.java index 1d3339e057..fa468e6ce9 100644 --- a/core/src/main/java/org/elasticsearch/search/suggest/SuggestionSearchContext.java +++ b/core/src/main/java/org/elasticsearch/search/suggest/SuggestionSearchContext.java @@ -20,6 +20,7 @@ package org.elasticsearch.search.suggest; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.index.query.QueryShardContext; import java.util.LinkedHashMap; import java.util.Map; @@ -36,20 +37,24 @@ public class SuggestionSearchContext { public Map<String, SuggestionContext> suggestions() { return suggestions; } - - public static class SuggestionContext { - + + public abstract static class SuggestionContext { + private BytesRef text; private BytesRef prefix; private BytesRef regex; - private final Suggester suggester; private String field; private Analyzer analyzer; private int size = 5; private int shardSize = -1; - private int shardId; - private String index; - + private QueryShardContext shardContext; + private Suggester<?> suggester; + + protected SuggestionContext(Suggester<?> suggester, QueryShardContext shardContext) { + this.suggester = suggester; + this.shardContext = shardContext; + } + public BytesRef getText() { return text; } @@ -74,12 +79,8 @@ public class SuggestionSearchContext { this.regex = regex; } - public SuggestionContext(Suggester suggester) { - this.suggester = suggester; - } - public Suggester<SuggestionContext> getSuggester() { - return this.suggester; + return ((Suggester<SuggestionContext>) suggester); } public Analyzer getAnalyzer() { @@ -119,21 +120,9 @@ public class SuggestionSearchContext { } this.shardSize = shardSize; } - - public void setShard(int shardId) { - this.shardId = shardId; - } - public void setIndex(String index) { - this.index = index; - } - - public String getIndex() { - return index; - } - - public int getShard() { - return shardId; + public QueryShardContext getShardContext() { + return this.shardContext; } } diff --git a/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestParser.java b/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestParser.java index 702b03f359..9d29525115 100644 --- a/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestParser.java +++ b/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestParser.java @@ -20,17 +20,16 @@ package org.elasticsearch.search.suggest.completion; import org.apache.lucene.analysis.Analyzer; import org.elasticsearch.ElasticsearchException; -import org.elasticsearch.common.ParseField; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.unit.Fuzziness; import org.elasticsearch.common.xcontent.ObjectParser; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.index.fielddata.IndexFieldDataService; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.core.CompletionFieldMapper; +import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.index.query.RegexpFlag; import org.elasticsearch.search.suggest.SuggestContextParser; import org.elasticsearch.search.suggest.SuggestUtils.Fields; @@ -135,8 +134,9 @@ public class CompletionSuggestParser implements SuggestContextParser { } @Override - public SuggestionSearchContext.SuggestionContext parse(XContentParser parser, MapperService mapperService, IndexFieldDataService fieldDataService) throws IOException { - final CompletionSuggestionContext suggestion = new CompletionSuggestionContext(completionSuggester, mapperService, fieldDataService); + public SuggestionSearchContext.SuggestionContext parse(XContentParser parser, QueryShardContext shardContext) throws IOException { + MapperService mapperService = shardContext.getMapperService(); + final CompletionSuggestionContext suggestion = new CompletionSuggestionContext(shardContext); final ContextAndSuggest contextAndSuggest = new ContextAndSuggest(mapperService); TLP_PARSER.parse(parser, suggestion, contextAndSuggest); final XContentParser contextParser = contextAndSuggest.contextParser; diff --git a/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggester.java b/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggester.java index 8cd9d386a1..be90a2e7e7 100644 --- a/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggester.java +++ b/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggester.java @@ -34,7 +34,9 @@ import org.elasticsearch.common.text.Text; import org.elasticsearch.index.fielddata.AtomicFieldData; import org.elasticsearch.index.fielddata.ScriptDocValues; import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.core.CompletionFieldMapper; +import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.search.suggest.Suggest; import org.elasticsearch.search.suggest.SuggestContextParser; import org.elasticsearch.search.suggest.Suggester; @@ -51,6 +53,8 @@ import java.util.Set; public class CompletionSuggester extends Suggester<CompletionSuggestionContext> { + public static final CompletionSuggester PROTOTYPE = new CompletionSuggester(); + @Override public SuggestContextParser getContextParser() { return new CompletionSuggestParser(this); @@ -86,9 +90,11 @@ public class CompletionSuggester extends Suggester<CompletionSuggestionContext> final LeafReaderContext subReaderContext = leaves.get(readerIndex); final int subDocId = suggestDoc.doc - subReaderContext.docBase; for (String field : payloadFields) { - MappedFieldType payloadFieldType = suggestionContext.getMapperService().fullName(field); + MapperService mapperService = suggestionContext.getShardContext().getMapperService(); + MappedFieldType payloadFieldType = mapperService.fullName(field); if (payloadFieldType != null) { - final AtomicFieldData data = suggestionContext.getIndexFieldDataService().getForField(payloadFieldType) + QueryShardContext shardContext = suggestionContext.getShardContext(); + final AtomicFieldData data = shardContext.getForField(payloadFieldType) .load(subReaderContext); final ScriptDocValues scriptValues = data.getScriptValues(); scriptValues.setNextDocId(subDocId); diff --git a/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestionBuilder.java b/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestionBuilder.java index 29992c1a07..0bd37be128 100644 --- a/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestionBuilder.java +++ b/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestionBuilder.java @@ -28,8 +28,10 @@ import org.elasticsearch.common.unit.Fuzziness; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.index.query.QueryParseContext; +import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.index.query.RegexpFlag; import org.elasticsearch.search.suggest.SuggestionBuilder; +import org.elasticsearch.search.suggest.SuggestionSearchContext.SuggestionContext; import org.elasticsearch.search.suggest.completion.context.CategoryQueryContext; import org.elasticsearch.search.suggest.completion.context.GeoQueryContext; @@ -372,10 +374,17 @@ public class CompletionSuggestionBuilder extends SuggestionBuilder<CompletionSug @Override protected CompletionSuggestionBuilder innerFromXContent(QueryParseContext parseContext, String name) throws IOException { + // NORELEASE return new CompletionSuggestionBuilder(name); } @Override + protected SuggestionContext build(QueryShardContext context) throws IOException { + // NORELEASE + throw new UnsupportedOperationException(); + } + + @Override public String getWriteableName() { return SUGGESTION_NAME; } diff --git a/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestionContext.java b/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestionContext.java index 535151b476..f6d6de88f4 100644 --- a/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestionContext.java +++ b/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestionContext.java @@ -20,10 +20,8 @@ package org.elasticsearch.search.suggest.completion; import org.apache.lucene.search.suggest.document.CompletionQuery; import org.elasticsearch.common.unit.Fuzziness; -import org.elasticsearch.index.fielddata.IndexFieldDataService; -import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.core.CompletionFieldMapper; -import org.elasticsearch.search.suggest.Suggester; +import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.search.suggest.SuggestionSearchContext; import org.elasticsearch.search.suggest.completion.context.ContextMapping; import org.elasticsearch.search.suggest.completion.context.ContextMappings; @@ -39,20 +37,16 @@ import java.util.Set; */ public class CompletionSuggestionContext extends SuggestionSearchContext.SuggestionContext { + protected CompletionSuggestionContext(QueryShardContext shardContext) { + super(CompletionSuggester.PROTOTYPE, shardContext); + } + private CompletionFieldMapper.CompletionFieldType fieldType; private CompletionSuggestionBuilder.FuzzyOptionsBuilder fuzzyOptionsBuilder; private CompletionSuggestionBuilder.RegexOptionsBuilder regexOptionsBuilder; private Map<String, List<ContextMapping.QueryContext>> queryContexts = Collections.emptyMap(); - private final MapperService mapperService; - private final IndexFieldDataService indexFieldDataService; private Set<String> payloadFields = Collections.emptySet(); - CompletionSuggestionContext(Suggester suggester, MapperService mapperService, IndexFieldDataService indexFieldDataService) { - super(suggester); - this.indexFieldDataService = indexFieldDataService; - this.mapperService = mapperService; - } - CompletionFieldMapper.CompletionFieldType getFieldType() { return this.fieldType; } @@ -73,15 +67,6 @@ public class CompletionSuggestionContext extends SuggestionSearchContext.Suggest this.queryContexts = queryContexts; } - - MapperService getMapperService() { - return mapperService; - } - - IndexFieldDataService getIndexFieldDataService() { - return indexFieldDataService; - } - void setPayloadFields(Set<String> fields) { this.payloadFields = fields; } diff --git a/core/src/main/java/org/elasticsearch/search/suggest/phrase/DirectCandidateGeneratorBuilder.java b/core/src/main/java/org/elasticsearch/search/suggest/phrase/DirectCandidateGeneratorBuilder.java index 8cc834ef0d..dd1be571bd 100644 --- a/core/src/main/java/org/elasticsearch/search/suggest/phrase/DirectCandidateGeneratorBuilder.java +++ b/core/src/main/java/org/elasticsearch/search/suggest/phrase/DirectCandidateGeneratorBuilder.java @@ -30,7 +30,6 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.query.QueryParseContext; -import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.search.suggest.SuggestUtils; import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder.CandidateGenerator; @@ -349,8 +348,7 @@ public final class DirectCandidateGeneratorBuilder return replaceField(tmpFieldName.iterator().next(), tempGenerator); } - public PhraseSuggestionContext.DirectCandidateGenerator build(QueryShardContext context) throws IOException { - MapperService mapperService = context.getMapperService(); + public PhraseSuggestionContext.DirectCandidateGenerator build(MapperService mapperService) throws IOException { PhraseSuggestionContext.DirectCandidateGenerator generator = new PhraseSuggestionContext.DirectCandidateGenerator(); generator.setField(this.field); transferIfNotNull(this.size, generator::size); diff --git a/core/src/main/java/org/elasticsearch/search/suggest/phrase/Laplace.java b/core/src/main/java/org/elasticsearch/search/suggest/phrase/Laplace.java new file mode 100644 index 0000000000..e11a920f96 --- /dev/null +++ b/core/src/main/java/org/elasticsearch/search/suggest/phrase/Laplace.java @@ -0,0 +1,126 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.suggest.phrase; + +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.Terms; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentParser.Token; +import org.elasticsearch.index.query.QueryParseContext; +import org.elasticsearch.search.suggest.phrase.WordScorer.WordScorerFactory; + +import java.io.IOException; +import java.util.Objects; + +/** + * An <a href="http://en.wikipedia.org/wiki/Additive_smoothing">additive + * smoothing</a> model. + * <p> + * See <a + * href="http://en.wikipedia.org/wiki/N-gram#Smoothing_techniques">N-Gram + * Smoothing</a> for details. + * </p> + */ +public final class Laplace extends SmoothingModel { + private double alpha = DEFAULT_LAPLACE_ALPHA; + private static final String NAME = "laplace"; + private static final ParseField ALPHA_FIELD = new ParseField("alpha"); + static final ParseField PARSE_FIELD = new ParseField(NAME); + /** + * Default alpha parameter for laplace smoothing + */ + public static final double DEFAULT_LAPLACE_ALPHA = 0.5; + public static final Laplace PROTOTYPE = new Laplace(DEFAULT_LAPLACE_ALPHA); + + /** + * Creates a Laplace smoothing model. + * + */ + public Laplace(double alpha) { + this.alpha = alpha; + } + + /** + * @return the laplace model alpha parameter + */ + public double getAlpha() { + return this.alpha; + } + + @Override + protected XContentBuilder innerToXContent(XContentBuilder builder, Params params) throws IOException { + builder.field(ALPHA_FIELD.getPreferredName(), alpha); + return builder; + } + + @Override + public String getWriteableName() { + return NAME; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeDouble(alpha); + } + + @Override + public SmoothingModel readFrom(StreamInput in) throws IOException { + return new Laplace(in.readDouble()); + } + + @Override + protected boolean doEquals(SmoothingModel other) { + Laplace otherModel = (Laplace) other; + return Objects.equals(alpha, otherModel.alpha); + } + + @Override + protected final int doHashCode() { + return Objects.hash(alpha); + } + + @Override + public SmoothingModel innerFromXContent(QueryParseContext parseContext) throws IOException { + XContentParser parser = parseContext.parser(); + XContentParser.Token token; + String fieldName = null; + double alpha = DEFAULT_LAPLACE_ALPHA; + while ((token = parser.nextToken()) != Token.END_OBJECT) { + if (token == XContentParser.Token.FIELD_NAME) { + fieldName = parser.currentName(); + } + if (token.isValue() && parseContext.parseFieldMatcher().match(fieldName, ALPHA_FIELD)) { + alpha = parser.doubleValue(); + } + } + return new Laplace(alpha); + } + + @Override + public WordScorerFactory buildWordScorerFactory() { + return (IndexReader reader, Terms terms, String field, double realWordLikelyhood, BytesRef separator) + -> new LaplaceScorer(reader, terms, field, realWordLikelyhood, separator, alpha); + } +}
\ No newline at end of file diff --git a/core/src/main/java/org/elasticsearch/search/suggest/phrase/LinearInterpolation.java b/core/src/main/java/org/elasticsearch/search/suggest/phrase/LinearInterpolation.java new file mode 100644 index 0000000000..b94ea333fd --- /dev/null +++ b/core/src/main/java/org/elasticsearch/search/suggest/phrase/LinearInterpolation.java @@ -0,0 +1,176 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.suggest.phrase; + +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.Terms; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.ParseFieldMatcher; +import org.elasticsearch.common.ParsingException; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentParser.Token; +import org.elasticsearch.index.query.QueryParseContext; +import org.elasticsearch.search.suggest.phrase.WordScorer.WordScorerFactory; + +import java.io.IOException; +import java.util.Objects; + +/** + * Linear interpolation smoothing model. + * <p> + * See <a + * href="http://en.wikipedia.org/wiki/N-gram#Smoothing_techniques">N-Gram + * Smoothing</a> for details. + * </p> + */ +public final class LinearInterpolation extends SmoothingModel { + private static final String NAME = "linear"; + public static final LinearInterpolation PROTOTYPE = new LinearInterpolation(0.8, 0.1, 0.1); + private final double trigramLambda; + private final double bigramLambda; + private final double unigramLambda; + static final ParseField PARSE_FIELD = new ParseField(NAME); + private static final ParseField TRIGRAM_FIELD = new ParseField("trigram_lambda"); + private static final ParseField BIGRAM_FIELD = new ParseField("bigram_lambda"); + private static final ParseField UNIGRAM_FIELD = new ParseField("unigram_lambda"); + + /** + * Creates a linear interpolation smoothing model. + * + * Note: the lambdas must sum up to one. + * + * @param trigramLambda + * the trigram lambda + * @param bigramLambda + * the bigram lambda + * @param unigramLambda + * the unigram lambda + */ + public LinearInterpolation(double trigramLambda, double bigramLambda, double unigramLambda) { + double sum = trigramLambda + bigramLambda + unigramLambda; + if (Math.abs(sum - 1.0) > 0.001) { + throw new IllegalArgumentException("linear smoothing lambdas must sum to 1"); + } + this.trigramLambda = trigramLambda; + this.bigramLambda = bigramLambda; + this.unigramLambda = unigramLambda; + } + + public double getTrigramLambda() { + return this.trigramLambda; + } + + public double getBigramLambda() { + return this.bigramLambda; + } + + public double getUnigramLambda() { + return this.unigramLambda; + } + + @Override + protected XContentBuilder innerToXContent(XContentBuilder builder, Params params) throws IOException { + builder.field(TRIGRAM_FIELD.getPreferredName(), trigramLambda); + builder.field(BIGRAM_FIELD.getPreferredName(), bigramLambda); + builder.field(UNIGRAM_FIELD.getPreferredName(), unigramLambda); + return builder; + } + + @Override + public String getWriteableName() { + return NAME; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeDouble(trigramLambda); + out.writeDouble(bigramLambda); + out.writeDouble(unigramLambda); + } + + @Override + public LinearInterpolation readFrom(StreamInput in) throws IOException { + return new LinearInterpolation(in.readDouble(), in.readDouble(), in.readDouble()); + } + + @Override + protected boolean doEquals(SmoothingModel other) { + final LinearInterpolation otherModel = (LinearInterpolation) other; + return Objects.equals(trigramLambda, otherModel.trigramLambda) && + Objects.equals(bigramLambda, otherModel.bigramLambda) && + Objects.equals(unigramLambda, otherModel.unigramLambda); + } + + @Override + protected final int doHashCode() { + return Objects.hash(trigramLambda, bigramLambda, unigramLambda); + } + + @Override + public LinearInterpolation innerFromXContent(QueryParseContext parseContext) throws IOException { + XContentParser parser = parseContext.parser(); + XContentParser.Token token; + String fieldName = null; + double trigramLambda = 0.0; + double bigramLambda = 0.0; + double unigramLambda = 0.0; + ParseFieldMatcher matcher = parseContext.parseFieldMatcher(); + while ((token = parser.nextToken()) != Token.END_OBJECT) { + if (token == XContentParser.Token.FIELD_NAME) { + fieldName = parser.currentName(); + } else if (token.isValue()) { + if (matcher.match(fieldName, TRIGRAM_FIELD)) { + trigramLambda = parser.doubleValue(); + if (trigramLambda < 0) { + throw new IllegalArgumentException("trigram_lambda must be positive"); + } + } else if (matcher.match(fieldName, BIGRAM_FIELD)) { + bigramLambda = parser.doubleValue(); + if (bigramLambda < 0) { + throw new IllegalArgumentException("bigram_lambda must be positive"); + } + } else if (matcher.match(fieldName, UNIGRAM_FIELD)) { + unigramLambda = parser.doubleValue(); + if (unigramLambda < 0) { + throw new IllegalArgumentException("unigram_lambda must be positive"); + } + } else { + throw new IllegalArgumentException( + "suggester[phrase][smoothing][linear] doesn't support field [" + fieldName + "]"); + } + } else { + throw new ParsingException(parser.getTokenLocation(), + "[" + NAME + "] unknown token [" + token + "] after [" + fieldName + "]"); + } + } + return new LinearInterpolation(trigramLambda, bigramLambda, unigramLambda); + } + + @Override + public WordScorerFactory buildWordScorerFactory() { + return (IndexReader reader, Terms terms, String field, double realWordLikelyhood, BytesRef separator) -> + new LinearInterpoatingScorer(reader, terms, field, realWordLikelyhood, separator, trigramLambda, bigramLambda, + unigramLambda); + } +}
\ No newline at end of file diff --git a/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestParser.java b/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestParser.java index fc60fc6fc8..e4400fb5cd 100644 --- a/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestParser.java +++ b/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestParser.java @@ -26,17 +26,16 @@ import org.elasticsearch.common.ParseFieldMatcher; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentParser.Token; import org.elasticsearch.index.analysis.ShingleTokenFilterFactory; -import org.elasticsearch.index.fielddata.IndexFieldDataService; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.script.CompiledScript; import org.elasticsearch.script.ScriptContext; +import org.elasticsearch.script.ScriptService; import org.elasticsearch.script.Template; import org.elasticsearch.search.suggest.SuggestContextParser; import org.elasticsearch.search.suggest.SuggestUtils; import org.elasticsearch.search.suggest.SuggestionSearchContext; -import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder.Laplace; -import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder.StupidBackoff; import org.elasticsearch.search.suggest.phrase.PhraseSuggestionContext.DirectCandidateGenerator; import java.io.IOException; @@ -51,8 +50,10 @@ public final class PhraseSuggestParser implements SuggestContextParser { } @Override - public SuggestionSearchContext.SuggestionContext parse(XContentParser parser, MapperService mapperService, IndexFieldDataService fieldDataService) throws IOException { - PhraseSuggestionContext suggestion = new PhraseSuggestionContext(suggester); + public SuggestionSearchContext.SuggestionContext parse(XContentParser parser, QueryShardContext shardContext) throws IOException { + MapperService mapperService = shardContext.getMapperService(); + ScriptService scriptService = shardContext.getScriptService(); + PhraseSuggestionContext suggestion = new PhraseSuggestionContext(shardContext); ParseFieldMatcher parseFieldMatcher = mapperService.getIndexSettings().getParseFieldMatcher(); XContentParser.Token token; String fieldName = null; @@ -135,7 +136,7 @@ public final class PhraseSuggestParser implements SuggestContextParser { throw new IllegalArgumentException("suggester[phrase][collate] query already set, doesn't support additional [" + fieldName + "]"); } Template template = Template.parse(parser, parseFieldMatcher); - CompiledScript compiledScript = suggester.scriptService().compile(template, ScriptContext.Standard.SEARCH, Collections.emptyMap()); + CompiledScript compiledScript = scriptService.compile(template, ScriptContext.Standard.SEARCH, Collections.emptyMap()); suggestion.setCollateQueryScript(compiledScript); } else if ("params".equals(fieldName)) { suggestion.setCollateScriptParams(parser.map()); @@ -199,9 +200,6 @@ public final class PhraseSuggestParser implements SuggestContextParser { suggestion.addGenerator(generator); } } - - - return suggestion; } diff --git a/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggester.java b/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggester.java index fbfa2b03ce..8f3e5164e4 100644 --- a/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggester.java +++ b/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggester.java @@ -31,9 +31,7 @@ import org.apache.lucene.util.CharsRefBuilder; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.text.Text; -import org.elasticsearch.index.IndexService; import org.elasticsearch.index.query.ParsedQuery; -import org.elasticsearch.indices.IndicesService; import org.elasticsearch.script.CompiledScript; import org.elasticsearch.script.ExecutableScript; import org.elasticsearch.script.ScriptService; @@ -44,6 +42,7 @@ import org.elasticsearch.search.suggest.SuggestContextParser; import org.elasticsearch.search.suggest.SuggestUtils; import org.elasticsearch.search.suggest.Suggester; import org.elasticsearch.search.suggest.SuggestionBuilder; +import org.elasticsearch.search.suggest.SuggestionSearchContext.SuggestionContext; import org.elasticsearch.search.suggest.phrase.NoisyChannelSpellChecker.Result; import java.io.IOException; @@ -54,13 +53,8 @@ import java.util.Map; public final class PhraseSuggester extends Suggester<PhraseSuggestionContext> { private final BytesRef SEPARATOR = new BytesRef(" "); private static final String SUGGESTION_TEMPLATE_VAR_NAME = "suggestion"; - private final ScriptService scriptService; - private final IndicesService indicesService; - public PhraseSuggester(ScriptService scriptService, IndicesService indicesService) { - this.scriptService = scriptService; - this.indicesService = indicesService; - } + public static final PhraseSuggester PROTOTYPE = new PhraseSuggester(); /* * More Ideas: @@ -118,10 +112,10 @@ public final class PhraseSuggester extends Suggester<PhraseSuggestionContext> { // from the index for a correction, collateMatch is updated final Map<String, Object> vars = suggestion.getCollateScriptParams(); vars.put(SUGGESTION_TEMPLATE_VAR_NAME, spare.toString()); + ScriptService scriptService = suggestion.getShardContext().getScriptService(); final ExecutableScript executable = scriptService.executable(collateScript, vars); final BytesReference querySource = (BytesReference) executable.run(); - IndexService indexService = indicesService.indexService(suggestion.getIndex()); - final ParsedQuery parsedQuery = indexService.newQueryShardContext().parse(querySource); + final ParsedQuery parsedQuery = suggestion.getShardContext().parse(querySource); collateMatch = Lucene.exists(searcher, parsedQuery.query()); } if (!collateMatch && !collatePrune) { @@ -145,15 +139,11 @@ public final class PhraseSuggester extends Suggester<PhraseSuggestionContext> { return response; } - private PhraseSuggestion.Entry buildResultEntry(PhraseSuggestionContext suggestion, CharsRefBuilder spare, double cutoffScore) { + private PhraseSuggestion.Entry buildResultEntry(SuggestionContext suggestion, CharsRefBuilder spare, double cutoffScore) { spare.copyUTF8Bytes(suggestion.getText()); return new PhraseSuggestion.Entry(new Text(spare.toString()), 0, spare.length(), cutoffScore); } - ScriptService scriptService() { - return scriptService; - } - @Override public SuggestContextParser getContextParser() { return new PhraseSuggestParser(this); diff --git a/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestionBuilder.java b/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestionBuilder.java index c079812afe..c83f68716b 100644 --- a/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestionBuilder.java +++ b/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestionBuilder.java @@ -19,27 +19,32 @@ package org.elasticsearch.search.suggest.phrase; -import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.Terms; -import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.ParseFieldMatcher; -import org.elasticsearch.common.ParsingException; -import org.elasticsearch.common.io.stream.NamedWriteable; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.lucene.BytesRefs; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentParser.Token; +import org.elasticsearch.index.analysis.ShingleTokenFilterFactory; +import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.query.QueryParseContext; +import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.script.CompiledScript; +import org.elasticsearch.script.ScriptContext; import org.elasticsearch.script.Template; +import org.elasticsearch.search.suggest.SuggestUtils; import org.elasticsearch.search.suggest.SuggestionBuilder; -import org.elasticsearch.search.suggest.phrase.WordScorer.WordScorerFactory; +import org.elasticsearch.search.suggest.SuggestionSearchContext.SuggestionContext; +import org.elasticsearch.search.suggest.phrase.PhraseSuggestionContext.DirectCandidateGenerator; import java.io.IOException; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -239,7 +244,7 @@ public final class PhraseSuggestionBuilder extends SuggestionBuilder<PhraseSugge /** * Sets an explicit smoothing model used for this suggester. The default is - * {@link PhraseSuggestionBuilder.StupidBackoff}. + * {@link StupidBackoff}. */ public PhraseSuggestionBuilder smoothingModel(SmoothingModel model) { this.model = model; @@ -254,6 +259,9 @@ public final class PhraseSuggestionBuilder extends SuggestionBuilder<PhraseSugge } public PhraseSuggestionBuilder tokenLimit(int tokenLimit) { + if (tokenLimit <= 0) { + throw new IllegalArgumentException("token_limit must be >= 1"); + } this.tokenLimit = tokenLimit; return this; } @@ -389,413 +397,6 @@ public final class PhraseSuggestionBuilder extends SuggestionBuilder<PhraseSugge return builder; } - /** - * Creates a new {@link DirectCandidateGeneratorBuilder} - * - * @param field - * the field this candidate generator operates on. - */ - public static DirectCandidateGeneratorBuilder candidateGenerator(String field) { - return new DirectCandidateGeneratorBuilder(field); - } - - /** - * A "stupid-backoff" smoothing model simialr to <a - * href="http://en.wikipedia.org/wiki/Katz's_back-off_model"> Katz's - * Backoff</a>. This model is used as the default if no model is configured. - * <p> - * See <a - * href="http://en.wikipedia.org/wiki/N-gram#Smoothing_techniques">N-Gram - * Smoothing</a> for details. - * </p> - */ - public static final class StupidBackoff extends SmoothingModel { - /** - * Default discount parameter for {@link StupidBackoff} smoothing - */ - public static final double DEFAULT_BACKOFF_DISCOUNT = 0.4; - public static final StupidBackoff PROTOTYPE = new StupidBackoff(DEFAULT_BACKOFF_DISCOUNT); - private double discount = DEFAULT_BACKOFF_DISCOUNT; - private static final String NAME = "stupid_backoff"; - private static final ParseField DISCOUNT_FIELD = new ParseField("discount"); - private static final ParseField PARSE_FIELD = new ParseField(NAME); - - /** - * Creates a Stupid-Backoff smoothing model. - * - * @param discount - * the discount given to lower order ngrams if the higher order ngram doesn't exits - */ - public StupidBackoff(double discount) { - this.discount = discount; - } - - /** - * @return the discount parameter of the model - */ - public double getDiscount() { - return this.discount; - } - - @Override - protected XContentBuilder innerToXContent(XContentBuilder builder, Params params) throws IOException { - builder.field(DISCOUNT_FIELD.getPreferredName(), discount); - return builder; - } - - @Override - public String getWriteableName() { - return NAME; - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - out.writeDouble(discount); - } - - @Override - public StupidBackoff readFrom(StreamInput in) throws IOException { - return new StupidBackoff(in.readDouble()); - } - - @Override - protected boolean doEquals(SmoothingModel other) { - StupidBackoff otherModel = (StupidBackoff) other; - return Objects.equals(discount, otherModel.discount); - } - - @Override - protected final int doHashCode() { - return Objects.hash(discount); - } - - @Override - public SmoothingModel innerFromXContent(QueryParseContext parseContext) throws IOException { - XContentParser parser = parseContext.parser(); - XContentParser.Token token; - String fieldName = null; - double discount = DEFAULT_BACKOFF_DISCOUNT; - while ((token = parser.nextToken()) != Token.END_OBJECT) { - if (token == XContentParser.Token.FIELD_NAME) { - fieldName = parser.currentName(); - } - if (token.isValue() && parseContext.parseFieldMatcher().match(fieldName, DISCOUNT_FIELD)) { - discount = parser.doubleValue(); - } - } - return new StupidBackoff(discount); - } - - @Override - public WordScorerFactory buildWordScorerFactory() { - return (IndexReader reader, Terms terms, String field, double realWordLikelyhood, BytesRef separator) - -> new StupidBackoffScorer(reader, terms, field, realWordLikelyhood, separator, discount); - } - } - - /** - * An <a href="http://en.wikipedia.org/wiki/Additive_smoothing">additive - * smoothing</a> model. - * <p> - * See <a - * href="http://en.wikipedia.org/wiki/N-gram#Smoothing_techniques">N-Gram - * Smoothing</a> for details. - * </p> - */ - public static final class Laplace extends SmoothingModel { - private double alpha = DEFAULT_LAPLACE_ALPHA; - private static final String NAME = "laplace"; - private static final ParseField ALPHA_FIELD = new ParseField("alpha"); - private static final ParseField PARSE_FIELD = new ParseField(NAME); - /** - * Default alpha parameter for laplace smoothing - */ - public static final double DEFAULT_LAPLACE_ALPHA = 0.5; - public static final Laplace PROTOTYPE = new Laplace(DEFAULT_LAPLACE_ALPHA); - - /** - * Creates a Laplace smoothing model. - * - */ - public Laplace(double alpha) { - this.alpha = alpha; - } - - /** - * @return the laplace model alpha parameter - */ - public double getAlpha() { - return this.alpha; - } - - @Override - protected XContentBuilder innerToXContent(XContentBuilder builder, Params params) throws IOException { - builder.field(ALPHA_FIELD.getPreferredName(), alpha); - return builder; - } - - @Override - public String getWriteableName() { - return NAME; - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - out.writeDouble(alpha); - } - - @Override - public SmoothingModel readFrom(StreamInput in) throws IOException { - return new Laplace(in.readDouble()); - } - - @Override - protected boolean doEquals(SmoothingModel other) { - Laplace otherModel = (Laplace) other; - return Objects.equals(alpha, otherModel.alpha); - } - - @Override - protected final int doHashCode() { - return Objects.hash(alpha); - } - - @Override - public SmoothingModel innerFromXContent(QueryParseContext parseContext) throws IOException { - XContentParser parser = parseContext.parser(); - XContentParser.Token token; - String fieldName = null; - double alpha = DEFAULT_LAPLACE_ALPHA; - while ((token = parser.nextToken()) != Token.END_OBJECT) { - if (token == XContentParser.Token.FIELD_NAME) { - fieldName = parser.currentName(); - } - if (token.isValue() && parseContext.parseFieldMatcher().match(fieldName, ALPHA_FIELD)) { - alpha = parser.doubleValue(); - } - } - return new Laplace(alpha); - } - - @Override - public WordScorerFactory buildWordScorerFactory() { - return (IndexReader reader, Terms terms, String field, double realWordLikelyhood, BytesRef separator) - -> new LaplaceScorer(reader, terms, field, realWordLikelyhood, separator, alpha); - } - } - - - public static abstract class SmoothingModel implements NamedWriteable<SmoothingModel>, ToXContent { - - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.startObject(getWriteableName()); - innerToXContent(builder,params); - builder.endObject(); - return builder; - } - - @Override - public final boolean equals(Object obj) { - if (this == obj) { - return true; - } - if (obj == null || getClass() != obj.getClass()) { - return false; - } - SmoothingModel other = (SmoothingModel) obj; - return doEquals(other); - } - - public static SmoothingModel fromXContent(QueryParseContext parseContext) throws IOException { - XContentParser parser = parseContext.parser(); - ParseFieldMatcher parseFieldMatcher = parseContext.parseFieldMatcher(); - XContentParser.Token token; - String fieldName = null; - SmoothingModel model = null; - while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { - if (token == XContentParser.Token.FIELD_NAME) { - fieldName = parser.currentName(); - } else if (token == XContentParser.Token.START_OBJECT) { - if (parseFieldMatcher.match(fieldName, LinearInterpolation.PARSE_FIELD)) { - model = LinearInterpolation.PROTOTYPE.innerFromXContent(parseContext); - } else if (parseFieldMatcher.match(fieldName, Laplace.PARSE_FIELD)) { - model = Laplace.PROTOTYPE.innerFromXContent(parseContext); - } else if (parseFieldMatcher.match(fieldName, StupidBackoff.PARSE_FIELD)) { - model = StupidBackoff.PROTOTYPE.innerFromXContent(parseContext); - } else { - throw new IllegalArgumentException("suggester[phrase] doesn't support object field [" + fieldName + "]"); - } - } else { - throw new ParsingException(parser.getTokenLocation(), - "[smoothing] unknown token [" + token + "] after [" + fieldName + "]"); - } - } - return model; - } - - public abstract SmoothingModel innerFromXContent(QueryParseContext parseContext) throws IOException; - - @Override - public final int hashCode() { - /* - * Override hashCode here and forward to an abstract method to force extensions of this class to override hashCode in the same - * way that we force them to override equals. This also prevents false positives in CheckStyle's EqualsHashCode check. - */ - return doHashCode(); - } - - public abstract WordScorerFactory buildWordScorerFactory(); - - /** - * subtype specific implementation of "equals". - */ - protected abstract boolean doEquals(SmoothingModel other); - - protected abstract int doHashCode(); - - protected abstract XContentBuilder innerToXContent(XContentBuilder builder, Params params) throws IOException; - } - - /** - * Linear interpolation smoothing model. - * <p> - * See <a - * href="http://en.wikipedia.org/wiki/N-gram#Smoothing_techniques">N-Gram - * Smoothing</a> for details. - * </p> - */ - public static final class LinearInterpolation extends SmoothingModel { - private static final String NAME = "linear"; - public static final LinearInterpolation PROTOTYPE = new LinearInterpolation(0.8, 0.1, 0.1); - private final double trigramLambda; - private final double bigramLambda; - private final double unigramLambda; - private static final ParseField PARSE_FIELD = new ParseField(NAME); - private static final ParseField TRIGRAM_FIELD = new ParseField("trigram_lambda"); - private static final ParseField BIGRAM_FIELD = new ParseField("bigram_lambda"); - private static final ParseField UNIGRAM_FIELD = new ParseField("unigram_lambda"); - - /** - * Creates a linear interpolation smoothing model. - * - * Note: the lambdas must sum up to one. - * - * @param trigramLambda - * the trigram lambda - * @param bigramLambda - * the bigram lambda - * @param unigramLambda - * the unigram lambda - */ - public LinearInterpolation(double trigramLambda, double bigramLambda, double unigramLambda) { - double sum = trigramLambda + bigramLambda + unigramLambda; - if (Math.abs(sum - 1.0) > 0.001) { - throw new IllegalArgumentException("linear smoothing lambdas must sum to 1"); - } - this.trigramLambda = trigramLambda; - this.bigramLambda = bigramLambda; - this.unigramLambda = unigramLambda; - } - - public double getTrigramLambda() { - return this.trigramLambda; - } - - public double getBigramLambda() { - return this.bigramLambda; - } - - public double getUnigramLambda() { - return this.unigramLambda; - } - - @Override - protected XContentBuilder innerToXContent(XContentBuilder builder, Params params) throws IOException { - builder.field(TRIGRAM_FIELD.getPreferredName(), trigramLambda); - builder.field(BIGRAM_FIELD.getPreferredName(), bigramLambda); - builder.field(UNIGRAM_FIELD.getPreferredName(), unigramLambda); - return builder; - } - - @Override - public String getWriteableName() { - return NAME; - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - out.writeDouble(trigramLambda); - out.writeDouble(bigramLambda); - out.writeDouble(unigramLambda); - } - - @Override - public LinearInterpolation readFrom(StreamInput in) throws IOException { - return new LinearInterpolation(in.readDouble(), in.readDouble(), in.readDouble()); - } - - @Override - protected boolean doEquals(SmoothingModel other) { - final LinearInterpolation otherModel = (LinearInterpolation) other; - return Objects.equals(trigramLambda, otherModel.trigramLambda) && - Objects.equals(bigramLambda, otherModel.bigramLambda) && - Objects.equals(unigramLambda, otherModel.unigramLambda); - } - - @Override - protected final int doHashCode() { - return Objects.hash(trigramLambda, bigramLambda, unigramLambda); - } - - @Override - public LinearInterpolation innerFromXContent(QueryParseContext parseContext) throws IOException { - XContentParser parser = parseContext.parser(); - XContentParser.Token token; - String fieldName = null; - double trigramLambda = 0.0; - double bigramLambda = 0.0; - double unigramLambda = 0.0; - ParseFieldMatcher matcher = parseContext.parseFieldMatcher(); - while ((token = parser.nextToken()) != Token.END_OBJECT) { - if (token == XContentParser.Token.FIELD_NAME) { - fieldName = parser.currentName(); - } else if (token.isValue()) { - if (matcher.match(fieldName, TRIGRAM_FIELD)) { - trigramLambda = parser.doubleValue(); - if (trigramLambda < 0) { - throw new IllegalArgumentException("trigram_lambda must be positive"); - } - } else if (matcher.match(fieldName, BIGRAM_FIELD)) { - bigramLambda = parser.doubleValue(); - if (bigramLambda < 0) { - throw new IllegalArgumentException("bigram_lambda must be positive"); - } - } else if (matcher.match(fieldName, UNIGRAM_FIELD)) { - unigramLambda = parser.doubleValue(); - if (unigramLambda < 0) { - throw new IllegalArgumentException("unigram_lambda must be positive"); - } - } else { - throw new IllegalArgumentException( - "suggester[phrase][smoothing][linear] doesn't support field [" + fieldName + "]"); - } - } else { - throw new ParsingException(parser.getTokenLocation(), - "[" + NAME + "] unknown token [" + token + "] after [" + fieldName + "]"); - } - } - return new LinearInterpolation(trigramLambda, bigramLambda, unigramLambda); - } - - @Override - public WordScorerFactory buildWordScorerFactory() { - return (IndexReader reader, Terms terms, String field, double realWordLikelyhood, BytesRef separator) -> - new LinearInterpoatingScorer(reader, terms, field, realWordLikelyhood, separator, trigramLambda, bigramLambda, - unigramLambda); - } - } - @Override protected PhraseSuggestionBuilder innerFromXContent(QueryParseContext parseContext, String suggestionName) throws IOException { XContentParser parser = parseContext.parser(); @@ -873,7 +474,6 @@ public final class PhraseSuggestionBuilder extends SuggestionBuilder<PhraseSugge "suggester[phrase][collate] query already set, doesn't support additional [" + fieldName + "]"); } Template template = Template.parse(parser, parseFieldMatcher); - // TODO remember to compile script in build() method suggestion.collateQuery(template); } else if (parseFieldMatcher.match(fieldName, PhraseSuggestionBuilder.COLLATE_QUERY_PARAMS)) { suggestion.collateParams(parser.map()); @@ -898,6 +498,98 @@ public final class PhraseSuggestionBuilder extends SuggestionBuilder<PhraseSugge return suggestion; } + + @Override + public SuggestionContext build(QueryShardContext context) throws IOException { + PhraseSuggestionContext suggestionContext = new PhraseSuggestionContext(context); + MapperService mapperService = context.getMapperService(); + populateCommonFields(mapperService, suggestionContext); + + suggestionContext.setSeparator(BytesRefs.toBytesRef(this.separator)); + suggestionContext.setRealWordErrorLikelihood(this.realWordErrorLikelihood); + suggestionContext.setConfidence(this.confidence); + suggestionContext.setMaxErrors(this.maxErrors); + suggestionContext.setSeparator(BytesRefs.toBytesRef(this.separator)); + suggestionContext.setRequireUnigram(this.forceUnigrams); + suggestionContext.setTokenLimit(this.tokenLimit); + suggestionContext.setPreTag(BytesRefs.toBytesRef(this.preTag)); + suggestionContext.setPostTag(BytesRefs.toBytesRef(this.postTag)); + + if (this.gramSize != null) { + suggestionContext.setGramSize(this.gramSize); + } + + for (List<CandidateGenerator> candidateGenerators : this.generators.values()) { + for (CandidateGenerator candidateGenerator : candidateGenerators) { + suggestionContext.addGenerator(candidateGenerator.build(mapperService)); + } + } + + if (this.model != null) { + suggestionContext.setModel(this.model.buildWordScorerFactory()); + } + + if (this.collateQuery != null) { + CompiledScript compiledScript = context.getScriptService().compile(this.collateQuery, ScriptContext.Standard.SEARCH, + Collections.emptyMap()); + suggestionContext.setCollateQueryScript(compiledScript); + if (this.collateParams != null) { + suggestionContext.setCollateScriptParams(this.collateParams); + } + suggestionContext.setCollatePrune(this.collatePrune); + } + + // TODO remove this when field is mandatory in builder ctor + if (suggestionContext.getField() == null) { + throw new IllegalArgumentException("The required field option is missing"); + } + + MappedFieldType fieldType = mapperService.fullName(suggestionContext.getField()); + if (fieldType == null) { + throw new IllegalArgumentException("No mapping found for field [" + suggestionContext.getField() + "]"); + } else if (suggestionContext.getAnalyzer() == null) { + // no analyzer name passed in, so try the field's analyzer, or the default analyzer + if (fieldType.searchAnalyzer() == null) { + suggestionContext.setAnalyzer(mapperService.searchAnalyzer()); + } else { + suggestionContext.setAnalyzer(fieldType.searchAnalyzer()); + } + } + + if (suggestionContext.model() == null) { + suggestionContext.setModel(StupidBackoffScorer.FACTORY); + } + + if (this.gramSize == null || suggestionContext.generators().isEmpty()) { + final ShingleTokenFilterFactory.Factory shingleFilterFactory = SuggestUtils + .getShingleFilterFactory(suggestionContext.getAnalyzer()); + if (this.gramSize == null) { + // try to detect the shingle size + if (shingleFilterFactory != null) { + suggestionContext.setGramSize(shingleFilterFactory.getMaxShingleSize()); + if (suggestionContext.getAnalyzer() == null && shingleFilterFactory.getMinShingleSize() > 1 + && !shingleFilterFactory.getOutputUnigrams()) { + throw new IllegalArgumentException("The default analyzer for field: [" + suggestionContext.getField() + + "] doesn't emit unigrams. If this is intentional try to set the analyzer explicitly"); + } + } + } + if (suggestionContext.generators().isEmpty()) { + if (shingleFilterFactory != null && shingleFilterFactory.getMinShingleSize() > 1 + && !shingleFilterFactory.getOutputUnigrams() && suggestionContext.getRequireUnigram()) { + throw new IllegalArgumentException("The default candidate generator for phrase suggest can't operate on field: [" + + suggestionContext.getField() + "] since it doesn't emit unigrams. " + + "If this is intentional try to set the candidate generator field explicitly"); + } + // use a default generator on the same field + DirectCandidateGenerator generator = new DirectCandidateGenerator(); + generator.setField(suggestionContext.getField()); + suggestionContext.addGenerator(generator); + } + } + return suggestionContext; + } + private static void ensureNoSmoothing(PhraseSuggestionBuilder suggestion) { if (suggestion.smoothingModel() != null) { throw new IllegalArgumentException("only one smoothing model supported"); @@ -1010,5 +702,7 @@ public final class PhraseSuggestionBuilder extends SuggestionBuilder<PhraseSugge String getType(); CandidateGenerator fromXContent(QueryParseContext parseContext) throws IOException; + + PhraseSuggestionContext.DirectCandidateGenerator build(MapperService mapperService) throws IOException; } } diff --git a/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestionContext.java b/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestionContext.java index 20af69b6a6..95c02d5add 100644 --- a/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestionContext.java +++ b/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestionContext.java @@ -20,9 +20,9 @@ package org.elasticsearch.search.suggest.phrase; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.script.CompiledScript; import org.elasticsearch.search.suggest.DirectSpellcheckerSettings; -import org.elasticsearch.search.suggest.Suggester; import org.elasticsearch.search.suggest.SuggestionSearchContext.SuggestionContext; import java.util.ArrayList; @@ -54,8 +54,8 @@ class PhraseSuggestionContext extends SuggestionContext { private Map<String, Object> collateScriptParams = new HashMap<>(1); private WordScorer.WordScorerFactory scorer; - public PhraseSuggestionContext(Suggester<? extends PhraseSuggestionContext> suggester) { - super(suggester); + public PhraseSuggestionContext(QueryShardContext shardContext) { + super(PhraseSuggester.PROTOTYPE, shardContext); } public float maxErrors() { @@ -154,8 +154,6 @@ class PhraseSuggestionContext extends SuggestionContext { public void postFilter(Analyzer postFilter) { this.postFilter = postFilter; } - - } public void setRequireUnigram(boolean requireUnigram) { @@ -213,5 +211,4 @@ class PhraseSuggestionContext extends SuggestionContext { boolean collatePrune() { return prune; } - } diff --git a/core/src/main/java/org/elasticsearch/search/suggest/phrase/SmoothingModel.java b/core/src/main/java/org/elasticsearch/search/suggest/phrase/SmoothingModel.java new file mode 100644 index 0000000000..0163c560de --- /dev/null +++ b/core/src/main/java/org/elasticsearch/search/suggest/phrase/SmoothingModel.java @@ -0,0 +1,105 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.suggest.phrase; + +import org.elasticsearch.common.ParseFieldMatcher; +import org.elasticsearch.common.ParsingException; +import org.elasticsearch.common.io.stream.NamedWriteable; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.index.query.QueryParseContext; +import org.elasticsearch.search.suggest.phrase.WordScorer.WordScorerFactory; + +import java.io.IOException; + +public abstract class SmoothingModel implements NamedWriteable<SmoothingModel>, ToXContent { + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(getWriteableName()); + innerToXContent(builder,params); + builder.endObject(); + return builder; + } + + @Override + public final boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + SmoothingModel other = (SmoothingModel) obj; + return doEquals(other); + } + + @Override + public final int hashCode() { + /* + * Override hashCode here and forward to an abstract method to force + * extensions of this class to override hashCode in the same way that we + * force them to override equals. This also prevents false positives in + * CheckStyle's EqualsHashCode check. + */ + return doHashCode(); + } + + protected abstract int doHashCode(); + + public static SmoothingModel fromXContent(QueryParseContext parseContext) throws IOException { + XContentParser parser = parseContext.parser(); + ParseFieldMatcher parseFieldMatcher = parseContext.parseFieldMatcher(); + XContentParser.Token token; + String fieldName = null; + SmoothingModel model = null; + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + if (token == XContentParser.Token.FIELD_NAME) { + fieldName = parser.currentName(); + } else if (token == XContentParser.Token.START_OBJECT) { + if (parseFieldMatcher.match(fieldName, LinearInterpolation.PARSE_FIELD)) { + model = LinearInterpolation.PROTOTYPE.innerFromXContent(parseContext); + } else if (parseFieldMatcher.match(fieldName, Laplace.PARSE_FIELD)) { + model = Laplace.PROTOTYPE.innerFromXContent(parseContext); + } else if (parseFieldMatcher.match(fieldName, StupidBackoff.PARSE_FIELD)) { + model = StupidBackoff.PROTOTYPE.innerFromXContent(parseContext); + } else { + throw new IllegalArgumentException("suggester[phrase] doesn't support object field [" + fieldName + "]"); + } + } else { + throw new ParsingException(parser.getTokenLocation(), + "[smoothing] unknown token [" + token + "] after [" + fieldName + "]"); + } + } + return model; + } + + public abstract SmoothingModel innerFromXContent(QueryParseContext parseContext) throws IOException; + + public abstract WordScorerFactory buildWordScorerFactory(); + + /** + * subtype specific implementation of "equals". + */ + protected abstract boolean doEquals(SmoothingModel other); + + protected abstract XContentBuilder innerToXContent(XContentBuilder builder, Params params) throws IOException; +}
\ No newline at end of file diff --git a/core/src/main/java/org/elasticsearch/search/suggest/phrase/StupidBackoff.java b/core/src/main/java/org/elasticsearch/search/suggest/phrase/StupidBackoff.java new file mode 100644 index 0000000000..9611622d8c --- /dev/null +++ b/core/src/main/java/org/elasticsearch/search/suggest/phrase/StupidBackoff.java @@ -0,0 +1,129 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.suggest.phrase; + +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.Terms; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentParser.Token; +import org.elasticsearch.index.query.QueryParseContext; +import org.elasticsearch.search.suggest.phrase.WordScorer.WordScorerFactory; + +import java.io.IOException; +import java.util.Objects; + +/** + * A "stupid-backoff" smoothing model simialr to <a + * href="http://en.wikipedia.org/wiki/Katz's_back-off_model"> Katz's + * Backoff</a>. This model is used as the default if no model is configured. + * <p> + * See <a + * href="http://en.wikipedia.org/wiki/N-gram#Smoothing_techniques">N-Gram + * Smoothing</a> for details. + * </p> + */ +public final class StupidBackoff extends SmoothingModel { + /** + * Default discount parameter for {@link StupidBackoff} smoothing + */ + public static final double DEFAULT_BACKOFF_DISCOUNT = 0.4; + public static final StupidBackoff PROTOTYPE = new StupidBackoff(DEFAULT_BACKOFF_DISCOUNT); + private double discount = DEFAULT_BACKOFF_DISCOUNT; + private static final String NAME = "stupid_backoff"; + private static final ParseField DISCOUNT_FIELD = new ParseField("discount"); + static final ParseField PARSE_FIELD = new ParseField(NAME); + + /** + * Creates a Stupid-Backoff smoothing model. + * + * @param discount + * the discount given to lower order ngrams if the higher order ngram doesn't exits + */ + public StupidBackoff(double discount) { + this.discount = discount; + } + + /** + * @return the discount parameter of the model + */ + public double getDiscount() { + return this.discount; + } + + @Override + protected XContentBuilder innerToXContent(XContentBuilder builder, Params params) throws IOException { + builder.field(DISCOUNT_FIELD.getPreferredName(), discount); + return builder; + } + + @Override + public String getWriteableName() { + return NAME; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeDouble(discount); + } + + @Override + public StupidBackoff readFrom(StreamInput in) throws IOException { + return new StupidBackoff(in.readDouble()); + } + + @Override + protected boolean doEquals(SmoothingModel other) { + StupidBackoff otherModel = (StupidBackoff) other; + return Objects.equals(discount, otherModel.discount); + } + + @Override + protected final int doHashCode() { + return Objects.hash(discount); + } + + @Override + public SmoothingModel innerFromXContent(QueryParseContext parseContext) throws IOException { + XContentParser parser = parseContext.parser(); + XContentParser.Token token; + String fieldName = null; + double discount = DEFAULT_BACKOFF_DISCOUNT; + while ((token = parser.nextToken()) != Token.END_OBJECT) { + if (token == XContentParser.Token.FIELD_NAME) { + fieldName = parser.currentName(); + } + if (token.isValue() && parseContext.parseFieldMatcher().match(fieldName, DISCOUNT_FIELD)) { + discount = parser.doubleValue(); + } + } + return new StupidBackoff(discount); + } + + @Override + public WordScorerFactory buildWordScorerFactory() { + return (IndexReader reader, Terms terms, String field, double realWordLikelyhood, BytesRef separator) + -> new StupidBackoffScorer(reader, terms, field, realWordLikelyhood, separator, discount); + } +}
\ No newline at end of file diff --git a/core/src/main/java/org/elasticsearch/search/suggest/term/TermSuggestParser.java b/core/src/main/java/org/elasticsearch/search/suggest/term/TermSuggestParser.java index a2fd680c21..7e75976d3a 100644 --- a/core/src/main/java/org/elasticsearch/search/suggest/term/TermSuggestParser.java +++ b/core/src/main/java/org/elasticsearch/search/suggest/term/TermSuggestParser.java @@ -20,8 +20,8 @@ package org.elasticsearch.search.suggest.term; import org.elasticsearch.common.ParseFieldMatcher; import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.index.fielddata.IndexFieldDataService; import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.search.suggest.DirectSpellcheckerSettings; import org.elasticsearch.search.suggest.SuggestContextParser; import org.elasticsearch.search.suggest.SuggestUtils; @@ -38,10 +38,11 @@ public final class TermSuggestParser implements SuggestContextParser { } @Override - public SuggestionSearchContext.SuggestionContext parse(XContentParser parser, MapperService mapperService, IndexFieldDataService fieldDataService) throws IOException { + public SuggestionSearchContext.SuggestionContext parse(XContentParser parser, QueryShardContext shardContext) throws IOException { + MapperService mapperService = shardContext.getMapperService(); XContentParser.Token token; String fieldName = null; - TermSuggestionContext suggestion = new TermSuggestionContext(suggester); + TermSuggestionContext suggestion = new TermSuggestionContext(shardContext); DirectSpellcheckerSettings settings = suggestion.getDirectSpellCheckerSettings(); while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { if (token == XContentParser.Token.FIELD_NAME) { diff --git a/core/src/main/java/org/elasticsearch/search/suggest/term/TermSuggester.java b/core/src/main/java/org/elasticsearch/search/suggest/term/TermSuggester.java index e67e619bf5..78ed8be6a2 100644 --- a/core/src/main/java/org/elasticsearch/search/suggest/term/TermSuggester.java +++ b/core/src/main/java/org/elasticsearch/search/suggest/term/TermSuggester.java @@ -40,6 +40,8 @@ import java.util.List; public final class TermSuggester extends Suggester<TermSuggestionContext> { + public static final TermSuggester PROTOTYPE = new TermSuggester(); + @Override public TermSuggestion innerExecute(String name, TermSuggestionContext suggestion, IndexSearcher searcher, CharsRefBuilder spare) throws IOException { diff --git a/core/src/main/java/org/elasticsearch/search/suggest/term/TermSuggestionBuilder.java b/core/src/main/java/org/elasticsearch/search/suggest/term/TermSuggestionBuilder.java index 1378c362c5..62d6718cd2 100644 --- a/core/src/main/java/org/elasticsearch/search/suggest/term/TermSuggestionBuilder.java +++ b/core/src/main/java/org/elasticsearch/search/suggest/term/TermSuggestionBuilder.java @@ -26,7 +26,9 @@ import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.query.QueryParseContext; +import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.search.suggest.SuggestionBuilder; +import org.elasticsearch.search.suggest.SuggestionSearchContext.SuggestionContext; import java.io.IOException; import java.util.Locale; @@ -382,6 +384,12 @@ public class TermSuggestionBuilder extends SuggestionBuilder<TermSuggestionBuild } @Override + protected SuggestionContext build(QueryShardContext context) throws IOException { + // NORELEASE + throw new UnsupportedOperationException(); + } + + @Override public String getWriteableName() { return SUGGESTION_NAME; } diff --git a/core/src/main/java/org/elasticsearch/search/suggest/term/TermSuggestionContext.java b/core/src/main/java/org/elasticsearch/search/suggest/term/TermSuggestionContext.java index 4ff32d797e..3af13ced8a 100644 --- a/core/src/main/java/org/elasticsearch/search/suggest/term/TermSuggestionContext.java +++ b/core/src/main/java/org/elasticsearch/search/suggest/term/TermSuggestionContext.java @@ -18,20 +18,19 @@ */ package org.elasticsearch.search.suggest.term; +import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.search.suggest.DirectSpellcheckerSettings; -import org.elasticsearch.search.suggest.Suggester; import org.elasticsearch.search.suggest.SuggestionSearchContext.SuggestionContext; final class TermSuggestionContext extends SuggestionContext { private final DirectSpellcheckerSettings settings = new DirectSpellcheckerSettings(); - public TermSuggestionContext(Suggester<? extends TermSuggestionContext> suggester) { - super(suggester); + public TermSuggestionContext(QueryShardContext shardContext) { + super(TermSuggester.PROTOTYPE, shardContext); } public DirectSpellcheckerSettings getDirectSpellCheckerSettings() { return settings; } - }
\ No newline at end of file diff --git a/core/src/test/java/org/elasticsearch/index/IndexModuleTests.java b/core/src/test/java/org/elasticsearch/index/IndexModuleTests.java index 0b0691bc58..656e3ab6a6 100644 --- a/core/src/test/java/org/elasticsearch/index/IndexModuleTests.java +++ b/core/src/test/java/org/elasticsearch/index/IndexModuleTests.java @@ -63,8 +63,8 @@ import org.elasticsearch.indices.query.IndicesQueriesRegistry; import org.elasticsearch.script.ScriptContextRegistry; import org.elasticsearch.script.ScriptEngineRegistry; import org.elasticsearch.script.ScriptEngineService; -import org.elasticsearch.script.ScriptSettings; import org.elasticsearch.script.ScriptService; +import org.elasticsearch.script.ScriptSettings; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.IndexSettingsModule; import org.elasticsearch.test.engine.MockEngineFactory; diff --git a/core/src/test/java/org/elasticsearch/search/suggest/AbstractSuggestionBuilderTestCase.java b/core/src/test/java/org/elasticsearch/search/suggest/AbstractSuggestionBuilderTestCase.java index 3c5797f1e4..71431e70f3 100644 --- a/core/src/test/java/org/elasticsearch/search/suggest/AbstractSuggestionBuilderTestCase.java +++ b/core/src/test/java/org/elasticsearch/search/suggest/AbstractSuggestionBuilderTestCase.java @@ -19,6 +19,7 @@ package org.elasticsearch.search.suggest; +import org.apache.lucene.analysis.core.WhitespaceAnalyzer; import org.elasticsearch.common.ParseFieldMatcher; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput; @@ -31,19 +32,48 @@ import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.env.Environment; +import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.analysis.AnalysisService; +import org.elasticsearch.index.analysis.NamedAnalyzer; +import org.elasticsearch.index.mapper.ContentPath; +import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.Mapper; +import org.elasticsearch.index.mapper.MapperBuilders; +import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.index.mapper.core.StringFieldMapper; +import org.elasticsearch.index.mapper.core.StringFieldMapper.StringFieldType; import org.elasticsearch.index.query.QueryParseContext; +import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.indices.IndicesModule; +import org.elasticsearch.script.CompiledScript; +import org.elasticsearch.script.Script; +import org.elasticsearch.script.ScriptContext; +import org.elasticsearch.script.ScriptContextRegistry; +import org.elasticsearch.script.ScriptEngineRegistry; +import org.elasticsearch.script.ScriptService; +import org.elasticsearch.script.ScriptServiceTests.TestEngineService; +import org.elasticsearch.script.ScriptSettings; +import org.elasticsearch.search.suggest.SuggestionSearchContext.SuggestionContext; import org.elasticsearch.search.suggest.completion.CompletionSuggestionBuilder; import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder; import org.elasticsearch.search.suggest.term.TermSuggestionBuilder; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.IndexSettingsModule; +import org.elasticsearch.watcher.ResourceWatcherService; import org.junit.AfterClass; import org.junit.BeforeClass; import java.io.IOException; +import java.nio.file.Path; import java.util.Collections; +import java.util.Iterator; +import java.util.Map; +import java.util.Map.Entry; import java.util.function.Consumer; import java.util.function.Supplier; +import static org.elasticsearch.common.settings.Settings.settingsBuilder; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.not; @@ -51,13 +81,35 @@ public abstract class AbstractSuggestionBuilderTestCase<SB extends SuggestionBui private static final int NUMBER_OF_TESTBUILDERS = 20; protected static NamedWriteableRegistry namedWriteableRegistry; - private static final Suggesters suggesters = new Suggesters(Collections.emptyMap(), null, null); + private static Suggesters suggesters; + private static ScriptService scriptService; + private static SuggestParseElement parseElement; /** * setup for the whole base test class */ @BeforeClass - public static void init() { + public static void init() throws IOException { + Path genericConfigFolder = createTempDir(); + Settings baseSettings = settingsBuilder() + .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()) + .put(Environment.PATH_CONF_SETTING.getKey(), genericConfigFolder) + .build(); + Environment environment = new Environment(baseSettings); + ScriptContextRegistry scriptContextRegistry = new ScriptContextRegistry(Collections.emptyList()); + ScriptEngineRegistry scriptEngineRegistry = new ScriptEngineRegistry(Collections.singletonList(new ScriptEngineRegistry + .ScriptEngineRegistration(TestEngineService.class, TestEngineService.TYPES))); + ScriptSettings scriptSettings = new ScriptSettings(scriptEngineRegistry, scriptContextRegistry); + scriptService = new ScriptService(baseSettings, environment, Collections.singleton(new TestEngineService()), + new ResourceWatcherService(baseSettings, null), scriptEngineRegistry, scriptContextRegistry, scriptSettings) { + @Override + public CompiledScript compile(Script script, ScriptContext scriptContext, Map<String, String> params) { + return new CompiledScript(ScriptType.INLINE, "mockName", "mocklang", script); + } + }; + suggesters = new Suggesters(Collections.emptyMap()); + parseElement = new SuggestParseElement(suggesters); + namedWriteableRegistry = new NamedWriteableRegistry(); namedWriteableRegistry.registerPrototype(SuggestionBuilder.class, TermSuggestionBuilder.PROTOTYPE); namedWriteableRegistry.registerPrototype(SuggestionBuilder.class, PhraseSuggestionBuilder.PROTOTYPE); @@ -69,7 +121,6 @@ public abstract class AbstractSuggestionBuilderTestCase<SB extends SuggestionBui namedWriteableRegistry = null; } - /** * Test serialization and deserialization of the suggestion builder */ @@ -88,13 +139,13 @@ public abstract class AbstractSuggestionBuilderTestCase<SB extends SuggestionBui */ protected SB randomTestBuilder() { SB randomSuggestion = randomSuggestionBuilder(); + randomSuggestion.field(randomAsciiOfLengthBetween(2, 20)); maybeSet(randomSuggestion::text, randomAsciiOfLengthBetween(2, 20)); maybeSet(randomSuggestion::prefix, randomAsciiOfLengthBetween(2, 20)); maybeSet(randomSuggestion::regex, randomAsciiOfLengthBetween(2, 20)); - maybeSet(randomSuggestion::field, randomAsciiOfLengthBetween(2, 20)); maybeSet(randomSuggestion::analyzer, randomAsciiOfLengthBetween(2, 20)); maybeSet(randomSuggestion::size, randomIntBetween(1, 20)); - maybeSet(randomSuggestion::shardSize, randomInt(20)); + maybeSet(randomSuggestion::shardSize, randomIntBetween(1, 20)); return randomSuggestion; } @@ -137,7 +188,8 @@ public abstract class AbstractSuggestionBuilderTestCase<SB extends SuggestionBui } /** - * creates random suggestion builder, renders it to xContent and back to new instance that should be equal to original + * creates random suggestion builder, renders it to xContent and back to new + * instance that should be equal to original */ public void testFromXContent() throws IOException { QueryParseContext context = new QueryParseContext(null); @@ -166,6 +218,97 @@ public abstract class AbstractSuggestionBuilderTestCase<SB extends SuggestionBui } } + /** + * parses random suggestion builder via old parseElement method and via + * build, comparing the results for equality + */ + public void testBuild() throws IOException { + IndexSettings idxSettings = IndexSettingsModule.newIndexSettings(randomAsciiOfLengthBetween(1, 10), Settings.EMPTY); + + AnalysisService mockAnalysisService = new AnalysisService(idxSettings, Collections.emptyMap(), Collections.emptyMap(), + Collections.emptyMap(), Collections.emptyMap()) { + @Override + public NamedAnalyzer analyzer(String name) { + return new NamedAnalyzer(name, new WhitespaceAnalyzer()); + } + }; + + MapperService mockMapperService = new MapperService(idxSettings, mockAnalysisService, null, new IndicesModule().getMapperRegistry(), + null) { + @Override + public MappedFieldType fullName(String fullName) { + return new StringFieldType(); + } + }; + + QueryShardContext mockShardContext = new QueryShardContext(idxSettings, null, null, null, mockMapperService, null, scriptService, + null) { + @Override + public MappedFieldType fieldMapper(String name) { + StringFieldMapper.Builder builder = MapperBuilders.stringField(name); + return builder.build(new Mapper.BuilderContext(idxSettings.getSettings(), new ContentPath(1))).fieldType(); + } + }; + mockShardContext.setMapUnmappedFieldAsString(true); + + for (int runs = 0; runs < NUMBER_OF_TESTBUILDERS; runs++) { + SuggestBuilder suggestBuilder = new SuggestBuilder(); + SB suggestionBuilder = randomTestBuilder(); + suggestBuilder.addSuggestion(suggestionBuilder); + + if (suggestionBuilder.text() == null) { + // we either need suggestion text or global text + suggestBuilder.setText(randomAsciiOfLengthBetween(5, 50)); + } + if (suggestionBuilder.text() != null && suggestionBuilder.prefix() != null) { + suggestionBuilder.prefix(null); + } + + XContentBuilder xContentBuilder = XContentFactory.contentBuilder(randomFrom(XContentType.values())); + if (randomBoolean()) { + xContentBuilder.prettyPrint(); + } + suggestBuilder.toXContent(xContentBuilder, ToXContent.EMPTY_PARAMS); + + XContentParser parser = XContentHelper.createParser(xContentBuilder.bytes()); + parser.nextToken(); // set cursor to START_OBJECT + SuggestionSearchContext parsedSuggestionSearchContext = parseElement.parseInternal(parser, mockShardContext); + + SuggestionSearchContext buildSuggestSearchContext = suggestBuilder.build(mockShardContext); + assertEquals(parsedSuggestionSearchContext.suggestions().size(), buildSuggestSearchContext.suggestions().size()); + Iterator<Entry<String, SuggestionContext>> iterator = buildSuggestSearchContext.suggestions().entrySet().iterator(); + for (Entry<String, SuggestionContext> entry : parsedSuggestionSearchContext.suggestions().entrySet()) { + Entry<String, SuggestionContext> other = iterator.next(); + assertEquals(entry.getKey(), other.getKey()); + + SuggestionContext oldSchoolContext = entry.getValue(); + SuggestionContext newSchoolContext = other.getValue(); + assertNotSame(oldSchoolContext, newSchoolContext); + // deep comparison of analyzers is difficult here, but we check they are set or not set + if (oldSchoolContext.getAnalyzer() != null) { + assertNotNull(newSchoolContext.getAnalyzer()); + } else { + assertNull(newSchoolContext.getAnalyzer()); + } + assertEquals(oldSchoolContext.getField(), newSchoolContext.getField()); + assertEquals(oldSchoolContext.getPrefix(), newSchoolContext.getPrefix()); + assertEquals(oldSchoolContext.getRegex(), newSchoolContext.getRegex()); + assertEquals(oldSchoolContext.getShardSize(), newSchoolContext.getShardSize()); + assertEquals(oldSchoolContext.getSize(), newSchoolContext.getSize()); + assertEquals(oldSchoolContext.getSuggester().getClass(), newSchoolContext.getSuggester().getClass()); + assertEquals(oldSchoolContext.getText(), newSchoolContext.getText()); + assertEquals(oldSchoolContext.getClass(), newSchoolContext.getClass()); + + assertSuggestionContext(oldSchoolContext, newSchoolContext); + } + } + } + + /** + * compare two SuggestionContexte implementations for the special suggestion type under test + */ + protected abstract void assertSuggestionContext(SuggestionContext oldSuggestion, SuggestionContext newSuggestion); + private SB mutate(SB firstBuilder) throws IOException { SB mutation = serializedCopy(firstBuilder); assertNotSame(mutation, firstBuilder); @@ -201,14 +344,16 @@ public abstract class AbstractSuggestionBuilderTestCase<SB extends SuggestionBui } /** - * take and input {@link SuggestBuilder} and return another one that is different in one aspect (to test non-equality) + * take and input {@link SuggestBuilder} and return another one that is + * different in one aspect (to test non-equality) */ protected abstract void mutateSpecificParameters(SB firstBuilder) throws IOException; @SuppressWarnings("unchecked") protected SB serializedCopy(SB original) throws IOException { try (BytesStreamOutput output = new BytesStreamOutput()) { - output.writeSuggestion(original);; + output.writeSuggestion(original); + ; try (StreamInput in = new NamedWriteableAwareStreamInput(StreamInput.wrap(output.bytes()), namedWriteableRegistry)) { return (SB) in.readSuggestion(); } @@ -222,7 +367,8 @@ public abstract class AbstractSuggestionBuilderTestCase<SB extends SuggestionBui } /** - * helper to get a random value in a certain range that's different from the input + * helper to get a random value in a certain range that's different from the + * input */ protected static <T> T randomValueOtherThan(T input, Supplier<T> randomSupplier) { T randomValue = null; diff --git a/core/src/test/java/org/elasticsearch/search/suggest/CustomSuggester.java b/core/src/test/java/org/elasticsearch/search/suggest/CustomSuggester.java index 4dbae08080..68e62983b5 100644 --- a/core/src/test/java/org/elasticsearch/search/suggest/CustomSuggester.java +++ b/core/src/test/java/org/elasticsearch/search/suggest/CustomSuggester.java @@ -21,6 +21,7 @@ package org.elasticsearch.search.suggest; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.util.CharsRefBuilder; import org.elasticsearch.common.text.Text; +import org.elasticsearch.index.query.QueryShardContext; import java.io.IOException; import java.util.Locale; @@ -54,9 +55,9 @@ public class CustomSuggester extends Suggester<CustomSuggester.CustomSuggestions @Override public SuggestContextParser getContextParser() { - return (parser, mapperService, fieldData) -> { + return (parser, shardContext) -> { Map<String, Object> options = parser.map(); - CustomSuggestionsContext suggestionContext = new CustomSuggestionsContext(CustomSuggester.this, options); + CustomSuggestionsContext suggestionContext = new CustomSuggestionsContext(shardContext, options); suggestionContext.setField((String) options.get("field")); return suggestionContext; }; @@ -66,8 +67,8 @@ public class CustomSuggester extends Suggester<CustomSuggester.CustomSuggestions public Map<String, Object> options; - public CustomSuggestionsContext(Suggester suggester, Map<String, Object> options) { - super(suggester); + public CustomSuggestionsContext(QueryShardContext context, Map<String, Object> options) { + super(new CustomSuggester(), context); this.options = options; } } diff --git a/core/src/test/java/org/elasticsearch/search/suggest/CustomSuggesterSearchIT.java b/core/src/test/java/org/elasticsearch/search/suggest/CustomSuggesterSearchIT.java index b3af0eee14..5bddad8bb0 100644 --- a/core/src/test/java/org/elasticsearch/search/suggest/CustomSuggesterSearchIT.java +++ b/core/src/test/java/org/elasticsearch/search/suggest/CustomSuggesterSearchIT.java @@ -25,7 +25,9 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.util.CollectionUtils; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.index.query.QueryParseContext; +import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.search.suggest.SuggestionSearchContext.SuggestionContext; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; import org.elasticsearch.test.ESIntegTestCase.Scope; @@ -132,6 +134,12 @@ public class CustomSuggesterSearchIT extends ESIntegTestCase { return new CustomSuggestionBuilder(name, randomField, randomSuffix); } + @Override + protected SuggestionContext build(QueryShardContext context) throws IOException { + // NORELEASE + return null; + } + } } diff --git a/core/src/test/java/org/elasticsearch/search/suggest/SuggestBuilderTests.java b/core/src/test/java/org/elasticsearch/search/suggest/SuggestBuilderTests.java index 2f53aaed9c..f4551b3de9 100644 --- a/core/src/test/java/org/elasticsearch/search/suggest/SuggestBuilderTests.java +++ b/core/src/test/java/org/elasticsearch/search/suggest/SuggestBuilderTests.java @@ -32,12 +32,12 @@ import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.query.QueryParseContext; import org.elasticsearch.search.suggest.completion.CompletionSuggestionBuilder; import org.elasticsearch.search.suggest.completion.WritableTestCase; +import org.elasticsearch.search.suggest.phrase.Laplace; +import org.elasticsearch.search.suggest.phrase.LinearInterpolation; import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder; -import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder.Laplace; -import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder.LinearInterpolation; -import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder.SmoothingModel; -import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder.StupidBackoff; import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilderTests; +import org.elasticsearch.search.suggest.phrase.SmoothingModel; +import org.elasticsearch.search.suggest.phrase.StupidBackoff; import org.elasticsearch.search.suggest.term.TermSuggestionBuilder; import java.io.IOException; @@ -62,7 +62,7 @@ public class SuggestBuilderTests extends WritableTestCase<SuggestBuilder> { * creates random suggestion builder, renders it to xContent and back to new instance that should be equal to original */ public void testFromXContent() throws IOException { - Suggesters suggesters = new Suggesters(Collections.emptyMap(), null, null); + Suggesters suggesters = new Suggesters(Collections.emptyMap()); QueryParseContext context = new QueryParseContext(null); context.parseFieldMatcher(new ParseFieldMatcher(Settings.EMPTY)); for (int runs = 0; runs < NUMBER_OF_RUNS; runs++) { diff --git a/core/src/test/java/org/elasticsearch/search/suggest/phrase/DirectCandidateGeneratorTests.java b/core/src/test/java/org/elasticsearch/search/suggest/phrase/DirectCandidateGeneratorTests.java index 02826b9a7e..9bf8447f8d 100644 --- a/core/src/test/java/org/elasticsearch/search/suggest/phrase/DirectCandidateGeneratorTests.java +++ b/core/src/test/java/org/elasticsearch/search/suggest/phrase/DirectCandidateGeneratorTests.java @@ -34,15 +34,10 @@ import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.analysis.AnalysisService; import org.elasticsearch.index.analysis.NamedAnalyzer; -import org.elasticsearch.index.mapper.ContentPath; import org.elasticsearch.index.mapper.MappedFieldType; -import org.elasticsearch.index.mapper.Mapper; -import org.elasticsearch.index.mapper.MapperBuilders; import org.elasticsearch.index.mapper.MapperService; -import org.elasticsearch.index.mapper.core.StringFieldMapper; import org.elasticsearch.index.mapper.core.StringFieldMapper.StringFieldType; import org.elasticsearch.index.query.QueryParseContext; -import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.indices.IndicesModule; import org.elasticsearch.indices.query.IndicesQueriesRegistry; import org.elasticsearch.search.suggest.phrase.PhraseSuggestionContext.DirectCandidateGenerator; @@ -171,19 +166,10 @@ public class DirectCandidateGeneratorTests extends ESTestCase{ } }; - QueryShardContext mockShardContext = new QueryShardContext(idxSettings, null, null, null, mockMapperService, null, null, null) { - @Override - public MappedFieldType fieldMapper(String name) { - StringFieldMapper.Builder builder = MapperBuilders.stringField(name); - return builder.build(new Mapper.BuilderContext(idxSettings.getSettings(), new ContentPath(1))).fieldType(); - } - }; - mockShardContext.setMapUnmappedFieldAsString(true); - for (int runs = 0; runs < NUMBER_OF_RUNS; runs++) { DirectCandidateGeneratorBuilder generator = randomCandidateGenerator(); // first, build via DirectCandidateGenerator#build() - DirectCandidateGenerator contextGenerator = generator.build(mockShardContext); + DirectCandidateGenerator contextGenerator = generator.build(mockMapperService); // second, render random test generator to xContent and parse using // PhraseSuggestParser @@ -195,28 +181,32 @@ public class DirectCandidateGeneratorTests extends ESTestCase{ XContentParser parser = XContentHelper.createParser(builder.bytes()); DirectCandidateGenerator secondGenerator = PhraseSuggestParser.parseCandidateGenerator(parser, - mockShardContext.getMapperService(), mockShardContext.parseFieldMatcher()); + mockMapperService, ParseFieldMatcher.EMPTY); // compare their properties assertNotSame(contextGenerator, secondGenerator); - assertEquals(contextGenerator.field(), secondGenerator.field()); - assertEquals(contextGenerator.accuracy(), secondGenerator.accuracy(), Float.MIN_VALUE); - assertEquals(contextGenerator.maxTermFreq(), secondGenerator.maxTermFreq(), Float.MIN_VALUE); - assertEquals(contextGenerator.maxEdits(), secondGenerator.maxEdits()); - assertEquals(contextGenerator.maxInspections(), secondGenerator.maxInspections()); - assertEquals(contextGenerator.minDocFreq(), secondGenerator.minDocFreq(), Float.MIN_VALUE); - assertEquals(contextGenerator.minWordLength(), secondGenerator.minWordLength()); - assertEquals(contextGenerator.postFilter(), secondGenerator.postFilter()); - assertEquals(contextGenerator.prefixLength(), secondGenerator.prefixLength()); - assertEquals(contextGenerator.preFilter(), secondGenerator.preFilter()); - assertEquals(contextGenerator.sort(), secondGenerator.sort()); - assertEquals(contextGenerator.size(), secondGenerator.size()); - // some instances of StringDistance don't support equals, just checking the class here - assertEquals(contextGenerator.stringDistance().getClass(), secondGenerator.stringDistance().getClass()); - assertEquals(contextGenerator.suggestMode(), secondGenerator.suggestMode()); + assertEqualGenerators(contextGenerator, secondGenerator); } } + public static void assertEqualGenerators(DirectCandidateGenerator first, DirectCandidateGenerator second) { + assertEquals(first.field(), second.field()); + assertEquals(first.accuracy(), second.accuracy(), Float.MIN_VALUE); + assertEquals(first.maxTermFreq(), second.maxTermFreq(), Float.MIN_VALUE); + assertEquals(first.maxEdits(), second.maxEdits()); + assertEquals(first.maxInspections(), second.maxInspections()); + assertEquals(first.minDocFreq(), second.minDocFreq(), Float.MIN_VALUE); + assertEquals(first.minWordLength(), second.minWordLength()); + assertEquals(first.postFilter(), second.postFilter()); + assertEquals(first.prefixLength(), second.prefixLength()); + assertEquals(first.preFilter(), second.preFilter()); + assertEquals(first.sort(), second.sort()); + assertEquals(first.size(), second.size()); + // some instances of StringDistance don't support equals, just checking the class here + assertEquals(first.stringDistance().getClass(), second.stringDistance().getClass()); + assertEquals(first.suggestMode(), second.suggestMode()); + } + /** * test that bad xContent throws exception */ diff --git a/core/src/test/java/org/elasticsearch/search/suggest/phrase/LaplaceModelTests.java b/core/src/test/java/org/elasticsearch/search/suggest/phrase/LaplaceModelTests.java index 1a939018b8..96ac0c9cb2 100644 --- a/core/src/test/java/org/elasticsearch/search/suggest/phrase/LaplaceModelTests.java +++ b/core/src/test/java/org/elasticsearch/search/suggest/phrase/LaplaceModelTests.java @@ -19,9 +19,6 @@ package org.elasticsearch.search.suggest.phrase; -import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder.Laplace; -import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder.SmoothingModel; - import static org.hamcrest.Matchers.instanceOf; public class LaplaceModelTests extends SmoothingModelTestCase { diff --git a/core/src/test/java/org/elasticsearch/search/suggest/phrase/LinearInterpolationModelTests.java b/core/src/test/java/org/elasticsearch/search/suggest/phrase/LinearInterpolationModelTests.java index 7984395abc..ed663ef524 100644 --- a/core/src/test/java/org/elasticsearch/search/suggest/phrase/LinearInterpolationModelTests.java +++ b/core/src/test/java/org/elasticsearch/search/suggest/phrase/LinearInterpolationModelTests.java @@ -19,9 +19,6 @@ package org.elasticsearch.search.suggest.phrase; -import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder.LinearInterpolation; -import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder.SmoothingModel; - import static org.hamcrest.Matchers.instanceOf; public class LinearInterpolationModelTests extends SmoothingModelTestCase { diff --git a/core/src/test/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestionBuilderTests.java b/core/src/test/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestionBuilderTests.java index d74719fa6f..2a553ef8cb 100644 --- a/core/src/test/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestionBuilderTests.java +++ b/core/src/test/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestionBuilderTests.java @@ -21,16 +21,17 @@ package org.elasticsearch.search.suggest.phrase; import org.elasticsearch.script.Template; import org.elasticsearch.search.suggest.AbstractSuggestionBuilderTestCase; -import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder.Laplace; -import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder.LinearInterpolation; -import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder.SmoothingModel; -import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder.StupidBackoff; +import org.elasticsearch.search.suggest.SuggestionSearchContext.SuggestionContext; +import org.elasticsearch.search.suggest.phrase.PhraseSuggestionContext.DirectCandidateGenerator; import org.junit.BeforeClass; import java.io.IOException; import java.util.HashMap; +import java.util.Iterator; import java.util.Map; +import static org.hamcrest.Matchers.instanceOf; + public class PhraseSuggestionBuilderTests extends AbstractSuggestionBuilderTestCase<PhraseSuggestionBuilder> { @BeforeClass @@ -70,7 +71,7 @@ public class PhraseSuggestionBuilderTests extends AbstractSuggestionBuilderTestC } maybeSet(testBuilder::gramSize, randomIntBetween(1, 5)); maybeSet(testBuilder::forceUnigrams, randomBoolean()); - maybeSet(testBuilder::tokenLimit, randomInt(20)); + maybeSet(testBuilder::tokenLimit, randomIntBetween(1, 20)); if (randomBoolean()) { testBuilder.smoothingModel(randomSmoothingModel()); } @@ -115,7 +116,7 @@ public class PhraseSuggestionBuilderTests extends AbstractSuggestionBuilderTestC builder.gramSize(randomValueOtherThan(builder.gramSize(), () -> randomIntBetween(1, 5))); break; case 4: - builder.tokenLimit(randomValueOtherThan(builder.tokenLimit(), () -> randomInt(20))); + builder.tokenLimit(randomValueOtherThan(builder.tokenLimit(), () -> randomIntBetween(1, 20))); break; case 5: builder.separator(randomValueOtherThan(builder.separator(), () -> randomAsciiOfLengthBetween(1, 10))); @@ -158,4 +159,38 @@ public class PhraseSuggestionBuilderTests extends AbstractSuggestionBuilderTestC } } + @Override + protected void assertSuggestionContext(SuggestionContext oldSuggestion, SuggestionContext newSuggestion) { + assertThat(oldSuggestion, instanceOf(PhraseSuggestionContext.class)); + assertThat(newSuggestion, instanceOf(PhraseSuggestionContext.class)); + PhraseSuggestionContext oldPhraseSuggestion = (PhraseSuggestionContext) oldSuggestion; + PhraseSuggestionContext newPhraseSuggestion = (PhraseSuggestionContext) newSuggestion; + assertEquals(oldPhraseSuggestion.confidence(), newPhraseSuggestion.confidence(), Float.MIN_VALUE); + assertEquals(oldPhraseSuggestion.collatePrune(), newPhraseSuggestion.collatePrune()); + assertEquals(oldPhraseSuggestion.gramSize(), newPhraseSuggestion.gramSize()); + assertEquals(oldPhraseSuggestion.realworldErrorLikelyhood(), newPhraseSuggestion.realworldErrorLikelyhood(), Float.MIN_VALUE); + assertEquals(oldPhraseSuggestion.maxErrors(), newPhraseSuggestion.maxErrors(), Float.MIN_VALUE); + assertEquals(oldPhraseSuggestion.separator(), newPhraseSuggestion.separator()); + assertEquals(oldPhraseSuggestion.getTokenLimit(), newPhraseSuggestion.getTokenLimit()); + assertEquals(oldPhraseSuggestion.getRequireUnigram(), newPhraseSuggestion.getRequireUnigram()); + assertEquals(oldPhraseSuggestion.getPreTag(), newPhraseSuggestion.getPreTag()); + assertEquals(oldPhraseSuggestion.getPostTag(), newPhraseSuggestion.getPostTag()); + if (oldPhraseSuggestion.getCollateQueryScript() != null) { + // only assert that we have a compiled script on the other side + assertNotNull(newPhraseSuggestion.getCollateQueryScript()); + } + if (oldPhraseSuggestion.generators() != null) { + assertNotNull(newPhraseSuggestion.generators()); + assertEquals(oldPhraseSuggestion.generators().size(), newPhraseSuggestion.generators().size()); + Iterator<DirectCandidateGenerator> secondList = newPhraseSuggestion.generators().iterator(); + for (DirectCandidateGenerator candidateGenerator : newPhraseSuggestion.generators()) { + DirectCandidateGeneratorTests.assertEqualGenerators(candidateGenerator, secondList.next()); + } + } + assertEquals(oldPhraseSuggestion.getCollateScriptParams(), newPhraseSuggestion.getCollateScriptParams()); + if (oldPhraseSuggestion.model() != null) { + assertNotNull(newPhraseSuggestion.model()); + } + } + } diff --git a/core/src/test/java/org/elasticsearch/search/suggest/phrase/SmoothingModelTestCase.java b/core/src/test/java/org/elasticsearch/search/suggest/phrase/SmoothingModelTestCase.java index 4672d9db97..b0912c3ac4 100644 --- a/core/src/test/java/org/elasticsearch/search/suggest/phrase/SmoothingModelTestCase.java +++ b/core/src/test/java/org/elasticsearch/search/suggest/phrase/SmoothingModelTestCase.java @@ -45,10 +45,6 @@ import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.query.QueryParseContext; import org.elasticsearch.indices.query.IndicesQueriesRegistry; -import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder.Laplace; -import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder.LinearInterpolation; -import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder.SmoothingModel; -import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder.StupidBackoff; import org.elasticsearch.test.ESTestCase; import org.junit.AfterClass; import org.junit.BeforeClass; diff --git a/core/src/test/java/org/elasticsearch/search/suggest/phrase/StupidBackoffModelTests.java b/core/src/test/java/org/elasticsearch/search/suggest/phrase/StupidBackoffModelTests.java index 3a59c19b13..1b6e1cf2c8 100644 --- a/core/src/test/java/org/elasticsearch/search/suggest/phrase/StupidBackoffModelTests.java +++ b/core/src/test/java/org/elasticsearch/search/suggest/phrase/StupidBackoffModelTests.java @@ -19,9 +19,6 @@ package org.elasticsearch.search.suggest.phrase; -import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder.SmoothingModel; -import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder.StupidBackoff; - import static org.hamcrest.Matchers.instanceOf; public class StupidBackoffModelTests extends SmoothingModelTestCase { diff --git a/core/src/test/java/org/elasticsearch/search/suggest/term/TermSuggestionBuilderTests.java b/core/src/test/java/org/elasticsearch/search/suggest/term/TermSuggestionBuilderTests.java index ca5f3f880e..f389b0cb18 100644 --- a/core/src/test/java/org/elasticsearch/search/suggest/term/TermSuggestionBuilderTests.java +++ b/core/src/test/java/org/elasticsearch/search/suggest/term/TermSuggestionBuilderTests.java @@ -20,6 +20,7 @@ package org.elasticsearch.search.suggest.term; import org.elasticsearch.search.suggest.AbstractSuggestionBuilderTestCase; +import org.elasticsearch.search.suggest.SuggestionSearchContext.SuggestionContext; import org.elasticsearch.search.suggest.term.TermSuggestionBuilder.SortBy; import org.elasticsearch.search.suggest.term.TermSuggestionBuilder.StringDistanceImpl; import org.elasticsearch.search.suggest.term.TermSuggestionBuilder.SuggestMode; @@ -33,6 +34,14 @@ import static org.hamcrest.Matchers.notNullValue; */ public class TermSuggestionBuilderTests extends AbstractSuggestionBuilderTestCase<TermSuggestionBuilder> { + /** + * creates random suggestion builder, renders it to xContent and back to new instance that should be equal to original + */ + @Override + public void testBuild() throws IOException { + // skip for now + } + @Override protected TermSuggestionBuilder randomSuggestionBuilder() { TermSuggestionBuilder testBuilder = new TermSuggestionBuilder(randomAsciiOfLength(10)); @@ -245,4 +254,9 @@ public class TermSuggestionBuilderTests extends AbstractSuggestionBuilderTestCas assertThat(builder.suggestMode(), notNullValue()); } + @Override + protected void assertSuggestionContext(SuggestionContext oldSuggestion, SuggestionContext newSuggestion) { + // put assertions on TermSuggestionContext here + } + } diff --git a/modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/SuggestSearchTests.java b/modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/SuggestSearchTests.java index d66ee0a6b3..1f2952e361 100644 --- a/modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/SuggestSearchTests.java +++ b/modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/SuggestSearchTests.java @@ -20,38 +20,6 @@ package org.elasticsearch.messy.tests; -import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS; -import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS; -import static org.elasticsearch.common.settings.Settings.settingsBuilder; -import static org.elasticsearch.index.query.QueryBuilders.matchQuery; -import static org.elasticsearch.search.suggest.SuggestBuilders.phraseSuggestion; -import static org.elasticsearch.search.suggest.SuggestBuilders.termSuggestion; -import static org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder.candidateGenerator; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSuggestion; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSuggestionPhraseCollateMatchExists; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSuggestionSize; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertThrows; -import static org.hamcrest.Matchers.anyOf; -import static org.hamcrest.Matchers.endsWith; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.instanceOf; -import static org.hamcrest.Matchers.nullValue; - -import java.io.IOException; -import java.net.URISyntaxException; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.ExecutionException; - import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder; import org.elasticsearch.action.index.IndexRequestBuilder; @@ -71,13 +39,47 @@ import org.elasticsearch.search.suggest.Suggest; import org.elasticsearch.search.suggest.SuggestBuilder; import org.elasticsearch.search.suggest.SuggestionBuilder; import org.elasticsearch.search.suggest.phrase.DirectCandidateGeneratorBuilder; +import org.elasticsearch.search.suggest.phrase.Laplace; +import org.elasticsearch.search.suggest.phrase.LinearInterpolation; import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder; +import org.elasticsearch.search.suggest.phrase.StupidBackoff; import org.elasticsearch.search.suggest.term.TermSuggestionBuilder; import org.elasticsearch.search.suggest.term.TermSuggestionBuilder.SortBy; import org.elasticsearch.search.suggest.term.TermSuggestionBuilder.SuggestMode; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.hamcrest.ElasticsearchAssertions; +import java.io.IOException; +import java.net.URISyntaxException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ExecutionException; + +import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS; +import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS; +import static org.elasticsearch.common.settings.Settings.settingsBuilder; +import static org.elasticsearch.index.query.QueryBuilders.matchQuery; +import static org.elasticsearch.search.suggest.SuggestBuilders.phraseSuggestion; +import static org.elasticsearch.search.suggest.SuggestBuilders.termSuggestion; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSuggestion; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSuggestionPhraseCollateMatchExists; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSuggestionSize; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertThrows; +import static org.hamcrest.Matchers.anyOf; +import static org.hamcrest.Matchers.endsWith; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.nullValue; + /** * Integration tests for term and phrase suggestions. Many of these tests many requests that vary only slightly from one another. Where * possible these tests should declare for the first request, make the request, modify the configuration for the next request, make that @@ -227,6 +229,16 @@ public class SuggestSearchTests extends ESIntegTestCase { assertSuggestionSize(searchSuggest, 0, 0, "did_you_mean"); } + /** + * Creates a new {@link DirectCandidateGeneratorBuilder} + * + * @param field + * the field this candidate generator operates on. + */ + private DirectCandidateGeneratorBuilder candidateGenerator(String field) { + return new DirectCandidateGeneratorBuilder(field); + } + // see #2729 public void testSizeOneShard() throws Exception { prepareCreate("test").setSettings( @@ -286,7 +298,7 @@ public class SuggestSearchTests extends ESIntegTestCase { refresh(); PhraseSuggestionBuilder phraseSuggestion = phraseSuggestion("did_you_mean").field("name.shingled") - .addCandidateGenerator(PhraseSuggestionBuilder.candidateGenerator("name").prefixLength(0).minWordLength(0).suggestMode("always").maxEdits(2)) + .addCandidateGenerator(candidateGenerator("name").prefixLength(0).minWordLength(0).suggestMode("always").maxEdits(2)) .gramSize(3); Suggest searchSuggest = searchSuggest( "ice tea", phraseSuggestion); assertSuggestion(searchSuggest, 0, 0, "did_you_mean", "iced tea"); @@ -439,7 +451,7 @@ public class SuggestSearchTests extends ESIntegTestCase { Suggest searchSuggest = searchSuggest( "a an the", phraseSuggestion("simple_phrase").field("body").gramSize(1) - .addCandidateGenerator(PhraseSuggestionBuilder.candidateGenerator("body").minWordLength(1).suggestMode("always")) + .addCandidateGenerator(candidateGenerator("body").minWordLength(1).suggestMode("always")) .size(1)); assertSuggestionSize(searchSuggest, 0, 0, "simple_phrase"); } @@ -475,13 +487,13 @@ public class SuggestSearchTests extends ESIntegTestCase { Suggest searchSuggest = searchSuggest( "hello word", phraseSuggestion("simple_phrase").field("body") - .addCandidateGenerator(PhraseSuggestionBuilder.candidateGenerator("body").prefixLength(4).minWordLength(1).suggestMode("always")) + .addCandidateGenerator(candidateGenerator("body").prefixLength(4).minWordLength(1).suggestMode("always")) .size(1).confidence(1.0f)); assertSuggestion(searchSuggest, 0, "simple_phrase", "hello words"); searchSuggest = searchSuggest( "hello word", phraseSuggestion("simple_phrase").field("body") - .addCandidateGenerator(PhraseSuggestionBuilder.candidateGenerator("body").prefixLength(2).minWordLength(1).suggestMode("always")) + .addCandidateGenerator(candidateGenerator("body").prefixLength(2).minWordLength(1).suggestMode("always")) .size(1).confidence(1.0f)); assertSuggestion(searchSuggest, 0, "simple_phrase", "hello world"); } @@ -573,17 +585,17 @@ public class SuggestSearchTests extends ESIntegTestCase { // set all mass to trigrams (not indexed) phraseSuggest.clearCandidateGenerators() .addCandidateGenerator(candidateGenerator("body").minWordLength(1).suggestMode("always")) - .smoothingModel(new PhraseSuggestionBuilder.LinearInterpolation(1,0,0)); + .smoothingModel(new LinearInterpolation(1,0,0)); searchSuggest = searchSuggest( "Xor the Got-Jewel", phraseSuggest); assertSuggestionSize(searchSuggest, 0, 0, "simple_phrase"); // set all mass to bigrams - phraseSuggest.smoothingModel(new PhraseSuggestionBuilder.LinearInterpolation(0,1,0)); + phraseSuggest.smoothingModel(new LinearInterpolation(0,1,0)); searchSuggest = searchSuggest( "Xor the Got-Jewel", phraseSuggest); assertSuggestion(searchSuggest, 0, "simple_phrase", "xorr the god jewel"); // distribute mass - phraseSuggest.smoothingModel(new PhraseSuggestionBuilder.LinearInterpolation(0.4,0.4,0.2)); + phraseSuggest.smoothingModel(new LinearInterpolation(0.4,0.4,0.2)); searchSuggest = searchSuggest( "Xor the Got-Jewel", phraseSuggest); assertSuggestion(searchSuggest, 0, "simple_phrase", "xorr the god jewel"); @@ -591,15 +603,15 @@ public class SuggestSearchTests extends ESIntegTestCase { assertSuggestion(searchSuggest, 0, "simple_phrase", "american ace"); // try all smoothing methods - phraseSuggest.smoothingModel(new PhraseSuggestionBuilder.LinearInterpolation(0.4,0.4,0.2)); + phraseSuggest.smoothingModel(new LinearInterpolation(0.4,0.4,0.2)); searchSuggest = searchSuggest( "Xor the Got-Jewel", phraseSuggest); assertSuggestion(searchSuggest, 0, "simple_phrase", "xorr the god jewel"); - phraseSuggest.smoothingModel(new PhraseSuggestionBuilder.Laplace(0.2)); + phraseSuggest.smoothingModel(new Laplace(0.2)); searchSuggest = searchSuggest( "Xor the Got-Jewel", phraseSuggest); assertSuggestion(searchSuggest, 0, "simple_phrase", "xorr the god jewel"); - phraseSuggest.smoothingModel(new PhraseSuggestionBuilder.StupidBackoff(0.1)); + phraseSuggest.smoothingModel(new StupidBackoff(0.1)); searchSuggest = searchSuggest( "Xor the Got-Jewel", phraseSuggest); assertSuggestion(searchSuggest, 0, "simple_phrase", "xorr the god jewel"); @@ -608,7 +620,7 @@ public class SuggestSearchTests extends ESIntegTestCase { searchSuggest = searchSuggest( "Xor the Got-Jewel", phraseSuggest); assertSuggestionSize(searchSuggest, 0, 0, "simple_phrase"); - phraseSuggest.tokenLimit(15).smoothingModel(new PhraseSuggestionBuilder.StupidBackoff(0.1)); + phraseSuggest.tokenLimit(15).smoothingModel(new StupidBackoff(0.1)); searchSuggest = searchSuggest( "Xor the Got-Jewel Xor the Got-Jewel Xor the Got-Jewel", phraseSuggest); assertSuggestion(searchSuggest, 0, "simple_phrase", "xorr the god jewel xorr the god jewel xorr the god jewel"); // Check the name this time because we're repeating it which is funky @@ -671,7 +683,7 @@ public class SuggestSearchTests extends ESIntegTestCase { .gramSize(2) .analyzer("body") .addCandidateGenerator(candidateGenerator("body").minWordLength(1).prefixLength(1).suggestMode("always").size(1).accuracy(0.1f)) - .smoothingModel(new PhraseSuggestionBuilder.StupidBackoff(0.1)) + .smoothingModel(new StupidBackoff(0.1)) .maxErrors(1.0f) .size(5); Suggest searchSuggest = searchSuggest( "Xorr the Gut-Jewel", phraseSuggestion); @@ -931,7 +943,7 @@ public class SuggestSearchTests extends ESIntegTestCase { Suggest searchSuggest = searchSuggest("nobel prize", phraseSuggestion("simple_phrase") .field("body") - .addCandidateGenerator(PhraseSuggestionBuilder.candidateGenerator("body").minWordLength(1).suggestMode("always").maxTermFreq(.99f)) + .addCandidateGenerator(candidateGenerator("body").minWordLength(1).suggestMode("always").maxTermFreq(.99f)) .confidence(2f) .maxErrors(5f) .size(1)); @@ -939,7 +951,7 @@ public class SuggestSearchTests extends ESIntegTestCase { searchSuggest = searchSuggest("noble prize", phraseSuggestion("simple_phrase") .field("body") - .addCandidateGenerator(PhraseSuggestionBuilder.candidateGenerator("body").minWordLength(1).suggestMode("always").maxTermFreq(.99f)) + .addCandidateGenerator(candidateGenerator("body").minWordLength(1).suggestMode("always").maxTermFreq(.99f)) .confidence(2f) .maxErrors(5f) .size(1)); @@ -1070,7 +1082,7 @@ public class SuggestSearchTests extends ESIntegTestCase { PhraseSuggestionBuilder suggest = phraseSuggestion("title") .field("title") - .addCandidateGenerator(PhraseSuggestionBuilder.candidateGenerator("title") + .addCandidateGenerator(candidateGenerator("title") .suggestMode("always") .maxTermFreq(.99f) .size(1000) // Setting a silly high size helps of generate a larger list of candidates for testing. @@ -1135,7 +1147,7 @@ public class SuggestSearchTests extends ESIntegTestCase { // suggest without collate PhraseSuggestionBuilder suggest = phraseSuggestion("title") .field("title") - .addCandidateGenerator(PhraseSuggestionBuilder.candidateGenerator("title") + .addCandidateGenerator(candidateGenerator("title") .suggestMode("always") .maxTermFreq(.99f) .size(10) |