diff options
author | Christoph Büscher <christoph@elastic.co> | 2016-02-10 19:05:13 +0100 |
---|---|---|
committer | Christoph Büscher <christoph@elastic.co> | 2016-02-10 19:05:13 +0100 |
commit | bbeb09eae7ac3c5d9837bb26eacfac6bba468929 (patch) | |
tree | 00a5f8b86fc78b8510c428888aa1011931fa18b9 /core/src/main/java/org | |
parent | 421ed1228b545a07db4710f0ee56959d70e64d45 (diff) | |
parent | 9e0f6e3f9c3ba7a539dcb9366529db5ab3295d61 (diff) |
Merge pull request #16507 from cbuescher/phrase-suggest-build
Add build method to create SuggestionContext to PhraseSuggestionBuilder
Diffstat (limited to 'core/src/main/java/org')
29 files changed, 825 insertions, 541 deletions
diff --git a/core/src/main/java/org/elasticsearch/action/suggest/TransportSuggestAction.java b/core/src/main/java/org/elasticsearch/action/suggest/TransportSuggestAction.java index 0ed9857855..616dbf9493 100644 --- a/core/src/main/java/org/elasticsearch/action/suggest/TransportSuggestAction.java +++ b/core/src/main/java/org/elasticsearch/action/suggest/TransportSuggestAction.java @@ -142,8 +142,7 @@ public class TransportSuggestAction extends TransportBroadcastAction<SuggestRequ if (parser.nextToken() != XContentParser.Token.START_OBJECT) { throw new IllegalArgumentException("suggest content missing"); } - final SuggestionSearchContext context = suggestPhase.parseElement().parseInternal(parser, indexService.mapperService(), - indexService.fieldData(), request.shardId().getIndexName(), request.shardId().id()); + final SuggestionSearchContext context = suggestPhase.parseElement().parseInternal(parser, indexService.newQueryShardContext()); final Suggest result = suggestPhase.execute(context, searcher.searcher()); return new ShardSuggestResponse(request.shardId(), result); } diff --git a/core/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java b/core/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java index d84328d68e..8c2dc444d2 100644 --- a/core/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java +++ b/core/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java @@ -39,7 +39,7 @@ import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.functionscore.ScoreFunctionBuilder; import org.elasticsearch.search.rescore.RescoreBuilder; import org.elasticsearch.search.suggest.SuggestionBuilder; -import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder.SmoothingModel; +import org.elasticsearch.search.suggest.phrase.SmoothingModel; import org.elasticsearch.tasks.Task; import org.joda.time.DateTime; import org.joda.time.DateTimeZone; diff --git a/core/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java b/core/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java index a7f1480623..d14d6e77ff 100644 --- a/core/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java +++ b/core/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java @@ -38,7 +38,7 @@ import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.functionscore.ScoreFunctionBuilder; import org.elasticsearch.search.rescore.RescoreBuilder; import org.elasticsearch.search.suggest.SuggestionBuilder; -import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder.SmoothingModel; +import org.elasticsearch.search.suggest.phrase.SmoothingModel; import org.elasticsearch.tasks.Task; import org.joda.time.ReadableInstant; diff --git a/core/src/main/java/org/elasticsearch/indices/query/IndicesQueriesRegistry.java b/core/src/main/java/org/elasticsearch/indices/query/IndicesQueriesRegistry.java index a9e90884a6..b0b212d2ab 100644 --- a/core/src/main/java/org/elasticsearch/indices/query/IndicesQueriesRegistry.java +++ b/core/src/main/java/org/elasticsearch/indices/query/IndicesQueriesRegistry.java @@ -19,12 +19,12 @@ package org.elasticsearch.indices.query; -import java.util.Map; - import org.elasticsearch.common.component.AbstractComponent; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.query.QueryParser; +import java.util.Map; + public class IndicesQueriesRegistry extends AbstractComponent { private Map<String, QueryParser<?>> queryParsers; diff --git a/core/src/main/java/org/elasticsearch/search/SearchService.java b/core/src/main/java/org/elasticsearch/search/SearchService.java index cf9c0cebce..ff6d8897d5 100644 --- a/core/src/main/java/org/elasticsearch/search/SearchService.java +++ b/core/src/main/java/org/elasticsearch/search/SearchService.java @@ -751,7 +751,7 @@ public class SearchService extends AbstractLifecycleComponent<SearchService> imp if (source.rescores() != null) { try { for (RescoreBuilder<?> rescore : source.rescores()) { - context.addRescore(rescore.build(context.getQueryShardContext())); + context.addRescore(rescore.build(queryShardContext)); } } catch (IOException e) { throw new SearchContextException(context, "failed to create RescoreSearchContext", e); @@ -776,7 +776,7 @@ public class SearchService extends AbstractLifecycleComponent<SearchService> imp if (source.highlighter() != null) { HighlightBuilder highlightBuilder = source.highlighter(); try { - context.highlight(highlightBuilder.build(context.getQueryShardContext())); + context.highlight(highlightBuilder.build(queryShardContext)); } catch (IOException e) { throw new SearchContextException(context, "failed to create SearchContextHighlighter", e); } diff --git a/core/src/main/java/org/elasticsearch/search/suggest/SuggestBuilder.java b/core/src/main/java/org/elasticsearch/search/suggest/SuggestBuilder.java index d16e8e1d84..2852204bb6 100644 --- a/core/src/main/java/org/elasticsearch/search/suggest/SuggestBuilder.java +++ b/core/src/main/java/org/elasticsearch/search/suggest/SuggestBuilder.java @@ -26,9 +26,12 @@ import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.lucene.BytesRefs; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.query.QueryParseContext; +import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.search.suggest.SuggestionSearchContext.SuggestionContext; import java.io.IOException; import java.util.ArrayList; @@ -137,6 +140,21 @@ public class SuggestBuilder extends ToXContentToBytes implements Writeable<Sugge return suggestBuilder; } + public SuggestionSearchContext build(QueryShardContext context) throws IOException { + SuggestionSearchContext suggestionSearchContext = new SuggestionSearchContext(); + for (SuggestionBuilder<?> suggestionBuilder : suggestions) { + SuggestionContext suggestionContext = suggestionBuilder.build(context); + if (suggestionContext.getText() == null) { + if (globalText == null) { + throw new IllegalArgumentException("The required text option is missing"); + } + suggestionContext.setText(BytesRefs.toBytesRef(globalText)); + } + suggestionSearchContext.addSuggestion(suggestionBuilder.name(), suggestionContext); + } + return suggestionSearchContext; + } + @Override public SuggestBuilder readFrom(StreamInput in) throws IOException { final SuggestBuilder builder = new SuggestBuilder(); diff --git a/core/src/main/java/org/elasticsearch/search/suggest/SuggestContextParser.java b/core/src/main/java/org/elasticsearch/search/suggest/SuggestContextParser.java index a7aa3fd60b..53d510bf53 100644 --- a/core/src/main/java/org/elasticsearch/search/suggest/SuggestContextParser.java +++ b/core/src/main/java/org/elasticsearch/search/suggest/SuggestContextParser.java @@ -19,12 +19,11 @@ package org.elasticsearch.search.suggest; import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.index.fielddata.IndexFieldDataService; -import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.index.query.QueryShardContext; import java.io.IOException; public interface SuggestContextParser { - SuggestionSearchContext.SuggestionContext parse(XContentParser parser, MapperService mapperService, IndexFieldDataService indexFieldDataService) throws IOException; + SuggestionSearchContext.SuggestionContext parse(XContentParser parser, QueryShardContext shardContext) throws IOException; } diff --git a/core/src/main/java/org/elasticsearch/search/suggest/SuggestParseElement.java b/core/src/main/java/org/elasticsearch/search/suggest/SuggestParseElement.java index cf6b391ec6..b9454dc264 100644 --- a/core/src/main/java/org/elasticsearch/search/suggest/SuggestParseElement.java +++ b/core/src/main/java/org/elasticsearch/search/suggest/SuggestParseElement.java @@ -21,8 +21,8 @@ package org.elasticsearch.search.suggest; import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.index.fielddata.IndexFieldDataService; import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.search.SearchParseElement; import org.elasticsearch.search.internal.SearchContext; import org.elasticsearch.search.suggest.SuggestionSearchContext.SuggestionContext; @@ -44,14 +44,13 @@ public final class SuggestParseElement implements SearchParseElement { @Override public void parse(XContentParser parser, SearchContext context) throws Exception { - SuggestionSearchContext suggestionSearchContext = parseInternal(parser, context.mapperService(), context.fieldData(), - context.shardTarget().index(), context.shardTarget().shardId()); + SuggestionSearchContext suggestionSearchContext = parseInternal(parser, context.getQueryShardContext()); context.suggest(suggestionSearchContext); } - public SuggestionSearchContext parseInternal(XContentParser parser, MapperService mapperService, IndexFieldDataService fieldDataService, - String index, int shardId) throws IOException { + public SuggestionSearchContext parseInternal(XContentParser parser, QueryShardContext shardContext) throws IOException { SuggestionSearchContext suggestionSearchContext = new SuggestionSearchContext(); + MapperService mapperService = shardContext.getMapperService(); BytesRef globalText = null; String fieldName = null; @@ -95,10 +94,20 @@ public final class SuggestParseElement implements SearchParseElement { throw new IllegalArgumentException("Suggester[" + fieldName + "] not supported"); } final SuggestContextParser contextParser = suggesters.get(fieldName).getContextParser(); - suggestionContext = contextParser.parse(parser, mapperService, fieldDataService); + suggestionContext = contextParser.parse(parser, shardContext); } } if (suggestionContext != null) { + if (suggestText != null) { + suggestionContext.setText(suggestText); + } + if (prefix != null) { + suggestionContext.setPrefix(prefix); + } + if (regex != null) { + suggestionContext.setRegex(regex); + } + if (suggestText != null && prefix == null) { suggestionContext.setPrefix(suggestText); suggestionContext.setText(suggestText); @@ -110,6 +119,8 @@ public final class SuggestParseElement implements SearchParseElement { suggestionContext.setText(regex); } suggestionContexts.put(suggestionName, suggestionContext); + } else { + throw new IllegalArgumentException("suggestion context could not be parsed correctly"); } } } @@ -117,9 +128,6 @@ public final class SuggestParseElement implements SearchParseElement { for (Map.Entry<String, SuggestionContext> entry : suggestionContexts.entrySet()) { String suggestionName = entry.getKey(); SuggestionContext suggestionContext = entry.getValue(); - - suggestionContext.setShard(shardId); - suggestionContext.setIndex(index); SuggestUtils.verifySuggestion(mapperService, globalText, suggestionContext); suggestionSearchContext.addSuggestion(suggestionName, suggestionContext); } diff --git a/core/src/main/java/org/elasticsearch/search/suggest/SuggestUtils.java b/core/src/main/java/org/elasticsearch/search/suggest/SuggestUtils.java index 989546d50b..03fb785b91 100644 --- a/core/src/main/java/org/elasticsearch/search/suggest/SuggestUtils.java +++ b/core/src/main/java/org/elasticsearch/search/suggest/SuggestUtils.java @@ -271,10 +271,10 @@ public final class SuggestUtils { return false; } return true; - } + public static void verifySuggestion(MapperService mapperService, BytesRef globalText, SuggestionContext suggestion) { // Verify options and set defaults if (suggestion.getField() == null) { @@ -294,7 +294,6 @@ public final class SuggestUtils { } } - public static ShingleTokenFilterFactory.Factory getShingleFilterFactory(Analyzer analyzer) { if (analyzer instanceof NamedAnalyzer) { analyzer = ((NamedAnalyzer)analyzer).analyzer(); diff --git a/core/src/main/java/org/elasticsearch/search/suggest/Suggesters.java b/core/src/main/java/org/elasticsearch/search/suggest/Suggesters.java index c26649f638..9857a06da6 100644 --- a/core/src/main/java/org/elasticsearch/search/suggest/Suggesters.java +++ b/core/src/main/java/org/elasticsearch/search/suggest/Suggesters.java @@ -20,8 +20,6 @@ package org.elasticsearch.search.suggest; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.util.ExtensionPoint; -import org.elasticsearch.indices.IndicesService; -import org.elasticsearch.script.ScriptService; import org.elasticsearch.search.suggest.completion.CompletionSuggester; import org.elasticsearch.search.suggest.phrase.PhraseSuggester; import org.elasticsearch.search.suggest.term.TermSuggester; @@ -42,21 +40,17 @@ public final class Suggesters extends ExtensionPoint.ClassMap<Suggester> { this(Collections.emptyMap()); } + @Inject public Suggesters(Map<String, Suggester> suggesters) { super("suggester", Suggester.class, new HashSet<>(Arrays.asList("phrase", "term", "completion")), Suggesters.class, SuggestParseElement.class, SuggestPhase.class); - this.parsers = Collections.unmodifiableMap(suggesters); - } - - @Inject - public Suggesters(Map<String, Suggester> suggesters, ScriptService scriptService, IndicesService indexServices) { - this(addBuildIns(suggesters, scriptService, indexServices)); + this.parsers = Collections.unmodifiableMap(addBuildIns(suggesters)); } - private static Map<String, Suggester> addBuildIns(Map<String, Suggester> suggesters, ScriptService scriptService, IndicesService indexServices) { + private static Map<String, Suggester> addBuildIns(Map<String, Suggester> suggesters) { final Map<String, Suggester> map = new HashMap<>(); - map.put("phrase", new PhraseSuggester(scriptService, indexServices)); - map.put("term", new TermSuggester()); - map.put("completion", new CompletionSuggester()); + map.put("phrase", PhraseSuggester.PROTOTYPE); + map.put("term", TermSuggester.PROTOTYPE); + map.put("completion", CompletionSuggester.PROTOTYPE); map.putAll(suggesters); return map; } diff --git a/core/src/main/java/org/elasticsearch/search/suggest/SuggestionBuilder.java b/core/src/main/java/org/elasticsearch/search/suggest/SuggestionBuilder.java index 1fdb38df88..70f3d061b4 100644 --- a/core/src/main/java/org/elasticsearch/search/suggest/SuggestionBuilder.java +++ b/core/src/main/java/org/elasticsearch/search/suggest/SuggestionBuilder.java @@ -19,15 +19,20 @@ package org.elasticsearch.search.suggest; +import org.apache.lucene.analysis.Analyzer; import org.elasticsearch.action.support.ToXContentToBytes; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.ParseFieldMatcher; import org.elasticsearch.common.io.stream.NamedWriteable; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.lucene.BytesRefs; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.query.QueryParseContext; +import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.search.suggest.SuggestionSearchContext.SuggestionContext; import java.io.IOException; import java.util.Objects; @@ -192,6 +197,56 @@ public abstract class SuggestionBuilder<T extends SuggestionBuilder<T>> extends protected abstract SuggestionBuilder<T> innerFromXContent(QueryParseContext parseContext, String name) throws IOException; + protected abstract SuggestionContext build(QueryShardContext context) throws IOException; + + /** + * Transfers the text, prefix, regex, analyzer, fieldname, size and shard size settings from the + * original {@link SuggestionBuilder} to the target {@link SuggestionContext} + */ + protected void populateCommonFields(MapperService mapperService, + SuggestionSearchContext.SuggestionContext suggestionContext) throws IOException { + + if (analyzer != null) { + Analyzer luceneAnalyzer = mapperService.analysisService().analyzer(analyzer); + if (luceneAnalyzer == null) { + throw new IllegalArgumentException("Analyzer [" + luceneAnalyzer + "] doesn't exists"); + } + suggestionContext.setAnalyzer(luceneAnalyzer); + } + + if (fieldname != null) { + suggestionContext.setField(fieldname); + } + + if (size != null) { + suggestionContext.setSize(size); + } + + if (shardSize != null) { + suggestionContext.setShardSize(shardSize); + } else { + // if no shard size is set in builder, use size (or at least 5) + suggestionContext.setShardSize(Math.max(suggestionContext.getSize(), 5)); + } + + if (text != null) { + suggestionContext.setText(BytesRefs.toBytesRef(text)); + } + if (prefix != null) { + suggestionContext.setPrefix(BytesRefs.toBytesRef(prefix)); + } + if (regex != null) { + suggestionContext.setRegex(BytesRefs.toBytesRef(regex)); + } + if (text != null && prefix == null) { + suggestionContext.setPrefix(BytesRefs.toBytesRef(text)); + } else if (text == null && prefix != null) { + suggestionContext.setText(BytesRefs.toBytesRef(prefix)); + } else if (text == null && regex != null) { + suggestionContext.setText(BytesRefs.toBytesRef(regex)); + } + } + private String getSuggesterName() { //default impl returns the same as writeable name, but we keep the distinction between the two just to make sure return getWriteableName(); diff --git a/core/src/main/java/org/elasticsearch/search/suggest/SuggestionSearchContext.java b/core/src/main/java/org/elasticsearch/search/suggest/SuggestionSearchContext.java index 1d3339e057..fa468e6ce9 100644 --- a/core/src/main/java/org/elasticsearch/search/suggest/SuggestionSearchContext.java +++ b/core/src/main/java/org/elasticsearch/search/suggest/SuggestionSearchContext.java @@ -20,6 +20,7 @@ package org.elasticsearch.search.suggest; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.index.query.QueryShardContext; import java.util.LinkedHashMap; import java.util.Map; @@ -36,20 +37,24 @@ public class SuggestionSearchContext { public Map<String, SuggestionContext> suggestions() { return suggestions; } - - public static class SuggestionContext { - + + public abstract static class SuggestionContext { + private BytesRef text; private BytesRef prefix; private BytesRef regex; - private final Suggester suggester; private String field; private Analyzer analyzer; private int size = 5; private int shardSize = -1; - private int shardId; - private String index; - + private QueryShardContext shardContext; + private Suggester<?> suggester; + + protected SuggestionContext(Suggester<?> suggester, QueryShardContext shardContext) { + this.suggester = suggester; + this.shardContext = shardContext; + } + public BytesRef getText() { return text; } @@ -74,12 +79,8 @@ public class SuggestionSearchContext { this.regex = regex; } - public SuggestionContext(Suggester suggester) { - this.suggester = suggester; - } - public Suggester<SuggestionContext> getSuggester() { - return this.suggester; + return ((Suggester<SuggestionContext>) suggester); } public Analyzer getAnalyzer() { @@ -119,21 +120,9 @@ public class SuggestionSearchContext { } this.shardSize = shardSize; } - - public void setShard(int shardId) { - this.shardId = shardId; - } - public void setIndex(String index) { - this.index = index; - } - - public String getIndex() { - return index; - } - - public int getShard() { - return shardId; + public QueryShardContext getShardContext() { + return this.shardContext; } } diff --git a/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestParser.java b/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestParser.java index 702b03f359..9d29525115 100644 --- a/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestParser.java +++ b/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestParser.java @@ -20,17 +20,16 @@ package org.elasticsearch.search.suggest.completion; import org.apache.lucene.analysis.Analyzer; import org.elasticsearch.ElasticsearchException; -import org.elasticsearch.common.ParseField; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.unit.Fuzziness; import org.elasticsearch.common.xcontent.ObjectParser; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.index.fielddata.IndexFieldDataService; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.core.CompletionFieldMapper; +import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.index.query.RegexpFlag; import org.elasticsearch.search.suggest.SuggestContextParser; import org.elasticsearch.search.suggest.SuggestUtils.Fields; @@ -135,8 +134,9 @@ public class CompletionSuggestParser implements SuggestContextParser { } @Override - public SuggestionSearchContext.SuggestionContext parse(XContentParser parser, MapperService mapperService, IndexFieldDataService fieldDataService) throws IOException { - final CompletionSuggestionContext suggestion = new CompletionSuggestionContext(completionSuggester, mapperService, fieldDataService); + public SuggestionSearchContext.SuggestionContext parse(XContentParser parser, QueryShardContext shardContext) throws IOException { + MapperService mapperService = shardContext.getMapperService(); + final CompletionSuggestionContext suggestion = new CompletionSuggestionContext(shardContext); final ContextAndSuggest contextAndSuggest = new ContextAndSuggest(mapperService); TLP_PARSER.parse(parser, suggestion, contextAndSuggest); final XContentParser contextParser = contextAndSuggest.contextParser; diff --git a/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggester.java b/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggester.java index 8cd9d386a1..be90a2e7e7 100644 --- a/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggester.java +++ b/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggester.java @@ -34,7 +34,9 @@ import org.elasticsearch.common.text.Text; import org.elasticsearch.index.fielddata.AtomicFieldData; import org.elasticsearch.index.fielddata.ScriptDocValues; import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.core.CompletionFieldMapper; +import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.search.suggest.Suggest; import org.elasticsearch.search.suggest.SuggestContextParser; import org.elasticsearch.search.suggest.Suggester; @@ -51,6 +53,8 @@ import java.util.Set; public class CompletionSuggester extends Suggester<CompletionSuggestionContext> { + public static final CompletionSuggester PROTOTYPE = new CompletionSuggester(); + @Override public SuggestContextParser getContextParser() { return new CompletionSuggestParser(this); @@ -86,9 +90,11 @@ public class CompletionSuggester extends Suggester<CompletionSuggestionContext> final LeafReaderContext subReaderContext = leaves.get(readerIndex); final int subDocId = suggestDoc.doc - subReaderContext.docBase; for (String field : payloadFields) { - MappedFieldType payloadFieldType = suggestionContext.getMapperService().fullName(field); + MapperService mapperService = suggestionContext.getShardContext().getMapperService(); + MappedFieldType payloadFieldType = mapperService.fullName(field); if (payloadFieldType != null) { - final AtomicFieldData data = suggestionContext.getIndexFieldDataService().getForField(payloadFieldType) + QueryShardContext shardContext = suggestionContext.getShardContext(); + final AtomicFieldData data = shardContext.getForField(payloadFieldType) .load(subReaderContext); final ScriptDocValues scriptValues = data.getScriptValues(); scriptValues.setNextDocId(subDocId); diff --git a/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestionBuilder.java b/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestionBuilder.java index 29992c1a07..0bd37be128 100644 --- a/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestionBuilder.java +++ b/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestionBuilder.java @@ -28,8 +28,10 @@ import org.elasticsearch.common.unit.Fuzziness; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.index.query.QueryParseContext; +import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.index.query.RegexpFlag; import org.elasticsearch.search.suggest.SuggestionBuilder; +import org.elasticsearch.search.suggest.SuggestionSearchContext.SuggestionContext; import org.elasticsearch.search.suggest.completion.context.CategoryQueryContext; import org.elasticsearch.search.suggest.completion.context.GeoQueryContext; @@ -372,10 +374,17 @@ public class CompletionSuggestionBuilder extends SuggestionBuilder<CompletionSug @Override protected CompletionSuggestionBuilder innerFromXContent(QueryParseContext parseContext, String name) throws IOException { + // NORELEASE return new CompletionSuggestionBuilder(name); } @Override + protected SuggestionContext build(QueryShardContext context) throws IOException { + // NORELEASE + throw new UnsupportedOperationException(); + } + + @Override public String getWriteableName() { return SUGGESTION_NAME; } diff --git a/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestionContext.java b/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestionContext.java index 535151b476..f6d6de88f4 100644 --- a/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestionContext.java +++ b/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestionContext.java @@ -20,10 +20,8 @@ package org.elasticsearch.search.suggest.completion; import org.apache.lucene.search.suggest.document.CompletionQuery; import org.elasticsearch.common.unit.Fuzziness; -import org.elasticsearch.index.fielddata.IndexFieldDataService; -import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.core.CompletionFieldMapper; -import org.elasticsearch.search.suggest.Suggester; +import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.search.suggest.SuggestionSearchContext; import org.elasticsearch.search.suggest.completion.context.ContextMapping; import org.elasticsearch.search.suggest.completion.context.ContextMappings; @@ -39,20 +37,16 @@ import java.util.Set; */ public class CompletionSuggestionContext extends SuggestionSearchContext.SuggestionContext { + protected CompletionSuggestionContext(QueryShardContext shardContext) { + super(CompletionSuggester.PROTOTYPE, shardContext); + } + private CompletionFieldMapper.CompletionFieldType fieldType; private CompletionSuggestionBuilder.FuzzyOptionsBuilder fuzzyOptionsBuilder; private CompletionSuggestionBuilder.RegexOptionsBuilder regexOptionsBuilder; private Map<String, List<ContextMapping.QueryContext>> queryContexts = Collections.emptyMap(); - private final MapperService mapperService; - private final IndexFieldDataService indexFieldDataService; private Set<String> payloadFields = Collections.emptySet(); - CompletionSuggestionContext(Suggester suggester, MapperService mapperService, IndexFieldDataService indexFieldDataService) { - super(suggester); - this.indexFieldDataService = indexFieldDataService; - this.mapperService = mapperService; - } - CompletionFieldMapper.CompletionFieldType getFieldType() { return this.fieldType; } @@ -73,15 +67,6 @@ public class CompletionSuggestionContext extends SuggestionSearchContext.Suggest this.queryContexts = queryContexts; } - - MapperService getMapperService() { - return mapperService; - } - - IndexFieldDataService getIndexFieldDataService() { - return indexFieldDataService; - } - void setPayloadFields(Set<String> fields) { this.payloadFields = fields; } diff --git a/core/src/main/java/org/elasticsearch/search/suggest/phrase/DirectCandidateGeneratorBuilder.java b/core/src/main/java/org/elasticsearch/search/suggest/phrase/DirectCandidateGeneratorBuilder.java index 8cc834ef0d..dd1be571bd 100644 --- a/core/src/main/java/org/elasticsearch/search/suggest/phrase/DirectCandidateGeneratorBuilder.java +++ b/core/src/main/java/org/elasticsearch/search/suggest/phrase/DirectCandidateGeneratorBuilder.java @@ -30,7 +30,6 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.query.QueryParseContext; -import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.search.suggest.SuggestUtils; import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder.CandidateGenerator; @@ -349,8 +348,7 @@ public final class DirectCandidateGeneratorBuilder return replaceField(tmpFieldName.iterator().next(), tempGenerator); } - public PhraseSuggestionContext.DirectCandidateGenerator build(QueryShardContext context) throws IOException { - MapperService mapperService = context.getMapperService(); + public PhraseSuggestionContext.DirectCandidateGenerator build(MapperService mapperService) throws IOException { PhraseSuggestionContext.DirectCandidateGenerator generator = new PhraseSuggestionContext.DirectCandidateGenerator(); generator.setField(this.field); transferIfNotNull(this.size, generator::size); diff --git a/core/src/main/java/org/elasticsearch/search/suggest/phrase/Laplace.java b/core/src/main/java/org/elasticsearch/search/suggest/phrase/Laplace.java new file mode 100644 index 0000000000..e11a920f96 --- /dev/null +++ b/core/src/main/java/org/elasticsearch/search/suggest/phrase/Laplace.java @@ -0,0 +1,126 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.suggest.phrase; + +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.Terms; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentParser.Token; +import org.elasticsearch.index.query.QueryParseContext; +import org.elasticsearch.search.suggest.phrase.WordScorer.WordScorerFactory; + +import java.io.IOException; +import java.util.Objects; + +/** + * An <a href="http://en.wikipedia.org/wiki/Additive_smoothing">additive + * smoothing</a> model. + * <p> + * See <a + * href="http://en.wikipedia.org/wiki/N-gram#Smoothing_techniques">N-Gram + * Smoothing</a> for details. + * </p> + */ +public final class Laplace extends SmoothingModel { + private double alpha = DEFAULT_LAPLACE_ALPHA; + private static final String NAME = "laplace"; + private static final ParseField ALPHA_FIELD = new ParseField("alpha"); + static final ParseField PARSE_FIELD = new ParseField(NAME); + /** + * Default alpha parameter for laplace smoothing + */ + public static final double DEFAULT_LAPLACE_ALPHA = 0.5; + public static final Laplace PROTOTYPE = new Laplace(DEFAULT_LAPLACE_ALPHA); + + /** + * Creates a Laplace smoothing model. + * + */ + public Laplace(double alpha) { + this.alpha = alpha; + } + + /** + * @return the laplace model alpha parameter + */ + public double getAlpha() { + return this.alpha; + } + + @Override + protected XContentBuilder innerToXContent(XContentBuilder builder, Params params) throws IOException { + builder.field(ALPHA_FIELD.getPreferredName(), alpha); + return builder; + } + + @Override + public String getWriteableName() { + return NAME; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeDouble(alpha); + } + + @Override + public SmoothingModel readFrom(StreamInput in) throws IOException { + return new Laplace(in.readDouble()); + } + + @Override + protected boolean doEquals(SmoothingModel other) { + Laplace otherModel = (Laplace) other; + return Objects.equals(alpha, otherModel.alpha); + } + + @Override + protected final int doHashCode() { + return Objects.hash(alpha); + } + + @Override + public SmoothingModel innerFromXContent(QueryParseContext parseContext) throws IOException { + XContentParser parser = parseContext.parser(); + XContentParser.Token token; + String fieldName = null; + double alpha = DEFAULT_LAPLACE_ALPHA; + while ((token = parser.nextToken()) != Token.END_OBJECT) { + if (token == XContentParser.Token.FIELD_NAME) { + fieldName = parser.currentName(); + } + if (token.isValue() && parseContext.parseFieldMatcher().match(fieldName, ALPHA_FIELD)) { + alpha = parser.doubleValue(); + } + } + return new Laplace(alpha); + } + + @Override + public WordScorerFactory buildWordScorerFactory() { + return (IndexReader reader, Terms terms, String field, double realWordLikelyhood, BytesRef separator) + -> new LaplaceScorer(reader, terms, field, realWordLikelyhood, separator, alpha); + } +}
\ No newline at end of file diff --git a/core/src/main/java/org/elasticsearch/search/suggest/phrase/LinearInterpolation.java b/core/src/main/java/org/elasticsearch/search/suggest/phrase/LinearInterpolation.java new file mode 100644 index 0000000000..b94ea333fd --- /dev/null +++ b/core/src/main/java/org/elasticsearch/search/suggest/phrase/LinearInterpolation.java @@ -0,0 +1,176 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.suggest.phrase; + +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.Terms; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.ParseFieldMatcher; +import org.elasticsearch.common.ParsingException; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentParser.Token; +import org.elasticsearch.index.query.QueryParseContext; +import org.elasticsearch.search.suggest.phrase.WordScorer.WordScorerFactory; + +import java.io.IOException; +import java.util.Objects; + +/** + * Linear interpolation smoothing model. + * <p> + * See <a + * href="http://en.wikipedia.org/wiki/N-gram#Smoothing_techniques">N-Gram + * Smoothing</a> for details. + * </p> + */ +public final class LinearInterpolation extends SmoothingModel { + private static final String NAME = "linear"; + public static final LinearInterpolation PROTOTYPE = new LinearInterpolation(0.8, 0.1, 0.1); + private final double trigramLambda; + private final double bigramLambda; + private final double unigramLambda; + static final ParseField PARSE_FIELD = new ParseField(NAME); + private static final ParseField TRIGRAM_FIELD = new ParseField("trigram_lambda"); + private static final ParseField BIGRAM_FIELD = new ParseField("bigram_lambda"); + private static final ParseField UNIGRAM_FIELD = new ParseField("unigram_lambda"); + + /** + * Creates a linear interpolation smoothing model. + * + * Note: the lambdas must sum up to one. + * + * @param trigramLambda + * the trigram lambda + * @param bigramLambda + * the bigram lambda + * @param unigramLambda + * the unigram lambda + */ + public LinearInterpolation(double trigramLambda, double bigramLambda, double unigramLambda) { + double sum = trigramLambda + bigramLambda + unigramLambda; + if (Math.abs(sum - 1.0) > 0.001) { + throw new IllegalArgumentException("linear smoothing lambdas must sum to 1"); + } + this.trigramLambda = trigramLambda; + this.bigramLambda = bigramLambda; + this.unigramLambda = unigramLambda; + } + + public double getTrigramLambda() { + return this.trigramLambda; + } + + public double getBigramLambda() { + return this.bigramLambda; + } + + public double getUnigramLambda() { + return this.unigramLambda; + } + + @Override + protected XContentBuilder innerToXContent(XContentBuilder builder, Params params) throws IOException { + builder.field(TRIGRAM_FIELD.getPreferredName(), trigramLambda); + builder.field(BIGRAM_FIELD.getPreferredName(), bigramLambda); + builder.field(UNIGRAM_FIELD.getPreferredName(), unigramLambda); + return builder; + } + + @Override + public String getWriteableName() { + return NAME; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeDouble(trigramLambda); + out.writeDouble(bigramLambda); + out.writeDouble(unigramLambda); + } + + @Override + public LinearInterpolation readFrom(StreamInput in) throws IOException { + return new LinearInterpolation(in.readDouble(), in.readDouble(), in.readDouble()); + } + + @Override + protected boolean doEquals(SmoothingModel other) { + final LinearInterpolation otherModel = (LinearInterpolation) other; + return Objects.equals(trigramLambda, otherModel.trigramLambda) && + Objects.equals(bigramLambda, otherModel.bigramLambda) && + Objects.equals(unigramLambda, otherModel.unigramLambda); + } + + @Override + protected final int doHashCode() { + return Objects.hash(trigramLambda, bigramLambda, unigramLambda); + } + + @Override + public LinearInterpolation innerFromXContent(QueryParseContext parseContext) throws IOException { + XContentParser parser = parseContext.parser(); + XContentParser.Token token; + String fieldName = null; + double trigramLambda = 0.0; + double bigramLambda = 0.0; + double unigramLambda = 0.0; + ParseFieldMatcher matcher = parseContext.parseFieldMatcher(); + while ((token = parser.nextToken()) != Token.END_OBJECT) { + if (token == XContentParser.Token.FIELD_NAME) { + fieldName = parser.currentName(); + } else if (token.isValue()) { + if (matcher.match(fieldName, TRIGRAM_FIELD)) { + trigramLambda = parser.doubleValue(); + if (trigramLambda < 0) { + throw new IllegalArgumentException("trigram_lambda must be positive"); + } + } else if (matcher.match(fieldName, BIGRAM_FIELD)) { + bigramLambda = parser.doubleValue(); + if (bigramLambda < 0) { + throw new IllegalArgumentException("bigram_lambda must be positive"); + } + } else if (matcher.match(fieldName, UNIGRAM_FIELD)) { + unigramLambda = parser.doubleValue(); + if (unigramLambda < 0) { + throw new IllegalArgumentException("unigram_lambda must be positive"); + } + } else { + throw new IllegalArgumentException( + "suggester[phrase][smoothing][linear] doesn't support field [" + fieldName + "]"); + } + } else { + throw new ParsingException(parser.getTokenLocation(), + "[" + NAME + "] unknown token [" + token + "] after [" + fieldName + "]"); + } + } + return new LinearInterpolation(trigramLambda, bigramLambda, unigramLambda); + } + + @Override + public WordScorerFactory buildWordScorerFactory() { + return (IndexReader reader, Terms terms, String field, double realWordLikelyhood, BytesRef separator) -> + new LinearInterpoatingScorer(reader, terms, field, realWordLikelyhood, separator, trigramLambda, bigramLambda, + unigramLambda); + } +}
\ No newline at end of file diff --git a/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestParser.java b/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestParser.java index fc60fc6fc8..e4400fb5cd 100644 --- a/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestParser.java +++ b/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestParser.java @@ -26,17 +26,16 @@ import org.elasticsearch.common.ParseFieldMatcher; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentParser.Token; import org.elasticsearch.index.analysis.ShingleTokenFilterFactory; -import org.elasticsearch.index.fielddata.IndexFieldDataService; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.script.CompiledScript; import org.elasticsearch.script.ScriptContext; +import org.elasticsearch.script.ScriptService; import org.elasticsearch.script.Template; import org.elasticsearch.search.suggest.SuggestContextParser; import org.elasticsearch.search.suggest.SuggestUtils; import org.elasticsearch.search.suggest.SuggestionSearchContext; -import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder.Laplace; -import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder.StupidBackoff; import org.elasticsearch.search.suggest.phrase.PhraseSuggestionContext.DirectCandidateGenerator; import java.io.IOException; @@ -51,8 +50,10 @@ public final class PhraseSuggestParser implements SuggestContextParser { } @Override - public SuggestionSearchContext.SuggestionContext parse(XContentParser parser, MapperService mapperService, IndexFieldDataService fieldDataService) throws IOException { - PhraseSuggestionContext suggestion = new PhraseSuggestionContext(suggester); + public SuggestionSearchContext.SuggestionContext parse(XContentParser parser, QueryShardContext shardContext) throws IOException { + MapperService mapperService = shardContext.getMapperService(); + ScriptService scriptService = shardContext.getScriptService(); + PhraseSuggestionContext suggestion = new PhraseSuggestionContext(shardContext); ParseFieldMatcher parseFieldMatcher = mapperService.getIndexSettings().getParseFieldMatcher(); XContentParser.Token token; String fieldName = null; @@ -135,7 +136,7 @@ public final class PhraseSuggestParser implements SuggestContextParser { throw new IllegalArgumentException("suggester[phrase][collate] query already set, doesn't support additional [" + fieldName + "]"); } Template template = Template.parse(parser, parseFieldMatcher); - CompiledScript compiledScript = suggester.scriptService().compile(template, ScriptContext.Standard.SEARCH, Collections.emptyMap()); + CompiledScript compiledScript = scriptService.compile(template, ScriptContext.Standard.SEARCH, Collections.emptyMap()); suggestion.setCollateQueryScript(compiledScript); } else if ("params".equals(fieldName)) { suggestion.setCollateScriptParams(parser.map()); @@ -199,9 +200,6 @@ public final class PhraseSuggestParser implements SuggestContextParser { suggestion.addGenerator(generator); } } - - - return suggestion; } diff --git a/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggester.java b/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggester.java index fbfa2b03ce..8f3e5164e4 100644 --- a/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggester.java +++ b/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggester.java @@ -31,9 +31,7 @@ import org.apache.lucene.util.CharsRefBuilder; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.text.Text; -import org.elasticsearch.index.IndexService; import org.elasticsearch.index.query.ParsedQuery; -import org.elasticsearch.indices.IndicesService; import org.elasticsearch.script.CompiledScript; import org.elasticsearch.script.ExecutableScript; import org.elasticsearch.script.ScriptService; @@ -44,6 +42,7 @@ import org.elasticsearch.search.suggest.SuggestContextParser; import org.elasticsearch.search.suggest.SuggestUtils; import org.elasticsearch.search.suggest.Suggester; import org.elasticsearch.search.suggest.SuggestionBuilder; +import org.elasticsearch.search.suggest.SuggestionSearchContext.SuggestionContext; import org.elasticsearch.search.suggest.phrase.NoisyChannelSpellChecker.Result; import java.io.IOException; @@ -54,13 +53,8 @@ import java.util.Map; public final class PhraseSuggester extends Suggester<PhraseSuggestionContext> { private final BytesRef SEPARATOR = new BytesRef(" "); private static final String SUGGESTION_TEMPLATE_VAR_NAME = "suggestion"; - private final ScriptService scriptService; - private final IndicesService indicesService; - public PhraseSuggester(ScriptService scriptService, IndicesService indicesService) { - this.scriptService = scriptService; - this.indicesService = indicesService; - } + public static final PhraseSuggester PROTOTYPE = new PhraseSuggester(); /* * More Ideas: @@ -118,10 +112,10 @@ public final class PhraseSuggester extends Suggester<PhraseSuggestionContext> { // from the index for a correction, collateMatch is updated final Map<String, Object> vars = suggestion.getCollateScriptParams(); vars.put(SUGGESTION_TEMPLATE_VAR_NAME, spare.toString()); + ScriptService scriptService = suggestion.getShardContext().getScriptService(); final ExecutableScript executable = scriptService.executable(collateScript, vars); final BytesReference querySource = (BytesReference) executable.run(); - IndexService indexService = indicesService.indexService(suggestion.getIndex()); - final ParsedQuery parsedQuery = indexService.newQueryShardContext().parse(querySource); + final ParsedQuery parsedQuery = suggestion.getShardContext().parse(querySource); collateMatch = Lucene.exists(searcher, parsedQuery.query()); } if (!collateMatch && !collatePrune) { @@ -145,15 +139,11 @@ public final class PhraseSuggester extends Suggester<PhraseSuggestionContext> { return response; } - private PhraseSuggestion.Entry buildResultEntry(PhraseSuggestionContext suggestion, CharsRefBuilder spare, double cutoffScore) { + private PhraseSuggestion.Entry buildResultEntry(SuggestionContext suggestion, CharsRefBuilder spare, double cutoffScore) { spare.copyUTF8Bytes(suggestion.getText()); return new PhraseSuggestion.Entry(new Text(spare.toString()), 0, spare.length(), cutoffScore); } - ScriptService scriptService() { - return scriptService; - } - @Override public SuggestContextParser getContextParser() { return new PhraseSuggestParser(this); diff --git a/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestionBuilder.java b/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestionBuilder.java index c079812afe..c83f68716b 100644 --- a/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestionBuilder.java +++ b/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestionBuilder.java @@ -19,27 +19,32 @@ package org.elasticsearch.search.suggest.phrase; -import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.Terms; -import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.ParseFieldMatcher; -import org.elasticsearch.common.ParsingException; -import org.elasticsearch.common.io.stream.NamedWriteable; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.lucene.BytesRefs; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentParser.Token; +import org.elasticsearch.index.analysis.ShingleTokenFilterFactory; +import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.query.QueryParseContext; +import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.script.CompiledScript; +import org.elasticsearch.script.ScriptContext; import org.elasticsearch.script.Template; +import org.elasticsearch.search.suggest.SuggestUtils; import org.elasticsearch.search.suggest.SuggestionBuilder; -import org.elasticsearch.search.suggest.phrase.WordScorer.WordScorerFactory; +import org.elasticsearch.search.suggest.SuggestionSearchContext.SuggestionContext; +import org.elasticsearch.search.suggest.phrase.PhraseSuggestionContext.DirectCandidateGenerator; import java.io.IOException; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -239,7 +244,7 @@ public final class PhraseSuggestionBuilder extends SuggestionBuilder<PhraseSugge /** * Sets an explicit smoothing model used for this suggester. The default is - * {@link PhraseSuggestionBuilder.StupidBackoff}. + * {@link StupidBackoff}. */ public PhraseSuggestionBuilder smoothingModel(SmoothingModel model) { this.model = model; @@ -254,6 +259,9 @@ public final class PhraseSuggestionBuilder extends SuggestionBuilder<PhraseSugge } public PhraseSuggestionBuilder tokenLimit(int tokenLimit) { + if (tokenLimit <= 0) { + throw new IllegalArgumentException("token_limit must be >= 1"); + } this.tokenLimit = tokenLimit; return this; } @@ -389,413 +397,6 @@ public final class PhraseSuggestionBuilder extends SuggestionBuilder<PhraseSugge return builder; } - /** - * Creates a new {@link DirectCandidateGeneratorBuilder} - * - * @param field - * the field this candidate generator operates on. - */ - public static DirectCandidateGeneratorBuilder candidateGenerator(String field) { - return new DirectCandidateGeneratorBuilder(field); - } - - /** - * A "stupid-backoff" smoothing model simialr to <a - * href="http://en.wikipedia.org/wiki/Katz's_back-off_model"> Katz's - * Backoff</a>. This model is used as the default if no model is configured. - * <p> - * See <a - * href="http://en.wikipedia.org/wiki/N-gram#Smoothing_techniques">N-Gram - * Smoothing</a> for details. - * </p> - */ - public static final class StupidBackoff extends SmoothingModel { - /** - * Default discount parameter for {@link StupidBackoff} smoothing - */ - public static final double DEFAULT_BACKOFF_DISCOUNT = 0.4; - public static final StupidBackoff PROTOTYPE = new StupidBackoff(DEFAULT_BACKOFF_DISCOUNT); - private double discount = DEFAULT_BACKOFF_DISCOUNT; - private static final String NAME = "stupid_backoff"; - private static final ParseField DISCOUNT_FIELD = new ParseField("discount"); - private static final ParseField PARSE_FIELD = new ParseField(NAME); - - /** - * Creates a Stupid-Backoff smoothing model. - * - * @param discount - * the discount given to lower order ngrams if the higher order ngram doesn't exits - */ - public StupidBackoff(double discount) { - this.discount = discount; - } - - /** - * @return the discount parameter of the model - */ - public double getDiscount() { - return this.discount; - } - - @Override - protected XContentBuilder innerToXContent(XContentBuilder builder, Params params) throws IOException { - builder.field(DISCOUNT_FIELD.getPreferredName(), discount); - return builder; - } - - @Override - public String getWriteableName() { - return NAME; - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - out.writeDouble(discount); - } - - @Override - public StupidBackoff readFrom(StreamInput in) throws IOException { - return new StupidBackoff(in.readDouble()); - } - - @Override - protected boolean doEquals(SmoothingModel other) { - StupidBackoff otherModel = (StupidBackoff) other; - return Objects.equals(discount, otherModel.discount); - } - - @Override - protected final int doHashCode() { - return Objects.hash(discount); - } - - @Override - public SmoothingModel innerFromXContent(QueryParseContext parseContext) throws IOException { - XContentParser parser = parseContext.parser(); - XContentParser.Token token; - String fieldName = null; - double discount = DEFAULT_BACKOFF_DISCOUNT; - while ((token = parser.nextToken()) != Token.END_OBJECT) { - if (token == XContentParser.Token.FIELD_NAME) { - fieldName = parser.currentName(); - } - if (token.isValue() && parseContext.parseFieldMatcher().match(fieldName, DISCOUNT_FIELD)) { - discount = parser.doubleValue(); - } - } - return new StupidBackoff(discount); - } - - @Override - public WordScorerFactory buildWordScorerFactory() { - return (IndexReader reader, Terms terms, String field, double realWordLikelyhood, BytesRef separator) - -> new StupidBackoffScorer(reader, terms, field, realWordLikelyhood, separator, discount); - } - } - - /** - * An <a href="http://en.wikipedia.org/wiki/Additive_smoothing">additive - * smoothing</a> model. - * <p> - * See <a - * href="http://en.wikipedia.org/wiki/N-gram#Smoothing_techniques">N-Gram - * Smoothing</a> for details. - * </p> - */ - public static final class Laplace extends SmoothingModel { - private double alpha = DEFAULT_LAPLACE_ALPHA; - private static final String NAME = "laplace"; - private static final ParseField ALPHA_FIELD = new ParseField("alpha"); - private static final ParseField PARSE_FIELD = new ParseField(NAME); - /** - * Default alpha parameter for laplace smoothing - */ - public static final double DEFAULT_LAPLACE_ALPHA = 0.5; - public static final Laplace PROTOTYPE = new Laplace(DEFAULT_LAPLACE_ALPHA); - - /** - * Creates a Laplace smoothing model. - * - */ - public Laplace(double alpha) { - this.alpha = alpha; - } - - /** - * @return the laplace model alpha parameter - */ - public double getAlpha() { - return this.alpha; - } - - @Override - protected XContentBuilder innerToXContent(XContentBuilder builder, Params params) throws IOException { - builder.field(ALPHA_FIELD.getPreferredName(), alpha); - return builder; - } - - @Override - public String getWriteableName() { - return NAME; - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - out.writeDouble(alpha); - } - - @Override - public SmoothingModel readFrom(StreamInput in) throws IOException { - return new Laplace(in.readDouble()); - } - - @Override - protected boolean doEquals(SmoothingModel other) { - Laplace otherModel = (Laplace) other; - return Objects.equals(alpha, otherModel.alpha); - } - - @Override - protected final int doHashCode() { - return Objects.hash(alpha); - } - - @Override - public SmoothingModel innerFromXContent(QueryParseContext parseContext) throws IOException { - XContentParser parser = parseContext.parser(); - XContentParser.Token token; - String fieldName = null; - double alpha = DEFAULT_LAPLACE_ALPHA; - while ((token = parser.nextToken()) != Token.END_OBJECT) { - if (token == XContentParser.Token.FIELD_NAME) { - fieldName = parser.currentName(); - } - if (token.isValue() && parseContext.parseFieldMatcher().match(fieldName, ALPHA_FIELD)) { - alpha = parser.doubleValue(); - } - } - return new Laplace(alpha); - } - - @Override - public WordScorerFactory buildWordScorerFactory() { - return (IndexReader reader, Terms terms, String field, double realWordLikelyhood, BytesRef separator) - -> new LaplaceScorer(reader, terms, field, realWordLikelyhood, separator, alpha); - } - } - - - public static abstract class SmoothingModel implements NamedWriteable<SmoothingModel>, ToXContent { - - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.startObject(getWriteableName()); - innerToXContent(builder,params); - builder.endObject(); - return builder; - } - - @Override - public final boolean equals(Object obj) { - if (this == obj) { - return true; - } - if (obj == null || getClass() != obj.getClass()) { - return false; - } - SmoothingModel other = (SmoothingModel) obj; - return doEquals(other); - } - - public static SmoothingModel fromXContent(QueryParseContext parseContext) throws IOException { - XContentParser parser = parseContext.parser(); - ParseFieldMatcher parseFieldMatcher = parseContext.parseFieldMatcher(); - XContentParser.Token token; - String fieldName = null; - SmoothingModel model = null; - while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { - if (token == XContentParser.Token.FIELD_NAME) { - fieldName = parser.currentName(); - } else if (token == XContentParser.Token.START_OBJECT) { - if (parseFieldMatcher.match(fieldName, LinearInterpolation.PARSE_FIELD)) { - model = LinearInterpolation.PROTOTYPE.innerFromXContent(parseContext); - } else if (parseFieldMatcher.match(fieldName, Laplace.PARSE_FIELD)) { - model = Laplace.PROTOTYPE.innerFromXContent(parseContext); - } else if (parseFieldMatcher.match(fieldName, StupidBackoff.PARSE_FIELD)) { - model = StupidBackoff.PROTOTYPE.innerFromXContent(parseContext); - } else { - throw new IllegalArgumentException("suggester[phrase] doesn't support object field [" + fieldName + "]"); - } - } else { - throw new ParsingException(parser.getTokenLocation(), - "[smoothing] unknown token [" + token + "] after [" + fieldName + "]"); - } - } - return model; - } - - public abstract SmoothingModel innerFromXContent(QueryParseContext parseContext) throws IOException; - - @Override - public final int hashCode() { - /* - * Override hashCode here and forward to an abstract method to force extensions of this class to override hashCode in the same - * way that we force them to override equals. This also prevents false positives in CheckStyle's EqualsHashCode check. - */ - return doHashCode(); - } - - public abstract WordScorerFactory buildWordScorerFactory(); - - /** - * subtype specific implementation of "equals". - */ - protected abstract boolean doEquals(SmoothingModel other); - - protected abstract int doHashCode(); - - protected abstract XContentBuilder innerToXContent(XContentBuilder builder, Params params) throws IOException; - } - - /** - * Linear interpolation smoothing model. - * <p> - * See <a - * href="http://en.wikipedia.org/wiki/N-gram#Smoothing_techniques">N-Gram - * Smoothing</a> for details. - * </p> - */ - public static final class LinearInterpolation extends SmoothingModel { - private static final String NAME = "linear"; - public static final LinearInterpolation PROTOTYPE = new LinearInterpolation(0.8, 0.1, 0.1); - private final double trigramLambda; - private final double bigramLambda; - private final double unigramLambda; - private static final ParseField PARSE_FIELD = new ParseField(NAME); - private static final ParseField TRIGRAM_FIELD = new ParseField("trigram_lambda"); - private static final ParseField BIGRAM_FIELD = new ParseField("bigram_lambda"); - private static final ParseField UNIGRAM_FIELD = new ParseField("unigram_lambda"); - - /** - * Creates a linear interpolation smoothing model. - * - * Note: the lambdas must sum up to one. - * - * @param trigramLambda - * the trigram lambda - * @param bigramLambda - * the bigram lambda - * @param unigramLambda - * the unigram lambda - */ - public LinearInterpolation(double trigramLambda, double bigramLambda, double unigramLambda) { - double sum = trigramLambda + bigramLambda + unigramLambda; - if (Math.abs(sum - 1.0) > 0.001) { - throw new IllegalArgumentException("linear smoothing lambdas must sum to 1"); - } - this.trigramLambda = trigramLambda; - this.bigramLambda = bigramLambda; - this.unigramLambda = unigramLambda; - } - - public double getTrigramLambda() { - return this.trigramLambda; - } - - public double getBigramLambda() { - return this.bigramLambda; - } - - public double getUnigramLambda() { - return this.unigramLambda; - } - - @Override - protected XContentBuilder innerToXContent(XContentBuilder builder, Params params) throws IOException { - builder.field(TRIGRAM_FIELD.getPreferredName(), trigramLambda); - builder.field(BIGRAM_FIELD.getPreferredName(), bigramLambda); - builder.field(UNIGRAM_FIELD.getPreferredName(), unigramLambda); - return builder; - } - - @Override - public String getWriteableName() { - return NAME; - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - out.writeDouble(trigramLambda); - out.writeDouble(bigramLambda); - out.writeDouble(unigramLambda); - } - - @Override - public LinearInterpolation readFrom(StreamInput in) throws IOException { - return new LinearInterpolation(in.readDouble(), in.readDouble(), in.readDouble()); - } - - @Override - protected boolean doEquals(SmoothingModel other) { - final LinearInterpolation otherModel = (LinearInterpolation) other; - return Objects.equals(trigramLambda, otherModel.trigramLambda) && - Objects.equals(bigramLambda, otherModel.bigramLambda) && - Objects.equals(unigramLambda, otherModel.unigramLambda); - } - - @Override - protected final int doHashCode() { - return Objects.hash(trigramLambda, bigramLambda, unigramLambda); - } - - @Override - public LinearInterpolation innerFromXContent(QueryParseContext parseContext) throws IOException { - XContentParser parser = parseContext.parser(); - XContentParser.Token token; - String fieldName = null; - double trigramLambda = 0.0; - double bigramLambda = 0.0; - double unigramLambda = 0.0; - ParseFieldMatcher matcher = parseContext.parseFieldMatcher(); - while ((token = parser.nextToken()) != Token.END_OBJECT) { - if (token == XContentParser.Token.FIELD_NAME) { - fieldName = parser.currentName(); - } else if (token.isValue()) { - if (matcher.match(fieldName, TRIGRAM_FIELD)) { - trigramLambda = parser.doubleValue(); - if (trigramLambda < 0) { - throw new IllegalArgumentException("trigram_lambda must be positive"); - } - } else if (matcher.match(fieldName, BIGRAM_FIELD)) { - bigramLambda = parser.doubleValue(); - if (bigramLambda < 0) { - throw new IllegalArgumentException("bigram_lambda must be positive"); - } - } else if (matcher.match(fieldName, UNIGRAM_FIELD)) { - unigramLambda = parser.doubleValue(); - if (unigramLambda < 0) { - throw new IllegalArgumentException("unigram_lambda must be positive"); - } - } else { - throw new IllegalArgumentException( - "suggester[phrase][smoothing][linear] doesn't support field [" + fieldName + "]"); - } - } else { - throw new ParsingException(parser.getTokenLocation(), - "[" + NAME + "] unknown token [" + token + "] after [" + fieldName + "]"); - } - } - return new LinearInterpolation(trigramLambda, bigramLambda, unigramLambda); - } - - @Override - public WordScorerFactory buildWordScorerFactory() { - return (IndexReader reader, Terms terms, String field, double realWordLikelyhood, BytesRef separator) -> - new LinearInterpoatingScorer(reader, terms, field, realWordLikelyhood, separator, trigramLambda, bigramLambda, - unigramLambda); - } - } - @Override protected PhraseSuggestionBuilder innerFromXContent(QueryParseContext parseContext, String suggestionName) throws IOException { XContentParser parser = parseContext.parser(); @@ -873,7 +474,6 @@ public final class PhraseSuggestionBuilder extends SuggestionBuilder<PhraseSugge "suggester[phrase][collate] query already set, doesn't support additional [" + fieldName + "]"); } Template template = Template.parse(parser, parseFieldMatcher); - // TODO remember to compile script in build() method suggestion.collateQuery(template); } else if (parseFieldMatcher.match(fieldName, PhraseSuggestionBuilder.COLLATE_QUERY_PARAMS)) { suggestion.collateParams(parser.map()); @@ -898,6 +498,98 @@ public final class PhraseSuggestionBuilder extends SuggestionBuilder<PhraseSugge return suggestion; } + + @Override + public SuggestionContext build(QueryShardContext context) throws IOException { + PhraseSuggestionContext suggestionContext = new PhraseSuggestionContext(context); + MapperService mapperService = context.getMapperService(); + populateCommonFields(mapperService, suggestionContext); + + suggestionContext.setSeparator(BytesRefs.toBytesRef(this.separator)); + suggestionContext.setRealWordErrorLikelihood(this.realWordErrorLikelihood); + suggestionContext.setConfidence(this.confidence); + suggestionContext.setMaxErrors(this.maxErrors); + suggestionContext.setSeparator(BytesRefs.toBytesRef(this.separator)); + suggestionContext.setRequireUnigram(this.forceUnigrams); + suggestionContext.setTokenLimit(this.tokenLimit); + suggestionContext.setPreTag(BytesRefs.toBytesRef(this.preTag)); + suggestionContext.setPostTag(BytesRefs.toBytesRef(this.postTag)); + + if (this.gramSize != null) { + suggestionContext.setGramSize(this.gramSize); + } + + for (List<CandidateGenerator> candidateGenerators : this.generators.values()) { + for (CandidateGenerator candidateGenerator : candidateGenerators) { + suggestionContext.addGenerator(candidateGenerator.build(mapperService)); + } + } + + if (this.model != null) { + suggestionContext.setModel(this.model.buildWordScorerFactory()); + } + + if (this.collateQuery != null) { + CompiledScript compiledScript = context.getScriptService().compile(this.collateQuery, ScriptContext.Standard.SEARCH, + Collections.emptyMap()); + suggestionContext.setCollateQueryScript(compiledScript); + if (this.collateParams != null) { + suggestionContext.setCollateScriptParams(this.collateParams); + } + suggestionContext.setCollatePrune(this.collatePrune); + } + + // TODO remove this when field is mandatory in builder ctor + if (suggestionContext.getField() == null) { + throw new IllegalArgumentException("The required field option is missing"); + } + + MappedFieldType fieldType = mapperService.fullName(suggestionContext.getField()); + if (fieldType == null) { + throw new IllegalArgumentException("No mapping found for field [" + suggestionContext.getField() + "]"); + } else if (suggestionContext.getAnalyzer() == null) { + // no analyzer name passed in, so try the field's analyzer, or the default analyzer + if (fieldType.searchAnalyzer() == null) { + suggestionContext.setAnalyzer(mapperService.searchAnalyzer()); + } else { + suggestionContext.setAnalyzer(fieldType.searchAnalyzer()); + } + } + + if (suggestionContext.model() == null) { + suggestionContext.setModel(StupidBackoffScorer.FACTORY); + } + + if (this.gramSize == null || suggestionContext.generators().isEmpty()) { + final ShingleTokenFilterFactory.Factory shingleFilterFactory = SuggestUtils + .getShingleFilterFactory(suggestionContext.getAnalyzer()); + if (this.gramSize == null) { + // try to detect the shingle size + if (shingleFilterFactory != null) { + suggestionContext.setGramSize(shingleFilterFactory.getMaxShingleSize()); + if (suggestionContext.getAnalyzer() == null && shingleFilterFactory.getMinShingleSize() > 1 + && !shingleFilterFactory.getOutputUnigrams()) { + throw new IllegalArgumentException("The default analyzer for field: [" + suggestionContext.getField() + + "] doesn't emit unigrams. If this is intentional try to set the analyzer explicitly"); + } + } + } + if (suggestionContext.generators().isEmpty()) { + if (shingleFilterFactory != null && shingleFilterFactory.getMinShingleSize() > 1 + && !shingleFilterFactory.getOutputUnigrams() && suggestionContext.getRequireUnigram()) { + throw new IllegalArgumentException("The default candidate generator for phrase suggest can't operate on field: [" + + suggestionContext.getField() + "] since it doesn't emit unigrams. " + + "If this is intentional try to set the candidate generator field explicitly"); + } + // use a default generator on the same field + DirectCandidateGenerator generator = new DirectCandidateGenerator(); + generator.setField(suggestionContext.getField()); + suggestionContext.addGenerator(generator); + } + } + return suggestionContext; + } + private static void ensureNoSmoothing(PhraseSuggestionBuilder suggestion) { if (suggestion.smoothingModel() != null) { throw new IllegalArgumentException("only one smoothing model supported"); @@ -1010,5 +702,7 @@ public final class PhraseSuggestionBuilder extends SuggestionBuilder<PhraseSugge String getType(); CandidateGenerator fromXContent(QueryParseContext parseContext) throws IOException; + + PhraseSuggestionContext.DirectCandidateGenerator build(MapperService mapperService) throws IOException; } } diff --git a/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestionContext.java b/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestionContext.java index 20af69b6a6..95c02d5add 100644 --- a/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestionContext.java +++ b/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestionContext.java @@ -20,9 +20,9 @@ package org.elasticsearch.search.suggest.phrase; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.script.CompiledScript; import org.elasticsearch.search.suggest.DirectSpellcheckerSettings; -import org.elasticsearch.search.suggest.Suggester; import org.elasticsearch.search.suggest.SuggestionSearchContext.SuggestionContext; import java.util.ArrayList; @@ -54,8 +54,8 @@ class PhraseSuggestionContext extends SuggestionContext { private Map<String, Object> collateScriptParams = new HashMap<>(1); private WordScorer.WordScorerFactory scorer; - public PhraseSuggestionContext(Suggester<? extends PhraseSuggestionContext> suggester) { - super(suggester); + public PhraseSuggestionContext(QueryShardContext shardContext) { + super(PhraseSuggester.PROTOTYPE, shardContext); } public float maxErrors() { @@ -154,8 +154,6 @@ class PhraseSuggestionContext extends SuggestionContext { public void postFilter(Analyzer postFilter) { this.postFilter = postFilter; } - - } public void setRequireUnigram(boolean requireUnigram) { @@ -213,5 +211,4 @@ class PhraseSuggestionContext extends SuggestionContext { boolean collatePrune() { return prune; } - } diff --git a/core/src/main/java/org/elasticsearch/search/suggest/phrase/SmoothingModel.java b/core/src/main/java/org/elasticsearch/search/suggest/phrase/SmoothingModel.java new file mode 100644 index 0000000000..0163c560de --- /dev/null +++ b/core/src/main/java/org/elasticsearch/search/suggest/phrase/SmoothingModel.java @@ -0,0 +1,105 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.suggest.phrase; + +import org.elasticsearch.common.ParseFieldMatcher; +import org.elasticsearch.common.ParsingException; +import org.elasticsearch.common.io.stream.NamedWriteable; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.index.query.QueryParseContext; +import org.elasticsearch.search.suggest.phrase.WordScorer.WordScorerFactory; + +import java.io.IOException; + +public abstract class SmoothingModel implements NamedWriteable<SmoothingModel>, ToXContent { + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(getWriteableName()); + innerToXContent(builder,params); + builder.endObject(); + return builder; + } + + @Override + public final boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + SmoothingModel other = (SmoothingModel) obj; + return doEquals(other); + } + + @Override + public final int hashCode() { + /* + * Override hashCode here and forward to an abstract method to force + * extensions of this class to override hashCode in the same way that we + * force them to override equals. This also prevents false positives in + * CheckStyle's EqualsHashCode check. + */ + return doHashCode(); + } + + protected abstract int doHashCode(); + + public static SmoothingModel fromXContent(QueryParseContext parseContext) throws IOException { + XContentParser parser = parseContext.parser(); + ParseFieldMatcher parseFieldMatcher = parseContext.parseFieldMatcher(); + XContentParser.Token token; + String fieldName = null; + SmoothingModel model = null; + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + if (token == XContentParser.Token.FIELD_NAME) { + fieldName = parser.currentName(); + } else if (token == XContentParser.Token.START_OBJECT) { + if (parseFieldMatcher.match(fieldName, LinearInterpolation.PARSE_FIELD)) { + model = LinearInterpolation.PROTOTYPE.innerFromXContent(parseContext); + } else if (parseFieldMatcher.match(fieldName, Laplace.PARSE_FIELD)) { + model = Laplace.PROTOTYPE.innerFromXContent(parseContext); + } else if (parseFieldMatcher.match(fieldName, StupidBackoff.PARSE_FIELD)) { + model = StupidBackoff.PROTOTYPE.innerFromXContent(parseContext); + } else { + throw new IllegalArgumentException("suggester[phrase] doesn't support object field [" + fieldName + "]"); + } + } else { + throw new ParsingException(parser.getTokenLocation(), + "[smoothing] unknown token [" + token + "] after [" + fieldName + "]"); + } + } + return model; + } + + public abstract SmoothingModel innerFromXContent(QueryParseContext parseContext) throws IOException; + + public abstract WordScorerFactory buildWordScorerFactory(); + + /** + * subtype specific implementation of "equals". + */ + protected abstract boolean doEquals(SmoothingModel other); + + protected abstract XContentBuilder innerToXContent(XContentBuilder builder, Params params) throws IOException; +}
\ No newline at end of file diff --git a/core/src/main/java/org/elasticsearch/search/suggest/phrase/StupidBackoff.java b/core/src/main/java/org/elasticsearch/search/suggest/phrase/StupidBackoff.java new file mode 100644 index 0000000000..9611622d8c --- /dev/null +++ b/core/src/main/java/org/elasticsearch/search/suggest/phrase/StupidBackoff.java @@ -0,0 +1,129 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.suggest.phrase; + +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.Terms; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentParser.Token; +import org.elasticsearch.index.query.QueryParseContext; +import org.elasticsearch.search.suggest.phrase.WordScorer.WordScorerFactory; + +import java.io.IOException; +import java.util.Objects; + +/** + * A "stupid-backoff" smoothing model simialr to <a + * href="http://en.wikipedia.org/wiki/Katz's_back-off_model"> Katz's + * Backoff</a>. This model is used as the default if no model is configured. + * <p> + * See <a + * href="http://en.wikipedia.org/wiki/N-gram#Smoothing_techniques">N-Gram + * Smoothing</a> for details. + * </p> + */ +public final class StupidBackoff extends SmoothingModel { + /** + * Default discount parameter for {@link StupidBackoff} smoothing + */ + public static final double DEFAULT_BACKOFF_DISCOUNT = 0.4; + public static final StupidBackoff PROTOTYPE = new StupidBackoff(DEFAULT_BACKOFF_DISCOUNT); + private double discount = DEFAULT_BACKOFF_DISCOUNT; + private static final String NAME = "stupid_backoff"; + private static final ParseField DISCOUNT_FIELD = new ParseField("discount"); + static final ParseField PARSE_FIELD = new ParseField(NAME); + + /** + * Creates a Stupid-Backoff smoothing model. + * + * @param discount + * the discount given to lower order ngrams if the higher order ngram doesn't exits + */ + public StupidBackoff(double discount) { + this.discount = discount; + } + + /** + * @return the discount parameter of the model + */ + public double getDiscount() { + return this.discount; + } + + @Override + protected XContentBuilder innerToXContent(XContentBuilder builder, Params params) throws IOException { + builder.field(DISCOUNT_FIELD.getPreferredName(), discount); + return builder; + } + + @Override + public String getWriteableName() { + return NAME; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeDouble(discount); + } + + @Override + public StupidBackoff readFrom(StreamInput in) throws IOException { + return new StupidBackoff(in.readDouble()); + } + + @Override + protected boolean doEquals(SmoothingModel other) { + StupidBackoff otherModel = (StupidBackoff) other; + return Objects.equals(discount, otherModel.discount); + } + + @Override + protected final int doHashCode() { + return Objects.hash(discount); + } + + @Override + public SmoothingModel innerFromXContent(QueryParseContext parseContext) throws IOException { + XContentParser parser = parseContext.parser(); + XContentParser.Token token; + String fieldName = null; + double discount = DEFAULT_BACKOFF_DISCOUNT; + while ((token = parser.nextToken()) != Token.END_OBJECT) { + if (token == XContentParser.Token.FIELD_NAME) { + fieldName = parser.currentName(); + } + if (token.isValue() && parseContext.parseFieldMatcher().match(fieldName, DISCOUNT_FIELD)) { + discount = parser.doubleValue(); + } + } + return new StupidBackoff(discount); + } + + @Override + public WordScorerFactory buildWordScorerFactory() { + return (IndexReader reader, Terms terms, String field, double realWordLikelyhood, BytesRef separator) + -> new StupidBackoffScorer(reader, terms, field, realWordLikelyhood, separator, discount); + } +}
\ No newline at end of file diff --git a/core/src/main/java/org/elasticsearch/search/suggest/term/TermSuggestParser.java b/core/src/main/java/org/elasticsearch/search/suggest/term/TermSuggestParser.java index a2fd680c21..7e75976d3a 100644 --- a/core/src/main/java/org/elasticsearch/search/suggest/term/TermSuggestParser.java +++ b/core/src/main/java/org/elasticsearch/search/suggest/term/TermSuggestParser.java @@ -20,8 +20,8 @@ package org.elasticsearch.search.suggest.term; import org.elasticsearch.common.ParseFieldMatcher; import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.index.fielddata.IndexFieldDataService; import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.search.suggest.DirectSpellcheckerSettings; import org.elasticsearch.search.suggest.SuggestContextParser; import org.elasticsearch.search.suggest.SuggestUtils; @@ -38,10 +38,11 @@ public final class TermSuggestParser implements SuggestContextParser { } @Override - public SuggestionSearchContext.SuggestionContext parse(XContentParser parser, MapperService mapperService, IndexFieldDataService fieldDataService) throws IOException { + public SuggestionSearchContext.SuggestionContext parse(XContentParser parser, QueryShardContext shardContext) throws IOException { + MapperService mapperService = shardContext.getMapperService(); XContentParser.Token token; String fieldName = null; - TermSuggestionContext suggestion = new TermSuggestionContext(suggester); + TermSuggestionContext suggestion = new TermSuggestionContext(shardContext); DirectSpellcheckerSettings settings = suggestion.getDirectSpellCheckerSettings(); while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { if (token == XContentParser.Token.FIELD_NAME) { diff --git a/core/src/main/java/org/elasticsearch/search/suggest/term/TermSuggester.java b/core/src/main/java/org/elasticsearch/search/suggest/term/TermSuggester.java index e67e619bf5..78ed8be6a2 100644 --- a/core/src/main/java/org/elasticsearch/search/suggest/term/TermSuggester.java +++ b/core/src/main/java/org/elasticsearch/search/suggest/term/TermSuggester.java @@ -40,6 +40,8 @@ import java.util.List; public final class TermSuggester extends Suggester<TermSuggestionContext> { + public static final TermSuggester PROTOTYPE = new TermSuggester(); + @Override public TermSuggestion innerExecute(String name, TermSuggestionContext suggestion, IndexSearcher searcher, CharsRefBuilder spare) throws IOException { diff --git a/core/src/main/java/org/elasticsearch/search/suggest/term/TermSuggestionBuilder.java b/core/src/main/java/org/elasticsearch/search/suggest/term/TermSuggestionBuilder.java index 1378c362c5..62d6718cd2 100644 --- a/core/src/main/java/org/elasticsearch/search/suggest/term/TermSuggestionBuilder.java +++ b/core/src/main/java/org/elasticsearch/search/suggest/term/TermSuggestionBuilder.java @@ -26,7 +26,9 @@ import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.query.QueryParseContext; +import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.search.suggest.SuggestionBuilder; +import org.elasticsearch.search.suggest.SuggestionSearchContext.SuggestionContext; import java.io.IOException; import java.util.Locale; @@ -382,6 +384,12 @@ public class TermSuggestionBuilder extends SuggestionBuilder<TermSuggestionBuild } @Override + protected SuggestionContext build(QueryShardContext context) throws IOException { + // NORELEASE + throw new UnsupportedOperationException(); + } + + @Override public String getWriteableName() { return SUGGESTION_NAME; } diff --git a/core/src/main/java/org/elasticsearch/search/suggest/term/TermSuggestionContext.java b/core/src/main/java/org/elasticsearch/search/suggest/term/TermSuggestionContext.java index 4ff32d797e..3af13ced8a 100644 --- a/core/src/main/java/org/elasticsearch/search/suggest/term/TermSuggestionContext.java +++ b/core/src/main/java/org/elasticsearch/search/suggest/term/TermSuggestionContext.java @@ -18,20 +18,19 @@ */ package org.elasticsearch.search.suggest.term; +import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.search.suggest.DirectSpellcheckerSettings; -import org.elasticsearch.search.suggest.Suggester; import org.elasticsearch.search.suggest.SuggestionSearchContext.SuggestionContext; final class TermSuggestionContext extends SuggestionContext { private final DirectSpellcheckerSettings settings = new DirectSpellcheckerSettings(); - public TermSuggestionContext(Suggester<? extends TermSuggestionContext> suggester) { - super(suggester); + public TermSuggestionContext(QueryShardContext shardContext) { + super(TermSuggester.PROTOTYPE, shardContext); } public DirectSpellcheckerSettings getDirectSpellCheckerSettings() { return settings; } - }
\ No newline at end of file |