diff options
| author | Adrien Grand <jpountz@gmail.com> | 2017-04-18 15:17:21 +0200 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2017-04-18 15:17:21 +0200 |
| commit | 4632661bc71bb22fc577df476e70e9dfabaaae66 (patch) | |
| tree | 6e10e1ffaa792c5fa46251d861736e59c9ed3404 | |
| parent | f217eb8ad8d3ecc82e2f926222b1f036b0d555b7 (diff) | |
Upgrade to a Lucene 7 snapshot (#24089)
We want to upgrade to Lucene 7 ahead of time in order to be able to check whether it causes any trouble to Elasticsearch before Lucene 7.0 gets released. From a user perspective, the main benefit of this upgrade is the enhanced support for sparse fields, whose resource consumption is now function of the number of docs that have a value rather than the total number of docs in the index.
Some notes about the change:
- it includes the deprecation of the `disable_coord` parameter of the `bool` and `common_terms` queries: Lucene has removed support for coord factors
- it includes the deprecation of the `index.similarity.base` expert setting, since it was only useful to configure coords and query norms, which have both been removed
- two tests have been marked with `@AwaitsFix` because of #23966, which we intend to address after the merge
269 files changed, 3973 insertions, 3848 deletions
diff --git a/buildSrc/src/main/resources/forbidden/es-all-signatures.txt b/buildSrc/src/main/resources/forbidden/es-all-signatures.txt index 64ae6784f1..f1d271d602 100644 --- a/buildSrc/src/main/resources/forbidden/es-all-signatures.txt +++ b/buildSrc/src/main/resources/forbidden/es-all-signatures.txt @@ -26,12 +26,6 @@ java.util.concurrent.ThreadLocalRandom java.security.MessageDigest#clone() @ use org.elasticsearch.common.hash.MessageDigests -@defaultMessage this should not have been added to lucene in the first place -org.apache.lucene.index.IndexReader#getCombinedCoreAndDeletesKey() - -@defaultMessage Soon to be removed -org.apache.lucene.document.FieldType#numericType() - @defaultMessage Don't use MethodHandles in slow ways, don't be lenient in tests. java.lang.invoke.MethodHandle#invoke(java.lang.Object[]) java.lang.invoke.MethodHandle#invokeWithArguments(java.lang.Object[]) diff --git a/buildSrc/src/main/resources/forbidden/es-core-signatures.txt b/buildSrc/src/main/resources/forbidden/es-core-signatures.txt index 059be403a6..6507f05be5 100644 --- a/buildSrc/src/main/resources/forbidden/es-core-signatures.txt +++ b/buildSrc/src/main/resources/forbidden/es-core-signatures.txt @@ -36,16 +36,6 @@ org.apache.lucene.index.IndexReader#decRef() org.apache.lucene.index.IndexReader#incRef() org.apache.lucene.index.IndexReader#tryIncRef() -@defaultMessage Close listeners can only installed via ElasticsearchDirectoryReader#addReaderCloseListener -org.apache.lucene.index.IndexReader#addReaderClosedListener(org.apache.lucene.index.IndexReader$ReaderClosedListener) -org.apache.lucene.index.IndexReader#removeReaderClosedListener(org.apache.lucene.index.IndexReader$ReaderClosedListener) - -@defaultMessage Pass the precision step from the mappings explicitly instead -org.apache.lucene.search.LegacyNumericRangeQuery#newDoubleRange(java.lang.String,java.lang.Double,java.lang.Double,boolean,boolean) -org.apache.lucene.search.LegacyNumericRangeQuery#newFloatRange(java.lang.String,java.lang.Float,java.lang.Float,boolean,boolean) -org.apache.lucene.search.LegacyNumericRangeQuery#newIntRange(java.lang.String,java.lang.Integer,java.lang.Integer,boolean,boolean) -org.apache.lucene.search.LegacyNumericRangeQuery#newLongRange(java.lang.String,java.lang.Long,java.lang.Long,boolean,boolean) - @defaultMessage Only use wait / notify when really needed try to use concurrency primitives, latches or callbacks instead. java.lang.Object#wait() java.lang.Object#wait(long) diff --git a/buildSrc/version.properties b/buildSrc/version.properties index a3a1681eb3..ca263b0efa 100644 --- a/buildSrc/version.properties +++ b/buildSrc/version.properties @@ -1,6 +1,6 @@ # When updating elasticsearch, please update 'rest' version in core/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy elasticsearch = 6.0.0-alpha1 -lucene = 6.5.0 +lucene = 7.0.0-snapshot-89f6d17 # optional dependencies spatial4j = 0.6 diff --git a/core/licenses/lucene-analyzers-common-6.5.0.jar.sha1 b/core/licenses/lucene-analyzers-common-6.5.0.jar.sha1 deleted file mode 100644 index 77d21a2377..0000000000 --- a/core/licenses/lucene-analyzers-common-6.5.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -3989779b05ecd0ace6affe19223b1c27156604f1
\ No newline at end of file diff --git a/core/licenses/lucene-analyzers-common-7.0.0-snapshot-89f6d17.jar.sha1 b/core/licenses/lucene-analyzers-common-7.0.0-snapshot-89f6d17.jar.sha1 new file mode 100644 index 0000000000..93afa8a73f --- /dev/null +++ b/core/licenses/lucene-analyzers-common-7.0.0-snapshot-89f6d17.jar.sha1 @@ -0,0 +1 @@ +e69234c2e898d86a53edbe8d22e33bebc45286cd
\ No newline at end of file diff --git a/core/licenses/lucene-backward-codecs-6.5.0.jar.sha1 b/core/licenses/lucene-backward-codecs-6.5.0.jar.sha1 deleted file mode 100644 index 9eaff80ac0..0000000000 --- a/core/licenses/lucene-backward-codecs-6.5.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -6a8660e7133f357ef40d9cac26316ccd9937a2eb
\ No newline at end of file diff --git a/core/licenses/lucene-backward-codecs-7.0.0-snapshot-89f6d17.jar.sha1 b/core/licenses/lucene-backward-codecs-7.0.0-snapshot-89f6d17.jar.sha1 new file mode 100644 index 0000000000..707ed41267 --- /dev/null +++ b/core/licenses/lucene-backward-codecs-7.0.0-snapshot-89f6d17.jar.sha1 @@ -0,0 +1 @@ +48172a8e1fe6562f55ab671d42af53652794d5df
\ No newline at end of file diff --git a/core/licenses/lucene-core-6.5.0.jar.sha1 b/core/licenses/lucene-core-6.5.0.jar.sha1 deleted file mode 100644 index 7af91ec15b..0000000000 --- a/core/licenses/lucene-core-6.5.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -ff176c9bde4228b43827849f5d2ff2e2717e3297
\ No newline at end of file diff --git a/core/licenses/lucene-core-7.0.0-snapshot-89f6d17.jar.sha1 b/core/licenses/lucene-core-7.0.0-snapshot-89f6d17.jar.sha1 new file mode 100644 index 0000000000..ef6a939a66 --- /dev/null +++ b/core/licenses/lucene-core-7.0.0-snapshot-89f6d17.jar.sha1 @@ -0,0 +1 @@ +3dab251d4c7ab4ff5095e5f1d1e127ec2cf3c07d
\ No newline at end of file diff --git a/core/licenses/lucene-grouping-6.5.0.jar.sha1 b/core/licenses/lucene-grouping-6.5.0.jar.sha1 deleted file mode 100644 index 08ccc2cd08..0000000000 --- a/core/licenses/lucene-grouping-6.5.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -10d2e5b36f460527ac9b948be0ec3077bde5b0ca
\ No newline at end of file diff --git a/core/licenses/lucene-grouping-7.0.0-snapshot-89f6d17.jar.sha1 b/core/licenses/lucene-grouping-7.0.0-snapshot-89f6d17.jar.sha1 new file mode 100644 index 0000000000..dfa06b60b4 --- /dev/null +++ b/core/licenses/lucene-grouping-7.0.0-snapshot-89f6d17.jar.sha1 @@ -0,0 +1 @@ +c01ae8a23b733d75d058a76bd85fcb49b9fd06fd
\ No newline at end of file diff --git a/core/licenses/lucene-highlighter-6.5.0.jar.sha1 b/core/licenses/lucene-highlighter-6.5.0.jar.sha1 deleted file mode 100644 index a8069723f1..0000000000 --- a/core/licenses/lucene-highlighter-6.5.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -0019bb6a631ea0123e8e553b0510fa81c9d3c3eb
\ No newline at end of file diff --git a/core/licenses/lucene-highlighter-7.0.0-snapshot-89f6d17.jar.sha1 b/core/licenses/lucene-highlighter-7.0.0-snapshot-89f6d17.jar.sha1 new file mode 100644 index 0000000000..51c8a47474 --- /dev/null +++ b/core/licenses/lucene-highlighter-7.0.0-snapshot-89f6d17.jar.sha1 @@ -0,0 +1 @@ +c53df048b97946fe66035505306b5651b702adb1
\ No newline at end of file diff --git a/core/licenses/lucene-join-6.5.0.jar.sha1 b/core/licenses/lucene-join-6.5.0.jar.sha1 deleted file mode 100644 index cbad6199d7..0000000000 --- a/core/licenses/lucene-join-6.5.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -dad85baba266793b9ceb80a9b08c4ee9838e09df
\ No newline at end of file diff --git a/core/licenses/lucene-join-7.0.0-snapshot-89f6d17.jar.sha1 b/core/licenses/lucene-join-7.0.0-snapshot-89f6d17.jar.sha1 new file mode 100644 index 0000000000..ad7e123f6d --- /dev/null +++ b/core/licenses/lucene-join-7.0.0-snapshot-89f6d17.jar.sha1 @@ -0,0 +1 @@ +1ecb349ba29abab75359e5125ac8a94fc81441d5
\ No newline at end of file diff --git a/core/licenses/lucene-memory-6.5.0.jar.sha1 b/core/licenses/lucene-memory-6.5.0.jar.sha1 deleted file mode 100644 index 5f22c0d9cf..0000000000 --- a/core/licenses/lucene-memory-6.5.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -938f9f7efe8a403fd57c99aedd75d040d9caa896
\ No newline at end of file diff --git a/core/licenses/lucene-memory-7.0.0-snapshot-89f6d17.jar.sha1 b/core/licenses/lucene-memory-7.0.0-snapshot-89f6d17.jar.sha1 new file mode 100644 index 0000000000..6c9fc8be42 --- /dev/null +++ b/core/licenses/lucene-memory-7.0.0-snapshot-89f6d17.jar.sha1 @@ -0,0 +1 @@ +e5f53b38652b1284ff254fba39e624ec117aef7d
\ No newline at end of file diff --git a/core/licenses/lucene-misc-6.5.0.jar.sha1 b/core/licenses/lucene-misc-6.5.0.jar.sha1 deleted file mode 100644 index 2b405d7f16..0000000000 --- a/core/licenses/lucene-misc-6.5.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -afdff39ecb30f6e2c6f056a5bdfcb13d928a25af
\ No newline at end of file diff --git a/core/licenses/lucene-misc-7.0.0-snapshot-89f6d17.jar.sha1 b/core/licenses/lucene-misc-7.0.0-snapshot-89f6d17.jar.sha1 new file mode 100644 index 0000000000..d69e68cdde --- /dev/null +++ b/core/licenses/lucene-misc-7.0.0-snapshot-89f6d17.jar.sha1 @@ -0,0 +1 @@ +2f340ed3f46d6b4c89fa31975b675c19028c15eb
\ No newline at end of file diff --git a/core/licenses/lucene-queries-6.5.0.jar.sha1 b/core/licenses/lucene-queries-6.5.0.jar.sha1 deleted file mode 100644 index 9a046ce204..0000000000 --- a/core/licenses/lucene-queries-6.5.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -8e3971a008070712d57b59cf1f7b44c0d9d3df25
\ No newline at end of file diff --git a/core/licenses/lucene-queries-7.0.0-snapshot-89f6d17.jar.sha1 b/core/licenses/lucene-queries-7.0.0-snapshot-89f6d17.jar.sha1 new file mode 100644 index 0000000000..1d2f98e6c8 --- /dev/null +++ b/core/licenses/lucene-queries-7.0.0-snapshot-89f6d17.jar.sha1 @@ -0,0 +1 @@ +a13862fb62cc1e516d16d6b6bb3cdb906c4925f6
\ No newline at end of file diff --git a/core/licenses/lucene-queryparser-6.5.0.jar.sha1 b/core/licenses/lucene-queryparser-6.5.0.jar.sha1 deleted file mode 100644 index 3136885ab9..0000000000 --- a/core/licenses/lucene-queryparser-6.5.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -225b904edf91ccdffffa398e1924ebadd5677c09
\ No newline at end of file diff --git a/core/licenses/lucene-queryparser-7.0.0-snapshot-89f6d17.jar.sha1 b/core/licenses/lucene-queryparser-7.0.0-snapshot-89f6d17.jar.sha1 new file mode 100644 index 0000000000..29492f1ed6 --- /dev/null +++ b/core/licenses/lucene-queryparser-7.0.0-snapshot-89f6d17.jar.sha1 @@ -0,0 +1 @@ +4e014f72a588453bae7dd1a555d741cf3bf39032
\ No newline at end of file diff --git a/core/licenses/lucene-sandbox-6.5.0.jar.sha1 b/core/licenses/lucene-sandbox-6.5.0.jar.sha1 deleted file mode 100644 index e3787e336d..0000000000 --- a/core/licenses/lucene-sandbox-6.5.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -5c994fc5dc4f37133a861571211303d81c5d51ff
\ No newline at end of file diff --git a/core/licenses/lucene-sandbox-7.0.0-snapshot-89f6d17.jar.sha1 b/core/licenses/lucene-sandbox-7.0.0-snapshot-89f6d17.jar.sha1 new file mode 100644 index 0000000000..80794b3e27 --- /dev/null +++ b/core/licenses/lucene-sandbox-7.0.0-snapshot-89f6d17.jar.sha1 @@ -0,0 +1 @@ +5e87d61c604d6b1c0ee5c38f09441d1b8b9c8c2b
\ No newline at end of file diff --git a/core/licenses/lucene-spatial-6.5.0.jar.sha1 b/core/licenses/lucene-spatial-6.5.0.jar.sha1 deleted file mode 100644 index cbadbfc42d..0000000000 --- a/core/licenses/lucene-spatial-6.5.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -553b7b13bef994f14076a85557df03cad67322e9
\ No newline at end of file diff --git a/core/licenses/lucene-spatial-7.0.0-snapshot-89f6d17.jar.sha1 b/core/licenses/lucene-spatial-7.0.0-snapshot-89f6d17.jar.sha1 new file mode 100644 index 0000000000..859a89f5a0 --- /dev/null +++ b/core/licenses/lucene-spatial-7.0.0-snapshot-89f6d17.jar.sha1 @@ -0,0 +1 @@ +be14aa163b339403d8ec904493c1be5dfa9baeaf
\ No newline at end of file diff --git a/core/licenses/lucene-spatial-extras-6.5.0.jar.sha1 b/core/licenses/lucene-spatial-extras-6.5.0.jar.sha1 deleted file mode 100644 index f2ad71855f..0000000000 --- a/core/licenses/lucene-spatial-extras-6.5.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -73deae791d861820974600705ba06e9f801cbe56
\ No newline at end of file diff --git a/core/licenses/lucene-spatial-extras-7.0.0-snapshot-89f6d17.jar.sha1 b/core/licenses/lucene-spatial-extras-7.0.0-snapshot-89f6d17.jar.sha1 new file mode 100644 index 0000000000..ee3c729288 --- /dev/null +++ b/core/licenses/lucene-spatial-extras-7.0.0-snapshot-89f6d17.jar.sha1 @@ -0,0 +1 @@ +a2c13be0fe4c5a98a30ec6ae673be1442409817c
\ No newline at end of file diff --git a/core/licenses/lucene-spatial3d-6.5.0.jar.sha1 b/core/licenses/lucene-spatial3d-6.5.0.jar.sha1 deleted file mode 100644 index 8fca9ac1eb..0000000000 --- a/core/licenses/lucene-spatial3d-6.5.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -c2aad69500dac79338ef45f570cab47bec3d2724
\ No newline at end of file diff --git a/core/licenses/lucene-spatial3d-7.0.0-snapshot-89f6d17.jar.sha1 b/core/licenses/lucene-spatial3d-7.0.0-snapshot-89f6d17.jar.sha1 new file mode 100644 index 0000000000..28fab4a381 --- /dev/null +++ b/core/licenses/lucene-spatial3d-7.0.0-snapshot-89f6d17.jar.sha1 @@ -0,0 +1 @@ +92b8282e474845fdae31f9f239f953bc7164401f
\ No newline at end of file diff --git a/core/licenses/lucene-suggest-6.5.0.jar.sha1 b/core/licenses/lucene-suggest-6.5.0.jar.sha1 deleted file mode 100644 index 62764fbbc3..0000000000 --- a/core/licenses/lucene-suggest-6.5.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -acf211f2bf901dfc8155a46c5a42c5650edf74ef
\ No newline at end of file diff --git a/core/licenses/lucene-suggest-7.0.0-snapshot-89f6d17.jar.sha1 b/core/licenses/lucene-suggest-7.0.0-snapshot-89f6d17.jar.sha1 new file mode 100644 index 0000000000..e928966f27 --- /dev/null +++ b/core/licenses/lucene-suggest-7.0.0-snapshot-89f6d17.jar.sha1 @@ -0,0 +1 @@ +1c4aaea267ed41657ebf01769bfddbcab5b27414
\ No newline at end of file diff --git a/core/src/main/java/org/apache/lucene/queries/BlendedTermQuery.java b/core/src/main/java/org/apache/lucene/queries/BlendedTermQuery.java index 0b34a95710..dbad7e0bf7 100644 --- a/core/src/main/java/org/apache/lucene/queries/BlendedTermQuery.java +++ b/core/src/main/java/org/apache/lucene/queries/BlendedTermQuery.java @@ -296,16 +296,15 @@ public abstract class BlendedTermQuery extends Query { return Objects.hash(classHash(), Arrays.hashCode(equalsTerms())); } - public static BlendedTermQuery booleanBlendedQuery(Term[] terms, final boolean disableCoord) { - return booleanBlendedQuery(terms, null, disableCoord); + public static BlendedTermQuery booleanBlendedQuery(Term[] terms) { + return booleanBlendedQuery(terms, null); } - public static BlendedTermQuery booleanBlendedQuery(Term[] terms, final float[] boosts, final boolean disableCoord) { + public static BlendedTermQuery booleanBlendedQuery(Term[] terms, final float[] boosts) { return new BlendedTermQuery(terms, boosts) { @Override protected Query topLevelQuery(Term[] terms, TermContext[] ctx, int[] docFreqs, int maxDoc) { BooleanQuery.Builder booleanQueryBuilder = new BooleanQuery.Builder(); - booleanQueryBuilder.setDisableCoord(disableCoord); for (int i = 0; i < terms.length; i++) { Query query = new TermQuery(terms[i], ctx[i]); if (boosts != null && boosts[i] != 1f) { @@ -318,14 +317,12 @@ public abstract class BlendedTermQuery extends Query { }; } - public static BlendedTermQuery commonTermsBlendedQuery(Term[] terms, final float[] boosts, final boolean disableCoord, final float maxTermFrequency) { + public static BlendedTermQuery commonTermsBlendedQuery(Term[] terms, final float[] boosts, final float maxTermFrequency) { return new BlendedTermQuery(terms, boosts) { @Override protected Query topLevelQuery(Term[] terms, TermContext[] ctx, int[] docFreqs, int maxDoc) { BooleanQuery.Builder highBuilder = new BooleanQuery.Builder(); - highBuilder.setDisableCoord(disableCoord); BooleanQuery.Builder lowBuilder = new BooleanQuery.Builder(); - lowBuilder.setDisableCoord(disableCoord); for (int i = 0; i < terms.length; i++) { Query query = new TermQuery(terms[i], ctx[i]); if (boosts != null && boosts[i] != 1f) { @@ -343,7 +340,6 @@ public abstract class BlendedTermQuery extends Query { BooleanQuery low = lowBuilder.build(); if (low.clauses().isEmpty()) { BooleanQuery.Builder queryBuilder = new BooleanQuery.Builder(); - queryBuilder.setDisableCoord(disableCoord); for (BooleanClause booleanClause : high) { queryBuilder.add(booleanClause.getQuery(), Occur.MUST); } @@ -352,7 +348,6 @@ public abstract class BlendedTermQuery extends Query { return low; } else { return new BooleanQuery.Builder() - .setDisableCoord(true) .add(high, BooleanClause.Occur.SHOULD) .add(low, BooleanClause.Occur.MUST) .build(); diff --git a/core/src/main/java/org/apache/lucene/queries/ExtendedCommonTermsQuery.java b/core/src/main/java/org/apache/lucene/queries/ExtendedCommonTermsQuery.java index 1889c6e759..4580de4cc4 100644 --- a/core/src/main/java/org/apache/lucene/queries/ExtendedCommonTermsQuery.java +++ b/core/src/main/java/org/apache/lucene/queries/ExtendedCommonTermsQuery.java @@ -35,8 +35,8 @@ public class ExtendedCommonTermsQuery extends CommonTermsQuery { private final MappedFieldType fieldType; - public ExtendedCommonTermsQuery(Occur highFreqOccur, Occur lowFreqOccur, float maxTermFrequency, boolean disableCoord, MappedFieldType fieldType) { - super(highFreqOccur, lowFreqOccur, maxTermFrequency, disableCoord); + public ExtendedCommonTermsQuery(Occur highFreqOccur, Occur lowFreqOccur, float maxTermFrequency, MappedFieldType fieldType) { + super(highFreqOccur, lowFreqOccur, maxTermFrequency); this.fieldType = fieldType; } diff --git a/core/src/main/java/org/apache/lucene/queries/MinDocQuery.java b/core/src/main/java/org/apache/lucene/queries/MinDocQuery.java index a8b7dc9299..d4f9ab7297 100644 --- a/core/src/main/java/org/apache/lucene/queries/MinDocQuery.java +++ b/core/src/main/java/org/apache/lucene/queries/MinDocQuery.java @@ -57,8 +57,8 @@ public final class MinDocQuery extends Query { } @Override - public Weight createWeight(IndexSearcher searcher, boolean needsScores) throws IOException { - return new ConstantScoreWeight(this) { + public Weight createWeight(IndexSearcher searcher, boolean needsScores, float boost) throws IOException { + return new ConstantScoreWeight(this, boost) { @Override public Scorer scorer(LeafReaderContext context) throws IOException { final int maxDoc = context.reader().maxDoc(); diff --git a/core/src/main/java/org/apache/lucene/queryparser/classic/MapperQueryParser.java b/core/src/main/java/org/apache/lucene/queryparser/classic/MapperQueryParser.java index 2b592aa935..79f522e8c1 100644 --- a/core/src/main/java/org/apache/lucene/queryparser/classic/MapperQueryParser.java +++ b/core/src/main/java/org/apache/lucene/queryparser/classic/MapperQueryParser.java @@ -25,9 +25,7 @@ import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute; import org.apache.lucene.index.Term; -import org.apache.lucene.queryparser.analyzing.AnalyzingQueryParser; import org.apache.lucene.search.BooleanClause; -import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.BoostQuery; import org.apache.lucene.search.DisjunctionMaxQuery; import org.apache.lucene.search.FuzzyQuery; @@ -70,7 +68,7 @@ import static org.elasticsearch.common.lucene.search.Queries.fixNegativeQueryIfN * Also breaks fields with [type].[name] into a boolean query that must include the type * as well as the query on the name. */ -public class MapperQueryParser extends AnalyzingQueryParser { +public class MapperQueryParser extends QueryParser { public static final Map<String, FieldQueryExtension> FIELD_QUERY_EXTENSIONS; @@ -103,14 +101,13 @@ public class MapperQueryParser extends AnalyzingQueryParser { setAnalyzer(settings.analyzer()); setMultiTermRewriteMethod(settings.rewriteMethod()); setEnablePositionIncrements(settings.enablePositionIncrements()); + setSplitOnWhitespace(settings.splitOnWhitespace()); setAutoGeneratePhraseQueries(settings.autoGeneratePhraseQueries()); setMaxDeterminizedStates(settings.maxDeterminizedStates()); setAllowLeadingWildcard(settings.allowLeadingWildcard()); - setLowercaseExpandedTerms(false); setPhraseSlop(settings.phraseSlop()); setDefaultOperator(settings.defaultOperator()); setFuzzyPrefixLength(settings.fuzzyPrefixLength()); - setSplitOnWhitespace(settings.splitOnWhitespace()); } /** @@ -175,7 +172,7 @@ public class MapperQueryParser extends AnalyzingQueryParser { } } if (clauses.isEmpty()) return null; // happens for stopwords - return getBooleanQueryCoordDisabled(clauses); + return getBooleanQuery(clauses); } } else { return getFieldQuerySingle(field, queryText, quoted); @@ -277,7 +274,7 @@ public class MapperQueryParser extends AnalyzingQueryParser { } } if (clauses.isEmpty()) return null; // happens for stopwords - return getBooleanQueryCoordDisabled(clauses); + return getBooleanQuery(clauses); } } else { return super.getFieldQuery(field, queryText, slop); @@ -328,7 +325,7 @@ public class MapperQueryParser extends AnalyzingQueryParser { } } if (clauses.isEmpty()) return null; // happens for stopwords - return getBooleanQueryCoordDisabled(clauses); + return getBooleanQuery(clauses); } } @@ -386,7 +383,7 @@ public class MapperQueryParser extends AnalyzingQueryParser { clauses.add(new BooleanClause(applyBoost(mField, q), BooleanClause.Occur.SHOULD)); } } - return getBooleanQueryCoordDisabled(clauses); + return getBooleanQuery(clauses); } } else { return getFuzzyQuerySingle(field, termStr, minSimilarity); @@ -450,7 +447,7 @@ public class MapperQueryParser extends AnalyzingQueryParser { } } if (clauses.isEmpty()) return null; // happens for stopwords - return getBooleanQueryCoordDisabled(clauses); + return getBooleanQuery(clauses); } } else { return getPrefixQuerySingle(field, termStr); @@ -559,7 +556,7 @@ public class MapperQueryParser extends AnalyzingQueryParser { innerClauses.add(new BooleanClause(super.getPrefixQuery(field, token), BooleanClause.Occur.SHOULD)); } - posQuery = getBooleanQueryCoordDisabled(innerClauses); + posQuery = getBooleanQuery(innerClauses); } clauses.add(new BooleanClause(posQuery, getDefaultOperator() == Operator.AND ? BooleanClause.Occur.MUST : BooleanClause.Occur.SHOULD)); @@ -612,7 +609,7 @@ public class MapperQueryParser extends AnalyzingQueryParser { } } if (clauses.isEmpty()) return null; // happens for stopwords - return getBooleanQueryCoordDisabled(clauses); + return getBooleanQuery(clauses); } } else { return getWildcardQuerySingle(field, termStr); @@ -676,7 +673,7 @@ public class MapperQueryParser extends AnalyzingQueryParser { } } if (clauses.isEmpty()) return null; // happens for stopwords - return getBooleanQueryCoordDisabled(clauses); + return getBooleanQuery(clauses); } } else { return getRegexpQuerySingle(field, termStr); @@ -713,19 +710,6 @@ public class MapperQueryParser extends AnalyzingQueryParser { } } - /** - * @deprecated review all use of this, don't rely on coord - */ - @Deprecated - protected Query getBooleanQueryCoordDisabled(List<BooleanClause> clauses) throws ParseException { - BooleanQuery.Builder builder = new BooleanQuery.Builder(); - builder.setDisableCoord(true); - for (BooleanClause clause : clauses) { - builder.add(clause); - } - return fixNegativeQueryIfNeeded(builder.build()); - } - @Override protected Query getBooleanQuery(List<BooleanClause> clauses) throws ParseException { diff --git a/core/src/main/java/org/apache/lucene/search/grouping/CollapsingDocValuesSource.java b/core/src/main/java/org/apache/lucene/search/grouping/CollapsingDocValuesSource.java index 5bc8afb347..e487733890 100644 --- a/core/src/main/java/org/apache/lucene/search/grouping/CollapsingDocValuesSource.java +++ b/core/src/main/java/org/apache/lucene/search/grouping/CollapsingDocValuesSource.java @@ -22,31 +22,32 @@ import org.apache.lucene.index.DocValues; import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.LeafReader; +import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.NumericDocValues; import org.apache.lucene.index.SortedDocValues; import org.apache.lucene.index.SortedNumericDocValues; import org.apache.lucene.index.SortedSetDocValues; -import org.apache.lucene.util.ArrayUtil; -import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.index.fielddata.AbstractNumericDocValues; +import org.elasticsearch.index.fielddata.AbstractSortedDocValues; import java.io.IOException; +import java.util.Collection; /** * Utility class that ensures that a single collapse key is extracted per document. */ -abstract class CollapsingDocValuesSource<T> { +abstract class CollapsingDocValuesSource<T> extends GroupSelector<T> { protected final String field; CollapsingDocValuesSource(String field) throws IOException { this.field = field; } - abstract T get(int doc); - - abstract T copy(T value, T reuse); - - abstract void setNextReader(LeafReader reader) throws IOException; + @Override + public void setGroups(Collection<SearchGroup<T>> groups) { + throw new UnsupportedOperationException(); + } /** * Implementation for {@link NumericDocValues} and {@link SortedNumericDocValues}. @@ -54,35 +55,43 @@ abstract class CollapsingDocValuesSource<T> { */ static class Numeric extends CollapsingDocValuesSource<Long> { private NumericDocValues values; - private Bits docsWithField; + private long value; + private boolean hasValue; Numeric(String field) throws IOException { super(field); } @Override - public Long get(int doc) { - if (docsWithField.get(doc)) { - return values.get(doc); + public State advanceTo(int doc) throws IOException { + if (values.advanceExact(doc)) { + hasValue = true; + value = values.longValue(); + return State.ACCEPT; } else { - return null; + hasValue = false; + return State.SKIP; } } @Override - public Long copy(Long value, Long reuse) { - return value; + public Long currentValue() { + return hasValue ? value : null; } @Override - public void setNextReader(LeafReader reader) throws IOException { + public Long copyValue() { + return currentValue(); + } + + @Override + public void setNextReader(LeafReaderContext readerContext) throws IOException { + LeafReader reader = readerContext.reader(); DocValuesType type = getDocValuesType(reader, field); if (type == null || type == DocValuesType.NONE) { values = DocValues.emptyNumeric(); - docsWithField = new Bits.MatchNoBits(reader.maxDoc()); return ; } - docsWithField = DocValues.getDocsWithField(reader, field); switch (type) { case NUMERIC: values = DocValues.getNumeric(reader, field); @@ -92,17 +101,34 @@ abstract class CollapsingDocValuesSource<T> { final SortedNumericDocValues sorted = DocValues.getSortedNumeric(reader, field); values = DocValues.unwrapSingleton(sorted); if (values == null) { - values = new NumericDocValues() { + values = new AbstractNumericDocValues() { + + private long value; + @Override - public long get(int docID) { - sorted.setDocument(docID); - assert sorted.count() > 0; - if (sorted.count() > 1) { - throw new IllegalStateException("failed to collapse " + docID + - ", the collapse field must be single valued"); + public boolean advanceExact(int target) throws IOException { + if (sorted.advanceExact(target)) { + if (sorted.docValueCount() > 1) { + throw new IllegalStateException("failed to collapse " + target + + ", the collapse field must be single valued"); + } + value = sorted.nextValue(); + return true; + } else { + return false; } - return sorted.valueAt(0); } + + @Override + public int docID() { + return sorted.docID(); + } + + @Override + public long longValue() throws IOException { + return value; + } + }; } break; @@ -119,47 +145,56 @@ abstract class CollapsingDocValuesSource<T> { * Fails with an {@link IllegalStateException} if a document contains multiple values for the specified field. */ static class Keyword extends CollapsingDocValuesSource<BytesRef> { - private Bits docsWithField; private SortedDocValues values; + private int ord; Keyword(String field) throws IOException { super(field); } @Override - public BytesRef get(int doc) { - if (docsWithField.get(doc)) { - return values.get(doc); + public org.apache.lucene.search.grouping.GroupSelector.State advanceTo(int doc) + throws IOException { + if (values.advanceExact(doc)) { + ord = values.ordValue(); + return State.ACCEPT; } else { + ord = -1; + return State.SKIP; + } + } + + @Override + public BytesRef currentValue() { + if (ord == -1) { return null; + } else { + try { + return values.lookupOrd(ord); + } catch (IOException e) { + throw new RuntimeException(e); + } } } @Override - public BytesRef copy(BytesRef value, BytesRef reuse) { + public BytesRef copyValue() { + BytesRef value = currentValue(); if (value == null) { return null; - } - if (reuse != null) { - reuse.bytes = ArrayUtil.grow(reuse.bytes, value.length); - reuse.offset = 0; - reuse.length = value.length; - System.arraycopy(value.bytes, value.offset, reuse.bytes, 0, value.length); - return reuse; } else { return BytesRef.deepCopyOf(value); } } @Override - public void setNextReader(LeafReader reader) throws IOException { + public void setNextReader(LeafReaderContext readerContext) throws IOException { + LeafReader reader = readerContext.reader(); DocValuesType type = getDocValuesType(reader, field); if (type == null || type == DocValuesType.NONE) { values = DocValues.emptySorted(); - docsWithField = new Bits.MatchNoBits(reader.maxDoc()); return ; } - docsWithField = DocValues.getDocsWithField(reader, field); switch (type) { case SORTED: values = DocValues.getSorted(reader, field); @@ -169,20 +204,36 @@ abstract class CollapsingDocValuesSource<T> { final SortedSetDocValues sorted = DocValues.getSortedSet(reader, field); values = DocValues.unwrapSingleton(sorted); if (values == null) { - values = new SortedDocValues() { + values = new AbstractSortedDocValues() { + + private int ord; + @Override - public int getOrd(int docID) { - sorted.setDocument(docID); - int ord = (int) sorted.nextOrd(); - if (sorted.nextOrd() != SortedSetDocValues.NO_MORE_ORDS) { - throw new IllegalStateException("failed to collapse " + docID + - ", the collapse field must be single valued"); + public boolean advanceExact(int target) throws IOException { + if (sorted.advanceExact(target)) { + ord = (int) sorted.nextOrd(); + if (sorted.nextOrd() != SortedSetDocValues.NO_MORE_ORDS) { + throw new IllegalStateException("failed to collapse " + target + + ", the collapse field must be single valued"); + } + return true; + } else { + return false; } + } + + @Override + public int docID() { + return sorted.docID(); + } + + @Override + public int ordValue() { return ord; } @Override - public BytesRef lookupOrd(int ord) { + public BytesRef lookupOrd(int ord) throws IOException { return sorted.lookupOrd(ord); } diff --git a/core/src/main/java/org/apache/lucene/search/grouping/CollapsingTopDocsCollector.java b/core/src/main/java/org/apache/lucene/search/grouping/CollapsingTopDocsCollector.java index 955a63e548..b5cb02bcd6 100644 --- a/core/src/main/java/org/apache/lucene/search/grouping/CollapsingTopDocsCollector.java +++ b/core/src/main/java/org/apache/lucene/search/grouping/CollapsingTopDocsCollector.java @@ -18,13 +18,11 @@ */ package org.apache.lucene.search.grouping; -import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.search.FieldDoc; import org.apache.lucene.search.ScoreDoc; import org.apache.lucene.search.Scorer; import org.apache.lucene.search.Sort; import org.apache.lucene.search.SortField; -import org.apache.lucene.util.BytesRef; import java.io.IOException; import java.util.Collection; @@ -37,7 +35,7 @@ import static org.apache.lucene.search.SortField.Type.SCORE; * output. The collapsing is done in a single pass by selecting only the top sorted document per collapse key. * The value used for the collapse key of each group can be found in {@link CollapseTopFieldDocs#collapseValues}. */ -public abstract class CollapsingTopDocsCollector<T> extends FirstPassGroupingCollector<T> { +public final class CollapsingTopDocsCollector<T> extends FirstPassGroupingCollector<T> { protected final String collapseField; protected final Sort sort; @@ -47,9 +45,9 @@ public abstract class CollapsingTopDocsCollector<T> extends FirstPassGroupingCol private float maxScore; private final boolean trackMaxScore; - private CollapsingTopDocsCollector(String collapseField, Sort sort, + CollapsingTopDocsCollector(GroupSelector<T> groupSelector, String collapseField, Sort sort, int topN, boolean trackMaxScore) throws IOException { - super(sort, topN); + super(groupSelector, sort, topN); this.collapseField = collapseField; this.trackMaxScore = trackMaxScore; if (trackMaxScore) { @@ -65,7 +63,7 @@ public abstract class CollapsingTopDocsCollector<T> extends FirstPassGroupingCol * {@link CollapseTopFieldDocs}. The collapsing needs only one pass so we can create the final top docs at the end * of the first pass. */ - public CollapseTopFieldDocs getTopDocs() { + public CollapseTopFieldDocs getTopDocs() throws IOException { Collection<SearchGroup<T>> groups = super.getTopGroups(0, true); if (groups == null) { return new CollapseTopFieldDocs(collapseField, totalHitCount, new ScoreDoc[0], @@ -121,57 +119,6 @@ public abstract class CollapsingTopDocsCollector<T> extends FirstPassGroupingCol totalHitCount++; } - private static class Numeric extends CollapsingTopDocsCollector<Long> { - private final CollapsingDocValuesSource.Numeric source; - - private Numeric(String collapseField, Sort sort, int topN, boolean trackMaxScore) throws IOException { - super(collapseField, sort, topN, trackMaxScore); - source = new CollapsingDocValuesSource.Numeric(collapseField); - } - - @Override - protected void doSetNextReader(LeafReaderContext readerContext) throws IOException { - super.doSetNextReader(readerContext); - source.setNextReader(readerContext.reader()); - } - - @Override - protected Long getDocGroupValue(int doc) { - return source.get(doc); - } - - @Override - protected Long copyDocGroupValue(Long groupValue, Long reuse) { - return source.copy(groupValue, reuse); - } - } - - private static class Keyword extends CollapsingTopDocsCollector<BytesRef> { - private final CollapsingDocValuesSource.Keyword source; - - private Keyword(String collapseField, Sort sort, int topN, boolean trackMaxScore) throws IOException { - super(collapseField, sort, topN, trackMaxScore); - source = new CollapsingDocValuesSource.Keyword(collapseField); - - } - - @Override - protected void doSetNextReader(LeafReaderContext readerContext) throws IOException { - super.doSetNextReader(readerContext); - source.setNextReader(readerContext.reader()); - } - - @Override - protected BytesRef getDocGroupValue(int doc) { - return source.get(doc); - } - - @Override - protected BytesRef copyDocGroupValue(BytesRef groupValue, BytesRef reuse) { - return source.copy(groupValue, reuse); - } - } - /** * Create a collapsing top docs collector on a {@link org.apache.lucene.index.NumericDocValues} field. * It accepts also {@link org.apache.lucene.index.SortedNumericDocValues} field but @@ -189,7 +136,8 @@ public abstract class CollapsingTopDocsCollector<T> extends FirstPassGroupingCol */ public static CollapsingTopDocsCollector<?> createNumeric(String collapseField, Sort sort, int topN, boolean trackMaxScore) throws IOException { - return new Numeric(collapseField, sort, topN, trackMaxScore); + return new CollapsingTopDocsCollector<>(new CollapsingDocValuesSource.Numeric(collapseField), + collapseField, sort, topN, trackMaxScore); } /** @@ -208,7 +156,8 @@ public abstract class CollapsingTopDocsCollector<T> extends FirstPassGroupingCol */ public static CollapsingTopDocsCollector<?> createKeyword(String collapseField, Sort sort, int topN, boolean trackMaxScore) throws IOException { - return new Keyword(collapseField, sort, topN, trackMaxScore); + return new CollapsingTopDocsCollector<>(new CollapsingDocValuesSource.Keyword(collapseField), + collapseField, sort, topN, trackMaxScore); } } diff --git a/core/src/main/java/org/elasticsearch/Version.java b/core/src/main/java/org/elasticsearch/Version.java index fad0d403d2..b3f38e57b5 100644 --- a/core/src/main/java/org/elasticsearch/Version.java +++ b/core/src/main/java/org/elasticsearch/Version.java @@ -82,7 +82,7 @@ public class Version implements Comparable<Version> { public static final Version V_5_5_0_UNRELEASED = new Version(V_5_5_0_ID_UNRELEASED, org.apache.lucene.util.Version.LUCENE_6_5_0); public static final int V_6_0_0_alpha1_ID_UNRELEASED = 6000001; public static final Version V_6_0_0_alpha1_UNRELEASED = - new Version(V_6_0_0_alpha1_ID_UNRELEASED, org.apache.lucene.util.Version.LUCENE_6_5_0); + new Version(V_6_0_0_alpha1_ID_UNRELEASED, org.apache.lucene.util.Version.LUCENE_7_0_0); public static final Version CURRENT = V_6_0_0_alpha1_UNRELEASED; // unreleased versions must be added to the above list with the suffix _UNRELEASED (with the exception of CURRENT) diff --git a/core/src/main/java/org/elasticsearch/common/geo/GeoHashUtils.java b/core/src/main/java/org/elasticsearch/common/geo/GeoHashUtils.java index 0f1ddcd376..cb31940a49 100644 --- a/core/src/main/java/org/elasticsearch/common/geo/GeoHashUtils.java +++ b/core/src/main/java/org/elasticsearch/common/geo/GeoHashUtils.java @@ -18,13 +18,13 @@ */ package org.elasticsearch.common.geo; -import java.util.ArrayList; -import java.util.Collection; - -import org.apache.lucene.spatial.geopoint.document.GeoPointField; import org.apache.lucene.geo.Rectangle; +import org.apache.lucene.spatial.util.MortonEncoder; import org.apache.lucene.util.BitUtil; +import java.util.ArrayList; +import java.util.Collection; + /** * Utilities for converting to/from the GeoHash standard * @@ -42,19 +42,35 @@ public class GeoHashUtils { /** maximum precision for geohash strings */ public static final int PRECISION = 12; - private static final short MORTON_OFFSET = (GeoPointField.BITS<<1) - (PRECISION*5); + /** number of bits used for quantizing latitude and longitude values */ + public static final short BITS = 31; + /** scaling factors to convert lat/lon into unsigned space */ + private static final double LAT_SCALE = (0x1L<<BITS)/180.0D; + private static final double LON_SCALE = (0x1L<<BITS)/360.0D; + private static final short MORTON_OFFSET = (BITS<<1) - (PRECISION*5); // No instance: private GeoHashUtils() { } + /************************* + * 31 bit encoding utils * + *************************/ + public static long encodeLatLon(final double lat, final double lon) { + long result = MortonEncoder.encode(lat, lon); + if (result == 0xFFFFFFFFFFFFFFFFL) { + return result & 0xC000000000000000L; + } + return result >>> 2; + } + /** * Encode lon/lat to the geohash based long format (lon/lat interleaved, 4 least significant bits = level) */ public static final long longEncode(final double lon, final double lat, final int level) { // shift to appropriate level final short msf = (short)(((12 - level) * 5) + MORTON_OFFSET); - return ((BitUtil.flipFlop(GeoPointField.encodeLatLon(lat, lon)) >>> msf) << 4) | level; + return ((BitUtil.flipFlop(encodeLatLon(lat, lon)) >>> msf) << 4) | level; } /** @@ -120,7 +136,7 @@ public class GeoHashUtils { */ public static final String stringEncode(final double lon, final double lat, final int level) { // convert to geohashlong - final long ghLong = fromMorton(GeoPointField.encodeLatLon(lat, lon), level); + final long ghLong = fromMorton(encodeLatLon(lat, lon), level); return stringEncode(ghLong); } @@ -141,7 +157,7 @@ public class GeoHashUtils { StringBuilder geoHash = new StringBuilder(); short precision = 0; - final short msf = (GeoPointField.BITS<<1)-5; + final short msf = (BITS<<1)-5; long mask = 31L<<msf; do { geoHash.append(BASE_32[(int)((mask & hashedVal)>>>(msf-(precision*5)))]); @@ -303,13 +319,31 @@ public class GeoHashUtils { return neighbors; } + /** decode longitude value from morton encoded geo point */ + public static final double decodeLongitude(final long hash) { + return unscaleLon(BitUtil.deinterleave(hash)); + } + + /** decode latitude value from morton encoded geo point */ + public static final double decodeLatitude(final long hash) { + return unscaleLat(BitUtil.deinterleave(hash >>> 1)); + } + + private static double unscaleLon(final long val) { + return (val / LON_SCALE) - 180; + } + + private static double unscaleLat(final long val) { + return (val / LAT_SCALE) - 90; + } + /** returns the latitude value from the string based geohash */ public static final double decodeLatitude(final String geohash) { - return GeoPointField.decodeLatitude(mortonEncode(geohash)); + return decodeLatitude(mortonEncode(geohash)); } /** returns the latitude value from the string based geohash */ public static final double decodeLongitude(final String geohash) { - return GeoPointField.decodeLongitude(mortonEncode(geohash)); + return decodeLongitude(mortonEncode(geohash)); } } diff --git a/core/src/main/java/org/elasticsearch/common/geo/GeoPoint.java b/core/src/main/java/org/elasticsearch/common/geo/GeoPoint.java index f76720b9ed..125bc5aefc 100644 --- a/core/src/main/java/org/elasticsearch/common/geo/GeoPoint.java +++ b/core/src/main/java/org/elasticsearch/common/geo/GeoPoint.java @@ -23,7 +23,6 @@ import org.apache.lucene.document.LatLonDocValuesField; import org.apache.lucene.document.LatLonPoint; import org.apache.lucene.geo.GeoEncodingUtils; import org.apache.lucene.index.IndexableField; -import org.apache.lucene.spatial.geopoint.document.GeoPointField; import org.apache.lucene.util.BitUtil; import org.apache.lucene.util.BytesRef; @@ -87,8 +86,8 @@ public final class GeoPoint { } public GeoPoint resetFromIndexHash(long hash) { - lon = GeoPointField.decodeLongitude(hash); - lat = GeoPointField.decodeLatitude(hash); + lon = GeoHashUtils.decodeLongitude(hash); + lat = GeoHashUtils.decodeLatitude(hash); return this; } @@ -112,7 +111,7 @@ public final class GeoPoint { public GeoPoint resetFromGeoHash(String geohash) { final long hash = mortonEncode(geohash); - return this.reset(GeoPointField.decodeLatitude(hash), GeoPointField.decodeLongitude(hash)); + return this.reset(GeoHashUtils.decodeLatitude(hash), GeoHashUtils.decodeLongitude(hash)); } public GeoPoint resetFromGeoHash(long geohashLong) { diff --git a/core/src/main/java/org/elasticsearch/common/geo/GeoUtils.java b/core/src/main/java/org/elasticsearch/common/geo/GeoUtils.java index 81215906a0..a586414631 100644 --- a/core/src/main/java/org/elasticsearch/common/geo/GeoUtils.java +++ b/core/src/main/java/org/elasticsearch/common/geo/GeoUtils.java @@ -22,7 +22,6 @@ package org.elasticsearch.common.geo; import org.apache.lucene.geo.Rectangle; import org.apache.lucene.spatial.prefix.tree.GeohashPrefixTree; import org.apache.lucene.spatial.prefix.tree.QuadPrefixTree; -import org.apache.lucene.util.Bits; import org.apache.lucene.util.SloppyMath; import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.common.unit.DistanceUnit; @@ -511,35 +510,40 @@ public class GeoUtils { final GeoPoint... fromPoints) { final GeoPointValues singleValues = FieldData.unwrapSingleton(geoPointValues); if (singleValues != null && fromPoints.length == 1) { - final Bits docsWithField = FieldData.unwrapSingletonBits(geoPointValues); return FieldData.singleton(new NumericDoubleValues() { @Override - public double get(int docID) { - if (docsWithField != null && !docsWithField.get(docID)) { - return 0d; - } - final GeoPoint to = singleValues.get(docID); + public boolean advanceExact(int doc) throws IOException { + return singleValues.advanceExact(doc); + } + + @Override + public double doubleValue() throws IOException { final GeoPoint from = fromPoints[0]; + final GeoPoint to = singleValues.geoPointValue(); return distance.calculate(from.lat(), from.lon(), to.lat(), to.lon(), unit); } - }, docsWithField); + }); } else { return new SortingNumericDoubleValues() { @Override - public void setDocument(int doc) { - geoPointValues.setDocument(doc); - resize(geoPointValues.count() * fromPoints.length); - int v = 0; - for (GeoPoint from : fromPoints) { - for (int i = 0; i < geoPointValues.count(); ++i) { - final GeoPoint point = geoPointValues.valueAt(i); - values[v] = distance.calculate(from.lat(), from.lon(), point.lat(), point.lon(), unit); - v++; + public boolean advanceExact(int target) throws IOException { + if (geoPointValues.advanceExact(target)) { + resize(geoPointValues.docValueCount() * fromPoints.length); + int v = 0; + for (int i = 0; i < geoPointValues.docValueCount(); ++i) { + final GeoPoint point = geoPointValues.nextValue(); + for (GeoPoint from : fromPoints) { + values[v] = distance.calculate(from.lat(), from.lon(), point.lat(), point.lon(), unit); + v++; + } } + sort(); + return true; + } else { + return false; } - sort(); } }; } diff --git a/core/src/main/java/org/elasticsearch/common/lucene/Lucene.java b/core/src/main/java/org/elasticsearch/common/lucene/Lucene.java index bf448b6153..c213c38461 100644 --- a/core/src/main/java/org/elasticsearch/common/lucene/Lucene.java +++ b/core/src/main/java/org/elasticsearch/common/lucene/Lucene.java @@ -51,14 +51,14 @@ import org.apache.lucene.search.ScoreDoc; import org.apache.lucene.search.Scorer; import org.apache.lucene.search.SimpleCollector; import org.apache.lucene.search.SortField; +import org.apache.lucene.search.SortedNumericSortField; +import org.apache.lucene.search.SortedSetSortField; import org.apache.lucene.search.TimeLimitingCollector; import org.apache.lucene.search.TopDocs; import org.apache.lucene.search.TopFieldDocs; import org.apache.lucene.search.TwoPhaseIterator; import org.apache.lucene.search.Weight; import org.apache.lucene.search.grouping.CollapseTopFieldDocs; -import org.apache.lucene.search.SortedNumericSortField; -import org.apache.lucene.search.SortedSetSortField; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexInput; @@ -89,9 +89,9 @@ import java.util.Map; import java.util.Objects; public class Lucene { - public static final String LATEST_DOC_VALUES_FORMAT = "Lucene54"; + public static final String LATEST_DOC_VALUES_FORMAT = "Lucene70"; public static final String LATEST_POSTINGS_FORMAT = "Lucene50"; - public static final String LATEST_CODEC = "Lucene62"; + public static final String LATEST_CODEC = "Lucene70"; static { Deprecated annotation = PostingsFormat.forName(LATEST_POSTINGS_FORMAT).getClass().getAnnotation(Deprecated.class); diff --git a/core/src/main/java/org/elasticsearch/common/lucene/ShardCoreKeyMap.java b/core/src/main/java/org/elasticsearch/common/lucene/ShardCoreKeyMap.java index 38e7691f39..73255aef13 100644 --- a/core/src/main/java/org/elasticsearch/common/lucene/ShardCoreKeyMap.java +++ b/core/src/main/java/org/elasticsearch/common/lucene/ShardCoreKeyMap.java @@ -19,8 +19,8 @@ package org.elasticsearch.common.lucene; +import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.LeafReader; -import org.apache.lucene.index.LeafReader.CoreClosedListener; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.index.shard.ShardUtils; @@ -46,8 +46,8 @@ import java.util.concurrent.ConcurrentHashMap; */ public final class ShardCoreKeyMap { - private final Map<Object, ShardId> coreKeyToShard; - private final Map<String, Set<Object>> indexToCoreKey; + private final Map<IndexReader.CacheKey, ShardId> coreKeyToShard; + private final Map<String, Set<IndexReader.CacheKey>> indexToCoreKey; public ShardCoreKeyMap() { coreKeyToShard = new ConcurrentHashMap<>(); @@ -63,7 +63,11 @@ public final class ShardCoreKeyMap { if (shardId == null) { throw new IllegalArgumentException("Could not extract shard id from " + reader); } - final Object coreKey = reader.getCoreCacheKey(); + final IndexReader.CacheHelper cacheHelper = reader.getCoreCacheHelper(); + if (cacheHelper == null) { + throw new IllegalArgumentException("Reader " + reader + " does not support caching"); + } + final IndexReader.CacheKey coreKey = cacheHelper.getKey(); if (coreKeyToShard.containsKey(coreKey)) { // Do this check before entering the synchronized block in order to @@ -75,18 +79,18 @@ public final class ShardCoreKeyMap { final String index = shardId.getIndexName(); synchronized (this) { if (coreKeyToShard.containsKey(coreKey) == false) { - Set<Object> objects = indexToCoreKey.get(index); + Set<IndexReader.CacheKey> objects = indexToCoreKey.get(index); if (objects == null) { objects = new HashSet<>(); indexToCoreKey.put(index, objects); } final boolean added = objects.add(coreKey); assert added; - CoreClosedListener listener = ownerCoreCacheKey -> { + IndexReader.ClosedListener listener = ownerCoreCacheKey -> { assert coreKey == ownerCoreCacheKey; synchronized (ShardCoreKeyMap.this) { coreKeyToShard.remove(ownerCoreCacheKey); - final Set<Object> coreKeys = indexToCoreKey.get(index); + final Set<IndexReader.CacheKey> coreKeys = indexToCoreKey.get(index); final boolean removed = coreKeys.remove(coreKey); assert removed; if (coreKeys.isEmpty()) { @@ -96,7 +100,7 @@ public final class ShardCoreKeyMap { }; boolean addedListener = false; try { - reader.addCoreClosedListener(listener); + cacheHelper.addClosedListener(listener); addedListener = true; // Only add the core key to the map as a last operation so that @@ -131,7 +135,7 @@ public final class ShardCoreKeyMap { * Get the set of core cache keys associated with the given index. */ public synchronized Set<Object> getCoreKeysForIndex(String index) { - final Set<Object> objects = indexToCoreKey.get(index); + final Set<IndexReader.CacheKey> objects = indexToCoreKey.get(index); if (objects == null) { return Collections.emptySet(); } @@ -154,9 +158,9 @@ public final class ShardCoreKeyMap { if (assertionsEnabled == false) { throw new AssertionError("only run this if assertions are enabled"); } - Collection<Set<Object>> values = indexToCoreKey.values(); + Collection<Set<IndexReader.CacheKey>> values = indexToCoreKey.values(); int size = 0; - for (Set<Object> value : values) { + for (Set<IndexReader.CacheKey> value : values) { size += value.size(); } return size == coreKeyToShard.size(); diff --git a/core/src/main/java/org/elasticsearch/common/lucene/all/AllTermQuery.java b/core/src/main/java/org/elasticsearch/common/lucene/all/AllTermQuery.java index 5307a417e1..7df146a11c 100644 --- a/core/src/main/java/org/elasticsearch/common/lucene/all/AllTermQuery.java +++ b/core/src/main/java/org/elasticsearch/common/lucene/all/AllTermQuery.java @@ -105,28 +105,18 @@ public final class AllTermQuery extends Query { } @Override - public Weight createWeight(IndexSearcher searcher, boolean needsScores) throws IOException { + public Weight createWeight(IndexSearcher searcher, boolean needsScores, float boost) throws IOException { if (needsScores == false) { - return new TermQuery(term).createWeight(searcher, needsScores); + return new TermQuery(term).createWeight(searcher, needsScores, boost); } final TermContext termStates = TermContext.build(searcher.getTopReaderContext(), term); final CollectionStatistics collectionStats = searcher.collectionStatistics(term.field()); final TermStatistics termStats = searcher.termStatistics(term, termStates); final Similarity similarity = searcher.getSimilarity(needsScores); - final SimWeight stats = similarity.computeWeight(collectionStats, termStats); + final SimWeight stats = similarity.computeWeight(boost, collectionStats, termStats); return new Weight(this) { @Override - public float getValueForNormalization() throws IOException { - return stats.getValueForNormalization(); - } - - @Override - public void normalize(float norm, float topLevelBoost) { - stats.normalize(norm, topLevelBoost); - } - - @Override public void extractTerms(Set<Term> terms) { terms.add(term); } diff --git a/core/src/main/java/org/elasticsearch/common/lucene/index/ElasticsearchDirectoryReader.java b/core/src/main/java/org/elasticsearch/common/lucene/index/ElasticsearchDirectoryReader.java index b9caecea96..0f83e07874 100644 --- a/core/src/main/java/org/elasticsearch/common/lucene/index/ElasticsearchDirectoryReader.java +++ b/core/src/main/java/org/elasticsearch/common/lucene/index/ElasticsearchDirectoryReader.java @@ -50,6 +50,12 @@ public final class ElasticsearchDirectoryReader extends FilterDirectoryReader { } @Override + public CacheHelper getReaderCacheHelper() { + // safe to delegate since this reader does not alter the index + return in.getReaderCacheHelper(); + } + + @Override protected DirectoryReader doWrapDirectoryReader(DirectoryReader in) throws IOException { return new ElasticsearchDirectoryReader(in, wrapper, shardId); } @@ -84,14 +90,17 @@ public final class ElasticsearchDirectoryReader extends FilterDirectoryReader { * @throws IllegalArgumentException if the reader doesn't contain an {@link ElasticsearchDirectoryReader} in it's hierarchy */ @SuppressForbidden(reason = "This is the only sane way to add a ReaderClosedListener") - public static void addReaderCloseListener(DirectoryReader reader, IndexReader.ReaderClosedListener listener) { + public static void addReaderCloseListener(DirectoryReader reader, IndexReader.ClosedListener listener) { ElasticsearchDirectoryReader elasticsearchDirectoryReader = getElasticsearchDirectoryReader(reader); - if (elasticsearchDirectoryReader != null) { - assert reader.getCoreCacheKey() == elasticsearchDirectoryReader.getCoreCacheKey(); - elasticsearchDirectoryReader.addReaderClosedListener(listener); - return; + if (elasticsearchDirectoryReader == null) { + throw new IllegalArgumentException("Can't install close listener reader is not an ElasticsearchDirectoryReader/ElasticsearchLeafReader"); + } + IndexReader.CacheHelper cacheHelper = elasticsearchDirectoryReader.getReaderCacheHelper(); + if (cacheHelper == null) { + throw new IllegalArgumentException("Reader " + elasticsearchDirectoryReader + " does not support caching"); } - throw new IllegalArgumentException("Can't install close listener reader is not an ElasticsearchDirectoryReader/ElasticsearchLeafReader"); + assert cacheHelper.getKey() == reader.getReaderCacheHelper().getKey(); + cacheHelper.addClosedListener(listener); } /** diff --git a/core/src/main/java/org/elasticsearch/common/lucene/index/ElasticsearchLeafReader.java b/core/src/main/java/org/elasticsearch/common/lucene/index/ElasticsearchLeafReader.java index 1094a7b184..0f13ee43d3 100644 --- a/core/src/main/java/org/elasticsearch/common/lucene/index/ElasticsearchLeafReader.java +++ b/core/src/main/java/org/elasticsearch/common/lucene/index/ElasticsearchLeafReader.java @@ -49,8 +49,13 @@ public final class ElasticsearchLeafReader extends FilterLeafReader { } @Override - public Object getCoreCacheKey() { - return in.getCoreCacheKey(); + public CacheHelper getCoreCacheHelper() { + return in.getCoreCacheHelper(); + } + + @Override + public CacheHelper getReaderCacheHelper() { + return in.getReaderCacheHelper(); } public static ElasticsearchLeafReader getElasticsearchLeafReader(LeafReader reader) { diff --git a/core/src/main/java/org/elasticsearch/common/lucene/search/Queries.java b/core/src/main/java/org/elasticsearch/common/lucene/search/Queries.java index 68a02ed256..acf3f9ffdf 100644 --- a/core/src/main/java/org/elasticsearch/common/lucene/search/Queries.java +++ b/core/src/main/java/org/elasticsearch/common/lucene/search/Queries.java @@ -121,7 +121,6 @@ public class Queries { if (isNegativeQuery(q)) { BooleanQuery bq = (BooleanQuery) q; BooleanQuery.Builder builder = new BooleanQuery.Builder(); - builder.setDisableCoord(bq.isCoordDisabled()); for (BooleanClause clause : bq) { builder.add(clause); } @@ -154,7 +153,6 @@ public class Queries { int msm = calculateMinShouldMatch(optionalClauses, minimumShouldMatch); if (0 < msm) { BooleanQuery.Builder builder = new BooleanQuery.Builder(); - builder.setDisableCoord(query.isCoordDisabled()); for (BooleanClause clause : query) { builder.add(clause); } @@ -170,10 +168,7 @@ public class Queries { * otherwise return the original query. */ public static Query maybeApplyMinimumShouldMatch(Query query, @Nullable String minimumShouldMatch) { - // If the coordination factor is disabled on a boolean query we don't apply the minimum should match. - // This is done to make sure that the minimum_should_match doesn't get applied when there is only one word - // and multiple variations of the same word in the query (synonyms for instance). - if (query instanceof BooleanQuery && !((BooleanQuery) query).isCoordDisabled()) { + if (query instanceof BooleanQuery) { return applyMinimumShouldMatch((BooleanQuery) query, minimumShouldMatch); } else if (query instanceof ExtendedCommonTermsQuery) { ((ExtendedCommonTermsQuery)query).setLowFreqMinimumNumberShouldMatch(minimumShouldMatch); diff --git a/core/src/main/java/org/elasticsearch/common/lucene/search/function/FieldValueFactorFunction.java b/core/src/main/java/org/elasticsearch/common/lucene/search/function/FieldValueFactorFunction.java index 3bc5542c2a..e225df040a 100644 --- a/core/src/main/java/org/elasticsearch/common/lucene/search/function/FieldValueFactorFunction.java +++ b/core/src/main/java/org/elasticsearch/common/lucene/search/function/FieldValueFactorFunction.java @@ -62,7 +62,7 @@ public class FieldValueFactorFunction extends ScoreFunction { public LeafScoreFunction getLeafScoreFunction(LeafReaderContext ctx) { final SortedNumericDoubleValues values; if(indexFieldData == null) { - values = FieldData.emptySortedNumericDoubles(ctx.reader().maxDoc()); + values = FieldData.emptySortedNumericDoubles(); } else { values = this.indexFieldData.load(ctx).getDoubleValues(); } @@ -70,16 +70,16 @@ public class FieldValueFactorFunction extends ScoreFunction { return new LeafScoreFunction() { @Override - public double score(int docId, float subQueryScore) { - values.setDocument(docId); - final int numValues = values.count(); + public double score(int docId, float subQueryScore) throws IOException { double value; - if (numValues > 0) { - value = values.valueAt(0); - } else if (missing != null) { - value = missing; + if (values.advanceExact(docId)) { + value = values.nextValue(); } else { - throw new ElasticsearchException("Missing value for field [" + field + "]"); + if (missing != null) { + value = missing; + } else { + throw new ElasticsearchException("Missing value for field [" + field + "]"); + } } double val = value * boostFactor; double result = modifier.apply(val); @@ -91,7 +91,7 @@ public class FieldValueFactorFunction extends ScoreFunction { } @Override - public Explanation explainScore(int docId, Explanation subQueryScore) { + public Explanation explainScore(int docId, Explanation subQueryScore) throws IOException { String modifierStr = modifier != null ? modifier.toString() : ""; String defaultStr = missing != null ? "?:" + missing : ""; double score = score(docId, subQueryScore.getValue()); diff --git a/core/src/main/java/org/elasticsearch/common/lucene/search/function/FiltersFunctionScoreQuery.java b/core/src/main/java/org/elasticsearch/common/lucene/search/function/FiltersFunctionScoreQuery.java index fe12622748..abf145406c 100644 --- a/core/src/main/java/org/elasticsearch/common/lucene/search/function/FiltersFunctionScoreQuery.java +++ b/core/src/main/java/org/elasticsearch/common/lucene/search/function/FiltersFunctionScoreQuery.java @@ -135,9 +135,9 @@ public class FiltersFunctionScoreQuery extends Query { } @Override - public Weight createWeight(IndexSearcher searcher, boolean needsScores) throws IOException { + public Weight createWeight(IndexSearcher searcher, boolean needsScores, float boost) throws IOException { if (needsScores == false && minScore == null) { - return subQuery.createWeight(searcher, needsScores); + return subQuery.createWeight(searcher, needsScores, boost); } boolean subQueryNeedsScores = combineFunction != CombineFunction.REPLACE; @@ -146,7 +146,7 @@ public class FiltersFunctionScoreQuery extends Query { subQueryNeedsScores |= filterFunctions[i].function.needsScores(); filterWeights[i] = searcher.createNormalizedWeight(filterFunctions[i].filter, false); } - Weight subQueryWeight = subQuery.createWeight(searcher, subQueryNeedsScores); + Weight subQueryWeight = subQuery.createWeight(searcher, subQueryNeedsScores, boost); return new CustomBoostFactorWeight(this, subQueryWeight, filterWeights, subQueryNeedsScores); } @@ -168,16 +168,6 @@ public class FiltersFunctionScoreQuery extends Query { subQueryWeight.extractTerms(terms); } - @Override - public float getValueForNormalization() throws IOException { - return subQueryWeight.getValueForNormalization(); - } - - @Override - public void normalize(float norm, float boost) { - subQueryWeight.normalize(norm, boost); - } - private FiltersFunctionFactorScorer functionScorer(LeafReaderContext context) throws IOException { Scorer subQueryScorer = subQueryWeight.scorer(context); if (subQueryScorer == null) { @@ -281,7 +271,7 @@ public class FiltersFunctionScoreQuery extends Query { return scoreCombiner.combine(subQueryScore, factor, maxBoost); } - protected double computeScore(int docId, float subQueryScore) { + protected double computeScore(int docId, float subQueryScore) throws IOException { double factor = 1d; switch(scoreMode) { case FIRST: diff --git a/core/src/main/java/org/elasticsearch/common/lucene/search/function/FunctionScoreQuery.java b/core/src/main/java/org/elasticsearch/common/lucene/search/function/FunctionScoreQuery.java index 61de1ab303..95fc58dce5 100644 --- a/core/src/main/java/org/elasticsearch/common/lucene/search/function/FunctionScoreQuery.java +++ b/core/src/main/java/org/elasticsearch/common/lucene/search/function/FunctionScoreQuery.java @@ -91,16 +91,16 @@ public class FunctionScoreQuery extends Query { } @Override - public Weight createWeight(IndexSearcher searcher, boolean needsScores) throws IOException { + public Weight createWeight(IndexSearcher searcher, boolean needsScores, float boost) throws IOException { if (needsScores == false && minScore == null) { - return subQuery.createWeight(searcher, needsScores); + return subQuery.createWeight(searcher, needsScores, boost); } boolean subQueryNeedsScores = combineFunction != CombineFunction.REPLACE // if we don't replace we need the original score || function == null // when the function is null, we just multiply the score, so we need it || function.needsScores(); // some scripts can replace with a script that returns eg. 1/_score - Weight subQueryWeight = subQuery.createWeight(searcher, subQueryNeedsScores); + Weight subQueryWeight = subQuery.createWeight(searcher, subQueryNeedsScores, boost); return new CustomBoostFactorWeight(this, subQueryWeight, subQueryNeedsScores); } @@ -120,16 +120,6 @@ public class FunctionScoreQuery extends Query { subQueryWeight.extractTerms(terms); } - @Override - public float getValueForNormalization() throws IOException { - return subQueryWeight.getValueForNormalization(); - } - - @Override - public void normalize(float norm, float boost) { - subQueryWeight.normalize(norm, boost); - } - private FunctionFactorScorer functionScorer(LeafReaderContext context) throws IOException { Scorer subQueryScorer = subQueryWeight.scorer(context); if (subQueryScorer == null) { diff --git a/core/src/main/java/org/elasticsearch/common/lucene/search/function/LeafScoreFunction.java b/core/src/main/java/org/elasticsearch/common/lucene/search/function/LeafScoreFunction.java index 7df35fa580..c00431a34b 100644 --- a/core/src/main/java/org/elasticsearch/common/lucene/search/function/LeafScoreFunction.java +++ b/core/src/main/java/org/elasticsearch/common/lucene/search/function/LeafScoreFunction.java @@ -26,7 +26,7 @@ import java.io.IOException; /** Per-leaf {@link ScoreFunction}. */ public abstract class LeafScoreFunction { - public abstract double score(int docId, float subQueryScore); + public abstract double score(int docId, float subQueryScore) throws IOException; public abstract Explanation explainScore(int docId, Explanation subQueryScore) throws IOException; diff --git a/core/src/main/java/org/elasticsearch/common/lucene/search/function/RandomScoreFunction.java b/core/src/main/java/org/elasticsearch/common/lucene/search/function/RandomScoreFunction.java index 3810c16bc0..43e71f4dff 100644 --- a/core/src/main/java/org/elasticsearch/common/lucene/search/function/RandomScoreFunction.java +++ b/core/src/main/java/org/elasticsearch/common/lucene/search/function/RandomScoreFunction.java @@ -25,6 +25,7 @@ import org.elasticsearch.index.fielddata.AtomicFieldData; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.SortedBinaryDocValues; +import java.io.IOException; import java.util.Objects; /** @@ -68,14 +69,16 @@ public class RandomScoreFunction extends ScoreFunction { return new LeafScoreFunction() { @Override - public double score(int docId, float subQueryScore) { - uidByteData.setDocument(docId); - int hash = StringHelper.murmurhash3_x86_32(uidByteData.valueAt(0), saltedSeed); + public double score(int docId, float subQueryScore) throws IOException { + if (uidByteData.advanceExact(docId) == false) { + throw new AssertionError("Document without a _uid"); + } + int hash = StringHelper.murmurhash3_x86_32(uidByteData.nextValue(), saltedSeed); return (hash & 0x00FFFFFF) / (float)(1 << 24); // only use the lower 24 bits to construct a float from 0.0-1.0 } @Override - public Explanation explainScore(int docId, Explanation subQueryScore) { + public Explanation explainScore(int docId, Explanation subQueryScore) throws IOException { return Explanation.match( CombineFunction.toFloat(score(docId, subQueryScore.getValue())), "random score function (seed: " + originalSeed + ")"); diff --git a/core/src/main/java/org/elasticsearch/common/lucene/search/function/WeightFactorFunction.java b/core/src/main/java/org/elasticsearch/common/lucene/search/function/WeightFactorFunction.java index 44c6245a81..c69a5efdd5 100644 --- a/core/src/main/java/org/elasticsearch/common/lucene/search/function/WeightFactorFunction.java +++ b/core/src/main/java/org/elasticsearch/common/lucene/search/function/WeightFactorFunction.java @@ -52,7 +52,7 @@ public class WeightFactorFunction extends ScoreFunction { final LeafScoreFunction leafFunction = scoreFunction.getLeafScoreFunction(ctx); return new LeafScoreFunction() { @Override - public double score(int docId, float subQueryScore) { + public double score(int docId, float subQueryScore) throws IOException { return leafFunction.score(docId, subQueryScore) * getWeight(); } diff --git a/core/src/main/java/org/elasticsearch/common/lucene/uid/PerThreadIDVersionAndSeqNoLookup.java b/core/src/main/java/org/elasticsearch/common/lucene/uid/PerThreadIDVersionAndSeqNoLookup.java index 80977618c4..96c9f30a95 100644 --- a/core/src/main/java/org/elasticsearch/common/lucene/uid/PerThreadIDVersionAndSeqNoLookup.java +++ b/core/src/main/java/org/elasticsearch/common/lucene/uid/PerThreadIDVersionAndSeqNoLookup.java @@ -52,12 +52,7 @@ final class PerThreadIDVersionAndSeqNoLookup { /** terms enum for uid field */ private final TermsEnum termsEnum; - /** _version data */ - private final NumericDocValues versions; - /** _seq_no data */ - private final NumericDocValues seqNos; - /** _primary_term data */ - private final NumericDocValues primaryTerms; + /** Reused for iteration (when the term exists) */ private PostingsEnum docsEnum; @@ -72,30 +67,33 @@ final class PerThreadIDVersionAndSeqNoLookup { Terms terms = fields.terms(UidFieldMapper.NAME); termsEnum = terms.iterator(); if (termsEnum == null) { - throw new IllegalArgumentException("reader misses the [" + UidFieldMapper.NAME + - "] field"); + throw new IllegalArgumentException("reader misses the [" + UidFieldMapper.NAME + "] field"); } - versions = reader.getNumericDocValues(VersionFieldMapper.NAME); - if (versions == null) { - throw new IllegalArgumentException("reader misses the [" + VersionFieldMapper.NAME + - "] field"); + if (reader.getNumericDocValues(VersionFieldMapper.NAME) == null) { + throw new IllegalArgumentException("reader misses the [" + VersionFieldMapper.NAME + "] field"); } - seqNos = reader.getNumericDocValues(SeqNoFieldMapper.NAME); - primaryTerms = reader.getNumericDocValues(SeqNoFieldMapper.PRIMARY_TERM_NAME); + Object readerKey = null; - assert (readerKey = reader.getCoreCacheKey()) != null; + assert (readerKey = reader.getCoreCacheHelper().getKey()) != null; this.readerKey = readerKey; } /** Return null if id is not found. */ public DocIdAndVersion lookupVersion(BytesRef id, Bits liveDocs, LeafReaderContext context) throws IOException { - assert context.reader().getCoreCacheKey().equals(readerKey) : + assert context.reader().getCoreCacheHelper().getKey().equals(readerKey) : "context's reader is not the same as the reader class was initialized on."; int docID = getDocID(id, liveDocs); if (docID != DocIdSetIterator.NO_MORE_DOCS) { - return new DocIdAndVersion(docID, versions.get(docID), context); + final NumericDocValues versions = context.reader().getNumericDocValues(VersionFieldMapper.NAME); + if (versions == null) { + throw new IllegalArgumentException("reader misses the [" + VersionFieldMapper.NAME + "] field"); + } + if (versions.advanceExact(docID) == false) { + throw new IllegalArgumentException("Document [" + docID + "] misses the [" + VersionFieldMapper.NAME + "] field"); + } + return new DocIdAndVersion(docID, versions.longValue(), context); } else { return null; } @@ -124,11 +122,18 @@ final class PerThreadIDVersionAndSeqNoLookup { /** Return null if id is not found. */ DocIdAndSeqNo lookupSeqNo(BytesRef id, Bits liveDocs, LeafReaderContext context) throws IOException { - assert context.reader().getCoreCacheKey().equals(readerKey) : + assert context.reader().getCoreCacheHelper().getKey().equals(readerKey) : "context's reader is not the same as the reader class was initialized on."; int docID = getDocID(id, liveDocs); if (docID != DocIdSetIterator.NO_MORE_DOCS) { - return new DocIdAndSeqNo(docID, seqNos == null ? SequenceNumbersService.UNASSIGNED_SEQ_NO : seqNos.get(docID), context); + NumericDocValues seqNos = context.reader().getNumericDocValues(SeqNoFieldMapper.NAME); + long seqNo; + if (seqNos != null && seqNos.advanceExact(docID)) { + seqNo = seqNos.longValue(); + } else { + seqNo = SequenceNumbersService.UNASSIGNED_SEQ_NO; + } + return new DocIdAndSeqNo(docID, seqNo, context); } else { return null; } @@ -139,7 +144,12 @@ final class PerThreadIDVersionAndSeqNoLookup { * * Note that 0 is an illegal primary term. See {@link org.elasticsearch.cluster.metadata.IndexMetaData#primaryTerm(int)} **/ - long lookUpPrimaryTerm(int docID) throws IOException { - return primaryTerms == null ? 0 : primaryTerms.get(docID); + long lookUpPrimaryTerm(int docID, LeafReader reader) throws IOException { + NumericDocValues primaryTerms = reader.getNumericDocValues(SeqNoFieldMapper.PRIMARY_TERM_NAME); + if (primaryTerms != null && primaryTerms.advanceExact(docID)) { + return primaryTerms.longValue(); + } else { + return 0; + } } } diff --git a/core/src/main/java/org/elasticsearch/common/lucene/uid/VersionsAndSeqNoResolver.java b/core/src/main/java/org/elasticsearch/common/lucene/uid/VersionsAndSeqNoResolver.java index 1cbae29a3d..409ce8dec2 100644 --- a/core/src/main/java/org/elasticsearch/common/lucene/uid/VersionsAndSeqNoResolver.java +++ b/core/src/main/java/org/elasticsearch/common/lucene/uid/VersionsAndSeqNoResolver.java @@ -21,7 +21,6 @@ package org.elasticsearch.common.lucene.uid; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.LeafReader; -import org.apache.lucene.index.LeafReader.CoreClosedListener; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.Term; import org.apache.lucene.util.CloseableThreadLocal; @@ -41,7 +40,7 @@ public final class VersionsAndSeqNoResolver { ConcurrentCollections.newConcurrentMapWithAggressiveConcurrency(); // Evict this reader from lookupStates once it's closed: - private static final CoreClosedListener removeLookupState = key -> { + private static final IndexReader.ClosedListener removeLookupState = key -> { CloseableThreadLocal<PerThreadIDVersionAndSeqNoLookup> ctl = lookupStates.remove(key); if (ctl != null) { ctl.close(); @@ -49,15 +48,15 @@ public final class VersionsAndSeqNoResolver { }; private static PerThreadIDVersionAndSeqNoLookup getLookupState(LeafReader reader) throws IOException { - Object key = reader.getCoreCacheKey(); - CloseableThreadLocal<PerThreadIDVersionAndSeqNoLookup> ctl = lookupStates.get(key); + IndexReader.CacheHelper cacheHelper = reader.getCoreCacheHelper(); + CloseableThreadLocal<PerThreadIDVersionAndSeqNoLookup> ctl = lookupStates.get(cacheHelper.getKey()); if (ctl == null) { // First time we are seeing this reader's core; make a new CTL: ctl = new CloseableThreadLocal<>(); - CloseableThreadLocal<PerThreadIDVersionAndSeqNoLookup> other = lookupStates.putIfAbsent(key, ctl); + CloseableThreadLocal<PerThreadIDVersionAndSeqNoLookup> other = lookupStates.putIfAbsent(cacheHelper.getKey(), ctl); if (other == null) { // Our CTL won, we must remove it when the core is closed: - reader.addCoreClosedListener(removeLookupState); + cacheHelper.addClosedListener(removeLookupState); } else { // Another thread beat us to it: just use their CTL: ctl = other; @@ -161,7 +160,7 @@ public final class VersionsAndSeqNoResolver { public static long loadPrimaryTerm(DocIdAndSeqNo docIdAndSeqNo) throws IOException { LeafReader leaf = docIdAndSeqNo.context.reader(); PerThreadIDVersionAndSeqNoLookup lookup = getLookupState(leaf); - long result = lookup.lookUpPrimaryTerm(docIdAndSeqNo.docId); + long result = lookup.lookUpPrimaryTerm(docIdAndSeqNo.docId, leaf); assert result > 0 : "should always resolve a primary term for a resolved sequence number. primary_term [" + result + "]" + " docId [" + docIdAndSeqNo.docId + "] seqNo [" + docIdAndSeqNo.seqNo + "]"; return result; diff --git a/core/src/main/java/org/elasticsearch/index/cache/bitset/BitsetFilterCache.java b/core/src/main/java/org/elasticsearch/index/cache/bitset/BitsetFilterCache.java index 0e4c54e7a7..04d2ac47d1 100644 --- a/core/src/main/java/org/elasticsearch/index/cache/bitset/BitsetFilterCache.java +++ b/core/src/main/java/org/elasticsearch/index/cache/bitset/BitsetFilterCache.java @@ -21,8 +21,8 @@ package org.elasticsearch.index.cache.bitset; import org.apache.logging.log4j.message.ParameterizedMessage; import org.apache.logging.log4j.util.Supplier; +import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexReaderContext; -import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.ReaderUtil; import org.apache.lucene.search.IndexSearcher; @@ -71,13 +71,13 @@ import java.util.concurrent.Executor; * and require that it should always be around should use this cache, otherwise the * {@link org.elasticsearch.index.cache.query.QueryCache} should be used instead. */ -public final class BitsetFilterCache extends AbstractIndexComponent implements LeafReader.CoreClosedListener, RemovalListener<Object, Cache<Query, BitsetFilterCache.Value>>, Closeable { +public final class BitsetFilterCache extends AbstractIndexComponent implements IndexReader.ClosedListener, RemovalListener<IndexReader.CacheKey, Cache<Query, BitsetFilterCache.Value>>, Closeable { public static final Setting<Boolean> INDEX_LOAD_RANDOM_ACCESS_FILTERS_EAGERLY_SETTING = Setting.boolSetting("index.load_fixed_bitset_filters_eagerly", true, Property.IndexScope); private final boolean loadRandomAccessFiltersEagerly; - private final Cache<Object, Cache<Query, Value>> loadedFilters; + private final Cache<IndexReader.CacheKey, Cache<Query, Value>> loadedFilters; private final Listener listener; public BitsetFilterCache(IndexSettings indexSettings, Listener listener) { @@ -86,7 +86,7 @@ public final class BitsetFilterCache extends AbstractIndexComponent implements L throw new IllegalArgumentException("listener must not be null"); } this.loadRandomAccessFiltersEagerly = this.indexSettings.getValue(INDEX_LOAD_RANDOM_ACCESS_FILTERS_EAGERLY_SETTING); - this.loadedFilters = CacheBuilder.<Object, Cache<Query, Value>>builder().removalListener(this).build(); + this.loadedFilters = CacheBuilder.<IndexReader.CacheKey, Cache<Query, Value>>builder().removalListener(this).build(); this.listener = listener; } @@ -100,7 +100,7 @@ public final class BitsetFilterCache extends AbstractIndexComponent implements L } @Override - public void onClose(Object ownerCoreCacheKey) { + public void onClose(IndexReader.CacheKey ownerCoreCacheKey) { loadedFilters.invalidate(ownerCoreCacheKey); } @@ -115,7 +115,11 @@ public final class BitsetFilterCache extends AbstractIndexComponent implements L } private BitSet getAndLoadIfNotPresent(final Query query, final LeafReaderContext context) throws IOException, ExecutionException { - final Object coreCacheReader = context.reader().getCoreCacheKey(); + final IndexReader.CacheHelper cacheHelper = context.reader().getCoreCacheHelper(); + if (cacheHelper == null) { + throw new IllegalArgumentException("Reader " + context.reader() + " does not support caching"); + } + final IndexReader.CacheKey coreCacheReader = cacheHelper.getKey(); final ShardId shardId = ShardUtils.extractShardId(context.reader()); if (shardId != null // can't require it because of the percolator && indexSettings.getIndex().equals(shardId.getIndex()) == false) { @@ -124,7 +128,7 @@ public final class BitsetFilterCache extends AbstractIndexComponent implements L + " with cache of index " + indexSettings.getIndex()); } Cache<Query, Value> filterToFbs = loadedFilters.computeIfAbsent(coreCacheReader, key -> { - context.reader().addCoreClosedListener(BitsetFilterCache.this); + cacheHelper.addClosedListener(BitsetFilterCache.this); return CacheBuilder.<Query, Value>builder().build(); }); @@ -148,7 +152,7 @@ public final class BitsetFilterCache extends AbstractIndexComponent implements L } @Override - public void onRemoval(RemovalNotification<Object, Cache<Query, Value>> notification) { + public void onRemoval(RemovalNotification<IndexReader.CacheKey, Cache<Query, Value>> notification) { if (notification.getKey() == null) { return; } @@ -272,7 +276,7 @@ public final class BitsetFilterCache extends AbstractIndexComponent implements L } - Cache<Object, Cache<Query, Value>> getLoadedFilters() { + Cache<IndexReader.CacheKey, Cache<Query, Value>> getLoadedFilters() { return loadedFilters; } diff --git a/core/src/main/java/org/elasticsearch/index/codec/CodecService.java b/core/src/main/java/org/elasticsearch/index/codec/CodecService.java index 59be64a85d..3034284897 100644 --- a/core/src/main/java/org/elasticsearch/index/codec/CodecService.java +++ b/core/src/main/java/org/elasticsearch/index/codec/CodecService.java @@ -22,7 +22,7 @@ package org.elasticsearch.index.codec; import org.apache.logging.log4j.Logger; import org.apache.lucene.codecs.Codec; import org.apache.lucene.codecs.lucene50.Lucene50StoredFieldsFormat.Mode; -import org.apache.lucene.codecs.lucene62.Lucene62Codec; +import org.apache.lucene.codecs.lucene70.Lucene70Codec; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.collect.MapBuilder; import org.elasticsearch.index.mapper.MapperService; @@ -47,8 +47,8 @@ public class CodecService { public CodecService(@Nullable MapperService mapperService, Logger logger) { final MapBuilder<String, Codec> codecs = MapBuilder.<String, Codec>newMapBuilder(); if (mapperService == null) { - codecs.put(DEFAULT_CODEC, new Lucene62Codec()); - codecs.put(BEST_COMPRESSION_CODEC, new Lucene62Codec(Mode.BEST_COMPRESSION)); + codecs.put(DEFAULT_CODEC, new Lucene70Codec()); + codecs.put(BEST_COMPRESSION_CODEC, new Lucene70Codec(Mode.BEST_COMPRESSION)); } else { codecs.put(DEFAULT_CODEC, new PerFieldMappingPostingFormatCodec(Mode.BEST_SPEED, mapperService, logger)); diff --git a/core/src/main/java/org/elasticsearch/index/codec/PerFieldMappingPostingFormatCodec.java b/core/src/main/java/org/elasticsearch/index/codec/PerFieldMappingPostingFormatCodec.java index 490c837e85..bf1e48e7a6 100644 --- a/core/src/main/java/org/elasticsearch/index/codec/PerFieldMappingPostingFormatCodec.java +++ b/core/src/main/java/org/elasticsearch/index/codec/PerFieldMappingPostingFormatCodec.java @@ -23,7 +23,7 @@ import org.apache.logging.log4j.Logger; import org.apache.lucene.codecs.Codec; import org.apache.lucene.codecs.PostingsFormat; import org.apache.lucene.codecs.lucene50.Lucene50StoredFieldsFormat; -import org.apache.lucene.codecs.lucene62.Lucene62Codec; +import org.apache.lucene.codecs.lucene70.Lucene70Codec; import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.index.mapper.CompletionFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; @@ -38,7 +38,7 @@ import org.elasticsearch.index.mapper.MapperService; * configured for a specific field the default postings format is used. */ // LUCENE UPGRADE: make sure to move to a new codec depending on the lucene version -public class PerFieldMappingPostingFormatCodec extends Lucene62Codec { +public class PerFieldMappingPostingFormatCodec extends Lucene70Codec { private final Logger logger; private final MapperService mapperService; diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/AbstractBinaryDocValues.java b/core/src/main/java/org/elasticsearch/index/fielddata/AbstractBinaryDocValues.java new file mode 100644 index 0000000000..a3ce33a4e6 --- /dev/null +++ b/core/src/main/java/org/elasticsearch/index/fielddata/AbstractBinaryDocValues.java @@ -0,0 +1,55 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.fielddata; + +import org.apache.lucene.index.BinaryDocValues; +import org.apache.lucene.search.DocIdSetIterator; + +import java.io.IOException; + +/** + * Base implementation that throws an {@link IOException} for the + * {@link DocIdSetIterator} APIs. This impl is safe to use for sorting and + * aggregations, which only use {@link #advanceExact(int)} and + * {@link #binaryValue()}. + */ +public abstract class AbstractBinaryDocValues extends BinaryDocValues { + + @Override + public int docID() { + throw new UnsupportedOperationException(); + } + + @Override + public int nextDoc() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public int advance(int target) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public long cost() { + throw new UnsupportedOperationException(); + } + +} diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/AbstractNumericDocValues.java b/core/src/main/java/org/elasticsearch/index/fielddata/AbstractNumericDocValues.java new file mode 100644 index 0000000000..21c96f3090 --- /dev/null +++ b/core/src/main/java/org/elasticsearch/index/fielddata/AbstractNumericDocValues.java @@ -0,0 +1,50 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.fielddata; + +import org.apache.lucene.index.NumericDocValues; +import org.apache.lucene.search.DocIdSetIterator; + +import java.io.IOException; + +/** + * Base implementation that throws an {@link IOException} for the + * {@link DocIdSetIterator} APIs. This impl is safe to use for sorting and + * aggregations, which only use {@link #advanceExact(int)} and + * {@link #longValue()}. + */ +public abstract class AbstractNumericDocValues extends NumericDocValues { + + @Override + public int nextDoc() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public int advance(int target) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public long cost() { + throw new UnsupportedOperationException(); + } + +} diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/AbstractRandomAccessOrds.java b/core/src/main/java/org/elasticsearch/index/fielddata/AbstractSortedDocValues.java index ef8954b97e..b1ace7e932 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/AbstractRandomAccessOrds.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/AbstractSortedDocValues.java @@ -19,30 +19,32 @@ package org.elasticsearch.index.fielddata; -import org.apache.lucene.index.RandomAccessOrds; +import org.apache.lucene.index.SortedDocValues; +import org.apache.lucene.search.DocIdSetIterator; + +import java.io.IOException; /** - * Base implementation of a {@link RandomAccessOrds} instance. + * Base implementation that throws an {@link IOException} for the + * {@link DocIdSetIterator} APIs. This impl is safe to use for sorting and + * aggregations, which only use {@link #advanceExact(int)} and + * {@link #ordValue()}. */ -public abstract class AbstractRandomAccessOrds extends RandomAccessOrds { - - int i = 0; +public abstract class AbstractSortedDocValues extends SortedDocValues { - protected abstract void doSetDocument(int docID); + @Override + public int nextDoc() throws IOException { + throw new UnsupportedOperationException(); + } @Override - public final void setDocument(int docID) { - doSetDocument(docID); - i = 0; + public int advance(int target) throws IOException { + throw new UnsupportedOperationException(); } @Override - public long nextOrd() { - if (i < cardinality()) { - return ordAt(i++); - } else { - return NO_MORE_ORDS; - } + public long cost() { + throw new UnsupportedOperationException(); } } diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/AbstractSortedNumericDocValues.java b/core/src/main/java/org/elasticsearch/index/fielddata/AbstractSortedNumericDocValues.java new file mode 100644 index 0000000000..c7f7814779 --- /dev/null +++ b/core/src/main/java/org/elasticsearch/index/fielddata/AbstractSortedNumericDocValues.java @@ -0,0 +1,55 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.fielddata; + +import org.apache.lucene.index.SortedNumericDocValues; +import org.apache.lucene.search.DocIdSetIterator; + +import java.io.IOException; + +/** + * Base implementation that throws an {@link IOException} for the + * {@link DocIdSetIterator} APIs. This impl is safe to use for sorting and + * aggregations, which only use {@link #advanceExact(int)} and + * {@link #docValueCount()} and {@link #nextValue()}. + */ +public abstract class AbstractSortedNumericDocValues extends SortedNumericDocValues { + + @Override + public int docID() { + throw new UnsupportedOperationException(); + } + + @Override + public int nextDoc() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public int advance(int target) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public long cost() { + throw new UnsupportedOperationException(); + } + +} diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/AbstractSortedSetDocValues.java b/core/src/main/java/org/elasticsearch/index/fielddata/AbstractSortedSetDocValues.java new file mode 100644 index 0000000000..fedf815f26 --- /dev/null +++ b/core/src/main/java/org/elasticsearch/index/fielddata/AbstractSortedSetDocValues.java @@ -0,0 +1,56 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.fielddata; + +import org.apache.lucene.index.SortedSetDocValues; +import org.apache.lucene.search.DocIdSetIterator; + +import java.io.IOException; + +/** + * Base implementation that throws an {@link IOException} for the + * {@link DocIdSetIterator} APIs. This impl is safe to use for sorting and + * aggregations, which only use {@link #advanceExact(int)} and + * {@link #getValueCount()} and {@link #nextOrd()} and {@link #lookupOrd(long)}. + */ +public abstract class AbstractSortedSetDocValues extends SortedSetDocValues { + + @Override + public int docID() { + throw new UnsupportedOperationException(); + } + + @Override + public int nextDoc() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public int advance(int target) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public long cost() { + throw new UnsupportedOperationException(); + } + + +} diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/AbstractSortingNumericDocValues.java b/core/src/main/java/org/elasticsearch/index/fielddata/AbstractSortingNumericDocValues.java new file mode 100644 index 0000000000..ad4b8c08cc --- /dev/null +++ b/core/src/main/java/org/elasticsearch/index/fielddata/AbstractSortingNumericDocValues.java @@ -0,0 +1,54 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.fielddata; + +import org.apache.lucene.search.DocIdSetIterator; + +import java.io.IOException; + +/** + * Base implementation that throws an {@link IOException} for the + * {@link DocIdSetIterator} APIs. This impl is safe to use for sorting and + * aggregations, which only use {@link #advanceExact(int)} and + * {@link #docValueCount()} and {@link #nextValue()}. + */ +public abstract class AbstractSortingNumericDocValues extends SortingNumericDocValues { + + @Override + public int docID() { + throw new UnsupportedOperationException(); + } + + @Override + public int nextDoc() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public int advance(int target) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public long cost() { + throw new UnsupportedOperationException(); + } + +} diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/AtomicOrdinalsFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/AtomicOrdinalsFieldData.java index 8d1fd31057..14ed16ecb3 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/AtomicOrdinalsFieldData.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/AtomicOrdinalsFieldData.java @@ -19,7 +19,7 @@ package org.elasticsearch.index.fielddata; -import org.apache.lucene.index.RandomAccessOrds; +import org.apache.lucene.index.SortedSetDocValues; /** * Specialization of {@link AtomicFieldData} for data that is indexed with @@ -30,6 +30,6 @@ public interface AtomicOrdinalsFieldData extends AtomicFieldData { /** * Return the ordinals values for the current atomic reader. */ - RandomAccessOrds getOrdinalsValues(); + SortedSetDocValues getOrdinalsValues(); } diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/FieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/FieldData.java index fce29c09ba..a1ac341b56 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/FieldData.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/FieldData.java @@ -22,15 +22,13 @@ package org.elasticsearch.index.fielddata; import org.apache.lucene.index.BinaryDocValues; import org.apache.lucene.index.DocValues; import org.apache.lucene.index.NumericDocValues; -import org.apache.lucene.index.RandomAccessOrds; import org.apache.lucene.index.SortedNumericDocValues; import org.apache.lucene.index.SortedSetDocValues; -import org.apache.lucene.util.ArrayUtil; import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.NumericUtils; import org.elasticsearch.common.geo.GeoPoint; +import java.io.IOException; import java.util.ArrayList; import java.util.List; @@ -43,8 +41,8 @@ public enum FieldData { /** * Return a {@link SortedBinaryDocValues} that doesn't contain any value. */ - public static SortedBinaryDocValues emptySortedBinary(int maxDoc) { - return singleton(DocValues.emptyBinary(), new Bits.MatchNoBits(maxDoc)); + public static SortedBinaryDocValues emptySortedBinary() { + return singleton(DocValues.emptyBinary()); } /** @@ -53,8 +51,13 @@ public enum FieldData { public static NumericDoubleValues emptyNumericDouble() { return new NumericDoubleValues() { @Override - public double get(int docID) { - return 0; + public boolean advanceExact(int doc) throws IOException { + return false; + } + + @Override + public double doubleValue() throws IOException { + throw new UnsupportedOperationException(); } }; @@ -63,16 +66,20 @@ public enum FieldData { /** * Return a {@link SortedNumericDoubleValues} that doesn't contain any value. */ - public static SortedNumericDoubleValues emptySortedNumericDoubles(int maxDoc) { - return singleton(emptyNumericDouble(), new Bits.MatchNoBits(maxDoc)); + public static SortedNumericDoubleValues emptySortedNumericDoubles() { + return singleton(emptyNumericDouble()); } public static GeoPointValues emptyGeoPoint() { - final GeoPoint point = new GeoPoint(); return new GeoPointValues() { @Override - public GeoPoint get(int docID) { - return point; + public boolean advanceExact(int doc) throws IOException { + return false; + } + + @Override + public GeoPoint geoPointValue() { + throw new UnsupportedOperationException(); } }; } @@ -80,68 +87,123 @@ public enum FieldData { /** * Return a {@link SortedNumericDoubleValues} that doesn't contain any value. */ - public static MultiGeoPointValues emptyMultiGeoPoints(int maxDoc) { - return singleton(emptyGeoPoint(), new Bits.MatchNoBits(maxDoc)); + public static MultiGeoPointValues emptyMultiGeoPoints() { + return singleton(emptyGeoPoint()); } /** * Returns a {@link Bits} representing all documents from <code>dv</code> that have a value. */ public static Bits docsWithValue(final SortedBinaryDocValues dv, final int maxDoc) { - return new Bits() { - @Override - public boolean get(int index) { - dv.setDocument(index); - return dv.count() != 0; - } + return new Bits() { + @Override + public boolean get(int index) { + try { + return dv.advanceExact(index); + } catch (IOException e) { + throw new RuntimeException(e); + } + } - @Override - public int length() { - return maxDoc; - } - }; + @Override + public int length() { + return maxDoc; + } + }; } /** - * Returns a Bits representing all documents from <code>dv</code> that have a value. + * Returns a {@link Bits} representing all documents from <code>dv</code> + * that have a value. + */ + public static Bits docsWithValue(final SortedSetDocValues dv, final int maxDoc) { + return new Bits() { + @Override + public boolean get(int index) { + try { + return dv.advanceExact(index); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + @Override + public int length() { + return maxDoc; + } + }; + } + + /** + * Returns a Bits representing all documents from <code>dv</code> that have + * a value. */ public static Bits docsWithValue(final MultiGeoPointValues dv, final int maxDoc) { - return new Bits() { - @Override - public boolean get(int index) { - dv.setDocument(index); - return dv.count() != 0; - } + return new Bits() { + @Override + public boolean get(int index) { + try { + return dv.advanceExact(index); + } catch (IOException e) { + throw new RuntimeException(e); + } + } - @Override - public int length() { - return maxDoc; - } - }; + @Override + public int length() { + return maxDoc; + } + }; } /** * Returns a Bits representing all documents from <code>dv</code> that have a value. */ public static Bits docsWithValue(final SortedNumericDoubleValues dv, final int maxDoc) { - return new Bits() { - @Override - public boolean get(int index) { - dv.setDocument(index); - return dv.count() != 0; - } + return new Bits() { + @Override + public boolean get(int index) { + try { + return dv.advanceExact(index); + } catch (IOException e) { + throw new RuntimeException(e); + } + } - @Override - public int length() { - return maxDoc; - } - }; + @Override + public int length() { + return maxDoc; + } + }; } /** - * Given a {@link SortedNumericDoubleValues}, return a {@link SortedNumericDocValues} - * instance that will translate double values to sortable long bits using - * {@link NumericUtils#doubleToSortableLong(double)}. + * Returns a Bits representing all documents from <code>dv</code> that have + * a value. + */ + public static Bits docsWithValue(final SortedNumericDocValues dv, final int maxDoc) { + return new Bits() { + @Override + public boolean get(int index) { + try { + return dv.advanceExact(index); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + @Override + public int length() { + return maxDoc; + } + }; + } + + /** + * Given a {@link SortedNumericDoubleValues}, return a + * {@link SortedNumericDocValues} instance that will translate double values + * to sortable long bits using + * {@link org.apache.lucene.util.NumericUtils#doubleToSortableLong(double)}. */ public static SortedNumericDocValues toSortableLongBits(SortedNumericDoubleValues values) { final NumericDoubleValues singleton = unwrapSingleton(values); @@ -152,8 +214,7 @@ public enum FieldData { } else { longBits = new SortableLongBitsNumericDocValues(singleton); } - final Bits docsWithField = unwrapSingletonBits(values); - return DocValues.singleton(longBits, docsWithField); + return DocValues.singleton(longBits); } else { if (values instanceof SortableLongBitsToSortedNumericDoubleValues) { return ((SortableLongBitsToSortedNumericDoubleValues) values).getLongValues(); @@ -166,7 +227,7 @@ public enum FieldData { /** * Given a {@link SortedNumericDocValues}, return a {@link SortedNumericDoubleValues} * instance that will translate long values to doubles using - * {@link NumericUtils#sortableLongToDouble(long)}. + * {@link org.apache.lucene.util.NumericUtils#sortableLongToDouble(long)}. */ public static SortedNumericDoubleValues sortableLongBitsToDoubles(SortedNumericDocValues values) { final NumericDocValues singleton = DocValues.unwrapSingleton(values); @@ -177,8 +238,7 @@ public enum FieldData { } else { doubles = new SortableLongBitsToNumericDoubleValues(singleton); } - final Bits docsWithField = DocValues.unwrapSingletonBits(values); - return singleton(doubles, docsWithField); + return singleton(doubles); } else { if (values instanceof SortableLongBitsSortedNumericDocValues) { return ((SortableLongBitsSortedNumericDocValues) values).getDoubleValues(); @@ -194,8 +254,7 @@ public enum FieldData { public static SortedNumericDoubleValues castToDouble(final SortedNumericDocValues values) { final NumericDocValues singleton = DocValues.unwrapSingleton(values); if (singleton != null) { - final Bits docsWithField = DocValues.unwrapSingletonBits(values); - return singleton(new DoubleCastedValues(singleton), docsWithField); + return singleton(new DoubleCastedValues(singleton)); } else { return new SortedDoubleCastedValues(values); } @@ -207,8 +266,7 @@ public enum FieldData { public static SortedNumericDocValues castToLong(final SortedNumericDoubleValues values) { final NumericDoubleValues singleton = unwrapSingleton(values); if (singleton != null) { - final Bits docsWithField = unwrapSingletonBits(values); - return DocValues.singleton(new LongCastedValues(singleton), docsWithField); + return DocValues.singleton(new LongCastedValues(singleton)); } else { return new SortedLongCastedValues(values); } @@ -217,15 +275,14 @@ public enum FieldData { /** * Returns a multi-valued view over the provided {@link NumericDoubleValues}. */ - public static SortedNumericDoubleValues singleton(NumericDoubleValues values, Bits docsWithField) { - return new SingletonSortedNumericDoubleValues(values, docsWithField); + public static SortedNumericDoubleValues singleton(NumericDoubleValues values) { + return new SingletonSortedNumericDoubleValues(values); } /** * Returns a single-valued view of the {@link SortedNumericDoubleValues}, - * if it was previously wrapped with {@link DocValues#singleton(NumericDocValues, Bits)}, + * if it was previously wrapped with {@link DocValues#singleton(NumericDocValues)}, * or null. - * @see DocValues#unwrapSingletonBits(SortedNumericDocValues) */ public static NumericDoubleValues unwrapSingleton(SortedNumericDoubleValues values) { if (values instanceof SingletonSortedNumericDoubleValues) { @@ -235,30 +292,16 @@ public enum FieldData { } /** - * Returns the documents with a value for the {@link SortedNumericDoubleValues}, - * if it was previously wrapped with {@link #singleton(NumericDoubleValues, Bits)}, - * or null. - */ - public static Bits unwrapSingletonBits(SortedNumericDoubleValues dv) { - if (dv instanceof SingletonSortedNumericDoubleValues) { - return ((SingletonSortedNumericDoubleValues)dv).getDocsWithField(); - } else { - return null; - } - } - - /** * Returns a multi-valued view over the provided {@link GeoPointValues}. */ - public static MultiGeoPointValues singleton(GeoPointValues values, Bits docsWithField) { - return new SingletonMultiGeoPointValues(values, docsWithField); + public static MultiGeoPointValues singleton(GeoPointValues values) { + return new SingletonMultiGeoPointValues(values); } /** * Returns a single-valued view of the {@link MultiGeoPointValues}, - * if it was previously wrapped with {@link #singleton(GeoPointValues, Bits)}, + * if it was previously wrapped with {@link #singleton(GeoPointValues)}, * or null. - * @see #unwrapSingletonBits(MultiGeoPointValues) */ public static GeoPointValues unwrapSingleton(MultiGeoPointValues values) { if (values instanceof SingletonMultiGeoPointValues) { @@ -268,29 +311,16 @@ public enum FieldData { } /** - * Returns the documents with a value for the {@link MultiGeoPointValues}, - * if it was previously wrapped with {@link #singleton(GeoPointValues, Bits)}, - * or null. - */ - public static Bits unwrapSingletonBits(MultiGeoPointValues values) { - if (values instanceof SingletonMultiGeoPointValues) { - return ((SingletonMultiGeoPointValues) values).getDocsWithField(); - } - return null; - } - - /** * Returns a multi-valued view over the provided {@link BinaryDocValues}. */ - public static SortedBinaryDocValues singleton(BinaryDocValues values, Bits docsWithField) { - return new SingletonSortedBinaryDocValues(values, docsWithField); + public static SortedBinaryDocValues singleton(BinaryDocValues values) { + return new SingletonSortedBinaryDocValues(values); } /** * Returns a single-valued view of the {@link SortedBinaryDocValues}, - * if it was previously wrapped with {@link #singleton(BinaryDocValues, Bits)}, + * if it was previously wrapped with {@link #singleton(BinaryDocValues)}, * or null. - * @see #unwrapSingletonBits(SortedBinaryDocValues) */ public static BinaryDocValues unwrapSingleton(SortedBinaryDocValues values) { if (values instanceof SingletonSortedBinaryDocValues) { @@ -300,18 +330,6 @@ public enum FieldData { } /** - * Returns the documents with a value for the {@link SortedBinaryDocValues}, - * if it was previously wrapped with {@link #singleton(BinaryDocValues, Bits)}, - * or null. - */ - public static Bits unwrapSingletonBits(SortedBinaryDocValues values) { - if (values instanceof SingletonSortedBinaryDocValues) { - return ((SingletonSortedBinaryDocValues) values).getDocsWithField(); - } - return null; - } - - /** * Returns whether the provided values *might* be multi-valued. There is no * guarantee that this method will return <tt>false</tt> in the single-valued case. */ @@ -359,10 +377,13 @@ public enum FieldData { public static SortedBinaryDocValues toString(final SortedNumericDocValues values) { return toString(new ToStringValues() { @Override - public void get(int docID, List<CharSequence> list) { - values.setDocument(docID); - for (int i = 0, count = values.count(); i < count; ++i) { - list.add(Long.toString(values.valueAt(i))); + public boolean advanceExact(int doc) throws IOException { + return values.advanceExact(doc); + } + @Override + public void get(List<CharSequence> list) throws IOException { + for (int i = 0, count = values.docValueCount(); i < count; ++i) { + list.add(Long.toString(values.nextValue())); } } }); @@ -376,10 +397,13 @@ public enum FieldData { public static SortedBinaryDocValues toString(final SortedNumericDoubleValues values) { return toString(new ToStringValues() { @Override - public void get(int docID, List<CharSequence> list) { - values.setDocument(docID); - for (int i = 0, count = values.count(); i < count; ++i) { - list.add(Double.toString(values.valueAt(i))); + public boolean advanceExact(int doc) throws IOException { + return values.advanceExact(doc); + } + @Override + public void get(List<CharSequence> list) throws IOException { + for (int i = 0, count = values.docValueCount(); i < count; ++i) { + list.add(Double.toString(values.nextValue())); } } }); @@ -390,23 +414,37 @@ public enum FieldData { * typically used for scripts or for the `map` execution mode of terms aggs. * NOTE: this is slow! */ - public static SortedBinaryDocValues toString(final RandomAccessOrds values) { + public static SortedBinaryDocValues toString(final SortedSetDocValues values) { return new SortedBinaryDocValues() { + private int count = 0; @Override - public BytesRef valueAt(int index) { - return values.lookupOrd(values.ordAt(index)); + public boolean advanceExact(int doc) throws IOException { + if (values.advanceExact(doc) == false) { + return false; + } + for (int i = 0; ; ++i) { + if (values.nextOrd() == SortedSetDocValues.NO_MORE_ORDS) { + count = i; + break; + } + } + // reset the iterator on the current doc + boolean advanced = values.advanceExact(doc); + assert advanced; + return true; } @Override - public void setDocument(int docId) { - values.setDocument(docId); + public int docValueCount() { + return count; } @Override - public int count() { - return values.cardinality(); + public BytesRef nextValue() throws IOException { + return values.lookupOrd(values.nextOrd()); } + }; } @@ -418,78 +456,30 @@ public enum FieldData { public static SortedBinaryDocValues toString(final MultiGeoPointValues values) { return toString(new ToStringValues() { @Override - public void get(int docID, List<CharSequence> list) { - values.setDocument(docID); - for (int i = 0, count = values.count(); i < count; ++i) { - list.add(values.valueAt(i).toString()); + public boolean advanceExact(int doc) throws IOException { + return values.advanceExact(doc); + } + @Override + public void get(List<CharSequence> list) throws IOException { + for (int i = 0, count = values.docValueCount(); i < count; ++i) { + list.add(values.nextValue().toString()); } } }); } - /** - * If <code>dv</code> is an instance of {@link RandomAccessOrds}, then return - * it, otherwise wrap it into a slow wrapper that implements random access. - */ - public static RandomAccessOrds maybeSlowRandomAccessOrds(final SortedSetDocValues dv) { - if (dv instanceof RandomAccessOrds) { - return (RandomAccessOrds) dv; - } else { - assert DocValues.unwrapSingleton(dv) == null : "this method expect singleton to return random-access ords"; - return new RandomAccessOrds() { - - int cardinality; - long[] ords = new long[0]; - int ord; - - @Override - public void setDocument(int docID) { - cardinality = 0; - dv.setDocument(docID); - for (long ord = dv.nextOrd(); ord != SortedSetDocValues.NO_MORE_ORDS; ord = dv.nextOrd()) { - ords = ArrayUtil.grow(ords, cardinality + 1); - ords[cardinality++] = ord; - } - ord = 0; - } - - @Override - public long nextOrd() { - return ords[ord++]; - } - - @Override - public BytesRef lookupOrd(long ord) { - return dv.lookupOrd(ord); - } - - @Override - public long getValueCount() { - return dv.getValueCount(); - } - - @Override - public long ordAt(int index) { - return ords[index]; - } - - @Override - public int cardinality() { - return cardinality; - } - }; - } - } - private static SortedBinaryDocValues toString(final ToStringValues toStringValues) { return new SortingBinaryDocValues() { final List<CharSequence> list = new ArrayList<>(); @Override - public void setDocument(int docID) { + public boolean advanceExact(int docID) throws IOException { + if (toStringValues.advanceExact(docID) == false) { + return false; + } list.clear(); - toStringValues.get(docID, list); + toStringValues.get(list); count = list.size(); grow(); for (int i = 0; i < count; ++i) { @@ -497,6 +487,7 @@ public enum FieldData { values[i].copyChars(s); } sort(); + return true; } }; @@ -504,7 +495,14 @@ public enum FieldData { private interface ToStringValues { - void get(int docID, List<CharSequence> values); + /** + * Advance this instance to the given document id + * @return true if there is a value for this document + */ + boolean advanceExact(int doc) throws IOException; + + /** Fill the list of charsquences with the list of values for the current document. */ + void get(List<CharSequence> values) throws IOException; } @@ -517,8 +515,13 @@ public enum FieldData { } @Override - public double get(int docID) { - return values.get(docID); + public double doubleValue() throws IOException { + return values.longValue(); + } + + @Override + public boolean advanceExact(int doc) throws IOException { + return values.advanceExact(doc); } } @@ -532,38 +535,49 @@ public enum FieldData { } @Override - public double valueAt(int index) { - return values.valueAt(index); + public boolean advanceExact(int target) throws IOException { + return values.advanceExact(target); } @Override - public void setDocument(int doc) { - values.setDocument(doc); + public double nextValue() throws IOException { + return values.nextValue(); } @Override - public int count() { - return values.count(); + public int docValueCount() { + return values.docValueCount(); } } - private static class LongCastedValues extends NumericDocValues { + private static class LongCastedValues extends AbstractNumericDocValues { private final NumericDoubleValues values; + private int docID = -1; LongCastedValues(NumericDoubleValues values) { this.values = values; } @Override - public long get(int docID) { - return (long) values.get(docID); + public boolean advanceExact(int target) throws IOException { + docID = target; + return values.advanceExact(target); } + @Override + public long longValue() throws IOException { + return (long) values.doubleValue(); + } + + @Override + public int docID() { + return docID; + } } - private static class SortedLongCastedValues extends SortedNumericDocValues { + private static class SortedLongCastedValues extends AbstractSortedNumericDocValues { private final SortedNumericDoubleValues values; @@ -572,18 +586,18 @@ public enum FieldData { } @Override - public long valueAt(int index) { - return (long) values.valueAt(index); + public boolean advanceExact(int target) throws IOException { + return values.advanceExact(target); } @Override - public void setDocument(int doc) { - values.setDocument(doc); + public int docValueCount() { + return values.docValueCount(); } @Override - public int count() { - return values.count(); + public long nextValue() throws IOException { + return (long) values.nextValue(); } } diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/GeoPointValues.java b/core/src/main/java/org/elasticsearch/index/fielddata/GeoPointValues.java index 8e8c8e67c9..8a679f18cf 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/GeoPointValues.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/GeoPointValues.java @@ -21,17 +21,23 @@ package org.elasticsearch.index.fielddata; import org.elasticsearch.common.geo.GeoPoint; +import java.io.IOException; + /** * Per-document geo-point values. */ public abstract class GeoPointValues { /** - * Get the {@link GeoPoint} associated with <code>docID</code>. + * Advance this instance to the given document id + * @return true if there is a value for this document + */ + public abstract boolean advanceExact(int doc) throws IOException; + + /** + * Get the {@link GeoPoint} associated with the current document. * The returned {@link GeoPoint} might be reused across calls. - * If the given <code>docID</code> does not have a value then the returned - * geo point mught have both latitude and longitude set to 0. */ - public abstract GeoPoint get(int docID); + public abstract GeoPoint geoPointValue(); } diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/MultiGeoPointValues.java b/core/src/main/java/org/elasticsearch/index/fielddata/MultiGeoPointValues.java index 6fa9c799dd..c80c337c6d 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/MultiGeoPointValues.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/MultiGeoPointValues.java @@ -20,6 +20,8 @@ package org.elasticsearch.index.fielddata; import org.elasticsearch.common.geo.GeoPoint; +import java.io.IOException; + /** * A stateful lightweight per document set of {@link GeoPoint} values. * To iterate over values in a document use the following pattern: @@ -44,28 +46,24 @@ public abstract class MultiGeoPointValues { } /** - * Sets iteration to the specified docID. - * @param docId document ID - * - * @see #valueAt(int) - * @see #count() + * Advance this instance to the given document id + * @return true if there is a value for this document */ - public abstract void setDocument(int docId); + public abstract boolean advanceExact(int doc) throws IOException; /** * Return the number of geo points the current document has. */ - public abstract int count(); + public abstract int docValueCount(); /** - * Return the <code>i-th</code> value associated with the current document. - * Behavior is undefined when <code>i</code> is undefined or greater than - * or equal to {@link #count()}. + * Return the next value associated with the current document. This must not be + * called more than {@link #docValueCount()} times. * * Note: the returned {@link GeoPoint} might be shared across invocations. * - * @return the next value for the current docID set to {@link #setDocument(int)}. + * @return the next value for the current docID set to {@link #advanceExact(int)}. */ - public abstract GeoPoint valueAt(int i); + public abstract GeoPoint nextValue() throws IOException; } diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/NumericDoubleValues.java b/core/src/main/java/org/elasticsearch/index/fielddata/NumericDoubleValues.java index 2cbbb0064f..0d78d3c790 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/NumericDoubleValues.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/NumericDoubleValues.java @@ -20,43 +20,58 @@ package org.elasticsearch.index.fielddata; import org.apache.lucene.index.NumericDocValues; +import org.apache.lucene.search.DoubleValues; + +import java.io.IOException; /** * A per-document numeric value. */ -public abstract class NumericDoubleValues { +public abstract class NumericDoubleValues extends DoubleValues { /** Sole constructor. (For invocation by subclass * constructors, typically implicit.) */ protected NumericDoubleValues() {} - - /** - * Returns the numeric value for the specified document ID. This must return - * <tt>0d</tt> if the given doc ID has no value. - * @param docID document ID to lookup - * @return numeric value - */ - public abstract double get(int docID); // TODO: this interaction with sort comparators is really ugly... /** Returns numeric docvalues view of raw double bits */ public NumericDocValues getRawDoubleValues() { - return new NumericDocValues() { - @Override - public long get(int docID) { - return Double.doubleToRawLongBits(NumericDoubleValues.this.get(docID)); - } + return new AbstractNumericDocValues() { + private int docID = -1; + @Override + public boolean advanceExact(int target) throws IOException { + docID = target; + return NumericDoubleValues.this.advanceExact(target); + } + @Override + public long longValue() throws IOException { + return Double.doubleToRawLongBits(NumericDoubleValues.this.doubleValue()); + } + @Override + public int docID() { + return docID; + } }; } // yes... this is doing what the previous code was doing... /** Returns numeric docvalues view of raw float bits */ public NumericDocValues getRawFloatValues() { - return new NumericDocValues() { - @Override - public long get(int docID) { - return Float.floatToRawIntBits((float)NumericDoubleValues.this.get(docID)); - } + return new AbstractNumericDocValues() { + private int docID = -1; + @Override + public boolean advanceExact(int target) throws IOException { + docID = target; + return NumericDoubleValues.this.advanceExact(target); + } + @Override + public long longValue() throws IOException { + return Float.floatToRawIntBits((float)NumericDoubleValues.this.doubleValue()); + } + @Override + public int docID() { + return docID; + } }; } } diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java b/core/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java index bfa23eb8f0..339e70c50b 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java @@ -21,7 +21,9 @@ package org.elasticsearch.index.fielddata; import org.apache.lucene.index.SortedNumericDocValues; +import org.apache.lucene.util.ArrayUtil; import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.BytesRefBuilder; import org.elasticsearch.common.geo.GeoHashUtils; import org.elasticsearch.common.geo.GeoPoint; import org.elasticsearch.common.geo.GeoUtils; @@ -32,7 +34,9 @@ import org.joda.time.DateTimeZone; import org.joda.time.MutableDateTime; import org.joda.time.ReadableDateTime; +import java.io.IOException; import java.util.AbstractList; +import java.util.Arrays; import java.util.Comparator; import java.util.List; import java.util.function.UnaryOperator; @@ -46,7 +50,7 @@ public abstract class ScriptDocValues<T> extends AbstractList<T> { /** * Set the current doc ID. */ - public abstract void setNextDocId(int docId); + public abstract void setNextDocId(int docId) throws IOException; /** * Return a copy of the list of the values for the current document. @@ -83,24 +87,48 @@ public abstract class ScriptDocValues<T> extends AbstractList<T> { public static final class Strings extends ScriptDocValues<String> { - private final SortedBinaryDocValues values; + private final SortedBinaryDocValues in; + private BytesRefBuilder[] values = new BytesRefBuilder[0]; + private int count; - public Strings(SortedBinaryDocValues values) { - this.values = values; + public Strings(SortedBinaryDocValues in) { + this.in = in; } @Override - public void setNextDocId(int docId) { - values.setDocument(docId); + public void setNextDocId(int docId) throws IOException { + if (in.advanceExact(docId)) { + resize(in.docValueCount()); + for (int i = 0; i < count; i++) { + values[i].copyBytes(in.nextValue()); + } + } else { + resize(0); + } + } + + /** + * Set the {@link #size()} and ensure that the {@link #values} array can + * store at least that many entries. + */ + protected void resize(int newSize) { + count = newSize; + if (newSize > values.length) { + final int oldLength = values.length; + values = ArrayUtil.grow(values, count); + for (int i = oldLength; i < values.length; ++i) { + values[i] = new BytesRefBuilder(); + } + } } public SortedBinaryDocValues getInternalValues() { - return this.values; + return this.in; } public BytesRef getBytesValue() { - if (values.count() > 0) { - return values.valueAt(0); + if (size() > 0) { + return values[0].get(); } else { return null; } @@ -117,12 +145,12 @@ public abstract class ScriptDocValues<T> extends AbstractList<T> { @Override public String get(int index) { - return values.valueAt(index).utf8ToString(); + return values[index].get().utf8ToString(); } @Override public int size() { - return values.count(); + return count; } } @@ -130,61 +158,81 @@ public abstract class ScriptDocValues<T> extends AbstractList<T> { public static final class Longs extends ScriptDocValues<Long> { protected static final DeprecationLogger deprecationLogger = new DeprecationLogger(ESLoggerFactory.getLogger(Longs.class)); - private final SortedNumericDocValues values; + private final SortedNumericDocValues in; + private long[] values = new long[0]; + private int count; private Dates dates; + private int docId = -1; + + public Longs(SortedNumericDocValues in) { + this.in = in; - public Longs(SortedNumericDocValues values) { - this.values = values; } @Override - public void setNextDocId(int docId) { - values.setDocument(docId); + public void setNextDocId(int docId) throws IOException { + this.docId = docId; + if (in.advanceExact(docId)) { + resize(in.docValueCount()); + for (int i = 0; i < count; i++) { + values[i] = in.nextValue(); + } + } else { + resize(0); + } if (dates != null) { - dates.refreshArray(); + dates.setNextDocId(docId); } } + /** + * Set the {@link #size()} and ensure that the {@link #values} array can + * store at least that many entries. + */ + protected void resize(int newSize) { + count = newSize; + values = ArrayUtil.grow(values, count); + } + public SortedNumericDocValues getInternalValues() { - return this.values; + return this.in; } public long getValue() { - int numValues = values.count(); - if (numValues == 0) { + if (count == 0) { return 0L; } - return values.valueAt(0); + return values[0]; } @Deprecated - public ReadableDateTime getDate() { + public ReadableDateTime getDate() throws IOException { deprecationLogger.deprecated("getDate on numeric fields is deprecated. Use a date field to get dates."); if (dates == null) { - dates = new Dates(values); - dates.refreshArray(); + dates = new Dates(in); + dates.setNextDocId(docId); } return dates.getValue(); } @Deprecated - public List<ReadableDateTime> getDates() { + public List<ReadableDateTime> getDates() throws IOException { deprecationLogger.deprecated("getDates on numeric fields is deprecated. Use a date field to get dates."); if (dates == null) { - dates = new Dates(values); - dates.refreshArray(); + dates = new Dates(in); + dates.setNextDocId(docId); } return dates; } @Override public Long get(int index) { - return values.valueAt(index); + return values[index]; } @Override public int size() { - return values.count(); + return count; } } @@ -193,22 +241,24 @@ public abstract class ScriptDocValues<T> extends AbstractList<T> { private static final ReadableDateTime EPOCH = new DateTime(0, DateTimeZone.UTC); - private final SortedNumericDocValues values; + private final SortedNumericDocValues in; /** * Values wrapped in {@link MutableDateTime}. Null by default an allocated on first usage so we allocate a reasonably size. We keep * this array so we don't have allocate new {@link MutableDateTime}s on every usage. Instead we reuse them for every document. */ private MutableDateTime[] dates; + private int count; - public Dates(SortedNumericDocValues values) { - this.values = values; + public Dates(SortedNumericDocValues in) { + this.in = in; } /** - * Fetch the first field value or 0 millis after epoch if there are no values. + * Fetch the first field value or 0 millis after epoch if there are no + * in. */ public ReadableDateTime getValue() { - if (values.count() == 0) { + if (count == 0) { return EPOCH; } return get(0); @@ -234,113 +284,159 @@ public abstract class ScriptDocValues<T> extends AbstractList<T> { @Override public ReadableDateTime get(int index) { - if (index >= values.count()) { + if (index >= count) { throw new IndexOutOfBoundsException( - "attempted to fetch the [" + index + "] date when there are only [" + values.count() + "] dates."); + "attempted to fetch the [" + index + "] date when there are only [" + + count + "] dates."); } return dates[index]; } @Override public int size() { - return values.count(); + return count; } @Override - public void setNextDocId(int docId) { - values.setDocument(docId); + public void setNextDocId(int docId) throws IOException { + if (in.advanceExact(docId)) { + count = in.docValueCount(); + } else { + count = 0; + } refreshArray(); } /** * Refresh the backing array. Package private so it can be called when {@link Longs} loads dates. */ - void refreshArray() { - if (values.count() == 0) { + void refreshArray() throws IOException { + if (count == 0) { return; } if (dates == null) { // Happens for the document. We delay allocating dates so we can allocate it with a reasonable size. - dates = new MutableDateTime[values.count()]; + dates = new MutableDateTime[count]; for (int i = 0; i < dates.length; i++) { - dates[i] = new MutableDateTime(values.valueAt(i), DateTimeZone.UTC); + dates[i] = new MutableDateTime(in.nextValue(), DateTimeZone.UTC); } return; } - if (values.count() > dates.length) { + if (count > dates.length) { // Happens when we move to a new document and it has more dates than any documents before it. MutableDateTime[] backup = dates; - dates = new MutableDateTime[values.count()]; + dates = new MutableDateTime[count]; System.arraycopy(backup, 0, dates, 0, backup.length); for (int i = 0; i < backup.length; i++) { - dates[i].setMillis(values.valueAt(i)); + dates[i].setMillis(in.nextValue()); } for (int i = backup.length; i < dates.length; i++) { - dates[i] = new MutableDateTime(values.valueAt(i), DateTimeZone.UTC); + dates[i] = new MutableDateTime(in.nextValue(), DateTimeZone.UTC); } return; } - for (int i = 0; i < values.count(); i++) { - dates[i].setMillis(values.valueAt(i)); + for (int i = 0; i < count; i++) { + dates[i].setMillis(in.nextValue()); } } } public static final class Doubles extends ScriptDocValues<Double> { - private final SortedNumericDoubleValues values; + private final SortedNumericDoubleValues in; + private double[] values = new double[0]; + private int count; - public Doubles(SortedNumericDoubleValues values) { - this.values = values; + public Doubles(SortedNumericDoubleValues in) { + this.in = in; } @Override - public void setNextDocId(int docId) { - values.setDocument(docId); + public void setNextDocId(int docId) throws IOException { + if (in.advanceExact(docId)) { + resize(in.docValueCount()); + for (int i = 0; i < count; i++) { + values[i] = in.nextValue(); + } + } else { + resize(0); + } + } + + /** + * Set the {@link #size()} and ensure that the {@link #values} array can + * store at least that many entries. + */ + protected void resize(int newSize) { + count = newSize; + values = ArrayUtil.grow(values, count); } public SortedNumericDoubleValues getInternalValues() { - return this.values; + return this.in; } public double getValue() { - int numValues = values.count(); - if (numValues == 0) { + if (count == 0) { return 0d; } - return values.valueAt(0); + return values[0]; } @Override public Double get(int index) { - return values.valueAt(index); + return values[index]; } @Override public int size() { - return values.count(); + return count; } } public static final class GeoPoints extends ScriptDocValues<GeoPoint> { - private final MultiGeoPointValues values; + private final MultiGeoPointValues in; + private GeoPoint[] values = new GeoPoint[0]; + private int count; - public GeoPoints(MultiGeoPointValues values) { - this.values = values; + public GeoPoints(MultiGeoPointValues in) { + this.in = in; } @Override - public void setNextDocId(int docId) { - values.setDocument(docId); + public void setNextDocId(int docId) throws IOException { + if (in.advanceExact(docId)) { + resize(in.docValueCount()); + for (int i = 0; i < count; i++) { + GeoPoint point = in.nextValue(); + values[i].reset(point.lat(), point.lon()); + } + } else { + resize(0); + } + } + + /** + * Set the {@link #size()} and ensure that the {@link #values} array can + * store at least that many entries. + */ + protected void resize(int newSize) { + count = newSize; + if (newSize > values.length) { + int oldLength = values.length; + values = ArrayUtil.grow(values, count); + for (int i = oldLength; i < values.length; ++i) { + values[i] = new GeoPoint(); + } + } } public GeoPoint getValue() { - int numValues = values.count(); - if (numValues == 0) { + if (count == 0) { return null; } - return values.valueAt(0); + return values[0]; } public double getLat() { @@ -371,13 +467,13 @@ public abstract class ScriptDocValues<T> extends AbstractList<T> { @Override public GeoPoint get(int index) { - final GeoPoint point = values.valueAt(index); + final GeoPoint point = values[index]; return new GeoPoint(point.lat(), point.lon()); } @Override public int size() { - return values.count(); + return count; } public double arcDistance(double lat, double lon) { @@ -420,66 +516,114 @@ public abstract class ScriptDocValues<T> extends AbstractList<T> { public static final class Booleans extends ScriptDocValues<Boolean> { - private final SortedNumericDocValues values; + private final SortedNumericDocValues in; + private boolean[] values = new boolean[0]; + private int count; - public Booleans(SortedNumericDocValues values) { - this.values = values; + public Booleans(SortedNumericDocValues in) { + this.in = in; } @Override - public void setNextDocId(int docId) { - values.setDocument(docId); + public void setNextDocId(int docId) throws IOException { + if (in.advanceExact(docId)) { + resize(in.docValueCount()); + for (int i = 0; i < count; i++) { + values[i] = in.nextValue() == 1; + } + } else { + resize(0); + } + } + + /** + * Set the {@link #size()} and ensure that the {@link #values} array can + * store at least that many entries. + */ + protected void resize(int newSize) { + count = newSize; + values = grow(values, count); } public boolean getValue() { - return values.count() != 0 && values.valueAt(0) == 1; + return count != 0 && values[0]; } @Override public Boolean get(int index) { - return values.valueAt(index) == 1; + return values[index]; } @Override public int size() { - return values.count(); + return count; + } + + private static boolean[] grow(boolean[] array, int minSize) { + assert minSize >= 0 : "size must be positive (got " + minSize + + "): likely integer overflow?"; + if (array.length < minSize) { + return Arrays.copyOf(array, ArrayUtil.oversize(minSize, 1)); + } else + return array; } } public static final class BytesRefs extends ScriptDocValues<BytesRef> { - private final SortedBinaryDocValues values; + private final SortedBinaryDocValues in; + private BytesRef[] values; + private int count; - public BytesRefs(SortedBinaryDocValues values) { - this.values = values; + public BytesRefs(SortedBinaryDocValues in) { + this.in = in; } @Override - public void setNextDocId(int docId) { - values.setDocument(docId); + public void setNextDocId(int docId) throws IOException { + if (in.advanceExact(docId)) { + resize(in.docValueCount()); + for (int i = 0; i < count; i++) { + values[i] = in.nextValue(); + } + } else { + resize(0); + } + } + + /** + * Set the {@link #size()} and ensure that the {@link #values} array can + * store at least that many entries. + */ + protected void resize(int newSize) { + count = newSize; + if (values == null) { + values = new BytesRef[newSize]; + } else { + values = ArrayUtil.grow(values, count); + } } public SortedBinaryDocValues getInternalValues() { - return this.values; + return this.in; } public BytesRef getValue() { - int numValues = values.count(); - if (numValues == 0) { + if (count == 0) { return new BytesRef(); } - return values.valueAt(0); + return values[0]; } @Override public BytesRef get(int index) { - return values.valueAt(index); + return values[index]; } @Override public int size() { - return values.count(); + return count; } } } diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/SingletonMultiGeoPointValues.java b/core/src/main/java/org/elasticsearch/index/fielddata/SingletonMultiGeoPointValues.java index 883444fabc..bae522e7b5 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/SingletonMultiGeoPointValues.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/SingletonMultiGeoPointValues.java @@ -19,48 +19,34 @@ package org.elasticsearch.index.fielddata; -import org.apache.lucene.util.Bits; import org.elasticsearch.common.geo.GeoPoint; +import java.io.IOException; + final class SingletonMultiGeoPointValues extends MultiGeoPointValues { private final GeoPointValues in; - private final Bits docsWithField; - private GeoPoint value; - private int count; - SingletonMultiGeoPointValues(GeoPointValues in, Bits docsWithField) { + SingletonMultiGeoPointValues(GeoPointValues in) { this.in = in; - this.docsWithField = docsWithField; } @Override - public void setDocument(int docID) { - value = in.get(docID); - if (value.lat() == Double.NaN && value.lon() == Double.NaN || (docsWithField != null && !docsWithField.get(docID))) { - count = 0; - } else { - count = 1; - } + public boolean advanceExact(int doc) throws IOException { + return in.advanceExact(doc); } @Override - public int count() { - return count; + public int docValueCount() { + return 1; } @Override - public GeoPoint valueAt(int index) { - assert index == 0; - return value; + public GeoPoint nextValue() { + return in.geoPointValue(); } - public GeoPointValues getGeoPointValues() { + GeoPointValues getGeoPointValues() { return in; } - - public Bits getDocsWithField() { - return docsWithField; - } - } diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/SingletonSortedBinaryDocValues.java b/core/src/main/java/org/elasticsearch/index/fielddata/SingletonSortedBinaryDocValues.java index 1a35f056f9..01f110a7b3 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/SingletonSortedBinaryDocValues.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/SingletonSortedBinaryDocValues.java @@ -20,49 +20,35 @@ package org.elasticsearch.index.fielddata; import org.apache.lucene.index.BinaryDocValues; -import org.apache.lucene.util.Bits; -import org.apache.lucene.util.Bits.MatchAllBits; import org.apache.lucene.util.BytesRef; +import java.io.IOException; + final class SingletonSortedBinaryDocValues extends SortedBinaryDocValues { private final BinaryDocValues in; - private final Bits docsWithField; - private BytesRef value; - private int count; - SingletonSortedBinaryDocValues(BinaryDocValues in, Bits docsWithField) { + SingletonSortedBinaryDocValues(BinaryDocValues in) { this.in = in; - this.docsWithField = docsWithField instanceof MatchAllBits ? null : docsWithField; } @Override - public void setDocument(int docID) { - value = in.get(docID); - if (value.length == 0 && docsWithField != null && !docsWithField.get(docID)) { - count = 0; - } else { - count = 1; - } + public boolean advanceExact(int doc) throws IOException { + return in.advanceExact(doc); } @Override - public int count() { - return count; + public int docValueCount() { + return 1; } @Override - public BytesRef valueAt(int index) { - assert index == 0; - return value; + public BytesRef nextValue() throws IOException { + return in.binaryValue(); } public BinaryDocValues getBinaryDocValues() { return in; } - public Bits getDocsWithField() { - return docsWithField; - } - } diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/SingletonSortedNumericDoubleValues.java b/core/src/main/java/org/elasticsearch/index/fielddata/SingletonSortedNumericDoubleValues.java index 4207ac73a1..af0768ac2c 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/SingletonSortedNumericDoubleValues.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/SingletonSortedNumericDoubleValues.java @@ -19,8 +19,7 @@ package org.elasticsearch.index.fielddata; -import org.apache.lucene.util.Bits; -import org.apache.lucene.util.Bits.MatchAllBits; +import java.io.IOException; /** * Exposes multi-valued view over a single-valued instance. @@ -29,43 +28,30 @@ import org.apache.lucene.util.Bits.MatchAllBits; * that works for single or multi-valued types. */ final class SingletonSortedNumericDoubleValues extends SortedNumericDoubleValues { - private final NumericDoubleValues in; - private final Bits docsWithField; - private double value; - private int count; + private final NumericDoubleValues in; - SingletonSortedNumericDoubleValues(NumericDoubleValues in, Bits docsWithField) { - this.in = in; - this.docsWithField = docsWithField instanceof MatchAllBits ? null : docsWithField; - } + SingletonSortedNumericDoubleValues(NumericDoubleValues in) { + this.in = in; + } - /** Return the wrapped {@link NumericDoubleValues} */ - public NumericDoubleValues getNumericDoubleValues() { - return in; - } + /** Return the wrapped {@link NumericDoubleValues} */ + public NumericDoubleValues getNumericDoubleValues() { + return in; + } - /** Return the wrapped {@link Bits} */ - public Bits getDocsWithField() { - return docsWithField; - } + @Override + public boolean advanceExact(int target) throws IOException { + return in.advanceExact(target); + } - @Override - public void setDocument(int doc) { - value = in.get(doc); - if (docsWithField != null && value == 0 && docsWithField.get(doc) == false) { - count = 0; - } else { - count = 1; + @Override + public int docValueCount() { + return 1; } - } - @Override - public double valueAt(int index) { - return value; - } + @Override + public double nextValue() throws IOException { + return in.doubleValue(); + } - @Override - public int count() { - return count; - } } diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/SortableLongBitsNumericDocValues.java b/core/src/main/java/org/elasticsearch/index/fielddata/SortableLongBitsNumericDocValues.java index 678b420613..259070cfc9 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/SortableLongBitsNumericDocValues.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/SortableLongBitsNumericDocValues.java @@ -22,13 +22,16 @@ package org.elasticsearch.index.fielddata; import org.apache.lucene.index.NumericDocValues; import org.apache.lucene.util.NumericUtils; +import java.io.IOException; + /** * {@link NumericDocValues} instance that wraps a {@link NumericDoubleValues} * and converts the doubles to sortable long bits using * {@link NumericUtils#doubleToSortableLong(double)}. */ -final class SortableLongBitsNumericDocValues extends NumericDocValues { +final class SortableLongBitsNumericDocValues extends AbstractNumericDocValues { + private int docID = -1; private final NumericDoubleValues values; SortableLongBitsNumericDocValues(NumericDoubleValues values) { @@ -36,8 +39,19 @@ final class SortableLongBitsNumericDocValues extends NumericDocValues { } @Override - public long get(int docID) { - return NumericUtils.doubleToSortableLong(values.get(docID)); + public long longValue() throws IOException { + return NumericUtils.doubleToSortableLong(values.doubleValue()); + } + + @Override + public boolean advanceExact(int target) throws IOException { + docID = target; + return values.advanceExact(target); + } + + @Override + public int docID() { + return docID; } /** Return the wrapped values. */ diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/SortableLongBitsSortedNumericDocValues.java b/core/src/main/java/org/elasticsearch/index/fielddata/SortableLongBitsSortedNumericDocValues.java index 9dbf44f14f..81d33bc432 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/SortableLongBitsSortedNumericDocValues.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/SortableLongBitsSortedNumericDocValues.java @@ -22,12 +22,14 @@ package org.elasticsearch.index.fielddata; import org.apache.lucene.index.SortedNumericDocValues; import org.apache.lucene.util.NumericUtils; +import java.io.IOException; + /** * {@link SortedNumericDocValues} instance that wraps a {@link SortedNumericDoubleValues} * and converts the doubles to sortable long bits using * {@link NumericUtils#doubleToSortableLong(double)}. */ -final class SortableLongBitsSortedNumericDocValues extends SortedNumericDocValues { +final class SortableLongBitsSortedNumericDocValues extends AbstractSortedNumericDocValues { private final SortedNumericDoubleValues values; @@ -36,18 +38,18 @@ final class SortableLongBitsSortedNumericDocValues extends SortedNumericDocValue } @Override - public void setDocument(int doc) { - values.setDocument(doc); + public boolean advanceExact(int target) throws IOException { + return values.advanceExact(target); } @Override - public long valueAt(int index) { - return NumericUtils.doubleToSortableLong(values.valueAt(index)); + public long nextValue() throws IOException { + return NumericUtils.doubleToSortableLong(values.nextValue()); } @Override - public int count() { - return values.count(); + public int docValueCount() { + return values.docValueCount(); } /** Return the wrapped values. */ diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/SortableLongBitsToNumericDoubleValues.java b/core/src/main/java/org/elasticsearch/index/fielddata/SortableLongBitsToNumericDoubleValues.java index 7c150cf319..94bc7168dc 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/SortableLongBitsToNumericDoubleValues.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/SortableLongBitsToNumericDoubleValues.java @@ -22,6 +22,8 @@ package org.elasticsearch.index.fielddata; import org.apache.lucene.index.NumericDocValues; import org.apache.lucene.util.NumericUtils; +import java.io.IOException; + /** * {@link NumericDoubleValues} instance that wraps a {@link NumericDocValues} * and converts the doubles to sortable long bits using @@ -36,8 +38,13 @@ final class SortableLongBitsToNumericDoubleValues extends NumericDoubleValues { } @Override - public double get(int docID) { - return NumericUtils.sortableLongToDouble(values.get(docID)); + public double doubleValue() throws IOException { + return NumericUtils.sortableLongToDouble(values.longValue()); + } + + @Override + public boolean advanceExact(int doc) throws IOException { + return values.advanceExact(doc); } /** Return the wrapped values. */ diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/SortableLongBitsToSortedNumericDoubleValues.java b/core/src/main/java/org/elasticsearch/index/fielddata/SortableLongBitsToSortedNumericDoubleValues.java index 2669463170..d78b66114b 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/SortableLongBitsToSortedNumericDoubleValues.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/SortableLongBitsToSortedNumericDoubleValues.java @@ -22,6 +22,8 @@ package org.elasticsearch.index.fielddata; import org.apache.lucene.index.SortedNumericDocValues; import org.apache.lucene.util.NumericUtils; +import java.io.IOException; + /** * {@link SortedNumericDoubleValues} instance that wraps a {@link SortedNumericDocValues} * and converts the doubles to sortable long bits using @@ -36,18 +38,18 @@ final class SortableLongBitsToSortedNumericDoubleValues extends SortedNumericDou } @Override - public void setDocument(int doc) { - values.setDocument(doc); + public boolean advanceExact(int target) throws IOException { + return values.advanceExact(target); } @Override - public double valueAt(int index) { - return NumericUtils.sortableLongToDouble(values.valueAt(index)); + public double nextValue() throws IOException { + return NumericUtils.sortableLongToDouble(values.nextValue()); } @Override - public int count() { - return values.count(); + public int docValueCount() { + return values.docValueCount(); } /** Return the wrapped values. */ diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/SortedBinaryDocValues.java b/core/src/main/java/org/elasticsearch/index/fielddata/SortedBinaryDocValues.java index b3c51141e2..119fda1bda 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/SortedBinaryDocValues.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/SortedBinaryDocValues.java @@ -21,28 +21,35 @@ package org.elasticsearch.index.fielddata; import org.apache.lucene.util.BytesRef; +import java.io.IOException; + /** * A list of per-document binary values, sorted * according to {@link BytesRef#compareTo(BytesRef)}. * There might be dups however. */ +// TODO: Should it expose a count (current approach) or return null when there are no more values? public abstract class SortedBinaryDocValues { /** - * Positions to the specified document + * Advance this instance to the given document id + * @return true if there is a value for this document */ - public abstract void setDocument(int docId); + public abstract boolean advanceExact(int doc) throws IOException; - /** - * Return the number of values of the current document. + /** + * Retrieves the number of values for the current document. This must always + * be greater than zero. + * It is illegal to call this method after {@link #advanceExact(int)} + * returned {@code false}. */ - public abstract int count(); + public abstract int docValueCount(); - /** - * Retrieve the value for the current document at the specified index. - * An index ranges from {@code 0} to {@code count()-1}. + /** + * Iterates to the next value in the current document. Do not call this more than + * {@link #docValueCount} times for the document. * Note that the returned {@link BytesRef} might be reused across invocations. */ - public abstract BytesRef valueAt(int index); + public abstract BytesRef nextValue() throws IOException; } diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/SortedNumericDoubleValues.java b/core/src/main/java/org/elasticsearch/index/fielddata/SortedNumericDoubleValues.java index 23cdfa7c0d..d0d9fc4b4c 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/SortedNumericDoubleValues.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/SortedNumericDoubleValues.java @@ -21,6 +21,8 @@ package org.elasticsearch.index.fielddata; import org.apache.lucene.index.SortedNumericDocValues; +import java.io.IOException; + /** * Clone of {@link SortedNumericDocValues} for double values. */ @@ -30,21 +32,25 @@ public abstract class SortedNumericDoubleValues { * constructors, typically implicit.) */ protected SortedNumericDoubleValues() {} - /** - * Positions to the specified document - */ - public abstract void setDocument(int doc); + /** Advance the iterator to exactly {@code target} and return whether + * {@code target} has a value. + * {@code target} must be greater than or equal to the current + * doc ID and must be a valid doc ID, ie. ≥ 0 and + * < {@code maxDoc}.*/ + public abstract boolean advanceExact(int target) throws IOException; - /** - * Retrieve the value for the current document at the specified index. - * An index ranges from {@code 0} to {@code count()-1}. + /** + * Iterates to the next value in the current document. Do not call this more than + * {@link #docValueCount} times for the document. */ - public abstract double valueAt(int index); - - /** - * Retrieves the count of values for the current document. - * This may be zero if a document has no values. + public abstract double nextValue() throws IOException; + + /** + * Retrieves the number of values for the current document. This must always + * be greater than zero. + * It is illegal to call this method after {@link #advanceExact(int)} + * returned {@code false}. */ - public abstract int count(); + public abstract int docValueCount(); } diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/SortingBinaryDocValues.java b/core/src/main/java/org/elasticsearch/index/fielddata/SortingBinaryDocValues.java index c35d59fc69..99f0d47725 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/SortingBinaryDocValues.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/SortingBinaryDocValues.java @@ -33,6 +33,7 @@ import java.util.Arrays; */ public abstract class SortingBinaryDocValues extends SortedBinaryDocValues { + private int index; protected int count; protected BytesRefBuilder[] values; private final Sorter sorter; @@ -73,15 +74,17 @@ public abstract class SortingBinaryDocValues extends SortedBinaryDocValues { */ protected final void sort() { sorter.sort(0, count); + index = 0; } @Override - public final int count() { + public int docValueCount() { return count; } @Override - public final BytesRef valueAt(int index) { - return values[index].get(); + public final BytesRef nextValue() { + assert index < count; + return values[index++].get(); } } diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/SortingNumericDocValues.java b/core/src/main/java/org/elasticsearch/index/fielddata/SortingNumericDocValues.java index 7c1d1fff2a..0049faaf2b 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/SortingNumericDocValues.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/SortingNumericDocValues.java @@ -31,10 +31,12 @@ public abstract class SortingNumericDocValues extends SortedNumericDocValues { private int count; protected long[] values; + protected int valuesCursor; private final Sorter sorter; protected SortingNumericDocValues() { values = new long[1]; + valuesCursor = 0; sorter = new InPlaceMergeSorter() { @Override @@ -52,12 +54,13 @@ public abstract class SortingNumericDocValues extends SortedNumericDocValues { } /** - * Set the {@link #count()} and ensure that the {@link #values} array can + * Set the {@link #docValueCount()} and ensure that the {@link #values} array can * store at least that many entries. */ protected final void resize(int newSize) { count = newSize; values = ArrayUtil.grow(values, count); + valuesCursor = 0; } /** @@ -69,12 +72,12 @@ public abstract class SortingNumericDocValues extends SortedNumericDocValues { } @Override - public final int count() { + public final int docValueCount() { return count; } @Override - public final long valueAt(int index) { - return values[index]; + public final long nextValue() { + return values[valuesCursor++]; } } diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/SortingNumericDoubleValues.java b/core/src/main/java/org/elasticsearch/index/fielddata/SortingNumericDoubleValues.java index 674a86719e..7a39f64745 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/SortingNumericDoubleValues.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/SortingNumericDoubleValues.java @@ -29,11 +29,13 @@ import org.apache.lucene.util.Sorter; public abstract class SortingNumericDoubleValues extends SortedNumericDoubleValues { private int count; + private int valuesCursor; protected double[] values; private final Sorter sorter; protected SortingNumericDoubleValues() { values = new double[1]; + valuesCursor = 0; sorter = new InPlaceMergeSorter() { @Override @@ -51,29 +53,30 @@ public abstract class SortingNumericDoubleValues extends SortedNumericDoubleValu } /** - * Set the {@link #count()} and ensure that the {@link #values} array can + * Set the {@link #docValueCount()} and ensure that the {@link #values} array can * store at least that many entries. */ protected final void resize(int newSize) { count = newSize; values = ArrayUtil.grow(values, count); + valuesCursor = 0; } /** * Sort values that are stored between offsets <code>0</code> and - * {@link #count} of {@link #values}. + * {@link #docValueCount} of {@link #values}. */ protected final void sort() { sorter.sort(0, count); } @Override - public final int count() { + public final int docValueCount() { return count; } @Override - public final double valueAt(int index) { - return values[index]; + public final double nextValue() { + return values[valuesCursor++]; } } diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/BytesRefFieldComparatorSource.java b/core/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/BytesRefFieldComparatorSource.java index 48b6a1127a..c433836417 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/BytesRefFieldComparatorSource.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/BytesRefFieldComparatorSource.java @@ -21,15 +21,15 @@ package org.elasticsearch.index.fielddata.fieldcomparator; import org.apache.lucene.index.BinaryDocValues; import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.index.RandomAccessOrds; import org.apache.lucene.index.SortedDocValues; +import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.FieldComparator; import org.apache.lucene.search.Scorer; import org.apache.lucene.search.SortField; import org.apache.lucene.util.BitSet; -import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.index.fielddata.AbstractSortedDocValues; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.IndexOrdinalsFieldData; import org.elasticsearch.index.fielddata.SortedBinaryDocValues; @@ -89,7 +89,7 @@ public class BytesRefFieldComparatorSource extends IndexFieldData.XFieldComparat @Override protected SortedDocValues getSortedDocValues(LeafReaderContext context, String field) throws IOException { - final RandomAccessOrds values = ((IndexOrdinalsFieldData) indexFieldData).load(context).getOrdinalsValues(); + final SortedSetDocValues values = ((IndexOrdinalsFieldData) indexFieldData).load(context).getOrdinalsValues(); final SortedDocValues selectedValues; if (nested == null) { selectedValues = sortMode.select(values); @@ -113,8 +113,6 @@ public class BytesRefFieldComparatorSource extends IndexFieldData.XFieldComparat }; } - final BytesRef nullPlaceHolder = new BytesRef(); - final BytesRef nonNullMissingBytes = missingBytes == null ? nullPlaceHolder : missingBytes; return new FieldComparator.TermValComparator(numHits, null, sortMissingLast) { @Override @@ -122,26 +120,16 @@ public class BytesRefFieldComparatorSource extends IndexFieldData.XFieldComparat final SortedBinaryDocValues values = getValues(context); final BinaryDocValues selectedValues; if (nested == null) { - selectedValues = sortMode.select(values, nonNullMissingBytes); + selectedValues = sortMode.select(values, missingBytes); } else { final BitSet rootDocs = nested.rootDocs(context); final DocIdSetIterator innerDocs = nested.innerDocs(context); - selectedValues = sortMode.select(values, nonNullMissingBytes, rootDocs, innerDocs, context.reader().maxDoc()); + selectedValues = sortMode.select(values, missingBytes, rootDocs, innerDocs, context.reader().maxDoc()); } return selectedValues; } @Override - protected Bits getDocsWithField(LeafReaderContext context, String field) throws IOException { - return new Bits.MatchAllBits(context.reader().maxDoc()); - } - - @Override - protected boolean isNull(int doc, BytesRef term) { - return term == nullPlaceHolder; - } - - @Override public void setScorer(Scorer scorer) { BytesRefFieldComparatorSource.this.setScorer(scorer); } @@ -154,13 +142,14 @@ public class BytesRefFieldComparatorSource extends IndexFieldData.XFieldComparat * are replaced with the specified term */ // TODO: move this out if we need it for other reasons - static class ReplaceMissing extends SortedDocValues { + static class ReplaceMissing extends AbstractSortedDocValues { final SortedDocValues in; final int substituteOrd; final BytesRef substituteTerm; final boolean exists; + boolean hasValue; - ReplaceMissing(SortedDocValues in, BytesRef term) { + ReplaceMissing(SortedDocValues in, BytesRef term) throws IOException { this.in = in; this.substituteTerm = term; int sub = in.lookupTerm(term); @@ -174,11 +163,12 @@ public class BytesRefFieldComparatorSource extends IndexFieldData.XFieldComparat } @Override - public int getOrd(int docID) { - int ord = in.getOrd(docID); - if (ord < 0) { + public int ordValue() throws IOException { + if (hasValue == false) { return substituteOrd; - } else if (exists == false && ord >= substituteOrd) { + } + int ord = in.ordValue(); + if (exists == false && ord >= substituteOrd) { return ord + 1; } else { return ord; @@ -186,6 +176,17 @@ public class BytesRefFieldComparatorSource extends IndexFieldData.XFieldComparat } @Override + public boolean advanceExact(int target) throws IOException { + hasValue = in.advanceExact(target); + return true; + } + + @Override + public int docID() { + return in.docID(); + } + + @Override public int getValueCount() { if (exists) { return in.getValueCount(); @@ -195,7 +196,7 @@ public class BytesRefFieldComparatorSource extends IndexFieldData.XFieldComparat } @Override - public BytesRef lookupOrd(int ord) { + public BytesRef lookupOrd(int ord) throws IOException { if (ord == substituteOrd) { return substituteTerm; } else if (exists == false && ord > substituteOrd) { diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/GlobalOrdinalMapping.java b/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/GlobalOrdinalMapping.java index 293ddf430d..3eda112674 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/GlobalOrdinalMapping.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/GlobalOrdinalMapping.java @@ -20,22 +20,23 @@ package org.elasticsearch.index.fielddata.ordinals; import org.apache.lucene.index.MultiDocValues.OrdinalMap; -import org.apache.lucene.index.RandomAccessOrds; +import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.LongValues; -import org.elasticsearch.index.fielddata.AbstractRandomAccessOrds; + +import java.io.IOException; /** - * A {@link RandomAccessOrds} implementation that returns ordinals that are global. + * A {@link SortedSetDocValues} implementation that returns ordinals that are global. */ -public class GlobalOrdinalMapping extends AbstractRandomAccessOrds { +public class GlobalOrdinalMapping extends SortedSetDocValues { - private final RandomAccessOrds values; + private final SortedSetDocValues values; private final OrdinalMap ordinalMap; private final LongValues mapping; - private final RandomAccessOrds[] bytesValues; + private final SortedSetDocValues[] bytesValues; - GlobalOrdinalMapping(OrdinalMap ordinalMap, RandomAccessOrds[] bytesValues, int segmentIndex) { + GlobalOrdinalMapping(OrdinalMap ordinalMap, SortedSetDocValues[] bytesValues, int segmentIndex) { super(); this.values = bytesValues[segmentIndex]; this.bytesValues = bytesValues; @@ -53,25 +54,45 @@ public class GlobalOrdinalMapping extends AbstractRandomAccessOrds { } @Override - public long ordAt(int index) { - return getGlobalOrd(values.ordAt(index)); - } - - @Override - public void doSetDocument(int docId) { - values.setDocument(docId); + public boolean advanceExact(int target) throws IOException { + return values.advanceExact(target); } @Override - public int cardinality() { - return values.cardinality(); + public long nextOrd() throws IOException { + long segmentOrd = values.nextOrd(); + if (segmentOrd == SortedSetDocValues.NO_MORE_ORDS) { + return SortedSetDocValues.NO_MORE_ORDS; + } else { + return getGlobalOrd(segmentOrd); + } } @Override - public BytesRef lookupOrd(long globalOrd) { + public BytesRef lookupOrd(long globalOrd) throws IOException { final long segmentOrd = ordinalMap.getFirstSegmentOrd(globalOrd); int readerIndex = ordinalMap.getFirstSegmentNumber(globalOrd); return bytesValues[readerIndex].lookupOrd(segmentOrd); } + @Override + public int docID() { + return values.docID(); + } + + @Override + public int nextDoc() throws IOException { + return values.nextDoc(); + } + + @Override + public int advance(int target) throws IOException { + return values.advance(target); + } + + @Override + public long cost() { + return values.cost(); + } + } diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/GlobalOrdinalsBuilder.java b/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/GlobalOrdinalsBuilder.java index 7b3b064689..49140968ca 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/GlobalOrdinalsBuilder.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/GlobalOrdinalsBuilder.java @@ -23,7 +23,7 @@ import org.apache.logging.log4j.Logger; import org.apache.lucene.index.DocValues; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.MultiDocValues.OrdinalMap; -import org.apache.lucene.index.RandomAccessOrds; +import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.util.Accountable; import org.apache.lucene.util.packed.PackedInts; import org.elasticsearch.common.breaker.CircuitBreaker; @@ -52,12 +52,12 @@ public enum GlobalOrdinalsBuilder { */ public static IndexOrdinalsFieldData build(final IndexReader indexReader, IndexOrdinalsFieldData indexFieldData, IndexSettings indexSettings, CircuitBreakerService breakerService, Logger logger, - Function<RandomAccessOrds, ScriptDocValues<?>> scriptFunction) throws IOException { + Function<SortedSetDocValues, ScriptDocValues<?>> scriptFunction) throws IOException { assert indexReader.leaves().size() > 1; long startTimeNS = System.nanoTime(); final AtomicOrdinalsFieldData[] atomicFD = new AtomicOrdinalsFieldData[indexReader.leaves().size()]; - final RandomAccessOrds[] subs = new RandomAccessOrds[indexReader.leaves().size()]; + final SortedSetDocValues[] subs = new SortedSetDocValues[indexReader.leaves().size()]; for (int i = 0; i < indexReader.leaves().size(); ++i) { atomicFD[i] = indexFieldData.load(indexReader.leaves().get(i)); subs[i] = atomicFD[i].getOrdinalsValues(); @@ -83,11 +83,11 @@ public enum GlobalOrdinalsBuilder { assert indexReader.leaves().size() > 1; final AtomicOrdinalsFieldData[] atomicFD = new AtomicOrdinalsFieldData[indexReader.leaves().size()]; - final RandomAccessOrds[] subs = new RandomAccessOrds[indexReader.leaves().size()]; + final SortedSetDocValues[] subs = new SortedSetDocValues[indexReader.leaves().size()]; for (int i = 0; i < indexReader.leaves().size(); ++i) { atomicFD[i] = new AbstractAtomicOrdinalsFieldData(AbstractAtomicOrdinalsFieldData.DEFAULT_SCRIPT_FUNCTION) { @Override - public RandomAccessOrds getOrdinalsValues() { + public SortedSetDocValues getOrdinalsValues() { return DocValues.emptySortedSet(); } diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/InternalGlobalOrdinalsIndexFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/InternalGlobalOrdinalsIndexFieldData.java index 284e22b21d..23ecc06fed 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/InternalGlobalOrdinalsIndexFieldData.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/InternalGlobalOrdinalsIndexFieldData.java @@ -20,7 +20,7 @@ package org.elasticsearch.index.fielddata.ordinals; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.MultiDocValues.OrdinalMap; -import org.apache.lucene.index.RandomAccessOrds; +import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.util.Accountable; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.fielddata.AtomicOrdinalsFieldData; @@ -36,10 +36,10 @@ import java.util.function.Function; final class InternalGlobalOrdinalsIndexFieldData extends GlobalOrdinalsIndexFieldData { private final Atomic[] atomicReaders; - private final Function<RandomAccessOrds, ScriptDocValues<?>> scriptFunction; + private final Function<SortedSetDocValues, ScriptDocValues<?>> scriptFunction; InternalGlobalOrdinalsIndexFieldData(IndexSettings indexSettings, String fieldName, AtomicOrdinalsFieldData[] segmentAfd, - OrdinalMap ordinalMap, long memorySizeInBytes, Function<RandomAccessOrds, ScriptDocValues<?>> scriptFunction) { + OrdinalMap ordinalMap, long memorySizeInBytes, Function<SortedSetDocValues, ScriptDocValues<?>> scriptFunction) { super(indexSettings, fieldName, memorySizeInBytes); this.atomicReaders = new Atomic[segmentAfd.length]; for (int i = 0; i < segmentAfd.length; i++) { @@ -67,13 +67,13 @@ final class InternalGlobalOrdinalsIndexFieldData extends GlobalOrdinalsIndexFiel } @Override - public RandomAccessOrds getOrdinalsValues() { - final RandomAccessOrds values = afd.getOrdinalsValues(); + public SortedSetDocValues getOrdinalsValues() { + final SortedSetDocValues values = afd.getOrdinalsValues(); if (values.getValueCount() == ordinalMap.getValueCount()) { // segment ordinals match global ordinals return values; } - final RandomAccessOrds[] bytesValues = new RandomAccessOrds[atomicReaders.length]; + final SortedSetDocValues[] bytesValues = new SortedSetDocValues[atomicReaders.length]; for (int i = 0; i < bytesValues.length; i++) { bytesValues[i] = atomicReaders[i].afd.getOrdinalsValues(); } diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/MultiOrdinals.java b/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/MultiOrdinals.java index d85073a068..86e2787658 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/MultiOrdinals.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/MultiOrdinals.java @@ -20,16 +20,17 @@ package org.elasticsearch.index.fielddata.ordinals; import org.apache.lucene.index.DocValues; -import org.apache.lucene.index.RandomAccessOrds; -import org.apache.lucene.index.SortedDocValues; +import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.util.Accountable; import org.apache.lucene.util.Accountables; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.LongsRef; import org.apache.lucene.util.packed.PackedInts; import org.apache.lucene.util.packed.PackedLongValues; -import org.elasticsearch.index.fielddata.AbstractRandomAccessOrds; +import org.elasticsearch.index.fielddata.AbstractSortedDocValues; +import org.elasticsearch.index.fielddata.AbstractSortedSetDocValues; +import java.io.IOException; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -100,21 +101,25 @@ public class MultiOrdinals extends Ordinals { } @Override - public RandomAccessOrds ordinals(ValuesHolder values) { + public SortedSetDocValues ordinals(ValuesHolder values) { if (multiValued) { return new MultiDocs(this, values); } else { - return (RandomAccessOrds) DocValues.singleton(new SingleDocs(this, values)); + return (SortedSetDocValues) DocValues.singleton(new SingleDocs(this, values)); } } - private static class SingleDocs extends SortedDocValues { + private static class SingleDocs extends AbstractSortedDocValues { private final int valueCount; private final PackedLongValues endOffsets; private final PackedLongValues ords; private final ValuesHolder values; + private int currentDoc = -1; + private long currentStartOffset; + private long currentEndOffset; + SingleDocs(MultiOrdinals ordinals, ValuesHolder values) { this.valueCount = (int) ordinals.valueCount; this.endOffsets = ordinals.endOffsets; @@ -123,10 +128,21 @@ public class MultiOrdinals extends Ordinals { } @Override - public int getOrd(int docId) { - final long startOffset = docId != 0 ? endOffsets.get(docId - 1) : 0; - final long endOffset = endOffsets.get(docId); - return startOffset == endOffset ? -1 : (int) ords.get(startOffset); + public int ordValue() { + return (int) ords.get(currentStartOffset); + } + + @Override + public boolean advanceExact(int docId) throws IOException { + currentDoc = docId; + currentStartOffset = docId != 0 ? endOffsets.get(docId - 1) : 0; + currentEndOffset = endOffsets.get(docId); + return currentStartOffset != currentEndOffset; + } + + @Override + public int docID() { + return currentDoc; } @Override @@ -141,15 +157,16 @@ public class MultiOrdinals extends Ordinals { } - private static class MultiDocs extends AbstractRandomAccessOrds { + private static class MultiDocs extends AbstractSortedSetDocValues { private final long valueCount; private final PackedLongValues endOffsets; private final PackedLongValues ords; - private long offset; - private int cardinality; private final ValuesHolder values; + private long currentOffset; + private long currentEndOffset; + MultiDocs(MultiOrdinals ordinals, ValuesHolder values) { this.valueCount = ordinals.valueCount; this.endOffsets = ordinals.endOffsets; @@ -163,21 +180,19 @@ public class MultiOrdinals extends Ordinals { } @Override - public void doSetDocument(int docId) { - final long startOffset = docId != 0 ? endOffsets.get(docId - 1) : 0; - final long endOffset = endOffsets.get(docId); - offset = startOffset; - cardinality = (int) (endOffset - startOffset); - } - - @Override - public int cardinality() { - return cardinality; + public boolean advanceExact(int docId) throws IOException { + currentOffset = docId != 0 ? endOffsets.get(docId - 1) : 0; + currentEndOffset = endOffsets.get(docId); + return currentOffset != currentEndOffset; } @Override - public long ordAt(int index) { - return ords.get(offset + index); + public long nextOrd() throws IOException { + if (currentOffset == currentEndOffset) { + return SortedSetDocValues.NO_MORE_ORDS; + } else { + return ords.get(currentOffset++); + } } @Override diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/Ordinals.java b/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/Ordinals.java index 77c0fd6bf5..7e59b70823 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/Ordinals.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/Ordinals.java @@ -19,7 +19,7 @@ package org.elasticsearch.index.fielddata.ordinals; -import org.apache.lucene.index.RandomAccessOrds; +import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.util.Accountable; import org.apache.lucene.util.BytesRef; @@ -41,9 +41,9 @@ public abstract class Ordinals implements Accountable { @Override public abstract long ramBytesUsed(); - public abstract RandomAccessOrds ordinals(ValuesHolder values); + public abstract SortedSetDocValues ordinals(ValuesHolder values); - public final RandomAccessOrds ordinals() { + public final SortedSetDocValues ordinals() { return ordinals(NO_VALUES); } diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/OrdinalsBuilder.java b/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/OrdinalsBuilder.java index 967d07174b..b2e1b9e310 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/OrdinalsBuilder.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/OrdinalsBuilder.java @@ -19,17 +19,14 @@ package org.elasticsearch.index.fielddata.ordinals; -import org.apache.lucene.index.FilteredTermsEnum; import org.apache.lucene.index.PostingsEnum; import org.apache.lucene.index.TermsEnum; import org.apache.lucene.search.DocIdSetIterator; -import org.apache.lucene.spatial.geopoint.document.GeoPointField; import org.apache.lucene.util.ArrayUtil; import org.apache.lucene.util.BitSet; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BytesRefIterator; import org.apache.lucene.util.FixedBitSet; -import org.apache.lucene.util.LegacyNumericUtils; import org.apache.lucene.util.LongsRef; import org.apache.lucene.util.packed.GrowableWriter; import org.apache.lucene.util.packed.PackedInts; @@ -414,23 +411,6 @@ public final class OrdinalsBuilder implements Closeable { } } - /** - * A {@link TermsEnum} that iterates only highest resolution geo prefix coded terms. - * - * @see #buildFromTerms(TermsEnum) - */ - public static TermsEnum wrapGeoPointTerms(TermsEnum termsEnum) { - return new FilteredTermsEnum(termsEnum, false) { - @Override - protected AcceptStatus accept(BytesRef term) throws IOException { - // accept only the max resolution terms - // todo is this necessary? - return GeoPointField.getPrefixCodedShift(term) == GeoPointField.PRECISION_STEP * 4 ? - AcceptStatus.YES : AcceptStatus.END; - } - }; - } - /** * Returns the maximum document ID this builder can associate with an ordinal @@ -440,50 +420,11 @@ public final class OrdinalsBuilder implements Closeable { } /** - * A {@link TermsEnum} that iterates only full precision prefix coded 64 bit values. - * - * @see #buildFromTerms(TermsEnum) - */ - public static TermsEnum wrapNumeric64Bit(TermsEnum termsEnum) { - return new FilteredTermsEnum(termsEnum, false) { - @Override - protected AcceptStatus accept(BytesRef term) throws IOException { - // we stop accepting terms once we moved across the prefix codec terms - redundant values! - return LegacyNumericUtils.getPrefixCodedLongShift(term) == 0 ? AcceptStatus.YES : AcceptStatus.END; - } - }; - } - - /** - * A {@link TermsEnum} that iterates only full precision prefix coded 32 bit values. - * - * @see #buildFromTerms(TermsEnum) - */ - public static TermsEnum wrapNumeric32Bit(TermsEnum termsEnum) { - return new FilteredTermsEnum(termsEnum, false) { - - @Override - protected AcceptStatus accept(BytesRef term) throws IOException { - // we stop accepting terms once we moved across the prefix codec terms - redundant values! - return LegacyNumericUtils.getPrefixCodedIntShift(term) == 0 ? AcceptStatus.YES : AcceptStatus.END; - } - }; - } - - /** * This method iterates all terms in the given {@link TermsEnum} and * associates each terms ordinal with the terms documents. The caller must * exhaust the returned {@link BytesRefIterator} which returns all values * where the first returned value is associated with the ordinal <tt>1</tt> * etc. - * <p> - * If the {@link TermsEnum} contains prefix coded numerical values the terms - * enum should be wrapped with either {@link #wrapNumeric32Bit(TermsEnum)} - * or {@link #wrapNumeric64Bit(TermsEnum)} depending on its precision. If - * the {@link TermsEnum} is not wrapped the returned - * {@link BytesRefIterator} will contain partial precision terms rather than - * only full-precision terms. - * </p> */ public BytesRefIterator buildFromTerms(final TermsEnum termsEnum) throws IOException { return new BytesRefIterator() { diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/SinglePackedOrdinals.java b/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/SinglePackedOrdinals.java index f7db724c35..27f0aadee8 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/SinglePackedOrdinals.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/SinglePackedOrdinals.java @@ -20,14 +20,15 @@ package org.elasticsearch.index.fielddata.ordinals; import org.apache.lucene.index.DocValues; -import org.apache.lucene.index.RandomAccessOrds; -import org.apache.lucene.index.SortedDocValues; +import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.util.Accountable; import org.apache.lucene.util.Accountables; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.RamUsageEstimator; import org.apache.lucene.util.packed.PackedInts; +import org.elasticsearch.index.fielddata.AbstractSortedDocValues; +import java.io.IOException; import java.util.Collection; import java.util.Collections; @@ -57,16 +58,19 @@ public class SinglePackedOrdinals extends Ordinals { } @Override - public RandomAccessOrds ordinals(ValuesHolder values) { - return (RandomAccessOrds) DocValues.singleton(new Docs(this, values)); + public SortedSetDocValues ordinals(ValuesHolder values) { + return (SortedSetDocValues) DocValues.singleton(new Docs(this, values)); } - private static class Docs extends SortedDocValues { + private static class Docs extends AbstractSortedDocValues { private final int maxOrd; private final PackedInts.Reader reader; private final ValuesHolder values; + private int currentDoc = -1; + private int currentOrd; + Docs(SinglePackedOrdinals parent, ValuesHolder values) { this.maxOrd = parent.valueCount; this.reader = parent.reader; @@ -84,8 +88,20 @@ public class SinglePackedOrdinals extends Ordinals { } @Override - public int getOrd(int docID) { - return (int) (reader.get(docID) - 1); + public int ordValue() { + return currentOrd; + } + + @Override + public boolean advanceExact(int docID) throws IOException { + currentDoc = docID; + currentOrd = (int) (reader.get(docID) - 1); + return currentOrd != -1; + } + + @Override + public int docID() { + return currentDoc; } } } diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractAtomicGeoPointFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractAtomicGeoPointFieldData.java index 9660d9f868..5d6575e437 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractAtomicGeoPointFieldData.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractAtomicGeoPointFieldData.java @@ -59,7 +59,7 @@ public abstract class AbstractAtomicGeoPointFieldData implements AtomicGeoPointF @Override public MultiGeoPointValues getGeoPointValues() { - return FieldData.emptyMultiGeoPoints(maxDoc); + return FieldData.emptyMultiGeoPoints(); } }; } diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractAtomicOrdinalsFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractAtomicOrdinalsFieldData.java index 7959bf3578..9130e19fb2 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractAtomicOrdinalsFieldData.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractAtomicOrdinalsFieldData.java @@ -20,7 +20,7 @@ package org.elasticsearch.index.fielddata.plain; import org.apache.lucene.index.DocValues; -import org.apache.lucene.index.RandomAccessOrds; +import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.util.Accountable; import org.elasticsearch.index.fielddata.AtomicOrdinalsFieldData; import org.elasticsearch.index.fielddata.FieldData; @@ -34,13 +34,13 @@ import java.util.function.Function; public abstract class AbstractAtomicOrdinalsFieldData implements AtomicOrdinalsFieldData { - public static final Function<RandomAccessOrds, ScriptDocValues<?>> DEFAULT_SCRIPT_FUNCTION = - ((Function<RandomAccessOrds, SortedBinaryDocValues>) FieldData::toString) + public static final Function<SortedSetDocValues, ScriptDocValues<?>> DEFAULT_SCRIPT_FUNCTION = + ((Function<SortedSetDocValues, SortedBinaryDocValues>) FieldData::toString) .andThen(ScriptDocValues.Strings::new); - private final Function<RandomAccessOrds, ScriptDocValues<?>> scriptFunction; + private final Function<SortedSetDocValues, ScriptDocValues<?>> scriptFunction; - protected AbstractAtomicOrdinalsFieldData(Function<RandomAccessOrds, ScriptDocValues<?>> scriptFunction) { + protected AbstractAtomicOrdinalsFieldData(Function<SortedSetDocValues, ScriptDocValues<?>> scriptFunction) { this.scriptFunction = scriptFunction; } @@ -72,7 +72,7 @@ public abstract class AbstractAtomicOrdinalsFieldData implements AtomicOrdinalsF } @Override - public RandomAccessOrds getOrdinalsValues() { + public SortedSetDocValues getOrdinalsValues() { return DocValues.emptySortedSet(); } }; diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractAtomicParentChildFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractAtomicParentChildFieldData.java index 7c03e1a794..2df5aa6bb6 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractAtomicParentChildFieldData.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractAtomicParentChildFieldData.java @@ -28,6 +28,7 @@ import org.elasticsearch.index.fielddata.AtomicParentChildFieldData; import org.elasticsearch.index.fielddata.ScriptDocValues; import org.elasticsearch.index.fielddata.SortedBinaryDocValues; +import java.io.IOException; import java.util.Collection; import java.util.Collections; import java.util.Set; @@ -46,16 +47,28 @@ abstract class AbstractAtomicParentChildFieldData implements AtomicParentChildFi public final SortedBinaryDocValues getBytesValues() { return new SortedBinaryDocValues() { + private final SortedDocValues[] perTypeValues; private final BytesRef[] terms = new BytesRef[2]; private int count; + private int termsCursor; + + { + Set<String> types = types(); + perTypeValues = new SortedDocValues[types.size()]; + int i = 0; + for (String type : types) { + perTypeValues[i++] = getOrdinalsValues(type); + } + } @Override - public void setDocument(int docId) { + public boolean advanceExact(int docId) throws IOException { count = 0; - for (String type : types()) { - final SortedDocValues values = getOrdinalsValues(type); - final int ord = values.getOrd(docId); - if (ord >= 0) { + termsCursor = 0; + + for (SortedDocValues values : perTypeValues) { + if (values.advanceExact(docId)) { + final int ord = values.ordValue(); terms[count++] = values.lookupOrd(ord); } } @@ -69,16 +82,17 @@ abstract class AbstractAtomicParentChildFieldData implements AtomicParentChildFi count = 1; } } + return count != 0; } @Override - public int count() { + public int docValueCount() { return count; } @Override - public BytesRef valueAt(int index) { - return terms[index]; + public BytesRef nextValue() throws IOException { + return terms[termsCursor++]; } }; } diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractGeoPointDVIndexFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractGeoPointDVIndexFieldData.java deleted file mode 100644 index 8db38e59ce..0000000000 --- a/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractGeoPointDVIndexFieldData.java +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.index.fielddata.plain; - -import org.apache.lucene.index.DocValues; -import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.search.SortField; -import org.elasticsearch.common.Nullable; -import org.elasticsearch.index.Index; -import org.elasticsearch.index.IndexSettings; -import org.elasticsearch.index.fielddata.AtomicGeoPointFieldData; -import org.elasticsearch.index.fielddata.IndexFieldData; -import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested; -import org.elasticsearch.index.fielddata.IndexFieldDataCache; -import org.elasticsearch.index.fielddata.IndexGeoPointFieldData; -import org.elasticsearch.index.mapper.MappedFieldType; -import org.elasticsearch.index.mapper.MapperService; -import org.elasticsearch.indices.breaker.CircuitBreakerService; -import org.elasticsearch.search.MultiValueMode; - -import java.io.IOException; - -public abstract class AbstractGeoPointDVIndexFieldData extends DocValuesIndexFieldData implements IndexGeoPointFieldData { - - AbstractGeoPointDVIndexFieldData(Index index, String fieldName) { - super(index, fieldName); - } - - @Override - public SortField sortField(@Nullable Object missingValue, MultiValueMode sortMode, Nested nested, boolean reverse) { - throw new IllegalArgumentException("can't sort on geo_point field without using specific sorting feature, like geo_distance"); - } - - /** - * Lucene 5.4 GeoPointFieldType - */ - public static class GeoPointDVIndexFieldData extends AbstractGeoPointDVIndexFieldData { - - public GeoPointDVIndexFieldData(Index index, String fieldName) { - super(index, fieldName); - } - - @Override - public AtomicGeoPointFieldData load(LeafReaderContext context) { - try { - return new GeoPointDVAtomicFieldData(DocValues.getSortedNumeric(context.reader(), fieldName)); - } catch (IOException e) { - throw new IllegalStateException("Cannot load doc values", e); - } - } - - @Override - public AtomicGeoPointFieldData loadDirect(LeafReaderContext context) throws Exception { - return load(context); - } - } - - public static class Builder implements IndexFieldData.Builder { - @Override - public IndexFieldData<?> build(IndexSettings indexSettings, MappedFieldType fieldType, IndexFieldDataCache cache, - CircuitBreakerService breakerService, MapperService mapperService) { - // Ignore breaker - return new GeoPointDVIndexFieldData(indexSettings.getIndex(), fieldType.name()); - } - } -} diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractIndexGeoPointFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractIndexGeoPointFieldData.java deleted file mode 100644 index bdf1bbac33..0000000000 --- a/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractIndexGeoPointFieldData.java +++ /dev/null @@ -1,117 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.index.fielddata.plain; - -import org.apache.lucene.search.SortField; -import org.apache.lucene.spatial.geopoint.document.GeoPointField; -import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.BytesRefIterator; -import org.apache.lucene.util.CharsRefBuilder; -import org.apache.lucene.util.LegacyNumericUtils; -import org.elasticsearch.common.Nullable; -import org.elasticsearch.common.geo.GeoPoint; -import org.elasticsearch.index.IndexSettings; -import org.elasticsearch.index.fielddata.AtomicGeoPointFieldData; -import org.elasticsearch.index.fielddata.IndexFieldData; -import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested; -import org.elasticsearch.index.fielddata.IndexFieldDataCache; -import org.elasticsearch.index.fielddata.IndexGeoPointFieldData; -import org.elasticsearch.search.MultiValueMode; - -import java.io.IOException; - -abstract class AbstractIndexGeoPointFieldData extends AbstractIndexFieldData<AtomicGeoPointFieldData> implements IndexGeoPointFieldData { - protected abstract static class BaseGeoPointTermsEnum { - protected final BytesRefIterator termsEnum; - - protected BaseGeoPointTermsEnum(BytesRefIterator termsEnum) { - this.termsEnum = termsEnum; - } - } - - protected static class GeoPointTermsEnum extends BaseGeoPointTermsEnum { - private final GeoPointField.TermEncoding termEncoding; - protected GeoPointTermsEnum(BytesRefIterator termsEnum, GeoPointField.TermEncoding termEncoding) { - super(termsEnum); - this.termEncoding = termEncoding; - } - - public Long next() throws IOException { - final BytesRef term = termsEnum.next(); - if (term == null) { - return null; - } - if (termEncoding == GeoPointField.TermEncoding.PREFIX) { - return GeoPointField.prefixCodedToGeoCoded(term); - } else if (termEncoding == GeoPointField.TermEncoding.NUMERIC) { - return LegacyNumericUtils.prefixCodedToLong(term); - } - throw new IllegalArgumentException("GeoPoint.TermEncoding should be one of: " + GeoPointField.TermEncoding.PREFIX - + " or " + GeoPointField.TermEncoding.NUMERIC + " found: " + termEncoding); - } - } - - protected static class GeoPointTermsEnumLegacy extends BaseGeoPointTermsEnum { - private final GeoPoint next; - private final CharsRefBuilder spare; - - protected GeoPointTermsEnumLegacy(BytesRefIterator termsEnum) { - super(termsEnum); - next = new GeoPoint(); - spare = new CharsRefBuilder(); - } - - public GeoPoint next() throws IOException { - final BytesRef term = termsEnum.next(); - if (term == null) { - return null; - } - spare.copyUTF8Bytes(term); - int commaIndex = -1; - for (int i = 0; i < spare.length(); i++) { - if (spare.charAt(i) == ',') { // saves a string creation - commaIndex = i; - break; - } - } - if (commaIndex == -1) { - assert false; - return next.reset(0, 0); - } - final double lat = Double.parseDouble(new String(spare.chars(), 0, commaIndex)); - final double lon = Double.parseDouble(new String(spare.chars(), commaIndex + 1, spare.length() - (commaIndex + 1))); - return next.reset(lat, lon); - } - } - - AbstractIndexGeoPointFieldData(IndexSettings indexSettings, String fieldName, IndexFieldDataCache cache) { - super(indexSettings, fieldName, cache); - } - - @Override - public SortField sortField(@Nullable Object missingValue, MultiValueMode sortMode, Nested nested, boolean reverse) { - throw new IllegalArgumentException("can't sort on geo_point field without using specific sorting feature, like geo_distance"); - } - - @Override - protected AtomicGeoPointFieldData empty(int maxDoc) { - return AbstractAtomicGeoPointFieldData.empty(maxDoc); - } -} diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractLatLonPointDVIndexFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractLatLonPointDVIndexFieldData.java index 3b4ac58e0e..6c92d57119 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractLatLonPointDVIndexFieldData.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractLatLonPointDVIndexFieldData.java @@ -19,13 +19,11 @@ package org.elasticsearch.index.fielddata.plain; import org.apache.lucene.document.LatLonDocValuesField; -import org.apache.lucene.index.DocValues; import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.search.SortField; -import org.elasticsearch.ElasticsearchException; import org.elasticsearch.common.Nullable; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexSettings; @@ -38,8 +36,6 @@ import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.indices.breaker.CircuitBreakerService; import org.elasticsearch.search.MultiValueMode; -import java.io.IOException; - public abstract class AbstractLatLonPointDVIndexFieldData extends DocValuesIndexFieldData implements IndexGeoPointFieldData { AbstractLatLonPointDVIndexFieldData(Index index, String fieldName) { @@ -58,16 +54,12 @@ public abstract class AbstractLatLonPointDVIndexFieldData extends DocValuesIndex @Override public AtomicGeoPointFieldData load(LeafReaderContext context) { - try { - LeafReader reader = context.reader(); - FieldInfo info = reader.getFieldInfos().fieldInfo(fieldName); - if (info != null) { - checkCompatible(info); - } - return new LatLonPointDVAtomicFieldData(DocValues.getSortedNumeric(reader, fieldName)); - } catch (IOException e) { - throw new IllegalStateException("Cannot load doc values", e); + LeafReader reader = context.reader(); + FieldInfo info = reader.getFieldInfos().fieldInfo(fieldName); + if (info != null) { + checkCompatible(info); } + return new LatLonPointDVAtomicFieldData(reader, fieldName); } @Override diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/plain/AtomicDoubleFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/plain/AtomicDoubleFieldData.java index 821fcc1bf2..82ae0bb5bf 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/plain/AtomicDoubleFieldData.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/plain/AtomicDoubleFieldData.java @@ -67,7 +67,7 @@ abstract class AtomicDoubleFieldData implements AtomicNumericFieldData { @Override public SortedNumericDoubleValues getDoubleValues() { - return FieldData.emptySortedNumericDoubles(maxDoc); + return FieldData.emptySortedNumericDoubles(); } @Override diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/plain/BinaryDVAtomicFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/plain/BinaryDVAtomicFieldData.java index c82ac69f28..014220304d 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/plain/BinaryDVAtomicFieldData.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/plain/BinaryDVAtomicFieldData.java @@ -23,7 +23,6 @@ import org.apache.lucene.index.BinaryDocValues; import org.apache.lucene.index.DocValues; import org.apache.lucene.index.LeafReader; import org.apache.lucene.util.Accountable; -import org.apache.lucene.util.Bits; import org.elasticsearch.index.fielddata.AtomicFieldData; import org.elasticsearch.index.fielddata.FieldData; import org.elasticsearch.index.fielddata.ScriptDocValues; @@ -49,8 +48,7 @@ public class BinaryDVAtomicFieldData implements AtomicFieldData { public SortedBinaryDocValues getBytesValues() { try { final BinaryDocValues values = DocValues.getBinary(reader, field); - final Bits docsWithField = DocValues.getDocsWithField(reader, field); - return FieldData.singleton(values, docsWithField); + return FieldData.singleton(values); } catch (IOException e) { throw new IllegalStateException("Cannot load doc values", e); } diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/plain/BytesBinaryDVAtomicFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/plain/BytesBinaryDVAtomicFieldData.java index 8d43241ba7..aa2775046f 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/plain/BytesBinaryDVAtomicFieldData.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/plain/BytesBinaryDVAtomicFieldData.java @@ -22,15 +22,12 @@ package org.elasticsearch.index.fielddata.plain; import org.apache.lucene.index.BinaryDocValues; import org.apache.lucene.store.ByteArrayDataInput; import org.apache.lucene.util.Accountable; -import org.apache.lucene.util.ArrayUtil; import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.BytesRefBuilder; -import org.apache.lucene.util.RamUsageEstimator; import org.elasticsearch.index.fielddata.AtomicFieldData; import org.elasticsearch.index.fielddata.ScriptDocValues; import org.elasticsearch.index.fielddata.SortedBinaryDocValues; -import java.util.Arrays; +import java.io.IOException; import java.util.Collection; import java.util.Collections; @@ -58,42 +55,34 @@ final class BytesBinaryDVAtomicFieldData implements AtomicFieldData { return new SortedBinaryDocValues() { int count; - BytesRefBuilder[] refs = new BytesRefBuilder[0]; final ByteArrayDataInput in = new ByteArrayDataInput(); + final BytesRef scratch = new BytesRef(); @Override - public void setDocument(int docId) { - final BytesRef bytes = values.get(docId); - in.reset(bytes.bytes, bytes.offset, bytes.length); - if (bytes.length == 0) { - count = 0; - } else { + public boolean advanceExact(int doc) throws IOException { + if (values.advanceExact(doc)) { + final BytesRef bytes = values.binaryValue(); + assert bytes.length > 0; + in.reset(bytes.bytes, bytes.offset, bytes.length); count = in.readVInt(); - if (count > refs.length) { - final int previousLength = refs.length; - refs = Arrays.copyOf(refs, ArrayUtil.oversize(count, RamUsageEstimator.NUM_BYTES_OBJECT_REF)); - for (int i = previousLength; i < refs.length; ++i) { - refs[i] = new BytesRefBuilder(); - } - } - for (int i = 0; i < count; ++i) { - final int length = in.readVInt(); - final BytesRefBuilder scratch = refs[i]; - scratch.grow(length); - in.readBytes(scratch.bytes(), 0, length); - scratch.setLength(length); - } + scratch.bytes = bytes.bytes; + return true; + } else { + return false; } } @Override - public int count() { + public int docValueCount() { return count; } @Override - public BytesRef valueAt(int index) { - return refs[index].get(); + public BytesRef nextValue() throws IOException { + scratch.length = in.readVInt(); + scratch.offset = in.getPosition(); + in.setPosition(scratch.offset + scratch.length); + return scratch; } }; diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/plain/DocValuesIndexFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/plain/DocValuesIndexFieldData.java index c77ceb5745..4a066328bc 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/plain/DocValuesIndexFieldData.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/plain/DocValuesIndexFieldData.java @@ -21,7 +21,7 @@ package org.elasticsearch.index.fielddata.plain; import org.apache.logging.log4j.Logger; import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.RandomAccessOrds; +import org.apache.lucene.index.SortedSetDocValues; import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexSettings; @@ -75,14 +75,14 @@ public abstract class DocValuesIndexFieldData { private static final Set<String> BINARY_INDEX_FIELD_NAMES = unmodifiableSet(newHashSet(UidFieldMapper.NAME, IdFieldMapper.NAME)); private NumericType numericType; - private Function<RandomAccessOrds, ScriptDocValues<?>> scriptFunction = AbstractAtomicOrdinalsFieldData.DEFAULT_SCRIPT_FUNCTION; + private Function<SortedSetDocValues, ScriptDocValues<?>> scriptFunction = AbstractAtomicOrdinalsFieldData.DEFAULT_SCRIPT_FUNCTION; public Builder numericType(NumericType type) { this.numericType = type; return this; } - public Builder scriptFunction(Function<RandomAccessOrds, ScriptDocValues<?>> scriptFunction) { + public Builder scriptFunction(Function<SortedSetDocValues, ScriptDocValues<?>> scriptFunction) { this.scriptFunction = scriptFunction; return this; } diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/plain/GeoPointDVAtomicFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/plain/GeoPointDVAtomicFieldData.java deleted file mode 100644 index a71ea3a552..0000000000 --- a/core/src/main/java/org/elasticsearch/index/fielddata/plain/GeoPointDVAtomicFieldData.java +++ /dev/null @@ -1,90 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.index.fielddata.plain; - -import org.apache.lucene.index.SortedNumericDocValues; -import org.apache.lucene.util.Accountable; -import org.apache.lucene.util.ArrayUtil; -import org.apache.lucene.util.RamUsageEstimator; -import org.elasticsearch.common.geo.GeoPoint; -import org.elasticsearch.index.fielddata.MultiGeoPointValues; - -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; - -final class GeoPointDVAtomicFieldData extends AbstractAtomicGeoPointFieldData { - - private final SortedNumericDocValues values; - - GeoPointDVAtomicFieldData(SortedNumericDocValues values) { - super(); - this.values = values; - } - - @Override - public long ramBytesUsed() { - return 0; // not exposed by Lucene - } - - @Override - public Collection<Accountable> getChildResources() { - return Collections.emptyList(); - } - - @Override - public void close() { - // no-op - } - - @Override - public MultiGeoPointValues getGeoPointValues() { - return new MultiGeoPointValues() { - GeoPoint[] points = new GeoPoint[0]; - private int count = 0; - - @Override - public void setDocument(int docId) { - values.setDocument(docId); - count = values.count(); - if (count > points.length) { - final int previousLength = points.length; - points = Arrays.copyOf(points, ArrayUtil.oversize(count, RamUsageEstimator.NUM_BYTES_OBJECT_REF)); - for (int i = previousLength; i < points.length; ++i) { - points[i] = new GeoPoint(Double.NaN, Double.NaN); - } - } - for (int i=0; i<count; ++i) { - points[i].resetFromIndexHash(values.valueAt(i)); - } - } - - @Override - public int count() { - return count; - } - - @Override - public GeoPoint valueAt(int index) { - return points[index]; - } - }; - } -}
\ No newline at end of file diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/plain/IndexIndexFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/plain/IndexIndexFieldData.java index 28c0752965..53832c0b5b 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/plain/IndexIndexFieldData.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/plain/IndexIndexFieldData.java @@ -22,13 +22,14 @@ package org.elasticsearch.index.fielddata.plain; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.DocValues; import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.index.RandomAccessOrds; import org.apache.lucene.index.SortedDocValues; +import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.search.SortField; import org.apache.lucene.util.Accountable; import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.Nullable; import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.fielddata.AbstractSortedDocValues; import org.elasticsearch.index.fielddata.AtomicOrdinalsFieldData; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.IndexFieldDataCache; @@ -40,6 +41,7 @@ import org.elasticsearch.index.mapper.TextFieldMapper; import org.elasticsearch.indices.breaker.CircuitBreakerService; import org.elasticsearch.search.MultiValueMode; +import java.io.IOException; import java.util.Collection; import java.util.Collections; @@ -75,9 +77,11 @@ public class IndexIndexFieldData extends AbstractIndexOrdinalsFieldData { } @Override - public RandomAccessOrds getOrdinalsValues() { + public SortedSetDocValues getOrdinalsValues() { final BytesRef term = new BytesRef(index); - final SortedDocValues sortedValues = new SortedDocValues() { + final SortedDocValues sortedValues = new AbstractSortedDocValues() { + + private int docID = -1; @Override public BytesRef lookupOrd(int ord) { @@ -90,11 +94,22 @@ public class IndexIndexFieldData extends AbstractIndexOrdinalsFieldData { } @Override - public int getOrd(int docID) { + public int ordValue() { return 0; } + + @Override + public boolean advanceExact(int target) throws IOException { + docID = target; + return true; + } + + @Override + public int docID() { + return docID; + } }; - return (RandomAccessOrds) DocValues.singleton(sortedValues); + return (SortedSetDocValues) DocValues.singleton(sortedValues); } @Override diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/plain/LatLonPointDVAtomicFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/plain/LatLonPointDVAtomicFieldData.java index d11a79c255..93fa5bf22a 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/plain/LatLonPointDVAtomicFieldData.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/plain/LatLonPointDVAtomicFieldData.java @@ -19,6 +19,8 @@ package org.elasticsearch.index.fielddata.plain; import org.apache.lucene.geo.GeoEncodingUtils; +import org.apache.lucene.index.DocValues; +import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.SortedNumericDocValues; import org.apache.lucene.util.Accountable; import org.apache.lucene.util.ArrayUtil; @@ -26,16 +28,18 @@ import org.apache.lucene.util.RamUsageEstimator; import org.elasticsearch.common.geo.GeoPoint; import org.elasticsearch.index.fielddata.MultiGeoPointValues; -import java.util.Arrays; +import java.io.IOException; import java.util.Collection; import java.util.Collections; final class LatLonPointDVAtomicFieldData extends AbstractAtomicGeoPointFieldData { - private final SortedNumericDocValues values; + private final LeafReader reader; + private final String fieldName; - LatLonPointDVAtomicFieldData(SortedNumericDocValues values) { + LatLonPointDVAtomicFieldData(LeafReader reader, String fieldName) { super(); - this.values = values; + this.reader = reader; + this.fieldName = fieldName; } @Override @@ -55,37 +59,32 @@ final class LatLonPointDVAtomicFieldData extends AbstractAtomicGeoPointFieldData @Override public MultiGeoPointValues getGeoPointValues() { - return new MultiGeoPointValues() { - GeoPoint[] points = new GeoPoint[0]; - private int count = 0; + try { + final SortedNumericDocValues numericValues = DocValues.getSortedNumeric(reader, fieldName); + return new MultiGeoPointValues() { - @Override - public void setDocument(int docId) { - values.setDocument(docId); - count = values.count(); - if (count > points.length) { - final int previousLength = points.length; - points = Arrays.copyOf(points, ArrayUtil.oversize(count, RamUsageEstimator.NUM_BYTES_OBJECT_REF)); - for (int i = previousLength; i < points.length; ++i) { - points[i] = new GeoPoint(Double.NaN, Double.NaN); - } - } - long encoded; - for (int i=0; i<count; ++i) { - encoded = values.valueAt(i); - points[i].reset(GeoEncodingUtils.decodeLatitude((int)(encoded >>> 32)), GeoEncodingUtils.decodeLongitude((int)encoded)); + final GeoPoint point = new GeoPoint(); + + @Override + public boolean advanceExact(int doc) throws IOException { + return numericValues.advanceExact(doc); } - } - @Override - public int count() { - return count; - } + @Override + public int docValueCount() { + return numericValues.docValueCount(); + } - @Override - public GeoPoint valueAt(int index) { - return points[index]; - } - }; + @Override + public GeoPoint nextValue() throws IOException { + final long encoded = numericValues.nextValue(); + point.reset(GeoEncodingUtils.decodeLatitude((int) (encoded >>> 32)), + GeoEncodingUtils.decodeLongitude((int) encoded)); + return point; + } + }; + } catch (IOException e) { + throw new IllegalStateException("Cannot load doc values", e); + } } } diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/plain/PagedBytesAtomicFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/plain/PagedBytesAtomicFieldData.java index 02e7da8886..c9e81ca4f2 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/plain/PagedBytesAtomicFieldData.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/plain/PagedBytesAtomicFieldData.java @@ -18,7 +18,7 @@ */ package org.elasticsearch.index.fielddata.plain; -import org.apache.lucene.index.RandomAccessOrds; +import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.util.Accountable; import org.apache.lucene.util.Accountables; import org.apache.lucene.util.BytesRef; @@ -68,7 +68,7 @@ public class PagedBytesAtomicFieldData extends AbstractAtomicOrdinalsFieldData { } @Override - public RandomAccessOrds getOrdinalsValues() { + public SortedSetDocValues getOrdinalsValues() { return ordinals.ordinals(new ValuesHolder(bytes, termOrdToBytesOffset)); } diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/plain/ParentChildIndexFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/plain/ParentChildIndexFieldData.java index ee451ff023..74b180f2c1 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/plain/ParentChildIndexFieldData.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/plain/ParentChildIndexFieldData.java @@ -40,6 +40,7 @@ import org.elasticsearch.common.lease.Releasables; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.fielddata.AbstractSortedDocValues; import org.elasticsearch.index.fielddata.AtomicParentChildFieldData; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested; @@ -254,10 +255,10 @@ public class ParentChildIndexFieldData extends AbstractIndexFieldData<AtomicPare return segmentValues; } final LongValues globalOrds = ordMap.getGlobalOrds(segmentIndex); - return new SortedDocValues() { + return new AbstractSortedDocValues() { @Override - public BytesRef lookupOrd(int ord) { + public BytesRef lookupOrd(int ord) throws IOException { final int segmentIndex = ordMap.getFirstSegmentNumber(ord); final int segmentOrd = (int) ordMap.getFirstSegmentOrd(ord); return allSegmentValues[segmentIndex].lookupOrd(segmentOrd); @@ -269,14 +270,18 @@ public class ParentChildIndexFieldData extends AbstractIndexFieldData<AtomicPare } @Override - public int getOrd(int docID) { - final int segmentOrd = segmentValues.getOrd(docID); - // TODO: is there a way we can get rid of this branch? - if (segmentOrd >= 0) { - return (int) globalOrds.get(segmentOrd); - } else { - return segmentOrd; - } + public int ordValue() throws IOException { + return (int) globalOrds.get(segmentValues.ordValue()); + } + + @Override + public boolean advanceExact(int target) throws IOException { + return segmentValues.advanceExact(target); + } + + @Override + public int docID() { + return segmentValues.docID(); } }; } @@ -313,7 +318,7 @@ public class ParentChildIndexFieldData extends AbstractIndexFieldData<AtomicPare private final Map<String, OrdinalMapAndAtomicFieldData> ordinalMapPerType; GlobalFieldData(IndexReader reader, AtomicParentChildFieldData[] fielddata, long ramBytesUsed, Map<String, OrdinalMapAndAtomicFieldData> ordinalMapPerType) { - this.coreCacheKey = reader.getCoreCacheKey(); + this.coreCacheKey = reader.getReaderCacheHelper().getKey(); this.leaves = reader.leaves(); this.ramBytesUsed = ramBytesUsed; this.fielddata = fielddata; @@ -327,7 +332,8 @@ public class ParentChildIndexFieldData extends AbstractIndexFieldData<AtomicPare @Override public AtomicParentChildFieldData load(LeafReaderContext context) { - assert context.reader().getCoreCacheKey() == leaves.get(context.ord).reader().getCoreCacheKey(); + assert context.reader().getCoreCacheHelper().getKey() == leaves.get(context.ord) + .reader().getCoreCacheHelper().getKey(); return fielddata[context.ord]; } @@ -363,7 +369,7 @@ public class ParentChildIndexFieldData extends AbstractIndexFieldData<AtomicPare @Override public IndexParentChildFieldData loadGlobal(DirectoryReader indexReader) { - if (indexReader.getCoreCacheKey() == coreCacheKey) { + if (indexReader.getReaderCacheHelper().getKey() == coreCacheKey) { return this; } throw new IllegalStateException(); diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/plain/SortedNumericDVIndexFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/plain/SortedNumericDVIndexFieldData.java index 82f3de0f72..1b28d3a8c7 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/plain/SortedNumericDVIndexFieldData.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/plain/SortedNumericDVIndexFieldData.java @@ -148,9 +148,7 @@ public class SortedNumericDVIndexFieldData extends DocValuesIndexFieldData imple * Although the API is multi-valued, most codecs in Lucene specialize * for the case where documents have at most one value. In this case * {@link DocValues#unwrapSingleton(SortedNumericDocValues)} will return - * the underlying single-valued NumericDocValues representation, and - * {@link DocValues#unwrapSingletonBits(SortedNumericDocValues)} will return - * a Bits matching documents that have a real value (as opposed to missing). + * the underlying single-valued NumericDocValues representation. */ static final class SortedNumericLongFieldData extends AtomicLongFieldData { final LeafReader reader; @@ -188,9 +186,7 @@ public class SortedNumericDVIndexFieldData extends DocValuesIndexFieldData imple * Although the API is multi-valued, most codecs in Lucene specialize * for the case where documents have at most one value. In this case * {@link FieldData#unwrapSingleton(SortedNumericDoubleValues)} will return - * the underlying single-valued NumericDoubleValues representation, and - * {@link FieldData#unwrapSingletonBits(SortedNumericDoubleValues)} will return - * a Bits matching documents that have a real value (as opposed to missing). + * the underlying single-valued NumericDoubleValues representation. */ static final class SortedNumericHalfFloatFieldData extends AtomicDoubleFieldData { final LeafReader reader; @@ -209,7 +205,7 @@ public class SortedNumericDVIndexFieldData extends DocValuesIndexFieldData imple NumericDocValues single = DocValues.unwrapSingleton(raw); if (single != null) { - return FieldData.singleton(new SingleHalfFloatValues(single), DocValues.unwrapSingletonBits(raw)); + return FieldData.singleton(new SingleHalfFloatValues(single)); } else { return new MultiHalfFloatValues(raw); } @@ -235,8 +231,13 @@ public class SortedNumericDVIndexFieldData extends DocValuesIndexFieldData imple } @Override - public double get(int docID) { - return HalfFloatPoint.sortableShortToHalfFloat((short) in.get(docID)); + public double doubleValue() throws IOException { + return HalfFloatPoint.sortableShortToHalfFloat((short) in.longValue()); + } + + @Override + public boolean advanceExact(int doc) throws IOException { + return in.advanceExact(doc); } } @@ -251,18 +252,18 @@ public class SortedNumericDVIndexFieldData extends DocValuesIndexFieldData imple } @Override - public void setDocument(int doc) { - in.setDocument(doc); + public boolean advanceExact(int target) throws IOException { + return in.advanceExact(target); } @Override - public double valueAt(int index) { - return HalfFloatPoint.sortableShortToHalfFloat((short) in.valueAt(index)); + public double nextValue() throws IOException { + return HalfFloatPoint.sortableShortToHalfFloat((short) in.nextValue()); } @Override - public int count() { - return in.count(); + public int docValueCount() { + return in.docValueCount(); } } @@ -277,9 +278,7 @@ public class SortedNumericDVIndexFieldData extends DocValuesIndexFieldData imple * Although the API is multi-valued, most codecs in Lucene specialize * for the case where documents have at most one value. In this case * {@link FieldData#unwrapSingleton(SortedNumericDoubleValues)} will return - * the underlying single-valued NumericDoubleValues representation, and - * {@link FieldData#unwrapSingletonBits(SortedNumericDoubleValues)} will return - * a Bits matching documents that have a real value (as opposed to missing). + * the underlying single-valued NumericDoubleValues representation. */ static final class SortedNumericFloatFieldData extends AtomicDoubleFieldData { final LeafReader reader; @@ -298,7 +297,7 @@ public class SortedNumericDVIndexFieldData extends DocValuesIndexFieldData imple NumericDocValues single = DocValues.unwrapSingleton(raw); if (single != null) { - return FieldData.singleton(new SingleFloatValues(single), DocValues.unwrapSingletonBits(raw)); + return FieldData.singleton(new SingleFloatValues(single)); } else { return new MultiFloatValues(raw); } @@ -324,8 +323,13 @@ public class SortedNumericDVIndexFieldData extends DocValuesIndexFieldData imple } @Override - public double get(int docID) { - return NumericUtils.sortableIntToFloat((int) in.get(docID)); + public double doubleValue() throws IOException { + return NumericUtils.sortableIntToFloat((int) in.longValue()); + } + + @Override + public boolean advanceExact(int doc) throws IOException { + return in.advanceExact(doc); } } @@ -340,18 +344,18 @@ public class SortedNumericDVIndexFieldData extends DocValuesIndexFieldData imple } @Override - public void setDocument(int doc) { - in.setDocument(doc); + public boolean advanceExact(int target) throws IOException { + return in.advanceExact(target); } @Override - public double valueAt(int index) { - return NumericUtils.sortableIntToFloat((int) in.valueAt(index)); + public double nextValue() throws IOException { + return NumericUtils.sortableIntToFloat((int) in.nextValue()); } @Override - public int count() { - return in.count(); + public int docValueCount() { + return in.docValueCount(); } } @@ -366,9 +370,7 @@ public class SortedNumericDVIndexFieldData extends DocValuesIndexFieldData imple * Although the API is multi-valued, most codecs in Lucene specialize * for the case where documents have at most one value. In this case * {@link FieldData#unwrapSingleton(SortedNumericDoubleValues)} will return - * the underlying single-valued NumericDoubleValues representation, and - * {@link FieldData#unwrapSingletonBits(SortedNumericDoubleValues)} will return - * a Bits matching documents that have a real value (as opposed to missing). + * the underlying single-valued NumericDoubleValues representation. */ static final class SortedNumericDoubleFieldData extends AtomicDoubleFieldData { final LeafReader reader; diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/plain/SortedSetDVBytesAtomicFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/plain/SortedSetDVBytesAtomicFieldData.java index 114031cd35..d336c1d5cd 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/plain/SortedSetDVBytesAtomicFieldData.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/plain/SortedSetDVBytesAtomicFieldData.java @@ -21,10 +21,9 @@ package org.elasticsearch.index.fielddata.plain; import org.apache.lucene.index.DocValues; import org.apache.lucene.index.LeafReader; -import org.apache.lucene.index.RandomAccessOrds; +import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.util.Accountable; import org.elasticsearch.index.fielddata.AtomicFieldData; -import org.elasticsearch.index.fielddata.FieldData; import org.elasticsearch.index.fielddata.ScriptDocValues; import java.io.IOException; @@ -40,7 +39,7 @@ public final class SortedSetDVBytesAtomicFieldData extends AbstractAtomicOrdinal private final LeafReader reader; private final String field; - SortedSetDVBytesAtomicFieldData(LeafReader reader, String field, Function<RandomAccessOrds, + SortedSetDVBytesAtomicFieldData(LeafReader reader, String field, Function<SortedSetDocValues, ScriptDocValues<?>> scriptFunction) { super(scriptFunction); this.reader = reader; @@ -48,9 +47,9 @@ public final class SortedSetDVBytesAtomicFieldData extends AbstractAtomicOrdinal } @Override - public RandomAccessOrds getOrdinalsValues() { + public SortedSetDocValues getOrdinalsValues() { try { - return FieldData.maybeSlowRandomAccessOrds(DocValues.getSortedSet(reader, field)); + return DocValues.getSortedSet(reader, field); } catch (IOException e) { throw new IllegalStateException("cannot load docvalues", e); } diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/plain/SortedSetDVOrdinalsIndexFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/plain/SortedSetDVOrdinalsIndexFieldData.java index 2c59e8559c..ea076d476d 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/plain/SortedSetDVOrdinalsIndexFieldData.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/plain/SortedSetDVOrdinalsIndexFieldData.java @@ -21,10 +21,10 @@ package org.elasticsearch.index.fielddata.plain; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.index.RandomAccessOrds; +import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.search.SortField; -import org.apache.lucene.search.SortedSetSortField; import org.apache.lucene.search.SortedSetSelector; +import org.apache.lucene.search.SortedSetSortField; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.common.Nullable; import org.elasticsearch.index.IndexSettings; @@ -46,10 +46,10 @@ public class SortedSetDVOrdinalsIndexFieldData extends DocValuesIndexFieldData i private final IndexSettings indexSettings; private final IndexFieldDataCache cache; private final CircuitBreakerService breakerService; - private final Function<RandomAccessOrds, ScriptDocValues<?>> scriptFunction; + private final Function<SortedSetDocValues, ScriptDocValues<?>> scriptFunction; public SortedSetDVOrdinalsIndexFieldData(IndexSettings indexSettings, IndexFieldDataCache cache, String fieldName, - CircuitBreakerService breakerService, Function<RandomAccessOrds, ScriptDocValues<?>> scriptFunction) { + CircuitBreakerService breakerService, Function<SortedSetDocValues, ScriptDocValues<?>> scriptFunction) { super(indexSettings.getIndex(), fieldName); this.indexSettings = indexSettings; this.cache = cache; diff --git a/core/src/main/java/org/elasticsearch/index/mapper/CustomDocValuesField.java b/core/src/main/java/org/elasticsearch/index/mapper/CustomDocValuesField.java index 8e6cee222d..60fbfc0698 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/CustomDocValuesField.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/CustomDocValuesField.java @@ -54,11 +54,6 @@ abstract class CustomDocValuesField implements IndexableField { } @Override - public float boost() { - return 1f; - } - - @Override public String stringValue() { return null; } diff --git a/core/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java index 8b7355dca4..39280bcee2 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java @@ -277,7 +277,7 @@ public class DocumentMapper implements ToXContent { } // We can pass down 'null' as acceptedDocs, because nestedDocId is a doc to be fetched and // therefor is guaranteed to be a live doc. - final Weight nestedWeight = filter.createWeight(sc.searcher(), false); + final Weight nestedWeight = filter.createWeight(sc.searcher(), false, 1f); Scorer scorer = nestedWeight.scorer(context); if (scorer == null) { continue; diff --git a/core/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java index 0ed093b3a8..b8c958f41f 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java @@ -21,7 +21,6 @@ package org.elasticsearch.index.mapper; import com.carrotsearch.hppc.cursors.ObjectCursor; import com.carrotsearch.hppc.cursors.ObjectObjectCursor; -import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.IndexableField; @@ -286,12 +285,6 @@ public abstract class FieldMapper extends Mapper implements Cloneable { try { parseCreateField(context, fields); for (IndexableField field : fields) { - if (!customBoost() - // don't set boosts eg. on dv fields - && field.fieldType().indexOptions() != IndexOptions.NONE - && indexCreatedVersion.before(Version.V_5_0_0_alpha1)) { - ((Field)(field)).setBoost(fieldType().boost()); - } context.doc().add(field); } } catch (Exception e) { @@ -306,13 +299,6 @@ public abstract class FieldMapper extends Mapper implements Cloneable { */ protected abstract void parseCreateField(ParseContext context, List<IndexableField> fields) throws IOException; - /** - * Derived classes can override it to specify that boost value is set by derived classes. - */ - protected boolean customBoost() { - return false; - } - @Override public Iterator<Mapper> iterator() { return multiFields.iterator(); diff --git a/core/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java index c46fe227d6..2f09027c4f 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java @@ -31,7 +31,6 @@ import org.apache.lucene.spatial.prefix.tree.GeohashPrefixTree; import org.apache.lucene.spatial.prefix.tree.PackedQuadPrefixTree; import org.apache.lucene.spatial.prefix.tree.QuadPrefixTree; import org.apache.lucene.spatial.prefix.tree.SpatialPrefixTree; -import org.elasticsearch.Version; import org.elasticsearch.action.fieldstats.FieldStats; import org.elasticsearch.common.Explicit; import org.elasticsearch.common.geo.GeoUtils; @@ -463,10 +462,6 @@ public class GeoShapeFieldMapper extends FieldMapper { return null; } for (Field field : fields) { - if (!customBoost() && - fieldType.boost() != 1f && Version.indexCreated(context.indexSettings()).before(Version.V_5_0_0_alpha1)) { - field.setBoost(fieldType().boost()); - } context.doc().add(field); } } catch (Exception e) { diff --git a/core/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java index a15432c635..3d6f4efba1 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java @@ -27,9 +27,10 @@ import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexableField; import org.apache.lucene.index.PointValues; -import org.apache.lucene.index.RandomAccessOrds; +import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.Query; +import org.apache.lucene.util.ArrayUtil; import org.apache.lucene.util.BytesRef; import org.elasticsearch.action.fieldstats.FieldStats; import org.elasticsearch.common.Explicit; @@ -233,19 +234,27 @@ public class IpFieldMapper extends FieldMapper { public static final class IpScriptDocValues extends ScriptDocValues<String> { - private final RandomAccessOrds values; + private final SortedSetDocValues in; + private long[] ords = new long[0]; + private int count; - public IpScriptDocValues(RandomAccessOrds values) { - this.values = values; + public IpScriptDocValues(SortedSetDocValues in) { + this.in = in; } @Override - public void setNextDocId(int docId) { - values.setDocument(docId); + public void setNextDocId(int docId) throws IOException { + count = 0; + if (in.advanceExact(docId)) { + for (long ord = in.nextOrd(); ord != SortedSetDocValues.NO_MORE_ORDS; ord = in.nextOrd()) { + ords = ArrayUtil.grow(ords, count + 1); + ords[count++] = ord; + } + } } public String getValue() { - if (isEmpty()) { + if (count == 0) { return null; } else { return get(0); @@ -254,15 +263,19 @@ public class IpFieldMapper extends FieldMapper { @Override public String get(int index) { - BytesRef encoded = values.lookupOrd(values.ordAt(0)); - InetAddress address = InetAddressPoint.decode( - Arrays.copyOfRange(encoded.bytes, encoded.offset, encoded.offset + encoded.length)); - return InetAddresses.toAddrString(address); + try { + BytesRef encoded = in.lookupOrd(ords[index]); + InetAddress address = InetAddressPoint.decode( + Arrays.copyOfRange(encoded.bytes, encoded.offset, encoded.offset + encoded.length)); + return InetAddresses.toAddrString(address); + } catch (IOException e) { + throw new RuntimeException(e); + } } @Override public int size() { - return values.cardinality(); + return count; } } diff --git a/core/src/main/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapper.java index 226ab905a2..c69e68b24b 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapper.java @@ -219,7 +219,7 @@ public class ScaledFloatFieldMapper extends FieldMapper { } @Override - public Query termsQuery(List values, QueryShardContext context) { + public Query termsQuery(List<?> values, QueryShardContext context) { failIfNotIndexed(); List<Long> scaledValues = new ArrayList<>(values.size()); for (Object value : values) { @@ -561,26 +561,30 @@ public class ScaledFloatFieldMapper extends FieldMapper { if (singleValues != null) { return FieldData.singleton(new NumericDoubleValues() { @Override - public double get(int docID) { - return singleValues.get(docID) * scalingFactorInverse; + public boolean advanceExact(int doc) throws IOException { + return singleValues.advanceExact(doc); } - }, DocValues.unwrapSingletonBits(values)); + @Override + public double doubleValue() throws IOException { + return singleValues.longValue() * scalingFactorInverse; + } + }); } else { return new SortedNumericDoubleValues() { @Override - public double valueAt(int index) { - return values.valueAt(index) * scalingFactorInverse; + public boolean advanceExact(int target) throws IOException { + return values.advanceExact(target); } @Override - public void setDocument(int doc) { - values.setDocument(doc); + public double nextValue() throws IOException { + return values.nextValue() * scalingFactorInverse; } @Override - public int count() { - return values.count(); + public int docValueCount() { + return values.docValueCount(); } }; } diff --git a/core/src/main/java/org/elasticsearch/index/query/BoolQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/BoolQueryBuilder.java index 5eb8b81009..dcab99ce6a 100644 --- a/core/src/main/java/org/elasticsearch/index/query/BoolQueryBuilder.java +++ b/core/src/main/java/org/elasticsearch/index/query/BoolQueryBuilder.java @@ -24,6 +24,7 @@ import org.apache.lucene.search.BooleanClause.Occur; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.Query; +import org.elasticsearch.Version; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.io.stream.StreamInput; @@ -48,14 +49,14 @@ public class BoolQueryBuilder extends AbstractQueryBuilder<BoolQueryBuilder> { public static final String NAME = "bool"; public static final boolean ADJUST_PURE_NEGATIVE_DEFAULT = true; - public static final boolean DISABLE_COORD_DEFAULT = false; private static final String MUSTNOT = "mustNot"; private static final String MUST_NOT = "must_not"; private static final String FILTER = "filter"; private static final String SHOULD = "should"; private static final String MUST = "must"; - private static final ParseField DISABLE_COORD_FIELD = new ParseField("disable_coord"); + private static final ParseField DISABLE_COORD_FIELD = new ParseField("disable_coord") + .withAllDeprecated("disable_coord has been removed"); private static final ParseField MINIMUM_SHOULD_MATCH = new ParseField("minimum_should_match"); private static final ParseField ADJUST_PURE_NEGATIVE = new ParseField("adjust_pure_negative"); @@ -67,8 +68,6 @@ public class BoolQueryBuilder extends AbstractQueryBuilder<BoolQueryBuilder> { private final List<QueryBuilder> shouldClauses = new ArrayList<>(); - private boolean disableCoord = DISABLE_COORD_DEFAULT; - private boolean adjustPureNegative = ADJUST_PURE_NEGATIVE_DEFAULT; private String minimumShouldMatch; @@ -89,7 +88,9 @@ public class BoolQueryBuilder extends AbstractQueryBuilder<BoolQueryBuilder> { shouldClauses.addAll(readQueries(in)); filterClauses.addAll(readQueries(in)); adjustPureNegative = in.readBoolean(); - disableCoord = in.readBoolean(); + if (in.getVersion().before(Version.V_6_0_0_alpha1_UNRELEASED)) { + in.readBoolean(); // disable_coord + } minimumShouldMatch = in.readOptionalString(); } @@ -100,7 +101,9 @@ public class BoolQueryBuilder extends AbstractQueryBuilder<BoolQueryBuilder> { writeQueries(out, shouldClauses); writeQueries(out, filterClauses); out.writeBoolean(adjustPureNegative); - out.writeBoolean(disableCoord); + if (out.getVersion().before(Version.V_6_0_0_alpha1_UNRELEASED)) { + out.writeBoolean(true); // disable_coord + } out.writeOptionalString(minimumShouldMatch); } @@ -187,21 +190,6 @@ public class BoolQueryBuilder extends AbstractQueryBuilder<BoolQueryBuilder> { } /** - * Disables <tt>Similarity#coord(int,int)</tt> in scoring. Defaults to <tt>false</tt>. - */ - public BoolQueryBuilder disableCoord(boolean disableCoord) { - this.disableCoord = disableCoord; - return this; - } - - /** - * @return whether the <tt>Similarity#coord(int,int)</tt> in scoring are disabled. Defaults to <tt>false</tt>. - */ - public boolean disableCoord() { - return this.disableCoord; - } - - /** * @return the string representation of the minimumShouldMatch settings for this query */ public String minimumShouldMatch() { @@ -267,7 +255,6 @@ public class BoolQueryBuilder extends AbstractQueryBuilder<BoolQueryBuilder> { doXArrayContent(FILTER, filterClauses, builder, params); doXArrayContent(MUST_NOT, mustNotClauses, builder, params); doXArrayContent(SHOULD, shouldClauses, builder, params); - builder.field(DISABLE_COORD_FIELD.getPreferredName(), disableCoord); builder.field(ADJUST_PURE_NEGATIVE.getPreferredName(), adjustPureNegative); if (minimumShouldMatch != null) { builder.field(MINIMUM_SHOULD_MATCH.getPreferredName(), minimumShouldMatch); @@ -291,7 +278,6 @@ public class BoolQueryBuilder extends AbstractQueryBuilder<BoolQueryBuilder> { public static BoolQueryBuilder fromXContent(QueryParseContext parseContext) throws IOException, ParsingException { XContentParser parser = parseContext.parser(); - boolean disableCoord = BoolQueryBuilder.DISABLE_COORD_DEFAULT; boolean adjustPureNegative = BoolQueryBuilder.ADJUST_PURE_NEGATIVE_DEFAULT; float boost = AbstractQueryBuilder.DEFAULT_BOOST; String minimumShouldMatch = null; @@ -349,7 +335,7 @@ public class BoolQueryBuilder extends AbstractQueryBuilder<BoolQueryBuilder> { } } else if (token.isValue()) { if (DISABLE_COORD_FIELD.match(currentFieldName)) { - disableCoord = parser.booleanValue(); + // ignore } else if (MINIMUM_SHOULD_MATCH.match(currentFieldName)) { minimumShouldMatch = parser.textOrNull(); } else if (AbstractQueryBuilder.BOOST_FIELD.match(currentFieldName)) { @@ -377,7 +363,6 @@ public class BoolQueryBuilder extends AbstractQueryBuilder<BoolQueryBuilder> { boolQuery.filter(queryBuilder); } boolQuery.boost(boost); - boolQuery.disableCoord(disableCoord); boolQuery.adjustPureNegative(adjustPureNegative); boolQuery.minimumShouldMatch(minimumShouldMatch); boolQuery.queryName(queryName); @@ -392,7 +377,6 @@ public class BoolQueryBuilder extends AbstractQueryBuilder<BoolQueryBuilder> { @Override protected Query doToQuery(QueryShardContext context) throws IOException { BooleanQuery.Builder booleanQueryBuilder = new BooleanQuery.Builder(); - booleanQueryBuilder.setDisableCoord(disableCoord); addBooleanClauses(context, booleanQueryBuilder, mustClauses, BooleanClause.Occur.MUST); addBooleanClauses(context, booleanQueryBuilder, mustNotClauses, BooleanClause.Occur.MUST_NOT); addBooleanClauses(context, booleanQueryBuilder, shouldClauses, BooleanClause.Occur.SHOULD); @@ -432,14 +416,13 @@ public class BoolQueryBuilder extends AbstractQueryBuilder<BoolQueryBuilder> { @Override protected int doHashCode() { - return Objects.hash(adjustPureNegative, disableCoord, + return Objects.hash(adjustPureNegative, minimumShouldMatch, mustClauses, shouldClauses, mustNotClauses, filterClauses); } @Override protected boolean doEquals(BoolQueryBuilder other) { return Objects.equals(adjustPureNegative, other.adjustPureNegative) && - Objects.equals(disableCoord, other.disableCoord) && Objects.equals(minimumShouldMatch, other.minimumShouldMatch) && Objects.equals(mustClauses, other.mustClauses) && Objects.equals(shouldClauses, other.shouldClauses) && @@ -462,7 +445,6 @@ public class BoolQueryBuilder extends AbstractQueryBuilder<BoolQueryBuilder> { if (changed) { newBuilder.adjustPureNegative = adjustPureNegative; - newBuilder.disableCoord = disableCoord; newBuilder.minimumShouldMatch = minimumShouldMatch; newBuilder.boost(boost()); newBuilder.queryName(queryName()); diff --git a/core/src/main/java/org/elasticsearch/index/query/CommonTermsQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/CommonTermsQueryBuilder.java index a91d8fe44c..e5fff943cd 100644 --- a/core/src/main/java/org/elasticsearch/index/query/CommonTermsQueryBuilder.java +++ b/core/src/main/java/org/elasticsearch/index/query/CommonTermsQueryBuilder.java @@ -25,10 +25,9 @@ import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; import org.apache.lucene.index.Term; import org.apache.lucene.queries.ExtendedCommonTermsQuery; import org.apache.lucene.search.BooleanClause.Occur; -import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.Query; -import org.apache.lucene.search.similarities.Similarity; import org.apache.lucene.util.BytesRefBuilder; +import org.elasticsearch.Version; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.Strings; @@ -48,13 +47,7 @@ import java.util.Objects; * added} terms where low-frequency terms are added to a required boolean clause * and high-frequency terms are added to an optional boolean clause. The * optional clause is only executed if the required "low-frequency' clause - * matches. Scores produced by this query will be slightly different to plain - * {@link BooleanQuery} scorer mainly due to differences in the - * {@link Similarity#coord(int,int) number of leave queries} in the required - * boolean clause. In the most cases high-frequency terms are unlikely to - * significantly contribute to the document score unless at least one of the - * low-frequency terms are matched such that this query can improve query - * execution times significantly if applicable. + * matches. */ public class CommonTermsQueryBuilder extends AbstractQueryBuilder<CommonTermsQueryBuilder> { @@ -69,7 +62,8 @@ public class CommonTermsQueryBuilder extends AbstractQueryBuilder<CommonTermsQue private static final ParseField MINIMUM_SHOULD_MATCH_FIELD = new ParseField("minimum_should_match"); private static final ParseField LOW_FREQ_OPERATOR_FIELD = new ParseField("low_freq_operator"); private static final ParseField HIGH_FREQ_OPERATOR_FIELD = new ParseField("high_freq_operator"); - private static final ParseField DISABLE_COORD_FIELD = new ParseField("disable_coord"); + private static final ParseField DISABLE_COORD_FIELD = new ParseField("disable_coord") + .withAllDeprecated("disable_coord has been removed"); private static final ParseField ANALYZER_FIELD = new ParseField("analyzer"); private static final ParseField QUERY_FIELD = new ParseField("query"); private static final ParseField HIGH_FREQ_FIELD = new ParseField("high_freq"); @@ -89,8 +83,6 @@ public class CommonTermsQueryBuilder extends AbstractQueryBuilder<CommonTermsQue private String highFreqMinimumShouldMatch = null; - private boolean disableCoord = DEFAULT_DISABLE_COORD; - private float cutoffFrequency = DEFAULT_CUTOFF_FREQ; /** @@ -119,7 +111,9 @@ public class CommonTermsQueryBuilder extends AbstractQueryBuilder<CommonTermsQue analyzer = in.readOptionalString(); lowFreqMinimumShouldMatch = in.readOptionalString(); highFreqMinimumShouldMatch = in.readOptionalString(); - disableCoord = in.readBoolean(); + if (in.getVersion().before(Version.V_6_0_0_alpha1_UNRELEASED)) { + in.readBoolean(); // disable_coord + } cutoffFrequency = in.readFloat(); } @@ -132,7 +126,9 @@ public class CommonTermsQueryBuilder extends AbstractQueryBuilder<CommonTermsQue out.writeOptionalString(analyzer); out.writeOptionalString(lowFreqMinimumShouldMatch); out.writeOptionalString(highFreqMinimumShouldMatch); - out.writeBoolean(disableCoord); + if (out.getVersion().before(Version.V_6_0_0_alpha1_UNRELEASED)) { + out.writeBoolean(true); // disable_coord + } out.writeFloat(cutoffFrequency); } @@ -226,21 +222,11 @@ public class CommonTermsQueryBuilder extends AbstractQueryBuilder<CommonTermsQue return this.lowFreqMinimumShouldMatch; } - public CommonTermsQueryBuilder disableCoord(boolean disableCoord) { - this.disableCoord = disableCoord; - return this; - } - - public boolean disableCoord() { - return this.disableCoord; - } - @Override protected void doXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(NAME); builder.startObject(fieldName); builder.field(QUERY_FIELD.getPreferredName(), text); - builder.field(DISABLE_COORD_FIELD.getPreferredName(), disableCoord); builder.field(HIGH_FREQ_OPERATOR_FIELD.getPreferredName(), highFreqOperator.toString()); builder.field(LOW_FREQ_OPERATOR_FIELD.getPreferredName(), lowFreqOperator.toString()); if (analyzer != null) { @@ -271,7 +257,6 @@ public class CommonTermsQueryBuilder extends AbstractQueryBuilder<CommonTermsQue String analyzer = null; String lowFreqMinimumShouldMatch = null; String highFreqMinimumShouldMatch = null; - boolean disableCoord = CommonTermsQueryBuilder.DEFAULT_DISABLE_COORD; Operator highFreqOperator = CommonTermsQueryBuilder.DEFAULT_HIGH_FREQ_OCCUR; Operator lowFreqOperator = CommonTermsQueryBuilder.DEFAULT_LOW_FREQ_OCCUR; float cutoffFrequency = CommonTermsQueryBuilder.DEFAULT_CUTOFF_FREQ; @@ -321,7 +306,7 @@ public class CommonTermsQueryBuilder extends AbstractQueryBuilder<CommonTermsQue } else if (ANALYZER_FIELD.match(currentFieldName)) { analyzer = parser.text(); } else if (DISABLE_COORD_FIELD.match(currentFieldName)) { - disableCoord = parser.booleanValue(); + // ignore } else if (AbstractQueryBuilder.BOOST_FIELD.match(currentFieldName)) { boost = parser.floatValue(); } else if (HIGH_FREQ_OPERATOR_FIELD.match(currentFieldName)) { @@ -353,7 +338,6 @@ public class CommonTermsQueryBuilder extends AbstractQueryBuilder<CommonTermsQue .analyzer(analyzer) .highFreqOperator(highFreqOperator) .lowFreqOperator(lowFreqOperator) - .disableCoord(disableCoord) .cutoffFrequency(cutoffFrequency) .boost(boost) .queryName(queryName); @@ -392,7 +376,7 @@ public class CommonTermsQueryBuilder extends AbstractQueryBuilder<CommonTermsQue Occur lowFreqOccur = lowFreqOperator.toBooleanClauseOccur(); ExtendedCommonTermsQuery commonsQuery = new ExtendedCommonTermsQuery(highFreqOccur, lowFreqOccur, - cutoffFrequency, disableCoord, fieldType); + cutoffFrequency, fieldType); return parseQueryString(commonsQuery, text, field, analyzerObj, lowFreqMinimumShouldMatch, highFreqMinimumShouldMatch); } @@ -418,7 +402,7 @@ public class CommonTermsQueryBuilder extends AbstractQueryBuilder<CommonTermsQue @Override protected int doHashCode() { return Objects.hash(fieldName, text, highFreqOperator, lowFreqOperator, analyzer, - lowFreqMinimumShouldMatch, highFreqMinimumShouldMatch, disableCoord, cutoffFrequency); + lowFreqMinimumShouldMatch, highFreqMinimumShouldMatch, cutoffFrequency); } @Override @@ -430,7 +414,6 @@ public class CommonTermsQueryBuilder extends AbstractQueryBuilder<CommonTermsQue Objects.equals(analyzer, other.analyzer) && Objects.equals(lowFreqMinimumShouldMatch, other.lowFreqMinimumShouldMatch) && Objects.equals(highFreqMinimumShouldMatch, other.highFreqMinimumShouldMatch) && - Objects.equals(disableCoord, other.disableCoord) && Objects.equals(cutoffFrequency, other.cutoffFrequency); } } diff --git a/core/src/main/java/org/elasticsearch/index/query/ScriptQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/ScriptQueryBuilder.java index f12e9b0676..4ad2e7824e 100644 --- a/core/src/main/java/org/elasticsearch/index/query/ScriptQueryBuilder.java +++ b/core/src/main/java/org/elasticsearch/index/query/ScriptQueryBuilder.java @@ -20,11 +20,14 @@ package org.elasticsearch.index.query; import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.search.ConstantScoreScorer; +import org.apache.lucene.search.ConstantScoreWeight; +import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; -import org.apache.lucene.search.RandomAccessWeight; +import org.apache.lucene.search.Scorer; +import org.apache.lucene.search.TwoPhaseIterator; import org.apache.lucene.search.Weight; -import org.apache.lucene.util.Bits; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.io.stream.StreamInput; @@ -172,16 +175,18 @@ public class ScriptQueryBuilder extends AbstractQueryBuilder<ScriptQueryBuilder> } @Override - public Weight createWeight(IndexSearcher searcher, boolean needsScores) throws IOException { - return new RandomAccessWeight(this) { + public Weight createWeight(IndexSearcher searcher, boolean needsScores, float boost) throws IOException { + return new ConstantScoreWeight(this, boost) { + @Override - protected Bits getMatchingDocs(final LeafReaderContext context) throws IOException { + public Scorer scorer(LeafReaderContext context) throws IOException { + DocIdSetIterator approximation = DocIdSetIterator.all(context.reader().maxDoc()); final LeafSearchScript leafScript = searchScript.getLeafSearchScript(context); - return new Bits() { + TwoPhaseIterator twoPhase = new TwoPhaseIterator(approximation) { @Override - public boolean get(int doc) { - leafScript.setDocument(doc); + public boolean matches() throws IOException { + leafScript.setDocument(approximation.docID()); Object val = leafScript.run(); if (val == null) { return false; @@ -196,11 +201,12 @@ public class ScriptQueryBuilder extends AbstractQueryBuilder<ScriptQueryBuilder> } @Override - public int length() { - return context.reader().maxDoc(); + public float matchCost() { + // TODO: how can we compute this? + return 1000f; } - }; + return new ConstantScoreScorer(this, score(), twoPhase); } }; } diff --git a/core/src/main/java/org/elasticsearch/index/query/SimpleQueryParser.java b/core/src/main/java/org/elasticsearch/index/query/SimpleQueryParser.java index a147e49604..9899ba9a74 100644 --- a/core/src/main/java/org/elasticsearch/index/query/SimpleQueryParser.java +++ b/core/src/main/java/org/elasticsearch/index/query/SimpleQueryParser.java @@ -80,7 +80,6 @@ public class SimpleQueryParser extends org.apache.lucene.queryparser.simple.Simp @Override public Query newDefaultQuery(String text) { BooleanQuery.Builder bq = new BooleanQuery.Builder(); - bq.setDisableCoord(true); for (Map.Entry<String,Float> entry : weights.entrySet()) { try { Query q = createBooleanQuery(entry.getKey(), text, super.getDefaultOperator()); @@ -101,7 +100,6 @@ public class SimpleQueryParser extends org.apache.lucene.queryparser.simple.Simp @Override public Query newFuzzyQuery(String text, int fuzziness) { BooleanQuery.Builder bq = new BooleanQuery.Builder(); - bq.setDisableCoord(true); for (Map.Entry<String,Float> entry : weights.entrySet()) { final String fieldName = entry.getKey(); try { @@ -118,7 +116,6 @@ public class SimpleQueryParser extends org.apache.lucene.queryparser.simple.Simp @Override public Query newPhraseQuery(String text, int slop) { BooleanQuery.Builder bq = new BooleanQuery.Builder(); - bq.setDisableCoord(true); for (Map.Entry<String,Float> entry : weights.entrySet()) { try { String field = entry.getKey(); @@ -148,7 +145,6 @@ public class SimpleQueryParser extends org.apache.lucene.queryparser.simple.Simp @Override public Query newPrefixQuery(String text) { BooleanQuery.Builder bq = new BooleanQuery.Builder(); - bq.setDisableCoord(true); for (Map.Entry<String,Float> entry : weights.entrySet()) { final String fieldName = entry.getKey(); try { @@ -272,7 +268,7 @@ public class SimpleQueryParser extends org.apache.lucene.queryparser.simple.Simp innerBuilder.add(new BooleanClause(new PrefixQuery(new Term(field, token)), BooleanClause.Occur.SHOULD)); } - posQuery = innerBuilder.setDisableCoord(true).build(); + posQuery = innerBuilder.build(); } builder.add(new BooleanClause(posQuery, getDefaultOperator())); } diff --git a/core/src/main/java/org/elasticsearch/index/query/functionscore/DecayFunctionBuilder.java b/core/src/main/java/org/elasticsearch/index/query/functionscore/DecayFunctionBuilder.java index 77b67b89fc..e296c10218 100644 --- a/core/src/main/java/org/elasticsearch/index/query/functionscore/DecayFunctionBuilder.java +++ b/core/src/main/java/org/elasticsearch/index/query/functionscore/DecayFunctionBuilder.java @@ -348,18 +348,18 @@ public abstract class DecayFunctionBuilder<DFB extends DecayFunctionBuilder<DFB> final MultiGeoPointValues geoPointValues = fieldData.load(context).getGeoPointValues(); return mode.select(new MultiValueMode.UnsortedNumericDoubleValues() { @Override - public int count() { - return geoPointValues.count(); + public int docValueCount() { + return geoPointValues.docValueCount(); } @Override - public void setDocument(int docId) { - geoPointValues.setDocument(docId); + public boolean advanceExact(int docId) throws IOException { + return geoPointValues.advanceExact(docId); } @Override - public double valueAt(int index) { - GeoPoint other = geoPointValues.valueAt(index); + public double nextValue() throws IOException { + GeoPoint other = geoPointValues.nextValue(); return Math.max(0.0d, distFunction.calculate(origin.lat(), origin.lon(), other.lat(), other.lon(), DistanceUnit.METERS) - offset); } @@ -367,15 +367,14 @@ public abstract class DecayFunctionBuilder<DFB extends DecayFunctionBuilder<DFB> } @Override - protected String getDistanceString(LeafReaderContext ctx, int docId) { + protected String getDistanceString(LeafReaderContext ctx, int docId) throws IOException { StringBuilder values = new StringBuilder(mode.name()); values.append(" of: ["); final MultiGeoPointValues geoPointValues = fieldData.load(ctx).getGeoPointValues(); - geoPointValues.setDocument(docId); - final int num = geoPointValues.count(); - if (num > 0) { + if (geoPointValues.advanceExact(docId)) { + final int num = geoPointValues.docValueCount(); for (int i = 0; i < num; i++) { - GeoPoint value = geoPointValues.valueAt(i); + GeoPoint value = geoPointValues.nextValue(); values.append("Math.max(arcDistance("); values.append(value).append("(=doc value),"); values.append(origin).append("(=origin)) - ").append(offset).append("(=offset), 0)"); @@ -430,33 +429,32 @@ public abstract class DecayFunctionBuilder<DFB extends DecayFunctionBuilder<DFB> final SortedNumericDoubleValues doubleValues = fieldData.load(context).getDoubleValues(); return mode.select(new MultiValueMode.UnsortedNumericDoubleValues() { @Override - public int count() { - return doubleValues.count(); + public int docValueCount() { + return doubleValues.docValueCount(); } @Override - public void setDocument(int docId) { - doubleValues.setDocument(docId); + public boolean advanceExact(int doc) throws IOException { + return doubleValues.advanceExact(doc); } @Override - public double valueAt(int index) { - return Math.max(0.0d, Math.abs(doubleValues.valueAt(index) - origin) - offset); + public double nextValue() throws IOException { + return Math.max(0.0d, Math.abs(doubleValues.nextValue() - origin) - offset); } }, 0.0); } @Override - protected String getDistanceString(LeafReaderContext ctx, int docId) { + protected String getDistanceString(LeafReaderContext ctx, int docId) throws IOException { StringBuilder values = new StringBuilder(mode.name()); values.append("["); final SortedNumericDoubleValues doubleValues = fieldData.load(ctx).getDoubleValues(); - doubleValues.setDocument(docId); - final int num = doubleValues.count(); - if (num > 0) { + if (doubleValues.advanceExact(docId)) { + final int num = doubleValues.docValueCount(); for (int i = 0; i < num; i++) { - double value = doubleValues.valueAt(i); + double value = doubleValues.nextValue(); values.append("Math.max(Math.abs("); values.append(value).append("(=doc value) - "); values.append(origin).append("(=origin))) - "); @@ -531,21 +529,28 @@ public abstract class DecayFunctionBuilder<DFB extends DecayFunctionBuilder<DFB> return new LeafScoreFunction() { @Override - public double score(int docId, float subQueryScore) { - return func.evaluate(distance.get(docId), scale); + public double score(int docId, float subQueryScore) throws IOException { + if (distance.advanceExact(docId)) { + return func.evaluate(distance.doubleValue(), scale); + } else { + return 0; + } } @Override public Explanation explainScore(int docId, Explanation subQueryScore) throws IOException { + if (distance.advanceExact(docId) == false) { + return Explanation.noMatch("No value for the distance"); + } return Explanation.match( CombineFunction.toFloat(score(docId, subQueryScore.getValue())), "Function for field " + getFieldName() + ":", - func.explainFunction(getDistanceString(ctx, docId), distance.get(docId), scale)); + func.explainFunction(getDistanceString(ctx, docId), distance.doubleValue(), scale)); } }; } - protected abstract String getDistanceString(LeafReaderContext ctx, int docId); + protected abstract String getDistanceString(LeafReaderContext ctx, int docId) throws IOException; protected abstract String getFieldName(); diff --git a/core/src/main/java/org/elasticsearch/index/search/ESToParentBlockJoinQuery.java b/core/src/main/java/org/elasticsearch/index/search/ESToParentBlockJoinQuery.java index 1ee427599c..3762b1fffc 100644 --- a/core/src/main/java/org/elasticsearch/index/search/ESToParentBlockJoinQuery.java +++ b/core/src/main/java/org/elasticsearch/index/search/ESToParentBlockJoinQuery.java @@ -75,8 +75,8 @@ public final class ESToParentBlockJoinQuery extends Query { } @Override - public Weight createWeight(IndexSearcher searcher, boolean needsScores) throws IOException { - return query.createWeight(searcher, needsScores); + public Weight createWeight(IndexSearcher searcher, boolean needsScores, float boost) throws IOException { + return query.createWeight(searcher, needsScores, boost); } @Override diff --git a/core/src/main/java/org/elasticsearch/index/search/MatchQuery.java b/core/src/main/java/org/elasticsearch/index/search/MatchQuery.java index f3ea9447db..c4630f338c 100644 --- a/core/src/main/java/org/elasticsearch/index/search/MatchQuery.java +++ b/core/src/main/java/org/elasticsearch/index/search/MatchQuery.java @@ -434,7 +434,7 @@ public class MatchQuery { private Query boolToExtendedCommonTermsQuery(BooleanQuery bq, Occur highFreqOccur, Occur lowFreqOccur, float maxTermFrequency, MappedFieldType fieldType) { ExtendedCommonTermsQuery query = new ExtendedCommonTermsQuery(highFreqOccur, lowFreqOccur, maxTermFrequency, - bq.isCoordDisabled(), fieldType); + fieldType); for (BooleanClause clause : bq.clauses()) { if (!(clause.getQuery() instanceof TermQuery)) { return bq; diff --git a/core/src/main/java/org/elasticsearch/index/search/MultiMatchQuery.java b/core/src/main/java/org/elasticsearch/index/search/MultiMatchQuery.java index 15e8561c8f..7c1f91d358 100644 --- a/core/src/main/java/org/elasticsearch/index/search/MultiMatchQuery.java +++ b/core/src/main/java/org/elasticsearch/index/search/MultiMatchQuery.java @@ -305,9 +305,9 @@ public class MultiMatchQuery extends MatchQuery { terms = Arrays.copyOf(terms, i); blendedBoost = Arrays.copyOf(blendedBoost, i); if (commonTermsCutoff != null) { - queries.add(BlendedTermQuery.commonTermsBlendedQuery(terms, blendedBoost, false, commonTermsCutoff)); + queries.add(BlendedTermQuery.commonTermsBlendedQuery(terms, blendedBoost, commonTermsCutoff)); } else if (tieBreaker == 1.0f) { - queries.add(BlendedTermQuery.booleanBlendedQuery(terms, blendedBoost, false)); + queries.add(BlendedTermQuery.booleanBlendedQuery(terms, blendedBoost)); } else { queries.add(BlendedTermQuery.dismaxBlendedQuery(terms, blendedBoost, tieBreaker)); } @@ -319,7 +319,6 @@ public class MultiMatchQuery extends MatchQuery { // however their score contribution will be different // TODO: can we improve this? BooleanQuery.Builder bq = new BooleanQuery.Builder(); - bq.setDisableCoord(true); for (Query query : queries) { bq.add(query, Occur.SHOULD); } diff --git a/core/src/main/java/org/elasticsearch/index/shard/FilterDocValuesProducer.java b/core/src/main/java/org/elasticsearch/index/shard/FilterDocValuesProducer.java deleted file mode 100644 index 149915458e..0000000000 --- a/core/src/main/java/org/elasticsearch/index/shard/FilterDocValuesProducer.java +++ /dev/null @@ -1,157 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.index.shard; - -import org.apache.lucene.codecs.DocValuesProducer; -import org.apache.lucene.index.BinaryDocValues; -import org.apache.lucene.index.FieldInfo; -import org.apache.lucene.index.NumericDocValues; -import org.apache.lucene.index.SortedDocValues; -import org.apache.lucene.index.SortedNumericDocValues; -import org.apache.lucene.index.SortedSetDocValues; -import org.apache.lucene.util.Accountable; -import org.apache.lucene.util.Bits; - -import java.io.IOException; -import java.util.Collection; -import java.util.Collections; - -// TODO: move this to lucene's FilterCodecReader - -/** - * Base class for filtering DocValuesProducer implementations. - * <p> - * NOTE: just like with DocValuesProducer, the default {@link #getMergeInstance()} - * is unoptimized. overriding this method when possible can improve performance. - */ -class FilterDocValuesProducer extends DocValuesProducer { - /** The underlying Producer instance. */ - protected final DocValuesProducer in; - - /** - * Creates a new FilterDocValuesProducer - * @param in the underlying producer. - */ - FilterDocValuesProducer(DocValuesProducer in) { - this.in = in; - } - - @Override - public void close() throws IOException { - in.close(); - } - - @Override - public long ramBytesUsed() { - return in.ramBytesUsed(); - } - - @Override - public Collection<Accountable> getChildResources() { - return in.getChildResources(); - } - - @Override - public NumericDocValues getNumeric(FieldInfo field) throws IOException { - return in.getNumeric(field); - } - - @Override - public BinaryDocValues getBinary(FieldInfo field) throws IOException { - return in.getBinary(field); - } - - @Override - public SortedDocValues getSorted(FieldInfo field) throws IOException { - return in.getSorted(field); - } - - @Override - public SortedNumericDocValues getSortedNumeric(FieldInfo field) throws IOException { - return in.getSortedNumeric(field); - } - - @Override - public SortedSetDocValues getSortedSet(FieldInfo field) throws IOException { - return in.getSortedSet(field); - } - - @Override - public Bits getDocsWithField(FieldInfo field) throws IOException { - return in.getDocsWithField(field); - } - - @Override - public void checkIntegrity() throws IOException { - in.checkIntegrity(); - } - - // TODO: move this out somewhere else (and can fix all these null producers in lucene?) - // we shouldn't need nullness for any reason. - - public static final DocValuesProducer EMPTY = new DocValuesProducer() { - - @Override - public void close() throws IOException {} - - @Override - public long ramBytesUsed() { - return 0; - } - - @Override - public Collection<Accountable> getChildResources() { - return Collections.emptyList(); - } - - @Override - public NumericDocValues getNumeric(FieldInfo field) throws IOException { - throw new IllegalStateException(); // we don't have any docvalues - } - - @Override - public BinaryDocValues getBinary(FieldInfo field) throws IOException { - throw new IllegalStateException(); // we don't have any docvalues - } - - @Override - public SortedDocValues getSorted(FieldInfo field) throws IOException { - throw new IllegalStateException(); // we don't have any docvalues - } - - @Override - public SortedNumericDocValues getSortedNumeric(FieldInfo field) throws IOException { - throw new IllegalStateException(); // we don't have any docvalues - } - - @Override - public SortedSetDocValues getSortedSet(FieldInfo field) throws IOException { - throw new IllegalStateException(); // we don't have any docvalues - } - - @Override - public Bits getDocsWithField(FieldInfo field) throws IOException { - throw new IllegalStateException(); // we don't have any docvalues - } - - @Override - public void checkIntegrity() throws IOException {} - }; -} diff --git a/core/src/main/java/org/elasticsearch/index/shard/IndexSearcherWrapper.java b/core/src/main/java/org/elasticsearch/index/shard/IndexSearcherWrapper.java index 5603001a29..a2e738128e 100644 --- a/core/src/main/java/org/elasticsearch/index/shard/IndexSearcherWrapper.java +++ b/core/src/main/java/org/elasticsearch/index/shard/IndexSearcherWrapper.java @@ -40,8 +40,9 @@ public class IndexSearcherWrapper { * Wraps the given {@link DirectoryReader}. The wrapped reader can filter out document just like delete documents etc. but * must not change any term or document content. * <p> - * NOTE: The wrapper has a per-request lifecycle, must delegate {@link IndexReader#getCoreCacheKey()} and must be an instance - * of {@link FilterDirectoryReader} that eventually exposes the original reader via {@link FilterDirectoryReader#getDelegate()}. + * NOTE: The wrapper has a per-request lifecycle, must delegate {@link IndexReader#getReaderCacheHelper()}, + * {@link LeafReader#getCoreCacheHelper()} and must be an instance of {@link FilterDirectoryReader} that + * eventually exposes the original reader via {@link FilterDirectoryReader#getDelegate()}. * The returned reader is closed once it goes out of scope. * </p> * @param reader The provided directory reader to be wrapped to add custom functionality @@ -74,7 +75,7 @@ public class IndexSearcherWrapper { NonClosingReaderWrapper nonClosingReaderWrapper = new NonClosingReaderWrapper(engineSearcher.getDirectoryReader()); DirectoryReader reader = wrap(nonClosingReaderWrapper); if (reader != nonClosingReaderWrapper) { - if (reader.getCoreCacheKey() != elasticsearchDirectoryReader.getCoreCacheKey()) { + if (reader.getReaderCacheHelper() != elasticsearchDirectoryReader.getReaderCacheHelper()) { throw new IllegalStateException("wrapped directory reader doesn't delegate IndexReader#getCoreCacheKey, wrappers must override this method and delegate" + " to the original readers core cache key. Wrapped readers can't be used as cache keys since their are used only per request which would lead to subtle bugs"); } @@ -136,9 +137,10 @@ public class IndexSearcherWrapper { } @Override - public Object getCoreCacheKey() { - return in.getCoreCacheKey(); + public CacheHelper getReaderCacheHelper() { + return in.getReaderCacheHelper(); } + } } diff --git a/core/src/main/java/org/elasticsearch/index/similarity/SimilarityService.java b/core/src/main/java/org/elasticsearch/index/similarity/SimilarityService.java index e8203af852..2f3abf7a61 100644 --- a/core/src/main/java/org/elasticsearch/index/similarity/SimilarityService.java +++ b/core/src/main/java/org/elasticsearch/index/similarity/SimilarityService.java @@ -23,6 +23,8 @@ import org.apache.lucene.search.similarities.PerFieldSimilarityWrapper; import org.apache.lucene.search.similarities.Similarity; import org.elasticsearch.Version; import org.elasticsearch.common.TriFunction; +import org.elasticsearch.common.logging.DeprecationLogger; +import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.AbstractIndexComponent; import org.elasticsearch.index.IndexModule; @@ -36,9 +38,9 @@ import java.util.Map; public final class SimilarityService extends AbstractIndexComponent { + private static final DeprecationLogger DEPRECATION_LOGGER = new DeprecationLogger(Loggers.getLogger(SimilarityService.class)); public static final String DEFAULT_SIMILARITY = "BM25"; private final Similarity defaultSimilarity; - private final Similarity baseSimilarity; private final Map<String, SimilarityProvider> similarities; private static final Map<String, TriFunction<String, Settings, Settings, SimilarityProvider>> DEFAULTS; public static final Map<String, TriFunction<String, Settings, Settings, SimilarityProvider>> BUILT_IN; @@ -95,14 +97,14 @@ public final class SimilarityService extends AbstractIndexComponent { this.similarities = providers; defaultSimilarity = (providers.get("default") != null) ? providers.get("default").get() : providers.get(SimilarityService.DEFAULT_SIMILARITY).get(); - // Expert users can configure the base type as being different to default, but out-of-box we use default. - baseSimilarity = (providers.get("base") != null) ? providers.get("base").get() : - defaultSimilarity; + if (providers.get("base") != null) { + DEPRECATION_LOGGER.deprecated("The [base] similarity is ignored since query normalization and coords have been removed"); + } } public Similarity similarity(MapperService mapperService) { // TODO we can maybe factor out MapperService here entirely by introducing an interface for the lookup? - return (mapperService != null) ? new PerFieldSimilarity(defaultSimilarity, baseSimilarity, mapperService) : + return (mapperService != null) ? new PerFieldSimilarity(defaultSimilarity, mapperService) : defaultSimilarity; } @@ -134,8 +136,8 @@ public final class SimilarityService extends AbstractIndexComponent { private final Similarity defaultSimilarity; private final MapperService mapperService; - PerFieldSimilarity(Similarity defaultSimilarity, Similarity baseSimilarity, MapperService mapperService) { - super(baseSimilarity); + PerFieldSimilarity(Similarity defaultSimilarity, MapperService mapperService) { + super(); this.defaultSimilarity = defaultSimilarity; this.mapperService = mapperService; } diff --git a/core/src/main/java/org/elasticsearch/indices/IndicesQueryCache.java b/core/src/main/java/org/elasticsearch/indices/IndicesQueryCache.java index f33818ca32..3fda3d3f80 100644 --- a/core/src/main/java/org/elasticsearch/indices/IndicesQueryCache.java +++ b/core/src/main/java/org/elasticsearch/indices/IndicesQueryCache.java @@ -140,16 +140,6 @@ public class IndicesQueryCache extends AbstractComponent implements QueryCache, } @Override - public float getValueForNormalization() throws IOException { - return in.getValueForNormalization(); - } - - @Override - public void normalize(float norm, float topLevelBoost) { - in.normalize(norm, topLevelBoost); - } - - @Override public Scorer scorer(LeafReaderContext context) throws IOException { shardKeyMap.add(context.reader()); return in.scorer(context); diff --git a/core/src/main/java/org/elasticsearch/indices/IndicesRequestCache.java b/core/src/main/java/org/elasticsearch/indices/IndicesRequestCache.java index 48560f102d..bd6d75ea3d 100644 --- a/core/src/main/java/org/elasticsearch/indices/IndicesRequestCache.java +++ b/core/src/main/java/org/elasticsearch/indices/IndicesRequestCache.java @@ -245,7 +245,7 @@ public final class IndicesRequestCache extends AbstractComponent implements Remo } } - private class CleanupKey implements IndexReader.ReaderClosedListener { + private class CleanupKey implements IndexReader.ClosedListener { final CacheEntity entity; final long readerVersion; // use the reader version to now keep a reference to a "short" lived reader until its reaped @@ -255,7 +255,7 @@ public final class IndicesRequestCache extends AbstractComponent implements Remo } @Override - public void onClose(IndexReader reader) { + public void onClose(IndexReader.CacheKey cacheKey) { Boolean remove = registeredClosedListeners.remove(this); if (remove != null) { keysToClean.add(this); @@ -265,6 +265,9 @@ public final class IndicesRequestCache extends AbstractComponent implements Remo @Override public boolean equals(Object o) { if (this == o) return true; + if (o == null || getClass() != o.getClass()) { + return false; + } CleanupKey that = (CleanupKey) o; if (readerVersion != that.readerVersion) return false; if (!entity.getCacheIdentity().equals(that.entity.getCacheIdentity())) return false; diff --git a/core/src/main/java/org/elasticsearch/indices/fielddata/cache/IndicesFieldDataCache.java b/core/src/main/java/org/elasticsearch/indices/fielddata/cache/IndicesFieldDataCache.java index 860bdeb8ff..fcd925c258 100644 --- a/core/src/main/java/org/elasticsearch/indices/fielddata/cache/IndicesFieldDataCache.java +++ b/core/src/main/java/org/elasticsearch/indices/fielddata/cache/IndicesFieldDataCache.java @@ -23,7 +23,7 @@ import org.apache.logging.log4j.Logger; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.index.SegmentReader; +import org.apache.lucene.index.IndexReader.CacheKey; import org.apache.lucene.util.Accountable; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.cache.Cache; @@ -44,6 +44,7 @@ import org.elasticsearch.index.fielddata.IndexFieldDataCache; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.index.shard.ShardUtils; +import java.io.IOException; import java.util.ArrayList; import java.util.List; import java.util.function.ToLongBiFunction; @@ -107,7 +108,7 @@ public class IndicesFieldDataCache extends AbstractComponent implements RemovalL /** * A specific cache instance for the relevant parameters of it (index, fieldNames, fieldType). */ - static class IndexFieldCache implements IndexFieldDataCache, SegmentReader.CoreClosedListener, IndexReader.ReaderClosedListener { + static class IndexFieldCache implements IndexFieldDataCache, IndexReader.ClosedListener { private final Logger logger; final Index index; final String fieldName; @@ -125,10 +126,14 @@ public class IndicesFieldDataCache extends AbstractComponent implements RemovalL @Override public <FD extends AtomicFieldData, IFD extends IndexFieldData<FD>> FD load(final LeafReaderContext context, final IFD indexFieldData) throws Exception { final ShardId shardId = ShardUtils.extractShardId(context.reader()); - final Key key = new Key(this, context.reader().getCoreCacheKey(), shardId); + final IndexReader.CacheHelper cacheHelper = context.reader().getCoreCacheHelper(); + if (cacheHelper == null) { + throw new IllegalArgumentException("Reader " + context.reader() + " does not support caching"); + } + final Key key = new Key(this, cacheHelper.getKey(), shardId); //noinspection unchecked final Accountable accountable = cache.computeIfAbsent(key, k -> { - context.reader().addCoreClosedListener(IndexFieldCache.this); + cacheHelper.addClosedListener(IndexFieldCache.this); for (Listener listener : this.listeners) { k.listeners.add(listener); } @@ -149,7 +154,11 @@ public class IndicesFieldDataCache extends AbstractComponent implements RemovalL @Override public <FD extends AtomicFieldData, IFD extends IndexFieldData.Global<FD>> IFD load(final DirectoryReader indexReader, final IFD indexFieldData) throws Exception { final ShardId shardId = ShardUtils.extractShardId(indexReader); - final Key key = new Key(this, indexReader.getCoreCacheKey(), shardId); + final IndexReader.CacheHelper cacheHelper = indexReader.getReaderCacheHelper(); + if (cacheHelper == null) { + throw new IllegalArgumentException("Reader " + indexReader + " does not support caching"); + } + final Key key = new Key(this, cacheHelper.getKey(), shardId); //noinspection unchecked final Accountable accountable = cache.computeIfAbsent(key, k -> { ElasticsearchDirectoryReader.addReaderCloseListener(indexReader, IndexFieldCache.this); @@ -171,14 +180,8 @@ public class IndicesFieldDataCache extends AbstractComponent implements RemovalL } @Override - public void onClose(Object coreKey) { - cache.invalidate(new Key(this, coreKey, null)); - // don't call cache.cleanUp here as it would have bad performance implications - } - - @Override - public void onClose(IndexReader reader) { - cache.invalidate(new Key(this, reader.getCoreCacheKey(), null)); + public void onClose(CacheKey key) throws IOException { + cache.invalidate(new Key(this, key, null)); // don't call cache.cleanUp here as it would have bad performance implications } @@ -211,12 +214,12 @@ public class IndicesFieldDataCache extends AbstractComponent implements RemovalL public static class Key { public final IndexFieldCache indexCache; - public final Object readerKey; + public final IndexReader.CacheKey readerKey; public final ShardId shardId; public final List<IndexFieldDataCache.Listener> listeners = new ArrayList<>(); - Key(IndexFieldCache indexCache, Object readerKey, @Nullable ShardId shardId) { + Key(IndexFieldCache indexCache, IndexReader.CacheKey readerKey, @Nullable ShardId shardId) { this.indexCache = indexCache; this.readerKey = readerKey; this.shardId = shardId; diff --git a/core/src/main/java/org/elasticsearch/search/MultiValueMode.java b/core/src/main/java/org/elasticsearch/search/MultiValueMode.java index 90e3417ad1..2d6fd8a2b6 100644 --- a/core/src/main/java/org/elasticsearch/search/MultiValueMode.java +++ b/core/src/main/java/org/elasticsearch/search/MultiValueMode.java @@ -23,17 +23,19 @@ package org.elasticsearch.search; import org.apache.lucene.index.BinaryDocValues; import org.apache.lucene.index.DocValues; import org.apache.lucene.index.NumericDocValues; -import org.apache.lucene.index.RandomAccessOrds; import org.apache.lucene.index.SortedDocValues; import org.apache.lucene.index.SortedNumericDocValues; +import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.util.BitSet; -import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BytesRefBuilder; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.index.fielddata.AbstractBinaryDocValues; +import org.elasticsearch.index.fielddata.AbstractNumericDocValues; +import org.elasticsearch.index.fielddata.AbstractSortedDocValues; import org.elasticsearch.index.fielddata.FieldData; import org.elasticsearch.index.fielddata.NumericDoubleValues; import org.elasticsearch.index.fielddata.SortedBinaryDocValues; @@ -52,86 +54,65 @@ public enum MultiValueMode implements Writeable { */ SUM { @Override - protected long pick(SortedNumericDocValues values, long missingValue, int doc) { - values.setDocument(doc); - final int count = values.count(); - if (count > 0) { - long total = 0; - for (int index = 0; index < count; ++index) { - total += values.valueAt(index); - } - return total; - } else { - return missingValue; + protected long pick(SortedNumericDocValues values) throws IOException { + final int count = values.docValueCount(); + long total = 0; + for (int index = 0; index < count; ++index) { + total += values.nextValue(); } + return total; } @Override - protected long pick(SortedNumericDocValues values, long missingValue, DocIdSetIterator docItr, int startDoc, int endDoc) { - try { - int totalCount = 0; - long totalValue = 0; - for (int doc = startDoc; doc < endDoc; doc = docItr.nextDoc()) { - values.setDocument(doc); - final int count = values.count(); + protected long pick(SortedNumericDocValues values, long missingValue, DocIdSetIterator docItr, int startDoc, int endDoc) throws IOException { + int totalCount = 0; + long totalValue = 0; + for (int doc = startDoc; doc < endDoc; doc = docItr.nextDoc()) { + if (values.advanceExact(doc)) { + final int count = values.docValueCount(); for (int index = 0; index < count; ++index) { - totalValue += values.valueAt(index); + totalValue += values.nextValue(); } totalCount += count; } - return totalCount > 0 ? totalValue : missingValue; - } catch (IOException ioException) { - throw new RuntimeException(ioException); } + return totalCount > 0 ? totalValue : missingValue; } @Override - protected double pick(SortedNumericDoubleValues values, double missingValue, int doc) { - values.setDocument(doc); - final int count = values.count(); - if (count > 0) { - double total = 0; - for (int index = 0; index < count; ++index) { - total += values.valueAt(index); - } - return total; - } else { - return missingValue; + protected double pick(SortedNumericDoubleValues values) throws IOException { + final int count = values.docValueCount(); + double total = 0; + for (int index = 0; index < count; ++index) { + total += values.nextValue(); } + return total; } @Override - protected double pick(SortedNumericDoubleValues values, double missingValue, DocIdSetIterator docItr, int startDoc, int endDoc) { - try { - int totalCount = 0; - double totalValue = 0; - for (int doc = startDoc; doc < endDoc; doc = docItr.nextDoc()) { - values.setDocument(doc); - final int count = values.count(); + protected double pick(SortedNumericDoubleValues values, double missingValue, DocIdSetIterator docItr, int startDoc, int endDoc) throws IOException { + int totalCount = 0; + double totalValue = 0; + for (int doc = startDoc; doc < endDoc; doc = docItr.nextDoc()) { + if (values.advanceExact(doc)) { + final int count = values.docValueCount(); for (int index = 0; index < count; ++index) { - totalValue += values.valueAt(index); + totalValue += values.nextValue(); } totalCount += count; } - return totalCount > 0 ? totalValue : missingValue; - } catch (IOException ioException) { - throw new RuntimeException(ioException); } + return totalCount > 0 ? totalValue : missingValue; } @Override - protected double pick(UnsortedNumericDoubleValues values, double missingValue, int doc) { - values.setDocument(doc); - final int count = values.count(); - if (count > 0) { - double total = 0; - for (int index = 0; index < count; ++index) { - total += values.valueAt(index); - } - return total; - } else { - return missingValue; + protected double pick(UnsortedNumericDoubleValues values) throws IOException { + final int count = values.docValueCount(); + double total = 0; + for (int index = 0; index < count; ++index) { + total += values.nextValue(); } + return total; } }, @@ -140,92 +121,71 @@ public enum MultiValueMode implements Writeable { */ AVG { @Override - protected long pick(SortedNumericDocValues values, long missingValue, int doc) { - values.setDocument(doc); - final int count = values.count(); - if (count > 0) { - long total = 0; - for (int index = 0; index < count; ++index) { - total += values.valueAt(index); - } - return count > 1 ? Math.round((double)total/(double)count) : total; - } else { - return missingValue; + protected long pick(SortedNumericDocValues values) throws IOException { + final int count = values.docValueCount(); + long total = 0; + for (int index = 0; index < count; ++index) { + total += values.nextValue(); } + return count > 1 ? Math.round((double)total/(double)count) : total; } @Override - protected long pick(SortedNumericDocValues values, long missingValue, DocIdSetIterator docItr, int startDoc, int endDoc) { - try { - int totalCount = 0; - long totalValue = 0; - for (int doc = startDoc; doc < endDoc; doc = docItr.nextDoc()) { - values.setDocument(doc); - final int count = values.count(); + protected long pick(SortedNumericDocValues values, long missingValue, DocIdSetIterator docItr, int startDoc, int endDoc) throws IOException { + int totalCount = 0; + long totalValue = 0; + for (int doc = startDoc; doc < endDoc; doc = docItr.nextDoc()) { + if (values.advanceExact(doc)) { + final int count = values.docValueCount(); for (int index = 0; index < count; ++index) { - totalValue += values.valueAt(index); + totalValue += values.nextValue(); } totalCount += count; } - if (totalCount < 1) { - return missingValue; - } - return totalCount > 1 ? Math.round((double)totalValue/(double)totalCount) : totalValue; - } catch (IOException ioException) { - throw new RuntimeException(ioException); } + if (totalCount < 1) { + return missingValue; + } + return totalCount > 1 ? Math.round((double)totalValue/(double)totalCount) : totalValue; } @Override - protected double pick(SortedNumericDoubleValues values, double missingValue, int doc) { - values.setDocument(doc); - final int count = values.count(); - if (count > 0) { - double total = 0; - for (int index = 0; index < count; ++index) { - total += values.valueAt(index); - } - return total/count; - } else { - return missingValue; + protected double pick(SortedNumericDoubleValues values) throws IOException { + final int count = values.docValueCount(); + double total = 0; + for (int index = 0; index < count; ++index) { + total += values.nextValue(); } + return total/count; } @Override - protected double pick(SortedNumericDoubleValues values, double missingValue, DocIdSetIterator docItr, int startDoc, int endDoc) { - try { - int totalCount = 0; - double totalValue = 0; - for (int doc = startDoc; doc < endDoc; doc = docItr.nextDoc()) { - values.setDocument(doc); - final int count = values.count(); + protected double pick(SortedNumericDoubleValues values, double missingValue, DocIdSetIterator docItr, int startDoc, int endDoc) throws IOException { + int totalCount = 0; + double totalValue = 0; + for (int doc = startDoc; doc < endDoc; doc = docItr.nextDoc()) { + if (values.advanceExact(doc)) { + final int count = values.docValueCount(); for (int index = 0; index < count; ++index) { - totalValue += values.valueAt(index); + totalValue += values.nextValue(); } totalCount += count; } - if (totalCount < 1) { - return missingValue; - } - return totalValue/totalCount; - } catch (IOException ioException) { - throw new RuntimeException(ioException); } + if (totalCount < 1) { + return missingValue; + } + return totalValue/totalCount; } @Override - protected double pick(UnsortedNumericDoubleValues values, double missingValue, int doc) { - values.setDocument(doc); - final int count = values.count(); - if (count > 0) { - double total = 0; - for (int index = 0; index < count; ++index) { - total += values.valueAt(index); - } - return total/count; - } else { - return missingValue; + protected double pick(UnsortedNumericDoubleValues values) throws IOException { + final int count = values.docValueCount(); + double total = 0; + for (int index = 0; index < count; ++index) { + total += values.nextValue(); } + return total/count; } }, @@ -234,36 +194,28 @@ public enum MultiValueMode implements Writeable { */ MEDIAN { @Override - protected long pick(SortedNumericDocValues values, long missingValue, int doc) { - values.setDocument(doc); - int count = values.count(); - if (count > 0) { - if (count % 2 == 0) { - count /= 2; - return Math.round((values.valueAt(count - 1) + values.valueAt(count))/2.0); - } else { - count /= 2; - return values.valueAt(count); - } + protected long pick(SortedNumericDocValues values) throws IOException { + int count = values.docValueCount(); + for (int i = 0; i < (count - 1) / 2; ++i) { + values.nextValue(); + } + if (count % 2 == 0) { + return Math.round(((double) values.nextValue() + values.nextValue()) / 2); } else { - return missingValue; + return values.nextValue(); } } @Override - protected double pick(SortedNumericDoubleValues values, double missingValue, int doc) { - values.setDocument(doc); - int count = values.count(); - if (count > 0) { - if (count % 2 == 0) { - count /= 2; - return (values.valueAt(count - 1) + values.valueAt(count))/2.0; - } else { - count /= 2; - return values.valueAt(count); - } + protected double pick(SortedNumericDoubleValues values) throws IOException { + int count = values.docValueCount(); + for (int i = 0; i < (count - 1) / 2; ++i) { + values.nextValue(); + } + if (count % 2 == 0) { + return (values.nextValue() + values.nextValue()) / 2; } else { - return missingValue; + return values.nextValue(); } } }, @@ -273,119 +225,93 @@ public enum MultiValueMode implements Writeable { */ MIN { @Override - protected long pick(SortedNumericDocValues values, long missingValue, int doc) { - values.setDocument(doc); - final int count = values.count(); - return count > 0 ? values.valueAt(0) : missingValue; + protected long pick(SortedNumericDocValues values) throws IOException { + return values.nextValue(); } @Override - protected long pick(SortedNumericDocValues values, long missingValue, DocIdSetIterator docItr, int startDoc, int endDoc) { - try { - int totalCount = 0; - long minValue = Long.MAX_VALUE; - for (int doc = startDoc; doc < endDoc; doc = docItr.nextDoc()) { - values.setDocument(doc); - final int count = values.count(); - if (count > 0) { - minValue = Math.min(minValue, values.valueAt(0)); - } - totalCount += count; + protected long pick(SortedNumericDocValues values, long missingValue, DocIdSetIterator docItr, int startDoc, int endDoc) throws IOException { + boolean hasValue = false; + long minValue = Long.MAX_VALUE; + for (int doc = startDoc; doc < endDoc; doc = docItr.nextDoc()) { + if (values.advanceExact(doc)) { + minValue = Math.min(minValue, values.nextValue()); + hasValue = true; } - return totalCount > 0 ? minValue : missingValue; - } catch (IOException ioException) { - throw new RuntimeException(ioException); } + return hasValue ? minValue : missingValue; } @Override - protected double pick(SortedNumericDoubleValues values, double missingValue, int doc) { - values.setDocument(doc); - int count = values.count(); - return count > 0 ? values.valueAt(0) : missingValue; + protected double pick(SortedNumericDoubleValues values) throws IOException { + return values.nextValue(); } @Override - protected double pick(SortedNumericDoubleValues values, double missingValue, DocIdSetIterator docItr, int startDoc, int endDoc) { - try { - int totalCount = 0; - double minValue = Double.MAX_VALUE; - for (int doc = startDoc; doc < endDoc; doc = docItr.nextDoc()) { - values.setDocument(doc); - final int count = values.count(); - if (count > 0) { - minValue = Math.min(minValue, values.valueAt(0)); - } - totalCount += count; + protected double pick(SortedNumericDoubleValues values, double missingValue, DocIdSetIterator docItr, int startDoc, int endDoc) throws IOException { + boolean hasValue = false; + double minValue = Double.POSITIVE_INFINITY; + for (int doc = startDoc; doc < endDoc; doc = docItr.nextDoc()) { + if (values.advanceExact(doc)) { + minValue = Math.min(minValue, values.nextValue()); + hasValue = true; } - return totalCount > 0 ? minValue : missingValue; - } catch (IOException ioException) { - throw new RuntimeException(ioException); } + return hasValue ? minValue : missingValue; } @Override - protected BytesRef pick(SortedBinaryDocValues values, BytesRef missingValue, int doc) { - values.setDocument(doc); - final int count = values.count(); - return count > 0 ? values.valueAt(0) : missingValue; + protected BytesRef pick(SortedBinaryDocValues values) throws IOException { + return values.nextValue(); } @Override - protected BytesRef pick(BinaryDocValues values, BytesRefBuilder builder, DocIdSetIterator docItr, int startDoc, int endDoc) { - try { - BytesRefBuilder value = null; - for (int doc = startDoc; doc < endDoc; doc = docItr.nextDoc()) { - final BytesRef innerValue = values.get(doc); - if (innerValue != null) { - if (value == null) { - builder.copyBytes(innerValue); - value = builder; - } else { - final BytesRef min = value.get().compareTo(innerValue) <= 0 ? value.get() : innerValue; - if (min == innerValue) { - value.copyBytes(min); - } + protected BytesRef pick(BinaryDocValues values, BytesRefBuilder builder, DocIdSetIterator docItr, int startDoc, int endDoc) throws IOException { + BytesRefBuilder value = null; + for (int doc = startDoc; doc < endDoc; doc = docItr.nextDoc()) { + if (values.advanceExact(doc)) { + final BytesRef innerValue = values.binaryValue(); + if (value == null) { + builder.copyBytes(innerValue); + value = builder; + } else { + final BytesRef min = value.get().compareTo(innerValue) <= 0 ? value.get() : innerValue; + if (min == innerValue) { + value.copyBytes(min); } } } - return value == null ? null : value.get(); - } catch (IOException ioException) { - throw new RuntimeException(ioException); } + return value == null ? null : value.get(); } @Override - protected int pick(RandomAccessOrds values, int doc) { - values.setDocument(doc); - return values.cardinality() > 0 ? (int)values.ordAt(0) : -1; + protected int pick(SortedSetDocValues values) throws IOException { + return Math.toIntExact(values.nextOrd()); } @Override - protected int pick(SortedDocValues values, DocIdSetIterator docItr, int startDoc, int endDoc) { - try { - int ord = -1; - for (int doc = startDoc; doc < endDoc; doc = docItr.nextDoc()) { - final int innerOrd = values.getOrd(doc); - if (innerOrd != -1) { - ord = ord == -1 ? innerOrd : Math.min(ord, innerOrd); - } + protected int pick(SortedDocValues values, DocIdSetIterator docItr, int startDoc, int endDoc) throws IOException { + int ord = Integer.MAX_VALUE; + boolean hasValue = false; + for (int doc = startDoc; doc < endDoc; doc = docItr.nextDoc()) { + if (values.advanceExact(doc)) { + final int innerOrd = values.ordValue(); + ord = Math.min(ord, innerOrd); + hasValue = true; } - return ord; - } catch (IOException ioException) { - throw new RuntimeException(ioException); } + return hasValue ? ord : -1; } @Override - protected double pick(UnsortedNumericDoubleValues values, double missingValue, int doc) { - values.setDocument(doc); - int count = values.count(); - double min = Double.MAX_VALUE; + protected double pick(UnsortedNumericDoubleValues values) throws IOException { + int count = values.docValueCount(); + double min = Double.POSITIVE_INFINITY; for (int index = 0; index < count; ++index) { - min = Math.min(values.valueAt(index), min); + min = Math.min(values.nextValue(), min); } - return count > 0 ? min : missingValue; + return min; } }, @@ -394,120 +320,114 @@ public enum MultiValueMode implements Writeable { */ MAX { @Override - protected long pick(SortedNumericDocValues values, long missingValue, int doc) { - values.setDocument(doc); - final int count = values.count(); - return count > 0 ? values.valueAt(count - 1) : missingValue; + protected long pick(SortedNumericDocValues values) throws IOException { + final int count = values.docValueCount(); + for (int i = 0; i < count - 1; ++i) { + values.nextValue(); + } + return values.nextValue(); } @Override - protected long pick(SortedNumericDocValues values, long missingValue, DocIdSetIterator docItr, int startDoc, int endDoc) { - try { - int totalCount = 0; - long maxValue = Long.MIN_VALUE; - for (int doc = startDoc; doc < endDoc; doc = docItr.nextDoc()) { - values.setDocument(doc); - final int count = values.count(); - if (count > 0) { - maxValue = Math.max(maxValue, values.valueAt(count - 1)); + protected long pick(SortedNumericDocValues values, long missingValue, DocIdSetIterator docItr, int startDoc, int endDoc) throws IOException { + boolean hasValue = false; + long maxValue = Long.MIN_VALUE; + for (int doc = startDoc; doc < endDoc; doc = docItr.nextDoc()) { + if (values.advanceExact(doc)) { + final int count = values.docValueCount(); + for (int i = 0; i < count - 1; ++i) { + values.nextValue(); } - totalCount += count; + maxValue = Math.max(maxValue, values.nextValue()); + hasValue = true; } - return totalCount > 0 ? maxValue : missingValue; - } catch (IOException ioException) { - throw new RuntimeException(ioException); } + return hasValue ? maxValue : missingValue; } @Override - protected double pick(SortedNumericDoubleValues values, double missingValue, int doc) { - values.setDocument(doc); - final int count = values.count(); - return count > 0 ? values.valueAt(count - 1) : missingValue; + protected double pick(SortedNumericDoubleValues values) throws IOException { + final int count = values.docValueCount(); + for (int i = 0; i < count - 1; ++i) { + values.nextValue(); + } + return values.nextValue(); } @Override - protected double pick(SortedNumericDoubleValues values, double missingValue, DocIdSetIterator docItr, int startDoc, int endDoc) { - try { - int totalCount = 0; - double maxValue = Double.MIN_VALUE; - for (int doc = startDoc; doc < endDoc; doc = docItr.nextDoc()) { - values.setDocument(doc); - final int count = values.count(); - if (count > 0) { - maxValue = Math.max(maxValue, values.valueAt(count - 1)); + protected double pick(SortedNumericDoubleValues values, double missingValue, DocIdSetIterator docItr, int startDoc, int endDoc) throws IOException { + boolean hasValue = false; + double maxValue = Double.NEGATIVE_INFINITY; + for (int doc = startDoc; doc < endDoc; doc = docItr.nextDoc()) { + if (values.advanceExact(doc)) { + final int count = values.docValueCount(); + for (int i = 0; i < count - 1; ++i) { + values.nextValue(); } - totalCount += count; + maxValue = Math.max(maxValue, values.nextValue()); + hasValue = true; } - return totalCount > 0 ? maxValue : missingValue; - } catch (IOException ioException) { - throw new RuntimeException(ioException); } + return hasValue ? maxValue : missingValue; } @Override - protected BytesRef pick(SortedBinaryDocValues values, BytesRef missingValue, int doc) { - values.setDocument(doc); - final int count = values.count(); - return count > 0 ? values.valueAt(count - 1) : missingValue; + protected BytesRef pick(SortedBinaryDocValues values) throws IOException { + int count = values.docValueCount(); + for (int i = 0; i < count - 1; ++i) { + values.nextValue(); + } + return values.nextValue(); } @Override - protected BytesRef pick(BinaryDocValues values, BytesRefBuilder builder, DocIdSetIterator docItr, int startDoc, int endDoc) { - try { - BytesRefBuilder value = null; - for (int doc = startDoc; doc < endDoc; doc = docItr.nextDoc()) { - final BytesRef innerValue = values.get(doc); - if (innerValue != null) { - if (value == null) { - builder.copyBytes(innerValue); - value = builder; - } else { - final BytesRef max = value.get().compareTo(innerValue) > 0 ? value.get() : innerValue; - if (max == innerValue) { - value.copyBytes(max); - } + protected BytesRef pick(BinaryDocValues values, BytesRefBuilder builder, DocIdSetIterator docItr, int startDoc, int endDoc) throws IOException { + BytesRefBuilder value = null; + for (int doc = startDoc; doc < endDoc; doc = docItr.nextDoc()) { + if (values.advanceExact(doc)) { + final BytesRef innerValue = values.binaryValue(); + if (value == null) { + builder.copyBytes(innerValue); + value = builder; + } else { + final BytesRef max = value.get().compareTo(innerValue) > 0 ? value.get() : innerValue; + if (max == innerValue) { + value.copyBytes(max); } } } - return value == null ? null : value.get(); - } catch (IOException ioException) { - throw new RuntimeException(ioException); } + return value == null ? null : value.get(); } @Override - protected int pick(RandomAccessOrds values, int doc) { - values.setDocument(doc); - final int count = values.cardinality(); - return count > 0 ? (int)values.ordAt(count - 1) : -1; + protected int pick(SortedSetDocValues values) throws IOException { + long maxOrd = -1; + for (long ord = values.nextOrd(); ord != SortedSetDocValues.NO_MORE_ORDS; ord = values.nextOrd()) { + maxOrd = ord; + } + return Math.toIntExact(maxOrd); } @Override - protected int pick(SortedDocValues values, DocIdSetIterator docItr, int startDoc, int endDoc) { - try { - int ord = -1; - for (int doc = startDoc; doc < endDoc; doc = docItr.nextDoc()) { - final int innerOrd = values.getOrd(doc); - if (innerOrd != -1) { - ord = Math.max(ord, innerOrd); - } + protected int pick(SortedDocValues values, DocIdSetIterator docItr, int startDoc, int endDoc) throws IOException { + int ord = -1; + for (int doc = startDoc; doc < endDoc; doc = docItr.nextDoc()) { + if (values.advanceExact(doc)) { + ord = Math.max(ord, values.ordValue()); } - return ord; - } catch (IOException ioException) { - throw new RuntimeException(ioException); } + return ord; } @Override - protected double pick(UnsortedNumericDoubleValues values, double missingValue, int doc) { - values.setDocument(doc); - int count = values.count(); - double max = Double.MIN_VALUE; + protected double pick(UnsortedNumericDoubleValues values) throws IOException { + int count = values.docValueCount(); + double max = Double.NEGATIVE_INFINITY; for (int index = 0; index < count; ++index) { - max = Math.max(values.valueAt(index), max); + max = Math.max(values.nextValue(), max); } - return count > 0 ? max : missingValue; + return max; } }; @@ -534,32 +454,51 @@ public enum MultiValueMode implements Writeable { public NumericDocValues select(final SortedNumericDocValues values, final long missingValue) { final NumericDocValues singleton = DocValues.unwrapSingleton(values); if (singleton != null) { - final Bits docsWithField = DocValues.unwrapSingletonBits(values); - if (docsWithField == null || missingValue == 0) { - return singleton; - } else { - return new NumericDocValues() { - @Override - public long get(int docID) { - final long value = singleton.get(docID); - if (value == 0 && docsWithField.get(docID) == false) { - return missingValue; - } - return value; - } - }; - } + return new AbstractNumericDocValues() { + + private boolean hasValue; + + @Override + public boolean advanceExact(int target) throws IOException { + hasValue = singleton.advanceExact(target); + return true; + } + + @Override + public int docID() { + return singleton.docID(); + } + + @Override + public long longValue() throws IOException { + return hasValue ? singleton.longValue() : missingValue; + } + }; } else { - return new NumericDocValues() { + return new AbstractNumericDocValues() { + + private boolean hasValue; + + @Override + public boolean advanceExact(int target) throws IOException { + hasValue = values.advanceExact(target); + return true; + } + + @Override + public int docID() { + return values.docID(); + } + @Override - public long get(int docID) { - return pick(values, missingValue, docID); + public long longValue() throws IOException { + return hasValue ? pick(values) : missingValue; } }; } } - protected long pick(SortedNumericDocValues values, long missingValue, int doc) { + protected long pick(SortedNumericDocValues values) throws IOException { throw new IllegalArgumentException("Unsupported sort mode: " + this); } @@ -580,38 +519,47 @@ public enum MultiValueMode implements Writeable { return select(DocValues.emptySortedNumeric(maxDoc), missingValue); } - return new NumericDocValues() { + return new AbstractNumericDocValues() { - int lastSeenRootDoc = 0; + int lastSeenRootDoc = -1; long lastEmittedValue = missingValue; @Override - public long get(int rootDoc) { + public boolean advanceExact(int rootDoc) throws IOException { assert rootDocs.get(rootDoc) : "can only sort root documents"; assert rootDoc >= lastSeenRootDoc : "can only evaluate current and upcoming root docs"; if (rootDoc == lastSeenRootDoc) { - return lastEmittedValue; + return true; + } else if (rootDoc == 0) { + lastEmittedValue = missingValue; + return true; + } + final int prevRootDoc = rootDocs.prevSetBit(rootDoc - 1); + final int firstNestedDoc; + if (innerDocs.docID() > prevRootDoc) { + firstNestedDoc = innerDocs.docID(); + } else { + firstNestedDoc = innerDocs.advance(prevRootDoc + 1); } - try { - final int prevRootDoc = rootDocs.prevSetBit(rootDoc - 1); - final int firstNestedDoc; - if (innerDocs.docID() > prevRootDoc) { - firstNestedDoc = innerDocs.docID(); - } else { - firstNestedDoc = innerDocs.advance(prevRootDoc + 1); - } - lastSeenRootDoc = rootDoc; - lastEmittedValue = pick(values, missingValue, innerDocs, firstNestedDoc, rootDoc); - return lastEmittedValue; - } catch (IOException e) { - throw new RuntimeException(e); - } + lastSeenRootDoc = rootDoc; + lastEmittedValue = pick(values, missingValue, innerDocs, firstNestedDoc, rootDoc); + return true; + } + + @Override + public int docID() { + return lastSeenRootDoc; + } + + @Override + public long longValue() { + return lastEmittedValue; } }; } - protected long pick(SortedNumericDocValues values, long missingValue, DocIdSetIterator docItr, int startDoc, int endDoc) { + protected long pick(SortedNumericDocValues values, long missingValue, DocIdSetIterator docItr, int startDoc, int endDoc) throws IOException { throw new IllegalArgumentException("Unsupported sort mode: " + this); } @@ -625,32 +573,42 @@ public enum MultiValueMode implements Writeable { public NumericDoubleValues select(final SortedNumericDoubleValues values, final double missingValue) { final NumericDoubleValues singleton = FieldData.unwrapSingleton(values); if (singleton != null) { - final Bits docsWithField = FieldData.unwrapSingletonBits(values); - if (docsWithField == null || missingValue == 0) { - return singleton; - } else { - return new NumericDoubleValues() { - @Override - public double get(int docID) { - final double value = singleton.get(docID); - if (value == 0 && docsWithField.get(docID) == false) { - return missingValue; - } - return value; - } - }; - } + return new NumericDoubleValues() { + + private boolean hasValue; + + @Override + public boolean advanceExact(int doc) throws IOException { + hasValue = singleton.advanceExact(doc); + return true; + } + + @Override + public double doubleValue() throws IOException { + return hasValue ? singleton.doubleValue() : missingValue; + } + + }; } else { return new NumericDoubleValues() { + + private boolean hasValue; + + @Override + public boolean advanceExact(int target) throws IOException { + hasValue = values.advanceExact(target); + return true; + } + @Override - public double get(int docID) { - return pick(values, missingValue, docID); + public double doubleValue() throws IOException { + return hasValue ? pick(values) : missingValue; } }; } } - protected double pick(SortedNumericDoubleValues values, double missingValue, int doc) { + protected double pick(SortedNumericDoubleValues values) throws IOException { throw new IllegalArgumentException("Unsupported sort mode: " + this); } @@ -668,7 +626,7 @@ public enum MultiValueMode implements Writeable { */ public NumericDoubleValues select(final SortedNumericDoubleValues values, final double missingValue, final BitSet rootDocs, final DocIdSetIterator innerDocs, int maxDoc) throws IOException { if (rootDocs == null || innerDocs == null) { - return select(FieldData.emptySortedNumericDoubles(maxDoc), missingValue); + return select(FieldData.emptySortedNumericDoubles(), missingValue); } return new NumericDoubleValues() { @@ -677,32 +635,33 @@ public enum MultiValueMode implements Writeable { double lastEmittedValue = missingValue; @Override - public double get(int rootDoc) { + public boolean advanceExact(int rootDoc) throws IOException { assert rootDocs.get(rootDoc) : "can only sort root documents"; assert rootDoc >= lastSeenRootDoc : "can only evaluate current and upcoming root docs"; if (rootDoc == lastSeenRootDoc) { - return lastEmittedValue; + return true; } - try { - final int prevRootDoc = rootDocs.prevSetBit(rootDoc - 1); - final int firstNestedDoc; - if (innerDocs.docID() > prevRootDoc) { - firstNestedDoc = innerDocs.docID(); - } else { - firstNestedDoc = innerDocs.advance(prevRootDoc + 1); - } - - lastSeenRootDoc = rootDoc; - lastEmittedValue = pick(values, missingValue, innerDocs, firstNestedDoc, rootDoc); - return lastEmittedValue; - } catch (IOException e) { - throw new RuntimeException(e); + final int prevRootDoc = rootDocs.prevSetBit(rootDoc - 1); + final int firstNestedDoc; + if (innerDocs.docID() > prevRootDoc) { + firstNestedDoc = innerDocs.docID(); + } else { + firstNestedDoc = innerDocs.advance(prevRootDoc + 1); } + + lastSeenRootDoc = rootDoc; + lastEmittedValue = pick(values, missingValue, innerDocs, firstNestedDoc, rootDoc); + return true; + } + + @Override + public double doubleValue() throws IOException { + return lastEmittedValue; } }; } - protected double pick(SortedNumericDoubleValues values, double missingValue, DocIdSetIterator docItr, int startDoc, int endDoc) { + protected double pick(SortedNumericDoubleValues values, double missingValue, DocIdSetIterator docItr, int startDoc, int endDoc) throws IOException { throw new IllegalArgumentException("Unsupported sort mode: " + this); } @@ -716,32 +675,49 @@ public enum MultiValueMode implements Writeable { public BinaryDocValues select(final SortedBinaryDocValues values, final BytesRef missingValue) { final BinaryDocValues singleton = FieldData.unwrapSingleton(values); if (singleton != null) { - final Bits docsWithField = FieldData.unwrapSingletonBits(values); - if (docsWithField == null) { + if (missingValue == null) { return singleton; - } else { - return new BinaryDocValues() { - @Override - public BytesRef get(int docID) { - final BytesRef value = singleton.get(docID); - if (value.length == 0 && docsWithField.get(docID) == false) { - return missingValue; - } - return value; - } - }; } + return new AbstractBinaryDocValues() { + + private boolean hasValue; + + @Override + public boolean advanceExact(int target) throws IOException { + hasValue = singleton.advanceExact(target); + return true; + } + + @Override + public BytesRef binaryValue() throws IOException { + return hasValue ? singleton.binaryValue() : missingValue; + } + }; } else { - return new BinaryDocValues() { + return new AbstractBinaryDocValues() { + + private BytesRef value; + + @Override + public boolean advanceExact(int target) throws IOException { + if (values.advanceExact(target)) { + value = pick(values); + return true; + } else { + value = missingValue; + return missingValue != null; + } + } + @Override - public BytesRef get(int docID) { - return pick(values, missingValue, docID); + public BytesRef binaryValue() throws IOException { + return value; } }; } } - protected BytesRef pick(SortedBinaryDocValues values, BytesRef missingValue, int doc) { + protected BytesRef pick(SortedBinaryDocValues values) throws IOException { throw new IllegalArgumentException("Unsupported sort mode: " + this); } @@ -759,11 +735,11 @@ public enum MultiValueMode implements Writeable { */ public BinaryDocValues select(final SortedBinaryDocValues values, final BytesRef missingValue, final BitSet rootDocs, final DocIdSetIterator innerDocs, int maxDoc) throws IOException { if (rootDocs == null || innerDocs == null) { - return select(FieldData.emptySortedBinary(maxDoc), missingValue); + return select(FieldData.emptySortedBinary(), missingValue); } final BinaryDocValues selectedValues = select(values, null); - return new BinaryDocValues() { + return new AbstractBinaryDocValues() { final BytesRefBuilder builder = new BytesRefBuilder(); @@ -771,36 +747,37 @@ public enum MultiValueMode implements Writeable { BytesRef lastEmittedValue = missingValue; @Override - public BytesRef get(int rootDoc) { + public boolean advanceExact(int rootDoc) throws IOException { assert rootDocs.get(rootDoc) : "can only sort root documents"; assert rootDoc >= lastSeenRootDoc : "can only evaluate current and upcoming root docs"; if (rootDoc == lastSeenRootDoc) { - return lastEmittedValue; + return true; } - try { - final int prevRootDoc = rootDocs.prevSetBit(rootDoc - 1); - final int firstNestedDoc; - if (innerDocs.docID() > prevRootDoc) { - firstNestedDoc = innerDocs.docID(); - } else { - firstNestedDoc = innerDocs.advance(prevRootDoc + 1); - } + final int prevRootDoc = rootDocs.prevSetBit(rootDoc - 1); + final int firstNestedDoc; + if (innerDocs.docID() > prevRootDoc) { + firstNestedDoc = innerDocs.docID(); + } else { + firstNestedDoc = innerDocs.advance(prevRootDoc + 1); + } - lastSeenRootDoc = rootDoc; - lastEmittedValue = pick(selectedValues, builder, innerDocs, firstNestedDoc, rootDoc); - if (lastEmittedValue == null) { - lastEmittedValue = missingValue; - } - return lastEmittedValue; - } catch (IOException e) { - throw new RuntimeException(e); + lastSeenRootDoc = rootDoc; + lastEmittedValue = pick(selectedValues, builder, innerDocs, firstNestedDoc, rootDoc); + if (lastEmittedValue == null) { + lastEmittedValue = missingValue; } + return true; + } + + @Override + public BytesRef binaryValue() throws IOException { + return lastEmittedValue; } }; } - protected BytesRef pick(BinaryDocValues values, BytesRefBuilder builder, DocIdSetIterator docItr, int startDoc, int endDoc) { + protected BytesRef pick(BinaryDocValues values, BytesRefBuilder builder, DocIdSetIterator docItr, int startDoc, int endDoc) throws IOException { throw new IllegalArgumentException("Unsupported sort mode: " + this); } @@ -810,7 +787,7 @@ public enum MultiValueMode implements Writeable { * * Allowed Modes: MIN, MAX */ - public SortedDocValues select(final RandomAccessOrds values) { + public SortedDocValues select(final SortedSetDocValues values) { if (values.getValueCount() >= Integer.MAX_VALUE) { throw new UnsupportedOperationException("fields containing more than " + (Integer.MAX_VALUE-1) + " unique terms are unsupported"); } @@ -819,26 +796,46 @@ public enum MultiValueMode implements Writeable { if (singleton != null) { return singleton; } else { - return new SortedDocValues() { + return new AbstractSortedDocValues() { + + int ord; + @Override - public int getOrd(int docID) { - return pick(values, docID); + public boolean advanceExact(int target) throws IOException { + if (values.advanceExact(target)) { + ord = pick(values); + return true; + } else { + ord = -1; + return false; + } } @Override - public BytesRef lookupOrd(int ord) { + public int docID() { + return values.docID(); + } + + @Override + public int ordValue() { + assert ord != -1; + return ord; + } + + @Override + public BytesRef lookupOrd(int ord) throws IOException { return values.lookupOrd(ord); } @Override public int getValueCount() { - return (int)values.getValueCount(); + return (int) values.getValueCount(); } }; } } - protected int pick(RandomAccessOrds values, int doc) { + protected int pick(SortedSetDocValues values) throws IOException { throw new IllegalArgumentException("Unsupported sort mode: " + this); } @@ -853,19 +850,20 @@ public enum MultiValueMode implements Writeable { * NOTE: Calling the returned instance on docs that are not root docs is illegal * The returned instance can only be evaluate the current and upcoming docs */ - public SortedDocValues select(final RandomAccessOrds values, final BitSet rootDocs, final DocIdSetIterator innerDocs) throws IOException { + public SortedDocValues select(final SortedSetDocValues values, final BitSet rootDocs, final DocIdSetIterator innerDocs) throws IOException { if (rootDocs == null || innerDocs == null) { return select(DocValues.emptySortedSet()); } final SortedDocValues selectedValues = select(values); - return new SortedDocValues() { + return new AbstractSortedDocValues() { + int docID = -1; int lastSeenRootDoc = 0; int lastEmittedOrd = -1; @Override - public BytesRef lookupOrd(int ord) { + public BytesRef lookupOrd(int ord) throws IOException { return selectedValues.lookupOrd(ord); } @@ -875,32 +873,39 @@ public enum MultiValueMode implements Writeable { } @Override - public int getOrd(int rootDoc) { + public boolean advanceExact(int rootDoc) throws IOException { assert rootDocs.get(rootDoc) : "can only sort root documents"; assert rootDoc >= lastSeenRootDoc : "can only evaluate current and upcoming root docs"; if (rootDoc == lastSeenRootDoc) { - return lastEmittedOrd; + return lastEmittedOrd != -1; } - try { - final int prevRootDoc = rootDocs.prevSetBit(rootDoc - 1); - final int firstNestedDoc; - if (innerDocs.docID() > prevRootDoc) { - firstNestedDoc = innerDocs.docID(); - } else { - firstNestedDoc = innerDocs.advance(prevRootDoc + 1); - } - - lastSeenRootDoc = rootDoc; - return lastEmittedOrd = pick(selectedValues, innerDocs, firstNestedDoc, rootDoc); - } catch (IOException e) { - throw new RuntimeException(e); + final int prevRootDoc = rootDocs.prevSetBit(rootDoc - 1); + final int firstNestedDoc; + if (innerDocs.docID() > prevRootDoc) { + firstNestedDoc = innerDocs.docID(); + } else { + firstNestedDoc = innerDocs.advance(prevRootDoc + 1); } + + docID = lastSeenRootDoc = rootDoc; + lastEmittedOrd = pick(selectedValues, innerDocs, firstNestedDoc, rootDoc); + return lastEmittedOrd != -1; + } + + @Override + public int docID() { + return docID; + } + + @Override + public int ordValue() { + return lastEmittedOrd; } }; } - protected int pick(SortedDocValues values, DocIdSetIterator docItr, int startDoc, int endDoc) { + protected int pick(SortedDocValues values, DocIdSetIterator docItr, int startDoc, int endDoc) throws IOException { throw new IllegalArgumentException("Unsupported sort mode: " + this); } @@ -913,24 +918,32 @@ public enum MultiValueMode implements Writeable { */ public NumericDoubleValues select(final UnsortedNumericDoubleValues values, final double missingValue) { return new NumericDoubleValues() { + private boolean hasValue; + + @Override + public boolean advanceExact(int doc) throws IOException { + hasValue = values.advanceExact(doc); + return true; + } @Override - public double get(int docID) { - return pick(values, missingValue, docID); + public double doubleValue() throws IOException { + return hasValue ? pick(values) : missingValue; } }; } - protected double pick(UnsortedNumericDoubleValues values, final double missingValue, int doc) { + protected double pick(UnsortedNumericDoubleValues values) throws IOException { throw new IllegalArgumentException("Unsupported sort mode: " + this); } /** * Interface allowing custom value generators to be used in MultiValueMode. */ + // TODO: why do we need it??? public interface UnsortedNumericDoubleValues { - int count(); - void setDocument(int docId); - double valueAt(int index); + boolean advanceExact(int doc) throws IOException; + int docValueCount() throws IOException; + double nextValue() throws IOException; } @Override diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/children/ParentToChildrenAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/children/ParentToChildrenAggregator.java index 5aa55c771b..37a443e9ba 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/children/ParentToChildrenAggregator.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/children/ParentToChildrenAggregator.java @@ -57,20 +57,25 @@ public class ParentToChildrenAggregator extends SingleBucketAggregator { private final Weight parentFilter; private final ValuesSource.Bytes.WithOrdinals.ParentChild valuesSource; - // Maybe use PagedGrowableWriter? This will be less wasteful than LongArray, but then we don't have the reuse feature of BigArrays. - // Also if we know the highest possible value that a parent agg will create then we store multiple values into one slot + // Maybe use PagedGrowableWriter? This will be less wasteful than LongArray, + // but then we don't have the reuse feature of BigArrays. + // Also if we know the highest possible value that a parent agg will create + // then we store multiple values into one slot private final LongArray parentOrdToBuckets; // Only pay the extra storage price if the a parentOrd has multiple buckets - // Most of the times a parent doesn't have multiple buckets, since there is only one document per parent ord, - // only in the case of terms agg if a parent doc has multiple terms per field this is needed: + // Most of the times a parent doesn't have multiple buckets, since there is + // only one document per parent ord, + // only in the case of terms agg if a parent doc has multiple terms per + // field this is needed: private final LongObjectPagedHashMap<long[]> parentOrdToOtherBuckets; private boolean multipleBucketsPerParentOrd = false; - public ParentToChildrenAggregator(String name, AggregatorFactories factories, SearchContext context, - Aggregator parent, String parentType, Query childFilter, Query parentFilter, - ValuesSource.Bytes.WithOrdinals.ParentChild valuesSource, - long maxOrd, List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData) throws IOException { + public ParentToChildrenAggregator(String name, AggregatorFactories factories, + SearchContext context, Aggregator parent, String parentType, Query childFilter, + Query parentFilter, ValuesSource.Bytes.WithOrdinals.ParentChild valuesSource, + long maxOrd, List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData) + throws IOException { super(name, factories, context, parent, pipelineAggregators, metaData); this.parentType = parentType; // these two filters are cached in the parser @@ -84,13 +89,14 @@ public class ParentToChildrenAggregator extends SingleBucketAggregator { @Override public InternalAggregation buildAggregation(long owningBucketOrdinal) throws IOException { - return new InternalChildren(name, bucketDocCount(owningBucketOrdinal), bucketAggregations(owningBucketOrdinal), pipelineAggregators(), - metaData()); + return new InternalChildren(name, bucketDocCount(owningBucketOrdinal), + bucketAggregations(owningBucketOrdinal), pipelineAggregators(), metaData()); } @Override public InternalAggregation buildEmptyAggregation() { - return new InternalChildren(name, 0, buildEmptySubAggregations(), pipelineAggregators(), metaData()); + return new InternalChildren(name, 0, buildEmptySubAggregations(), pipelineAggregators(), + metaData()); } @Override @@ -108,8 +114,8 @@ public class ParentToChildrenAggregator extends SingleBucketAggregator { @Override public void collect(int docId, long bucket) throws IOException { - if (parentDocs.get(docId)) { - long globalOrdinal = globalOrdinals.getOrd(docId); + if (parentDocs.get(docId) && globalOrdinals.advanceExact(docId)) { + long globalOrdinal = globalOrdinals.ordValue(); if (globalOrdinal != -1) { if (parentOrdToBuckets.get(globalOrdinal) == -1) { parentOrdToBuckets.set(globalOrdinal, bucket); @@ -120,7 +126,7 @@ public class ParentToChildrenAggregator extends SingleBucketAggregator { bucketOrds[bucketOrds.length - 1] = bucket; parentOrdToOtherBuckets.put(globalOrdinal, bucketOrds); } else { - parentOrdToOtherBuckets.put(globalOrdinal, new long[]{bucket}); + parentOrdToOtherBuckets.put(globalOrdinal, new long[] { bucket }); } multipleBucketsPerParentOrd = true; } @@ -141,18 +147,21 @@ public class ParentToChildrenAggregator extends SingleBucketAggregator { DocIdSetIterator childDocsIter = childDocsScorer.iterator(); final LeafBucketCollector sub = collectableSubAggregators.getLeafCollector(ctx); - final SortedDocValues globalOrdinals = valuesSource.globalOrdinalsValues(parentType, ctx); + final SortedDocValues globalOrdinals = valuesSource.globalOrdinalsValues(parentType, + ctx); // Set the scorer, since we now replay only the child docIds - sub.setScorer(new ConstantScoreScorer(null, 1f,childDocsIter)); + sub.setScorer(new ConstantScoreScorer(null, 1f, childDocsIter)); final Bits liveDocs = ctx.reader().getLiveDocs(); - for (int docId = childDocsIter.nextDoc(); docId != DocIdSetIterator.NO_MORE_DOCS; docId = childDocsIter.nextDoc()) { + for (int docId = childDocsIter + .nextDoc(); docId != DocIdSetIterator.NO_MORE_DOCS; docId = childDocsIter + .nextDoc()) { if (liveDocs != null && liveDocs.get(docId) == false) { continue; } - long globalOrdinal = globalOrdinals.getOrd(docId); - if (globalOrdinal != -1) { + if (globalOrdinals.advanceExact(docId)) { + long globalOrdinal = globalOrdinals.ordValue(); long bucketOrd = parentOrdToBuckets.get(globalOrdinal); if (bucketOrd != -1) { collectBucket(sub, docId, bucketOrd); diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridAggregationBuilder.java index 602c3a81c6..1c60247446 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridAggregationBuilder.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridAggregationBuilder.java @@ -28,10 +28,10 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.xcontent.ObjectParser; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.index.fielddata.AbstractSortingNumericDocValues; import org.elasticsearch.index.fielddata.MultiGeoPointValues; import org.elasticsearch.index.fielddata.SortedBinaryDocValues; import org.elasticsearch.index.fielddata.SortedNumericDoubleValues; -import org.elasticsearch.index.fielddata.SortingNumericDocValues; import org.elasticsearch.index.query.QueryParseContext; import org.elasticsearch.search.aggregations.AggregatorFactories.Builder; import org.elasticsearch.search.aggregations.AggregatorFactory; @@ -187,7 +187,7 @@ public class GeoGridAggregationBuilder extends ValuesSourceAggregationBuilder<Va return NAME; } - private static class CellValues extends SortingNumericDocValues { + private static class CellValues extends AbstractSortingNumericDocValues { private MultiGeoPointValues geoValues; private int precision; @@ -197,14 +197,19 @@ public class GeoGridAggregationBuilder extends ValuesSourceAggregationBuilder<Va } @Override - public void setDocument(int docId) { - geoValues.setDocument(docId); - resize(geoValues.count()); - for (int i = 0; i < count(); ++i) { - GeoPoint target = geoValues.valueAt(i); - values[i] = GeoHashUtils.longEncode(target.getLon(), target.getLat(), precision); + public boolean advanceExact(int docId) throws IOException { + if (geoValues.advanceExact(docId)) { + resize(geoValues.docValueCount()); + for (int i = 0; i < docValueCount(); ++i) { + GeoPoint target = geoValues.nextValue(); + values[i] = GeoHashUtils.longEncode(target.getLon(), target.getLat(), + precision); + } + sort(); + return true; + } else { + return false; } - sort(); } } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridAggregator.java index d20b216cd0..ebcb7b39db 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridAggregator.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridAggregator.java @@ -69,21 +69,22 @@ public class GeoHashGridAggregator extends BucketsAggregator { @Override public void collect(int doc, long bucket) throws IOException { assert bucket == 0; - values.setDocument(doc); - final int valuesCount = values.count(); - - long previous = Long.MAX_VALUE; - for (int i = 0; i < valuesCount; ++i) { - final long val = values.valueAt(i); - if (previous != val || i == 0) { - long bucketOrdinal = bucketOrds.add(val); - if (bucketOrdinal < 0) { // already seen - bucketOrdinal = - 1 - bucketOrdinal; - collectExistingBucket(sub, doc, bucketOrdinal); - } else { - collectBucket(sub, doc, bucketOrdinal); + if (values.advanceExact(doc)) { + final int valuesCount = values.docValueCount(); + + long previous = Long.MAX_VALUE; + for (int i = 0; i < valuesCount; ++i) { + final long val = values.nextValue(); + if (previous != val || i == 0) { + long bucketOrdinal = bucketOrds.add(val); + if (bucketOrdinal < 0) { // already seen + bucketOrdinal = -1 - bucketOrdinal; + collectExistingBucket(sub, doc, bucketOrdinal); + } else { + collectBucket(sub, doc, bucketOrdinal); + } + previous = val; } - previous = val; } } } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregator.java index a52f2b2cfb..871cade50c 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregator.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregator.java @@ -97,25 +97,26 @@ class DateHistogramAggregator extends BucketsAggregator { @Override public void collect(int doc, long bucket) throws IOException { assert bucket == 0; - values.setDocument(doc); - final int valuesCount = values.count(); - - long previousRounded = Long.MIN_VALUE; - for (int i = 0; i < valuesCount; ++i) { - long value = values.valueAt(i); - long rounded = rounding.round(value - offset) + offset; - assert rounded >= previousRounded; - if (rounded == previousRounded) { - continue; + if (values.advanceExact(doc)) { + final int valuesCount = values.docValueCount(); + + long previousRounded = Long.MIN_VALUE; + for (int i = 0; i < valuesCount; ++i) { + long value = values.nextValue(); + long rounded = rounding.round(value - offset) + offset; + assert rounded >= previousRounded; + if (rounded == previousRounded) { + continue; + } + long bucketOrd = bucketOrds.add(rounded); + if (bucketOrd < 0) { // already seen + bucketOrd = -1 - bucketOrd; + collectExistingBucket(sub, doc, bucketOrd); + } else { + collectBucket(sub, doc, bucketOrd); + } + previousRounded = rounded; } - long bucketOrd = bucketOrds.add(rounded); - if (bucketOrd < 0) { // already seen - bucketOrd = -1 - bucketOrd; - collectExistingBucket(sub, doc, bucketOrd); - } else { - collectBucket(sub, doc, bucketOrd); - } - previousRounded = rounded; } } }; diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/HistogramAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/HistogramAggregator.java index 4b547989d8..a3a038cfa3 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/HistogramAggregator.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/HistogramAggregator.java @@ -101,25 +101,26 @@ class HistogramAggregator extends BucketsAggregator { @Override public void collect(int doc, long bucket) throws IOException { assert bucket == 0; - values.setDocument(doc); - final int valuesCount = values.count(); - - double previousKey = Double.NEGATIVE_INFINITY; - for (int i = 0; i < valuesCount; ++i) { - double value = values.valueAt(i); - double key = Math.floor((value - offset) / interval); - assert key >= previousKey; - if (key == previousKey) { - continue; + if (values.advanceExact(doc)) { + final int valuesCount = values.docValueCount(); + + double previousKey = Double.NEGATIVE_INFINITY; + for (int i = 0; i < valuesCount; ++i) { + double value = values.nextValue(); + double key = Math.floor((value - offset) / interval); + assert key >= previousKey; + if (key == previousKey) { + continue; + } + long bucketOrd = bucketOrds.add(Double.doubleToLongBits(key)); + if (bucketOrd < 0) { // already seen + bucketOrd = -1 - bucketOrd; + collectExistingBucket(sub, doc, bucketOrd); + } else { + collectBucket(sub, doc, bucketOrd); + } + previousKey = key; } - long bucketOrd = bucketOrds.add(Double.doubleToLongBits(key)); - if (bucketOrd < 0) { // already seen - bucketOrd = -1 - bucketOrd; - collectExistingBucket(sub, doc, bucketOrd); - } else { - collectBucket(sub, doc, bucketOrd); - } - previousKey = key; } } }; diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/BinaryRangeAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/BinaryRangeAggregator.java index 873eef2f71..9151ebc985 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/BinaryRangeAggregator.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/BinaryRangeAggregator.java @@ -127,7 +127,7 @@ public final class BinaryRangeAggregator extends BucketsAggregator { final LeafBucketCollector sub; SortedSetRangeLeafCollector(SortedSetDocValues values, - Range[] ranges, LeafBucketCollector sub) { + Range[] ranges, LeafBucketCollector sub) throws IOException { super(sub, values); for (int i = 1; i < ranges.length; ++i) { if (RANGE_COMPARATOR.compare(ranges[i-1], ranges[i]) > 0) { @@ -167,10 +167,13 @@ public final class BinaryRangeAggregator extends BucketsAggregator { @Override public void collect(int doc, long bucket) throws IOException { - values.setDocument(doc); - int lo = 0; - for (long ord = values.nextOrd(); ord != SortedSetDocValues.NO_MORE_ORDS; ord = values.nextOrd()) { - lo = collect(doc, ord, bucket, lo); + if (values.advanceExact(doc)) { + int lo = 0; + for (long ord = values + .nextOrd(); ord != SortedSetDocValues.NO_MORE_ORDS; ord = values + .nextOrd()) { + lo = collect(doc, ord, bucket, lo); + } } } @@ -259,11 +262,12 @@ public final class BinaryRangeAggregator extends BucketsAggregator { @Override public void collect(int doc, long bucket) throws IOException { - values.setDocument(doc); - final int valuesCount = values.count(); - for (int i = 0, lo = 0; i < valuesCount; ++i) { - final BytesRef value = values.valueAt(i); - lo = collect(doc, value, bucket, lo); + if (values.advanceExact(doc)) { + final int valuesCount = values.docValueCount(); + for (int i = 0, lo = 0; i < valuesCount; ++i) { + final BytesRef value = values.nextValue(); + lo = collect(doc, value, bucket, lo); + } } } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregator.java index a2e55c18fb..4512803075 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregator.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregator.java @@ -236,11 +236,12 @@ public class RangeAggregator extends BucketsAggregator { return new LeafBucketCollectorBase(sub, values) { @Override public void collect(int doc, long bucket) throws IOException { - values.setDocument(doc); - final int valuesCount = values.count(); - for (int i = 0, lo = 0; i < valuesCount; ++i) { - final double value = values.valueAt(i); - lo = collect(doc, value, bucket, lo); + if (values.advanceExact(doc)) { + final int valuesCount = values.docValueCount(); + for (int i = 0, lo = 0; i < valuesCount; ++i) { + final double value = values.nextValue(); + lo = collect(doc, value, bucket, lo); + } } } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedBytesHashSamplerAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedBytesHashSamplerAggregator.java index 33532a9f43..472881b008 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedBytesHashSamplerAggregator.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedBytesHashSamplerAggregator.java @@ -26,6 +26,7 @@ import org.apache.lucene.search.DiversifiedTopDocsCollector.ScoreDocKey; import org.apache.lucene.search.TopDocsCollector; import org.apache.lucene.util.BytesRef; import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.index.fielddata.AbstractNumericDocValues; import org.elasticsearch.index.fielddata.SortedBinaryDocValues; import org.elasticsearch.search.aggregations.Aggregator; import org.elasticsearch.search.aggregations.AggregatorFactories; @@ -97,20 +98,33 @@ public class DiversifiedBytesHashSamplerAggregator extends SamplerAggregator { } catch (IOException e) { throw new ElasticsearchException("Error reading values", e); } - return new NumericDocValues() { - @Override - public long get(int doc) { + return new AbstractNumericDocValues() { - values.setDocument(doc); - final int valuesCount = values.count(); - if (valuesCount > 1) { - throw new IllegalArgumentException("Sample diversifying key must be a single valued-field"); - } - if (valuesCount == 1) { - final BytesRef bytes = values.valueAt(0); - return bytes.hashCode(); + private int docID = -1; + + @Override + public boolean advanceExact(int target) throws IOException { + docID = target; + if (values.advanceExact(target)) { + if (values.docValueCount() > 1) { + throw new IllegalArgumentException( + "Sample diversifying key must be a single valued-field"); + } + return true; + } else { + return false; } - return 0; + } + + @Override + public int docID() { + return docID; + } + + @Override + public long longValue() throws IOException { + final BytesRef bytes = values.nextValue(); + return bytes.hashCode(); } }; } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedMapSamplerAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedMapSamplerAggregator.java index d16e798b96..935da11e13 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedMapSamplerAggregator.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedMapSamplerAggregator.java @@ -28,6 +28,7 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.common.lease.Releasables; import org.elasticsearch.common.util.BytesRefHash; +import org.elasticsearch.index.fielddata.AbstractNumericDocValues; import org.elasticsearch.index.fielddata.SortedBinaryDocValues; import org.elasticsearch.search.aggregations.Aggregator; import org.elasticsearch.search.aggregations.AggregatorFactories; @@ -103,25 +104,38 @@ public class DiversifiedMapSamplerAggregator extends SamplerAggregator { } catch (IOException e) { throw new ElasticsearchException("Error reading values", e); } - return new NumericDocValues() { - @Override - public long get(int doc) { + return new AbstractNumericDocValues() { - values.setDocument(doc); - final int valuesCount = values.count(); - if (valuesCount > 1) { - throw new IllegalArgumentException("Sample diversifying key must be a single valued-field"); - } - if (valuesCount == 1) { - final BytesRef bytes = values.valueAt(0); + private int docID = -1; - long bucketOrdinal = bucketOrds.add(bytes); - if (bucketOrdinal < 0) { // already seen - bucketOrdinal = -1 - bucketOrdinal; + @Override + public boolean advanceExact(int target) throws IOException { + docID = target; + if (values.advanceExact(target)) { + if (values.docValueCount() > 1) { + throw new IllegalArgumentException( + "Sample diversifying key must be a single valued-field"); } - return bucketOrdinal; + return true; + } else { + return false; + } + } + + @Override + public int docID() { + return docID; + } + + @Override + public long longValue() throws IOException { + final BytesRef bytes = values.nextValue(); + + long bucketOrdinal = bucketOrds.add(bytes); + if (bucketOrdinal < 0) { // already seen + bucketOrdinal = -1 - bucketOrdinal; } - return 0; + return bucketOrdinal; } }; } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedNumericSamplerAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedNumericSamplerAggregator.java index c33a1a5553..8ce29c14bb 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedNumericSamplerAggregator.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedNumericSamplerAggregator.java @@ -26,6 +26,7 @@ import org.apache.lucene.search.DiversifiedTopDocsCollector; import org.apache.lucene.search.DiversifiedTopDocsCollector.ScoreDocKey; import org.apache.lucene.search.TopDocsCollector; import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.index.fielddata.AbstractNumericDocValues; import org.elasticsearch.search.aggregations.Aggregator; import org.elasticsearch.search.aggregations.AggregatorFactories; import org.elasticsearch.search.aggregations.bucket.DeferringBucketCollector; @@ -88,18 +89,29 @@ public class DiversifiedNumericSamplerAggregator extends SamplerAggregator { } catch (IOException e) { throw new ElasticsearchException("Error reading values", e); } - return new NumericDocValues() { + return new AbstractNumericDocValues() { + @Override - public long get(int doc) { - values.setDocument(doc); - final int valuesCount = values.count(); - if (valuesCount > 1) { - throw new IllegalArgumentException("Sample diversifying key must be a single valued-field"); - } - if (valuesCount == 1) { - return values.valueAt(0); + public boolean advanceExact(int target) throws IOException { + if (values.advanceExact(target)) { + if (values.docValueCount() > 1) { + throw new IllegalArgumentException( + "Sample diversifying key must be a single valued-field"); + } + return true; + } else { + return false; } - return Long.MIN_VALUE; + } + + @Override + public int docID() { + return values.docID(); + } + + @Override + public long longValue() throws IOException { + return values.nextValue(); } }; } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedOrdinalsSamplerAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedOrdinalsSamplerAggregator.java index a5d027debe..74f7a16d62 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedOrdinalsSamplerAggregator.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedOrdinalsSamplerAggregator.java @@ -22,11 +22,12 @@ package org.elasticsearch.search.aggregations.bucket.sampler; import org.apache.lucene.index.DocValues; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.NumericDocValues; -import org.apache.lucene.index.RandomAccessOrds; import org.apache.lucene.index.SortedDocValues; +import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.search.DiversifiedTopDocsCollector; import org.apache.lucene.search.DiversifiedTopDocsCollector.ScoreDocKey; import org.apache.lucene.search.TopDocsCollector; +import org.elasticsearch.index.fielddata.AbstractNumericDocValues; import org.elasticsearch.search.aggregations.Aggregator; import org.elasticsearch.search.aggregations.AggregatorFactories; import org.elasticsearch.search.aggregations.bucket.DeferringBucketCollector; @@ -84,29 +85,55 @@ public class DiversifiedOrdinalsSamplerAggregator extends SamplerAggregator { @Override protected NumericDocValues getKeys(LeafReaderContext context) { - final RandomAccessOrds globalOrds = valuesSource.globalOrdinalsValues(context); + final SortedSetDocValues globalOrds = valuesSource.globalOrdinalsValues(context); final SortedDocValues singleValues = DocValues.unwrapSingleton(globalOrds); if (singleValues != null) { - return new NumericDocValues() { + return new AbstractNumericDocValues() { + + @Override + public boolean advanceExact(int target) throws IOException { + return singleValues.advanceExact(target); + } + @Override - public long get(int doc) { - return singleValues.getOrd(doc); + public int docID() { + return singleValues.docID(); + } + + @Override + public long longValue() throws IOException { + return singleValues.ordValue(); } }; } - return new NumericDocValues() { + return new AbstractNumericDocValues() { + + long value; + @Override - public long get(int doc) { - globalOrds.setDocument(doc); - final long valuesCount = globalOrds.cardinality(); - if (valuesCount > 1) { - throw new IllegalArgumentException("Sample diversifying key must be a single valued-field"); - } - if (valuesCount == 1) { - long result = globalOrds.ordAt(0); - return result; + public boolean advanceExact(int target) throws IOException { + if (globalOrds.advanceExact(target)) { + value = globalOrds.nextOrd(); + // Check there isn't a second value for this + // document + if (globalOrds.nextOrd() != SortedSetDocValues.NO_MORE_ORDS) { + throw new IllegalArgumentException( + "Sample diversifying key must be a single valued-field"); + } + return true; + } else { + return false; } - return Long.MIN_VALUE; + } + + @Override + public int docID() { + return globalOrds.docID(); + } + + @Override + public long longValue() throws IOException { + return value; } }; diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/GlobalOrdinalsSignificantTermsAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/GlobalOrdinalsSignificantTermsAggregator.java index bf1714b14c..98effdcfd5 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/GlobalOrdinalsSignificantTermsAggregator.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/GlobalOrdinalsSignificantTermsAggregator.java @@ -20,6 +20,7 @@ package org.elasticsearch.search.aggregations.bucket.significant; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.lease.Releasables; import org.elasticsearch.common.util.LongHash; @@ -177,16 +178,17 @@ public class GlobalOrdinalsSignificantTermsAggregator extends GlobalOrdinalsStri public void collect(int doc, long bucket) throws IOException { assert bucket == 0; numCollectedDocs++; - globalOrds.setDocument(doc); - final int numOrds = globalOrds.cardinality(); - for (int i = 0; i < numOrds; i++) { - final long globalOrd = globalOrds.ordAt(i); - long bucketOrd = bucketOrds.add(globalOrd); - if (bucketOrd < 0) { - bucketOrd = -1 - bucketOrd; - collectExistingBucket(sub, doc, bucketOrd); - } else { - collectBucket(sub, doc, bucketOrd); + if (globalOrds.advanceExact(doc)) { + for (long globalOrd = globalOrds.nextOrd(); + globalOrd != SortedSetDocValues.NO_MORE_ORDS; + globalOrd = globalOrds.nextOrd()) { + long bucketOrd = bucketOrds.add(globalOrd); + if (bucketOrd < 0) { + bucketOrd = -1 - bucketOrd; + collectExistingBucket(sub, doc, bucketOrd); + } else { + collectBucket(sub, doc, bucketOrd); + } } } } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/GlobalOrdinalsStringTermsAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/GlobalOrdinalsStringTermsAggregator.java index 8e9d779cd0..5ef96d1925 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/GlobalOrdinalsStringTermsAggregator.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/GlobalOrdinalsStringTermsAggregator.java @@ -21,8 +21,8 @@ package org.elasticsearch.search.aggregations.bucket.terms; import org.apache.lucene.index.DocValues; import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.index.RandomAccessOrds; import org.apache.lucene.index.SortedDocValues; +import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.util.ArrayUtil; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.LongBitSet; @@ -31,7 +31,7 @@ import org.elasticsearch.common.lease.Releasables; import org.elasticsearch.common.util.IntArray; import org.elasticsearch.common.util.LongHash; import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.index.fielddata.AbstractRandomAccessOrds; +import org.elasticsearch.index.fielddata.AbstractSortedSetDocValues; import org.elasticsearch.index.fielddata.ordinals.GlobalOrdinalMapping; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.aggregations.Aggregator; @@ -67,7 +67,7 @@ public class GlobalOrdinalsStringTermsAggregator extends AbstractStringTermsAggr // especially if this agg is on a second layer or deeper. protected LongBitSet acceptedGlobalOrdinals; - protected RandomAccessOrds globalOrds; + protected SortedSetDocValues globalOrds; public GlobalOrdinalsStringTermsAggregator(String name, AggregatorFactories factories, ValuesSource.Bytes.WithOrdinals valuesSource, Terms.Order order, DocValueFormat format, BucketCountThresholds bucketCountThresholds, @@ -101,7 +101,8 @@ public class GlobalOrdinalsStringTermsAggregator extends AbstractStringTermsAggr return newCollector(globalOrds, sub); } - protected LeafBucketCollector newCollector(final RandomAccessOrds ords, final LeafBucketCollector sub) { + protected LeafBucketCollector newCollector(final SortedSetDocValues ords, + final LeafBucketCollector sub) { grow(ords.getValueCount()); final SortedDocValues singleValues = DocValues.unwrapSingleton(ords); if (singleValues != null) { @@ -109,8 +110,8 @@ public class GlobalOrdinalsStringTermsAggregator extends AbstractStringTermsAggr @Override public void collect(int doc, long bucket) throws IOException { assert bucket == 0; - final int ord = singleValues.getOrd(doc); - if (ord >= 0) { + if (singleValues.advanceExact(doc)) { + final int ord = singleValues.ordValue(); collectExistingBucket(sub, doc, ord); } } @@ -120,11 +121,12 @@ public class GlobalOrdinalsStringTermsAggregator extends AbstractStringTermsAggr @Override public void collect(int doc, long bucket) throws IOException { assert bucket == 0; - ords.setDocument(doc); - final int numOrds = ords.cardinality(); - for (int i = 0; i < numOrds; i++) { - final long globalOrd = ords.ordAt(i); - collectExistingBucket(sub, doc, globalOrd); + if (ords.advanceExact(doc)) { + for (long globalOrd = ords.nextOrd(); + globalOrd != SortedSetDocValues.NO_MORE_ORDS; + globalOrd = ords.nextOrd()) { + collectExistingBucket(sub, doc, globalOrd); + } } } }; @@ -270,14 +272,15 @@ public class GlobalOrdinalsStringTermsAggregator extends AbstractStringTermsAggr } @Override - protected LeafBucketCollector newCollector(final RandomAccessOrds ords, final LeafBucketCollector sub) { + protected LeafBucketCollector newCollector(final SortedSetDocValues ords, + final LeafBucketCollector sub) { final SortedDocValues singleValues = DocValues.unwrapSingleton(ords); if (singleValues != null) { return new LeafBucketCollectorBase(sub, ords) { @Override public void collect(int doc, long bucket) throws IOException { - final int globalOrd = singleValues.getOrd(doc); - if (globalOrd >= 0) { + if (singleValues.advanceExact(doc)) { + final int globalOrd = singleValues.ordValue(); long bucketOrd = bucketOrds.add(globalOrd); if (bucketOrd < 0) { bucketOrd = -1 - bucketOrd; @@ -292,16 +295,17 @@ public class GlobalOrdinalsStringTermsAggregator extends AbstractStringTermsAggr return new LeafBucketCollectorBase(sub, ords) { @Override public void collect(int doc, long bucket) throws IOException { - ords.setDocument(doc); - final int numOrds = ords.cardinality(); - for (int i = 0; i < numOrds; i++) { - final long globalOrd = ords.ordAt(i); - long bucketOrd = bucketOrds.add(globalOrd); - if (bucketOrd < 0) { - bucketOrd = -1 - bucketOrd; - collectExistingBucket(sub, doc, bucketOrd); - } else { - collectBucket(sub, doc, bucketOrd); + if (ords.advanceExact(doc)) { + for (long globalOrd = ords.nextOrd(); + globalOrd != SortedSetDocValues.NO_MORE_ORDS; + globalOrd = ords.nextOrd()) { + long bucketOrd = bucketOrds.add(globalOrd); + if (bucketOrd < 0) { + bucketOrd = -1 - bucketOrd; + collectExistingBucket(sub, doc, bucketOrd); + } else { + collectBucket(sub, doc, bucketOrd); + } } } } @@ -330,7 +334,7 @@ public class GlobalOrdinalsStringTermsAggregator extends AbstractStringTermsAggr private IntArray segmentDocCounts; - private RandomAccessOrds segmentOrds; + private SortedSetDocValues segmentOrds; public LowCardinality(String name, AggregatorFactories factories, ValuesSource.Bytes.WithOrdinals valuesSource, Terms.Order order, DocValueFormat format, @@ -345,7 +349,8 @@ public class GlobalOrdinalsStringTermsAggregator extends AbstractStringTermsAggr // bucketOrd is ord + 1 to avoid a branch to deal with the missing ord @Override - protected LeafBucketCollector newCollector(final RandomAccessOrds ords, LeafBucketCollector sub) { + protected LeafBucketCollector newCollector(final SortedSetDocValues ords, + LeafBucketCollector sub) { segmentDocCounts = context.bigArrays().grow(segmentDocCounts, 1 + ords.getValueCount()); assert sub == LeafBucketCollector.NO_OP_COLLECTOR; final SortedDocValues singleValues = DocValues.unwrapSingleton(ords); @@ -354,8 +359,10 @@ public class GlobalOrdinalsStringTermsAggregator extends AbstractStringTermsAggr @Override public void collect(int doc, long bucket) throws IOException { assert bucket == 0; - final int ord = singleValues.getOrd(doc); - segmentDocCounts.increment(ord + 1, 1); + if (singleValues.advanceExact(doc)) { + final int ord = singleValues.ordValue(); + segmentDocCounts.increment(ord + 1, 1); + } } }; } else { @@ -363,11 +370,12 @@ public class GlobalOrdinalsStringTermsAggregator extends AbstractStringTermsAggr @Override public void collect(int doc, long bucket) throws IOException { assert bucket == 0; - ords.setDocument(doc); - final int numOrds = ords.cardinality(); - for (int i = 0; i < numOrds; i++) { - final long segmentOrd = ords.ordAt(i); - segmentDocCounts.increment(segmentOrd + 1, 1); + if (ords.advanceExact(doc)) { + for (long segmentOrd = ords.nextOrd(); + segmentOrd != SortedSetDocValues.NO_MORE_ORDS; + segmentOrd = ords.nextOrd()) { + segmentDocCounts.increment(segmentOrd + 1, 1); + } } } }; @@ -422,15 +430,12 @@ public class GlobalOrdinalsStringTermsAggregator extends AbstractStringTermsAggr } } - private static final class FilteredOrdinals extends AbstractRandomAccessOrds { + private static final class FilteredOrdinals extends AbstractSortedSetDocValues { - private final RandomAccessOrds inner; + private final SortedSetDocValues inner; private final LongBitSet accepted; - private int cardinality; - private long[] ords = new long[0]; - - private FilteredOrdinals(RandomAccessOrds inner, LongBitSet accepted) { + private FilteredOrdinals(SortedSetDocValues inner, LongBitSet accepted) { this.inner = inner; this.accepted = accepted; } @@ -441,33 +446,33 @@ public class GlobalOrdinalsStringTermsAggregator extends AbstractStringTermsAggr } @Override - public long ordAt(int index) { - return ords[index]; + public BytesRef lookupOrd(long ord) throws IOException { + return inner.lookupOrd(ord); } @Override - public void doSetDocument(int docId) { - inner.setDocument(docId); - final int innerCardinality = inner.cardinality(); - ords = ArrayUtil.grow(ords, innerCardinality); - - cardinality = 0; - for (int slot = 0; slot < innerCardinality; slot++) { - long ord = inner.ordAt(slot); + public long nextOrd() throws IOException { + for (long ord = inner.nextOrd(); ord != NO_MORE_ORDS; ord = inner.nextOrd()) { if (accepted.get(ord)) { - ords[cardinality++] = ord; + return ord; } } + return NO_MORE_ORDS; } @Override - public int cardinality() { - return cardinality; - } - - @Override - public BytesRef lookupOrd(long ord) { - return inner.lookupOrd(ord); + public boolean advanceExact(int target) throws IOException { + if (inner.advanceExact(target)) { + for (long ord = inner.nextOrd(); ord != NO_MORE_ORDS; ord = inner.nextOrd()) { + if (accepted.get(ord)) { + // reset the iterator + boolean advanced = inner.advanceExact(target); + assert advanced; + return true; + } + } + } + return false; } } } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/LongTermsAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/LongTermsAggregator.java index e041ada417..8f8e2f3079 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/LongTermsAggregator.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/LongTermsAggregator.java @@ -77,24 +77,25 @@ public class LongTermsAggregator extends TermsAggregator { @Override public void collect(int doc, long owningBucketOrdinal) throws IOException { assert owningBucketOrdinal == 0; - values.setDocument(doc); - final int valuesCount = values.count(); - - long previous = Long.MAX_VALUE; - for (int i = 0; i < valuesCount; ++i) { - final long val = values.valueAt(i); - if (previous != val || i == 0) { - if ((longFilter == null) || (longFilter.accept(val))) { - long bucketOrdinal = bucketOrds.add(val); - if (bucketOrdinal < 0) { // already seen - bucketOrdinal = - 1 - bucketOrdinal; - collectExistingBucket(sub, doc, bucketOrdinal); - } else { - collectBucket(sub, doc, bucketOrdinal); + if (values.advanceExact(doc)) { + final int valuesCount = values.docValueCount(); + + long previous = Long.MAX_VALUE; + for (int i = 0; i < valuesCount; ++i) { + final long val = values.nextValue(); + if (previous != val || i == 0) { + if ((longFilter == null) || (longFilter.accept(val))) { + long bucketOrdinal = bucketOrds.add(val); + if (bucketOrdinal < 0) { // already seen + bucketOrdinal = -1 - bucketOrdinal; + collectExistingBucket(sub, doc, bucketOrdinal); + } else { + collectBucket(sub, doc, bucketOrdinal); + } } - } - previous = val; + previous = val; + } } } } @@ -110,12 +111,13 @@ public class LongTermsAggregator extends TermsAggregator { for (LeafReaderContext ctx : context.searcher().getTopReaderContext().leaves()) { final SortedNumericDocValues values = getValues(valuesSource, ctx); for (int docId = 0; docId < ctx.reader().maxDoc(); ++docId) { - values.setDocument(docId); - final int valueCount = values.count(); - for (int i = 0; i < valueCount; ++i) { - long value = values.valueAt(i); - if (longFilter == null || longFilter.accept(value)) { - bucketOrds.add(value); + if (values.advanceExact(docId)) { + final int valueCount = values.docValueCount(); + for (int i = 0; i < valueCount; ++i) { + long value = values.nextValue(); + if (longFilter == null || longFilter.accept(value)) { + bucketOrds.add(value); + } } } } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/StringTermsAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/StringTermsAggregator.java index c93fc94ff6..61c46cdfd6 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/StringTermsAggregator.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/StringTermsAggregator.java @@ -78,27 +78,29 @@ public class StringTermsAggregator extends AbstractStringTermsAggregator { @Override public void collect(int doc, long bucket) throws IOException { assert bucket == 0; - values.setDocument(doc); - final int valuesCount = values.count(); - - // SortedBinaryDocValues don't guarantee uniqueness so we need to take care of dups - previous.clear(); - for (int i = 0; i < valuesCount; ++i) { - final BytesRef bytes = values.valueAt(i); - if (includeExclude != null && !includeExclude.accept(bytes)) { - continue; - } - if (previous.get().equals(bytes)) { - continue; - } - long bucketOrdinal = bucketOrds.add(bytes); - if (bucketOrdinal < 0) { // already seen - bucketOrdinal = - 1 - bucketOrdinal; - collectExistingBucket(sub, doc, bucketOrdinal); - } else { - collectBucket(sub, doc, bucketOrdinal); + if (values.advanceExact(doc)) { + final int valuesCount = values.docValueCount(); + + // SortedBinaryDocValues don't guarantee uniqueness so we + // need to take care of dups + previous.clear(); + for (int i = 0; i < valuesCount; ++i) { + final BytesRef bytes = values.nextValue(); + if (includeExclude != null && !includeExclude.accept(bytes)) { + continue; + } + if (previous.get().equals(bytes)) { + continue; + } + long bucketOrdinal = bucketOrds.add(bytes); + if (bucketOrdinal < 0) { // already seen + bucketOrdinal = -1 - bucketOrdinal; + collectExistingBucket(sub, doc, bucketOrdinal); + } else { + collectBucket(sub, doc, bucketOrdinal); + } + previous.copyBytes(bytes); } - previous.copyBytes(bytes); } } }; @@ -114,12 +116,13 @@ public class StringTermsAggregator extends AbstractStringTermsAggregator { final SortedBinaryDocValues values = valuesSource.bytesValues(ctx); // brute force for (int docId = 0; docId < ctx.reader().maxDoc(); ++docId) { - values.setDocument(docId); - final int valueCount = values.count(); - for (int i = 0; i < valueCount; ++i) { - final BytesRef term = values.valueAt(i); - if (includeExclude == null || includeExclude.accept(term)) { - bucketOrds.add(term); + if (values.advanceExact(docId)) { + final int valueCount = values.docValueCount(); + for (int i = 0; i < valueCount; ++i) { + final BytesRef term = values.nextValue(); + if (includeExclude == null || includeExclude.accept(term)) { + bucketOrds.add(term); + } } } } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/support/IncludeExclude.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/support/IncludeExclude.java index 1520710210..aabe5f585d 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/support/IncludeExclude.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/support/IncludeExclude.java @@ -21,7 +21,7 @@ package org.elasticsearch.search.aggregations.bucket.terms.support; import com.carrotsearch.hppc.BitMixer; import com.carrotsearch.hppc.LongHashSet; import com.carrotsearch.hppc.LongSet; -import org.apache.lucene.index.RandomAccessOrds; + import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; @@ -233,14 +233,16 @@ public class IncludeExclude implements Writeable, ToXContent { } public abstract static class OrdinalsFilter { - public abstract LongBitSet acceptedGlobalOrdinals(RandomAccessOrds globalOrdinals) throws IOException; + public abstract LongBitSet acceptedGlobalOrdinals(SortedSetDocValues globalOrdinals) + throws IOException; } class PartitionedOrdinalsFilter extends OrdinalsFilter { @Override - public LongBitSet acceptedGlobalOrdinals(RandomAccessOrds globalOrdinals) throws IOException { + public LongBitSet acceptedGlobalOrdinals(SortedSetDocValues globalOrdinals) + throws IOException { final long numOrds = globalOrdinals.getValueCount(); final LongBitSet acceptedGlobalOrdinals = new LongBitSet(numOrds); final TermsEnum termEnum = globalOrdinals.termsEnum(); @@ -269,7 +271,7 @@ public class IncludeExclude implements Writeable, ToXContent { * */ @Override - public LongBitSet acceptedGlobalOrdinals(RandomAccessOrds globalOrdinals) + public LongBitSet acceptedGlobalOrdinals(SortedSetDocValues globalOrdinals) throws IOException { LongBitSet acceptedGlobalOrdinals = new LongBitSet(globalOrdinals.getValueCount()); TermsEnum globalTermsEnum; @@ -295,7 +297,8 @@ public class IncludeExclude implements Writeable, ToXContent { } @Override - public LongBitSet acceptedGlobalOrdinals(RandomAccessOrds globalOrdinals) throws IOException { + public LongBitSet acceptedGlobalOrdinals(SortedSetDocValues globalOrdinals) + throws IOException { LongBitSet acceptedGlobalOrdinals = new LongBitSet(globalOrdinals.getValueCount()); if (includeValues != null) { for (BytesRef term : includeValues) { diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/avg/AvgAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/avg/AvgAggregator.java index eb75bae268..0decfa0557 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/avg/AvgAggregator.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/avg/AvgAggregator.java @@ -77,14 +77,15 @@ public class AvgAggregator extends NumericMetricsAggregator.SingleValue { counts = bigArrays.grow(counts, bucket + 1); sums = bigArrays.grow(sums, bucket + 1); - values.setDocument(doc); - final int valueCount = values.count(); - counts.increment(bucket, valueCount); - double sum = 0; - for (int i = 0; i < valueCount; i++) { - sum += values.valueAt(i); + if (values.advanceExact(doc)) { + final int valueCount = values.docValueCount(); + counts.increment(bucket, valueCount); + double sum = 0; + for (int i = 0; i < valueCount; i++) { + sum += values.nextValue(); + } + sums.increment(bucket, sum); } - sums.increment(bucket, sum); } }; } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/CardinalityAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/CardinalityAggregator.java index 7d5db460ae..7a8483b1b2 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/CardinalityAggregator.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/CardinalityAggregator.java @@ -22,8 +22,8 @@ package org.elasticsearch.search.aggregations.metrics.cardinality; import com.carrotsearch.hppc.BitMixer; import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.index.RandomAccessOrds; import org.apache.lucene.index.SortedNumericDocValues; +import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.FixedBitSet; @@ -88,7 +88,7 @@ public class CardinalityAggregator extends NumericMetricsAggregator.SingleValue if (valuesSource instanceof ValuesSource.Bytes.WithOrdinals) { ValuesSource.Bytes.WithOrdinals source = (ValuesSource.Bytes.WithOrdinals) valuesSource; - final RandomAccessOrds ordinalValues = source.ordinalsValues(ctx); + final SortedSetDocValues ordinalValues = source.ordinalsValues(ctx); final long maxOrd = ordinalValues.getValueCount(); if (maxOrd == 0) { return new EmptyCollector(); @@ -114,7 +114,7 @@ public class CardinalityAggregator extends NumericMetricsAggregator.SingleValue return collector; } - private void postCollectLastCollector() { + private void postCollectLastCollector() throws IOException { if (collector != null) { try { collector.postCollect(); @@ -126,7 +126,7 @@ public class CardinalityAggregator extends NumericMetricsAggregator.SingleValue } @Override - protected void doPostCollection() { + protected void doPostCollection() throws IOException { postCollectLastCollector(); } @@ -159,7 +159,7 @@ public class CardinalityAggregator extends NumericMetricsAggregator.SingleValue private abstract static class Collector extends LeafBucketCollector implements Releasable { - public abstract void postCollect(); + public abstract void postCollect() throws IOException; } @@ -192,11 +192,12 @@ public class CardinalityAggregator extends NumericMetricsAggregator.SingleValue } @Override - public void collect(int doc, long bucketOrd) { - hashes.setDocument(doc); - final int valueCount = hashes.count(); - for (int i = 0; i < valueCount; ++i) { - counts.collect(bucketOrd, hashes.valueAt(i)); + public void collect(int doc, long bucketOrd) throws IOException { + if (hashes.advanceExact(doc)) { + final int valueCount = hashes.count(); + for (int i = 0; i < valueCount; ++i) { + counts.collect(bucketOrd, hashes.nextValue()); + } } } @@ -224,12 +225,13 @@ public class CardinalityAggregator extends NumericMetricsAggregator.SingleValue } private final BigArrays bigArrays; - private final RandomAccessOrds values; + private final SortedSetDocValues values; private final int maxOrd; private final HyperLogLogPlusPlus counts; private ObjectArray<FixedBitSet> visitedOrds; - OrdinalsCollector(HyperLogLogPlusPlus counts, RandomAccessOrds values, BigArrays bigArrays) { + OrdinalsCollector(HyperLogLogPlusPlus counts, SortedSetDocValues values, + BigArrays bigArrays) { if (values.getValueCount() > Integer.MAX_VALUE) { throw new IllegalArgumentException(); } @@ -241,22 +243,22 @@ public class CardinalityAggregator extends NumericMetricsAggregator.SingleValue } @Override - public void collect(int doc, long bucketOrd) { + public void collect(int doc, long bucketOrd) throws IOException { visitedOrds = bigArrays.grow(visitedOrds, bucketOrd + 1); FixedBitSet bits = visitedOrds.get(bucketOrd); if (bits == null) { bits = new FixedBitSet(maxOrd); visitedOrds.set(bucketOrd, bits); } - values.setDocument(doc); - final int valueCount = values.cardinality(); - for (int i = 0; i < valueCount; ++i) { - bits.set((int) values.ordAt(i)); + if (values.advanceExact(doc)) { + for (long ord = values.nextOrd(); ord != SortedSetDocValues.NO_MORE_ORDS; ord = values.nextOrd()) { + bits.set((int) ord); + } } } @Override - public void postCollect() { + public void postCollect() throws IOException { final FixedBitSet allVisitedOrds = new FixedBitSet(maxOrd); for (long bucket = visitedOrds.size() - 1; bucket >= 0; --bucket) { final FixedBitSet bits = visitedOrds.get(bucket); @@ -296,11 +298,11 @@ public class CardinalityAggregator extends NumericMetricsAggregator.SingleValue */ abstract static class MurmurHash3Values { - public abstract void setDocument(int docId); + public abstract boolean advanceExact(int docId) throws IOException; public abstract int count(); - public abstract long valueAt(int index); + public abstract long nextValue() throws IOException; /** * Return a {@link MurmurHash3Values} instance that computes hashes on the fly for each double value. @@ -332,18 +334,18 @@ public class CardinalityAggregator extends NumericMetricsAggregator.SingleValue } @Override - public void setDocument(int docId) { - values.setDocument(docId); + public boolean advanceExact(int docId) throws IOException { + return values.advanceExact(docId); } @Override public int count() { - return values.count(); + return values.docValueCount(); } @Override - public long valueAt(int index) { - return BitMixer.mix64(values.valueAt(index)); + public long nextValue() throws IOException { + return BitMixer.mix64(values.nextValue()); } } @@ -356,18 +358,18 @@ public class CardinalityAggregator extends NumericMetricsAggregator.SingleValue } @Override - public void setDocument(int docId) { - values.setDocument(docId); + public boolean advanceExact(int docId) throws IOException { + return values.advanceExact(docId); } @Override public int count() { - return values.count(); + return values.docValueCount(); } @Override - public long valueAt(int index) { - return BitMixer.mix64(java.lang.Double.doubleToLongBits(values.valueAt(index))); + public long nextValue() throws IOException { + return BitMixer.mix64(java.lang.Double.doubleToLongBits(values.nextValue())); } } @@ -382,18 +384,18 @@ public class CardinalityAggregator extends NumericMetricsAggregator.SingleValue } @Override - public void setDocument(int docId) { - values.setDocument(docId); + public boolean advanceExact(int docId) throws IOException { + return values.advanceExact(docId); } @Override public int count() { - return values.count(); + return values.docValueCount(); } @Override - public long valueAt(int index) { - final BytesRef bytes = values.valueAt(index); + public long nextValue() throws IOException { + final BytesRef bytes = values.nextValue(); org.elasticsearch.common.hash.MurmurHash3.hash128(bytes.bytes, bytes.offset, bytes.length, 0, hash); return hash.h1; } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geobounds/GeoBoundsAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geobounds/GeoBoundsAggregator.java index 2083ea570d..5c0cb4ba60 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geobounds/GeoBoundsAggregator.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geobounds/GeoBoundsAggregator.java @@ -102,41 +102,42 @@ public final class GeoBoundsAggregator extends MetricsAggregator { negRights.fill(from, negRights.size(), Double.NEGATIVE_INFINITY); } - values.setDocument(doc); - final int valuesCount = values.count(); + if (values.advanceExact(doc)) { + final int valuesCount = values.docValueCount(); - for (int i = 0; i < valuesCount; ++i) { - GeoPoint value = values.valueAt(i); - double top = tops.get(bucket); - if (value.lat() > top) { - top = value.lat(); + for (int i = 0; i < valuesCount; ++i) { + GeoPoint value = values.nextValue(); + double top = tops.get(bucket); + if (value.lat() > top) { + top = value.lat(); + } + double bottom = bottoms.get(bucket); + if (value.lat() < bottom) { + bottom = value.lat(); + } + double posLeft = posLefts.get(bucket); + if (value.lon() >= 0 && value.lon() < posLeft) { + posLeft = value.lon(); + } + double posRight = posRights.get(bucket); + if (value.lon() >= 0 && value.lon() > posRight) { + posRight = value.lon(); + } + double negLeft = negLefts.get(bucket); + if (value.lon() < 0 && value.lon() < negLeft) { + negLeft = value.lon(); + } + double negRight = negRights.get(bucket); + if (value.lon() < 0 && value.lon() > negRight) { + negRight = value.lon(); + } + tops.set(bucket, top); + bottoms.set(bucket, bottom); + posLefts.set(bucket, posLeft); + posRights.set(bucket, posRight); + negLefts.set(bucket, negLeft); + negRights.set(bucket, negRight); } - double bottom = bottoms.get(bucket); - if (value.lat() < bottom) { - bottom = value.lat(); - } - double posLeft = posLefts.get(bucket); - if (value.lon() >= 0 && value.lon() < posLeft) { - posLeft = value.lon(); - } - double posRight = posRights.get(bucket); - if (value.lon() >= 0 && value.lon() > posRight) { - posRight = value.lon(); - } - double negLeft = negLefts.get(bucket); - if (value.lon() < 0 && value.lon() < negLeft) { - negLeft = value.lon(); - } - double negRight = negRights.get(bucket); - if (value.lon() < 0 && value.lon() > negRight) { - negRight = value.lon(); - } - tops.set(bucket, top); - bottoms.set(bucket, bottom); - posLefts.set(bucket, posLeft); - posRights.set(bucket, posRight); - negLefts.set(bucket, negLeft); - negRights.set(bucket, negRight); } } }; diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geocentroid/GeoCentroidAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geocentroid/GeoCentroidAggregator.java index 4b23d042d5..795524e5a0 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geocentroid/GeoCentroidAggregator.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geocentroid/GeoCentroidAggregator.java @@ -20,7 +20,6 @@ package org.elasticsearch.search.aggregations.metrics.geocentroid; import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.spatial.geopoint.document.GeoPointField; import org.elasticsearch.common.geo.GeoPoint; import org.elasticsearch.common.lease.Releasables; import org.elasticsearch.common.util.BigArrays; @@ -72,29 +71,30 @@ final class GeoCentroidAggregator extends MetricsAggregator { centroids = bigArrays.grow(centroids, bucket + 1); counts = bigArrays.grow(counts, bucket + 1); - values.setDocument(doc); - final int valueCount = values.count(); - if (valueCount > 0) { + if (values.advanceExact(doc)) { + final int valueCount = values.docValueCount(); double[] pt = new double[2]; // get the previously accumulated number of counts long prevCounts = counts.get(bucket); // increment by the number of points for this document counts.increment(bucket, valueCount); - // get the previous GeoPoint if a moving avg was computed + // get the previous GeoPoint if a moving avg was + // computed if (prevCounts > 0) { final long mortonCode = centroids.get(bucket); - pt[0] = GeoPointField.decodeLongitude(mortonCode); - pt[1] = GeoPointField.decodeLatitude(mortonCode); + pt[0] = InternalGeoCentroid.decodeLongitude(mortonCode); + pt[1] = InternalGeoCentroid.decodeLatitude(mortonCode); } // update the moving average for (int i = 0; i < valueCount; ++i) { - GeoPoint value = values.valueAt(i); + GeoPoint value = values.nextValue(); pt[0] = pt[0] + (value.getLon() - pt[0]) / ++prevCounts; pt[1] = pt[1] + (value.getLat() - pt[1]) / prevCounts; } - // TODO: we do not need to interleave the lat and lon bits here + // TODO: we do not need to interleave the lat and lon + // bits here // should we just store them contiguously? - centroids.set(bucket, GeoPointField.encodeLatLon(pt[1], pt[0])); + centroids.set(bucket, InternalGeoCentroid.encodeLatLon(pt[1], pt[0])); } } }; @@ -108,7 +108,8 @@ final class GeoCentroidAggregator extends MetricsAggregator { final long bucketCount = counts.get(bucket); final long mortonCode = centroids.get(bucket); final GeoPoint bucketCentroid = (bucketCount > 0) - ? new GeoPoint(GeoPointField.decodeLatitude(mortonCode), GeoPointField.decodeLongitude(mortonCode)) + ? new GeoPoint(InternalGeoCentroid.decodeLatitude(mortonCode), + InternalGeoCentroid.decodeLongitude(mortonCode)) : null; return new InternalGeoCentroid(name, bucketCentroid , bucketCount, pipelineAggregators(), metaData()); } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geocentroid/InternalGeoCentroid.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geocentroid/InternalGeoCentroid.java index 597ad6176b..bd65cd28af 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geocentroid/InternalGeoCentroid.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geocentroid/InternalGeoCentroid.java @@ -19,7 +19,7 @@ package org.elasticsearch.search.aggregations.metrics.geocentroid; -import org.apache.lucene.spatial.geopoint.document.GeoPointField; +import org.apache.lucene.geo.GeoEncodingUtils; import org.elasticsearch.common.geo.GeoPoint; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -38,6 +38,18 @@ public class InternalGeoCentroid extends InternalAggregation implements GeoCentr protected final GeoPoint centroid; protected final long count; + public static long encodeLatLon(double lat, double lon) { + return (Integer.toUnsignedLong(GeoEncodingUtils.encodeLatitude(lat)) << 32) | Integer.toUnsignedLong(GeoEncodingUtils.encodeLongitude(lon)); + } + + public static double decodeLatitude(long encodedLatLon) { + return GeoEncodingUtils.decodeLatitude((int) (encodedLatLon >>> 32)); + } + + public static double decodeLongitude(long encodedLatLon) { + return GeoEncodingUtils.decodeLongitude((int) (encodedLatLon & 0xFFFFFFFFL)); + } + InternalGeoCentroid(String name, GeoPoint centroid, long count, List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData) { super(name, pipelineAggregators, metaData); @@ -55,7 +67,7 @@ public class InternalGeoCentroid extends InternalAggregation implements GeoCentr count = in.readVLong(); if (in.readBoolean()) { final long hash = in.readLong(); - centroid = new GeoPoint(GeoPointField.decodeLatitude(hash), GeoPointField.decodeLongitude(hash)); + centroid = new GeoPoint(decodeLatitude(hash), decodeLongitude(hash)); } else { centroid = null; } @@ -67,7 +79,7 @@ public class InternalGeoCentroid extends InternalAggregation implements GeoCentr if (centroid != null) { out.writeBoolean(true); // should we just write lat and lon separately? - out.writeLong(GeoPointField.encodeLatLon(centroid.lat(), centroid.lon())); + out.writeLong(encodeLatLon(centroid.lat(), centroid.lon())); } else { out.writeBoolean(false); } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/max/MaxAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/max/MaxAggregator.java index cef7f37328..8ef4d0b7e2 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/max/MaxAggregator.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/max/MaxAggregator.java @@ -82,10 +82,12 @@ public class MaxAggregator extends NumericMetricsAggregator.SingleValue { maxes = bigArrays.grow(maxes, bucket + 1); maxes.fill(from, maxes.size(), Double.NEGATIVE_INFINITY); } - final double value = values.get(doc); - double max = maxes.get(bucket); - max = Math.max(max, value); - maxes.set(bucket, max); + if (values.advanceExact(doc)) { + final double value = values.doubleValue(); + double max = maxes.get(bucket); + max = Math.max(max, value); + maxes.set(bucket, max); + } } }; diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/min/MinAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/min/MinAggregator.java index 31e87b12a6..f355f55139 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/min/MinAggregator.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/min/MinAggregator.java @@ -81,10 +81,12 @@ public class MinAggregator extends NumericMetricsAggregator.SingleValue { mins = bigArrays.grow(mins, bucket + 1); mins.fill(from, mins.size(), Double.POSITIVE_INFINITY); } - final double value = values.get(doc); - double min = mins.get(bucket); - min = Math.min(min, value); - mins.set(bucket, min); + if (values.advanceExact(doc)) { + final double value = values.doubleValue(); + double min = mins.get(bucket); + min = Math.min(min, value); + mins.set(bucket, min); + } } }; diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/hdr/AbstractHDRPercentilesAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/hdr/AbstractHDRPercentilesAggregator.java index bf4443c887..47c267aae9 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/hdr/AbstractHDRPercentilesAggregator.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/hdr/AbstractHDRPercentilesAggregator.java @@ -95,10 +95,11 @@ public abstract class AbstractHDRPercentilesAggregator extends NumericMetricsAgg states.set(bucket, state); } - values.setDocument(doc); - final int valueCount = values.count(); - for (int i = 0; i < valueCount; i++) { - state.recordValue(values.valueAt(i)); + if (values.advanceExact(doc)) { + final int valueCount = values.docValueCount(); + for (int i = 0; i < valueCount; i++) { + state.recordValue(values.nextValue()); + } } } }; diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/tdigest/AbstractTDigestPercentilesAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/tdigest/AbstractTDigestPercentilesAggregator.java index 2c68d580e1..1b5ed510f8 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/tdigest/AbstractTDigestPercentilesAggregator.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/tdigest/AbstractTDigestPercentilesAggregator.java @@ -87,10 +87,11 @@ public abstract class AbstractTDigestPercentilesAggregator extends NumericMetric states.set(bucket, state); } - values.setDocument(doc); - final int valueCount = values.count(); - for (int i = 0; i < valueCount; i++) { - state.add(values.valueAt(i)); + if (values.advanceExact(doc)) { + final int valueCount = values.docValueCount(); + for (int i = 0; i < valueCount; i++) { + state.add(values.nextValue()); + } } } }; diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/StatsAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/StatsAggregator.java index e3657c3851..cca176bd1a 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/StatsAggregator.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/StatsAggregator.java @@ -94,21 +94,22 @@ public class StatsAggregator extends NumericMetricsAggregator.MultiValue { maxes.fill(from, overSize, Double.NEGATIVE_INFINITY); } - values.setDocument(doc); - final int valuesCount = values.count(); - counts.increment(bucket, valuesCount); - double sum = 0; - double min = mins.get(bucket); - double max = maxes.get(bucket); - for (int i = 0; i < valuesCount; i++) { - double value = values.valueAt(i); - sum += value; - min = Math.min(min, value); - max = Math.max(max, value); + if (values.advanceExact(doc)) { + final int valuesCount = values.docValueCount(); + counts.increment(bucket, valuesCount); + double sum = 0; + double min = mins.get(bucket); + double max = maxes.get(bucket); + for (int i = 0; i < valuesCount; i++) { + double value = values.nextValue(); + sum += value; + min = Math.min(min, value); + max = Math.max(max, value); + } + sums.increment(bucket, sum); + mins.set(bucket, min); + maxes.set(bucket, max); } - sums.increment(bucket, sum); - mins.set(bucket, min); - maxes.set(bucket, max); } }; } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/extended/ExtendedStatsAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/extended/ExtendedStatsAggregator.java index d6faf5cbb7..8dd78bf137 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/extended/ExtendedStatsAggregator.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/extended/ExtendedStatsAggregator.java @@ -102,24 +102,25 @@ public class ExtendedStatsAggregator extends NumericMetricsAggregator.MultiValue maxes.fill(from, overSize, Double.NEGATIVE_INFINITY); } - values.setDocument(doc); - final int valuesCount = values.count(); - counts.increment(bucket, valuesCount); - double sum = 0; - double sumOfSqr = 0; - double min = mins.get(bucket); - double max = maxes.get(bucket); - for (int i = 0; i < valuesCount; i++) { - double value = values.valueAt(i); - sum += value; - sumOfSqr += value * value; - min = Math.min(min, value); - max = Math.max(max, value); + if (values.advanceExact(doc)) { + final int valuesCount = values.docValueCount(); + counts.increment(bucket, valuesCount); + double sum = 0; + double sumOfSqr = 0; + double min = mins.get(bucket); + double max = maxes.get(bucket); + for (int i = 0; i < valuesCount; i++) { + double value = values.nextValue(); + sum += value; + sumOfSqr += value * value; + min = Math.min(min, value); + max = Math.max(max, value); + } + sums.increment(bucket, sum); + sumOfSqrs.increment(bucket, sumOfSqr); + mins.set(bucket, min); + maxes.set(bucket, max); } - sums.increment(bucket, sum); - sumOfSqrs.increment(bucket, sumOfSqr); - mins.set(bucket, min); - maxes.set(bucket, max); } }; diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/sum/SumAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/sum/SumAggregator.java index f7f2ee5501..bd325b3937 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/sum/SumAggregator.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/sum/SumAggregator.java @@ -71,13 +71,14 @@ public class SumAggregator extends NumericMetricsAggregator.SingleValue { @Override public void collect(int doc, long bucket) throws IOException { sums = bigArrays.grow(sums, bucket + 1); - values.setDocument(doc); - final int valuesCount = values.count(); - double sum = 0; - for (int i = 0; i < valuesCount; i++) { - sum += values.valueAt(i); + if (values.advanceExact(doc)) { + final int valuesCount = values.docValueCount(); + double sum = 0; + for (int i = 0; i < valuesCount; i++) { + sum += values.nextValue(); + } + sums.increment(bucket, sum); } - sums.increment(bucket, sum); } }; } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/valuecount/ValueCountAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/valuecount/ValueCountAggregator.java index e0ed6d1ade..929e26d04b 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/valuecount/ValueCountAggregator.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/valuecount/ValueCountAggregator.java @@ -73,8 +73,9 @@ public class ValueCountAggregator extends NumericMetricsAggregator.SingleValue { @Override public void collect(int doc, long bucket) throws IOException { counts = bigArrays.grow(counts, bucket + 1); - values.setDocument(doc); - counts.increment(bucket, values.count()); + if (values.advanceExact(doc)) { + counts.increment(bucket, values.docValueCount()); + } } }; diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/support/MissingValues.java b/core/src/main/java/org/elasticsearch/search/aggregations/support/MissingValues.java index 4a01b67b78..d5ff2cde97 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/support/MissingValues.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/support/MissingValues.java @@ -20,11 +20,12 @@ package org.elasticsearch.search.aggregations.support; import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.index.RandomAccessOrds; import org.apache.lucene.index.SortedNumericDocValues; +import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.geo.GeoPoint; -import org.elasticsearch.index.fielddata.AbstractRandomAccessOrds; +import org.elasticsearch.index.fielddata.AbstractSortedNumericDocValues; +import org.elasticsearch.index.fielddata.AbstractSortedSetDocValues; import org.elasticsearch.index.fielddata.MultiGeoPointValues; import org.elasticsearch.index.fielddata.SortedBinaryDocValues; import org.elasticsearch.index.fielddata.SortedNumericDoubleValues; @@ -56,25 +57,29 @@ public enum MissingValues { private int count; @Override - public BytesRef valueAt(int index) { - if (count > 0) { - return values.valueAt(index); - } else if (index == 0) { - return missing; + public boolean advanceExact(int doc) throws IOException { + if (values.advanceExact(doc)) { + count = values.docValueCount(); } else { - throw new IndexOutOfBoundsException(); + count = 0; } + // always return true because we want to return a value even if + // the document does not have a value + return true; } @Override - public void setDocument(int docId) { - values.setDocument(docId); - count = values.count(); + public int docValueCount() { + return count == 0 ? 1 : count; } @Override - public int count() { - return count == 0 ? 1 : count; + public BytesRef nextValue() throws IOException { + if (count > 0) { + return values.nextValue(); + } else { + return missing; + } } }; } @@ -109,32 +114,36 @@ public enum MissingValues { } static SortedNumericDocValues replaceMissing(final SortedNumericDocValues values, final long missing) { - return new SortedNumericDocValues() { + return new AbstractSortedNumericDocValues() { private int count; @Override - public void setDocument(int doc) { - values.setDocument(doc); - count = values.count(); - } - - @Override - public long valueAt(int index) { + public long nextValue() throws IOException { if (count > 0) { - return values.valueAt(index); - } else if (index == 0) { - return missing; + return values.nextValue(); } else { - throw new IndexOutOfBoundsException(); + return missing; } } @Override - public int count() { + public int docValueCount() { return count == 0 ? 1 : count; } + @Override + public boolean advanceExact(int doc) throws IOException { + if (values.advanceExact(doc)) { + count = values.docValueCount(); + } else { + count = 0; + } + // always return true because we want to return a value even if + // the document does not have a value + return true; + } + }; } @@ -144,24 +153,28 @@ public enum MissingValues { private int count; @Override - public void setDocument(int doc) { - values.setDocument(doc); - count = values.count(); + public boolean advanceExact(int doc) throws IOException { + if (values.advanceExact(doc)) { + count = values.docValueCount(); + } else { + count = 0; + } + // always return true because we want to return a value even if + // the document does not have a value + return true; } @Override - public double valueAt(int index) { + public double nextValue() throws IOException { if (count > 0) { - return values.valueAt(index); - } else if (index == 0) { - return missing; + return values.nextValue(); } else { - throw new IndexOutOfBoundsException(); + return missing; } } @Override - public int count() { + public int docValueCount() { return count == 0 ? 1 : count; } @@ -177,20 +190,22 @@ public enum MissingValues { } @Override - public RandomAccessOrds ordinalsValues(LeafReaderContext context) { - RandomAccessOrds values = valuesSource.ordinalsValues(context); + public SortedSetDocValues ordinalsValues(LeafReaderContext context) throws IOException { + SortedSetDocValues values = valuesSource.ordinalsValues(context); return replaceMissing(values, missing); } @Override - public RandomAccessOrds globalOrdinalsValues(LeafReaderContext context) { - RandomAccessOrds values = valuesSource.globalOrdinalsValues(context); + public SortedSetDocValues globalOrdinalsValues(LeafReaderContext context) + throws IOException { + SortedSetDocValues values = valuesSource.globalOrdinalsValues(context); return replaceMissing(values, missing); } }; } - static RandomAccessOrds replaceMissing(final RandomAccessOrds values, final BytesRef missing) { + static SortedSetDocValues replaceMissing(final SortedSetDocValues values, + final BytesRef missing) throws IOException { final long missingOrd = values.lookupTerm(missing); if (missingOrd >= 0) { // The value already exists @@ -201,19 +216,15 @@ public enum MissingValues { } } - static RandomAccessOrds replaceMissingOrd(final RandomAccessOrds values, final long missingOrd) { - return new AbstractRandomAccessOrds() { + static SortedSetDocValues replaceMissingOrd(final SortedSetDocValues values, + final long missingOrd) { + return new AbstractSortedSetDocValues() { - private int cardinality = 0; + private boolean hasOrds; + private long nextMissingOrd; @Override - public void doSetDocument(int docID) { - values.setDocument(docID); - cardinality = values.cardinality(); - } - - @Override - public BytesRef lookupOrd(long ord) { + public BytesRef lookupOrd(long ord) throws IOException { return values.lookupOrd(ord); } @@ -223,36 +234,39 @@ public enum MissingValues { } @Override - public long ordAt(int index) { - if (cardinality > 0) { - return values.ordAt(index); - } else if (index == 0) { - return missingOrd; + public long nextOrd() throws IOException { + if (hasOrds) { + return values.nextOrd(); } else { - throw new IndexOutOfBoundsException(); + // we want to return the next missing ord but set this to + // NO_MORE_ORDS so on the next call we indicate there are no + // more values + long ordToReturn = nextMissingOrd; + nextMissingOrd = SortedSetDocValues.NO_MORE_ORDS; + return ordToReturn; } } @Override - public int cardinality() { - return cardinality == 0 ? 1 : cardinality; + public boolean advanceExact(int doc) throws IOException { + hasOrds = values.advanceExact(doc); + nextMissingOrd = missingOrd; + // always return true because we want to return a value even if + // the document does not have a value + return true; } }; } - static RandomAccessOrds insertOrd(final RandomAccessOrds values, final long insertedOrd, final BytesRef missingValue) { - return new AbstractRandomAccessOrds() { + static SortedSetDocValues insertOrd(final SortedSetDocValues values, final long insertedOrd, + final BytesRef missingValue) { + return new AbstractSortedSetDocValues() { - private int cardinality = 0; - - @Override - public void doSetDocument(int docID) { - values.setDocument(docID); - cardinality = values.cardinality(); - } + private boolean hasOrds; + private long nextMissingOrd; @Override - public BytesRef lookupOrd(long ord) { + public BytesRef lookupOrd(long ord) throws IOException { if (ord < insertedOrd) { return values.lookupOrd(ord); } else if (ord > insertedOrd) { @@ -268,24 +282,31 @@ public enum MissingValues { } @Override - public long ordAt(int index) { - if (cardinality > 0) { - final long ord = values.ordAt(index); + public long nextOrd() throws IOException { + if (hasOrds) { + final long ord = values.nextOrd(); if (ord < insertedOrd) { return ord; } else { return ord + 1; } - } else if (index == 0) { - return insertedOrd; } else { - throw new IndexOutOfBoundsException(); + // we want to return the next missing ord but set this to + // NO_MORE_ORDS so on the next call we indicate there are no + // more values + long ordToReturn = nextMissingOrd; + nextMissingOrd = SortedSetDocValues.NO_MORE_ORDS; + return ordToReturn; } } @Override - public int cardinality() { - return cardinality == 0 ? 1 : cardinality; + public boolean advanceExact(int doc) throws IOException { + hasOrds = values.advanceExact(doc); + nextMissingOrd = insertedOrd; + // always return true because we want to return a value even if + // the document does not have a value + return true; } }; } @@ -312,25 +333,29 @@ public enum MissingValues { private int count; @Override - public GeoPoint valueAt(int index) { - if (count > 0) { - return values.valueAt(index); - } else if (index == 0) { - return missing; + public boolean advanceExact(int doc) throws IOException { + if (values.advanceExact(doc)) { + count = values.docValueCount(); } else { - throw new IndexOutOfBoundsException(); + count = 0; } + // always return true because we want to return a value even if + // the document does not have a value + return true; } @Override - public void setDocument(int docId) { - values.setDocument(docId); - count = values.count(); + public int docValueCount() { + return count == 0 ? 1 : count; } @Override - public int count() { - return count == 0 ? 1 : count; + public GeoPoint nextValue() throws IOException { + if (count > 0) { + return values.nextValue(); + } else { + return missing; + } } }; } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSource.java b/core/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSource.java index f3de52ae65..e6ed81c2d8 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSource.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSource.java @@ -22,14 +22,15 @@ import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.DocValues; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.index.RandomAccessOrds; import org.apache.lucene.index.SortedDocValues; import org.apache.lucene.index.SortedNumericDocValues; +import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Scorer; import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.lucene.ScorerAware; +import org.elasticsearch.index.fielddata.AbstractSortingNumericDocValues; import org.elasticsearch.index.fielddata.AtomicOrdinalsFieldData; import org.elasticsearch.index.fielddata.AtomicParentChildFieldData; import org.elasticsearch.index.fielddata.IndexFieldData; @@ -41,7 +42,6 @@ import org.elasticsearch.index.fielddata.MultiGeoPointValues; import org.elasticsearch.index.fielddata.SortedBinaryDocValues; import org.elasticsearch.index.fielddata.SortedNumericDoubleValues; import org.elasticsearch.index.fielddata.SortingBinaryDocValues; -import org.elasticsearch.index.fielddata.SortingNumericDocValues; import org.elasticsearch.index.fielddata.SortingNumericDoubleValues; import org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData; import org.elasticsearch.script.LeafSearchScript; @@ -72,11 +72,8 @@ public abstract class ValuesSource { @Override public Bits docsWithValue(LeafReaderContext context) throws IOException { final SortedBinaryDocValues bytes = bytesValues(context); - if (org.elasticsearch.index.fielddata.FieldData.unwrapSingleton(bytes) != null) { - return org.elasticsearch.index.fielddata.FieldData.unwrapSingletonBits(bytes); - } else { - return org.elasticsearch.index.fielddata.FieldData.docsWithValue(bytes, context.reader().maxDoc()); - } + return org.elasticsearch.index.fielddata.FieldData.docsWithValue(bytes, + context.reader().maxDoc()); } public abstract static class WithOrdinals extends Bytes { @@ -84,43 +81,42 @@ public abstract class ValuesSource { public static final WithOrdinals EMPTY = new WithOrdinals() { @Override - public RandomAccessOrds ordinalsValues(LeafReaderContext context) { + public SortedSetDocValues ordinalsValues(LeafReaderContext context) { return DocValues.emptySortedSet(); } @Override - public RandomAccessOrds globalOrdinalsValues(LeafReaderContext context) { + public SortedSetDocValues globalOrdinalsValues(LeafReaderContext context) { return DocValues.emptySortedSet(); } @Override public SortedBinaryDocValues bytesValues(LeafReaderContext context) throws IOException { - return org.elasticsearch.index.fielddata.FieldData.emptySortedBinary(context.reader().maxDoc()); + return org.elasticsearch.index.fielddata.FieldData.emptySortedBinary(); } }; @Override - public Bits docsWithValue(LeafReaderContext context) { - final RandomAccessOrds ordinals = ordinalsValues(context); - if (DocValues.unwrapSingleton(ordinals) != null) { - return DocValues.docsWithValue(DocValues.unwrapSingleton(ordinals), context.reader().maxDoc()); - } else { - return DocValues.docsWithValue(ordinals, context.reader().maxDoc()); - } + public Bits docsWithValue(LeafReaderContext context) throws IOException { + final SortedSetDocValues ordinals = ordinalsValues(context); + return org.elasticsearch.index.fielddata.FieldData.docsWithValue(ordinals, + context.reader().maxDoc()); } - public abstract RandomAccessOrds ordinalsValues(LeafReaderContext context); + public abstract SortedSetDocValues ordinalsValues(LeafReaderContext context) + throws IOException; - public abstract RandomAccessOrds globalOrdinalsValues(LeafReaderContext context); + public abstract SortedSetDocValues globalOrdinalsValues(LeafReaderContext context) + throws IOException; - public long globalMaxOrd(IndexSearcher indexSearcher) { + public long globalMaxOrd(IndexSearcher indexSearcher) throws IOException { IndexReader indexReader = indexSearcher.getIndexReader(); if (indexReader.leaves().isEmpty()) { return 0; } else { LeafReaderContext atomicReaderContext = indexReader.leaves().get(0); - RandomAccessOrds values = globalOrdinalsValues(atomicReaderContext); + SortedSetDocValues values = globalOrdinalsValues(atomicReaderContext); return values.getValueCount(); } } @@ -140,13 +136,13 @@ public abstract class ValuesSource { } @Override - public RandomAccessOrds ordinalsValues(LeafReaderContext context) { + public SortedSetDocValues ordinalsValues(LeafReaderContext context) { final AtomicOrdinalsFieldData atomicFieldData = indexFieldData.load(context); return atomicFieldData.getOrdinalsValues(); } @Override - public RandomAccessOrds globalOrdinalsValues(LeafReaderContext context) { + public SortedSetDocValues globalOrdinalsValues(LeafReaderContext context) { final IndexOrdinalsFieldData global = indexFieldData.loadGlobal((DirectoryReader)context.parent.reader()); final AtomicOrdinalsFieldData atomicFieldData = global.load(context); return atomicFieldData.getOrdinalsValues(); @@ -240,12 +236,12 @@ public abstract class ValuesSource { @Override public SortedNumericDoubleValues doubleValues(LeafReaderContext context) throws IOException { - return org.elasticsearch.index.fielddata.FieldData.emptySortedNumericDoubles(context.reader().maxDoc()); + return org.elasticsearch.index.fielddata.FieldData.emptySortedNumericDoubles(); } @Override public SortedBinaryDocValues bytesValues(LeafReaderContext context) throws IOException { - return org.elasticsearch.index.fielddata.FieldData.emptySortedBinary(context.reader().maxDoc()); + return org.elasticsearch.index.fielddata.FieldData.emptySortedBinary(); } }; @@ -263,18 +259,12 @@ public abstract class ValuesSource { public Bits docsWithValue(LeafReaderContext context) throws IOException { if (isFloatingPoint()) { final SortedNumericDoubleValues values = doubleValues(context); - if (org.elasticsearch.index.fielddata.FieldData.unwrapSingleton(values) != null) { - return org.elasticsearch.index.fielddata.FieldData.unwrapSingletonBits(values); - } else { - return org.elasticsearch.index.fielddata.FieldData.docsWithValue(values, context.reader().maxDoc()); - } + return org.elasticsearch.index.fielddata.FieldData.docsWithValue(values, + context.reader().maxDoc()); } else { final SortedNumericDocValues values = longValues(context); - if (DocValues.unwrapSingleton(values) != null) { - return DocValues.unwrapSingletonBits(values); - } else { - return DocValues.docsWithValue(values, context.reader().maxDoc()); - } + return org.elasticsearch.index.fielddata.FieldData.docsWithValue(values, + context.reader().maxDoc()); } } @@ -313,7 +303,7 @@ public abstract class ValuesSource { return new DoubleValues(delegate.doubleValues(context), script.getLeafSearchScript(context)); } - static class LongValues extends SortingNumericDocValues implements ScorerAware { + static class LongValues extends AbstractSortingNumericDocValues implements ScorerAware { private final SortedNumericDocValues longValues; private final LeafSearchScript script; @@ -324,20 +314,23 @@ public abstract class ValuesSource { } @Override - public void setDocument(int doc) { - longValues.setDocument(doc); - resize(longValues.count()); - script.setDocument(doc); - for (int i = 0; i < count(); ++i) { - script.setNextAggregationValue(longValues.valueAt(i)); - values[i] = script.runAsLong(); - } - sort(); + public void setScorer(Scorer scorer) { + script.setScorer(scorer); } @Override - public void setScorer(Scorer scorer) { - script.setScorer(scorer); + public boolean advanceExact(int target) throws IOException { + if (longValues.advanceExact(target)) { + resize(longValues.docValueCount()); + script.setDocument(target); + for (int i = 0; i < docValueCount(); ++i) { + script.setNextAggregationValue(longValues.nextValue()); + values[i] = script.runAsLong(); + } + sort(); + return true; + } + return false; } } @@ -352,20 +345,23 @@ public abstract class ValuesSource { } @Override - public void setDocument(int doc) { - doubleValues.setDocument(doc); - resize(doubleValues.count()); - script.setDocument(doc); - for (int i = 0; i < count(); ++i) { - script.setNextAggregationValue(doubleValues.valueAt(i)); - values[i] = script.runAsDouble(); - } - sort(); + public void setScorer(Scorer scorer) { + script.setScorer(scorer); } @Override - public void setScorer(Scorer scorer) { - script.setScorer(scorer); + public boolean advanceExact(int target) throws IOException { + if (doubleValues.advanceExact(target)) { + resize(doubleValues.docValueCount()); + script.setDocument(target); + for (int i = 0; i < docValueCount(); ++i) { + script.setNextAggregationValue(doubleValues.nextValue()); + values[i] = script.runAsDouble(); + } + sort(); + return true; + } + return false; } } } @@ -468,21 +464,27 @@ public abstract class ValuesSource { } @Override - public void setDocument(int docId) { - bytesValues.setDocument(docId); - count = bytesValues.count(); - grow(); - for (int i = 0; i < count; ++i) { - final BytesRef value = bytesValues.valueAt(i); - script.setNextAggregationValue(value.utf8ToString()); - values[i].copyChars(script.run().toString()); - } - sort(); + public void setScorer(Scorer scorer) { + script.setScorer(scorer); } @Override - public void setScorer(Scorer scorer) { - script.setScorer(scorer); + public boolean advanceExact(int doc) throws IOException { + if (bytesValues.advanceExact(doc)) { + count = bytesValues.docValueCount(); + grow(); + for (int i = 0; i < count; ++i) { + final BytesRef value = bytesValues.nextValue(); + script.setNextAggregationValue(value.utf8ToString()); + values[i].copyChars(script.run().toString()); + } + sort(); + return true; + } else { + count = 0; + grow(); + return false; + } } } } @@ -493,12 +495,12 @@ public abstract class ValuesSource { @Override public MultiGeoPointValues geoPointValues(LeafReaderContext context) { - return org.elasticsearch.index.fielddata.FieldData.emptyMultiGeoPoints(context.reader().maxDoc()); + return org.elasticsearch.index.fielddata.FieldData.emptyMultiGeoPoints(); } @Override public SortedBinaryDocValues bytesValues(LeafReaderContext context) throws IOException { - return org.elasticsearch.index.fielddata.FieldData.emptySortedBinary(context.reader().maxDoc()); + return org.elasticsearch.index.fielddata.FieldData.emptySortedBinary(); } }; @@ -506,11 +508,8 @@ public abstract class ValuesSource { @Override public Bits docsWithValue(LeafReaderContext context) { final MultiGeoPointValues geoPoints = geoPointValues(context); - if (org.elasticsearch.index.fielddata.FieldData.unwrapSingleton(geoPoints) != null) { - return org.elasticsearch.index.fielddata.FieldData.unwrapSingletonBits(geoPoints); - } else { - return org.elasticsearch.index.fielddata.FieldData.docsWithValue(geoPoints, context.reader().maxDoc()); - } + return org.elasticsearch.index.fielddata.FieldData.docsWithValue(geoPoints, + context.reader().maxDoc()); } public abstract MultiGeoPointValues geoPointValues(LeafReaderContext context); diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/support/values/ScriptBytesValues.java b/core/src/main/java/org/elasticsearch/search/aggregations/support/values/ScriptBytesValues.java index ce36fec941..78685ed0e8 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/support/values/ScriptBytesValues.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/support/values/ScriptBytesValues.java @@ -24,6 +24,7 @@ import org.elasticsearch.index.fielddata.SortedBinaryDocValues; import org.elasticsearch.index.fielddata.SortingBinaryDocValues; import org.elasticsearch.script.LeafSearchScript; +import java.io.IOException; import java.lang.reflect.Array; import java.util.Collection; @@ -48,14 +49,16 @@ public class ScriptBytesValues extends SortingBinaryDocValues implements ScorerA } @Override - public void setDocument(int docId) { - script.setDocument(docId); + public boolean advanceExact(int doc) throws IOException { + script.setDocument(doc); final Object value = script.run(); - if (value == null) { - count = 0; + return false; } else if (value.getClass().isArray()) { count = Array.getLength(value); + if (count == 0) { + return false; + } grow(); for (int i = 0; i < count; ++i) { set(i, Array.get(value, i)); @@ -63,6 +66,9 @@ public class ScriptBytesValues extends SortingBinaryDocValues implements ScorerA } else if (value instanceof Collection) { final Collection<?> coll = (Collection<?>) value; count = coll.size(); + if (count == 0) { + return false; + } grow(); int i = 0; for (Object v : coll) { @@ -73,6 +79,7 @@ public class ScriptBytesValues extends SortingBinaryDocValues implements ScorerA set(0, value); } sort(); + return true; } @Override diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/support/values/ScriptDoubleValues.java b/core/src/main/java/org/elasticsearch/search/aggregations/support/values/ScriptDoubleValues.java index 4f91f59ebe..619ffde0a1 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/support/values/ScriptDoubleValues.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/support/values/ScriptDoubleValues.java @@ -25,6 +25,7 @@ import org.elasticsearch.script.LeafSearchScript; import org.elasticsearch.search.aggregations.AggregationExecutionException; import org.joda.time.ReadableInstant; +import java.io.IOException; import java.lang.reflect.Array; import java.util.Collection; @@ -41,12 +42,12 @@ public class ScriptDoubleValues extends SortingNumericDoubleValues implements Sc } @Override - public void setDocument(int docId) { - script.setDocument(docId); + public boolean advanceExact(int target) throws IOException { + script.setDocument(target); final Object value = script.run(); if (value == null) { - resize(0); + return false; } else if (value instanceof Number) { resize(1); values[0] = ((Number) value).doubleValue(); @@ -54,23 +55,32 @@ public class ScriptDoubleValues extends SortingNumericDoubleValues implements Sc resize(1); values[0] = ((ReadableInstant) value).getMillis(); } else if (value.getClass().isArray()) { - resize(Array.getLength(value)); - for (int i = 0; i < count(); ++i) { + int length = Array.getLength(value); + if (length == 0) { + return false; + } + resize(length); + for (int i = 0; i < length; ++i) { values[i] = toDoubleValue(Array.get(value, i)); } } else if (value instanceof Collection) { - resize(((Collection<?>) value).size()); + Collection<?> coll = (Collection<?>) value; + if (coll.isEmpty()) { + return false; + } + resize(coll.size()); int i = 0; - for (Object v : (Collection<?>) value) { + for (Object v : coll) { values[i++] = toDoubleValue(v); } - assert i == count(); + assert i == docValueCount(); } else { resize(1); values[0] = toDoubleValue(value); } sort(); + return true; } private static double toDoubleValue(Object o) { diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/support/values/ScriptLongValues.java b/core/src/main/java/org/elasticsearch/search/aggregations/support/values/ScriptLongValues.java index cd515401c5..6247e96c7e 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/support/values/ScriptLongValues.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/support/values/ScriptLongValues.java @@ -21,11 +21,12 @@ package org.elasticsearch.search.aggregations.support.values; import org.apache.lucene.search.Scorer; import org.apache.lucene.util.LongValues; import org.elasticsearch.common.lucene.ScorerAware; -import org.elasticsearch.index.fielddata.SortingNumericDocValues; +import org.elasticsearch.index.fielddata.AbstractSortingNumericDocValues; import org.elasticsearch.script.LeafSearchScript; import org.elasticsearch.search.aggregations.AggregationExecutionException; import org.joda.time.ReadableInstant; +import java.io.IOException; import java.lang.reflect.Array; import java.util.Collection; import java.util.Iterator; @@ -33,7 +34,7 @@ import java.util.Iterator; /** * {@link LongValues} implementation which is based on a script */ -public class ScriptLongValues extends SortingNumericDocValues implements ScorerAware { +public class ScriptLongValues extends AbstractSortingNumericDocValues implements ScorerAware { final LeafSearchScript script; @@ -43,28 +44,36 @@ public class ScriptLongValues extends SortingNumericDocValues implements ScorerA } @Override - public void setDocument(int docId) { - script.setDocument(docId); + public boolean advanceExact(int target) throws IOException { + script.setDocument(target); final Object value = script.run(); if (value == null) { - resize(0); + return false; } else if (value.getClass().isArray()) { - resize(Array.getLength(value)); - for (int i = 0; i < count(); ++i) { + int length = Array.getLength(value); + if (length == 0) { + return false; + } + resize(length); + for (int i = 0; i < length; ++i) { values[i] = toLongValue(Array.get(value, i)); } } else if (value instanceof Collection) { - resize(((Collection<?>) value).size()); + Collection<?> coll = (Collection<?>) value; + if (coll.isEmpty()) { + return false; + } + resize(coll.size()); int i = 0; - for (Iterator<?> it = ((Collection<?>) value).iterator(); it.hasNext(); ++i) { + for (Iterator<?> it = coll.iterator(); it.hasNext(); ++i) { values[i] = toLongValue(it.next()); } - assert i == count(); + assert i == docValueCount(); } else { @@ -73,6 +82,7 @@ public class ScriptLongValues extends SortingNumericDocValues implements ScorerA } sort(); + return true; } private static long toLongValue(Object o) { diff --git a/core/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java b/core/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java index 97f2681252..e5bfcfa6df 100644 --- a/core/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java +++ b/core/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java @@ -132,41 +132,41 @@ public class FetchPhase implements SearchPhase { } } - SearchHit[] hits = new SearchHit[context.docIdsToLoadSize()]; - FetchSubPhase.HitContext hitContext = new FetchSubPhase.HitContext(); - for (int index = 0; index < context.docIdsToLoadSize(); index++) { - if(context.isCancelled()) { - throw new TaskCancelledException("cancelled"); - } - int docId = context.docIdsToLoad()[context.docIdsToLoadFrom() + index]; - int readerIndex = ReaderUtil.subIndex(docId, context.searcher().getIndexReader().leaves()); - LeafReaderContext subReaderContext = context.searcher().getIndexReader().leaves().get(readerIndex); - int subDocId = docId - subReaderContext.docBase; + try { + SearchHit[] hits = new SearchHit[context.docIdsToLoadSize()]; + FetchSubPhase.HitContext hitContext = new FetchSubPhase.HitContext(); + for (int index = 0; index < context.docIdsToLoadSize(); index++) { + if(context.isCancelled()) { + throw new TaskCancelledException("cancelled"); + } + int docId = context.docIdsToLoad()[context.docIdsToLoadFrom() + index]; + int readerIndex = ReaderUtil.subIndex(docId, context.searcher().getIndexReader().leaves()); + LeafReaderContext subReaderContext = context.searcher().getIndexReader().leaves().get(readerIndex); + int subDocId = docId - subReaderContext.docBase; - final SearchHit searchHit; - try { + final SearchHit searchHit; int rootDocId = findRootDocumentIfNested(context, subReaderContext, subDocId); if (rootDocId != -1) { searchHit = createNestedSearchHit(context, docId, subDocId, rootDocId, fieldNames, fieldNamePatterns, subReaderContext); } else { searchHit = createSearchHit(context, fieldsVisitor, docId, subDocId, subReaderContext); } - } catch (IOException e) { - throw ExceptionsHelper.convertToElastic(e); + + hits[index] = searchHit; + hitContext.reset(searchHit, subReaderContext, subDocId, context.searcher()); + for (FetchSubPhase fetchSubPhase : fetchSubPhases) { + fetchSubPhase.hitExecute(context, hitContext); + } } - hits[index] = searchHit; - hitContext.reset(searchHit, subReaderContext, subDocId, context.searcher()); for (FetchSubPhase fetchSubPhase : fetchSubPhases) { - fetchSubPhase.hitExecute(context, hitContext); + fetchSubPhase.hitsExecute(context, hits); } - } - for (FetchSubPhase fetchSubPhase : fetchSubPhases) { - fetchSubPhase.hitsExecute(context, hits); + context.fetchResult().hits(new SearchHits(hits, context.queryResult().getTotalHits(), context.queryResult().getMaxScore())); + } catch (IOException e) { + throw ExceptionsHelper.convertToElastic(e); } - - context.fetchResult().hits(new SearchHits(hits, context.queryResult().getTotalHits(), context.queryResult().getMaxScore())); } private int findRootDocumentIfNested(SearchContext context, LeafReaderContext subReaderContext, int subDocId) throws IOException { diff --git a/core/src/main/java/org/elasticsearch/search/fetch/FetchSubPhase.java b/core/src/main/java/org/elasticsearch/search/fetch/FetchSubPhase.java index 6f34eba212..84154926bf 100644 --- a/core/src/main/java/org/elasticsearch/search/fetch/FetchSubPhase.java +++ b/core/src/main/java/org/elasticsearch/search/fetch/FetchSubPhase.java @@ -25,6 +25,7 @@ import org.apache.lucene.search.IndexSearcher; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.internal.SearchContext; +import java.io.IOException; import java.util.HashMap; import java.util.Map; @@ -79,8 +80,8 @@ public interface FetchSubPhase { /** * Executes the hit level phase, with a reader and doc id (note, its a low level reader, and the matching doc). */ - default void hitExecute(SearchContext context, HitContext hitContext) {} + default void hitExecute(SearchContext context, HitContext hitContext) throws IOException {} - default void hitsExecute(SearchContext context, SearchHit[] hits) {} + default void hitsExecute(SearchContext context, SearchHit[] hits) throws IOException {} } diff --git a/core/src/main/java/org/elasticsearch/search/fetch/subphase/DocValueFieldsFetchSubPhase.java b/core/src/main/java/org/elasticsearch/search/fetch/subphase/DocValueFieldsFetchSubPhase.java index 031f7a6240..42cee23d39 100644 --- a/core/src/main/java/org/elasticsearch/search/fetch/subphase/DocValueFieldsFetchSubPhase.java +++ b/core/src/main/java/org/elasticsearch/search/fetch/subphase/DocValueFieldsFetchSubPhase.java @@ -25,6 +25,7 @@ import org.elasticsearch.search.SearchHitField; import org.elasticsearch.search.fetch.FetchSubPhase; import org.elasticsearch.search.internal.SearchContext; +import java.io.IOException; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -37,7 +38,7 @@ import java.util.HashMap; public final class DocValueFieldsFetchSubPhase implements FetchSubPhase { @Override - public void hitExecute(SearchContext context, HitContext hitContext) { + public void hitExecute(SearchContext context, HitContext hitContext) throws IOException { if (context.collapse() != null) { // retrieve the `doc_value` associated with the collapse field String name = context.collapse().getFieldType().name(); diff --git a/core/src/main/java/org/elasticsearch/search/fetch/subphase/InnerHitsContext.java b/core/src/main/java/org/elasticsearch/search/fetch/subphase/InnerHitsContext.java index 4edde7f9bc..aa4168bd04 100644 --- a/core/src/main/java/org/elasticsearch/search/fetch/subphase/InnerHitsContext.java +++ b/core/src/main/java/org/elasticsearch/search/fetch/subphase/InnerHitsContext.java @@ -130,13 +130,9 @@ public final class InnerHitsContext { return new TopDocs(context.searcher().count(q), Lucene.EMPTY_SCORE_DOCS, 0); } else { int topN = Math.min(from() + size(), context.searcher().getIndexReader().maxDoc()); - TopDocsCollector topDocsCollector; + TopDocsCollector<?> topDocsCollector; if (sort() != null) { - try { - topDocsCollector = TopFieldCollector.create(sort().sort, topN, true, trackScores(), trackScores()); - } catch (IOException e) { - throw ExceptionsHelper.convertToElastic(e); - } + topDocsCollector = TopFieldCollector.create(sort().sort, topN, true, trackScores(), trackScores()); } else { topDocsCollector = TopScoreDocCollector.create(topN); } @@ -148,7 +144,6 @@ public final class InnerHitsContext { return topDocsCollector.topDocs(from(), size()); } } - } public static final class ParentChildInnerHits extends BaseInnerHits { diff --git a/core/src/main/java/org/elasticsearch/search/fetch/subphase/ParentFieldSubFetchPhase.java b/core/src/main/java/org/elasticsearch/search/fetch/subphase/ParentFieldSubFetchPhase.java index 00b9686702..0ffef32e42 100644 --- a/core/src/main/java/org/elasticsearch/search/fetch/subphase/ParentFieldSubFetchPhase.java +++ b/core/src/main/java/org/elasticsearch/search/fetch/subphase/ParentFieldSubFetchPhase.java @@ -62,11 +62,11 @@ public final class ParentFieldSubFetchPhase implements FetchSubPhase { public static String getParentId(ParentFieldMapper fieldMapper, LeafReader reader, int docId) { try { SortedDocValues docValues = reader.getSortedDocValues(fieldMapper.name()); - if (docValues == null) { + if (docValues == null || docValues.advanceExact(docId) == false) { // hit has no _parent field. return null; } - BytesRef parentId = docValues.get(docId); + BytesRef parentId = docValues.binaryValue(); return parentId.length > 0 ? parentId.utf8ToString() : null; } catch (IOException e) { throw ExceptionsHelper.convertToElastic(e); diff --git a/core/src/main/java/org/elasticsearch/search/fetch/subphase/VersionFetchSubPhase.java b/core/src/main/java/org/elasticsearch/search/fetch/subphase/VersionFetchSubPhase.java index 1ce102e364..5f69230ca4 100644 --- a/core/src/main/java/org/elasticsearch/search/fetch/subphase/VersionFetchSubPhase.java +++ b/core/src/main/java/org/elasticsearch/search/fetch/subphase/VersionFetchSubPhase.java @@ -38,8 +38,8 @@ public final class VersionFetchSubPhase implements FetchSubPhase { long version = Versions.NOT_FOUND; try { NumericDocValues versions = hitContext.reader().getNumericDocValues(VersionFieldMapper.NAME); - if (versions != null) { - version = versions.get(hitContext.docId()); + if (versions != null && versions.advanceExact(hitContext.docId())) { + version = versions.longValue(); } } catch (IOException e) { throw new ElasticsearchException("Could not retrieve version", e); diff --git a/core/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java b/core/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java index 8d33140e3e..d3b1951846 100644 --- a/core/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java +++ b/core/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java @@ -110,7 +110,7 @@ public class ContextIndexSearcher extends IndexSearcher implements Releasable { } @Override - public Weight createWeight(Query query, boolean needsScores) throws IOException { + public Weight createWeight(Query query, boolean needsScores, float boost) throws IOException { if (profiler != null) { // createWeight() is called for each query in the tree, so we tell the queryProfiler // each invocation so that it can build an internal representation of the query @@ -119,7 +119,7 @@ public class ContextIndexSearcher extends IndexSearcher implements Releasable { profile.startTime(QueryTimingType.CREATE_WEIGHT); final Weight weight; try { - weight = super.createWeight(query, needsScores); + weight = super.createWeight(query, needsScores, boost); } finally { profile.stopAndRecordTime(); profiler.pollLastElement(); @@ -127,7 +127,7 @@ public class ContextIndexSearcher extends IndexSearcher implements Releasable { return new ProfileWeight(query, weight, profile); } else { // needs to be 'super', not 'in' in order to use aggregated DFS - return super.createWeight(query, needsScores); + return super.createWeight(query, needsScores, boost); } } diff --git a/core/src/main/java/org/elasticsearch/search/lookup/LeafDocLookup.java b/core/src/main/java/org/elasticsearch/search/lookup/LeafDocLookup.java index 6ce060c493..f02cf14f4b 100644 --- a/core/src/main/java/org/elasticsearch/search/lookup/LeafDocLookup.java +++ b/core/src/main/java/org/elasticsearch/search/lookup/LeafDocLookup.java @@ -19,12 +19,14 @@ package org.elasticsearch.search.lookup; import org.apache.lucene.index.LeafReaderContext; +import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.common.Nullable; import org.elasticsearch.index.fielddata.IndexFieldDataService; import org.elasticsearch.index.fielddata.ScriptDocValues; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperService; +import java.io.IOException; import java.security.AccessController; import java.security.PrivilegedAction; import java.util.Arrays; @@ -86,7 +88,11 @@ public class LeafDocLookup implements Map<String, ScriptDocValues<?>> { }); localCacheFieldData.put(fieldName, scriptValues); } - scriptValues.setNextDocId(docId); + try { + scriptValues.setNextDocId(docId); + } catch (IOException e) { + throw ExceptionsHelper.convertToElastic(e); + } return scriptValues; } diff --git a/core/src/main/java/org/elasticsearch/search/profile/query/ProfileWeight.java b/core/src/main/java/org/elasticsearch/search/profile/query/ProfileWeight.java index 9ca33f8454..716f68da86 100644 --- a/core/src/main/java/org/elasticsearch/search/profile/query/ProfileWeight.java +++ b/core/src/main/java/org/elasticsearch/search/profile/query/ProfileWeight.java @@ -80,16 +80,6 @@ public final class ProfileWeight extends Weight { } @Override - public float getValueForNormalization() throws IOException { - return subQueryWeight.getValueForNormalization(); - } - - @Override - public void normalize(float norm, float topLevelBoost) { - subQueryWeight.normalize(norm, topLevelBoost); - } - - @Override public void extractTerms(Set<Term> set) { subQueryWeight.extractTerms(set); } diff --git a/core/src/main/java/org/elasticsearch/search/slice/DocValuesSliceQuery.java b/core/src/main/java/org/elasticsearch/search/slice/DocValuesSliceQuery.java index 95cafacedd..796bcb2dac 100644 --- a/core/src/main/java/org/elasticsearch/search/slice/DocValuesSliceQuery.java +++ b/core/src/main/java/org/elasticsearch/search/slice/DocValuesSliceQuery.java @@ -23,10 +23,13 @@ import com.carrotsearch.hppc.BitMixer; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.DocValues; import org.apache.lucene.index.SortedNumericDocValues; +import org.apache.lucene.search.ConstantScoreScorer; +import org.apache.lucene.search.ConstantScoreWeight; +import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Scorer; +import org.apache.lucene.search.TwoPhaseIterator; import org.apache.lucene.search.Weight; -import org.apache.lucene.search.RandomAccessWeight; -import org.apache.lucene.util.Bits; import java.io.IOException; @@ -42,27 +45,38 @@ public final class DocValuesSliceQuery extends SliceQuery { } @Override - public Weight createWeight(IndexSearcher searcher, boolean needsScores) throws IOException { - return new RandomAccessWeight(this) { + public Weight createWeight(IndexSearcher searcher, boolean needsScores, float boost) throws IOException { + return new ConstantScoreWeight(this, boost) { + @Override - protected Bits getMatchingDocs(final LeafReaderContext context) throws IOException { + public Scorer scorer(LeafReaderContext context) throws IOException { final SortedNumericDocValues values = DocValues.getSortedNumeric(context.reader(), getField()); - return new Bits() { + final DocIdSetIterator approximation = DocIdSetIterator.all(context.reader().maxDoc()); + final TwoPhaseIterator twoPhase = new TwoPhaseIterator(approximation) { + @Override - public boolean get(int doc) { - values.setDocument(doc); - for (int i = 0; i < values.count(); i++) { - return contains(BitMixer.mix(values.valueAt(i))); + public boolean matches() throws IOException { + if (values.advanceExact(approximation.docID())) { + for (int i = 0; i < values.docValueCount(); i++) { + if (contains(BitMixer.mix(values.nextValue()))) { + return true; + } + } + return false; + } else { + return contains(0); } - return contains(0); } @Override - public int length() { - return context.reader().maxDoc(); + public float matchCost() { + // BitMixer.mix seems to be about 10 ops + return 10; } }; + return new ConstantScoreScorer(this, score(), twoPhase); } + }; } } diff --git a/core/src/main/java/org/elasticsearch/search/slice/TermsSliceQuery.java b/core/src/main/java/org/elasticsearch/search/slice/TermsSliceQuery.java index ddc02d32e5..bb6908c99f 100644 --- a/core/src/main/java/org/elasticsearch/search/slice/TermsSliceQuery.java +++ b/core/src/main/java/org/elasticsearch/search/slice/TermsSliceQuery.java @@ -55,8 +55,8 @@ public final class TermsSliceQuery extends SliceQuery { } @Override - public Weight createWeight(IndexSearcher searcher, boolean needsScores) throws IOException { - return new ConstantScoreWeight(this) { + public Weight createWeight(IndexSearcher searcher, boolean needsScores, float boost) throws IOException { + return new ConstantScoreWeight(this, boost) { @Override public Scorer scorer(LeafReaderContext context) throws IOException { final DocIdSet disi = build(context.reader()); diff --git a/core/src/main/java/org/elasticsearch/search/sort/ScriptSortBuilder.java b/core/src/main/java/org/elasticsearch/search/sort/ScriptSortBuilder.java index ec4e19d1ee..2901e05f05 100644 --- a/core/src/main/java/org/elasticsearch/search/sort/ScriptSortBuilder.java +++ b/core/src/main/java/org/elasticsearch/search/sort/ScriptSortBuilder.java @@ -32,6 +32,7 @@ import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.xcontent.ConstructingObjectParser; import org.elasticsearch.common.xcontent.ObjectParser.ValueType; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.index.fielddata.AbstractBinaryDocValues; import org.elasticsearch.index.fielddata.FieldData; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested; @@ -261,16 +262,20 @@ public class ScriptSortBuilder extends SortBuilder<ScriptSortBuilder> { @Override protected SortedBinaryDocValues getValues(LeafReaderContext context) throws IOException { leafScript = searchScript.getLeafSearchScript(context); - final BinaryDocValues values = new BinaryDocValues() { + final BinaryDocValues values = new AbstractBinaryDocValues() { final BytesRefBuilder spare = new BytesRefBuilder(); @Override - public BytesRef get(int docID) { - leafScript.setDocument(docID); + public boolean advanceExact(int doc) throws IOException { + leafScript.setDocument(doc); + return true; + } + @Override + public BytesRef binaryValue() { spare.copyChars(leafScript.run().toString()); return spare.get(); } }; - return FieldData.singleton(values, null); + return FieldData.singleton(values); } @Override protected void setScorer(Scorer scorer) { @@ -286,12 +291,16 @@ public class ScriptSortBuilder extends SortBuilder<ScriptSortBuilder> { leafScript = searchScript.getLeafSearchScript(context); final NumericDoubleValues values = new NumericDoubleValues() { @Override - public double get(int docID) { - leafScript.setDocument(docID); + public boolean advanceExact(int doc) throws IOException { + leafScript.setDocument(doc); + return false; + } + @Override + public double doubleValue() { return leafScript.runAsDouble(); } }; - return FieldData.singleton(values, null); + return FieldData.singleton(values); } @Override protected void setScorer(Scorer scorer) { diff --git a/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggester.java b/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggester.java index abc8d209cd..0b127b2eee 100644 --- a/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggester.java +++ b/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggester.java @@ -83,7 +83,7 @@ public class CompletionSuggester extends Suggester<CompletionSuggestionContext> private static void suggest(IndexSearcher searcher, CompletionQuery query, TopSuggestDocsCollector collector) throws IOException { query = (CompletionQuery) query.rewrite(searcher.getIndexReader()); - Weight weight = query.createWeight(searcher, collector.needsScores()); + Weight weight = query.createWeight(searcher, collector.needsScores(), 1f); for (LeafReaderContext context : searcher.getIndexReader().leaves()) { BulkScorer scorer = weight.bulkScorer(context); if (scorer != null) { @@ -185,7 +185,7 @@ public class CompletionSuggester extends Suggester<CompletionSuggestionContext> private final Map<Integer, SuggestDoc> scoreDocMap; // TODO: expose dup removal - + TopDocumentsCollector(int num) { super(1, false); // TODO hack, we don't use the underlying pq, so we allocate a size of 1 this.num = num; diff --git a/core/src/main/resources/org/elasticsearch/bootstrap/security.policy b/core/src/main/resources/org/elasticsearch/bootstrap/security.policy index 5c9c56de18..6d28944680 100644 --- a/core/src/main/resources/org/elasticsearch/bootstrap/security.policy +++ b/core/src/main/resources/org/elasticsearch/bootstrap/security.policy @@ -31,7 +31,7 @@ grant codeBase "${codebase.securesm-1.1.jar}" { //// Very special jar permissions: //// These are dangerous permissions that we don't want to grant to everything. -grant codeBase "${codebase.lucene-core-6.5.0.jar}" { +grant codeBase "${codebase.lucene-core-7.0.0-snapshot-89f6d17.jar}" { // needed to allow MMapDirectory's "unmap hack" (die unmap hack, die) // java 8 package permission java.lang.RuntimePermission "accessClassInPackage.sun.misc"; @@ -42,7 +42,7 @@ grant codeBase "${codebase.lucene-core-6.5.0.jar}" { permission java.lang.RuntimePermission "accessDeclaredMembers"; }; -grant codeBase "${codebase.lucene-misc-6.5.0.jar}" { +grant codeBase "${codebase.lucene-misc-7.0.0-snapshot-89f6d17.jar}" { // needed to allow shard shrinking to use hard-links if possible via lucenes HardlinkCopyDirectoryWrapper permission java.nio.file.LinkPermission "hard"; }; diff --git a/core/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy b/core/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy index 2c39ccb350..9a089ef281 100644 --- a/core/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy +++ b/core/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy @@ -33,7 +33,7 @@ grant codeBase "${codebase.securemock-1.2.jar}" { permission java.lang.reflect.ReflectPermission "suppressAccessChecks"; }; -grant codeBase "${codebase.lucene-test-framework-6.5.0.jar}" { +grant codeBase "${codebase.lucene-test-framework-7.0.0-snapshot-89f6d17.jar}" { // needed by RamUsageTester permission java.lang.reflect.ReflectPermission "suppressAccessChecks"; // needed for testing hardlinks in StoreRecoveryTests since we install MockFS diff --git a/core/src/test/java/org/apache/lucene/queries/BlendedTermQueryTests.java b/core/src/test/java/org/apache/lucene/queries/BlendedTermQueryTests.java index 3c727ddf6e..b047716956 100644 --- a/core/src/test/java/org/apache/lucene/queries/BlendedTermQueryTests.java +++ b/core/src/test/java/org/apache/lucene/queries/BlendedTermQueryTests.java @@ -85,7 +85,7 @@ public class BlendedTermQueryTests extends ESTestCase { { Term[] terms = new Term[]{new Term("firstname", "simon"), new Term("surname", "simon")}; - BlendedTermQuery query = BlendedTermQuery.booleanBlendedQuery(terms, true); + BlendedTermQuery query = BlendedTermQuery.booleanBlendedQuery(terms); TopDocs search = searcher.search(query, 3); ScoreDoc[] scoreDocs = search.scoreDocs; assertEquals(3, scoreDocs.length); @@ -93,7 +93,6 @@ public class BlendedTermQueryTests extends ESTestCase { } { BooleanQuery.Builder query = new BooleanQuery.Builder(); - query.setDisableCoord(true); query.add(new TermQuery(new Term("firstname", "simon")), BooleanClause.Occur.SHOULD); query.add(new TermQuery(new Term("surname", "simon")), BooleanClause.Occur.SHOULD); TopDocs search = searcher.search(query.build(), 1); @@ -146,7 +145,6 @@ public class BlendedTermQueryTests extends ESTestCase { { String[] fields = new String[]{"username", "song"}; BooleanQuery.Builder query = new BooleanQuery.Builder(); - query.setDisableCoord(true); query.add(BlendedTermQuery.dismaxBlendedQuery(toTerms(fields, "foo"), 0.1f), BooleanClause.Occur.SHOULD); query.add(BlendedTermQuery.dismaxBlendedQuery(toTerms(fields, "fighters"), 0.1f), BooleanClause.Occur.SHOULD); query.add(BlendedTermQuery.dismaxBlendedQuery(toTerms(fields, "generator"), 0.1f), BooleanClause.Occur.SHOULD); @@ -156,7 +154,6 @@ public class BlendedTermQueryTests extends ESTestCase { } { BooleanQuery.Builder query = new BooleanQuery.Builder(); - query.setDisableCoord(true); DisjunctionMaxQuery uname = new DisjunctionMaxQuery( Arrays.asList(new TermQuery(new Term("username", "foo")), new TermQuery(new Term("song", "foo"))), 0.0f); @@ -186,13 +183,12 @@ public class BlendedTermQueryTests extends ESTestCase { } String term = TestUtil.randomRealisticUnicodeString(random(), 1, 10); Term[] terms = toTerms(fields, term); - boolean disableCoord = random().nextBoolean(); boolean useBoolean = random().nextBoolean(); float tieBreaker = random().nextFloat(); - BlendedTermQuery query = useBoolean ? BlendedTermQuery.booleanBlendedQuery(terms, disableCoord) : BlendedTermQuery.dismaxBlendedQuery(terms, tieBreaker); + BlendedTermQuery query = useBoolean ? BlendedTermQuery.booleanBlendedQuery(terms) : BlendedTermQuery.dismaxBlendedQuery(terms, tieBreaker); QueryUtils.check(query); terms = toTerms(fields, term); - BlendedTermQuery query2 = useBoolean ? BlendedTermQuery.booleanBlendedQuery(terms, disableCoord) : BlendedTermQuery.dismaxBlendedQuery(terms, tieBreaker); + BlendedTermQuery query2 = useBoolean ? BlendedTermQuery.booleanBlendedQuery(terms) : BlendedTermQuery.dismaxBlendedQuery(terms, tieBreaker); assertEquals(query, query2); } } @@ -222,7 +218,7 @@ public class BlendedTermQueryTests extends ESTestCase { } BlendedTermQuery blendedTermQuery = random().nextBoolean() ? BlendedTermQuery.dismaxBlendedQuery(terms.toArray(new Term[0]), random().nextFloat()) : - BlendedTermQuery.booleanBlendedQuery(terms.toArray(new Term[0]), random().nextBoolean()); + BlendedTermQuery.booleanBlendedQuery(terms.toArray(new Term[0])); Set<Term> extracted = new HashSet<>(); IndexSearcher searcher = new IndexSearcher(new MultiReader()); searcher.createNormalizedWeight(blendedTermQuery, false).extractTerms(extracted); diff --git a/core/src/test/java/org/elasticsearch/common/lucene/LuceneTests.java b/core/src/test/java/org/elasticsearch/common/lucene/LuceneTests.java index 33a0b855a7..a1132647c7 100644 --- a/core/src/test/java/org/elasticsearch/common/lucene/LuceneTests.java +++ b/core/src/test/java/org/elasticsearch/common/lucene/LuceneTests.java @@ -28,7 +28,6 @@ import org.apache.lucene.document.TextField; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; -import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.NoDeletionPolicy; import org.apache.lucene.index.NoMergePolicy; @@ -375,7 +374,7 @@ public class LuceneTests extends ESTestCase { try (DirectoryReader reader = DirectoryReader.open(w)) { IndexSearcher searcher = newSearcher(reader); - Weight termWeight = new TermQuery(new Term("foo", "bar")).createWeight(searcher, false); + Weight termWeight = new TermQuery(new Term("foo", "bar")).createWeight(searcher, false, 1f); assertEquals(1, reader.leaves().size()); LeafReaderContext leafReaderContext = searcher.getIndexReader().leaves().get(0); Bits bits = Lucene.asSequentialAccessBits(leafReaderContext.reader().maxDoc(), termWeight.scorer(leafReaderContext)); diff --git a/core/src/test/java/org/elasticsearch/common/lucene/ShardCoreKeyMapTests.java b/core/src/test/java/org/elasticsearch/common/lucene/ShardCoreKeyMapTests.java index 5cba06741a..ca24a1346f 100644 --- a/core/src/test/java/org/elasticsearch/common/lucene/ShardCoreKeyMapTests.java +++ b/core/src/test/java/org/elasticsearch/common/lucene/ShardCoreKeyMapTests.java @@ -112,7 +112,7 @@ public class ShardCoreKeyMapTests extends ESTestCase { assertEquals(3, map.size()); for (LeafReaderContext ctx : reader2.leaves()) { - assertEquals(shardId2, map.getShardId(ctx.reader().getCoreCacheKey())); + assertEquals(shardId2, map.getShardId(ctx.reader().getCoreCacheHelper().getKey())); } w1.addDocument(new Document()); @@ -136,7 +136,7 @@ public class ShardCoreKeyMapTests extends ESTestCase { final Set<Object> index1Keys = new HashSet<>(); for (DirectoryReader reader : Arrays.asList(reader1, reader2)) { for (LeafReaderContext ctx : reader.leaves()) { - index1Keys.add(ctx.reader().getCoreCacheKey()); + index1Keys.add(ctx.reader().getCoreCacheHelper().getKey()); } } index1Keys.removeAll(map.getCoreKeysForIndex("index1")); diff --git a/core/src/test/java/org/elasticsearch/common/lucene/index/ESDirectoryReaderTests.java b/core/src/test/java/org/elasticsearch/common/lucene/index/ESDirectoryReaderTests.java index 817dabfece..397dd28460 100644 --- a/core/src/test/java/org/elasticsearch/common/lucene/index/ESDirectoryReaderTests.java +++ b/core/src/test/java/org/elasticsearch/common/lucene/index/ESDirectoryReaderTests.java @@ -29,7 +29,6 @@ import org.apache.lucene.index.NoMergePolicy; import org.apache.lucene.index.Term; import org.apache.lucene.store.Directory; import org.apache.lucene.util.IOUtils; -import org.elasticsearch.index.Index; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.test.ESTestCase; @@ -66,7 +65,7 @@ public class ESDirectoryReaderTests extends ESTestCase { // we should have the same cache key as before assertEquals(1, ir2.numDocs()); assertEquals(1, ir2.leaves().size()); - assertSame(ir.leaves().get(0).reader().getCoreCacheKey(), ir2.leaves().get(0).reader().getCoreCacheKey()); + assertSame(ir.leaves().get(0).reader().getCoreCacheHelper().getKey(), ir2.leaves().get(0).reader().getCoreCacheHelper().getKey()); IOUtils.close(ir, ir2, iw, dir); } } diff --git a/core/src/test/java/org/elasticsearch/deps/lucene/SimpleLuceneTests.java b/core/src/test/java/org/elasticsearch/deps/lucene/SimpleLuceneTests.java index 2b08213ab4..6b117fc7e9 100644 --- a/core/src/test/java/org/elasticsearch/deps/lucene/SimpleLuceneTests.java +++ b/core/src/test/java/org/elasticsearch/deps/lucene/SimpleLuceneTests.java @@ -21,24 +21,19 @@ package org.elasticsearch.deps.lucene; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; -import org.apache.lucene.document.FieldType; -import org.apache.lucene.document.LegacyIntField; +import org.apache.lucene.document.IntPoint; import org.apache.lucene.document.SortedDocValuesField; +import org.apache.lucene.document.StoredField; import org.apache.lucene.document.TextField; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.FieldInfo; -import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.IndexableField; -import org.apache.lucene.index.LeafReader; -import org.apache.lucene.index.PostingsEnum; -import org.apache.lucene.index.SlowCompositeReaderWrapper; +import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.StoredFieldVisitor; import org.apache.lucene.index.Term; -import org.apache.lucene.index.Terms; -import org.apache.lucene.index.TermsEnum; import org.apache.lucene.search.FieldDoc; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.MatchAllDocsQuery; @@ -50,8 +45,6 @@ import org.apache.lucene.search.TopFieldDocs; import org.apache.lucene.store.Directory; import org.apache.lucene.store.RAMDirectory; import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.BytesRefBuilder; -import org.apache.lucene.util.LegacyNumericUtils; import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.test.ESTestCase; @@ -71,7 +64,7 @@ public class SimpleLuceneTests extends ESTestCase { document.add(new SortedDocValuesField("str", new BytesRef(text))); indexWriter.addDocument(document); } - IndexReader reader = SlowCompositeReaderWrapper.wrap(DirectoryReader.open(indexWriter)); + IndexReader reader = DirectoryReader.open(indexWriter); IndexSearcher searcher = new IndexSearcher(reader); TopFieldDocs docs = searcher.search(new MatchAllDocsQuery(), 10, new Sort(new SortField("str", SortField.Type.STRING))); for (int i = 0; i < 10; i++) { @@ -86,7 +79,8 @@ public class SimpleLuceneTests extends ESTestCase { Document document = new Document(); document.add(new TextField("_id", "1", Field.Store.YES)); - document.add(new LegacyIntField("test", 2, LegacyIntField.TYPE_STORED)); + document.add(new IntPoint("test", 2)); + document.add(new StoredField("test", 2)); indexWriter.addDocument(document); IndexReader reader = DirectoryReader.open(indexWriter); @@ -94,11 +88,9 @@ public class SimpleLuceneTests extends ESTestCase { TopDocs topDocs = searcher.search(new TermQuery(new Term("_id", "1")), 1); Document doc = searcher.doc(topDocs.scoreDocs[0].doc); IndexableField f = doc.getField("test"); - assertThat(f.stringValue(), equalTo("2")); + assertThat(f.numericValue(), equalTo(2)); - BytesRefBuilder bytes = new BytesRefBuilder(); - LegacyNumericUtils.intToPrefixCoded(2, 0, bytes); - topDocs = searcher.search(new TermQuery(new Term("test", bytes.get())), 1); + topDocs = searcher.search(IntPoint.newExactQuery("test", 2), 1); doc = searcher.doc(topDocs.scoreDocs[0].doc); f = doc.getField("test"); assertThat(f.stringValue(), equalTo("2")); @@ -139,40 +131,6 @@ public class SimpleLuceneTests extends ESTestCase { indexWriter.close(); } - public void testBoost() throws Exception { - Directory dir = new RAMDirectory(); - IndexWriter indexWriter = new IndexWriter(dir, new IndexWriterConfig(Lucene.STANDARD_ANALYZER)); - - for (int i = 0; i < 100; i++) { - // TODO (just setting the boost value does not seem to work...) - StringBuilder value = new StringBuilder().append("value"); - for (int j = 0; j < i; j++) { - value.append(" ").append("value"); - } - Document document = new Document(); - TextField textField = new TextField("_id", Integer.toString(i), Field.Store.YES); - textField.setBoost(i); - document.add(textField); - textField = new TextField("value", value.toString(), Field.Store.YES); - textField.setBoost(i); - document.add(textField); - indexWriter.addDocument(document); - } - - IndexReader reader = DirectoryReader.open(indexWriter); - IndexSearcher searcher = new IndexSearcher(reader); - TermQuery query = new TermQuery(new Term("value", "value")); - TopDocs topDocs = searcher.search(query, 100); - assertThat(100, equalTo(topDocs.totalHits)); - for (int i = 0; i < topDocs.scoreDocs.length; i++) { - Document doc = searcher.doc(topDocs.scoreDocs[i].doc); -// System.out.println(doc.get("id") + ": " + searcher.explain(query, topDocs.scoreDocs[i].doc)); - assertThat(doc.get("_id"), equalTo(Integer.toString(100 - i - 1))); - } - - indexWriter.close(); - } - public void testNRTSearchOnClosedWriter() throws Exception { Directory dir = new RAMDirectory(); IndexWriter indexWriter = new IndexWriter(dir, new IndexWriterConfig(Lucene.STANDARD_ANALYZER)); @@ -181,7 +139,6 @@ public class SimpleLuceneTests extends ESTestCase { for (int i = 0; i < 100; i++) { Document document = new Document(); TextField field = new TextField("_id", Integer.toString(i), Field.Store.YES); - field.setBoost(i); document.add(field); indexWriter.addDocument(document); } @@ -189,60 +146,9 @@ public class SimpleLuceneTests extends ESTestCase { indexWriter.close(); - TermsEnum termDocs = SlowCompositeReaderWrapper.wrap(reader).terms("_id").iterator(); - termDocs.next(); - } - - /** - * A test just to verify that term freqs are not stored for numeric fields. <tt>int1</tt> is not storing termFreq - * and <tt>int2</tt> does. - */ - public void testNumericTermDocsFreqs() throws Exception { - Directory dir = new RAMDirectory(); - IndexWriter indexWriter = new IndexWriter(dir, new IndexWriterConfig(Lucene.STANDARD_ANALYZER)); - - Document doc = new Document(); - FieldType type = LegacyIntField.TYPE_NOT_STORED; - LegacyIntField field = new LegacyIntField("int1", 1, type); - doc.add(field); - - type = new FieldType(LegacyIntField.TYPE_NOT_STORED); - type.setIndexOptions(IndexOptions.DOCS_AND_FREQS); - type.freeze(); - - field = new LegacyIntField("int1", 1, type); - doc.add(field); - - field = new LegacyIntField("int2", 1, type); - doc.add(field); - - field = new LegacyIntField("int2", 1, type); - doc.add(field); - - indexWriter.addDocument(doc); - - IndexReader reader = DirectoryReader.open(indexWriter); - LeafReader atomicReader = SlowCompositeReaderWrapper.wrap(reader); - - Terms terms = atomicReader.terms("int1"); - TermsEnum termsEnum = terms.iterator(); - termsEnum.next(); - - PostingsEnum termDocs = termsEnum.postings(null); - assertThat(termDocs.nextDoc(), equalTo(0)); - assertThat(termDocs.docID(), equalTo(0)); - assertThat(termDocs.freq(), equalTo(1)); - - terms = atomicReader.terms("int2"); - termsEnum = terms.iterator(); - termsEnum.next(); - termDocs = termsEnum.postings(termDocs); - assertThat(termDocs.nextDoc(), equalTo(0)); - assertThat(termDocs.docID(), equalTo(0)); - assertThat(termDocs.freq(), equalTo(2)); - - reader.close(); - indexWriter.close(); + for (LeafReaderContext leaf : reader.leaves()) { + leaf.reader().terms("_id").iterator().next(); + } } private DirectoryReader refreshReader(DirectoryReader reader) throws IOException { diff --git a/core/src/test/java/org/elasticsearch/index/IndexModuleTests.java b/core/src/test/java/org/elasticsearch/index/IndexModuleTests.java index 106dc9274d..6f80da2d63 100644 --- a/core/src/test/java/org/elasticsearch/index/IndexModuleTests.java +++ b/core/src/test/java/org/elasticsearch/index/IndexModuleTests.java @@ -431,8 +431,8 @@ public class IndexModuleTests extends ESTestCase { } @Override - public SimWeight computeWeight(CollectionStatistics collectionStats, TermStatistics... termStats) { - return delegate.computeWeight(collectionStats, termStats); + public SimWeight computeWeight(float boost, CollectionStatistics collectionStats, TermStatistics... termStats) { + return delegate.computeWeight(boost, collectionStats, termStats); } @Override diff --git a/core/src/test/java/org/elasticsearch/index/codec/CodecTests.java b/core/src/test/java/org/elasticsearch/index/codec/CodecTests.java index e49e21bd2f..57dfd19185 100644 --- a/core/src/test/java/org/elasticsearch/index/codec/CodecTests.java +++ b/core/src/test/java/org/elasticsearch/index/codec/CodecTests.java @@ -20,12 +20,10 @@ package org.elasticsearch.index.codec; import org.apache.lucene.codecs.Codec; -import org.apache.lucene.codecs.lucene50.Lucene50Codec; import org.apache.lucene.codecs.lucene50.Lucene50StoredFieldsFormat; import org.apache.lucene.codecs.lucene50.Lucene50StoredFieldsFormat.Mode; -import org.apache.lucene.codecs.lucene53.Lucene53Codec; -import org.apache.lucene.codecs.lucene54.Lucene54Codec; import org.apache.lucene.codecs.lucene62.Lucene62Codec; +import org.apache.lucene.codecs.lucene70.Lucene70Codec; import org.apache.lucene.document.Document; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexWriter; @@ -55,10 +53,8 @@ public class CodecTests extends ESTestCase { public void testResolveDefaultCodecs() throws Exception { CodecService codecService = createCodecService(); assertThat(codecService.codec("default"), instanceOf(PerFieldMappingPostingFormatCodec.class)); - assertThat(codecService.codec("default"), instanceOf(Lucene62Codec.class)); - assertThat(codecService.codec("Lucene54"), instanceOf(Lucene54Codec.class)); - assertThat(codecService.codec("Lucene53"), instanceOf(Lucene53Codec.class)); - assertThat(codecService.codec("Lucene50"), instanceOf(Lucene50Codec.class)); + assertThat(codecService.codec("default"), instanceOf(Lucene70Codec.class)); + assertThat(codecService.codec("Lucene62"), instanceOf(Lucene62Codec.class)); } public void testDefault() throws Exception { diff --git a/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java b/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java index 30d22dfb73..3f9965c066 100644 --- a/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java +++ b/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java @@ -2171,7 +2171,10 @@ public class InternalEngineTests extends ESTestCase { final Bits bits = leaf.getLiveDocs(); for (int docID = 0; docID < leaf.maxDoc(); docID++) { if (bits == null || bits.get(docID)) { - final long seqNo = values.get(docID); + if (values.advanceExact(docID) == false) { + throw new AssertionError("Document does not have a seq number: " + docID); + } + final long seqNo = values.longValue(); assertFalse("should not have more than one document with the same seq_no[" + seqNo + "]", bitSet.get((int) seqNo)); bitSet.set((int) seqNo); } diff --git a/core/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataImplTestCase.java b/core/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataImplTestCase.java index c96292d90e..c22114e28a 100644 --- a/core/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataImplTestCase.java +++ b/core/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataImplTestCase.java @@ -31,6 +31,7 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.Strings; import org.elasticsearch.search.MultiValueMode; +import java.io.IOException; import java.util.List; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThanOrEqualTo; @@ -77,8 +78,8 @@ public abstract class AbstractFieldDataImplTestCase extends AbstractFieldDataTes AtomicFieldData fieldData = indexFieldData.load(readerContext); SortedBinaryDocValues values = fieldData.getBytesValues(); for (int i = 0; i < readerContext.reader().maxDoc(); ++i) { - values.setDocument(i); - assertThat(values.count(), greaterThanOrEqualTo(1)); + assertTrue(values.advanceExact(i)); + assertThat(values.docValueCount(), greaterThanOrEqualTo(1)); } } } @@ -93,16 +94,17 @@ public abstract class AbstractFieldDataImplTestCase extends AbstractFieldDataTes SortedBinaryDocValues bytesValues = fieldData.getBytesValues(); - bytesValues.setDocument(0); - assertThat(bytesValues.count(), equalTo(1)); - assertThat(bytesValues.valueAt(0), equalTo(new BytesRef(two()))); - bytesValues.setDocument(1); - assertThat(bytesValues.count(), equalTo(1)); - assertThat(bytesValues.valueAt(0), equalTo(new BytesRef(one()))); - bytesValues.setDocument(2); - assertThat(bytesValues.count(), equalTo(1)); - assertThat(bytesValues.valueAt(0), equalTo(new BytesRef(three()))); - + assertTrue(bytesValues.advanceExact(0)); + assertThat(bytesValues.docValueCount(), equalTo(1)); + assertThat(bytesValues.nextValue(), equalTo(new BytesRef(two()))); + assertTrue(bytesValues.advanceExact(1)); + assertThat(bytesValues.docValueCount(), equalTo(1)); + assertThat(bytesValues.nextValue(), equalTo(new BytesRef(one()))); + assertTrue(bytesValues.advanceExact(2)); + assertThat(bytesValues.docValueCount(), equalTo(1)); + assertThat(bytesValues.nextValue(), equalTo(new BytesRef(three()))); + + bytesValues = fieldData.getBytesValues(); assertValues(bytesValues, 0, two()); assertValues(bytesValues, 1, one()); assertValues(bytesValues, 2, three()); @@ -132,19 +134,23 @@ public abstract class AbstractFieldDataImplTestCase extends AbstractFieldDataTes protected abstract void fillSingleValueWithMissing() throws Exception; - public void assertValues(SortedBinaryDocValues values, int docId, BytesRef... actualValues) { - values.setDocument(docId); - assertThat(values.count(), equalTo(actualValues.length)); - for (int i = 0; i < actualValues.length; i++) { - assertThat(values.valueAt(i), equalTo(actualValues[i])); + public void assertValues(SortedBinaryDocValues values, int docId, BytesRef... actualValues) throws IOException { + assertEquals(actualValues.length > 0, values.advanceExact(docId)); + if (actualValues.length > 0) { + assertThat(values.docValueCount(), equalTo(actualValues.length)); + for (int i = 0; i < actualValues.length; i++) { + assertThat(values.nextValue(), equalTo(actualValues[i])); + } } } - public void assertValues(SortedBinaryDocValues values, int docId, String... actualValues) { - values.setDocument(docId); - assertThat(values.count(), equalTo(actualValues.length)); - for (int i = 0; i < actualValues.length; i++) { - assertThat(values.valueAt(i), equalTo(new BytesRef(actualValues[i]))); + public void assertValues(SortedBinaryDocValues values, int docId, String... actualValues) throws IOException { + assertEquals(actualValues.length > 0, values.advanceExact(docId)); + if (actualValues.length > 0) { + assertThat(values.docValueCount(), equalTo(actualValues.length)); + for (int i = 0; i < actualValues.length; i++) { + assertThat(values.nextValue(), equalTo(new BytesRef(actualValues[i]))); + } } } diff --git a/core/src/test/java/org/elasticsearch/index/fielddata/AbstractGeoFieldDataTestCase.java b/core/src/test/java/org/elasticsearch/index/fielddata/AbstractGeoFieldDataTestCase.java index 21d816e83e..a46fd68a29 100644 --- a/core/src/test/java/org/elasticsearch/index/fielddata/AbstractGeoFieldDataTestCase.java +++ b/core/src/test/java/org/elasticsearch/index/fielddata/AbstractGeoFieldDataTestCase.java @@ -25,10 +25,10 @@ import org.apache.lucene.document.StringField; import org.elasticsearch.common.geo.GeoPoint; import org.elasticsearch.common.geo.GeoUtils; +import java.io.IOException; + import static org.elasticsearch.test.geo.RandomShapeGenerator.randomPoint; import static org.hamcrest.Matchers.allOf; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.hamcrest.Matchers.lessThanOrEqualTo; @@ -71,23 +71,20 @@ public abstract class AbstractGeoFieldDataTestCase extends AbstractFieldDataImpl assumeFalse("Only test on non geo_point fields", getFieldDataType().equals("geo_point")); } - protected void assertValues(MultiGeoPointValues values, int docId) { + protected void assertValues(MultiGeoPointValues values, int docId) throws IOException { assertValues(values, docId, false); } - protected void assertMissing(MultiGeoPointValues values, int docId) { + protected void assertMissing(MultiGeoPointValues values, int docId) throws IOException { assertValues(values, docId, true); } - private void assertValues(MultiGeoPointValues values, int docId, boolean missing) { - values.setDocument(docId); - int docCount = values.count(); - if (missing) { - assertThat(docCount, equalTo(0)); - } else { - assertThat(docCount, greaterThan(0)); + private void assertValues(MultiGeoPointValues values, int docId, boolean missing) throws IOException { + assertEquals(missing == false, values.advanceExact(docId)); + if (missing == false) { + final int docCount = values.docValueCount(); for (int i = 0; i < docCount; ++i) { - final GeoPoint point = values.valueAt(i); + final GeoPoint point = values.nextValue(); assertThat(point.lat(), allOf(greaterThanOrEqualTo(GeoUtils.MIN_LAT), lessThanOrEqualTo(GeoUtils.MAX_LAT))); assertThat(point.lon(), allOf(greaterThanOrEqualTo(GeoUtils.MIN_LON), lessThanOrEqualTo(GeoUtils.MAX_LON))); } diff --git a/core/src/test/java/org/elasticsearch/index/fielddata/AbstractStringFieldDataTestCase.java b/core/src/test/java/org/elasticsearch/index/fielddata/AbstractStringFieldDataTestCase.java index c241627887..aebb0a802e 100644 --- a/core/src/test/java/org/elasticsearch/index/fielddata/AbstractStringFieldDataTestCase.java +++ b/core/src/test/java/org/elasticsearch/index/fielddata/AbstractStringFieldDataTestCase.java @@ -20,6 +20,7 @@ package org.elasticsearch.index.fielddata; import com.carrotsearch.randomizedtesting.generators.RandomPicks; + import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.document.Field.Store; @@ -27,7 +28,7 @@ import org.apache.lucene.document.SortedSetDocValuesField; import org.apache.lucene.document.StringField; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.index.RandomAccessOrds; +import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.index.Term; import org.apache.lucene.index.TermsEnum; import org.apache.lucene.search.ConstantScoreQuery; @@ -48,8 +49,6 @@ import org.apache.lucene.util.TestUtil; import org.apache.lucene.util.UnicodeUtil; import org.elasticsearch.common.lucene.index.ElasticsearchDirectoryReader; import org.elasticsearch.common.lucene.search.Queries; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource; import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested; import org.elasticsearch.index.fielddata.fieldcomparator.BytesRefFieldComparatorSource; import org.elasticsearch.index.fielddata.ordinals.GlobalOrdinalsIndexFieldData; @@ -453,19 +452,6 @@ public abstract class AbstractStringFieldDataTestCase extends AbstractFieldDataI searcher.getIndexReader().close(); } - private void assertIteratorConsistentWithRandomAccess(RandomAccessOrds ords, int maxDoc) { - for (int doc = 0; doc < maxDoc; ++doc) { - ords.setDocument(doc); - final int cardinality = ords.cardinality(); - for (int i = 0; i < cardinality; ++i) { - assertEquals(ords.nextOrd(), ords.ordAt(i)); - } - for (int i = 0; i < 3; ++i) { - assertEquals(ords.nextOrd(), -1); - } - } - } - public void testGlobalOrdinals() throws Exception { fillExtendedMvSet(); refreshReader(); @@ -477,31 +463,29 @@ public abstract class AbstractStringFieldDataTestCase extends AbstractFieldDataI assertThat(globalOrdinals, instanceOf(GlobalOrdinalsIndexFieldData.class)); LeafReaderContext leaf = topLevelReader.leaves().get(0); AtomicOrdinalsFieldData afd = globalOrdinals.load(leaf); - RandomAccessOrds values = afd.getOrdinalsValues(); - assertIteratorConsistentWithRandomAccess(values, leaf.reader().maxDoc()); - values.setDocument(0); - assertThat(values.cardinality(), equalTo(2)); + SortedSetDocValues values = afd.getOrdinalsValues(); + assertTrue(values.advanceExact(0)); long ord = values.nextOrd(); assertThat(ord, equalTo(3L)); assertThat(values.lookupOrd(ord).utf8ToString(), equalTo("02")); ord = values.nextOrd(); assertThat(ord, equalTo(5L)); assertThat(values.lookupOrd(ord).utf8ToString(), equalTo("04")); - values.setDocument(1); - assertThat(values.cardinality(), equalTo(0)); - values.setDocument(2); - assertThat(values.cardinality(), equalTo(1)); + ord = values.nextOrd(); + assertThat(ord, equalTo(SortedSetDocValues.NO_MORE_ORDS)); + assertFalse(values.advanceExact(1)); + assertTrue(values.advanceExact(2)); ord = values.nextOrd(); assertThat(ord, equalTo(4L)); assertThat(values.lookupOrd(ord).utf8ToString(), equalTo("03")); + ord = values.nextOrd(); + assertThat(ord, equalTo(SortedSetDocValues.NO_MORE_ORDS)); // Second segment leaf = topLevelReader.leaves().get(1); afd = globalOrdinals.load(leaf); values = afd.getOrdinalsValues(); - assertIteratorConsistentWithRandomAccess(values, leaf.reader().maxDoc()); - values.setDocument(0); - assertThat(values.cardinality(), equalTo(3)); + assertTrue(values.advanceExact(0)); ord = values.nextOrd(); assertThat(ord, equalTo(5L)); assertThat(values.lookupOrd(ord).utf8ToString(), equalTo("04")); @@ -511,8 +495,9 @@ public abstract class AbstractStringFieldDataTestCase extends AbstractFieldDataI ord = values.nextOrd(); assertThat(ord, equalTo(7L)); assertThat(values.lookupOrd(ord).utf8ToString(), equalTo("06")); - values.setDocument(1); - assertThat(values.cardinality(), equalTo(3)); + ord = values.nextOrd(); + assertThat(ord, equalTo(SortedSetDocValues.NO_MORE_ORDS)); + assertTrue(values.advanceExact(1)); ord = values.nextOrd(); assertThat(ord, equalTo(7L)); assertThat(values.lookupOrd(ord).utf8ToString(), equalTo("06")); @@ -522,10 +507,10 @@ public abstract class AbstractStringFieldDataTestCase extends AbstractFieldDataI ord = values.nextOrd(); assertThat(ord, equalTo(9L)); assertThat(values.lookupOrd(ord).utf8ToString(), equalTo("08")); - values.setDocument(2); - assertThat(values.cardinality(), equalTo(0)); - values.setDocument(3); - assertThat(values.cardinality(), equalTo(3)); + ord = values.nextOrd(); + assertThat(ord, equalTo(SortedSetDocValues.NO_MORE_ORDS)); + assertFalse(values.advanceExact(2)); + assertTrue(values.advanceExact(3)); ord = values.nextOrd(); assertThat(ord, equalTo(9L)); assertThat(values.lookupOrd(ord).utf8ToString(), equalTo("08")); @@ -535,15 +520,14 @@ public abstract class AbstractStringFieldDataTestCase extends AbstractFieldDataI ord = values.nextOrd(); assertThat(ord, equalTo(11L)); assertThat(values.lookupOrd(ord).utf8ToString(), equalTo("10")); + ord = values.nextOrd(); + assertThat(ord, equalTo(SortedSetDocValues.NO_MORE_ORDS)); // Third segment leaf = topLevelReader.leaves().get(2); afd = globalOrdinals.load(leaf); values = afd.getOrdinalsValues(); - assertIteratorConsistentWithRandomAccess(values, leaf.reader().maxDoc()); - values.setDocument(0); - values.setDocument(0); - assertThat(values.cardinality(), equalTo(3)); + assertTrue(values.advanceExact(0)); ord = values.nextOrd(); assertThat(ord, equalTo(0L)); assertThat(values.lookupOrd(ord).utf8ToString(), equalTo("!08")); @@ -553,6 +537,8 @@ public abstract class AbstractStringFieldDataTestCase extends AbstractFieldDataI ord = values.nextOrd(); assertThat(ord, equalTo(2L)); assertThat(values.lookupOrd(ord).utf8ToString(), equalTo("!10")); + ord = values.nextOrd(); + assertThat(ord, equalTo(SortedSetDocValues.NO_MORE_ORDS)); } public void testTermsEnum() throws Exception { diff --git a/core/src/test/java/org/elasticsearch/index/fielddata/BinaryDVFieldDataTests.java b/core/src/test/java/org/elasticsearch/index/fielddata/BinaryDVFieldDataTests.java index 70c1486fe1..dfb3a3c1b3 100644 --- a/core/src/test/java/org/elasticsearch/index/fielddata/BinaryDVFieldDataTests.java +++ b/core/src/test/java/org/elasticsearch/index/fielddata/BinaryDVFieldDataTests.java @@ -23,7 +23,6 @@ import com.carrotsearch.hppc.ObjectArrayList; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.compress.CompressedXContent; -import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.CollectionUtils; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; @@ -87,23 +86,22 @@ public class BinaryDVFieldDataTests extends AbstractFieldDataTestCase { SortedBinaryDocValues bytesValues = fieldData.getBytesValues(); CollectionUtils.sortAndDedup(bytesList1); - bytesValues.setDocument(0); - assertThat(bytesValues.count(), equalTo(2)); - assertThat(bytesValues.valueAt(0), equalTo(new BytesRef(bytesList1.get(0)))); - assertThat(bytesValues.valueAt(1), equalTo(new BytesRef(bytesList1.get(1)))); + assertTrue(bytesValues.advanceExact(0)); + assertThat(bytesValues.docValueCount(), equalTo(2)); + assertThat(bytesValues.nextValue(), equalTo(new BytesRef(bytesList1.get(0)))); + assertThat(bytesValues.nextValue(), equalTo(new BytesRef(bytesList1.get(1)))); - bytesValues.setDocument(1); - assertThat(bytesValues.count(), equalTo(1)); - assertThat(bytesValues.valueAt(0), equalTo(new BytesRef(bytes1))); + assertTrue(bytesValues.advanceExact(1)); + assertThat(bytesValues.docValueCount(), equalTo(1)); + assertThat(bytesValues.nextValue(), equalTo(new BytesRef(bytes1))); - bytesValues.setDocument(2); - assertThat(bytesValues.count(), equalTo(0)); + assertFalse(bytesValues.advanceExact(2)); CollectionUtils.sortAndDedup(bytesList2); - bytesValues.setDocument(3); - assertThat(bytesValues.count(), equalTo(2)); - assertThat(bytesValues.valueAt(0), equalTo(new BytesRef(bytesList2.get(0)))); - assertThat(bytesValues.valueAt(1), equalTo(new BytesRef(bytesList2.get(1)))); + assertTrue(bytesValues.advanceExact(3)); + assertThat(bytesValues.docValueCount(), equalTo(2)); + assertThat(bytesValues.nextValue(), equalTo(new BytesRef(bytesList2.get(0)))); + assertThat(bytesValues.nextValue(), equalTo(new BytesRef(bytesList2.get(1)))); } } diff --git a/core/src/test/java/org/elasticsearch/index/fielddata/FieldDataTests.java b/core/src/test/java/org/elasticsearch/index/fielddata/FieldDataTests.java index 33ddc80868..6236517dde 100644 --- a/core/src/test/java/org/elasticsearch/index/fielddata/FieldDataTests.java +++ b/core/src/test/java/org/elasticsearch/index/fielddata/FieldDataTests.java @@ -25,83 +25,116 @@ import org.apache.lucene.index.SortedNumericDocValues; import org.apache.lucene.util.NumericUtils; import org.elasticsearch.test.ESTestCase; +import java.io.IOException; + public class FieldDataTests extends ESTestCase { - public void testSortableLongBitsToDoubles() { + private static class DummyValues extends AbstractNumericDocValues { + + private final long value; + private int docID = -1; + + DummyValues(long value) { + this.value = value; + } + + @Override + public boolean advanceExact(int target) throws IOException { + docID = target; + return true; + } + + @Override + public int docID() { + return docID; + } + + @Override + public long longValue() throws IOException { + return value; + } + } + + public void testSortableLongBitsToDoubles() throws IOException { final double value = randomDouble(); final long valueBits = NumericUtils.doubleToSortableLong(value); - NumericDocValues values = new NumericDocValues() { - @Override - public long get(int docID) { - return valueBits; - } - }; + NumericDocValues values = new DummyValues(valueBits); - SortedNumericDoubleValues asMultiDoubles = FieldData.sortableLongBitsToDoubles(DocValues.singleton(values, null)); + SortedNumericDoubleValues asMultiDoubles = FieldData.sortableLongBitsToDoubles(DocValues.singleton(values)); NumericDoubleValues asDoubles = FieldData.unwrapSingleton(asMultiDoubles); assertNotNull(asDoubles); - assertEquals(value, asDoubles.get(0), 0); + assertTrue(asDoubles.advanceExact(0)); + assertEquals(value, asDoubles.doubleValue(), 0); + values = new DummyValues(valueBits); + asMultiDoubles = FieldData.sortableLongBitsToDoubles(DocValues.singleton(values)); NumericDocValues backToLongs = DocValues.unwrapSingleton(FieldData.toSortableLongBits(asMultiDoubles)); assertSame(values, backToLongs); - SortedNumericDocValues multiValues = new SortedNumericDocValues() { + SortedNumericDocValues multiValues = new AbstractSortedNumericDocValues() { @Override - public long valueAt(int index) { - return valueBits; + public boolean advanceExact(int target) throws IOException { + return true; } @Override - public void setDocument(int doc) { + public long nextValue() { + return valueBits; } @Override - public int count() { + public int docValueCount() { return 1; } }; asMultiDoubles = FieldData.sortableLongBitsToDoubles(multiValues); - assertEquals(value, asMultiDoubles.valueAt(0), 0); + assertEquals(value, asMultiDoubles.nextValue(), 0); assertSame(multiValues, FieldData.toSortableLongBits(asMultiDoubles)); } - public void testDoublesToSortableLongBits() { + public void testDoublesToSortableLongBits() throws IOException { final double value = randomDouble(); final long valueBits = NumericUtils.doubleToSortableLong(value); NumericDoubleValues values = new NumericDoubleValues() { @Override - public double get(int docID) { + public boolean advanceExact(int doc) throws IOException { + return true; + } + @Override + public double doubleValue() { return value; } }; - SortedNumericDocValues asMultiLongs = FieldData.toSortableLongBits(FieldData.singleton(values, null)); + SortedNumericDocValues asMultiLongs = FieldData.toSortableLongBits(FieldData.singleton(values)); NumericDocValues asLongs = DocValues.unwrapSingleton(asMultiLongs); assertNotNull(asLongs); - assertEquals(valueBits, asLongs.get(0)); + assertTrue(asLongs.advanceExact(0)); + assertEquals(valueBits, asLongs.longValue()); SortedNumericDoubleValues multiValues = new SortedNumericDoubleValues() { @Override - public double valueAt(int index) { + public double nextValue() { return value; } @Override - public void setDocument(int doc) { + public boolean advanceExact(int target) throws IOException { + return true; } @Override - public int count() { + public int docValueCount() { return 1; } }; asMultiLongs = FieldData.toSortableLongBits(multiValues); - assertEquals(valueBits, asMultiLongs.valueAt(0)); + assertEquals(valueBits, asMultiLongs.nextValue()); assertSame(multiValues, FieldData.sortableLongBitsToDoubles(asMultiLongs)); } } diff --git a/core/src/test/java/org/elasticsearch/index/fielddata/FilterFieldDataTests.java b/core/src/test/java/org/elasticsearch/index/fielddata/FilterFieldDataTests.java index 33ecd57b13..19725aca52 100644 --- a/core/src/test/java/org/elasticsearch/index/fielddata/FilterFieldDataTests.java +++ b/core/src/test/java/org/elasticsearch/index/fielddata/FilterFieldDataTests.java @@ -22,11 +22,11 @@ import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.document.StringField; import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.index.RandomAccessOrds; +import org.apache.lucene.index.SortedSetDocValues; import org.elasticsearch.index.mapper.ContentPath; import org.elasticsearch.index.mapper.MappedFieldType; -import org.elasticsearch.index.mapper.TextFieldMapper; import org.elasticsearch.index.mapper.Mapper.BuilderContext; +import org.elasticsearch.index.mapper.TextFieldMapper; import java.util.List; import java.util.Random; @@ -71,7 +71,7 @@ public class FilterFieldDataTests extends AbstractFieldDataTestCase { IndexOrdinalsFieldData fieldData = ifdService.getForField(ft); for (LeafReaderContext context : contexts) { AtomicOrdinalsFieldData loadDirect = fieldData.loadDirect(context); - RandomAccessOrds bytesValues = loadDirect.getOrdinalsValues(); + SortedSetDocValues bytesValues = loadDirect.getOrdinalsValues(); assertThat(2L, equalTo(bytesValues.getValueCount())); assertThat(bytesValues.lookupOrd(0).utf8ToString(), equalTo("10")); assertThat(bytesValues.lookupOrd(1).utf8ToString(), equalTo("100")); @@ -86,7 +86,7 @@ public class FilterFieldDataTests extends AbstractFieldDataTestCase { IndexOrdinalsFieldData fieldData = ifdService.getForField(ft); for (LeafReaderContext context : contexts) { AtomicOrdinalsFieldData loadDirect = fieldData.loadDirect(context); - RandomAccessOrds bytesValues = loadDirect.getOrdinalsValues(); + SortedSetDocValues bytesValues = loadDirect.getOrdinalsValues(); assertThat(1L, equalTo(bytesValues.getValueCount())); assertThat(bytesValues.lookupOrd(0).utf8ToString(), equalTo("5")); } @@ -101,7 +101,7 @@ public class FilterFieldDataTests extends AbstractFieldDataTestCase { IndexOrdinalsFieldData fieldData = ifdService.getForField(ft); for (LeafReaderContext context : contexts) { AtomicOrdinalsFieldData loadDirect = fieldData.loadDirect(context); - RandomAccessOrds bytesValues = loadDirect.getOrdinalsValues(); + SortedSetDocValues bytesValues = loadDirect.getOrdinalsValues(); assertThat(2L, equalTo(bytesValues.getValueCount())); assertThat(bytesValues.lookupOrd(0).utf8ToString(), equalTo("10")); assertThat(bytesValues.lookupOrd(1).utf8ToString(), equalTo("100")); @@ -117,7 +117,7 @@ public class FilterFieldDataTests extends AbstractFieldDataTestCase { IndexOrdinalsFieldData fieldData = ifdService.getForField(ft); for (LeafReaderContext context : contexts) { AtomicOrdinalsFieldData loadDirect = fieldData.loadDirect(context); - RandomAccessOrds bytesValues = loadDirect.getOrdinalsValues(); + SortedSetDocValues bytesValues = loadDirect.getOrdinalsValues(); assertThat(2L, equalTo(bytesValues.getValueCount())); assertThat(bytesValues.lookupOrd(0).utf8ToString(), equalTo("10")); assertThat(bytesValues.lookupOrd(1).utf8ToString(), equalTo("100")); diff --git a/core/src/test/java/org/elasticsearch/index/fielddata/ParentChildFieldDataTests.java b/core/src/test/java/org/elasticsearch/index/fielddata/ParentChildFieldDataTests.java index f426e5433c..f7bdc8efed 100644 --- a/core/src/test/java/org/elasticsearch/index/fielddata/ParentChildFieldDataTests.java +++ b/core/src/test/java/org/elasticsearch/index/fielddata/ParentChildFieldDataTests.java @@ -132,40 +132,39 @@ public class ParentChildFieldDataTests extends AbstractFieldDataTestCase { AtomicFieldData fieldData = indexFieldData.load(readerContext); SortedBinaryDocValues bytesValues = fieldData.getBytesValues(); - bytesValues.setDocument(0); - assertThat(bytesValues.count(), equalTo(1)); - assertThat(bytesValues.valueAt(0).utf8ToString(), equalTo("1")); + assertTrue(bytesValues.advanceExact(0)); + assertThat(bytesValues.docValueCount(), equalTo(1)); + assertThat(bytesValues.nextValue().utf8ToString(), equalTo("1")); - bytesValues.setDocument(1); - assertThat(bytesValues.count(), equalTo(2)); - assertThat(bytesValues.valueAt(0).utf8ToString(), equalTo("1")); - assertThat(bytesValues.valueAt(1).utf8ToString(), equalTo("2")); + assertTrue(bytesValues.advanceExact(1)); + assertThat(bytesValues.docValueCount(), equalTo(2)); + assertThat(bytesValues.nextValue().utf8ToString(), equalTo("1")); + assertThat(bytesValues.nextValue().utf8ToString(), equalTo("2")); - bytesValues.setDocument(2); - assertThat(bytesValues.count(), equalTo(2)); - assertThat(bytesValues.valueAt(0).utf8ToString(), equalTo("1")); - assertThat(bytesValues.valueAt(1).utf8ToString(), equalTo("3")); + assertTrue(bytesValues.advanceExact(2)); + assertThat(bytesValues.docValueCount(), equalTo(2)); + assertThat(bytesValues.nextValue().utf8ToString(), equalTo("1")); + assertThat(bytesValues.nextValue().utf8ToString(), equalTo("3")); - bytesValues.setDocument(3); - assertThat(bytesValues.count(), equalTo(1)); - assertThat(bytesValues.valueAt(0).utf8ToString(), equalTo("2")); + assertTrue(bytesValues.advanceExact(3)); + assertThat(bytesValues.docValueCount(), equalTo(1)); + assertThat(bytesValues.nextValue().utf8ToString(), equalTo("2")); - bytesValues.setDocument(4); - assertThat(bytesValues.count(), equalTo(2)); - assertThat(bytesValues.valueAt(0).utf8ToString(), equalTo("2")); - assertThat(bytesValues.valueAt(1).utf8ToString(), equalTo("4")); + assertTrue(bytesValues.advanceExact(4)); + assertThat(bytesValues.docValueCount(), equalTo(2)); + assertThat(bytesValues.nextValue().utf8ToString(), equalTo("2")); + assertThat(bytesValues.nextValue().utf8ToString(), equalTo("4")); - bytesValues.setDocument(5); - assertThat(bytesValues.count(), equalTo(2)); - assertThat(bytesValues.valueAt(0).utf8ToString(), equalTo("1")); - assertThat(bytesValues.valueAt(1).utf8ToString(), equalTo("5")); + assertTrue(bytesValues.advanceExact(5)); + assertThat(bytesValues.docValueCount(), equalTo(2)); + assertThat(bytesValues.nextValue().utf8ToString(), equalTo("1")); + assertThat(bytesValues.nextValue().utf8ToString(), equalTo("5")); - bytesValues.setDocument(6); - assertThat(bytesValues.count(), equalTo(1)); - assertThat(bytesValues.valueAt(0).utf8ToString(), equalTo("2")); + assertTrue(bytesValues.advanceExact(6)); + assertThat(bytesValues.docValueCount(), equalTo(1)); + assertThat(bytesValues.nextValue().utf8ToString(), equalTo("2")); - bytesValues.setDocument(7); - assertThat(bytesValues.count(), equalTo(0)); + assertFalse(bytesValues.advanceExact(7)); } } @@ -236,7 +235,7 @@ public class ParentChildFieldDataTests extends AbstractFieldDataTestCase { ids[j] = BytesRef.deepCopyOf(id); } } - expected.put(context.reader().getCoreCacheKey(), ids); + expected.put(context.reader().getCoreCacheHelper().getKey(), ids); } for (int i = 0; i < numThreads; ++i) { @@ -249,7 +248,7 @@ public class ParentChildFieldDataTests extends AbstractFieldDataTestCase { for (LeafReaderContext context : reader.leaves()) { AtomicParentChildFieldData leafData = global.load(context); SortedDocValues parentIds = leafData.getOrdinalsValues(parentType); - final BytesRef[] expectedIds = expected.get(context.reader().getCoreCacheKey()); + final BytesRef[] expectedIds = expected.get(context.reader().getCoreCacheHelper().getKey()); for (int j = 0; j < parentIds.getValueCount(); ++j) { final BytesRef id = parentIds.lookupOrd(j); assertEquals(expectedIds[j], id); diff --git a/core/src/test/java/org/elasticsearch/index/fielddata/ScriptDocValuesDatesTests.java b/core/src/test/java/org/elasticsearch/index/fielddata/ScriptDocValuesDatesTests.java index f8579efae7..626327d454 100644 --- a/core/src/test/java/org/elasticsearch/index/fielddata/ScriptDocValuesDatesTests.java +++ b/core/src/test/java/org/elasticsearch/index/fielddata/ScriptDocValuesDatesTests.java @@ -19,15 +19,16 @@ package org.elasticsearch.index.fielddata; -import org.apache.lucene.index.SortedNumericDocValues; import org.elasticsearch.index.fielddata.ScriptDocValues.Dates; import org.elasticsearch.test.ESTestCase; import org.joda.time.DateTime; import org.joda.time.DateTimeZone; import org.joda.time.ReadableDateTime; +import java.io.IOException; + public class ScriptDocValuesDatesTests extends ESTestCase { - public void test() { + public void test() throws IOException { long[][] values = new long[between(3, 10)][]; ReadableDateTime[][] expectedDates = new ReadableDateTime[values.length][]; for (int d = 0; d < values.length; d++) { @@ -56,20 +57,23 @@ public class ScriptDocValuesDatesTests extends ESTestCase { } private Dates wrap(long[][] values) { - return new Dates(new SortedNumericDocValues() { + return new Dates(new AbstractSortedNumericDocValues() { long[] current; + int i; @Override - public void setDocument(int doc) { + public boolean advanceExact(int doc) { current = values[doc]; + i = 0; + return current.length > 0; } @Override - public int count() { + public int docValueCount() { return current.length; } @Override - public long valueAt(int index) { - return current[index]; + public long nextValue() { + return current[i++]; } }); } diff --git a/core/src/test/java/org/elasticsearch/index/fielddata/ScriptDocValuesGeoPointsTests.java b/core/src/test/java/org/elasticsearch/index/fielddata/ScriptDocValuesGeoPointsTests.java index 466ca0b699..1f71a808ab 100644 --- a/core/src/test/java/org/elasticsearch/index/fielddata/ScriptDocValuesGeoPointsTests.java +++ b/core/src/test/java/org/elasticsearch/index/fielddata/ScriptDocValuesGeoPointsTests.java @@ -23,6 +23,7 @@ import org.elasticsearch.common.geo.GeoPoint; import org.elasticsearch.common.geo.GeoUtils; import org.elasticsearch.test.ESTestCase; +import java.io.IOException; import java.util.Arrays; public class ScriptDocValuesGeoPointsTests extends ESTestCase { @@ -30,22 +31,24 @@ public class ScriptDocValuesGeoPointsTests extends ESTestCase { private static MultiGeoPointValues wrap(final GeoPoint... points) { return new MultiGeoPointValues() { int docID = -1; + int i; @Override - public GeoPoint valueAt(int i) { + public GeoPoint nextValue() { if (docID != 0) { fail(); } - return points[i]; + return points[i++]; } @Override - public void setDocument(int docId) { - this.docID = docId; + public boolean advanceExact(int docId) { + docID = docId; + return points.length > 0; } @Override - public int count() { + public int docValueCount() { if (docID != 0) { return 0; } @@ -62,7 +65,7 @@ public class ScriptDocValuesGeoPointsTests extends ESTestCase { return randomDouble() * 360 - 180; } - public void testGeoGetLatLon() { + public void testGeoGetLatLon() throws IOException { final double lat1 = randomLat(); final double lat2 = randomLat(); final double lon1 = randomLon(); @@ -81,7 +84,7 @@ public class ScriptDocValuesGeoPointsTests extends ESTestCase { assertTrue(Arrays.equals(new double[] {lon1, lon2}, script.getLons())); } - public void testGeoDistance() { + public void testGeoDistance() throws IOException { final double lat = randomLat(); final double lon = randomLon(); final MultiGeoPointValues values = wrap(new GeoPoint(lat, lon)); diff --git a/core/src/test/java/org/elasticsearch/index/fielddata/ScriptDocValuesLongsTests.java b/core/src/test/java/org/elasticsearch/index/fielddata/ScriptDocValuesLongsTests.java index c6ec8ce749..1b3e8fa227 100644 --- a/core/src/test/java/org/elasticsearch/index/fielddata/ScriptDocValuesLongsTests.java +++ b/core/src/test/java/org/elasticsearch/index/fielddata/ScriptDocValuesLongsTests.java @@ -19,15 +19,16 @@ package org.elasticsearch.index.fielddata; -import org.apache.lucene.index.SortedNumericDocValues; import org.elasticsearch.index.fielddata.ScriptDocValues.Longs; import org.elasticsearch.test.ESTestCase; import org.joda.time.DateTime; import org.joda.time.DateTimeZone; import org.joda.time.ReadableDateTime; +import java.io.IOException; + public class ScriptDocValuesLongsTests extends ESTestCase { - public void testLongs() { + public void testLongs() throws IOException { long[][] values = new long[between(3, 10)][]; for (int d = 0; d < values.length; d++) { values[d] = new long[randomBoolean() ? randomBoolean() ? 0 : 1 : between(2, 100)]; @@ -54,7 +55,7 @@ public class ScriptDocValuesLongsTests extends ESTestCase { } } - public void testDates() { + public void testDates() throws IOException { long[][] values = new long[between(3, 10)][]; ReadableDateTime[][] dates = new ReadableDateTime[values.length][]; for (int d = 0; d < values.length; d++) { @@ -87,20 +88,23 @@ public class ScriptDocValuesLongsTests extends ESTestCase { } private Longs wrap(long[][] values) { - return new Longs(new SortedNumericDocValues() { + return new Longs(new AbstractSortedNumericDocValues() { long[] current; + int i; @Override - public void setDocument(int doc) { + public boolean advanceExact(int doc) { + i = 0; current = values[doc]; + return current.length > 0; } @Override - public int count() { + public int docValueCount() { return current.length; } @Override - public long valueAt(int index) { - return current[index]; + public long nextValue() { + return current[i++]; } }); } diff --git a/core/src/test/java/org/elasticsearch/index/fielddata/fieldcomparator/ReplaceMissingTests.java b/core/src/test/java/org/elasticsearch/index/fielddata/fieldcomparator/ReplaceMissingTests.java index a291311c3b..65c6335ebb 100644 --- a/core/src/test/java/org/elasticsearch/index/fielddata/fieldcomparator/ReplaceMissingTests.java +++ b/core/src/test/java/org/elasticsearch/index/fielddata/fieldcomparator/ReplaceMissingTests.java @@ -62,49 +62,68 @@ public class ReplaceMissingTests extends ESTestCase { assertEquals("cat", dv.lookupOrd(0).utf8ToString()); assertEquals("dog", dv.lookupOrd(1).utf8ToString()); - assertEquals(0, dv.getOrd(0)); - assertEquals(0, dv.getOrd(1)); - assertEquals(1, dv.getOrd(2)); - + assertTrue(dv.advanceExact(0)); + assertEquals(0, dv.ordValue()); + assertTrue(dv.advanceExact(1)); + assertEquals(0, dv.ordValue()); + assertTrue(dv.advanceExact(2)); + assertEquals(1, dv.ordValue()); + + raw = ar.getSortedDocValues("field"); dv = new BytesRefFieldComparatorSource.ReplaceMissing(raw, new BytesRef("dog")); assertEquals(2, dv.getValueCount()); assertEquals("cat", dv.lookupOrd(0).utf8ToString()); assertEquals("dog", dv.lookupOrd(1).utf8ToString()); - assertEquals(0, dv.getOrd(0)); - assertEquals(1, dv.getOrd(1)); - assertEquals(1, dv.getOrd(2)); + assertTrue(dv.advanceExact(0)); + assertEquals(0, dv.ordValue()); + assertTrue(dv.advanceExact(1)); + assertEquals(1, dv.ordValue()); + assertTrue(dv.advanceExact(2)); + assertEquals(1, dv.ordValue()); // non-existing values + raw = ar.getSortedDocValues("field"); dv = new BytesRefFieldComparatorSource.ReplaceMissing(raw, new BytesRef("apple")); assertEquals(3, dv.getValueCount()); assertEquals("apple", dv.lookupOrd(0).utf8ToString()); assertEquals("cat", dv.lookupOrd(1).utf8ToString()); assertEquals("dog", dv.lookupOrd(2).utf8ToString()); - assertEquals(1, dv.getOrd(0)); - assertEquals(0, dv.getOrd(1)); - assertEquals(2, dv.getOrd(2)); + assertTrue(dv.advanceExact(0)); + assertEquals(1, dv.ordValue()); + assertTrue(dv.advanceExact(1)); + assertEquals(0, dv.ordValue()); + assertTrue(dv.advanceExact(2)); + assertEquals(2, dv.ordValue()); + raw = ar.getSortedDocValues("field"); dv = new BytesRefFieldComparatorSource.ReplaceMissing(raw, new BytesRef("company")); assertEquals(3, dv.getValueCount()); assertEquals("cat", dv.lookupOrd(0).utf8ToString()); assertEquals("company", dv.lookupOrd(1).utf8ToString()); assertEquals("dog", dv.lookupOrd(2).utf8ToString()); - assertEquals(0, dv.getOrd(0)); - assertEquals(1, dv.getOrd(1)); - assertEquals(2, dv.getOrd(2)); + assertTrue(dv.advanceExact(0)); + assertEquals(0, dv.ordValue()); + assertTrue(dv.advanceExact(1)); + assertEquals(1, dv.ordValue()); + assertTrue(dv.advanceExact(2)); + assertEquals(2, dv.ordValue()); + raw = ar.getSortedDocValues("field"); dv = new BytesRefFieldComparatorSource.ReplaceMissing(raw, new BytesRef("ebay")); assertEquals(3, dv.getValueCount()); assertEquals("cat", dv.lookupOrd(0).utf8ToString()); assertEquals("dog", dv.lookupOrd(1).utf8ToString()); assertEquals("ebay", dv.lookupOrd(2).utf8ToString()); - assertEquals(0, dv.getOrd(0)); - assertEquals(2, dv.getOrd(1)); - assertEquals(1, dv.getOrd(2)); + assertTrue(dv.advanceExact(0)); + assertEquals(0, dv.ordValue()); + assertTrue(dv.advanceExact(1)); + assertEquals(2, dv.ordValue()); + assertTrue(dv.advanceExact(2)); + assertEquals(1, dv.ordValue()); reader.close(); dir.close(); diff --git a/core/src/test/java/org/elasticsearch/index/fielddata/ordinals/MultiOrdinalsTests.java b/core/src/test/java/org/elasticsearch/index/fielddata/ordinals/MultiOrdinalsTests.java index c594427b7e..1ae6197c54 100644 --- a/core/src/test/java/org/elasticsearch/index/fielddata/ordinals/MultiOrdinalsTests.java +++ b/core/src/test/java/org/elasticsearch/index/fielddata/ordinals/MultiOrdinalsTests.java @@ -18,8 +18,8 @@ */ package org.elasticsearch.index.fielddata.ordinals; -import org.apache.lucene.index.RandomAccessOrds; import org.apache.lucene.index.SortedDocValues; +import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.util.packed.PackedInts; import org.elasticsearch.index.fielddata.FieldData; import org.elasticsearch.search.MultiValueMode; @@ -102,7 +102,7 @@ public class MultiOrdinalsTests extends ESTestCase { } }); Ordinals ords = creationMultiOrdinals(builder); - RandomAccessOrds docs = ords.ordinals(); + SortedSetDocValues docs = ords.ordinals(); final SortedDocValues singleOrds = MultiValueMode.MIN.select(docs); int docId = ordsAndIds.get(0).id; List<Long> docOrds = new ArrayList<>(); @@ -111,22 +111,18 @@ public class MultiOrdinalsTests extends ESTestCase { docOrds.add(ordAndId.ord); } else { if (!docOrds.isEmpty()) { - assertThat((long) singleOrds.getOrd(docId), equalTo(docOrds.get(0))); + assertTrue(singleOrds.advanceExact(docId)); + assertThat((long) singleOrds.ordValue(), equalTo(docOrds.get(0))); - docs.setDocument(docId); - final int numOrds = docs.cardinality(); - assertThat(numOrds, equalTo(docOrds.size())); - for (int i = 0; i < numOrds; i++) { - assertThat(docs.nextOrd(), equalTo(docOrds.get(i))); + assertTrue(docs.advanceExact(docId)); + for (Long ord : docOrds) { + assertThat(docs.nextOrd(), equalTo(ord)); } - final long[] array = new long[docOrds.size()]; - for (int i = 0; i < array.length; i++) { - array[i] = docOrds.get(i); - } - assertIter(docs, docId, array); + assertEquals(SortedSetDocValues.NO_MORE_ORDS, docs.nextOrd()); } for (int i = docId + 1; i < ordAndId.id; i++) { - assertThat((long) singleOrds.getOrd(i), equalTo(RandomAccessOrds.NO_MORE_ORDS)); + assertFalse(singleOrds.advanceExact(i)); + assertFalse(docs.advanceExact(i)); } docId = ordAndId.id; docOrds.clear(); @@ -209,18 +205,10 @@ public class MultiOrdinalsTests extends ESTestCase { }; Ordinals ordinals = creationMultiOrdinals(builder); - RandomAccessOrds docs = ordinals.ordinals(); + SortedSetDocValues docs = ordinals.ordinals(); assertEquals(docs, ordinalPlan); } - protected static void assertIter(RandomAccessOrds docs, int docId, long... expectedOrdinals) { - docs.setDocument(docId); - assertThat(docs.cardinality(), equalTo(expectedOrdinals.length)); - for (long expectedOrdinal : expectedOrdinals) { - assertThat(docs.nextOrd(), equalTo(expectedOrdinal)); - } - } - public void testMultiValuesDocsWithOverlappingStorageArrays() throws Exception { int maxDoc = 7; long maxOrds = 15; @@ -261,11 +249,11 @@ public class MultiOrdinalsTests extends ESTestCase { }; Ordinals ordinals = new MultiOrdinals(builder, PackedInts.FASTEST); - RandomAccessOrds docs = ordinals.ordinals(); + SortedSetDocValues docs = ordinals.ordinals(); assertEquals(docs, ordinalPlan); } - private void assertEquals(RandomAccessOrds docs, long[][] ordinalPlan) { + private void assertEquals(SortedSetDocValues docs, long[][] ordinalPlan) throws IOException { long maxOrd = 0; for (int doc = 0; doc < ordinalPlan.length; ++doc) { if (ordinalPlan[doc].length > 0) { @@ -276,10 +264,12 @@ public class MultiOrdinalsTests extends ESTestCase { assertThat(FieldData.isMultiValued(docs), equalTo(true)); for (int doc = 0; doc < ordinalPlan.length; ++doc) { long[] ords = ordinalPlan[doc]; - docs.setDocument(doc); - assertThat(docs.cardinality(), equalTo(ords.length)); - for (int i = 0; i < ords.length; ++i) { - assertThat(docs.ordAt(i), equalTo(ords[i])); + assertEquals(ords.length > 0, docs.advanceExact(doc)); + if (ords.length > 0) { + for (long ord : ords) { + assertThat(docs.nextOrd(), equalTo(ord)); + } + assertThat(docs.nextOrd(), equalTo(SortedSetDocValues.NO_MORE_ORDS)); } } } diff --git a/core/src/test/java/org/elasticsearch/index/fielddata/ordinals/SingleOrdinalsTests.java b/core/src/test/java/org/elasticsearch/index/fielddata/ordinals/SingleOrdinalsTests.java index 9eb73e9332..b0fbda6a94 100644 --- a/core/src/test/java/org/elasticsearch/index/fielddata/ordinals/SingleOrdinalsTests.java +++ b/core/src/test/java/org/elasticsearch/index/fielddata/ordinals/SingleOrdinalsTests.java @@ -19,8 +19,8 @@ package org.elasticsearch.index.fielddata.ordinals; import org.apache.lucene.index.DocValues; -import org.apache.lucene.index.RandomAccessOrds; import org.apache.lucene.index.SortedDocValues; +import org.apache.lucene.index.SortedSetDocValues; import org.elasticsearch.test.ESTestCase; import java.io.IOException; @@ -48,12 +48,13 @@ public class SingleOrdinalsTests extends ESTestCase { Ordinals ords = builder.build(); assertThat(ords, instanceOf(SinglePackedOrdinals.class)); - RandomAccessOrds docs = ords.ordinals(); + SortedSetDocValues docs = ords.ordinals(); final SortedDocValues singleOrds = DocValues.unwrapSingleton(docs); assertNotNull(singleOrds); for (Map.Entry<Integer, Long> entry : controlDocToOrdinal.entrySet()) { - assertThat(entry.getValue(), equalTo((long) singleOrds.getOrd(entry.getKey()))); + assertTrue(singleOrds.advanceExact(entry.getKey())); + assertEquals(singleOrds.ordValue(), (long) entry.getValue()); } } diff --git a/core/src/test/java/org/elasticsearch/index/fielddata/plain/HalfFloatFielddataTests.java b/core/src/test/java/org/elasticsearch/index/fielddata/plain/HalfFloatFielddataTests.java index af15114eae..19972e389b 100644 --- a/core/src/test/java/org/elasticsearch/index/fielddata/plain/HalfFloatFielddataTests.java +++ b/core/src/test/java/org/elasticsearch/index/fielddata/plain/HalfFloatFielddataTests.java @@ -51,9 +51,9 @@ public class HalfFloatFielddataTests extends ESTestCase { SortedNumericDoubleValues values = new SortedNumericDVIndexFieldData.SortedNumericHalfFloatFieldData( reader, "half_float").getDoubleValues(); assertNotNull(FieldData.unwrapSingleton(values)); - values.setDocument(0); - assertEquals(1, values.count()); - assertEquals(3f, values.valueAt(0), 0f); + assertTrue(values.advanceExact(0)); + assertEquals(1, values.docValueCount()); + assertEquals(3f, values.nextValue(), 0f); IOUtils.close(dirReader, w, dir); } @@ -73,10 +73,10 @@ public class HalfFloatFielddataTests extends ESTestCase { SortedNumericDoubleValues values = new SortedNumericDVIndexFieldData.SortedNumericHalfFloatFieldData( reader, "half_float").getDoubleValues(); assertNull(FieldData.unwrapSingleton(values)); - values.setDocument(0); - assertEquals(2, values.count()); - assertEquals(2f, values.valueAt(0), 0f); - assertEquals(3f, values.valueAt(1), 0f); + assertTrue(values.advanceExact(0)); + assertEquals(2, values.docValueCount()); + assertEquals(2f, values.nextValue(), 0f); + assertEquals(3f, values.nextValue(), 0f); IOUtils.close(dirReader, w, dir); } } diff --git a/core/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java index 9c67292431..e6a1c0a69d 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java @@ -97,9 +97,9 @@ public class BooleanFieldMapperTests extends ESSingleNodeTestCase { assertEquals(new BytesRef("T"), leaf.terms("field").iterator().next()); SortedNumericDocValues values = leaf.getSortedNumericDocValues("field"); assertNotNull(values); - values.setDocument(0); - assertEquals(1, values.count()); - assertEquals(1, values.valueAt(0)); + assertTrue(values.advanceExact(0)); + assertEquals(1, values.docValueCount()); + assertEquals(1, values.nextValue()); } } } diff --git a/core/src/test/java/org/elasticsearch/index/mapper/FakeStringFieldMapper.java b/core/src/test/java/org/elasticsearch/index/mapper/FakeStringFieldMapper.java index 2969b8392b..642282c9d5 100755 --- a/core/src/test/java/org/elasticsearch/index/mapper/FakeStringFieldMapper.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/FakeStringFieldMapper.java @@ -122,11 +122,6 @@ public class FakeStringFieldMapper extends FieldMapper { } @Override - protected boolean customBoost() { - return true; - } - - @Override protected void parseCreateField(ParseContext context, List<IndexableField> fields) throws IOException { String value; if (context.externalValueSet()) { diff --git a/core/src/test/java/org/elasticsearch/index/mapper/ScaledFloatFieldTypeTests.java b/core/src/test/java/org/elasticsearch/index/mapper/ScaledFloatFieldTypeTests.java index dd66421986..1945c164fc 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/ScaledFloatFieldTypeTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/ScaledFloatFieldTypeTests.java @@ -195,19 +195,19 @@ public class ScaledFloatFieldTypeTests extends FieldTypeTestCase { assertEquals(fielddata.getNumericType(), IndexNumericFieldData.NumericType.DOUBLE); AtomicNumericFieldData leafFieldData = fielddata.load(reader.leaves().get(0)); SortedNumericDoubleValues values = leafFieldData.getDoubleValues(); - values.setDocument(0); - assertEquals(1, values.count()); - assertEquals(10/ft.getScalingFactor(), values.valueAt(0), 10e-5); + assertTrue(values.advanceExact(0)); + assertEquals(1, values.docValueCount()); + assertEquals(10/ft.getScalingFactor(), values.nextValue(), 10e-5); // multi-valued ft.setName("scaled_float2"); fielddata = (IndexNumericFieldData) ft.fielddataBuilder().build(indexSettings, ft, null, null, null); leafFieldData = fielddata.load(reader.leaves().get(0)); values = leafFieldData.getDoubleValues(); - values.setDocument(0); - assertEquals(2, values.count()); - assertEquals(5/ft.getScalingFactor(), values.valueAt(0), 10e-5); - assertEquals(12/ft.getScalingFactor(), values.valueAt(1), 10e-5); + assertTrue(values.advanceExact(0)); + assertEquals(2, values.docValueCount()); + assertEquals(5/ft.getScalingFactor(), values.nextValue(), 10e-5); + assertEquals(12/ft.getScalingFactor(), values.nextValue(), 10e-5); } IOUtils.close(w, dir); } diff --git a/core/src/test/java/org/elasticsearch/index/query/BoolQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/BoolQueryBuilderTests.java index 339c9a9136..68d2d369bb 100644 --- a/core/src/test/java/org/elasticsearch/index/query/BoolQueryBuilderTests.java +++ b/core/src/test/java/org/elasticsearch/index/query/BoolQueryBuilderTests.java @@ -54,9 +54,6 @@ public class BoolQueryBuilderTests extends AbstractQueryTestCase<BoolQueryBuilde query.adjustPureNegative(randomBoolean()); } if (randomBoolean()) { - query.disableCoord(randomBoolean()); - } - if (randomBoolean()) { query.minimumShouldMatch(randomMinimumShouldMatch()); } int mustClauses = randomIntBetween(0, 3); @@ -95,7 +92,6 @@ public class BoolQueryBuilderTests extends AbstractQueryTestCase<BoolQueryBuilde } else { assertThat(query, instanceOf(BooleanQuery.class)); BooleanQuery booleanQuery = (BooleanQuery) query; - assertThat(booleanQuery.isCoordDisabled(), equalTo(queryBuilder.disableCoord())); if (queryBuilder.adjustPureNegative()) { boolean isNegative = true; for (BooleanClause clause : clauses) { @@ -259,8 +255,7 @@ public class BoolQueryBuilderTests extends AbstractQueryTestCase<BoolQueryBuilde boolQuery() .should(termQuery("foo", "bar")) .should(termQuery("foo2", "bar2")) - .minimumShouldMatch("3") - .disableCoord(true)).toQuery(createShardContext()); + .minimumShouldMatch("3")).toQuery(createShardContext()); assertEquals(3, bq.getMinimumNumberShouldMatch()); } @@ -310,7 +305,6 @@ public class BoolQueryBuilderTests extends AbstractQueryTestCase<BoolQueryBuilde " }" + " }" + " } ]," + - " \"disable_coord\" : false," + " \"adjust_pure_negative\" : true," + " \"minimum_should_match\" : \"23\"," + " \"boost\" : 42.0" + diff --git a/core/src/test/java/org/elasticsearch/index/query/CommonTermsQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/CommonTermsQueryBuilderTests.java index 66c36e0418..5478e172fe 100644 --- a/core/src/test/java/org/elasticsearch/index/query/CommonTermsQueryBuilderTests.java +++ b/core/src/test/java/org/elasticsearch/index/query/CommonTermsQueryBuilderTests.java @@ -79,9 +79,6 @@ public class CommonTermsQueryBuilderTests extends AbstractQueryTestCase<CommonTe query.analyzer(randomAnalyzer()); } - if (randomBoolean()) { - query.disableCoord(randomBoolean()); - } return query; } @@ -122,7 +119,6 @@ public class CommonTermsQueryBuilderTests extends AbstractQueryTestCase<CommonTe " \"common\" : {\n" + " \"body\" : {\n" + " \"query\" : \"nelly the elephant not as a cartoon\",\n" + - " \"disable_coord\" : true,\n" + " \"high_freq_operator\" : \"AND\",\n" + " \"low_freq_operator\" : \"OR\",\n" + " \"cutoff_frequency\" : 0.001,\n" + @@ -174,11 +170,9 @@ public class CommonTermsQueryBuilderTests extends AbstractQueryTestCase<CommonTe // see #11730 public void testCommonTermsQuery4() throws IOException { - boolean disableCoord = randomBoolean(); - Query parsedQuery = parseQuery(commonTermsQuery("field", "text").disableCoord(disableCoord)).toQuery(createShardContext()); + Query parsedQuery = parseQuery(commonTermsQuery("field", "text")).toQuery(createShardContext()); assertThat(parsedQuery, instanceOf(ExtendedCommonTermsQuery.class)); ExtendedCommonTermsQuery ectQuery = (ExtendedCommonTermsQuery) parsedQuery; - assertThat(ectQuery.isCoordDisabled(), equalTo(disableCoord)); } public void testParseFailsWithMultipleFields() throws IOException { diff --git a/core/src/test/java/org/elasticsearch/index/query/MultiMatchQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/MultiMatchQueryBuilderTests.java index 990137184f..85a368bf2b 100644 --- a/core/src/test/java/org/elasticsearch/index/query/MultiMatchQueryBuilderTests.java +++ b/core/src/test/java/org/elasticsearch/index/query/MultiMatchQueryBuilderTests.java @@ -26,7 +26,6 @@ import org.apache.lucene.search.BoostQuery; import org.apache.lucene.search.DisjunctionMaxQuery; import org.apache.lucene.search.FuzzyQuery; import org.apache.lucene.search.IndexOrDocValuesQuery; -import org.apache.lucene.search.LegacyNumericRangeQuery; import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.PhraseQuery; @@ -146,7 +145,6 @@ public class MultiMatchQueryBuilderTests extends AbstractQueryTestCase<MultiMatc .or(instanceOf(FuzzyQuery.class)).or(instanceOf(MultiPhrasePrefixQuery.class)) .or(instanceOf(MatchAllDocsQuery.class)).or(instanceOf(ExtendedCommonTermsQuery.class)) .or(instanceOf(MatchNoDocsQuery.class)).or(instanceOf(PhraseQuery.class)) - .or(instanceOf(LegacyNumericRangeQuery.class)) .or(instanceOf(PointRangeQuery.class)).or(instanceOf(IndexOrDocValuesQuery.class))); } diff --git a/core/src/test/java/org/elasticsearch/index/query/NestedQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/NestedQueryBuilderTests.java index d8cdf221ca..267963878e 100644 --- a/core/src/test/java/org/elasticsearch/index/query/NestedQueryBuilderTests.java +++ b/core/src/test/java/org/elasticsearch/index/query/NestedQueryBuilderTests.java @@ -152,7 +152,6 @@ public class NestedQueryBuilderTests extends AbstractQueryTestCase<NestedQueryBu " }\n" + " }\n" + " } ],\n" + - " \"disable_coord\" : false,\n" + " \"adjust_pure_negative\" : true,\n" + " \"boost\" : 1.0\n" + " }\n" + diff --git a/core/src/test/java/org/elasticsearch/index/query/QueryStringQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/QueryStringQueryBuilderTests.java index 28bc9c83a5..86647233aa 100644 --- a/core/src/test/java/org/elasticsearch/index/query/QueryStringQueryBuilderTests.java +++ b/core/src/test/java/org/elasticsearch/index/query/QueryStringQueryBuilderTests.java @@ -374,7 +374,6 @@ public class QueryStringQueryBuilderTests extends AbstractQueryTestCase<QueryStr BooleanClause.Occur.SHOULD)) .add(new BooleanClause(new PrefixQuery(new Term(STRING_FIELD_NAME, "foobar")), BooleanClause.Occur.SHOULD)) - .setDisableCoord(true) .build(), defaultOp) .build(), defaultOp) .add(new BooleanClause(new SynonymQuery(new Term(STRING_FIELD_NAME, "last"), @@ -463,7 +462,6 @@ public class QueryStringQueryBuilderTests extends AbstractQueryTestCase<QueryStr new SpanTermQuery(new Term(STRING_FIELD_NAME, "cavy")))) .addClause(new SpanTermQuery(new Term(STRING_FIELD_NAME, "smells"))) .build(), Occur.SHOULD) - .setDisableCoord(true) .build(); assertThat(query, Matchers.equalTo(expectedQuery)); @@ -481,7 +479,6 @@ public class QueryStringQueryBuilderTests extends AbstractQueryTestCase<QueryStr .setSlop(2) .build(), Occur.SHOULD) - .setDisableCoord(true) .build(); assertThat(query, Matchers.equalTo(expectedQuery)); } diff --git a/core/src/test/java/org/elasticsearch/index/query/SimpleQueryParserTests.java b/core/src/test/java/org/elasticsearch/index/query/SimpleQueryParserTests.java index de423db442..53ba0f6d2b 100644 --- a/core/src/test/java/org/elasticsearch/index/query/SimpleQueryParserTests.java +++ b/core/src/test/java/org/elasticsearch/index/query/SimpleQueryParserTests.java @@ -108,7 +108,6 @@ public class SimpleQueryParserTests extends ESTestCase { BooleanClause.Occur.SHOULD)) .add(new BooleanClause(new PrefixQuery(new Term("field1", "foobar")), BooleanClause.Occur.SHOULD)) - .setDisableCoord(true) .build(), defaultOp) .build(), defaultOp) .add(new BooleanClause(new SynonymQuery(new Term("field1", "last"), diff --git a/core/src/test/java/org/elasticsearch/index/query/SimpleQueryStringBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/SimpleQueryStringBuilderTests.java index 5e221748e2..e08221a168 100644 --- a/core/src/test/java/org/elasticsearch/index/query/SimpleQueryStringBuilderTests.java +++ b/core/src/test/java/org/elasticsearch/index/query/SimpleQueryStringBuilderTests.java @@ -47,7 +47,6 @@ import java.util.Set; import static org.hamcrest.Matchers.anyOf; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.notNullValue; @@ -244,9 +243,16 @@ public class SimpleQueryStringBuilderTests extends AbstractQueryTestCase<SimpleQ Map.Entry<String, Float> field = fieldsIterator.next(); assertTermOrBoostQuery(booleanClause.getQuery(), field.getKey(), queryBuilder.value(), field.getValue()); } - if (queryBuilder.minimumShouldMatch() != null && !boolQuery.isCoordDisabled()) { + /** + * TODO: + * Test disabled because we cannot check min should match consistently: + * https://github.com/elastic/elasticsearch/issues/23966 + * + if (queryBuilder.minimumShouldMatch() != null && !boolQuery.isCoordDisabled()) { assertThat(boolQuery.getMinimumNumberShouldMatch(), greaterThan(0)); - } + } + * + **/ } else if (queryBuilder.fields().size() == 1) { Map.Entry<String, Float> field = queryBuilder.fields().entrySet().iterator().next(); assertTermOrBoostQuery(query, field.getKey(), queryBuilder.value(), field.getValue()); @@ -331,6 +337,7 @@ public class SimpleQueryStringBuilderTests extends AbstractQueryTestCase<SimpleQ assertEquals(json, ".quote", parsed.quoteFieldSuffix()); } + @AwaitsFix(bugUrl = "Waiting on fix for minimumShouldMatch https://github.com/elastic/elasticsearch/issues/23966") public void testMinimumShouldMatch() throws IOException { QueryShardContext shardContext = createShardContext(); int numberOfTerms = randomIntBetween(1, 4); @@ -357,7 +364,8 @@ public class SimpleQueryStringBuilderTests extends AbstractQueryTestCase<SimpleQ assertThat(query, instanceOf(BooleanQuery.class)); BooleanQuery boolQuery = (BooleanQuery) query; int expectedMinimumShouldMatch = numberOfTerms * percent / 100; - if (numberOfTerms == 1 || simpleQueryStringBuilder.defaultOperator().equals(Operator.AND)) { + if (numberOfTerms == 1 + || simpleQueryStringBuilder.defaultOperator().equals(Operator.AND)) { expectedMinimumShouldMatch = 0; } assertEquals(expectedMinimumShouldMatch, boolQuery.getMinimumNumberShouldMatch()); diff --git a/core/src/test/java/org/elasticsearch/index/query/functionscore/FunctionScoreTests.java b/core/src/test/java/org/elasticsearch/index/query/functionscore/FunctionScoreTests.java index 16e66f50e0..341f4d87bd 100644 --- a/core/src/test/java/org/elasticsearch/index/query/functionscore/FunctionScoreTests.java +++ b/core/src/test/java/org/elasticsearch/index/query/functionscore/FunctionScoreTests.java @@ -99,16 +99,17 @@ public class FunctionScoreTests extends ESTestCase { public SortedBinaryDocValues getBytesValues() { return new SortedBinaryDocValues() { @Override - public void setDocument(int docId) { + public boolean advanceExact(int docId) { + return true; } @Override - public int count() { + public int docValueCount() { return 1; } @Override - public BytesRef valueAt(int index) { + public BytesRef nextValue() { return new BytesRef("0"); } }; @@ -178,17 +179,18 @@ public class FunctionScoreTests extends ESTestCase { public SortedNumericDoubleValues getDoubleValues() { return new SortedNumericDoubleValues() { @Override - public void setDocument(int doc) { + public boolean advanceExact(int docId) { + return true; } @Override - public double valueAt(int index) { + public int docValueCount() { return 1; } @Override - public int count() { - return 1; + public double nextValue() { + return 1d; } }; } @@ -614,7 +616,7 @@ public class FunctionScoreTests extends ESTestCase { FunctionScoreQuery fsq = new FunctionScoreQuery(query, null, null, null, Float.POSITIVE_INFINITY); for (boolean needsScores : new boolean[] {true, false}) { - Weight weight = searcher.createWeight(fsq, needsScores); + Weight weight = searcher.createWeight(fsq, needsScores, 1f); Scorer scorer = weight.scorer(reader.leaves().get(0)); assertNotNull(scorer.twoPhaseIterator()); } @@ -622,7 +624,7 @@ public class FunctionScoreTests extends ESTestCase { FiltersFunctionScoreQuery ffsq = new FiltersFunctionScoreQuery(query, ScoreMode.SUM, new FilterFunction[0], Float.POSITIVE_INFINITY, null, CombineFunction.MULTIPLY); for (boolean needsScores : new boolean[] {true, false}) { - Weight weight = searcher.createWeight(ffsq, needsScores); + Weight weight = searcher.createWeight(ffsq, needsScores, 1f); Scorer scorer = weight.scorer(reader.leaves().get(0)); assertNotNull(scorer.twoPhaseIterator()); } diff --git a/core/src/test/java/org/elasticsearch/index/query/plugin/DummyQueryParserPlugin.java b/core/src/test/java/org/elasticsearch/index/query/plugin/DummyQueryParserPlugin.java index 86c03c7273..3d0eee7959 100644 --- a/core/src/test/java/org/elasticsearch/index/query/plugin/DummyQueryParserPlugin.java +++ b/core/src/test/java/org/elasticsearch/index/query/plugin/DummyQueryParserPlugin.java @@ -52,8 +52,8 @@ public class DummyQueryParserPlugin extends Plugin implements SearchPlugin { } @Override - public Weight createWeight(IndexSearcher searcher, boolean needsScores) throws IOException { - return matchAllDocsQuery.createWeight(searcher, needsScores); + public Weight createWeight(IndexSearcher searcher, boolean needsScores, float boost) throws IOException { + return matchAllDocsQuery.createWeight(searcher, needsScores, boost); } @Override diff --git a/core/src/test/java/org/elasticsearch/index/search/MultiMatchQueryTests.java b/core/src/test/java/org/elasticsearch/index/search/MultiMatchQueryTests.java index 992667a505..be6b0c3865 100644 --- a/core/src/test/java/org/elasticsearch/index/search/MultiMatchQueryTests.java +++ b/core/src/test/java/org/elasticsearch/index/search/MultiMatchQueryTests.java @@ -110,7 +110,7 @@ public class MultiMatchQueryTests extends ESSingleNodeTestCase { ft2.setName("bar"); Term[] terms = new Term[] { new Term("foo", "baz"), new Term("bar", "baz") }; float[] boosts = new float[] {2, 3}; - Query expected = BlendedTermQuery.booleanBlendedQuery(terms, boosts, false); + Query expected = BlendedTermQuery.booleanBlendedQuery(terms, boosts); Query actual = MultiMatchQuery.blendTerm( indexService.newQueryShardContext(randomInt(20), null, () -> { throw new UnsupportedOperationException(); }), new BytesRef("baz"), null, 1f, new FieldAndFieldType(ft1, 2), new FieldAndFieldType(ft2, 3)); @@ -126,7 +126,7 @@ public class MultiMatchQueryTests extends ESSingleNodeTestCase { ft2.setBoost(10); Term[] terms = new Term[] { new Term("foo", "baz"), new Term("bar", "baz") }; float[] boosts = new float[] {200, 30}; - Query expected = BlendedTermQuery.booleanBlendedQuery(terms, boosts, false); + Query expected = BlendedTermQuery.booleanBlendedQuery(terms, boosts); Query actual = MultiMatchQuery.blendTerm( indexService.newQueryShardContext(randomInt(20), null, () -> { throw new UnsupportedOperationException(); }), new BytesRef("baz"), null, 1f, new FieldAndFieldType(ft1, 2), new FieldAndFieldType(ft2, 3)); @@ -145,7 +145,7 @@ public class MultiMatchQueryTests extends ESSingleNodeTestCase { ft2.setName("bar"); Term[] terms = new Term[] { new Term("foo", "baz") }; float[] boosts = new float[] {2}; - Query expected = BlendedTermQuery.booleanBlendedQuery(terms, boosts, false); + Query expected = BlendedTermQuery.booleanBlendedQuery(terms, boosts); Query actual = MultiMatchQuery.blendTerm( indexService.newQueryShardContext(randomInt(20), null, () -> { throw new UnsupportedOperationException(); }), new BytesRef("baz"), null, 1f, new FieldAndFieldType(ft1, 2), new FieldAndFieldType(ft2, 3)); @@ -164,9 +164,9 @@ public class MultiMatchQueryTests extends ESSingleNodeTestCase { ft2.setName("bar"); Term[] terms = new Term[] { new Term("foo", "baz") }; float[] boosts = new float[] {2}; - Query expectedClause1 = BlendedTermQuery.booleanBlendedQuery(terms, boosts, false); + Query expectedClause1 = BlendedTermQuery.booleanBlendedQuery(terms, boosts); Query expectedClause2 = new BoostQuery(new MatchAllDocsQuery(), 3); - Query expected = new BooleanQuery.Builder().setDisableCoord(true) + Query expected = new BooleanQuery.Builder() .add(expectedClause1, Occur.SHOULD) .add(expectedClause2, Occur.SHOULD) .build(); diff --git a/core/src/test/java/org/elasticsearch/index/shard/IndexSearcherWrapperTests.java b/core/src/test/java/org/elasticsearch/index/shard/IndexSearcherWrapperTests.java index 6b7c77fdf5..f5d4f048c2 100644 --- a/core/src/test/java/org/elasticsearch/index/shard/IndexSearcherWrapperTests.java +++ b/core/src/test/java/org/elasticsearch/index/shard/IndexSearcherWrapperTests.java @@ -76,8 +76,8 @@ public class IndexSearcherWrapperTests extends ESTestCase { try (Engine.Searcher engineSearcher = new Engine.Searcher("foo", searcher)) { final Engine.Searcher wrap = wrapper.wrap(engineSearcher); assertEquals(1, wrap.reader().getRefCount()); - ElasticsearchDirectoryReader.addReaderCloseListener(wrap.getDirectoryReader(), reader -> { - if (reader == open) { + ElasticsearchDirectoryReader.addReaderCloseListener(wrap.getDirectoryReader(), key -> { + if (key == open.getReaderCacheHelper().getKey()) { count.incrementAndGet(); } outerCount.incrementAndGet(); @@ -123,11 +123,11 @@ public class IndexSearcherWrapperTests extends ESTestCase { final ConcurrentHashMap<Object, TopDocs> cache = new ConcurrentHashMap<>(); try (Engine.Searcher engineSearcher = new Engine.Searcher("foo", searcher)) { try (Engine.Searcher wrap = wrapper.wrap(engineSearcher)) { - ElasticsearchDirectoryReader.addReaderCloseListener(wrap.getDirectoryReader(), reader -> { - cache.remove(reader.getCoreCacheKey()); + ElasticsearchDirectoryReader.addReaderCloseListener(wrap.getDirectoryReader(), key -> { + cache.remove(key); }); TopDocs search = wrap.searcher().search(new TermQuery(new Term("field", "doc")), 1); - cache.put(wrap.reader().getCoreCacheKey(), search); + cache.put(wrap.reader().getReaderCacheHelper().getKey(), search); } } assertEquals(1, closeCalls.get()); @@ -158,49 +158,6 @@ public class IndexSearcherWrapperTests extends ESTestCase { IOUtils.close(open, writer, dir); } - public void testWrappedReaderMustDelegateCoreCacheKey() throws IOException { - Directory dir = newDirectory(); - IndexWriterConfig iwc = newIndexWriterConfig(); - IndexWriter writer = new IndexWriter(dir, iwc); - Document doc = new Document(); - doc.add(new StringField("id", "1", random().nextBoolean() ? Field.Store.YES : Field.Store.NO)); - doc.add(new TextField("field", "doc", random().nextBoolean() ? Field.Store.YES : Field.Store.NO)); - writer.addDocument(doc); - DirectoryReader open = ElasticsearchDirectoryReader.wrap(DirectoryReader.open(writer), new ShardId("foo", "_na_", 1)); - IndexSearcher searcher = new IndexSearcher(open); - assertEquals(1, searcher.search(new TermQuery(new Term("field", "doc")), 1).totalHits); - searcher.setSimilarity(iwc.getSimilarity()); - IndexSearcherWrapper wrapper = new IndexSearcherWrapper() { - @Override - protected DirectoryReader wrap(DirectoryReader reader) throws IOException { - return new BrokenWrapper(reader, false); - } - }; - try (Engine.Searcher engineSearcher = new Engine.Searcher("foo", searcher)) { - try { - wrapper.wrap(engineSearcher); - fail("reader must delegate cache key"); - } catch (IllegalStateException ex) { - // all is well - } - } - wrapper = new IndexSearcherWrapper() { - @Override - protected DirectoryReader wrap(DirectoryReader reader) throws IOException { - return new BrokenWrapper(reader, true); - } - }; - try (Engine.Searcher engineSearcher = new Engine.Searcher("foo", searcher)) { - try { - wrapper.wrap(engineSearcher); - fail("reader must delegate cache key"); - } catch (IllegalStateException ex) { - // all is well - } - } - IOUtils.close(open, writer, dir); - } - private static class FieldMaskingReader extends FilterDirectoryReader { private final String field; private final AtomicInteger closeCalls; @@ -222,8 +179,8 @@ public class IndexSearcherWrapperTests extends ESTestCase { } @Override - public Object getCoreCacheKey() { - return in.getCoreCacheKey(); + public CacheHelper getReaderCacheHelper() { + return in.getReaderCacheHelper(); } @Override @@ -233,44 +190,4 @@ public class IndexSearcherWrapperTests extends ESTestCase { } } - private static class BrokenWrapper extends FilterDirectoryReader { - - private final boolean hideDelegate; - - BrokenWrapper(DirectoryReader in, boolean hideDelegate) throws IOException { - super(in, new SubReaderWrapper() { - @Override - public LeafReader wrap(LeafReader reader) { - return reader; - } - }); - this.hideDelegate = hideDelegate; - } - - @Override - protected DirectoryReader doWrapDirectoryReader(DirectoryReader in) throws IOException { - return new BrokenWrapper(in, hideDelegate); - } - - @Override - public DirectoryReader getDelegate() { - if (hideDelegate) { - try { - return ElasticsearchDirectoryReader.wrap(super.getDelegate(), new ShardId("foo", "_na_", 1)); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - return super.getDelegate(); - } - - @Override - public Object getCoreCacheKey() { - if (hideDelegate == false) { - return super.getCoreCacheKey(); - } else { - return in.getCoreCacheKey(); - } - } - } } diff --git a/core/src/test/java/org/elasticsearch/index/store/StoreTests.java b/core/src/test/java/org/elasticsearch/index/store/StoreTests.java index dfc24d73c9..1a164154df 100644 --- a/core/src/test/java/org/elasticsearch/index/store/StoreTests.java +++ b/core/src/test/java/org/elasticsearch/index/store/StoreTests.java @@ -865,7 +865,7 @@ public class StoreTests extends ESTestCase { String translogId = "a translog id"; commitData.put(Engine.SYNC_COMMIT_ID, syncId); commitData.put(Translog.TRANSLOG_GENERATION_KEY, translogId); - writer.setCommitData(commitData); + writer.setLiveCommitData(commitData.entrySet()); writer.commit(); writer.close(); Store.MetadataSnapshot metadata; diff --git a/core/src/test/java/org/elasticsearch/indices/IndicesQueryCacheTests.java b/core/src/test/java/org/elasticsearch/indices/IndicesQueryCacheTests.java index cd94ee0f8e..10f098787c 100644 --- a/core/src/test/java/org/elasticsearch/indices/IndicesQueryCacheTests.java +++ b/core/src/test/java/org/elasticsearch/indices/IndicesQueryCacheTests.java @@ -69,9 +69,9 @@ public class IndicesQueryCacheTests extends ESTestCase { } @Override - public Weight createWeight(IndexSearcher searcher, boolean needsScores) + public Weight createWeight(IndexSearcher searcher, boolean needsScores, float boost) throws IOException { - return new ConstantScoreWeight(this) { + return new ConstantScoreWeight(this, boost) { @Override public Scorer scorer(LeafReaderContext context) throws IOException { return new ConstantScoreScorer(this, score(), DocIdSetIterator.all(context.reader().maxDoc())); diff --git a/core/src/test/java/org/elasticsearch/indices/memory/breaker/RandomExceptionCircuitBreakerIT.java b/core/src/test/java/org/elasticsearch/indices/memory/breaker/RandomExceptionCircuitBreakerIT.java index 2f336d8d69..6dd9126a51 100644 --- a/core/src/test/java/org/elasticsearch/indices/memory/breaker/RandomExceptionCircuitBreakerIT.java +++ b/core/src/test/java/org/elasticsearch/indices/memory/breaker/RandomExceptionCircuitBreakerIT.java @@ -41,7 +41,6 @@ import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.indices.IndicesService; import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache; import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.search.basic.SearchWithRandomExceptionsIT; import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.engine.MockEngineSupport; @@ -298,5 +297,10 @@ public class RandomExceptionCircuitBreakerIT extends ESIntegTestCase { protected DirectoryReader doWrapDirectoryReader(DirectoryReader in) throws IOException { return new RandomExceptionDirectoryReaderWrapper(in, settings); } + + @Override + public CacheHelper getReaderCacheHelper() { + return in.getReaderCacheHelper(); + } } } diff --git a/core/src/test/java/org/elasticsearch/search/MultiValueModeTests.java b/core/src/test/java/org/elasticsearch/search/MultiValueModeTests.java index 5caba0fb44..1a357c55eb 100644 --- a/core/src/test/java/org/elasticsearch/search/MultiValueModeTests.java +++ b/core/src/test/java/org/elasticsearch/search/MultiValueModeTests.java @@ -20,21 +20,28 @@ package org.elasticsearch.search; import com.carrotsearch.randomizedtesting.generators.RandomStrings; + import org.apache.lucene.index.BinaryDocValues; import org.apache.lucene.index.DocValues; import org.apache.lucene.index.NumericDocValues; -import org.apache.lucene.index.RandomAccessOrds; import org.apache.lucene.index.SortedDocValues; import org.apache.lucene.index.SortedNumericDocValues; +import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.util.BitSetIterator; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.FixedBitSet; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.index.fielddata.AbstractBinaryDocValues; +import org.elasticsearch.index.fielddata.AbstractNumericDocValues; +import org.elasticsearch.index.fielddata.AbstractSortedDocValues; +import org.elasticsearch.index.fielddata.AbstractSortedNumericDocValues; +import org.elasticsearch.index.fielddata.AbstractSortedSetDocValues; import org.elasticsearch.index.fielddata.FieldData; import org.elasticsearch.index.fielddata.NumericDoubleValues; import org.elasticsearch.index.fielddata.SortedBinaryDocValues; import org.elasticsearch.index.fielddata.SortedNumericDoubleValues; +import org.elasticsearch.search.MultiValueMode.UnsortedNumericDoubleValues; import org.elasticsearch.test.ESTestCase; import java.io.IOException; @@ -44,6 +51,11 @@ import static org.hamcrest.Matchers.equalTo; public class MultiValueModeTests extends ESTestCase { + @FunctionalInterface + private interface Supplier<T> { + T get() throws IOException; + } + private static FixedBitSet randomRootDocs(int maxDoc) { FixedBitSet set = new FixedBitSet(maxDoc); for (int i = 0; i < maxDoc; ++i) { @@ -80,17 +92,27 @@ public class MultiValueModeTests extends ESTestCase { docsWithValue.set(i); } } - final NumericDocValues singleValues = new NumericDocValues() { + + final Supplier<SortedNumericDocValues> multiValues = () -> DocValues.singleton(new AbstractNumericDocValues() { + int docId = -1; @Override - public long get(int docID) { - return array[docID]; + public boolean advanceExact(int target) throws IOException { + this.docId = target; + return docsWithValue == null ? true : docsWithValue.get(docId); } - }; - final SortedNumericDocValues multiValues = DocValues.singleton(singleValues, docsWithValue); - verify(multiValues, numDocs); + @Override + public int docID() { + return docId; + } + @Override + public long longValue() { + return array[docId]; + } + }); + verifySortedNumeric(multiValues, numDocs); final FixedBitSet rootDocs = randomRootDocs(numDocs); final FixedBitSet innerDocs = randomInnerDocs(rootDocs); - verify(multiValues, numDocs, rootDocs, innerDocs); + verifySortedNumeric(multiValues, numDocs, rootDocs, innerDocs); } public void testMultiValuedLongs() throws Exception { @@ -104,42 +126,46 @@ public class MultiValueModeTests extends ESTestCase { Arrays.sort(values); array[i] = values; } - final SortedNumericDocValues multiValues = new SortedNumericDocValues() { + final Supplier<SortedNumericDocValues> multiValues = () -> new AbstractSortedNumericDocValues() { int doc; + int i; @Override - public long valueAt(int index) { - return array[doc][index]; + public long nextValue() { + return array[doc][i++]; } @Override - public void setDocument(int doc) { + public boolean advanceExact(int doc) { this.doc = doc; + i = 0; + return array[doc].length > 0; } @Override - public int count() { + public int docValueCount() { return array[doc].length; } }; - verify(multiValues, numDocs); + verifySortedNumeric(multiValues, numDocs); final FixedBitSet rootDocs = randomRootDocs(numDocs); final FixedBitSet innerDocs = randomInnerDocs(rootDocs); - verify(multiValues, numDocs, rootDocs, innerDocs); + verifySortedNumeric(multiValues, numDocs, rootDocs, innerDocs); } - private void verify(SortedNumericDocValues values, int maxDoc) { + private void verifySortedNumeric(Supplier<SortedNumericDocValues> supplier, int maxDoc) throws IOException { for (long missingValue : new long[] { 0, randomLong() }) { for (MultiValueMode mode : MultiValueMode.values()) { + SortedNumericDocValues values = supplier.get(); final NumericDocValues selected = mode.select(values, missingValue); for (int i = 0; i < maxDoc; ++i) { - final long actual = selected.get(i); + assertTrue(selected.advanceExact(i)); + final long actual = selected.longValue(); long expected = 0; - values.setDocument(i); - int numValues = values.count(); - if (numValues == 0) { + if (values.advanceExact(i) == false) { expected = missingValue; } else { + int numValues = values.docValueCount(); if (mode == MultiValueMode.MAX) { expected = Long.MIN_VALUE; } else if (mode == MultiValueMode.MIN) { @@ -147,11 +173,11 @@ public class MultiValueModeTests extends ESTestCase { } for (int j = 0; j < numValues; ++j) { if (mode == MultiValueMode.SUM || mode == MultiValueMode.AVG) { - expected += values.valueAt(j); + expected += values.nextValue(); } else if (mode == MultiValueMode.MIN) { - expected = Math.min(expected, values.valueAt(j)); + expected = Math.min(expected, values.nextValue()); } else if (mode == MultiValueMode.MAX) { - expected = Math.max(expected, values.valueAt(j)); + expected = Math.max(expected, values.nextValue()); } } if (mode == MultiValueMode.AVG) { @@ -159,9 +185,15 @@ public class MultiValueModeTests extends ESTestCase { } else if (mode == MultiValueMode.MEDIAN) { int value = numValues/2; if (numValues % 2 == 0) { - expected = Math.round((values.valueAt(value - 1) + values.valueAt(value))/2.0); + for (int j = 0; j < value - 1; ++j) { + values.nextValue(); + } + expected = Math.round(((double) values.nextValue() + values.nextValue())/2.0); } else { - expected = values.valueAt(value); + for (int j = 0; j < value; ++j) { + values.nextValue(); + } + expected = values.nextValue(); } } } @@ -172,13 +204,15 @@ public class MultiValueModeTests extends ESTestCase { } } - private void verify(SortedNumericDocValues values, int maxDoc, FixedBitSet rootDocs, FixedBitSet innerDocs) throws IOException { + private void verifySortedNumeric(Supplier<SortedNumericDocValues> supplier, int maxDoc, FixedBitSet rootDocs, FixedBitSet innerDocs) throws IOException { for (long missingValue : new long[] { 0, randomLong() }) { for (MultiValueMode mode : new MultiValueMode[] {MultiValueMode.MIN, MultiValueMode.MAX, MultiValueMode.SUM, MultiValueMode.AVG}) { + SortedNumericDocValues values = supplier.get(); final NumericDocValues selected = mode.select(values, missingValue, rootDocs, new BitSetIterator(innerDocs, 0L), maxDoc); int prevRoot = -1; for (int root = rootDocs.nextSetBit(0); root != -1; root = root + 1 < maxDoc ? rootDocs.nextSetBit(root + 1) : -1) { - final long actual = selected.get(root); + assertTrue(selected.advanceExact(root)); + final long actual = selected.longValue(); long expected = 0; if (mode == MultiValueMode.MAX) { expected = Long.MIN_VALUE; @@ -187,16 +221,17 @@ public class MultiValueModeTests extends ESTestCase { } int numValues = 0; for (int child = innerDocs.nextSetBit(prevRoot + 1); child != -1 && child < root; child = innerDocs.nextSetBit(child + 1)) { - values.setDocument(child); - for (int j = 0; j < values.count(); ++j) { - if (mode == MultiValueMode.SUM || mode == MultiValueMode.AVG) { - expected += values.valueAt(j); - } else if (mode == MultiValueMode.MIN) { - expected = Math.min(expected, values.valueAt(j)); - } else if (mode == MultiValueMode.MAX) { - expected = Math.max(expected, values.valueAt(j)); + if (values.advanceExact(child)) { + for (int j = 0; j < values.docValueCount(); ++j) { + if (mode == MultiValueMode.SUM || mode == MultiValueMode.AVG) { + expected += values.nextValue(); + } else if (mode == MultiValueMode.MIN) { + expected = Math.min(expected, values.nextValue()); + } else if (mode == MultiValueMode.MAX) { + expected = Math.max(expected, values.nextValue()); + } + ++numValues; } - ++numValues; } } if (numValues == 0) { @@ -227,17 +262,22 @@ public class MultiValueModeTests extends ESTestCase { docsWithValue.set(i); } } - final NumericDoubleValues singleValues = new NumericDoubleValues() { + final Supplier<SortedNumericDoubleValues> multiValues = () -> FieldData.singleton(new NumericDoubleValues() { + int docID; @Override - public double get(int docID) { + public boolean advanceExact(int doc) throws IOException { + docID = doc; + return docsWithValue == null || docsWithValue.get(doc); + } + @Override + public double doubleValue() { return array[docID]; } - }; - final SortedNumericDoubleValues multiValues = FieldData.singleton(singleValues, docsWithValue); - verify(multiValues, numDocs); + }); + verifySortedNumericDouble(multiValues, numDocs); final FixedBitSet rootDocs = randomRootDocs(numDocs); final FixedBitSet innerDocs = randomInnerDocs(rootDocs); - verify(multiValues, numDocs, rootDocs, innerDocs); + verifySortedNumericDouble(multiValues, numDocs, rootDocs, innerDocs); } public void testMultiValuedDoubles() throws Exception { @@ -251,45 +291,49 @@ public class MultiValueModeTests extends ESTestCase { Arrays.sort(values); array[i] = values; } - final SortedNumericDoubleValues multiValues = new SortedNumericDoubleValues() { + final Supplier<SortedNumericDoubleValues> multiValues = () -> new SortedNumericDoubleValues() { int doc; + int i; @Override - public double valueAt(int index) { - return array[doc][index]; + public double nextValue() { + return array[doc][i++]; } @Override - public void setDocument(int doc) { + public boolean advanceExact(int doc) { this.doc = doc; + i = 0; + return array[doc].length > 0; } @Override - public int count() { + public int docValueCount() { return array[doc].length; } }; - verify(multiValues, numDocs); + verifySortedNumericDouble(multiValues, numDocs); final FixedBitSet rootDocs = randomRootDocs(numDocs); final FixedBitSet innerDocs = randomInnerDocs(rootDocs); - verify(multiValues, numDocs, rootDocs, innerDocs); + verifySortedNumericDouble(multiValues, numDocs, rootDocs, innerDocs); } - private void verify(SortedNumericDoubleValues values, int maxDoc) { + private void verifySortedNumericDouble(Supplier<SortedNumericDoubleValues> supplier, int maxDoc) throws IOException { for (long missingValue : new long[] { 0, randomLong() }) { for (MultiValueMode mode : MultiValueMode.values()) { if (MultiValueMode.MEDIAN.equals(mode)) { continue; } + SortedNumericDoubleValues values = supplier.get(); final NumericDoubleValues selected = mode.select(values, missingValue); for (int i = 0; i < maxDoc; ++i) { - final double actual = selected.get(i); + assertTrue(selected.advanceExact(i)); + final double actual = selected.doubleValue(); double expected = 0.0; - values.setDocument(i); - int numValues = values.count(); - if (numValues == 0) { + if (values.advanceExact(i) == false) { expected = missingValue; } else { + int numValues = values.docValueCount(); if (mode == MultiValueMode.MAX) { expected = Long.MIN_VALUE; } else if (mode == MultiValueMode.MIN) { @@ -297,11 +341,11 @@ public class MultiValueModeTests extends ESTestCase { } for (int j = 0; j < numValues; ++j) { if (mode == MultiValueMode.SUM || mode == MultiValueMode.AVG) { - expected += values.valueAt(j); + expected += values.nextValue(); } else if (mode == MultiValueMode.MIN) { - expected = Math.min(expected, values.valueAt(j)); + expected = Math.min(expected, values.nextValue()); } else if (mode == MultiValueMode.MAX) { - expected = Math.max(expected, values.valueAt(j)); + expected = Math.max(expected, values.nextValue()); } } if (mode == MultiValueMode.AVG) { @@ -309,9 +353,15 @@ public class MultiValueModeTests extends ESTestCase { } else if (mode == MultiValueMode.MEDIAN) { int value = numValues/2; if (numValues % 2 == 0) { - expected = (values.valueAt(value - 1) + values.valueAt(value))/2.0; + for (int j = 0; j < value - 1; ++j) { + values.nextValue(); + } + expected = (values.nextValue() + values.nextValue())/2.0; } else { - expected = values.valueAt(value); + for (int j = 0; j < value; ++j) { + values.nextValue(); + } + expected = values.nextValue(); } } } @@ -322,13 +372,15 @@ public class MultiValueModeTests extends ESTestCase { } } - private void verify(SortedNumericDoubleValues values, int maxDoc, FixedBitSet rootDocs, FixedBitSet innerDocs) throws IOException { + private void verifySortedNumericDouble(Supplier<SortedNumericDoubleValues> supplier, int maxDoc, FixedBitSet rootDocs, FixedBitSet innerDocs) throws IOException { for (long missingValue : new long[] { 0, randomLong() }) { for (MultiValueMode mode : new MultiValueMode[] {MultiValueMode.MIN, MultiValueMode.MAX, MultiValueMode.SUM, MultiValueMode.AVG}) { + SortedNumericDoubleValues values = supplier.get(); final NumericDoubleValues selected = mode.select(values, missingValue, rootDocs, new BitSetIterator(innerDocs, 0L), maxDoc); int prevRoot = -1; for (int root = rootDocs.nextSetBit(0); root != -1; root = root + 1 < maxDoc ? rootDocs.nextSetBit(root + 1) : -1) { - final double actual = selected.get(root); + assertTrue(selected.advanceExact(root)); + final double actual = selected.doubleValue();; double expected = 0.0; if (mode == MultiValueMode.MAX) { expected = Long.MIN_VALUE; @@ -337,16 +389,17 @@ public class MultiValueModeTests extends ESTestCase { } int numValues = 0; for (int child = innerDocs.nextSetBit(prevRoot + 1); child != -1 && child < root; child = innerDocs.nextSetBit(child + 1)) { - values.setDocument(child); - for (int j = 0; j < values.count(); ++j) { - if (mode == MultiValueMode.SUM || mode == MultiValueMode.AVG) { - expected += values.valueAt(j); - } else if (mode == MultiValueMode.MIN) { - expected = Math.min(expected, values.valueAt(j)); - } else if (mode == MultiValueMode.MAX) { - expected = Math.max(expected, values.valueAt(j)); + if (values.advanceExact(child)) { + for (int j = 0; j < values.docValueCount(); ++j) { + if (mode == MultiValueMode.SUM || mode == MultiValueMode.AVG) { + expected += values.nextValue(); + } else if (mode == MultiValueMode.MIN) { + expected = Math.min(expected, values.nextValue()); + } else if (mode == MultiValueMode.MAX) { + expected = Math.max(expected, values.nextValue()); + } + ++numValues; } - ++numValues; } } if (numValues == 0) { @@ -380,17 +433,22 @@ public class MultiValueModeTests extends ESTestCase { } } } - final BinaryDocValues singleValues = new BinaryDocValues() { + final Supplier<SortedBinaryDocValues> multiValues = () -> FieldData.singleton(new AbstractBinaryDocValues() { + int docID; + @Override + public boolean advanceExact(int target) throws IOException { + docID = target; + return docsWithValue == null || docsWithValue.get(docID); + } @Override - public BytesRef get(int docID) { + public BytesRef binaryValue() { return BytesRef.deepCopyOf(array[docID]); } - }; - final SortedBinaryDocValues multiValues = FieldData.singleton(singleValues, docsWithValue); - verify(multiValues, numDocs); + }); + verifySortedBinary(multiValues, numDocs); final FixedBitSet rootDocs = randomRootDocs(numDocs); final FixedBitSet innerDocs = randomInnerDocs(rootDocs); - verify(multiValues, numDocs, rootDocs, innerDocs); + verifySortedBinary(multiValues, numDocs, rootDocs, innerDocs); } public void testMultiValuedStrings() throws Exception { @@ -404,50 +462,55 @@ public class MultiValueModeTests extends ESTestCase { Arrays.sort(values); array[i] = values; } - final SortedBinaryDocValues multiValues = new SortedBinaryDocValues() { + final Supplier<SortedBinaryDocValues> multiValues = () -> new SortedBinaryDocValues() { int doc; + int i; @Override - public BytesRef valueAt(int index) { - return BytesRef.deepCopyOf(array[doc][index]); + public BytesRef nextValue() { + return BytesRef.deepCopyOf(array[doc][i++]); } @Override - public void setDocument(int doc) { + public boolean advanceExact(int doc) { this.doc = doc; + i = 0; + return array[doc].length > 0; } @Override - public int count() { + public int docValueCount() { return array[doc].length; } }; - verify(multiValues, numDocs); + verifySortedBinary(multiValues, numDocs); final FixedBitSet rootDocs = randomRootDocs(numDocs); final FixedBitSet innerDocs = randomInnerDocs(rootDocs); - verify(multiValues, numDocs, rootDocs, innerDocs); + verifySortedBinary(multiValues, numDocs, rootDocs, innerDocs); } - private void verify(SortedBinaryDocValues values, int maxDoc) { + private void verifySortedBinary(Supplier<SortedBinaryDocValues> supplier, int maxDoc) throws IOException { for (BytesRef missingValue : new BytesRef[] { new BytesRef(), new BytesRef(RandomStrings.randomAsciiOfLength(random(), 8)) }) { for (MultiValueMode mode : new MultiValueMode[] {MultiValueMode.MIN, MultiValueMode.MAX}) { + SortedBinaryDocValues values = supplier.get(); final BinaryDocValues selected = mode.select(values, missingValue); for (int i = 0; i < maxDoc; ++i) { - final BytesRef actual = selected.get(i); + assertTrue(selected.advanceExact(i)); + final BytesRef actual = selected.binaryValue(); BytesRef expected = null; - values.setDocument(i); - int numValues = values.count(); - if (numValues == 0) { + if (values.advanceExact(i) == false) { expected = missingValue; } else { + int numValues = values.docValueCount(); for (int j = 0; j < numValues; ++j) { if (expected == null) { - expected = BytesRef.deepCopyOf(values.valueAt(j)); + expected = BytesRef.deepCopyOf(values.nextValue()); } else { + BytesRef value = values.nextValue(); if (mode == MultiValueMode.MIN) { - expected = expected.compareTo(values.valueAt(j)) <= 0 ? expected : BytesRef.deepCopyOf(values.valueAt(j)); + expected = expected.compareTo(value) <= 0 ? expected : BytesRef.deepCopyOf(value); } else if (mode == MultiValueMode.MAX) { - expected = expected.compareTo(values.valueAt(j)) > 0 ? expected : BytesRef.deepCopyOf(values.valueAt(j)); + expected = expected.compareTo(value) > 0 ? expected : BytesRef.deepCopyOf(value); } } } @@ -462,24 +525,28 @@ public class MultiValueModeTests extends ESTestCase { } } - private void verify(SortedBinaryDocValues values, int maxDoc, FixedBitSet rootDocs, FixedBitSet innerDocs) throws IOException { + private void verifySortedBinary(Supplier<SortedBinaryDocValues> supplier, int maxDoc, FixedBitSet rootDocs, FixedBitSet innerDocs) throws IOException { for (BytesRef missingValue : new BytesRef[] { new BytesRef(), new BytesRef(RandomStrings.randomAsciiOfLength(random(), 8)) }) { for (MultiValueMode mode : new MultiValueMode[] {MultiValueMode.MIN, MultiValueMode.MAX}) { + SortedBinaryDocValues values = supplier.get(); final BinaryDocValues selected = mode.select(values, missingValue, rootDocs, new BitSetIterator(innerDocs, 0L), maxDoc); int prevRoot = -1; for (int root = rootDocs.nextSetBit(0); root != -1; root = root + 1 < maxDoc ? rootDocs.nextSetBit(root + 1) : -1) { - final BytesRef actual = selected.get(root); + assertTrue(selected.advanceExact(root)); + final BytesRef actual = selected.binaryValue(); BytesRef expected = null; for (int child = innerDocs.nextSetBit(prevRoot + 1); child != -1 && child < root; child = innerDocs.nextSetBit(child + 1)) { - values.setDocument(child); - for (int j = 0; j < values.count(); ++j) { - if (expected == null) { - expected = BytesRef.deepCopyOf(values.valueAt(j)); - } else { - if (mode == MultiValueMode.MIN) { - expected = expected.compareTo(values.valueAt(j)) <= 0 ? expected : BytesRef.deepCopyOf(values.valueAt(j)); - } else if (mode == MultiValueMode.MAX) { - expected = expected.compareTo(values.valueAt(j)) > 0 ? expected : BytesRef.deepCopyOf(values.valueAt(j)); + if (values.advanceExact(child)) { + for (int j = 0; j < values.docValueCount(); ++j) { + if (expected == null) { + expected = BytesRef.deepCopyOf(values.nextValue()); + } else { + BytesRef value = values.nextValue(); + if (mode == MultiValueMode.MIN) { + expected = expected.compareTo(value) <= 0 ? expected : BytesRef.deepCopyOf(value); + } else if (mode == MultiValueMode.MAX) { + expected = expected.compareTo(value) > 0 ? expected : BytesRef.deepCopyOf(value); + } } } } @@ -507,9 +574,21 @@ public class MultiValueModeTests extends ESTestCase { array[i] = -1; } } - final SortedDocValues singleValues = new SortedDocValues() { + final Supplier<SortedSetDocValues> multiValues = () -> DocValues.singleton(new AbstractSortedDocValues() { + private int docID = -1; + @Override + public boolean advanceExact(int target) throws IOException { + docID = target; + return array[docID] != -1; + } + @Override - public int getOrd(int docID) { + public int docID() { + return docID; + } + + @Override + public int ordValue() { return array[docID]; } @@ -522,12 +601,11 @@ public class MultiValueModeTests extends ESTestCase { public int getValueCount() { return 1 << 20; } - }; - final RandomAccessOrds multiValues = (RandomAccessOrds) DocValues.singleton(singleValues); - verify(multiValues, numDocs); + }); + verifySortedSet(multiValues, numDocs); final FixedBitSet rootDocs = randomRootDocs(numDocs); final FixedBitSet innerDocs = randomInnerDocs(rootDocs); - verify(multiValues, numDocs, rootDocs, innerDocs); + verifySortedSet(multiValues, numDocs, rootDocs, innerDocs); } public void testMultiValuedOrds() throws Exception { @@ -540,27 +618,24 @@ public class MultiValueModeTests extends ESTestCase { } array[i] = values; } - final RandomAccessOrds multiValues = new RandomAccessOrds() { + final Supplier<SortedSetDocValues> multiValues = () -> new AbstractSortedSetDocValues() { int doc; - - @Override - public long ordAt(int index) { - return array[doc][index]; - } - - @Override - public int cardinality() { - return array[doc].length; - } + int i; @Override public long nextOrd() { - throw new UnsupportedOperationException(); + if (i < array[doc].length) { + return array[doc][i++]; + } else { + return NO_MORE_ORDS; + } } @Override - public void setDocument(int docID) { + public boolean advanceExact(int docID) { this.doc = docID; + i = 0; + return array[doc].length > 0; } @Override @@ -573,27 +648,29 @@ public class MultiValueModeTests extends ESTestCase { return 1 << 20; } }; - verify(multiValues, numDocs); + verifySortedSet(multiValues, numDocs); final FixedBitSet rootDocs = randomRootDocs(numDocs); final FixedBitSet innerDocs = randomInnerDocs(rootDocs); - verify(multiValues, numDocs, rootDocs, innerDocs); + verifySortedSet(multiValues, numDocs, rootDocs, innerDocs); } - private void verify(RandomAccessOrds values, int maxDoc) { + private void verifySortedSet(Supplier<SortedSetDocValues> supplier, int maxDoc) throws IOException { for (MultiValueMode mode : new MultiValueMode[] {MultiValueMode.MIN, MultiValueMode.MAX}) { + SortedSetDocValues values = supplier.get(); final SortedDocValues selected = mode.select(values); for (int i = 0; i < maxDoc; ++i) { - final long actual = selected.getOrd(i); + final long actual = selected.advanceExact(i) ? selected.ordValue() : -1; int expected = -1; - values.setDocument(i); - for (int j = 0; j < values.cardinality(); ++j) { - if (expected == -1) { - expected = (int) values.ordAt(j); - } else { - if (mode == MultiValueMode.MIN) { - expected = Math.min(expected, (int)values.ordAt(j)); - } else if (mode == MultiValueMode.MAX) { - expected = Math.max(expected, (int)values.ordAt(j)); + if (values.advanceExact(i)) { + for (long ord = values.nextOrd(); ord != SortedSetDocValues.NO_MORE_ORDS; ord = values.nextOrd()) { + if (expected == -1) { + expected = (int) ord; + } else { + if (mode == MultiValueMode.MIN) { + expected = Math.min(expected, (int) ord); + } else if (mode == MultiValueMode.MAX) { + expected = Math.max(expected, (int) ord); + } } } } @@ -603,23 +680,25 @@ public class MultiValueModeTests extends ESTestCase { } } - private void verify(RandomAccessOrds values, int maxDoc, FixedBitSet rootDocs, FixedBitSet innerDocs) throws IOException { + private void verifySortedSet(Supplier<SortedSetDocValues> supplier, int maxDoc, FixedBitSet rootDocs, FixedBitSet innerDocs) throws IOException { for (MultiValueMode mode : new MultiValueMode[] {MultiValueMode.MIN, MultiValueMode.MAX}) { + SortedSetDocValues values = supplier.get(); final SortedDocValues selected = mode.select(values, rootDocs, new BitSetIterator(innerDocs, 0L)); int prevRoot = -1; for (int root = rootDocs.nextSetBit(0); root != -1; root = root + 1 < maxDoc ? rootDocs.nextSetBit(root + 1) : -1) { - final int actual = selected.getOrd(root); + final int actual = selected.advanceExact(root) ? selected.ordValue() : -1; int expected = -1; for (int child = innerDocs.nextSetBit(prevRoot + 1); child != -1 && child < root; child = innerDocs.nextSetBit(child + 1)) { - values.setDocument(child); - for (int j = 0; j < values.cardinality(); ++j) { - if (expected == -1) { - expected = (int) values.ordAt(j); - } else { - if (mode == MultiValueMode.MIN) { - expected = Math.min(expected, (int)values.ordAt(j)); - } else if (mode == MultiValueMode.MAX) { - expected = Math.max(expected, (int)values.ordAt(j)); + if (values.advanceExact(child)) { + for (long ord = values.nextOrd(); ord != SortedSetDocValues.NO_MORE_ORDS; ord = values.nextOrd()) { + if (expected == -1) { + expected = (int) ord; + } else { + if (mode == MultiValueMode.MIN) { + expected = Math.min(expected, (int) ord); + } else if (mode == MultiValueMode.MAX) { + expected = Math.max(expected, (int) ord); + } } } } @@ -647,30 +726,36 @@ public class MultiValueModeTests extends ESTestCase { } } final NumericDoubleValues singleValues = new NumericDoubleValues() { + private int docID; + @Override + public boolean advanceExact(int doc) throws IOException { + docID = doc; + return docsWithValue == null || docsWithValue.get(docID); + } @Override - public double get(int docID) { + public double doubleValue() { return array[docID]; } }; - final SortedNumericDoubleValues singletonValues = FieldData.singleton(singleValues, docsWithValue); + final SortedNumericDoubleValues singletonValues = FieldData.singleton(singleValues); final MultiValueMode.UnsortedNumericDoubleValues multiValues = new MultiValueMode.UnsortedNumericDoubleValues() { @Override - public int count() { - return singletonValues.count(); + public int docValueCount() { + return singletonValues.docValueCount(); } @Override - public void setDocument(int doc) { - singletonValues.setDocument(doc); + public boolean advanceExact(int doc) throws IOException { + return singletonValues.advanceExact(doc); } @Override - public double valueAt(int index) { - return Math.cos(singletonValues.valueAt(index)); + public double nextValue() throws IOException { + return Math.cos(singletonValues.nextValue()); } }; - verify(multiValues, numDocs); + verifyUnsortedNumeric(() -> multiValues, numDocs); } public void testUnsortedMultiValuedDoubles() throws Exception { @@ -686,37 +771,41 @@ public class MultiValueModeTests extends ESTestCase { } final MultiValueMode.UnsortedNumericDoubleValues multiValues = new MultiValueMode.UnsortedNumericDoubleValues() { int doc; + int i; @Override - public int count() { + public int docValueCount() { return array[doc].length; } @Override - public void setDocument(int doc) { + public boolean advanceExact(int doc) { this.doc = doc; + i = 0; + return array[doc].length > 0; } @Override - public double valueAt(int index) { - return Math.sin(array[doc][index]); + public double nextValue() { + return Math.sin(array[doc][i++]); } }; - verify(multiValues, numDocs); + verifyUnsortedNumeric(() -> multiValues, numDocs); } - private void verify(MultiValueMode.UnsortedNumericDoubleValues values, int maxDoc) { + private void verifyUnsortedNumeric(Supplier<MultiValueMode.UnsortedNumericDoubleValues> supplier, int maxDoc) throws IOException { for (double missingValue : new double[] { 0, randomDouble() }) { for (MultiValueMode mode : new MultiValueMode[] {MultiValueMode.MIN, MultiValueMode.MAX, MultiValueMode.SUM, MultiValueMode.AVG}) { + UnsortedNumericDoubleValues values = supplier.get(); final NumericDoubleValues selected = mode.select(values, missingValue); for (int i = 0; i < maxDoc; ++i) { - final double actual = selected.get(i); + assertTrue(selected.advanceExact(i)); + final double actual = selected.doubleValue(); double expected = 0.0; - values.setDocument(i); - int numValues = values.count(); - if (numValues == 0) { + if (values.advanceExact(i) == false) { expected = missingValue; } else { + int numValues = values.docValueCount(); if (mode == MultiValueMode.MAX) { expected = Long.MIN_VALUE; } else if (mode == MultiValueMode.MIN) { @@ -724,11 +813,11 @@ public class MultiValueModeTests extends ESTestCase { } for (int j = 0; j < numValues; ++j) { if (mode == MultiValueMode.SUM || mode == MultiValueMode.AVG) { - expected += values.valueAt(j); + expected += values.nextValue(); } else if (mode == MultiValueMode.MIN) { - expected = Math.min(expected, values.valueAt(j)); + expected = Math.min(expected, values.nextValue()); } else if (mode == MultiValueMode.MAX) { - expected = Math.max(expected, values.valueAt(j)); + expected = Math.max(expected, values.nextValue()); } } if (mode == MultiValueMode.AVG) { diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java b/core/src/test/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java index 9ebfb623b6..3dc530204b 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java @@ -236,7 +236,7 @@ public abstract class AggregatorTestCase extends ESTestCase { List<InternalAggregation> aggs = new ArrayList<> (); Query rewritten = searcher.rewrite(query); - Weight weight = searcher.createWeight(rewritten, true); + Weight weight = searcher.createWeight(rewritten, true, 1f); C root = createAggregator(builder, searcher, fieldTypes); try { for (ShardSearcher subSearcher : subSearchers) { diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/range/BinaryRangeAggregatorTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/range/BinaryRangeAggregatorTests.java index 6ba6da3b55..6ed2c1a3a8 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/range/BinaryRangeAggregatorTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/range/BinaryRangeAggregatorTests.java @@ -23,9 +23,9 @@ import java.util.Arrays; import java.util.HashSet; import java.util.Set; -import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.TestUtil; +import org.elasticsearch.index.fielddata.AbstractSortedSetDocValues; import org.elasticsearch.index.fielddata.SortedBinaryDocValues; import org.elasticsearch.search.aggregations.LeafBucketCollector; import org.elasticsearch.search.aggregations.bucket.range.BinaryRangeAggregator.SortedBinaryRangeLeafCollector; @@ -36,7 +36,7 @@ import com.carrotsearch.hppc.LongHashSet; public class BinaryRangeAggregatorTests extends ESTestCase { - private static class FakeSortedSetDocValues extends SortedSetDocValues { + private static class FakeSortedSetDocValues extends AbstractSortedSetDocValues { private final BytesRef[] terms; long[] ords; @@ -47,8 +47,9 @@ public class BinaryRangeAggregatorTests extends ESTestCase { } @Override - public void setDocument(int docID) { + public boolean advanceExact(int docID) { i = 0; + return true; } @Override @@ -145,6 +146,7 @@ public class BinaryRangeAggregatorTests extends ESTestCase { private static class FakeSortedBinaryDocValues extends SortedBinaryDocValues { private final BytesRef[] terms; + int i; long[] ords; FakeSortedBinaryDocValues(BytesRef[] terms) { @@ -152,18 +154,19 @@ public class BinaryRangeAggregatorTests extends ESTestCase { } @Override - public void setDocument(int docID) { - // no-op + public boolean advanceExact(int docID) { + i = 0; + return true; } @Override - public int count() { + public int docValueCount() { return ords.length; } @Override - public BytesRef valueAt(int index) { - return terms[(int) ords[index]]; + public BytesRef nextValue() { + return terms[(int) ords[i++]]; } } diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/support/IncludeExcludeTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/support/IncludeExcludeTests.java index 42f0df4beb..c211f754d1 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/support/IncludeExcludeTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/support/IncludeExcludeTests.java @@ -20,7 +20,6 @@ package org.elasticsearch.search.aggregations.support; import org.apache.lucene.index.DocValues; -import org.apache.lucene.index.RandomAccessOrds; import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.LongBitSet; @@ -30,6 +29,7 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.index.fielddata.AbstractSortedSetDocValues; import org.elasticsearch.index.query.QueryParseContext; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.aggregations.bucket.terms.support.IncludeExclude; @@ -58,13 +58,14 @@ public class IncludeExcludeTests extends ESTestCase { } public void testSingleTermWithOrds() throws IOException { - RandomAccessOrds ords = new RandomAccessOrds() { + SortedSetDocValues ords = new AbstractSortedSetDocValues() { boolean consumed = true; @Override - public void setDocument(int docID) { + public boolean advanceExact(int docID) { consumed = false; + return true; } @Override @@ -88,15 +89,6 @@ public class IncludeExcludeTests extends ESTestCase { return 1; } - @Override - public long ordAt(int index) { - return 0; - } - - @Override - public int cardinality() { - return 1; - } }; IncludeExclude inexcl = new IncludeExclude( new TreeSet<>(Collections.singleton(new BytesRef("foo"))), diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/support/MissingValuesTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/support/MissingValuesTests.java index 2c819a671e..568b8e7996 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/support/MissingValuesTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/support/MissingValuesTests.java @@ -21,24 +21,27 @@ package org.elasticsearch.search.aggregations.support; import com.carrotsearch.randomizedtesting.generators.RandomPicks; import com.carrotsearch.randomizedtesting.generators.RandomStrings; -import org.apache.lucene.index.RandomAccessOrds; + import org.apache.lucene.index.SortedNumericDocValues; +import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.TestUtil; import org.elasticsearch.common.geo.GeoPoint; -import org.elasticsearch.index.fielddata.AbstractRandomAccessOrds; +import org.elasticsearch.index.fielddata.AbstractSortedNumericDocValues; +import org.elasticsearch.index.fielddata.AbstractSortedSetDocValues; import org.elasticsearch.index.fielddata.MultiGeoPointValues; import org.elasticsearch.index.fielddata.SortedBinaryDocValues; import org.elasticsearch.index.fielddata.SortedNumericDoubleValues; import org.elasticsearch.test.ESTestCase; +import java.io.IOException; import java.util.Arrays; import java.util.HashSet; import java.util.Set; public class MissingValuesTests extends ESTestCase { - public void testMissingBytes() { + public void testMissingBytes() throws IOException { final int numDocs = TestUtil.nextInt(random(), 1, 100); final BytesRef[][] values = new BytesRef[numDocs][]; for (int i = 0; i < numDocs; ++i) { @@ -50,40 +53,43 @@ public class MissingValuesTests extends ESTestCase { } SortedBinaryDocValues asBinaryValues = new SortedBinaryDocValues() { - int i = -1; + int doc = -1; + int i; @Override - public BytesRef valueAt(int index) { - return values[i][index]; + public BytesRef nextValue() { + return values[doc][i++]; } @Override - public void setDocument(int docId) { - i = docId; + public boolean advanceExact(int docId) { + doc = docId; + i = 0; + return values[doc].length > 0; } @Override - public int count() { - return values[i].length; + public int docValueCount() { + return values[doc].length; } }; final BytesRef missing = new BytesRef(RandomStrings.randomAsciiOfLength(random(), 2)); SortedBinaryDocValues withMissingReplaced = MissingValues.replaceMissing(asBinaryValues, missing); for (int i = 0; i < numDocs; ++i) { - withMissingReplaced.setDocument(i); + assertTrue(withMissingReplaced.advanceExact(i)); if (values[i].length > 0) { - assertEquals(values[i].length, withMissingReplaced.count()); + assertEquals(values[i].length, withMissingReplaced.docValueCount()); for (int j = 0; j < values[i].length; ++j) { - assertEquals(values[i][j], withMissingReplaced.valueAt(j)); + assertEquals(values[i][j], withMissingReplaced.nextValue()); } } else { - assertEquals(1, withMissingReplaced.count()); - assertEquals(missing, withMissingReplaced.valueAt(0)); + assertEquals(1, withMissingReplaced.docValueCount()); + assertEquals(missing, withMissingReplaced.nextValue()); } } } - public void testMissingOrds() { + public void testMissingOrds() throws IOException { final int numDocs = TestUtil.nextInt(random(), 1, 100); final int numOrds = TestUtil.nextInt(random(), 1, 10); @@ -105,13 +111,16 @@ public class MissingValuesTests extends ESTestCase { ords[i][j] = TestUtil.nextInt(random(), ords[i][j], maxOrd - 1); } } - RandomAccessOrds asRandomAccessOrds = new AbstractRandomAccessOrds() { + SortedSetDocValues asRandomAccessOrds = new AbstractSortedSetDocValues() { - int i = -1; + int doc = -1; + int i; @Override - public void doSetDocument(int docID) { - i = docID; + public boolean advanceExact(int docID) { + doc = docID; + i = 0; + return ords[doc].length > 0; } @Override @@ -125,13 +134,12 @@ public class MissingValuesTests extends ESTestCase { } @Override - public long ordAt(int index) { - return ords[i][index]; - } - - @Override - public int cardinality() { - return ords[i].length; + public long nextOrd() { + if (i < ords[doc].length) { + return ords[doc][i++]; + } else { + return NO_MORE_ORDS; + } } }; @@ -139,28 +147,29 @@ public class MissingValuesTests extends ESTestCase { final BytesRef missingMissing = new BytesRef(RandomStrings.randomAsciiOfLength(random(), 5)); for (BytesRef missing : Arrays.asList(existingMissing, missingMissing)) { - RandomAccessOrds withMissingReplaced = MissingValues.replaceMissing(asRandomAccessOrds, missing); + SortedSetDocValues withMissingReplaced = MissingValues.replaceMissing(asRandomAccessOrds, missing); if (valueSet.contains(missing)) { assertEquals(values.length, withMissingReplaced.getValueCount()); } else { assertEquals(values.length + 1, withMissingReplaced.getValueCount()); } for (int i = 0; i < numDocs; ++i) { - withMissingReplaced.setDocument(i); + assertTrue(withMissingReplaced.advanceExact(i)); if (ords[i].length > 0) { - assertEquals(ords[i].length, withMissingReplaced.cardinality()); - for (int j = 0; j < ords[i].length; ++j) { - assertEquals(values[ords[i][j]], withMissingReplaced.lookupOrd(withMissingReplaced.ordAt(j))); + for (int ord : ords[i]) { + assertEquals(values[ord], + withMissingReplaced.lookupOrd(withMissingReplaced.nextOrd())); } + assertEquals(SortedSetDocValues.NO_MORE_ORDS, withMissingReplaced.nextOrd()); } else { - assertEquals(1, withMissingReplaced.cardinality()); - assertEquals(missing, withMissingReplaced.lookupOrd(withMissingReplaced.ordAt(0))); + assertEquals(missing, withMissingReplaced.lookupOrd(withMissingReplaced.nextOrd())); + assertEquals(SortedSetDocValues.NO_MORE_ORDS, withMissingReplaced.nextOrd()); } } } } - public void testMissingLongs() { + public void testMissingLongs() throws IOException { final int numDocs = TestUtil.nextInt(random(), 1, 100); final int[][] values = new int[numDocs][]; for (int i = 0; i < numDocs; ++i) { @@ -170,42 +179,45 @@ public class MissingValuesTests extends ESTestCase { } Arrays.sort(values[i]); } - SortedNumericDocValues asNumericValues = new SortedNumericDocValues() { + SortedNumericDocValues asNumericValues = new AbstractSortedNumericDocValues() { - int i = -1; + int doc = -1; + int i; @Override - public long valueAt(int index) { - return values[i][index]; + public long nextValue() { + return values[doc][i++]; } @Override - public void setDocument(int docId) { - i = docId; + public boolean advanceExact(int docId) { + doc = docId; + i = 0; + return values[doc].length > 0; } @Override - public int count() { - return values[i].length; + public int docValueCount() { + return values[doc].length; } }; final long missing = randomInt(); SortedNumericDocValues withMissingReplaced = MissingValues.replaceMissing(asNumericValues, missing); for (int i = 0; i < numDocs; ++i) { - withMissingReplaced.setDocument(i); + assertTrue(withMissingReplaced.advanceExact(i)); if (values[i].length > 0) { - assertEquals(values[i].length, withMissingReplaced.count()); + assertEquals(values[i].length, withMissingReplaced.docValueCount()); for (int j = 0; j < values[i].length; ++j) { - assertEquals(values[i][j], withMissingReplaced.valueAt(j)); + assertEquals(values[i][j], withMissingReplaced.nextValue()); } } else { - assertEquals(1, withMissingReplaced.count()); - assertEquals(missing, withMissingReplaced.valueAt(0)); + assertEquals(1, withMissingReplaced.docValueCount()); + assertEquals(missing, withMissingReplaced.nextValue()); } } } - public void testMissingDoubles() { + public void testMissingDoubles() throws IOException { final int numDocs = TestUtil.nextInt(random(), 1, 100); final double[][] values = new double[numDocs][]; for (int i = 0; i < numDocs; ++i) { @@ -217,40 +229,43 @@ public class MissingValuesTests extends ESTestCase { } SortedNumericDoubleValues asNumericValues = new SortedNumericDoubleValues() { - int i = -1; + int doc = -1; + int i; @Override - public double valueAt(int index) { - return values[i][index]; + public double nextValue() { + return values[doc][i++]; } @Override - public void setDocument(int docId) { - i = docId; + public boolean advanceExact(int docId) { + doc = docId; + i = 0; + return true; } @Override - public int count() { - return values[i].length; + public int docValueCount() { + return values[doc].length; } }; final long missing = randomInt(); SortedNumericDoubleValues withMissingReplaced = MissingValues.replaceMissing(asNumericValues, missing); for (int i = 0; i < numDocs; ++i) { - withMissingReplaced.setDocument(i); + assertTrue(withMissingReplaced.advanceExact(i)); if (values[i].length > 0) { - assertEquals(values[i].length, withMissingReplaced.count()); + assertEquals(values[i].length, withMissingReplaced.docValueCount()); for (int j = 0; j < values[i].length; ++j) { - assertEquals(values[i][j], withMissingReplaced.valueAt(j), 0); + assertEquals(values[i][j], withMissingReplaced.nextValue(), 0); } } else { - assertEquals(1, withMissingReplaced.count()); - assertEquals(missing, withMissingReplaced.valueAt(0), 0); + assertEquals(1, withMissingReplaced.docValueCount()); + assertEquals(missing, withMissingReplaced.nextValue(), 0); } } } - public void testMissingGeoPoints() { + public void testMissingGeoPoints() throws IOException { final int numDocs = TestUtil.nextInt(random(), 1, 100); final GeoPoint[][] values = new GeoPoint[numDocs][]; for (int i = 0; i < numDocs; ++i) { @@ -261,35 +276,38 @@ public class MissingValuesTests extends ESTestCase { } MultiGeoPointValues asGeoValues = new MultiGeoPointValues() { - int i = -1; + int doc = -1; + int i; @Override - public GeoPoint valueAt(int index) { - return values[i][index]; + public GeoPoint nextValue() { + return values[doc][i++]; } @Override - public void setDocument(int docId) { - i = docId; + public boolean advanceExact(int docId) { + doc = docId; + i = 0; + return values[doc].length > 0; } @Override - public int count() { - return values[i].length; + public int docValueCount() { + return values[doc].length; } }; final GeoPoint missing = new GeoPoint(randomDouble() * 90, randomDouble() * 180); MultiGeoPointValues withMissingReplaced = MissingValues.replaceMissing(asGeoValues, missing); for (int i = 0; i < numDocs; ++i) { - withMissingReplaced.setDocument(i); + assertTrue(withMissingReplaced.advanceExact(i)); if (values[i].length > 0) { - assertEquals(values[i].length, withMissingReplaced.count()); + assertEquals(values[i].length, withMissingReplaced.docValueCount()); for (int j = 0; j < values[i].length; ++j) { - assertEquals(values[i][j], withMissingReplaced.valueAt(j)); + assertEquals(values[i][j], withMissingReplaced.nextValue()); } } else { - assertEquals(1, withMissingReplaced.count()); - assertEquals(missing, withMissingReplaced.valueAt(0)); + assertEquals(1, withMissingReplaced.docValueCount()); + assertEquals(missing, withMissingReplaced.nextValue()); } } } diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/support/ScriptValuesTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/support/ScriptValuesTests.java index f338e96131..11e03a969d 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/support/ScriptValuesTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/support/ScriptValuesTests.java @@ -28,6 +28,7 @@ import org.elasticsearch.search.aggregations.support.values.ScriptDoubleValues; import org.elasticsearch.search.aggregations.support.values.ScriptLongValues; import org.elasticsearch.test.ESTestCase; +import java.io.IOException; import java.util.Arrays; import java.util.Map; @@ -87,7 +88,7 @@ public class ScriptValuesTests extends ESTestCase { } - public void testLongs() { + public void testLongs() throws IOException { final Object[][] values = new Long[randomInt(10)][]; for (int i = 0; i < values.length; ++i) { Long[] longs = new Long[randomInt(8)]; @@ -100,15 +101,17 @@ public class ScriptValuesTests extends ESTestCase { FakeSearchScript script = new FakeSearchScript(values); ScriptLongValues scriptValues = new ScriptLongValues(script); for (int i = 0; i < values.length; ++i) { - scriptValues.setDocument(i); - assertEquals(values[i].length, scriptValues.count()); - for (int j = 0; j < values[i].length; ++j) { - assertEquals(values[i][j], scriptValues.valueAt(j)); + assertEquals(values[i].length > 0, scriptValues.advanceExact(i)); + if (values[i].length > 0) { + assertEquals(values[i].length, scriptValues.docValueCount()); + for (int j = 0; j < values[i].length; ++j) { + assertEquals(values[i][j], scriptValues.nextValue()); + } } } } - public void testBooleans() { + public void testBooleans() throws IOException { final Object[][] values = new Boolean[randomInt(10)][]; for (int i = 0; i < values.length; ++i) { Boolean[] booleans = new Boolean[randomInt(8)]; @@ -121,15 +124,17 @@ public class ScriptValuesTests extends ESTestCase { FakeSearchScript script = new FakeSearchScript(values); ScriptLongValues scriptValues = new ScriptLongValues(script); for (int i = 0; i < values.length; ++i) { - scriptValues.setDocument(i); - assertEquals(values[i].length, scriptValues.count()); - for (int j = 0; j < values[i].length; ++j) { - assertEquals(values[i][j], scriptValues.valueAt(j) == 1L); + assertEquals(values[i].length > 0, scriptValues.advanceExact(i)); + if (values[i].length > 0) { + assertEquals(values[i].length, scriptValues.docValueCount()); + for (int j = 0; j < values[i].length; ++j) { + assertEquals(values[i][j], scriptValues.nextValue() == 1L); + } } } } - public void testDoubles() { + public void testDoubles() throws IOException { final Object[][] values = new Double[randomInt(10)][]; for (int i = 0; i < values.length; ++i) { Double[] doubles = new Double[randomInt(8)]; @@ -142,15 +147,17 @@ public class ScriptValuesTests extends ESTestCase { FakeSearchScript script = new FakeSearchScript(values); ScriptDoubleValues scriptValues = new ScriptDoubleValues(script); for (int i = 0; i < values.length; ++i) { - scriptValues.setDocument(i); - assertEquals(values[i].length, scriptValues.count()); - for (int j = 0; j < values[i].length; ++j) { - assertEquals(values[i][j], scriptValues.valueAt(j)); + assertEquals(values[i].length > 0, scriptValues.advanceExact(i)); + if (values[i].length > 0) { + assertEquals(values[i].length, scriptValues.docValueCount()); + for (int j = 0; j < values[i].length; ++j) { + assertEquals(values[i][j], scriptValues.nextValue()); + } } } } - public void testBytes() { + public void testBytes() throws IOException { final String[][] values = new String[randomInt(10)][]; for (int i = 0; i < values.length; ++i) { String[] strings = new String[randomInt(8)]; @@ -163,10 +170,12 @@ public class ScriptValuesTests extends ESTestCase { FakeSearchScript script = new FakeSearchScript(values); ScriptBytesValues scriptValues = new ScriptBytesValues(script); for (int i = 0; i < values.length; ++i) { - scriptValues.setDocument(i); - assertEquals(values[i].length, scriptValues.count()); - for (int j = 0; j < values[i].length; ++j) { - assertEquals(new BytesRef(values[i][j]), scriptValues.valueAt(j)); + assertEquals(values[i].length > 0, scriptValues.advanceExact(i)); + if (values[i].length > 0) { + assertEquals(values[i].length, scriptValues.docValueCount()); + for (int j = 0; j < values[i].length; ++j) { + assertEquals(new BytesRef(values[i][j]), scriptValues.nextValue()); + } } } } diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/support/ValuesSourceConfigTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/support/ValuesSourceConfigTests.java index 5f8c658f23..d0c583257a 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/support/ValuesSourceConfigTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/support/ValuesSourceConfigTests.java @@ -48,9 +48,9 @@ public class ValuesSourceConfigTests extends ESSingleNodeTestCase { ValuesSource.Bytes valuesSource = config.toValuesSource(context); LeafReaderContext ctx = searcher.reader().leaves().get(0); SortedBinaryDocValues values = valuesSource.bytesValues(ctx); - values.setDocument(0); - assertEquals(1, values.count()); - assertEquals(new BytesRef("abc"), values.valueAt(0)); + assertTrue(values.advanceExact(0)); + assertEquals(1, values.docValueCount()); + assertEquals(new BytesRef("abc"), values.nextValue()); } } @@ -70,16 +70,15 @@ public class ValuesSourceConfigTests extends ESSingleNodeTestCase { ValuesSource.Bytes valuesSource = config.toValuesSource(context); LeafReaderContext ctx = searcher.reader().leaves().get(0); SortedBinaryDocValues values = valuesSource.bytesValues(ctx); - values.setDocument(0); - assertEquals(0, values.count()); + assertFalse(values.advanceExact(0)); config = ValuesSourceConfig.resolve( context, null, "bytes", null, "abc", null, null); valuesSource = config.toValuesSource(context); values = valuesSource.bytesValues(ctx); - values.setDocument(0); - assertEquals(1, values.count()); - assertEquals(new BytesRef("abc"), values.valueAt(0)); + assertTrue(values.advanceExact(0)); + assertEquals(1, values.docValueCount()); + assertEquals(new BytesRef("abc"), values.nextValue()); } } @@ -102,9 +101,9 @@ public class ValuesSourceConfigTests extends ESSingleNodeTestCase { valuesSource = config.toValuesSource(context); LeafReaderContext ctx = searcher.reader().leaves().get(0); SortedBinaryDocValues values = valuesSource.bytesValues(ctx); - values.setDocument(0); - assertEquals(1, values.count()); - assertEquals(new BytesRef("abc"), values.valueAt(0)); + assertTrue(values.advanceExact(0)); + assertEquals(1, values.docValueCount()); + assertEquals(new BytesRef("abc"), values.nextValue()); } } @@ -124,9 +123,9 @@ public class ValuesSourceConfigTests extends ESSingleNodeTestCase { ValuesSource.Numeric valuesSource = config.toValuesSource(context); LeafReaderContext ctx = searcher.reader().leaves().get(0); SortedNumericDocValues values = valuesSource.longValues(ctx); - values.setDocument(0); - assertEquals(1, values.count()); - assertEquals(42, values.valueAt(0)); + assertTrue(values.advanceExact(0)); + assertEquals(1, values.docValueCount()); + assertEquals(42, values.nextValue()); } } @@ -146,16 +145,15 @@ public class ValuesSourceConfigTests extends ESSingleNodeTestCase { ValuesSource.Numeric valuesSource = config.toValuesSource(context); LeafReaderContext ctx = searcher.reader().leaves().get(0); SortedNumericDocValues values = valuesSource.longValues(ctx); - values.setDocument(0); - assertEquals(0, values.count()); + assertFalse(values.advanceExact(0)); config = ValuesSourceConfig.resolve( context, null, "long", null, 42, null, null); valuesSource = config.toValuesSource(context); values = valuesSource.longValues(ctx); - values.setDocument(0); - assertEquals(1, values.count()); - assertEquals(42, values.valueAt(0)); + assertTrue(values.advanceExact(0)); + assertEquals(1, values.docValueCount()); + assertEquals(42, values.nextValue()); } } @@ -179,9 +177,9 @@ public class ValuesSourceConfigTests extends ESSingleNodeTestCase { valuesSource = config.toValuesSource(context); LeafReaderContext ctx = searcher.reader().leaves().get(0); SortedNumericDocValues values = valuesSource.longValues(ctx); - values.setDocument(0); - assertEquals(1, values.count()); - assertEquals(42, values.valueAt(0)); + assertTrue(values.advanceExact(0)); + assertEquals(1, values.docValueCount()); + assertEquals(42, values.nextValue()); } } @@ -201,9 +199,9 @@ public class ValuesSourceConfigTests extends ESSingleNodeTestCase { ValuesSource.Numeric valuesSource = config.toValuesSource(context); LeafReaderContext ctx = searcher.reader().leaves().get(0); SortedNumericDocValues values = valuesSource.longValues(ctx); - values.setDocument(0); - assertEquals(1, values.count()); - assertEquals(1, values.valueAt(0)); + assertTrue(values.advanceExact(0)); + assertEquals(1, values.docValueCount()); + assertEquals(1, values.nextValue()); } } @@ -223,16 +221,15 @@ public class ValuesSourceConfigTests extends ESSingleNodeTestCase { ValuesSource.Numeric valuesSource = config.toValuesSource(context); LeafReaderContext ctx = searcher.reader().leaves().get(0); SortedNumericDocValues values = valuesSource.longValues(ctx); - values.setDocument(0); - assertEquals(0, values.count()); + assertFalse(values.advanceExact(0)); config = ValuesSourceConfig.resolve( context, null, "bool", null, true, null, null); valuesSource = config.toValuesSource(context); values = valuesSource.longValues(ctx); - values.setDocument(0); - assertEquals(1, values.count()); - assertEquals(1, values.valueAt(0)); + assertTrue(values.advanceExact(0)); + assertEquals(1, values.docValueCount()); + assertEquals(1, values.nextValue()); } } @@ -256,9 +253,9 @@ public class ValuesSourceConfigTests extends ESSingleNodeTestCase { valuesSource = config.toValuesSource(context); LeafReaderContext ctx = searcher.reader().leaves().get(0); SortedNumericDocValues values = valuesSource.longValues(ctx); - values.setDocument(0); - assertEquals(1, values.count()); - assertEquals(1, values.valueAt(0)); + assertTrue(values.advanceExact(0)); + assertEquals(1, values.docValueCount()); + assertEquals(1, values.nextValue()); } } } diff --git a/core/src/test/java/org/elasticsearch/search/basic/SearchWithRandomExceptionsIT.java b/core/src/test/java/org/elasticsearch/search/basic/SearchWithRandomExceptionsIT.java index 067064dfc1..5b68d3cd1f 100644 --- a/core/src/test/java/org/elasticsearch/search/basic/SearchWithRandomExceptionsIT.java +++ b/core/src/test/java/org/elasticsearch/search/basic/SearchWithRandomExceptionsIT.java @@ -243,6 +243,11 @@ public class SearchWithRandomExceptionsIT extends ESIntegTestCase { protected DirectoryReader doWrapDirectoryReader(DirectoryReader in) throws IOException { return new RandomExceptionDirectoryReaderWrapper(in, settings); } + + @Override + public CacheHelper getReaderCacheHelper() { + return in.getReaderCacheHelper(); + } } diff --git a/core/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/PlainHighlighterTests.java b/core/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/PlainHighlighterTests.java index b923c2464d..18ba716f70 100644 --- a/core/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/PlainHighlighterTests.java +++ b/core/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/PlainHighlighterTests.java @@ -19,29 +19,12 @@ package org.elasticsearch.search.fetch.subphase.highlight; -import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.MockAnalyzer; -import org.apache.lucene.analysis.standard.StandardAnalyzer; import org.apache.lucene.index.Term; -import org.apache.lucene.search.BooleanClause; -import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.PhraseQuery; import org.apache.lucene.search.Query; -import org.apache.lucene.search.TermQuery; -import org.apache.lucene.search.highlight.InvalidTokenOffsetsException; import org.apache.lucene.search.highlight.QueryScorer; -import org.apache.lucene.spatial.geopoint.search.GeoPointDistanceQuery; -import org.apache.lucene.spatial.geopoint.search.GeoPointInBBoxQuery; -import org.apache.lucene.spatial.geopoint.search.GeoPointInPolygonQuery; import org.apache.lucene.util.LuceneTestCase; -import org.elasticsearch.index.analysis.FieldNameAnalyzer; -import org.elasticsearch.search.fetch.subphase.highlight.CustomQueryScorer; - -import java.io.IOException; -import java.util.HashMap; -import java.util.Map; - -import static org.hamcrest.Matchers.equalTo; public class PlainHighlighterTests extends LuceneTestCase { @@ -55,40 +38,4 @@ public class PlainHighlighterTests extends LuceneTestCase { String[] frags = highlighter.getBestFragments(new MockAnalyzer(random()), "field", "bar foo bar foo", 10); assertArrayEquals(new String[] {"bar <B>foo</B> <B>bar</B> foo"}, frags); } - - public void checkGeoQueryHighlighting(Query geoQuery) throws IOException, InvalidTokenOffsetsException { - Map analysers = new HashMap<String, Analyzer>(); - analysers.put("text", new StandardAnalyzer()); - FieldNameAnalyzer fieldNameAnalyzer = new FieldNameAnalyzer(analysers); - Query termQuery = new TermQuery(new Term("text", "failure")); - Query boolQuery = new BooleanQuery.Builder().add(new BooleanClause(geoQuery, BooleanClause.Occur.SHOULD)) - .add(new BooleanClause(termQuery, BooleanClause.Occur.SHOULD)).build(); - org.apache.lucene.search.highlight.Highlighter highlighter = - new org.apache.lucene.search.highlight.Highlighter(new CustomQueryScorer(boolQuery)); - String fragment = highlighter.getBestFragment(fieldNameAnalyzer.tokenStream("text", "Arbitrary text field which should not cause " + - "a failure"), "Arbitrary text field which should not cause a failure"); - assertThat(fragment, equalTo("Arbitrary text field which should not cause a <B>failure</B>")); - Query rewritten = boolQuery.rewrite(null); - highlighter = new org.apache.lucene.search.highlight.Highlighter(new CustomQueryScorer(rewritten)); - fragment = highlighter.getBestFragment(fieldNameAnalyzer.tokenStream("text", "Arbitrary text field which should not cause " + - "a failure"), "Arbitrary text field which should not cause a failure"); - assertThat(fragment, equalTo("Arbitrary text field which should not cause a <B>failure</B>")); - } - - public void testGeoPointInBBoxQueryHighlighting() throws IOException, InvalidTokenOffsetsException { - Query geoQuery = new GeoPointDistanceQuery("geo_point", -64.92354174306496, -170.15625, 5576757); - checkGeoQueryHighlighting(geoQuery); - } - - public void testGeoPointDistanceQueryHighlighting() throws IOException, InvalidTokenOffsetsException { - Query geoQuery = new GeoPointInBBoxQuery("geo_point", -64.92354174306496, 61.10078883158897, -170.15625, 118.47656249999999); - checkGeoQueryHighlighting(geoQuery); - } - - public void testGeoPointInPolygonQueryHighlighting() throws IOException, InvalidTokenOffsetsException { - double[] polyLats = new double[]{0, 60, 0, 0}; - double[] polyLons = new double[]{0, 60, 90, 0}; - Query geoQuery = new GeoPointInPolygonQuery("geo_point", polyLats, polyLons); - checkGeoQueryHighlighting(geoQuery); - } } diff --git a/core/src/test/java/org/elasticsearch/search/functionscore/QueryRescorerIT.java b/core/src/test/java/org/elasticsearch/search/functionscore/QueryRescorerIT.java index fc9dfb8844..18db8cd539 100644 --- a/core/src/test/java/org/elasticsearch/search/functionscore/QueryRescorerIT.java +++ b/core/src/test/java/org/elasticsearch/search/functionscore/QueryRescorerIT.java @@ -587,12 +587,12 @@ public class QueryRescorerIT extends ESIntegTestCase { String[] intToEnglish = new String[] { English.intToEnglish(i), English.intToEnglish(i + 1), English.intToEnglish(i + 2), English.intToEnglish(i + 3) }; - QueryBuilder query = boolQuery().disableCoord(true) + QueryBuilder query = boolQuery() .should(functionScoreQuery(termQuery("field1", intToEnglish[0]), weightFactorFunction(2.0f)).boostMode(REPLACE)) .should(functionScoreQuery(termQuery("field1", intToEnglish[1]), weightFactorFunction(3.0f)).boostMode(REPLACE)) .should(functionScoreQuery(termQuery("field1", intToEnglish[2]), weightFactorFunction(5.0f)).boostMode(REPLACE)) .should(functionScoreQuery(termQuery("field1", intToEnglish[3]), weightFactorFunction(0.2f)).boostMode(REPLACE)); - QueryRescorerBuilder rescoreQuery = queryRescorer(boolQuery().disableCoord(true) + QueryRescorerBuilder rescoreQuery = queryRescorer(boolQuery() .should(functionScoreQuery(termQuery("field1", intToEnglish[0]), weightFactorFunction(5.0f)).boostMode(REPLACE)) .should(functionScoreQuery(termQuery("field1", intToEnglish[1]), weightFactorFunction(7.0f)).boostMode(REPLACE)) .should(functionScoreQuery(termQuery("field1", intToEnglish[3]), weightFactorFunction(0.0f)).boostMode(REPLACE))); diff --git a/core/src/test/java/org/elasticsearch/search/query/SimpleQueryStringIT.java b/core/src/test/java/org/elasticsearch/search/query/SimpleQueryStringIT.java index f22ec392b9..58c0bf82e9 100644 --- a/core/src/test/java/org/elasticsearch/search/query/SimpleQueryStringIT.java +++ b/core/src/test/java/org/elasticsearch/search/query/SimpleQueryStringIT.java @@ -120,6 +120,7 @@ public class SimpleQueryStringIT extends ESIntegTestCase { assertSearchHits(searchResponse, "5", "6"); } + @AwaitsFix(bugUrl="https://github.com/elastic/elasticsearch/issues/23966") public void testSimpleQueryStringMinimumShouldMatch() throws Exception { createIndex("test"); ensureGreen("test"); diff --git a/docs/reference/aggregations/metrics/geocentroid-aggregation.asciidoc b/docs/reference/aggregations/metrics/geocentroid-aggregation.asciidoc index f215857bf8..4756ebdafc 100644 --- a/docs/reference/aggregations/metrics/geocentroid-aggregation.asciidoc +++ b/docs/reference/aggregations/metrics/geocentroid-aggregation.asciidoc @@ -60,8 +60,8 @@ The response for the above aggregation: "aggregations": { "centroid": { "location": { - "lat": 51.009829603135586, - "lon": 3.966213036328554 + "lat": 51.00982963107526, + "lon": 3.9662130922079086 } } } @@ -113,7 +113,7 @@ The response for the above aggregation: "centroid": { "location": { "lat": 52.371655656024814, - "lon": 4.909563269466162 + "lon": 4.909563297405839 } } }, @@ -122,8 +122,8 @@ The response for the above aggregation: "doc_count": 2, "centroid": { "location": { - "lat": 48.86055544484407, - "lon": 2.331694420427084 + "lat": 48.86055548675358, + "lon": 2.3316944623366 } } }, @@ -132,8 +132,8 @@ The response for the above aggregation: "doc_count": 1, "centroid": { "location": { - "lat": 51.222899928689, - "lon": 4.405199903994799 + "lat": 51.22289997059852, + "lon": 4.40519998781383 } } } diff --git a/docs/reference/index-modules/similarity.asciidoc b/docs/reference/index-modules/similarity.asciidoc index 7930ed573b..61333c41a0 100644 --- a/docs/reference/index-modules/similarity.asciidoc +++ b/docs/reference/index-modules/similarity.asciidoc @@ -165,14 +165,10 @@ Type name: `LMJelinekMercer` [float] [[default-base]] -==== Default and Base Similarities +==== Default Similarity By default, Elasticsearch will use whatever similarity is configured as -`default`. However, the similarity functions `queryNorm()` and `coord()` -are not per-field. Consequently, for expert users wanting to change the -implementation used for these two methods, while not changing the -`default`, it is possible to configure a similarity with the name -`base`. This similarity will then be used for the two methods. +`default`. You can change the default similarity for all fields in an index when it is <<indices-create-index,created>>: @@ -185,7 +181,7 @@ PUT /my_index "index": { "similarity": { "default": { - "type": "classic" + "type": "boolean" } } } @@ -205,7 +201,7 @@ PUT /my_index/_settings "index": { "similarity": { "default": { - "type": "classic" + "type": "boolean" } } } diff --git a/docs/reference/migration/migrate_6_0/search.asciidoc b/docs/reference/migration/migrate_6_0/search.asciidoc index f94d93b6ad..7b080eeb3b 100644 --- a/docs/reference/migration/migrate_6_0/search.asciidoc +++ b/docs/reference/migration/migrate_6_0/search.asciidoc @@ -48,6 +48,9 @@ * The `ignore_malformed` and `coerce` parameters have been removed from `geo_bounding_box`, `geo_polygon`, and `geo_distance` queries. +* The `disable_coord` parameter of the `bool` and `common_terms` queries has + been removed. If provided, it will be ignored and issue a deprecation warning. + ==== Search shards API The search shards API no longer accepts the `type` url parameter, which didn't diff --git a/docs/reference/migration/migrate_6_0/settings.asciidoc b/docs/reference/migration/migrate_6_0/settings.asciidoc index 44acb99939..fa010aab1f 100644 --- a/docs/reference/migration/migrate_6_0/settings.asciidoc +++ b/docs/reference/migration/migrate_6_0/settings.asciidoc @@ -59,4 +59,10 @@ will use the best `store` implementation for your operating system. The blocking TCP client, blocking TCP server, and blocking HTTP server have been removed. As a consequence, the `network.tcp.blocking_server`, `network.tcp.blocking_client`, `network.tcp.blocking`,`transport.tcp.blocking_client`, `transport.tcp.blocking_server`, -and `http.tcp.blocking_server` settings are not recognized anymore.
\ No newline at end of file +and `http.tcp.blocking_server` settings are not recognized anymore. + +==== Similarity settings + +The `base` similarity is now ignored as coords and query normalization have +been removed. If provided, this setting will be ignored and issue a +deprecation warning. diff --git a/docs/reference/query-dsl/bool-query.asciidoc b/docs/reference/query-dsl/bool-query.asciidoc index 977953affc..4d66e5e7f6 100644 --- a/docs/reference/query-dsl/bool-query.asciidoc +++ b/docs/reference/query-dsl/bool-query.asciidoc @@ -37,11 +37,6 @@ If this query is used in a filter context and it has `should` clauses then at least one `should` clause is required to match. ======================================================================== -The bool query also supports `disable_coord` parameter (defaults to -`false`). Basically the coord similarity computes a score factor based -on the fraction of all query terms that a document contains. See Lucene -`BooleanQuery` for more details. - The `bool` query takes a _more-matches-is-better_ approach, so the score from each matching `must` or `should` clause will be added together to provide the final `_score` for each document. diff --git a/docs/reference/query-dsl/common-terms-query.asciidoc b/docs/reference/query-dsl/common-terms-query.asciidoc index fcc4ace2ec..a0c58597f7 100644 --- a/docs/reference/query-dsl/common-terms-query.asciidoc +++ b/docs/reference/query-dsl/common-terms-query.asciidoc @@ -297,5 +297,5 @@ GET /_search The high frequency generated query is then slightly less restrictive than with an `AND`. -The `common` terms query also supports `boost`, `analyzer` and -`disable_coord` as parameters. +The `common` terms query also supports `boost` and `analyzer` as +parameters. diff --git a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsAggregator.java b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsAggregator.java index 4665e74daa..5c193828c5 100644 --- a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsAggregator.java +++ b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsAggregator.java @@ -100,16 +100,20 @@ public class MatrixStatsAggregator extends MetricsAggregator { /** * return a map of field names and data */ - private boolean includeDocument(int doc) { + private boolean includeDocument(int doc) throws IOException { // loop over fields for (int i = 0; i < fieldVals.length; ++i) { final NumericDoubleValues doubleValues = values[i]; - final double value = doubleValues.get(doc); - // skip if value is missing - if (value == Double.NEGATIVE_INFINITY) { + if (doubleValues.advanceExact(doc)) { + final double value = doubleValues.doubleValue(); + if (value == Double.NEGATIVE_INFINITY) { + // TODO: Fix matrix stats to treat neg inf as any other value + return false; + } + fieldVals[i] = value; + } else { return false; } - fieldVals[i] = value; } return true; } diff --git a/modules/lang-expression/licenses/lucene-expressions-6.5.0.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-6.5.0.jar.sha1 deleted file mode 100644 index 10514bb3d1..0000000000 --- a/modules/lang-expression/licenses/lucene-expressions-6.5.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -5dfd44932fc77187a233a1cbf228c1a96ac8924f
\ No newline at end of file diff --git a/modules/lang-expression/licenses/lucene-expressions-7.0.0-snapshot-89f6d17.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-7.0.0-snapshot-89f6d17.jar.sha1 new file mode 100644 index 0000000000..c47ca7d206 --- /dev/null +++ b/modules/lang-expression/licenses/lucene-expressions-7.0.0-snapshot-89f6d17.jar.sha1 @@ -0,0 +1 @@ +52bc12bbc30db614056896495f30699d69eabae4
\ No newline at end of file diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/CountMethodValueSource.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/CountMethodValueSource.java index 1aa85ef685..6a36ff7a53 100644 --- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/CountMethodValueSource.java +++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/CountMethodValueSource.java @@ -50,11 +50,14 @@ final class CountMethodValueSource extends ValueSource { final SortedNumericDoubleValues values = leafData.getDoubleValues(); return new DoubleDocValues(this) { - @Override - public double doubleVal(int doc) { - values.setDocument(doc); - return values.count(); - } + @Override + public double doubleVal(int doc) throws IOException { + if (values.advanceExact(doc)) { + return values.docValueCount(); + } else { + return 0; + } + } }; } diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/DateMethodValueSource.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/DateMethodValueSource.java index 98909f4401..b0bc7c203b 100644 --- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/DateMethodValueSource.java +++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/DateMethodValueSource.java @@ -56,12 +56,16 @@ class DateMethodValueSource extends FieldDataValueSource { final Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("UTC"), Locale.ROOT); NumericDoubleValues docValues = multiValueMode.select(leafData.getDoubleValues(), 0d); return new DoubleDocValues(this) { - @Override - public double doubleVal(int docId) { - long millis = (long)docValues.get(docId); - calendar.setTimeInMillis(millis); - return calendar.get(calendarType); - } + @Override + public double doubleVal(int docId) throws IOException { + if (docValues.advanceExact(docId)) { + long millis = (long)docValues.doubleValue(); + calendar.setTimeInMillis(millis); + return calendar.get(calendarType); + } else { + return 0; + } + } }; } diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/DateObjectValueSource.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/DateObjectValueSource.java index a9f2018d39..86abcbcbef 100644 --- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/DateObjectValueSource.java +++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/DateObjectValueSource.java @@ -58,12 +58,16 @@ class DateObjectValueSource extends FieldDataValueSource { MutableDateTime joda = new MutableDateTime(0, DateTimeZone.UTC); NumericDoubleValues docValues = multiValueMode.select(leafData.getDoubleValues(), 0d); return new DoubleDocValues(this) { - @Override - public double doubleVal(int docId) { - long millis = (long)docValues.get(docId); - joda.setMillis(millis); - return function.applyAsInt(joda); - } + @Override + public double doubleVal(int docId) throws IOException { + if (docValues.advanceExact(docId)) { + long millis = (long)docValues.doubleValue(); + joda.setMillis(millis); + return function.applyAsInt(joda); + } else { + return 0; + } + } }; } diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/EmptyMemberValueSource.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/EmptyMemberValueSource.java index b4c8582e0d..aa12560e1f 100644 --- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/EmptyMemberValueSource.java +++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/EmptyMemberValueSource.java @@ -19,10 +19,6 @@ package org.elasticsearch.script.expression; -import java.io.IOException; -import java.util.Map; -import java.util.Objects; - import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.queries.function.FunctionValues; import org.apache.lucene.queries.function.ValueSource; @@ -31,6 +27,10 @@ import org.elasticsearch.index.fielddata.AtomicNumericFieldData; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.SortedNumericDoubleValues; +import java.io.IOException; +import java.util.Map; +import java.util.Objects; + /** * ValueSource to return non-zero if a field is missing. * <p> @@ -50,12 +50,11 @@ final class EmptyMemberValueSource extends ValueSource { final SortedNumericDoubleValues values = leafData.getDoubleValues(); return new DoubleDocValues(this) { @Override - public double doubleVal(int doc) { - values.setDocument(doc); - if (values.count() == 0) { - return 1; - } else { + public double doubleVal(int doc) throws IOException { + if (values.advanceExact(doc)) { return 0; + } else { + return 1; } } }; diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/FieldDataValueSource.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/FieldDataValueSource.java index 3ac885e49d..f2aaced376 100644 --- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/FieldDataValueSource.java +++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/FieldDataValueSource.java @@ -71,8 +71,12 @@ class FieldDataValueSource extends ValueSource { NumericDoubleValues docValues = multiValueMode.select(leafData.getDoubleValues(), 0d); return new DoubleDocValues(this) { @Override - public double doubleVal(int doc) { - return docValues.get(doc); + public double doubleVal(int doc) throws IOException { + if (docValues.advanceExact(doc)) { + return docValues.doubleValue(); + } else { + return 0; + } } }; } diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/GeoEmptyValueSource.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/GeoEmptyValueSource.java index 2a9b09ba10..0b16aaf9dc 100644 --- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/GeoEmptyValueSource.java +++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/GeoEmptyValueSource.java @@ -48,9 +48,8 @@ final class GeoEmptyValueSource extends ValueSource { final MultiGeoPointValues values = leafData.getGeoPointValues(); return new DoubleDocValues(this) { @Override - public double doubleVal(int doc) { - values.setDocument(doc); - if (values.count() == 0) { + public double doubleVal(int doc) throws IOException { + if (values.advanceExact(doc)) { return 1; } else { return 0; diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/GeoLatitudeValueSource.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/GeoLatitudeValueSource.java index d23eceda2f..fd812dac5a 100644 --- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/GeoLatitudeValueSource.java +++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/GeoLatitudeValueSource.java @@ -48,12 +48,11 @@ final class GeoLatitudeValueSource extends ValueSource { final MultiGeoPointValues values = leafData.getGeoPointValues(); return new DoubleDocValues(this) { @Override - public double doubleVal(int doc) { - values.setDocument(doc); - if (values.count() == 0) { - return 0.0; + public double doubleVal(int doc) throws IOException { + if (values.advanceExact(doc)) { + return values.nextValue().getLat(); } else { - return values.valueAt(0).getLat(); + return 0.0; } } }; diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/GeoLongitudeValueSource.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/GeoLongitudeValueSource.java index 4fbc8fd936..fd05d92d62 100644 --- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/GeoLongitudeValueSource.java +++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/GeoLongitudeValueSource.java @@ -48,12 +48,11 @@ final class GeoLongitudeValueSource extends ValueSource { final MultiGeoPointValues values = leafData.getGeoPointValues(); return new DoubleDocValues(this) { @Override - public double doubleVal(int doc) { - values.setDocument(doc); - if (values.count() == 0) { - return 0.0; + public double doubleVal(int doc) throws IOException { + if (values.advanceExact(doc)) { + return values.nextValue().getLon(); } else { - return values.valueAt(0).getLon(); + return 0.0; } } }; diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQuery.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQuery.java index 848b882d1a..c4f321c2ea 100644 --- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQuery.java +++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQuery.java @@ -74,9 +74,9 @@ final class PercolateQuery extends Query implements Accountable { } @Override - public Weight createWeight(IndexSearcher searcher, boolean needsScores) throws IOException { - final Weight verifiedMatchesWeight = verifiedMatchesQuery.createWeight(searcher, false); - final Weight candidateMatchesWeight = candidateMatchesQuery.createWeight(searcher, false); + public Weight createWeight(IndexSearcher searcher, boolean needsScores, float boost) throws IOException { + final Weight verifiedMatchesWeight = verifiedMatchesQuery.createWeight(searcher, false, boost); + final Weight candidateMatchesWeight = candidateMatchesQuery.createWeight(searcher, false, boost); return new Weight(this) { @Override public void extractTerms(Set<Term> set) { @@ -105,16 +105,6 @@ final class PercolateQuery extends Query implements Accountable { } @Override - public float getValueForNormalization() throws IOException { - return candidateMatchesWeight.getValueForNormalization(); - } - - @Override - public void normalize(float v, float v1) { - candidateMatchesWeight.normalize(v, v1); - } - - @Override public Scorer scorer(LeafReaderContext leafReaderContext) throws IOException { final Scorer approximation = candidateMatchesWeight.scorer(leafReaderContext); if (approximation == null) { diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java index d548a3f354..9ebb65c327 100644 --- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java +++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java @@ -33,7 +33,6 @@ import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; import org.apache.lucene.search.Weight; import org.apache.lucene.store.RAMDirectory; -import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ResourceNotFoundException; @@ -485,10 +484,9 @@ public class PercolateQueryBuilder extends AbstractQueryBuilder<PercolateQueryBu return docId -> null; } - Bits bits = leafReader.getDocsWithField(fieldType.queryBuilderField.name()); return docId -> { - if (bits.get(docId)) { - BytesRef qbSource = binaryDocValues.get(docId); + if (binaryDocValues.advanceExact(docId)) { + BytesRef qbSource = binaryDocValues.binaryValue(); if (qbSource.length > 0) { XContent xContent = PercolatorFieldMapper.QUERY_BUILDER_CONTENT_TYPE.xContent(); try (XContentParser sourceParser = xContent.createParser(context.getXContentRegistry(), qbSource.bytes, diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/CandidateQueryTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/CandidateQueryTests.java index 65005e957a..8c594a0545 100644 --- a/modules/percolator/src/test/java/org/elasticsearch/percolator/CandidateQueryTests.java +++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/CandidateQueryTests.java @@ -79,6 +79,7 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.List; +import java.util.Set; import java.util.function.Function; import static org.hamcrest.Matchers.equalTo; @@ -245,7 +246,7 @@ public class CandidateQueryTests extends ESSingleNodeTestCase { addQuery(commonTermsQuery, documents); BlendedTermQuery blendedTermQuery = BlendedTermQuery.booleanBlendedQuery(new Term[]{new Term("field", "quick"), - new Term("field", "brown"), new Term("field", "fox")}, false); + new Term("field", "brown"), new Term("field", "fox")}); addQuery(blendedTermQuery, documents); SpanNearQuery spanNearQuery = new SpanNearQuery.Builder("field", true) @@ -360,12 +361,15 @@ public class CandidateQueryTests extends ESSingleNodeTestCase { } @Override - public Weight createWeight(IndexSearcher searcher, boolean needsScores) { - return new ConstantScoreWeight(this) { + public Weight createWeight(IndexSearcher searcher, boolean needsScores, float boost) { + return new Weight(this) { float _score; @Override + public void extractTerms(Set<Term> terms) {} + + @Override public Explanation explain(LeafReaderContext context, int doc) throws IOException { Scorer scorer = scorer(context); if (scorer != null) { @@ -406,7 +410,7 @@ public class CandidateQueryTests extends ESSingleNodeTestCase { } } }; - return new FilterScorer(new ConstantScoreScorer(this, score(), memoryIndexIterator)) { + return new FilterScorer(new ConstantScoreScorer(this, 1f, memoryIndexIterator)) { @Override public float score() throws IOException { diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/QueryAnalyzerTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/QueryAnalyzerTests.java index 558d237f12..ea18bd38d1 100644 --- a/modules/percolator/src/test/java/org/elasticsearch/percolator/QueryAnalyzerTests.java +++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/QueryAnalyzerTests.java @@ -276,7 +276,7 @@ public class QueryAnalyzerTests extends ESTestCase { public void testExtractQueryMetadata_blendedTermQuery() { Term[] termsArr = new Term[]{new Term("_field", "_term1"), new Term("_field", "_term2")}; - BlendedTermQuery commonTermsQuery = BlendedTermQuery.booleanBlendedQuery(termsArr, false); + BlendedTermQuery commonTermsQuery = BlendedTermQuery.booleanBlendedQuery(termsArr); Result result = analyze(commonTermsQuery); assertThat(result.verified, is(true)); List<Term> terms = new ArrayList<>(result.terms); diff --git a/plugins/analysis-icu/licenses/lucene-analyzers-icu-6.5.0.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analyzers-icu-6.5.0.jar.sha1 deleted file mode 100644 index 95df77a752..0000000000 --- a/plugins/analysis-icu/licenses/lucene-analyzers-icu-6.5.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -3a71465f63887f871bc377d87a0838c29b0a857d
\ No newline at end of file diff --git a/plugins/analysis-icu/licenses/lucene-analyzers-icu-7.0.0-snapshot-89f6d17.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analyzers-icu-7.0.0-snapshot-89f6d17.jar.sha1 new file mode 100644 index 0000000000..1ac5e494f0 --- /dev/null +++ b/plugins/analysis-icu/licenses/lucene-analyzers-icu-7.0.0-snapshot-89f6d17.jar.sha1 @@ -0,0 +1 @@ +e6e6d743c100e4d7bc55480d5333d634e41856ca
\ No newline at end of file diff --git a/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-6.5.0.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-6.5.0.jar.sha1 deleted file mode 100644 index 0c928699fc..0000000000 --- a/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-6.5.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -03353b0d030f6d5a63c4c0d5b64c770f5ba9d829
\ No newline at end of file diff --git a/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-7.0.0-snapshot-89f6d17.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-7.0.0-snapshot-89f6d17.jar.sha1 new file mode 100644 index 0000000000..8be206e2c1 --- /dev/null +++ b/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-7.0.0-snapshot-89f6d17.jar.sha1 @@ -0,0 +1 @@ +e520a9c7f8a2fc9f7c575940d9b24834a592ca25
\ No newline at end of file diff --git a/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-6.5.0.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-6.5.0.jar.sha1 deleted file mode 100644 index ba2bee2847..0000000000 --- a/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-6.5.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -77ce4fb8c62688d8a094f08a07685c464ec46345
\ No newline at end of file diff --git a/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-7.0.0-snapshot-89f6d17.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-7.0.0-snapshot-89f6d17.jar.sha1 new file mode 100644 index 0000000000..2b69050029 --- /dev/null +++ b/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-7.0.0-snapshot-89f6d17.jar.sha1 @@ -0,0 +1 @@ +32fe29deb1c10cb7ae70d5f4d95fcc414f9813d6
\ No newline at end of file diff --git a/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-6.5.0.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-6.5.0.jar.sha1 deleted file mode 100644 index 0a0ae4cf40..0000000000 --- a/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-6.5.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -60a780d900e48b0cead42d82fe405ad54bd658c3
\ No newline at end of file diff --git a/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-7.0.0-snapshot-89f6d17.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-7.0.0-snapshot-89f6d17.jar.sha1 new file mode 100644 index 0000000000..5412498f1c --- /dev/null +++ b/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-7.0.0-snapshot-89f6d17.jar.sha1 @@ -0,0 +1 @@ +e50f4ab0d6ebf85b282a86707309343e3260c4a2
\ No newline at end of file diff --git a/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-6.5.0.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-6.5.0.jar.sha1 deleted file mode 100644 index 9a1387fa22..0000000000 --- a/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-6.5.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -894c42c011d291e72d14db660499c75281de9efd
\ No newline at end of file diff --git a/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-7.0.0-snapshot-89f6d17.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-7.0.0-snapshot-89f6d17.jar.sha1 new file mode 100644 index 0000000000..c845088ffb --- /dev/null +++ b/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-7.0.0-snapshot-89f6d17.jar.sha1 @@ -0,0 +1 @@ +a41c22ef7dd43991e1f3555ff527ac79eb47fdca
\ No newline at end of file diff --git a/plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-6.5.0.jar.sha1 b/plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-6.5.0.jar.sha1 deleted file mode 100644 index 89a0283d52..0000000000 --- a/plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-6.5.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -72f0172cf947ab563a7c8166855cf7cbdfe33136
\ No newline at end of file diff --git a/plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-7.0.0-snapshot-89f6d17.jar.sha1 b/plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-7.0.0-snapshot-89f6d17.jar.sha1 new file mode 100644 index 0000000000..973047cdf8 --- /dev/null +++ b/plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-7.0.0-snapshot-89f6d17.jar.sha1 @@ -0,0 +1 @@ +01fe11b45d9f6a68ef1e9994bebd81d26632efc5
\ No newline at end of file diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search_shards/10_basic.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/search_shards/10_basic.yaml index d48d508875..42189883b1 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search_shards/10_basic.yaml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search_shards/10_basic.yaml @@ -76,6 +76,5 @@ - lte: { indices.test_index.filter.bool.should.1.term.field.boost: 1.0 } - gte: { indices.test_index.filter.bool.should.1.term.field.boost: 1.0 } - match: { indices.test_index.filter.bool.adjust_pure_negative: true} - - match: { indices.test_index.filter.bool.disable_coord: false} - lte: { indices.test_index.filter.bool.boost: 1.0 } - gte: { indices.test_index.filter.bool.boost: 1.0 } diff --git a/test/framework/src/main/java/org/elasticsearch/AnalysisFactoryTestCase.java b/test/framework/src/main/java/org/elasticsearch/AnalysisFactoryTestCase.java index 76d33a6c42..83f955296b 100644 --- a/test/framework/src/main/java/org/elasticsearch/AnalysisFactoryTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/AnalysisFactoryTestCase.java @@ -285,6 +285,8 @@ public class AnalysisFactoryTestCase extends ESTestCase { .put("fingerprint", Void.class) // for tee-sinks .put("daterecognizer", Void.class) + // for token filters that generate bad offsets, which are now rejected since Lucene 7 + .put("fixbrokenoffsets", Void.class) .immutableMap(); diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java index 146e62cc61..fab12d2606 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java @@ -37,7 +37,6 @@ import org.apache.logging.log4j.core.config.Configurator; import org.apache.logging.log4j.status.StatusConsoleListener; import org.apache.logging.log4j.status.StatusData; import org.apache.logging.log4j.status.StatusLogger; -import org.apache.lucene.uninverting.UninvertingReader; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.LuceneTestCase.SuppressCodecs; import org.apache.lucene.util.TestRuleMarkFailure; @@ -387,9 +386,6 @@ public abstract class ESTestCase extends LuceneTestCase { protected static void checkStaticState() throws Exception { MockPageCacheRecycler.ensureAllPagesAreReleased(); MockBigArrays.ensureAllArraysAreReleased(); - // field cache should NEVER get loaded. - String[] entries = UninvertingReader.getUninvertedStats(); - assertEquals("fieldcache must never be used, got=" + Arrays.toString(entries), 0, entries.length); // ensure no one changed the status logger level on us assertThat(StatusLogger.getLogger().getLevel(), equalTo(Level.WARN)); diff --git a/test/framework/src/main/java/org/elasticsearch/test/FieldMaskingReader.java b/test/framework/src/main/java/org/elasticsearch/test/FieldMaskingReader.java index 5ce620166c..3aba68868f 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/FieldMaskingReader.java +++ b/test/framework/src/main/java/org/elasticsearch/test/FieldMaskingReader.java @@ -21,7 +21,9 @@ package org.elasticsearch.test; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.FieldFilterLeafReader; import org.apache.lucene.index.FilterDirectoryReader; +import org.apache.lucene.index.FilterLeafReader; import org.apache.lucene.index.LeafReader; +import org.apache.lucene.index.IndexReader.CacheHelper; import java.io.IOException; import java.util.Collections; @@ -32,7 +34,24 @@ public class FieldMaskingReader extends FilterDirectoryReader { super(in, new FilterDirectoryReader.SubReaderWrapper() { @Override public LeafReader wrap(LeafReader reader) { - return new FieldFilterLeafReader(reader, Collections.singleton(field), true); + return new FilterLeafReader(new FieldFilterLeafReader(reader, Collections.singleton(field), true)) { + + // FieldFilterLeafReader does not forward cache helpers + // since it considers it is illegal because of the fact + // that it changes the content of the index. However we + // want this behavior for tests, and security plugins + // are careful to only use the cache when it's valid + + @Override + public CacheHelper getReaderCacheHelper() { + return reader.getReaderCacheHelper(); + } + + @Override + public CacheHelper getCoreCacheHelper() { + return reader.getCoreCacheHelper(); + } + }; } }); this.field = field; @@ -45,7 +64,7 @@ public class FieldMaskingReader extends FilterDirectoryReader { } @Override - public Object getCoreCacheKey() { - return in.getCoreCacheKey(); + public CacheHelper getReaderCacheHelper() { + return in.getReaderCacheHelper(); } }
\ No newline at end of file diff --git a/test/framework/src/main/java/org/elasticsearch/test/engine/MockEngineSupport.java b/test/framework/src/main/java/org/elasticsearch/test/engine/MockEngineSupport.java index a7fc61e691..4c5bd0d326 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/engine/MockEngineSupport.java +++ b/test/framework/src/main/java/org/elasticsearch/test/engine/MockEngineSupport.java @@ -180,11 +180,6 @@ public final class MockEngineSupport { this.subReaderWrapper = subReaderWrapper; } - @Override - public Object getCoreCacheKey() { - return in.getCoreCacheKey(); - } - } public Engine.Searcher wrapSearcher(String source, Engine.Searcher engineSearcher, IndexSearcher searcher, SearcherManager manager) { diff --git a/test/framework/src/main/java/org/elasticsearch/test/engine/ThrowingLeafReaderWrapper.java b/test/framework/src/main/java/org/elasticsearch/test/engine/ThrowingLeafReaderWrapper.java index 9da5d7515c..314f1b5285 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/engine/ThrowingLeafReaderWrapper.java +++ b/test/framework/src/main/java/org/elasticsearch/test/engine/ThrowingLeafReaderWrapper.java @@ -195,4 +195,15 @@ public class ThrowingLeafReaderWrapper extends FilterLeafReader { thrower.maybeThrow(Flags.Norms); return super.getNormValues(field); } + + + @Override + public CacheHelper getCoreCacheHelper() { + return in.getCoreCacheHelper(); + } + + @Override + public CacheHelper getReaderCacheHelper() { + return in.getReaderCacheHelper(); + } } |