diff options
Diffstat (limited to 'core')
2 files changed, 181 insertions, 10 deletions
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/support/IncludeExclude.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/support/IncludeExclude.java index ea4797780c..725a8c437c 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/support/IncludeExclude.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/support/IncludeExclude.java @@ -29,6 +29,7 @@ import org.apache.lucene.index.TermsEnum; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.LongBitSet; import org.apache.lucene.util.NumericUtils; +import org.apache.lucene.util.StringHelper; import org.apache.lucene.util.automaton.Automata; import org.apache.lucene.util.automaton.Automaton; import org.apache.lucene.util.automaton.ByteRunAutomaton; @@ -49,7 +50,6 @@ import org.elasticsearch.index.query.QueryParseContext; import org.elasticsearch.search.DocValueFormat; import java.io.IOException; -import java.nio.ByteBuffer; import java.util.HashSet; import java.util.Objects; import java.util.Set; @@ -66,6 +66,10 @@ public class IncludeExclude implements Writeable, ToXContent { public static final ParseField PATTERN_FIELD = new ParseField("pattern"); public static final ParseField PARTITION_FIELD = new ParseField("partition"); public static final ParseField NUM_PARTITIONS_FIELD = new ParseField("num_partitions"); + // Needed to add this seed for a deterministic term hashing policy + // otherwise tests fail to get expected results and worse, shards + // can disagree on which terms hash to the required partition. + private static final int HASH_PARTITIONING_SEED = 31; // for parsing purposes only // TODO: move all aggs to the same package so that this stuff could be pkg-private @@ -196,7 +200,7 @@ public class IncludeExclude implements Writeable, ToXContent { class PartitionedStringFilter extends StringFilter { @Override public boolean accept(BytesRef value) { - return Math.floorMod(value.hashCode(), incNumPartitions) == incZeroBasedPartition; + return Math.floorMod(StringHelper.murmurhash3_x86_32(value, HASH_PARTITIONING_SEED), incNumPartitions) == incZeroBasedPartition; } } @@ -252,7 +256,7 @@ public class IncludeExclude implements Writeable, ToXContent { BytesRef term = termEnum.next(); while (term != null) { - if (Math.floorMod(term.hashCode(), incNumPartitions) == incZeroBasedPartition) { + if (Math.floorMod(StringHelper.murmurhash3_x86_32(term, HASH_PARTITIONING_SEED), incNumPartitions) == incZeroBasedPartition) { acceptedGlobalOrdinals.set(termEnum.ord()); } term = termEnum.next(); diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/support/IncludeExcludeTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/support/IncludeExcludeTests.java index 8929417c26..52233fc874 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/support/IncludeExcludeTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/support/IncludeExcludeTests.java @@ -24,6 +24,15 @@ import org.apache.lucene.index.RandomAccessOrds; import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.LongBitSet; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.ParseFieldMatcher; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.index.query.QueryParseContext; +import org.elasticsearch.indices.query.IndicesQueriesRegistry; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.aggregations.bucket.terms.support.IncludeExclude; import org.elasticsearch.search.aggregations.bucket.terms.support.IncludeExclude.OrdinalsFilter; @@ -34,6 +43,9 @@ import java.util.Collections; import java.util.TreeSet; public class IncludeExcludeTests extends ESTestCase { + + private final ParseFieldMatcher parseFieldMatcher = ParseFieldMatcher.STRICT; + private final IndicesQueriesRegistry queriesRegistry = new IndicesQueriesRegistry(); public void testEmptyTermsWithOrds() throws IOException { IncludeExclude inexcl = new IncludeExclude( @@ -125,13 +137,168 @@ public class IncludeExcludeTests extends ESTestCase { assertFalse(acceptedOrds.get(0)); } - public void testEquals() { - assertEquals(new IncludeExclude(3, 20), new IncludeExclude(3, 20)); - assertEquals(new IncludeExclude(3, 20).hashCode(), new IncludeExclude(3, 20).hashCode()); - assertFalse(new IncludeExclude(3, 20).equals(new IncludeExclude(4, 20))); - assertTrue(new IncludeExclude(3, 20).hashCode() != new IncludeExclude(4, 20).hashCode()); - assertFalse(new IncludeExclude(3, 20).equals(new IncludeExclude(3, 21))); - assertTrue(new IncludeExclude(3, 20).hashCode() != new IncludeExclude(3, 21).hashCode()); + public void testPartitionedEquals() throws IOException { + IncludeExclude serialized = serialize(new IncludeExclude(3, 20), IncludeExclude.INCLUDE_FIELD); + assertFalse(serialized.isRegexBased()); + assertTrue(serialized.isPartitionBased()); + + IncludeExclude same = new IncludeExclude(3, 20); + assertEquals(serialized, same); + assertEquals(serialized.hashCode(), same.hashCode()); + + IncludeExclude differentParam1 = new IncludeExclude(4, 20); + assertFalse(serialized.equals(differentParam1)); + assertTrue(serialized.hashCode() != differentParam1.hashCode()); + + IncludeExclude differentParam2 = new IncludeExclude(3, 21); + assertFalse(serialized.equals(differentParam2)); + assertTrue(serialized.hashCode() != differentParam2.hashCode()); + } + + public void testExactIncludeValuesEquals() throws IOException { + String[] incValues = { "a", "b" }; + String[] differentIncValues = { "a", "c" }; + IncludeExclude serialized = serialize(new IncludeExclude(incValues, null), IncludeExclude.INCLUDE_FIELD); + assertFalse(serialized.isPartitionBased()); + assertFalse(serialized.isRegexBased()); + + IncludeExclude same = new IncludeExclude(incValues, null); + assertEquals(serialized, same); + assertEquals(serialized.hashCode(), same.hashCode()); + + IncludeExclude different = new IncludeExclude(differentIncValues, null); + assertFalse(serialized.equals(different)); + assertTrue(serialized.hashCode() != different.hashCode()); + } + + public void testExactExcludeValuesEquals() throws IOException { + String[] excValues = { "a", "b" }; + String[] differentExcValues = { "a", "c" }; + IncludeExclude serialized = serialize(new IncludeExclude(null, excValues), IncludeExclude.EXCLUDE_FIELD); + assertFalse(serialized.isPartitionBased()); + assertFalse(serialized.isRegexBased()); + + IncludeExclude same = new IncludeExclude(null, excValues); + assertEquals(serialized, same); + assertEquals(serialized.hashCode(), same.hashCode()); + + IncludeExclude different = new IncludeExclude(null, differentExcValues); + assertFalse(serialized.equals(different)); + assertTrue(serialized.hashCode() != different.hashCode()); + } + + public void testRegexInclude() throws IOException { + String incRegex = "foo.*"; + String differentRegex = "bar.*"; + IncludeExclude serialized = serialize(new IncludeExclude(incRegex, null), IncludeExclude.INCLUDE_FIELD); + assertFalse(serialized.isPartitionBased()); + assertTrue(serialized.isRegexBased()); + + IncludeExclude same = new IncludeExclude(incRegex, null); + assertEquals(serialized, same); + assertEquals(serialized.hashCode(), same.hashCode()); + + IncludeExclude different = new IncludeExclude(differentRegex, null); + assertFalse(serialized.equals(different)); + assertTrue(serialized.hashCode() != different.hashCode()); + } + + public void testRegexExclude() throws IOException { + String excRegex = "foo.*"; + String differentRegex = "bar.*"; + IncludeExclude serialized = serialize(new IncludeExclude(null, excRegex), IncludeExclude.EXCLUDE_FIELD); + assertFalse(serialized.isPartitionBased()); + assertTrue(serialized.isRegexBased()); + + IncludeExclude same = new IncludeExclude(null, excRegex); + assertEquals(serialized, same); + assertEquals(serialized.hashCode(), same.hashCode()); + + IncludeExclude different = new IncludeExclude(null, differentRegex); + assertFalse(serialized.equals(different)); + assertTrue(serialized.hashCode() != different.hashCode()); + } + + // Serializes/deserializes an IncludeExclude statement with a single clause + private IncludeExclude serialize(IncludeExclude incExc, ParseField field) throws IOException { + XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values())); + if (randomBoolean()) { + builder.prettyPrint(); + } + builder.startObject(); + incExc.toXContent(builder, ToXContent.EMPTY_PARAMS); + builder.endObject(); + + XContentParser parser = XContentFactory.xContent(builder.bytes()).createParser(builder.bytes()); + XContentParser.Token token = parser.nextToken(); + assertEquals(token, XContentParser.Token.START_OBJECT); + token = parser.nextToken(); + assertEquals(token, XContentParser.Token.FIELD_NAME); + assertEquals(field.getPreferredName(), parser.currentName()); + token = parser.nextToken(); + + QueryParseContext parseContext = new QueryParseContext(queriesRegistry, parser, ParseFieldMatcher.STRICT); + if (field.getPreferredName().equalsIgnoreCase("include")) { + return IncludeExclude.parseInclude(parser, parseContext); + } else if (field.getPreferredName().equalsIgnoreCase("exclude")) { + return IncludeExclude.parseExclude(parser, parseContext); + } else { + throw new IllegalArgumentException( + "Unexpected field name serialized in test: " + field.getPreferredName()); + } + } + + public void testRegexIncludeAndExclude() throws IOException { + String incRegex = "foo.*"; + String excRegex = "football"; + String differentExcRegex = "foosball"; + IncludeExclude serialized = serializeMixedRegex(new IncludeExclude(incRegex, excRegex)); + assertFalse(serialized.isPartitionBased()); + assertTrue(serialized.isRegexBased()); + + IncludeExclude same = new IncludeExclude(incRegex, excRegex); + assertEquals(serialized, same); + assertEquals(serialized.hashCode(), same.hashCode()); + + IncludeExclude different = new IncludeExclude(incRegex, differentExcRegex); + assertFalse(serialized.equals(different)); + assertTrue(serialized.hashCode() != different.hashCode()); + } + + // Serializes/deserializes the IncludeExclude statement with include AND + // exclude clauses + private IncludeExclude serializeMixedRegex(IncludeExclude incExc) throws IOException { + XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values())); + if (randomBoolean()) { + builder.prettyPrint(); + } + builder.startObject(); + incExc.toXContent(builder, ToXContent.EMPTY_PARAMS); + builder.endObject(); + + XContentParser parser = XContentFactory.xContent(builder.bytes()).createParser(builder.bytes()); + QueryParseContext parseContext = new QueryParseContext(queriesRegistry, parser, parseFieldMatcher); + XContentParser.Token token = parser.nextToken(); + assertEquals(token, XContentParser.Token.START_OBJECT); + + IncludeExclude inc = null; + IncludeExclude exc = null; + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + assertEquals(XContentParser.Token.FIELD_NAME, token); + if (parseFieldMatcher.match(parser.currentName(), IncludeExclude.INCLUDE_FIELD)) { + token = parser.nextToken(); + inc = IncludeExclude.parseInclude(parser, parseContext); + } else if (parseFieldMatcher.match(parser.currentName(), IncludeExclude.EXCLUDE_FIELD)) { + token = parser.nextToken(); + exc = IncludeExclude.parseExclude(parser, parseContext); + } else { + throw new IllegalArgumentException("Unexpected field name serialized in test: " + parser.currentName()); + } + } + assertNotNull(inc); + assertNotNull(exc); + // Include and Exclude clauses are parsed independently and then merged + return IncludeExclude.merge(inc, exc); } } |