Added tests for toXContent and fromXContent for IncludeExclude class.

New REST test revealed an issue with inconsistent hashing in partitioned
term tests which is also fixed in this change.

Closes #22102

2 files changed, 181 insertions, 10 deletions

diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/support/IncludeExclude.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/support/IncludeExclude.java
index ea4797780c..725a8c437c 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/support/IncludeExclude.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/support/IncludeExclude.java
@@ -29,6 +29,7 @@ import org.apache.lucene.index.TermsEnum;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.LongBitSet;
 import org.apache.lucene.util.NumericUtils;
+import org.apache.lucene.util.StringHelper;
 import org.apache.lucene.util.automaton.Automata;
 import org.apache.lucene.util.automaton.Automaton;
 import org.apache.lucene.util.automaton.ByteRunAutomaton;
@@ -49,7 +50,6 @@ import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.search.DocValueFormat;
 
 import java.io.IOException;
-import java.nio.ByteBuffer;
 import java.util.HashSet;
 import java.util.Objects;
 import java.util.Set;
@@ -66,6 +66,10 @@ public class IncludeExclude implements Writeable, ToXContent {
     public static final ParseField PATTERN_FIELD = new ParseField("pattern");
     public static final ParseField PARTITION_FIELD = new ParseField("partition");
     public static final ParseField NUM_PARTITIONS_FIELD = new ParseField("num_partitions");
+    // Needed to add this seed for a deterministic term hashing policy
+    // otherwise tests fail to get expected results and worse, shards 
+    // can disagree on which terms hash to the required partition. 
+    private static final int HASH_PARTITIONING_SEED = 31;
 
     // for parsing purposes only
     // TODO: move all aggs to the same package so that this stuff could be pkg-private
@@ -196,7 +200,7 @@ public class IncludeExclude implements Writeable, ToXContent {
     class PartitionedStringFilter extends StringFilter {
         @Override
         public boolean accept(BytesRef value) {
-            return Math.floorMod(value.hashCode(), incNumPartitions) == incZeroBasedPartition;
+            return Math.floorMod(StringHelper.murmurhash3_x86_32(value, HASH_PARTITIONING_SEED), incNumPartitions) == incZeroBasedPartition;
         }
     }
 
@@ -252,7 +256,7 @@ public class IncludeExclude implements Writeable, ToXContent {
 
             BytesRef term = termEnum.next();
             while (term != null) {
-                if (Math.floorMod(term.hashCode(), incNumPartitions) == incZeroBasedPartition) {
+                if (Math.floorMod(StringHelper.murmurhash3_x86_32(term, HASH_PARTITIONING_SEED), incNumPartitions) == incZeroBasedPartition) {
                     acceptedGlobalOrdinals.set(termEnum.ord());
                 }
                 term = termEnum.next();
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/support/IncludeExcludeTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/support/IncludeExcludeTests.java
index 8929417c26..52233fc874 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/support/IncludeExcludeTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/support/IncludeExcludeTests.java
@@ -24,6 +24,15 @@ import org.apache.lucene.index.RandomAccessOrds;
 import org.apache.lucene.index.SortedSetDocValues;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.LongBitSet;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.ParseFieldMatcher;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.index.query.QueryParseContext;
+import org.elasticsearch.indices.query.IndicesQueriesRegistry;
 import org.elasticsearch.search.DocValueFormat;
 import org.elasticsearch.search.aggregations.bucket.terms.support.IncludeExclude;
 import org.elasticsearch.search.aggregations.bucket.terms.support.IncludeExclude.OrdinalsFilter;
@@ -34,6 +43,9 @@ import java.util.Collections;
 import java.util.TreeSet;
 
 public class IncludeExcludeTests extends ESTestCase {
+    
+    private final ParseFieldMatcher parseFieldMatcher = ParseFieldMatcher.STRICT;
+    private final IndicesQueriesRegistry queriesRegistry = new IndicesQueriesRegistry();    
 
     public void testEmptyTermsWithOrds() throws IOException {
         IncludeExclude inexcl = new IncludeExclude(
@@ -125,13 +137,168 @@ public class IncludeExcludeTests extends ESTestCase {
         assertFalse(acceptedOrds.get(0));
     }
 
-    public void testEquals() {
-        assertEquals(new IncludeExclude(3, 20), new IncludeExclude(3, 20));
-        assertEquals(new IncludeExclude(3, 20).hashCode(), new IncludeExclude(3, 20).hashCode());
-        assertFalse(new IncludeExclude(3, 20).equals(new IncludeExclude(4, 20)));
-        assertTrue(new IncludeExclude(3, 20).hashCode() != new IncludeExclude(4, 20).hashCode());
-        assertFalse(new IncludeExclude(3, 20).equals(new IncludeExclude(3, 21)));
-        assertTrue(new IncludeExclude(3, 20).hashCode() != new IncludeExclude(3, 21).hashCode());
+    public void testPartitionedEquals() throws IOException {
+        IncludeExclude serialized = serialize(new IncludeExclude(3, 20), IncludeExclude.INCLUDE_FIELD);
+        assertFalse(serialized.isRegexBased());
+        assertTrue(serialized.isPartitionBased());
+
+        IncludeExclude same = new IncludeExclude(3, 20);
+        assertEquals(serialized, same);
+        assertEquals(serialized.hashCode(), same.hashCode());
+
+        IncludeExclude differentParam1 = new IncludeExclude(4, 20);
+        assertFalse(serialized.equals(differentParam1));
+        assertTrue(serialized.hashCode() != differentParam1.hashCode());
+
+        IncludeExclude differentParam2 = new IncludeExclude(3, 21);
+        assertFalse(serialized.equals(differentParam2));
+        assertTrue(serialized.hashCode() != differentParam2.hashCode());
+    }
+
+    public void testExactIncludeValuesEquals() throws IOException {
+        String[] incValues = { "a", "b" };
+        String[] differentIncValues = { "a", "c" };
+        IncludeExclude serialized = serialize(new IncludeExclude(incValues, null), IncludeExclude.INCLUDE_FIELD);
+        assertFalse(serialized.isPartitionBased());
+        assertFalse(serialized.isRegexBased());
+
+        IncludeExclude same = new IncludeExclude(incValues, null);
+        assertEquals(serialized, same);
+        assertEquals(serialized.hashCode(), same.hashCode());
+
+        IncludeExclude different = new IncludeExclude(differentIncValues, null);
+        assertFalse(serialized.equals(different));
+        assertTrue(serialized.hashCode() != different.hashCode());
+    }
+
+    public void testExactExcludeValuesEquals() throws IOException {
+        String[] excValues = { "a", "b" };
+        String[] differentExcValues = { "a", "c" };
+        IncludeExclude serialized = serialize(new IncludeExclude(null, excValues), IncludeExclude.EXCLUDE_FIELD);
+        assertFalse(serialized.isPartitionBased());
+        assertFalse(serialized.isRegexBased());
+
+        IncludeExclude same = new IncludeExclude(null, excValues);
+        assertEquals(serialized, same);
+        assertEquals(serialized.hashCode(), same.hashCode());
+
+        IncludeExclude different = new IncludeExclude(null, differentExcValues);
+        assertFalse(serialized.equals(different));
+        assertTrue(serialized.hashCode() != different.hashCode());
+    }
+
+    public void testRegexInclude() throws IOException {
+        String incRegex = "foo.*";
+        String differentRegex = "bar.*";
+        IncludeExclude serialized = serialize(new IncludeExclude(incRegex, null), IncludeExclude.INCLUDE_FIELD);
+        assertFalse(serialized.isPartitionBased());
+        assertTrue(serialized.isRegexBased());
+
+        IncludeExclude same = new IncludeExclude(incRegex, null);
+        assertEquals(serialized, same);
+        assertEquals(serialized.hashCode(), same.hashCode());
+
+        IncludeExclude different = new IncludeExclude(differentRegex, null);
+        assertFalse(serialized.equals(different));
+        assertTrue(serialized.hashCode() != different.hashCode());
+    }
+
+    public void testRegexExclude() throws IOException {
+        String excRegex = "foo.*";
+        String differentRegex = "bar.*";
+        IncludeExclude serialized = serialize(new IncludeExclude(null, excRegex), IncludeExclude.EXCLUDE_FIELD);
+        assertFalse(serialized.isPartitionBased());
+        assertTrue(serialized.isRegexBased());
+
+        IncludeExclude same = new IncludeExclude(null, excRegex);
+        assertEquals(serialized, same);
+        assertEquals(serialized.hashCode(), same.hashCode());
+
+        IncludeExclude different = new IncludeExclude(null, differentRegex);
+        assertFalse(serialized.equals(different));
+        assertTrue(serialized.hashCode() != different.hashCode());
+    }
+
+    // Serializes/deserializes an IncludeExclude statement with a single clause
+    private IncludeExclude serialize(IncludeExclude incExc, ParseField field) throws IOException {
+        XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values()));
+        if (randomBoolean()) {
+            builder.prettyPrint();
+        }
+        builder.startObject();
+        incExc.toXContent(builder, ToXContent.EMPTY_PARAMS);
+        builder.endObject();
+
+        XContentParser parser = XContentFactory.xContent(builder.bytes()).createParser(builder.bytes());
+        XContentParser.Token token = parser.nextToken();
+        assertEquals(token, XContentParser.Token.START_OBJECT);
+        token = parser.nextToken();
+        assertEquals(token, XContentParser.Token.FIELD_NAME);
+        assertEquals(field.getPreferredName(), parser.currentName());
+        token = parser.nextToken();
+
+        QueryParseContext parseContext = new QueryParseContext(queriesRegistry, parser, ParseFieldMatcher.STRICT);
+        if (field.getPreferredName().equalsIgnoreCase("include")) {
+            return IncludeExclude.parseInclude(parser, parseContext);
+        } else if (field.getPreferredName().equalsIgnoreCase("exclude")) {
+            return IncludeExclude.parseExclude(parser, parseContext);
+        } else {
+            throw new IllegalArgumentException(
+                    "Unexpected field name serialized in test: " + field.getPreferredName());
+        }
+    }
+
+    public void testRegexIncludeAndExclude() throws IOException {
+        String incRegex = "foo.*";
+        String excRegex = "football";
+        String differentExcRegex = "foosball";
+        IncludeExclude serialized = serializeMixedRegex(new IncludeExclude(incRegex, excRegex));
+        assertFalse(serialized.isPartitionBased());
+        assertTrue(serialized.isRegexBased());
+
+        IncludeExclude same = new IncludeExclude(incRegex, excRegex);
+        assertEquals(serialized, same);
+        assertEquals(serialized.hashCode(), same.hashCode());
+
+        IncludeExclude different = new IncludeExclude(incRegex, differentExcRegex);
+        assertFalse(serialized.equals(different));
+        assertTrue(serialized.hashCode() != different.hashCode());
+    }
+
+    // Serializes/deserializes the IncludeExclude statement with include AND
+    // exclude clauses
+    private IncludeExclude serializeMixedRegex(IncludeExclude incExc) throws IOException {
+        XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values()));
+        if (randomBoolean()) {
+            builder.prettyPrint();
+        }
+        builder.startObject();
+        incExc.toXContent(builder, ToXContent.EMPTY_PARAMS);
+        builder.endObject();
+
+        XContentParser parser = XContentFactory.xContent(builder.bytes()).createParser(builder.bytes());
+        QueryParseContext parseContext = new QueryParseContext(queriesRegistry, parser, parseFieldMatcher);
+        XContentParser.Token token = parser.nextToken();
+        assertEquals(token, XContentParser.Token.START_OBJECT);
+
+        IncludeExclude inc = null;
+        IncludeExclude exc = null;
+        while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+            assertEquals(XContentParser.Token.FIELD_NAME, token);
+            if (parseFieldMatcher.match(parser.currentName(), IncludeExclude.INCLUDE_FIELD)) {
+                token = parser.nextToken();
+                inc = IncludeExclude.parseInclude(parser, parseContext);
+            } else if (parseFieldMatcher.match(parser.currentName(), IncludeExclude.EXCLUDE_FIELD)) {
+                token = parser.nextToken();
+                exc = IncludeExclude.parseExclude(parser, parseContext);
+            } else {
+                throw new IllegalArgumentException("Unexpected field name serialized in test: " + parser.currentName());
+            }
+        }
+        assertNotNull(inc);
+        assertNotNull(exc);
+        // Include and Exclude clauses are parsed independently and then merged
+        return IncludeExclude.merge(inc, exc);
     }
 
 }