[Test] Extending checks for Suggestion parsing (#25132)

When parsing responses we should be ignoring any new unknown fields or inner
objects in most cases to be forward compatible with changes in core on the
client side. This change adds test for this for Suggestions and its various
subclasses to check if we are able to ignore new fields and objects in the
xContent.

6 files changed, 103 insertions, 13 deletions

diff --git a/core/src/test/java/org/elasticsearch/search/suggest/CompletionSuggestionOptionTests.java b/core/src/test/java/org/elasticsearch/search/suggest/CompletionSuggestionOptionTests.java
index 18a24e0011..df4c6898cc 100644
--- a/core/src/test/java/org/elasticsearch/search/suggest/CompletionSuggestionOptionTests.java
+++ b/core/src/test/java/org/elasticsearch/search/suggest/CompletionSuggestionOptionTests.java
@@ -36,8 +36,10 @@ import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Map;
 import java.util.Set;
+import java.util.function.Predicate;
 
 import static org.elasticsearch.common.xcontent.XContentHelper.toXContent;
+import static org.elasticsearch.test.XContentTestUtils.insertRandomFields;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertToXContentEquivalent;
 
 public class CompletionSuggestionOptionTests extends ESTestCase {
@@ -67,17 +69,31 @@ public class CompletionSuggestionOptionTests extends ESTestCase {
     }
 
     public void testFromXContent() throws IOException {
+        doTestFromXContent(false);
+    }
+
+    public void testFromXContentWithRandomFields() throws IOException {
+        doTestFromXContent(true);
+    }
+
+    private void doTestFromXContent(boolean addRandomFields) throws IOException {
         Option option = createTestItem();
         XContentType xContentType = randomFrom(XContentType.values());
         boolean humanReadable = randomBoolean();
         BytesReference originalBytes = toShuffledXContent(option, xContentType, ToXContent.EMPTY_PARAMS, humanReadable);
-        if (randomBoolean()) {
-            try (XContentParser parser = createParser(xContentType.xContent(), originalBytes)) {
-                originalBytes = shuffleXContent(parser, randomBoolean()).bytes();
-            }
+        BytesReference mutated;
+        if (addRandomFields) {
+            // "contexts" is an object consisting of key/array pairs, we shouldn't add anything random there
+            // also there can be inner search hits fields inside this option, we need to exclude another couple of paths
+            // where we cannot add random stuff
+            Predicate<String> excludeFilter = (path) -> (path.endsWith(CompletionSuggestion.Entry.Option.CONTEXTS.getPreferredName())
+                    || path.endsWith("highlight") || path.endsWith("fields") || path.contains("_source") || path.contains("inner_hits"));
+            mutated = insertRandomFields(xContentType, originalBytes, excludeFilter, random());
+        } else {
+            mutated = originalBytes;
         }
         Option parsed;
-        try (XContentParser parser = createParser(xContentType.xContent(), originalBytes)) {
+        try (XContentParser parser = createParser(xContentType.xContent(), mutated)) {
             parsed = Option.fromXContent(parser);
             assertNull(parser.nextToken());
         }
diff --git a/core/src/test/java/org/elasticsearch/search/suggest/SuggestTests.java b/core/src/test/java/org/elasticsearch/search/suggest/SuggestTests.java
index c565836adb..8c938caa47 100644
--- a/core/src/test/java/org/elasticsearch/search/suggest/SuggestTests.java
+++ b/core/src/test/java/org/elasticsearch/search/suggest/SuggestTests.java
@@ -90,7 +90,7 @@ public class SuggestTests extends ESTestCase {
         Suggest suggest = createTestItem();
         XContentType xContentType = randomFrom(XContentType.values());
         boolean humanReadable = randomBoolean();
-        BytesReference originalBytes = toXContent(suggest, xContentType, params, humanReadable);
+        BytesReference originalBytes = toShuffledXContent(suggest, xContentType, params, humanReadable);
         Suggest parsed;
         try (XContentParser parser = createParser(xContentType.xContent(), originalBytes)) {
             ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser::getTokenLocation);
diff --git a/core/src/test/java/org/elasticsearch/search/suggest/SuggestionEntryTests.java b/core/src/test/java/org/elasticsearch/search/suggest/SuggestionEntryTests.java
index 770fd2f6e6..46971a5537 100644
--- a/core/src/test/java/org/elasticsearch/search/suggest/SuggestionEntryTests.java
+++ b/core/src/test/java/org/elasticsearch/search/suggest/SuggestionEntryTests.java
@@ -36,10 +36,12 @@ import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.function.Function;
+import java.util.function.Predicate;
 import java.util.function.Supplier;
 
 import static org.elasticsearch.common.xcontent.XContentHelper.toXContent;
 import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken;
+import static org.elasticsearch.test.XContentTestUtils.insertRandomFields;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertToXContentEquivalent;
 
 public class SuggestionEntryTests extends ESTestCase {
@@ -80,15 +82,35 @@ public class SuggestionEntryTests extends ESTestCase {
         return entry;
     }
 
-    @SuppressWarnings("unchecked")
     public void testFromXContent() throws IOException {
+        doTestFromXContent(false);
+    }
+
+    public void testFromXContentWithRandomFields() throws IOException {
+        doTestFromXContent(true);
+    }
+
+    @SuppressWarnings("unchecked")
+    private void doTestFromXContent(boolean addRandomFields) throws IOException {
         for (Class<? extends Entry> entryType : ENTRY_PARSERS.keySet()) {
             Entry<Option> entry = createTestItem(entryType);
             XContentType xContentType = randomFrom(XContentType.values());
             boolean humanReadable = randomBoolean();
             BytesReference originalBytes = toShuffledXContent(entry, xContentType, ToXContent.EMPTY_PARAMS, humanReadable);
+            BytesReference mutated;
+            if (addRandomFields) {
+                // "contexts" is an object consisting of key/array pairs, we shouldn't add anything random there
+                // also there can be inner search hits fields inside this option, we need to exclude another couple of paths
+                // where we cannot add random stuff
+                Predicate<String> excludeFilter = (
+                        path) -> (path.endsWith(CompletionSuggestion.Entry.Option.CONTEXTS.getPreferredName()) || path.endsWith("highlight")
+                                || path.endsWith("fields") || path.contains("_source") || path.contains("inner_hits"));
+                mutated = insertRandomFields(xContentType, originalBytes, excludeFilter, random());
+            } else {
+                mutated = originalBytes;
+            }
             Entry<Option> parsed;
-            try (XContentParser parser = createParser(xContentType.xContent(), originalBytes)) {
+            try (XContentParser parser = createParser(xContentType.xContent(), mutated)) {
                 ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser::getTokenLocation);
                 parsed = ENTRY_PARSERS.get(entry.getClass()).apply(parser);
                 assertEquals(XContentParser.Token.END_OBJECT, parser.currentToken());
diff --git a/core/src/test/java/org/elasticsearch/search/suggest/SuggestionOptionTests.java b/core/src/test/java/org/elasticsearch/search/suggest/SuggestionOptionTests.java
index 32a3d92584..7385dd38f4 100644
--- a/core/src/test/java/org/elasticsearch/search/suggest/SuggestionOptionTests.java
+++ b/core/src/test/java/org/elasticsearch/search/suggest/SuggestionOptionTests.java
@@ -31,6 +31,7 @@ import java.io.IOException;
 
 import static org.elasticsearch.common.xcontent.XContentHelper.toXContent;
 import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken;
+import static org.elasticsearch.test.XContentTestUtils.insertRandomFields;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertToXContentEquivalent;
 
 public class SuggestionOptionTests extends ESTestCase {
@@ -44,12 +45,26 @@ public class SuggestionOptionTests extends ESTestCase {
     }
 
     public void testFromXContent() throws IOException {
+        doTestFromXContent(false);
+    }
+
+    public void testFromXContentWithRandomFields() throws IOException {
+        doTestFromXContent(true);
+    }
+
+    private void doTestFromXContent(boolean addRandomFields) throws IOException {
         Option option = createTestItem();
         XContentType xContentType = randomFrom(XContentType.values());
         boolean humanReadable = randomBoolean();
         BytesReference originalBytes = toShuffledXContent(option, xContentType, ToXContent.EMPTY_PARAMS, humanReadable);
+        BytesReference mutated;
+        if (addRandomFields) {
+            mutated = insertRandomFields(xContentType, originalBytes, null, random());
+        } else {
+            mutated = originalBytes;
+        }
         Option parsed;
-        try (XContentParser parser = createParser(xContentType.xContent(), originalBytes)) {
+        try (XContentParser parser = createParser(xContentType.xContent(), mutated)) {
             ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser::getTokenLocation);
             parsed = Option.fromXContent(parser);
             assertEquals(XContentParser.Token.END_OBJECT, parser.currentToken());
diff --git a/core/src/test/java/org/elasticsearch/search/suggest/SuggestionTests.java b/core/src/test/java/org/elasticsearch/search/suggest/SuggestionTests.java
index ddec50b8d4..fbf6a88922 100644
--- a/core/src/test/java/org/elasticsearch/search/suggest/SuggestionTests.java
+++ b/core/src/test/java/org/elasticsearch/search/suggest/SuggestionTests.java
@@ -41,10 +41,12 @@ import java.io.IOException;
 import java.util.Collections;
 import java.util.Map;
 import java.util.Set;
+import java.util.function.Predicate;
 import java.util.function.Supplier;
 
 import static org.elasticsearch.common.xcontent.XContentHelper.toXContent;
 import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken;
+import static org.elasticsearch.test.XContentTestUtils.insertRandomFields;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertToXContentEquivalent;
 
 public class SuggestionTests extends ESTestCase {
@@ -98,16 +100,36 @@ public class SuggestionTests extends ESTestCase {
         return suggestion;
     }
 
-    @SuppressWarnings({ "rawtypes" })
     public void testFromXContent() throws IOException {
+        doTestFromXContent(false);
+    }
+
+    public void testFromXContentWithRandomFields() throws IOException {
+        doTestFromXContent(true);
+    }
+
+    @SuppressWarnings({ "rawtypes" })
+    private void doTestFromXContent(boolean addRandomFields) throws IOException {
         ToXContent.Params params = new ToXContent.MapParams(Collections.singletonMap(RestSearchAction.TYPED_KEYS_PARAM, "true"));
         for (Class<Suggestion<? extends Entry<? extends Option>>> type : SUGGESTION_TYPES) {
             Suggestion suggestion = createTestItem(type);
             XContentType xContentType = randomFrom(XContentType.values());
             boolean humanReadable = randomBoolean();
-            BytesReference originalBytes = toXContent(suggestion, xContentType, params, humanReadable);
+            BytesReference originalBytes = toShuffledXContent(suggestion, xContentType, params, humanReadable);
+            BytesReference mutated;
+            if (addRandomFields) {
+                // - "contexts" is an object consisting of key/array pairs, we shouldn't add anything random there
+                // - there can be inner search hits fields inside this option where we cannot add random stuff
+                // - the root object should be excluded since it contains the named suggestion arrays
+                Predicate<String> excludeFilter = path -> (path.isEmpty()
+                        || path.endsWith(CompletionSuggestion.Entry.Option.CONTEXTS.getPreferredName()) || path.endsWith("highlight")
+                        || path.endsWith("fields") || path.contains("_source") || path.contains("inner_hits"));
+                mutated = insertRandomFields(xContentType, originalBytes, excludeFilter, random());
+            } else {
+                mutated = originalBytes;
+            }
             Suggestion parsed;
-            try (XContentParser parser = createParser(xContentType.xContent(), originalBytes)) {
+            try (XContentParser parser = createParser(xContentType.xContent(), mutated)) {
                 ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser::getTokenLocation);
                 ensureExpectedToken(XContentParser.Token.FIELD_NAME, parser.nextToken(), parser::getTokenLocation);
                 parsed = Suggestion.fromXContent(parser);
diff --git a/core/src/test/java/org/elasticsearch/search/suggest/TermSuggestionOptionTests.java b/core/src/test/java/org/elasticsearch/search/suggest/TermSuggestionOptionTests.java
index 75ced3cb80..f25fe09341 100644
--- a/core/src/test/java/org/elasticsearch/search/suggest/TermSuggestionOptionTests.java
+++ b/core/src/test/java/org/elasticsearch/search/suggest/TermSuggestionOptionTests.java
@@ -31,6 +31,7 @@ import java.io.IOException;
 
 import static org.elasticsearch.common.xcontent.XContentHelper.toXContent;
 import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken;
+import static org.elasticsearch.test.XContentTestUtils.insertRandomFields;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertToXContentEquivalent;
 
 public class TermSuggestionOptionTests extends ESTestCase {
@@ -43,12 +44,26 @@ public class TermSuggestionOptionTests extends ESTestCase {
     }
 
     public void testFromXContent() throws IOException {
+        doTestFromXContent(false);
+    }
+
+    public void testFromXContentWithRandomFields() throws IOException {
+        doTestFromXContent(true);
+    }
+
+    private void doTestFromXContent(boolean addRandomFields) throws IOException {
         Option option = createTestItem();
         XContentType xContentType = randomFrom(XContentType.values());
         boolean humanReadable = randomBoolean();
         BytesReference originalBytes = toShuffledXContent(option, xContentType, ToXContent.EMPTY_PARAMS, humanReadable);
+        BytesReference mutated;
+        if (addRandomFields) {
+            mutated = insertRandomFields(xContentType, originalBytes, null, random());
+        } else {
+            mutated = originalBytes;
+        }
         Option parsed;
-        try (XContentParser parser = createParser(xContentType.xContent(), originalBytes)) {
+        try (XContentParser parser = createParser(xContentType.xContent(), mutated)) {
             ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser::getTokenLocation);
             parsed = Option.fromXContent(parser);
             assertEquals(XContentParser.Token.END_OBJECT, parser.currentToken());