Remove much ceremony from parsing client yaml test suites (#22311)

* Remove a checked exception, replacing it with `ParsingException`.
* Remove all Parser classes for the yaml sections, replacing them with static methods.
* Remove `ClientYamlTestFragmentParser`. Isn't used any more.
* Remove `ClientYamlTestSuiteParseContext`, replacing it with some static utility methods.

I did not rewrite the parsers using `ObjectParser` because I don't think it is worth it right now.

93 files changed, 1373 insertions, 2174 deletions

diff --git a/core/src/test/java/org/elasticsearch/ExceptionSerializationTests.java b/core/src/test/java/org/elasticsearch/ExceptionSerializationTests.java
index 786426822d..f7660e3a2e 100644
--- a/core/src/test/java/org/elasticsearch/ExceptionSerializationTests.java
+++ b/core/src/test/java/org/elasticsearch/ExceptionSerializationTests.java
@@ -121,7 +121,6 @@ public class ExceptionSerializationTests extends ESTestCase {
         final Path startPath = PathUtils.get(ElasticsearchException.class.getProtectionDomain().getCodeSource().getLocation().toURI())
                 .resolve("org").resolve("elasticsearch");
         final Set<? extends Class<?>> ignore = Sets.newHashSet(
-                org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException.class,
                 CancellableThreadsTests.CustomException.class,
                 org.elasticsearch.rest.BytesRestResponseTests.WithHeadersException.class,
                 AbstractClientHeadersTestCase.InternalException.class);
diff --git a/distribution/deb/src/test/java/org/elasticsearch/test/rest/DebClientYamlTestSuiteIT.java b/distribution/deb/src/test/java/org/elasticsearch/test/rest/DebClientYamlTestSuiteIT.java
index a63b304a1d..9a7978f69b 100644
--- a/distribution/deb/src/test/java/org/elasticsearch/test/rest/DebClientYamlTestSuiteIT.java
+++ b/distribution/deb/src/test/java/org/elasticsearch/test/rest/DebClientYamlTestSuiteIT.java
@@ -23,7 +23,6 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
 
 import java.io.IOException;
 
@@ -34,7 +33,7 @@ public class DebClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException, ClientYamlTestParseException {
+    public static Iterable<Object[]> parameters() throws IOException {
         return createParameters();
     }
 }
diff --git a/distribution/integ-test-zip/src/test/java/org/elasticsearch/test/rest/IntegTestZipClientYamlTestSuiteIT.java b/distribution/integ-test-zip/src/test/java/org/elasticsearch/test/rest/IntegTestZipClientYamlTestSuiteIT.java
index c81ff7439f..c3a038392c 100644
--- a/distribution/integ-test-zip/src/test/java/org/elasticsearch/test/rest/IntegTestZipClientYamlTestSuiteIT.java
+++ b/distribution/integ-test-zip/src/test/java/org/elasticsearch/test/rest/IntegTestZipClientYamlTestSuiteIT.java
@@ -23,7 +23,6 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
 
 import java.io.IOException;
 
@@ -34,7 +33,7 @@ public class IntegTestZipClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException, ClientYamlTestParseException {
+    public static Iterable<Object[]> parameters() throws IOException {
         return createParameters();
     }
 }
diff --git a/distribution/rpm/src/test/java/org/elasticsearch/test/rest/RpmClientYamlTestSuiteIT.java b/distribution/rpm/src/test/java/org/elasticsearch/test/rest/RpmClientYamlTestSuiteIT.java
index 9569dfe4d1..86b4932180 100644
--- a/distribution/rpm/src/test/java/org/elasticsearch/test/rest/RpmClientYamlTestSuiteIT.java
+++ b/distribution/rpm/src/test/java/org/elasticsearch/test/rest/RpmClientYamlTestSuiteIT.java
@@ -23,7 +23,6 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
 
 import java.io.IOException;
 
@@ -34,7 +33,7 @@ public class RpmClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException, ClientYamlTestParseException {
+    public static Iterable<Object[]> parameters() throws IOException {
         return createParameters();
     }
 }
diff --git a/distribution/tar/src/test/java/org/elasticsearch/test/rest/TarClientYamlTestSuiteIT.java b/distribution/tar/src/test/java/org/elasticsearch/test/rest/TarClientYamlTestSuiteIT.java
index 0c811c383d..a86e398830 100644
--- a/distribution/tar/src/test/java/org/elasticsearch/test/rest/TarClientYamlTestSuiteIT.java
+++ b/distribution/tar/src/test/java/org/elasticsearch/test/rest/TarClientYamlTestSuiteIT.java
@@ -23,7 +23,6 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
 
 import java.io.IOException;
 
@@ -34,7 +33,7 @@ public class TarClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException, ClientYamlTestParseException {
+    public static Iterable<Object[]> parameters() throws IOException {
         return createParameters();
     }
 }
diff --git a/distribution/zip/src/test/java/org/elasticsearch/test/rest/ZipClientYamlTestSuiteIT.java b/distribution/zip/src/test/java/org/elasticsearch/test/rest/ZipClientYamlTestSuiteIT.java
index 52581c8e76..93c31c4ab6 100644
--- a/distribution/zip/src/test/java/org/elasticsearch/test/rest/ZipClientYamlTestSuiteIT.java
+++ b/distribution/zip/src/test/java/org/elasticsearch/test/rest/ZipClientYamlTestSuiteIT.java
@@ -23,7 +23,6 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
 
 import java.io.IOException;
 
@@ -34,7 +33,7 @@ public class ZipClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException, ClientYamlTestParseException {
+    public static Iterable<Object[]> parameters() throws IOException {
         return createParameters();
     }
 }
diff --git a/docs/src/test/java/org/elasticsearch/smoketest/DocsClientYamlTestSuiteIT.java b/docs/src/test/java/org/elasticsearch/smoketest/DocsClientYamlTestSuiteIT.java
index 87ca5acf1c..a79730339e 100644
--- a/docs/src/test/java/org/elasticsearch/smoketest/DocsClientYamlTestSuiteIT.java
+++ b/docs/src/test/java/org/elasticsearch/smoketest/DocsClientYamlTestSuiteIT.java
@@ -24,7 +24,6 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
 
 import java.io.IOException;
 import java.util.List;
@@ -36,7 +35,7 @@ public class DocsClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException, ClientYamlTestParseException {
+    public static Iterable<Object[]> parameters() throws IOException {
         return ESClientYamlSuiteTestCase.createParameters();
     }
 
diff --git a/modules/aggs-matrix-stats/src/test/java/org/elasticsearch/search/aggregations/matrix/MatrixStatsClientYamlTestSuiteIT.java b/modules/aggs-matrix-stats/src/test/java/org/elasticsearch/search/aggregations/matrix/MatrixStatsClientYamlTestSuiteIT.java
index 11ddd2dfd4..6e11a56d2c 100644
--- a/modules/aggs-matrix-stats/src/test/java/org/elasticsearch/search/aggregations/matrix/MatrixStatsClientYamlTestSuiteIT.java
+++ b/modules/aggs-matrix-stats/src/test/java/org/elasticsearch/search/aggregations/matrix/MatrixStatsClientYamlTestSuiteIT.java
@@ -23,7 +23,6 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
 
 import java.io.IOException;
 
@@ -33,7 +32,7 @@ public class MatrixStatsClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException, ClientYamlTestParseException {
+    public static Iterable<Object[]> parameters() throws IOException {
         return ESClientYamlSuiteTestCase.createParameters();
     }
 }
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/IngestCommonClientYamlTestSuiteIT.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/IngestCommonClientYamlTestSuiteIT.java
index 1b678835c4..4027a75a2d 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/IngestCommonClientYamlTestSuiteIT.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/IngestCommonClientYamlTestSuiteIT.java
@@ -24,7 +24,6 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
 
 import java.io.IOException;
 
@@ -35,7 +34,7 @@ public class IngestCommonClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException, ClientYamlTestParseException {
+    public static Iterable<Object[]> parameters() throws IOException {
         return ESClientYamlSuiteTestCase.createParameters();
     }
 }
diff --git a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/LangExpressionClientYamlTestSuiteIT.java b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/LangExpressionClientYamlTestSuiteIT.java
index 9a30def83e..a20050b80a 100644
--- a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/LangExpressionClientYamlTestSuiteIT.java
+++ b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/LangExpressionClientYamlTestSuiteIT.java
@@ -24,7 +24,6 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
 
 import java.io.IOException;
 
@@ -35,7 +34,7 @@ public class LangExpressionClientYamlTestSuiteIT extends ESClientYamlSuiteTestCa
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException, ClientYamlTestParseException {
+    public static Iterable<Object[]> parameters() throws IOException {
         return ESClientYamlSuiteTestCase.createParameters();
     }
 }
diff --git a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/LangMustacheClientYamlTestSuiteIT.java b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/LangMustacheClientYamlTestSuiteIT.java
index 377fa870c4..72eb9f2ad7 100644
--- a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/LangMustacheClientYamlTestSuiteIT.java
+++ b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/LangMustacheClientYamlTestSuiteIT.java
@@ -24,7 +24,6 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
 
 import java.io.IOException;
 
@@ -35,7 +34,7 @@ public class LangMustacheClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException, ClientYamlTestParseException {
+    public static Iterable<Object[]> parameters() throws IOException {
         return ESClientYamlSuiteTestCase.createParameters();
     }
 }
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/LangPainlessClientYamlTestSuiteIT.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/LangPainlessClientYamlTestSuiteIT.java
index 55d3f1c810..9d055b74cc 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/LangPainlessClientYamlTestSuiteIT.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/LangPainlessClientYamlTestSuiteIT.java
@@ -24,7 +24,6 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
 
 import java.io.IOException;
 
@@ -36,7 +35,7 @@ public class LangPainlessClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException, ClientYamlTestParseException {
+    public static Iterable<Object[]> parameters() throws IOException {
         return ESClientYamlSuiteTestCase.createParameters();
     }
 }
diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorClientYamlTestSuiteIT.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorClientYamlTestSuiteIT.java
index db1bbf13c8..8efd350839 100644
--- a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorClientYamlTestSuiteIT.java
+++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorClientYamlTestSuiteIT.java
@@ -24,7 +24,6 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
 
 import java.io.IOException;
 
@@ -34,7 +33,7 @@ public class PercolatorClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException, ClientYamlTestParseException {
+    public static Iterable<Object[]> parameters() throws IOException {
         return ESClientYamlSuiteTestCase.createParameters();
     }
 }
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexClientYamlTestSuiteIT.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexClientYamlTestSuiteIT.java
index 54483eae56..0975cad96a 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexClientYamlTestSuiteIT.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexClientYamlTestSuiteIT.java
@@ -24,7 +24,6 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
 
 import java.io.IOException;
 
@@ -34,7 +33,7 @@ public class ReindexClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException, ClientYamlTestParseException {
+    public static Iterable<Object[]> parameters() throws IOException {
         return ESClientYamlSuiteTestCase.createParameters();
     }
 }
diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4ClientYamlTestSuiteIT.java b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4ClientYamlTestSuiteIT.java
index 237227cd4d..2341f3905e 100644
--- a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4ClientYamlTestSuiteIT.java
+++ b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4ClientYamlTestSuiteIT.java
@@ -26,7 +26,6 @@ import com.carrotsearch.randomizedtesting.annotations.TimeoutSuite;
 import org.apache.lucene.util.TimeUnits;
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
 
 import java.io.IOException;
 
@@ -39,7 +38,7 @@ public class Netty4ClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException, ClientYamlTestParseException {
+    public static Iterable<Object[]> parameters() throws IOException {
         return ESClientYamlSuiteTestCase.createParameters();
     }
 
diff --git a/plugins/analysis-icu/src/test/java/org/elasticsearch/index/analysis/IcuClientYamlTestSuiteIT.java b/plugins/analysis-icu/src/test/java/org/elasticsearch/index/analysis/IcuClientYamlTestSuiteIT.java
index ce2e660ecf..04fc222063 100644
--- a/plugins/analysis-icu/src/test/java/org/elasticsearch/index/analysis/IcuClientYamlTestSuiteIT.java
+++ b/plugins/analysis-icu/src/test/java/org/elasticsearch/index/analysis/IcuClientYamlTestSuiteIT.java
@@ -24,7 +24,6 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
 
 import java.io.IOException;
 
@@ -35,7 +34,7 @@ public class IcuClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException, ClientYamlTestParseException {
+    public static Iterable<Object[]> parameters() throws IOException {
         return ESClientYamlSuiteTestCase.createParameters();
     }
 }
diff --git a/plugins/analysis-kuromoji/src/test/java/org/elasticsearch/index/analysis/KuromojiClientYamlTestSuiteIT.java b/plugins/analysis-kuromoji/src/test/java/org/elasticsearch/index/analysis/KuromojiClientYamlTestSuiteIT.java
index 0797b10d77..bc5360f94d 100644
--- a/plugins/analysis-kuromoji/src/test/java/org/elasticsearch/index/analysis/KuromojiClientYamlTestSuiteIT.java
+++ b/plugins/analysis-kuromoji/src/test/java/org/elasticsearch/index/analysis/KuromojiClientYamlTestSuiteIT.java
@@ -24,7 +24,6 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
 
 import java.io.IOException;
 
@@ -35,7 +34,7 @@ public class KuromojiClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException, ClientYamlTestParseException {
+    public static Iterable<Object[]> parameters() throws IOException {
         return ESClientYamlSuiteTestCase.createParameters();
     }
 }
diff --git a/plugins/analysis-phonetic/src/test/java/org/elasticsearch/index/analysis/PhoneticClientYamlTestSuiteIT.java b/plugins/analysis-phonetic/src/test/java/org/elasticsearch/index/analysis/PhoneticClientYamlTestSuiteIT.java
index 447eb1d6cd..3d892bddee 100644
--- a/plugins/analysis-phonetic/src/test/java/org/elasticsearch/index/analysis/PhoneticClientYamlTestSuiteIT.java
+++ b/plugins/analysis-phonetic/src/test/java/org/elasticsearch/index/analysis/PhoneticClientYamlTestSuiteIT.java
@@ -24,7 +24,6 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
 
 import java.io.IOException;
 
@@ -35,7 +34,7 @@ public class PhoneticClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException, ClientYamlTestParseException {
+    public static Iterable<Object[]> parameters() throws IOException {
         return ESClientYamlSuiteTestCase.createParameters();
     }
 }
diff --git a/plugins/analysis-smartcn/src/test/java/org/elasticsearch/index/analysis/SmartCNClientYamlTestSuiteIT.java b/plugins/analysis-smartcn/src/test/java/org/elasticsearch/index/analysis/SmartCNClientYamlTestSuiteIT.java
index 534af79a19..24a581d9e2 100644
--- a/plugins/analysis-smartcn/src/test/java/org/elasticsearch/index/analysis/SmartCNClientYamlTestSuiteIT.java
+++ b/plugins/analysis-smartcn/src/test/java/org/elasticsearch/index/analysis/SmartCNClientYamlTestSuiteIT.java
@@ -24,7 +24,6 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
 
 import java.io.IOException;
 
@@ -35,7 +34,7 @@ public class SmartCNClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException, ClientYamlTestParseException {
+    public static Iterable<Object[]> parameters() throws IOException {
         return ESClientYamlSuiteTestCase.createParameters();
     }
 }
diff --git a/plugins/analysis-stempel/src/test/java/org/elasticsearch/index/analysis/StempelClientYamlTestSuiteIT.java b/plugins/analysis-stempel/src/test/java/org/elasticsearch/index/analysis/StempelClientYamlTestSuiteIT.java
index 56edcdb692..371431e1c2 100644
--- a/plugins/analysis-stempel/src/test/java/org/elasticsearch/index/analysis/StempelClientYamlTestSuiteIT.java
+++ b/plugins/analysis-stempel/src/test/java/org/elasticsearch/index/analysis/StempelClientYamlTestSuiteIT.java
@@ -24,7 +24,6 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
 
 import java.io.IOException;
 
@@ -35,7 +34,7 @@ public class StempelClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException, ClientYamlTestParseException {
+    public static Iterable<Object[]> parameters() throws IOException {
         return ESClientYamlSuiteTestCase.createParameters();
     }
 }
diff --git a/plugins/analysis-ukrainian/src/test/java/org/elasticsearch/index/analysis/UkrainianClientYamlTestSuiteIT.java b/plugins/analysis-ukrainian/src/test/java/org/elasticsearch/index/analysis/UkrainianClientYamlTestSuiteIT.java
index dd77fdf74a..50d935e522 100644
--- a/plugins/analysis-ukrainian/src/test/java/org/elasticsearch/index/analysis/UkrainianClientYamlTestSuiteIT.java
+++ b/plugins/analysis-ukrainian/src/test/java/org/elasticsearch/index/analysis/UkrainianClientYamlTestSuiteIT.java
@@ -24,7 +24,6 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
 
 import java.io.IOException;
 
@@ -35,7 +34,7 @@ public class UkrainianClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException, ClientYamlTestParseException {
+    public static Iterable<Object[]> parameters() throws IOException {
         return ESClientYamlSuiteTestCase.createParameters();
     }
 }
diff --git a/plugins/discovery-azure-classic/src/test/java/org/elasticsearch/discovery/azure/classic/DiscoveryAzureClassicClientYamlTestSuiteIT.java b/plugins/discovery-azure-classic/src/test/java/org/elasticsearch/discovery/azure/classic/DiscoveryAzureClassicClientYamlTestSuiteIT.java
index 33c5d41f70..4f6a44ef23 100644
--- a/plugins/discovery-azure-classic/src/test/java/org/elasticsearch/discovery/azure/classic/DiscoveryAzureClassicClientYamlTestSuiteIT.java
+++ b/plugins/discovery-azure-classic/src/test/java/org/elasticsearch/discovery/azure/classic/DiscoveryAzureClassicClientYamlTestSuiteIT.java
@@ -24,7 +24,6 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
 
 import java.io.IOException;
 
@@ -35,7 +34,7 @@ public class DiscoveryAzureClassicClientYamlTestSuiteIT extends ESClientYamlSuit
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException, ClientYamlTestParseException {
+    public static Iterable<Object[]> parameters() throws IOException {
         return ESClientYamlSuiteTestCase.createParameters();
     }
 }
diff --git a/plugins/discovery-ec2/src/test/java/org/elasticsearch/cloud/aws/CloudAwsClientYamlTestSuiteIT.java b/plugins/discovery-ec2/src/test/java/org/elasticsearch/cloud/aws/CloudAwsClientYamlTestSuiteIT.java
index 3cd30c187d..3488b5ea49 100644
--- a/plugins/discovery-ec2/src/test/java/org/elasticsearch/cloud/aws/CloudAwsClientYamlTestSuiteIT.java
+++ b/plugins/discovery-ec2/src/test/java/org/elasticsearch/cloud/aws/CloudAwsClientYamlTestSuiteIT.java
@@ -24,7 +24,6 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
 
 import java.io.IOException;
 
@@ -35,7 +34,7 @@ public class CloudAwsClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException, ClientYamlTestParseException {
+    public static Iterable<Object[]> parameters() throws IOException {
         return ESClientYamlSuiteTestCase.createParameters();
     }
 }
diff --git a/plugins/discovery-file/src/test/java/org/elasticsearch/discovery/file/FileBasedDiscoveryClientYamlTestSuiteIT.java b/plugins/discovery-file/src/test/java/org/elasticsearch/discovery/file/FileBasedDiscoveryClientYamlTestSuiteIT.java
index 8a0bd808db..dbc6212af9 100644
--- a/plugins/discovery-file/src/test/java/org/elasticsearch/discovery/file/FileBasedDiscoveryClientYamlTestSuiteIT.java
+++ b/plugins/discovery-file/src/test/java/org/elasticsearch/discovery/file/FileBasedDiscoveryClientYamlTestSuiteIT.java
@@ -21,9 +21,9 @@ package org.elasticsearch.discovery.file;
 
 import com.carrotsearch.randomizedtesting.annotations.Name;
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
 
 import java.io.IOException;
 
@@ -37,7 +37,7 @@ public class FileBasedDiscoveryClientYamlTestSuiteIT extends ESClientYamlSuiteTe
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException, ClientYamlTestParseException {
+    public static Iterable<Object[]> parameters() throws IOException {
         return ESClientYamlSuiteTestCase.createParameters();
     }
 }
diff --git a/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/DiscoveryGceClientYamlTestSuiteIT.java b/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/DiscoveryGceClientYamlTestSuiteIT.java
index 3af39b6da5..67ef297609 100644
--- a/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/DiscoveryGceClientYamlTestSuiteIT.java
+++ b/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/DiscoveryGceClientYamlTestSuiteIT.java
@@ -24,7 +24,6 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
 
 import java.io.IOException;
 
@@ -35,7 +34,7 @@ public class DiscoveryGceClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException, ClientYamlTestParseException {
+    public static Iterable<Object[]> parameters() throws IOException {
         return ESClientYamlSuiteTestCase.createParameters();
     }
 }
diff --git a/plugins/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/IngestAttachmentClientYamlTestSuiteIT.java b/plugins/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/IngestAttachmentClientYamlTestSuiteIT.java
index 40e95451e4..43e9cd394f 100644
--- a/plugins/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/IngestAttachmentClientYamlTestSuiteIT.java
+++ b/plugins/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/IngestAttachmentClientYamlTestSuiteIT.java
@@ -24,7 +24,6 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
 
 import java.io.IOException;
 
@@ -35,7 +34,7 @@ public class IngestAttachmentClientYamlTestSuiteIT extends ESClientYamlSuiteTest
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException, ClientYamlTestParseException {
+    public static Iterable<Object[]> parameters() throws IOException {
         return ESClientYamlSuiteTestCase.createParameters();
     }
 }
diff --git a/plugins/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/IngestGeoIpClientYamlTestSuiteIT.java b/plugins/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/IngestGeoIpClientYamlTestSuiteIT.java
index ed381dab0b..33f7c67e01 100644
--- a/plugins/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/IngestGeoIpClientYamlTestSuiteIT.java
+++ b/plugins/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/IngestGeoIpClientYamlTestSuiteIT.java
@@ -24,7 +24,6 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
 
 import java.io.IOException;
 
@@ -35,7 +34,7 @@ public class IngestGeoIpClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException, ClientYamlTestParseException {
+    public static Iterable<Object[]> parameters() throws IOException {
         return ESClientYamlSuiteTestCase.createParameters();
     }
 }
diff --git a/plugins/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/IngestUserAgentClientYamlTestSuiteIT.java b/plugins/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/IngestUserAgentClientYamlTestSuiteIT.java
index 2acac87363..cc70c1ef6b 100644
--- a/plugins/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/IngestUserAgentClientYamlTestSuiteIT.java
+++ b/plugins/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/IngestUserAgentClientYamlTestSuiteIT.java
@@ -24,7 +24,6 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
 
 import java.io.IOException;
 
@@ -35,7 +34,7 @@ public class IngestUserAgentClientYamlTestSuiteIT extends ESClientYamlSuiteTestC
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException, ClientYamlTestParseException {
+    public static Iterable<Object[]> parameters() throws IOException {
         return ESClientYamlSuiteTestCase.createParameters();
     }
 }
diff --git a/plugins/jvm-example/src/test/java/org/elasticsearch/plugin/example/JvmExampleClientYamlTestSuiteIT.java b/plugins/jvm-example/src/test/java/org/elasticsearch/plugin/example/JvmExampleClientYamlTestSuiteIT.java
index b7bae90817..d3e84316b1 100644
--- a/plugins/jvm-example/src/test/java/org/elasticsearch/plugin/example/JvmExampleClientYamlTestSuiteIT.java
+++ b/plugins/jvm-example/src/test/java/org/elasticsearch/plugin/example/JvmExampleClientYamlTestSuiteIT.java
@@ -24,7 +24,6 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
 
 import java.io.IOException;
 
@@ -35,7 +34,7 @@ public class JvmExampleClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException, ClientYamlTestParseException {
+    public static Iterable<Object[]> parameters() throws IOException {
         return ESClientYamlSuiteTestCase.createParameters();
     }
 }
diff --git a/plugins/mapper-murmur3/src/test/java/org/elasticsearch/index/mapper/murmur3/MapperMurmur3ClientYamlTestSuiteIT.java b/plugins/mapper-murmur3/src/test/java/org/elasticsearch/index/mapper/murmur3/MapperMurmur3ClientYamlTestSuiteIT.java
index 3e9a5f1392..9ca9f677b8 100644
--- a/plugins/mapper-murmur3/src/test/java/org/elasticsearch/index/mapper/murmur3/MapperMurmur3ClientYamlTestSuiteIT.java
+++ b/plugins/mapper-murmur3/src/test/java/org/elasticsearch/index/mapper/murmur3/MapperMurmur3ClientYamlTestSuiteIT.java
@@ -24,7 +24,6 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
 
 import java.io.IOException;
 
@@ -35,7 +34,7 @@ public class MapperMurmur3ClientYamlTestSuiteIT extends ESClientYamlSuiteTestCas
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException, ClientYamlTestParseException {
+    public static Iterable<Object[]> parameters() throws IOException {
         return createParameters();
     }
 }
diff --git a/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/MapperSizeClientYamlTestSuiteIT.java b/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/MapperSizeClientYamlTestSuiteIT.java
index d8de3635b7..3741c5626f 100644
--- a/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/MapperSizeClientYamlTestSuiteIT.java
+++ b/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/MapperSizeClientYamlTestSuiteIT.java
@@ -24,7 +24,6 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
 
 import java.io.IOException;
 
@@ -35,7 +34,7 @@ public class MapperSizeClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException, ClientYamlTestParseException {
+    public static Iterable<Object[]> parameters() throws IOException {
         return createParameters();
     }
 }
diff --git a/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/RepositoryAzureClientYamlTestSuiteIT.java b/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/RepositoryAzureClientYamlTestSuiteIT.java
index 2919d07397..5d3d051fb3 100644
--- a/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/RepositoryAzureClientYamlTestSuiteIT.java
+++ b/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/RepositoryAzureClientYamlTestSuiteIT.java
@@ -24,7 +24,6 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
 
 import java.io.IOException;
 
@@ -35,7 +34,7 @@ public class RepositoryAzureClientYamlTestSuiteIT extends ESClientYamlSuiteTestC
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException, ClientYamlTestParseException {
+    public static Iterable<Object[]> parameters() throws IOException {
         return ESClientYamlSuiteTestCase.createParameters();
     }
 }
diff --git a/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/RepositoryGcsClientYamlTestSuiteIT.java b/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/RepositoryGcsClientYamlTestSuiteIT.java
index 6ed036e277..cdd86bc48b 100644
--- a/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/RepositoryGcsClientYamlTestSuiteIT.java
+++ b/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/RepositoryGcsClientYamlTestSuiteIT.java
@@ -24,7 +24,6 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
 
 import java.io.IOException;
 
@@ -35,7 +34,7 @@ public class RepositoryGcsClientYamlTestSuiteIT extends ESClientYamlSuiteTestCas
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException, ClientYamlTestParseException {
+    public static Iterable<Object[]> parameters() throws IOException {
         return createParameters();
     }
 }
diff --git a/plugins/repository-hdfs/src/test/java/org/elasticsearch/repositories/hdfs/RepositoryHdfsClientYamlTestSuiteIT.java b/plugins/repository-hdfs/src/test/java/org/elasticsearch/repositories/hdfs/RepositoryHdfsClientYamlTestSuiteIT.java
index 264a350d51..6e66b0d49f 100644
--- a/plugins/repository-hdfs/src/test/java/org/elasticsearch/repositories/hdfs/RepositoryHdfsClientYamlTestSuiteIT.java
+++ b/plugins/repository-hdfs/src/test/java/org/elasticsearch/repositories/hdfs/RepositoryHdfsClientYamlTestSuiteIT.java
@@ -18,14 +18,13 @@
  */
 package org.elasticsearch.repositories.hdfs;
 
-import java.io.IOException;
-
 import com.carrotsearch.randomizedtesting.annotations.Name;
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
+
+import java.io.IOException;
 
 public class RepositoryHdfsClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
 
@@ -34,7 +33,7 @@ public class RepositoryHdfsClientYamlTestSuiteIT extends ESClientYamlSuiteTestCa
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException, ClientYamlTestParseException {
+    public static Iterable<Object[]> parameters() throws IOException {
         return ESClientYamlSuiteTestCase.createParameters();
     }
 }
diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/RepositoryS3ClientYamlTestSuiteIT.java b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/RepositoryS3ClientYamlTestSuiteIT.java
index 9c567a570f..30056f67d2 100644
--- a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/RepositoryS3ClientYamlTestSuiteIT.java
+++ b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/RepositoryS3ClientYamlTestSuiteIT.java
@@ -24,7 +24,6 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
 
 import java.io.IOException;
 
@@ -35,7 +34,7 @@ public class RepositoryS3ClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException, ClientYamlTestParseException {
+    public static Iterable<Object[]> parameters() throws IOException {
         return ESClientYamlSuiteTestCase.createParameters();
     }
 }
diff --git a/plugins/store-smb/src/test/java/org/elasticsearch/index/store/StoreSmbClientYamlTestSuiteIT.java b/plugins/store-smb/src/test/java/org/elasticsearch/index/store/StoreSmbClientYamlTestSuiteIT.java
index 0b9de745ca..0216083c95 100644
--- a/plugins/store-smb/src/test/java/org/elasticsearch/index/store/StoreSmbClientYamlTestSuiteIT.java
+++ b/plugins/store-smb/src/test/java/org/elasticsearch/index/store/StoreSmbClientYamlTestSuiteIT.java
@@ -24,7 +24,6 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
 
 import java.io.IOException;
 
@@ -35,7 +34,7 @@ public class StoreSmbClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException, ClientYamlTestParseException {
+    public static Iterable<Object[]> parameters() throws IOException {
         return ESClientYamlSuiteTestCase.createParameters();
     }
 }
diff --git a/qa/backwards-5.0/src/test/java/org/elasticsearch/backwards/Backwards50ClientYamlTestSuiteIT.java b/qa/backwards-5.0/src/test/java/org/elasticsearch/backwards/Backwards50ClientYamlTestSuiteIT.java
index af77b216bc..cac063fce7 100644
--- a/qa/backwards-5.0/src/test/java/org/elasticsearch/backwards/Backwards50ClientYamlTestSuiteIT.java
+++ b/qa/backwards-5.0/src/test/java/org/elasticsearch/backwards/Backwards50ClientYamlTestSuiteIT.java
@@ -21,10 +21,10 @@ package org.elasticsearch.backwards;
 
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 import com.carrotsearch.randomizedtesting.annotations.TimeoutSuite;
+
 import org.apache.lucene.util.TimeUnits;
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
 
 import java.io.IOException;
 
@@ -36,7 +36,7 @@ public class Backwards50ClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException, ClientYamlTestParseException {
+    public static Iterable<Object[]> parameters() throws IOException {
         return createParameters();
     }
 }
diff --git a/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/UpgradeClusterClientYamlTestSuiteIT.java b/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/UpgradeClusterClientYamlTestSuiteIT.java
index f58d618adf..b7b825da47 100644
--- a/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/UpgradeClusterClientYamlTestSuiteIT.java
+++ b/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/UpgradeClusterClientYamlTestSuiteIT.java
@@ -21,10 +21,10 @@ package org.elasticsearch.upgrades;
 
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 import com.carrotsearch.randomizedtesting.annotations.TimeoutSuite;
+
 import org.apache.lucene.util.TimeUnits;
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
 
 import java.io.IOException;
 
@@ -41,7 +41,7 @@ public class UpgradeClusterClientYamlTestSuiteIT extends ESClientYamlSuiteTestCa
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException, ClientYamlTestParseException {
+    public static Iterable<Object[]> parameters() throws IOException {
         return createParameters();
     }
 }
diff --git a/qa/smoke-test-ingest-disabled/src/test/java/org/elasticsearch/smoketest/SmokeTestIngestDisabledClientYamlTestSuiteIT.java b/qa/smoke-test-ingest-disabled/src/test/java/org/elasticsearch/smoketest/SmokeTestIngestDisabledClientYamlTestSuiteIT.java
index beb7499bf7..7f4c2c4a4d 100644
--- a/qa/smoke-test-ingest-disabled/src/test/java/org/elasticsearch/smoketest/SmokeTestIngestDisabledClientYamlTestSuiteIT.java
+++ b/qa/smoke-test-ingest-disabled/src/test/java/org/elasticsearch/smoketest/SmokeTestIngestDisabledClientYamlTestSuiteIT.java
@@ -24,7 +24,6 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
 
 import java.io.IOException;
 
@@ -35,7 +34,7 @@ public class SmokeTestIngestDisabledClientYamlTestSuiteIT extends ESClientYamlSu
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException, ClientYamlTestParseException {
+    public static Iterable<Object[]> parameters() throws IOException {
         return ESClientYamlSuiteTestCase.createParameters();
     }
 
diff --git a/qa/smoke-test-ingest-with-all-dependencies/src/test/java/org/elasticsearch/smoketest/SmokeTestIngestWithAllDepsClientYamlTestSuiteIT.java b/qa/smoke-test-ingest-with-all-dependencies/src/test/java/org/elasticsearch/smoketest/SmokeTestIngestWithAllDepsClientYamlTestSuiteIT.java
index 0bd7b9ac02..d1e1adabfd 100644
--- a/qa/smoke-test-ingest-with-all-dependencies/src/test/java/org/elasticsearch/smoketest/SmokeTestIngestWithAllDepsClientYamlTestSuiteIT.java
+++ b/qa/smoke-test-ingest-with-all-dependencies/src/test/java/org/elasticsearch/smoketest/SmokeTestIngestWithAllDepsClientYamlTestSuiteIT.java
@@ -24,7 +24,6 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
 
 import java.io.IOException;
 
@@ -35,7 +34,7 @@ public class SmokeTestIngestWithAllDepsClientYamlTestSuiteIT extends ESClientYam
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException, ClientYamlTestParseException {
+    public static Iterable<Object[]> parameters() throws IOException {
         return ESClientYamlSuiteTestCase.createParameters();
     }
 
diff --git a/qa/smoke-test-multinode/src/test/java/org/elasticsearch/smoketest/SmokeTestMultiNodeClientYamlTestSuiteIT.java b/qa/smoke-test-multinode/src/test/java/org/elasticsearch/smoketest/SmokeTestMultiNodeClientYamlTestSuiteIT.java
index 225e12f65f..69abd0b3cc 100644
--- a/qa/smoke-test-multinode/src/test/java/org/elasticsearch/smoketest/SmokeTestMultiNodeClientYamlTestSuiteIT.java
+++ b/qa/smoke-test-multinode/src/test/java/org/elasticsearch/smoketest/SmokeTestMultiNodeClientYamlTestSuiteIT.java
@@ -21,12 +21,11 @@ package org.elasticsearch.smoketest;
 
 import com.carrotsearch.randomizedtesting.annotations.Name;
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
-
 import com.carrotsearch.randomizedtesting.annotations.TimeoutSuite;
+
 import org.apache.lucene.util.TimeUnits;
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
 
 import java.io.IOException;
 
@@ -38,7 +37,7 @@ public class SmokeTestMultiNodeClientYamlTestSuiteIT extends ESClientYamlSuiteTe
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException, ClientYamlTestParseException {
+    public static Iterable<Object[]> parameters() throws IOException {
         return ESClientYamlSuiteTestCase.createParameters();
     }
 }
diff --git a/qa/smoke-test-plugins/src/test/java/org/elasticsearch/smoketest/SmokeTestPluginsClientYamlTestSuiteIT.java b/qa/smoke-test-plugins/src/test/java/org/elasticsearch/smoketest/SmokeTestPluginsClientYamlTestSuiteIT.java
index 0f5636095e..2ae7be8fb1 100644
--- a/qa/smoke-test-plugins/src/test/java/org/elasticsearch/smoketest/SmokeTestPluginsClientYamlTestSuiteIT.java
+++ b/qa/smoke-test-plugins/src/test/java/org/elasticsearch/smoketest/SmokeTestPluginsClientYamlTestSuiteIT.java
@@ -24,7 +24,6 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
 
 import java.io.IOException;
 
@@ -35,7 +34,7 @@ public class SmokeTestPluginsClientYamlTestSuiteIT extends ESClientYamlSuiteTest
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException, ClientYamlTestParseException {
+    public static Iterable<Object[]> parameters() throws IOException {
         return ESClientYamlSuiteTestCase.createParameters();
     }
 }
diff --git a/qa/smoke-test-reindex-with-painless/src/test/java/org/elasticsearch/smoketest/SmokeTestReindexWithPainlessClientYamlTestSuiteIT.java b/qa/smoke-test-reindex-with-painless/src/test/java/org/elasticsearch/smoketest/SmokeTestReindexWithPainlessClientYamlTestSuiteIT.java
index fb4f27210d..5366eaf4bd 100644
--- a/qa/smoke-test-reindex-with-painless/src/test/java/org/elasticsearch/smoketest/SmokeTestReindexWithPainlessClientYamlTestSuiteIT.java
+++ b/qa/smoke-test-reindex-with-painless/src/test/java/org/elasticsearch/smoketest/SmokeTestReindexWithPainlessClientYamlTestSuiteIT.java
@@ -24,7 +24,6 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
 
 import java.io.IOException;
 
@@ -34,7 +33,7 @@ public class SmokeTestReindexWithPainlessClientYamlTestSuiteIT extends ESClientY
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException, ClientYamlTestParseException {
+    public static Iterable<Object[]> parameters() throws IOException {
         return ESClientYamlSuiteTestCase.createParameters();
     }
 }
diff --git a/qa/smoke-test-tribe-node/src/test/java/org/elasticsearch/tribe/TribeClientYamlTestSuiteIT.java b/qa/smoke-test-tribe-node/src/test/java/org/elasticsearch/tribe/TribeClientYamlTestSuiteIT.java
index 6013913bdc..211043ed4b 100644
--- a/qa/smoke-test-tribe-node/src/test/java/org/elasticsearch/tribe/TribeClientYamlTestSuiteIT.java
+++ b/qa/smoke-test-tribe-node/src/test/java/org/elasticsearch/tribe/TribeClientYamlTestSuiteIT.java
@@ -21,9 +21,9 @@ package org.elasticsearch.tribe;
 
 import com.carrotsearch.randomizedtesting.annotations.Name;
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
 
 import java.io.IOException;
 
@@ -46,7 +46,7 @@ public class TribeClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException, ClientYamlTestParseException {
+    public static Iterable<Object[]> parameters() throws IOException {
         return createParameters();
     }
 
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCase.java
index e4434d1365..f025056a4e 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCase.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.test.rest.yaml;
 
 import com.carrotsearch.randomizedtesting.RandomizedTest;
+
 import org.apache.http.HttpHost;
 import org.apache.lucene.util.IOUtils;
 import org.elasticsearch.Version;
@@ -31,8 +32,6 @@ import org.elasticsearch.common.SuppressForbidden;
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.test.rest.ESRestTestCase;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestSuiteParser;
 import org.elasticsearch.test.rest.yaml.restspec.ClientYamlSuiteRestApi;
 import org.elasticsearch.test.rest.yaml.restspec.ClientYamlSuiteRestSpec;
 import org.elasticsearch.test.rest.yaml.section.ClientYamlTestSection;
@@ -186,7 +185,7 @@ public abstract class ESClientYamlSuiteTestCase extends ESRestTestCase {
         super.afterIfFailed(errors);
     }
 
-    public static Iterable<Object[]> createParameters() throws IOException, ClientYamlTestParseException {
+    public static Iterable<Object[]> createParameters() throws IOException {
         List<ClientYamlTestCandidate> restTestCandidates = collectTestCandidates();
         List<Object[]> objects = new ArrayList<>();
         for (ClientYamlTestCandidate restTestCandidate : restTestCandidates) {
@@ -195,7 +194,7 @@ public abstract class ESClientYamlSuiteTestCase extends ESRestTestCase {
         return objects;
     }
 
-    private static List<ClientYamlTestCandidate> collectTestCandidates() throws ClientYamlTestParseException, IOException {
+    private static List<ClientYamlTestCandidate> collectTestCandidates() throws IOException {
         List<ClientYamlTestCandidate> testCandidates = new ArrayList<>();
         FileSystem fileSystem = getFileSystem();
         // don't make a try-with, getFileSystem returns null
@@ -203,12 +202,11 @@ public abstract class ESClientYamlSuiteTestCase extends ESRestTestCase {
         try {
             String[] paths = resolvePathsProperty(REST_TESTS_SUITE, DEFAULT_TESTS_PATH);
             Map<String, Set<Path>> yamlSuites = FileUtils.findYamlSuites(fileSystem, DEFAULT_TESTS_PATH, paths);
-            ClientYamlTestSuiteParser restTestSuiteParser = new ClientYamlTestSuiteParser();
             //yaml suites are grouped by directory (effectively by api)
             for (String api : yamlSuites.keySet()) {
                 List<Path> yamlFiles = new ArrayList<>(yamlSuites.get(api));
                 for (Path yamlFile : yamlFiles) {
-                    ClientYamlTestSuite restTestSuite = restTestSuiteParser.parse(api, yamlFile);
+                    ClientYamlTestSuite restTestSuite = ClientYamlTestSuite.parse(api, yamlFile);
                     for (ClientYamlTestSection testSection : restTestSuite.getTestSections()) {
                         testCandidates.add(new ClientYamlTestCandidate(restTestSuite, testSection));
                     }
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/ClientYamlTestFragmentParser.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/ClientYamlTestFragmentParser.java
deleted file mode 100644
index 390ac1ce36..0000000000
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/ClientYamlTestFragmentParser.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.test.rest.yaml.parser;
-
-import java.io.IOException;
-
-/**
- * Base parser for a REST test suite fragment
- * @param <T> the test fragment's type that gets parsed and returned
- */
-public interface ClientYamlTestFragmentParser<T> {
-
-    /**
-     * Parses a test fragment given the current {@link ClientYamlTestSuiteParseContext}
-     */
-    T parse(ClientYamlTestSuiteParseContext parseContext) throws IOException, ClientYamlTestParseException;
-}
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/ClientYamlTestParseException.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/ClientYamlTestParseException.java
deleted file mode 100644
index 594f701c79..0000000000
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/ClientYamlTestParseException.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.test.rest.yaml.parser;
-
-/**
- * Exception thrown whenever there is a problem parsing any of the REST test suite fragment
- */
-public class ClientYamlTestParseException extends Exception {
-
-    ClientYamlTestParseException(String message) {
-        super(message);
-    }
-
-    ClientYamlTestParseException(String message, Throwable cause) {
-        super(message, cause);
-    }
-}
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/ClientYamlTestSectionParser.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/ClientYamlTestSectionParser.java
deleted file mode 100644
index b6b6adfd03..0000000000
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/ClientYamlTestSectionParser.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.test.rest.yaml.parser;
-
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.test.rest.yaml.section.ClientYamlTestSection;
-
-import java.io.IOException;
-
-/**
- * Parser for a complete test section
- */
-public class ClientYamlTestSectionParser implements ClientYamlTestFragmentParser<ClientYamlTestSection> {
-
-    @Override
-    public ClientYamlTestSection parse(ClientYamlTestSuiteParseContext parseContext) throws IOException, ClientYamlTestParseException {
-        XContentParser parser = parseContext.parser();
-        parseContext.advanceToFieldName();
-        ClientYamlTestSection testSection = new ClientYamlTestSection(parser.currentName());
-        try {
-            parser.nextToken();
-            testSection.setSkipSection(parseContext.parseSkipSection());
-
-            while ( parser.currentToken() != XContentParser.Token.END_ARRAY) {
-                parseContext.advanceToFieldName();
-                testSection.addExecutableSection(parseContext.parseExecutableSection());
-            }
-
-            parser.nextToken();
-            assert parser.currentToken() == XContentParser.Token.END_OBJECT : "malformed section [" + testSection.getName() + "] expected "
-                + XContentParser.Token.END_OBJECT  + " but was " + parser.currentToken();
-            parser.nextToken();
-
-            return testSection;
-        } catch (Exception e) {
-            throw new ClientYamlTestParseException("Error parsing test named [" + testSection.getName() + "]", e);
-        }
-    }
-
-}
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/ClientYamlTestSuiteParseContext.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/ClientYamlTestSuiteParseContext.java
deleted file mode 100644
index e466c1010f..0000000000
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/ClientYamlTestSuiteParseContext.java
+++ /dev/null
@@ -1,195 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.test.rest.yaml.parser;
-
-import org.elasticsearch.common.ParseFieldMatcher;
-import org.elasticsearch.common.ParseFieldMatcherSupplier;
-import org.elasticsearch.common.collect.Tuple;
-import org.elasticsearch.common.xcontent.XContentLocation;
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.test.rest.yaml.section.DoSection;
-import org.elasticsearch.test.rest.yaml.section.ExecutableSection;
-import org.elasticsearch.test.rest.yaml.section.SetupSection;
-import org.elasticsearch.test.rest.yaml.section.SkipSection;
-import org.elasticsearch.test.rest.yaml.section.TeardownSection;
-import org.elasticsearch.test.rest.yaml.section.ClientYamlTestSection;
-
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
-
-/**
- * Context shared across the whole tests parse phase.
- * Provides shared parse methods and holds information needed to parse the test sections (e.g. es version)
- */
-public class ClientYamlTestSuiteParseContext implements ParseFieldMatcherSupplier {
-
-    private static final SetupSectionParser SETUP_SECTION_PARSER = new SetupSectionParser();
-    private static final TeardownSectionParser TEARDOWN_SECTION_PARSER = new TeardownSectionParser();
-    private static final ClientYamlTestSectionParser TEST_SECTION_PARSER = new ClientYamlTestSectionParser();
-    private static final SkipSectionParser SKIP_SECTION_PARSER = new SkipSectionParser();
-    private static final DoSectionParser DO_SECTION_PARSER = new DoSectionParser();
-    private static final Map<String, ClientYamlTestFragmentParser<? extends ExecutableSection>> EXECUTABLE_SECTIONS_PARSERS =
-            new HashMap<>();
-    static {
-        EXECUTABLE_SECTIONS_PARSERS.put("do", DO_SECTION_PARSER);
-        EXECUTABLE_SECTIONS_PARSERS.put("set", new SetSectionParser());
-        EXECUTABLE_SECTIONS_PARSERS.put("match", new MatchParser());
-        EXECUTABLE_SECTIONS_PARSERS.put("is_true", new IsTrueParser());
-        EXECUTABLE_SECTIONS_PARSERS.put("is_false", new IsFalseParser());
-        EXECUTABLE_SECTIONS_PARSERS.put("gt", new GreaterThanParser());
-        EXECUTABLE_SECTIONS_PARSERS.put("gte", new GreaterThanEqualToParser());
-        EXECUTABLE_SECTIONS_PARSERS.put("lt", new LessThanParser());
-        EXECUTABLE_SECTIONS_PARSERS.put("lte", new LessThanOrEqualToParser());
-        EXECUTABLE_SECTIONS_PARSERS.put("length", new LengthParser());
-    }
-
-    private final String api;
-    private final String suiteName;
-    private final XContentParser parser;
-
-    public ClientYamlTestSuiteParseContext(String api, String suiteName, XContentParser parser) {
-        this.api = api;
-        this.suiteName = suiteName;
-        this.parser = parser;
-    }
-
-    public String getApi() {
-        return api;
-    }
-
-    public String getSuiteName() {
-        return suiteName;
-    }
-
-    public XContentParser parser() {
-        return parser;
-    }
-
-    public SetupSection parseSetupSection() throws IOException, ClientYamlTestParseException {
-
-        advanceToFieldName();
-
-        if ("setup".equals(parser.currentName())) {
-            parser.nextToken();
-            SetupSection setupSection = SETUP_SECTION_PARSER.parse(this);
-            parser.nextToken();
-            return setupSection;
-        }
-
-        return SetupSection.EMPTY;
-    }
-
-    public TeardownSection parseTeardownSection() throws IOException, ClientYamlTestParseException {
-        advanceToFieldName();
-
-        if ("teardown".equals(parser.currentName())) {
-            parser.nextToken();
-            TeardownSection teardownSection = TEARDOWN_SECTION_PARSER.parse(this);
-            parser.nextToken();
-            return teardownSection;
-        }
-
-        return TeardownSection.EMPTY;
-    }
-
-    public ClientYamlTestSection parseTestSection() throws IOException, ClientYamlTestParseException {
-        return TEST_SECTION_PARSER.parse(this);
-    }
-
-    public SkipSection parseSkipSection() throws IOException, ClientYamlTestParseException {
-
-        advanceToFieldName();
-
-        if ("skip".equals(parser.currentName())) {
-            SkipSection skipSection = SKIP_SECTION_PARSER.parse(this);
-            parser.nextToken();
-            return skipSection;
-        }
-
-        return SkipSection.EMPTY;
-    }
-
-    public ExecutableSection parseExecutableSection() throws IOException, ClientYamlTestParseException {
-        advanceToFieldName();
-        String section = parser.currentName();
-        ClientYamlTestFragmentParser<? extends ExecutableSection> execSectionParser = EXECUTABLE_SECTIONS_PARSERS.get(section);
-        if (execSectionParser == null) {
-            throw new ClientYamlTestParseException("no parser found for executable section [" + section + "]");
-        }
-        XContentLocation location = parser.getTokenLocation();
-        try {
-            ExecutableSection executableSection = execSectionParser.parse(this);
-            parser.nextToken();
-            return executableSection;
-        } catch (Exception e) {
-            throw new IOException("Error parsing section starting at ["+ location + "]", e);
-        }
-    }
-
-    public DoSection parseDoSection() throws IOException, ClientYamlTestParseException {
-        return DO_SECTION_PARSER.parse(this);
-    }
-
-    public void advanceToFieldName() throws IOException, ClientYamlTestParseException {
-        XContentParser.Token token = parser.currentToken();
-        //we are in the beginning, haven't called nextToken yet
-        if (token == null) {
-            token = parser.nextToken();
-        }
-        if (token == XContentParser.Token.START_ARRAY) {
-            token = parser.nextToken();
-        }
-        if (token == XContentParser.Token.START_OBJECT) {
-            token = parser.nextToken();
-        }
-        if (token != XContentParser.Token.FIELD_NAME) {
-            throw new ClientYamlTestParseException("malformed test section: field name expected but found " + token + " at "
-                    + parser.getTokenLocation());
-        }
-    }
-
-    public String parseField() throws IOException, ClientYamlTestParseException {
-        parser.nextToken();
-        assert parser.currentToken().isValue();
-        String field = parser.text();
-        parser.nextToken();
-        return field;
-    }
-
-    public Tuple<String, Object> parseTuple() throws IOException, ClientYamlTestParseException {
-        parser.nextToken();
-        advanceToFieldName();
-        Map<String,Object> map = parser.map();
-        assert parser.currentToken() == XContentParser.Token.END_OBJECT;
-        parser.nextToken();
-
-        if (map.size() != 1) {
-            throw new ClientYamlTestParseException("expected key value pair but found " + map.size() + " ");
-        }
-
-        Map.Entry<String, Object> entry = map.entrySet().iterator().next();
-        return Tuple.tuple(entry.getKey(), entry.getValue());
-    }
-
-    @Override
-    public ParseFieldMatcher getParseFieldMatcher() {
-        return ParseFieldMatcher.STRICT;
-    }
-}
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/ClientYamlTestSuiteParser.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/ClientYamlTestSuiteParser.java
deleted file mode 100644
index 3c09452fc8..0000000000
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/ClientYamlTestSuiteParser.java
+++ /dev/null
@@ -1,101 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.test.rest.yaml.parser;
-
-import org.elasticsearch.common.xcontent.NamedXContentRegistry;
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.common.xcontent.yaml.YamlXContent;
-import org.elasticsearch.test.rest.yaml.section.ClientYamlTestSuite;
-import org.elasticsearch.test.rest.yaml.section.ClientYamlTestSection;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.nio.channels.FileChannel;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.StandardOpenOption;
-
-/**
- * Parser for a complete test suite (yaml file)
- */
-public class ClientYamlTestSuiteParser implements ClientYamlTestFragmentParser<ClientYamlTestSuite> {
-
-    public ClientYamlTestSuite parse(String api, Path file) throws IOException, ClientYamlTestParseException {
-
-        if (!Files.isRegularFile(file)) {
-            throw new IllegalArgumentException(file.toAbsolutePath() + " is not a file");
-        }
-
-        String filename = file.getFileName().toString();
-        //remove the file extension
-        int i = filename.lastIndexOf('.');
-        if (i > 0) {
-            filename = filename.substring(0, i);
-        }
-
-        //our yaml parser seems to be too tolerant. Each yaml suite must end with \n, otherwise clients tests might break.
-        try (FileChannel channel = FileChannel.open(file, StandardOpenOption.READ)) {
-            ByteBuffer bb = ByteBuffer.wrap(new byte[1]);
-            channel.read(bb, channel.size() - 1);
-            if (bb.get(0) != 10) {
-                throw new ClientYamlTestParseException("test suite [" + api + "/" + filename + "] doesn't end with line feed (\\n)");
-            }
-        }
-
-        try (XContentParser parser = YamlXContent.yamlXContent.createParser(NamedXContentRegistry.EMPTY, Files.newInputStream(file))) {
-            ClientYamlTestSuiteParseContext testParseContext = new ClientYamlTestSuiteParseContext(api, filename, parser);
-            return parse(testParseContext);
-        } catch(Exception e) {
-            throw new ClientYamlTestParseException("Error parsing " + api + "/" + filename, e);
-        }
-    }
-
-    @Override
-    public ClientYamlTestSuite parse(ClientYamlTestSuiteParseContext parseContext) throws IOException, ClientYamlTestParseException {
-        XContentParser parser = parseContext.parser();
-
-        parser.nextToken();
-        assert parser.currentToken() == XContentParser.Token.START_OBJECT : "expected token to be START_OBJECT but was "
-                + parser.currentToken();
-
-        ClientYamlTestSuite restTestSuite = new ClientYamlTestSuite(parseContext.getApi(), parseContext.getSuiteName());
-
-        restTestSuite.setSetupSection(parseContext.parseSetupSection());
-        restTestSuite.setTeardownSection(parseContext.parseTeardownSection());
-
-        while(true) {
-            //the "---" section separator is not understood by the yaml parser. null is returned, same as when the parser is closed
-            //we need to somehow distinguish between a null in the middle of a test ("---")
-            // and a null at the end of the file (at least two consecutive null tokens)
-            if(parser.currentToken() == null) {
-                if (parser.nextToken() == null) {
-                    break;
-                }
-            }
-
-            ClientYamlTestSection testSection = parseContext.parseTestSection();
-            if (!restTestSuite.addTestSection(testSection)) {
-                throw new ClientYamlTestParseException("duplicate test section [" + testSection.getName() + "] found in ["
-                        + restTestSuite.getPath() + "]");
-            }
-        }
-
-        return restTestSuite;
-    }
-}
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/DoSectionParser.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/DoSectionParser.java
deleted file mode 100644
index ba746c129b..0000000000
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/DoSectionParser.java
+++ /dev/null
@@ -1,125 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.test.rest.yaml.parser;
-
-import org.elasticsearch.common.ParsingException;
-import org.elasticsearch.common.xcontent.NamedXContentRegistry;
-import org.elasticsearch.common.xcontent.XContentFactory;
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.common.xcontent.XContentType;
-import org.elasticsearch.test.rest.yaml.section.ApiCallSection;
-import org.elasticsearch.test.rest.yaml.section.DoSection;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import static java.util.Collections.unmodifiableList;
-
-/**
- * Parser for do sections
- */
-public class DoSectionParser implements ClientYamlTestFragmentParser<DoSection> {
-
-    @Override
-    public DoSection parse(ClientYamlTestSuiteParseContext parseContext) throws IOException, ClientYamlTestParseException {
-
-        XContentParser parser = parseContext.parser();
-
-        String currentFieldName = null;
-        XContentParser.Token token;
-
-        DoSection doSection = new DoSection(parseContext.parser().getTokenLocation());
-        ApiCallSection apiCallSection = null;
-        Map<String, String> headers = new HashMap<>();
-        List<String> expectedWarnings = new ArrayList<>();
-
-        while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
-            if (token == XContentParser.Token.FIELD_NAME) {
-                currentFieldName = parser.currentName();
-            } else if (token.isValue()) {
-                if ("catch".equals(currentFieldName)) {
-                    doSection.setCatch(parser.text());
-                }
-            } else if (token == XContentParser.Token.START_ARRAY) {
-                if ("warnings".equals(currentFieldName)) {
-                    while ((token = parser.nextToken()) == XContentParser.Token.VALUE_STRING) {
-                        expectedWarnings.add(parser.text());
-                    }
-                    if (token != XContentParser.Token.END_ARRAY) {
-                        throw new ParsingException(parser.getTokenLocation(), "[warnings] must be a string array but saw [" + token + "]");
-                    }
-                } else {
-                    throw new ParsingException(parser.getTokenLocation(), "unknown array [" + currentFieldName + "]");
-                }
-            } else if (token == XContentParser.Token.START_OBJECT) {
-                if ("headers".equals(currentFieldName)) {
-                    String headerName = null;
-                    while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
-                        if (token == XContentParser.Token.FIELD_NAME) {
-                            headerName = parser.currentName();
-                        } else if (token.isValue()) {
-                            headers.put(headerName, parser.text());
-                        }
-                    }
-                } else if (currentFieldName != null) { // must be part of API call then
-                    apiCallSection = new ApiCallSection(currentFieldName);
-                    String paramName = null;
-                    while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
-                        if (token == XContentParser.Token.FIELD_NAME) {
-                            paramName = parser.currentName();
-                        } else if (token.isValue()) {
-                            if ("body".equals(paramName)) {
-                                String body = parser.text();
-                                XContentType bodyContentType = XContentFactory.xContentType(body);
-                                XContentParser bodyParser = XContentFactory.xContent(bodyContentType).createParser(
-                                        NamedXContentRegistry.EMPTY, body);
-                                //multiple bodies are supported e.g. in case of bulk provided as a whole string
-                                while(bodyParser.nextToken() != null) {
-                                    apiCallSection.addBody(bodyParser.mapOrdered());
-                                }
-                            } else {
-                                apiCallSection.addParam(paramName, parser.text());
-                            }
-                        } else if (token == XContentParser.Token.START_OBJECT) {
-                            if ("body".equals(paramName)) {
-                                apiCallSection.addBody(parser.mapOrdered());
-                            }
-                        }
-                    }
-                }
-            }
-        }
-        try {
-            if (apiCallSection == null) {
-                throw new ClientYamlTestParseException("client call section is mandatory within a do section");
-            }
-            if (headers.isEmpty() == false) {
-                apiCallSection.addHeaders(headers);
-            }
-            doSection.setApiCallSection(apiCallSection);
-            doSection.setExpectedWarningHeaders(unmodifiableList(expectedWarnings));
-        } finally {
-            parser.nextToken();
-        }
-        return doSection;
-    }
-}
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/GreaterThanEqualToParser.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/GreaterThanEqualToParser.java
deleted file mode 100644
index 65a46d139b..0000000000
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/GreaterThanEqualToParser.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.test.rest.yaml.parser;
-
-import org.elasticsearch.common.collect.Tuple;
-import org.elasticsearch.common.xcontent.XContentLocation;
-import org.elasticsearch.test.rest.yaml.section.GreaterThanEqualToAssertion;
-
-import java.io.IOException;
-
-/**
- * Parser for gte assert sections
- */
-public class GreaterThanEqualToParser implements ClientYamlTestFragmentParser<GreaterThanEqualToAssertion> {
-    @Override
-    public GreaterThanEqualToAssertion parse(ClientYamlTestSuiteParseContext parseContext)
-            throws IOException, ClientYamlTestParseException {
-        XContentLocation location = parseContext.parser().getTokenLocation();
-        Tuple<String,Object> stringObjectTuple = parseContext.parseTuple();
-        if (! (stringObjectTuple.v2() instanceof Comparable) ) {
-            throw new ClientYamlTestParseException("gte section can only be used with objects that support natural ordering, found "
-                    + stringObjectTuple.v2().getClass().getSimpleName());
-        }
-        return new GreaterThanEqualToAssertion(location, stringObjectTuple.v1(), stringObjectTuple.v2());
-    }
-}
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/GreaterThanParser.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/GreaterThanParser.java
deleted file mode 100644
index f6c5327206..0000000000
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/GreaterThanParser.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.test.rest.yaml.parser;
-
-import org.elasticsearch.common.collect.Tuple;
-import org.elasticsearch.common.xcontent.XContentLocation;
-import org.elasticsearch.test.rest.yaml.section.GreaterThanAssertion;
-
-import java.io.IOException;
-
-/**
- * Parser for gt assert sections
- */
-public class GreaterThanParser implements ClientYamlTestFragmentParser<GreaterThanAssertion> {
-
-    @Override
-    public GreaterThanAssertion parse(ClientYamlTestSuiteParseContext parseContext) throws IOException, ClientYamlTestParseException {
-        XContentLocation location = parseContext.parser().getTokenLocation();
-        Tuple<String,Object> stringObjectTuple = parseContext.parseTuple();
-        if (! (stringObjectTuple.v2() instanceof Comparable) ) {
-            throw new ClientYamlTestParseException("gt section can only be used with objects that support natural ordering, found "
-                    + stringObjectTuple.v2().getClass().getSimpleName());
-        }
-        return new GreaterThanAssertion(location, stringObjectTuple.v1(), stringObjectTuple.v2());
-    }
-}
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/IsFalseParser.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/IsFalseParser.java
deleted file mode 100644
index 3d9593ce29..0000000000
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/IsFalseParser.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.test.rest.yaml.parser;
-
-import org.elasticsearch.test.rest.yaml.section.IsFalseAssertion;
-
-import java.io.IOException;
-
-/**
- * Parser for is_false assert sections
- */
-public class IsFalseParser implements ClientYamlTestFragmentParser<IsFalseAssertion> {
-
-    @Override
-    public IsFalseAssertion parse(ClientYamlTestSuiteParseContext parseContext) throws IOException, ClientYamlTestParseException {
-        return new IsFalseAssertion(parseContext.parser().getTokenLocation(), parseContext.parseField());
-    }
-}
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/IsTrueParser.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/IsTrueParser.java
deleted file mode 100644
index 9dfc3be57e..0000000000
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/IsTrueParser.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.test.rest.yaml.parser;
-
-import org.elasticsearch.test.rest.yaml.section.IsTrueAssertion;
-
-import java.io.IOException;
-
-/**
- * Parser for is_true assert sections
- */
-public class IsTrueParser implements ClientYamlTestFragmentParser<IsTrueAssertion> {
-
-    @Override
-    public IsTrueAssertion parse(ClientYamlTestSuiteParseContext parseContext) throws IOException, ClientYamlTestParseException {
-        return new IsTrueAssertion(parseContext.parser().getTokenLocation(), parseContext.parseField());
-    }
-}
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/LengthParser.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/LengthParser.java
deleted file mode 100644
index 98c030cd6a..0000000000
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/LengthParser.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.test.rest.yaml.parser;
-
-import org.elasticsearch.common.collect.Tuple;
-import org.elasticsearch.common.xcontent.XContentLocation;
-import org.elasticsearch.test.rest.yaml.section.LengthAssertion;
-
-import java.io.IOException;
-
-/**
- * Parser for length assert sections
- */
-public class LengthParser implements ClientYamlTestFragmentParser<LengthAssertion> {
-
-    @Override
-    public LengthAssertion parse(ClientYamlTestSuiteParseContext parseContext) throws IOException, ClientYamlTestParseException {
-        XContentLocation location = parseContext.parser().getTokenLocation();
-        Tuple<String,Object> stringObjectTuple = parseContext.parseTuple();
-        assert stringObjectTuple.v2() != null;
-        int value;
-        if (stringObjectTuple.v2() instanceof  Number) {
-            value = ((Number) stringObjectTuple.v2()).intValue();
-        } else {
-            try {
-                value = Integer.valueOf(stringObjectTuple.v2().toString());
-            } catch(NumberFormatException e) {
-                throw new ClientYamlTestParseException("length is not a valid number", e);
-            }
-
-        }
-        return new LengthAssertion(location, stringObjectTuple.v1(), value);
-    }
-}
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/LessThanOrEqualToParser.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/LessThanOrEqualToParser.java
deleted file mode 100644
index 5dfe4c742f..0000000000
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/LessThanOrEqualToParser.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.test.rest.yaml.parser;
-
-import org.elasticsearch.common.collect.Tuple;
-import org.elasticsearch.common.xcontent.XContentLocation;
-import org.elasticsearch.test.rest.yaml.section.LessThanOrEqualToAssertion;
-
-import java.io.IOException;
-
-/**
- * Parser for lte assert section
- */
-public class LessThanOrEqualToParser implements ClientYamlTestFragmentParser<LessThanOrEqualToAssertion> {
-
-    @Override
-    public LessThanOrEqualToAssertion parse(ClientYamlTestSuiteParseContext parseContext) throws IOException, ClientYamlTestParseException {
-        XContentLocation location = parseContext.parser().getTokenLocation();
-        Tuple<String,Object> stringObjectTuple = parseContext.parseTuple();
-        if (! (stringObjectTuple.v2() instanceof Comparable) ) {
-            throw new ClientYamlTestParseException("lte section can only be used with objects that support natural ordering, found "
-                    + stringObjectTuple.v2().getClass().getSimpleName());
-        }
-        return new LessThanOrEqualToAssertion(location, stringObjectTuple.v1(), stringObjectTuple.v2());
-    }
-}
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/LessThanParser.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/LessThanParser.java
deleted file mode 100644
index 0caf12ea91..0000000000
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/LessThanParser.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.test.rest.yaml.parser;
-
-import org.elasticsearch.common.collect.Tuple;
-import org.elasticsearch.common.xcontent.XContentLocation;
-import org.elasticsearch.test.rest.yaml.section.LessThanAssertion;
-
-import java.io.IOException;
-
-/**
- * Parser for lt assert sections
- */
-public class LessThanParser implements ClientYamlTestFragmentParser<LessThanAssertion> {
-
-    @Override
-    public LessThanAssertion parse(ClientYamlTestSuiteParseContext parseContext) throws IOException, ClientYamlTestParseException {
-        XContentLocation location = parseContext.parser().getTokenLocation();
-        Tuple<String,Object> stringObjectTuple = parseContext.parseTuple();
-        if (! (stringObjectTuple.v2() instanceof Comparable) ) {
-            throw new ClientYamlTestParseException("lt section can only be used with objects that support natural ordering, found "
-                    + stringObjectTuple.v2().getClass().getSimpleName());
-        }
-        return new LessThanAssertion(location, stringObjectTuple.v1(), stringObjectTuple.v2());
-    }
-}
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/MatchParser.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/MatchParser.java
deleted file mode 100644
index f456c32d09..0000000000
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/MatchParser.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.test.rest.yaml.parser;
-
-import org.elasticsearch.common.collect.Tuple;
-import org.elasticsearch.common.xcontent.XContentLocation;
-import org.elasticsearch.test.rest.yaml.section.MatchAssertion;
-
-import java.io.IOException;
-
-/**
- * Parser for match assert sections
- */
-public class MatchParser implements ClientYamlTestFragmentParser<MatchAssertion> {
-
-    @Override
-    public MatchAssertion parse(ClientYamlTestSuiteParseContext parseContext) throws IOException, ClientYamlTestParseException {
-        XContentLocation location = parseContext.parser().getTokenLocation();
-        Tuple<String,Object> stringObjectTuple = parseContext.parseTuple();
-        return new MatchAssertion(location, stringObjectTuple.v1(), stringObjectTuple.v2());
-    }
-}
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/SetSectionParser.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/SetSectionParser.java
deleted file mode 100644
index c7797e42d5..0000000000
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/SetSectionParser.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.test.rest.yaml.parser;
-
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.test.rest.yaml.section.SetSection;
-
-import java.io.IOException;
-
-/**
- * Parser for set sections
- */
-public class SetSectionParser implements ClientYamlTestFragmentParser<SetSection> {
-
-    @Override
-    public SetSection parse(ClientYamlTestSuiteParseContext parseContext) throws IOException, ClientYamlTestParseException {
-
-        XContentParser parser = parseContext.parser();
-
-        String currentFieldName = null;
-        XContentParser.Token token;
-
-        SetSection setSection = new SetSection(parser.getTokenLocation());
-
-        while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
-            if (token == XContentParser.Token.FIELD_NAME) {
-                currentFieldName = parser.currentName();
-            } else if (token.isValue()) {
-                setSection.addSet(currentFieldName, parser.text());
-            }
-        }
-
-        parser.nextToken();
-
-        if (setSection.getStash().isEmpty()) {
-            throw new ClientYamlTestParseException("set section must set at least a value");
-        }
-
-        return setSection;
-    }
-}
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/SetupSectionParser.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/SetupSectionParser.java
deleted file mode 100644
index e62b3af525..0000000000
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/SetupSectionParser.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.test.rest.yaml.parser;
-
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.test.rest.yaml.section.SetupSection;
-
-import java.io.IOException;
-
-/**
- * Parser for setup sections
- */
-public class SetupSectionParser implements ClientYamlTestFragmentParser<SetupSection> {
-
-    @Override
-    public SetupSection parse(ClientYamlTestSuiteParseContext parseContext) throws IOException, ClientYamlTestParseException {
-
-        XContentParser parser = parseContext.parser();
-
-        SetupSection setupSection = new SetupSection();
-        setupSection.setSkipSection(parseContext.parseSkipSection());
-
-        while (parser.currentToken() != XContentParser.Token.END_ARRAY) {
-            parseContext.advanceToFieldName();
-            if (!"do".equals(parser.currentName())) {
-                throw new ClientYamlTestParseException("section [" + parser.currentName() + "] not supported within setup section");
-            }
-
-            parser.nextToken();
-            setupSection.addDoSection(parseContext.parseDoSection());
-            parser.nextToken();
-        }
-
-        parser.nextToken();
-
-        return setupSection;
-    }
-}
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/SkipSectionParser.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/SkipSectionParser.java
deleted file mode 100644
index b73edf7d2c..0000000000
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/SkipSectionParser.java
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.test.rest.yaml.parser;
-
-import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.test.rest.yaml.section.SkipSection;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * Parser for skip sections
- */
-public class SkipSectionParser implements ClientYamlTestFragmentParser<SkipSection> {
-
-    @Override
-    public SkipSection parse(ClientYamlTestSuiteParseContext parseContext) throws IOException, ClientYamlTestParseException {
-
-        XContentParser parser = parseContext.parser();
-
-        String currentFieldName = null;
-        XContentParser.Token token;
-        String version = null;
-        String reason = null;
-        List<String> features = new ArrayList<>();
-
-        while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
-            if (token == XContentParser.Token.FIELD_NAME) {
-                currentFieldName = parser.currentName();
-            } else if (token.isValue()) {
-                if ("version".equals(currentFieldName)) {
-                    version = parser.text();
-                } else if ("reason".equals(currentFieldName)) {
-                    reason = parser.text();
-                } else if ("features".equals(currentFieldName)) {
-                    features.add(parser.text());
-                }
-                else {
-                    throw new ClientYamlTestParseException("field " + currentFieldName + " not supported within skip section");
-                }
-            } else if (token == XContentParser.Token.START_ARRAY) {
-                if ("features".equals(currentFieldName)) {
-                    while(parser.nextToken() != XContentParser.Token.END_ARRAY) {
-                        features.add(parser.text());
-                    }
-                }
-            }
-        }
-
-        parser.nextToken();
-
-        if (!Strings.hasLength(version) && features.isEmpty()) {
-            throw new ClientYamlTestParseException("version or features is mandatory within skip section");
-        }
-        if (Strings.hasLength(version) && !Strings.hasLength(reason)) {
-            throw new ClientYamlTestParseException("reason is mandatory within skip version section");
-        }
-
-        return new SkipSection(version, features, reason);
-
-    }
-}
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/TeardownSectionParser.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/TeardownSectionParser.java
deleted file mode 100644
index ed1b42c3a9..0000000000
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/TeardownSectionParser.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.test.rest.yaml.parser;
-
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.test.rest.yaml.section.TeardownSection;
-
-import java.io.IOException;
-
-/**
- * Parser for teardown section
- */
-public class TeardownSectionParser implements ClientYamlTestFragmentParser<TeardownSection> {
-
-    @Override
-    public TeardownSection parse(ClientYamlTestSuiteParseContext parseContext) throws IOException, ClientYamlTestParseException {
-        XContentParser parser = parseContext.parser();
-
-        TeardownSection teardownSection = new TeardownSection();
-        teardownSection.setSkipSection(parseContext.parseSkipSection());
-
-        while (parser.currentToken() != XContentParser.Token.END_ARRAY) {
-            parseContext.advanceToFieldName();
-            if (!"do".equals(parser.currentName())) {
-                throw new ClientYamlTestParseException("section [" + parser.currentName() + "] not supported within teardown section");
-            }
-
-            parser.nextToken();
-            teardownSection.addDoSection(parseContext.parseDoSection());
-            parser.nextToken();
-        }
-
-        parser.nextToken();
-        return teardownSection;
-    }
-}
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSection.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSection.java
index cd9cc1fbe4..cbef40fbcb 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSection.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSection.java
@@ -18,6 +18,11 @@
  */
 package org.elasticsearch.test.rest.yaml.section;
 
+import org.elasticsearch.common.ParsingException;
+import org.elasticsearch.common.xcontent.XContentLocation;
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
 
@@ -25,15 +30,44 @@ import java.util.List;
  * Represents a test section, which is composed of a skip section and multiple executable sections.
  */
 public class ClientYamlTestSection implements Comparable<ClientYamlTestSection> {
+    public static ClientYamlTestSection parse(XContentParser parser) throws IOException {
+        ParserUtils.advanceToFieldName(parser);
+        ClientYamlTestSection testSection = new ClientYamlTestSection(parser.getTokenLocation(), parser.currentName());
+        try {
+            parser.nextToken();
+            testSection.setSkipSection(SkipSection.parseIfNext(parser));
+
+            while (parser.currentToken() != XContentParser.Token.END_ARRAY) {
+                ParserUtils.advanceToFieldName(parser);
+                testSection.addExecutableSection(ExecutableSection.parse(parser));
+            }
+
+            parser.nextToken();
+            assert parser.currentToken() == XContentParser.Token.END_OBJECT : "malformed section [" + testSection.getName() + "] expected "
+                + XContentParser.Token.END_OBJECT  + " but was " + parser.currentToken();
+            parser.nextToken();
+
+            return testSection;
+        } catch (Exception e) {
+            throw new ParsingException(parser.getTokenLocation(), "Error parsing test named [" + testSection.getName() + "]", e);
+        }
+    }
+
+    private final XContentLocation location;
     private final String name;
     private SkipSection skipSection;
     private final List<ExecutableSection> executableSections;
 
-    public ClientYamlTestSection(String name) {
+    public ClientYamlTestSection(XContentLocation location, String name) {
+        this.location = location;
         this.name = name;
         this.executableSections = new ArrayList<>();
     }
 
+    public XContentLocation getLocation() {
+        return location;
+    }
+
     public String getName() {
         return name;
     }
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSuite.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSuite.java
index af3e1be052..5f97eb4af6 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSuite.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSuite.java
@@ -18,6 +18,16 @@
  */
 package org.elasticsearch.test.rest.yaml.section;
 
+import org.elasticsearch.common.ParsingException;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.yaml.YamlXContent;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.channels.FileChannel;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardOpenOption;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Set;
@@ -28,6 +38,63 @@ import java.util.TreeSet;
  * Supports a setup section and multiple test sections.
  */
 public class ClientYamlTestSuite {
+    public static ClientYamlTestSuite parse(String api, Path file) throws IOException {
+        if (!Files.isRegularFile(file)) {
+            throw new IllegalArgumentException(file.toAbsolutePath() + " is not a file");
+        }
+
+        String filename = file.getFileName().toString();
+        //remove the file extension
+        int i = filename.lastIndexOf('.');
+        if (i > 0) {
+            filename = filename.substring(0, i);
+        }
+
+        //our yaml parser seems to be too tolerant. Each yaml suite must end with \n, otherwise clients tests might break.
+        try (FileChannel channel = FileChannel.open(file, StandardOpenOption.READ)) {
+            ByteBuffer bb = ByteBuffer.wrap(new byte[1]);
+            channel.read(bb, channel.size() - 1);
+            if (bb.get(0) != 10) {
+                throw new IOException("test suite [" + api + "/" + filename + "] doesn't end with line feed (\\n)");
+            }
+        }
+
+        try (XContentParser parser = YamlXContent.yamlXContent.createParser(ExecutableSection.XCONTENT_REGISTRY,
+                Files.newInputStream(file))) {
+            return parse(api, filename, parser);
+        } catch(Exception e) {
+            throw new IOException("Error parsing " + api + "/" + filename, e);
+        }
+    }
+
+    public static ClientYamlTestSuite parse(String api, String suiteName, XContentParser parser) throws IOException {
+        parser.nextToken();
+        assert parser.currentToken() == XContentParser.Token.START_OBJECT : "expected token to be START_OBJECT but was "
+                + parser.currentToken();
+
+        ClientYamlTestSuite restTestSuite = new ClientYamlTestSuite(api, suiteName);
+
+        restTestSuite.setSetupSection(SetupSection.parseIfNext(parser));
+        restTestSuite.setTeardownSection(TeardownSection.parseIfNext(parser));
+
+        while(true) {
+            //the "---" section separator is not understood by the yaml parser. null is returned, same as when the parser is closed
+            //we need to somehow distinguish between a null in the middle of a test ("---")
+            // and a null at the end of the file (at least two consecutive null tokens)
+            if(parser.currentToken() == null) {
+                if (parser.nextToken() == null) {
+                    break;
+                }
+            }
+
+            ClientYamlTestSection testSection = ClientYamlTestSection.parse(parser);
+            if (!restTestSuite.addTestSection(testSection)) {
+                throw new ParsingException(testSection.getLocation(), "duplicate test section [" + testSection.getName() + "]");
+            }
+        }
+
+        return restTestSuite;
+    }
 
     private final String api;
     private final String name;
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/DoSection.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/DoSection.java
index e233e9fab8..1729238040 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/DoSection.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/DoSection.java
@@ -19,15 +19,21 @@
 package org.elasticsearch.test.rest.yaml.section;
 
 import org.apache.logging.log4j.Logger;
+import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.logging.Loggers;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentLocation;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.test.rest.yaml.ClientYamlTestExecutionContext;
 import org.elasticsearch.test.rest.yaml.ClientYamlTestResponse;
 import org.elasticsearch.test.rest.yaml.ClientYamlTestResponseException;
 
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.LinkedHashSet;
 import java.util.List;
@@ -35,6 +41,7 @@ import java.util.Map;
 import java.util.Set;
 
 import static java.util.Collections.emptyList;
+import static java.util.Collections.unmodifiableList;
 import static org.elasticsearch.common.collect.Tuple.tuple;
 import static org.elasticsearch.test.hamcrest.RegexMatcher.matches;
 import static org.hamcrest.Matchers.allOf;
@@ -65,6 +72,86 @@ import static org.junit.Assert.fail;
  *
  */
 public class DoSection implements ExecutableSection {
+    public static DoSection parse(XContentParser parser) throws IOException {
+        String currentFieldName = null;
+        XContentParser.Token token;
+
+        DoSection doSection = new DoSection(parser.getTokenLocation());
+        ApiCallSection apiCallSection = null;
+        Map<String, String> headers = new HashMap<>();
+        List<String> expectedWarnings = new ArrayList<>();
+
+        while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+            if (token == XContentParser.Token.FIELD_NAME) {
+                currentFieldName = parser.currentName();
+            } else if (token.isValue()) {
+                if ("catch".equals(currentFieldName)) {
+                    doSection.setCatch(parser.text());
+                }
+            } else if (token == XContentParser.Token.START_ARRAY) {
+                if ("warnings".equals(currentFieldName)) {
+                    while ((token = parser.nextToken()) == XContentParser.Token.VALUE_STRING) {
+                        expectedWarnings.add(parser.text());
+                    }
+                    if (token != XContentParser.Token.END_ARRAY) {
+                        throw new ParsingException(parser.getTokenLocation(), "[warnings] must be a string array but saw [" + token + "]");
+                    }
+                } else {
+                    throw new ParsingException(parser.getTokenLocation(), "unknown array [" + currentFieldName + "]");
+                }
+            } else if (token == XContentParser.Token.START_OBJECT) {
+                if ("headers".equals(currentFieldName)) {
+                    String headerName = null;
+                    while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+                        if (token == XContentParser.Token.FIELD_NAME) {
+                            headerName = parser.currentName();
+                        } else if (token.isValue()) {
+                            headers.put(headerName, parser.text());
+                        }
+                    }
+                } else if (currentFieldName != null) { // must be part of API call then
+                    apiCallSection = new ApiCallSection(currentFieldName);
+                    String paramName = null;
+                    while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+                        if (token == XContentParser.Token.FIELD_NAME) {
+                            paramName = parser.currentName();
+                        } else if (token.isValue()) {
+                            if ("body".equals(paramName)) {
+                                String body = parser.text();
+                                XContentType bodyContentType = XContentFactory.xContentType(body);
+                                XContentParser bodyParser = XContentFactory.xContent(bodyContentType).createParser(
+                                        NamedXContentRegistry.EMPTY, body);
+                                //multiple bodies are supported e.g. in case of bulk provided as a whole string
+                                while(bodyParser.nextToken() != null) {
+                                    apiCallSection.addBody(bodyParser.mapOrdered());
+                                }
+                            } else {
+                                apiCallSection.addParam(paramName, parser.text());
+                            }
+                        } else if (token == XContentParser.Token.START_OBJECT) {
+                            if ("body".equals(paramName)) {
+                                apiCallSection.addBody(parser.mapOrdered());
+                            }
+                        }
+                    }
+                }
+            }
+        }
+        try {
+            if (apiCallSection == null) {
+                throw new IllegalArgumentException("client call section is mandatory within a do section");
+            }
+            if (headers.isEmpty() == false) {
+                apiCallSection.addHeaders(headers);
+            }
+            doSection.setApiCallSection(apiCallSection);
+            doSection.setExpectedWarningHeaders(unmodifiableList(expectedWarnings));
+        } finally {
+            parser.nextToken();
+        }
+        return doSection;
+    }
+
 
     private static final Logger logger = Loggers.getLogger(DoSection.class);
 
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ExecutableSection.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ExecutableSection.java
index 11b8f3e7a0..827457f4c2 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ExecutableSection.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ExecutableSection.java
@@ -18,16 +18,48 @@
  */
 package org.elasticsearch.test.rest.yaml.section;
 
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.XContentLocation;
+import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.test.rest.yaml.ClientYamlTestExecutionContext;
 
 import java.io.IOException;
+import java.util.Arrays;
 
 /**
  * Represents a test fragment that can be executed (e.g. api call, assertion)
  */
 public interface ExecutableSection {
     /**
+     * {@link NamedXContentRegistry} needed in the {@link XContentParser} before calling {@link ExecutableSection#parse(XContentParser)}.
+     */
+    NamedXContentRegistry XCONTENT_REGISTRY = new NamedXContentRegistry(Arrays.asList(
+            new NamedXContentRegistry.Entry(ExecutableSection.class, new ParseField("do"), DoSection::parse),
+            new NamedXContentRegistry.Entry(ExecutableSection.class, new ParseField("set"), SetSection::parse),
+            new NamedXContentRegistry.Entry(ExecutableSection.class, new ParseField("match"), MatchAssertion::parse),
+            new NamedXContentRegistry.Entry(ExecutableSection.class, new ParseField("is_true"), IsTrueAssertion::parse),
+            new NamedXContentRegistry.Entry(ExecutableSection.class, new ParseField("is_false"), IsFalseAssertion::parse),
+            new NamedXContentRegistry.Entry(ExecutableSection.class, new ParseField("gt"), GreaterThanAssertion::parse),
+            new NamedXContentRegistry.Entry(ExecutableSection.class, new ParseField("gte"), GreaterThanEqualToAssertion::parse),
+            new NamedXContentRegistry.Entry(ExecutableSection.class, new ParseField("lt"), LessThanAssertion::parse),
+            new NamedXContentRegistry.Entry(ExecutableSection.class, new ParseField("lte"), LessThanOrEqualToAssertion::parse),
+            new NamedXContentRegistry.Entry(ExecutableSection.class, new ParseField("length"), LengthAssertion::parse)));
+
+    static ExecutableSection parse(XContentParser parser) throws IOException {
+        ParserUtils.advanceToFieldName(parser);
+        String section = parser.currentName();
+        XContentLocation location = parser.getTokenLocation();
+        try {
+            ExecutableSection executableSection = parser.namedObject(ExecutableSection.class, section, null);
+            parser.nextToken();
+            return executableSection;
+        } catch (Exception e) {
+            throw new IOException("Error parsing section starting at [" + location + "]", e);
+        }
+    }
+
+    /**
      * Get the location in the test that this was defined. 
      */
     XContentLocation getLocation();
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/GreaterThanAssertion.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/GreaterThanAssertion.java
index b531f180fd..39a1f1d378 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/GreaterThanAssertion.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/GreaterThanAssertion.java
@@ -19,8 +19,12 @@
 package org.elasticsearch.test.rest.yaml.section;
 
 import org.apache.logging.log4j.Logger;
+import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.xcontent.XContentLocation;
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.io.IOException;
 
 import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.instanceOf;
@@ -33,6 +37,15 @@ import static org.junit.Assert.fail;
  * - gt:    { fields._ttl: 0}
  */
 public class GreaterThanAssertion extends Assertion {
+    public static GreaterThanAssertion parse(XContentParser parser) throws IOException {
+        XContentLocation location = parser.getTokenLocation();
+        Tuple<String,Object> stringObjectTuple = ParserUtils.parseTuple(parser);
+        if (! (stringObjectTuple.v2() instanceof Comparable) ) {
+            throw new IllegalArgumentException("gt section can only be used with objects that support natural ordering, found "
+                    + stringObjectTuple.v2().getClass().getSimpleName());
+        }
+        return new GreaterThanAssertion(location, stringObjectTuple.v1(), stringObjectTuple.v2());
+    }
 
     private static final Logger logger = Loggers.getLogger(GreaterThanAssertion.class);
 
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/GreaterThanEqualToAssertion.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/GreaterThanEqualToAssertion.java
index 14b1a08a87..3fd9bf7adf 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/GreaterThanEqualToAssertion.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/GreaterThanEqualToAssertion.java
@@ -20,8 +20,12 @@
 package org.elasticsearch.test.rest.yaml.section;
 
 import org.apache.logging.log4j.Logger;
+import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.xcontent.XContentLocation;
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.io.IOException;
 
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 import static org.hamcrest.Matchers.instanceOf;
@@ -34,6 +38,15 @@ import static org.junit.Assert.fail;
  *   - gte:     { fields._ttl: 0 }
  */
 public class GreaterThanEqualToAssertion extends Assertion {
+    public static GreaterThanEqualToAssertion parse(XContentParser parser) throws IOException {
+        XContentLocation location = parser.getTokenLocation();
+        Tuple<String,Object> stringObjectTuple = ParserUtils.parseTuple(parser);
+        if (! (stringObjectTuple.v2() instanceof Comparable) ) {
+            throw new IllegalArgumentException("gte section can only be used with objects that support natural ordering, found "
+                    + stringObjectTuple.v2().getClass().getSimpleName());
+        }
+        return new GreaterThanEqualToAssertion(location, stringObjectTuple.v1(), stringObjectTuple.v2());
+    }
 
     private static final Logger logger = Loggers.getLogger(GreaterThanEqualToAssertion.class);
 
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/IsFalseAssertion.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/IsFalseAssertion.java
index a356182ab4..56ee603c70 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/IsFalseAssertion.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/IsFalseAssertion.java
@@ -21,6 +21,9 @@ package org.elasticsearch.test.rest.yaml.section;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.xcontent.XContentLocation;
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.io.IOException;
 
 import static org.hamcrest.Matchers.anyOf;
 import static org.hamcrest.Matchers.equalTo;
@@ -34,6 +37,9 @@ import static org.junit.Assert.assertThat;
  *
  */
 public class IsFalseAssertion extends Assertion {
+    public static IsFalseAssertion parse(XContentParser parser) throws IOException {
+        return new IsFalseAssertion(parser.getTokenLocation(), ParserUtils.parseField(parser));
+    }
 
     private static final Logger logger = Loggers.getLogger(IsFalseAssertion.class);
 
@@ -42,7 +48,6 @@ public class IsFalseAssertion extends Assertion {
     }
 
     @Override
-    @SuppressWarnings("unchecked")
     protected void doAssert(Object actualValue, Object expectedValue) {
         logger.trace("assert that [{}] doesn't have a true value (field: [{}])", actualValue, getField());
 
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/IsTrueAssertion.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/IsTrueAssertion.java
index 76ca0de70d..9b3f37e1f5 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/IsTrueAssertion.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/IsTrueAssertion.java
@@ -21,6 +21,9 @@ package org.elasticsearch.test.rest.yaml.section;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.xcontent.XContentLocation;
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.io.IOException;
 
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.equalToIgnoringCase;
@@ -35,6 +38,9 @@ import static org.junit.Assert.assertThat;
  *
  */
 public class IsTrueAssertion extends Assertion {
+    public static IsTrueAssertion parse(XContentParser parser) throws IOException {
+        return new IsTrueAssertion(parser.getTokenLocation(), ParserUtils.parseField(parser));
+    }
 
     private static final Logger logger = Loggers.getLogger(IsTrueAssertion.class);
 
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/LengthAssertion.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/LengthAssertion.java
index 062b9ecd87..aeecc50b90 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/LengthAssertion.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/LengthAssertion.java
@@ -19,9 +19,12 @@
 package org.elasticsearch.test.rest.yaml.section;
 
 import org.apache.logging.log4j.Logger;
+import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.xcontent.XContentLocation;
+import org.elasticsearch.common.xcontent.XContentParser;
 
+import java.io.IOException;
 import java.util.List;
 import java.util.Map;
 
@@ -35,6 +38,22 @@ import static org.junit.Assert.assertThat;
  * - length:   { hits.hits: 1  }
  */
 public class LengthAssertion extends Assertion {
+    public static LengthAssertion parse(XContentParser parser) throws IOException {
+        XContentLocation location = parser.getTokenLocation();
+        Tuple<String,Object> stringObjectTuple = ParserUtils.parseTuple(parser);
+        assert stringObjectTuple.v2() != null;
+        int value;
+        if (stringObjectTuple.v2() instanceof Number) {
+            value = ((Number) stringObjectTuple.v2()).intValue();
+        } else {
+            try {
+                value = Integer.valueOf(stringObjectTuple.v2().toString());
+            } catch(NumberFormatException e) {
+                throw new IllegalArgumentException("length is not a valid number", e);
+            }
+        }
+        return new LengthAssertion(location, stringObjectTuple.v1(), value);
+    }
 
     private static final Logger logger = Loggers.getLogger(LengthAssertion.class);
 
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/LessThanAssertion.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/LessThanAssertion.java
index 591bd83fa6..75a1edcf81 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/LessThanAssertion.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/LessThanAssertion.java
@@ -19,8 +19,12 @@
 package org.elasticsearch.test.rest.yaml.section;
 
 import org.apache.logging.log4j.Logger;
+import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.xcontent.XContentLocation;
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.io.IOException;
 
 import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.lessThan;
@@ -34,6 +38,15 @@ import static org.junit.Assert.fail;
  *
  */
 public class LessThanAssertion extends Assertion {
+    public static LessThanAssertion parse(XContentParser parser) throws IOException {
+        XContentLocation location = parser.getTokenLocation();
+        Tuple<String,Object> stringObjectTuple = ParserUtils.parseTuple(parser);
+        if (false == stringObjectTuple.v2() instanceof Comparable) {
+            throw new IllegalArgumentException("lt section can only be used with objects that support natural ordering, found "
+                    + stringObjectTuple.v2().getClass().getSimpleName());
+        }
+        return new LessThanAssertion(location, stringObjectTuple.v1(), stringObjectTuple.v2());
+    }
 
     private static final Logger logger = Loggers.getLogger(LessThanAssertion.class);
 
@@ -42,7 +55,6 @@ public class LessThanAssertion extends Assertion {
     }
 
     @Override
-    @SuppressWarnings("unchecked")
     protected void doAssert(Object actualValue, Object expectedValue) {
         logger.trace("assert that [{}] is less than [{}] (field: [{}])", actualValue, expectedValue, getField());
         assertThat("value of [" + getField() + "] is not comparable (got [" + safeClass(actualValue) + "])",
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/LessThanOrEqualToAssertion.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/LessThanOrEqualToAssertion.java
index 7c5710f689..23b6a1e4ef 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/LessThanOrEqualToAssertion.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/LessThanOrEqualToAssertion.java
@@ -20,8 +20,12 @@
 package org.elasticsearch.test.rest.yaml.section;
 
 import org.apache.logging.log4j.Logger;
+import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.xcontent.XContentLocation;
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.io.IOException;
 
 import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.lessThanOrEqualTo;
@@ -34,6 +38,15 @@ import static org.junit.Assert.fail;
  *   - lte:     { fields._ttl: 0 }
  */
 public class LessThanOrEqualToAssertion  extends Assertion {
+    public static LessThanOrEqualToAssertion parse(XContentParser parser) throws IOException {
+        XContentLocation location = parser.getTokenLocation();
+        Tuple<String,Object> stringObjectTuple = ParserUtils.parseTuple(parser);
+        if (false == stringObjectTuple.v2() instanceof Comparable) {
+            throw new IllegalArgumentException("lte section can only be used with objects that support natural ordering, found "
+                    + stringObjectTuple.v2().getClass().getSimpleName());
+        }
+        return new LessThanOrEqualToAssertion(location, stringObjectTuple.v1(), stringObjectTuple.v2());
+    }
 
     private static final Logger logger = Loggers.getLogger(LessThanOrEqualToAssertion.class);
 
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/MatchAssertion.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/MatchAssertion.java
index 60e3424704..fb52e8ee69 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/MatchAssertion.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/MatchAssertion.java
@@ -20,9 +20,12 @@ package org.elasticsearch.test.rest.yaml.section;
 
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.xcontent.XContentLocation;
+import org.elasticsearch.common.xcontent.XContentParser;
 
+import java.io.IOException;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
@@ -43,6 +46,11 @@ import static org.junit.Assert.assertThat;
  *
  */
 public class MatchAssertion extends Assertion {
+    public static MatchAssertion parse(XContentParser parser) throws IOException {
+        XContentLocation location = parser.getTokenLocation();
+        Tuple<String,Object> stringObjectTuple = ParserUtils.parseTuple(parser);
+        return new MatchAssertion(location, stringObjectTuple.v1(), stringObjectTuple.v2());
+    }
 
     private static final Logger logger = Loggers.getLogger(MatchAssertion.class);
 
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ParserUtils.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ParserUtils.java
new file mode 100644
index 0000000000..b35450c66e
--- /dev/null
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ParserUtils.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.test.rest.yaml.section;
+
+import org.elasticsearch.common.collect.Tuple;
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.io.IOException;
+import java.util.Map;
+
+/**
+ * Utility methods used for parsing test sections.
+ */
+class ParserUtils {
+    private ParserUtils() {
+        // Do not build.
+    }
+
+    public static String parseField(XContentParser parser) throws IOException {
+        parser.nextToken();
+        assert parser.currentToken().isValue();
+        String field = parser.text();
+        parser.nextToken();
+        return field;
+    }
+
+    public static Tuple<String, Object> parseTuple(XContentParser parser) throws IOException {
+        parser.nextToken();
+        advanceToFieldName(parser);
+        Map<String,Object> map = parser.map();
+        assert parser.currentToken() == XContentParser.Token.END_OBJECT;
+        parser.nextToken();
+
+        if (map.size() != 1) {
+            throw new IllegalArgumentException("expected key value pair but found an object with " + map.size() + " fields");
+        }
+
+        Map.Entry<String, Object> entry = map.entrySet().iterator().next();
+        return Tuple.tuple(entry.getKey(), entry.getValue());
+    }
+
+    public static void advanceToFieldName(XContentParser parser) throws IOException {
+        XContentParser.Token token = parser.currentToken();
+        //we are in the beginning, haven't called nextToken yet
+        if (token == null) {
+            token = parser.nextToken();
+        }
+        if (token == XContentParser.Token.START_ARRAY) {
+            token = parser.nextToken();
+        }
+        if (token == XContentParser.Token.START_OBJECT) {
+            token = parser.nextToken();
+        }
+        if (token != XContentParser.Token.FIELD_NAME) {
+            throw new IllegalArgumentException("malformed test section: field name expected but found " + token + " at "
+                    + parser.getTokenLocation());
+        }
+    } 
+}
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/SetSection.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/SetSection.java
index aa39c4d30c..087f667ac9 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/SetSection.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/SetSection.java
@@ -18,7 +18,9 @@
  */
 package org.elasticsearch.test.rest.yaml.section;
 
+import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.xcontent.XContentLocation;
+import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.test.rest.yaml.ClientYamlTestExecutionContext;
 
 import java.io.IOException;
@@ -32,6 +34,28 @@ import java.util.Map;
  *
  */
 public class SetSection implements ExecutableSection {
+    public static SetSection parse(XContentParser parser) throws IOException {
+        String currentFieldName = null;
+        XContentParser.Token token;
+
+        SetSection setSection = new SetSection(parser.getTokenLocation());
+
+        while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+            if (token == XContentParser.Token.FIELD_NAME) {
+                currentFieldName = parser.currentName();
+            } else if (token.isValue()) {
+                setSection.addSet(currentFieldName, parser.text());
+            }
+        }
+
+        parser.nextToken();
+
+        if (setSection.getStash().isEmpty()) {
+            throw new ParsingException(setSection.location, "set section must set at least a value");
+        }
+
+        return setSection;
+    }
 
     private final Map<String, String> stash = new HashMap<>();
     private final XContentLocation location;
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/SetupSection.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/SetupSection.java
index c2bcaa3ecd..f0783cc1cc 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/SetupSection.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/SetupSection.java
@@ -18,6 +18,9 @@
  */
 package org.elasticsearch.test.rest.yaml.section;
 
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
 
@@ -25,6 +28,41 @@ import java.util.List;
  * Represents a setup section. Holds a skip section and multiple do sections.
  */
 public class SetupSection {
+    /**
+     * Parse a {@link SetupSection} if the next field is {@code skip}, otherwise returns {@link SetupSection#EMPTY}.
+     */
+    public static SetupSection parseIfNext(XContentParser parser) throws IOException {
+        ParserUtils.advanceToFieldName(parser);
+
+        if ("setup".equals(parser.currentName())) {
+            parser.nextToken();
+            SetupSection section = parse(parser);
+            parser.nextToken();
+            return section;
+        }
+
+        return EMPTY;
+    }
+
+    public static SetupSection parse(XContentParser parser) throws IOException {
+        SetupSection setupSection = new SetupSection();
+        setupSection.setSkipSection(SkipSection.parseIfNext(parser));
+
+        while (parser.currentToken() != XContentParser.Token.END_ARRAY) {
+            ParserUtils.advanceToFieldName(parser);
+            if (!"do".equals(parser.currentName())) {
+                throw new IllegalArgumentException("section [" + parser.currentName() + "] not supported within setup section");
+            }
+
+            parser.nextToken();
+            setupSection.addDoSection(DoSection.parse(parser));
+            parser.nextToken();
+        }
+
+        parser.nextToken();
+
+        return setupSection;
+    }
 
     public static final SetupSection EMPTY;
 
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/SkipSection.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/SkipSection.java
index e508c3c101..62588baf46 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/SkipSection.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/SkipSection.java
@@ -19,9 +19,13 @@
 package org.elasticsearch.test.rest.yaml.section;
 
 import org.elasticsearch.Version;
+import org.elasticsearch.common.ParsingException;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.test.VersionUtils;
 import org.elasticsearch.test.rest.yaml.Features;
 
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
 
@@ -32,6 +36,63 @@ import java.util.List;
  * - a specific test feature required that might not be implemented yet by the runner
  */
 public class SkipSection {
+    /**
+     * Parse a {@link SkipSection} if the next field is {@code skip}, otherwise returns {@link SkipSection#EMPTY}.
+     */
+    public static SkipSection parseIfNext(XContentParser parser) throws IOException {
+        ParserUtils.advanceToFieldName(parser);
+
+        if ("skip".equals(parser.currentName())) {
+            SkipSection section = parse(parser);
+            parser.nextToken();
+            return section;
+        }
+
+        return EMPTY;
+    }
+
+    public static SkipSection parse(XContentParser parser) throws IOException {
+        String currentFieldName = null;
+        XContentParser.Token token;
+        String version = null;
+        String reason = null;
+        List<String> features = new ArrayList<>();
+
+        while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+            if (token == XContentParser.Token.FIELD_NAME) {
+                currentFieldName = parser.currentName();
+            } else if (token.isValue()) {
+                if ("version".equals(currentFieldName)) {
+                    version = parser.text();
+                } else if ("reason".equals(currentFieldName)) {
+                    reason = parser.text();
+                } else if ("features".equals(currentFieldName)) {
+                    features.add(parser.text());
+                }
+                else {
+                    throw new ParsingException(parser.getTokenLocation(),
+                            "field " + currentFieldName + " not supported within skip section");
+                }
+            } else if (token == XContentParser.Token.START_ARRAY) {
+                if ("features".equals(currentFieldName)) {
+                    while(parser.nextToken() != XContentParser.Token.END_ARRAY) {
+                        features.add(parser.text());
+                    }
+                }
+            }
+        }
+
+        parser.nextToken();
+
+        if (!Strings.hasLength(version) && features.isEmpty()) {
+            throw new ParsingException(parser.getTokenLocation(), "version or features is mandatory within skip section");
+        }
+        if (Strings.hasLength(version) && !Strings.hasLength(reason)) {
+            throw new ParsingException(parser.getTokenLocation(), "reason is mandatory within skip version section");
+        }
+
+        return new SkipSection(version, features, reason);
+    }
 
     public static final SkipSection EMPTY = new SkipSection();
 
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/TeardownSection.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/TeardownSection.java
index 1e49f8b503..1ae4d41a03 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/TeardownSection.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/TeardownSection.java
@@ -19,10 +19,49 @@
 
 package org.elasticsearch.test.rest.yaml.section;
 
+import org.elasticsearch.common.ParsingException;
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
 
 public class TeardownSection {
+    /**
+     * Parse a {@link TeardownSection} if the next field is {@code skip}, otherwise returns {@link TeardownSection#EMPTY}.
+     */
+    public static TeardownSection parseIfNext(XContentParser parser) throws IOException {
+        ParserUtils.advanceToFieldName(parser);
+
+        if ("teardown".equals(parser.currentName())) {
+            parser.nextToken();
+            TeardownSection section = parse(parser);
+            parser.nextToken();
+            return section;
+        }
+
+        return EMPTY;
+    }
+
+    public static TeardownSection parse(XContentParser parser) throws IOException {
+        TeardownSection teardownSection = new TeardownSection();
+        teardownSection.setSkipSection(SkipSection.parseIfNext(parser));
+
+        while (parser.currentToken() != XContentParser.Token.END_ARRAY) {
+            ParserUtils.advanceToFieldName(parser);
+            if (!"do".equals(parser.currentName())) {
+                throw new ParsingException(parser.getTokenLocation(),
+                        "section [" + parser.currentName() + "] not supported within teardown section");
+            }
+
+            parser.nextToken();
+            teardownSection.addDoSection(DoSection.parse(parser));
+            parser.nextToken();
+        }
+
+        parser.nextToken();
+        return teardownSection;
+    }
 
     public static final TeardownSection EMPTY;
 
diff --git a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/parser/DoSectionParserTests.java b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/parser/DoSectionParserTests.java
deleted file mode 100644
index 6057de6c70..0000000000
--- a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/parser/DoSectionParserTests.java
+++ /dev/null
@@ -1,455 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.test.rest.yaml.parser;
-
-import org.elasticsearch.common.xcontent.XContent;
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.common.xcontent.yaml.YamlXContent;
-import org.elasticsearch.test.rest.yaml.section.ApiCallSection;
-import org.elasticsearch.test.rest.yaml.section.DoSection;
-import org.hamcrest.MatcherAssert;
-
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.Map;
-
-import static java.util.Collections.singletonList;
-import static org.hamcrest.CoreMatchers.equalTo;
-import static org.hamcrest.Matchers.is;
-import static org.hamcrest.Matchers.notNullValue;
-import static org.hamcrest.Matchers.nullValue;
-
-public class DoSectionParserTests extends AbstractClientYamlTestFragmentParserTestCase {
-    public void testParseDoSectionNoBody() throws Exception {
-        parser = createParser(YamlXContent.yamlXContent,
-                "get:\n" +
-                "    index:    test_index\n" +
-                "    type:    test_type\n" +
-                "    id:        1"
-        );
-
-        DoSectionParser doSectionParser = new DoSectionParser();
-        DoSection doSection = doSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
-        ApiCallSection apiCallSection = doSection.getApiCallSection();
-
-        assertThat(apiCallSection, notNullValue());
-        assertThat(apiCallSection.getApi(), equalTo("get"));
-        assertThat(apiCallSection.getParams().size(), equalTo(3));
-        assertThat(apiCallSection.getParams().get("index"), equalTo("test_index"));
-        assertThat(apiCallSection.getParams().get("type"), equalTo("test_type"));
-        assertThat(apiCallSection.getParams().get("id"), equalTo("1"));
-        assertThat(apiCallSection.hasBody(), equalTo(false));
-    }
-
-    public void testParseDoSectionNoParamsNoBody() throws Exception {
-        parser = createParser(YamlXContent.yamlXContent,
-                "cluster.node_info: {}"
-        );
-
-        DoSectionParser doSectionParser = new DoSectionParser();
-        DoSection doSection = doSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
-        ApiCallSection apiCallSection = doSection.getApiCallSection();
-
-        assertThat(apiCallSection, notNullValue());
-        assertThat(apiCallSection.getApi(), equalTo("cluster.node_info"));
-        assertThat(apiCallSection.getParams().size(), equalTo(0));
-        assertThat(apiCallSection.hasBody(), equalTo(false));
-    }
-
-    public void testParseDoSectionWithJsonBody() throws Exception {
-        String body = "{ \"include\": { \"field1\": \"v1\", \"field2\": \"v2\" }, \"count\": 1 }";
-        parser = createParser(YamlXContent.yamlXContent,
-                "index:\n" +
-                "    index:  test_1\n" +
-                "    type:   test\n" +
-                "    id:     1\n" +
-                "    body:   " + body
-        );
-
-        DoSectionParser doSectionParser = new DoSectionParser();
-        DoSection doSection = doSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
-        ApiCallSection apiCallSection = doSection.getApiCallSection();
-
-        assertThat(apiCallSection, notNullValue());
-        assertThat(apiCallSection.getApi(), equalTo("index"));
-        assertThat(apiCallSection.getParams().size(), equalTo(3));
-        assertThat(apiCallSection.getParams().get("index"), equalTo("test_1"));
-        assertThat(apiCallSection.getParams().get("type"), equalTo("test"));
-        assertThat(apiCallSection.getParams().get("id"), equalTo("1"));
-        assertThat(apiCallSection.hasBody(), equalTo(true));
-
-        assertJsonEquals(apiCallSection.getBodies().get(0), body);
-    }
-
-    public void testParseDoSectionWithJsonMultipleBodiesAsLongString() throws Exception {
-        String bodies[] = new String[]{
-                "{ \"index\": { \"_index\":\"test_index\", \"_type\":\"test_type\", \"_id\":\"test_id\" } }\n",
-                "{ \"f1\":\"v1\", \"f2\":42 }\n",
-                "{ \"index\": { \"_index\":\"test_index2\", \"_type\":\"test_type2\", \"_id\":\"test_id2\" } }\n",
-                "{ \"f1\":\"v2\", \"f2\":47 }\n"
-        };
-        parser = createParser(YamlXContent.yamlXContent,
-                "bulk:\n" +
-                "    refresh: true\n" +
-                "    body: |\n" +
-                "        " + bodies[0] +
-                "        " + bodies[1] +
-                "        " + bodies[2] +
-                "        " + bodies[3]
-        );
-
-        DoSectionParser doSectionParser = new DoSectionParser();
-        DoSection doSection = doSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
-        ApiCallSection apiCallSection = doSection.getApiCallSection();
-
-        assertThat(apiCallSection, notNullValue());
-        assertThat(apiCallSection.getApi(), equalTo("bulk"));
-        assertThat(apiCallSection.getParams().size(), equalTo(1));
-        assertThat(apiCallSection.getParams().get("refresh"), equalTo("true"));
-        assertThat(apiCallSection.hasBody(), equalTo(true));
-        assertThat(apiCallSection.getBodies().size(), equalTo(4));
-    }
-
-    public void testParseDoSectionWithJsonMultipleBodiesRepeatedProperty() throws Exception {
-        assumeFalse("Test only makes sense if XContent parser doesn't have strict duplicate checks enabled",
-            XContent.isStrictDuplicateDetectionEnabled());
-
-        String[] bodies = new String[] {
-                "{ \"index\": { \"_index\":\"test_index\", \"_type\":\"test_type\", \"_id\":\"test_id\" } }",
-                "{ \"f1\":\"v1\", \"f2\":42 }",
-        };
-        parser = createParser(YamlXContent.yamlXContent,
-                "bulk:\n" +
-                "    refresh: true\n" +
-                "    body: \n" +
-                "        " + bodies[0] + "\n" +
-                "    body: \n" +
-                "        " + bodies[1]
-        );
-
-        DoSectionParser doSectionParser = new DoSectionParser();
-        DoSection doSection = doSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
-        ApiCallSection apiCallSection = doSection.getApiCallSection();
-
-        assertThat(apiCallSection, notNullValue());
-        assertThat(apiCallSection.getApi(), equalTo("bulk"));
-        assertThat(apiCallSection.getParams().size(), equalTo(1));
-        assertThat(apiCallSection.getParams().get("refresh"), equalTo("true"));
-        assertThat(apiCallSection.hasBody(), equalTo(true));
-        assertThat(apiCallSection.getBodies().size(), equalTo(bodies.length));
-        for (int i = 0; i < bodies.length; i++) {
-            assertJsonEquals(apiCallSection.getBodies().get(i), bodies[i]);
-        }
-    }
-
-    public void testParseDoSectionWithYamlBody() throws Exception {
-        parser = createParser(YamlXContent.yamlXContent,
-                "search:\n" +
-                "    body:\n" +
-                "        \"_source\": [ include.field1, include.field2 ]\n" +
-                "        \"query\": { \"match_all\": {} }"
-        );
-        String body = "{ \"_source\": [ \"include.field1\", \"include.field2\" ], \"query\": { \"match_all\": {} }}";
-
-        DoSectionParser doSectionParser = new DoSectionParser();
-        DoSection doSection = doSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
-        ApiCallSection apiCallSection = doSection.getApiCallSection();
-
-        assertThat(apiCallSection, notNullValue());
-        assertThat(apiCallSection.getApi(), equalTo("search"));
-        assertThat(apiCallSection.getParams().size(), equalTo(0));
-        assertThat(apiCallSection.hasBody(), equalTo(true));
-        assertThat(apiCallSection.getBodies().size(), equalTo(1));
-        assertJsonEquals(apiCallSection.getBodies().get(0), body);
-    }
-
-    public void testParseDoSectionWithYamlMultipleBodies() throws Exception {
-        parser = createParser(YamlXContent.yamlXContent,
-                "bulk:\n" +
-                "    refresh: true\n" +
-                "    body:\n" +
-                "        - index:\n" +
-                "            _index: test_index\n" +
-                "            _type:  test_type\n" +
-                "            _id:    test_id\n" +
-                "        - f1: v1\n" +
-                "          f2: 42\n" +
-                "        - index:\n" +
-                "            _index: test_index2\n" +
-                "            _type:  test_type2\n" +
-                "            _id:    test_id2\n" +
-                "        - f1: v2\n" +
-                "          f2: 47"
-        );
-        String[] bodies = new String[4];
-        bodies[0] = "{\"index\": {\"_index\": \"test_index\", \"_type\":  \"test_type\", \"_id\": \"test_id\"}}";
-        bodies[1] = "{ \"f1\":\"v1\", \"f2\": 42 }";
-        bodies[2] = "{\"index\": {\"_index\": \"test_index2\", \"_type\":  \"test_type2\", \"_id\": \"test_id2\"}}";
-        bodies[3] = "{ \"f1\":\"v2\", \"f2\": 47 }";
-
-        DoSectionParser doSectionParser = new DoSectionParser();
-        DoSection doSection = doSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
-        ApiCallSection apiCallSection = doSection.getApiCallSection();
-
-        assertThat(apiCallSection, notNullValue());
-        assertThat(apiCallSection.getApi(), equalTo("bulk"));
-        assertThat(apiCallSection.getParams().size(), equalTo(1));
-        assertThat(apiCallSection.getParams().get("refresh"), equalTo("true"));
-        assertThat(apiCallSection.hasBody(), equalTo(true));
-        assertThat(apiCallSection.getBodies().size(), equalTo(bodies.length));
-
-        for (int i = 0; i < bodies.length; i++) {
-            assertJsonEquals(apiCallSection.getBodies().get(i), bodies[i]);
-        }
-    }
-
-    public void testParseDoSectionWithYamlMultipleBodiesRepeatedProperty() throws Exception {
-        assumeFalse("Test only makes sense if XContent parser doesn't have strict duplicate checks enabled",
-            XContent.isStrictDuplicateDetectionEnabled());
-
-        parser = createParser(YamlXContent.yamlXContent,
-                "bulk:\n" +
-                "    refresh: true\n" +
-                "    body:\n" +
-                "        index:\n" +
-                "            _index: test_index\n" +
-                "            _type:  test_type\n" +
-                "            _id:    test_id\n" +
-                "    body:\n" +
-                "        f1: v1\n" +
-                "        f2: 42\n"
-        );
-        String[] bodies = new String[2];
-        bodies[0] = "{\"index\": {\"_index\": \"test_index\", \"_type\":  \"test_type\", \"_id\": \"test_id\"}}";
-        bodies[1] = "{ \"f1\":\"v1\", \"f2\": 42 }";
-
-        DoSectionParser doSectionParser = new DoSectionParser();
-        DoSection doSection = doSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
-        ApiCallSection apiCallSection = doSection.getApiCallSection();
-
-        assertThat(apiCallSection, notNullValue());
-        assertThat(apiCallSection.getApi(), equalTo("bulk"));
-        assertThat(apiCallSection.getParams().size(), equalTo(1));
-        assertThat(apiCallSection.getParams().get("refresh"), equalTo("true"));
-        assertThat(apiCallSection.hasBody(), equalTo(true));
-        assertThat(apiCallSection.getBodies().size(), equalTo(bodies.length));
-
-        for (int i = 0; i < bodies.length; i++) {
-            assertJsonEquals(apiCallSection.getBodies().get(i), bodies[i]);
-        }
-    }
-
-    public void testParseDoSectionWithYamlBodyMultiGet() throws Exception {
-        parser = createParser(YamlXContent.yamlXContent,
-                "mget:\n" +
-                "    body:\n" +
-                "        docs:\n" +
-                "            - { _index: test_2, _type: test, _id: 1}\n" +
-                "            - { _index: test_1, _type: none, _id: 1}"
-        );
-        String body = "{ \"docs\": [ " +
-                "{\"_index\": \"test_2\", \"_type\":\"test\", \"_id\":1}, " +
-                "{\"_index\": \"test_1\", \"_type\":\"none\", \"_id\":1} " +
-                "]}";
-
-        DoSectionParser doSectionParser = new DoSectionParser();
-        DoSection doSection = doSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
-        ApiCallSection apiCallSection = doSection.getApiCallSection();
-
-        assertThat(apiCallSection, notNullValue());
-        assertThat(apiCallSection.getApi(), equalTo("mget"));
-        assertThat(apiCallSection.getParams().size(), equalTo(0));
-        assertThat(apiCallSection.hasBody(), equalTo(true));
-        assertThat(apiCallSection.getBodies().size(), equalTo(1));
-        assertJsonEquals(apiCallSection.getBodies().get(0), body);
-    }
-
-    public void testParseDoSectionWithBodyStringified() throws Exception {
-        parser = createParser(YamlXContent.yamlXContent,
-                "index:\n" +
-                "    index:  test_1\n" +
-                "    type:   test\n" +
-                "    id:     1\n" +
-                "    body:   \"{ \\\"_source\\\": true, \\\"query\\\": { \\\"match_all\\\": {} } }\""
-        );
-
-        DoSectionParser doSectionParser = new DoSectionParser();
-        DoSection doSection = doSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
-        ApiCallSection apiCallSection = doSection.getApiCallSection();
-
-        assertThat(apiCallSection, notNullValue());
-        assertThat(apiCallSection.getApi(), equalTo("index"));
-        assertThat(apiCallSection.getParams().size(), equalTo(3));
-        assertThat(apiCallSection.getParams().get("index"), equalTo("test_1"));
-        assertThat(apiCallSection.getParams().get("type"), equalTo("test"));
-        assertThat(apiCallSection.getParams().get("id"), equalTo("1"));
-        assertThat(apiCallSection.hasBody(), equalTo(true));
-        assertThat(apiCallSection.getBodies().size(), equalTo(1));
-        //stringified body is taken as is
-        assertJsonEquals(apiCallSection.getBodies().get(0), "{ \"_source\": true, \"query\": { \"match_all\": {} } }");
-    }
-
-    public void testParseDoSectionWithBodiesStringifiedAndNot() throws Exception {
-        parser = createParser(YamlXContent.yamlXContent,
-                "index:\n" +
-                "    body:\n" +
-                "        - \"{ \\\"_source\\\": true, \\\"query\\\": { \\\"match_all\\\": {} } }\"\n" +
-                "        - { size: 100, query: { match_all: {} } }"
-        );
-
-        String body = "{ \"size\": 100, \"query\": { \"match_all\": {} } }";
-
-        DoSectionParser doSectionParser = new DoSectionParser();
-        DoSection doSection = doSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
-        ApiCallSection apiCallSection = doSection.getApiCallSection();
-
-        assertThat(apiCallSection.getApi(), equalTo("index"));
-        assertThat(apiCallSection.getParams().size(), equalTo(0));
-        assertThat(apiCallSection.hasBody(), equalTo(true));
-        assertThat(apiCallSection.getBodies().size(), equalTo(2));
-        //stringified body is taken as is
-        assertJsonEquals(apiCallSection.getBodies().get(0), "{ \"_source\": true, \"query\": { \"match_all\": {} } }");
-        assertJsonEquals(apiCallSection.getBodies().get(1), body);
-    }
-
-    public void testParseDoSectionWithCatch() throws Exception {
-        parser = createParser(YamlXContent.yamlXContent,
-                "catch: missing\n" +
-                "indices.get_warmer:\n" +
-                "    index: test_index\n" +
-                "    name: test_warmer"
-        );
-
-        DoSectionParser doSectionParser = new DoSectionParser();
-        DoSection doSection = doSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
-
-        assertThat(doSection.getCatch(), equalTo("missing"));
-        assertThat(doSection.getApiCallSection(), notNullValue());
-        assertThat(doSection.getApiCallSection().getApi(), equalTo("indices.get_warmer"));
-        assertThat(doSection.getApiCallSection().getParams().size(), equalTo(2));
-        assertThat(doSection.getApiCallSection().hasBody(), equalTo(false));
-    }
-
-    public void testParseDoSectionWithHeaders() throws Exception {
-        parser = createParser(YamlXContent.yamlXContent,
-                "headers:\n" +
-                "    Authorization: \"thing one\"\n" +
-                "    Content-Type: \"application/json\"\n" +
-                "indices.get_warmer:\n" +
-                "    index: test_index\n" +
-                "    name: test_warmer"
-        );
-
-        DoSectionParser doSectionParser = new DoSectionParser();
-        DoSection doSection = doSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
-
-        assertThat(doSection.getApiCallSection(), notNullValue());
-        assertThat(doSection.getApiCallSection().getApi(), equalTo("indices.get_warmer"));
-        assertThat(doSection.getApiCallSection().getParams().size(), equalTo(2));
-        assertThat(doSection.getApiCallSection().hasBody(), equalTo(false));
-        assertThat(doSection.getApiCallSection().getHeaders(), notNullValue());
-        assertThat(doSection.getApiCallSection().getHeaders().size(), equalTo(2));
-        assertThat(doSection.getApiCallSection().getHeaders().get("Authorization"), equalTo("thing one"));
-        assertThat(doSection.getApiCallSection().getHeaders().get("Content-Type"), equalTo("application/json"));
-    }
-
-    public void testParseDoSectionWithoutClientCallSection() throws Exception {
-        parser = createParser(YamlXContent.yamlXContent,
-                "catch: missing\n"
-        );
-
-        DoSectionParser doSectionParser = new DoSectionParser();
-        try {
-            doSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
-            fail("Expected RestTestParseException");
-        } catch (ClientYamlTestParseException e) {
-            assertThat(e.getMessage(), is("client call section is mandatory within a do section"));
-        }
-    }
-
-    public void testParseDoSectionMultivaluedField() throws Exception {
-        parser = createParser(YamlXContent.yamlXContent,
-                "indices.get_field_mapping:\n" +
-                "        index: test_index\n" +
-                "        type: test_type\n" +
-                "        field: [ text , text1 ]"
-        );
-
-        DoSectionParser doSectionParser = new DoSectionParser();
-        DoSection doSection = doSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
-
-        assertThat(doSection.getCatch(), nullValue());
-        assertThat(doSection.getApiCallSection(), notNullValue());
-        assertThat(doSection.getApiCallSection().getApi(), equalTo("indices.get_field_mapping"));
-        assertThat(doSection.getApiCallSection().getParams().size(), equalTo(3));
-        assertThat(doSection.getApiCallSection().getParams().get("index"), equalTo("test_index"));
-        assertThat(doSection.getApiCallSection().getParams().get("type"), equalTo("test_type"));
-        assertThat(doSection.getApiCallSection().getParams().get("field"), equalTo("text,text1"));
-        assertThat(doSection.getApiCallSection().hasBody(), equalTo(false));
-        assertThat(doSection.getApiCallSection().getBodies().size(), equalTo(0));
-    }
-
-    public void testParseDoSectionExpectedWarnings() throws Exception {
-        parser = createParser(YamlXContent.yamlXContent,
-                "indices.get_field_mapping:\n" +
-                "        index: test_index\n" +
-                "        type: test_type\n" +
-                "warnings:\n" +
-                "    - some test warning they are typically pretty long\n" +
-                "    - some other test warning somtimes they have [in] them"
-        );
-
-        DoSectionParser doSectionParser = new DoSectionParser();
-        DoSection doSection = doSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
-
-        assertThat(doSection.getCatch(), nullValue());
-        assertThat(doSection.getApiCallSection(), notNullValue());
-        assertThat(doSection.getApiCallSection().getApi(), equalTo("indices.get_field_mapping"));
-        assertThat(doSection.getApiCallSection().getParams().size(), equalTo(2));
-        assertThat(doSection.getApiCallSection().getParams().get("index"), equalTo("test_index"));
-        assertThat(doSection.getApiCallSection().getParams().get("type"), equalTo("test_type"));
-        assertThat(doSection.getApiCallSection().hasBody(), equalTo(false));
-        assertThat(doSection.getApiCallSection().getBodies().size(), equalTo(0));
-        assertThat(doSection.getExpectedWarningHeaders(), equalTo(Arrays.asList(
-                "some test warning they are typically pretty long",
-                "some other test warning somtimes they have [in] them")));
-
-        parser = createParser(YamlXContent.yamlXContent,
-                "indices.get_field_mapping:\n" +
-                "        index: test_index\n" +
-                "warnings:\n" +
-                "    - just one entry this time"
-        );
-
-        doSection = doSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
-        assertThat(doSection.getCatch(), nullValue());
-        assertThat(doSection.getApiCallSection(), notNullValue());
-        assertThat(doSection.getExpectedWarningHeaders(), equalTo(singletonList(
-                "just one entry this time")));
-
-    }
-
-    private void assertJsonEquals(Map<String, Object> actual, String expected) throws IOException {
-        Map<String,Object> expectedMap;
-        try (XContentParser parser = createParser(YamlXContent.yamlXContent, expected)) {
-            expectedMap = parser.mapOrdered();
-        }
-        MatcherAssert.assertThat(actual, equalTo(expectedMap));
-    }
-}
diff --git a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/parser/SkipSectionParserTests.java b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/parser/SkipSectionParserTests.java
deleted file mode 100644
index 9f488fd9d5..0000000000
--- a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/parser/SkipSectionParserTests.java
+++ /dev/null
@@ -1,144 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.test.rest.yaml.parser;
-
-import org.elasticsearch.Version;
-import org.elasticsearch.common.xcontent.yaml.YamlXContent;
-import org.elasticsearch.test.VersionUtils;
-import org.elasticsearch.test.rest.yaml.section.SkipSection;
-
-import java.util.Arrays;
-
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.is;
-import static org.hamcrest.Matchers.notNullValue;
-import static org.hamcrest.Matchers.nullValue;
-
-public class SkipSectionParserTests extends AbstractClientYamlTestFragmentParserTestCase {
-    public void testParseSkipSectionVersionNoFeature() throws Exception {
-        parser = createParser(YamlXContent.yamlXContent,
-                "version:     \" - 2.1.0\"\n" +
-                "reason:      Delete ignores the parent param"
-        );
-
-        SkipSectionParser skipSectionParser = new SkipSectionParser();
-
-        SkipSection skipSection = skipSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
-
-        assertThat(skipSection, notNullValue());
-        assertThat(skipSection.getLowerVersion(), equalTo(VersionUtils.getFirstVersion()));
-        assertThat(skipSection.getUpperVersion(), equalTo(Version.V_2_1_0));
-        assertThat(skipSection.getFeatures().size(), equalTo(0));
-        assertThat(skipSection.getReason(), equalTo("Delete ignores the parent param"));
-    }
-
-    public void testParseSkipSectionAllVersions() throws Exception {
-        parser = createParser(YamlXContent.yamlXContent,
-            "version:     \" all \"\n" +
-            "reason:      Delete ignores the parent param"
-        );
-
-        SkipSectionParser skipSectionParser = new SkipSectionParser();
-
-        SkipSection skipSection = skipSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
-
-        assertThat(skipSection, notNullValue());
-        assertThat(skipSection.getLowerVersion(), equalTo(VersionUtils.getFirstVersion()));
-        assertThat(skipSection.getUpperVersion(), equalTo(Version.CURRENT));
-        assertThat(skipSection.getFeatures().size(), equalTo(0));
-        assertThat(skipSection.getReason(), equalTo("Delete ignores the parent param"));
-    }
-
-    public void testParseSkipSectionFeatureNoVersion() throws Exception {
-        parser = createParser(YamlXContent.yamlXContent,
-                "features:     regex"
-        );
-
-        SkipSectionParser skipSectionParser = new SkipSectionParser();
-
-        SkipSection skipSection = skipSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
-
-        assertThat(skipSection, notNullValue());
-        assertThat(skipSection.isVersionCheck(), equalTo(false));
-        assertThat(skipSection.getFeatures().size(), equalTo(1));
-        assertThat(skipSection.getFeatures().get(0), equalTo("regex"));
-        assertThat(skipSection.getReason(), nullValue());
-    }
-
-    public void testParseSkipSectionFeaturesNoVersion() throws Exception {
-        parser = createParser(YamlXContent.yamlXContent,
-                "features:     [regex1,regex2,regex3]"
-        );
-
-        SkipSectionParser skipSectionParser = new SkipSectionParser();
-
-        SkipSection skipSection = skipSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
-
-        assertThat(skipSection, notNullValue());
-        assertThat(skipSection.isVersionCheck(), equalTo(false));
-        assertThat(skipSection.getFeatures().size(), equalTo(3));
-        assertThat(skipSection.getFeatures().get(0), equalTo("regex1"));
-        assertThat(skipSection.getFeatures().get(1), equalTo("regex2"));
-        assertThat(skipSection.getFeatures().get(2), equalTo("regex3"));
-        assertThat(skipSection.getReason(), nullValue());
-    }
-
-    public void testParseSkipSectionBothFeatureAndVersion() throws Exception {
-        parser = createParser(YamlXContent.yamlXContent,
-                "version:     \" - 0.90.2\"\n" +
-                "features:     regex\n" +
-                "reason:      Delete ignores the parent param"
-        );
-
-        SkipSectionParser skipSectionParser = new SkipSectionParser();
-        SkipSection parse = skipSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
-        assertEquals(VersionUtils.getFirstVersion(), parse.getLowerVersion());
-        assertEquals(Version.fromString("0.90.2"), parse.getUpperVersion());
-        assertEquals(Arrays.asList("regex"), parse.getFeatures());
-        assertEquals("Delete ignores the parent param", parse.getReason());
-    }
-
-    public void testParseSkipSectionNoReason() throws Exception {
-        parser = createParser(YamlXContent.yamlXContent,
-                "version:     \" - 0.90.2\"\n"
-        );
-
-        SkipSectionParser skipSectionParser = new SkipSectionParser();
-        try {
-            skipSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
-            fail("Expected RestTestParseException");
-        } catch (ClientYamlTestParseException e) {
-            assertThat(e.getMessage(), is("reason is mandatory within skip version section"));
-        }
-    }
-
-    public void testParseSkipSectionNoVersionNorFeature() throws Exception {
-        parser = createParser(YamlXContent.yamlXContent,
-                "reason:      Delete ignores the parent param\n"
-        );
-
-        SkipSectionParser skipSectionParser = new SkipSectionParser();
-        try {
-            skipSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
-            fail("Expected RestTestParseException");
-        } catch (ClientYamlTestParseException e) {
-            assertThat(e.getMessage(), is("version or features is mandatory within skip section"));
-        }
-    }
-}
diff --git a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/parser/TestSectionParserTests.java b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/parser/TestSectionParserTests.java
deleted file mode 100644
index 5673f20f06..0000000000
--- a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/parser/TestSectionParserTests.java
+++ /dev/null
@@ -1,249 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.test.rest.yaml.parser;
-
-import org.elasticsearch.Version;
-import org.elasticsearch.common.xcontent.yaml.YamlXContent;
-import org.elasticsearch.test.rest.yaml.section.ClientYamlTestSection;
-import org.elasticsearch.test.rest.yaml.section.DoSection;
-import org.elasticsearch.test.rest.yaml.section.GreaterThanAssertion;
-import org.elasticsearch.test.rest.yaml.section.IsFalseAssertion;
-import org.elasticsearch.test.rest.yaml.section.IsTrueAssertion;
-import org.elasticsearch.test.rest.yaml.section.LengthAssertion;
-import org.elasticsearch.test.rest.yaml.section.LessThanAssertion;
-import org.elasticsearch.test.rest.yaml.section.MatchAssertion;
-import org.elasticsearch.test.rest.yaml.section.SetSection;
-import org.elasticsearch.test.rest.yaml.section.SkipSection;
-
-import java.util.Map;
-
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.instanceOf;
-import static org.hamcrest.Matchers.notNullValue;
-import static org.hamcrest.Matchers.nullValue;
-
-public class TestSectionParserTests extends AbstractClientYamlTestFragmentParserTestCase {
-    public void testParseTestSectionWithDoSection() throws Exception {
-        parser = createParser(YamlXContent.yamlXContent,
-                "\"First test section\": \n" +
-                " - do :\n" +
-                "     catch: missing\n" +
-                "     indices.get_warmer:\n" +
-                "         index: test_index\n" +
-                "         name: test_warmer"
-        );
-
-        ClientYamlTestSectionParser testSectionParser = new ClientYamlTestSectionParser();
-        ClientYamlTestSection testSection = testSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
-
-        assertThat(testSection, notNullValue());
-        assertThat(testSection.getName(), equalTo("First test section"));
-        assertThat(testSection.getSkipSection(), equalTo(SkipSection.EMPTY));
-        assertThat(testSection.getExecutableSections().size(), equalTo(1));
-        DoSection doSection = (DoSection)testSection.getExecutableSections().get(0);
-        assertThat(doSection.getCatch(), equalTo("missing"));
-        assertThat(doSection.getApiCallSection(), notNullValue());
-        assertThat(doSection.getApiCallSection().getApi(), equalTo("indices.get_warmer"));
-        assertThat(doSection.getApiCallSection().getParams().size(), equalTo(2));
-        assertThat(doSection.getApiCallSection().hasBody(), equalTo(false));
-    }
-
-    public void testParseTestSectionWithDoSetAndSkipSectionsNoSkip() throws Exception {
-        String yaml =
-                "\"First test section\": \n" +
-                        "  - skip:\n" +
-                        "      version:  \"2.0.0 - 2.2.0\"\n" +
-                        "      reason:   \"Update doesn't return metadata fields, waiting for #3259\"\n" +
-                        "  - do :\n" +
-                        "      catch: missing\n" +
-                        "      indices.get_warmer:\n" +
-                        "          index: test_index\n" +
-                        "          name: test_warmer\n" +
-                        "  - set: {_scroll_id: scroll_id}";
-
-
-        ClientYamlTestSectionParser testSectionParser = new ClientYamlTestSectionParser();
-        parser = createParser(YamlXContent.yamlXContent,yaml);
-        ClientYamlTestSection testSection = testSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
-
-        assertThat(testSection, notNullValue());
-        assertThat(testSection.getName(), equalTo("First test section"));
-        assertThat(testSection.getSkipSection(), notNullValue());
-        assertThat(testSection.getSkipSection().getLowerVersion(), equalTo(Version.V_2_0_0));
-        assertThat(testSection.getSkipSection().getUpperVersion(), equalTo(Version.V_2_2_0));
-        assertThat(testSection.getSkipSection().getReason(), equalTo("Update doesn't return metadata fields, waiting for #3259"));
-        assertThat(testSection.getExecutableSections().size(), equalTo(2));
-        DoSection doSection = (DoSection)testSection.getExecutableSections().get(0);
-        assertThat(doSection.getCatch(), equalTo("missing"));
-        assertThat(doSection.getApiCallSection(), notNullValue());
-        assertThat(doSection.getApiCallSection().getApi(), equalTo("indices.get_warmer"));
-        assertThat(doSection.getApiCallSection().getParams().size(), equalTo(2));
-        assertThat(doSection.getApiCallSection().hasBody(), equalTo(false));
-        SetSection setSection = (SetSection) testSection.getExecutableSections().get(1);
-        assertThat(setSection.getStash().size(), equalTo(1));
-        assertThat(setSection.getStash().get("_scroll_id"), equalTo("scroll_id"));
-    }
-
-    public void testParseTestSectionWithMultipleDoSections() throws Exception {
-        parser = createParser(YamlXContent.yamlXContent,
-                "\"Basic\":\n" +
-                        "\n" +
-                        "  - do:\n" +
-                        "      index:\n" +
-                        "        index: test_1\n" +
-                        "        type:  test\n" +
-                        "        id:    中文\n" +
-                        "        body:  { \"foo\": \"Hello: 中文\" }\n" +
-                        "  - do:\n" +
-                        "      get:\n" +
-                        "        index: test_1\n" +
-                        "        type:  test\n" +
-                        "        id:    中文"
-        );
-
-        ClientYamlTestSectionParser testSectionParser = new ClientYamlTestSectionParser();
-        ClientYamlTestSection testSection = testSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
-
-        assertThat(testSection, notNullValue());
-        assertThat(testSection.getName(), equalTo("Basic"));
-        assertThat(testSection.getSkipSection(), equalTo(SkipSection.EMPTY));
-        assertThat(testSection.getExecutableSections().size(), equalTo(2));
-        DoSection doSection = (DoSection)testSection.getExecutableSections().get(0);
-        assertThat(doSection.getCatch(), nullValue());
-        assertThat(doSection.getApiCallSection(), notNullValue());
-        assertThat(doSection.getApiCallSection().getApi(), equalTo("index"));
-        assertThat(doSection.getApiCallSection().getParams().size(), equalTo(3));
-        assertThat(doSection.getApiCallSection().hasBody(), equalTo(true));
-        doSection = (DoSection)testSection.getExecutableSections().get(1);
-        assertThat(doSection.getCatch(), nullValue());
-        assertThat(doSection.getApiCallSection(), notNullValue());
-        assertThat(doSection.getApiCallSection().getApi(), equalTo("get"));
-        assertThat(doSection.getApiCallSection().getParams().size(), equalTo(3));
-        assertThat(doSection.getApiCallSection().hasBody(), equalTo(false));
-    }
-
-    public void testParseTestSectionWithDoSectionsAndAssertions() throws Exception {
-        parser = createParser(YamlXContent.yamlXContent,
-                "\"Basic\":\n" +
-                        "\n" +
-                        "  - do:\n" +
-                        "      index:\n" +
-                        "        index: test_1\n" +
-                        "        type:  test\n" +
-                        "        id:    中文\n" +
-                        "        body:  { \"foo\": \"Hello: 中文\" }\n" +
-                        "\n" +
-                        "  - do:\n" +
-                        "      get:\n" +
-                        "        index: test_1\n" +
-                        "        type:  test\n" +
-                        "        id:    中文\n" +
-                        "\n" +
-                        "  - match: { _index:   test_1 }\n" +
-                        "  - is_true: _source\n" +
-                        "  - match: { _source:  { foo: \"Hello: 中文\" } }\n" +
-                        "\n" +
-                        "  - do:\n" +
-                        "      get:\n" +
-                        "        index: test_1\n" +
-                        "        id:    中文\n" +
-                        "\n" +
-                        "  - length: { _index:   6 }\n" +
-                        "  - is_false: whatever\n" +
-                        "  - gt: { size: 5      }\n" +
-                        "  - lt: { size: 10      }"
-        );
-
-        ClientYamlTestSectionParser testSectionParser = new ClientYamlTestSectionParser();
-        ClientYamlTestSection testSection = testSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
-
-        assertThat(testSection, notNullValue());
-        assertThat(testSection.getName(), equalTo("Basic"));
-        assertThat(testSection.getSkipSection(), equalTo(SkipSection.EMPTY));
-        assertThat(testSection.getExecutableSections().size(), equalTo(10));
-
-        DoSection doSection = (DoSection)testSection.getExecutableSections().get(0);
-        assertThat(doSection.getCatch(), nullValue());
-        assertThat(doSection.getApiCallSection(), notNullValue());
-        assertThat(doSection.getApiCallSection().getApi(), equalTo("index"));
-        assertThat(doSection.getApiCallSection().getParams().size(), equalTo(3));
-        assertThat(doSection.getApiCallSection().hasBody(), equalTo(true));
-
-        doSection = (DoSection)testSection.getExecutableSections().get(1);
-        assertThat(doSection.getCatch(), nullValue());
-        assertThat(doSection.getApiCallSection(), notNullValue());
-        assertThat(doSection.getApiCallSection().getApi(), equalTo("get"));
-        assertThat(doSection.getApiCallSection().getParams().size(), equalTo(3));
-        assertThat(doSection.getApiCallSection().hasBody(), equalTo(false));
-
-        MatchAssertion matchAssertion = (MatchAssertion)testSection.getExecutableSections().get(2);
-        assertThat(matchAssertion.getField(), equalTo("_index"));
-        assertThat(matchAssertion.getExpectedValue().toString(), equalTo("test_1"));
-
-        IsTrueAssertion trueAssertion = (IsTrueAssertion)testSection.getExecutableSections().get(3);
-        assertThat(trueAssertion.getField(), equalTo("_source"));
-
-        matchAssertion = (MatchAssertion)testSection.getExecutableSections().get(4);
-        assertThat(matchAssertion.getField(), equalTo("_source"));
-        assertThat(matchAssertion.getExpectedValue(), instanceOf(Map.class));
-        Map map = (Map) matchAssertion.getExpectedValue();
-        assertThat(map.size(), equalTo(1));
-        assertThat(map.get("foo").toString(), equalTo("Hello: 中文"));
-
-        doSection = (DoSection)testSection.getExecutableSections().get(5);
-        assertThat(doSection.getCatch(), nullValue());
-        assertThat(doSection.getApiCallSection(), notNullValue());
-        assertThat(doSection.getApiCallSection().getApi(), equalTo("get"));
-        assertThat(doSection.getApiCallSection().getParams().size(), equalTo(2));
-        assertThat(doSection.getApiCallSection().hasBody(), equalTo(false));
-
-        LengthAssertion lengthAssertion = (LengthAssertion) testSection.getExecutableSections().get(6);
-        assertThat(lengthAssertion.getField(), equalTo("_index"));
-        assertThat(lengthAssertion.getExpectedValue(), instanceOf(Integer.class));
-        assertThat((Integer) lengthAssertion.getExpectedValue(), equalTo(6));
-
-        IsFalseAssertion falseAssertion = (IsFalseAssertion)testSection.getExecutableSections().get(7);
-        assertThat(falseAssertion.getField(), equalTo("whatever"));
-
-        GreaterThanAssertion greaterThanAssertion = (GreaterThanAssertion) testSection.getExecutableSections().get(8);
-        assertThat(greaterThanAssertion.getField(), equalTo("size"));
-        assertThat(greaterThanAssertion.getExpectedValue(), instanceOf(Integer.class));
-        assertThat((Integer) greaterThanAssertion.getExpectedValue(), equalTo(5));
-
-        LessThanAssertion lessThanAssertion = (LessThanAssertion) testSection.getExecutableSections().get(9);
-        assertThat(lessThanAssertion.getField(), equalTo("size"));
-        assertThat(lessThanAssertion.getExpectedValue(), instanceOf(Integer.class));
-        assertThat((Integer) lessThanAssertion.getExpectedValue(), equalTo(10));
-    }
-
-    public void testSmallSection() throws Exception {
-        parser = createParser(YamlXContent.yamlXContent,
-                "\"node_info test\":\n" +
-                "  - do:\n" +
-                "      cluster.node_info: {}\n" +
-                "  \n" +
-                "  - is_true: nodes\n" +
-                "  - is_true: cluster_name\n");
-        ClientYamlTestSectionParser testSectionParser = new ClientYamlTestSectionParser();
-        ClientYamlTestSection testSection = testSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
-        assertThat(testSection, notNullValue());
-        assertThat(testSection.getName(), equalTo("node_info test"));
-        assertThat(testSection.getExecutableSections().size(), equalTo(3));
-    }
-}
diff --git a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/restspec/ClientYamlSuiteRestApiParserTests.java b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/restspec/ClientYamlSuiteRestApiParserTests.java
index caca95b131..6acdd93540 100644
--- a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/restspec/ClientYamlSuiteRestApiParserTests.java
+++ b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/restspec/ClientYamlSuiteRestApiParserTests.java
@@ -19,7 +19,7 @@
 package org.elasticsearch.test.rest.yaml.restspec;
 
 import org.elasticsearch.common.xcontent.yaml.YamlXContent;
-import org.elasticsearch.test.rest.yaml.parser.AbstractClientYamlTestFragmentParserTestCase;
+import org.elasticsearch.test.rest.yaml.section.AbstractClientYamlTestFragmentParserTestCase;
 
 import static org.hamcrest.Matchers.contains;
 import static org.hamcrest.Matchers.equalTo;
diff --git a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/parser/AbstractClientYamlTestFragmentParserTestCase.java b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/AbstractClientYamlTestFragmentParserTestCase.java
index 3b0859d2d8..b7038a7d7a 100644
--- a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/parser/AbstractClientYamlTestFragmentParserTestCase.java
+++ b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/AbstractClientYamlTestFragmentParserTestCase.java
@@ -17,19 +17,20 @@
  * under the License.
  */
 
-package org.elasticsearch.test.rest.yaml.parser;
+package org.elasticsearch.test.rest.yaml.section;
 
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.rest.yaml.section.ExecutableSection;
 import org.junit.After;
 
 import static org.hamcrest.Matchers.nullValue;
 
 /**
- * Superclass for tests of subclasses of {@link ClientYamlTestFragmentParser}.
+ * Superclass for tests that parse parts of the test suite.
  */
 public abstract class AbstractClientYamlTestFragmentParserTestCase extends ESTestCase {
-
     protected XContentParser parser;
 
     @Override
@@ -38,9 +39,16 @@ public abstract class AbstractClientYamlTestFragmentParserTestCase extends ESTes
         super.tearDown();
         // test may be skipped so we did not create a parser instance
         if (parser != null) {
-            //this is the way to make sure that we consumed the whole yaml
+            //next token can be null even in the middle of the document (e.g. with "---"), but not too many consecutive times
             assertThat(parser.currentToken(), nullValue());
+            assertThat(parser.nextToken(), nullValue());
+            assertThat(parser.nextToken(), nullValue());
             parser.close();
         }
     }
+
+    @Override
+    protected NamedXContentRegistry xContentRegistry() {
+        return ExecutableSection.XCONTENT_REGISTRY;
+    }
 }
diff --git a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/parser/AssertionParsersTests.java b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/AssertionTests.java
index c908f07914..fcef746783 100644
--- a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/parser/AssertionParsersTests.java
+++ b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/AssertionTests.java
@@ -16,7 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.elasticsearch.test.rest.yaml.parser;
+package org.elasticsearch.test.rest.yaml.section;
 
 import org.elasticsearch.common.xcontent.yaml.YamlXContent;
 import org.elasticsearch.test.rest.yaml.section.GreaterThanAssertion;
@@ -33,14 +33,13 @@ import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.notNullValue;
 
-public class AssertionParsersTests extends AbstractClientYamlTestFragmentParserTestCase {
+public class AssertionTests extends AbstractClientYamlTestFragmentParserTestCase {
     public void testParseIsTrue() throws Exception {
         parser = createParser(YamlXContent.yamlXContent,
                 "get.fields._timestamp"
         );
 
-        IsTrueParser isTrueParser = new IsTrueParser();
-        IsTrueAssertion trueAssertion = isTrueParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
+        IsTrueAssertion trueAssertion = IsTrueAssertion.parse(parser);
 
         assertThat(trueAssertion, notNullValue());
         assertThat(trueAssertion.getField(), equalTo("get.fields._timestamp"));
@@ -51,8 +50,7 @@ public class AssertionParsersTests extends AbstractClientYamlTestFragmentParserT
                 "docs.1._source"
         );
 
-        IsFalseParser isFalseParser = new IsFalseParser();
-        IsFalseAssertion falseAssertion = isFalseParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
+        IsFalseAssertion falseAssertion = IsFalseAssertion.parse(parser);
 
         assertThat(falseAssertion, notNullValue());
         assertThat(falseAssertion.getField(), equalTo("docs.1._source"));
@@ -63,8 +61,7 @@ public class AssertionParsersTests extends AbstractClientYamlTestFragmentParserT
                 "{ field: 3}"
         );
 
-        GreaterThanParser greaterThanParser = new GreaterThanParser();
-        GreaterThanAssertion greaterThanAssertion = greaterThanParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
+        GreaterThanAssertion greaterThanAssertion = GreaterThanAssertion.parse(parser);
         assertThat(greaterThanAssertion, notNullValue());
         assertThat(greaterThanAssertion.getField(), equalTo("field"));
         assertThat(greaterThanAssertion.getExpectedValue(), instanceOf(Integer.class));
@@ -76,8 +73,7 @@ public class AssertionParsersTests extends AbstractClientYamlTestFragmentParserT
                 "{ field: 3}"
         );
 
-        LessThanParser lessThanParser = new LessThanParser();
-        LessThanAssertion lessThanAssertion = lessThanParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
+        LessThanAssertion lessThanAssertion = LessThanAssertion.parse(parser);
         assertThat(lessThanAssertion, notNullValue());
         assertThat(lessThanAssertion.getField(), equalTo("field"));
         assertThat(lessThanAssertion.getExpectedValue(), instanceOf(Integer.class));
@@ -89,8 +85,7 @@ public class AssertionParsersTests extends AbstractClientYamlTestFragmentParserT
                 "{ _id: 22}"
         );
 
-        LengthParser lengthParser = new LengthParser();
-        LengthAssertion lengthAssertion = lengthParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
+        LengthAssertion lengthAssertion = LengthAssertion.parse(parser);
         assertThat(lengthAssertion, notNullValue());
         assertThat(lengthAssertion.getField(), equalTo("_id"));
         assertThat(lengthAssertion.getExpectedValue(), instanceOf(Integer.class));
@@ -102,8 +97,7 @@ public class AssertionParsersTests extends AbstractClientYamlTestFragmentParserT
                 "{ field: 10 }"
         );
 
-        MatchParser matchParser = new MatchParser();
-        MatchAssertion matchAssertion = matchParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
+        MatchAssertion matchAssertion = MatchAssertion.parse(parser);
 
         assertThat(matchAssertion, notNullValue());
         assertThat(matchAssertion.getField(), equalTo("field"));
@@ -116,8 +110,7 @@ public class AssertionParsersTests extends AbstractClientYamlTestFragmentParserT
                 "{ foo: bar }"
         );
 
-        MatchParser matchParser = new MatchParser();
-        MatchAssertion matchAssertion = matchParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
+        MatchAssertion matchAssertion = MatchAssertion.parse(parser);
 
         assertThat(matchAssertion, notNullValue());
         assertThat(matchAssertion.getField(), equalTo("foo"));
@@ -130,13 +123,12 @@ public class AssertionParsersTests extends AbstractClientYamlTestFragmentParserT
                 "{'matches': ['test_percolator_1', 'test_percolator_2']}"
         );
 
-        MatchParser matchParser = new MatchParser();
-        MatchAssertion matchAssertion = matchParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
+        MatchAssertion matchAssertion = MatchAssertion.parse(parser);
 
         assertThat(matchAssertion, notNullValue());
         assertThat(matchAssertion.getField(), equalTo("matches"));
         assertThat(matchAssertion.getExpectedValue(), instanceOf(List.class));
-        List strings = (List) matchAssertion.getExpectedValue();
+        List<?> strings = (List<?>) matchAssertion.getExpectedValue();
         assertThat(strings.size(), equalTo(2));
         assertThat(strings.get(0).toString(), equalTo("test_percolator_1"));
         assertThat(strings.get(1).toString(), equalTo("test_percolator_2"));
@@ -148,8 +140,7 @@ public class AssertionParsersTests extends AbstractClientYamlTestFragmentParserT
                 "{ _source: { responses.0.hits.total: 3, foo: bar  }}"
         );
 
-        MatchParser matchParser = new MatchParser();
-        MatchAssertion matchAssertion = matchParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
+        MatchAssertion matchAssertion = MatchAssertion.parse(parser);
 
         assertThat(matchAssertion, notNullValue());
         assertThat(matchAssertion.getField(), equalTo("_source"));
diff --git a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSectionTests.java b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSectionTests.java
index db12c3af9b..0947c83451 100644
--- a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSectionTests.java
+++ b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSectionTests.java
@@ -19,15 +19,22 @@
 
 package org.elasticsearch.test.rest.yaml.section;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.common.xcontent.XContentLocation;
-import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.common.xcontent.yaml.YamlXContent;
+
+import java.util.Map;
 
 import static java.util.Collections.singletonList;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.hamcrest.Matchers.notNullValue;
+import static org.hamcrest.Matchers.nullValue;
 
-public class ClientYamlTestSectionTests extends ESTestCase {
+public class ClientYamlTestSectionTests extends AbstractClientYamlTestFragmentParserTestCase {
     public void testAddingDoWithoutWarningWithoutSkip() {
         int lineNumber = between(1, 10000);
-        ClientYamlTestSection section = new ClientYamlTestSection("test");
+        ClientYamlTestSection section = new ClientYamlTestSection(new XContentLocation(0, 0), "test");
         section.setSkipSection(SkipSection.EMPTY);
         DoSection doSection = new DoSection(new XContentLocation(lineNumber, 0));
         section.addExecutableSection(doSection);
@@ -35,7 +42,7 @@ public class ClientYamlTestSectionTests extends ESTestCase {
 
     public void testAddingDoWithWarningWithSkip() {
         int lineNumber = between(1, 10000);
-        ClientYamlTestSection section = new ClientYamlTestSection("test");
+        ClientYamlTestSection section = new ClientYamlTestSection(new XContentLocation(0, 0), "test");
         section.setSkipSection(new SkipSection(null, singletonList("warnings"), null));
         DoSection doSection = new DoSection(new XContentLocation(lineNumber, 0));
         doSection.setExpectedWarningHeaders(singletonList("foo"));
@@ -44,7 +51,7 @@ public class ClientYamlTestSectionTests extends ESTestCase {
 
     public void testAddingDoWithWarningWithSkipButNotWarnings() {
         int lineNumber = between(1, 10000);
-        ClientYamlTestSection section = new ClientYamlTestSection("test");
+        ClientYamlTestSection section = new ClientYamlTestSection(new XContentLocation(0, 0), "test");
         section.setSkipSection(new SkipSection(null, singletonList("yaml"), null));
         DoSection doSection = new DoSection(new XContentLocation(lineNumber, 0));
         doSection.setExpectedWarningHeaders(singletonList("foo"));
@@ -53,4 +60,207 @@ public class ClientYamlTestSectionTests extends ESTestCase {
                 + " [warnings] section can skip the test at line [" + lineNumber + "]", e.getMessage());
     }
 
+    public void testParseTestSectionWithDoSection() throws Exception {
+        parser = createParser(YamlXContent.yamlXContent,
+                "\"First test section\": \n" +
+                " - do :\n" +
+                "     catch: missing\n" +
+                "     indices.get_warmer:\n" +
+                "         index: test_index\n" +
+                "         name: test_warmer"
+        );
+
+        ClientYamlTestSection testSection = ClientYamlTestSection.parse(parser);
+
+        assertThat(testSection, notNullValue());
+        assertThat(testSection.getName(), equalTo("First test section"));
+        assertThat(testSection.getSkipSection(), equalTo(SkipSection.EMPTY));
+        assertThat(testSection.getExecutableSections().size(), equalTo(1));
+        DoSection doSection = (DoSection)testSection.getExecutableSections().get(0);
+        assertThat(doSection.getCatch(), equalTo("missing"));
+        assertThat(doSection.getApiCallSection(), notNullValue());
+        assertThat(doSection.getApiCallSection().getApi(), equalTo("indices.get_warmer"));
+        assertThat(doSection.getApiCallSection().getParams().size(), equalTo(2));
+        assertThat(doSection.getApiCallSection().hasBody(), equalTo(false));
+    }
+
+    public void testParseTestSectionWithDoSetAndSkipSectionsNoSkip() throws Exception {
+        parser = createParser(YamlXContent.yamlXContent,
+                "\"First test section\": \n" +
+                        "  - skip:\n" +
+                        "      version:  \"2.0.0 - 2.2.0\"\n" +
+                        "      reason:   \"Update doesn't return metadata fields, waiting for #3259\"\n" +
+                        "  - do :\n" +
+                        "      catch: missing\n" +
+                        "      indices.get_warmer:\n" +
+                        "          index: test_index\n" +
+                        "          name: test_warmer\n" +
+                        "  - set: {_scroll_id: scroll_id}");
+
+
+        ClientYamlTestSection testSection = ClientYamlTestSection.parse(parser);
+
+        assertThat(testSection, notNullValue());
+        assertThat(testSection.getName(), equalTo("First test section"));
+        assertThat(testSection.getSkipSection(), notNullValue());
+        assertThat(testSection.getSkipSection().getLowerVersion(), equalTo(Version.V_2_0_0));
+        assertThat(testSection.getSkipSection().getUpperVersion(), equalTo(Version.V_2_2_0));
+        assertThat(testSection.getSkipSection().getReason(), equalTo("Update doesn't return metadata fields, waiting for #3259"));
+        assertThat(testSection.getExecutableSections().size(), equalTo(2));
+        DoSection doSection = (DoSection)testSection.getExecutableSections().get(0);
+        assertThat(doSection.getCatch(), equalTo("missing"));
+        assertThat(doSection.getApiCallSection(), notNullValue());
+        assertThat(doSection.getApiCallSection().getApi(), equalTo("indices.get_warmer"));
+        assertThat(doSection.getApiCallSection().getParams().size(), equalTo(2));
+        assertThat(doSection.getApiCallSection().hasBody(), equalTo(false));
+        SetSection setSection = (SetSection) testSection.getExecutableSections().get(1);
+        assertThat(setSection.getStash().size(), equalTo(1));
+        assertThat(setSection.getStash().get("_scroll_id"), equalTo("scroll_id"));
+    }
+
+    public void testParseTestSectionWithMultipleDoSections() throws Exception {
+        parser = createParser(YamlXContent.yamlXContent,
+                "\"Basic\":\n" +
+                        "\n" +
+                        "  - do:\n" +
+                        "      index:\n" +
+                        "        index: test_1\n" +
+                        "        type:  test\n" +
+                        "        id:    中文\n" +
+                        "        body:  { \"foo\": \"Hello: 中文\" }\n" +
+                        "  - do:\n" +
+                        "      get:\n" +
+                        "        index: test_1\n" +
+                        "        type:  test\n" +
+                        "        id:    中文"
+        );
+
+        ClientYamlTestSection testSection = ClientYamlTestSection.parse(parser);
+
+        assertThat(testSection, notNullValue());
+        assertThat(testSection.getName(), equalTo("Basic"));
+        assertThat(testSection.getSkipSection(), equalTo(SkipSection.EMPTY));
+        assertThat(testSection.getExecutableSections().size(), equalTo(2));
+        DoSection doSection = (DoSection)testSection.getExecutableSections().get(0);
+        assertThat(doSection.getCatch(), nullValue());
+        assertThat(doSection.getApiCallSection(), notNullValue());
+        assertThat(doSection.getApiCallSection().getApi(), equalTo("index"));
+        assertThat(doSection.getApiCallSection().getParams().size(), equalTo(3));
+        assertThat(doSection.getApiCallSection().hasBody(), equalTo(true));
+        doSection = (DoSection)testSection.getExecutableSections().get(1);
+        assertThat(doSection.getCatch(), nullValue());
+        assertThat(doSection.getApiCallSection(), notNullValue());
+        assertThat(doSection.getApiCallSection().getApi(), equalTo("get"));
+        assertThat(doSection.getApiCallSection().getParams().size(), equalTo(3));
+        assertThat(doSection.getApiCallSection().hasBody(), equalTo(false));
+    }
+
+    public void testParseTestSectionWithDoSectionsAndAssertions() throws Exception {
+        parser = createParser(YamlXContent.yamlXContent,
+                "\"Basic\":\n" +
+                        "\n" +
+                        "  - do:\n" +
+                        "      index:\n" +
+                        "        index: test_1\n" +
+                        "        type:  test\n" +
+                        "        id:    中文\n" +
+                        "        body:  { \"foo\": \"Hello: 中文\" }\n" +
+                        "\n" +
+                        "  - do:\n" +
+                        "      get:\n" +
+                        "        index: test_1\n" +
+                        "        type:  test\n" +
+                        "        id:    中文\n" +
+                        "\n" +
+                        "  - match: { _index:   test_1 }\n" +
+                        "  - is_true: _source\n" +
+                        "  - match: { _source:  { foo: \"Hello: 中文\" } }\n" +
+                        "\n" +
+                        "  - do:\n" +
+                        "      get:\n" +
+                        "        index: test_1\n" +
+                        "        id:    中文\n" +
+                        "\n" +
+                        "  - length: { _index:   6 }\n" +
+                        "  - is_false: whatever\n" +
+                        "  - gt: { size: 5      }\n" +
+                        "  - lt: { size: 10      }"
+        );
+
+        ClientYamlTestSection testSection = ClientYamlTestSection.parse(parser);
+
+        assertThat(testSection, notNullValue());
+        assertThat(testSection.getName(), equalTo("Basic"));
+        assertThat(testSection.getSkipSection(), equalTo(SkipSection.EMPTY));
+        assertThat(testSection.getExecutableSections().size(), equalTo(10));
+
+        DoSection doSection = (DoSection)testSection.getExecutableSections().get(0);
+        assertThat(doSection.getCatch(), nullValue());
+        assertThat(doSection.getApiCallSection(), notNullValue());
+        assertThat(doSection.getApiCallSection().getApi(), equalTo("index"));
+        assertThat(doSection.getApiCallSection().getParams().size(), equalTo(3));
+        assertThat(doSection.getApiCallSection().hasBody(), equalTo(true));
+
+        doSection = (DoSection)testSection.getExecutableSections().get(1);
+        assertThat(doSection.getCatch(), nullValue());
+        assertThat(doSection.getApiCallSection(), notNullValue());
+        assertThat(doSection.getApiCallSection().getApi(), equalTo("get"));
+        assertThat(doSection.getApiCallSection().getParams().size(), equalTo(3));
+        assertThat(doSection.getApiCallSection().hasBody(), equalTo(false));
+
+        MatchAssertion matchAssertion = (MatchAssertion)testSection.getExecutableSections().get(2);
+        assertThat(matchAssertion.getField(), equalTo("_index"));
+        assertThat(matchAssertion.getExpectedValue().toString(), equalTo("test_1"));
+
+        IsTrueAssertion trueAssertion = (IsTrueAssertion)testSection.getExecutableSections().get(3);
+        assertThat(trueAssertion.getField(), equalTo("_source"));
+
+        matchAssertion = (MatchAssertion)testSection.getExecutableSections().get(4);
+        assertThat(matchAssertion.getField(), equalTo("_source"));
+        assertThat(matchAssertion.getExpectedValue(), instanceOf(Map.class));
+        Map map = (Map) matchAssertion.getExpectedValue();
+        assertThat(map.size(), equalTo(1));
+        assertThat(map.get("foo").toString(), equalTo("Hello: 中文"));
+
+        doSection = (DoSection)testSection.getExecutableSections().get(5);
+        assertThat(doSection.getCatch(), nullValue());
+        assertThat(doSection.getApiCallSection(), notNullValue());
+        assertThat(doSection.getApiCallSection().getApi(), equalTo("get"));
+        assertThat(doSection.getApiCallSection().getParams().size(), equalTo(2));
+        assertThat(doSection.getApiCallSection().hasBody(), equalTo(false));
+
+        LengthAssertion lengthAssertion = (LengthAssertion) testSection.getExecutableSections().get(6);
+        assertThat(lengthAssertion.getField(), equalTo("_index"));
+        assertThat(lengthAssertion.getExpectedValue(), instanceOf(Integer.class));
+        assertThat((Integer) lengthAssertion.getExpectedValue(), equalTo(6));
+
+        IsFalseAssertion falseAssertion = (IsFalseAssertion)testSection.getExecutableSections().get(7);
+        assertThat(falseAssertion.getField(), equalTo("whatever"));
+
+        GreaterThanAssertion greaterThanAssertion = (GreaterThanAssertion) testSection.getExecutableSections().get(8);
+        assertThat(greaterThanAssertion.getField(), equalTo("size"));
+        assertThat(greaterThanAssertion.getExpectedValue(), instanceOf(Integer.class));
+        assertThat((Integer) greaterThanAssertion.getExpectedValue(), equalTo(5));
+
+        LessThanAssertion lessThanAssertion = (LessThanAssertion) testSection.getExecutableSections().get(9);
+        assertThat(lessThanAssertion.getField(), equalTo("size"));
+        assertThat(lessThanAssertion.getExpectedValue(), instanceOf(Integer.class));
+        assertThat((Integer) lessThanAssertion.getExpectedValue(), equalTo(10));
+    }
+
+    public void testSmallSection() throws Exception {
+        parser = createParser(YamlXContent.yamlXContent,
+                "\"node_info test\":\n" +
+                "  - do:\n" +
+                "      cluster.node_info: {}\n" +
+                "  \n" +
+                "  - is_true: nodes\n" +
+                "  - is_true: cluster_name\n");
+
+        ClientYamlTestSection testSection = ClientYamlTestSection.parse(parser);
+        assertThat(testSection, notNullValue());
+        assertThat(testSection.getName(), equalTo("node_info test"));
+        assertThat(testSection.getExecutableSections().size(), equalTo(3));
+    }
+
 }
diff --git a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/parser/ClientYamlSuiteTestParserTests.java b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSuiteTests.java
index 801bc4e06e..4c96986146 100644
--- a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/parser/ClientYamlSuiteTestParserTests.java
+++ b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSuiteTests.java
@@ -16,20 +16,12 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.elasticsearch.test.rest.yaml.parser;
+
+package org.elasticsearch.test.rest.yaml.section;
 
 import org.elasticsearch.Version;
-import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.xcontent.yaml.YamlXContent;
-import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestSuiteParseContext;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestSuiteParser;
-import org.elasticsearch.test.rest.yaml.section.ClientYamlTestSuite;
-import org.elasticsearch.test.rest.yaml.section.DoSection;
-import org.elasticsearch.test.rest.yaml.section.IsTrueAssertion;
-import org.elasticsearch.test.rest.yaml.section.MatchAssertion;
-import org.junit.After;
 
 import java.util.Map;
 
@@ -39,21 +31,7 @@ import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.notNullValue;
 import static org.hamcrest.Matchers.nullValue;
 
-public class ClientYamlSuiteTestParserTests extends ESTestCase {
-    private XContentParser parser;
-
-    @Override
-    @After
-    public void tearDown() throws Exception {
-        super.tearDown();
-        //makes sure that we consumed the whole stream, XContentParser doesn't expose isClosed method
-        //next token can be null even in the middle of the document (e.g. with "---"), but not too many consecutive times
-        assertThat(parser.currentToken(), nullValue());
-        assertThat(parser.nextToken(), nullValue());
-        assertThat(parser.nextToken(), nullValue());
-        parser.close();
-    }
-
+public class ClientYamlTestSuiteTests extends AbstractClientYamlTestFragmentParserTestCase {
     public void testParseTestSetupTeardownAndSections() throws Exception {
         final boolean includeSetup = randomBoolean();
         final boolean includeTeardown = randomBoolean();
@@ -103,11 +81,10 @@ public class ClientYamlSuiteTestParserTests extends ESTestCase {
                         "  - match: {test_type.properties.text.analyzer: whitespace}\n"
         );
 
-        ClientYamlTestSuiteParser testParser = new ClientYamlTestSuiteParser();
-        ClientYamlTestSuite restTestSuite = testParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
+        ClientYamlTestSuite restTestSuite = ClientYamlTestSuite.parse(getTestClass().getName(), getTestName(), parser);
 
         assertThat(restTestSuite, notNullValue());
-        assertThat(restTestSuite.getName(), equalTo("suite"));
+        assertThat(restTestSuite.getName(), equalTo(getTestName()));
         assertThat(restTestSuite.getSetupSection(), notNullValue());
         if (includeSetup) {
             assertThat(restTestSuite.getSetupSection().isEmpty(), equalTo(false));
@@ -207,11 +184,10 @@ public class ClientYamlSuiteTestParserTests extends ESTestCase {
                 " - match:   { _source: { foo: bar }}"
         );
 
-        ClientYamlTestSuiteParser testParser = new ClientYamlTestSuiteParser();
-        ClientYamlTestSuite restTestSuite = testParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
+        ClientYamlTestSuite restTestSuite = ClientYamlTestSuite.parse(getTestClass().getName(), getTestName(), parser);
 
         assertThat(restTestSuite, notNullValue());
-        assertThat(restTestSuite.getName(), equalTo("suite"));
+        assertThat(restTestSuite.getName(), equalTo(getTestName()));
 
         assertThat(restTestSuite.getSetupSection().isEmpty(), equalTo(true));
 
@@ -320,11 +296,10 @@ public class ClientYamlSuiteTestParserTests extends ESTestCase {
                 "            params:       { bar: 'xxx' }\n"
         );
 
-        ClientYamlTestSuiteParser testParser = new ClientYamlTestSuiteParser();
-        ClientYamlTestSuite restTestSuite = testParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
+        ClientYamlTestSuite restTestSuite = ClientYamlTestSuite.parse(getTestClass().getName(), getTestName(), parser);
 
         assertThat(restTestSuite, notNullValue());
-        assertThat(restTestSuite.getName(), equalTo("suite"));
+        assertThat(restTestSuite.getName(), equalTo(getTestName()));
 
         assertThat(restTestSuite.getSetupSection().isEmpty(), equalTo(true));
 
@@ -394,12 +369,8 @@ public class ClientYamlSuiteTestParserTests extends ESTestCase {
                         "\n"
         );
 
-        ClientYamlTestSuiteParser testParser = new ClientYamlTestSuiteParser();
-        try {
-            testParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
-            fail("Expected RestTestParseException");
-        } catch (ClientYamlTestParseException e) {
-            assertThat(e.getMessage(), containsString("duplicate test section"));
-        }
+        Exception e = expectThrows(ParsingException.class, () ->
+            ClientYamlTestSuite.parse(getTestClass().getName(), getTestName(), parser));
+        assertThat(e.getMessage(), containsString("duplicate test section"));
     }
 }
diff --git a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/DoSectionTests.java b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/DoSectionTests.java
index e981b2d999..463e71341b 100644
--- a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/DoSectionTests.java
+++ b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/DoSectionTests.java
@@ -19,16 +19,24 @@
 
 package org.elasticsearch.test.rest.yaml.section;
 
+import org.elasticsearch.common.xcontent.XContent;
 import org.elasticsearch.common.xcontent.XContentLocation;
-import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.yaml.YamlXContent;
+import org.hamcrest.MatcherAssert;
 
 import java.io.IOException;
 import java.util.Arrays;
+import java.util.Map;
 
 import static java.util.Collections.emptyList;
 import static java.util.Collections.singletonList;
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.notNullValue;
+import static org.hamcrest.Matchers.nullValue;
 
-public class DoSectionTests extends ESTestCase {
+public class DoSectionTests extends AbstractClientYamlTestFragmentParserTestCase {
     public void testWarningHeaders() throws IOException {
         DoSection section = new DoSection(new XContentLocation(1, 1));
 
@@ -63,4 +71,397 @@ public class DoSectionTests extends ESTestCase {
         assertEquals("got unexpected warning headers [\ncat\n] didn't get expected warning headers [\nanother\nsome more\n]",
                 e.getMessage());
     }
+
+    public void testParseDoSectionNoBody() throws Exception {
+        parser = createParser(YamlXContent.yamlXContent,
+                "get:\n" +
+                "    index:    test_index\n" +
+                "    type:    test_type\n" +
+                "    id:        1"
+        );
+
+        DoSection doSection = DoSection.parse(parser);
+        ApiCallSection apiCallSection = doSection.getApiCallSection();
+
+        assertThat(apiCallSection, notNullValue());
+        assertThat(apiCallSection.getApi(), equalTo("get"));
+        assertThat(apiCallSection.getParams().size(), equalTo(3));
+        assertThat(apiCallSection.getParams().get("index"), equalTo("test_index"));
+        assertThat(apiCallSection.getParams().get("type"), equalTo("test_type"));
+        assertThat(apiCallSection.getParams().get("id"), equalTo("1"));
+        assertThat(apiCallSection.hasBody(), equalTo(false));
+    }
+
+    public void testParseDoSectionNoParamsNoBody() throws Exception {
+        parser = createParser(YamlXContent.yamlXContent,
+                "cluster.node_info: {}"
+        );
+
+        DoSection doSection = DoSection.parse(parser);
+        ApiCallSection apiCallSection = doSection.getApiCallSection();
+
+        assertThat(apiCallSection, notNullValue());
+        assertThat(apiCallSection.getApi(), equalTo("cluster.node_info"));
+        assertThat(apiCallSection.getParams().size(), equalTo(0));
+        assertThat(apiCallSection.hasBody(), equalTo(false));
+    }
+
+    public void testParseDoSectionWithJsonBody() throws Exception {
+        String body = "{ \"include\": { \"field1\": \"v1\", \"field2\": \"v2\" }, \"count\": 1 }";
+        parser = createParser(YamlXContent.yamlXContent,
+                "index:\n" +
+                "    index:  test_1\n" +
+                "    type:   test\n" +
+                "    id:     1\n" +
+                "    body:   " + body
+        );
+
+        DoSection doSection = DoSection.parse(parser);
+        ApiCallSection apiCallSection = doSection.getApiCallSection();
+
+        assertThat(apiCallSection, notNullValue());
+        assertThat(apiCallSection.getApi(), equalTo("index"));
+        assertThat(apiCallSection.getParams().size(), equalTo(3));
+        assertThat(apiCallSection.getParams().get("index"), equalTo("test_1"));
+        assertThat(apiCallSection.getParams().get("type"), equalTo("test"));
+        assertThat(apiCallSection.getParams().get("id"), equalTo("1"));
+        assertThat(apiCallSection.hasBody(), equalTo(true));
+
+        assertJsonEquals(apiCallSection.getBodies().get(0), body);
+    }
+
+    public void testParseDoSectionWithJsonMultipleBodiesAsLongString() throws Exception {
+        String bodies[] = new String[]{
+                "{ \"index\": { \"_index\":\"test_index\", \"_type\":\"test_type\", \"_id\":\"test_id\" } }\n",
+                "{ \"f1\":\"v1\", \"f2\":42 }\n",
+                "{ \"index\": { \"_index\":\"test_index2\", \"_type\":\"test_type2\", \"_id\":\"test_id2\" } }\n",
+                "{ \"f1\":\"v2\", \"f2\":47 }\n"
+        };
+        parser = createParser(YamlXContent.yamlXContent,
+                "bulk:\n" +
+                "    refresh: true\n" +
+                "    body: |\n" +
+                "        " + bodies[0] +
+                "        " + bodies[1] +
+                "        " + bodies[2] +
+                "        " + bodies[3]
+        );
+
+        DoSection doSection = DoSection.parse(parser);
+        ApiCallSection apiCallSection = doSection.getApiCallSection();
+
+        assertThat(apiCallSection, notNullValue());
+        assertThat(apiCallSection.getApi(), equalTo("bulk"));
+        assertThat(apiCallSection.getParams().size(), equalTo(1));
+        assertThat(apiCallSection.getParams().get("refresh"), equalTo("true"));
+        assertThat(apiCallSection.hasBody(), equalTo(true));
+        assertThat(apiCallSection.getBodies().size(), equalTo(4));
+    }
+
+    public void testParseDoSectionWithJsonMultipleBodiesRepeatedProperty() throws Exception {
+        assumeFalse("Test only makes sense if XContent parser doesn't have strict duplicate checks enabled",
+            XContent.isStrictDuplicateDetectionEnabled());
+
+        String[] bodies = new String[] {
+                "{ \"index\": { \"_index\":\"test_index\", \"_type\":\"test_type\", \"_id\":\"test_id\" } }",
+                "{ \"f1\":\"v1\", \"f2\":42 }",
+        };
+        parser = createParser(YamlXContent.yamlXContent,
+                "bulk:\n" +
+                "    refresh: true\n" +
+                "    body: \n" +
+                "        " + bodies[0] + "\n" +
+                "    body: \n" +
+                "        " + bodies[1]
+        );
+
+        DoSection doSection = DoSection.parse(parser);
+        ApiCallSection apiCallSection = doSection.getApiCallSection();
+
+        assertThat(apiCallSection, notNullValue());
+        assertThat(apiCallSection.getApi(), equalTo("bulk"));
+        assertThat(apiCallSection.getParams().size(), equalTo(1));
+        assertThat(apiCallSection.getParams().get("refresh"), equalTo("true"));
+        assertThat(apiCallSection.hasBody(), equalTo(true));
+        assertThat(apiCallSection.getBodies().size(), equalTo(bodies.length));
+        for (int i = 0; i < bodies.length; i++) {
+            assertJsonEquals(apiCallSection.getBodies().get(i), bodies[i]);
+        }
+    }
+
+    public void testParseDoSectionWithYamlBody() throws Exception {
+        parser = createParser(YamlXContent.yamlXContent,
+                "search:\n" +
+                "    body:\n" +
+                "        \"_source\": [ include.field1, include.field2 ]\n" +
+                "        \"query\": { \"match_all\": {} }"
+        );
+        String body = "{ \"_source\": [ \"include.field1\", \"include.field2\" ], \"query\": { \"match_all\": {} }}";
+
+        DoSection doSection = DoSection.parse(parser);
+        ApiCallSection apiCallSection = doSection.getApiCallSection();
+
+        assertThat(apiCallSection, notNullValue());
+        assertThat(apiCallSection.getApi(), equalTo("search"));
+        assertThat(apiCallSection.getParams().size(), equalTo(0));
+        assertThat(apiCallSection.hasBody(), equalTo(true));
+        assertThat(apiCallSection.getBodies().size(), equalTo(1));
+        assertJsonEquals(apiCallSection.getBodies().get(0), body);
+    }
+
+    public void testParseDoSectionWithYamlMultipleBodies() throws Exception {
+        parser = createParser(YamlXContent.yamlXContent,
+                "bulk:\n" +
+                "    refresh: true\n" +
+                "    body:\n" +
+                "        - index:\n" +
+                "            _index: test_index\n" +
+                "            _type:  test_type\n" +
+                "            _id:    test_id\n" +
+                "        - f1: v1\n" +
+                "          f2: 42\n" +
+                "        - index:\n" +
+                "            _index: test_index2\n" +
+                "            _type:  test_type2\n" +
+                "            _id:    test_id2\n" +
+                "        - f1: v2\n" +
+                "          f2: 47"
+        );
+        String[] bodies = new String[4];
+        bodies[0] = "{\"index\": {\"_index\": \"test_index\", \"_type\":  \"test_type\", \"_id\": \"test_id\"}}";
+        bodies[1] = "{ \"f1\":\"v1\", \"f2\": 42 }";
+        bodies[2] = "{\"index\": {\"_index\": \"test_index2\", \"_type\":  \"test_type2\", \"_id\": \"test_id2\"}}";
+        bodies[3] = "{ \"f1\":\"v2\", \"f2\": 47 }";
+
+        DoSection doSection = DoSection.parse(parser);
+        ApiCallSection apiCallSection = doSection.getApiCallSection();
+
+        assertThat(apiCallSection, notNullValue());
+        assertThat(apiCallSection.getApi(), equalTo("bulk"));
+        assertThat(apiCallSection.getParams().size(), equalTo(1));
+        assertThat(apiCallSection.getParams().get("refresh"), equalTo("true"));
+        assertThat(apiCallSection.hasBody(), equalTo(true));
+        assertThat(apiCallSection.getBodies().size(), equalTo(bodies.length));
+
+        for (int i = 0; i < bodies.length; i++) {
+            assertJsonEquals(apiCallSection.getBodies().get(i), bodies[i]);
+        }
+    }
+
+    public void testParseDoSectionWithYamlMultipleBodiesRepeatedProperty() throws Exception {
+        assumeFalse("Test only makes sense if XContent parser doesn't have strict duplicate checks enabled",
+            XContent.isStrictDuplicateDetectionEnabled());
+
+        parser = createParser(YamlXContent.yamlXContent,
+                "bulk:\n" +
+                "    refresh: true\n" +
+                "    body:\n" +
+                "        index:\n" +
+                "            _index: test_index\n" +
+                "            _type:  test_type\n" +
+                "            _id:    test_id\n" +
+                "    body:\n" +
+                "        f1: v1\n" +
+                "        f2: 42\n"
+        );
+        String[] bodies = new String[2];
+        bodies[0] = "{\"index\": {\"_index\": \"test_index\", \"_type\":  \"test_type\", \"_id\": \"test_id\"}}";
+        bodies[1] = "{ \"f1\":\"v1\", \"f2\": 42 }";
+
+        DoSection doSection = DoSection.parse(parser);
+        ApiCallSection apiCallSection = doSection.getApiCallSection();
+
+        assertThat(apiCallSection, notNullValue());
+        assertThat(apiCallSection.getApi(), equalTo("bulk"));
+        assertThat(apiCallSection.getParams().size(), equalTo(1));
+        assertThat(apiCallSection.getParams().get("refresh"), equalTo("true"));
+        assertThat(apiCallSection.hasBody(), equalTo(true));
+        assertThat(apiCallSection.getBodies().size(), equalTo(bodies.length));
+
+        for (int i = 0; i < bodies.length; i++) {
+            assertJsonEquals(apiCallSection.getBodies().get(i), bodies[i]);
+        }
+    }
+
+    public void testParseDoSectionWithYamlBodyMultiGet() throws Exception {
+        parser = createParser(YamlXContent.yamlXContent,
+                "mget:\n" +
+                "    body:\n" +
+                "        docs:\n" +
+                "            - { _index: test_2, _type: test, _id: 1}\n" +
+                "            - { _index: test_1, _type: none, _id: 1}"
+        );
+        String body = "{ \"docs\": [ " +
+                "{\"_index\": \"test_2\", \"_type\":\"test\", \"_id\":1}, " +
+                "{\"_index\": \"test_1\", \"_type\":\"none\", \"_id\":1} " +
+                "]}";
+
+        DoSection doSection = DoSection.parse(parser);
+        ApiCallSection apiCallSection = doSection.getApiCallSection();
+
+        assertThat(apiCallSection, notNullValue());
+        assertThat(apiCallSection.getApi(), equalTo("mget"));
+        assertThat(apiCallSection.getParams().size(), equalTo(0));
+        assertThat(apiCallSection.hasBody(), equalTo(true));
+        assertThat(apiCallSection.getBodies().size(), equalTo(1));
+        assertJsonEquals(apiCallSection.getBodies().get(0), body);
+    }
+
+    public void testParseDoSectionWithBodyStringified() throws Exception {
+        parser = createParser(YamlXContent.yamlXContent,
+                "index:\n" +
+                "    index:  test_1\n" +
+                "    type:   test\n" +
+                "    id:     1\n" +
+                "    body:   \"{ \\\"_source\\\": true, \\\"query\\\": { \\\"match_all\\\": {} } }\""
+        );
+
+        DoSection doSection = DoSection.parse(parser);
+        ApiCallSection apiCallSection = doSection.getApiCallSection();
+
+        assertThat(apiCallSection, notNullValue());
+        assertThat(apiCallSection.getApi(), equalTo("index"));
+        assertThat(apiCallSection.getParams().size(), equalTo(3));
+        assertThat(apiCallSection.getParams().get("index"), equalTo("test_1"));
+        assertThat(apiCallSection.getParams().get("type"), equalTo("test"));
+        assertThat(apiCallSection.getParams().get("id"), equalTo("1"));
+        assertThat(apiCallSection.hasBody(), equalTo(true));
+        assertThat(apiCallSection.getBodies().size(), equalTo(1));
+        //stringified body is taken as is
+        assertJsonEquals(apiCallSection.getBodies().get(0), "{ \"_source\": true, \"query\": { \"match_all\": {} } }");
+    }
+
+    public void testParseDoSectionWithBodiesStringifiedAndNot() throws Exception {
+        parser = createParser(YamlXContent.yamlXContent,
+                "index:\n" +
+                "    body:\n" +
+                "        - \"{ \\\"_source\\\": true, \\\"query\\\": { \\\"match_all\\\": {} } }\"\n" +
+                "        - { size: 100, query: { match_all: {} } }"
+        );
+
+        String body = "{ \"size\": 100, \"query\": { \"match_all\": {} } }";
+
+        DoSection doSection = DoSection.parse(parser);
+        ApiCallSection apiCallSection = doSection.getApiCallSection();
+
+        assertThat(apiCallSection.getApi(), equalTo("index"));
+        assertThat(apiCallSection.getParams().size(), equalTo(0));
+        assertThat(apiCallSection.hasBody(), equalTo(true));
+        assertThat(apiCallSection.getBodies().size(), equalTo(2));
+        //stringified body is taken as is
+        assertJsonEquals(apiCallSection.getBodies().get(0), "{ \"_source\": true, \"query\": { \"match_all\": {} } }");
+        assertJsonEquals(apiCallSection.getBodies().get(1), body);
+    }
+
+    public void testParseDoSectionWithCatch() throws Exception {
+        parser = createParser(YamlXContent.yamlXContent,
+                "catch: missing\n" +
+                "indices.get_warmer:\n" +
+                "    index: test_index\n" +
+                "    name: test_warmer"
+        );
+
+        DoSection doSection = DoSection.parse(parser);
+        assertThat(doSection.getCatch(), equalTo("missing"));
+        assertThat(doSection.getApiCallSection(), notNullValue());
+        assertThat(doSection.getApiCallSection().getApi(), equalTo("indices.get_warmer"));
+        assertThat(doSection.getApiCallSection().getParams().size(), equalTo(2));
+        assertThat(doSection.getApiCallSection().hasBody(), equalTo(false));
+    }
+
+    public void testParseDoSectionWithHeaders() throws Exception {
+        parser = createParser(YamlXContent.yamlXContent,
+                "headers:\n" +
+                "    Authorization: \"thing one\"\n" +
+                "    Content-Type: \"application/json\"\n" +
+                "indices.get_warmer:\n" +
+                "    index: test_index\n" +
+                "    name: test_warmer"
+        );
+
+        DoSection doSection = DoSection.parse(parser);
+        assertThat(doSection.getApiCallSection(), notNullValue());
+        assertThat(doSection.getApiCallSection().getApi(), equalTo("indices.get_warmer"));
+        assertThat(doSection.getApiCallSection().getParams().size(), equalTo(2));
+        assertThat(doSection.getApiCallSection().hasBody(), equalTo(false));
+        assertThat(doSection.getApiCallSection().getHeaders(), notNullValue());
+        assertThat(doSection.getApiCallSection().getHeaders().size(), equalTo(2));
+        assertThat(doSection.getApiCallSection().getHeaders().get("Authorization"), equalTo("thing one"));
+        assertThat(doSection.getApiCallSection().getHeaders().get("Content-Type"), equalTo("application/json"));
+    }
+
+    public void testParseDoSectionWithoutClientCallSection() throws Exception {
+        parser = createParser(YamlXContent.yamlXContent,
+                "catch: missing\n"
+        );
+
+        Exception e = expectThrows(IllegalArgumentException.class, () -> DoSection.parse(parser));
+        assertThat(e.getMessage(), is("client call section is mandatory within a do section"));
+    }
+
+    public void testParseDoSectionMultivaluedField() throws Exception {
+        parser = createParser(YamlXContent.yamlXContent,
+                "indices.get_field_mapping:\n" +
+                "        index: test_index\n" +
+                "        type: test_type\n" +
+                "        field: [ text , text1 ]"
+        );
+
+        DoSection doSection = DoSection.parse(parser);
+        assertThat(doSection.getCatch(), nullValue());
+        assertThat(doSection.getApiCallSection(), notNullValue());
+        assertThat(doSection.getApiCallSection().getApi(), equalTo("indices.get_field_mapping"));
+        assertThat(doSection.getApiCallSection().getParams().size(), equalTo(3));
+        assertThat(doSection.getApiCallSection().getParams().get("index"), equalTo("test_index"));
+        assertThat(doSection.getApiCallSection().getParams().get("type"), equalTo("test_type"));
+        assertThat(doSection.getApiCallSection().getParams().get("field"), equalTo("text,text1"));
+        assertThat(doSection.getApiCallSection().hasBody(), equalTo(false));
+        assertThat(doSection.getApiCallSection().getBodies().size(), equalTo(0));
+    }
+
+    public void testParseDoSectionExpectedWarnings() throws Exception {
+        parser = createParser(YamlXContent.yamlXContent,
+                "indices.get_field_mapping:\n" +
+                "        index: test_index\n" +
+                "        type: test_type\n" +
+                "warnings:\n" +
+                "    - some test warning they are typically pretty long\n" +
+                "    - some other test warning somtimes they have [in] them"
+        );
+
+        DoSection doSection = DoSection.parse(parser);
+        assertThat(doSection.getCatch(), nullValue());
+        assertThat(doSection.getApiCallSection(), notNullValue());
+        assertThat(doSection.getApiCallSection().getApi(), equalTo("indices.get_field_mapping"));
+        assertThat(doSection.getApiCallSection().getParams().size(), equalTo(2));
+        assertThat(doSection.getApiCallSection().getParams().get("index"), equalTo("test_index"));
+        assertThat(doSection.getApiCallSection().getParams().get("type"), equalTo("test_type"));
+        assertThat(doSection.getApiCallSection().hasBody(), equalTo(false));
+        assertThat(doSection.getApiCallSection().getBodies().size(), equalTo(0));
+        assertThat(doSection.getExpectedWarningHeaders(), equalTo(Arrays.asList(
+                "some test warning they are typically pretty long",
+                "some other test warning somtimes they have [in] them")));
+
+        parser = createParser(YamlXContent.yamlXContent,
+                "indices.get_field_mapping:\n" +
+                "        index: test_index\n" +
+                "warnings:\n" +
+                "    - just one entry this time"
+        );
+
+        doSection = DoSection.parse(parser);
+        assertThat(doSection.getCatch(), nullValue());
+        assertThat(doSection.getApiCallSection(), notNullValue());
+        assertThat(doSection.getExpectedWarningHeaders(), equalTo(singletonList(
+                "just one entry this time")));
+
+    }
+
+    private void assertJsonEquals(Map<String, Object> actual, String expected) throws IOException {
+        Map<String,Object> expectedMap;
+        try (XContentParser parser = createParser(YamlXContent.yamlXContent, expected)) {
+            expectedMap = parser.mapOrdered();
+        }
+        MatcherAssert.assertThat(actual, equalTo(expectedMap));
+    }
 }
diff --git a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/parser/SetSectionParserTests.java b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/SetSectionTests.java
index 91d3575878..74b3d774bd 100644
--- a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/parser/SetSectionParserTests.java
+++ b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/SetSectionTests.java
@@ -16,25 +16,22 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.elasticsearch.test.rest.yaml.parser;
+package org.elasticsearch.test.rest.yaml.section;
 
+import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.xcontent.yaml.YamlXContent;
-import org.elasticsearch.test.rest.yaml.section.SetSection;
 
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.notNullValue;
 
-public class SetSectionParserTests extends AbstractClientYamlTestFragmentParserTestCase {
+public class SetSectionTests extends AbstractClientYamlTestFragmentParserTestCase {
     public void testParseSetSectionSingleValue() throws Exception {
         parser = createParser(YamlXContent.yamlXContent,
                         "{ _id: id }"
         );
 
-        SetSectionParser setSectionParser = new SetSectionParser();
-
-        SetSection setSection = setSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
-
+        SetSection setSection = SetSection.parse(parser);
         assertThat(setSection, notNullValue());
         assertThat(setSection.getStash(), notNullValue());
         assertThat(setSection.getStash().size(), equalTo(1));
@@ -46,10 +43,7 @@ public class SetSectionParserTests extends AbstractClientYamlTestFragmentParserT
                 "{ _id: id, _type: type, _index: index }"
         );
 
-        SetSectionParser setSectionParser = new SetSectionParser();
-
-        SetSection setSection = setSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
-
+        SetSection setSection = SetSection.parse(parser);
         assertThat(setSection, notNullValue());
         assertThat(setSection.getStash(), notNullValue());
         assertThat(setSection.getStash().size(), equalTo(3));
@@ -63,12 +57,7 @@ public class SetSectionParserTests extends AbstractClientYamlTestFragmentParserT
                 "{ }"
         );
 
-        SetSectionParser setSectionParser = new SetSectionParser();
-        try {
-            setSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
-            fail("Expected RestTestParseException");
-        } catch (ClientYamlTestParseException e) {
-            assertThat(e.getMessage(), is("set section must set at least a value"));
-        }
+        Exception e = expectThrows(ParsingException.class, () -> SetSection.parse(parser));
+        assertThat(e.getMessage(), is("set section must set at least a value"));
     }
 }
\ No newline at end of file
diff --git a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/parser/SetupSectionParserTests.java b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/SetupSectionTests.java
index e18e4a3e78..f6174cf0be 100644
--- a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/parser/SetupSectionParserTests.java
+++ b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/SetupSectionTests.java
@@ -16,7 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.elasticsearch.test.rest.yaml.parser;
+package org.elasticsearch.test.rest.yaml.section;
 
 import org.elasticsearch.Version;
 import org.elasticsearch.common.xcontent.yaml.YamlXContent;
@@ -25,7 +25,7 @@ import org.elasticsearch.test.rest.yaml.section.SetupSection;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.notNullValue;
 
-public class SetupSectionParserTests extends AbstractClientYamlTestFragmentParserTestCase {
+public class SetupSectionTests extends AbstractClientYamlTestFragmentParserTestCase {
     public void testParseSetupSection() throws Exception {
         parser = createParser(YamlXContent.yamlXContent,
                 "  - do:\n" +
@@ -42,8 +42,7 @@ public class SetupSectionParserTests extends AbstractClientYamlTestFragmentParse
                 "        body:   { \"include\": { \"field1\": \"v1\", \"field2\": \"v2\" }, \"count\": 1 }\n"
         );
 
-        SetupSectionParser setupSectionParser = new SetupSectionParser();
-        SetupSection setupSection = setupSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
+        SetupSection setupSection = SetupSection.parse(parser);
 
         assertThat(setupSection, notNullValue());
         assertThat(setupSection.getSkipSection().isEmpty(), equalTo(true));
@@ -71,8 +70,7 @@ public class SetupSectionParserTests extends AbstractClientYamlTestFragmentParse
                         "        body:   { \"include\": { \"field1\": \"v1\", \"field2\": \"v2\" }, \"count\": 1 }\n"
         );
 
-        SetupSectionParser setupSectionParser = new SetupSectionParser();
-        SetupSection setupSection = setupSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
+        SetupSection setupSection = SetupSection.parse(parser);
 
         assertThat(setupSection, notNullValue());
         assertThat(setupSection.getSkipSection().isEmpty(), equalTo(false));
diff --git a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/SkipSectionTests.java b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/SkipSectionTests.java
index c8f7b35128..984cbf1715 100644
--- a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/SkipSectionTests.java
+++ b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/SkipSectionTests.java
@@ -20,12 +20,19 @@
 package org.elasticsearch.test.rest.yaml.section;
 
 import org.elasticsearch.Version;
-import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.common.ParsingException;
+import org.elasticsearch.common.xcontent.yaml.YamlXContent;
+import org.elasticsearch.test.VersionUtils;
 
 import java.util.Arrays;
 import java.util.Collections;
 
-public class SkipSectionTests extends ESTestCase {
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.notNullValue;
+import static org.hamcrest.Matchers.nullValue;
+
+public class SkipSectionTests extends AbstractClientYamlTestFragmentParserTestCase {
 
     public void testSkip() {
         SkipSection section = new SkipSection("2.0.0 - 2.1.0", randomBoolean() ? Collections.emptyList() :
@@ -44,4 +51,92 @@ public class SkipSectionTests extends ESTestCase {
         section = new SkipSection(null, Arrays.asList("warnings"), null);
         assertEquals("[FOOBAR] skipped, unsupported features [warnings]", section.getSkipMessage("FOOBAR"));
     }
+
+    public void testParseSkipSectionVersionNoFeature() throws Exception {
+        parser = createParser(YamlXContent.yamlXContent,
+                "version:     \" - 2.1.0\"\n" +
+                "reason:      Delete ignores the parent param"
+        );
+
+        SkipSection skipSection = SkipSection.parse(parser);
+        assertThat(skipSection, notNullValue());
+        assertThat(skipSection.getLowerVersion(), equalTo(VersionUtils.getFirstVersion()));
+        assertThat(skipSection.getUpperVersion(), equalTo(Version.V_2_1_0));
+        assertThat(skipSection.getFeatures().size(), equalTo(0));
+        assertThat(skipSection.getReason(), equalTo("Delete ignores the parent param"));
+    }
+
+    public void testParseSkipSectionAllVersions() throws Exception {
+        parser = createParser(YamlXContent.yamlXContent,
+            "version:     \" all \"\n" +
+            "reason:      Delete ignores the parent param"
+        );
+
+        SkipSection skipSection = SkipSection.parse(parser);
+        assertThat(skipSection, notNullValue());
+        assertThat(skipSection.getLowerVersion(), equalTo(VersionUtils.getFirstVersion()));
+        assertThat(skipSection.getUpperVersion(), equalTo(Version.CURRENT));
+        assertThat(skipSection.getFeatures().size(), equalTo(0));
+        assertThat(skipSection.getReason(), equalTo("Delete ignores the parent param"));
+    }
+
+    public void testParseSkipSectionFeatureNoVersion() throws Exception {
+        parser = createParser(YamlXContent.yamlXContent,
+                "features:     regex"
+        );
+
+        SkipSection skipSection = SkipSection.parse(parser);
+        assertThat(skipSection, notNullValue());
+        assertThat(skipSection.isVersionCheck(), equalTo(false));
+        assertThat(skipSection.getFeatures().size(), equalTo(1));
+        assertThat(skipSection.getFeatures().get(0), equalTo("regex"));
+        assertThat(skipSection.getReason(), nullValue());
+    }
+
+    public void testParseSkipSectionFeaturesNoVersion() throws Exception {
+        parser = createParser(YamlXContent.yamlXContent,
+                "features:     [regex1,regex2,regex3]"
+        );
+
+        SkipSection skipSection = SkipSection.parse(parser);
+        assertThat(skipSection, notNullValue());
+        assertThat(skipSection.isVersionCheck(), equalTo(false));
+        assertThat(skipSection.getFeatures().size(), equalTo(3));
+        assertThat(skipSection.getFeatures().get(0), equalTo("regex1"));
+        assertThat(skipSection.getFeatures().get(1), equalTo("regex2"));
+        assertThat(skipSection.getFeatures().get(2), equalTo("regex3"));
+        assertThat(skipSection.getReason(), nullValue());
+    }
+
+    public void testParseSkipSectionBothFeatureAndVersion() throws Exception {
+        parser = createParser(YamlXContent.yamlXContent,
+                "version:     \" - 0.90.2\"\n" +
+                "features:     regex\n" +
+                "reason:      Delete ignores the parent param"
+        );
+
+        SkipSection skipSection = SkipSection.parse(parser);
+        assertEquals(VersionUtils.getFirstVersion(), skipSection.getLowerVersion());
+        assertEquals(Version.fromString("0.90.2"), skipSection.getUpperVersion());
+        assertEquals(Arrays.asList("regex"), skipSection.getFeatures());
+        assertEquals("Delete ignores the parent param", skipSection.getReason());
+    }
+
+    public void testParseSkipSectionNoReason() throws Exception {
+        parser = createParser(YamlXContent.yamlXContent,
+                "version:     \" - 0.90.2\"\n"
+        );
+
+        Exception e = expectThrows(ParsingException.class, () -> SkipSection.parse(parser));
+        assertThat(e.getMessage(), is("reason is mandatory within skip version section"));
+    }
+
+    public void testParseSkipSectionNoVersionNorFeature() throws Exception {
+        parser = createParser(YamlXContent.yamlXContent,
+                "reason:      Delete ignores the parent param\n"
+        );
+
+        Exception e = expectThrows(ParsingException.class, () -> SkipSection.parse(parser));
+        assertThat(e.getMessage(), is("version or features is mandatory within skip section"));
+    }
 }
diff --git a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/parser/TeardownSectionParserTests.java b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/TeardownSectionTests.java
index e84eb6b681..f057d0d370 100644
--- a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/parser/TeardownSectionParserTests.java
+++ b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/TeardownSectionTests.java
@@ -17,22 +17,18 @@
  * under the License.
  */
 
-package org.elasticsearch.test.rest.yaml.parser;
+package org.elasticsearch.test.rest.yaml.section;
 
 import org.elasticsearch.Version;
 import org.elasticsearch.common.xcontent.yaml.YamlXContent;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestSuiteParseContext;
-import org.elasticsearch.test.rest.yaml.parser.TeardownSectionParser;
-import org.elasticsearch.test.rest.yaml.section.TeardownSection;
 
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.notNullValue;
 
 /**
- * Unit tests for the teardown section parser
+ * Unit tests for the teardown section.
  */
-public class TeardownSectionParserTests extends AbstractClientYamlTestFragmentParserTestCase {
-
+public class TeardownSectionTests extends AbstractClientYamlTestFragmentParserTestCase {
     public void testParseTeardownSection() throws Exception {
         parser = createParser(YamlXContent.yamlXContent,
                 "  - do:\n" +
@@ -49,9 +45,7 @@ public class TeardownSectionParserTests extends AbstractClientYamlTestFragmentPa
                 "        ignore: 404"
         );
 
-        TeardownSectionParser teardownSectionParser = new TeardownSectionParser();
-        TeardownSection section = teardownSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
-
+        TeardownSection section = TeardownSection.parse(parser);
         assertThat(section, notNullValue());
         assertThat(section.getSkipSection().isEmpty(), equalTo(true));
         assertThat(section.getDoSections().size(), equalTo(2));
@@ -78,9 +72,7 @@ public class TeardownSectionParserTests extends AbstractClientYamlTestFragmentPa
                 "        ignore: 404"
         );
 
-        TeardownSectionParser teardownSectionParser = new TeardownSectionParser();
-        TeardownSection section = teardownSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
-
+        TeardownSection section = TeardownSection.parse(parser);
         assertThat(section, notNullValue());
         assertThat(section.getSkipSection().isEmpty(), equalTo(false));
         assertThat(section.getSkipSection().getLowerVersion(), equalTo(Version.V_2_0_0));