Stop returning "es." internal exception headers as http response headers (#22703)

move "es." internal headers to separate metadata set in ElasticsearchException and stop returning them as response headers

Closes #17593

* [TEST] remove ESExceptionTests, move its methods to ElasticsearchExceptionTests or ExceptionSerializationTests

1 files changed, 337 insertions, 68 deletions

diff --git a/core/src/test/java/org/elasticsearch/ElasticsearchExceptionTests.java b/core/src/test/java/org/elasticsearch/ElasticsearchExceptionTests.java
index 576b53ad0e..4a8a90afab 100644
--- a/core/src/test/java/org/elasticsearch/ElasticsearchExceptionTests.java
+++ b/core/src/test/java/org/elasticsearch/ElasticsearchExceptionTests.java
@@ -21,73 +21,338 @@ package org.elasticsearch;
 
 import org.apache.lucene.util.Constants;
 import org.elasticsearch.action.RoutingMissingException;
+import org.elasticsearch.action.search.SearchPhaseExecutionException;
+import org.elasticsearch.action.search.ShardSearchFailure;
 import org.elasticsearch.action.support.broadcast.BroadcastShardOperationFailedException;
 import org.elasticsearch.cluster.block.ClusterBlockException;
+import org.elasticsearch.common.ParsingException;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.common.xcontent.XContentLocation;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.discovery.DiscoverySettings;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.index.IndexNotFoundException;
+import org.elasticsearch.index.query.QueryShardException;
 import org.elasticsearch.index.shard.IndexShardRecoveringException;
 import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.search.SearchParseException;
+import org.elasticsearch.search.SearchShardTarget;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.TestSearchContext;
+import org.elasticsearch.transport.RemoteTransportException;
 import org.hamcrest.Matcher;
 
+import java.io.EOFException;
+import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.util.Collections;
 
 import static java.util.Collections.singleton;
-import static org.hamcrest.CoreMatchers.equalTo;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertToXContentEquivalent;
 import static org.hamcrest.CoreMatchers.hasItem;
-import static org.hamcrest.CoreMatchers.startsWith;
+import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasSize;
+import static org.hamcrest.Matchers.startsWith;
 
 public class ElasticsearchExceptionTests extends ESTestCase {
 
+    public void testStatus() {
+        ElasticsearchException exception = new ElasticsearchException("test");
+        assertThat(exception.status(), equalTo(RestStatus.INTERNAL_SERVER_ERROR));
+
+        exception = new ElasticsearchException("test", new RuntimeException());
+        assertThat(exception.status(), equalTo(RestStatus.INTERNAL_SERVER_ERROR));
+
+        exception = new ElasticsearchException("test", new ResourceNotFoundException("test"));
+        assertThat(exception.status(), equalTo(RestStatus.INTERNAL_SERVER_ERROR));
+
+        exception = new RemoteTransportException("test", new ResourceNotFoundException("test"));
+        assertThat(exception.status(), equalTo(RestStatus.NOT_FOUND));
+
+        exception = new RemoteTransportException("test", new ResourceAlreadyExistsException("test"));
+        assertThat(exception.status(), equalTo(RestStatus.BAD_REQUEST));
+
+        exception = new RemoteTransportException("test", new IllegalArgumentException("foobar"));
+        assertThat(exception.status(), equalTo(RestStatus.BAD_REQUEST));
+
+        exception = new RemoteTransportException("test", new IllegalStateException("foobar"));
+        assertThat(exception.status(), equalTo(RestStatus.INTERNAL_SERVER_ERROR));
+    }
+
+    public void testGuessRootCause() {
+        {
+            ElasticsearchException exception = new ElasticsearchException("foo", new ElasticsearchException("bar",
+                    new IndexNotFoundException("foo", new RuntimeException("foobar"))));
+            ElasticsearchException[] rootCauses = exception.guessRootCauses();
+            assertEquals(rootCauses.length, 1);
+            assertEquals(ElasticsearchException.getExceptionName(rootCauses[0]), "index_not_found_exception");
+            assertEquals(rootCauses[0].getMessage(), "no such index");
+            ShardSearchFailure failure = new ShardSearchFailure(new ParsingException(1, 2, "foobar", null),
+                    new SearchShardTarget("node_1", new Index("foo", "_na_"), 1));
+            ShardSearchFailure failure1 = new ShardSearchFailure(new ParsingException(1, 2, "foobar", null),
+                    new SearchShardTarget("node_1", new Index("foo", "_na_"), 2));
+            SearchPhaseExecutionException ex = new SearchPhaseExecutionException("search", "all shards failed",
+                    new ShardSearchFailure[]{failure, failure1});
+            if (randomBoolean()) {
+                rootCauses = (randomBoolean() ? new RemoteTransportException("remoteboom", ex) : ex).guessRootCauses();
+            } else {
+                rootCauses = ElasticsearchException.guessRootCauses(randomBoolean() ? new RemoteTransportException("remoteboom", ex) : ex);
+            }
+            assertEquals(ElasticsearchException.getExceptionName(rootCauses[0]), "parsing_exception");
+            assertEquals(rootCauses[0].getMessage(), "foobar");
+
+            ElasticsearchException oneLevel = new ElasticsearchException("foo", new RuntimeException("foobar"));
+            rootCauses = oneLevel.guessRootCauses();
+            assertEquals(ElasticsearchException.getExceptionName(rootCauses[0]), "exception");
+            assertEquals(rootCauses[0].getMessage(), "foo");
+        }
+        {
+            ShardSearchFailure failure = new ShardSearchFailure(
+                    new ParsingException(1, 2, "foobar", null),
+                    new SearchShardTarget("node_1", new Index("foo", "_na_"), 1));
+            ShardSearchFailure failure1 = new ShardSearchFailure(new QueryShardException(new Index("foo1", "_na_"), "foobar", null),
+                    new SearchShardTarget("node_1", new Index("foo1", "_na_"), 1));
+            ShardSearchFailure failure2 = new ShardSearchFailure(new QueryShardException(new Index("foo1", "_na_"), "foobar", null),
+                    new SearchShardTarget("node_1", new Index("foo1", "_na_"), 2));
+            SearchPhaseExecutionException ex = new SearchPhaseExecutionException("search", "all shards failed",
+                    new ShardSearchFailure[]{failure, failure1, failure2});
+            final ElasticsearchException[] rootCauses = ex.guessRootCauses();
+            assertEquals(rootCauses.length, 2);
+            assertEquals(ElasticsearchException.getExceptionName(rootCauses[0]), "parsing_exception");
+            assertEquals(rootCauses[0].getMessage(), "foobar");
+            assertEquals(((ParsingException) rootCauses[0]).getLineNumber(), 1);
+            assertEquals(((ParsingException) rootCauses[0]).getColumnNumber(), 2);
+            assertEquals(ElasticsearchException.getExceptionName(rootCauses[1]), "query_shard_exception");
+            assertEquals((rootCauses[1]).getIndex().getName(), "foo1");
+            assertEquals(rootCauses[1].getMessage(), "foobar");
+        }
+
+        {
+            final ElasticsearchException[] foobars = ElasticsearchException.guessRootCauses(new IllegalArgumentException("foobar"));
+            assertEquals(foobars.length, 1);
+            assertTrue(foobars[0] instanceof ElasticsearchException);
+            assertEquals(foobars[0].getMessage(), "foobar");
+            assertEquals(foobars[0].getCause().getClass(), IllegalArgumentException.class);
+            assertEquals(foobars[0].getExceptionName(), "illegal_argument_exception");
+        }
+    }
+
+    public void testDeduplicate() throws IOException {
+        {
+            ShardSearchFailure failure = new ShardSearchFailure(new ParsingException(1, 2, "foobar", null),
+                    new SearchShardTarget("node_1", new Index("foo", "_na_"), 1));
+            ShardSearchFailure failure1 = new ShardSearchFailure(new ParsingException(1, 2, "foobar", null),
+                    new SearchShardTarget("node_1", new Index("foo", "_na_"), 2));
+            SearchPhaseExecutionException ex = new SearchPhaseExecutionException("search", "all shards failed",
+                    randomBoolean() ? failure1.getCause() : failure.getCause(), new ShardSearchFailure[]{failure, failure1});
+            XContentBuilder builder = XContentFactory.jsonBuilder();
+            builder.startObject();
+            ex.toXContent(builder, ToXContent.EMPTY_PARAMS);
+            builder.endObject();
+            String expected = "{\"type\":\"search_phase_execution_exception\",\"reason\":\"all shards failed\",\"phase\":\"search\"," +
+                    "\"grouped\":true,\"failed_shards\":[{\"shard\":1,\"index\":\"foo\",\"node\":\"node_1\",\"reason\":" +
+                    "{\"type\":\"parsing_exception\",\"reason\":\"foobar\",\"line\":1,\"col\":2}}]}";
+            assertEquals(expected, builder.string());
+        }
+        {
+            ShardSearchFailure failure = new ShardSearchFailure(new ParsingException(1, 2, "foobar", null),
+                    new SearchShardTarget("node_1", new Index("foo", "_na_"), 1));
+            ShardSearchFailure failure1 = new ShardSearchFailure(new QueryShardException(new Index("foo1", "_na_"), "foobar", null),
+                    new SearchShardTarget("node_1", new Index("foo1", "_na_"), 1));
+            ShardSearchFailure failure2 = new ShardSearchFailure(new QueryShardException(new Index("foo1", "_na_"), "foobar", null),
+                    new SearchShardTarget("node_1", new Index("foo1", "_na_"), 2));
+            SearchPhaseExecutionException ex = new SearchPhaseExecutionException("search", "all shards failed",
+                    new ShardSearchFailure[]{failure, failure1, failure2});
+            XContentBuilder builder = XContentFactory.jsonBuilder();
+            builder.startObject();
+            ex.toXContent(builder, ToXContent.EMPTY_PARAMS);
+            builder.endObject();
+            String expected = "{\"type\":\"search_phase_execution_exception\",\"reason\":\"all shards failed\"," +
+                    "\"phase\":\"search\",\"grouped\":true,\"failed_shards\":[{\"shard\":1,\"index\":\"foo\",\"node\":\"node_1\"," +
+                    "\"reason\":{\"type\":\"parsing_exception\",\"reason\":\"foobar\",\"line\":1,\"col\":2}},{\"shard\":1," +
+                    "\"index\":\"foo1\",\"node\":\"node_1\",\"reason\":{\"type\":\"query_shard_exception\",\"reason\":\"foobar\"," +
+                    "\"index_uuid\":\"_na_\",\"index\":\"foo1\"}}]}";
+            assertEquals(expected, builder.string());
+        }
+        {
+            ShardSearchFailure failure = new ShardSearchFailure(new ParsingException(1, 2, "foobar", null),
+                    new SearchShardTarget("node_1", new Index("foo", "_na_"), 1));
+            ShardSearchFailure failure1 = new ShardSearchFailure(new ParsingException(1, 2, "foobar", null),
+                    new SearchShardTarget("node_1", new Index("foo", "_na_"), 2));
+            NullPointerException nullPointerException = new NullPointerException();
+            SearchPhaseExecutionException ex = new SearchPhaseExecutionException("search", "all shards failed", nullPointerException,
+                    new ShardSearchFailure[]{failure, failure1});
+            assertEquals(nullPointerException, ex.getCause());
+            XContentBuilder builder = XContentFactory.jsonBuilder();
+            builder.startObject();
+            ex.toXContent(builder, ToXContent.EMPTY_PARAMS);
+            builder.endObject();
+            String expected = "{\"type\":\"search_phase_execution_exception\",\"reason\":\"all shards failed\"," +
+                    "\"phase\":\"search\",\"grouped\":true,\"failed_shards\":[{\"shard\":1,\"index\":\"foo\",\"node\":\"node_1\"," +
+                    "\"reason\":{\"type\":\"parsing_exception\",\"reason\":\"foobar\",\"line\":1,\"col\":2}}]," +
+                    "\"caused_by\":{\"type\":\"null_pointer_exception\",\"reason\":null}}";
+            assertEquals(expected, builder.string());
+        }
+    }
+
+    /**
+     * Check whether this exception contains an exception of the given type:
+     * either it is of the given class itself or it contains a nested cause
+     * of the given type.
+     *
+     * @param exType the exception type to look for
+     * @return whether there is a nested exception of the specified type
+     */
+    private static boolean contains(Throwable t, Class<? extends Throwable> exType) {
+        if (exType == null) {
+            return false;
+        }
+        for (Throwable cause = t; t != null; t = t.getCause()) {
+            if (exType.isInstance(cause)) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+    public void testGetRootCause() {
+        Exception root = new RuntimeException("foobar");
+        ElasticsearchException exception = new ElasticsearchException("foo", new ElasticsearchException("bar",
+                new IllegalArgumentException("index is closed", root)));
+        assertEquals(root, exception.getRootCause());
+        assertTrue(contains(exception, RuntimeException.class));
+        assertFalse(contains(exception, EOFException.class));
+    }
+
+    public void testToString() {
+        ElasticsearchException exception = new ElasticsearchException("foo", new ElasticsearchException("bar",
+                new IllegalArgumentException("index is closed", new RuntimeException("foobar"))));
+        assertEquals("ElasticsearchException[foo]; nested: ElasticsearchException[bar]; nested: IllegalArgumentException" +
+                "[index is closed]; nested: RuntimeException[foobar];", exception.toString());
+    }
+
     public void testToXContent() throws IOException {
-        ElasticsearchException e = new ElasticsearchException("test");
-        assertExceptionAsJson(e, false, equalTo("{\"type\":\"exception\",\"reason\":\"test\"}"));
-
-        e = new IndexShardRecoveringException(new ShardId("_test", "_0", 5));
-        assertExceptionAsJson(e, false, equalTo("{\"type\":\"index_shard_recovering_exception\"," +
-                "\"reason\":\"CurrentState[RECOVERING] Already recovering\",\"index_uuid\":\"_0\",\"shard\":\"5\",\"index\":\"_test\"}"));
-
-        e = new BroadcastShardOperationFailedException(new ShardId("_index", "_uuid", 12), "foo", new IllegalStateException("bar"));
-        assertExceptionAsJson(e, false, equalTo("{\"type\":\"illegal_state_exception\",\"reason\":\"bar\"}"));
-
-        e = new ElasticsearchException(new IllegalArgumentException("foo"));
-        assertExceptionAsJson(e, false, equalTo("{\"type\":\"exception\",\"reason\":\"java.lang.IllegalArgumentException: foo\"," +
-                "\"caused_by\":{\"type\":\"illegal_argument_exception\",\"reason\":\"foo\"}}"));
-
-        e = new ElasticsearchException("foo", new IllegalStateException("bar"));
-        assertExceptionAsJson(e, false, equalTo("{\"type\":\"exception\",\"reason\":\"foo\"," +
-                "\"caused_by\":{\"type\":\"illegal_state_exception\",\"reason\":\"bar\"}}"));
-
-        // Test the same exception but with the "rest.exception.stacktrace.skip" parameter disabled: the stack_trace must be present
-        // in the JSON. Since the stack can be large, it only checks the beginning of the JSON.
-        assertExceptionAsJson(e, true, startsWith("{\"type\":\"exception\",\"reason\":\"foo\"," +
-                "\"caused_by\":{\"type\":\"illegal_state_exception\",\"reason\":\"bar\"," +
-                "\"stack_trace\":\"java.lang.IllegalStateException: bar" +
-                (Constants.WINDOWS ? "\\r\\n" : "\\n") +
-                "\\tat org.elasticsearch."));
+        {
+            ElasticsearchException e = new ElasticsearchException("test");
+            assertExceptionAsJson(e, "{\"type\":\"exception\",\"reason\":\"test\"}");
+        }
+        {
+            ElasticsearchException e = new IndexShardRecoveringException(new ShardId("_test", "_0", 5));
+            assertExceptionAsJson(e, "{\"type\":\"index_shard_recovering_exception\"," +
+                    "\"reason\":\"CurrentState[RECOVERING] Already recovering\",\"index_uuid\":\"_0\"," +
+                    "\"shard\":\"5\",\"index\":\"_test\"}");
+        }
+        {
+            ElasticsearchException e = new BroadcastShardOperationFailedException(new ShardId("_index", "_uuid", 12), "foo",
+                    new IllegalStateException("bar"));
+            assertExceptionAsJson(e, "{\"type\":\"illegal_state_exception\",\"reason\":\"bar\"}");
+        }
+        {
+            ElasticsearchException e = new ElasticsearchException(new IllegalArgumentException("foo"));
+            assertExceptionAsJson(e, "{\"type\":\"exception\",\"reason\":\"java.lang.IllegalArgumentException: foo\"," +
+                    "\"caused_by\":{\"type\":\"illegal_argument_exception\",\"reason\":\"foo\"}}");
+        }
+        {
+            ElasticsearchException e = new SearchParseException(new TestSearchContext(null), "foo", new XContentLocation(1,0));
+            assertExceptionAsJson(e, "{\"type\":\"search_parse_exception\",\"reason\":\"foo\",\"line\":1,\"col\":0}");
+        }
+        {
+            ElasticsearchException ex = new ElasticsearchException("foo",
+                    new ElasticsearchException("bar", new IllegalArgumentException("index is closed", new RuntimeException("foobar"))));
+            assertExceptionAsJson(ex, "{\"type\":\"exception\",\"reason\":\"foo\",\"caused_by\":{\"type\":\"exception\"," +
+                    "\"reason\":\"bar\",\"caused_by\":{\"type\":\"illegal_argument_exception\",\"reason\":\"index is closed\"," +
+                    "\"caused_by\":{\"type\":\"runtime_exception\",\"reason\":\"foobar\"}}}}");
+        }
+        {
+            ElasticsearchException e = new ElasticsearchException("foo", new IllegalStateException("bar"));
+            assertExceptionAsJson(e, "{\"type\":\"exception\",\"reason\":\"foo\"," +
+                    "\"caused_by\":{\"type\":\"illegal_state_exception\",\"reason\":\"bar\"}}");
+
+            // Test the same exception but with the "rest.exception.stacktrace.skip" parameter disabled: the stack_trace must be present
+            // in the JSON. Since the stack can be large, it only checks the beginning of the JSON.
+            ToXContent.Params params = new ToXContent.MapParams(
+                    Collections.singletonMap(ElasticsearchException.REST_EXCEPTION_SKIP_STACK_TRACE, "false"));
+            String actual;
+            try (XContentBuilder builder = XContentBuilder.builder(XContentType.JSON.xContent())) {
+                builder.startObject();
+                e.toXContent(builder, params);
+                builder.endObject();
+                actual = builder.string();
+            }
+            assertThat(actual, startsWith("{\"type\":\"exception\",\"reason\":\"foo\"," +
+                    "\"caused_by\":{\"type\":\"illegal_state_exception\",\"reason\":\"bar\"," +
+                    "\"stack_trace\":\"java.lang.IllegalStateException: bar" +
+                    (Constants.WINDOWS ? "\\r\\n" : "\\n") +
+                    "\\tat org.elasticsearch."));
+        }
     }
 
-    public void testToXContentWithHeaders() throws IOException {
+    public void testGenerateThrowableToXContent() throws IOException {
+        {
+            Exception ex;
+            if (randomBoolean()) {
+                // just a wrapper which is omitted
+                ex = new RemoteTransportException("foobar", new FileNotFoundException("foo not found"));
+            } else {
+                ex = new FileNotFoundException("foo not found");
+            }
+            assertExceptionAsJson(ex, "{\"type\":\"file_not_found_exception\",\"reason\":\"foo not found\"}");
+        }
+        {
+            ParsingException ex = new ParsingException(1, 2, "foobar", null);
+            assertExceptionAsJson(ex, "{\"type\":\"parsing_exception\",\"reason\":\"foobar\",\"line\":1,\"col\":2}");
+        }
+
+        { // test equivalence
+            ElasticsearchException ex =  new RemoteTransportException("foobar", new FileNotFoundException("foo not found"));
+            String toXContentString = Strings.toString(ex);
+            String throwableString = Strings.toString((builder, params) -> {
+                ElasticsearchException.generateThrowableXContent(builder, params, ex);
+                return builder;
+            });
+
+            assertEquals(throwableString, toXContentString);
+            assertEquals("{\"type\":\"file_not_found_exception\",\"reason\":\"foo not found\"}", toXContentString);
+        }
+
+        { // render header and metadata
+            ParsingException ex = new ParsingException(1, 2, "foobar", null);
+            ex.addMetadata("es.test1", "value1");
+            ex.addMetadata("es.test2", "value2");
+            ex.addHeader("test", "some value");
+            ex.addHeader("test_multi", "some value", "another value");
+            String expected = "{\"type\":\"parsing_exception\",\"reason\":\"foobar\",\"line\":1,\"col\":2," +
+                    "\"test1\":\"value1\",\"test2\":\"value2\"," +
+                    "\"header\":{\"test_multi\":" +
+                    "[\"some value\",\"another value\"],\"test\":\"some value\"}}";
+            assertExceptionAsJson(ex, expected);
+        }
+    }
+
+    public void testToXContentWithHeadersAndMetadata() throws IOException {
         ElasticsearchException e = new ElasticsearchException("foo",
                                         new ElasticsearchException("bar",
                                                 new ElasticsearchException("baz",
                                                         new ClusterBlockException(singleton(DiscoverySettings.NO_MASTER_BLOCK_WRITES)))));
         e.addHeader("foo_0", "0");
         e.addHeader("foo_1", "1");
-        e.addHeader("es.header_foo_0", "foo_0");
-        e.addHeader("es.header_foo_1", "foo_1");
+        e.addMetadata("es.metadata_foo_0", "foo_0");
+        e.addMetadata("es.metadata_foo_1", "foo_1");
 
         final String expectedJson = "{"
             + "\"type\":\"exception\","
             + "\"reason\":\"foo\","
-            + "\"header_foo_0\":\"foo_0\","
-            + "\"header_foo_1\":\"foo_1\","
+            + "\"metadata_foo_0\":\"foo_0\","
+            + "\"metadata_foo_1\":\"foo_1\","
             + "\"caused_by\":{"
                 + "\"type\":\"exception\","
                 + "\"reason\":\"bar\","
@@ -106,7 +371,7 @@ public class ElasticsearchExceptionTests extends ESTestCase {
                 + "}"
         + "}";
 
-        assertExceptionAsJson(e, false, equalTo(expectedJson));
+        assertExceptionAsJson(e, expectedJson);
 
         ElasticsearchException parsed;
         try (XContentParser parser = createParser(XContentType.JSON.xContent(), expectedJson)) {
@@ -118,11 +383,12 @@ public class ElasticsearchExceptionTests extends ESTestCase {
 
         assertNotNull(parsed);
         assertEquals(parsed.getMessage(), "Elasticsearch exception [type=exception, reason=foo]");
-        assertThat(parsed.getHeaderKeys(), hasSize(4));
-        assertEquals(parsed.getHeader("header_foo_0").get(0), "foo_0");
-        assertEquals(parsed.getHeader("header_foo_1").get(0), "foo_1");
+        assertThat(parsed.getHeaderKeys(), hasSize(2));
         assertEquals(parsed.getHeader("foo_0").get(0), "0");
         assertEquals(parsed.getHeader("foo_1").get(0), "1");
+        assertThat(parsed.getMetadataKeys(), hasSize(2));
+        assertEquals(parsed.getMetadata("es.metadata_foo_0").get(0), "foo_0");
+        assertEquals(parsed.getMetadata("es.metadata_foo_1").get(0), "foo_1");
 
         ElasticsearchException cause = (ElasticsearchException) parsed.getCause();
         assertEquals(cause.getMessage(), "Elasticsearch exception [type=exception, reason=bar]");
@@ -185,24 +451,25 @@ public class ElasticsearchExceptionTests extends ESTestCase {
         cause = (ElasticsearchException) cause.getCause();
         assertEquals(cause.getMessage(),
                 "Elasticsearch exception [type=routing_missing_exception, reason=routing is required for [_test]/[_type]/[_id]]");
-        assertThat(cause.getHeaderKeys(), hasSize(2));
-        assertThat(cause.getHeader("index"), hasItem("_test"));
-        assertThat(cause.getHeader("index_uuid"), hasItem("_na_"));
+        assertThat(cause.getHeaderKeys(), hasSize(0));
+        assertThat(cause.getMetadataKeys(), hasSize(2));
+        assertThat(cause.getMetadata("es.index"), hasItem("_test"));
+        assertThat(cause.getMetadata("es.index_uuid"), hasItem("_na_"));
     }
 
-    public void testFromXContentWithHeaders() throws IOException {
+    public void testFromXContentWithHeadersAndMetadata() throws IOException {
         RoutingMissingException routing = new RoutingMissingException("_test", "_type", "_id");
         ElasticsearchException baz = new ElasticsearchException("baz", routing);
         baz.addHeader("baz_0", "baz0");
-        baz.addHeader("es.baz_1", "baz1");
+        baz.addMetadata("es.baz_1", "baz1");
         baz.addHeader("baz_2", "baz2");
-        baz.addHeader("es.baz_3", "baz3");
+        baz.addMetadata("es.baz_3", "baz3");
         ElasticsearchException bar = new ElasticsearchException("bar", baz);
-        bar.addHeader("es.bar_0", "bar0");
+        bar.addMetadata("es.bar_0", "bar0");
         bar.addHeader("bar_1", "bar1");
-        bar.addHeader("es.bar_2", "bar2");
+        bar.addMetadata("es.bar_2", "bar2");
         ElasticsearchException foo = new ElasticsearchException("foo", bar);
-        foo.addHeader("es.foo_0", "foo0");
+        foo.addMetadata("es.foo_0", "foo0");
         foo.addHeader("foo_1", "foo1");
 
         final XContent xContent = randomFrom(XContentType.values()).xContent();
@@ -218,31 +485,35 @@ public class ElasticsearchExceptionTests extends ESTestCase {
 
         assertNotNull(parsed);
         assertEquals(parsed.getMessage(), "Elasticsearch exception [type=exception, reason=foo]");
-        assertThat(parsed.getHeaderKeys(), hasSize(2));
-        assertThat(parsed.getHeader("foo_0"), hasItem("foo0"));
+        assertThat(parsed.getHeaderKeys(), hasSize(1));
         assertThat(parsed.getHeader("foo_1"), hasItem("foo1"));
+        assertThat(parsed.getMetadataKeys(), hasSize(1));
+        assertThat(parsed.getMetadata("es.foo_0"), hasItem("foo0"));
 
         ElasticsearchException cause = (ElasticsearchException) parsed.getCause();
         assertEquals(cause.getMessage(), "Elasticsearch exception [type=exception, reason=bar]");
-        assertThat(cause.getHeaderKeys(), hasSize(3));
-        assertThat(cause.getHeader("bar_0"), hasItem("bar0"));
+        assertThat(cause.getHeaderKeys(), hasSize(1));
         assertThat(cause.getHeader("bar_1"), hasItem("bar1"));
-        assertThat(cause.getHeader("bar_2"), hasItem("bar2"));
+        assertThat(cause.getMetadataKeys(), hasSize(2));
+        assertThat(cause.getMetadata("es.bar_0"), hasItem("bar0"));
+        assertThat(cause.getMetadata("es.bar_2"), hasItem("bar2"));
 
         cause = (ElasticsearchException) cause.getCause();
         assertEquals(cause.getMessage(), "Elasticsearch exception [type=exception, reason=baz]");
-        assertThat(cause.getHeaderKeys(), hasSize(4));
+        assertThat(cause.getHeaderKeys(), hasSize(2));
         assertThat(cause.getHeader("baz_0"), hasItem("baz0"));
-        assertThat(cause.getHeader("baz_1"), hasItem("baz1"));
         assertThat(cause.getHeader("baz_2"), hasItem("baz2"));
-        assertThat(cause.getHeader("baz_3"), hasItem("baz3"));
+        assertThat(cause.getMetadataKeys(), hasSize(2));
+        assertThat(cause.getMetadata("es.baz_1"), hasItem("baz1"));
+        assertThat(cause.getMetadata("es.baz_3"), hasItem("baz3"));
 
         cause = (ElasticsearchException) cause.getCause();
         assertEquals(cause.getMessage(),
                 "Elasticsearch exception [type=routing_missing_exception, reason=routing is required for [_test]/[_type]/[_id]]");
-        assertThat(cause.getHeaderKeys(), hasSize(2));
-        assertThat(cause.getHeader("index"), hasItem("_test"));
-        assertThat(cause.getHeader("index_uuid"), hasItem("_na_"));
+        assertThat(cause.getHeaderKeys(), hasSize(0));
+        assertThat(cause.getMetadataKeys(), hasSize(2));
+        assertThat(cause.getMetadata("es.index"), hasItem("_test"));
+        assertThat(cause.getMetadata("es.index_uuid"), hasItem("_na_"));
     }
 
     /**
@@ -251,17 +522,15 @@ public class ElasticsearchExceptionTests extends ESTestCase {
      * By default, the stack trace of the exception is not rendered. The parameter `errorTrace` forces the stack trace to
      * be rendered like the REST API does when the "error_trace" parameter is set to true.
      */
-    private static void assertExceptionAsJson(ElasticsearchException e, boolean errorTrace, Matcher<String> expected)
-            throws IOException {
-        ToXContent.Params params = ToXContent.EMPTY_PARAMS;
-        if (errorTrace) {
-            params = new ToXContent.MapParams(Collections.singletonMap(ElasticsearchException.REST_EXCEPTION_SKIP_STACK_TRACE, "false"));
-        }
-        try (XContentBuilder builder = XContentBuilder.builder(XContentType.JSON.xContent())) {
-            builder.startObject();
-            e.toXContent(builder, params);
-            builder.endObject();
-            assertThat(builder.bytes().utf8ToString(), expected);
-        }
+    private static void assertToXContentAsJson(ToXContent e, String expectedJson) throws IOException {
+        BytesReference actual = XContentHelper.toXContent(e, XContentType.JSON, randomBoolean());
+        assertToXContentEquivalent(new BytesArray(expectedJson), actual, XContentType.JSON);
+    }
+
+    private static void assertExceptionAsJson(Exception e, String expectedJson) throws IOException {
+        assertToXContentAsJson((builder, params) -> {
+            ElasticsearchException.generateThrowableXContent(builder, params, e);
+            return builder;
+        }, expectedJson);
     }
 }