Upgrading HDFS Repository Plugin to use HDFS 2.8.1 Client (#25497)

Hadoop 2.7.x libraries fail when running on JDK9 due to the version string changing to a single 
character. On Hadoop 2.8, this is no longer a problem, and it is unclear on whether the fix will be 
backported to the 2.7 branch. This commit upgrades our dependency of Hadoop for the HDFS 
Repository to 2.8.1.

22 files changed, 136 insertions, 57 deletions

diff --git a/plugins/repository-hdfs/build.gradle b/plugins/repository-hdfs/build.gradle
index df35a93b57..1dc625563e 100644
--- a/plugins/repository-hdfs/build.gradle
+++ b/plugins/repository-hdfs/build.gradle
@@ -32,7 +32,7 @@ esplugin {
 apply plugin: 'elasticsearch.vagrantsupport'
 
 versions << [
-  'hadoop2': '2.7.1'
+  'hadoop2': '2.8.1'
 ]
 
 configurations {
@@ -45,7 +45,8 @@ dependencies {
   compile "org.apache.hadoop:hadoop-annotations:${versions.hadoop2}"
   compile "org.apache.hadoop:hadoop-auth:${versions.hadoop2}"
   compile "org.apache.hadoop:hadoop-hdfs:${versions.hadoop2}"
-  compile 'org.apache.htrace:htrace-core:3.1.0-incubating'
+  compile "org.apache.hadoop:hadoop-hdfs-client:${versions.hadoop2}"
+  compile 'org.apache.htrace:htrace-core4:4.0.1-incubating'
   compile 'com.google.guava:guava:11.0.2'
   compile 'com.google.protobuf:protobuf-java:2.5.0'
   compile 'commons-logging:commons-logging:1.1.3'
@@ -210,12 +211,15 @@ if (secureFixtureSupported) {
 thirdPartyAudit.excludes = [
   // classes are missing, because we added hadoop jars one by one until tests pass.
   'com.google.gson.stream.JsonReader', 
-  'com.google.gson.stream.JsonWriter', 
-  'com.jcraft.jsch.ChannelExec', 
-  'com.jcraft.jsch.JSch', 
-  'com.jcraft.jsch.Logger', 
-  'com.jcraft.jsch.Session', 
-  'com.sun.jersey.api.ParamException', 
+  'com.google.gson.stream.JsonWriter',
+  'com.jcraft.jsch.ChannelExec',
+  'com.jcraft.jsch.ChannelSftp',
+  'com.jcraft.jsch.ChannelSftp$LsEntry',
+  'com.jcraft.jsch.JSch',
+  'com.jcraft.jsch.Logger',
+  'com.jcraft.jsch.Session',
+  'com.jcraft.jsch.SftpATTRS',
+  'com.sun.jersey.api.ParamException',
   'com.sun.jersey.api.core.HttpContext', 
   'com.sun.jersey.core.spi.component.ComponentContext', 
   'com.sun.jersey.core.spi.component.ComponentScope', 
@@ -239,6 +243,7 @@ thirdPartyAudit.excludes = [
   'io.netty.channel.ChannelHandlerContext',
   'io.netty.channel.ChannelInboundHandlerAdapter',
   'io.netty.channel.ChannelInitializer',
+  'io.netty.channel.ChannelOption',
   'io.netty.channel.ChannelPipeline',
   'io.netty.channel.EventLoopGroup',
   'io.netty.channel.SimpleChannelInboundHandler',
@@ -267,7 +272,8 @@ thirdPartyAudit.excludes = [
   'io.netty.handler.stream.ChunkedStream',
   'io.netty.handler.stream.ChunkedWriteHandler',
   'io.netty.util.concurrent.GlobalEventExecutor',
-  'javax.ws.rs.core.Context', 
+  'io.netty.util.ReferenceCountUtil',
+  'javax.ws.rs.core.Context',
   'javax.ws.rs.core.MediaType', 
   'javax.ws.rs.core.MultivaluedMap', 
   'javax.ws.rs.core.Response$ResponseBuilder', 
@@ -317,8 +323,7 @@ thirdPartyAudit.excludes = [
   'org.apache.commons.digester.substitution.MultiVariableExpander', 
   'org.apache.commons.digester.substitution.VariableSubstitutor', 
   'org.apache.commons.digester.xmlrules.DigesterLoader', 
-  'org.apache.commons.httpclient.util.URIUtil', 
-  'org.apache.commons.jxpath.JXPathContext', 
+  'org.apache.commons.jxpath.JXPathContext',
   'org.apache.commons.jxpath.ri.JXPathContextReferenceImpl', 
   'org.apache.commons.jxpath.ri.QName', 
   'org.apache.commons.jxpath.ri.compiler.NodeNameTest', 
@@ -368,7 +373,8 @@ thirdPartyAudit.excludes = [
   'org.apache.curator.utils.EnsurePath', 
   'org.apache.curator.utils.PathUtils', 
   'org.apache.curator.utils.ThreadUtils', 
-  'org.apache.curator.utils.ZKPaths', 
+  'org.apache.curator.utils.ZKPaths',
+  'org.apache.directory.shared.kerberos.components.EncryptionKey',
   'org.apache.directory.server.kerberos.shared.keytab.Keytab', 
   'org.apache.directory.server.kerberos.shared.keytab.KeytabEntry', 
   'org.apache.http.NameValuePair', 
@@ -402,33 +408,32 @@ thirdPartyAudit.excludes = [
   'org.codehaus.jackson.JsonFactory', 
   'org.codehaus.jackson.JsonGenerator', 
   'org.codehaus.jackson.JsonGenerator$Feature', 
-  'org.codehaus.jackson.JsonNode', 
-  'org.codehaus.jackson.map.MappingJsonFactory', 
+  'org.codehaus.jackson.map.MappingJsonFactory',
   'org.codehaus.jackson.map.ObjectMapper', 
   'org.codehaus.jackson.map.ObjectReader', 
   'org.codehaus.jackson.map.ObjectWriter', 
   'org.codehaus.jackson.node.ContainerNode', 
-  'org.codehaus.jackson.type.TypeReference', 
-  'org.codehaus.jackson.util.MinimalPrettyPrinter', 
+  'org.codehaus.jackson.util.MinimalPrettyPrinter',
   'org.fusesource.leveldbjni.JniDBFactory', 
   'org.iq80.leveldb.DB', 
   'org.iq80.leveldb.Options', 
-  'org.iq80.leveldb.WriteBatch', 
-  'org.mortbay.jetty.Connector', 
-  'org.mortbay.jetty.Handler', 
-  'org.mortbay.jetty.InclusiveByteRange', 
-  'org.mortbay.jetty.MimeTypes', 
-  'org.mortbay.jetty.NCSARequestLog', 
-  'org.mortbay.jetty.RequestLog', 
-  'org.mortbay.jetty.Server', 
-  'org.mortbay.jetty.handler.ContextHandler$SContext', 
-  'org.mortbay.jetty.handler.ContextHandler', 
-  'org.mortbay.jetty.handler.ContextHandlerCollection', 
-  'org.mortbay.jetty.handler.HandlerCollection', 
-  'org.mortbay.jetty.handler.RequestLogHandler', 
-  'org.mortbay.jetty.nio.SelectChannelConnector', 
-  'org.mortbay.jetty.security.SslSocketConnector', 
-  'org.mortbay.jetty.servlet.AbstractSessionManager', 
+  'org.iq80.leveldb.WriteBatch',
+  'org.mortbay.jetty.Connector',
+  'org.mortbay.jetty.Handler',
+  'org.mortbay.jetty.InclusiveByteRange',
+  'org.mortbay.jetty.MimeTypes',
+  'org.mortbay.jetty.NCSARequestLog',
+  'org.mortbay.jetty.RequestLog',
+  'org.mortbay.jetty.Server',
+  'org.mortbay.jetty.handler.ContextHandler$SContext',
+  'org.mortbay.jetty.handler.ContextHandler',
+  'org.mortbay.jetty.handler.ContextHandlerCollection',
+  'org.mortbay.jetty.handler.HandlerCollection',
+  'org.mortbay.jetty.handler.RequestLogHandler',
+  'org.mortbay.jetty.nio.SelectChannelConnector',
+  'org.mortbay.jetty.security.SslSelectChannelConnector',
+  'org.mortbay.jetty.security.SslSocketConnector',
+  'org.mortbay.jetty.servlet.AbstractSessionManager',
   'org.mortbay.jetty.servlet.Context', 
   'org.mortbay.jetty.servlet.DefaultServlet', 
   'org.mortbay.jetty.servlet.FilterHolder', 
@@ -437,8 +442,7 @@ thirdPartyAudit.excludes = [
   'org.mortbay.jetty.servlet.ServletHolder', 
   'org.mortbay.jetty.servlet.SessionHandler', 
   'org.mortbay.jetty.webapp.WebAppContext', 
-  'org.mortbay.log.Log', 
-  'org.mortbay.thread.QueuedThreadPool', 
+  'org.mortbay.thread.QueuedThreadPool',
   'org.mortbay.util.MultiException', 
   'org.mortbay.util.ajax.JSON$Convertible', 
   'org.mortbay.util.ajax.JSON$Output', 
@@ -473,9 +477,26 @@ thirdPartyAudit.excludes = [
   'org.apache.log4j.AppenderSkeleton',
   'org.apache.log4j.AsyncAppender',
   'org.apache.log4j.helpers.ISO8601DateFormat',
-  'org.apache.log4j.spi.ThrowableInformation'
+  'org.apache.log4j.spi.ThrowableInformation',
+
+  // New optional dependencies in 2.8
+  'com.nimbusds.jose.JWSObject$State',
+  'com.nimbusds.jose.crypto.RSASSAVerifier',
+  'com.nimbusds.jwt.ReadOnlyJWTClaimsSet',
+  'com.nimbusds.jwt.SignedJWT',
+  'com.squareup.okhttp.Call',
+  'com.squareup.okhttp.MediaType',
+  'com.squareup.okhttp.OkHttpClient',
+  'com.squareup.okhttp.Request$Builder',
+  'com.squareup.okhttp.RequestBody',
+  'com.squareup.okhttp.Response',
+  'com.squareup.okhttp.ResponseBody'
 ]
 
+if (project.rootProject.ext.javaVersion == JavaVersion.VERSION_1_9) {
+  thirdPartyAudit.excludes.add('javax.xml.bind.annotation.adapters.HexBinaryAdapter')
+}
+
 // Gradle 2.13 bundles org.slf4j.impl.StaticLoggerBinder in its core.jar which leaks into the forbidden APIs ant task
 // Gradle 2.14+ does not bundle this class anymore so we need to properly exclude it here.
 if (GradleVersion.current() > GradleVersion.version("2.13")) {
diff --git a/plugins/repository-hdfs/licenses/hadoop-annotations-2.7.1.jar.sha1 b/plugins/repository-hdfs/licenses/hadoop-annotations-2.7.1.jar.sha1
deleted file mode 100644
index 660467a4c6..0000000000
--- a/plugins/repository-hdfs/licenses/hadoop-annotations-2.7.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-2a77fe74ee056bf45598cf7e20cd624e8388e627
\ No newline at end of file
diff --git a/plugins/repository-hdfs/licenses/hadoop-annotations-2.8.1.jar.sha1 b/plugins/repository-hdfs/licenses/hadoop-annotations-2.8.1.jar.sha1
new file mode 100644
index 0000000000..bb000d4ddb
--- /dev/null
+++ b/plugins/repository-hdfs/licenses/hadoop-annotations-2.8.1.jar.sha1
@@ -0,0 +1 @@
+335a867cf42bf789919bfc3229ff26747124e8f1
\ No newline at end of file
diff --git a/plugins/repository-hdfs/licenses/hadoop-auth-2.7.1.jar.sha1 b/plugins/repository-hdfs/licenses/hadoop-auth-2.7.1.jar.sha1
deleted file mode 100644
index 0161301ead..0000000000
--- a/plugins/repository-hdfs/licenses/hadoop-auth-2.7.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-2515f339f97f1d7ba850485e06e395a58586bc2e
\ No newline at end of file
diff --git a/plugins/repository-hdfs/licenses/hadoop-auth-2.8.1.jar.sha1 b/plugins/repository-hdfs/licenses/hadoop-auth-2.8.1.jar.sha1
new file mode 100644
index 0000000000..67a1c4f3fa
--- /dev/null
+++ b/plugins/repository-hdfs/licenses/hadoop-auth-2.8.1.jar.sha1
@@ -0,0 +1 @@
+688ccccc0e0739d8737a93b0039a4a661e52084b
\ No newline at end of file
diff --git a/plugins/repository-hdfs/licenses/hadoop-client-2.7.1.jar.sha1 b/plugins/repository-hdfs/licenses/hadoop-client-2.7.1.jar.sha1
deleted file mode 100644
index 4c6dca8af4..0000000000
--- a/plugins/repository-hdfs/licenses/hadoop-client-2.7.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-dbc2faacd210e6a1e3eb7def6e42065c7457d960
\ No newline at end of file
diff --git a/plugins/repository-hdfs/licenses/hadoop-client-2.8.1.jar.sha1 b/plugins/repository-hdfs/licenses/hadoop-client-2.8.1.jar.sha1
new file mode 100644
index 0000000000..e7be8d468d
--- /dev/null
+++ b/plugins/repository-hdfs/licenses/hadoop-client-2.8.1.jar.sha1
@@ -0,0 +1 @@
+4812f251f8100fd4722c3cec5d7353f71f69cda9
\ No newline at end of file
diff --git a/plugins/repository-hdfs/licenses/hadoop-common-2.7.1.jar.sha1 b/plugins/repository-hdfs/licenses/hadoop-common-2.7.1.jar.sha1
deleted file mode 100644
index 64ff368db6..0000000000
--- a/plugins/repository-hdfs/licenses/hadoop-common-2.7.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-50580f5ebab60b1b318ad157f668d8e40a1cc0da
\ No newline at end of file
diff --git a/plugins/repository-hdfs/licenses/hadoop-common-2.8.1.jar.sha1 b/plugins/repository-hdfs/licenses/hadoop-common-2.8.1.jar.sha1
new file mode 100644
index 0000000000..6e6c625b7d
--- /dev/null
+++ b/plugins/repository-hdfs/licenses/hadoop-common-2.8.1.jar.sha1
@@ -0,0 +1 @@
+a4df18b79e4d0349ce4b58a52d314e7ae1d6be99
\ No newline at end of file
diff --git a/plugins/repository-hdfs/licenses/hadoop-hdfs-2.7.1.jar.sha1 b/plugins/repository-hdfs/licenses/hadoop-hdfs-2.7.1.jar.sha1
deleted file mode 100644
index 2d4954befa..0000000000
--- a/plugins/repository-hdfs/licenses/hadoop-hdfs-2.7.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-11681de93a4cd76c841e352b7094f839b072a21f
\ No newline at end of file
diff --git a/plugins/repository-hdfs/licenses/hadoop-hdfs-2.8.1.jar.sha1 b/plugins/repository-hdfs/licenses/hadoop-hdfs-2.8.1.jar.sha1
new file mode 100644
index 0000000000..e419a3124f
--- /dev/null
+++ b/plugins/repository-hdfs/licenses/hadoop-hdfs-2.8.1.jar.sha1
@@ -0,0 +1 @@
+a378f4bc8e6cd779d779c9f512e0e31edd771633
\ No newline at end of file
diff --git a/plugins/repository-hdfs/licenses/hadoop-hdfs-client-2.8.1.jar.sha1 b/plugins/repository-hdfs/licenses/hadoop-hdfs-client-2.8.1.jar.sha1
new file mode 100644
index 0000000000..20c7963cd3
--- /dev/null
+++ b/plugins/repository-hdfs/licenses/hadoop-hdfs-client-2.8.1.jar.sha1
@@ -0,0 +1 @@
+6b0100e4f58ecf7ce75817fce1ffdfbec947337a
\ No newline at end of file
diff --git a/plugins/repository-hdfs/licenses/htrace-core-3.1.0-incubating.jar.sha1 b/plugins/repository-hdfs/licenses/htrace-core-3.1.0-incubating.jar.sha1
deleted file mode 100644
index c742d8397c..0000000000
--- a/plugins/repository-hdfs/licenses/htrace-core-3.1.0-incubating.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-f73606e7c9ede5802335c290bf47490ad6d51df3
\ No newline at end of file
diff --git a/plugins/repository-hdfs/licenses/htrace-core4-4.0.1-incubating.jar.sha1 b/plugins/repository-hdfs/licenses/htrace-core4-4.0.1-incubating.jar.sha1
new file mode 100644
index 0000000000..a0a562b2a4
--- /dev/null
+++ b/plugins/repository-hdfs/licenses/htrace-core4-4.0.1-incubating.jar.sha1
@@ -0,0 +1 @@
+f4ef727cb4675788ac66f48e217020acc1690960
\ No newline at end of file
diff --git a/plugins/repository-hdfs/licenses/htrace-core-LICENSE.txt b/plugins/repository-hdfs/licenses/htrace-core4-LICENSE.txt
index 0befae88d4..5e4a6e6fa7 100644
--- a/plugins/repository-hdfs/licenses/htrace-core-LICENSE.txt
+++ b/plugins/repository-hdfs/licenses/htrace-core4-LICENSE.txt
@@ -203,8 +203,8 @@ of dependencies that are NOT Apache Licensed.
    See the License for the specific language governing permissions and
    limitations under the License.
 
-The HTrace Owl logo is from http://www.clker.com/clipart-13653.html.  It is
-public domain.
+Units, a string formatting go library, is Copyright (c) 2014 Alec Thomas
+and MIT licensed: https://github.com/alecthomas/units/blob/master/COPYING
 
 D3, a javascript library for manipulating data, used by htrace-hbase
 is Copyright 2010-2014, Michael Bostock and BSD licensed:
@@ -239,4 +239,7 @@ https://github.com/moment/moment/blob/develop/LICENSE
 CMP is an implementation of the MessagePack serialization format in
 C.  It is licensed under the MIT license:
 https://github.com/camgunz/cmp/blob/master/LICENSE
-See ./htrace-c/src/util/cmp.c and ./htrace-c/src/util/cmp.h.
+
+go-codec is an implementation of several serialization and deserialization
+codecs in Go.  It is licensed under the MIT license:
+https://github.com/ugorji/go/blob/master/LICENSE
\ No newline at end of file
diff --git a/plugins/repository-hdfs/licenses/htrace-core-NOTICE.txt b/plugins/repository-hdfs/licenses/htrace-core4-NOTICE.txt
index 845b6965e0..c96631aab6 100644
--- a/plugins/repository-hdfs/licenses/htrace-core-NOTICE.txt
+++ b/plugins/repository-hdfs/licenses/htrace-core4-NOTICE.txt
@@ -11,3 +11,6 @@ that are NOT Apache licensed (with pointers to their licensing)
 Apache HTrace includes an Apache Thrift connector to Zipkin. Zipkin
 is a distributed tracing system that is Apache 2.0 Licensed.
 Copyright 2012 Twitter, Inc.
+
+Our Owl logo we took from http://www.clker.com/clipart-13653.html.
+It is public domain/free.
\ No newline at end of file
diff --git a/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsPlugin.java b/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsPlugin.java
index 4e51ab23b8..c0b3d805bc 100644
--- a/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsPlugin.java
+++ b/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsPlugin.java
@@ -62,8 +62,6 @@ public final class HdfsPlugin extends Plugin implements RepositoryPlugin {
             Class.forName("org.apache.hadoop.util.StringUtils");
             Class.forName("org.apache.hadoop.util.ShutdownHookManager");
             Class.forName("org.apache.hadoop.conf.Configuration");
-            Class.forName("org.apache.hadoop.hdfs.protocol.HdfsConstants");
-            Class.forName("org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck");
         } catch (ClassNotFoundException | IOException e) {
             throw new RuntimeException(e);
         } finally {
diff --git a/plugins/repository-hdfs/src/test/java/org/elasticsearch/repositories/hdfs/HdfsBlobStoreContainerTests.java b/plugins/repository-hdfs/src/test/java/org/elasticsearch/repositories/hdfs/HdfsBlobStoreContainerTests.java
index 2bfe6843da..195dea9810 100644
--- a/plugins/repository-hdfs/src/test/java/org/elasticsearch/repositories/hdfs/HdfsBlobStoreContainerTests.java
+++ b/plugins/repository-hdfs/src/test/java/org/elasticsearch/repositories/hdfs/HdfsBlobStoreContainerTests.java
@@ -19,15 +19,6 @@
 
 package org.elasticsearch.repositories.hdfs;
 
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.AbstractFileSystem;
-import org.apache.hadoop.fs.FileContext;
-import org.apache.hadoop.fs.UnsupportedFileSystemException;
-import org.elasticsearch.common.SuppressForbidden;
-import org.elasticsearch.common.blobstore.BlobStore;
-import org.elasticsearch.repositories.ESBlobStoreContainerTestCase;
-
-import javax.security.auth.Subject;
 import java.io.IOException;
 import java.lang.reflect.Constructor;
 import java.lang.reflect.InvocationTargetException;
@@ -38,7 +29,18 @@ import java.security.PrivilegedAction;
 import java.security.PrivilegedActionException;
 import java.security.PrivilegedExceptionAction;
 import java.util.Collections;
+import javax.security.auth.Subject;
+
+import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.AbstractFileSystem;
+import org.apache.hadoop.fs.FileContext;
+import org.apache.hadoop.fs.UnsupportedFileSystemException;
+import org.elasticsearch.common.SuppressForbidden;
+import org.elasticsearch.common.blobstore.BlobStore;
+import org.elasticsearch.repositories.ESBlobStoreContainerTestCase;
 
+@ThreadLeakFilters(filters = {HdfsClientThreadLeakFilter.class})
 public class HdfsBlobStoreContainerTests extends ESBlobStoreContainerTestCase {
 
     @Override
diff --git a/plugins/repository-hdfs/src/test/java/org/elasticsearch/repositories/hdfs/HdfsClientThreadLeakFilter.java b/plugins/repository-hdfs/src/test/java/org/elasticsearch/repositories/hdfs/HdfsClientThreadLeakFilter.java
new file mode 100644
index 0000000000..5db9635aa8
--- /dev/null
+++ b/plugins/repository-hdfs/src/test/java/org/elasticsearch/repositories/hdfs/HdfsClientThreadLeakFilter.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.repositories.hdfs;
+
+import com.carrotsearch.randomizedtesting.ThreadFilter;
+
+/**
+ * In Hadoop 2.8.0, there is a thread that is started by the filesystem to clean up old execution stats.
+ * This thread ignores all interrupts, catching InterruptedException, logging it, and continuing on
+ * with its work. The thread is a daemon, so it thankfully does not stop the JVM from closing, and it
+ * is started only once in a class's static initialization. This currently breaks our testing as this
+ * thread leaks out of the client and is picked up by the test framework. This thread filter is meant
+ * to ignore the offending thread until a version of Hadoop is released that addresses the incorrect
+ * interrupt handling.
+ *
+ * @see <a href="https://issues.apache.org/jira/browse/HADOOP-12829">https://issues.apache.org/jira/browse/HADOOP-12829</a>
+ * @see "org.apache.hadoop.fs.FileSystem.Statistics.StatisticsDataReferenceCleaner"
+ * @see "org.apache.hadoop.fs.FileSystem.Statistics"
+ */
+public final class HdfsClientThreadLeakFilter implements ThreadFilter {
+
+    private static final String OFFENDING_THREAD_NAME =
+        "org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner";
+
+    @Override
+    public boolean reject(Thread t) {
+        return t.getName().equals(OFFENDING_THREAD_NAME);
+    }
+}
diff --git a/plugins/repository-hdfs/src/test/java/org/elasticsearch/repositories/hdfs/HdfsTests.java b/plugins/repository-hdfs/src/test/java/org/elasticsearch/repositories/hdfs/HdfsTests.java
index 511aafdd29..9d0520205e 100644
--- a/plugins/repository-hdfs/src/test/java/org/elasticsearch/repositories/hdfs/HdfsTests.java
+++ b/plugins/repository-hdfs/src/test/java/org/elasticsearch/repositories/hdfs/HdfsTests.java
@@ -23,6 +23,7 @@ import static org.hamcrest.Matchers.greaterThan;
 
 import java.util.Collection;
 
+import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
 import org.elasticsearch.action.admin.cluster.repositories.put.PutRepositoryResponse;
 import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse;
 import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse;
@@ -34,6 +35,7 @@ import org.elasticsearch.repositories.RepositoryException;
 import org.elasticsearch.snapshots.SnapshotState;
 import org.elasticsearch.test.ESSingleNodeTestCase;
 
+@ThreadLeakFilters(filters = {HdfsClientThreadLeakFilter.class})
 public class HdfsTests extends ESSingleNodeTestCase {
 
     @Override
diff --git a/test/fixtures/hdfs-fixture/build.gradle b/test/fixtures/hdfs-fixture/build.gradle
index 3d63939f66..70cb2b6bed 100644
--- a/test/fixtures/hdfs-fixture/build.gradle
+++ b/test/fixtures/hdfs-fixture/build.gradle
@@ -20,7 +20,7 @@
 apply plugin: 'elasticsearch.build'
 
 versions << [
-  'hadoop2': '2.7.1'
+  'hadoop2': '2.8.1'
 ]
 
 // we create MiniHdfsCluster with the hadoop artifact
diff --git a/test/fixtures/hdfs-fixture/src/main/java/hdfs/MiniHDFS.java b/test/fixtures/hdfs-fixture/src/main/java/hdfs/MiniHDFS.java
index f868f4da97..7d41d94e99 100644
--- a/test/fixtures/hdfs-fixture/src/main/java/hdfs/MiniHDFS.java
+++ b/test/fixtures/hdfs-fixture/src/main/java/hdfs/MiniHDFS.java
@@ -26,6 +26,7 @@ import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.nio.file.StandardCopyOption;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.List;
 
 import org.apache.hadoop.conf.Configuration;
@@ -49,7 +50,8 @@ public class MiniHDFS {
 
     public static void main(String[] args) throws Exception {
         if (args.length != 1 && args.length != 3) {
-            throw new IllegalArgumentException("MiniHDFS <baseDirectory> [<kerberosPrincipal> <kerberosKeytab>]");
+            throw new IllegalArgumentException("Expected: MiniHDFS <baseDirectory> [<kerberosPrincipal> <kerberosKeytab>], " +
+                "got: " + Arrays.toString(args));
         }
         boolean secure = args.length == 3;
 
@@ -83,6 +85,7 @@ public class MiniHDFS {
             cfg.set(DFSConfigKeys.DFS_NAMENODE_KEYTAB_FILE_KEY, keytabFile);
             cfg.set(DFSConfigKeys.DFS_DATANODE_KEYTAB_FILE_KEY, keytabFile);
             cfg.set(DFSConfigKeys.DFS_NAMENODE_ACLS_ENABLED_KEY, "true");
+            cfg.set(DFSConfigKeys.DFS_BLOCK_ACCESS_TOKEN_ENABLE_KEY, "true");
             cfg.set(DFSConfigKeys.IGNORE_SECURE_PORTS_FOR_TESTING_KEY, "true");
         }