Sense for VirtualBox and $HOME when deciding to turn on vagrant testing. (#24636)

We're using Vagrant in more places now than before. This commit includes a plugin that verifies
the Vagrant and Virtualbox installations for projects that depend on them. This shared code
should fix up the errors we've seen from CI builds relating to the new Kerberos fixture.

5 files changed, 153 insertions, 82 deletions

diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantSupportPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantSupportPlugin.groovy
new file mode 100644
index 0000000000..d3b7e3aa88
--- /dev/null
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantSupportPlugin.groovy
@@ -0,0 +1,127 @@
+package org.elasticsearch.gradle.vagrant
+
+import org.gradle.api.GradleException
+import org.gradle.api.InvalidUserDataException
+import org.gradle.api.Plugin
+import org.gradle.api.Project
+import org.gradle.process.ExecResult
+import org.gradle.process.internal.ExecException
+
+/**
+ * Global configuration for if Vagrant tasks are supported in this
+ * build environment.
+ */
+class VagrantSupportPlugin implements Plugin<Project> {
+
+    @Override
+    void apply(Project project) {
+        if (project.rootProject.ext.has('vagrantEnvChecksDone') == false) {
+            Map vagrantInstallation = getVagrantInstallation(project)
+            Map virtualBoxInstallation = getVirtualBoxInstallation(project)
+
+            project.rootProject.ext.vagrantInstallation = vagrantInstallation
+            project.rootProject.ext.virtualBoxInstallation = virtualBoxInstallation
+            project.rootProject.ext.vagrantSupported = vagrantInstallation.supported && virtualBoxInstallation.supported
+            project.rootProject.ext.vagrantEnvChecksDone = true
+
+            // Finding that HOME needs to be set when performing vagrant updates
+            String homeLocation = System.getenv("HOME")
+            if (project.rootProject.ext.vagrantSupported && homeLocation == null) {
+                throw new GradleException("Could not locate \$HOME environment variable. Vagrant is enabled " +
+                        "and requires \$HOME to be set to function properly.")
+            }
+        }
+
+        addVerifyInstallationTasks(project)
+    }
+
+    private Map getVagrantInstallation(Project project) {
+        try {
+            ByteArrayOutputStream pipe = new ByteArrayOutputStream()
+            ExecResult runResult = project.exec {
+                commandLine 'vagrant', '--version'
+                standardOutput pipe
+                ignoreExitValue true
+            }
+            String version = pipe.toString().trim()
+            if (runResult.exitValue == 0) {
+                if (version ==~ /Vagrant 1\.(8\.[6-9]|9\.[0-9])+/) {
+                    return [ 'supported' : true ]
+                } else {
+                    return [ 'supported' : false,
+                             'info' : "Illegal version of vagrant [${version}]. Need [Vagrant 1.8.6+]" ]
+                }
+            } else {
+                return [ 'supported' : false,
+                         'info' : "Could not read installed vagrant version:\n" + version ]
+            }
+        } catch (ExecException e) {
+            // Exec still throws this if it cannot find the command, regardless if ignoreExitValue is set.
+            // Swallow error. Vagrant isn't installed. Don't halt the build here.
+            return [ 'supported' : false, 'info' : "Could not find vagrant: " + e.message ]
+        }
+    }
+
+    private Map getVirtualBoxInstallation(Project project) {
+        try {
+            ByteArrayOutputStream pipe = new ByteArrayOutputStream()
+            ExecResult runResult = project.exec {
+                commandLine 'vboxmanage', '--version'
+                standardOutput = pipe
+                ignoreExitValue true
+            }
+            String version = pipe.toString().trim()
+            if (runResult.exitValue == 0) {
+                try {
+                    String[] versions = version.split('\\.')
+                    int major = Integer.parseInt(versions[0])
+                    int minor = Integer.parseInt(versions[1])
+                    if ((major < 5) || (major == 5 && minor < 1)) {
+                        return [ 'supported' : false,
+                                 'info' : "Illegal version of virtualbox [${version}]. Need [5.1+]" ]
+                    } else {
+                        return [ 'supported' : true ]
+                    }
+                } catch (NumberFormatException | ArrayIndexOutOfBoundsException e) {
+                    return [ 'supported' : false,
+                             'info' : "Unable to parse version of virtualbox [${version}]. Required [5.1+]" ]
+                }
+            } else {
+                return [ 'supported': false, 'info': "Could not read installed virtualbox version:\n" + version ]
+            }
+        } catch (ExecException e) {
+            // Exec still throws this if it cannot find the command, regardless if ignoreExitValue is set.
+            // Swallow error. VirtualBox isn't installed. Don't halt the build here.
+            return [ 'supported' : false, 'info' : "Could not find virtualbox: " + e.message ]
+        }
+    }
+
+    private void addVerifyInstallationTasks(Project project) {
+        createCheckVagrantVersionTask(project)
+        createCheckVirtualBoxVersionTask(project)
+    }
+
+    private void createCheckVagrantVersionTask(Project project) {
+        project.tasks.create('vagrantCheckVersion') {
+            description 'Check the Vagrant version'
+            group 'Verification'
+            doLast {
+                if (project.rootProject.vagrantInstallation.supported == false) {
+                    throw new InvalidUserDataException(project.rootProject.vagrantInstallation.info)
+                }
+            }
+        }
+    }
+
+    private void createCheckVirtualBoxVersionTask(Project project) {
+        project.tasks.create('virtualboxCheckVersion') {
+            description 'Check the Virtualbox version'
+            group 'Verification'
+            doLast {
+                if (project.rootProject.virtualBoxInstallation.supported == false) {
+                    throw new InvalidUserDataException(project.rootProject.virtualBoxInstallation.info)
+                }
+            }
+        }
+    }
+}
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy
index 45049acb23..1c90f4607b 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy
@@ -227,43 +227,6 @@ class VagrantTestPlugin implements Plugin<Project> {
         vagrantSetUpTask.dependsOn copyBatsTests, copyBatsUtils, copyBatsArchives, createVersionFile, createUpgradeFromFile
     }
 
-    private static void createCheckVagrantVersionTask(Project project) {
-        project.tasks.create('vagrantCheckVersion', Exec) {
-            description 'Check the Vagrant version'
-            group 'Verification'
-            commandLine 'vagrant', '--version'
-            standardOutput = new ByteArrayOutputStream()
-            doLast {
-                String version = standardOutput.toString().trim()
-                if ((version ==~ /Vagrant 1\.(8\.[6-9]|9\.[0-9])+/) == false) {
-                    throw new InvalidUserDataException("Illegal version of vagrant [${version}]. Need [Vagrant 1.8.6+]")
-                }
-            }
-        }
-    }
-
-    private static void createCheckVirtualBoxVersionTask(Project project) {
-        project.tasks.create('virtualboxCheckVersion', Exec) {
-            description 'Check the Virtualbox version'
-            group 'Verification'
-            commandLine 'vboxmanage', '--version'
-            standardOutput = new ByteArrayOutputStream()
-            doLast {
-                String version = standardOutput.toString().trim()
-                try {
-                    String[] versions = version.split('\\.')
-                    int major = Integer.parseInt(versions[0])
-                    int minor = Integer.parseInt(versions[1])
-                    if ((major < 5) || (major == 5 && minor < 1)) {
-                        throw new InvalidUserDataException("Illegal version of virtualbox [${version}]. Need [5.1+]")
-                    }
-                } catch (NumberFormatException | ArrayIndexOutOfBoundsException e) {
-                    throw new InvalidUserDataException("Unable to parse version of virtualbox [${version}]. Required [5.1+]", e)
-                }
-            }
-        }
-    }
-
     private static void createPackagingTestTask(Project project) {
         project.tasks.create('packagingTest') {
             group 'Verification'
@@ -291,8 +254,6 @@ class VagrantTestPlugin implements Plugin<Project> {
         createCleanTask(project)
         createStopTask(project)
         createSmokeTestTask(project)
-        createCheckVagrantVersionTask(project)
-        createCheckVirtualBoxVersionTask(project)
         createPrepareVagrantTestEnvTask(project)
         createPackagingTestTask(project)
         createPlatformTestTask(project)
diff --git a/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.vagrantsupport.properties b/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.vagrantsupport.properties
new file mode 100644
index 0000000000..73a3f41234
--- /dev/null
+++ b/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.vagrantsupport.properties
@@ -0,0 +1 @@
+implementation-class=org.elasticsearch.gradle.vagrant.VagrantSupportPlugin
\ No newline at end of file
diff --git a/plugins/repository-hdfs/build.gradle b/plugins/repository-hdfs/build.gradle
index cf4a400a75..dbacd73e9d 100644
--- a/plugins/repository-hdfs/build.gradle
+++ b/plugins/repository-hdfs/build.gradle
@@ -29,6 +29,8 @@ esplugin {
   classname 'org.elasticsearch.repositories.hdfs.HdfsPlugin'
 }
 
+apply plugin: 'elasticsearch.vagrantsupport'
+
 versions << [
   'hadoop2': '2.7.1'
 ]
@@ -83,6 +85,7 @@ task krb5kdcUpdate(type: org.elasticsearch.gradle.vagrant.VagrantCommandTask) {
   subcommand 'update'
   boxName box
   environmentVars vagrantEnvVars
+  dependsOn "vagrantCheckVersion", "virtualboxCheckVersion"
 }
 
 task krb5kdcFixture(type: org.elasticsearch.gradle.test.VagrantFixture) {
@@ -125,7 +128,7 @@ task secureHdfsFixture(type: org.elasticsearch.gradle.test.AntFixture) {
           "${keytabPath}"
 }
 
-boolean fixtureSupported = false;
+boolean fixtureSupported = false
 if (Os.isFamily(Os.FAMILY_WINDOWS)) {
   // hdfs fixture will not start without hadoop native libraries on windows
   String nativePath = System.getenv("HADOOP_HOME")
@@ -153,60 +156,38 @@ if (fixtureSupported) {
   integTestRunner.systemProperty 'tests.rest.suite', 'hdfs_repository/10_basic'
 }
 
-boolean secureFixtureSupported = false;
+boolean secureFixtureSupported = false
 if (fixtureSupported) {
-  // Only do secure fixture support if the regular fixture is supported,
-  // and if vagrant is installed. The ignoreExitValue on exec only matters
-  // in cases where the command can be found and successfully started. In
-  // situations where the vagrant command isn't able to be started at all
-  // (it's not installed) then Gradle still throws ExecException.
-  ByteArrayOutputStream pipe = new ByteArrayOutputStream()
-  try {
-    ExecResult runResult = exec {
-      commandLine 'vagrant', '--version'
-      standardOutput pipe
-      ignoreExitValue true
-    }
-    String output = pipe.toString().trim()
-    if (runResult.exitValue == 0) {
-      secureFixtureSupported = (output ==~ /Vagrant 1\.(8\.[6-9]|9\.[0-9])+/)
-    } else {
-      logger.warn("Could not read installed vagrant version:\n" + output)
-    }
-  } catch (org.gradle.process.internal.ExecException e) {
-    logger.warn("Could not find vagrant: " + e.message)
-    // Swallow error. Vagrant isn't installed. Leave secure fixture support off.
-  }
+  secureFixtureSupported = project.rootProject.vagrantSupported
 }
 
 // Create a Integration Test suite just for security based tests
-if (secureFixtureSupported && false) { // This fails due to a vagrant configuration issue - remove the false check to re-enable
-  // This must execute before the afterEvaluate block from integTestSecure
-  project.afterEvaluate {
-    Path elasticsearchKT = project(':test:fixtures:krb5kdc-fixture').buildDir.toPath().resolve("keytabs").resolve("elasticsearch.keytab").toAbsolutePath()
-    Path krb5conf = project(':test:fixtures:krb5kdc-fixture').buildDir.toPath().resolve("conf").resolve("krb5.conf").toAbsolutePath()
-
-    project.integTestSecureCluster.dependsOn(project.bundlePlugin)
-    project.integTestSecure.clusterConfig.plugin(project.path)
-    project.integTestSecure.clusterConfig.extraConfigFile("repository-hdfs/krb5.keytab", "${elasticsearchKT}")
-    project.integTestSecure.clusterConfig.jvmArgs = "-Xms" + System.getProperty('tests.heap.size', '512m') +
-            " " + "-Xmx" + System.getProperty('tests.heap.size', '512m') +
-            " " + "-Djava.security.krb5.conf=${krb5conf}" +
-            " " + System.getProperty('tests.jvm.argline', '')
-  }
+// This must execute before the afterEvaluate block from integTestSecure
+project.afterEvaluate {
+  Path elasticsearchKT = project(':test:fixtures:krb5kdc-fixture').buildDir.toPath().resolve("keytabs").resolve("elasticsearch.keytab").toAbsolutePath()
+  Path krb5conf = project(':test:fixtures:krb5kdc-fixture').buildDir.toPath().resolve("conf").resolve("krb5.conf").toAbsolutePath()
+
+  project.integTestSecureCluster.dependsOn(project.bundlePlugin)
+  project.integTestSecure.clusterConfig.plugin(project.path)
+  project.integTestSecure.clusterConfig.extraConfigFile("repository-hdfs/krb5.keytab", "${elasticsearchKT}")
+  project.integTestSecure.clusterConfig.jvmArgs = "-Xms" + System.getProperty('tests.heap.size', '512m') +
+          " " + "-Xmx" + System.getProperty('tests.heap.size', '512m') +
+          " " + "-Djava.security.krb5.conf=${krb5conf}" +
+          " " + System.getProperty('tests.jvm.argline', '')
+}
 
-  RestIntegTestTask integTestSecure = project.tasks.create('integTestSecure', RestIntegTestTask.class) {
-    description = "Runs rest tests against an elasticsearch cluster with HDFS secured by MIT Kerberos."
-  }
+RestIntegTestTask integTestSecure = project.tasks.create('integTestSecure', RestIntegTestTask.class) {
+  description = "Runs rest tests against an elasticsearch cluster with HDFS secured by MIT Kerberos."
+}
 
-  integTestSecure.mustRunAfter(project.integTest)
+if (secureFixtureSupported) {
   project.check.dependsOn(integTestSecure)
 
   // Fixture dependencies
   integTestSecureCluster.dependsOn secureHdfsFixture, krb5kdcFixture
   integTestSecureRunner.systemProperty 'tests.rest.suite', 'secure_hdfs_repository'
 } else {
-  logger.warn("secured hdfsFixture is unsupported, please install Vagrant 1.8.6+ to enable")
+  integTestSecure.enabled = false
 }
 
 thirdPartyAudit.excludes = [
diff --git a/qa/vagrant/build.gradle b/qa/vagrant/build.gradle
index 52b6b2e511..40fc399818 100644
--- a/qa/vagrant/build.gradle
+++ b/qa/vagrant/build.gradle
@@ -17,6 +17,7 @@
  * under the License.
  */
 
+apply plugin: 'elasticsearch.vagrantsupport'
 apply plugin: 'elasticsearch.vagrant'
 
 List<String> plugins = []