Build: Pin the random seed at startup (#24990)

Pins the random testing seed at build start rather than letting
it vary with every randomized testing invocation. This is useful
for projects where random decisions in one randomized testing run
can effect the outcome of a second randomized testing run such as
the full cluster restart tests.

The goal isn't for tests to be able to assume that random decision
will be the same in both tests. It is more to make sure that the
seed printed when a test fails reproduces the appropriate random
decisions. And pinning the seed at startup should do just that.

This works by taking the key passed as a system property if one
is passed, otherwise picking a random long and getting it into
appropriate key format. The build just calls
`new Random().nextLong()` to get the seed while randomized testing
uses a Murmur3 hash of `System.nanoTime`.

5 files changed, 47 insertions, 27 deletions

diff --git a/buildSrc/src/main/groovy/com/carrotsearch/gradle/junit4/RandomizedTestingPlugin.groovy b/buildSrc/src/main/groovy/com/carrotsearch/gradle/junit4/RandomizedTestingPlugin.groovy
index e2230b116c..d3d07db0d2 100644
--- a/buildSrc/src/main/groovy/com/carrotsearch/gradle/junit4/RandomizedTestingPlugin.groovy
+++ b/buildSrc/src/main/groovy/com/carrotsearch/gradle/junit4/RandomizedTestingPlugin.groovy
@@ -12,10 +12,38 @@ import org.gradle.api.tasks.testing.Test
 class RandomizedTestingPlugin implements Plugin<Project> {
 
     void apply(Project project) {
+        setupSeed(project)
         replaceTestTask(project.tasks)
         configureAnt(project.ant)
     }
 
+    /**
+     * Pins the test seed at configuration time so it isn't different on every
+     * {@link RandomizedTestingTask} execution. This is useful if random
+     * decisions in one run of {@linkplain RandomizedTestingTask} influence the
+     * outcome of subsequent runs. Pinning the seed up front like this makes
+     * the reproduction line from one run be useful on another run.
+     */
+    static void setupSeed(Project project) {
+        if (project.rootProject.ext.has('testSeed')) {
+            /* Skip this if we've already pinned the testSeed. It is important
+             * that this checks the rootProject so that we know we've only ever
+             * initialized one time. */
+            return
+        }
+        String testSeed = System.getProperty('tests.seed')
+        if (testSeed == null) {
+            long seed = new Random(System.currentTimeMillis()).nextLong()
+            testSeed = Long.toUnsignedString(seed, 16).toUpperCase(Locale.ROOT)
+        }
+        /* Set the testSeed on the root project first so other projects can use
+         * it during initialization. */
+        project.rootProject.ext.testSeed = testSeed
+        project.rootProject.subprojects {
+            project.ext.testSeed = testSeed
+        }
+    }
+
     static void replaceTestTask(TaskContainer tasks) {
         Test oldTestTask = tasks.findByPath('test')
         if (oldTestTask == null) {
diff --git a/buildSrc/src/main/groovy/com/carrotsearch/gradle/junit4/RandomizedTestingTask.groovy b/buildSrc/src/main/groovy/com/carrotsearch/gradle/junit4/RandomizedTestingTask.groovy
index e24c226837..1817ea57e7 100644
--- a/buildSrc/src/main/groovy/com/carrotsearch/gradle/junit4/RandomizedTestingTask.groovy
+++ b/buildSrc/src/main/groovy/com/carrotsearch/gradle/junit4/RandomizedTestingTask.groovy
@@ -9,6 +9,7 @@ import org.apache.tools.ant.DefaultLogger
 import org.apache.tools.ant.RuntimeConfigurable
 import org.apache.tools.ant.UnknownElement
 import org.gradle.api.DefaultTask
+import org.gradle.api.InvalidUserDataException
 import org.gradle.api.file.FileCollection
 import org.gradle.api.file.FileTreeElement
 import org.gradle.api.internal.tasks.options.Option
@@ -259,8 +260,13 @@ class RandomizedTestingTask extends DefaultTask {
                     }
                 }
                 for (Map.Entry<String, Object> prop : systemProperties) {
+                    if (prop.getKey().equals('tests.seed')) {
+                        throw new InvalidUserDataException('Seed should be ' +
+                            'set on the project instead of a system property')
+                    }
                     sysproperty key: prop.getKey(), value: prop.getValue().toString()
                 }
+                systemProperty 'tests.seed', project.testSeed
                 for (Map.Entry<String, Object> envvar : environmentVariables) {
                     env key: envvar.getKey(), value: envvar.getValue().toString()
                 }
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy
index 87d5ec9ae5..cfcb8cfa88 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy
@@ -120,6 +120,7 @@ class BuildPlugin implements Plugin<Project> {
                 println "  JDK Version           : ${gradleJavaVersionDetails}"
                 println "  JAVA_HOME             : ${gradleJavaHome}"
             }
+            println " Random Testing Seed    : ${project.testSeed}"
 
             // enforce gradle version
             GradleVersion minGradle = GradleVersion.version('3.3')
@@ -525,7 +526,12 @@ class BuildPlugin implements Plugin<Project> {
             systemProperty 'tests.logger.level', 'WARN'
             for (Map.Entry<String, String> property : System.properties.entrySet()) {
                 if (property.getKey().startsWith('tests.') ||
-                    property.getKey().startsWith('es.')) {
+                        property.getKey().startsWith('es.')) {
+                    if (property.getKey().equals('tests.seed')) {
+                        /* The seed is already set on the project so we
+                         * shouldn't attempt to override it. */
+                        continue;
+                    }
                     systemProperty property.getKey(), property.getValue()
                 }
             }
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantPropertiesExtension.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantPropertiesExtension.groovy
index f16913d5be..e6e7fca62f 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantPropertiesExtension.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantPropertiesExtension.groovy
@@ -26,12 +26,6 @@ class VagrantPropertiesExtension {
     List<String> boxes
 
     @Input
-    Long testSeed
-
-    @Input
-    String formattedTestSeed
-
-    @Input
     String upgradeFromVersion
 
     @Input
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy
index 45049acb23..c8d77ea2fb 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy
@@ -1,5 +1,6 @@
 package org.elasticsearch.gradle.vagrant
 
+import com.carrotsearch.gradle.junit4.RandomizedTestingPlugin
 import org.elasticsearch.gradle.FileContentsTask
 import org.gradle.api.*
 import org.gradle.api.artifacts.dsl.RepositoryHandler
@@ -100,23 +101,10 @@ class VagrantTestPlugin implements Plugin<Project> {
     private static void createBatsConfiguration(Project project) {
         project.configurations.create(BATS)
 
-        final long seed
-        final String formattedSeed
-        String maybeTestsSeed = System.getProperty("tests.seed")
-        if (maybeTestsSeed != null) {
-            if (maybeTestsSeed.trim().isEmpty()) {
-                throw new GradleException("explicit tests.seed cannot be empty")
-            }
-            String masterSeed = maybeTestsSeed.tokenize(':').get(0)
-            seed = new BigInteger(masterSeed, 16).longValue()
-            formattedSeed = maybeTestsSeed
-        } else {
-            seed = new Random().nextLong()
-            formattedSeed = String.format("%016X", seed)
-        }
-
         String upgradeFromVersion = System.getProperty("tests.packaging.upgradeVersion");
         if (upgradeFromVersion == null) {
+            String firstPartOfSeed = project.rootProject.testSeed.tokenize(':').get(0)
+            final long seed = Long.parseUnsignedLong(firstPartOfSeed, 16)
             upgradeFromVersion = project.indexCompatVersions[new Random(seed).nextInt(project.indexCompatVersions.size())]
         }
 
@@ -130,8 +118,6 @@ class VagrantTestPlugin implements Plugin<Project> {
             project.dependencies.add(BATS, "org.elasticsearch.distribution.${it}:elasticsearch:${upgradeFromVersion}@${it}")
         }
 
-        project.extensions.esvagrant.testSeed = seed
-        project.extensions.esvagrant.formattedTestSeed = formattedSeed
         project.extensions.esvagrant.upgradeFromVersion = upgradeFromVersion
     }
 
@@ -395,7 +381,7 @@ class VagrantTestPlugin implements Plugin<Project> {
                 void afterExecute(Task task, TaskState state) {
                     if (state.failure != null) {
                         println "REPRODUCE WITH: gradle ${packaging.path} " +
-                            "-Dtests.seed=${project.extensions.esvagrant.formattedTestSeed} "
+                            "-Dtests.seed=${project.testSeed} "
                     }
                 }
             }
@@ -415,14 +401,14 @@ class VagrantTestPlugin implements Plugin<Project> {
                 environmentVars vagrantEnvVars
                 dependsOn up
                 finalizedBy halt
-                args '--command', PLATFORM_TEST_COMMAND + " -Dtests.seed=${-> project.extensions.esvagrant.formattedTestSeed}"
+                args '--command', PLATFORM_TEST_COMMAND + " -Dtests.seed=${-> project.testSeed}"
             }
             TaskExecutionAdapter platformReproListener = new TaskExecutionAdapter() {
                 @Override
                 void afterExecute(Task task, TaskState state) {
                     if (state.failure != null) {
                         println "REPRODUCE WITH: gradle ${platform.path} " +
-                            "-Dtests.seed=${project.extensions.esvagrant.formattedTestSeed} "
+                            "-Dtests.seed=${project.testSeed} "
                     }
                 }
             }