Add microbenchmarking infrastructure (#18891)

With this commit we add a benchmarks project that contains the necessary build
infrastructure and an example benchmark. It is added as a separate project to avoid
interfering with the regular build too much (especially sanity checks) and to keep
the microbenchmarks isolated.

Microbenchmarks are generated with `gradle :benchmarks:jmhJar` and can be run with
` gradle :benchmarks:jmh`.

We intentionally do not use the
[jmh-gradle-plugin](https://github.com/melix/jmh-gradle-plugin) as it causes all
sorts of problems (dependencies are not properly excluded, not all JMH parameters
can be set) and it adds another abstraction layer that is not needed.

Closes #18242

4 files changed, 374 insertions, 0 deletions

diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/DateBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/DateBenchmark.java
new file mode 100644
index 0000000000..87f4f0ccbd
--- /dev/null
+++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/DateBenchmark.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.benchmark;
+
+import org.joda.time.DateTimeZone;
+import org.joda.time.MutableDateTime;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.State;
+
+import java.time.Duration;
+import java.time.Instant;
+import java.time.ZoneOffset;
+import java.time.ZonedDateTime;
+import java.time.temporal.TemporalAmount;
+import java.util.Calendar;
+import java.util.Locale;
+import java.util.TimeZone;
+
+@SuppressWarnings("unused") //invoked by benchmarking framework
+@State(Scope.Benchmark)
+public class DateBenchmark {
+    private long instant;
+
+    private MutableDateTime jodaDate = new MutableDateTime(0, DateTimeZone.UTC);
+
+    private Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("UTC"), Locale.ROOT);
+
+    private ZonedDateTime javaDateTime = ZonedDateTime.ofInstant(Instant.ofEpochMilli(0L), ZoneOffset.UTC);
+
+    private TemporalAmount diff = Duration.ofMillis(1L);
+
+    @Benchmark
+    public int mutableDateTimeSetMillisGetDayOfMonth() {
+        jodaDate.setMillis(instant++);
+        return jodaDate.getDayOfMonth();
+    }
+
+    @Benchmark
+    public int calendarSetMillisGetDayOfMonth() {
+        calendar.setTimeInMillis(instant++);
+        return calendar.get(Calendar.DAY_OF_MONTH);
+    }
+
+    @Benchmark
+    public int javaDateTimeSetMillisGetDayOfMonth() {
+        // all classes in java.time are immutable, we have to use a new instance
+        javaDateTime = javaDateTime.plus(diff);
+        return javaDateTime.getDayOfMonth();
+    }
+}
diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/HelloBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/HelloBenchmark.java
new file mode 100644
index 0000000000..0af853c18f
--- /dev/null
+++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/HelloBenchmark.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.benchmark;
+
+import org.openjdk.jmh.annotations.Benchmark;
+
+@SuppressWarnings("unused") //invoked by benchmarking framework
+public class HelloBenchmark {
+    @Benchmark
+    public void benchmarkRuntimeOverhead() {
+        //intentionally left blank
+    }
+}
diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/AllocationBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/AllocationBenchmark.java
new file mode 100644
index 0000000000..d5bb0916ec
--- /dev/null
+++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/AllocationBenchmark.java
@@ -0,0 +1,170 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.benchmark.routing.allocation;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.ClusterName;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.cluster.routing.RoutingTable;
+import org.elasticsearch.cluster.routing.ShardRoutingState;
+import org.elasticsearch.cluster.routing.allocation.AllocationService;
+import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
+import org.elasticsearch.common.settings.Settings;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Warmup;
+
+import java.util.Collections;
+import java.util.concurrent.TimeUnit;
+
+@Fork(3)
+@Warmup(iterations = 10)
+@Measurement(iterations = 10)
+@BenchmarkMode(Mode.AverageTime)
+@OutputTimeUnit(TimeUnit.MILLISECONDS)
+@State(Scope.Benchmark)
+@SuppressWarnings("unused") //invoked by benchmarking framework
+public class AllocationBenchmark {
+    // Do NOT make any field final (even if it is not annotated with @Param)! See also
+    // http://hg.openjdk.java.net/code-tools/jmh/file/tip/jmh-samples/src/main/java/org/openjdk/jmh/samples/JMHSample_10_ConstantFold.java
+
+    // we cannot use individual @Params as some will lead to invalid combinations which do not let the benchmark terminate. JMH offers no
+    // support to constrain the combinations of benchmark parameters and we do not want to rely on OptionsBuilder as each benchmark would
+    // need its own main method and we cannot execute more than one class with a main method per JAR.
+    @Param({
+        // indices, shards, replicas, nodes
+        "       10,      1,        0,     1",
+        "       10,      3,        0,     1",
+        "       10,     10,        0,     1",
+        "      100,      1,        0,     1",
+        "      100,      3,        0,     1",
+        "      100,     10,        0,     1",
+
+        "       10,      1,        0,    10",
+        "       10,      3,        0,    10",
+        "       10,     10,        0,    10",
+        "      100,      1,        0,    10",
+        "      100,      3,        0,    10",
+        "      100,     10,        0,    10",
+
+        "       10,      1,        1,    10",
+        "       10,      3,        1,    10",
+        "       10,     10,        1,    10",
+        "      100,      1,        1,    10",
+        "      100,      3,        1,    10",
+        "      100,     10,        1,    10",
+
+        "       10,      1,        2,    10",
+        "       10,      3,        2,    10",
+        "       10,     10,        2,    10",
+        "      100,      1,        2,    10",
+        "      100,      3,        2,    10",
+        "      100,     10,        2,    10",
+
+        "       10,      1,        0,    50",
+        "       10,      3,        0,    50",
+        "       10,     10,        0,    50",
+        "      100,      1,        0,    50",
+        "      100,      3,        0,    50",
+        "      100,     10,        0,    50",
+
+        "       10,      1,        1,    50",
+        "       10,      3,        1,    50",
+        "       10,     10,        1,    50",
+        "      100,      1,        1,    50",
+        "      100,      3,        1,    50",
+        "      100,     10,        1,    50",
+
+        "       10,      1,        2,    50",
+        "       10,      3,        2,    50",
+        "       10,     10,        2,    50",
+        "      100,      1,        2,    50",
+        "      100,      3,        2,    50",
+        "      100,     10,        2,    50"
+    })
+    public String indicesShardsReplicasNodes = "10,1,0,1";
+
+    public int numTags = 2;
+
+    private AllocationService strategy;
+    private ClusterState initialClusterState;
+
+    @Setup
+    public void setUp() throws Exception {
+        final String[] params = indicesShardsReplicasNodes.split(",");
+
+        int numIndices = toInt(params[0]);
+        int numShards = toInt(params[1]);
+        int numReplicas = toInt(params[2]);
+        int numNodes = toInt(params[3]);
+
+        strategy = Allocators.createAllocationService(Settings.builder()
+                .put("cluster.routing.allocation.awareness.attributes", "tag")
+                .build());
+
+        MetaData.Builder mb = MetaData.builder();
+        for (int i = 1; i <= numIndices; i++) {
+            mb.put(IndexMetaData.builder("test_" + i)
+                    .settings(Settings.builder().put("index.version.created", Version.CURRENT))
+                    .numberOfShards(numShards)
+                    .numberOfReplicas(numReplicas)
+            );
+        }
+        MetaData metaData = mb.build();
+        RoutingTable.Builder rb = RoutingTable.builder();
+        for (int i = 1; i <= numIndices; i++) {
+            rb.addAsNew(metaData.index("test_" + i));
+        }
+        RoutingTable routingTable = rb.build();
+        DiscoveryNodes.Builder nb = DiscoveryNodes.builder();
+        for (int i = 1; i <= numNodes; i++) {
+            nb.put(Allocators.newNode("node" + i, Collections.singletonMap("tag", "tag_" + (i % numTags))));
+        }
+        initialClusterState = ClusterState.builder(ClusterName.DEFAULT).metaData(metaData).routingTable(routingTable).nodes
+                (nb).build();
+    }
+
+    private int toInt(String v) {
+        return Integer.valueOf(v.trim());
+    }
+
+    @Benchmark
+    public ClusterState measureAllocation() {
+        ClusterState clusterState = initialClusterState;
+        while (clusterState.getRoutingNodes().hasUnassignedShards()) {
+            RoutingAllocation.Result result = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes()
+                    .shardsWithState(ShardRoutingState.INITIALIZING));
+            clusterState = ClusterState.builder(clusterState).routingResult(result).build();
+            result = strategy.reroute(clusterState, "reroute");
+            clusterState = ClusterState.builder(clusterState).routingResult(result).build();
+        }
+        return clusterState;
+    }
+}
diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/Allocators.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/Allocators.java
new file mode 100644
index 0000000000..aba7fda102
--- /dev/null
+++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/Allocators.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.benchmark.routing.allocation;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.ClusterModule;
+import org.elasticsearch.cluster.EmptyClusterInfoService;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.routing.allocation.AllocationService;
+import org.elasticsearch.cluster.routing.allocation.FailedRerouteAllocation;
+import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
+import org.elasticsearch.cluster.routing.allocation.StartedRerouteAllocation;
+import org.elasticsearch.cluster.routing.allocation.allocator.BalancedShardsAllocator;
+import org.elasticsearch.cluster.routing.allocation.decider.AllocationDecider;
+import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders;
+import org.elasticsearch.common.settings.ClusterSettings;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.transport.DummyTransportAddress;
+import org.elasticsearch.common.util.set.Sets;
+import org.elasticsearch.gateway.GatewayAllocator;
+
+import java.lang.reflect.Constructor;
+import java.lang.reflect.InvocationTargetException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+public final class Allocators {
+    private static class NoopGatewayAllocator extends GatewayAllocator {
+        public static final NoopGatewayAllocator INSTANCE = new NoopGatewayAllocator();
+
+        protected NoopGatewayAllocator() {
+            super(Settings.EMPTY, null, null);
+        }
+
+        @Override
+        public void applyStartedShards(StartedRerouteAllocation allocation) {
+            // noop
+        }
+
+        @Override
+        public void applyFailedShards(FailedRerouteAllocation allocation) {
+            // noop
+        }
+
+        @Override
+        public boolean allocateUnassigned(RoutingAllocation allocation) {
+            return false;
+        }
+    }
+
+    private Allocators() {
+        throw new AssertionError("Do not instantiate");
+    }
+
+
+    public static AllocationService createAllocationService(Settings settings) throws NoSuchMethodException, InstantiationException,
+        IllegalAccessException, InvocationTargetException {
+        return createAllocationService(settings, new ClusterSettings(Settings.Builder.EMPTY_SETTINGS, ClusterSettings
+            .BUILT_IN_CLUSTER_SETTINGS));
+    }
+
+    public static AllocationService createAllocationService(Settings settings, ClusterSettings clusterSettings) throws
+        InvocationTargetException, NoSuchMethodException, InstantiationException, IllegalAccessException {
+        return new AllocationService(settings,
+            defaultAllocationDeciders(settings, clusterSettings),
+            NoopGatewayAllocator.INSTANCE, new BalancedShardsAllocator(settings), EmptyClusterInfoService.INSTANCE);
+    }
+
+    public static AllocationDeciders defaultAllocationDeciders(Settings settings, ClusterSettings clusterSettings) throws
+        IllegalAccessException, InvocationTargetException, InstantiationException, NoSuchMethodException {
+        List<AllocationDecider> list = new ArrayList<>();
+        // Keep a deterministic order of allocation deciders for the benchmark
+        for (Class<? extends AllocationDecider> deciderClass : ClusterModule.DEFAULT_ALLOCATION_DECIDERS) {
+            try {
+                Constructor<? extends AllocationDecider> constructor = deciderClass.getConstructor(Settings.class, ClusterSettings
+                    .class);
+                list.add(constructor.newInstance(settings, clusterSettings));
+            } catch (NoSuchMethodException e) {
+                Constructor<? extends AllocationDecider> constructor = deciderClass.getConstructor(Settings.class);
+                list.add(constructor.newInstance(settings));
+            }
+        }
+        return new AllocationDeciders(settings, list.toArray(new AllocationDecider[0]));
+
+    }
+
+    public static DiscoveryNode newNode(String nodeId, Map<String, String> attributes) {
+        return new DiscoveryNode("", nodeId, DummyTransportAddress.INSTANCE, attributes, Sets.newHashSet(DiscoveryNode.Role.MASTER,
+            DiscoveryNode.Role.DATA), Version.CURRENT);
+    }
+}