diff options
Diffstat (limited to 'core/src/test/java')
58 files changed, 1552 insertions, 455 deletions
diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainTests.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainTests.java new file mode 100644 index 0000000000..95415ecdbd --- /dev/null +++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainTests.java @@ -0,0 +1,86 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.action.admin.cluster.allocation; + +import org.elasticsearch.client.Requests; +import org.elasticsearch.cluster.routing.allocation.decider.Decision; +import org.elasticsearch.test.ESSingleNodeTestCase; + + +/** + * Tests for the cluster allocation explanation + */ +public final class ClusterAllocationExplainTests extends ESSingleNodeTestCase { + + public void testShardExplain() throws Exception { + client().admin().indices().prepareCreate("test") + .setSettings("index.number_of_shards", 1, "index.number_of_replicas", 1).get(); + client().admin().cluster().health(Requests.clusterHealthRequest("test").waitForYellowStatus()).get(); + ClusterAllocationExplainResponse resp = client().admin().cluster().prepareAllocationExplain() + .setIndex("test").setShard(0).setPrimary(false).get(); + + ClusterAllocationExplanation cae = resp.getExplanation(); + assertNotNull("should always have an explanation", cae); + assertEquals("test", cae.getShard().getIndexName()); + assertEquals(0, cae.getShard().getId()); + assertEquals(false, cae.isPrimary()); + assertNull(cae.getAssignedNodeId()); + assertNotNull(cae.getUnassignedInfo()); + Decision d = cae.getNodeDecisions().values().iterator().next(); + assertNotNull("should have a decision", d); + assertEquals(Decision.Type.NO, d.type()); + assertTrue(d.toString(), d.toString().contains("NO(the shard cannot be allocated on the same node id")); + assertTrue(d instanceof Decision.Multi); + Decision.Multi md = (Decision.Multi) d; + Decision ssd = md.getDecisions().get(0); + assertEquals(Decision.Type.NO, ssd.type()); + assertTrue(ssd.toString(), ssd.toString().contains("NO(the shard cannot be allocated on the same node id")); + Float weight = cae.getNodeWeights().values().iterator().next(); + assertNotNull("should have a weight", weight); + + resp = client().admin().cluster().prepareAllocationExplain().setIndex("test").setShard(0).setPrimary(true).get(); + + cae = resp.getExplanation(); + assertNotNull("should always have an explanation", cae); + assertEquals("test", cae.getShard().getIndexName()); + assertEquals(0, cae.getShard().getId()); + assertEquals(true, cae.isPrimary()); + assertNotNull("shard should have assigned node id", cae.getAssignedNodeId()); + assertNull("assigned shard should not have unassigned info", cae.getUnassignedInfo()); + d = cae.getNodeDecisions().values().iterator().next(); + assertNotNull("should have a decision", d); + assertEquals(Decision.Type.NO, d.type()); + assertTrue(d.toString(), d.toString().contains("NO(the shard cannot be allocated on the same node id")); + assertTrue(d instanceof Decision.Multi); + md = (Decision.Multi) d; + ssd = md.getDecisions().get(0); + assertEquals(Decision.Type.NO, ssd.type()); + assertTrue(ssd.toString(), ssd.toString().contains("NO(the shard cannot be allocated on the same node id")); + weight = cae.getNodeWeights().values().iterator().next(); + assertNotNull("should have a weight", weight); + + resp = client().admin().cluster().prepareAllocationExplain().useAnyUnassignedShard().get(); + cae = resp.getExplanation(); + assertNotNull("should always have an explanation", cae); + assertEquals("test", cae.getShard().getIndexName()); + assertEquals(0, cae.getShard().getId()); + assertEquals(false, cae.isPrimary()); + } +} diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplanationTests.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplanationTests.java new file mode 100644 index 0000000000..060fb73fbf --- /dev/null +++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplanationTests.java @@ -0,0 +1,91 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.action.admin.cluster.allocation; + +import org.elasticsearch.Version; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.routing.RoutingNode; +import org.elasticsearch.cluster.routing.ShardRouting; +import org.elasticsearch.cluster.routing.allocation.RoutingAllocation; +import org.elasticsearch.cluster.routing.allocation.decider.AllocationDecider; +import org.elasticsearch.cluster.routing.allocation.decider.Decision; +import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.transport.DummyTransportAddress; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.index.Index; +import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.test.ESTestCase; + +import java.util.HashMap; +import java.util.Map; + +import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; + +/** + * Tests for the cluster allocation explanation + */ +public final class ClusterAllocationExplanationTests extends ESTestCase { + + public void testDecisionEquality() { + Decision.Multi d = new Decision.Multi(); + Decision.Multi d2 = new Decision.Multi(); + d.add(Decision.single(Decision.Type.NO, "no label", "because I said no")); + d.add(Decision.single(Decision.Type.YES, "yes label", "yes please")); + d.add(Decision.single(Decision.Type.THROTTLE, "throttle label", "wait a sec")); + d2.add(Decision.single(Decision.Type.NO, "no label", "because I said no")); + d2.add(Decision.single(Decision.Type.YES, "yes label", "yes please")); + d2.add(Decision.single(Decision.Type.THROTTLE, "throttle label", "wait a sec")); + assertEquals(d, d2); + } + + public void testExplanationSerialization() throws Exception { + ShardId shard = new ShardId("test", "uuid", 0); + Map<DiscoveryNode, Decision> nodeToDecisions = new HashMap<>(); + Map<DiscoveryNode, Float> nodeToWeight = new HashMap<>(); + for (int i = randomIntBetween(2, 5); i > 0; i--) { + DiscoveryNode dn = new DiscoveryNode("node-" + i, DummyTransportAddress.INSTANCE, Version.CURRENT); + Decision.Multi d = new Decision.Multi(); + d.add(Decision.single(Decision.Type.NO, "no label", "because I said no")); + d.add(Decision.single(Decision.Type.YES, "yes label", "yes please")); + d.add(Decision.single(Decision.Type.THROTTLE, "throttle label", "wait a sec")); + nodeToDecisions.put(dn, d); + nodeToWeight.put(dn, randomFloat()); + } + + ClusterAllocationExplanation cae = new ClusterAllocationExplanation(shard, true, "assignedNode", null, + nodeToDecisions, nodeToWeight); + BytesStreamOutput out = new BytesStreamOutput(); + cae.writeTo(out); + StreamInput in = StreamInput.wrap(out.bytes()); + ClusterAllocationExplanation cae2 = new ClusterAllocationExplanation(in); + assertEquals(shard, cae2.getShard()); + assertTrue(cae2.isPrimary()); + assertTrue(cae2.isAssigned()); + assertEquals("assignedNode", cae2.getAssignedNodeId()); + assertNull(cae2.getUnassignedInfo()); + for (Map.Entry<DiscoveryNode, Decision> entry : cae2.getNodeDecisions().entrySet()) { + assertEquals(nodeToDecisions.get(entry.getKey()), entry.getValue()); + } + assertEquals(nodeToWeight, cae2.getNodeWeights()); + } +} diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TaskManagerTestCase.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TaskManagerTestCase.java index 48d9f8fed4..3d996becba 100644 --- a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TaskManagerTestCase.java +++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TaskManagerTestCase.java @@ -219,6 +219,10 @@ public abstract class TaskManagerTestCase extends ESTestCase { clusterService.close(); transportService.close(); } + + public String getNodeId() { + return discoveryNode.getId(); + } } public static void connectNodes(TestNode... nodes) { diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TransportTasksActionTests.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TransportTasksActionTests.java index 4b478b52bd..972d9735ef 100644 --- a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TransportTasksActionTests.java +++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TransportTasksActionTests.java @@ -38,6 +38,7 @@ import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; @@ -54,6 +55,7 @@ import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; @@ -629,4 +631,76 @@ public class TransportTasksActionTests extends TaskManagerTestCase { NodesResponse responses = future.get(); assertEquals(0, responses.failureCount()); } + + + /** + * This test starts nodes actions that blocks on all nodes. While node actions are blocked in the middle of execution + * it executes a tasks action that targets these blocked node actions. The test verifies that task actions are only + * getting executed on nodes that are not listed in the node filter. + */ + public void testTaskNodeFiltering() throws ExecutionException, InterruptedException, IOException { + setupTestNodes(Settings.EMPTY); + connectNodes(testNodes); + CountDownLatch checkLatch = new CountDownLatch(1); + // Start some test nodes action so we could have something to run tasks actions on + ActionFuture<NodesResponse> future = startBlockingTestNodesAction(checkLatch); + + String[] allNodes = new String[testNodes.length]; + for (int i = 0; i < testNodes.length; i++) { + allNodes[i] = testNodes[i].getNodeId(); + } + + int filterNodesSize = randomInt(allNodes.length); + Set<String> filterNodes = new HashSet<>(randomSubsetOf(filterNodesSize, allNodes)); + logger.info("Filtering out nodes {} size: {}", filterNodes, filterNodesSize); + + TestTasksAction[] tasksActions = new TestTasksAction[nodesCount]; + for (int i = 0; i < testNodes.length; i++) { + final int node = i; + // Simulate a task action that works on all nodes except nodes listed in filterNodes. + // We are testing that it works. + tasksActions[i] = new TestTasksAction(Settings.EMPTY, "testTasksAction", clusterName, threadPool, + testNodes[i].clusterService, testNodes[i].transportService) { + + @Override + protected String[] filterNodeIds(DiscoveryNodes nodes, String[] nodesIds) { + String[] superNodes = super.filterNodeIds(nodes, nodesIds); + List<String> filteredNodes = new ArrayList<>(); + for (String node : superNodes) { + if (filterNodes.contains(node) == false) { + filteredNodes.add(node); + } + } + return filteredNodes.toArray(new String[filteredNodes.size()]); + } + + @Override + protected TestTaskResponse taskOperation(TestTasksRequest request, Task task) { + return new TestTaskResponse(testNodes[node].getNodeId()); + } + }; + } + + // Run task action on node tasks that are currently running + // should be successful on all nodes except nodes that we filtered out + TestTasksRequest testTasksRequest = new TestTasksRequest(); + testTasksRequest.setActions("testAction[n]"); // pick all test actions + TestTasksResponse response = tasksActions[randomIntBetween(0, nodesCount - 1)].execute(testTasksRequest).get(); + + // Get successful responses from all nodes except nodes that we filtered out + assertEquals(testNodes.length - filterNodes.size(), response.tasks.size()); + assertEquals(0, response.getTaskFailures().size()); // no task failed + assertEquals(0, response.getNodeFailures().size()); // no nodes failed + + // Make sure that filtered nodes didn't send any responses + for (TestTaskResponse taskResponse : response.tasks) { + String nodeId = taskResponse.getStatus(); + assertFalse("Found response from filtered node " + nodeId, filterNodes.contains(nodeId)); + } + + // Release all node tasks and wait for response + checkLatch.countDown(); + NodesResponse responses = future.get(); + assertEquals(0, responses.failureCount()); + } } diff --git a/core/src/test/java/org/elasticsearch/action/ingest/SimulateExecutionServiceTests.java b/core/src/test/java/org/elasticsearch/action/ingest/SimulateExecutionServiceTests.java index cf1cab2416..f66dfa81ea 100644 --- a/core/src/test/java/org/elasticsearch/action/ingest/SimulateExecutionServiceTests.java +++ b/core/src/test/java/org/elasticsearch/action/ingest/SimulateExecutionServiceTests.java @@ -31,10 +31,8 @@ import org.elasticsearch.threadpool.ThreadPool; import org.junit.After; import org.junit.Before; -import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; -import java.util.List; +import java.util.Map; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; @@ -46,7 +44,6 @@ public class SimulateExecutionServiceTests extends ESTestCase { private ThreadPool threadPool; private SimulateExecutionService executionService; - private Pipeline pipeline; private Processor processor; private IngestDocument ingestDocument; @@ -59,7 +56,6 @@ public class SimulateExecutionServiceTests extends ESTestCase { ); executionService = new SimulateExecutionService(threadPool); processor = new TestProcessor("id", "mock", ingestDocument -> {}); - pipeline = new Pipeline("_id", "_description", new CompoundProcessor(processor, processor)); ingestDocument = RandomDocumentPicks.randomIngestDocument(random()); } @@ -68,74 +64,6 @@ public class SimulateExecutionServiceTests extends ESTestCase { threadPool.shutdown(); } - public void testExecuteVerboseDocumentSimple() throws Exception { - List<SimulateProcessorResult> processorResultList = new ArrayList<>(); - executionService.executeVerboseDocument(processor, ingestDocument, processorResultList); - SimulateProcessorResult result = new SimulateProcessorResult("id", ingestDocument); - assertThat(processorResultList.size(), equalTo(1)); - assertThat(processorResultList.get(0).getProcessorTag(), equalTo(result.getProcessorTag())); - assertThat(processorResultList.get(0).getIngestDocument(), equalTo(result.getIngestDocument())); - assertThat(processorResultList.get(0).getFailure(), nullValue()); - } - - public void testExecuteVerboseDocumentSimpleException() throws Exception { - RuntimeException exception = new RuntimeException("mock_exception"); - TestProcessor processor = new TestProcessor("id", "mock", ingestDocument -> { throw exception; }); - List<SimulateProcessorResult> processorResultList = new ArrayList<>(); - try { - executionService.executeVerboseDocument(processor, ingestDocument, processorResultList); - fail("should throw exception"); - } catch (RuntimeException e) { - assertThat(e.getMessage(), equalTo("mock_exception")); - } - SimulateProcessorResult result = new SimulateProcessorResult("id", exception); - assertThat(processorResultList.size(), equalTo(1)); - assertThat(processorResultList.get(0).getProcessorTag(), equalTo(result.getProcessorTag())); - assertThat(processorResultList.get(0).getFailure(), equalTo(result.getFailure())); - } - - public void testExecuteVerboseDocumentCompoundSuccess() throws Exception { - TestProcessor processor1 = new TestProcessor("p1", "mock", ingestDocument -> { }); - TestProcessor processor2 = new TestProcessor("p2", "mock", ingestDocument -> { }); - - Processor compoundProcessor = new CompoundProcessor(processor1, processor2); - List<SimulateProcessorResult> processorResultList = new ArrayList<>(); - executionService.executeVerboseDocument(compoundProcessor, ingestDocument, processorResultList); - assertThat(processor1.getInvokedCounter(), equalTo(1)); - assertThat(processor2.getInvokedCounter(), equalTo(1)); - assertThat(processorResultList.size(), equalTo(2)); - assertThat(processorResultList.get(0).getProcessorTag(), equalTo("p1")); - assertThat(processorResultList.get(0).getIngestDocument(), equalTo(ingestDocument)); - assertThat(processorResultList.get(0).getFailure(), nullValue()); - assertThat(processorResultList.get(1).getProcessorTag(), equalTo("p2")); - assertThat(processorResultList.get(1).getIngestDocument(), equalTo(ingestDocument)); - assertThat(processorResultList.get(1).getFailure(), nullValue()); - } - - public void testExecuteVerboseDocumentCompoundOnFailure() throws Exception { - TestProcessor processor1 = new TestProcessor("p1", "mock", ingestDocument -> { }); - TestProcessor processor2 = new TestProcessor("p2", "mock", ingestDocument -> { throw new RuntimeException("p2_exception"); }); - TestProcessor onFailureProcessor1 = new TestProcessor("fail_p1", "mock", ingestDocument -> { }); - TestProcessor onFailureProcessor2 = new TestProcessor("fail_p2", "mock", ingestDocument -> { throw new RuntimeException("fail_p2_exception"); }); - TestProcessor onFailureProcessor3 = new TestProcessor("fail_p3", "mock", ingestDocument -> { }); - CompoundProcessor onFailureCompoundProcessor = new CompoundProcessor(Collections.singletonList(onFailureProcessor2), Collections.singletonList(onFailureProcessor3)); - - Processor compoundProcessor = new CompoundProcessor(Arrays.asList(processor1, processor2), Arrays.asList(onFailureProcessor1, onFailureCompoundProcessor)); - List<SimulateProcessorResult> processorResultList = new ArrayList<>(); - executionService.executeVerboseDocument(compoundProcessor, ingestDocument, processorResultList); - assertThat(processor1.getInvokedCounter(), equalTo(1)); - assertThat(processor2.getInvokedCounter(), equalTo(1)); - assertThat(onFailureProcessor1.getInvokedCounter(), equalTo(1)); - assertThat(onFailureProcessor2.getInvokedCounter(), equalTo(1)); - assertThat(onFailureProcessor3.getInvokedCounter(), equalTo(1)); - assertThat(processorResultList.size(), equalTo(5)); - assertThat(processorResultList.get(0).getProcessorTag(), equalTo("p1")); - assertThat(processorResultList.get(1).getProcessorTag(), equalTo("p2")); - assertThat(processorResultList.get(2).getProcessorTag(), equalTo("fail_p1")); - assertThat(processorResultList.get(3).getProcessorTag(), equalTo("fail_p2")); - assertThat(processorResultList.get(4).getProcessorTag(), equalTo("fail_p3")); - } - public void testExecuteVerboseItem() throws Exception { TestProcessor processor = new TestProcessor("test-id", "mock", ingestDocument -> {}); Pipeline pipeline = new Pipeline("_id", "_description", new CompoundProcessor(processor, processor)); @@ -170,16 +98,43 @@ public class SimulateExecutionServiceTests extends ESTestCase { assertThat(simulateDocumentBaseResult.getFailure(), nullValue()); } - public void testExecuteVerboseItemWithFailure() throws Exception { + public void testExecuteVerboseItemExceptionWithoutOnFailure() throws Exception { + TestProcessor processor1 = new TestProcessor("processor_0", "mock", ingestDocument -> {}); + TestProcessor processor2 = new TestProcessor("processor_1", "mock", ingestDocument -> { throw new RuntimeException("processor failed"); }); + TestProcessor processor3 = new TestProcessor("processor_2", "mock", ingestDocument -> {}); + Pipeline pipeline = new Pipeline("_id", "_description", new CompoundProcessor(processor1, processor2, processor3)); + SimulateDocumentResult actualItemResponse = executionService.executeDocument(pipeline, ingestDocument, true); + assertThat(processor1.getInvokedCounter(), equalTo(1)); + assertThat(processor2.getInvokedCounter(), equalTo(1)); + assertThat(processor3.getInvokedCounter(), equalTo(0)); + assertThat(actualItemResponse, instanceOf(SimulateDocumentVerboseResult.class)); + SimulateDocumentVerboseResult simulateDocumentVerboseResult = (SimulateDocumentVerboseResult) actualItemResponse; + assertThat(simulateDocumentVerboseResult.getProcessorResults().size(), equalTo(2)); + assertThat(simulateDocumentVerboseResult.getProcessorResults().get(0).getProcessorTag(), equalTo("processor_0")); + assertThat(simulateDocumentVerboseResult.getProcessorResults().get(0).getFailure(), nullValue()); + assertThat(simulateDocumentVerboseResult.getProcessorResults().get(0).getIngestDocument(), not(sameInstance(ingestDocument))); + assertThat(simulateDocumentVerboseResult.getProcessorResults().get(0).getIngestDocument(), equalTo(ingestDocument)); + assertThat(simulateDocumentVerboseResult.getProcessorResults().get(0).getIngestDocument().getSourceAndMetadata(), not(sameInstance(ingestDocument.getSourceAndMetadata()))); + assertThat(simulateDocumentVerboseResult.getProcessorResults().get(1).getProcessorTag(), equalTo("processor_1")); + assertThat(simulateDocumentVerboseResult.getProcessorResults().get(1).getIngestDocument(), nullValue()); + assertThat(simulateDocumentVerboseResult.getProcessorResults().get(1).getFailure(), instanceOf(RuntimeException.class)); + RuntimeException runtimeException = (RuntimeException) simulateDocumentVerboseResult.getProcessorResults().get(1).getFailure(); + assertThat(runtimeException.getMessage(), equalTo("processor failed")); + } + + public void testExecuteVerboseItemWithOnFailure() throws Exception { TestProcessor processor1 = new TestProcessor("processor_0", "mock", ingestDocument -> { throw new RuntimeException("processor failed"); }); TestProcessor processor2 = new TestProcessor("processor_1", "mock", ingestDocument -> {}); - Pipeline pipeline = new Pipeline("_id", "_description", new CompoundProcessor(Collections.singletonList(processor1), Collections.singletonList(processor2))); + TestProcessor processor3 = new TestProcessor("processor_2", "mock", ingestDocument -> {}); + Pipeline pipeline = new Pipeline("_id", "_description", + new CompoundProcessor(new CompoundProcessor(Collections.singletonList(processor1), + Collections.singletonList(processor2)), processor3)); SimulateDocumentResult actualItemResponse = executionService.executeDocument(pipeline, ingestDocument, true); assertThat(processor1.getInvokedCounter(), equalTo(1)); assertThat(processor2.getInvokedCounter(), equalTo(1)); assertThat(actualItemResponse, instanceOf(SimulateDocumentVerboseResult.class)); SimulateDocumentVerboseResult simulateDocumentVerboseResult = (SimulateDocumentVerboseResult) actualItemResponse; - assertThat(simulateDocumentVerboseResult.getProcessorResults().size(), equalTo(2)); + assertThat(simulateDocumentVerboseResult.getProcessorResults().size(), equalTo(3)); assertThat(simulateDocumentVerboseResult.getProcessorResults().get(0).getProcessorTag(), equalTo("processor_0")); assertThat(simulateDocumentVerboseResult.getProcessorResults().get(0).getIngestDocument(), nullValue()); assertThat(simulateDocumentVerboseResult.getProcessorResults().get(0).getFailure(), instanceOf(RuntimeException.class)); @@ -187,8 +142,20 @@ public class SimulateExecutionServiceTests extends ESTestCase { assertThat(runtimeException.getMessage(), equalTo("processor failed")); assertThat(simulateDocumentVerboseResult.getProcessorResults().get(1).getProcessorTag(), equalTo("processor_1")); assertThat(simulateDocumentVerboseResult.getProcessorResults().get(1).getIngestDocument(), not(sameInstance(ingestDocument))); - assertThat(simulateDocumentVerboseResult.getProcessorResults().get(1).getIngestDocument(), equalTo(ingestDocument)); + + IngestDocument ingestDocumentWithOnFailureMetadata = new IngestDocument(ingestDocument); + Map<String, String> metadata = ingestDocumentWithOnFailureMetadata.getIngestMetadata(); + metadata.put(CompoundProcessor.ON_FAILURE_PROCESSOR_TYPE_FIELD, "mock"); + metadata.put(CompoundProcessor.ON_FAILURE_PROCESSOR_TAG_FIELD, "processor_0"); + metadata.put(CompoundProcessor.ON_FAILURE_MESSAGE_FIELD, "processor failed"); + assertThat(simulateDocumentVerboseResult.getProcessorResults().get(1).getIngestDocument(), equalTo(ingestDocumentWithOnFailureMetadata)); + assertThat(simulateDocumentVerboseResult.getProcessorResults().get(1).getFailure(), nullValue()); + + assertThat(simulateDocumentVerboseResult.getProcessorResults().get(2).getProcessorTag(), equalTo("processor_2")); + assertThat(simulateDocumentVerboseResult.getProcessorResults().get(2).getIngestDocument(), not(sameInstance(ingestDocument))); + assertThat(simulateDocumentVerboseResult.getProcessorResults().get(2).getIngestDocument(), equalTo(ingestDocument)); + assertThat(simulateDocumentVerboseResult.getProcessorResults().get(2).getFailure(), nullValue()); } public void testExecuteItemWithFailure() throws Exception { diff --git a/core/src/test/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeActionTests.java b/core/src/test/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeActionTests.java index b166f5f45c..5d17232735 100644 --- a/core/src/test/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeActionTests.java +++ b/core/src/test/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeActionTests.java @@ -34,7 +34,9 @@ import org.elasticsearch.cluster.block.ClusterBlock; import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.block.ClusterBlocks; +import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.routing.IndexRoutingTable; @@ -212,10 +214,12 @@ public class TransportBroadcastByNodeActionTests extends ESTestCase { IndexRoutingTable.Builder indexRoutingTable = IndexRoutingTable.builder(new Index(index, "_na_")); int shardIndex = -1; + int totalIndexShards = 0; for (int i = 0; i < numberOfNodes; i++) { final DiscoveryNode node = newNode(i); discoBuilder = discoBuilder.put(node); int numberOfShards = randomIntBetween(1, 10); + totalIndexShards += numberOfShards; for (int j = 0; j < numberOfShards; j++) { final ShardId shardId = new ShardId(index, "_na_", ++shardIndex); ShardRouting shard = TestShardRouting.newShardRouting(index, shardId.getId(), node.id(), true, ShardRoutingState.STARTED); @@ -228,6 +232,12 @@ public class TransportBroadcastByNodeActionTests extends ESTestCase { discoBuilder.masterNodeId(newNode(numberOfNodes - 1).id()); ClusterState.Builder stateBuilder = ClusterState.builder(new ClusterName(TEST_CLUSTER)); stateBuilder.nodes(discoBuilder); + final IndexMetaData.Builder indexMetaData = IndexMetaData.builder(index) + .settings(Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)) + .numberOfReplicas(0) + .numberOfShards(totalIndexShards); + + stateBuilder.metaData(MetaData.builder().put(indexMetaData)); stateBuilder.routingTable(RoutingTable.builder().add(indexRoutingTable.build()).build()); ClusterState clusterState = stateBuilder.build(); setState(clusterService, clusterState); diff --git a/core/src/test/java/org/elasticsearch/action/support/replication/BroadcastReplicationTests.java b/core/src/test/java/org/elasticsearch/action/support/replication/BroadcastReplicationTests.java index 4125f02b95..9170ff2e5a 100644 --- a/core/src/test/java/org/elasticsearch/action/support/replication/BroadcastReplicationTests.java +++ b/core/src/test/java/org/elasticsearch/action/support/replication/BroadcastReplicationTests.java @@ -142,7 +142,7 @@ public class BroadcastReplicationTests extends ESTestCase { public void testResultCombine() throws InterruptedException, ExecutionException, IOException { final String index = "test"; - int numShards = randomInt(3); + int numShards = 1 + randomInt(3); setState(clusterService, stateWithAssignedPrimariesAndOneReplica(index, numShards)); logger.debug("--> using initial state:\n{}", clusterService.state().prettyPrint()); Future<BroadcastResponse> response = (broadcastReplicationAction.execute(new DummyBroadcastRequest().indices(index))); diff --git a/core/src/test/java/org/elasticsearch/action/support/replication/ClusterStateCreationUtils.java b/core/src/test/java/org/elasticsearch/action/support/replication/ClusterStateCreationUtils.java index 6b38d35c63..bfe2922906 100644 --- a/core/src/test/java/org/elasticsearch/action/support/replication/ClusterStateCreationUtils.java +++ b/core/src/test/java/org/elasticsearch/action/support/replication/ClusterStateCreationUtils.java @@ -45,6 +45,7 @@ import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS; import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_VERSION_CREATED; import static org.elasticsearch.test.ESTestCase.randomFrom; +import static org.elasticsearch.test.ESTestCase.randomInt; import static org.elasticsearch.test.ESTestCase.randomIntBetween; /** @@ -84,10 +85,11 @@ public class ClusterStateCreationUtils { } discoBuilder.localNodeId(newNode(0).id()); discoBuilder.masterNodeId(newNode(1).id()); // we need a non-local master to test shard failures + final int primaryTerm = randomInt(200); IndexMetaData indexMetaData = IndexMetaData.builder(index).settings(Settings.builder() .put(SETTING_VERSION_CREATED, Version.CURRENT) .put(SETTING_NUMBER_OF_SHARDS, 1).put(SETTING_NUMBER_OF_REPLICAS, numberOfReplicas) - .put(SETTING_CREATION_DATE, System.currentTimeMillis())).build(); + .put(SETTING_CREATION_DATE, System.currentTimeMillis())).primaryTerm(0, primaryTerm).build(); RoutingTable.Builder routing = new RoutingTable.Builder(); routing.addAsNew(indexMetaData); @@ -111,7 +113,8 @@ public class ClusterStateCreationUtils { } else { unassignedInfo = new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null); } - indexShardRoutingBuilder.addShard(TestShardRouting.newShardRouting(index, 0, primaryNode, relocatingNode, null, true, primaryState, unassignedInfo)); + indexShardRoutingBuilder.addShard(TestShardRouting.newShardRouting(index, 0, primaryNode, relocatingNode, null, true, + primaryState, unassignedInfo)); for (ShardRoutingState replicaState : replicaStates) { String replicaNode = null; @@ -152,7 +155,7 @@ public class ClusterStateCreationUtils { discoBuilder.masterNodeId(newNode(1).id()); // we need a non-local master to test shard failures IndexMetaData indexMetaData = IndexMetaData.builder(index).settings(Settings.builder() .put(SETTING_VERSION_CREATED, Version.CURRENT) - .put(SETTING_NUMBER_OF_SHARDS, 1).put(SETTING_NUMBER_OF_REPLICAS, 1) + .put(SETTING_NUMBER_OF_SHARDS, numberOfShards).put(SETTING_NUMBER_OF_REPLICAS, 1) .put(SETTING_CREATION_DATE, System.currentTimeMillis())).build(); ClusterState.Builder state = ClusterState.builder(new ClusterName("test")); state.nodes(discoBuilder); @@ -163,8 +166,10 @@ public class ClusterStateCreationUtils { routing.addAsNew(indexMetaData); final ShardId shardId = new ShardId(index, "_na_", i); IndexShardRoutingTable.Builder indexShardRoutingBuilder = new IndexShardRoutingTable.Builder(shardId); - indexShardRoutingBuilder.addShard(TestShardRouting.newShardRouting(index, i, newNode(0).id(), null, null, true, ShardRoutingState.STARTED, null)); - indexShardRoutingBuilder.addShard(TestShardRouting.newShardRouting(index, i, newNode(1).id(), null, null, false, ShardRoutingState.STARTED, null)); + indexShardRoutingBuilder.addShard(TestShardRouting.newShardRouting(index, i, newNode(0).id(), null, null, true, + ShardRoutingState.STARTED, null)); + indexShardRoutingBuilder.addShard(TestShardRouting.newShardRouting(index, i, newNode(1).id(), null, null, false, + ShardRoutingState.STARTED, null)); indexRoutingTableBuilder.addIndexShard(indexShardRoutingBuilder.build()); } state.routingTable(RoutingTable.builder().add(indexRoutingTableBuilder.build()).build()); @@ -229,12 +234,13 @@ public class ClusterStateCreationUtils { /** * Creates a cluster state where local node and master node can be specified + * * @param localNode node in allNodes that is the local node * @param masterNode node in allNodes that is the master node. Can be null if no master exists * @param allNodes all nodes in the cluster * @return cluster state */ - public static ClusterState state(DiscoveryNode localNode, DiscoveryNode masterNode, DiscoveryNode... allNodes) { + public static ClusterState state(DiscoveryNode localNode, DiscoveryNode masterNode, DiscoveryNode... allNodes) { DiscoveryNodes.Builder discoBuilder = DiscoveryNodes.builder(); for (DiscoveryNode node : allNodes) { discoBuilder.put(node); diff --git a/core/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationActionTests.java b/core/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationActionTests.java index 1fc94dcb53..446ad74e8b 100644 --- a/core/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationActionTests.java +++ b/core/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationActionTests.java @@ -630,11 +630,13 @@ public class TransportReplicationActionTests extends ESTestCase { final ShardIterator shardIt = shardRoutingTable.shardsIt(); final ShardId shardId = shardIt.shardId(); final Request request = new Request(shardId); + final long primaryTerm = randomInt(200); + request.primaryTerm(primaryTerm); final PlainActionFuture<Response> listener = new PlainActionFuture<>(); ReplicationTask task = maybeTask(); logger.debug("expecting [{}] assigned replicas, [{}] total shards. using state: \n{}", assignedReplicas, totalShards, clusterService.state().prettyPrint()); - TransportReplicationAction.IndexShardReference reference = getOrCreateIndexShardOperationsCounter(); + TransportReplicationAction.IndexShardReference reference = getOrCreateIndexShardOperationsCounter(0); ShardRouting primaryShard = state.getRoutingTable().shardRoutingTable(shardId).primaryShard(); indexShardRouting.set(primaryShard); @@ -767,6 +769,9 @@ public class TransportReplicationActionTests extends ESTestCase { } // all replicas have responded so the counter should be decreased again assertIndexShardCounter(1); + + // assert that nothing in the replica logic changes the primary term of the operation + assertThat(request.primaryTerm(), equalTo(primaryTerm)); } public void testCounterOnPrimary() throws Exception { @@ -989,7 +994,7 @@ public class TransportReplicationActionTests extends ESTestCase { /** * Returns testIndexShardOperationsCounter or initializes it if it was already created in this test run. */ - private synchronized TransportReplicationAction.IndexShardReference getOrCreateIndexShardOperationsCounter() { + private synchronized TransportReplicationAction.IndexShardReference getOrCreateIndexShardOperationsCounter(long primaryTerm) { count.incrementAndGet(); return new TransportReplicationAction.IndexShardReference() { @Override @@ -1010,6 +1015,11 @@ public class TransportReplicationActionTests extends ESTestCase { } @Override + public long opPrimaryTerm() { + return primaryTerm; + } + + @Override public void close() { count.decrementAndGet(); } @@ -1104,13 +1114,15 @@ public class TransportReplicationActionTests extends ESTestCase { return false; } + @Override protected IndexShardReference getIndexShardReferenceOnPrimary(ShardId shardId) { - return getOrCreateIndexShardOperationsCounter(); + final IndexMetaData indexMetaData = clusterService.state().metaData().index(shardId.getIndex()); + return getOrCreateIndexShardOperationsCounter(indexMetaData.primaryTerm(shardId.id())); } - protected IndexShardReference getIndexShardReferenceOnReplica(ShardId shardId) { - return getOrCreateIndexShardOperationsCounter(); + protected IndexShardReference getIndexShardReferenceOnReplica(ShardId shardId, long opPrimaryTerm) { + return getOrCreateIndexShardOperationsCounter(opPrimaryTerm); } } diff --git a/core/src/test/java/org/elasticsearch/bootstrap/BootstrapCheckTests.java b/core/src/test/java/org/elasticsearch/bootstrap/BootstrapCheckTests.java index 4816c6038e..27f26e7c8c 100644 --- a/core/src/test/java/org/elasticsearch/bootstrap/BootstrapCheckTests.java +++ b/core/src/test/java/org/elasticsearch/bootstrap/BootstrapCheckTests.java @@ -32,6 +32,8 @@ import java.util.Set; import java.util.concurrent.atomic.AtomicLong; import static org.hamcrest.CoreMatchers.containsString; +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.Matchers.not; public class BootstrapCheckTests extends ESTestCase { @@ -80,9 +82,9 @@ public class BootstrapCheckTests extends ESTestCase { public void testFileDescriptorLimitsThrowsOnInvalidLimit() { final IllegalArgumentException e = - expectThrows( - IllegalArgumentException.class, - () -> new BootstrapCheck.FileDescriptorCheck(-randomIntBetween(0, Integer.MAX_VALUE))); + expectThrows( + IllegalArgumentException.class, + () -> new BootstrapCheck.FileDescriptorCheck(-randomIntBetween(0, Integer.MAX_VALUE))); assertThat(e.getMessage(), containsString("limit must be positive but was")); } @@ -121,8 +123,8 @@ public class BootstrapCheckTests extends ESTestCase { fail("should have failed due to memory not being locked"); } catch (final RuntimeException e) { assertThat( - e.getMessage(), - containsString("memory locking requested for elasticsearch process but memory is not locked")); + e.getMessage(), + containsString("memory locking requested for elasticsearch process but memory is not locked")); } } else { // nothing should happen @@ -197,4 +199,12 @@ public class BootstrapCheckTests extends ESTestCase { assertTrue(BootstrapCheck.enforceLimits(settings)); } + public void testMinMasterNodes() { + boolean isSet = randomBoolean(); + BootstrapCheck.Check check = new BootstrapCheck.MinMasterNodesCheck(isSet); + assertThat(check.check(), not(equalTo(isSet))); + List<BootstrapCheck.Check> defaultChecks = BootstrapCheck.checks(Settings.EMPTY); + + expectThrows(RuntimeException.class, () -> BootstrapCheck.check(true, defaultChecks)); + } } diff --git a/core/src/test/java/org/elasticsearch/cluster/ClusterModuleTests.java b/core/src/test/java/org/elasticsearch/cluster/ClusterModuleTests.java index 4a930bc9c2..016f70f51b 100644 --- a/core/src/test/java/org/elasticsearch/cluster/ClusterModuleTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/ClusterModuleTests.java @@ -22,6 +22,7 @@ package org.elasticsearch.cluster; import org.elasticsearch.action.admin.indices.create.CreateIndexClusterStateUpdateRequest; import org.elasticsearch.cluster.metadata.IndexTemplateFilter; import org.elasticsearch.cluster.metadata.IndexTemplateMetaData; +import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.routing.RoutingNode; import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.cluster.routing.allocation.FailedRerouteAllocation; @@ -39,6 +40,8 @@ import org.elasticsearch.common.settings.Setting.Property; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsModule; +import java.util.HashMap; +import java.util.Map; public class ClusterModuleTests extends ModuleTestCase { public static class FakeAllocationDecider extends AllocationDecider { @@ -52,6 +55,11 @@ public class ClusterModuleTests extends ModuleTestCase { public boolean allocate(RoutingAllocation allocation) { return false; } + + @Override + public Map<DiscoveryNode, Float> weighShard(RoutingAllocation allocation, ShardRouting shard) { + return new HashMap<>(); + } } static class FakeIndexTemplateFilter implements IndexTemplateFilter { diff --git a/core/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetaDataTests.java b/core/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetaDataTests.java index 5886158506..f7e8b18196 100644 --- a/core/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetaDataTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetaDataTests.java @@ -41,11 +41,14 @@ public class ToAndFromJsonMetaDataTests extends ESTestCase { .put(IndexMetaData.builder("test1") .settings(settings(Version.CURRENT)) .numberOfShards(1) - .numberOfReplicas(2)) + .numberOfReplicas(2) + .primaryTerm(0, 1)) .put(IndexMetaData.builder("test2") .settings(settings(Version.CURRENT).put("setting1", "value1").put("setting2", "value2")) .numberOfShards(2) - .numberOfReplicas(3)) + .numberOfReplicas(3) + .primaryTerm(0, 2) + .primaryTerm(1, 2)) .put(IndexMetaData.builder("test3") .settings(settings(Version.CURRENT)) .numberOfShards(1) @@ -112,15 +115,15 @@ public class ToAndFromJsonMetaDataTests extends ESTestCase { .putAlias(newAliasMetaDataBuilder("alias1").filter(ALIAS_FILTER1)) .putAlias(newAliasMetaDataBuilder("alias2")) .putAlias(newAliasMetaDataBuilder("alias4").filter(ALIAS_FILTER2))) - .put(IndexTemplateMetaData.builder("foo") - .template("bar") - .order(1) - .settings(settingsBuilder() - .put("setting1", "value1") - .put("setting2", "value2")) - .putAlias(newAliasMetaDataBuilder("alias-bar1")) - .putAlias(newAliasMetaDataBuilder("alias-bar2").filter("{\"term\":{\"user\":\"kimchy\"}}")) - .putAlias(newAliasMetaDataBuilder("alias-bar3").routing("routing-bar"))) + .put(IndexTemplateMetaData.builder("foo") + .template("bar") + .order(1) + .settings(settingsBuilder() + .put("setting1", "value1") + .put("setting2", "value2")) + .putAlias(newAliasMetaDataBuilder("alias-bar1")) + .putAlias(newAliasMetaDataBuilder("alias-bar2").filter("{\"term\":{\"user\":\"kimchy\"}}")) + .putAlias(newAliasMetaDataBuilder("alias-bar3").routing("routing-bar"))) .put(IndexMetaData.builder("test12") .settings(settings(Version.CURRENT) .put("setting1", "value1") @@ -133,15 +136,15 @@ public class ToAndFromJsonMetaDataTests extends ESTestCase { .putAlias(newAliasMetaDataBuilder("alias1").filter(ALIAS_FILTER1)) .putAlias(newAliasMetaDataBuilder("alias2")) .putAlias(newAliasMetaDataBuilder("alias4").filter(ALIAS_FILTER2))) - .put(IndexTemplateMetaData.builder("foo") - .template("bar") - .order(1) - .settings(settingsBuilder() - .put("setting1", "value1") - .put("setting2", "value2")) - .putAlias(newAliasMetaDataBuilder("alias-bar1")) - .putAlias(newAliasMetaDataBuilder("alias-bar2").filter("{\"term\":{\"user\":\"kimchy\"}}")) - .putAlias(newAliasMetaDataBuilder("alias-bar3").routing("routing-bar"))) + .put(IndexTemplateMetaData.builder("foo") + .template("bar") + .order(1) + .settings(settingsBuilder() + .put("setting1", "value1") + .put("setting2", "value2")) + .putAlias(newAliasMetaDataBuilder("alias-bar1")) + .putAlias(newAliasMetaDataBuilder("alias-bar2").filter("{\"term\":{\"user\":\"kimchy\"}}")) + .putAlias(newAliasMetaDataBuilder("alias-bar3").routing("routing-bar"))) .build(); String metaDataSource = MetaData.Builder.toXContent(metaData); @@ -150,6 +153,7 @@ public class ToAndFromJsonMetaDataTests extends ESTestCase { MetaData parsedMetaData = MetaData.Builder.fromXContent(XContentFactory.xContent(XContentType.JSON).createParser(metaDataSource)); IndexMetaData indexMetaData = parsedMetaData.index("test1"); + assertThat(indexMetaData.primaryTerm(0), equalTo(1L)); assertThat(indexMetaData.getNumberOfShards(), equalTo(1)); assertThat(indexMetaData.getNumberOfReplicas(), equalTo(2)); assertThat(indexMetaData.getCreationDate(), equalTo(-1L)); @@ -159,6 +163,8 @@ public class ToAndFromJsonMetaDataTests extends ESTestCase { indexMetaData = parsedMetaData.index("test2"); assertThat(indexMetaData.getNumberOfShards(), equalTo(2)); assertThat(indexMetaData.getNumberOfReplicas(), equalTo(3)); + assertThat(indexMetaData.primaryTerm(0), equalTo(2L)); + assertThat(indexMetaData.primaryTerm(1), equalTo(2L)); assertThat(indexMetaData.getCreationDate(), equalTo(-1L)); assertThat(indexMetaData.getSettings().getAsMap().size(), equalTo(5)); assertThat(indexMetaData.getSettings().get("setting1"), equalTo("value1")); diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/PrimaryTermsTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/PrimaryTermsTests.java new file mode 100644 index 0000000000..d9b74621cc --- /dev/null +++ b/core/src/test/java/org/elasticsearch/cluster/routing/PrimaryTermsTests.java @@ -0,0 +1,241 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.cluster.routing; + +import org.elasticsearch.Version; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.health.ClusterStateHealth; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.cluster.node.DiscoveryNodes; +import org.elasticsearch.cluster.node.DiscoveryNodes.Builder; +import org.elasticsearch.cluster.routing.allocation.AllocationService; +import org.elasticsearch.cluster.routing.allocation.FailedRerouteAllocation; +import org.elasticsearch.cluster.routing.allocation.RoutingAllocation; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.test.ESAllocationTestCase; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import static org.elasticsearch.cluster.routing.ShardRoutingState.INITIALIZING; +import static org.elasticsearch.common.settings.Settings.settingsBuilder; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; + +public class PrimaryTermsTests extends ESAllocationTestCase { + + private static final String TEST_INDEX_1 = "test1"; + private static final String TEST_INDEX_2 = "test2"; + private RoutingTable testRoutingTable; + private int numberOfShards; + private int numberOfReplicas; + private final static Settings DEFAULT_SETTINGS = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build(); + private AllocationService allocationService; + private ClusterState clusterState; + + private final Map<String, long[]> primaryTermsPerIndex = new HashMap<>(); + + @Override + public void setUp() throws Exception { + super.setUp(); + this.allocationService = createAllocationService(settingsBuilder() + .put("cluster.routing.allocation.node_concurrent_recoveries", Integer.MAX_VALUE) // don't limit recoveries + .put("cluster.routing.allocation.node_initial_primaries_recoveries", Integer.MAX_VALUE) + .build()); + this.numberOfShards = randomIntBetween(1, 5); + this.numberOfReplicas = randomIntBetween(1, 5); + logger.info("Setup test with " + this.numberOfShards + " shards and " + this.numberOfReplicas + " replicas."); + this.primaryTermsPerIndex.clear(); + MetaData metaData = MetaData.builder() + .put(createIndexMetaData(TEST_INDEX_1)) + .put(createIndexMetaData(TEST_INDEX_2)) + .build(); + + this.testRoutingTable = new RoutingTable.Builder() + .add(new IndexRoutingTable.Builder(metaData.index(TEST_INDEX_1).getIndex()).initializeAsNew(metaData.index(TEST_INDEX_1)) + .build()) + .add(new IndexRoutingTable.Builder(metaData.index(TEST_INDEX_2).getIndex()).initializeAsNew(metaData.index(TEST_INDEX_2)) + .build()) + .build(); + + this.clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT).metaData(metaData) + .routingTable(testRoutingTable).build(); + } + + /** + * puts primary shard routings into initializing state + */ + private void initPrimaries() { + logger.info("adding " + (this.numberOfReplicas + 1) + " nodes and performing rerouting"); + Builder discoBuilder = DiscoveryNodes.builder(); + for (int i = 0; i < this.numberOfReplicas + 1; i++) { + discoBuilder = discoBuilder.put(newNode("node" + i)); + } + this.clusterState = ClusterState.builder(clusterState).nodes(discoBuilder).build(); + RoutingAllocation.Result rerouteResult = allocationService.reroute(clusterState, "reroute"); + this.testRoutingTable = rerouteResult.routingTable(); + assertThat(rerouteResult.changed(), is(true)); + applyRerouteResult(rerouteResult); + primaryTermsPerIndex.keySet().forEach(this::incrementPrimaryTerm); + } + + private void incrementPrimaryTerm(String index) { + final long[] primaryTerms = primaryTermsPerIndex.get(index); + for (int i = 0; i < primaryTerms.length; i++) { + primaryTerms[i]++; + } + } + + private void incrementPrimaryTerm(String index, int shard) { + primaryTermsPerIndex.get(index)[shard]++; + } + + private boolean startInitializingShards(String index) { + this.clusterState = ClusterState.builder(clusterState).routingTable(this.testRoutingTable).build(); + final List<ShardRouting> startedShards = this.clusterState.getRoutingNodes().shardsWithState(index, INITIALIZING); + logger.info("start primary shards for index [{}]: {} ", index, startedShards); + RoutingAllocation.Result rerouteResult = allocationService.applyStartedShards(this.clusterState, startedShards); + applyRerouteResult(rerouteResult); + return rerouteResult.changed(); + } + + private void applyRerouteResult(RoutingAllocation.Result rerouteResult) { + ClusterState previousClusterState = this.clusterState; + ClusterState newClusterState = ClusterState.builder(previousClusterState).routingResult(rerouteResult).build(); + ClusterState.Builder builder = ClusterState.builder(newClusterState).incrementVersion(); + if (previousClusterState.routingTable() != newClusterState.routingTable()) { + builder.routingTable(RoutingTable.builder(newClusterState.routingTable()).version(newClusterState.routingTable().version() + 1) + .build()); + } + if (previousClusterState.metaData() != newClusterState.metaData()) { + builder.metaData(MetaData.builder(newClusterState.metaData()).version(newClusterState.metaData().version() + 1)); + } + this.clusterState = builder.build(); + this.testRoutingTable = rerouteResult.routingTable(); + final ClusterStateHealth clusterHealth = new ClusterStateHealth(clusterState); + logger.info("applied reroute. active shards: p [{}], t [{}], init shards: [{}], relocating: [{}]", + clusterHealth.getActivePrimaryShards(), clusterHealth.getActiveShards(), + clusterHealth.getInitializingShards(), clusterHealth.getRelocatingShards()); + } + + private void failSomePrimaries(String index) { + this.clusterState = ClusterState.builder(clusterState).routingTable(this.testRoutingTable).build(); + final IndexRoutingTable indexShardRoutingTable = testRoutingTable.index(index); + Set<Integer> shardIdsToFail = new HashSet<>(); + for (int i = 1 + randomInt(numberOfShards - 1); i > 0; i--) { + shardIdsToFail.add(randomInt(numberOfShards - 1)); + } + logger.info("failing primary shards {} for index [{}]", shardIdsToFail, index); + List<FailedRerouteAllocation.FailedShard> failedShards = new ArrayList<>(); + for (int shard : shardIdsToFail) { + failedShards.add(new FailedRerouteAllocation.FailedShard(indexShardRoutingTable.shard(shard).primaryShard(), "test", null)); + incrementPrimaryTerm(index, shard); // the primary failure should increment the primary term; + } + RoutingAllocation.Result rerouteResult = allocationService.applyFailedShards(this.clusterState, failedShards); + applyRerouteResult(rerouteResult); + } + + private void addNodes() { + DiscoveryNodes.Builder nodesBuilder = DiscoveryNodes.builder(clusterState.nodes()); + final int newNodes = randomInt(10); + logger.info("adding [{}] nodes", newNodes); + for (int i = 0; i < newNodes; i++) { + nodesBuilder.put(newNode("extra_" + i)); + } + this.clusterState = ClusterState.builder(clusterState).nodes(nodesBuilder).build(); + RoutingAllocation.Result rerouteResult = allocationService.reroute(this.clusterState, "nodes added"); + applyRerouteResult(rerouteResult); + + } + + private IndexMetaData.Builder createIndexMetaData(String indexName) { + primaryTermsPerIndex.put(indexName, new long[numberOfShards]); + final IndexMetaData.Builder builder = new IndexMetaData.Builder(indexName) + .settings(DEFAULT_SETTINGS) + .numberOfReplicas(this.numberOfReplicas) + .numberOfShards(this.numberOfShards); + for (int i = 0; i < numberOfShards; i++) { + builder.primaryTerm(i, randomInt(200)); + primaryTermsPerIndex.get(indexName)[i] = builder.primaryTerm(i); + } + return builder; + } + + private void assertAllPrimaryTerm() { + primaryTermsPerIndex.keySet().forEach(this::assertPrimaryTerm); + } + + private void assertPrimaryTerm(String index) { + final long[] terms = primaryTermsPerIndex.get(index); + final IndexMetaData indexMetaData = clusterState.metaData().index(index); + for (IndexShardRoutingTable shardRoutingTable : this.testRoutingTable.index(index)) { + final int shard = shardRoutingTable.shardId().id(); + assertThat("primary term mismatch between indexMetaData of [" + index + "] and shard [" + shard + "]'s routing", + indexMetaData.primaryTerm(shard), equalTo(terms[shard])); + } + } + + public void testPrimaryTermMetaDataSync() { + assertAllPrimaryTerm(); + + initPrimaries(); + assertAllPrimaryTerm(); + + startInitializingShards(TEST_INDEX_1); + assertAllPrimaryTerm(); + + startInitializingShards(TEST_INDEX_2); + assertAllPrimaryTerm(); + + // now start all replicas too + startInitializingShards(TEST_INDEX_1); + startInitializingShards(TEST_INDEX_2); + assertAllPrimaryTerm(); + + // relocations shouldn't change much + addNodes(); + assertAllPrimaryTerm(); + boolean changed = true; + while (changed) { + changed = startInitializingShards(TEST_INDEX_1); + assertAllPrimaryTerm(); + changed |= startInitializingShards(TEST_INDEX_2); + assertAllPrimaryTerm(); + } + + // primary promotion + failSomePrimaries(TEST_INDEX_1); + assertAllPrimaryTerm(); + + // stablize cluster + changed = true; + while (changed) { + changed = startInitializingShards(TEST_INDEX_1); + assertAllPrimaryTerm(); + changed |= startInitializingShards(TEST_INDEX_2); + assertAllPrimaryTerm(); + } + } +} diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/BalanceConfigurationTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/BalanceConfigurationTests.java index 56a66b52d6..f1495bb5e7 100644 --- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/BalanceConfigurationTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/BalanceConfigurationTests.java @@ -33,6 +33,7 @@ import org.elasticsearch.cluster.routing.RoutingNodes; import org.elasticsearch.cluster.routing.RoutingTable; import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.cluster.routing.ShardRoutingState; +import org.elasticsearch.cluster.routing.allocation.RoutingAllocation; import org.elasticsearch.cluster.routing.allocation.allocator.BalancedShardsAllocator; import org.elasticsearch.cluster.routing.allocation.allocator.ShardsAllocator; import org.elasticsearch.cluster.routing.allocation.decider.ClusterRebalanceAllocationDecider; @@ -44,6 +45,8 @@ import org.elasticsearch.test.ESAllocationTestCase; import org.elasticsearch.test.gateway.NoopGatewayAllocator; import org.hamcrest.Matchers; +import java.util.HashMap; +import java.util.Map; import static org.elasticsearch.cluster.routing.ShardRoutingState.INITIALIZING; import static org.elasticsearch.cluster.routing.ShardRoutingState.STARTED; import static org.elasticsearch.common.settings.Settings.settingsBuilder; @@ -313,6 +316,9 @@ public class BalanceConfigurationTests extends ESAllocationTestCase { new ClusterSettings(Settings.Builder.EMPTY_SETTINGS, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), getRandom()), NoopGatewayAllocator.INSTANCE, new ShardsAllocator() { + public Map<DiscoveryNode, Float> weighShard(RoutingAllocation allocation, ShardRouting shard) { + return new HashMap<DiscoveryNode, Float>(); + } /* * // this allocator tries to rebuild this scenario where a rebalance is * // triggered solely by the primary overload on node [1] where a shard diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/CatAllocationTestCase.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/CatAllocationTestCase.java index be40351019..0bd8441312 100644 --- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/CatAllocationTestCase.java +++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/CatAllocationTestCase.java @@ -30,7 +30,6 @@ import org.elasticsearch.cluster.routing.RoutingTable; import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.cluster.routing.ShardRoutingState; import org.elasticsearch.cluster.routing.TestShardRouting; -import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.test.ESAllocationTestCase; import java.io.BufferedReader; diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/PrimaryElectionRoutingTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/PrimaryElectionRoutingTests.java index 7e59ab8a6b..b18ee32ff5 100644 --- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/PrimaryElectionRoutingTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/PrimaryElectionRoutingTests.java @@ -58,29 +58,31 @@ public class PrimaryElectionRoutingTests extends ESAllocationTestCase { ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT).metaData(metaData).routingTable(routingTable).build(); logger.info("Adding two nodes and performing rerouting"); - clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().put(newNode("node1")).put(newNode("node2"))).build(); - RoutingTable prevRoutingTable = routingTable; - routingTable = strategy.reroute(clusterState, "reroute").routingTable(); - clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); + clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().put(newNode("node1"))).build(); + RoutingAllocation.Result result = strategy.reroute(clusterState, "reroute"); + clusterState = ClusterState.builder(clusterState).routingResult(result).build(); + + clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()).put(newNode("node2"))).build(); + result = strategy.reroute(clusterState, "reroute"); + clusterState = ClusterState.builder(clusterState).routingResult(result).build(); logger.info("Start the primary shard (on node1)"); RoutingNodes routingNodes = clusterState.getRoutingNodes(); - prevRoutingTable = routingTable; - routingTable = strategy.applyStartedShards(clusterState, routingNodes.node("node1").shardsWithState(INITIALIZING)).routingTable(); - clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); + result = strategy.applyStartedShards(clusterState, routingNodes.node("node1").shardsWithState(INITIALIZING)); + clusterState = ClusterState.builder(clusterState).routingResult(result).build(); logger.info("Start the backup shard (on node2)"); routingNodes = clusterState.getRoutingNodes(); - prevRoutingTable = routingTable; - routingTable = strategy.applyStartedShards(clusterState, routingNodes.node("node2").shardsWithState(INITIALIZING)).routingTable(); - clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); + result = strategy.applyStartedShards(clusterState, routingNodes.node("node2").shardsWithState(INITIALIZING)); + clusterState = ClusterState.builder(clusterState).routingResult(result).build(); logger.info("Adding third node and reroute and kill first node"); clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()).put(newNode("node3")).remove("node1")).build(); - prevRoutingTable = routingTable; - routingTable = strategy.reroute(clusterState, "reroute").routingTable(); - clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); + RoutingTable prevRoutingTable = clusterState.routingTable(); + result = strategy.reroute(clusterState, "reroute"); + clusterState = ClusterState.builder(clusterState).routingResult(result).build(); routingNodes = clusterState.getRoutingNodes(); + routingTable = clusterState.routingTable(); assertThat(prevRoutingTable != routingTable, equalTo(true)); assertThat(routingTable.index("test").shards().size(), equalTo(1)); @@ -89,6 +91,7 @@ public class PrimaryElectionRoutingTests extends ESAllocationTestCase { assertThat(routingNodes.node("node3").numberOfShardsWithState(INITIALIZING), equalTo(1)); // verify where the primary is assertThat(routingTable.index("test").shard(0).primaryShard().currentNodeId(), equalTo("node2")); + assertThat(clusterState.metaData().index("test").primaryTerm(0), equalTo(2L)); assertThat(routingTable.index("test").shard(0).replicaShards().get(0).currentNodeId(), equalTo("node3")); } @@ -110,16 +113,18 @@ public class PrimaryElectionRoutingTests extends ESAllocationTestCase { logger.info("Adding two nodes and performing rerouting"); clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().put(newNode("node1")).put(newNode("node2"))).build(); RoutingAllocation.Result rerouteResult = allocation.reroute(clusterState, "reroute"); - clusterState = ClusterState.builder(clusterState).routingTable(rerouteResult.routingTable()).build(); + clusterState = ClusterState.builder(clusterState).routingResult(rerouteResult).build(); logger.info("Start the primary shards"); RoutingNodes routingNodes = clusterState.getRoutingNodes(); rerouteResult = allocation.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING)); - clusterState = ClusterState.builder(clusterState).routingTable(rerouteResult.routingTable()).build(); + clusterState = ClusterState.builder(clusterState).routingResult(rerouteResult).build(); routingNodes = clusterState.getRoutingNodes(); assertThat(routingNodes.shardsWithState(STARTED).size(), equalTo(2)); assertThat(routingNodes.shardsWithState(INITIALIZING).size(), equalTo(2)); + assertThat(clusterState.metaData().index("test").primaryTerm(0), equalTo(1L)); + assertThat(clusterState.metaData().index("test").primaryTerm(1), equalTo(1L)); // now, fail one node, while the replica is initializing, and it also holds a primary logger.info("--> fail node with primary"); @@ -129,12 +134,13 @@ public class PrimaryElectionRoutingTests extends ESAllocationTestCase { .put(newNode(nodeIdRemaining)) ).build(); rerouteResult = allocation.reroute(clusterState, "reroute"); - clusterState = ClusterState.builder(clusterState).routingTable(rerouteResult.routingTable()).build(); + clusterState = ClusterState.builder(clusterState).routingResult(rerouteResult).build(); routingNodes = clusterState.getRoutingNodes(); assertThat(routingNodes.shardsWithState(STARTED).size(), equalTo(1)); assertThat(routingNodes.shardsWithState(INITIALIZING).size(), equalTo(1)); assertThat(routingNodes.node(nodeIdRemaining).shardsWithState(INITIALIZING).get(0).primary(), equalTo(true)); + assertThat(clusterState.metaData().index("test").primaryTerm(0), equalTo(2L)); } } diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ShardStateIT.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ShardStateIT.java new file mode 100644 index 0000000000..38c1575042 --- /dev/null +++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ShardStateIT.java @@ -0,0 +1,80 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.cluster.routing.allocation; + +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.health.ClusterHealthStatus; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.index.IndexService; +import org.elasticsearch.index.shard.IndexShard; +import org.elasticsearch.indices.IndicesService; +import org.elasticsearch.test.ESIntegTestCase; + +import static org.hamcrest.Matchers.equalTo; + +public class ShardStateIT extends ESIntegTestCase { + + public void testPrimaryFailureIncreasesTerm() throws Exception { + internalCluster().ensureAtLeastNumDataNodes(2); + prepareCreate("test").setSettings(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 2, IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1).get(); + ensureGreen(); + assertPrimaryTerms(1, 1); + + logger.info("--> disabling allocation to capture shard failure"); + disableAllocation("test"); + + ClusterState state = client().admin().cluster().prepareState().get().getState(); + final int shard = randomBoolean() ? 0 : 1; + final String nodeId = state.routingTable().index("test").shard(shard).primaryShard().currentNodeId(); + final String node = state.nodes().get(nodeId).name(); + logger.info("--> failing primary of [{}] on node [{}]", shard, node); + IndicesService indicesService = internalCluster().getInstance(IndicesService.class, node); + indicesService.indexService(resolveIndex("test")).getShard(shard).failShard("simulated test failure", null); + + logger.info("--> waiting for a yellow index"); + assertBusy(() -> assertThat(client().admin().cluster().prepareHealth().get().getStatus(), equalTo(ClusterHealthStatus.YELLOW))); + + final long term0 = shard == 0 ? 2 : 1; + final long term1 = shard == 1 ? 2 : 1; + assertPrimaryTerms(term0, term1); + + logger.info("--> enabling allocation"); + enableAllocation("test"); + ensureGreen(); + assertPrimaryTerms(term0, term1); + } + + protected void assertPrimaryTerms(long term0, long term1) { + for (String node : internalCluster().getNodeNames()) { + logger.debug("--> asserting primary terms terms on [{}]", node); + ClusterState state = client(node).admin().cluster().prepareState().setLocal(true).get().getState(); + IndexMetaData metaData = state.metaData().index("test"); + assertThat(metaData.primaryTerm(0), equalTo(term0)); + assertThat(metaData.primaryTerm(1), equalTo(term1)); + IndicesService indicesService = internalCluster().getInstance(IndicesService.class, node); + IndexService indexService = indicesService.indexService(metaData.getIndex()); + if (indexService != null) { + for (IndexShard shard : indexService) { + assertThat("term mismatch for shard " + shard.shardId(), + shard.getPrimaryTerm(), equalTo(metaData.primaryTerm(shard.shardId().id()))); + } + } + } + } +} diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java index 928756fec0..260a33780a 100644 --- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java @@ -59,6 +59,7 @@ import static org.elasticsearch.cluster.routing.ShardRoutingState.RELOCATING; import static org.elasticsearch.cluster.routing.ShardRoutingState.STARTED; import static org.elasticsearch.cluster.routing.ShardRoutingState.UNASSIGNED; import static org.elasticsearch.common.settings.Settings.settingsBuilder; +import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.nullValue; @@ -794,7 +795,8 @@ public class DiskThresholdDeciderTests extends ESAllocationTestCase { fail("should not have been able to reroute the shard"); } catch (IllegalArgumentException e) { assertThat("can't allocated because there isn't enough room: " + e.getMessage(), - e.getMessage().contains("more than allowed [70.0%] used disk on node, free: [26.0%]"), equalTo(true)); + e.getMessage(), + containsString("the node is above the low watermark and has more than allowed [70.0%] used disk, free: [26.0%]")); } } diff --git a/core/src/test/java/org/elasticsearch/common/geo/builders/AbstractShapeBuilderTestCase.java b/core/src/test/java/org/elasticsearch/common/geo/builders/AbstractShapeBuilderTestCase.java index 9311db44da..63f6ecd0e6 100644 --- a/core/src/test/java/org/elasticsearch/common/geo/builders/AbstractShapeBuilderTestCase.java +++ b/core/src/test/java/org/elasticsearch/common/geo/builders/AbstractShapeBuilderTestCase.java @@ -50,15 +50,7 @@ public abstract class AbstractShapeBuilderTestCase<SB extends ShapeBuilder> exte public static void init() { if (namedWriteableRegistry == null) { namedWriteableRegistry = new NamedWriteableRegistry(); - namedWriteableRegistry.registerPrototype(ShapeBuilder.class, PointBuilder.PROTOTYPE); - namedWriteableRegistry.registerPrototype(ShapeBuilder.class, CircleBuilder.PROTOTYPE); - namedWriteableRegistry.registerPrototype(ShapeBuilder.class, EnvelopeBuilder.PROTOTYPE); - namedWriteableRegistry.registerPrototype(ShapeBuilder.class, MultiPointBuilder.PROTOTYPE); - namedWriteableRegistry.registerPrototype(ShapeBuilder.class, LineStringBuilder.PROTOTYPE); - namedWriteableRegistry.registerPrototype(ShapeBuilder.class, MultiLineStringBuilder.PROTOTYPE); - namedWriteableRegistry.registerPrototype(ShapeBuilder.class, PolygonBuilder.PROTOTYPE); - namedWriteableRegistry.registerPrototype(ShapeBuilder.class, MultiPolygonBuilder.PROTOTYPE); - namedWriteableRegistry.registerPrototype(ShapeBuilder.class, GeometryCollectionBuilder.PROTOTYPE); + ShapeBuilders.register(namedWriteableRegistry); } } @@ -146,8 +138,7 @@ public abstract class AbstractShapeBuilderTestCase<SB extends ShapeBuilder> exte try (BytesStreamOutput output = new BytesStreamOutput()) { original.writeTo(output); try (StreamInput in = new NamedWriteableAwareStreamInput(StreamInput.wrap(output.bytes()), namedWriteableRegistry)) { - ShapeBuilder prototype = (ShapeBuilder) namedWriteableRegistry.getPrototype(ShapeBuilder.class, original.getWriteableName()); - return prototype.readFrom(in); + return namedWriteableRegistry.getReader(ShapeBuilder.class, original.getWriteableName()).read(in); } } } diff --git a/core/src/test/java/org/elasticsearch/common/geo/builders/EnvelopeBuilderTests.java b/core/src/test/java/org/elasticsearch/common/geo/builders/EnvelopeBuilderTests.java index 881db868ef..c2730f91df 100644 --- a/core/src/test/java/org/elasticsearch/common/geo/builders/EnvelopeBuilderTests.java +++ b/core/src/test/java/org/elasticsearch/common/geo/builders/EnvelopeBuilderTests.java @@ -19,30 +19,21 @@ package org.elasticsearch.common.geo.builders; -import org.locationtech.spatial4j.shape.Rectangle; import com.vividsolutions.jts.geom.Coordinate; + import org.elasticsearch.test.geo.RandomShapeGenerator; +import org.locationtech.spatial4j.shape.Rectangle; import java.io.IOException; -import static org.hamcrest.Matchers.equalTo; - public class EnvelopeBuilderTests extends AbstractShapeBuilderTestCase<EnvelopeBuilder> { public void testInvalidConstructorArgs() { - try { - new EnvelopeBuilder(null, new Coordinate(1.0, -1.0)); - fail("Exception expected"); - } catch (NullPointerException e) { - assertThat("topLeft of envelope cannot be null", equalTo(e.getMessage())); - } - - try { - new EnvelopeBuilder(new Coordinate(1.0, -1.0), null); - fail("Exception expected"); - } catch (NullPointerException e) { - assertThat("bottomRight of envelope cannot be null", equalTo(e.getMessage())); - } + NullPointerException e; + e = expectThrows(NullPointerException.class, () -> new EnvelopeBuilder(null, new Coordinate(1.0, -1.0))); + assertEquals("topLeft of envelope cannot be null", e.getMessage()); + e = expectThrows(NullPointerException.class, () -> new EnvelopeBuilder(new Coordinate(1.0, -1.0), null)); + assertEquals("bottomRight of envelope cannot be null", e.getMessage()); } @Override @@ -60,16 +51,21 @@ public class EnvelopeBuilderTests extends AbstractShapeBuilderTestCase<EnvelopeB // move one corner to the middle of original switch (randomIntBetween(0, 3)) { case 0: - mutation = new EnvelopeBuilder(new Coordinate(randomDoubleBetween(-180.0, original.bottomRight().x, true), original.topLeft().y), original.bottomRight()); + mutation = new EnvelopeBuilder( + new Coordinate(randomDoubleBetween(-180.0, original.bottomRight().x, true), original.topLeft().y), + original.bottomRight()); break; case 1: - mutation = new EnvelopeBuilder(new Coordinate(original.topLeft().x, randomDoubleBetween(original.bottomRight().y, 90.0, true)), original.bottomRight()); + mutation = new EnvelopeBuilder(new Coordinate(original.topLeft().x, randomDoubleBetween(original.bottomRight().y, 90.0, true)), + original.bottomRight()); break; case 2: - mutation = new EnvelopeBuilder(original.topLeft(), new Coordinate(randomDoubleBetween(original.topLeft().x, 180.0, true), original.bottomRight().y)); + mutation = new EnvelopeBuilder(original.topLeft(), + new Coordinate(randomDoubleBetween(original.topLeft().x, 180.0, true), original.bottomRight().y)); break; case 3: - mutation = new EnvelopeBuilder(original.topLeft(), new Coordinate(original.bottomRight().x, randomDoubleBetween(-90.0, original.topLeft().y, true))); + mutation = new EnvelopeBuilder(original.topLeft(), + new Coordinate(original.bottomRight().x, randomDoubleBetween(-90.0, original.topLeft().y, true))); break; } return mutation; diff --git a/core/src/test/java/org/elasticsearch/common/geo/builders/LineStringBuilderTests.java b/core/src/test/java/org/elasticsearch/common/geo/builders/LineStringBuilderTests.java index f6fcf8449d..e96c35287c 100644 --- a/core/src/test/java/org/elasticsearch/common/geo/builders/LineStringBuilderTests.java +++ b/core/src/test/java/org/elasticsearch/common/geo/builders/LineStringBuilderTests.java @@ -27,31 +27,15 @@ import org.elasticsearch.test.geo.RandomShapeGenerator.ShapeType; import java.io.IOException; import java.util.List; -import static org.hamcrest.Matchers.equalTo; - public class LineStringBuilderTests extends AbstractShapeBuilderTestCase<LineStringBuilder> { public void testInvalidConstructorArgs() { - try { - new LineStringBuilder((List<Coordinate>) null); - fail("Exception expected"); - } catch (IllegalArgumentException e) { - assertThat("cannot create point collection with empty set of points", equalTo(e.getMessage())); - } - - try { - new LineStringBuilder(new CoordinatesBuilder()); - fail("Exception expected"); - } catch (IllegalArgumentException e) { - assertThat("cannot create point collection with empty set of points", equalTo(e.getMessage())); - } - - try { - new LineStringBuilder(new CoordinatesBuilder().coordinate(0.0, 0.0)); - fail("Exception expected"); - } catch (IllegalArgumentException e) { - assertThat("invalid number of points in LineString (found [1] - must be >= 2)", equalTo(e.getMessage())); - } + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> new LineStringBuilder((List<Coordinate>) null)); + assertEquals("cannot create point collection with empty set of points", e.getMessage()); + e = expectThrows(IllegalArgumentException.class, () -> new LineStringBuilder(new CoordinatesBuilder())); + assertEquals("cannot create point collection with empty set of points", e.getMessage()); + e = expectThrows(IllegalArgumentException.class, () -> new LineStringBuilder(new CoordinatesBuilder().coordinate(0.0, 0.0))); + assertEquals("invalid number of points in LineString (found [1] - must be >= 2)", e.getMessage()); } @Override diff --git a/core/src/test/java/org/elasticsearch/common/geo/builders/MultiLineStringBuilderTests.java b/core/src/test/java/org/elasticsearch/common/geo/builders/MultiLineStringBuilderTests.java index 3c618fd369..925d177c57 100644 --- a/core/src/test/java/org/elasticsearch/common/geo/builders/MultiLineStringBuilderTests.java +++ b/core/src/test/java/org/elasticsearch/common/geo/builders/MultiLineStringBuilderTests.java @@ -68,9 +68,6 @@ public class MultiLineStringBuilderTests extends AbstractShapeBuilderTestCase<Mu } static MultiLineStringBuilder createRandomShape() { - if (true) { - return new MultiLineStringBuilder(); - } - return (MultiLineStringBuilder) RandomShapeGenerator.createShape(getRandom(), ShapeType.MULTILINESTRING); + return new MultiLineStringBuilder(); } } diff --git a/core/src/test/java/org/elasticsearch/common/geo/builders/MultiPointBuilderTests.java b/core/src/test/java/org/elasticsearch/common/geo/builders/MultiPointBuilderTests.java index 006064578e..ec2eb50bd3 100644 --- a/core/src/test/java/org/elasticsearch/common/geo/builders/MultiPointBuilderTests.java +++ b/core/src/test/java/org/elasticsearch/common/geo/builders/MultiPointBuilderTests.java @@ -20,29 +20,20 @@ package org.elasticsearch.common.geo.builders; import com.vividsolutions.jts.geom.Coordinate; + import org.elasticsearch.test.geo.RandomShapeGenerator; import org.elasticsearch.test.geo.RandomShapeGenerator.ShapeType; import java.io.IOException; - -import static org.hamcrest.Matchers.equalTo; +import java.util.List; public class MultiPointBuilderTests extends AbstractShapeBuilderTestCase<MultiPointBuilder> { public void testInvalidBuilderException() { - try { - new MultiPointBuilder(null); - fail("IllegalArgumentException expected"); - } catch (IllegalArgumentException e) { - assertThat("cannot create point collection with empty set of points", equalTo(e.getMessage())); - } - - try { - new MultiPointBuilder(new CoordinatesBuilder().build()); - fail("IllegalArgumentException expected"); - } catch (IllegalArgumentException e) { - assertThat("cannot create point collection with empty set of points", equalTo(e.getMessage())); - } + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> new MultiPointBuilder((List<Coordinate>) null)); + assertEquals("cannot create point collection with empty set of points", e.getMessage()); + e = expectThrows(IllegalArgumentException.class, () -> new MultiPointBuilder(new CoordinatesBuilder().build())); + assertEquals("cannot create point collection with empty set of points", e.getMessage()); // one point is minimum new MultiPointBuilder(new CoordinatesBuilder().coordinate(0.0, 0.0).build()); diff --git a/core/src/test/java/org/elasticsearch/common/geo/builders/PolygonBuilderTests.java b/core/src/test/java/org/elasticsearch/common/geo/builders/PolygonBuilderTests.java index 24e0bc8571..9a35690f94 100644 --- a/core/src/test/java/org/elasticsearch/common/geo/builders/PolygonBuilderTests.java +++ b/core/src/test/java/org/elasticsearch/common/geo/builders/PolygonBuilderTests.java @@ -80,7 +80,8 @@ public class PolygonBuilderTests extends AbstractShapeBuilderTestCase<PolygonBui * This is done so we don't have to expose a setter for orientation in the actual class */ private static PolygonBuilder polyWithOposingOrientation(PolygonBuilder pb) { - PolygonBuilder mutation = new PolygonBuilder(pb.shell(), pb.orientation() == Orientation.LEFT ? Orientation.RIGHT : Orientation.LEFT); + PolygonBuilder mutation = new PolygonBuilder(pb.shell(), + pb.orientation() == Orientation.LEFT ? Orientation.RIGHT : Orientation.LEFT); for (LineStringBuilder hole : pb.holes()) { mutation.hole(hole); } diff --git a/core/src/test/java/org/elasticsearch/common/io/stream/BytesStreamsTests.java b/core/src/test/java/org/elasticsearch/common/io/stream/BytesStreamsTests.java index 80bad2e1ec..7a2828c0a1 100644 --- a/core/src/test/java/org/elasticsearch/common/io/stream/BytesStreamsTests.java +++ b/core/src/test/java/org/elasticsearch/common/io/stream/BytesStreamsTests.java @@ -29,8 +29,10 @@ import java.io.IOException; import java.util.Objects; import static org.hamcrest.Matchers.closeTo; +import static org.hamcrest.Matchers.endsWith; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.startsWith; /** * Tests for {@link BytesStreamOutput} paging behaviour. @@ -301,7 +303,7 @@ public class BytesStreamsTests extends ESTestCase { public void testNamedWriteable() throws IOException { BytesStreamOutput out = new BytesStreamOutput(); NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry(); - namedWriteableRegistry.registerPrototype(BaseNamedWriteable.class, new TestNamedWriteable(null, null)); + namedWriteableRegistry.register(BaseNamedWriteable.class, TestNamedWriteable.NAME, TestNamedWriteable::new); TestNamedWriteable namedWriteableIn = new TestNamedWriteable(randomAsciiOfLengthBetween(1, 10), randomAsciiOfLengthBetween(1, 10)); out.writeNamedWriteable(namedWriteableIn); byte[] bytes = out.bytes().toBytes(); @@ -314,32 +316,25 @@ public class BytesStreamsTests extends ESTestCase { public void testNamedWriteableDuplicates() throws IOException { NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry(); - namedWriteableRegistry.registerPrototype(BaseNamedWriteable.class, new TestNamedWriteable(null, null)); - try { - namedWriteableRegistry.registerPrototype(BaseNamedWriteable.class, new TestNamedWriteable(null, null)); - fail("registerPrototype should have failed"); - } catch(IllegalArgumentException e) { - assertThat(e.getMessage(), equalTo("named writeable of type [" + TestNamedWriteable.class.getName() + "] with name [" + TestNamedWriteable.NAME + "] is already registered by type [" - + TestNamedWriteable.class.getName() + "] within category [" + BaseNamedWriteable.class.getName() + "]")); - } + namedWriteableRegistry.register(BaseNamedWriteable.class, TestNamedWriteable.NAME, TestNamedWriteable::new); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, + () -> namedWriteableRegistry.register(BaseNamedWriteable.class, TestNamedWriteable.NAME, TestNamedWriteable::new)); + assertThat(e.getMessage(), startsWith("named writeable [" + BaseNamedWriteable.class.getName() + "][" + TestNamedWriteable.NAME + + "] is already registered by [")); } public void testNamedWriteableUnknownCategory() throws IOException { BytesStreamOutput out = new BytesStreamOutput(); out.writeNamedWriteable(new TestNamedWriteable("test1", "test2")); StreamInput in = new NamedWriteableAwareStreamInput(StreamInput.wrap(out.bytes().toBytes()), new NamedWriteableRegistry()); - try { - //no named writeable registered with given name, can write but cannot read it back - in.readNamedWriteable(BaseNamedWriteable.class); - fail("read should have failed"); - } catch(IllegalArgumentException e) { - assertThat(e.getMessage(), equalTo("unknown named writeable category [" + BaseNamedWriteable.class.getName() + "]")); - } + //no named writeable registered with given name, can write but cannot read it back + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> in.readNamedWriteable(BaseNamedWriteable.class)); + assertThat(e.getMessage(), equalTo("unknown named writeable category [" + BaseNamedWriteable.class.getName() + "]")); } public void testNamedWriteableUnknownNamedWriteable() throws IOException { NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry(); - namedWriteableRegistry.registerPrototype(BaseNamedWriteable.class, new TestNamedWriteable(null, null)); + namedWriteableRegistry.register(BaseNamedWriteable.class, TestNamedWriteable.NAME, TestNamedWriteable::new); BytesStreamOutput out = new BytesStreamOutput(); out.writeNamedWriteable(new NamedWriteable() { @Override @@ -362,7 +357,7 @@ public class BytesStreamsTests extends ESTestCase { in.readNamedWriteable(BaseNamedWriteable.class); fail("read should have failed"); } catch(IllegalArgumentException e) { - assertThat(e.getMessage(), equalTo("unknown named writeable with name [unknown] within category [" + BaseNamedWriteable.class.getName() + "]")); + assertThat(e.getMessage(), equalTo("unknown named writeable [" + BaseNamedWriteable.class.getName() + "][unknown]")); } } @@ -379,6 +374,27 @@ public class BytesStreamsTests extends ESTestCase { } } + public void testNamedWriteableReaderReturnsNull() throws IOException { + BytesStreamOutput out = new BytesStreamOutput(); + NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry(); + namedWriteableRegistry.register(BaseNamedWriteable.class, TestNamedWriteable.NAME, (StreamInput in) -> null); + TestNamedWriteable namedWriteableIn = new TestNamedWriteable(randomAsciiOfLengthBetween(1, 10), randomAsciiOfLengthBetween(1, 10)); + out.writeNamedWriteable(namedWriteableIn); + byte[] bytes = out.bytes().toBytes(); + StreamInput in = new NamedWriteableAwareStreamInput(StreamInput.wrap(bytes), namedWriteableRegistry); + assertEquals(in.available(), bytes.length); + IOException e = expectThrows(IOException.class, () -> in.readNamedWriteable(BaseNamedWriteable.class)); + assertThat(e.getMessage(), endsWith("] returned null which is not allowed and probably means it screwed up the stream.")); + } + + public void testOptionalWriteableReaderReturnsNull() throws IOException { + BytesStreamOutput out = new BytesStreamOutput(); + out.writeOptionalWriteable(new TestNamedWriteable(randomAsciiOfLengthBetween(1, 10), randomAsciiOfLengthBetween(1, 10))); + StreamInput in = StreamInput.wrap(out.bytes().toBytes()); + IOException e = expectThrows(IOException.class, () -> in.readOptionalWriteable((StreamInput ignored) -> null)); + assertThat(e.getMessage(), endsWith("] returned null which is not allowed and probably means it screwed up the stream.")); + } + private static abstract class BaseNamedWriteable<T> implements NamedWriteable<T> { } @@ -395,6 +411,11 @@ public class BytesStreamsTests extends ESTestCase { this.field2 = field2; } + public TestNamedWriteable(StreamInput in) throws IOException { + field1 = in.readString(); + field2 = in.readString(); + } + @Override public String getWriteableName() { return NAME; @@ -407,11 +428,6 @@ public class BytesStreamsTests extends ESTestCase { } @Override - public TestNamedWriteable readFrom(StreamInput in) throws IOException { - return new TestNamedWriteable(in.readString(), in.readString()); - } - - @Override public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; diff --git a/core/src/test/java/org/elasticsearch/common/network/NetworkModuleTests.java b/core/src/test/java/org/elasticsearch/common/network/NetworkModuleTests.java index bd794f96da..4d8f3e6e58 100644 --- a/core/src/test/java/org/elasticsearch/common/network/NetworkModuleTests.java +++ b/core/src/test/java/org/elasticsearch/common/network/NetworkModuleTests.java @@ -191,23 +191,24 @@ public class NetworkModuleTests extends ModuleTestCase { Settings settings = Settings.EMPTY; NetworkModule module = new NetworkModule(new NetworkService(settings), settings, false, registry); - // Builtin prototype comes back - assertNotNull(registry.getPrototype(Task.Status.class, ReplicationTask.Status.PROTOTYPE.getWriteableName())); + // Builtin reader comes back + assertNotNull(registry.getReader(Task.Status.class, ReplicationTask.Status.NAME)); - Task.Status dummy = new DummyTaskStatus(); - module.registerTaskStatus(dummy); - assertThat(registry.getPrototype(Task.Status.class, "dummy"), sameInstance(dummy)); + module.registerTaskStatus(DummyTaskStatus.NAME, DummyTaskStatus::new); + assertEquals("test", expectThrows(UnsupportedOperationException.class, + () -> registry.getReader(Task.Status.class, DummyTaskStatus.NAME).read(null)).getMessage()); } private class DummyTaskStatus implements Task.Status { - @Override - public String getWriteableName() { - return "dummy"; + public static final String NAME = "dummy"; + + public DummyTaskStatus(StreamInput in) { + throw new UnsupportedOperationException("test"); } @Override - public Status readFrom(StreamInput in) throws IOException { - throw new UnsupportedOperationException(); + public String getWriteableName() { + return NAME; } @Override diff --git a/core/src/test/java/org/elasticsearch/common/settings/SettingsTests.java b/core/src/test/java/org/elasticsearch/common/settings/SettingsTests.java index e3f2bc1bb2..eb6cc56816 100644 --- a/core/src/test/java/org/elasticsearch/common/settings/SettingsTests.java +++ b/core/src/test/java/org/elasticsearch/common/settings/SettingsTests.java @@ -201,8 +201,8 @@ public class SettingsTests extends ESTestCase { assertThat(settings.getAsArray("value"), arrayContaining("2", "3")); settings = settingsBuilder() - .put(new YamlSettingsLoader().load("value: 1")) - .put(new YamlSettingsLoader().load("value: [ 2, 3 ]")) + .put(new YamlSettingsLoader(false).load("value: 1")) + .put(new YamlSettingsLoader(false).load("value: [ 2, 3 ]")) .build(); assertThat(settings.getAsArray("value"), arrayContaining("2", "3")); diff --git a/core/src/test/java/org/elasticsearch/common/settings/loader/JsonSettingsLoaderTests.java b/core/src/test/java/org/elasticsearch/common/settings/loader/JsonSettingsLoaderTests.java index d7f10891f2..154ef8ee03 100644 --- a/core/src/test/java/org/elasticsearch/common/settings/loader/JsonSettingsLoaderTests.java +++ b/core/src/test/java/org/elasticsearch/common/settings/loader/JsonSettingsLoaderTests.java @@ -25,15 +25,14 @@ import org.elasticsearch.common.settings.SettingsException; import org.elasticsearch.test.ESTestCase; import static org.elasticsearch.common.settings.Settings.settingsBuilder; +import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.Matchers.equalTo; -/** - * - */ public class JsonSettingsLoaderTests extends ESTestCase { + public void testSimpleJsonSettings() throws Exception { - String json = "/org/elasticsearch/common/settings/loader/test-settings.json"; - Settings settings = settingsBuilder() + final String json = "/org/elasticsearch/common/settings/loader/test-settings.json"; + final Settings settings = settingsBuilder() .loadFromStream(json, getClass().getResourceAsStream(json)) .build(); @@ -50,15 +49,23 @@ public class JsonSettingsLoaderTests extends ESTestCase { } public void testDuplicateKeysThrowsException() { - String json = "{\"foo\":\"bar\",\"foo\":\"baz\"}"; - try { - settingsBuilder() - .loadFromSource(json) - .build(); - fail("expected exception"); - } catch (SettingsException e) { - assertEquals(e.getCause().getClass(), ElasticsearchParseException.class); - assertTrue(e.toString().contains("duplicate settings key [foo] found at line number [1], column number [20], previous value [bar], current value [baz]")); - } + final String json = "{\"foo\":\"bar\",\"foo\":\"baz\"}"; + final SettingsException e = expectThrows(SettingsException.class, () -> settingsBuilder().loadFromSource(json).build()); + assertEquals(e.getCause().getClass(), ElasticsearchParseException.class); + assertThat( + e.toString(), + containsString("duplicate settings key [foo] " + + "found at line number [1], " + + "column number [20], " + + "previous value [bar], " + + "current value [baz]")); } + + public void testNullValuedSettingThrowsException() { + final String json = "{\"foo\":null}"; + final ElasticsearchParseException e = + expectThrows(ElasticsearchParseException.class, () -> new JsonSettingsLoader(false).load(json)); + assertThat(e.toString(), containsString("null-valued setting found for key [foo] found at line number [1], column number [8]")); + } + } diff --git a/core/src/test/java/org/elasticsearch/common/settings/loader/PropertiesSettingsLoaderTests.java b/core/src/test/java/org/elasticsearch/common/settings/loader/PropertiesSettingsLoaderTests.java index 7a1897fbaf..c13ae7cc68 100644 --- a/core/src/test/java/org/elasticsearch/common/settings/loader/PropertiesSettingsLoaderTests.java +++ b/core/src/test/java/org/elasticsearch/common/settings/loader/PropertiesSettingsLoaderTests.java @@ -21,27 +21,37 @@ package org.elasticsearch.common.settings.loader; import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.test.ESTestCase; +import org.junit.Before; import java.io.IOException; import java.nio.charset.Charset; public class PropertiesSettingsLoaderTests extends ESTestCase { + + private PropertiesSettingsLoader loader; + + @Before + public void setUp() throws Exception { + super.setUp(); + loader = new PropertiesSettingsLoader(); + } + public void testDuplicateKeyFromStringThrowsException() throws IOException { - PropertiesSettingsLoader loader = new PropertiesSettingsLoader(); - try { - loader.load("foo=bar\nfoo=baz"); - fail("expected exception"); - } catch (ElasticsearchParseException e) { - assertEquals(e.getMessage(), "duplicate settings key [foo] found, previous value [bar], current value [baz]"); - } + final ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> loader.load("foo=bar\nfoo=baz")); + assertEquals(e.getMessage(), "duplicate settings key [foo] found, previous value [bar], current value [baz]"); } public void testDuplicateKeysFromBytesThrowsException() throws IOException { - PropertiesSettingsLoader loader = new PropertiesSettingsLoader(); - try { - loader.load("foo=bar\nfoo=baz".getBytes(Charset.defaultCharset())); - } catch (ElasticsearchParseException e) { - assertEquals(e.getMessage(), "duplicate settings key [foo] found, previous value [bar], current value [baz]"); - } + final ElasticsearchParseException e = expectThrows( + ElasticsearchParseException.class, + () -> loader.load("foo=bar\nfoo=baz".getBytes(Charset.defaultCharset())) + ); + assertEquals(e.getMessage(), "duplicate settings key [foo] found, previous value [bar], current value [baz]"); } + + public void testThatNoDuplicatesPropertiesDoesNotAcceptNullValues() { + final PropertiesSettingsLoader.NoDuplicatesProperties properties = loader.new NoDuplicatesProperties(); + expectThrows(NullPointerException.class, () -> properties.put("key", null)); + } + } diff --git a/core/src/test/java/org/elasticsearch/common/settings/loader/YamlSettingsLoaderTests.java b/core/src/test/java/org/elasticsearch/common/settings/loader/YamlSettingsLoaderTests.java index 48703044ec..2e2a187da0 100644 --- a/core/src/test/java/org/elasticsearch/common/settings/loader/YamlSettingsLoaderTests.java +++ b/core/src/test/java/org/elasticsearch/common/settings/loader/YamlSettingsLoaderTests.java @@ -28,13 +28,11 @@ import static org.elasticsearch.common.settings.Settings.settingsBuilder; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; -/** - * - */ public class YamlSettingsLoaderTests extends ESTestCase { + public void testSimpleYamlSettings() throws Exception { - String yaml = "/org/elasticsearch/common/settings/loader/test-settings.yml"; - Settings settings = settingsBuilder() + final String yaml = "/org/elasticsearch/common/settings/loader/test-settings.yml"; + final Settings settings = settingsBuilder() .loadFromStream(yaml, getClass().getResourceAsStream(yaml)) .build(); @@ -51,39 +49,41 @@ public class YamlSettingsLoaderTests extends ESTestCase { } public void testIndentation() { - String yaml = "/org/elasticsearch/common/settings/loader/indentation-settings.yml"; - try { - settingsBuilder() - .loadFromStream(yaml, getClass().getResourceAsStream(yaml)) - .build(); - fail("Expected SettingsException"); - } catch(SettingsException e ) { - assertThat(e.getMessage(), containsString("Failed to load settings")); - } + final String yaml = "/org/elasticsearch/common/settings/loader/indentation-settings.yml"; + final SettingsException e = + expectThrows( + SettingsException.class, + () -> settingsBuilder().loadFromStream(yaml, getClass().getResourceAsStream(yaml)).build()); + assertThat(e.getMessage(), containsString("Failed to load settings")); } public void testIndentationWithExplicitDocumentStart() { - String yaml = "/org/elasticsearch/common/settings/loader/indentation-with-explicit-document-start-settings.yml"; - try { - settingsBuilder() - .loadFromStream(yaml, getClass().getResourceAsStream(yaml)) - .build(); - fail("Expected SettingsException"); - } catch (SettingsException e) { - assertThat(e.getMessage(), containsString("Failed to load settings")); - } + final String yaml = "/org/elasticsearch/common/settings/loader/indentation-with-explicit-document-start-settings.yml"; + final SettingsException e = + expectThrows( + SettingsException.class, + () -> settingsBuilder().loadFromStream(yaml, getClass().getResourceAsStream(yaml)).build()); + assertThat(e.getMessage(), containsString("Failed to load settings")); } public void testDuplicateKeysThrowsException() { - String yaml = "foo: bar\nfoo: baz"; - try { - settingsBuilder() - .loadFromSource(yaml) - .build(); - fail("expected exception"); - } catch (SettingsException e) { - assertEquals(e.getCause().getClass(), ElasticsearchParseException.class); - assertTrue(e.toString().contains("duplicate settings key [foo] found at line number [2], column number [6], previous value [bar], current value [baz]")); - } + final String yaml = "foo: bar\nfoo: baz"; + final SettingsException e = expectThrows(SettingsException.class, () -> settingsBuilder().loadFromSource(yaml).build()); + assertEquals(e.getCause().getClass(), ElasticsearchParseException.class); + assertThat( + e.toString(), + containsString("duplicate settings key [foo] " + + "found at line number [2], " + + "column number [6], " + + "previous value [bar], " + + "current value [baz]")); + } + + public void testNullValuedSettingThrowsException() { + final String yaml = "foo:"; + final ElasticsearchParseException e = + expectThrows(ElasticsearchParseException.class, () -> new YamlSettingsLoader(false).load(yaml)); + assertThat(e.toString(), containsString("null-valued setting found for key [foo] found at line number [1], column number [5]")); } + } diff --git a/core/src/test/java/org/elasticsearch/common/unit/DistanceUnitTests.java b/core/src/test/java/org/elasticsearch/common/unit/DistanceUnitTests.java index 5d7bbb3ca1..f9a4d3f22a 100644 --- a/core/src/test/java/org/elasticsearch/common/unit/DistanceUnitTests.java +++ b/core/src/test/java/org/elasticsearch/common/unit/DistanceUnitTests.java @@ -83,7 +83,7 @@ public class DistanceUnitTests extends ESTestCase { try (BytesStreamOutput out = new BytesStreamOutput()) { unit.writeTo(out); try (StreamInput in = StreamInput.wrap(out.bytes())) { - assertThat("Roundtrip serialisation failed.", DistanceUnit.readDistanceUnit(in), equalTo(unit)); + assertThat("Roundtrip serialisation failed.", DistanceUnit.readFromStream(in), equalTo(unit)); } } } diff --git a/core/src/test/java/org/elasticsearch/discovery/zen/ZenDiscoveryUnitTests.java b/core/src/test/java/org/elasticsearch/discovery/zen/ZenDiscoveryUnitTests.java index e1d6beecaa..468d01aaec 100644 --- a/core/src/test/java/org/elasticsearch/discovery/zen/ZenDiscoveryUnitTests.java +++ b/core/src/test/java/org/elasticsearch/discovery/zen/ZenDiscoveryUnitTests.java @@ -25,10 +25,19 @@ import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.common.transport.DummyTransportAddress; +import org.elasticsearch.discovery.zen.ping.ZenPing; import org.elasticsearch.test.ESTestCase; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + import static org.elasticsearch.discovery.zen.ZenDiscovery.shouldIgnoreOrRejectNewClusterState; import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; /** */ @@ -89,4 +98,34 @@ public class ZenDiscoveryUnitTests extends ESTestCase { } assertFalse("should not ignore, because current state doesn't have a master", shouldIgnoreOrRejectNewClusterState(logger, currentState.build(), newState.build())); } + + public void testFilterNonMasterPingResponse() { + ArrayList<ZenPing.PingResponse> responses = new ArrayList<>(); + ArrayList<DiscoveryNode> masterNodes = new ArrayList<>(); + ArrayList<DiscoveryNode> allNodes = new ArrayList<>(); + for (int i = randomIntBetween(10, 20); i >= 0; i--) { + Map<String, String> attrs = new HashMap<>(); + for (String attr : randomSubsetOf( + Arrays.asList(DiscoveryNode.INGEST_ATTR, DiscoveryNode.DATA_ATTR, DiscoveryNode.MASTER_ATTR))) { + attrs.put(attr, randomBoolean() + ""); + } + + DiscoveryNode node = new DiscoveryNode("node_" + i, "id_" + i, DummyTransportAddress.INSTANCE, attrs, Version.CURRENT); + responses.add(new ZenPing.PingResponse(node, randomBoolean() ? null : node, new ClusterName("test"), randomBoolean())); + allNodes.add(node); + if (node.isMasterNode()) { + masterNodes.add(node); + } + } + + boolean ignore = randomBoolean(); + List<ZenPing.PingResponse> filtered = ZenDiscovery.filterPingResponses( + responses.toArray(new ZenPing.PingResponse[responses.size()]), ignore, logger); + final List<DiscoveryNode> filteredNodes = filtered.stream().map(ZenPing.PingResponse::node).collect(Collectors.toList()); + if (ignore) { + assertThat(filteredNodes, equalTo(masterNodes)); + } else { + assertThat(filteredNodes, equalTo(allNodes)); + } + } } diff --git a/core/src/test/java/org/elasticsearch/gateway/RecoveryFromGatewayIT.java b/core/src/test/java/org/elasticsearch/gateway/RecoveryFromGatewayIT.java index 4da9c2df17..dac284ee59 100644 --- a/core/src/test/java/org/elasticsearch/gateway/RecoveryFromGatewayIT.java +++ b/core/src/test/java/org/elasticsearch/gateway/RecoveryFromGatewayIT.java @@ -19,11 +19,13 @@ package org.elasticsearch.gateway; +import com.carrotsearch.hppc.cursors.ObjectCursor; import org.elasticsearch.action.admin.indices.recovery.RecoveryResponse; import org.elasticsearch.action.admin.indices.stats.IndexStats; import org.elasticsearch.action.admin.indices.stats.ShardStats; import org.elasticsearch.client.Client; import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider; import org.elasticsearch.cluster.routing.allocation.decider.ThrottlingAllocationDecider; import org.elasticsearch.common.settings.Settings; @@ -37,11 +39,13 @@ import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; import org.elasticsearch.test.ESIntegTestCase.Scope; import org.elasticsearch.test.InternalTestCluster.RestartCallback; -import org.elasticsearch.test.junit.annotations.TestLogging; import org.elasticsearch.test.store.MockFSDirectoryService; import org.elasticsearch.test.store.MockFSIndexStore; import java.util.Collection; +import java.util.HashMap; +import java.util.Map; +import java.util.stream.IntStream; import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS; import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS; @@ -88,10 +92,13 @@ public class RecoveryFromGatewayIT extends ESIntegTestCase { assertHitCount(client().prepareSearch().setSize(0).setQuery(termQuery("appAccountIds", 179)).execute().actionGet(), 2); ensureYellow("test"); // wait for primary allocations here otherwise if we have a lot of shards we might have a // shard that is still in post recovery when we restart and the ensureYellow() below will timeout + + Map<String, long[]> primaryTerms = assertAndCapturePrimaryTerms(null); internalCluster().fullRestart(); logger.info("Running Cluster Health (wait for the shards to startup)"); ensureYellow(); + primaryTerms = assertAndCapturePrimaryTerms(primaryTerms); client().admin().indices().prepareRefresh().execute().actionGet(); assertHitCount(client().prepareSearch().setSize(0).setQuery(termQuery("appAccountIds", 179)).execute().actionGet(), 2); @@ -100,11 +107,37 @@ public class RecoveryFromGatewayIT extends ESIntegTestCase { logger.info("Running Cluster Health (wait for the shards to startup)"); ensureYellow(); + primaryTerms = assertAndCapturePrimaryTerms(primaryTerms); client().admin().indices().prepareRefresh().execute().actionGet(); assertHitCount(client().prepareSearch().setSize(0).setQuery(termQuery("appAccountIds", 179)).execute().actionGet(), 2); } + private Map<String, long[]> assertAndCapturePrimaryTerms(Map<String, long[]> previousTerms) { + if (previousTerms == null) { + previousTerms = new HashMap<>(); + } + final Map<String, long[]> result = new HashMap<>(); + final ClusterState state = client().admin().cluster().prepareState().get().getState(); + for (ObjectCursor<IndexMetaData> cursor : state.metaData().indices().values()) { + final IndexMetaData indexMetaData = cursor.value; + final String index = indexMetaData.getIndex().getName(); + final long[] previous = previousTerms.get(index); + final long[] current = IntStream.range(0, indexMetaData.getNumberOfShards()).mapToLong(indexMetaData::primaryTerm).toArray(); + if (previous == null) { + result.put(index, current); + } else { + assertThat("number of terms changed for index [" + index + "]", current.length, equalTo(previous.length)); + for (int shard = 0; shard < current.length; shard++) { + assertThat("primary term didn't increase for [" + index + "][" + shard + "]", current[shard], greaterThan(previous[shard])); + } + result.put(index, current); + } + } + + return result; + } + public void testSingleNodeNoFlush() throws Exception { internalCluster().startNode(); @@ -163,10 +196,14 @@ public class RecoveryFromGatewayIT extends ESIntegTestCase { logger.info("Ensure all primaries have been started"); ensureYellow(); } + + Map<String, long[]> primaryTerms = assertAndCapturePrimaryTerms(null); + internalCluster().fullRestart(); logger.info("Running Cluster Health (wait for the shards to startup)"); ensureYellow(); + primaryTerms = assertAndCapturePrimaryTerms(primaryTerms); for (int i = 0; i <= randomInt(10); i++) { assertHitCount(client().prepareSearch().setSize(0).setQuery(matchAllQuery()).get(), value1Docs + value2Docs); @@ -180,6 +217,7 @@ public class RecoveryFromGatewayIT extends ESIntegTestCase { logger.info("Running Cluster Health (wait for the shards to startup)"); ensureYellow(); + primaryTerms = assertAndCapturePrimaryTerms(primaryTerms); for (int i = 0; i <= randomInt(10); i++) { assertHitCount(client().prepareSearch().setSize(0).setQuery(matchAllQuery()).get(), value1Docs + value2Docs); @@ -201,10 +239,13 @@ public class RecoveryFromGatewayIT extends ESIntegTestCase { ensureYellow("test"); // wait for primary allocations here otherwise if we have a lot of shards we might have a // shard that is still in post recovery when we restart and the ensureYellow() below will timeout + Map<String, long[]> primaryTerms = assertAndCapturePrimaryTerms(null); + internalCluster().fullRestart(); logger.info("Running Cluster Health (wait for the shards to startup)"); ensureYellow(); + primaryTerms = assertAndCapturePrimaryTerms(primaryTerms); for (int i = 0; i < 10; i++) { assertHitCount(client().prepareSearch().setSize(0).setQuery(matchAllQuery()).execute().actionGet(), 2); @@ -214,6 +255,7 @@ public class RecoveryFromGatewayIT extends ESIntegTestCase { logger.info("Running Cluster Health (wait for the shards to startup)"); ensureYellow(); + primaryTerms = assertAndCapturePrimaryTerms(primaryTerms); for (int i = 0; i < 10; i++) { assertHitCount(client().prepareSearch().setSize(0).setQuery(matchAllQuery()).execute().actionGet(), 2); @@ -236,6 +278,8 @@ public class RecoveryFromGatewayIT extends ESIntegTestCase { assertHitCount(client().prepareSearch().setSize(0).setQuery(matchAllQuery()).execute().actionGet(), 2); } + Map<String, long[]> primaryTerms = assertAndCapturePrimaryTerms(null); + internalCluster().fullRestart(new RestartCallback() { @Override public Settings onNodeStopped(String nodeName) throws Exception { @@ -251,6 +295,7 @@ public class RecoveryFromGatewayIT extends ESIntegTestCase { logger.info("Running Cluster Health (wait for the shards to startup)"); ensureGreen(); + primaryTerms = assertAndCapturePrimaryTerms(primaryTerms); for (int i = 0; i < 10; i++) { assertHitCount(client().prepareSearch().setSize(0).setQuery(matchAllQuery()).execute().actionGet(), 2); @@ -276,6 +321,8 @@ public class RecoveryFromGatewayIT extends ESIntegTestCase { String metaDataUuid = client().admin().cluster().prepareState().execute().get().getState().getMetaData().clusterUUID(); assertThat(metaDataUuid, not(equalTo("_na_"))); + Map<String, long[]> primaryTerms = assertAndCapturePrimaryTerms(null); + logger.info("--> closing first node, and indexing more data to the second node"); internalCluster().fullRestart(new RestartCallback() { @@ -315,6 +362,7 @@ public class RecoveryFromGatewayIT extends ESIntegTestCase { logger.info("--> running cluster_health (wait for the shards to startup)"); ensureGreen(); + primaryTerms = assertAndCapturePrimaryTerms(primaryTerms); assertThat(client().admin().cluster().prepareState().execute().get().getState().getMetaData().clusterUUID(), equalTo(metaDataUuid)); @@ -386,11 +434,15 @@ public class RecoveryFromGatewayIT extends ESIntegTestCase { .setTransientSettings(settingsBuilder() .put(EnableAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ENABLE_SETTING.getKey(), EnableAllocationDecider.Allocation.NONE)) .get(); + + Map<String, long[]> primaryTerms = assertAndCapturePrimaryTerms(null); + logger.info("--> full cluster restart"); internalCluster().fullRestart(); logger.info("--> waiting for cluster to return to green after {}shutdown", useSyncIds ? "" : "second "); ensureGreen(); + primaryTerms = assertAndCapturePrimaryTerms(primaryTerms); if (useSyncIds) { assertSyncIdsNotNull(); @@ -445,6 +497,8 @@ public class RecoveryFromGatewayIT extends ESIntegTestCase { internalCluster().startNode(settingsBuilder().put(Environment.PATH_DATA_SETTING.getKey(), createTempDir()).build()); ensureGreen(); + Map<String, long[]> primaryTerms = assertAndCapturePrimaryTerms(null); + internalCluster().fullRestart(new RestartCallback() { @@ -455,6 +509,7 @@ public class RecoveryFromGatewayIT extends ESIntegTestCase { }); ensureYellow(); + primaryTerms = assertAndCapturePrimaryTerms(primaryTerms); assertThat(client().admin().indices().prepareExists("test").execute().actionGet().isExists(), equalTo(true)); assertHitCount(client().prepareSearch("test").setSize(0).setQuery(QueryBuilders.matchAllQuery()).execute().actionGet(), 1); diff --git a/core/src/test/java/org/elasticsearch/index/IndexServiceTests.java b/core/src/test/java/org/elasticsearch/index/IndexServiceTests.java index 3a4020e410..97258b12a3 100644 --- a/core/src/test/java/org/elasticsearch/index/IndexServiceTests.java +++ b/core/src/test/java/org/elasticsearch/index/IndexServiceTests.java @@ -187,12 +187,13 @@ public class IndexServiceTests extends ESSingleNodeTestCase { return ThreadPool.Names.GENERIC; } }; + latch.get().await(); latch.set(new CountDownLatch(1)); assertEquals(1, count.get()); - latch2.get().countDown(); - latch2.set(new CountDownLatch(1)); - + // here we need to swap first before we let it go otherwise threads might be very fast and run that task twice due to + // random exception and the schedule interval is 1ms + latch2.getAndSet(new CountDownLatch(1)).countDown(); latch.get().await(); assertEquals(2, count.get()); task.close(); diff --git a/core/src/test/java/org/elasticsearch/index/mapper/DynamicMappingTests.java b/core/src/test/java/org/elasticsearch/index/mapper/DynamicMappingTests.java index e3a81d2a06..477e48ae5d 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/DynamicMappingTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/DynamicMappingTests.java @@ -42,7 +42,6 @@ import org.elasticsearch.index.mapper.core.TextFieldMapper; import org.elasticsearch.test.ESSingleNodeTestCase; import java.io.IOException; -import java.util.List; import static java.util.Collections.emptyMap; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; @@ -245,7 +244,17 @@ public class DynamicMappingTests extends ESSingleNodeTestCase { // original mapping not modified assertEquals(mapping, serialize(mapper)); // but we have an update - assertEquals("{\"type\":{\"properties\":{\"foo\":{\"type\":\"text\"}}}}", serialize(update)); + assertEquals(XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties") + .startObject("foo") + .field("type", "text") + .startObject("fields") + .startObject("keyword") + .field("type", "keyword") + .field("ignore_above", 256) + .endObject() + .endObject() + .endObject() + .endObject().endObject().endObject().string(), serialize(update)); } public void testIncremental() throws Exception { @@ -267,7 +276,14 @@ public class DynamicMappingTests extends ESSingleNodeTestCase { // but we have an update assertEquals(XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties") // foo is NOT in the update - .startObject("bar").field("type", "text").endObject() + .startObject("bar").field("type", "text") + .startObject("fields") + .startObject("keyword") + .field("type", "keyword") + .field("ignore_above", 256) + .endObject() + .endObject() + .endObject() .endObject().endObject().string(), serialize(update)); } @@ -287,8 +303,22 @@ public class DynamicMappingTests extends ESSingleNodeTestCase { assertEquals(mapping, serialize(mapper)); // but we have an update assertEquals(XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties") - .startObject("bar").field("type", "text").endObject() - .startObject("foo").field("type", "text").endObject() + .startObject("bar").field("type", "text") + .startObject("fields") + .startObject("keyword") + .field("type", "keyword") + .field("ignore_above", 256) + .endObject() + .endObject() + .endObject() + .startObject("foo").field("type", "text") + .startObject("fields") + .startObject("keyword") + .field("type", "keyword") + .field("ignore_above", 256) + .endObject() + .endObject() + .endObject() .endObject().endObject().string(), serialize(update)); } @@ -308,7 +338,9 @@ public class DynamicMappingTests extends ESSingleNodeTestCase { assertEquals(mapping, serialize(mapper)); // but we have an update assertEquals(XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties") - .startObject("foo").startObject("properties").startObject("bar").startObject("properties").startObject("baz").field("type", "text").endObject().endObject().endObject().endObject().endObject() + .startObject("foo").startObject("properties").startObject("bar").startObject("properties").startObject("baz").field("type", "text") + .startObject("fields").startObject("keyword").field("type", "keyword").field("ignore_above", 256).endObject() + .endObject().endObject().endObject().endObject().endObject().endObject() .endObject().endObject().endObject().string(), serialize(update)); } @@ -328,7 +360,15 @@ public class DynamicMappingTests extends ESSingleNodeTestCase { assertEquals(mapping, serialize(mapper)); // but we have an update assertEquals(XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties") - .startObject("foo").field("type", "text").endObject() + .startObject("foo") + .field("type", "text") + .startObject("fields") + .startObject("keyword") + .field("type", "keyword") + .field("ignore_above", 256) + .endObject() + .endObject() + .endObject() .endObject().endObject().endObject().string(), serialize(update)); } @@ -348,7 +388,9 @@ public class DynamicMappingTests extends ESSingleNodeTestCase { assertEquals(mapping, serialize(mapper)); // but we have an update assertEquals(XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties") - .startObject("foo").startObject("properties").startObject("bar").startObject("properties").startObject("baz").field("type", "text").endObject().endObject().endObject().endObject().endObject() + .startObject("foo").startObject("properties").startObject("bar").startObject("properties").startObject("baz").field("type", "text").startObject("fields") + .startObject("keyword").field("type", "keyword").field("ignore_above", 256).endObject() + .endObject().endObject().endObject().endObject().endObject().endObject() .endObject().endObject().endObject().string(), serialize(update)); } @@ -369,7 +411,14 @@ public class DynamicMappingTests extends ESSingleNodeTestCase { assertEquals(mapping, serialize(mapper)); assertEquals(XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties") .startObject("foo").startObject("properties") - .startObject("bar").field("type", "text").endObject() + .startObject("bar").field("type", "text") + .startObject("fields") + .startObject("keyword") + .field("type", "keyword") + .field("ignore_above", 256) + .endObject() + .endObject() + .endObject() .startObject("baz").field("type", "long").endObject() .endObject().endObject() .endObject().endObject().endObject().string(), serialize(update)); diff --git a/core/src/test/java/org/elasticsearch/index/mapper/internal/FieldNamesFieldMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/internal/FieldNamesFieldMapperTests.java index 2646d94471..6fc4c4a02a 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/internal/FieldNamesFieldMapperTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/internal/FieldNamesFieldMapperTests.java @@ -102,7 +102,7 @@ public class FieldNamesFieldMapperTests extends ESSingleNodeTestCase { .endObject() .bytes()); - assertFieldNames(set("a", "b", "b.c", "_uid", "_type", "_version", "_source", "_all"), doc); + assertFieldNames(set("a", "a.keyword", "b", "b.c", "_uid", "_type", "_version", "_source", "_all"), doc); } public void testExplicitEnabled() throws Exception { @@ -119,7 +119,7 @@ public class FieldNamesFieldMapperTests extends ESSingleNodeTestCase { .endObject() .bytes()); - assertFieldNames(set("field", "_uid", "_type", "_version", "_source", "_all"), doc); + assertFieldNames(set("field", "field.keyword", "_uid", "_type", "_version", "_source", "_all"), doc); } public void testDisabled() throws Exception { diff --git a/core/src/test/java/org/elasticsearch/index/mapper/internal/TypeFieldTypeTests.java b/core/src/test/java/org/elasticsearch/index/mapper/internal/TypeFieldTypeTests.java index c01b04584e..91216983b7 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/internal/TypeFieldTypeTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/internal/TypeFieldTypeTests.java @@ -18,6 +18,23 @@ */ package org.elasticsearch.index.mapper.internal; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field.Store; +import org.apache.lucene.document.StringField; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.BooleanClause.Occur; +import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.ConstantScoreQuery; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.MatchAllDocsQuery; +import org.apache.lucene.search.PhraseQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.TermQuery; +import org.apache.lucene.store.Directory; +import org.apache.lucene.util.IOUtils; import org.elasticsearch.index.mapper.FieldTypeTestCase; import org.elasticsearch.index.mapper.MappedFieldType; @@ -26,4 +43,36 @@ public class TypeFieldTypeTests extends FieldTypeTestCase { protected MappedFieldType createDefaultFieldType() { return new TypeFieldMapper.TypeFieldType(); } + + public void testTermQuery() throws Exception { + Directory dir = newDirectory(); + IndexWriter w = new IndexWriter(dir, newIndexWriterConfig()); + Document doc = new Document(); + StringField type = new StringField(TypeFieldMapper.NAME, "my_type", Store.NO); + doc.add(type); + w.addDocument(doc); + w.addDocument(doc); + IndexReader reader = DirectoryReader.open(w); + + TypeFieldMapper.TypeFieldType ft = new TypeFieldMapper.TypeFieldType(); + ft.setName(TypeFieldMapper.NAME); + Query query = ft.termQuery("my_type", null); + + assertEquals(new MatchAllDocsQuery(), query.rewrite(reader)); + + // Make sure that Lucene actually simplifies the query when there is a single type + Query userQuery = new PhraseQuery("body", "quick", "fox"); + Query filteredQuery = new BooleanQuery.Builder().add(userQuery, Occur.MUST).add(query, Occur.FILTER).build(); + Query rewritten = new IndexSearcher(reader).rewrite(filteredQuery); + assertEquals(userQuery, rewritten); + + type.setStringValue("my_type2"); + w.addDocument(doc); + reader.close(); + reader = DirectoryReader.open(w); + + assertEquals(new ConstantScoreQuery(new TermQuery(new Term(TypeFieldMapper.NAME, "my_type"))), query.rewrite(reader)); + + IOUtils.close(reader, w, dir); + } } diff --git a/core/src/test/java/org/elasticsearch/index/percolator/ExtractQueryTermsServiceTests.java b/core/src/test/java/org/elasticsearch/index/percolator/ExtractQueryTermsServiceTests.java index f17a4fc664..572eb1b46c 100644 --- a/core/src/test/java/org/elasticsearch/index/percolator/ExtractQueryTermsServiceTests.java +++ b/core/src/test/java/org/elasticsearch/index/percolator/ExtractQueryTermsServiceTests.java @@ -24,6 +24,8 @@ import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.Term; import org.apache.lucene.index.memory.MemoryIndex; +import org.apache.lucene.queries.BlendedTermQuery; +import org.apache.lucene.queries.CommonTermsQuery; import org.apache.lucene.queries.TermsQuery; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; @@ -39,16 +41,13 @@ import org.elasticsearch.test.ESTestCase; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Set; import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.hasKey; import static org.hamcrest.Matchers.instanceOf; -import static org.hamcrest.Matchers.nullValue; import static org.hamcrest.Matchers.sameInstance; public class ExtractQueryTermsServiceTests extends ESTestCase { @@ -218,6 +217,31 @@ public class ExtractQueryTermsServiceTests extends ESTestCase { assertThat(terms.get(0).bytes(), equalTo(termQuery1.getTerm().bytes())); } + public void testExtractQueryMetadata_commonTermsQuery() { + CommonTermsQuery commonTermsQuery = new CommonTermsQuery(BooleanClause.Occur.SHOULD, BooleanClause.Occur.SHOULD, 100); + commonTermsQuery.add(new Term("_field", "_term1")); + commonTermsQuery.add(new Term("_field", "_term2")); + List<Term> terms = new ArrayList<>(ExtractQueryTermsService.extractQueryTerms(commonTermsQuery)); + Collections.sort(terms); + assertThat(terms.size(), equalTo(2)); + assertThat(terms.get(0).field(), equalTo("_field")); + assertThat(terms.get(0).text(), equalTo("_term1")); + assertThat(terms.get(1).field(), equalTo("_field")); + assertThat(terms.get(1).text(), equalTo("_term2")); + } + + public void testExtractQueryMetadata_blendedTermQuery() { + Term[] terms = new Term[]{new Term("_field", "_term1"), new Term("_field", "_term2")}; + BlendedTermQuery commonTermsQuery = BlendedTermQuery.booleanBlendedQuery(terms, false); + List<Term> result = new ArrayList<>(ExtractQueryTermsService.extractQueryTerms(commonTermsQuery)); + Collections.sort(result); + assertThat(result.size(), equalTo(2)); + assertThat(result.get(0).field(), equalTo("_field")); + assertThat(result.get(0).text(), equalTo("_term1")); + assertThat(result.get(1).field(), equalTo("_field")); + assertThat(result.get(1).text(), equalTo("_term2")); + } + public void testExtractQueryMetadata_unsupportedQuery() { TermRangeQuery termRangeQuery = new TermRangeQuery("_field", null, null, true, false); @@ -229,7 +253,7 @@ public class ExtractQueryTermsServiceTests extends ESTestCase { } TermQuery termQuery1 = new TermQuery(new Term("_field", "_term")); - BooleanQuery.Builder builder = new BooleanQuery.Builder();; + BooleanQuery.Builder builder = new BooleanQuery.Builder(); builder.add(termQuery1, BooleanClause.Occur.SHOULD); builder.add(termRangeQuery, BooleanClause.Occur.SHOULD); BooleanQuery bq = builder.build(); diff --git a/core/src/test/java/org/elasticsearch/index/query/HasChildQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/HasChildQueryBuilderTests.java index 4715574b5a..675ad954e0 100644 --- a/core/src/test/java/org/elasticsearch/index/query/HasChildQueryBuilderTests.java +++ b/core/src/test/java/org/elasticsearch/index/query/HasChildQueryBuilderTests.java @@ -275,12 +275,7 @@ public class HasChildQueryBuilderTests extends AbstractQueryTestCase<HasChildQue assertThat(termQuery.getTerm().bytes(), equalTo(ids[0])); //check the type filter assertThat(booleanQuery.clauses().get(1).getOccur(), equalTo(BooleanClause.Occur.FILTER)); - assertThat(booleanQuery.clauses().get(1).getQuery(), instanceOf(ConstantScoreQuery.class)); - ConstantScoreQuery typeConstantScoreQuery = (ConstantScoreQuery) booleanQuery.clauses().get(1).getQuery(); - assertThat(typeConstantScoreQuery.getQuery(), instanceOf(TermQuery.class)); - TermQuery typeTermQuery = (TermQuery) typeConstantScoreQuery.getQuery(); - assertThat(typeTermQuery.getTerm().field(), equalTo(TypeFieldMapper.NAME)); - assertThat(typeTermQuery.getTerm().text(), equalTo(type)); + assertEquals(new TypeFieldMapper.TypeQuery(new BytesRef(type)), booleanQuery.clauses().get(1).getQuery()); } /** diff --git a/core/src/test/java/org/elasticsearch/index/query/PercolatorQueryTests.java b/core/src/test/java/org/elasticsearch/index/query/PercolatorQueryTests.java index 347802d377..cbcfba55b9 100644 --- a/core/src/test/java/org/elasticsearch/index/query/PercolatorQueryTests.java +++ b/core/src/test/java/org/elasticsearch/index/query/PercolatorQueryTests.java @@ -32,9 +32,10 @@ import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.NoMergePolicy; import org.apache.lucene.index.Term; import org.apache.lucene.index.memory.MemoryIndex; +import org.apache.lucene.queries.BlendedTermQuery; +import org.apache.lucene.queries.CommonTermsQuery; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; -import org.apache.lucene.search.Explanation; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.PhraseQuery; @@ -59,8 +60,6 @@ import java.util.HashMap; import java.util.Map; import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.is; -import static org.hamcrest.Matchers.not; public class PercolatorQueryTests extends ESTestCase { @@ -179,36 +178,30 @@ public class PercolatorQueryTests extends ESTestCase { MemoryIndex memoryIndex = new MemoryIndex(); String id = Integer.toString(i); memoryIndex.addField("field", id, new WhitespaceAnalyzer()); - IndexSearcher percolateSearcher = memoryIndex.createSearcher(); - - PercolatorQuery.Builder builder1 = new PercolatorQuery.Builder( - "docType", - queryRegistry, - new BytesArray("{}"), - percolateSearcher, - new MatchAllDocsQuery() - ); - // enables the optimization that prevents queries from being evaluated that don't match - builder1.extractQueryTermsQuery(EXTRACTED_TERMS_FIELD_NAME, UNKNOWN_QUERY_FIELD_NAME); - TopDocs topDocs1 = shardSearcher.search(builder1.build(), 10); - - PercolatorQuery.Builder builder2 = new PercolatorQuery.Builder( - "docType", - queryRegistry, - new BytesArray("{}"), - percolateSearcher, - new MatchAllDocsQuery() - ); - TopDocs topDocs2 = shardSearcher.search(builder2.build(), 10); - - assertThat(topDocs1.totalHits, equalTo(topDocs2.totalHits)); - assertThat(topDocs1.scoreDocs.length, equalTo(topDocs2.scoreDocs.length)); - for (int j = 0; j < topDocs1.scoreDocs.length; j++) { - assertThat(topDocs1.scoreDocs[j].doc, equalTo(topDocs2.scoreDocs[j].doc)); - } + duelRun(memoryIndex, shardSearcher); } } + public void testDuelSpecificQueries() throws Exception { + CommonTermsQuery commonTermsQuery = new CommonTermsQuery(BooleanClause.Occur.SHOULD, BooleanClause.Occur.SHOULD, 128); + commonTermsQuery.add(new Term("field", "quick")); + commonTermsQuery.add(new Term("field", "brown")); + commonTermsQuery.add(new Term("field", "fox")); + addPercolatorQuery("_id1", commonTermsQuery); + + BlendedTermQuery blendedTermQuery = BlendedTermQuery.booleanBlendedQuery(new Term[]{new Term("field", "quick"), + new Term("field", "brown"), new Term("field", "fox")}, false); + addPercolatorQuery("_id2", blendedTermQuery); + + indexWriter.close(); + directoryReader = DirectoryReader.open(directory); + IndexSearcher shardSearcher = newSearcher(directoryReader); + + MemoryIndex memoryIndex = new MemoryIndex(); + memoryIndex.addField("field", "the quick brown fox jumps over the lazy dog", new WhitespaceAnalyzer()); + duelRun(memoryIndex, shardSearcher); + } + void addPercolatorQuery(String id, Query query, String... extraFields) throws IOException { queries.put(id, query); ParseContext.Document document = new ParseContext.Document(); @@ -222,6 +215,35 @@ public class PercolatorQueryTests extends ESTestCase { indexWriter.addDocument(document); } + private void duelRun(MemoryIndex memoryIndex, IndexSearcher shardSearcher) throws IOException { + IndexSearcher percolateSearcher = memoryIndex.createSearcher(); + PercolatorQuery.Builder builder1 = new PercolatorQuery.Builder( + "docType", + queryRegistry, + new BytesArray("{}"), + percolateSearcher, + new MatchAllDocsQuery() + ); + // enables the optimization that prevents queries from being evaluated that don't match + builder1.extractQueryTermsQuery(EXTRACTED_TERMS_FIELD_NAME, UNKNOWN_QUERY_FIELD_NAME); + TopDocs topDocs1 = shardSearcher.search(builder1.build(), 10); + + PercolatorQuery.Builder builder2 = new PercolatorQuery.Builder( + "docType", + queryRegistry, + new BytesArray("{}"), + percolateSearcher, + new MatchAllDocsQuery() + ); + TopDocs topDocs2 = shardSearcher.search(builder2.build(), 10); + + assertThat(topDocs1.totalHits, equalTo(topDocs2.totalHits)); + assertThat(topDocs1.scoreDocs.length, equalTo(topDocs2.scoreDocs.length)); + for (int j = 0; j < topDocs1.scoreDocs.length; j++) { + assertThat(topDocs1.scoreDocs[j].doc, equalTo(topDocs2.scoreDocs[j].doc)); + } + } + private final static class CustomQuery extends Query { private final Term term; diff --git a/core/src/test/java/org/elasticsearch/index/query/TypeQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/TypeQueryBuilderTests.java index 4621390e5f..674b6aed78 100644 --- a/core/src/test/java/org/elasticsearch/index/query/TypeQueryBuilderTests.java +++ b/core/src/test/java/org/elasticsearch/index/query/TypeQueryBuilderTests.java @@ -19,17 +19,12 @@ package org.elasticsearch.index.query; -import org.apache.lucene.search.ConstantScoreQuery; import org.apache.lucene.search.Query; -import org.apache.lucene.search.TermQuery; +import org.apache.lucene.util.BytesRef; import org.elasticsearch.index.mapper.internal.TypeFieldMapper; import java.io.IOException; -import static org.hamcrest.Matchers.either; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.instanceOf; - public class TypeQueryBuilderTests extends AbstractQueryTestCase<TypeQueryBuilder> { @Override @@ -39,14 +34,7 @@ public class TypeQueryBuilderTests extends AbstractQueryTestCase<TypeQueryBuilde @Override protected void doAssertLuceneQuery(TypeQueryBuilder queryBuilder, Query query, QueryShardContext context) throws IOException { - assertThat(query, either(instanceOf(TermQuery.class)).or(instanceOf(ConstantScoreQuery.class))); - if (query instanceof ConstantScoreQuery) { - query = ((ConstantScoreQuery) query).getQuery(); - assertThat(query, instanceOf(TermQuery.class)); - } - TermQuery termQuery = (TermQuery) query; - assertThat(termQuery.getTerm().field(), equalTo(TypeFieldMapper.NAME)); - assertThat(termQuery.getTerm().text(), equalTo(queryBuilder.type())); + assertEquals(new TypeFieldMapper.TypeQuery(new BytesRef(queryBuilder.type())), query); } public void testIllegalArgument() { diff --git a/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java b/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java index 72f58f104e..1839df5dd3 100644 --- a/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java +++ b/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java @@ -37,18 +37,22 @@ import org.elasticsearch.action.admin.indices.stats.CommonStats; import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags; import org.elasticsearch.action.admin.indices.stats.IndexStats; import org.elasticsearch.action.admin.indices.stats.ShardStats; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.index.TransportIndexAction; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.cluster.ClusterInfoService; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.InternalClusterInfoService; import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.metadata.SnapshotId; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.routing.AllocationId; import org.elasticsearch.cluster.routing.RestoreSource; import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.cluster.routing.ShardRoutingHelper; +import org.elasticsearch.cluster.routing.ShardRoutingState; import org.elasticsearch.cluster.routing.TestShardRouting; import org.elasticsearch.cluster.routing.UnassignedInfo; import org.elasticsearch.cluster.service.ClusterService; @@ -59,7 +63,6 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.lease.Releasable; import org.elasticsearch.common.lease.Releasables; import org.elasticsearch.common.logging.ESLogger; -import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.DummyTransportAddress; import org.elasticsearch.common.unit.ByteSizeUnit; @@ -119,6 +122,7 @@ import java.util.concurrent.atomic.AtomicInteger; import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS; import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS; import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_VERSION_CREATED; +import static org.elasticsearch.common.lucene.Lucene.cleanLuceneIndex; import static org.elasticsearch.common.settings.Settings.settingsBuilder; import static org.elasticsearch.common.xcontent.ToXContent.EMPTY_PARAMS; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; @@ -127,6 +131,7 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcke import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchHits; +import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; @@ -168,6 +173,7 @@ public class IndexShardTests extends ESSingleNodeTestCase { createIndex("test"); ensureGreen(); NodeEnvironment env = getInstanceFromNode(NodeEnvironment.class); + ClusterService cs = getInstanceFromNode(ClusterService.class); final Index index = cs.state().metaData().index("test").getIndex(); Path[] shardPaths = env.availableShardPaths(new ShardId(index, 0)); @@ -295,31 +301,133 @@ public class IndexShardTests extends ESSingleNodeTestCase { // expected } try { - indexShard.acquireReplicaOperationLock(); + indexShard.acquireReplicaOperationLock(indexShard.getPrimaryTerm()); fail("we should not be able to increment anymore"); } catch (IndexShardClosedException e) { // expected } } - public void testIndexOperationsCounter() throws InterruptedException, ExecutionException, IOException { + public void testOperationLocksOnPrimaryShards() throws InterruptedException, ExecutionException, IOException { assertAcked(client().admin().indices().prepareCreate("test").setSettings(Settings.builder().put("index.number_of_shards", 1).put("index.number_of_replicas", 0)).get()); ensureGreen("test"); IndicesService indicesService = getInstanceFromNode(IndicesService.class); IndexService indexService = indicesService.indexServiceSafe(resolveIndex("test")); IndexShard indexShard = indexService.getShardOrNull(0); + long primaryTerm = indexShard.getPrimaryTerm(); + + ShardRouting temp = indexShard.routingEntry(); + final ShardRouting newPrimaryShardRouting; + if (randomBoolean()) { + // relocation target + newPrimaryShardRouting = TestShardRouting.newShardRouting(temp.index(), temp.id(), temp.currentNodeId(), "other node", + true, ShardRoutingState.INITIALIZING, AllocationId.newRelocation(temp.allocationId())); + } else if (randomBoolean()) { + // simulate promotion + ShardRouting newReplicaShardRouting = TestShardRouting.newShardRouting(temp.index(), temp.id(), temp.currentNodeId(), null, + false, ShardRoutingState.STARTED, temp.allocationId()); + indexShard.updateRoutingEntry(newReplicaShardRouting, false); + primaryTerm = primaryTerm + 1; + indexShard.updatePrimaryTerm(primaryTerm); + newPrimaryShardRouting = TestShardRouting.newShardRouting(temp.index(), temp.id(), temp.currentNodeId(), null, + true, ShardRoutingState.STARTED, temp.allocationId()); + } else { + newPrimaryShardRouting = temp; + } + indexShard.updateRoutingEntry(newPrimaryShardRouting, false); + assertEquals(0, indexShard.getActiveOperationsCount()); + if (newPrimaryShardRouting.isRelocationTarget() == false) { + try { + indexShard.acquireReplicaOperationLock(primaryTerm); + fail("shard shouldn't accept operations as replica"); + } catch (IllegalStateException ignored) { + + } + } Releasable operation1 = indexShard.acquirePrimaryOperationLock(); assertEquals(1, indexShard.getActiveOperationsCount()); Releasable operation2 = indexShard.acquirePrimaryOperationLock(); assertEquals(2, indexShard.getActiveOperationsCount()); + + Releasables.close(operation1, operation2); + assertEquals(0, indexShard.getActiveOperationsCount()); + } + + public void testOperationLocksOnReplicaShards() throws InterruptedException, ExecutionException, IOException { + assertAcked(client().admin().indices().prepareCreate("test").setSettings(Settings.builder().put("index.number_of_shards", 1).put("index.number_of_replicas", 0)).get()); + ensureGreen("test"); + IndicesService indicesService = getInstanceFromNode(IndicesService.class); + IndexService indexService = indicesService.indexServiceSafe(resolveIndex("test")); + IndexShard indexShard = indexService.getShardOrNull(0); + long primaryTerm = indexShard.getPrimaryTerm(); + + // ugly hack to allow the shard to operated as a replica + final ShardRouting temp = indexShard.routingEntry(); + final ShardRouting newShardRouting; + switch (randomInt(2)) { + case 0: + // started replica + newShardRouting = TestShardRouting.newShardRouting(temp.index(), temp.id(), temp.currentNodeId(), null, + false, ShardRoutingState.STARTED, AllocationId.newRelocation(temp.allocationId())); + + indexShard.updateRoutingEntry(newShardRouting, false); + break; + case 1: + // initializing replica / primary + final boolean relocating = randomBoolean(); + newShardRouting = TestShardRouting.newShardRouting(temp.index(), temp.id(), temp.currentNodeId(), + relocating ? "sourceNode" : null, + relocating ? randomBoolean() : false, + ShardRoutingState.INITIALIZING, + relocating ? AllocationId.newRelocation(temp.allocationId()) : temp.allocationId()); + indexShard.updateRoutingEntry(newShardRouting, false); + break; + case 2: + // relocation source + newShardRouting = TestShardRouting.newShardRouting(temp.index(), temp.id(), temp.currentNodeId(), "otherNode", + false, ShardRoutingState.RELOCATING, AllocationId.newRelocation(temp.allocationId())); + indexShard.updateRoutingEntry(newShardRouting, false); + indexShard.relocated("test"); + break; + default: + throw new UnsupportedOperationException("get your numbers straight"); + + } + logger.info("updated shard routing to {}", newShardRouting); + + assertEquals(0, indexShard.getActiveOperationsCount()); + if (newShardRouting.primary() == false) { + try { + indexShard.acquirePrimaryOperationLock(); + fail("shard shouldn't accept primary ops"); + } catch (IllegalStateException ignored) { + + } + } + + Releasable operation1 = indexShard.acquireReplicaOperationLock(primaryTerm); + assertEquals(1, indexShard.getActiveOperationsCount()); + Releasable operation2 = indexShard.acquireReplicaOperationLock(primaryTerm); + assertEquals(2, indexShard.getActiveOperationsCount()); + + try { + indexShard.acquireReplicaOperationLock(primaryTerm - 1); + fail("you can not increment the operation counter with an older primary term"); + } catch (IllegalArgumentException e) { + assertThat(e.getMessage(), containsString("operation term")); + assertThat(e.getMessage(), containsString("too old")); + } + + // but you can increment with a newer one.. + indexShard.acquireReplicaOperationLock(primaryTerm + 1 + randomInt(20)).close(); Releasables.close(operation1, operation2); assertEquals(0, indexShard.getActiveOperationsCount()); } public void testMarkAsInactiveTriggersSyncedFlush() throws Exception { assertAcked(client().admin().indices().prepareCreate("test") - .setSettings(SETTING_NUMBER_OF_SHARDS, 1, SETTING_NUMBER_OF_REPLICAS, 0)); + .setSettings(SETTING_NUMBER_OF_SHARDS, 1, SETTING_NUMBER_OF_REPLICAS, 0)); client().prepareIndex("test", "test").setSource("{}").get(); ensureGreen("test"); IndicesService indicesService = getInstanceFromNode(IndicesService.class); @@ -364,14 +472,14 @@ public class IndexShardTests extends ESSingleNodeTestCase { assertTrue(shard.getEngine().getTranslog().syncNeeded()); setDurability(shard, Translog.Durability.REQUEST); assertNoFailures(client().prepareBulk() - .add(client().prepareIndex("test", "bar", "3").setSource("{}")) - .add(client().prepareDelete("test", "bar", "1")).get()); + .add(client().prepareIndex("test", "bar", "3").setSource("{}")) + .add(client().prepareDelete("test", "bar", "1")).get()); assertFalse(shard.getEngine().getTranslog().syncNeeded()); setDurability(shard, Translog.Durability.ASYNC); assertNoFailures(client().prepareBulk() - .add(client().prepareIndex("test", "bar", "4").setSource("{}")) - .add(client().prepareDelete("test", "bar", "3")).get()); + .add(client().prepareIndex("test", "bar", "4").setSource("{}")) + .add(client().prepareDelete("test", "bar", "3")).get()); setDurability(shard, Translog.Durability.REQUEST); assertTrue(shard.getEngine().getTranslog().syncNeeded()); } @@ -384,7 +492,7 @@ public class IndexShardTests extends ESSingleNodeTestCase { public void testMinimumCompatVersion() { Version versionCreated = VersionUtils.randomVersion(random()); assertAcked(client().admin().indices().prepareCreate("test") - .setSettings(SETTING_NUMBER_OF_SHARDS, 1, SETTING_NUMBER_OF_REPLICAS, 0, SETTING_VERSION_CREATED, versionCreated.id)); + .setSettings(SETTING_NUMBER_OF_SHARDS, 1, SETTING_NUMBER_OF_REPLICAS, 0, SETTING_VERSION_CREATED, versionCreated.id)); client().prepareIndex("test", "test").setSource("{}").get(); ensureGreen("test"); IndicesService indicesService = getInstanceFromNode(IndicesService.class); @@ -398,7 +506,7 @@ public class IndexShardTests extends ESSingleNodeTestCase { public void testUpdatePriority() { assertAcked(client().admin().indices().prepareCreate("test") - .setSettings(IndexMetaData.SETTING_PRIORITY, 200)); + .setSettings(IndexMetaData.SETTING_PRIORITY, 200)); IndexService indexService = getInstanceFromNode(IndicesService.class).indexService(resolveIndex("test")); assertEquals(200, indexService.getIndexSettings().getSettings().getAsInt(IndexMetaData.SETTING_PRIORITY, 0).intValue()); client().admin().indices().prepareUpdateSettings("test").setSettings(Settings.builder().put(IndexMetaData.SETTING_PRIORITY, 400).build()).get(); @@ -434,8 +542,8 @@ public class IndexShardTests extends ESSingleNodeTestCase { Path idxPath = env.sharedDataFile().resolve(randomAsciiOfLength(10)); logger.info("--> idxPath: [{}]", idxPath); Settings idxSettings = Settings.builder() - .put(IndexMetaData.SETTING_DATA_PATH, idxPath) - .build(); + .put(IndexMetaData.SETTING_DATA_PATH, idxPath) + .build(); createIndex("test", idxSettings); ensureGreen("test"); client().prepareIndex("test", "bar", "1").setSource("{}").setRefresh(true).get(); @@ -447,7 +555,7 @@ public class IndexShardTests extends ESSingleNodeTestCase { public void testExpectedShardSizeIsPresent() throws InterruptedException { assertAcked(client().admin().indices().prepareCreate("test") - .setSettings(SETTING_NUMBER_OF_SHARDS, 1, SETTING_NUMBER_OF_REPLICAS, 0)); + .setSettings(SETTING_NUMBER_OF_SHARDS, 1, SETTING_NUMBER_OF_REPLICAS, 0)); for (int i = 0; i < 50; i++) { client().prepareIndex("test", "test").setSource("{}").get(); } @@ -475,11 +583,11 @@ public class IndexShardTests extends ESSingleNodeTestCase { IOUtils.rm(endDir); Settings sb = Settings.builder() - .put(IndexMetaData.SETTING_DATA_PATH, startDir.toAbsolutePath().toString()) - .build(); + .put(IndexMetaData.SETTING_DATA_PATH, startDir.toAbsolutePath().toString()) + .build(); Settings sb2 = Settings.builder() - .put(IndexMetaData.SETTING_DATA_PATH, endDir.toAbsolutePath().toString()) - .build(); + .put(IndexMetaData.SETTING_DATA_PATH, endDir.toAbsolutePath().toString()) + .build(); logger.info("--> creating an index with data_path [{}]", startDir.toAbsolutePath().toString()); createIndex(INDEX, sb); @@ -510,9 +618,9 @@ public class IndexShardTests extends ESSingleNodeTestCase { logger.info("--> updating settings..."); client().admin().indices().prepareUpdateSettings(INDEX) - .setSettings(sb2) - .setIndicesOptions(IndicesOptions.fromOptions(true, false, true, true)) - .get(); + .setSettings(sb2) + .setIndicesOptions(IndicesOptions.fromOptions(true, false, true, true)) + .get(); assert Files.exists(startDir) == false : "start dir shouldn't exist"; @@ -642,7 +750,7 @@ public class IndexShardTests extends ESSingleNodeTestCase { try { shard.index(index); fail(); - }catch (IllegalIndexShardStateException e){ + } catch (IllegalIndexShardStateException e) { } @@ -655,7 +763,7 @@ public class IndexShardTests extends ESSingleNodeTestCase { try { shard.delete(delete); fail(); - }catch (IllegalIndexShardStateException e){ + } catch (IllegalIndexShardStateException e) { } @@ -692,7 +800,7 @@ public class IndexShardTests extends ESSingleNodeTestCase { long size = shard.getEngine().getTranslog().sizeInBytes(); logger.info("--> current translog size: [{}] num_ops [{}] generation [{}]", shard.getEngine().getTranslog().sizeInBytes(), shard.getEngine().getTranslog().totalOperations(), shard.getEngine().getTranslog().getGeneration()); client().admin().indices().prepareUpdateSettings("test").setSettings(settingsBuilder().put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), new ByteSizeValue(size, ByteSizeUnit.BYTES)) - .build()).get(); + .build()).get(); client().prepareDelete("test", "test", "2").get(); logger.info("--> translog size after delete: [{}] num_ops [{}] generation [{}]", shard.getEngine().getTranslog().sizeInBytes(), shard.getEngine().getTranslog().totalOperations(), shard.getEngine().getTranslog().getGeneration()); assertBusy(() -> { // this is async @@ -877,7 +985,7 @@ public class IndexShardTests extends ESSingleNodeTestCase { newShard.updateRoutingEntry(routing, false); DiscoveryNode localNode = new DiscoveryNode("foo", DummyTransportAddress.INSTANCE, Version.CURRENT); newShard.markAsRecovering("store", new RecoveryState(newShard.shardId(), routing.primary(), RecoveryState.Type.STORE, localNode, - localNode)); + localNode)); assertTrue(newShard.recoverFromStore(localNode)); assertEquals(0, newShard.recoveryState().getTranslog().recoveredOperations()); assertEquals(0, newShard.recoveryState().getTranslog().totalOperations()); @@ -890,7 +998,7 @@ public class IndexShardTests extends ESSingleNodeTestCase { assertHitCount(response, 0); } - public void testFailIfIndexNotPresentInRecoverFromStore() throws IOException { + public void testFailIfIndexNotPresentInRecoverFromStore() throws Exception { createIndex("test"); ensureGreen(); IndicesService indicesService = getInstanceFromNode(IndicesService.class); @@ -907,7 +1015,7 @@ public class IndexShardTests extends ESSingleNodeTestCase { Store store = shard.store(); store.incRef(); test.removeShard(0, "b/c simon says so"); - Lucene.cleanLuceneIndex(store.directory()); + cleanLuceneIndex(store.directory()); store.decRef(); ShardRoutingHelper.reinit(routing); IndexShard newShard = test.createShard(routing); @@ -940,7 +1048,12 @@ public class IndexShardTests extends ESSingleNodeTestCase { newShard.updateRoutingEntry(routing, true); SearchResponse response = client().prepareSearch().get(); assertHitCount(response, 0); - client().prepareIndex("test", "test", "0").setSource("{}").setRefresh(true).get(); + // we can't issue this request through a client because of the inconsistencies we created with the cluster state + // doing it directly instead + IndexRequest request = client().prepareIndex("test", "test", "0").setSource("{}").request(); + request.process(MetaData.builder().put(test.getMetaData(), false).build(), null, false, "test"); + TransportIndexAction.executeIndexRequestOnPrimary(request, newShard, null); + newShard.refresh("test"); assertHitCount(client().prepareSearch().get(), 1); } @@ -999,7 +1112,7 @@ public class IndexShardTests extends ESSingleNodeTestCase { @Override public void restore(SnapshotId snapshotId, Version version, ShardId shardId, ShardId snapshotShardId, RecoveryState recoveryState) { try { - Lucene.cleanLuceneIndex(targetStore.directory()); + cleanLuceneIndex(targetStore.directory()); for (String file : sourceStore.directory().listAll()) { if (file.equals("write.lock") || file.startsWith("extra")) { continue; @@ -1205,12 +1318,12 @@ public class IndexShardTests extends ESSingleNodeTestCase { public void testIndexingBufferDuringInternalRecovery() throws IOException { createIndex("index"); client().admin().indices().preparePutMapping("index").setType("testtype").setSource(jsonBuilder().startObject() - .startObject("testtype") - .startObject("properties") - .startObject("foo") - .field("type", "text") - .endObject() - .endObject().endObject().endObject()).get(); + .startObject("testtype") + .startObject("properties") + .startObject("foo") + .field("type", "text") + .endObject() + .endObject().endObject().endObject()).get(); ensureGreen(); IndicesService indicesService = getInstanceFromNode(IndicesService.class); IndexService test = indicesService.indexService(resolveIndex("index")); @@ -1234,12 +1347,12 @@ public class IndexShardTests extends ESSingleNodeTestCase { public void testIndexingBufferDuringPeerRecovery() throws IOException { createIndex("index"); client().admin().indices().preparePutMapping("index").setType("testtype").setSource(jsonBuilder().startObject() - .startObject("testtype") - .startObject("properties") - .startObject("foo") - .field("type", "text") - .endObject() - .endObject().endObject().endObject()).get(); + .startObject("testtype") + .startObject("properties") + .startObject("foo") + .field("type", "text") + .endObject() + .endObject().endObject().endObject()).get(); ensureGreen(); IndicesService indicesService = getInstanceFromNode(IndicesService.class); IndexService test = indicesService.indexService(resolveIndex("index")); diff --git a/core/src/test/java/org/elasticsearch/indices/recovery/IndexPrimaryRelocationIT.java b/core/src/test/java/org/elasticsearch/indices/recovery/IndexPrimaryRelocationIT.java index d105c98f4a..322ed269bc 100644 --- a/core/src/test/java/org/elasticsearch/indices/recovery/IndexPrimaryRelocationIT.java +++ b/core/src/test/java/org/elasticsearch/indices/recovery/IndexPrimaryRelocationIT.java @@ -40,6 +40,7 @@ public class IndexPrimaryRelocationIT extends ESIntegTestCase { private static final int RELOCATION_COUNT = 25; + @TestLogging("_root:DEBUG,action.delete:TRACE,action.index:TRACE,index.shard:TRACE,cluster.service:TRACE") public void testPrimaryRelocationWhileIndexing() throws Exception { internalCluster().ensureAtLeastNumDataNodes(randomIntBetween(2, 3)); client().admin().indices().prepareCreate("test") diff --git a/core/src/test/java/org/elasticsearch/ingest/processor/TrackingResultProcessorTests.java b/core/src/test/java/org/elasticsearch/ingest/processor/TrackingResultProcessorTests.java new file mode 100644 index 0000000000..e53eec56cf --- /dev/null +++ b/core/src/test/java/org/elasticsearch/ingest/processor/TrackingResultProcessorTests.java @@ -0,0 +1,129 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.ingest.processor; + +import org.elasticsearch.action.ingest.SimulateProcessorResult; +import org.elasticsearch.ingest.TestProcessor; +import org.elasticsearch.ingest.core.CompoundProcessor; +import org.elasticsearch.ingest.core.IngestDocument; +import org.elasticsearch.test.ESTestCase; +import org.junit.Before; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.elasticsearch.ingest.core.CompoundProcessor.ON_FAILURE_MESSAGE_FIELD; +import static org.elasticsearch.ingest.core.CompoundProcessor.ON_FAILURE_PROCESSOR_TAG_FIELD; +import static org.elasticsearch.ingest.core.CompoundProcessor.ON_FAILURE_PROCESSOR_TYPE_FIELD; +import static org.elasticsearch.ingest.processor.TrackingResultProcessor.decorate; +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.Matchers.nullValue; + +public class TrackingResultProcessorTests extends ESTestCase { + + private IngestDocument ingestDocument; + private List<SimulateProcessorResult> resultList; + + @Before + public void init() { + ingestDocument = new IngestDocument(new HashMap<>(), new HashMap<>()); + resultList = new ArrayList<>(); + } + + public void testActualProcessor() throws Exception { + TestProcessor actualProcessor = new TestProcessor(ingestDocument -> {}); + TrackingResultProcessor trackingProcessor = new TrackingResultProcessor(actualProcessor, resultList); + trackingProcessor.execute(ingestDocument); + + SimulateProcessorResult expectedResult = new SimulateProcessorResult(actualProcessor.getTag(), ingestDocument); + + assertThat(actualProcessor.getInvokedCounter(), equalTo(1)); + assertThat(resultList.size(), equalTo(1)); + + assertThat(resultList.get(0).getIngestDocument(), equalTo(expectedResult.getIngestDocument())); + assertThat(resultList.get(0).getFailure(), nullValue()); + assertThat(resultList.get(0).getProcessorTag(), equalTo(expectedResult.getProcessorTag())); + } + + public void testActualCompoundProcessorWithoutOnFailure() throws Exception { + RuntimeException exception = new RuntimeException("processor failed"); + TestProcessor testProcessor = new TestProcessor(ingestDocument -> { throw exception; }); + CompoundProcessor actualProcessor = new CompoundProcessor(testProcessor); + CompoundProcessor trackingProcessor = decorate(actualProcessor, resultList); + + try { + trackingProcessor.execute(ingestDocument); + } catch (Exception e) { + assertThat(e.getMessage(), equalTo(exception.getMessage())); + } + + SimulateProcessorResult expectedFirstResult = new SimulateProcessorResult(testProcessor.getTag(), ingestDocument); + assertThat(testProcessor.getInvokedCounter(), equalTo(1)); + assertThat(resultList.size(), equalTo(1)); + assertThat(resultList.get(0).getIngestDocument(), nullValue()); + assertThat(resultList.get(0).getFailure(), equalTo(exception)); + assertThat(resultList.get(0).getProcessorTag(), equalTo(expectedFirstResult.getProcessorTag())); + } + + public void testActualCompoundProcessorWithOnFailure() throws Exception { + RuntimeException exception = new RuntimeException("fail"); + TestProcessor failProcessor = new TestProcessor("fail", "test", ingestDocument -> { throw exception; }); + TestProcessor onFailureProcessor = new TestProcessor("success", "test", ingestDocument -> {}); + CompoundProcessor actualProcessor = new CompoundProcessor( + Arrays.asList(new CompoundProcessor( + Arrays.asList(failProcessor, onFailureProcessor), + Arrays.asList(onFailureProcessor, failProcessor))), + Arrays.asList(onFailureProcessor)); + CompoundProcessor trackingProcessor = decorate(actualProcessor, resultList); + trackingProcessor.execute(ingestDocument); + + SimulateProcessorResult expectedFailResult = new SimulateProcessorResult(failProcessor.getTag(), ingestDocument); + SimulateProcessorResult expectedSuccessResult = new SimulateProcessorResult(onFailureProcessor.getTag(), ingestDocument); + + assertThat(failProcessor.getInvokedCounter(), equalTo(2)); + assertThat(onFailureProcessor.getInvokedCounter(), equalTo(2)); + assertThat(resultList.size(), equalTo(4)); + + assertThat(resultList.get(0).getIngestDocument(), nullValue()); + assertThat(resultList.get(0).getFailure(), equalTo(exception)); + assertThat(resultList.get(0).getProcessorTag(), equalTo(expectedFailResult.getProcessorTag())); + + Map<String, String> metadata = resultList.get(1).getIngestDocument().getIngestMetadata(); + assertThat(metadata.get(ON_FAILURE_MESSAGE_FIELD), equalTo("fail")); + assertThat(metadata.get(ON_FAILURE_PROCESSOR_TYPE_FIELD), equalTo("test")); + assertThat(metadata.get(ON_FAILURE_PROCESSOR_TAG_FIELD), equalTo("fail")); + assertThat(resultList.get(1).getFailure(), nullValue()); + assertThat(resultList.get(1).getProcessorTag(), equalTo(expectedSuccessResult.getProcessorTag())); + + assertThat(resultList.get(2).getIngestDocument(), nullValue()); + assertThat(resultList.get(2).getFailure(), equalTo(exception)); + assertThat(resultList.get(2).getProcessorTag(), equalTo(expectedFailResult.getProcessorTag())); + + metadata = resultList.get(3).getIngestDocument().getIngestMetadata(); + assertThat(metadata.get(ON_FAILURE_MESSAGE_FIELD), equalTo("fail")); + assertThat(metadata.get(ON_FAILURE_PROCESSOR_TYPE_FIELD), equalTo("compound")); + assertThat(metadata.get(ON_FAILURE_PROCESSOR_TAG_FIELD), equalTo("CompoundProcessor-fail-success-success-fail")); + assertThat(resultList.get(3).getFailure(), nullValue()); + assertThat(resultList.get(3).getProcessorTag(), equalTo(expectedSuccessResult.getProcessorTag())); + } +} diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/BaseAggregationTestCase.java b/core/src/test/java/org/elasticsearch/search/aggregations/BaseAggregationTestCase.java index 4a12072da4..0d65bb2bf6 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/BaseAggregationTestCase.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/BaseAggregationTestCase.java @@ -252,9 +252,8 @@ public abstract class BaseAggregationTestCase<AB extends AggregatorBuilder<AB>> try (BytesStreamOutput output = new BytesStreamOutput()) { testAgg.writeTo(output); try (StreamInput in = new NamedWriteableAwareStreamInput(StreamInput.wrap(output.bytes()), namedWriteableRegistry)) { - AggregatorBuilder prototype = (AggregatorBuilder) namedWriteableRegistry.getPrototype(AggregatorBuilder.class, - testAgg.getWriteableName()); - AggregatorBuilder deserializedQuery = prototype.readFrom(in); + AggregatorBuilder deserializedQuery = namedWriteableRegistry.getReader(AggregatorBuilder.class, testAgg.getWriteableName()) + .read(in); assertEquals(deserializedQuery, testAgg); assertEquals(deserializedQuery.hashCode(), testAgg.hashCode()); assertNotSame(deserializedQuery, testAgg); @@ -294,10 +293,8 @@ public abstract class BaseAggregationTestCase<AB extends AggregatorBuilder<AB>> try (BytesStreamOutput output = new BytesStreamOutput()) { agg.writeTo(output); try (StreamInput in = new NamedWriteableAwareStreamInput(StreamInput.wrap(output.bytes()), namedWriteableRegistry)) { - AggregatorBuilder prototype = (AggregatorBuilder) namedWriteableRegistry.getPrototype(AggregatorBuilder.class, - agg.getWriteableName()); @SuppressWarnings("unchecked") - AB secondAgg = (AB) prototype.readFrom(in); + AB secondAgg = (AB) namedWriteableRegistry.getReader(AggregatorBuilder.class, agg.getWriteableName()).read(in); return secondAgg; } } diff --git a/core/src/test/java/org/elasticsearch/search/highlight/HighlighterSearchIT.java b/core/src/test/java/org/elasticsearch/search/highlight/HighlighterSearchIT.java index dbe2714d05..fa8caaf0ed 100644 --- a/core/src/test/java/org/elasticsearch/search/highlight/HighlighterSearchIT.java +++ b/core/src/test/java/org/elasticsearch/search/highlight/HighlighterSearchIT.java @@ -564,6 +564,7 @@ public class HighlighterSearchIT extends ESIntegTestCase { .startObject("properties") .startObject("field1").field("type", "text").field("store", true).field("index_options", "offsets") .field("term_vector", "with_positions_offsets").endObject() + .startObject("field2").field("type", "text").endObject() .endObject().endObject().endObject())); ensureGreen(); diff --git a/core/src/test/java/org/elasticsearch/search/percolator/PercolatorQuerySearchIT.java b/core/src/test/java/org/elasticsearch/search/percolator/PercolatorQuerySearchIT.java index 55f2ab8012..fc2a6d02b8 100644 --- a/core/src/test/java/org/elasticsearch/search/percolator/PercolatorQuerySearchIT.java +++ b/core/src/test/java/org/elasticsearch/search/percolator/PercolatorQuerySearchIT.java @@ -21,15 +21,17 @@ package org.elasticsearch.search.percolator; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.index.percolator.PercolatorFieldMapper; +import org.elasticsearch.index.query.MultiMatchQueryBuilder; import org.elasticsearch.search.highlight.HighlightBuilder; import org.elasticsearch.search.sort.SortOrder; -import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESSingleNodeTestCase; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.index.query.QueryBuilders.boolQuery; +import static org.elasticsearch.index.query.QueryBuilders.commonTermsQuery; import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; import static org.elasticsearch.index.query.QueryBuilders.matchQuery; +import static org.elasticsearch.index.query.QueryBuilders.multiMatchQuery; import static org.elasticsearch.index.query.QueryBuilders.percolatorQuery; import static org.elasticsearch.index.query.QueryBuilders.termQuery; import static org.hamcrest.Matchers.equalTo; @@ -85,6 +87,32 @@ public class PercolatorQuerySearchIT extends ESSingleNodeTestCase { assertThat(response.getHits().getAt(2).getId(), equalTo("3")); } + public void testPercolatorSpecificQueries() throws Exception { + createIndex("test", client().admin().indices().prepareCreate("test") + .addMapping("type", "field1", "type=text", "field2", "type=text") + ); + + client().prepareIndex("test", PercolatorFieldMapper.TYPE_NAME, "1") + .setSource(jsonBuilder().startObject().field("query", commonTermsQuery("field1", "quick brown fox")).endObject()) + .get(); + client().prepareIndex("test", PercolatorFieldMapper.TYPE_NAME, "2") + .setSource(jsonBuilder().startObject().field("query", multiMatchQuery("quick brown fox", "field1", "field2") + .type(MultiMatchQueryBuilder.Type.CROSS_FIELDS)).endObject()) + .get(); + client().admin().indices().prepareRefresh().get(); + + BytesReference source = jsonBuilder().startObject() + .field("field1", "the quick brown fox jumps over the lazy dog") + .field("field2", "the quick brown fox falls down into the well") + .endObject().bytes(); + SearchResponse response = client().prepareSearch() + .setQuery(percolatorQuery("type", source)) + .get(); + assertHitCount(response, 2); + assertThat(response.getHits().getAt(0).getId(), equalTo("1")); + assertThat(response.getHits().getAt(1).getId(), equalTo("2")); + } + public void testPercolatorQueryWithHighlighting() throws Exception { createIndex("test", client().admin().indices().prepareCreate("test") .addMapping("type", "field1", "type=text") @@ -125,7 +153,7 @@ public class PercolatorQuerySearchIT extends ESSingleNodeTestCase { assertThat(searchResponse.getHits().getAt(3).getHighlightFields().get("field1").fragments()[0].string(), equalTo("The quick brown fox jumps over the lazy <em>dog</em>")); assertThat(searchResponse.getHits().getAt(4).getHighlightFields().get("field1").fragments()[0].string(), - equalTo("The quick brown <em>fox</em> jumps over the lazy dog"));; + equalTo("The quick brown <em>fox</em> jumps over the lazy dog")); } } diff --git a/core/src/test/java/org/elasticsearch/search/rescore/QueryRescoreBuilderTests.java b/core/src/test/java/org/elasticsearch/search/rescore/QueryRescoreBuilderTests.java index e0673a64ee..0386dd847f 100644 --- a/core/src/test/java/org/elasticsearch/search/rescore/QueryRescoreBuilderTests.java +++ b/core/src/test/java/org/elasticsearch/search/rescore/QueryRescoreBuilderTests.java @@ -69,7 +69,6 @@ public class QueryRescoreBuilderTests extends ESTestCase { @BeforeClass public static void init() { namedWriteableRegistry = new NamedWriteableRegistry(); - namedWriteableRegistry.registerPrototype(RescoreBuilder.class, QueryRescorerBuilder.PROTOTYPE); indicesQueriesRegistry = new SearchModule(Settings.EMPTY, namedWriteableRegistry).buildQueryParserRegistry(); } diff --git a/core/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java b/core/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java index 935dfb178b..7cc44f22a2 100644 --- a/core/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java +++ b/core/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java @@ -100,10 +100,6 @@ public abstract class AbstractSortTestCase<T extends SortBuilder<T>> extends EST }; namedWriteableRegistry = new NamedWriteableRegistry(); - namedWriteableRegistry.registerPrototype(SortBuilder.class, GeoDistanceSortBuilder.PROTOTYPE); - namedWriteableRegistry.registerPrototype(SortBuilder.class, ScoreSortBuilder.PROTOTYPE); - namedWriteableRegistry.registerPrototype(SortBuilder.class, ScriptSortBuilder.PROTOTYPE); - namedWriteableRegistry.registerPrototype(SortBuilder.class, FieldSortBuilder.PROTOTYPE); indicesQueriesRegistry = new SearchModule(Settings.EMPTY, namedWriteableRegistry).buildQueryParserRegistry(); } @@ -118,6 +114,9 @@ public abstract class AbstractSortTestCase<T extends SortBuilder<T>> extends EST /** Returns mutated version of original so the returned sort is different in terms of equals/hashcode */ protected abstract T mutate(T original) throws IOException; + /** Parse the sort from xContent. Just delegate to the SortBuilder's static fromXContent method. */ + protected abstract T fromXContent(QueryParseContext context, String fieldName) throws IOException; + /** * Test that creates new sort from a random test sort and checks both for equality */ @@ -142,7 +141,7 @@ public abstract class AbstractSortTestCase<T extends SortBuilder<T>> extends EST QueryParseContext context = new QueryParseContext(indicesQueriesRegistry); context.reset(itemParser); - T parsedItem = testItem.fromXContent(context, elementName); + T parsedItem = fromXContent(context, elementName); assertNotSame(testItem, parsedItem); assertEquals(testItem, parsedItem); assertEquals(testItem.hashCode(), parsedItem.hashCode()); @@ -255,10 +254,7 @@ public abstract class AbstractSortTestCase<T extends SortBuilder<T>> extends EST try (BytesStreamOutput output = new BytesStreamOutput()) { original.writeTo(output); try (StreamInput in = new NamedWriteableAwareStreamInput(StreamInput.wrap(output.bytes()), namedWriteableRegistry)) { - T prototype = (T) namedWriteableRegistry.getPrototype(SortBuilder.class, - original.getWriteableName()); - T copy = prototype.readFrom(in); - return copy; + return (T) namedWriteableRegistry.getReader(SortBuilder.class, original.getWriteableName()).read(in); } } } diff --git a/core/src/test/java/org/elasticsearch/search/sort/FieldSortBuilderTests.java b/core/src/test/java/org/elasticsearch/search/sort/FieldSortBuilderTests.java index f28ec8797c..f4d960aab0 100644 --- a/core/src/test/java/org/elasticsearch/search/sort/FieldSortBuilderTests.java +++ b/core/src/test/java/org/elasticsearch/search/sort/FieldSortBuilderTests.java @@ -131,4 +131,9 @@ public class FieldSortBuilderTests extends AbstractSortTestCase<FieldSortBuilder } } + + @Override + protected FieldSortBuilder fromXContent(QueryParseContext context, String fieldName) throws IOException { + return FieldSortBuilder.fromXContent(context, fieldName); + } } diff --git a/core/src/test/java/org/elasticsearch/search/sort/GeoDistanceSortBuilderTests.java b/core/src/test/java/org/elasticsearch/search/sort/GeoDistanceSortBuilderTests.java index e63cb5ad29..995623ed0a 100644 --- a/core/src/test/java/org/elasticsearch/search/sort/GeoDistanceSortBuilderTests.java +++ b/core/src/test/java/org/elasticsearch/search/sort/GeoDistanceSortBuilderTests.java @@ -243,12 +243,8 @@ public class GeoDistanceSortBuilderTests extends AbstractSortTestCase<GeoDistanc QueryParseContext context = new QueryParseContext(indicesQueriesRegistry); context.reset(itemParser); - try { - GeoDistanceSortBuilder.PROTOTYPE.fromXContent(context, ""); - fail("sort mode sum should not be supported"); - } catch (IllegalArgumentException e) { - // all good - } + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> GeoDistanceSortBuilder.fromXContent(context, "")); + assertEquals("sort_mode [sum] isn't supported for sorting by geo distance", e.getMessage()); } public void testGeoDistanceSortCanBeParsedFromGeoHash() throws IOException { @@ -274,7 +270,7 @@ public class GeoDistanceSortBuilderTests extends AbstractSortTestCase<GeoDistanc QueryParseContext context = new QueryParseContext(indicesQueriesRegistry); context.reset(itemParser); - GeoDistanceSortBuilder result = GeoDistanceSortBuilder.PROTOTYPE.fromXContent(context, json); + GeoDistanceSortBuilder result = GeoDistanceSortBuilder.fromXContent(context, json); assertEquals("[-19.700583312660456, -2.8225036337971687, " + "31.537466906011105, -74.63590376079082, " + "43.71844606474042, -5.548660643398762, " @@ -282,4 +278,9 @@ public class GeoDistanceSortBuilderTests extends AbstractSortTestCase<GeoDistanc + "-69.44606635719538, 84.25200328230858, " + "-39.03717711567879, 44.74099852144718]", Arrays.toString(result.points())); } + + @Override + protected GeoDistanceSortBuilder fromXContent(QueryParseContext context, String fieldName) throws IOException { + return GeoDistanceSortBuilder.fromXContent(context, fieldName); + } } diff --git a/core/src/test/java/org/elasticsearch/search/sort/ScoreSortBuilderTests.java b/core/src/test/java/org/elasticsearch/search/sort/ScoreSortBuilderTests.java index 6d605fd625..372d984539 100644 --- a/core/src/test/java/org/elasticsearch/search/sort/ScoreSortBuilderTests.java +++ b/core/src/test/java/org/elasticsearch/search/sort/ScoreSortBuilderTests.java @@ -78,7 +78,7 @@ public class ScoreSortBuilderTests extends AbstractSortTestCase<ScoreSortBuilder parser.nextToken(); context.reset(parser); - ScoreSortBuilder scoreSort = ScoreSortBuilder.PROTOTYPE.fromXContent(context, "_score"); + ScoreSortBuilder scoreSort = ScoreSortBuilder.fromXContent(context, "_score"); assertEquals(order, scoreSort.order()); } @@ -107,4 +107,9 @@ public class ScoreSortBuilderTests extends AbstractSortTestCase<ScoreSortBuilder assertEquals(SortField.Type.SCORE, sortField.getType()); assertEquals(builder.order() == SortOrder.DESC ? false : true, sortField.getReverse()); } + + @Override + protected ScoreSortBuilder fromXContent(QueryParseContext context, String fieldName) throws IOException { + return ScoreSortBuilder.fromXContent(context, fieldName); + } } diff --git a/core/src/test/java/org/elasticsearch/search/sort/ScriptSortBuilderTests.java b/core/src/test/java/org/elasticsearch/search/sort/ScriptSortBuilderTests.java index 5e22667a9b..0e04dc9e4d 100644 --- a/core/src/test/java/org/elasticsearch/search/sort/ScriptSortBuilderTests.java +++ b/core/src/test/java/org/elasticsearch/search/sort/ScriptSortBuilderTests.java @@ -180,7 +180,7 @@ public class ScriptSortBuilderTests extends AbstractSortTestCase<ScriptSortBuild parser.nextToken(); context.reset(parser); - ScriptSortBuilder builder = ScriptSortBuilder.PROTOTYPE.fromXContent(context, null); + ScriptSortBuilder builder = ScriptSortBuilder.fromXContent(context, null); assertEquals("doc['field_name'].value * factor", builder.script().getScript()); assertNull(builder.script().getLang()); assertEquals(1.1, builder.script().getParams().get("factor")); @@ -211,7 +211,7 @@ public class ScriptSortBuilderTests extends AbstractSortTestCase<ScriptSortBuild parser.nextToken(); context.reset(parser); - ScriptSortBuilder builder = ScriptSortBuilder.PROTOTYPE.fromXContent(context, null); + ScriptSortBuilder builder = ScriptSortBuilder.fromXContent(context, null); assertEquals("doc['field_name'].value * factor", builder.script().getScript()); assertNull(builder.script().getLang()); assertEquals(1.1, builder.script().getParams().get("factor")); @@ -235,7 +235,7 @@ public class ScriptSortBuilderTests extends AbstractSortTestCase<ScriptSortBuild context.reset(parser); exceptionRule.expect(ParsingException.class); exceptionRule.expectMessage("failed to parse field [bad_field]"); - ScriptSortBuilder.PROTOTYPE.fromXContent(context, null); + ScriptSortBuilder.fromXContent(context, null); } public void testParseBadFieldNameExceptionsOnStartObject() throws IOException { @@ -251,7 +251,7 @@ public class ScriptSortBuilderTests extends AbstractSortTestCase<ScriptSortBuild context.reset(parser); exceptionRule.expect(ParsingException.class); exceptionRule.expectMessage("failed to parse field [bad_field]"); - ScriptSortBuilder.PROTOTYPE.fromXContent(context, null); + ScriptSortBuilder.fromXContent(context, null); } public void testParseUnexpectedToken() throws IOException { @@ -267,7 +267,7 @@ public class ScriptSortBuilderTests extends AbstractSortTestCase<ScriptSortBuild context.reset(parser); exceptionRule.expect(ParsingException.class); exceptionRule.expectMessage("unexpected token [START_ARRAY]"); - ScriptSortBuilder.PROTOTYPE.fromXContent(context, null); + ScriptSortBuilder.fromXContent(context, null); } /** @@ -279,4 +279,9 @@ public class ScriptSortBuilderTests extends AbstractSortTestCase<ScriptSortBuild exceptionRule.expectMessage("script sort of type [string] doesn't support mode"); builder.sortMode(SortMode.fromString(randomFrom(new String[]{"avg", "median", "sum"}))); } + + @Override + protected ScriptSortBuilder fromXContent(QueryParseContext context, String fieldName) throws IOException { + return ScriptSortBuilder.fromXContent(context, fieldName); + } } diff --git a/core/src/test/java/org/elasticsearch/search/sort/SortBuilderTests.java b/core/src/test/java/org/elasticsearch/search/sort/SortBuilderTests.java index b8100b5815..7d182a5a8b 100644 --- a/core/src/test/java/org/elasticsearch/search/sort/SortBuilderTests.java +++ b/core/src/test/java/org/elasticsearch/search/sort/SortBuilderTests.java @@ -51,10 +51,6 @@ public class SortBuilderTests extends ESTestCase { @BeforeClass public static void init() { namedWriteableRegistry = new NamedWriteableRegistry(); - namedWriteableRegistry.registerPrototype(SortBuilder.class, GeoDistanceSortBuilder.PROTOTYPE); - namedWriteableRegistry.registerPrototype(SortBuilder.class, ScoreSortBuilder.PROTOTYPE); - namedWriteableRegistry.registerPrototype(SortBuilder.class, ScriptSortBuilder.PROTOTYPE); - namedWriteableRegistry.registerPrototype(SortBuilder.class, FieldSortBuilder.PROTOTYPE); indicesQueriesRegistry = new SearchModule(Settings.EMPTY, namedWriteableRegistry).buildQueryParserRegistry(); } diff --git a/core/src/test/java/org/elasticsearch/search/sort/SortOrderTests.java b/core/src/test/java/org/elasticsearch/search/sort/SortOrderTests.java index e505ec68e6..2de48decbd 100644 --- a/core/src/test/java/org/elasticsearch/search/sort/SortOrderTests.java +++ b/core/src/test/java/org/elasticsearch/search/sort/SortOrderTests.java @@ -38,7 +38,7 @@ public class SortOrderTests extends ESTestCase { try (BytesStreamOutput out = new BytesStreamOutput()) { unit.writeTo(out); try (StreamInput in = StreamInput.wrap(out.bytes())) { - assertThat("Roundtrip serialisation failed.", SortOrder.readOrderFrom(in), equalTo(unit)); + assertThat("Roundtrip serialisation failed.", SortOrder.readFromStream(in), equalTo(unit)); } } } diff --git a/core/src/test/java/org/elasticsearch/search/sort/SortParserTests.java b/core/src/test/java/org/elasticsearch/search/sort/SortParserTests.java index 23d12fcfd0..e2c5411962 100644 --- a/core/src/test/java/org/elasticsearch/search/sort/SortParserTests.java +++ b/core/src/test/java/org/elasticsearch/search/sort/SortParserTests.java @@ -140,6 +140,6 @@ public class SortParserTests extends ESSingleNodeTestCase { parser.setParseFieldMatcher(ParseFieldMatcher.STRICT); parseContext.reset(parser); parser.nextToken(); - GeoDistanceSortBuilder.PROTOTYPE.fromXContent(parseContext, null); + GeoDistanceSortBuilder.fromXContent(parseContext, null); } } diff --git a/core/src/test/java/org/elasticsearch/search/suggest/phrase/SmoothingModelTestCase.java b/core/src/test/java/org/elasticsearch/search/suggest/phrase/SmoothingModelTestCase.java index 00009d1e76..c3890f0ead 100644 --- a/core/src/test/java/org/elasticsearch/search/suggest/phrase/SmoothingModelTestCase.java +++ b/core/src/test/java/org/elasticsearch/search/suggest/phrase/SmoothingModelTestCase.java @@ -108,9 +108,7 @@ public abstract class SmoothingModelTestCase extends ESTestCase { XContentParser parser = XContentHelper.createParser(contentBuilder.bytes()); context.reset(parser); parser.nextToken(); // go to start token, real parsing would do that in the outer element parser - SmoothingModel prototype = (SmoothingModel) namedWriteableRegistry.getPrototype(SmoothingModel.class, - testModel.getWriteableName()); - SmoothingModel parsedModel = prototype.innerFromXContent(context); + SmoothingModel parsedModel = testModel.innerFromXContent(context); assertNotSame(testModel, parsedModel); assertEquals(testModel, parsedModel); assertEquals(testModel.hashCode(), parsedModel.hashCode()); @@ -188,9 +186,7 @@ public abstract class SmoothingModelTestCase extends ESTestCase { try (BytesStreamOutput output = new BytesStreamOutput()) { original.writeTo(output); try (StreamInput in = new NamedWriteableAwareStreamInput(StreamInput.wrap(output.bytes()), namedWriteableRegistry)) { - SmoothingModel prototype = (SmoothingModel) namedWriteableRegistry.getPrototype(SmoothingModel.class, - original.getWriteableName()); - return prototype.readFrom(in); + return namedWriteableRegistry.getReader(SmoothingModel.class, original.getWriteableName()).read(in); } } } diff --git a/core/src/test/java/org/elasticsearch/test/geo/RandomShapeGenerator.java b/core/src/test/java/org/elasticsearch/test/geo/RandomShapeGenerator.java index 95984da55f..897fa44b59 100644 --- a/core/src/test/java/org/elasticsearch/test/geo/RandomShapeGenerator.java +++ b/core/src/test/java/org/elasticsearch/test/geo/RandomShapeGenerator.java @@ -293,8 +293,9 @@ public class RandomShapeGenerator extends RandomGeoGenerator { xDivisible(yRange.getMax()*10e3)/10e3); } + /** creates a small random rectangle by default to keep shape test performance at bay */ public static Rectangle xRandomRectangle(Random r, Point nearP) { - return xRandomRectangle(r, nearP, ctx.getWorldBounds(), false); + return xRandomRectangle(r, nearP, ctx.getWorldBounds(), true); } public static Rectangle xRandomRectangle(Random r, Point nearP, boolean small) { |