diff options
author | Tanguy Leroux <tlrx.dev@gmail.com> | 2017-05-19 13:13:00 +0200 |
---|---|---|
committer | Tanguy Leroux <tlrx.dev@gmail.com> | 2017-05-19 13:13:00 +0200 |
commit | 83aa00b3f665099b0f5dadf5a2fcd97219cdbde5 (patch) | |
tree | 04253d353603d3caa4098c09d870409564a7d4d1 /core | |
parent | 4c34ea8fc84ccd27af98b81f51408fcb53077a20 (diff) | |
parent | 55af1f7a2b503d7997b476c34f952bd9fce6eeb5 (diff) |
Merge remote-tracking branch 'origin/master' into feature/client_aggs_parsing
Diffstat (limited to 'core')
32 files changed, 542 insertions, 1151 deletions
diff --git a/core/build.gradle b/core/build.gradle index 72aaca6da1..516aeb1d0e 100644 --- a/core/build.gradle +++ b/core/build.gradle @@ -124,8 +124,8 @@ forbiddenPatterns { task generateModulesList { List<String> modules = project(':modules').subprojects.collect { it.name } File modulesFile = new File(buildDir, 'generated-resources/modules.txt') - processResources.from(modulesFile) - inputs.property('modules', modules) + processResources.from(modulesFile) + inputs.property('modules', modules) outputs.file(modulesFile) doLast { modulesFile.parentFile.mkdirs() @@ -138,8 +138,8 @@ task generatePluginsList { .findAll { it.name.contains('example') == false } .collect { it.name } File pluginsFile = new File(buildDir, 'generated-resources/plugins.txt') - processResources.from(pluginsFile) - inputs.property('plugins', plugins) + processResources.from(pluginsFile) + inputs.property('plugins', plugins) outputs.file(pluginsFile) doLast { pluginsFile.parentFile.mkdirs() @@ -256,7 +256,7 @@ thirdPartyAudit.excludes = [ 'org.zeromq.ZMQ', // from org.locationtech.spatial4j.io.GeoJSONReader (spatial4j) - 'org.noggit.JSONParser', + 'org.noggit.JSONParser', ] dependencyLicenses { @@ -277,3 +277,43 @@ if (isEclipse == false || project.path == ":core-tests") { check.dependsOn integTest integTest.mustRunAfter test } + +task('verifyVersions') { + description 'Verifies that all released versions that are indexed compatible are listed in Version.java.' + group 'Verification' + enabled = false == gradle.startParameter.isOffline() + doLast { + // Read the list from maven central + Node xml + new URL('https://repo1.maven.org/maven2/org/elasticsearch/elasticsearch/maven-metadata.xml').openStream().withStream { s -> + xml = new XmlParser().parse(s) + } + Set<String> knownVersions = new TreeSet<>(xml.versioning.versions.version.collect { it.text() }.findAll { it ==~ /\d\.\d\.\d/ }) + + // Limit the known versions to those that should be wire compatible + String currentVersion = versions.elasticsearch.minus('-SNAPSHOT') + int prevMajor = Integer.parseInt(currentVersion.split('\\.')[0]) - 1 + if (prevMajor == 4) { + // 4 didn't exist, it was 2. + prevMajor = 2; + } + knownVersions = knownVersions.findAll { Integer.parseInt(it.split('\\.')[0]) >= prevMajor } + + /* Limit the listed versions to those that have been marked as released. + * Versions not marked as released don't get the same testing and we want + * to make sure that we flip all unreleased versions to released as soon + * as possible after release. */ + Set<String> actualVersions = new TreeSet<>( + indexCompatVersions + .findAll { false == it.unreleased } + .collect { it.toString() }) + + // Finally, compare! + if (!knownVersions.equals(actualVersions)) { + throw new GradleException("out-of-date versions\nActual :" + + actualVersions + "\nExpected:" + knownVersions + + "; update Version.java") + } + } +} +check.dependsOn(verifyVersions) diff --git a/core/src/main/java/org/elasticsearch/Version.java b/core/src/main/java/org/elasticsearch/Version.java index 44b989a378..1e60a4ac83 100644 --- a/core/src/main/java/org/elasticsearch/Version.java +++ b/core/src/main/java/org/elasticsearch/Version.java @@ -278,9 +278,12 @@ public class Version implements Comparable<Version> { public Version minimumCompatibilityVersion() { final int bwcMajor; final int bwcMinor; - if (this.onOrAfter(Version.V_6_0_0_alpha1_UNRELEASED)) { - bwcMajor = major - 1; - bwcMinor = 4; + if (major == 6) { // we only specialize for current major here + bwcMajor = Version.V_5_4_0.major; + bwcMinor = Version.V_5_4_0.minor; + } else if (major > 6) { // all the future versions are compatible with first minor... + bwcMajor = major -1; + bwcMinor = 0; } else { bwcMajor = major; bwcMinor = 0; diff --git a/core/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java b/core/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java index 5b99ed02cf..f0aede639e 100644 --- a/core/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java +++ b/core/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java @@ -23,7 +23,6 @@ import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; import org.apache.logging.log4j.util.Supplier; import org.elasticsearch.ExceptionsHelper; -import org.elasticsearch.Version; import org.elasticsearch.action.DocWriteRequest; import org.elasticsearch.action.DocWriteResponse; import org.elasticsearch.action.delete.DeleteRequest; @@ -31,8 +30,8 @@ import org.elasticsearch.action.delete.DeleteResponse; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.index.IndexResponse; import org.elasticsearch.action.support.ActionFilters; -import org.elasticsearch.action.support.replication.ReplicationOperation; import org.elasticsearch.action.support.TransportActions; +import org.elasticsearch.action.support.replication.ReplicationOperation; import org.elasticsearch.action.support.replication.ReplicationResponse.ShardInfo; import org.elasticsearch.action.support.replication.TransportWriteAction; import org.elasticsearch.action.update.UpdateHelper; @@ -56,7 +55,6 @@ import org.elasticsearch.index.engine.Engine; import org.elasticsearch.index.engine.VersionConflictEngineException; import org.elasticsearch.index.get.GetResult; import org.elasticsearch.index.mapper.MapperParsingException; -import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.Mapping; import org.elasticsearch.index.mapper.SourceToParse; import org.elasticsearch.index.seqno.SequenceNumbersService; @@ -414,13 +412,6 @@ public class TransportShardBulkAction extends TransportWriteAction<BulkShardRequ FAILURE } - static { - assert Version.CURRENT.minimumCompatibilityVersion().after(Version.V_6_0_0_alpha1_UNRELEASED) == false: - "Remove logic handling NoOp result from primary response; see TODO in replicaItemExecutionMode" + - " as the current minimum compatible version [" + - Version.CURRENT.minimumCompatibilityVersion() + "] is after 6.0"; - } - /** * Determines whether a bulk item request should be executed on the replica. * @return {@link ReplicaItemExecutionMode#NORMAL} upon normal primary execution with no failures @@ -436,10 +427,11 @@ public class TransportShardBulkAction extends TransportWriteAction<BulkShardRequ ? ReplicaItemExecutionMode.FAILURE // we have a seq no generated with the failure, replicate as no-op : ReplicaItemExecutionMode.NOOP; // no seq no generated, ignore replication } else { - // NOTE: write requests originating from pre-6.0 nodes can send a no-op operation to - // the replica; we ignore replication - // TODO: remove noOp result check from primary response, when pre-6.0 nodes are not supported - // we should return ReplicationItemExecutionMode.NORMAL instead + // TODO: once we know for sure that every operation that has been processed on the primary is assigned a seq# + // (i.e., all nodes on the cluster are on v6.0.0 or higher) we can use the existence of a seq# to indicate whether + // an operation should be processed or be treated as a noop. This means we could remove this method and the + // ReplicaItemExecutionMode enum and have a simple boolean check for seq != UNASSIGNED_SEQ_NO which will work for + // both failures and indexing operations. return primaryResponse.getResponse().getResult() != DocWriteResponse.Result.NOOP ? ReplicaItemExecutionMode.NORMAL // execution successful on primary : ReplicaItemExecutionMode.NOOP; // ignore replication diff --git a/core/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java b/core/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java index 9b96f407f3..d8ee93fe88 100644 --- a/core/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java +++ b/core/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java @@ -19,8 +19,6 @@ package org.elasticsearch.common.settings; import org.elasticsearch.action.admin.indices.close.TransportCloseIndexAction; -import org.elasticsearch.transport.RemoteClusterService; -import org.elasticsearch.transport.RemoteClusterAware; import org.elasticsearch.action.search.TransportSearchAction; import org.elasticsearch.action.support.AutoCreateIndex; import org.elasticsearch.action.support.DestructiveOperations; @@ -88,6 +86,8 @@ import org.elasticsearch.search.SearchModule; import org.elasticsearch.search.SearchService; import org.elasticsearch.search.fetch.subphase.highlight.FastVectorHighlighter; import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.RemoteClusterAware; +import org.elasticsearch.transport.RemoteClusterService; import org.elasticsearch.transport.TcpTransport; import org.elasticsearch.transport.Transport; import org.elasticsearch.transport.TransportService; @@ -304,6 +304,8 @@ public final class ClusterSettings extends AbstractScopedSettings { ScriptService.SCRIPT_CACHE_EXPIRE_SETTING, ScriptService.SCRIPT_MAX_SIZE_IN_BYTES, ScriptService.SCRIPT_MAX_COMPILATIONS_PER_MINUTE, + ScriptService.TYPES_ALLOWED_SETTING, + ScriptService.CONTEXTS_ALLOWED_SETTING, IndicesService.INDICES_CACHE_CLEAN_INTERVAL_SETTING, IndicesFieldDataCache.INDICES_FIELDDATA_CACHE_SIZE_KEY, IndicesRequestCache.INDICES_CACHE_QUERY_SIZE, @@ -339,6 +341,7 @@ public final class ClusterSettings extends AbstractScopedSettings { ZenDiscovery.SEND_LEAVE_REQUEST_SETTING, ZenDiscovery.MASTER_ELECTION_WAIT_FOR_JOINS_TIMEOUT_SETTING, ZenDiscovery.MASTER_ELECTION_IGNORE_NON_MASTER_PINGS_SETTING, + ZenDiscovery.MAX_PENDING_CLUSTER_STATES_SETTING, UnicastZenPing.DISCOVERY_ZEN_PING_UNICAST_HOSTS_SETTING, UnicastZenPing.DISCOVERY_ZEN_PING_UNICAST_CONCURRENT_CONNECTS_SETTING, UnicastZenPing.DISCOVERY_ZEN_PING_UNICAST_HOSTS_RESOLVE_TIMEOUT, diff --git a/core/src/main/java/org/elasticsearch/discovery/zen/PublishClusterStateAction.java b/core/src/main/java/org/elasticsearch/discovery/zen/PublishClusterStateAction.java index 4150783a8f..ae469d162a 100644 --- a/core/src/main/java/org/elasticsearch/discovery/zen/PublishClusterStateAction.java +++ b/core/src/main/java/org/elasticsearch/discovery/zen/PublishClusterStateAction.java @@ -23,8 +23,8 @@ import org.apache.logging.log4j.message.ParameterizedMessage; import org.apache.lucene.util.IOUtils; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.Version; +import org.elasticsearch.action.ActionListener; import org.elasticsearch.cluster.ClusterChangedEvent; -import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.Diff; import org.elasticsearch.cluster.IncompatibleClusterStateVersionException; @@ -60,61 +60,53 @@ import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; -import java.util.Locale; import java.util.Map; import java.util.Set; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; -import java.util.function.Supplier; public class PublishClusterStateAction extends AbstractComponent { public static final String SEND_ACTION_NAME = "internal:discovery/zen/publish/send"; public static final String COMMIT_ACTION_NAME = "internal:discovery/zen/publish/commit"; - public static final String SETTINGS_MAX_PENDING_CLUSTER_STATES = "discovery.zen.publish.max_pending_cluster_states"; + public interface IncomingClusterStateListener { - public interface NewPendingClusterStateListener { + /** + * called when a new incoming cluster state has been received. + * Should validate the incoming state and throw an exception if it's not a valid successor state. + */ + void onIncomingClusterState(ClusterState incomingState); - /** a new cluster state has been committed and is ready to process via {@link #pendingStatesQueue()} */ - void onNewClusterState(String reason); + /** + * called when a cluster state has been committed and is ready to be processed + */ + void onClusterStateCommitted(String stateUUID, ActionListener<Void> processedListener); } private final TransportService transportService; private final NamedWriteableRegistry namedWriteableRegistry; - private final Supplier<ClusterState> clusterStateSupplier; - private final NewPendingClusterStateListener newPendingClusterStatelistener; + private final IncomingClusterStateListener incomingClusterStateListener; private final DiscoverySettings discoverySettings; - private final ClusterName clusterName; - private final PendingClusterStatesQueue pendingStatesQueue; public PublishClusterStateAction( Settings settings, TransportService transportService, NamedWriteableRegistry namedWriteableRegistry, - Supplier<ClusterState> clusterStateSupplier, - NewPendingClusterStateListener listener, - DiscoverySettings discoverySettings, - ClusterName clusterName) { + IncomingClusterStateListener incomingClusterStateListener, + DiscoverySettings discoverySettings) { super(settings); this.transportService = transportService; this.namedWriteableRegistry = namedWriteableRegistry; - this.clusterStateSupplier = clusterStateSupplier; - this.newPendingClusterStatelistener = listener; + this.incomingClusterStateListener = incomingClusterStateListener; this.discoverySettings = discoverySettings; - this.clusterName = clusterName; - this.pendingStatesQueue = new PendingClusterStatesQueue(logger, settings.getAsInt(SETTINGS_MAX_PENDING_CLUSTER_STATES, 25)); transportService.registerRequestHandler(SEND_ACTION_NAME, BytesTransportRequest::new, ThreadPool.Names.SAME, false, false, new SendClusterStateRequestHandler()); transportService.registerRequestHandler(COMMIT_ACTION_NAME, CommitClusterStateRequest::new, ThreadPool.Names.SAME, false, false, new CommitClusterStateRequestHandler()); } - public PendingClusterStatesQueue pendingStatesQueue() { - return pendingStatesQueue; - } - /** * publishes a cluster change event to other nodes. if at least minMasterNodes acknowledge the change it is committed and will * be processed by the master and the other nodes. @@ -387,7 +379,7 @@ public class PublishClusterStateAction extends AbstractComponent { final ClusterState incomingState; // If true we received full cluster state - otherwise diffs if (in.readBoolean()) { - incomingState = ClusterState.readFrom(in, clusterStateSupplier.get().nodes().getLocalNode()); + incomingState = ClusterState.readFrom(in, transportService.getLocalNode()); logger.debug("received full cluster state version [{}] with size [{}]", incomingState.version(), request.bytes().length()); } else if (lastSeenClusterState != null) { @@ -399,10 +391,7 @@ public class PublishClusterStateAction extends AbstractComponent { logger.debug("received diff for but don't have any local cluster state - requesting full state"); throw new IncompatibleClusterStateVersionException("have no local cluster state"); } - // sanity check incoming state - validateIncomingState(incomingState, lastSeenClusterState); - - pendingStatesQueue.addPending(incomingState); + incomingClusterStateListener.onIncomingClusterState(incomingState); lastSeenClusterState = incomingState; } } finally { @@ -411,56 +400,22 @@ public class PublishClusterStateAction extends AbstractComponent { channel.sendResponse(TransportResponse.Empty.INSTANCE); } - // package private for testing - - /** - * does simple sanity check of the incoming cluster state. Throws an exception on rejections. - */ - void validateIncomingState(ClusterState incomingState, ClusterState lastSeenClusterState) { - final ClusterName incomingClusterName = incomingState.getClusterName(); - if (!incomingClusterName.equals(this.clusterName)) { - logger.warn("received cluster state from [{}] which is also master but with a different cluster name [{}]", - incomingState.nodes().getMasterNode(), incomingClusterName); - throw new IllegalStateException("received state from a node that is not part of the cluster"); - } - final ClusterState clusterState = clusterStateSupplier.get(); - - if (clusterState.nodes().getLocalNode().equals(incomingState.nodes().getLocalNode()) == false) { - logger.warn("received a cluster state from [{}] and not part of the cluster, should not happen", - incomingState.nodes().getMasterNode()); - throw new IllegalStateException("received state with a local node that does not match the current local node"); - } - - if (ZenDiscovery.shouldIgnoreOrRejectNewClusterState(logger, clusterState, incomingState)) { - String message = String.format( - Locale.ROOT, - "rejecting cluster state version [%d] uuid [%s] received from [%s]", - incomingState.version(), - incomingState.stateUUID(), - incomingState.nodes().getMasterNodeId() - ); - logger.warn(message); - throw new IllegalStateException(message); - } - - } - protected void handleCommitRequest(CommitClusterStateRequest request, final TransportChannel channel) { - final ClusterState state = pendingStatesQueue.markAsCommitted(request.stateUUID, - new PendingClusterStatesQueue.StateProcessedListener() { + incomingClusterStateListener.onClusterStateCommitted(request.stateUUID, new ActionListener<Void>() { + @Override - public void onNewClusterStateProcessed() { + public void onResponse(Void ignore) { try { // send a response to the master to indicate that this cluster state has been processed post committing it. channel.sendResponse(TransportResponse.Empty.INSTANCE); } catch (Exception e) { logger.debug("failed to send response on cluster state processed", e); - onNewClusterStateFailed(e); + onFailure(e); } } @Override - public void onNewClusterStateFailed(Exception e) { + public void onFailure(Exception e) { try { channel.sendResponse(e); } catch (Exception inner) { @@ -469,10 +424,6 @@ public class PublishClusterStateAction extends AbstractComponent { } } }); - if (state != null) { - newPendingClusterStatelistener.onNewClusterState("master " + state.nodes().getMasterNode() + - " committed version [" + state.version() + "]"); - } } private class SendClusterStateRequestHandler implements TransportRequestHandler<BytesTransportRequest> { diff --git a/core/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java b/core/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java index 09e6357ba5..dfbf3f780b 100644 --- a/core/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java +++ b/core/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java @@ -25,6 +25,7 @@ import org.apache.logging.log4j.util.Supplier; import org.apache.lucene.util.IOUtils; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ExceptionsHelper; +import org.elasticsearch.action.ActionListener; import org.elasticsearch.cluster.ClusterChangedEvent; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; @@ -56,6 +57,7 @@ import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.discovery.Discovery; import org.elasticsearch.discovery.DiscoverySettings; import org.elasticsearch.discovery.DiscoveryStats; +import org.elasticsearch.discovery.zen.PublishClusterStateAction.IncomingClusterStateListener; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.EmptyTransportResponseHandler; import org.elasticsearch.transport.TransportChannel; @@ -82,7 +84,7 @@ import java.util.stream.Collectors; import static org.elasticsearch.common.unit.TimeValue.timeValueSeconds; import static org.elasticsearch.gateway.GatewayService.STATE_NOT_RECOVERED_BLOCK; -public class ZenDiscovery extends AbstractLifecycleComponent implements Discovery, PingContextProvider { +public class ZenDiscovery extends AbstractLifecycleComponent implements Discovery, PingContextProvider, IncomingClusterStateListener { public static final Setting<TimeValue> PING_TIMEOUT_SETTING = Setting.positiveTimeSetting("discovery.zen.ping_timeout", timeValueSeconds(3), Property.NodeScope); @@ -104,6 +106,8 @@ public class ZenDiscovery extends AbstractLifecycleComponent implements Discover Property.NodeScope); public static final Setting<Boolean> MASTER_ELECTION_IGNORE_NON_MASTER_PINGS_SETTING = Setting.boolSetting("discovery.zen.master_election.ignore_non_master_pings", false, Property.NodeScope); + public static final Setting<Integer> MAX_PENDING_CLUSTER_STATES_SETTING = + Setting.intSetting("discovery.zen.publish.max_pending_cluster_states", 25, 1, Property.NodeScope); public static final String DISCOVERY_REJOIN_ACTION_NAME = "internal:discovery/zen/rejoin"; @@ -139,6 +143,8 @@ public class ZenDiscovery extends AbstractLifecycleComponent implements Discover private final JoinThreadControl joinThreadControl; + private final PendingClusterStatesQueue pendingStatesQueue; + private final NodeJoinController nodeJoinController; private final NodeRemovalClusterStateTaskExecutor nodeRemovalExecutor; @@ -197,16 +203,15 @@ public class ZenDiscovery extends AbstractLifecycleComponent implements Discover this.masterFD.addListener(new MasterNodeFailureListener()); this.nodesFD = new NodesFaultDetection(settings, threadPool, transportService, clusterName); this.nodesFD.addListener(new NodeFaultDetectionListener()); + this.pendingStatesQueue = new PendingClusterStatesQueue(logger, MAX_PENDING_CLUSTER_STATES_SETTING.get(settings)); this.publishClusterState = new PublishClusterStateAction( settings, transportService, namedWriteableRegistry, - this::clusterState, - new NewPendingClusterStateListener(), - discoverySettings, - clusterName); + this, + discoverySettings); this.membership = new MembershipAction(settings, transportService, new MembershipListener()); this.joinThreadControl = new JoinThreadControl(); @@ -311,7 +316,7 @@ public class ZenDiscovery extends AbstractLifecycleComponent implements Discover throw new FailedToCommitClusterStateException("state was mutated while calculating new CS update"); } - publishClusterState.pendingStatesQueue().addPending(newState); + pendingStatesQueue.addPending(newState); try { publishClusterState.publish(clusterChangedEvent, electMaster.minimumMasterNodes(), ackListener); @@ -321,7 +326,7 @@ public class ZenDiscovery extends AbstractLifecycleComponent implements Discover newState.version(), electMaster.minimumMasterNodes()); synchronized (stateMutex) { - publishClusterState.pendingStatesQueue().failAllStatesAndClear( + pendingStatesQueue.failAllStatesAndClear( new ElasticsearchException("failed to publish cluster state")); rejoin("zen-disco-failed-to-publish"); @@ -332,7 +337,7 @@ public class ZenDiscovery extends AbstractLifecycleComponent implements Discover final DiscoveryNode localNode = newState.getNodes().getLocalNode(); final CountDownLatch latch = new CountDownLatch(1); final AtomicBoolean processedOrFailed = new AtomicBoolean(); - publishClusterState.pendingStatesQueue().markAsCommitted(newState.stateUUID(), + pendingStatesQueue.markAsCommitted(newState.stateUUID(), new PendingClusterStatesQueue.StateProcessedListener() { @Override public void onNewClusterStateProcessed() { @@ -391,7 +396,7 @@ public class ZenDiscovery extends AbstractLifecycleComponent implements Discover @Override public DiscoveryStats stats() { - PendingClusterStateStats queueStats = publishClusterState.pendingStatesQueue().stats(); + PendingClusterStateStats queueStats = pendingStatesQueue.stats(); return new DiscoveryStats(queueStats); } @@ -409,11 +414,11 @@ public class ZenDiscovery extends AbstractLifecycleComponent implements Discover // used for testing public ClusterState[] pendingClusterStates() { - return publishClusterState.pendingStatesQueue().pendingClusterStates(); + return pendingStatesQueue.pendingClusterStates(); } PendingClusterStatesQueue pendingClusterStatesQueue() { - return publishClusterState.pendingStatesQueue(); + return pendingStatesQueue; } /** @@ -703,7 +708,7 @@ public class ZenDiscovery extends AbstractLifecycleComponent implements Discover synchronized (stateMutex) { if (localNodeMaster() == false && masterNode.equals(committedState.get().nodes().getMasterNode())) { // flush any pending cluster states from old master, so it will not be set as master again - publishClusterState.pendingStatesQueue().failAllStatesAndClear(new ElasticsearchException("master left [{}]", reason)); + pendingStatesQueue.failAllStatesAndClear(new ElasticsearchException("master left [{}]", reason)); rejoin("master left (reason = " + reason + ")"); } } @@ -713,7 +718,7 @@ public class ZenDiscovery extends AbstractLifecycleComponent implements Discover boolean processNextCommittedClusterState(String reason) { assert Thread.holdsLock(stateMutex); - final ClusterState newClusterState = publishClusterState.pendingStatesQueue().getNextClusterStateToProcess(); + final ClusterState newClusterState = pendingStatesQueue.getNextClusterStateToProcess(); final ClusterState currentState = committedState.get(); final ClusterState adaptedNewClusterState; // all pending states have been processed @@ -742,7 +747,7 @@ public class ZenDiscovery extends AbstractLifecycleComponent implements Discover } } catch (Exception e) { try { - publishClusterState.pendingStatesQueue().markAsFailed(newClusterState, e); + pendingStatesQueue.markAsFailed(newClusterState, e); } catch (Exception inner) { inner.addSuppressed(e); logger.error((Supplier<?>) () -> new ParameterizedMessage("unexpected exception while failing [{}]", reason), inner); @@ -811,7 +816,7 @@ public class ZenDiscovery extends AbstractLifecycleComponent implements Discover @Override public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) { try { - publishClusterState.pendingStatesQueue().markAsProcessed(newClusterState); + pendingStatesQueue.markAsProcessed(newClusterState); } catch (Exception e) { onFailure(source, e); } @@ -823,7 +828,7 @@ public class ZenDiscovery extends AbstractLifecycleComponent implements Discover try { // TODO: use cluster state uuid instead of full cluster state so that we don't keep reference to CS around // for too long. - publishClusterState.pendingStatesQueue().markAsFailed(newClusterState, e); + pendingStatesQueue.markAsFailed(newClusterState, e); } catch (Exception inner) { inner.addSuppressed(e); logger.error((Supplier<?>) () -> new ParameterizedMessage("unexpected exception while failing [{}]", reason), inner); @@ -1066,16 +1071,64 @@ public class ZenDiscovery extends AbstractLifecycleComponent implements Discover } } - private class NewPendingClusterStateListener implements PublishClusterStateAction.NewPendingClusterStateListener { + @Override + public void onIncomingClusterState(ClusterState incomingState) { + validateIncomingState(logger, incomingState, committedState.get()); + pendingStatesQueue.addPending(incomingState); + } - @Override - public void onNewClusterState(String reason) { + @Override + public void onClusterStateCommitted(String stateUUID, ActionListener<Void> processedListener) { + final ClusterState state = pendingStatesQueue.markAsCommitted(stateUUID, + new PendingClusterStatesQueue.StateProcessedListener() { + @Override + public void onNewClusterStateProcessed() { + processedListener.onResponse(null); + } + + @Override + public void onNewClusterStateFailed(Exception e) { + processedListener.onFailure(e); + } + }); + if (state != null) { synchronized (stateMutex) { - processNextCommittedClusterState(reason); + processNextCommittedClusterState("master " + state.nodes().getMasterNode() + + " committed version [" + state.version() + "]"); } } } + /** + * does simple sanity check of the incoming cluster state. Throws an exception on rejections. + */ + static void validateIncomingState(Logger logger, ClusterState incomingState, ClusterState lastState) { + final ClusterName incomingClusterName = incomingState.getClusterName(); + if (!incomingClusterName.equals(lastState.getClusterName())) { + logger.warn("received cluster state from [{}] which is also master but with a different cluster name [{}]", + incomingState.nodes().getMasterNode(), incomingClusterName); + throw new IllegalStateException("received state from a node that is not part of the cluster"); + } + if (lastState.nodes().getLocalNode().equals(incomingState.nodes().getLocalNode()) == false) { + logger.warn("received a cluster state from [{}] and not part of the cluster, should not happen", + incomingState.nodes().getMasterNode()); + throw new IllegalStateException("received state with a local node that does not match the current local node"); + } + + if (shouldIgnoreOrRejectNewClusterState(logger, lastState, incomingState)) { + String message = String.format( + Locale.ROOT, + "rejecting cluster state version [%d] uuid [%s] received from [%s]", + incomingState.version(), + incomingState.stateUUID(), + incomingState.nodes().getMasterNodeId() + ); + logger.warn(message); + throw new IllegalStateException(message); + } + + } + private class MembershipListener implements MembershipAction.MembershipListener { @Override public void onJoin(DiscoveryNode node, MembershipAction.JoinCallback callback) { diff --git a/core/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java b/core/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java index 1b243bccd7..7371bde893 100644 --- a/core/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java +++ b/core/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java @@ -33,6 +33,7 @@ import org.elasticsearch.cluster.action.shard.ShardStateAction; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodes; +import org.elasticsearch.cluster.routing.AllocationId; import org.elasticsearch.cluster.routing.IndexShardRoutingTable; import org.elasticsearch.cluster.routing.RecoverySource.Type; import org.elasticsearch.cluster.routing.RoutingNode; @@ -551,18 +552,16 @@ public class IndicesClusterStateService extends AbstractLifecycleComponent imple try { shard.updateRoutingEntry(shardRouting); if (shardRouting.primary()) { - IndexShardRoutingTable indexShardRoutingTable = routingTable.shardRoutingTable(shardRouting.shardId()); - Set<String> activeIds = indexShardRoutingTable.activeShards().stream() - // filter to shards that track seq# and should be taken into consideration for checkpoint tracking - // shards on old nodes will go through a file based recovery which will also transfer seq# information. - .filter(sr -> nodes.get(sr.currentNodeId()).getVersion().onOrAfter(Version.V_6_0_0_alpha1_UNRELEASED)) - .map(r -> r.allocationId().getId()) - .collect(Collectors.toSet()); - Set<String> initializingIds = indexShardRoutingTable.getAllInitializingShards().stream() - .filter(sr -> nodes.get(sr.currentNodeId()).getVersion().onOrAfter(Version.V_6_0_0_alpha1_UNRELEASED)) - .map(r -> r.allocationId().getId()) - .collect(Collectors.toSet()); - shard.updateAllocationIdsFromMaster(activeIds, initializingIds); + final IndexShardRoutingTable indexShardRoutingTable = routingTable.shardRoutingTable(shardRouting.shardId()); + /* + * Filter to shards that track sequence numbers and should be taken into consideration for checkpoint tracking. Shards on + * old nodes will go through a file-based recovery which will also transfer sequence number information. + */ + final Set<String> activeIds = + allocationIdsForShardsOnNodesThatUnderstandSeqNos(indexShardRoutingTable.activeShards(), nodes); + final Set<String> initializingIds = + allocationIdsForShardsOnNodesThatUnderstandSeqNos(indexShardRoutingTable.getAllInitializingShards(), nodes); + shard.updateAllocationIdsFromMaster(activeIds, initializingIds); } } catch (Exception e) { failAndRemoveShard(shardRouting, true, "failed updating shard routing entry", e, clusterState); @@ -586,6 +585,17 @@ public class IndicesClusterStateService extends AbstractLifecycleComponent imple } } + private Set<String> allocationIdsForShardsOnNodesThatUnderstandSeqNos( + final List<ShardRouting> shardRoutings, + final DiscoveryNodes nodes) { + return shardRoutings + .stream() + .filter(sr -> nodes.get(sr.currentNodeId()).getVersion().onOrAfter(Version.V_6_0_0_alpha1_UNRELEASED)) + .map(ShardRouting::allocationId) + .map(AllocationId::getId) + .collect(Collectors.toSet()); + } + /** * Finds the routing source node for peer recovery, return null if its not found. Note, this method expects the shard * routing to *require* peer recovery, use {@link ShardRouting#recoverySource()} to check if its needed or not. diff --git a/core/src/main/java/org/elasticsearch/monitor/jvm/JvmInfo.java b/core/src/main/java/org/elasticsearch/monitor/jvm/JvmInfo.java index c18a1c5e3f..1674ba33c0 100644 --- a/core/src/main/java/org/elasticsearch/monitor/jvm/JvmInfo.java +++ b/core/src/main/java/org/elasticsearch/monitor/jvm/JvmInfo.java @@ -299,19 +299,19 @@ public class JvmInfo implements Writeable, ToXContent { public int versionAsInteger() { try { int i = 0; - String sVersion = ""; + StringBuilder sVersion = new StringBuilder(); for (; i < version.length(); i++) { if (!Character.isDigit(version.charAt(i)) && version.charAt(i) != '.') { break; } if (version.charAt(i) != '.') { - sVersion += version.charAt(i); + sVersion.append(version.charAt(i)); } } if (i == 0) { return -1; } - return Integer.parseInt(sVersion); + return Integer.parseInt(sVersion.toString()); } catch (Exception e) { return -1; } @@ -320,19 +320,19 @@ public class JvmInfo implements Writeable, ToXContent { public int versionUpdatePack() { try { int i = 0; - String sVersion = ""; + StringBuilder sVersion = new StringBuilder(); for (; i < version.length(); i++) { if (!Character.isDigit(version.charAt(i)) && version.charAt(i) != '.') { break; } if (version.charAt(i) != '.') { - sVersion += version.charAt(i); + sVersion.append(version.charAt(i)); } } if (i == 0) { return -1; } - Integer.parseInt(sVersion); + Integer.parseInt(sVersion.toString()); int from; if (version.charAt(i) == '_') { // 1.7.0_4 diff --git a/core/src/main/java/org/elasticsearch/node/Node.java b/core/src/main/java/org/elasticsearch/node/Node.java index bbda2869ff..dbf3d21181 100644 --- a/core/src/main/java/org/elasticsearch/node/Node.java +++ b/core/src/main/java/org/elasticsearch/node/Node.java @@ -327,7 +327,6 @@ public class Node implements Closeable { final ResourceWatcherService resourceWatcherService = new ResourceWatcherService(settings, threadPool); final ScriptModule scriptModule = ScriptModule.create(settings, pluginsService.filterPlugins(ScriptPlugin.class)); AnalysisModule analysisModule = new AnalysisModule(this.environment, pluginsService.filterPlugins(AnalysisPlugin.class)); - additionalSettings.addAll(scriptModule.getSettings()); // this is as early as we can validate settings at this point. we already pass them to ScriptModule as well as ThreadPool // so we might be late here already final SettingsModule settingsModule = new SettingsModule(this.settings, additionalSettings, additionalSettingsFilter); diff --git a/core/src/main/java/org/elasticsearch/rest/BaseRestHandler.java b/core/src/main/java/org/elasticsearch/rest/BaseRestHandler.java index 81620ec8a7..3db635b044 100644 --- a/core/src/main/java/org/elasticsearch/rest/BaseRestHandler.java +++ b/core/src/main/java/org/elasticsearch/rest/BaseRestHandler.java @@ -85,12 +85,12 @@ public abstract class BaseRestHandler extends AbstractComponent implements RestH final Set<String> invalids, final Set<String> candidates, final String detail) { - String message = String.format( + StringBuilder message = new StringBuilder(String.format( Locale.ROOT, "request [%s] contains unrecognized %s%s: ", request.path(), detail, - invalids.size() > 1 ? "s" : ""); + invalids.size() > 1 ? "s" : "")); boolean first = true; for (final String invalid : invalids) { final LevensteinDistance ld = new LevensteinDistance(); @@ -108,17 +108,23 @@ public abstract class BaseRestHandler extends AbstractComponent implements RestH else return a.v2().compareTo(b.v2()); }); if (first == false) { - message += ", "; + message.append(", "); } - message += "[" + invalid + "]"; + message.append("[").append(invalid).append("]"); final List<String> keys = scoredParams.stream().map(Tuple::v2).collect(Collectors.toList()); if (keys.isEmpty() == false) { - message += " -> did you mean " + (keys.size() == 1 ? "[" + keys.get(0) + "]" : "any of " + keys.toString()) + "?"; + message.append(" -> did you mean "); + if (keys.size() == 1) { + message.append("[").append(keys.get(0)).append("]"); + } else { + message.append("any of ").append(keys.toString()); + } + message.append("?"); } first = false; } - return message; + return message.toString(); } /** diff --git a/core/src/main/java/org/elasticsearch/script/ScriptModes.java b/core/src/main/java/org/elasticsearch/script/ScriptModes.java deleted file mode 100644 index ef1355d678..0000000000 --- a/core/src/main/java/org/elasticsearch/script/ScriptModes.java +++ /dev/null @@ -1,150 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.script; - -import org.apache.lucene.util.SetOnce; -import org.elasticsearch.common.settings.Setting; -import org.elasticsearch.common.settings.Settings; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.TreeMap; -import java.util.function.Function; - -/** - * Holds the boolean indicating the enabled mode for each of the different scripting languages available, each script source and each - * scripted operation. - */ -public class ScriptModes { - - private static final String SCRIPT_SETTINGS_PREFIX = "script"; - private static final String ENGINE_SETTINGS_PREFIX = "script.engine"; - - final Map<String, Boolean> scriptEnabled; - - private static final Setting<List<String>> TYPES_ALLOWED_SETTING = - Setting.listSetting("script.types_allowed", Collections.emptyList(), Function.identity(), Setting.Property.NodeScope); - private static final Setting<List<String>> CONTEXTS_ALLOWED_SETTING = - Setting.listSetting("script.contexts_allowed", Collections.emptyList(), Function.identity(), Setting.Property.NodeScope); - - private final Set<String> typesAllowed; - private final Set<String> contextsAllowed; - - ScriptModes(ScriptContextRegistry scriptContextRegistry, ScriptSettings scriptSettings, Settings settings) { - HashMap<String, Boolean> scriptModes = new HashMap<>(); - for (Setting<Boolean> scriptModeSetting : scriptSettings.getScriptLanguageSettings()) { - scriptModes.put(scriptModeSetting.getKey(), scriptModeSetting.get(settings)); - } - this.scriptEnabled = Collections.unmodifiableMap(scriptModes); - - typesAllowed = TYPES_ALLOWED_SETTING.exists(settings) ? new HashSet<>() : null; - - if (typesAllowed != null) { - for (String settingType : TYPES_ALLOWED_SETTING.get(settings)) { - boolean found = false; - - for (ScriptType scriptType : ScriptType.values()) { - if (scriptType.getName().equals(settingType)) { - found = true; - typesAllowed.add(settingType); - - break; - } - } - - if (!found) { - throw new IllegalArgumentException( - "unknown script type [" + settingType + "] found in setting [" + TYPES_ALLOWED_SETTING.getKey() + "]."); - } - } - } - - contextsAllowed = CONTEXTS_ALLOWED_SETTING.exists(settings) ? new HashSet<>() : null; - - if (contextsAllowed != null) { - for (String settingContext : CONTEXTS_ALLOWED_SETTING.get(settings)) { - if (scriptContextRegistry.isSupportedContext(settingContext)) { - contextsAllowed.add(settingContext); - } else { - throw new IllegalArgumentException( - "unknown script context [" + settingContext + "] found in setting [" + CONTEXTS_ALLOWED_SETTING.getKey() + "]."); - } - } - } - } - - /** - * Returns the script mode for a script of a certain written in a certain language, - * of a certain type and executing as part of a specific operation/api. - * - * @param lang the language that the script is written in - * @param scriptType the type of the script - * @param scriptContext the operation that requires the execution of the script - * @return whether scripts are enabled (true) or disabled (false) - */ - public boolean getScriptEnabled(String lang, ScriptType scriptType, ScriptContext scriptContext) { - if (typesAllowed != null && typesAllowed.contains(scriptType.getName()) == false) { - throw new IllegalArgumentException("[" + scriptType.getName() + "] scripts cannot be executed"); - } - - if (contextsAllowed != null && contextsAllowed.contains(scriptContext.getKey()) == false) { - throw new IllegalArgumentException("[" + scriptContext.getKey() + "] scripts cannot be executed"); - } - - Boolean scriptMode = scriptEnabled.get(getKey(lang, scriptType, scriptContext)); - if (scriptMode == null) { - throw new IllegalArgumentException("script mode not found for lang [" + lang + "], script_type [" + scriptType + "], operation [" + scriptContext.getKey() + "]"); - } - return scriptMode; - } - - static String operationKey(ScriptContext scriptContext) { - return SCRIPT_SETTINGS_PREFIX + "." + scriptContext.getKey(); - } - - static String sourceKey(ScriptType scriptType) { - return SCRIPT_SETTINGS_PREFIX + "." + scriptType.getName(); - } - - static String getGlobalKey(String lang, ScriptType scriptType) { - return ENGINE_SETTINGS_PREFIX + "." + lang + "." + scriptType; - } - - static String getKey(String lang, ScriptType scriptType, ScriptContext scriptContext) { - return ENGINE_SETTINGS_PREFIX + "." + lang + "." + scriptType + "." + scriptContext.getKey(); - } - - @Override - public String toString() { - //order settings by key before printing them out, for readability - TreeMap<String, Boolean> scriptModesTreeMap = new TreeMap<>(); - scriptModesTreeMap.putAll(scriptEnabled); - StringBuilder stringBuilder = new StringBuilder(); - for (Map.Entry<String, Boolean> stringScriptModeEntry : scriptModesTreeMap.entrySet()) { - stringBuilder.append(stringScriptModeEntry.getKey()).append(": ").append(stringScriptModeEntry.getValue()).append("\n"); - } - return stringBuilder.toString(); - } -} diff --git a/core/src/main/java/org/elasticsearch/script/ScriptModule.java b/core/src/main/java/org/elasticsearch/script/ScriptModule.java index 29c9c90e76..1d7ecf119e 100644 --- a/core/src/main/java/org/elasticsearch/script/ScriptModule.java +++ b/core/src/main/java/org/elasticsearch/script/ScriptModule.java @@ -20,24 +20,18 @@ package org.elasticsearch.script; import org.elasticsearch.common.settings.ClusterSettings; -import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.env.Environment; import org.elasticsearch.plugins.ScriptPlugin; -import org.elasticsearch.watcher.ResourceWatcherService; import java.io.IOException; import java.util.List; -import java.util.Map; import java.util.Objects; -import java.util.function.Function; import java.util.stream.Collectors; /** - * Manages building {@link ScriptService} and {@link ScriptSettings} from a list of plugins. + * Manages building {@link ScriptService}. */ public class ScriptModule { - private final ScriptSettings scriptSettings; private final ScriptService scriptService; /** @@ -59,22 +53,14 @@ public class ScriptModule { List<ScriptContext.Plugin> customScriptContexts) { ScriptContextRegistry scriptContextRegistry = new ScriptContextRegistry(customScriptContexts); ScriptEngineRegistry scriptEngineRegistry = new ScriptEngineRegistry(scriptEngines); - scriptSettings = new ScriptSettings(scriptEngineRegistry, scriptContextRegistry); try { - scriptService = new ScriptService(settings, scriptEngineRegistry, scriptContextRegistry, scriptSettings); + scriptService = new ScriptService(settings, scriptEngineRegistry, scriptContextRegistry); } catch (IOException e) { throw new RuntimeException("Couldn't setup ScriptService", e); } } /** - * Extra settings for scripts. - */ - public List<Setting<?>> getSettings() { - return scriptSettings.getSettings(); - } - - /** * Service responsible for managing scripts. */ public ScriptService getScriptService() { diff --git a/core/src/main/java/org/elasticsearch/script/ScriptService.java b/core/src/main/java/org/elasticsearch/script/ScriptService.java index 860bb31560..712fa5d3ea 100644 --- a/core/src/main/java/org/elasticsearch/script/ScriptService.java +++ b/core/src/main/java/org/elasticsearch/script/ScriptService.java @@ -19,12 +19,6 @@ package org.elasticsearch.script; -import java.io.Closeable; -import java.io.IOException; -import java.util.Collections; -import java.util.Map; -import java.util.Objects; - import org.apache.lucene.util.IOUtils; import org.elasticsearch.ResourceNotFoundException; import org.elasticsearch.action.ActionListener; @@ -41,7 +35,6 @@ import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.Strings; import org.elasticsearch.common.breaker.CircuitBreakingException; -import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.cache.Cache; import org.elasticsearch.common.cache.CacheBuilder; import org.elasticsearch.common.cache.RemovalListener; @@ -55,6 +48,16 @@ import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.search.lookup.SearchLookup; import org.elasticsearch.template.CompiledTemplate; +import java.io.Closeable; +import java.io.IOException; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.function.Function; + public class ScriptService extends AbstractComponent implements Closeable, ClusterStateListener { static final String DISABLE_DYNAMIC_SCRIPTING_SETTING = "script.disable_dynamic"; @@ -68,11 +71,20 @@ public class ScriptService extends AbstractComponent implements Closeable, Clust public static final Setting<Integer> SCRIPT_MAX_COMPILATIONS_PER_MINUTE = Setting.intSetting("script.max_compilations_per_minute", 15, 0, Property.Dynamic, Property.NodeScope); + public static final String ALLOW_NONE = "none"; + + public static final Setting<List<String>> TYPES_ALLOWED_SETTING = + Setting.listSetting("script.types_allowed", Collections.emptyList(), Function.identity(), Setting.Property.NodeScope); + public static final Setting<List<String>> CONTEXTS_ALLOWED_SETTING = + Setting.listSetting("script.contexts_allowed", Collections.emptyList(), Function.identity(), Setting.Property.NodeScope); + + private final Set<String> typesAllowed; + private final Set<String> contextsAllowed; + private final Map<String, ScriptEngine> engines; private final Cache<CacheKey, CompiledScript> cache; - private final ScriptModes scriptModes; private final ScriptContextRegistry scriptContextRegistry; private final ScriptMetrics scriptMetrics = new ScriptMetrics(); @@ -84,18 +96,87 @@ public class ScriptService extends AbstractComponent implements Closeable, Clust private double scriptsPerMinCounter; private double compilesAllowedPerNano; - public ScriptService(Settings settings, ScriptEngineRegistry scriptEngineRegistry, - ScriptContextRegistry scriptContextRegistry, ScriptSettings scriptSettings) throws IOException { + public ScriptService(Settings settings, ScriptEngineRegistry scriptEngineRegistry, ScriptContextRegistry scriptContextRegistry) throws IOException { super(settings); + + Objects.requireNonNull(settings); Objects.requireNonNull(scriptEngineRegistry); Objects.requireNonNull(scriptContextRegistry); - Objects.requireNonNull(scriptSettings); + if (Strings.hasLength(settings.get(DISABLE_DYNAMIC_SCRIPTING_SETTING))) { throw new IllegalArgumentException(DISABLE_DYNAMIC_SCRIPTING_SETTING + " is not a supported setting, replace with fine-grained script settings. \n" + - "Dynamic scripts can be enabled for all languages and all operations by replacing `script.disable_dynamic: false` with `script.inline: true` and `script.stored: true` in elasticsearch.yml"); + "Dynamic scripts can be enabled for all languages and all operations not using `script.disable_dynamic: false` in elasticsearch.yml"); + } + + this.typesAllowed = TYPES_ALLOWED_SETTING.exists(settings) ? new HashSet<>() : null; + + if (this.typesAllowed != null) { + List<String> typesAllowedList = TYPES_ALLOWED_SETTING.get(settings); + + if (typesAllowedList.isEmpty()) { + throw new IllegalArgumentException( + "must specify at least one script type or none for setting [" + TYPES_ALLOWED_SETTING.getKey() + "]."); + } + + for (String settingType : typesAllowedList) { + if (ALLOW_NONE.equals(settingType)) { + if (typesAllowedList.size() != 1) { + throw new IllegalArgumentException("cannot specify both [" + ALLOW_NONE + "]" + + " and other script types for setting [" + TYPES_ALLOWED_SETTING.getKey() + "]."); + } else { + break; + } + } + + boolean found = false; + + for (ScriptType scriptType : ScriptType.values()) { + if (scriptType.getName().equals(settingType)) { + found = true; + this.typesAllowed.add(settingType); + + break; + } + } + + if (found == false) { + throw new IllegalArgumentException( + "unknown script type [" + settingType + "] found in setting [" + TYPES_ALLOWED_SETTING.getKey() + "]."); + } + } + } + + this.contextsAllowed = CONTEXTS_ALLOWED_SETTING.exists(settings) ? new HashSet<>() : null; + + if (this.contextsAllowed != null) { + List<String> contextsAllowedList = CONTEXTS_ALLOWED_SETTING.get(settings); + + if (contextsAllowedList.isEmpty()) { + throw new IllegalArgumentException( + "must specify at least one script context or none for setting [" + CONTEXTS_ALLOWED_SETTING.getKey() + "]."); + } + + for (String settingContext : contextsAllowedList) { + if (ALLOW_NONE.equals(settingContext)) { + if (contextsAllowedList.size() != 1) { + throw new IllegalArgumentException("cannot specify both [" + ALLOW_NONE + "]" + + " and other script contexts for setting [" + CONTEXTS_ALLOWED_SETTING.getKey() + "]."); + } else { + break; + } + } + + if (scriptContextRegistry.isSupportedContext(settingContext)) { + this.contextsAllowed.add(settingContext); + } else { + throw new IllegalArgumentException( + "unknown script context [" + settingContext + "] found in setting [" + CONTEXTS_ALLOWED_SETTING.getKey() + "]."); + } + } } this.scriptContextRegistry = scriptContextRegistry; + int cacheMaxSize = SCRIPT_CACHE_SIZE_SETTING.get(settings); CacheBuilder<CacheKey, CompiledScript> cacheBuilder = CacheBuilder.builder(); @@ -110,8 +191,9 @@ public class ScriptService extends AbstractComponent implements Closeable, Clust logger.debug("using script cache with max_size [{}], expire [{}]", cacheMaxSize, cacheExpire); this.cache = cacheBuilder.removalListener(new ScriptCacheRemovalListener()).build(); + this.engines = scriptEngineRegistry.getRegisteredLanguages(); - this.scriptModes = new ScriptModes(scriptContextRegistry, scriptSettings, settings); + this.lastInlineCompileTime = System.nanoTime(); this.setMaxCompilationsPerMinute(SCRIPT_MAX_COMPILATIONS_PER_MINUTE.get(settings)); } @@ -194,9 +276,16 @@ public class ScriptService extends AbstractComponent implements Closeable, Clust ScriptEngine scriptEngine = getEngine(lang); - if (canExecuteScript(lang, type, scriptContext) == false) { - throw new IllegalStateException("scripts of type [" + script.getType() + "]," + - " operation [" + scriptContext.getKey() + "] and lang [" + lang + "] are disabled"); + if (isTypeEnabled(type) == false) { + throw new IllegalArgumentException("cannot execute [" + type + "] scripts"); + } + + if (scriptContextRegistry.isSupportedContext(scriptContext.getKey()) == false) { + throw new IllegalArgumentException("script context [" + scriptContext.getKey() + "] not supported"); + } + + if (isContextEnabled(scriptContext) == false) { + throw new IllegalArgumentException("cannot execute scripts using [" + scriptContext.getKey() + "] context"); } if (logger.isTraceEnabled()) { @@ -288,6 +377,18 @@ public class ScriptService extends AbstractComponent implements Closeable, Clust return engines.containsKey(lang); } + public boolean isTypeEnabled(ScriptType scriptType) { + return typesAllowed == null || typesAllowed.contains(scriptType.getName()); + } + + public boolean isContextEnabled(ScriptContext scriptContext) { + return contextsAllowed == null || contextsAllowed.contains(scriptContext.getKey()); + } + + public boolean isAnyContextEnabled() { + return contextsAllowed == null || contextsAllowed.isEmpty() == false; + } + StoredScriptSource getScriptFromClusterState(String id, String lang) { if (lang != null && isLangSupported(lang) == false) { throw new IllegalArgumentException("unable to get stored script with unsupported lang [" + lang + "]"); @@ -328,16 +429,19 @@ public class ScriptService extends AbstractComponent implements Closeable, Clust try { ScriptEngine scriptEngine = getEngine(source.getLang()); - if (isAnyScriptContextEnabled(source.getLang(), ScriptType.STORED)) { + if (isTypeEnabled(ScriptType.STORED) == false) { + throw new IllegalArgumentException( + "cannot put [" + ScriptType.STORED + "] script, [" + ScriptType.STORED + "] scripts are not enabled"); + } else if (isAnyContextEnabled() == false) { + throw new IllegalArgumentException( + "cannot put [" + ScriptType.STORED + "] script, no script contexts are enabled"); + } else { Object compiled = scriptEngine.compile(request.id(), source.getCode(), Collections.emptyMap()); if (compiled == null) { throw new IllegalArgumentException("failed to parse/compile stored script [" + request.id() + "]" + (source.getCode() == null ? "" : " using code [" + source.getCode() + "]")); } - } else { - throw new IllegalArgumentException( - "cannot put stored script [" + request.id() + "], stored scripts cannot be run under any context"); } } catch (ScriptException good) { throw good; @@ -422,23 +526,6 @@ public class ScriptService extends AbstractComponent implements Closeable, Clust return getEngine(compiledScript.lang()).search(compiledScript, lookup, params); } - private boolean isAnyScriptContextEnabled(String lang, ScriptType scriptType) { - for (ScriptContext scriptContext : scriptContextRegistry.scriptContexts()) { - if (canExecuteScript(lang, scriptType, scriptContext)) { - return true; - } - } - return false; - } - - private boolean canExecuteScript(String lang, ScriptType scriptType, ScriptContext scriptContext) { - assert lang != null; - if (scriptContextRegistry.isSupportedContext(scriptContext.getKey()) == false) { - throw new IllegalArgumentException("script context [" + scriptContext.getKey() + "] not supported"); - } - return scriptModes.getScriptEnabled(lang, scriptType, scriptContext); - } - public ScriptStats stats() { return scriptMetrics.stats(); } diff --git a/core/src/main/java/org/elasticsearch/script/ScriptSettings.java b/core/src/main/java/org/elasticsearch/script/ScriptSettings.java deleted file mode 100644 index 306f5dc068..0000000000 --- a/core/src/main/java/org/elasticsearch/script/ScriptSettings.java +++ /dev/null @@ -1,155 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.script; - -import org.elasticsearch.common.settings.Setting; -import org.elasticsearch.common.settings.Setting.Property; -import org.elasticsearch.common.settings.Settings; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.EnumMap; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.function.Function; - -public class ScriptSettings { - - private static final Map<ScriptType, Setting<Boolean>> SCRIPT_TYPE_SETTING_MAP; - - static { - Map<ScriptType, Setting<Boolean>> scriptTypeSettingMap = new EnumMap<>(ScriptType.class); - for (ScriptType scriptType : ScriptType.values()) { - scriptTypeSettingMap.put(scriptType, Setting.boolSetting( - ScriptModes.sourceKey(scriptType), - scriptType.isDefaultEnabled(), - Property.NodeScope, - Property.Deprecated)); - } - SCRIPT_TYPE_SETTING_MAP = Collections.unmodifiableMap(scriptTypeSettingMap); - } - - private final Map<ScriptContext, Setting<Boolean>> scriptContextSettingMap; - private final List<Setting<Boolean>> scriptLanguageSettings; - - public ScriptSettings(ScriptEngineRegistry scriptEngineRegistry, ScriptContextRegistry scriptContextRegistry) { - Map<ScriptContext, Setting<Boolean>> scriptContextSettingMap = contextSettings(scriptContextRegistry); - this.scriptContextSettingMap = Collections.unmodifiableMap(scriptContextSettingMap); - - List<Setting<Boolean>> scriptLanguageSettings = languageSettings(SCRIPT_TYPE_SETTING_MAP, scriptContextSettingMap, scriptEngineRegistry, scriptContextRegistry); - this.scriptLanguageSettings = Collections.unmodifiableList(scriptLanguageSettings); - } - - private static Map<ScriptContext, Setting<Boolean>> contextSettings(ScriptContextRegistry scriptContextRegistry) { - Map<ScriptContext, Setting<Boolean>> scriptContextSettingMap = new HashMap<>(); - for (ScriptContext scriptContext : scriptContextRegistry.scriptContexts()) { - scriptContextSettingMap.put(scriptContext, - Setting.boolSetting(ScriptModes.operationKey(scriptContext), false, Property.NodeScope, Property.Deprecated)); - } - return scriptContextSettingMap; - } - - private static List<Setting<Boolean>> languageSettings(Map<ScriptType, Setting<Boolean>> scriptTypeSettingMap, - Map<ScriptContext, Setting<Boolean>> scriptContextSettingMap, - ScriptEngineRegistry scriptEngineRegistry, - ScriptContextRegistry scriptContextRegistry) { - final List<Setting<Boolean>> scriptModeSettings = new ArrayList<>(); - - for (final Class<? extends ScriptEngine> scriptEngineService : scriptEngineRegistry.getRegisteredScriptEngineServices()) { - final String language = scriptEngineRegistry.getLanguage(scriptEngineService); - for (final ScriptType scriptType : ScriptType.values()) { - // Top level, like "script.engine.groovy.inline" - final boolean defaultNonFileScriptMode = scriptEngineRegistry.getDefaultInlineScriptEnableds().get(language); - boolean defaultLangAndType = defaultNonFileScriptMode; - // Files are treated differently because they are never default-deny - final boolean defaultIfNothingSet = defaultLangAndType; - - Function<Settings, String> defaultLangAndTypeFn = settings -> { - final Setting<Boolean> globalTypeSetting = scriptTypeSettingMap.get(scriptType); - final Setting<Boolean> langAndTypeSetting = Setting.boolSetting(ScriptModes.getGlobalKey(language, scriptType), - defaultIfNothingSet, Property.NodeScope, Property.Deprecated); - - if (langAndTypeSetting.exists(settings)) { - // fine-grained e.g. script.engine.groovy.inline - return langAndTypeSetting.get(settings).toString(); - } else if (globalTypeSetting.exists(settings)) { - // global type - script.inline - return globalTypeSetting.get(settings).toString(); - } else { - return Boolean.toString(defaultIfNothingSet); - } - }; - - // Setting for something like "script.engine.groovy.inline" - final Setting<Boolean> langAndTypeSetting = Setting.boolSetting(ScriptModes.getGlobalKey(language, scriptType), - defaultLangAndTypeFn, Property.NodeScope, Property.Deprecated); - scriptModeSettings.add(langAndTypeSetting); - - for (ScriptContext scriptContext : scriptContextRegistry.scriptContexts()) { - final String langAndTypeAndContextName = ScriptModes.getKey(language, scriptType, scriptContext); - // A function that, given a setting, will return what the default should be. Since the fine-grained script settings - // read from a bunch of different places this is implemented in this way. - Function<Settings, String> defaultSettingFn = settings -> { - final Setting<Boolean> globalOpSetting = scriptContextSettingMap.get(scriptContext); - final Setting<Boolean> globalTypeSetting = scriptTypeSettingMap.get(scriptType); - final Setting<Boolean> langAndTypeAndContextSetting = Setting.boolSetting(langAndTypeAndContextName, - defaultIfNothingSet, Property.NodeScope, Property.Deprecated); - - // fallback logic for script mode settings - if (langAndTypeAndContextSetting.exists(settings)) { - // like: "script.engine.groovy.inline.aggs: true" - return langAndTypeAndContextSetting.get(settings).toString(); - } else if (langAndTypeSetting.exists(settings)) { - // like: "script.engine.groovy.inline: true" - return langAndTypeSetting.get(settings).toString(); - } else if (globalOpSetting.exists(settings)) { - // like: "script.aggs: true" - return globalOpSetting.get(settings).toString(); - } else if (globalTypeSetting.exists(settings)) { - // like: "script.inline: true" - return globalTypeSetting.get(settings).toString(); - } else { - // Nothing is set! - return Boolean.toString(defaultIfNothingSet); - } - }; - // The actual setting for finest grained script settings - Setting<Boolean> setting = - Setting.boolSetting(langAndTypeAndContextName, defaultSettingFn, Property.NodeScope, Property.Deprecated); - scriptModeSettings.add(setting); - } - } - } - return scriptModeSettings; - } - - public List<Setting<?>> getSettings() { - List<Setting<?>> settings = new ArrayList<>(); - settings.addAll(SCRIPT_TYPE_SETTING_MAP.values()); - settings.addAll(scriptContextSettingMap.values()); - settings.addAll(scriptLanguageSettings); - return settings; - } - - public Iterable<Setting<Boolean>> getScriptLanguageSettings() { - return scriptLanguageSettings; - } -} diff --git a/core/src/main/java/org/elasticsearch/script/ScriptType.java b/core/src/main/java/org/elasticsearch/script/ScriptType.java index cb77e37404..c076ccfd88 100644 --- a/core/src/main/java/org/elasticsearch/script/ScriptType.java +++ b/core/src/main/java/org/elasticsearch/script/ScriptType.java @@ -28,8 +28,8 @@ import java.io.IOException; /** * ScriptType represents the way a script is stored and retrieved from the {@link ScriptService}. - * It's also used to by {@link ScriptSettings} and {@link ScriptModes} to determine whether or not - * a {@link Script} is allowed to be executed based on both default and user-defined settings. + * It's also used to by {@link ScriptService} to determine whether or not a {@link Script} is + * allowed to be executed based on both default and user-defined settings. */ public enum ScriptType implements Writeable { diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalDateHistogram.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalDateHistogram.java index c3eab06f28..6082366094 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalDateHistogram.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalDateHistogram.java @@ -369,8 +369,8 @@ public final class InternalDateHistogram extends InternalMultiBucketAggregation< Bucket firstBucket = iter.hasNext() ? list.get(iter.nextIndex()) : null; if (firstBucket == null) { if (bounds.getMin() != null && bounds.getMax() != null) { - long key = bounds.getMin(); - long max = bounds.getMax(); + long key = bounds.getMin() + offset; + long max = bounds.getMax() + offset; while (key <= max) { iter.add(new InternalDateHistogram.Bucket(key, 0, keyed, format, reducedEmptySubAggs)); key = nextKey(key).longValue(); @@ -378,7 +378,7 @@ public final class InternalDateHistogram extends InternalMultiBucketAggregation< } } else { if (bounds.getMin() != null) { - long key = bounds.getMin(); + long key = bounds.getMin() + offset; if (key < firstBucket.key) { while (key < firstBucket.key) { iter.add(new InternalDateHistogram.Bucket(key, 0, keyed, format, reducedEmptySubAggs)); @@ -405,12 +405,12 @@ public final class InternalDateHistogram extends InternalMultiBucketAggregation< } // finally, adding the empty buckets *after* the actual data (based on the extended_bounds.max requested by the user) - if (bounds != null && lastBucket != null && bounds.getMax() != null && bounds.getMax() > lastBucket.key) { - long key = emptyBucketInfo.rounding.nextRoundingValue(lastBucket.key); - long max = bounds.getMax(); + if (bounds != null && lastBucket != null && bounds.getMax() != null && bounds.getMax() + offset > lastBucket.key) { + long key = nextKey(lastBucket.key).longValue(); + long max = bounds.getMax() + offset; while (key <= max) { iter.add(new InternalDateHistogram.Bucket(key, 0, keyed, format, reducedEmptySubAggs)); - key = emptyBucketInfo.rounding.nextRoundingValue(key); + key = nextKey(key).longValue(); } } } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantLongTerms.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantLongTerms.java index c88c7c8e1f..29a8e16d9b 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantLongTerms.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantLongTerms.java @@ -77,11 +77,6 @@ public class SignificantLongTerms extends InternalMappedSignificantTerms<Signifi } @Override - public int compareTerm(SignificantTerms.Bucket other) { - return Long.compare(term, ((Number) other.getKey()).longValue()); - } - - @Override public String getKeyAsString() { return format.format(term); } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantStringTerms.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantStringTerms.java index 6627a2295d..b170171f5c 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantStringTerms.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantStringTerms.java @@ -28,7 +28,6 @@ import org.elasticsearch.search.aggregations.bucket.significant.heuristics.Signi import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; import java.io.IOException; -import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Objects; @@ -83,11 +82,6 @@ public class SignificantStringTerms extends InternalMappedSignificantTerms<Signi } @Override - public int compareTerm(SignificantTerms.Bucket other) { - return termBytes.compareTo(((Bucket) other).termBytes); - } - - @Override public String getKeyAsString() { return format.format(termBytes); } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTerms.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTerms.java index ae09feef93..8c0da8b890 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTerms.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTerms.java @@ -40,8 +40,6 @@ public interface SignificantTerms extends MultiBucketsAggregation, Iterable<Sign long getSupersetSize(); long getSubsetSize(); - - int compareTerm(SignificantTerms.Bucket other); } @Override diff --git a/core/src/test/java/org/elasticsearch/VersionTests.java b/core/src/test/java/org/elasticsearch/VersionTests.java index 96a0c9aa81..27782511de 100644 --- a/core/src/test/java/org/elasticsearch/VersionTests.java +++ b/core/src/test/java/org/elasticsearch/VersionTests.java @@ -293,7 +293,6 @@ public class VersionTests extends ESTestCase { if (maxBranchVersion == null) { maxBranchVersions.put(branchName, v); } else if (v.after(maxBranchVersion)) { - assertFalse("Version " + maxBranchVersion + " cannot be a snapshot because version " + v + " exists", VersionUtils.isSnapshot(maxBranchVersion)); maxBranchVersions.put(branchName, v); } @@ -329,6 +328,16 @@ public class VersionTests extends ESTestCase { assertTrue(isCompatible(Version.V_5_5_0_UNRELEASED, Version.V_6_0_0_alpha2_UNRELEASED)); assertFalse(isCompatible(Version.fromId(2000099), Version.V_6_0_0_alpha2_UNRELEASED)); assertFalse(isCompatible(Version.fromId(2000099), Version.V_5_0_0)); + assertTrue(isCompatible(Version.fromString("6.0.0"), Version.fromString("7.0.0"))); + if (Version.CURRENT.isRelease()) { + assertTrue(isCompatible(Version.CURRENT, Version.fromString("7.0.0"))); + } else { + assertFalse(isCompatible(Version.CURRENT, Version.fromString("7.0.0"))); + } + assertFalse("only compatible with the latest minor", + isCompatible(VersionUtils.getPreviousMinorVersion(), Version.fromString("7.0.0"))); + assertFalse(isCompatible(Version.V_5_0_0, Version.fromString("6.0.0"))); + assertFalse(isCompatible(Version.V_5_0_0, Version.fromString("7.0.0"))); } diff --git a/core/src/test/java/org/elasticsearch/action/update/UpdateRequestTests.java b/core/src/test/java/org/elasticsearch/action/update/UpdateRequestTests.java index e03f3ec45f..a3bb96a72f 100644 --- a/core/src/test/java/org/elasticsearch/action/update/UpdateRequestTests.java +++ b/core/src/test/java/org/elasticsearch/action/update/UpdateRequestTests.java @@ -45,7 +45,6 @@ import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptContextRegistry; import org.elasticsearch.script.ScriptEngineRegistry; import org.elasticsearch.script.ScriptService; -import org.elasticsearch.script.ScriptSettings; import org.elasticsearch.script.ScriptType; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.RandomObjects; @@ -64,7 +63,6 @@ import static java.util.Collections.emptyMap; import static java.util.Collections.singletonList; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.common.xcontent.XContentHelper.toXContent; -import static org.elasticsearch.common.xcontent.XContentHelper.update; import static org.elasticsearch.script.MockScriptEngine.mockInlineScript; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertToXContentEquivalent; import static org.hamcrest.Matchers.arrayContaining; @@ -145,15 +143,12 @@ public class UpdateRequestTests extends ESTestCase { final ScriptEngineRegistry scriptEngineRegistry = new ScriptEngineRegistry(singletonList(engine)); - final ScriptSettings scriptSettings = - new ScriptSettings(scriptEngineRegistry, scriptContextRegistry); final ResourceWatcherService watcherService = new ResourceWatcherService(baseSettings, null); ScriptService scriptService = new ScriptService( baseSettings, scriptEngineRegistry, - scriptContextRegistry, - scriptSettings); + scriptContextRegistry); final Settings settings = settings(Version.CURRENT).build(); updateHelper = new UpdateHelper(settings, scriptService); diff --git a/core/src/test/java/org/elasticsearch/discovery/zen/PublishClusterStateActionTests.java b/core/src/test/java/org/elasticsearch/discovery/zen/PublishClusterStateActionTests.java index 863bf80085..2e29347204 100644 --- a/core/src/test/java/org/elasticsearch/discovery/zen/PublishClusterStateActionTests.java +++ b/core/src/test/java/org/elasticsearch/discovery/zen/PublishClusterStateActionTests.java @@ -22,6 +22,7 @@ package org.elasticsearch.discovery.zen; import org.apache.logging.log4j.Logger; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.Version; +import org.elasticsearch.action.ActionListener; import org.elasticsearch.cluster.ClusterChangedEvent; import org.elasticsearch.cluster.ClusterModule; import org.elasticsearch.cluster.ClusterName; @@ -64,20 +65,17 @@ import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.List; -import java.util.Locale; import java.util.Map; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; -import java.util.function.Supplier; import static org.hamcrest.CoreMatchers.instanceOf; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.emptyIterable; import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.hasToString; import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; @@ -90,11 +88,12 @@ public class PublishClusterStateActionTests extends ESTestCase { protected ThreadPool threadPool; protected Map<String, MockNode> nodes = new HashMap<>(); - public static class MockNode implements PublishClusterStateAction.NewPendingClusterStateListener { + public static class MockNode implements PublishClusterStateAction.IncomingClusterStateListener { public final DiscoveryNode discoveryNode; public final MockTransportService service; public MockPublishAction action; public final ClusterStateListener listener; + private final PendingClusterStatesQueue pendingStatesQueue; public volatile ClusterState clusterState; @@ -108,6 +107,7 @@ public class PublishClusterStateActionTests extends ESTestCase { this.logger = logger; this.clusterState = ClusterState.builder(CLUSTER_NAME).nodes(DiscoveryNodes.builder() .add(discoveryNode).localNodeId(discoveryNode.getId()).build()).build(); + this.pendingStatesQueue = new PendingClusterStatesQueue(logger, 25); } public MockNode setAsMaster() { @@ -128,18 +128,37 @@ public class PublishClusterStateActionTests extends ESTestCase { } @Override - public void onNewClusterState(String reason) { - ClusterState newClusterState = action.pendingStatesQueue().getNextClusterStateToProcess(); - logger.debug("[{}] received version [{}], uuid [{}]", - discoveryNode.getName(), newClusterState.version(), newClusterState.stateUUID()); - if (listener != null) { - ClusterChangedEvent event = new ClusterChangedEvent("", newClusterState, clusterState); - listener.clusterChanged(event); - } - if (clusterState.nodes().getMasterNode() == null || newClusterState.supersedes(clusterState)) { - clusterState = newClusterState; + public void onIncomingClusterState(ClusterState incomingState) { + ZenDiscovery.validateIncomingState(logger, incomingState, clusterState); + pendingStatesQueue.addPending(incomingState); + } + + public void onClusterStateCommitted(String stateUUID, ActionListener<Void> processedListener) { + final ClusterState state = pendingStatesQueue.markAsCommitted(stateUUID, + new PendingClusterStatesQueue.StateProcessedListener() { + @Override + public void onNewClusterStateProcessed() { + processedListener.onResponse(null); + } + + @Override + public void onNewClusterStateFailed(Exception e) { + processedListener.onFailure(e); + } + }); + if (state != null) { + ClusterState newClusterState = pendingStatesQueue.getNextClusterStateToProcess(); + logger.debug("[{}] received version [{}], uuid [{}]", + discoveryNode.getName(), newClusterState.version(), newClusterState.stateUUID()); + if (listener != null) { + ClusterChangedEvent event = new ClusterChangedEvent("", newClusterState, clusterState); + listener.clusterChanged(event); + } + if (clusterState.nodes().getMasterNode() == null || newClusterState.supersedes(clusterState)) { + clusterState = newClusterState; + } + pendingStatesQueue.markAsProcessed(newClusterState); } - action.pendingStatesQueue().markAsProcessed(newClusterState); } public DiscoveryNodes nodes() { @@ -168,7 +187,7 @@ public class PublishClusterStateActionTests extends ESTestCase { MockTransportService service = buildTransportService(settings, threadPool); DiscoveryNode discoveryNode = service.getLocalDiscoNode(); MockNode node = new MockNode(discoveryNode, service, listener, logger); - node.action = buildPublishClusterStateAction(settings, service, () -> node.clusterState, node); + node.action = buildPublishClusterStateAction(settings, service, node); final CountDownLatch latch = new CountDownLatch(nodes.size() * 2); TransportConnectionListener waitForConnection = new TransportConnectionListener() { @Override @@ -241,8 +260,7 @@ public class PublishClusterStateActionTests extends ESTestCase { private static MockPublishAction buildPublishClusterStateAction( Settings settings, MockTransportService transportService, - Supplier<ClusterState> clusterStateSupplier, - PublishClusterStateAction.NewPendingClusterStateListener listener + PublishClusterStateAction.IncomingClusterStateListener listener ) { DiscoverySettings discoverySettings = new DiscoverySettings(settings, new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)); @@ -251,10 +269,8 @@ public class PublishClusterStateActionTests extends ESTestCase { settings, transportService, namedWriteableRegistry, - clusterStateSupplier, listener, - discoverySettings, - CLUSTER_NAME); + discoverySettings); } public void testSimpleClusterStatePublishing() throws Exception { @@ -607,86 +623,6 @@ public class PublishClusterStateActionTests extends ESTestCase { } } - public void testIncomingClusterStateValidation() throws Exception { - MockNode node = createMockNode("node"); - - logger.info("--> testing acceptances of any master when having no master"); - ClusterState state = ClusterState.builder(node.clusterState) - .nodes(DiscoveryNodes.builder(node.nodes()).masterNodeId(randomAlphaOfLength(10))).incrementVersion().build(); - node.action.validateIncomingState(state, null); - - // now set a master node - node.clusterState = ClusterState.builder(node.clusterState) - .nodes(DiscoveryNodes.builder(node.nodes()).masterNodeId("master")).build(); - logger.info("--> testing rejection of another master"); - try { - node.action.validateIncomingState(state, node.clusterState); - fail("node accepted state from another master"); - } catch (IllegalStateException OK) { - assertThat(OK.toString(), containsString("cluster state from a different master than the current one, rejecting")); - } - - logger.info("--> test state from the current master is accepted"); - node.action.validateIncomingState(ClusterState.builder(node.clusterState) - .nodes(DiscoveryNodes.builder(node.nodes()).masterNodeId("master")).incrementVersion().build(), node.clusterState); - - - logger.info("--> testing rejection of another cluster name"); - try { - node.action.validateIncomingState(ClusterState.builder(new ClusterName(randomAlphaOfLength(10))) - .nodes(node.nodes()).build(), node.clusterState); - fail("node accepted state with another cluster name"); - } catch (IllegalStateException OK) { - assertThat(OK.toString(), containsString("received state from a node that is not part of the cluster")); - } - - logger.info("--> testing rejection of a cluster state with wrong local node"); - try { - state = ClusterState.builder(node.clusterState) - .nodes(DiscoveryNodes.builder(node.nodes()).localNodeId("_non_existing_").build()) - .incrementVersion().build(); - node.action.validateIncomingState(state, node.clusterState); - fail("node accepted state with non-existence local node"); - } catch (IllegalStateException OK) { - assertThat(OK.toString(), containsString("received state with a local node that does not match the current local node")); - } - - try { - MockNode otherNode = createMockNode("otherNode"); - state = ClusterState.builder(node.clusterState).nodes( - DiscoveryNodes.builder(node.nodes()).add(otherNode.discoveryNode).localNodeId(otherNode.discoveryNode.getId()).build() - ).incrementVersion().build(); - node.action.validateIncomingState(state, node.clusterState); - fail("node accepted state with existent but wrong local node"); - } catch (IllegalStateException OK) { - assertThat(OK.toString(), containsString("received state with a local node that does not match the current local node")); - } - - logger.info("--> testing acceptance of an old cluster state"); - final ClusterState incomingState = node.clusterState; - node.clusterState = ClusterState.builder(node.clusterState).incrementVersion().build(); - final IllegalStateException e = - expectThrows(IllegalStateException.class, () -> node.action.validateIncomingState(incomingState, node.clusterState)); - final String message = String.format( - Locale.ROOT, - "rejecting cluster state version [%d] uuid [%s] received from [%s]", - incomingState.version(), - incomingState.stateUUID(), - incomingState.nodes().getMasterNodeId() - ); - assertThat(e, hasToString("java.lang.IllegalStateException: " + message)); - - // an older version from a *new* master is also OK! - ClusterState previousState = ClusterState.builder(node.clusterState).incrementVersion().build(); - state = ClusterState.builder(node.clusterState) - .nodes(DiscoveryNodes.builder(node.clusterState.nodes()).masterNodeId("_new_master_").build()) - .build(); - // remove the master of the node (but still have a previous cluster state with it)! - node.resetMasterId(); - - node.action.validateIncomingState(state, previousState); - } - public void testOutOfOrderCommitMessages() throws Throwable { MockNode node = createMockNode("node").setAsMaster(); final CapturingTransportChannel channel = new CapturingTransportChannel(); @@ -874,9 +810,8 @@ public class PublishClusterStateActionTests extends ESTestCase { AtomicBoolean errorOnCommit = new AtomicBoolean(); public MockPublishAction(Settings settings, TransportService transportService, NamedWriteableRegistry namedWriteableRegistry, - Supplier<ClusterState> clusterStateSupplier, NewPendingClusterStateListener listener, - DiscoverySettings discoverySettings, ClusterName clusterName) { - super(settings, transportService, namedWriteableRegistry, clusterStateSupplier, listener, discoverySettings, clusterName); + IncomingClusterStateListener listener, DiscoverySettings discoverySettings) { + super(settings, transportService, namedWriteableRegistry, listener, discoverySettings); } @Override diff --git a/core/src/test/java/org/elasticsearch/discovery/zen/ZenDiscoveryUnitTests.java b/core/src/test/java/org/elasticsearch/discovery/zen/ZenDiscoveryUnitTests.java index 65856add56..cb88213cfe 100644 --- a/core/src/test/java/org/elasticsearch/discovery/zen/ZenDiscoveryUnitTests.java +++ b/core/src/test/java/org/elasticsearch/discovery/zen/ZenDiscoveryUnitTests.java @@ -67,6 +67,7 @@ import java.util.Collections; import java.util.EnumSet; import java.util.HashSet; import java.util.List; +import java.util.Locale; import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; @@ -88,6 +89,7 @@ import static org.hamcrest.Matchers.arrayWithSize; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.emptyArray; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasToString; public class ZenDiscoveryUnitTests extends ESTestCase { @@ -405,4 +407,94 @@ public class ZenDiscoveryUnitTests extends ESTestCase { } } } + + public void testIncomingClusterStateValidation() throws Exception { + ClusterName clusterName = new ClusterName("abc"); + + DiscoveryNodes.Builder currentNodes = DiscoveryNodes.builder().add( + new DiscoveryNode("a", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT)).localNodeId("a"); + + ClusterState previousState = ClusterState.builder(clusterName).nodes(currentNodes).build(); + + logger.info("--> testing acceptances of any master when having no master"); + ClusterState state = ClusterState.builder(previousState) + .nodes(DiscoveryNodes.builder(previousState.nodes()).masterNodeId(randomAlphaOfLength(10))).incrementVersion().build(); + ZenDiscovery.validateIncomingState(logger, state, previousState); + + // now set a master node + previousState = state; + state = ClusterState.builder(previousState) + .nodes(DiscoveryNodes.builder(previousState.nodes()).masterNodeId("master")).build(); + logger.info("--> testing rejection of another master"); + try { + ZenDiscovery.validateIncomingState(logger, state, previousState); + fail("node accepted state from another master"); + } catch (IllegalStateException OK) { + assertThat(OK.toString(), containsString("cluster state from a different master than the current one, rejecting")); + } + + logger.info("--> test state from the current master is accepted"); + previousState = state; + ZenDiscovery.validateIncomingState(logger, ClusterState.builder(previousState) + .nodes(DiscoveryNodes.builder(previousState.nodes()).masterNodeId("master")).incrementVersion().build(), previousState); + + + logger.info("--> testing rejection of another cluster name"); + try { + ZenDiscovery.validateIncomingState(logger, ClusterState.builder(new ClusterName(randomAlphaOfLength(10))) + .nodes(previousState.nodes()).build(), previousState); + fail("node accepted state with another cluster name"); + } catch (IllegalStateException OK) { + assertThat(OK.toString(), containsString("received state from a node that is not part of the cluster")); + } + + logger.info("--> testing rejection of a cluster state with wrong local node"); + try { + state = ClusterState.builder(previousState) + .nodes(DiscoveryNodes.builder(previousState.nodes()).localNodeId("_non_existing_").build()) + .incrementVersion().build(); + ZenDiscovery.validateIncomingState(logger, state, previousState); + fail("node accepted state with non-existence local node"); + } catch (IllegalStateException OK) { + assertThat(OK.toString(), containsString("received state with a local node that does not match the current local node")); + } + + try { + DiscoveryNode otherNode = new DiscoveryNode("b", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT); + state = ClusterState.builder(previousState).nodes( + DiscoveryNodes.builder(previousState.nodes()).add(otherNode) + .localNodeId(otherNode.getId()).build() + ).incrementVersion().build(); + ZenDiscovery.validateIncomingState(logger, state, previousState); + fail("node accepted state with existent but wrong local node"); + } catch (IllegalStateException OK) { + assertThat(OK.toString(), containsString("received state with a local node that does not match the current local node")); + } + + logger.info("--> testing acceptance of an old cluster state"); + final ClusterState incomingState = previousState; + previousState = ClusterState.builder(previousState).incrementVersion().build(); + final ClusterState finalPreviousState = previousState; + final IllegalStateException e = + expectThrows(IllegalStateException.class, () -> ZenDiscovery.validateIncomingState(logger, incomingState, finalPreviousState)); + final String message = String.format( + Locale.ROOT, + "rejecting cluster state version [%d] uuid [%s] received from [%s]", + incomingState.version(), + incomingState.stateUUID(), + incomingState.nodes().getMasterNodeId() + ); + assertThat(e, hasToString("java.lang.IllegalStateException: " + message)); + + ClusterState higherVersionState = ClusterState.builder(previousState).incrementVersion().build(); + // remove the master of the node (but still have a previous cluster state with it)! + higherVersionState = ClusterState.builder(higherVersionState) + .nodes(DiscoveryNodes.builder(higherVersionState.nodes()).masterNodeId(null)).build(); + // an older version from a *new* master is also OK! + state = ClusterState.builder(previousState) + .nodes(DiscoveryNodes.builder(previousState.nodes()).masterNodeId("_new_master_").build()) + .build(); + + ZenDiscovery.validateIncomingState(logger, state, higherVersionState); + } } diff --git a/core/src/test/java/org/elasticsearch/index/IndexModuleTests.java b/core/src/test/java/org/elasticsearch/index/IndexModuleTests.java index 7dd97f8554..b1a9bc7501 100644 --- a/core/src/test/java/org/elasticsearch/index/IndexModuleTests.java +++ b/core/src/test/java/org/elasticsearch/index/IndexModuleTests.java @@ -70,7 +70,6 @@ import org.elasticsearch.indices.mapper.MapperRegistry; import org.elasticsearch.script.ScriptContextRegistry; import org.elasticsearch.script.ScriptEngineRegistry; import org.elasticsearch.script.ScriptService; -import org.elasticsearch.script.ScriptSettings; import org.elasticsearch.search.internal.SearchContext; import org.elasticsearch.test.ClusterServiceUtils; import org.elasticsearch.test.ESTestCase; @@ -130,8 +129,7 @@ public class IndexModuleTests extends ESTestCase { bigArrays = new BigArrays(settings, circuitBreakerService); ScriptEngineRegistry scriptEngineRegistry = new ScriptEngineRegistry(emptyList()); ScriptContextRegistry scriptContextRegistry = new ScriptContextRegistry(Collections.emptyList()); - ScriptSettings scriptSettings = new ScriptSettings(scriptEngineRegistry, scriptContextRegistry); - scriptService = new ScriptService(settings, scriptEngineRegistry, scriptContextRegistry, scriptSettings); + scriptService = new ScriptService(settings, scriptEngineRegistry, scriptContextRegistry); clusterService = ClusterServiceUtils.createClusterService(threadPool); nodeEnvironment = new NodeEnvironment(settings, environment); mapperRegistry = new IndicesModule(Collections.emptyList()).getMapperRegistry(); diff --git a/core/src/test/java/org/elasticsearch/script/ScriptContextTests.java b/core/src/test/java/org/elasticsearch/script/ScriptContextTests.java index 182a6c1af5..6fc77eeb9b 100644 --- a/core/src/test/java/org/elasticsearch/script/ScriptContextTests.java +++ b/core/src/test/java/org/elasticsearch/script/ScriptContextTests.java @@ -23,7 +23,6 @@ import org.elasticsearch.cluster.ClusterChangedEvent; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.MetaData; -import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.Environment; import org.elasticsearch.test.ESTestCase; @@ -32,21 +31,14 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; -import static org.hamcrest.Matchers.containsString; - public class ScriptContextTests extends ESTestCase { private static final String PLUGIN_NAME = "testplugin"; - private static final String SCRIPT_PLUGIN_CUSTOM_SETTING = "script." + PLUGIN_NAME + "_custom_globally_disabled_op"; - private static final String SCRIPT_ENGINE_CUSTOM_SETTING = "script.engine." + MockScriptEngine.NAME + ".inline." + PLUGIN_NAME + "_custom_exp_disabled_op"; - - private ScriptSettings scriptSettings; ScriptService makeScriptService() throws Exception { Settings settings = Settings.builder() .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir()) - .put(SCRIPT_PLUGIN_CUSTOM_SETTING, "false") - .put(SCRIPT_ENGINE_CUSTOM_SETTING, "false") + .put("script.contexts_allowed", "search, aggs, testplugin_custom_op") .build(); MockScriptEngine scriptEngine = new MockScriptEngine(MockScriptEngine.NAME, Collections.singletonMap("1", script -> "1")); @@ -56,8 +48,7 @@ public class ScriptContextTests extends ESTestCase { new ScriptContext.Plugin(PLUGIN_NAME, "custom_exp_disabled_op"), new ScriptContext.Plugin(PLUGIN_NAME, "custom_globally_disabled_op")); ScriptContextRegistry scriptContextRegistry = new ScriptContextRegistry(customContexts); - scriptSettings = new ScriptSettings(scriptEngineRegistry, scriptContextRegistry); - ScriptService scriptService = new ScriptService(settings, scriptEngineRegistry, scriptContextRegistry, scriptSettings); + ScriptService scriptService = new ScriptService(settings, scriptEngineRegistry, scriptContextRegistry); ClusterState empty = ClusterState.builder(new ClusterName("_name")).build(); ScriptMetaData smd = empty.metaData().custom(ScriptMetaData.TYPE); @@ -69,8 +60,6 @@ public class ScriptContextTests extends ESTestCase { return scriptService; } - - public void testCustomGlobalScriptContextSettings() throws Exception { ScriptService scriptService = makeScriptService(); for (ScriptType scriptType : ScriptType.values()) { @@ -78,12 +67,10 @@ public class ScriptContextTests extends ESTestCase { Script script = new Script(scriptType, MockScriptEngine.NAME, "1", Collections.emptyMap()); scriptService.compile(script, new ScriptContext.Plugin(PLUGIN_NAME, "custom_globally_disabled_op")); fail("script compilation should have been rejected"); - } catch (IllegalStateException e) { - assertThat(e.getMessage(), containsString("scripts of type [" + scriptType + "], operation [" + PLUGIN_NAME + "_custom_globally_disabled_op] and lang [" + MockScriptEngine.NAME + "] are disabled")); + } catch (IllegalArgumentException e) { + assertTrue(e.getMessage(), e.getMessage().contains("cannot execute scripts using [" + PLUGIN_NAME + "_custom_globally_disabled_op] context")); } } - assertSettingDeprecationsAndWarnings( - ScriptSettingsTests.buildDeprecatedSettingsArray(scriptSettings, SCRIPT_PLUGIN_CUSTOM_SETTING, SCRIPT_ENGINE_CUSTOM_SETTING)); } public void testCustomScriptContextSettings() throws Exception { @@ -92,16 +79,14 @@ public class ScriptContextTests extends ESTestCase { try { scriptService.compile(script, new ScriptContext.Plugin(PLUGIN_NAME, "custom_exp_disabled_op")); fail("script compilation should have been rejected"); - } catch (IllegalStateException e) { - assertTrue(e.getMessage(), e.getMessage().contains("scripts of type [inline], operation [" + PLUGIN_NAME + "_custom_exp_disabled_op] and lang [" + MockScriptEngine.NAME + "] are disabled")); + } catch (IllegalArgumentException e) { + assertTrue(e.getMessage(), e.getMessage().contains("cannot execute scripts using [" + PLUGIN_NAME + "_custom_exp_disabled_op] context")); } // still works for other script contexts assertNotNull(scriptService.compile(script, ScriptContext.Standard.AGGS)); assertNotNull(scriptService.compile(script, ScriptContext.Standard.SEARCH)); assertNotNull(scriptService.compile(script, new ScriptContext.Plugin(PLUGIN_NAME, "custom_op"))); - assertSettingDeprecationsAndWarnings( - ScriptSettingsTests.buildDeprecatedSettingsArray(scriptSettings, SCRIPT_PLUGIN_CUSTOM_SETTING, SCRIPT_ENGINE_CUSTOM_SETTING)); } public void testUnknownPluginScriptContext() throws Exception { @@ -115,8 +100,6 @@ public class ScriptContextTests extends ESTestCase { assertTrue(e.getMessage(), e.getMessage().contains("script context [" + PLUGIN_NAME + "_unknown] not supported")); } } - assertSettingDeprecationsAndWarnings( - ScriptSettingsTests.buildDeprecatedSettingsArray(scriptSettings, SCRIPT_PLUGIN_CUSTOM_SETTING, SCRIPT_ENGINE_CUSTOM_SETTING)); } public void testUnknownCustomScriptContext() throws Exception { @@ -136,7 +119,5 @@ public class ScriptContextTests extends ESTestCase { assertTrue(e.getMessage(), e.getMessage().contains("script context [test] not supported")); } } - assertSettingDeprecationsAndWarnings( - ScriptSettingsTests.buildDeprecatedSettingsArray(scriptSettings, SCRIPT_PLUGIN_CUSTOM_SETTING, SCRIPT_ENGINE_CUSTOM_SETTING)); } } diff --git a/core/src/test/java/org/elasticsearch/script/ScriptModesTests.java b/core/src/test/java/org/elasticsearch/script/ScriptModesTests.java deleted file mode 100644 index 653ac47485..0000000000 --- a/core/src/test/java/org/elasticsearch/script/ScriptModesTests.java +++ /dev/null @@ -1,254 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.script; - -import org.elasticsearch.common.Nullable; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.search.lookup.SearchLookup; -import org.elasticsearch.test.ESTestCase; -import org.junit.After; -import org.junit.Before; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; - -import static java.util.Collections.unmodifiableMap; -import static org.elasticsearch.common.util.set.Sets.newHashSet; -import static org.hamcrest.CoreMatchers.equalTo; -import static org.hamcrest.CoreMatchers.notNullValue; -import static org.hamcrest.Matchers.containsString; - -// TODO: this needs to be a base test class, and all scripting engines extend it -public class ScriptModesTests extends ESTestCase { - ScriptSettings scriptSettings; - ScriptContextRegistry scriptContextRegistry; - private ScriptContext[] scriptContexts; - private Map<String, ScriptEngine> scriptEngines; - private ScriptModes scriptModes; - private Set<String> checkedSettings; - private boolean assertAllSettingsWereChecked; - private boolean assertScriptModesNonNull; - - @Before - public void setupScriptEngines() { - //randomly register custom script contexts - int randomInt = randomIntBetween(0, 3); - //prevent duplicates using map - Map<String, ScriptContext.Plugin> contexts = new HashMap<>(); - for (int i = 0; i < randomInt; i++) { - String plugin = randomAlphaOfLength(randomIntBetween(1, 10)); - String operation = randomAlphaOfLength(randomIntBetween(1, 30)); - String context = plugin + "-" + operation; - contexts.put(context, new ScriptContext.Plugin(plugin, operation)); - } - scriptContextRegistry = new ScriptContextRegistry(contexts.values()); - scriptContexts = scriptContextRegistry.scriptContexts().toArray(new ScriptContext[scriptContextRegistry.scriptContexts().size()]); - scriptEngines = buildScriptEnginesByLangMap(newHashSet(new CustomScriptEngine())); - ScriptEngineRegistry scriptEngineRegistry = new ScriptEngineRegistry(scriptEngines.values()); - scriptSettings = new ScriptSettings(scriptEngineRegistry, scriptContextRegistry); - checkedSettings = new HashSet<>(); - assertAllSettingsWereChecked = true; - assertScriptModesNonNull = true; - } - - @After - public void assertAllSettingsWereChecked() { - if (assertScriptModesNonNull) { - assertThat(scriptModes, notNullValue()); - int numberOfSettings = ScriptType.values().length * scriptContextRegistry.scriptContexts().size(); - numberOfSettings += 2; // for top-level inline/store settings - assertThat(scriptModes.scriptEnabled.size(), equalTo(numberOfSettings)); - if (assertAllSettingsWereChecked) { - assertThat(checkedSettings.size(), equalTo(numberOfSettings)); - } - } - } - - public void testDefaultSettings() { - this.scriptModes = new ScriptModes(scriptContextRegistry, scriptSettings, Settings.EMPTY); - assertScriptModesAllOps(false, ScriptType.STORED, ScriptType.INLINE); - } - - public void testMissingSetting() { - assertAllSettingsWereChecked = false; - this.scriptModes = new ScriptModes(scriptContextRegistry, scriptSettings, Settings.EMPTY); - try { - scriptModes.getScriptEnabled("non_existing", randomFrom(ScriptType.values()), randomFrom(scriptContexts)); - fail("Expected IllegalArgumentException"); - } catch (IllegalArgumentException e) { - assertThat(e.getMessage(), containsString("not found for lang [non_existing]")); - } - } - - public void testScriptTypeGenericSettings() { - int randomInt = randomIntBetween(1, ScriptType.values().length - 1); - Set<ScriptType> randomScriptTypesSet = new HashSet<>(); - boolean[] randomScriptModes = new boolean[randomInt]; - for (int i = 0; i < randomInt; i++) { - boolean added = false; - while (added == false) { - added = randomScriptTypesSet.add(randomFrom(ScriptType.values())); - } - randomScriptModes[i] = randomBoolean(); - } - ScriptType[] randomScriptTypes = randomScriptTypesSet.toArray(new ScriptType[randomScriptTypesSet.size()]); - List<String> deprecated = new ArrayList<>(); - Settings.Builder builder = Settings.builder(); - for (int i = 0; i < randomInt; i++) { - builder.put("script" + "." + randomScriptTypes[i].getName(), randomScriptModes[i]); - deprecated.add("script" + "." + randomScriptTypes[i].getName()); - } - this.scriptModes = new ScriptModes(scriptContextRegistry, scriptSettings, builder.build()); - - for (int i = 0; i < randomInt; i++) { - assertScriptModesAllOps(randomScriptModes[i], randomScriptTypes[i]); - } - if (randomScriptTypesSet.contains(ScriptType.STORED) == false) { - assertScriptModesAllOps(false, ScriptType.STORED); - } - if (randomScriptTypesSet.contains(ScriptType.INLINE) == false) { - assertScriptModesAllOps(false, ScriptType.INLINE); - } - assertSettingDeprecationsAndWarnings( - ScriptSettingsTests.buildDeprecatedSettingsArray(scriptSettings, deprecated.toArray(new String[] {}))); - } - - public void testScriptContextGenericSettings() { - int randomInt = randomIntBetween(1, scriptContexts.length - 1); - Set<ScriptContext> randomScriptContextsSet = new HashSet<>(); - boolean[] randomScriptModes = new boolean[randomInt]; - for (int i = 0; i < randomInt; i++) { - boolean added = false; - while (added == false) { - added = randomScriptContextsSet.add(randomFrom(scriptContexts)); - } - randomScriptModes[i] = randomBoolean(); - } - ScriptContext[] randomScriptContexts = randomScriptContextsSet.toArray(new ScriptContext[randomScriptContextsSet.size()]); - List<String> deprecated = new ArrayList<>(); - Settings.Builder builder = Settings.builder(); - for (int i = 0; i < randomInt; i++) { - builder.put("script" + "." + randomScriptContexts[i].getKey(), randomScriptModes[i]); - deprecated.add("script" + "." + randomScriptContexts[i].getKey()); - } - this.scriptModes = new ScriptModes(scriptContextRegistry, scriptSettings, builder.build()); - - for (int i = 0; i < randomInt; i++) { - assertScriptModesAllTypes(randomScriptModes[i], randomScriptContexts[i]); - } - - ScriptContext[] complementOf = complementOf(randomScriptContexts); - assertScriptModes(false, new ScriptType[]{ScriptType.STORED, ScriptType.INLINE}, complementOf); - assertSettingDeprecationsAndWarnings( - ScriptSettingsTests.buildDeprecatedSettingsArray(scriptSettings, deprecated.toArray(new String[] {}))); - } - - public void testConflictingScriptTypeAndOpGenericSettings() { - ScriptContext scriptContext = randomFrom(scriptContexts); - Settings.Builder builder = Settings.builder() - .put("script." + scriptContext.getKey(), "false") - .put("script.stored", "true") - .put("script.inline", "true"); - //operations generic settings have precedence over script type generic settings - this.scriptModes = new ScriptModes(scriptContextRegistry, scriptSettings, builder.build()); - assertScriptModesAllTypes(false, scriptContext); - ScriptContext[] complementOf = complementOf(scriptContext); - assertScriptModes(true, new ScriptType[]{ScriptType.STORED}, complementOf); - assertScriptModes(true, new ScriptType[]{ScriptType.INLINE}, complementOf); - assertSettingDeprecationsAndWarnings( - ScriptSettingsTests.buildDeprecatedSettingsArray( - scriptSettings, "script." + scriptContext.getKey(), "script.stored", "script.inline")); - } - - private void assertScriptModesAllOps(boolean expectedScriptEnabled, ScriptType... scriptTypes) { - assertScriptModes(expectedScriptEnabled, scriptTypes, scriptContexts); - } - - private void assertScriptModesAllTypes(boolean expectedScriptEnabled, ScriptContext... scriptContexts) { - assertScriptModes(expectedScriptEnabled, ScriptType.values(), scriptContexts); - } - - private void assertScriptModes(boolean expectedScriptEnabled, ScriptType[] scriptTypes, ScriptContext... scriptContexts) { - assert scriptTypes.length > 0; - assert scriptContexts.length > 0; - for (ScriptType scriptType : scriptTypes) { - checkedSettings.add("script.engine.custom." + scriptType); - for (ScriptContext scriptContext : scriptContexts) { - assertThat("custom." + scriptType + "." + scriptContext.getKey() + " doesn't have the expected value", - scriptModes.getScriptEnabled("custom", scriptType, scriptContext), equalTo(expectedScriptEnabled)); - checkedSettings.add("custom." + scriptType + "." + scriptContext); - } - } - } - - private ScriptContext[] complementOf(ScriptContext... scriptContexts) { - Map<String, ScriptContext> copy = new HashMap<>(); - for (ScriptContext scriptContext : scriptContextRegistry.scriptContexts()) { - copy.put(scriptContext.getKey(), scriptContext); - } - for (ScriptContext scriptContext : scriptContexts) { - copy.remove(scriptContext.getKey()); - } - return copy.values().toArray(new ScriptContext[copy.size()]); - } - - static Map<String, ScriptEngine> buildScriptEnginesByLangMap(Set<ScriptEngine> scriptEngines) { - Map<String, ScriptEngine> builder = new HashMap<>(); - for (ScriptEngine scriptEngine : scriptEngines) { - String type = scriptEngine.getType(); - builder.put(type, scriptEngine); - } - return unmodifiableMap(builder); - } - - private static class CustomScriptEngine implements ScriptEngine { - - public static final String NAME = "custom"; - - @Override - public String getType() { - return NAME; - } - - @Override - public Object compile(String scriptName, String scriptSource, Map<String, String> params) { - return null; - } - - @Override - public ExecutableScript executable(CompiledScript compiledScript, @Nullable Map<String, Object> vars) { - return null; - } - - @Override - public SearchScript search(CompiledScript compiledScript, SearchLookup lookup, @Nullable Map<String, Object> vars) { - return null; - } - - @Override - public void close() { - } - } -} diff --git a/core/src/test/java/org/elasticsearch/script/ScriptServiceTests.java b/core/src/test/java/org/elasticsearch/script/ScriptServiceTests.java index 9b52ff81ef..671fdaf502 100644 --- a/core/src/test/java/org/elasticsearch/script/ScriptServiceTests.java +++ b/core/src/test/java/org/elasticsearch/script/ScriptServiceTests.java @@ -57,7 +57,6 @@ public class ScriptServiceTests extends ESTestCase { private Map<String, ScriptEngine> scriptEnginesByLangMap; private ScriptEngineRegistry scriptEngineRegistry; private ScriptContextRegistry scriptContextRegistry; - private ScriptSettings scriptSettings; private ScriptContext[] scriptContexts; private ScriptService scriptService; private Settings baseSettings; @@ -80,8 +79,6 @@ public class ScriptServiceTests extends ESTestCase { scriptEngine = new TestEngine(); dangerousScriptEngine = new TestDangerousEngine(); TestEngine defaultScriptServiceEngine = new TestEngine(Script.DEFAULT_SCRIPT_LANG) {}; - scriptEnginesByLangMap = ScriptModesTests.buildScriptEnginesByLangMap( - new HashSet<>(Arrays.asList(scriptEngine, defaultScriptServiceEngine))); //randomly register custom script contexts int randomInt = randomIntBetween(0, 3); //prevent duplicates using map @@ -101,15 +98,13 @@ public class ScriptServiceTests extends ESTestCase { scriptEngineRegistry = new ScriptEngineRegistry(Arrays.asList(scriptEngine, dangerousScriptEngine, defaultScriptServiceEngine)); scriptContextRegistry = new ScriptContextRegistry(contexts.values()); - scriptSettings = new ScriptSettings(scriptEngineRegistry, scriptContextRegistry); scriptContexts = scriptContextRegistry.scriptContexts().toArray(new ScriptContext[scriptContextRegistry.scriptContexts().size()]); logger.info("--> setup script service"); } private void buildScriptService(Settings additionalSettings) throws IOException { Settings finalSettings = Settings.builder().put(baseSettings).put(additionalSettings).build(); - // TODO: - scriptService = new ScriptService(finalSettings, scriptEngineRegistry, scriptContextRegistry, scriptSettings) { + scriptService = new ScriptService(finalSettings, scriptEngineRegistry, scriptContextRegistry) { @Override StoredScriptSource getScriptFromClusterState(String id, String lang) { //mock the script that gets retrieved from an index @@ -179,33 +174,25 @@ public class ScriptServiceTests extends ESTestCase { public void testAllowSomeScriptTypeSettings() throws IOException { Settings.Builder builder = Settings.builder(); builder.put("script.types_allowed", "inline"); - builder.put("script.engine.painless.stored", false); buildScriptService(builder.build()); assertCompileAccepted("painless", "script", ScriptType.INLINE, ScriptContext.Standard.SEARCH); assertCompileRejected("painless", "script", ScriptType.STORED, ScriptContext.Standard.SEARCH); - - assertSettingDeprecationsAndWarnings( - ScriptSettingsTests.buildDeprecatedSettingsArray(scriptSettings, "script.engine.painless.stored")); } public void testAllowSomeScriptContextSettings() throws IOException { Settings.Builder builder = Settings.builder(); builder.put("script.contexts_allowed", "search, aggs"); - builder.put("script.update", false); buildScriptService(builder.build()); assertCompileAccepted("painless", "script", ScriptType.INLINE, ScriptContext.Standard.SEARCH); assertCompileAccepted("painless", "script", ScriptType.INLINE, ScriptContext.Standard.AGGS); assertCompileRejected("painless", "script", ScriptType.INLINE, ScriptContext.Standard.UPDATE); - - assertSettingDeprecationsAndWarnings( - ScriptSettingsTests.buildDeprecatedSettingsArray(scriptSettings, "script.update")); } public void testAllowNoScriptTypeSettings() throws IOException { Settings.Builder builder = Settings.builder(); - builder.put("script.types_allowed", ""); + builder.put("script.types_allowed", "none"); buildScriptService(builder.build()); assertCompileRejected("painless", "script", ScriptType.INLINE, ScriptContext.Standard.SEARCH); @@ -214,7 +201,7 @@ public class ScriptServiceTests extends ESTestCase { public void testAllowNoScriptContextSettings() throws IOException { Settings.Builder builder = Settings.builder(); - builder.put("script.contexts_allowed", ""); + builder.put("script.contexts_allowed", "none"); buildScriptService(builder.build()); assertCompileRejected("painless", "script", ScriptType.INLINE, ScriptContext.Standard.SEARCH); @@ -223,109 +210,6 @@ public class ScriptServiceTests extends ESTestCase { assertCompileRejected("painless", "script", ScriptType.INLINE, ScriptContext.Standard.INGEST); } - public void testDefaultBehaviourFineGrainedSettings() throws IOException { - Settings.Builder builder = Settings.builder(); - buildScriptService(builder.build()); - - for (ScriptContext scriptContext : scriptContexts) { - assertCompileRejected("dtest", "script", ScriptType.INLINE, scriptContext); - assertCompileRejected("dtest", "script", ScriptType.STORED, scriptContext); - } - } - - public void testFineGrainedSettings() throws IOException { - //collect the fine-grained settings to set for this run - int numScriptSettings = randomIntBetween(0, ScriptType.values().length); - Map<ScriptType, Boolean> scriptSourceSettings = new HashMap<>(); - for (int i = 0; i < numScriptSettings; i++) { - ScriptType scriptType; - do { - scriptType = randomFrom(ScriptType.values()); - } while (scriptSourceSettings.containsKey(scriptType)); - scriptSourceSettings.put(scriptType, randomBoolean()); - } - int numScriptContextSettings = randomIntBetween(0, this.scriptContextRegistry.scriptContexts().size()); - Map<ScriptContext, Boolean> scriptContextSettings = new HashMap<>(); - for (int i = 0; i < numScriptContextSettings; i++) { - ScriptContext scriptContext; - do { - scriptContext = randomFrom(this.scriptContexts); - } while (scriptContextSettings.containsKey(scriptContext)); - scriptContextSettings.put(scriptContext, randomBoolean()); - } - int numEngineSettings = randomIntBetween(0, ScriptType.values().length * scriptContexts.length); - Map<String, Boolean> engineSettings = new HashMap<>(); - for (int i = 0; i < numEngineSettings; i++) { - String settingKey; - do { - ScriptType scriptType = randomFrom(ScriptType.values()); - ScriptContext scriptContext = randomFrom(this.scriptContexts); - settingKey = scriptEngine.getType() + "." + scriptType + "." + scriptContext.getKey(); - } while (engineSettings.containsKey(settingKey)); - engineSettings.put(settingKey, randomBoolean()); - } - List<String> deprecated = new ArrayList<>(); - //set the selected fine-grained settings - Settings.Builder builder = Settings.builder(); - for (Map.Entry<ScriptType, Boolean> entry : scriptSourceSettings.entrySet()) { - if (entry.getValue()) { - builder.put("script" + "." + entry.getKey().getName(), "true"); - } else { - builder.put("script" + "." + entry.getKey().getName(), "false"); - } - deprecated.add("script" + "." + entry.getKey().getName()); - } - for (Map.Entry<ScriptContext, Boolean> entry : scriptContextSettings.entrySet()) { - if (entry.getValue()) { - builder.put("script" + "." + entry.getKey().getKey(), "true"); - } else { - builder.put("script" + "." + entry.getKey().getKey(), "false"); - } - deprecated.add("script" + "." + entry.getKey().getKey()); - } - for (Map.Entry<String, Boolean> entry : engineSettings.entrySet()) { - int delimiter = entry.getKey().indexOf('.'); - String part1 = entry.getKey().substring(0, delimiter); - String part2 = entry.getKey().substring(delimiter + 1); - - String lang = randomFrom(scriptEnginesByLangMap.get(part1).getType()); - if (entry.getValue()) { - builder.put("script.engine" + "." + lang + "." + part2, "true"); - } else { - builder.put("script.engine" + "." + lang + "." + part2, "false"); - } - deprecated.add("script.engine" + "." + lang + "." + part2); - } - - buildScriptService(builder.build()); - - for (ScriptType scriptType : ScriptType.values()) { - String script = "script"; - for (ScriptContext scriptContext : this.scriptContexts) { - //fallback mechanism: 1) engine specific settings 2) op based settings 3) source based settings - Boolean scriptEnabled = engineSettings.get(dangerousScriptEngine.getType() + "." + scriptType + "." + scriptContext.getKey()); - if (scriptEnabled == null) { - scriptEnabled = scriptContextSettings.get(scriptContext); - } - if (scriptEnabled == null) { - scriptEnabled = scriptSourceSettings.get(scriptType); - } - if (scriptEnabled == null) { - scriptEnabled = DEFAULT_SCRIPT_ENABLED.get(scriptType); - } - - String lang = dangerousScriptEngine.getType(); - if (scriptEnabled) { - assertCompileAccepted(lang, script, scriptType, scriptContext); - } else { - assertCompileRejected(lang, script, scriptType, scriptContext); - } - } - } - assertSettingDeprecationsAndWarnings( - ScriptSettingsTests.buildDeprecatedSettingsArray(scriptSettings, deprecated.toArray(new String[] {}))); - } - public void testCompileNonRegisteredContext() throws IOException { buildScriptService(Settings.EMPTY); String pluginName; @@ -378,14 +262,11 @@ public class ScriptServiceTests extends ESTestCase { public void testCompilationStatsOnCacheHit() throws IOException { Settings.Builder builder = Settings.builder(); builder.put(ScriptService.SCRIPT_CACHE_SIZE_SETTING.getKey(), 1); - builder.put("script.inline", "true"); buildScriptService(builder.build()); Script script = new Script(ScriptType.INLINE, "test", "1+1", Collections.emptyMap()); scriptService.compile(script, randomFrom(scriptContexts)); scriptService.compile(script, randomFrom(scriptContexts)); assertEquals(1L, scriptService.stats().getCompilations()); - assertSettingDeprecationsAndWarnings( - ScriptSettingsTests.buildDeprecatedSettingsArray(scriptSettings, "script.inline")); } public void testIndexedScriptCountedInCompilationStats() throws IOException { @@ -397,25 +278,19 @@ public class ScriptServiceTests extends ESTestCase { public void testCacheEvictionCountedInCacheEvictionsStats() throws IOException { Settings.Builder builder = Settings.builder(); builder.put(ScriptService.SCRIPT_CACHE_SIZE_SETTING.getKey(), 1); - builder.put("script.inline", "true"); buildScriptService(builder.build()); scriptService.compile(new Script(ScriptType.INLINE, "test", "1+1", Collections.emptyMap()), randomFrom(scriptContexts)); scriptService.compile(new Script(ScriptType.INLINE, "test", "2+2", Collections.emptyMap()), randomFrom(scriptContexts)); assertEquals(2L, scriptService.stats().getCompilations()); assertEquals(1L, scriptService.stats().getCacheEvictions()); - assertSettingDeprecationsAndWarnings( - ScriptSettingsTests.buildDeprecatedSettingsArray(scriptSettings, "script.inline")); } public void testDefaultLanguage() throws IOException { Settings.Builder builder = Settings.builder(); - builder.put("script.inline", "true"); buildScriptService(builder.build()); CompiledScript script = scriptService.compile( new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, "1 + 1", Collections.emptyMap()), randomFrom(scriptContexts)); assertEquals(script.lang(), Script.DEFAULT_SCRIPT_LANG); - assertSettingDeprecationsAndWarnings( - ScriptSettingsTests.buildDeprecatedSettingsArray(scriptSettings, "script.inline")); } public void testStoreScript() throws Exception { diff --git a/core/src/test/java/org/elasticsearch/script/ScriptSettingsTests.java b/core/src/test/java/org/elasticsearch/script/ScriptSettingsTests.java deleted file mode 100644 index b435fff52d..0000000000 --- a/core/src/test/java/org/elasticsearch/script/ScriptSettingsTests.java +++ /dev/null @@ -1,98 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.script; - -import org.elasticsearch.common.Nullable; -import org.elasticsearch.common.settings.Setting; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.search.lookup.SearchLookup; -import org.elasticsearch.test.ESTestCase; - -import java.util.Collections; -import java.util.Iterator; -import java.util.Map; - -import static org.hamcrest.Matchers.equalTo; - -public class ScriptSettingsTests extends ESTestCase { - - - public static Setting<?>[] buildDeprecatedSettingsArray(ScriptSettings scriptSettings, String... keys) { - Setting<?>[] settings = new Setting[keys.length]; - int count = 0; - - for (Setting<?> setting : scriptSettings.getSettings()) { - for (String key : keys) { - if (setting.getKey().equals(key)) { - settings[count++] = setting; - } - } - } - - return settings; - } - - public void testSettingsAreProperlyPropogated() { - ScriptEngineRegistry scriptEngineRegistry = - new ScriptEngineRegistry(Collections.singletonList(new CustomScriptEngine())); - ScriptContextRegistry scriptContextRegistry = new ScriptContextRegistry(Collections.emptyList()); - ScriptSettings scriptSettings = new ScriptSettings(scriptEngineRegistry, scriptContextRegistry); - boolean enabled = randomBoolean(); - Settings s = Settings.builder().put("script.inline", enabled).build(); - for (Iterator<Setting<Boolean>> iter = scriptSettings.getScriptLanguageSettings().iterator(); iter.hasNext();) { - Setting<Boolean> setting = iter.next(); - if (setting.getKey().endsWith(".inline")) { - assertThat("inline settings should have propagated", setting.get(s), equalTo(enabled)); - assertThat(setting.getDefaultRaw(s), equalTo(Boolean.toString(enabled))); - } - } - assertSettingDeprecationsAndWarnings(buildDeprecatedSettingsArray(scriptSettings, "script.inline")); - } - - private static class CustomScriptEngine implements ScriptEngine { - - public static final String NAME = "custom"; - - @Override - public String getType() { - return NAME; - } - - @Override - public Object compile(String scriptName, String scriptSource, Map<String, String> params) { - return null; - } - - @Override - public ExecutableScript executable(CompiledScript compiledScript, @Nullable Map<String, Object> vars) { - return null; - } - - @Override - public SearchScript search(CompiledScript compiledScript, SearchLookup lookup, @Nullable Map<String, Object> vars) { - return null; - } - - @Override - public void close() { - } - } - -} diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DateHistogramIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DateHistogramIT.java index 283f964b68..db5a0a1cd8 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DateHistogramIT.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DateHistogramIT.java @@ -1162,7 +1162,61 @@ public class DateHistogramIT extends ESIntegTestCase { assertThat(bucket.getDocCount(), equalTo(0L)); } } - internalCluster().wipeIndices("test12278"); + internalCluster().wipeIndices(index); + } + + /** + * Test date histogram aggregation with day interval, offset and + * extended bounds (see https://github.com/elastic/elasticsearch/issues/23776) + */ + public void testSingleValueFieldWithExtendedBoundsOffset() throws Exception { + String index = "test23776"; + prepareCreate(index) + .setSettings(Settings.builder().put(indexSettings()).put("index.number_of_shards", 1).put("index.number_of_replicas", 0)) + .execute().actionGet(); + + List<IndexRequestBuilder> builders = new ArrayList<>(); + builders.add(indexDoc(index, DateTime.parse("2016-01-03T08:00:00.000Z"), 1)); + builders.add(indexDoc(index, DateTime.parse("2016-01-03T08:00:00.000Z"), 2)); + builders.add(indexDoc(index, DateTime.parse("2016-01-06T08:00:00.000Z"), 3)); + builders.add(indexDoc(index, DateTime.parse("2016-01-06T08:00:00.000Z"), 4)); + indexRandom(true, builders); + ensureSearchable(index); + + SearchResponse response = null; + // retrieve those docs with the same time zone and extended bounds + response = client() + .prepareSearch(index) + .addAggregation( + dateHistogram("histo").field("date").dateHistogramInterval(DateHistogramInterval.days(1)).offset("+6h").minDocCount(0) + .extendedBounds(new ExtendedBounds("2016-01-01T06:00:00Z", "2016-01-08T08:00:00Z")) + ).execute().actionGet(); + assertSearchResponse(response); + + Histogram histo = response.getAggregations().get("histo"); + assertThat(histo, notNullValue()); + assertThat(histo.getName(), equalTo("histo")); + List<? extends Bucket> buckets = histo.getBuckets(); + assertThat(buckets.size(), equalTo(8)); + + assertEquals("2016-01-01T06:00:00.000Z", buckets.get(0).getKeyAsString()); + assertEquals(0, buckets.get(0).getDocCount()); + assertEquals("2016-01-02T06:00:00.000Z", buckets.get(1).getKeyAsString()); + assertEquals(0, buckets.get(1).getDocCount()); + assertEquals("2016-01-03T06:00:00.000Z", buckets.get(2).getKeyAsString()); + assertEquals(2, buckets.get(2).getDocCount()); + assertEquals("2016-01-04T06:00:00.000Z", buckets.get(3).getKeyAsString()); + assertEquals(0, buckets.get(3).getDocCount()); + assertEquals("2016-01-05T06:00:00.000Z", buckets.get(4).getKeyAsString()); + assertEquals(0, buckets.get(4).getDocCount()); + assertEquals("2016-01-06T06:00:00.000Z", buckets.get(5).getKeyAsString()); + assertEquals(2, buckets.get(5).getDocCount()); + assertEquals("2016-01-07T06:00:00.000Z", buckets.get(6).getKeyAsString()); + assertEquals(0, buckets.get(6).getDocCount()); + assertEquals("2016-01-08T06:00:00.000Z", buckets.get(7).getKeyAsString()); + assertEquals(0, buckets.get(7).getDocCount()); + + internalCluster().wipeIndices(index); } public void testSingleValueWithMultipleDateFormatsFromMapping() throws Exception { diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/scripted/InternalScriptedMetricTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/scripted/InternalScriptedMetricTests.java index 33ba2684f7..093dab2f4f 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/scripted/InternalScriptedMetricTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/scripted/InternalScriptedMetricTests.java @@ -29,7 +29,6 @@ import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptContextRegistry; import org.elasticsearch.script.ScriptEngineRegistry; import org.elasticsearch.script.ScriptService; -import org.elasticsearch.script.ScriptSettings; import org.elasticsearch.script.ScriptType; import org.elasticsearch.search.aggregations.ParsedAggregation; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; @@ -122,10 +121,8 @@ public class InternalScriptedMetricTests extends InternalAggregationTestCase<Int })); ScriptEngineRegistry scriptEngineRegistry = new ScriptEngineRegistry(Collections.singletonList(scriptEngine)); ScriptContextRegistry scriptContextRegistry = new ScriptContextRegistry(Collections.emptyList()); - ScriptSettings scriptSettings = new ScriptSettings(scriptEngineRegistry, scriptContextRegistry); try { - return new ScriptService(Settings.EMPTY, scriptEngineRegistry, scriptContextRegistry, - scriptSettings); + return new ScriptService(Settings.EMPTY, scriptEngineRegistry, scriptContextRegistry); } catch (IOException e) { throw new ElasticsearchException(e); } diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/scripted/ScriptedMetricAggregatorTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/scripted/ScriptedMetricAggregatorTests.java index 58487a715e..441be5b0e8 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/scripted/ScriptedMetricAggregatorTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/scripted/ScriptedMetricAggregatorTests.java @@ -38,7 +38,6 @@ import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptContextRegistry; import org.elasticsearch.script.ScriptEngineRegistry; import org.elasticsearch.script.ScriptService; -import org.elasticsearch.script.ScriptSettings; import org.elasticsearch.script.ScriptType; import org.elasticsearch.search.aggregations.AggregatorTestCase; import org.junit.BeforeClass; @@ -201,10 +200,9 @@ public class ScriptedMetricAggregatorTests extends AggregatorTestCase { MockScriptEngine scriptEngine = new MockScriptEngine(MockScriptEngine.NAME, SCRIPTS); ScriptEngineRegistry scriptEngineRegistry = new ScriptEngineRegistry(Collections.singletonList(scriptEngine)); ScriptContextRegistry scriptContextRegistry = new ScriptContextRegistry(Collections.emptyList()); - ScriptSettings scriptSettings = new ScriptSettings(scriptEngineRegistry, scriptContextRegistry); ScriptService scriptService; try { - scriptService = new ScriptService(Settings.EMPTY, scriptEngineRegistry, scriptContextRegistry, scriptSettings); + scriptService = new ScriptService(Settings.EMPTY, scriptEngineRegistry, scriptContextRegistry); } catch (IOException e) { throw new ElasticsearchException(e); } diff --git a/core/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java b/core/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java index ee671ca3d0..83a3cf8514 100644 --- a/core/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java +++ b/core/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java @@ -57,7 +57,6 @@ import org.elasticsearch.script.ScriptContextRegistry; import org.elasticsearch.script.ScriptEngineRegistry; import org.elasticsearch.script.ScriptService; import org.elasticsearch.script.ScriptServiceTests.TestEngine; -import org.elasticsearch.script.ScriptSettings; import org.elasticsearch.script.ScriptType; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.SearchModule; @@ -89,11 +88,9 @@ public abstract class AbstractSortTestCase<T extends SortBuilder<T>> extends EST .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()) .put(Environment.PATH_CONF_SETTING.getKey(), genericConfigFolder) .build(); - Environment environment = new Environment(baseSettings); ScriptContextRegistry scriptContextRegistry = new ScriptContextRegistry(Collections.emptyList()); ScriptEngineRegistry scriptEngineRegistry = new ScriptEngineRegistry(Collections.singletonList(new TestEngine())); - ScriptSettings scriptSettings = new ScriptSettings(scriptEngineRegistry, scriptContextRegistry); - scriptService = new ScriptService(baseSettings, scriptEngineRegistry, scriptContextRegistry, scriptSettings) { + scriptService = new ScriptService(baseSettings, scriptEngineRegistry, scriptContextRegistry) { @Override public CompiledScript compile(Script script, ScriptContext scriptContext) { return new CompiledScript(ScriptType.INLINE, "mockName", "test", script); |