diff options
51 files changed, 453 insertions, 449 deletions
diff --git a/common/src/main/java/org/apache/drill/common/util/CoreDecimalUtility.java b/common/src/main/java/org/apache/drill/common/util/CoreDecimalUtility.java index 9f61ccf17..bf12cfb36 100644 --- a/common/src/main/java/org/apache/drill/common/util/CoreDecimalUtility.java +++ b/common/src/main/java/org/apache/drill/common/util/CoreDecimalUtility.java @@ -82,10 +82,9 @@ public class CoreDecimalUtility { } public static boolean isDecimalType(TypeProtos.MinorType minorType) { - if (minorType == TypeProtos.MinorType.DECIMAL9 || minorType == TypeProtos.MinorType.DECIMAL18 || - minorType == TypeProtos.MinorType.DECIMAL28SPARSE || minorType == TypeProtos.MinorType.DECIMAL38SPARSE) { - return true; - } - return false; + return minorType == TypeProtos.MinorType.DECIMAL9 || + minorType == TypeProtos.MinorType.DECIMAL18 || + minorType == TypeProtos.MinorType.DECIMAL28SPARSE || + minorType == TypeProtos.MinorType.DECIMAL38SPARSE; } } diff --git a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/TestHiveStorage.java b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/TestHiveStorage.java index efd26ff06..cf987974e 100644 --- a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/TestHiveStorage.java +++ b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/TestHiveStorage.java @@ -26,7 +26,6 @@ import org.apache.drill.common.exceptions.UserRemoteException; import org.apache.drill.exec.ExecConstants; import org.apache.drill.exec.planner.physical.PlannerSettings; import org.apache.drill.exec.proto.UserProtos; -import org.apache.drill.test.OperatorFixture; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.hive.common.type.HiveVarchar; import org.joda.time.DateTime; @@ -73,10 +72,8 @@ public class TestHiveStorage extends HiveTestBase { .baselineValues(200L) .go(); } finally { - final OperatorFixture.TestOptionSet testOptionSet = new OperatorFixture.TestOptionSet(); - test(String.format("alter session set `%s` = %s", - ExecConstants.HIVE_OPTIMIZE_SCAN_WITH_NATIVE_READERS, - Boolean.toString(testOptionSet.getDefault(ExecConstants.HIVE_OPTIMIZE_SCAN_WITH_NATIVE_READERS).bool_val))); + test("alter session reset `%s`", + ExecConstants.HIVE_OPTIMIZE_SCAN_WITH_NATIVE_READERS); } } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/BufferManagerImpl.java b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/BufferManagerImpl.java index aa00b1c55..d0dcace8d 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/BufferManagerImpl.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/BufferManagerImpl.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -44,6 +44,7 @@ public class BufferManagerImpl implements BufferManager { managedBuffers.clear(); } + @Override public DrillBuf replace(DrillBuf old, int newSize) { if (managedBuffers.remove(old.memoryAddress()) == null) { throw new IllegalStateException("Tried to remove unmanaged buffer."); @@ -52,10 +53,12 @@ public class BufferManagerImpl implements BufferManager { return getManagedBuffer(newSize); } + @Override public DrillBuf getManagedBuffer() { return getManagedBuffer(256); } + @Override public DrillBuf getManagedBuffer(int size) { DrillBuf newBuf = allocator.buffer(size, this); managedBuffers.put(newBuf.memoryAddress(), newBuf); diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ImplCreator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ImplCreator.java index 58bf383af..0871621e9 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ImplCreator.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ImplCreator.java @@ -105,6 +105,7 @@ public class ImplCreator { } /** Create RootExec and its children (RecordBatches) for given FragmentRoot */ + @SuppressWarnings("unchecked") private RootExec getRootExec(final FragmentRoot root, final FragmentContext context) throws ExecutionSetupException { final List<RecordBatch> childRecordBatches = getChildren(root, context); @@ -141,6 +142,7 @@ public class ImplCreator { return proxyUgi.doAs(new PrivilegedExceptionAction<RecordBatch>() { @Override public RecordBatch run() throws Exception { + @SuppressWarnings("unchecked") final CloseableRecordBatch batch = ((BatchCreator<PhysicalOperator>) getOpCreator(op, context)).getBatch( context, op, childRecordBatches); operators.addFirst(batch); @@ -153,6 +155,7 @@ public class ImplCreator { throw new ExecutionSetupException(errMsg, e); } } else { + @SuppressWarnings("unchecked") final CloseableRecordBatch batch = ((BatchCreator<PhysicalOperator>) getOpCreator(op, context)).getBatch(context, op, childRecordBatches); operators.addFirst(batch); diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/MergingReceiverCreator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/MergingReceiverCreator.java index 59e744802..690a6624e 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/MergingReceiverCreator.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/MergingReceiverCreator.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -31,6 +31,7 @@ import org.apache.drill.exec.work.batch.RawBatchBuffer; public class MergingReceiverCreator implements BatchCreator<MergingReceiverPOP> { static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(MergingReceiverCreator.class); + @SuppressWarnings("resource") @Override public MergingRecordBatch getBatch(FragmentContext context, MergingReceiverPOP receiver, @@ -45,6 +46,4 @@ public class MergingReceiverCreator implements BatchCreator<MergingReceiverPOP> return new MergingRecordBatch(context, receiver, buffers); } - - } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScanBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScanBatch.java index 64be129db..6c2c171bb 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScanBatch.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScanBatch.java @@ -165,9 +165,9 @@ public class ScanBatch implements CloseableRecordBatch { try { while (true) { if (currentReader == null && !getNextReaderIfHas()) { - releaseAssets(); // All data has been read. Release resource. - done = true; - return IterOutcome.NONE; + releaseAssets(); // All data has been read. Release resource. + done = true; + return IterOutcome.NONE; } injector.injectChecked(context.getExecutionControls(), "next-allocate", OutOfMemoryException.class); currentReader.allocate(mutator.fieldVectorMap()); @@ -235,15 +235,14 @@ public class ScanBatch implements CloseableRecordBatch { } private boolean getNextReaderIfHas() throws ExecutionSetupException { - if (readers.hasNext()) { - currentReader = readers.next(); - implicitValues = implicitColumns.hasNext() ? implicitColumns.next() : null; - currentReader.setup(oContext, mutator); - currentReaderClassName = currentReader.getClass().getSimpleName(); - return true; - } else { + if (!readers.hasNext()) { return false; } + currentReader = readers.next(); + implicitValues = implicitColumns.hasNext() ? implicitColumns.next() : null; + currentReader.setup(oContext, mutator); + currentReaderClassName = currentReader.getClass().getSimpleName(); + return true; } private void addImplicitVectors() { @@ -251,8 +250,7 @@ public class ScanBatch implements CloseableRecordBatch { if (!implicitColumnList.isEmpty()) { for (String column : implicitColumnList.get(0).keySet()) { final MaterializedField field = MaterializedField.create(column, Types.optional(MinorType.VARCHAR)); - @SuppressWarnings("resource") - final ValueVector v = mutator.addField(field, NullableVarCharVector.class, true /*implicit field*/); + mutator.addField(field, NullableVarCharVector.class, true /*implicit field*/); } } } catch(SchemaChangeException e) { @@ -338,7 +336,6 @@ public class ScanBatch implements CloseableRecordBatch { return implicitFieldVectorMap; } - @SuppressWarnings("resource") @Override public <T extends ValueVector> T addField(MaterializedField field, Class<T> clazz) throws SchemaChangeException { @@ -388,6 +385,7 @@ public class ScanBatch implements CloseableRecordBatch { schemaChanged = false; } + @SuppressWarnings("resource") private <T extends ValueVector> T addField(MaterializedField field, Class<T> clazz, boolean isImplicitField) throws SchemaChangeException { Map<String, ValueVector> fieldVectorMap; diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScreenCreator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScreenCreator.java index 60355fb6b..d9abf406c 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScreenCreator.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScreenCreator.java @@ -21,7 +21,6 @@ import java.util.List; import org.apache.drill.common.exceptions.ExecutionSetupException; import org.apache.drill.exec.exception.OutOfMemoryException; -import org.apache.drill.exec.exception.OutOfMemoryException; import org.apache.drill.exec.ops.AccountingUserConnection; import org.apache.drill.exec.ops.FragmentContext; import org.apache.drill.exec.ops.MetricDef; @@ -33,12 +32,11 @@ import org.apache.drill.exec.proto.UserBitShared.QueryData; import org.apache.drill.exec.proto.UserBitShared.RecordBatchDef; import org.apache.drill.exec.record.RecordBatch; import org.apache.drill.exec.record.RecordBatch.IterOutcome; - -import com.google.common.base.Preconditions; - import org.apache.drill.exec.testing.ControlsInjector; import org.apache.drill.exec.testing.ControlsInjectorFactory; +import com.google.common.base.Preconditions; + public class ScreenCreator implements RootCreator<Screen> { //private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ScreenCreator.class); private static final ControlsInjector injector = ControlsInjectorFactory.getInjector(ScreenCreator.class); diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/TraceInjector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/TraceInjector.java index 92d1882eb..0aac91eab 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/TraceInjector.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/TraceInjector.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -35,10 +35,7 @@ public class TraceInjector extends AbstractPhysicalVisitor<PhysicalOperator, Fra static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TraceInjector.class); static int traceTagCount = 0; - - RootExec root = null; - private ScreenCreator sc = new ScreenCreator(); public static PhysicalOperator getExec(FragmentContext context, FragmentRoot root) throws ExecutionSetupException { TraceInjector tI = new TraceInjector(); @@ -89,5 +86,4 @@ public class TraceInjector extends AbstractPhysicalVisitor<PhysicalOperator, Fra return newOp; } - } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/WriterRecordBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/WriterRecordBatch.java index 939832bca..0ea17d699 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/WriterRecordBatch.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/WriterRecordBatch.java @@ -131,11 +131,13 @@ public class WriterRecordBatch extends AbstractRecordBatch<Writer> { } private void addOutputContainerData() { + @SuppressWarnings("resource") final VarCharVector fragmentIdVector = (VarCharVector) container.getValueAccessorById( VarCharVector.class, container.getValueVectorId(SchemaPath.getSimplePath("Fragment")).getFieldIds()) .getValueVector(); AllocationHelper.allocate(fragmentIdVector, 1, 50); + @SuppressWarnings("resource") final BigIntVector summaryVector = (BigIntVector) container.getValueAccessorById(BigIntVector.class, container.getValueVectorId(SchemaPath.getSimplePath("Number of records written")).getFieldIds()) .getValueVector(); diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/BaseOptionManager.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/BaseOptionManager.java index 5a0fa2d91..a502e0793 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/BaseOptionManager.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/BaseOptionManager.java @@ -18,15 +18,87 @@ package org.apache.drill.exec.server.options; import org.apache.drill.common.exceptions.UserException; +import org.apache.drill.exec.server.options.OptionValue.Kind; import java.util.Iterator; /** - * This {@link OptionManager} implements some the basic methods and should be extended by concrete implementations. + * This {@link OptionManager} implements some the basic methods and should be + * extended by concrete implementations. */ -public abstract class BaseOptionManager extends BaseOptionSet implements OptionManager { + +public abstract class BaseOptionManager implements OptionManager { private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(BaseOptionManager.class); + /** + * Gets the current option value given a validator. + * + * @param validator the validator + * @return option value + * @throws IllegalArgumentException - if the validator is not found + */ + private OptionValue getOptionSafe(OptionValidator validator) { + final String optionName = validator.getOptionName(); + OptionValue value = getOption(optionName); + return value == null ? getDefault(optionName) : value; + } + + @Override + public boolean getOption(TypeValidators.BooleanValidator validator) { + return getOptionSafe(validator).bool_val; + } + + @Override + public double getOption(TypeValidators.DoubleValidator validator) { + return getOptionSafe(validator).float_val; + } + + @Override + public long getOption(TypeValidators.LongValidator validator) { + return getOptionSafe(validator).num_val; + } + + @Override + public String getOption(TypeValidators.StringValidator validator) { + return getOptionSafe(validator).string_val; + } + + @Override + public boolean getBoolean(String name) { + return getByType(name, Kind.BOOLEAN).bool_val; + } + + @Override + public long getLong(String name) { + return getByType(name, Kind.LONG).num_val; + } + + @Override + public double getDouble(String name) { + return getByType(name, Kind.DOUBLE).float_val; + } + + @Override + public String getString(String name) { + return getByType(name, Kind.STRING).string_val; + } + + private OptionValue getByType(String name, Kind dataType) { + OptionValue value = getOption(name); + if (value == null) { + throw UserException.systemError(null) + .addContext("Undefined option: " + name) + .build(logger); + } + if (value.kind != dataType) { + throw UserException.systemError(null) + .addContext("Option " + name + " is of data type " + + value.kind + " but was requested as " + dataType) + .build(logger); + } + return value; + } + @Override public OptionList getInternalOptionList() { return getAllOptionList(true); diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/BaseOptionSet.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/BaseOptionSet.java deleted file mode 100644 index f664401f8..000000000 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/BaseOptionSet.java +++ /dev/null @@ -1,57 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.drill.exec.server.options; - -/** - * A basic implementation of an {@link OptionSet}. - */ -public abstract class BaseOptionSet implements OptionSet { - /** - * Gets the current option value given a validator. - * - * @param validator the validator - * @return option value - * @throws IllegalArgumentException - if the validator is not found - */ - private OptionValue getOptionSafe(OptionValidator validator) { - final String optionName = validator.getOptionName(); - OptionValue value = getOption(optionName); - return value == null ? getDefault(optionName) : value; - } - - @Override - public boolean getOption(TypeValidators.BooleanValidator validator) { - return getOptionSafe(validator).bool_val; - } - - @Override - public double getOption(TypeValidators.DoubleValidator validator) { - return getOptionSafe(validator).float_val; - } - - @Override - public long getOption(TypeValidators.LongValidator validator) { - return getOptionSafe(validator).num_val; - } - - @Override - public String getOption(TypeValidators.StringValidator validator) { - return getOptionSafe(validator).string_val; - } -} diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/OptionSet.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/OptionSet.java index 702d7df19..e96d571e8 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/OptionSet.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/OptionSet.java @@ -74,4 +74,48 @@ public interface OptionSet { * @return the string value */ String getOption(TypeValidators.StringValidator validator); + + /** + * Return the value of a Boolean option. + * + * @param name option name + * @return the Boolean value + * @throws IllegalArgumentException if the option is undefined or + * is not of the correct data type + */ + + boolean getBoolean(String name); + + /** + * Return the value of a long option. + * + * @param name option name + * @return the long value + * @throws IllegalArgumentException if the option is undefined or + * is not of the correct data type + */ + + long getLong(String name); + + /** + * Return the value of a double option. + * + * @param name option name + * @return the double value + * @throws IllegalArgumentException if the option is undefined or + * is not of the correct data type + */ + + double getDouble(String name); + + /** + * Return the value of a String option. + * + * @param name option name + * @return the String value + * @throws IllegalArgumentException if the option is undefined or + * is not of the correct data type + */ + + String getString(String name); } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/OptionValidator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/OptionValidator.java index 541840853..91439278c 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/OptionValidator.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/OptionValidator.java @@ -17,7 +17,6 @@ */ package org.apache.drill.exec.server.options; -import org.apache.drill.common.config.DrillConfig; import org.apache.drill.common.exceptions.UserException; import org.apache.drill.exec.ExecConstants; import org.apache.drill.exec.server.options.OptionValue.Kind; diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/OptionValue.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/OptionValue.java index e14a84605..fc4a9f056 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/OptionValue.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/OptionValue.java @@ -133,10 +133,14 @@ public class OptionValue implements Comparable<OptionValue> { return create(type, name, ((Boolean) val).booleanValue(), scope); } else if (val instanceof Long) { return create(type, name, ((Long) val).longValue(), scope); + } else if (val instanceof Integer) { + return create(type, name, ((Integer) val).longValue(), scope); } else if (val instanceof String) { return create(type, name, (String) val, scope); } else if (val instanceof Double) { return create(type, name, ((Double) val).doubleValue(), scope); + } else if (val instanceof Float) { + return create(type, name, ((Float) val).doubleValue(), scope); } throw new IllegalArgumentException(String.format("Unsupported type %s", val.getClass())); diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/PersistedOptionValue.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/PersistedOptionValue.java index 685799cb6..720a040a4 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/PersistedOptionValue.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/PersistedOptionValue.java @@ -269,6 +269,7 @@ public class PersistedOptionValue { * value of an option. This deserializer is essentially future proof since it only requires a value * to be stored for an option. */ + @SuppressWarnings("serial") public static class Deserializer extends StdDeserializer<PersistedOptionValue> { private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(Deserializer.class); diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java index e81ed0b8e..1c4554747 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java @@ -36,8 +36,10 @@ import org.apache.drill.exec.planner.physical.PlannerSettings; import org.apache.drill.exec.store.sys.PersistentStore; import org.apache.drill.exec.store.sys.PersistentStoreConfig; import org.apache.drill.exec.store.sys.PersistentStoreProvider; +import org.apache.drill.exec.store.sys.store.provider.InMemoryStoreProvider; import org.apache.drill.exec.util.AssertionUtil; +import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Lists; import com.google.common.collect.Sets; @@ -223,7 +225,6 @@ public class SystemOptionManager extends BaseOptionManager implements AutoClosea private final PersistentStoreProvider provider; - private final DrillConfig bootConfig; /** * Persistent store for options that have been changed from default. * NOTE: CRUD operations must use lowercase keys. @@ -243,12 +244,25 @@ public class SystemOptionManager extends BaseOptionManager implements AutoClosea this.config = PersistentStoreConfig.newJacksonBuilder(lpPersistence.getMapper(), PersistedOptionValue.class) .name("sys.options") .build(); - this.bootConfig = bootConfig; this.definitions = definitions; this.defaults = populateDefaultValues(definitions, bootConfig); } /** + * Test-only, in-memory version of the system option manager. + * + * @param bootConfig Drill config + */ + + @VisibleForTesting + public SystemOptionManager(final DrillConfig bootConfig) { + this.provider = new InMemoryStoreProvider(100); + this.config = null; + this.definitions = SystemOptionManager.createDefaultOptionDefinitions(); + this.defaults = populateDefaultValues(definitions, bootConfig); + } + + /** * Initializes this option manager. * * @return this option manager @@ -315,7 +329,13 @@ public class SystemOptionManager extends BaseOptionManager implements AutoClosea @Override public OptionValue getDefault(String optionName) { - return defaults.get(optionName); + OptionValue value = defaults.get(optionName); + if (value == null) { + throw UserException.systemError(null) + .addContext("Undefined default value for option: " + optionName) + .build(logger); + } + return value; } @Override diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSelection.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSelection.java index 81c7ad2bc..293259ff2 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSelection.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSelection.java @@ -208,18 +208,6 @@ public class FileSelection { return this.wasAllPartitionsPruned; } - private static String commonPath(final List<FileStatus> statuses) { - if (statuses == null || statuses.isEmpty()) { - return ""; - } - - final List<String> files = Lists.newArrayList(); - for (final FileStatus status : statuses) { - files.add(status.getPath().toString()); - } - return commonPathForFiles(files); - } - /** * Returns longest common path for the given list of files. * diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemConfig.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemConfig.java index adc390947..3c8f3a7f7 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemConfig.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemConfig.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -19,7 +19,6 @@ package org.apache.drill.exec.store.dfs; import java.util.Map; -import com.fasterxml.jackson.annotation.JsonInclude; import org.apache.drill.common.logical.FormatPluginConfig; import org.apache.drill.common.logical.StoragePluginConfig; diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemSchemaFactory.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemSchemaFactory.java index cf30162cf..5d9937706 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemSchemaFactory.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemSchemaFactory.java @@ -46,7 +46,6 @@ import org.apache.hadoop.fs.Path; * This is the top level schema that responds to root level path requests. Also supports */ public class FileSystemSchemaFactory implements SchemaFactory{ - private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(FileSystemSchemaFactory.class); public static final String DEFAULT_WS_NAME = "default"; @@ -61,6 +60,7 @@ public class FileSystemSchemaFactory implements SchemaFactory{ @Override public void registerSchemas(SchemaConfig schemaConfig, SchemaPlus parent) throws IOException { + @SuppressWarnings("resource") FileSystemSchema schema = new FileSystemSchema(schemaName, schemaConfig); SchemaPlus plusOfThis = parent.add(schema.getName(), schema); schema.setPlus(plusOfThis); @@ -75,6 +75,7 @@ public class FileSystemSchemaFactory implements SchemaFactory{ super(ImmutableList.<String>of(), name); for(WorkspaceSchemaFactory f : factories){ if (f.accessible(schemaConfig.getUserName())) { + @SuppressWarnings("resource") WorkspaceSchema s = f.createSchema(getSchemaPath(), schemaConfig); schemaMap.put(s.getName(), s); } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/ReadEntryFromHDFS.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/ReadEntryFromHDFS.java index 97124bce6..e02d84170 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/ReadEntryFromHDFS.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/ReadEntryFromHDFS.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -34,10 +34,12 @@ public class ReadEntryFromHDFS extends ReadEntryWithPath implements FileWork{ this.length = length; } + @Override public long getStart() { return start; } + @Override public long getLength() { return length; } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetScanBatchCreator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetScanBatchCreator.java index 60179482f..bb0b65fc5 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetScanBatchCreator.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetScanBatchCreator.java @@ -57,6 +57,7 @@ public class ParquetScanBatchCreator implements BatchCreator<ParquetRowGroupScan private static final String ENABLE_BYTES_TOTAL_COUNTER = "parquet.benchmark.bytes.total"; private static final String ENABLE_TIME_READ_COUNTER = "parquet.benchmark.time.read"; + @SuppressWarnings("resource") @Override public ScanBatch getBatch(FragmentContext context, ParquetRowGroupScan rowGroupScan, List<RecordBatch> children) throws ExecutionSetupException { @@ -119,7 +120,7 @@ public class ParquetScanBatchCreator implements BatchCreator<ParquetRowGroupScan if (logger.isDebugEnabled()) { logger.debug(containsCorruptDates.toString()); } - if (!context.getOptions().getOption(ExecConstants.PARQUET_NEW_RECORD_READER).bool_val && !isComplex(footers.get(e.getPath()))) { + if (!context.getOptions().getBoolean(ExecConstants.PARQUET_NEW_RECORD_READER) && !isComplex(footers.get(e.getPath()))) { readers.add( new ParquetRecordReader( context, e.getPath(), e.getRowGroupIndex(), e.getNumRecordsToRead(), fs, diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetSchema.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetSchema.java index 10187b7a9..773f3d3ef 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetSchema.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetSchema.java @@ -118,7 +118,7 @@ public class ParquetSchema { loadParquetSchema(); computeFixedPart(); - if (! selectedColumnMetadata.isEmpty() && allFieldsFixedLength) { + if (! selectedColumnMetadata.isEmpty() && allFieldsFixedLength) { recordsPerBatch = (int) Math.min(Math.min(batchSize / bitWidthAllFixedFields, footer.getBlocks().get(0).getColumns().get(0).getValueCount()), ParquetRecordReader.DEFAULT_RECORDS_TO_READ_IF_FIXED_WIDTH); } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetGroupConverter.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetGroupConverter.java index 5c8db9180..5c7c8e112 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetGroupConverter.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetGroupConverter.java @@ -17,7 +17,7 @@ */ package org.apache.drill.exec.store.parquet2; -import io.netty.buffer.DrillBuf; +import static org.apache.drill.exec.store.parquet.ParquetReaderUtility.NanoTimeUtils.getDateTimeValueFromBinary; import java.math.BigDecimal; import java.util.ArrayList; @@ -48,7 +48,6 @@ import org.apache.drill.exec.physical.impl.OutputMutator; import org.apache.drill.exec.server.options.OptionManager; import org.apache.drill.exec.store.parquet.ParquetReaderUtility; import org.apache.drill.exec.store.parquet.columnreaders.ParquetColumnMetadata; -import org.apache.drill.exec.store.parquet.columnreaders.ParquetRecordReader; import org.apache.drill.exec.util.DecimalUtility; import org.apache.drill.exec.vector.complex.impl.ComplexWriterImpl; import org.apache.drill.exec.vector.complex.writer.BaseWriter.MapWriter; @@ -67,8 +66,6 @@ import org.apache.drill.exec.vector.complex.writer.TimeStampWriter; import org.apache.drill.exec.vector.complex.writer.TimeWriter; import org.apache.drill.exec.vector.complex.writer.VarBinaryWriter; import org.apache.drill.exec.vector.complex.writer.VarCharWriter; -import org.joda.time.DateTimeConstants; - import org.apache.parquet.io.api.Binary; import org.apache.parquet.io.api.Converter; import org.apache.parquet.io.api.GroupConverter; @@ -80,10 +77,11 @@ import org.apache.parquet.schema.OriginalType; import org.apache.parquet.schema.PrimitiveType; import org.apache.parquet.schema.Type; import org.apache.parquet.schema.Type.Repetition; +import org.joda.time.DateTimeConstants; import com.google.common.collect.Lists; -import static org.apache.drill.exec.store.parquet.ParquetReaderUtility.NanoTimeUtils.getDateTimeValueFromBinary; +import io.netty.buffer.DrillBuf; public class DrillParquetGroupConverter extends GroupConverter { @@ -182,7 +180,9 @@ public class DrillParquetGroupConverter extends GroupConverter { switch(type.getOriginalType()) { case DECIMAL: { ParquetReaderUtility.checkDecimalTypeEnabled(options); - Decimal9Writer writer = type.getRepetition() == Repetition.REPEATED ? mapWriter.list(name).decimal9() : mapWriter.decimal9(name); + Decimal9Writer writer = type.getRepetition() == Repetition.REPEATED + ? mapWriter.list(name).decimal9() + : mapWriter.decimal9(name, type.getDecimalMetadata().getScale(), type.getDecimalMetadata().getPrecision()); return new DrillDecimal9Converter(writer, type.getDecimalMetadata().getPrecision(), type.getDecimalMetadata().getScale()); } case DATE: { @@ -218,7 +218,9 @@ public class DrillParquetGroupConverter extends GroupConverter { switch(type.getOriginalType()) { case DECIMAL: { ParquetReaderUtility.checkDecimalTypeEnabled(options); - Decimal18Writer writer = type.getRepetition() == Repetition.REPEATED ? mapWriter.list(name).decimal18() : mapWriter.decimal18(name); + Decimal18Writer writer = type.getRepetition() == Repetition.REPEATED + ? mapWriter.list(name).decimal18() + : mapWriter.decimal18(name, type.getDecimalMetadata().getScale(), type.getDecimalMetadata().getPrecision()); return new DrillDecimal18Converter(writer, type.getDecimalMetadata().getPrecision(), type.getDecimalMetadata().getScale()); } case TIMESTAMP_MILLIS: { @@ -270,10 +272,14 @@ public class DrillParquetGroupConverter extends GroupConverter { ParquetReaderUtility.checkDecimalTypeEnabled(options); DecimalMetadata metadata = type.getDecimalMetadata(); if (metadata.getPrecision() <= 28) { - Decimal28SparseWriter writer = type.getRepetition() == Repetition.REPEATED ? mapWriter.list(name).decimal28Sparse() : mapWriter.decimal28Sparse(name, metadata.getScale(), metadata.getPrecision()); + Decimal28SparseWriter writer = type.getRepetition() == Repetition.REPEATED + ? mapWriter.list(name).decimal28Sparse() + : mapWriter.decimal28Sparse(name, metadata.getScale(), metadata.getPrecision()); return new DrillBinaryToDecimal28Converter(writer, metadata.getPrecision(), metadata.getScale(), mutator.getManagedBuffer()); } else { - Decimal38SparseWriter writer = type.getRepetition() == Repetition.REPEATED ? mapWriter.list(name).decimal38Sparse() : mapWriter.decimal38Sparse(name, metadata.getScale(), metadata.getPrecision()); + Decimal38SparseWriter writer = type.getRepetition() == Repetition.REPEATED + ? mapWriter.list(name).decimal38Sparse() + : mapWriter.decimal38Sparse(name, metadata.getScale(), metadata.getPrecision()); return new DrillBinaryToDecimal38Converter(writer, metadata.getPrecision(), metadata.getScale(), mutator.getManagedBuffer()); } } @@ -287,10 +293,14 @@ public class DrillParquetGroupConverter extends GroupConverter { ParquetReaderUtility.checkDecimalTypeEnabled(options); DecimalMetadata metadata = type.getDecimalMetadata(); if (metadata.getPrecision() <= 28) { - Decimal28SparseWriter writer = type.getRepetition() == Repetition.REPEATED ? mapWriter.list(name).decimal28Sparse() : mapWriter.decimal28Sparse(name, metadata.getScale(), metadata.getPrecision()); + Decimal28SparseWriter writer = type.getRepetition() == Repetition.REPEATED + ? mapWriter.list(name).decimal28Sparse() + : mapWriter.decimal28Sparse(name, metadata.getScale(), metadata.getPrecision()); return new DrillBinaryToDecimal28Converter(writer, metadata.getPrecision(), metadata.getScale(), mutator.getManagedBuffer()); } else { - Decimal38SparseWriter writer = type.getRepetition() == Repetition.REPEATED ? mapWriter.list(name).decimal38Sparse() : mapWriter.decimal38Sparse(name, metadata.getScale(), metadata.getPrecision()); + Decimal38SparseWriter writer = type.getRepetition() == Repetition.REPEATED + ? mapWriter.list(name).decimal38Sparse() + : mapWriter.decimal38Sparse(name, metadata.getScale(), metadata.getPrecision()); return new DrillBinaryToDecimal38Converter(writer, metadata.getPrecision(), metadata.getScale(), mutator.getManagedBuffer()); } } else if (type.getOriginalType() == OriginalType.INTERVAL) { @@ -620,6 +630,7 @@ public class DrillParquetGroupConverter extends GroupConverter { private VarBinaryWriter writer; private VarBinaryHolder holder = new VarBinaryHolder(); + @SuppressWarnings("resource") public DrillFixedBinaryToVarbinaryConverter(VarBinaryWriter writer, int length, DrillBuf buf) { this.writer = writer; holder.buffer = buf = buf.reallocIfNeeded(length); diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetReader.java index 9e66f6d7c..e74c621d6 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetReader.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetReader.java @@ -32,7 +32,6 @@ import org.apache.drill.common.exceptions.ExecutionSetupException; import org.apache.drill.common.expression.PathSegment; import org.apache.drill.common.expression.SchemaPath; import org.apache.drill.common.types.TypeProtos; -import org.apache.drill.exec.ExecConstants; import org.apache.drill.exec.exception.OutOfMemoryException; import org.apache.drill.exec.expr.TypeHelper; import org.apache.drill.exec.ops.FragmentContext; @@ -47,19 +46,18 @@ import org.apache.drill.exec.store.parquet.RowGroupReadEntry; import org.apache.drill.exec.vector.AllocationHelper; import org.apache.drill.exec.vector.NullableIntVector; import org.apache.drill.exec.vector.ValueVector; -import org.apache.drill.exec.vector.VariableWidthVector; import org.apache.drill.exec.vector.complex.impl.VectorContainerWriter; import org.apache.hadoop.fs.Path; import org.apache.parquet.column.ColumnDescriptor; import org.apache.parquet.hadoop.CodecFactory; -import org.apache.parquet.hadoop.metadata.ColumnPath; -import org.apache.parquet.io.RecordReader; import org.apache.parquet.hadoop.ColumnChunkIncReadStore; import org.apache.parquet.hadoop.metadata.BlockMetaData; import org.apache.parquet.hadoop.metadata.ColumnChunkMetaData; +import org.apache.parquet.hadoop.metadata.ColumnPath; import org.apache.parquet.hadoop.metadata.ParquetMetadata; import org.apache.parquet.io.ColumnIOFactory; import org.apache.parquet.io.MessageColumnIO; +import org.apache.parquet.io.RecordReader; import org.apache.parquet.schema.GroupType; import org.apache.parquet.schema.MessageType; import org.apache.parquet.schema.Type; @@ -84,14 +82,7 @@ public class DrillParquetReader extends AbstractRecordReader { private RecordReader<Void> recordReader; private DrillParquetRecordMaterializer recordMaterializer; private int recordCount; - private List<ValueVector> primitiveVectors; private OperatorContext operatorContext; - // The interface for the parquet-mr library does not allow re-winding, to enable us to write into our - // fixed size value vectors, we must check how full the vectors are after some number of reads, for performance - // we avoid doing this every record. These values are populated with system/session settings to allow users to optimize - // for performance or allow a wider record size to be suported - private final int fillLevelCheckFrequency; - private final int fillLevelCheckThreshold; private FragmentContext fragmentContext; // For columns not found in the file, we need to return a schema element with the correct number of values @@ -102,8 +93,8 @@ public class DrillParquetReader extends AbstractRecordReader { // No actual data needs to be read out of the file, we only need to return batches until we have 'read' the number of // records specified in the row group metadata long mockRecordsRead=0; - private List<SchemaPath> columnsNotFound=null; - boolean noColumnsFound = false; // true if none of the columns in the projection list is found in the schema + private List<SchemaPath> columnsNotFound; + boolean noColumnsFound; // true if none of the columns in the projection list is found in the schema // See DRILL-4203 private final ParquetReaderUtility.DateCorruptionStatus containsCorruptedDates; @@ -116,8 +107,6 @@ public class DrillParquetReader extends AbstractRecordReader { this.entry = entry; setColumns(columns); this.fragmentContext = fragmentContext; - fillLevelCheckFrequency = this.fragmentContext.getOptions().getOption(ExecConstants.PARQUET_VECTOR_FILL_CHECK_THRESHOLD).num_val.intValue(); - fillLevelCheckThreshold = this.fragmentContext.getOptions().getOption(ExecConstants.PARQUET_VECTOR_FILL_THRESHOLD).num_val.intValue(); } public static MessageType getProjection(MessageType schema, @@ -263,13 +252,13 @@ public class DrillParquetReader extends AbstractRecordReader { } } - if(!noColumnsFound) { + if (!noColumnsFound) { writer = new VectorContainerWriter(output); // Discard the columns not found in the schema when create DrillParquetRecordMaterializer, since they have been added to output already. + @SuppressWarnings("unchecked") final Collection<SchemaPath> columns = columnsNotFound == null || columnsNotFound.size() == 0 ? getColumns(): CollectionUtils.subtract(getColumns(), columnsNotFound); recordMaterializer = new DrillParquetRecordMaterializer(output, writer, projection, columns, fragmentContext.getOptions(), containsCorruptedDates); - primitiveVectors = writer.getMapVector().getPrimitiveVectors(); recordReader = columnIO.getRecordReader(pageReadStore, recordMaterializer); } } catch (Exception e) { @@ -331,22 +320,6 @@ public class DrillParquetReader extends AbstractRecordReader { return count; } - private int getPercentFilled() { - int filled = 0; - for (final ValueVector v : primitiveVectors) { - filled = Math.max(filled, v.getAccessor().getValueCount() * 100 / v.getValueCapacity()); - if (v instanceof VariableWidthVector) { - filled = Math.max(filled, ((VariableWidthVector) v).getCurrentSizeInBytes() * 100 / ((VariableWidthVector) v).getByteCapacity()); - } - // TODO - need to re-enable this -// if (v instanceof RepeatedFixedWidthVector) { -// filled = Math.max(filled, ((RepeatedFixedWidthVector) v).getAccessor().getGroupCount() * 100) -// } - } - logger.debug("Percent filled: {}", filled); - return filled; - } - @Override public void close() { try { diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetRecordMaterializer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetRecordMaterializer.java index 2d778bd4a..f6ffdbd46 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetRecordMaterializer.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetRecordMaterializer.java @@ -17,19 +17,17 @@ */ package org.apache.drill.exec.store.parquet2; +import java.util.Collection; + import org.apache.drill.common.expression.SchemaPath; import org.apache.drill.exec.physical.impl.OutputMutator; import org.apache.drill.exec.server.options.OptionManager; import org.apache.drill.exec.store.parquet.ParquetReaderUtility; import org.apache.drill.exec.vector.complex.writer.BaseWriter.ComplexWriter; - import org.apache.parquet.io.api.GroupConverter; import org.apache.parquet.io.api.RecordMaterializer; import org.apache.parquet.schema.MessageType; -import java.util.Collection; -import java.util.List; - public class DrillParquetRecordMaterializer extends RecordMaterializer<Void> { public DrillParquetGroupConverter root; diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/store/provider/InMemoryStoreProvider.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/store/provider/InMemoryStoreProvider.java index ffe7b1852..3ab85ec29 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/store/provider/InMemoryStoreProvider.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/store/provider/InMemoryStoreProvider.java @@ -32,10 +32,7 @@ public class InMemoryStoreProvider implements PersistentStoreProvider { } @Override - public void close() throws Exception { - // TODO Auto-generated method stub - - } + public void close() throws Exception { } @Override public <V> PersistentStore<V> getOrCreateStore(PersistentStoreConfig<V> config) throws StoreException { @@ -43,9 +40,5 @@ public class InMemoryStoreProvider implements PersistentStoreProvider { } @Override - public void start() throws Exception { - // TODO Auto-generated method stub - - } - + public void start() throws Exception { } } diff --git a/exec/java-exec/src/test/java/org/apache/drill/BaseTestQuery.java b/exec/java-exec/src/test/java/org/apache/drill/BaseTestQuery.java index a094f518d..05ca6ea0c 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/BaseTestQuery.java +++ b/exec/java-exec/src/test/java/org/apache/drill/BaseTestQuery.java @@ -78,6 +78,7 @@ import java.util.regex.Pattern; import org.apache.drill.exec.record.VectorWrapper; import org.apache.drill.exec.vector.ValueVector; +import org.apache.drill.test.ClusterFixture; public class BaseTestQuery extends ExecTest { private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(BaseTestQuery.class); @@ -215,6 +216,7 @@ public class BaseTestQuery extends ExecTest { bits[i] = new Drillbit(config, serviceSet, classpathScan); bits[i].run(); + @SuppressWarnings("resource") final StoragePluginRegistry pluginRegistry = bits[i].getContext().getStorage(); TestUtilities.updateDfsTestTmpSchemaLocation(pluginRegistry, dfsTestTmpSchemaLocation); TestUtilities.makeDfsTmpSchemaImmutable(pluginRegistry); @@ -377,6 +379,25 @@ public class BaseTestQuery extends ExecTest { testNoResult(1, query, args); } + public static void alterSession(String option, Object value) { + String valueStr = ClusterFixture.stringify(value); + try { + test("ALTER SESSION SET `%s` = %s", option, valueStr); + } catch(final Exception e) { + fail(String.format("Failed to set session option `%s` = %s, Error: %s", + option, valueStr, e.toString())); + } + } + + public static void resetSessionOption(String option) { + try { + test("ALTER SESSION RESET `%s`", option); + } catch(final Exception e) { + fail(String.format("Failed to reset session option `%s`, Error: %s", + option, e.toString())); + } + } + protected static void testNoResult(int interation, String query, Object... args) throws Exception { query = String.format(query, args); logger.debug("Running query:\n--------------\n" + query); @@ -440,9 +461,9 @@ public class BaseTestQuery extends ExecTest { } catch (AssertionError e) { e.addSuppressed(actualException); throw e; + } } } - } /** * Utility method which tests given query produces a {@link UserException} @@ -479,23 +500,19 @@ public class BaseTestQuery extends ExecTest { } protected static void setSessionOption(final String option, final boolean value) { - setSessionOption(option, Boolean.toString(value)); + alterSession(option, value); } protected static void setSessionOption(final String option, final long value) { - setSessionOption(option, Long.toString(value)); + alterSession(option, value); } protected static void setSessionOption(final String option, final double value) { - setSessionOption(option, Double.toString(value)); + alterSession(option, value); } protected static void setSessionOption(final String option, final String value) { - try { - runSQL(String.format("alter session set `%s` = %s", option, value)); - } catch(final Exception e) { - fail(String.format("Failed to set session option `%s` = %s, Error: %s", option, value, e.toString())); - } + alterSession(option, value); } public static class SilentListener implements UserResultsListener { diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/agg/TestHashAggrSpill.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/agg/TestHashAggrSpill.java index 20038578a..2e81acde9 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/agg/TestHashAggrSpill.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/agg/TestHashAggrSpill.java @@ -28,7 +28,7 @@ import org.apache.drill.exec.planner.physical.PlannerSettings; import org.apache.drill.exec.proto.UserBitShared; import org.apache.drill.test.ClientFixture; import org.apache.drill.test.ClusterFixture; -import org.apache.drill.test.FixtureBuilder; +import org.apache.drill.test.ClusterFixtureBuilder; import org.apache.drill.test.LogFixture; import org.apache.drill.test.ProfileParser; import org.apache.drill.test.QueryBuilder; @@ -94,7 +94,7 @@ public class TestHashAggrSpill { .logger("org.apache.drill", Level.WARN) ; - FixtureBuilder builder = ClusterFixture.builder() + ClusterFixtureBuilder builder = ClusterFixture.builder() .sessionOption(ExecConstants.HASHAGG_MAX_MEMORY_KEY,maxMem) .sessionOption(ExecConstants.HASHAGG_NUM_PARTITIONS_KEY,numPartitions) .sessionOption(ExecConstants.HASHAGG_MIN_BATCHES_PER_PARTITION_KEY,minBatches) diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/limit/TestLimitWithExchanges.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/limit/TestLimitWithExchanges.java index 8b75f3a5d..b49a12e90 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/limit/TestLimitWithExchanges.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/limit/TestLimitWithExchanges.java @@ -74,8 +74,7 @@ public class TestLimitWithExchanges extends BaseTestQuery { final String[] expectedPlan5 = {"(?s)Limit\\(fetch=\\[1\\].*UnionExchange.*Limit\\(fetch=\\[1\\]\\).*Join"}; testLimitHelper(sql5, expectedPlan5, excludedPlan, 1); } finally { - final OperatorFixture.TestOptionSet testOptionSet = new OperatorFixture.TestOptionSet(); - test("alter session set `%s` = %s", ExecConstants.SLICE_TARGET, testOptionSet.getDefault(ExecConstants.SLICE_TARGET).getValue()); + resetSessionOption(ExecConstants.SLICE_TARGET); } } @@ -96,8 +95,7 @@ public class TestLimitWithExchanges extends BaseTestQuery { testLimitHelper(sql2, expectedPlan, excludedPlan2, 5); } finally { - final OperatorFixture.TestOptionSet testOptionSet = new OperatorFixture.TestOptionSet(); - test("alter session set `planner.slice_target` = " + testOptionSet.getDefault(ExecConstants.SLICE_TARGET).getValue()); + resetSessionOption(ExecConstants.SLICE_TARGET); } } @@ -126,8 +124,7 @@ public class TestLimitWithExchanges extends BaseTestQuery { testLimitHelper(sql3, expectedPlan2, excludedPlan2, 10); testLimitHelper(sql4, expectedPlan2, excludedPlan2, 10); } finally { - final OperatorFixture.TestOptionSet testOptionSet = new OperatorFixture.TestOptionSet(); - test("alter session set `planner.slice_target` = " + testOptionSet.getDefault(ExecConstants.SLICE_TARGET).getValue()); + resetSessionOption(ExecConstants.SLICE_TARGET); } } diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/validate/TestValidationOptions.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/validate/TestValidationOptions.java index d4e33b097..4b3cbff9e 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/validate/TestValidationOptions.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/validate/TestValidationOptions.java @@ -24,7 +24,7 @@ import org.apache.drill.exec.store.easy.text.compliant.CompliantTextRecordReader import org.apache.drill.test.ClientFixture; import org.apache.drill.test.ClusterFixture; import org.apache.drill.test.DrillTest; -import org.apache.drill.test.FixtureBuilder; +import org.apache.drill.test.ClusterFixtureBuilder; import org.apache.drill.test.LogFixture; import org.junit.AfterClass; import org.junit.BeforeClass; @@ -59,7 +59,7 @@ public class TestValidationOptions extends DrillTest { @Test public void testOptions() throws Exception { - FixtureBuilder builder = ClusterFixture.builder() + ClusterFixtureBuilder builder = ClusterFixture.builder() .maxParallelization(1) .configProperty(ExecConstants.ENABLE_ITERATOR_VALIDATION, false) .configProperty(ExecConstants.ENABLE_VECTOR_VALIDATION, false) @@ -93,7 +93,7 @@ public class TestValidationOptions extends DrillTest { @Test public void testConfig() throws Exception { - FixtureBuilder builder = ClusterFixture.builder() + ClusterFixtureBuilder builder = ClusterFixture.builder() .maxParallelization(1) .configProperty(ExecConstants.ENABLE_ITERATOR_VALIDATION, true) .configProperty(ExecConstants.ENABLE_VECTOR_VALIDATION, true) @@ -119,7 +119,7 @@ public class TestValidationOptions extends DrillTest { @Test public void testDefaults() throws Exception { - FixtureBuilder builder = ClusterFixture.builder() + ClusterFixtureBuilder builder = ClusterFixture.builder() .maxParallelization(1) ; try (ClusterFixture cluster = builder.build(); diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestParquetWriter.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestParquetWriter.java index 41be0ba63..9715d142a 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestParquetWriter.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestParquetWriter.java @@ -17,8 +17,8 @@ */ package org.apache.drill.exec.physical.impl.writer; -import static org.apache.drill.exec.store.parquet.ParquetRecordWriter.DRILL_VERSION_PROPERTY; import static org.apache.drill.TestBuilder.convertToLocalTimestamp; +import static org.apache.drill.exec.store.parquet.ParquetRecordWriter.DRILL_VERSION_PROPERTY; import static org.apache.parquet.format.converter.ParquetMetadataConverter.SKIP_ROW_GROUPS; import static org.junit.Assert.assertEquals; @@ -26,14 +26,13 @@ import java.io.File; import java.io.FileWriter; import java.math.BigDecimal; import java.sql.Date; -import java.util.Arrays; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; -import com.google.common.base.Joiner; import org.apache.drill.BaseTestQuery; import org.apache.drill.categories.ParquetTest; import org.apache.drill.categories.SlowTest; @@ -44,7 +43,6 @@ import org.apache.drill.common.util.TestTools; import org.apache.drill.exec.ExecConstants; import org.apache.drill.exec.fn.interp.TestConstantFolding; import org.apache.drill.exec.planner.physical.PlannerSettings; -import org.apache.drill.test.OperatorFixture; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; @@ -63,10 +61,11 @@ import org.junit.rules.TemporaryFolder; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; +import com.google.common.base.Joiner; + @RunWith(Parameterized.class) @Category({SlowTest.class, ParquetTest.class}) public class TestParquetWriter extends BaseTestQuery { -// private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestParquetWriter.class); @Parameterized.Parameters public static Collection<Object[]> data() { @@ -126,12 +125,12 @@ public class TestParquetWriter extends BaseTestQuery { @BeforeClass public static void initFs() throws Exception { fs = getLocalFileSystem(); - test(String.format("alter session set `%s` = true", PlannerSettings.ENABLE_DECIMAL_DATA_TYPE_KEY)); + alterSession(PlannerSettings.ENABLE_DECIMAL_DATA_TYPE_KEY, true); } @AfterClass public static void disableDecimalDataType() throws Exception { - test(String.format("alter session set `%s` = false", PlannerSettings.ENABLE_DECIMAL_DATA_TYPE_KEY)); + alterSession(PlannerSettings.ENABLE_DECIMAL_DATA_TYPE_KEY, false); } @Test @@ -188,25 +187,26 @@ public class TestParquetWriter extends BaseTestQuery { try { // read all of the types with the complex reader - test(String.format("alter session set %s = true", ExecConstants.PARQUET_NEW_RECORD_READER)); + alterSession(ExecConstants.PARQUET_NEW_RECORD_READER, true); runTestAndValidate(allTypesSelection, "*", allTypesTable, "donuts_json"); } finally { - test(String.format("alter session set %s = false", ExecConstants.PARQUET_NEW_RECORD_READER)); + resetSessionOption(ExecConstants.PARQUET_NEW_RECORD_READER); } } @Test public void testAllScalarTypesDictionary() throws Exception { try { - test(String.format("alter session set %s = true", ExecConstants.PARQUET_WRITER_ENABLE_DICTIONARY_ENCODING)); + alterSession(ExecConstants.PARQUET_WRITER_ENABLE_DICTIONARY_ENCODING, true); /// read once with the flat reader runTestAndValidate(allTypesSelection, "*", allTypesTable, "donuts_json"); // read all of the types with the complex reader - test(String.format("alter session set %s = true", ExecConstants.PARQUET_NEW_RECORD_READER)); + alterSession(ExecConstants.PARQUET_NEW_RECORD_READER, true); runTestAndValidate(allTypesSelection, "*", allTypesTable, "donuts_json"); } finally { - test(String.format("alter session set %s = false", ExecConstants.PARQUET_WRITER_ENABLE_DICTIONARY_ENCODING)); + resetSessionOption(ExecConstants.PARQUET_NEW_RECORD_READER); + resetSessionOption(ExecConstants.PARQUET_WRITER_ENABLE_DICTIONARY_ENCODING); } } @@ -221,10 +221,10 @@ public class TestParquetWriter extends BaseTestQuery { String selection = "type"; String inputTable = "cp.`donuts.json`"; try { - test(String.format("alter session set %s = true", ExecConstants.PARQUET_WRITER_ENABLE_DICTIONARY_ENCODING)); + alterSession(ExecConstants.PARQUET_WRITER_ENABLE_DICTIONARY_ENCODING, true); runTestAndValidate(selection, selection, inputTable, "donuts_json"); } finally { - test(String.format("alter session set %s = false", ExecConstants.PARQUET_WRITER_ENABLE_DICTIONARY_ENCODING)); + resetSessionOption(ExecConstants.PARQUET_WRITER_ENABLE_DICTIONARY_ENCODING); } } @@ -262,7 +262,7 @@ public class TestParquetWriter extends BaseTestQuery { @Test public void testTPCHReadWrite1_date_convertedType() throws Exception { try { - test("alter session set `%s` = false", ExecConstants.PARQUET_WRITER_ENABLE_DICTIONARY_ENCODING); + alterSession(ExecConstants.PARQUET_WRITER_ENABLE_DICTIONARY_ENCODING, false); String selection = "L_ORDERKEY, L_PARTKEY, L_SUPPKEY, L_LINENUMBER, L_QUANTITY, L_EXTENDEDPRICE, L_DISCOUNT, L_TAX, " + "L_RETURNFLAG, L_LINESTATUS, L_SHIPDATE, cast(L_COMMITDATE as DATE) as L_COMMITDATE, cast(L_RECEIPTDATE as DATE) AS L_RECEIPTDATE, L_SHIPINSTRUCT, L_SHIPMODE, L_COMMENT"; String validationSelection = "L_ORDERKEY, L_PARTKEY, L_SUPPKEY, L_LINENUMBER, L_QUANTITY, L_EXTENDEDPRICE, L_DISCOUNT, L_TAX, " + @@ -270,9 +270,7 @@ public class TestParquetWriter extends BaseTestQuery { String inputTable = "cp.`tpch/lineitem.parquet`"; runTestAndValidate(selection, validationSelection, inputTable, "lineitem_parquet_converted"); } finally { - OperatorFixture.TestOptionSet optionSet = new OperatorFixture.TestOptionSet(); - test("alter session set `%s` = %b", ExecConstants.PARQUET_WRITER_ENABLE_DICTIONARY_ENCODING, - optionSet.getDefault(ExecConstants.PARQUET_WRITER_ENABLE_DICTIONARY_ENCODING).bool_val); + resetSessionOption(ExecConstants.PARQUET_WRITER_ENABLE_DICTIONARY_ENCODING); } } @@ -321,29 +319,24 @@ public class TestParquetWriter extends BaseTestQuery { @Test public void testTPCHReadWriteNoDictUncompressed() throws Exception { try { - test(String.format("alter session set `%s` = false", ExecConstants.PARQUET_WRITER_ENABLE_DICTIONARY_ENCODING)); - test(String.format("alter session set `%s` = 'none'", ExecConstants.PARQUET_WRITER_COMPRESSION_TYPE)); + alterSession(ExecConstants.PARQUET_WRITER_ENABLE_DICTIONARY_ENCODING, false); + alterSession(ExecConstants.PARQUET_WRITER_COMPRESSION_TYPE, "none"); String inputTable = "cp.`tpch/supplier.parquet`"; runTestAndValidate("*", "*", inputTable, "supplier_parquet_no_dict_uncompressed"); } finally { - final OperatorFixture.TestOptionSet optionSet = new OperatorFixture.TestOptionSet(); - test(String.format("alter session set `%s` = %b", ExecConstants.PARQUET_WRITER_ENABLE_DICTIONARY_ENCODING, - optionSet.getDefault(ExecConstants.PARQUET_WRITER_ENABLE_DICTIONARY_ENCODING).bool_val)); - test(String.format("alter session set `%s` = '%s'", ExecConstants.PARQUET_WRITER_COMPRESSION_TYPE, - optionSet.getDefault(ExecConstants.PARQUET_WRITER_COMPRESSION_TYPE).string_val)); + resetSessionOption(ExecConstants.PARQUET_WRITER_ENABLE_DICTIONARY_ENCODING); + resetSessionOption(ExecConstants.PARQUET_WRITER_COMPRESSION_TYPE); } } @Test public void testTPCHReadWriteDictGzip() throws Exception { try { - test(String.format("alter session set `%s` = 'gzip'", ExecConstants.PARQUET_WRITER_COMPRESSION_TYPE)); + alterSession(ExecConstants.PARQUET_WRITER_COMPRESSION_TYPE, "gzip"); String inputTable = "cp.`tpch/supplier.parquet`"; runTestAndValidate("*", "*", inputTable, "supplier_parquet_dict_gzip"); } finally { - final OperatorFixture.TestOptionSet optionSet = new OperatorFixture.TestOptionSet(); - test(String.format("alter session set `%s` = '%s'", ExecConstants.PARQUET_WRITER_COMPRESSION_TYPE, - optionSet.getDefault(ExecConstants.PARQUET_WRITER_COMPRESSION_TYPE).string_val)); + resetSessionOption(ExecConstants.PARQUET_WRITER_COMPRESSION_TYPE); } } @@ -393,22 +386,33 @@ public class TestParquetWriter extends BaseTestQuery { "cast(salary as decimal(24,2)) as decimal24, cast(salary as decimal(38,2)) as decimal38"; String validateSelection = "decimal8, decimal15, decimal24, decimal38"; String inputTable = "cp.`employee.json`"; - runTestAndValidate(selection, validateSelection, inputTable, "parquet_decimal"); + + // DRILL-5833: The "old" writer had a decimal bug, but the new one + // did not. The one used was random. Force the test to run both + // the old and new readers. + + try { + alterSession(ExecConstants.PARQUET_NEW_RECORD_READER, true); + runTestAndValidate(selection, validateSelection, inputTable, "parquet_decimal"); + alterSession(ExecConstants.PARQUET_NEW_RECORD_READER, false); + runTestAndValidate(selection, validateSelection, inputTable, "parquet_decimal"); + } finally { + resetSessionOption(ExecConstants.PARQUET_NEW_RECORD_READER); + } } @Test public void testMulipleRowGroups() throws Exception { try { - test(String.format("ALTER SESSION SET `%s` = %d", ExecConstants.PARQUET_BLOCK_SIZE, 1024*1024)); + alterSession(ExecConstants.PARQUET_BLOCK_SIZE, 1024*1024); String selection = "mi"; String inputTable = "cp.`customer.json`"; runTestAndValidate(selection, selection, inputTable, "foodmart_customer_parquet"); } finally { - test(String.format("ALTER SESSION SET `%s` = %d", ExecConstants.PARQUET_BLOCK_SIZE, 512*1024*1024)); + resetSessionOption(ExecConstants.PARQUET_BLOCK_SIZE); } } - @Test public void testDate() throws Exception { String selection = "cast(hire_date as DATE) as hire_date"; @@ -465,9 +469,7 @@ public class TestParquetWriter extends BaseTestQuery { .optionSettingQueriesForBaseline("alter system set `store.parquet.use_new_reader` = true") .build().run(); } finally { - final OperatorFixture.TestOptionSet optionSet = new OperatorFixture.TestOptionSet(); - test("alter system set `%s` = %b", ExecConstants.PARQUET_NEW_RECORD_READER, - optionSet.getDefault(ExecConstants.PARQUET_NEW_RECORD_READER).bool_val); + resetSessionOption(ExecConstants.PARQUET_NEW_RECORD_READER); } } @@ -486,10 +488,7 @@ public class TestParquetWriter extends BaseTestQuery { "alter system set `store.parquet.use_new_reader` = true") .build().run(); } finally { - final OperatorFixture.TestOptionSet optionSet = new OperatorFixture.TestOptionSet(); - test("alter system set `%s` = %b", - ExecConstants.PARQUET_NEW_RECORD_READER, - optionSet.getDefault(ExecConstants.PARQUET_NEW_RECORD_READER).bool_val); + resetSessionOption(ExecConstants.PARQUET_NEW_RECORD_READER); } } @@ -730,7 +729,6 @@ public class TestParquetWriter extends BaseTestQuery { try { deleteTableIfExists(outputFile); test("use dfs_test.tmp"); - // test("ALTER SESSION SET `planner.add_producer_consumer` = false"); String query = String.format("SELECT %s FROM %s", selection, inputTable); String create = "CREATE TABLE " + outputFile + " AS " + query; String validateQuery = String.format("SELECT %s FROM " + outputFile, validationSelection); @@ -738,13 +736,16 @@ public class TestParquetWriter extends BaseTestQuery { testBuilder() .unOrdered() - .sqlQuery(query) - .sqlBaselineQuery(validateQuery) + // Validate query is the query on the output file (the one to validate) + .sqlQuery(validateQuery) + // The basline query is the query on the input file (the expected values) + .sqlBaselineQuery(query) .go(); Configuration hadoopConf = new Configuration(); hadoopConf.set(FileSystem.FS_DEFAULT_NAME_KEY, FileSystem.DEFAULT_FS); Path output = new Path(getDfsTestTmpSchemaLocation(), outputFile); + @SuppressWarnings("resource") FileSystem fs = output.getFileSystem(hadoopConf); for (FileStatus file : fs.listStatus(output)) { ParquetMetadata footer = ParquetFileReader.readFooter(hadoopConf, file, SKIP_ROW_GROUPS); @@ -765,10 +766,10 @@ public class TestParquetWriter extends BaseTestQuery { public void testImpalaParquetInt96() throws Exception { compareParquetReadersColumnar("field_impala_ts", "cp.`parquet/int96_impala_1.parquet`"); try { - test("alter session set %s = true", ExecConstants.PARQUET_READER_INT96_AS_TIMESTAMP); + alterSession(ExecConstants.PARQUET_READER_INT96_AS_TIMESTAMP, true); compareParquetReadersColumnar("field_impala_ts", "cp.`parquet/int96_impala_1.parquet`"); } finally { - test("alter session reset %s", ExecConstants.PARQUET_READER_INT96_AS_TIMESTAMP); + resetSessionOption(ExecConstants.PARQUET_READER_INT96_AS_TIMESTAMP); } } @@ -799,7 +800,7 @@ public class TestParquetWriter extends BaseTestQuery { .baselineColumns("int96_ts") .build().run(); } finally { - test("alter system reset `%s`", ExecConstants.PARQUET_READER_INT96_AS_TIMESTAMP); + resetSessionOption(ExecConstants.PARQUET_READER_INT96_AS_TIMESTAMP); } } @@ -818,12 +819,12 @@ public class TestParquetWriter extends BaseTestQuery { @Test public void testImpalaParquetTimestampInt96AsTimeStamp() throws Exception { try { - test("alter session set %s = false", ExecConstants.PARQUET_NEW_RECORD_READER); + alterSession(ExecConstants.PARQUET_NEW_RECORD_READER, false); compareParquetInt96Converters("field_impala_ts", "cp.`parquet/int96_impala_1.parquet`"); - test("alter session set %s = true", ExecConstants.PARQUET_NEW_RECORD_READER); + alterSession(ExecConstants.PARQUET_NEW_RECORD_READER, true); compareParquetInt96Converters("field_impala_ts", "cp.`parquet/int96_impala_1.parquet`"); } finally { - test("alter session reset `%s`", ExecConstants.PARQUET_NEW_RECORD_READER); + resetSessionOption(ExecConstants.PARQUET_NEW_RECORD_READER); } } @@ -939,7 +940,7 @@ public class TestParquetWriter extends BaseTestQuery { .build() .run(); } finally { - test("alter system reset `%s`", ExecConstants.PARQUET_READER_INT96_AS_TIMESTAMP); + resetSessionOption(ExecConstants.PARQUET_READER_INT96_AS_TIMESTAMP); } } @@ -958,24 +959,22 @@ public class TestParquetWriter extends BaseTestQuery { @Test public void testTPCHReadWriteGzip() throws Exception { try { - test(String.format("alter session set `%s` = 'gzip'", ExecConstants.PARQUET_WRITER_COMPRESSION_TYPE)); + alterSession(ExecConstants.PARQUET_WRITER_COMPRESSION_TYPE, "gzip"); String inputTable = "cp.`tpch/supplier.parquet`"; runTestAndValidate("*", "*", inputTable, "suppkey_parquet_dict_gzip"); } finally { - final OperatorFixture.TestOptionSet optionSet = new OperatorFixture.TestOptionSet(); - test(String.format("alter session set `%s` = '%s'", ExecConstants.PARQUET_WRITER_COMPRESSION_TYPE, optionSet.getDefault(ExecConstants.PARQUET_WRITER_COMPRESSION_TYPE).string_val)); + resetSessionOption(ExecConstants.PARQUET_WRITER_COMPRESSION_TYPE); } } @Test public void testTPCHReadWriteSnappy() throws Exception { try { - test(String.format("alter session set `%s` = 'snappy'", ExecConstants.PARQUET_WRITER_COMPRESSION_TYPE)); + alterSession(ExecConstants.PARQUET_WRITER_COMPRESSION_TYPE, "snappy"); String inputTable = "cp.`supplier_snappy.parquet`"; runTestAndValidate("*", "*", inputTable, "suppkey_parquet_dict_snappy"); } finally { - final OperatorFixture.TestOptionSet optionSet = new OperatorFixture.TestOptionSet(); - test(String.format("alter session set `%s` = '%s'", ExecConstants.PARQUET_WRITER_COMPRESSION_TYPE, optionSet.getDefault(ExecConstants.PARQUET_WRITER_COMPRESSION_TYPE).string_val)); + resetSessionOption(ExecConstants.PARQUET_WRITER_COMPRESSION_TYPE); } } @@ -997,7 +996,7 @@ public class TestParquetWriter extends BaseTestQuery { .build() .run(); } finally { - test("alter system reset `%s`", ExecConstants.PARQUET_READER_INT96_AS_TIMESTAMP); + resetSessionOption(ExecConstants.PARQUET_READER_INT96_AS_TIMESTAMP); } } diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestParquetWriterEmptyFiles.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestParquetWriterEmptyFiles.java index c9d4300f6..2170f418d 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestParquetWriterEmptyFiles.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestParquetWriterEmptyFiles.java @@ -97,8 +97,7 @@ public class TestParquetWriterEmptyFiles extends BaseTestQuery { .go(); } finally { // restore the session option - final OperatorFixture.TestOptionSet optionSet = new OperatorFixture.TestOptionSet(); - test("ALTER SESSION SET `store.parquet.block-size` = %d", optionSet.getDefault(ExecConstants.PARQUET_BLOCK_SIZE).num_val); + resetSessionOption(ExecConstants.PARQUET_BLOCK_SIZE); deleteTableIfExists(outputFile); } } diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/xsort/TestSimpleExternalSort.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/xsort/TestSimpleExternalSort.java index d2d7b0c8d..aa0d4eec0 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/xsort/TestSimpleExternalSort.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/xsort/TestSimpleExternalSort.java @@ -34,7 +34,7 @@ import org.apache.drill.exec.vector.BigIntVector; import org.apache.drill.test.ClientFixture; import org.apache.drill.test.ClusterFixture; import org.apache.drill.test.DrillTest; -import org.apache.drill.test.FixtureBuilder; +import org.apache.drill.test.ClusterFixtureBuilder; import org.apache.drill.categories.SlowTest; import org.junit.Rule; import org.junit.Test; @@ -69,7 +69,7 @@ public class TestSimpleExternalSort extends DrillTest { */ private void mergeSortWithSv2(boolean testLegacy) throws Exception { - FixtureBuilder builder = ClusterFixture.builder() + ClusterFixtureBuilder builder = ClusterFixture.builder() .configProperty(ExecConstants.EXTERNAL_SORT_DISABLE_MANAGED, false) ; try (ClusterFixture cluster = builder.build(); @@ -96,7 +96,7 @@ public class TestSimpleExternalSort extends DrillTest { } private void sortOneKeyDescendingMergeSort(boolean testLegacy) throws Throwable { - FixtureBuilder builder = ClusterFixture.builder() + ClusterFixtureBuilder builder = ClusterFixture.builder() .configProperty(ExecConstants.EXTERNAL_SORT_DISABLE_MANAGED, false) ; try (ClusterFixture cluster = builder.build(); @@ -147,7 +147,7 @@ public class TestSimpleExternalSort extends DrillTest { } private void sortOneKeyDescendingExternalSort(boolean testLegacy) throws Throwable { - FixtureBuilder builder = ClusterFixture.builder() + ClusterFixtureBuilder builder = ClusterFixture.builder() .configProperty(ExecConstants.EXTERNAL_SORT_SPILL_THRESHOLD, 4) .configProperty(ExecConstants.EXTERNAL_SORT_SPILL_GROUP_SIZE, 4) .configProperty(ExecConstants.EXTERNAL_SORT_BATCH_LIMIT, 4) @@ -173,7 +173,7 @@ public class TestSimpleExternalSort extends DrillTest { } private void outOfMemoryExternalSort(boolean testLegacy) throws Throwable{ - FixtureBuilder builder = ClusterFixture.builder() + ClusterFixtureBuilder builder = ClusterFixture.builder() // Probably do nothing in modern Drill .configProperty("drill.memory.fragment.max", 50_000_000) .configProperty("drill.memory.fragment.initial", 2_000_000) diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/xsort/TestSortSpillWithException.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/xsort/TestSortSpillWithException.java index f6b3a8d44..2694f64b6 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/xsort/TestSortSpillWithException.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/xsort/TestSortSpillWithException.java @@ -32,7 +32,7 @@ import org.apache.drill.exec.testing.Controls; import org.apache.drill.exec.testing.ControlsInjectionUtil; import org.apache.drill.test.ClusterFixture; import org.apache.drill.test.ClusterTest; -import org.apache.drill.test.FixtureBuilder; +import org.apache.drill.test.ClusterFixtureBuilder; import org.junit.BeforeClass; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -51,7 +51,7 @@ public class TestSortSpillWithException extends ClusterTest { @BeforeClass public static void setup() throws Exception { - FixtureBuilder builder = ClusterFixture.builder() + ClusterFixtureBuilder builder = ClusterFixture.builder() .configProperty(ExecConstants.EXTERNAL_SORT_SPILL_THRESHOLD, 1) // Unmanaged .configProperty(ExecConstants.EXTERNAL_SORT_SPILL_GROUP_SIZE, 1) // Unmanaged .sessionOption(ExecConstants.MAX_QUERY_MEMORY_PER_NODE_KEY, 60 * 1024 * 1024) // Spill early diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/xsort/managed/SortTestUtilities.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/xsort/managed/SortTestUtilities.java index f525b2ae9..4c90769ef 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/xsort/managed/SortTestUtilities.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/xsort/managed/SortTestUtilities.java @@ -62,6 +62,7 @@ public class SortTestUtilities { return makeSchema(MinorType.INT, true); } + @SuppressWarnings("resource") public static PriorityQueueCopierWrapper makeCopier(OperatorFixture fixture, String sortOrder, String nullOrder) { FieldReference expr = FieldReference.getWithQuotedRef("key"); Ordering ordering = new Ordering(sortOrder, expr, nullOrder); diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/xsort/managed/TestShortArrays.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/xsort/managed/TestShortArrays.java index 1c9d49eca..ba5dfceb3 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/xsort/managed/TestShortArrays.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/xsort/managed/TestShortArrays.java @@ -83,7 +83,7 @@ public class TestShortArrays extends SubOperatorTest { assertEquals(bHint.elementCount, bCol.estElementCountPerArray, 0.001); // Create a new batch, and new vector, using the sizer and - // intializer inferred from the previoius batch. + // initializer inferred from the previous batch. SingleRowSet empty = fixture.rowSet(schema); vi.allocateBatch(empty.container(), 100); diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/server/TestDrillbitResilience.java b/exec/java-exec/src/test/java/org/apache/drill/exec/server/TestDrillbitResilience.java index bde67644e..1548d7ddc 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/server/TestDrillbitResilience.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/server/TestDrillbitResilience.java @@ -34,8 +34,8 @@ import org.apache.commons.math3.util.Pair; import org.apache.drill.BaseTestQuery; import org.apache.drill.QueryTestUtil; import org.apache.drill.SingleRowListener; -import org.apache.drill.common.concurrent.ExtendedLatch; import org.apache.drill.common.DrillAutoCloseables; +import org.apache.drill.common.concurrent.ExtendedLatch; import org.apache.drill.common.config.DrillConfig; import org.apache.drill.common.exceptions.UserException; import org.apache.drill.common.types.TypeProtos.MinorType; @@ -58,6 +58,7 @@ import org.apache.drill.exec.physical.impl.xsort.ExternalSortBatch; import org.apache.drill.exec.planner.sql.DrillSqlWorker; import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint; import org.apache.drill.exec.proto.GeneralRPCProtos.Ack; +import org.apache.drill.exec.proto.UserBitShared; import org.apache.drill.exec.proto.UserBitShared.DrillPBError; import org.apache.drill.exec.proto.UserBitShared.ExceptionWrapper; import org.apache.drill.exec.proto.UserBitShared.QueryData; @@ -74,15 +75,14 @@ import org.apache.drill.exec.rpc.RpcException; import org.apache.drill.exec.rpc.user.QueryDataBatch; import org.apache.drill.exec.rpc.user.UserResultsListener; import org.apache.drill.exec.store.pojo.PojoRecordReader; -import org.apache.drill.exec.testing.ControlsInjectionUtil; import org.apache.drill.exec.testing.Controls; +import org.apache.drill.exec.testing.ControlsInjectionUtil; import org.apache.drill.exec.util.Pointer; import org.apache.drill.exec.work.foreman.Foreman; import org.apache.drill.exec.work.foreman.ForemanException; import org.apache.drill.exec.work.foreman.ForemanSetupException; import org.apache.drill.exec.work.fragment.FragmentExecutor; import org.apache.drill.test.DrillTest; -import org.apache.drill.test.OperatorFixture; import org.apache.drill.categories.SlowTest; import org.junit.After; import org.junit.AfterClass; @@ -138,7 +138,6 @@ public class TestDrillbitResilience extends DrillTest { * @param name name of the drillbit */ private static void stopDrillbit(final String name) { - @SuppressWarnings("resource") final Drillbit drillbit = drillbits.get(name); if (drillbit == null) { throw new IllegalStateException("No Drillbit named \"" + name + "\" found"); @@ -321,6 +320,19 @@ public class TestDrillbitResilience extends DrillTest { ControlsInjectionUtil.setSessionOption(drillClient, option, value); } + private static void resetSessionOption(final String option) { + try { + final List<QueryDataBatch> results = drillClient.runQuery( + UserBitShared.QueryType.SQL, String.format("ALTER session RESET `%s`", + option)); + for (final QueryDataBatch data : results) { + data.release(); + } + } catch (final RpcException e) { + fail("Could not reset option: " + e.toString()); + } + } + /** * Check that the injected exception is what we were expecting. * @@ -812,9 +824,8 @@ public class TestDrillbitResilience extends DrillTest { final String query = "SELECT sales_city, COUNT(*) cnt FROM cp.`region.json` GROUP BY sales_city"; assertCancelledWithoutException(control, new ListenerThatCancelsQueryAfterFirstBatchOfData(), query); } finally { - final OperatorFixture.TestOptionSet testOptionSet = new OperatorFixture.TestOptionSet(); - setSessionOption(SLICE_TARGET, Long.toString(SLICE_TARGET_DEFAULT)); - setSessionOption(HASHAGG.getOptionName(), testOptionSet.getDefault(HASHAGG.getOptionName()).bool_val.toString()); + resetSessionOption(SLICE_TARGET); + resetSessionOption(HASHAGG.getOptionName()); } } @@ -844,11 +855,9 @@ public class TestDrillbitResilience extends DrillTest { final long after = countAllocatedMemory(); assertEquals(String.format("We are leaking %d bytes", after - before), before, after); } finally { - final OperatorFixture.TestOptionSet testOptionSet = new OperatorFixture.TestOptionSet(); - setSessionOption(SLICE_TARGET, Long.toString(SLICE_TARGET_DEFAULT)); - setSessionOption(HASHAGG.getOptionName(), testOptionSet.getDefault(HASHAGG.getOptionName()).bool_val.toString()); - setSessionOption(PARTITION_SENDER_SET_THREADS.getOptionName(), - Long.toString(testOptionSet.getDefault(PARTITION_SENDER_SET_THREADS.getOptionName()).num_val)); + resetSessionOption(SLICE_TARGET); + resetSessionOption(HASHAGG.getOptionName()); + resetSessionOption(PARTITION_SENDER_SET_THREADS.getOptionName()); } } diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/server/rest/StatusResourcesTest.java b/exec/java-exec/src/test/java/org/apache/drill/exec/server/rest/StatusResourcesTest.java index bf7578ff0..4f4390fac 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/server/rest/StatusResourcesTest.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/server/rest/StatusResourcesTest.java @@ -23,7 +23,7 @@ import org.apache.drill.exec.server.options.OptionDefinition; import org.apache.drill.exec.server.options.OptionValidator; import org.apache.drill.test.ClientFixture; import org.apache.drill.test.ClusterFixture; -import org.apache.drill.test.FixtureBuilder; +import org.apache.drill.test.ClusterFixtureBuilder; import org.apache.drill.test.RestClientFixture; import org.junit.Assert; import org.junit.Test; @@ -36,7 +36,7 @@ public class StatusResourcesTest { public void testRetrieveInternalOption() throws Exception { OptionDefinition optionDefinition = createMockPropOptionDefinition(); - FixtureBuilder builder = ClusterFixture.builder(). + ClusterFixtureBuilder builder = ClusterFixture.builder(). configProperty(ExecConstants.HTTP_ENABLE, true). configProperty(ExecConstants.bootDefaultFor(MOCK_PROPERTY), "a"). configProperty(ExecConstants.HTTP_PORT_HUNT, true). @@ -60,7 +60,7 @@ public class StatusResourcesTest { @Test public void testRetrievePublicOption() throws Exception { - FixtureBuilder builder = ClusterFixture.builder(). + ClusterFixtureBuilder builder = ClusterFixture.builder(). configProperty(ExecConstants.HTTP_ENABLE, true). configProperty(ExecConstants.HTTP_PORT_HUNT, true). configProperty(ExecConstants.SYS_STORE_PROVIDER_LOCAL_ENABLE_WRITE, false). diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetInternalsTest.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetInternalsTest.java index 60e466d8f..0f7c21326 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetInternalsTest.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetInternalsTest.java @@ -23,7 +23,7 @@ import org.apache.drill.common.types.TypeProtos; import org.apache.drill.common.types.Types; import org.apache.drill.test.ClusterFixture; import org.apache.drill.test.ClusterTest; -import org.apache.drill.test.FixtureBuilder; +import org.apache.drill.test.ClusterFixtureBuilder; import org.junit.BeforeClass; import org.junit.Test; @@ -34,7 +34,7 @@ public class ParquetInternalsTest extends ClusterTest { @BeforeClass public static void setup( ) throws Exception { - FixtureBuilder builder = ClusterFixture.builder() + ClusterFixtureBuilder builder = ClusterFixture.builder() // Set options, etc. ; startCluster(builder); diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetFilterPushDown.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetFilterPushDown.java index 8f56c4535..cb5000f32 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetFilterPushDown.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetFilterPushDown.java @@ -292,9 +292,7 @@ public class TestParquetFilterPushDown extends PlanTestBase { testParquetFilterPD(query1, 9, 3, false); } finally { - final OperatorFixture.TestOptionSet testOptionSet = new OperatorFixture.TestOptionSet(); - test("alter session set `" + PlannerSettings.PARQUET_ROWGROUP_FILTER_PUSHDOWN_PLANNING_KEY + "` = " + - testOptionSet.getDefault(PlannerSettings.PARQUET_ROWGROUP_FILTER_PUSHDOWN_PLANNING_KEY).bool_val); + resetSessionOption(PlannerSettings.PARQUET_ROWGROUP_FILTER_PUSHDOWN_PLANNING_KEY); deleteTableIfExists(tableName); } } @@ -317,9 +315,7 @@ public class TestParquetFilterPushDown extends PlanTestBase { testParquetFilterPD(query1, 9, 3, false); } finally { - final OperatorFixture.TestOptionSet testOptionSet = new OperatorFixture.TestOptionSet(); - test("alter session set `" + PlannerSettings.PARQUET_ROWGROUP_FILTER_PUSHDOWN_PLANNING_THRESHOLD_KEY + "` = " + - testOptionSet.getDefault(PlannerSettings.PARQUET_ROWGROUP_FILTER_PUSHDOWN_PLANNING_THRESHOLD_KEY).num_val); + resetSessionOption(PlannerSettings.PARQUET_ROWGROUP_FILTER_PUSHDOWN_PLANNING_THRESHOLD_KEY); deleteTableIfExists(tableName); } } diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/sys/TestPStoreProviders.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/sys/TestPStoreProviders.java index 79a0f45ab..e7b150c78 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/sys/TestPStoreProviders.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/sys/TestPStoreProviders.java @@ -34,7 +34,7 @@ import org.apache.drill.exec.store.sys.store.provider.LocalPersistentStoreProvid import org.apache.drill.exec.store.sys.store.provider.ZookeeperPersistentStoreProvider; import org.apache.drill.test.ClientFixture; import org.apache.drill.test.ClusterFixture; -import org.apache.drill.test.FixtureBuilder; +import org.apache.drill.test.ClusterFixtureBuilder; import org.apache.drill.testutils.DirTestWatcher; import org.apache.zookeeper.CreateMode; import org.junit.Assert; @@ -85,7 +85,7 @@ public class TestPStoreProviders extends TestWithZookeeper { org.apache.commons.io.FileUtils.copyDirectory(localOptionsResources, optionsDir); - FixtureBuilder builder = ClusterFixture.builder(). + ClusterFixtureBuilder builder = ClusterFixture.builder(). configProperty(ExecConstants.HTTP_ENABLE, false). configProperty(ExecConstants.SYS_STORE_PROVIDER_CLASS, LocalPersistentStoreProvider.class.getCanonicalName()). configProperty(ExecConstants.SYS_STORE_PROVIDER_LOCAL_PATH, String.format("file://%s", dirTestWatcher.getDirPath())). diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/util/TestQueryMemoryAlloc.java b/exec/java-exec/src/test/java/org/apache/drill/exec/util/TestQueryMemoryAlloc.java index 2476da77c..7a62ef336 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/util/TestQueryMemoryAlloc.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/util/TestQueryMemoryAlloc.java @@ -33,8 +33,8 @@ public class TestQueryMemoryAlloc extends DrillTest { @Test public void testDefaultOptions() throws Exception { OperatorFixtureBuilder builder = OperatorFixture.builder(); - builder.options().set(ExecConstants.PERCENT_MEMORY_PER_QUERY_KEY, 0.05); - builder.options().set(ExecConstants.MAX_QUERY_MEMORY_PER_NODE_KEY, 2 * ONE_GB); + builder.systemOption(ExecConstants.PERCENT_MEMORY_PER_QUERY_KEY, 0.05); + builder.systemOption(ExecConstants.MAX_QUERY_MEMORY_PER_NODE_KEY, 2 * ONE_GB); try (OperatorFixture fixture = builder.build()) { @@ -58,8 +58,8 @@ public class TestQueryMemoryAlloc extends DrillTest { @Test public void testCustomFloor() throws Exception { OperatorFixtureBuilder builder = OperatorFixture.builder(); - builder.options().set(ExecConstants.PERCENT_MEMORY_PER_QUERY_KEY, 0.05); - builder.options().set(ExecConstants.MAX_QUERY_MEMORY_PER_NODE_KEY, 3 * ONE_GB); + builder.systemOption(ExecConstants.PERCENT_MEMORY_PER_QUERY_KEY, 0.05); + builder.systemOption(ExecConstants.MAX_QUERY_MEMORY_PER_NODE_KEY, 3 * ONE_GB); try (OperatorFixture fixture = builder.build()) { @@ -83,8 +83,8 @@ public class TestQueryMemoryAlloc extends DrillTest { @Test public void testCustomPercent() throws Exception { OperatorFixtureBuilder builder = OperatorFixture.builder(); - builder.options().set(ExecConstants.PERCENT_MEMORY_PER_QUERY_KEY, 0.10); - builder.options().set(ExecConstants.MAX_QUERY_MEMORY_PER_NODE_KEY, 2 * ONE_GB); + builder.systemOption(ExecConstants.PERCENT_MEMORY_PER_QUERY_KEY, 0.10); + builder.systemOption(ExecConstants.MAX_QUERY_MEMORY_PER_NODE_KEY, 2 * ONE_GB); try (OperatorFixture fixture = builder.build()) { @@ -117,9 +117,9 @@ public class TestQueryMemoryAlloc extends DrillTest { @Test public void testOpMemory() throws Exception { OperatorFixtureBuilder builder = OperatorFixture.builder(); - builder.options().set(ExecConstants.CPU_LOAD_AVERAGE_KEY, 0.7); - builder.options().set(ExecConstants.MAX_WIDTH_PER_NODE_KEY, 10); - builder.options().set(ExecConstants.MIN_MEMORY_PER_BUFFERED_OP_KEY, 40 * ONE_MB); + builder.systemOption(ExecConstants.CPU_LOAD_AVERAGE_KEY, 0.7); + builder.systemOption(ExecConstants.MAX_WIDTH_PER_NODE_KEY, 10); + builder.systemOption(ExecConstants.MIN_MEMORY_PER_BUFFERED_OP_KEY, 40 * ONE_MB); try (OperatorFixture fixture = builder.build()) { diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestExtendedTypes.java b/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestExtendedTypes.java index 07108180d..58793a22e 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestExtendedTypes.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestExtendedTypes.java @@ -61,13 +61,8 @@ public class TestExtendedTypes extends BaseTestQuery { + "/0_0_0.json")); assertEquals(new String(originalData), new String(newData)); } finally { - final OperatorFixture.TestOptionSet testOptionSet = new OperatorFixture.TestOptionSet(); - testNoResult(String.format("ALTER SESSION SET `%s` = '%s'", - ExecConstants.OUTPUT_FORMAT_VALIDATOR.getOptionName(), - testOptionSet.getDefault(ExecConstants.OUTPUT_FORMAT_VALIDATOR.getOptionName()).getValue())); - testNoResult(String.format("ALTER SESSION SET `%s` = %s", - ExecConstants.JSON_EXTENDED_TYPES.getOptionName(), - testOptionSet.getDefault(ExecConstants.JSON_EXTENDED_TYPES.getOptionName()).getValue())); + resetSessionOption(ExecConstants.OUTPUT_FORMAT_VALIDATOR.getOptionName()); + resetSessionOption(ExecConstants.JSON_EXTENDED_TYPES.getOptionName()); } } @@ -92,13 +87,8 @@ public class TestExtendedTypes extends BaseTestQuery { String expected = "drill_timestamp_millies,bin,bin1\n2015-07-07T03:59:43.488,drill,drill\n"; Assert.assertEquals(expected, actual); } finally { - final OperatorFixture.TestOptionSet testOptionSet = new OperatorFixture.TestOptionSet(); - testNoResult(String.format("ALTER SESSION SET `%s` = '%s'", - ExecConstants.OUTPUT_FORMAT_VALIDATOR.getOptionName(), - testOptionSet.getDefault(ExecConstants.OUTPUT_FORMAT_VALIDATOR.getOptionName()).getValue())); - testNoResult(String.format("ALTER SESSION SET `%s` = %s", - ExecConstants.JSON_EXTENDED_TYPES.getOptionName(), - testOptionSet.getDefault(ExecConstants.JSON_EXTENDED_TYPES.getOptionName()).getValue())); + resetSessionOption(ExecConstants.OUTPUT_FORMAT_VALIDATOR.getOptionName()); + resetSessionOption(ExecConstants.JSON_EXTENDED_TYPES.getOptionName()); } } } diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/ClientFixture.java b/exec/java-exec/src/test/java/org/apache/drill/test/ClientFixture.java index 8ad0fbb48..a80d4f62d 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/test/ClientFixture.java +++ b/exec/java-exec/src/test/java/org/apache/drill/test/ClientFixture.java @@ -132,14 +132,12 @@ public class ClientFixture implements AutoCloseable { * @param key */ - public void resetSystem(String key) { - String sql = "ALTER SYSTEM RESET `" + key + "`"; - runSqlSilently(sql); + public void resetSession(String key) { + runSqlSilently("ALTER SESSION RESET `" + key + "`"); } - public void resetSession(String key) { - String sql = "ALTER SESSION RESET `" + key + "`"; - runSqlSilently(sql); + public void resetSystem(String key) { + runSqlSilently("ALTER SYSTEM RESET `" + key + "`"); } /** diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/ClusterFixture.java b/exec/java-exec/src/test/java/org/apache/drill/test/ClusterFixture.java index 5d53b59a0..f36c19357 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/test/ClusterFixture.java +++ b/exec/java-exec/src/test/java/org/apache/drill/test/ClusterFixture.java @@ -147,7 +147,7 @@ public class ClusterFixture extends BaseFixture implements AutoCloseable { private List<File> tempDirs = new ArrayList<>(); - ClusterFixture(FixtureBuilder builder) { + ClusterFixture(ClusterFixtureBuilder builder) { setClientProps(builder); configureZk(builder); @@ -166,9 +166,9 @@ public class ClusterFixture extends BaseFixture implements AutoCloseable { /** * Set the client properties to be used by client fixture. - * @param builder {@link FixtureBuilder#clientProps} + * @param builder {@link ClusterFixtureBuilder#clientProps} */ - private void setClientProps(FixtureBuilder builder) { + private void setClientProps(ClusterFixtureBuilder builder) { clientProps = builder.clientProps; } @@ -176,7 +176,7 @@ public class ClusterFixture extends BaseFixture implements AutoCloseable { return clientProps; } - private void configureZk(FixtureBuilder builder) { + private void configureZk(ClusterFixtureBuilder builder) { // Start ZK if requested. @@ -213,7 +213,7 @@ public class ClusterFixture extends BaseFixture implements AutoCloseable { } } - private void createConfig(FixtureBuilder builder) throws Exception { + private void createConfig(ClusterFixtureBuilder builder) throws Exception { // Create a config // Because of the way DrillConfig works, we can set the ZK @@ -235,7 +235,7 @@ public class ClusterFixture extends BaseFixture implements AutoCloseable { } } - private void startDrillbits(FixtureBuilder builder) throws Exception { + private void startDrillbits(ClusterFixtureBuilder builder) throws Exception { // // Ensure that Drill uses the log directory determined here rather than // // it's hard-coded defaults. WIP: seems to be needed some times but // // not others. @@ -319,12 +319,12 @@ public class ClusterFixture extends BaseFixture implements AutoCloseable { ((StoragePluginRegistryImpl) pluginRegistry).definePlugin(MockStorageEngineConfig.NAME, config, plugin); } - private void applyOptions(FixtureBuilder builder) throws Exception { + private void applyOptions(ClusterFixtureBuilder builder) throws Exception { // Apply system options if (builder.systemOptions != null) { - for (FixtureBuilder.RuntimeOption option : builder.systemOptions) { + for (ClusterFixtureBuilder.RuntimeOption option : builder.systemOptions) { clientFixture().alterSystem(option.key, option.value); } } @@ -332,7 +332,7 @@ public class ClusterFixture extends BaseFixture implements AutoCloseable { // Apply session options. if (builder.sessionOptions != null) { - for (FixtureBuilder.RuntimeOption option : builder.sessionOptions) { + for (ClusterFixtureBuilder.RuntimeOption option : builder.sessionOptions) { clientFixture().alterSession(option.key, option.value); } } @@ -584,8 +584,8 @@ public class ClusterFixture extends BaseFixture implements AutoCloseable { public static final String EXPLAIN_PLAN_TEXT = "text"; public static final String EXPLAIN_PLAN_JSON = "json"; - public static FixtureBuilder builder() { - FixtureBuilder builder = new FixtureBuilder() + public static ClusterFixtureBuilder builder() { + ClusterFixtureBuilder builder = new ClusterFixtureBuilder() .sessionOption(ExecConstants.MAX_WIDTH_PER_NODE_KEY, MAX_WIDTH_PER_NODE); Properties props = new Properties(); props.putAll(ClusterFixture.TEST_CONFIGURATIONS); @@ -604,8 +604,8 @@ public class ClusterFixture extends BaseFixture implements AutoCloseable { * @return a fixture builder with no default properties set */ - public static FixtureBuilder bareBuilder() { - return new FixtureBuilder(); + public static ClusterFixtureBuilder bareBuilder() { + return new ClusterFixtureBuilder(); } /** diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/FixtureBuilder.java b/exec/java-exec/src/test/java/org/apache/drill/test/ClusterFixtureBuilder.java index e73c2babb..829555406 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/test/FixtureBuilder.java +++ b/exec/java-exec/src/test/java/org/apache/drill/test/ClusterFixtureBuilder.java @@ -32,7 +32,7 @@ import org.apache.drill.exec.server.options.OptionDefinition; * a max width (parallelization) of 2. */ -public class FixtureBuilder { +public class ClusterFixtureBuilder { public static class RuntimeOption { public String key; @@ -88,7 +88,7 @@ public class FixtureBuilder { * @see {@link #configProperty(String, Object)} */ - public FixtureBuilder configResource(String configResource) { + public ClusterFixtureBuilder configResource(String configResource) { // TypeSafe gets unhappy about a leading slash, but other functions // require it. Silently discard the leading slash if given to @@ -101,7 +101,7 @@ public class FixtureBuilder { /** * */ - public FixtureBuilder setOptionDefault(String key, Object value) { + public ClusterFixtureBuilder setOptionDefault(String key, Object value) { String option_name = OPTION_DEFAULTS_ROOT + key; configBuilder().put(option_name, value.toString()); return this; @@ -113,12 +113,12 @@ public class FixtureBuilder { * @return this builder */ - public FixtureBuilder configProperty(String key, Object value) { + public ClusterFixtureBuilder configProperty(String key, Object value) { configBuilder.put(key, value.toString()); return this; } - public FixtureBuilder putDefinition(OptionDefinition definition) { + public ClusterFixtureBuilder putDefinition(OptionDefinition definition) { configBuilder.putDefinition(definition); return this; } @@ -130,7 +130,7 @@ public class FixtureBuilder { * @param value property value * @return this builder */ - public FixtureBuilder configClientProperty(String key, Object value) { + public ClusterFixtureBuilder configClientProperty(String key, Object value) { if (clientProps == null) { clientProps = new Properties(); } @@ -148,7 +148,7 @@ public class FixtureBuilder { * @see {@link ClusterFixture#alterSession(String, Object)} */ - public FixtureBuilder sessionOption(String key, Object value) { + public ClusterFixtureBuilder sessionOption(String key, Object value) { if (sessionOptions == null) { sessionOptions = new ArrayList<>(); } @@ -166,7 +166,7 @@ public class FixtureBuilder { * @see {@link ClusterFixture#alterSystem(String, Object)} */ - public FixtureBuilder systemOption(String key, Object value) { + public ClusterFixtureBuilder systemOption(String key, Object value) { if (systemOptions == null) { systemOptions = new ArrayList<>(); } @@ -181,7 +181,7 @@ public class FixtureBuilder { * @param n the "max width per node" parallelization option. * @return this builder */ - public FixtureBuilder maxParallelization(int n) { + public ClusterFixtureBuilder maxParallelization(int n) { return sessionOption(ExecConstants.MAX_WIDTH_PER_NODE_KEY, n); } @@ -191,7 +191,7 @@ public class FixtureBuilder { * @param n the desired cluster size * @return this builder */ - public FixtureBuilder clusterSize(int n) { + public ClusterFixtureBuilder clusterSize(int n) { bitCount = n; bitNames = null; return this; @@ -204,7 +204,7 @@ public class FixtureBuilder { * @param bitNames array of (unique) Drillbit names * @return this builder */ - public FixtureBuilder withBits(String bitNames[]) { + public ClusterFixtureBuilder withBits(String bitNames[]) { this.bitNames = bitNames; bitCount = bitNames.length; return this; @@ -216,11 +216,11 @@ public class FixtureBuilder { * Drillbits. * @return this builder */ - public FixtureBuilder withLocalZk() { + public ClusterFixtureBuilder withLocalZk() { return withLocalZk(1); } - public FixtureBuilder withLocalZk(int count) { + public ClusterFixtureBuilder withLocalZk(int count) { localZkCount = count; usingZk = true; @@ -229,7 +229,7 @@ public class FixtureBuilder { return configProperty(ExecConstants.ZK_REFRESH, DEFAULT_ZK_REFRESH); } - public FixtureBuilder withRemoteZk(String connStr) { + public ClusterFixtureBuilder withRemoteZk(String connStr) { usingZk = true; return configProperty(ExecConstants.ZK_CONNECTION, connStr); } @@ -242,7 +242,7 @@ public class FixtureBuilder { * @param zk the global Zookeeper to use * @return this builder */ - public FixtureBuilder withZk(ZookeeperHelper zk) { + public ClusterFixtureBuilder withZk(ZookeeperHelper zk) { zkHelper = zk; usingZk = true; @@ -252,7 +252,7 @@ public class FixtureBuilder { return this; } - public FixtureBuilder tempDir(File path) { + public ClusterFixtureBuilder tempDir(File path) { this.tempDir = path; return this; } @@ -274,7 +274,7 @@ public class FixtureBuilder { * @return this builder */ - public FixtureBuilder keepLocalFiles() { + public ClusterFixtureBuilder keepLocalFiles() { preserveLocalFiles = true; return this; } @@ -290,7 +290,7 @@ public class FixtureBuilder { * @return this builder */ - public FixtureBuilder saveProfiles() { + public ClusterFixtureBuilder saveProfiles() { configProperty(ExecConstants.SYS_STORE_PROVIDER_LOCAL_ENABLE_WRITE, true); systemOption(ExecConstants.ENABLE_QUERY_PROFILE_OPTION, true); systemOption(ExecConstants.QUERY_PROFILE_DEBUG_OPTION, true); diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/ClusterTest.java b/exec/java-exec/src/test/java/org/apache/drill/test/ClusterTest.java index e204fded5..4a3823c23 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/test/ClusterTest.java +++ b/exec/java-exec/src/test/java/org/apache/drill/test/ClusterTest.java @@ -77,7 +77,7 @@ public class ClusterTest extends DrillTest { protected static ClusterFixture cluster; protected static ClientFixture client; - protected static void startCluster(FixtureBuilder builder) throws Exception { + protected static void startCluster(ClusterFixtureBuilder builder) throws Exception { cluster = builder.build(); client = cluster.clientFixture(); } diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/ExampleTest.java b/exec/java-exec/src/test/java/org/apache/drill/test/ExampleTest.java index a770d3e02..67ae4a3a4 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/test/ExampleTest.java +++ b/exec/java-exec/src/test/java/org/apache/drill/test/ExampleTest.java @@ -88,7 +88,7 @@ public class ExampleTest { @Test public void secondTest() throws Exception { - FixtureBuilder builder = ClusterFixture.builder() + ClusterFixtureBuilder builder = ClusterFixture.builder() .configProperty(ExecConstants.SLICE_TARGET, 10) ; @@ -159,7 +159,7 @@ public class ExampleTest { // And trace messages for one class. .logger(ExternalSortBatch.class, Level.TRACE) ; - FixtureBuilder builder = ClusterFixture.builder() + ClusterFixtureBuilder builder = ClusterFixture.builder() // Easy way to run single threaded for easy debugging .maxParallelization(1) // Set some session options @@ -203,7 +203,7 @@ public class ExampleTest { @Test public void fifthTest() throws Exception { - FixtureBuilder builder = ClusterFixture.builder() + ClusterFixtureBuilder builder = ClusterFixture.builder() .maxParallelization(1) .configProperty(ExecConstants.SYS_STORE_PROVIDER_LOCAL_ENABLE_WRITE, true) ; diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/OperatorFixture.java b/exec/java-exec/src/test/java/org/apache/drill/test/OperatorFixture.java index 39b0dc72e..09abf1285 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/test/OperatorFixture.java +++ b/exec/java-exec/src/test/java/org/apache/drill/test/OperatorFixture.java @@ -18,12 +18,12 @@ package org.apache.drill.test; import java.io.IOException; +import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; import org.apache.drill.common.config.DrillConfig; -import org.apache.drill.common.map.CaseInsensitiveMap; import org.apache.drill.common.scanner.ClassPathScanner; import org.apache.drill.common.scanner.persistence.ScanResult; import org.apache.drill.exec.ExecConstants; @@ -43,14 +43,10 @@ import org.apache.drill.exec.ops.OperatorStatReceiver; import org.apache.drill.exec.physical.base.PhysicalOperator; import org.apache.drill.exec.record.BatchSchema; import org.apache.drill.exec.record.VectorContainer; -import org.apache.drill.exec.server.options.BaseOptionSet; -import org.apache.drill.exec.server.options.OptionDefinition; import org.apache.drill.exec.server.options.OptionSet; -import org.apache.drill.exec.server.options.OptionValue; -import org.apache.drill.exec.server.options.OptionValue.AccessibleScopes; -import org.apache.drill.exec.server.options.OptionValue.OptionScope; import org.apache.drill.exec.server.options.SystemOptionManager; import org.apache.drill.exec.testing.ExecutionControls; +import org.apache.drill.test.ClusterFixtureBuilder.RuntimeOption; import org.apache.drill.test.rowSet.DirectRowSet; import org.apache.drill.test.rowSet.HyperRowSetImpl; import org.apache.drill.test.rowSet.IndirectRowSet; @@ -92,84 +88,29 @@ public class OperatorFixture extends BaseFixture implements AutoCloseable { public static class OperatorFixtureBuilder { - ConfigBuilder configBuilder = new ConfigBuilder(); - TestOptionSet options = new TestOptionSet(); + protected ConfigBuilder configBuilder = new ConfigBuilder(); + protected List<RuntimeOption> systemOptions; + protected ExecutionControls controls; public ConfigBuilder configBuilder() { return configBuilder; } - public TestOptionSet options() { - return options; - } - - public OperatorFixture build() { - return new OperatorFixture(this); - } - } - - /** - * Test-time implementation of the system and session options. Provides - * a simple storage and a simple set interface, then implements the standard - * system and session option read interface. - */ - - public static class TestOptionSet extends BaseOptionSet { - - private Map<String,OptionValue> values = CaseInsensitiveMap.newHashMap(); - private Map<String,OptionValue> defaults = CaseInsensitiveMap.newHashMap(); - - public TestOptionSet() { - DrillConfig defaultConfig = DrillConfig.create(); - Map<String, OptionDefinition> definitions = SystemOptionManager.createDefaultOptionDefinitions(); - CaseInsensitiveMap<OptionValue> defaultValues = SystemOptionManager.populateDefaultValues(definitions, defaultConfig); - - for (Map.Entry<String, OptionValue> entry: defaultValues.entrySet()) { - String optionName = entry.getKey(); - OptionValue optionValue = entry.getValue(); - defaults.put(optionName, optionValue); + public OperatorFixtureBuilder systemOption(String key, Object value) { + if (systemOptions == null) { + systemOptions = new ArrayList<>(); } - - // Crashes in FunctionImplementationRegistry if not set - set(ExecConstants.CAST_TO_NULLABLE_NUMERIC, false); - // Crashes in the Dynamic UDF code if not disabled - set(ExecConstants.USE_DYNAMIC_UDFS_KEY, false); - } - - public void set(String key, int value) { - set(key, (long) value); - } - - public void set(String key, long value) { - values.put(key, OptionValue.create(AccessibleScopes.SYSTEM, key, value, OptionScope.SYSTEM)); + systemOptions.add(new RuntimeOption(key, value)); + return this; } - public void set(String key, boolean value) { - values.put(key, OptionValue.create(AccessibleScopes.SYSTEM, key, value, OptionScope.SYSTEM)); - } - - public void set(String key, double value) { - values.put(key, OptionValue.create(AccessibleScopes.SYSTEM, key, value, OptionScope.SYSTEM)); - } - - public void set(String key, String value) { - values.put(key, OptionValue.create(AccessibleScopes.SYSTEM, key, value, OptionScope.SYSTEM)); - } - - @Override - public OptionValue getOption(String name) { - final OptionValue value = values.get(name); - - if (value != null) { - return value; - } - - return getDefault(name); + public OperatorFixtureBuilder setControls(ExecutionControls controls) { + this.controls = controls; + return this; } - @Override - public OptionValue getDefault(String optionName) { - return defaults.get(optionName); + public OperatorFixture build() { + return new OperatorFixture(this); } } @@ -255,7 +196,7 @@ public class OperatorFixture extends BaseFixture implements AutoCloseable { public static class MockStats implements OperatorStatReceiver { - public Map<Integer,Double> stats = new HashMap<>(); + public Map<Integer, Double> stats = new HashMap<>(); @Override public void addLongStat(MetricDef metric, long value) { @@ -295,26 +236,39 @@ public class OperatorFixture extends BaseFixture implements AutoCloseable { } } - private final TestOptionSet options; + private final SystemOptionManager options; private final TestCodeGenContext context; private final OperatorStatReceiver stats; protected OperatorFixture(OperatorFixtureBuilder builder) { config = builder.configBuilder().build(); allocator = RootAllocatorFactory.newRoot(config); - options = builder.options(); + options = new SystemOptionManager(config); + try { + options.init(); + } catch (Exception e) { + throw new IllegalStateException("Failed to initialize the system option manager", e); + } + if (builder.systemOptions != null) { + applySystemOptions(builder.systemOptions); + } context = new TestCodeGenContext(config, options); stats = new MockStats(); } - public TestOptionSet options() { return options; } - + private void applySystemOptions(List<RuntimeOption> systemOptions) { + for (RuntimeOption option : systemOptions) { + options.setLocalOption(option.key, option.value); + } + } - public FragmentExecContext codeGenContext() { return context; } + public SystemOptionManager options() { return options; } + public FragmentExecContext fragmentExecContext() { return context; } @Override public void close() throws Exception { allocator.close(); + options.close(); } public static OperatorFixtureBuilder builder() { diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/TestConfigLinkage.java b/exec/java-exec/src/test/java/org/apache/drill/test/TestConfigLinkage.java index 19b8ed5ce..83c60b394 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/test/TestConfigLinkage.java +++ b/exec/java-exec/src/test/java/org/apache/drill/test/TestConfigLinkage.java @@ -52,7 +52,7 @@ public class TestConfigLinkage { public void testDefaultInternalValue() throws Exception { OptionDefinition optionDefinition = createMockPropOptionDefinition(); - FixtureBuilder builder = ClusterFixture.builder(). + ClusterFixtureBuilder builder = ClusterFixture.builder(). configProperty(ExecConstants.bootDefaultFor(MOCK_PROPERTY), "a"). putDefinition(optionDefinition); @@ -72,7 +72,7 @@ public class TestConfigLinkage { public void testDefaultValidatorInternalValue() throws Exception { OptionDefinition optionDefinition = createMockPropOptionDefinition(); - FixtureBuilder builder = ClusterFixture.builder(). + ClusterFixtureBuilder builder = ClusterFixture.builder(). putDefinition(optionDefinition). configProperty(ExecConstants.bootDefaultFor(MOCK_PROPERTY), "a"); @@ -91,7 +91,7 @@ public class TestConfigLinkage { /* Test if session option takes precedence */ @Test public void testSessionOption() throws Exception { - FixtureBuilder builder = ClusterFixture.builder().sessionOption(ExecConstants.SLICE_TARGET, 10); + ClusterFixtureBuilder builder = ClusterFixture.builder().sessionOption(ExecConstants.SLICE_TARGET, 10); try (ClusterFixture cluster = builder.build(); ClientFixture client = cluster.clientFixture()) { String slice_target = client.queryBuilder().sql("SELECT val FROM sys.%s where name='planner.slice_target' and optionScope = 'SESSION'", SystemTable.OPTION_VAL @@ -104,7 +104,7 @@ public class TestConfigLinkage { /* Test if system option takes precedence over the boot option */ @Test public void testSystemOption() throws Exception { - FixtureBuilder builder = ClusterFixture.builder().systemOption(ExecConstants.SLICE_TARGET, 10000); + ClusterFixtureBuilder builder = ClusterFixture.builder().systemOption(ExecConstants.SLICE_TARGET, 10000); try (ClusterFixture cluster = builder.build(); ClientFixture client = cluster.clientFixture()) { String slice_target = client.queryBuilder().sql("SELECT val FROM sys.%s where name='planner.slice_target' and optionScope = 'SYSTEM'", SystemTable.OPTION_VAL.getTableName()) @@ -117,7 +117,7 @@ public class TestConfigLinkage { public void testInternalSystemOption() throws Exception { OptionDefinition optionDefinition = createMockPropOptionDefinition(); - FixtureBuilder builder = ClusterFixture.builder(). + ClusterFixtureBuilder builder = ClusterFixture.builder(). putDefinition(optionDefinition). configProperty(ExecConstants.bootDefaultFor(MOCK_PROPERTY), "a"). systemOption(MOCK_PROPERTY, "blah"); @@ -139,7 +139,7 @@ public class TestConfigLinkage { /* Test if config option takes precedence if config option is not set */ @Test public void testConfigOption() throws Exception { - FixtureBuilder builder = ClusterFixture.builder() + ClusterFixtureBuilder builder = ClusterFixture.builder() .setOptionDefault(ExecConstants.SLICE_TARGET, 1000); try (ClusterFixture cluster = builder.build(); ClientFixture client = cluster.clientFixture()) { @@ -164,7 +164,7 @@ public class TestConfigLinkage { /* Test if altering session option takes precedence over system option */ @Test public void testSessionPrecedence() throws Exception { - FixtureBuilder builder = ClusterFixture.builder().systemOption(ExecConstants.SLICE_TARGET, 100000); + ClusterFixtureBuilder builder = ClusterFixture.builder().systemOption(ExecConstants.SLICE_TARGET, 100000); try (ClusterFixture cluster = builder.build(); ClientFixture client = cluster.clientFixture()) { client.queryBuilder().sql("ALTER SESSION SET `planner.slice_target` = 10000").run(); @@ -177,7 +177,7 @@ public class TestConfigLinkage { /* Test if setting maxwidth option through config takes effect */ @Test public void testMaxWidthPerNodeConfig() throws Exception { - FixtureBuilder builder = ClusterFixture.bareBuilder().setOptionDefault(ExecConstants.MAX_WIDTH_PER_NODE_KEY, 2); + ClusterFixtureBuilder builder = ClusterFixture.bareBuilder().setOptionDefault(ExecConstants.MAX_WIDTH_PER_NODE_KEY, 2); try (ClusterFixture cluster = builder.build(); ClientFixture client = cluster.clientFixture()) { String maxWidth = client.queryBuilder().sql("SELECT val FROM sys.%s where name='planner.width.max_per_node' and optionScope = 'BOOT'", SystemTable.OPTION_VAL.getTableName()) @@ -189,7 +189,7 @@ public class TestConfigLinkage { /* Test if setting maxwidth at system level takes precedence */ @Test public void testMaxWidthPerNodeSystem() throws Exception { - FixtureBuilder builder = ClusterFixture.bareBuilder().systemOption(ExecConstants.MAX_WIDTH_PER_NODE_KEY, 3); + ClusterFixtureBuilder builder = ClusterFixture.bareBuilder().systemOption(ExecConstants.MAX_WIDTH_PER_NODE_KEY, 3); try (ClusterFixture cluster = builder.build(); ClientFixture client = cluster.clientFixture()) { String maxWidth = client.queryBuilder().sql("SELECT val FROM sys.%s where name='planner.width.max_per_node' and optionScope = 'SYSTEM'", SystemTable.OPTION_VAL.getTableName()) @@ -201,7 +201,7 @@ public class TestConfigLinkage { /* Test if setting maxwidth at session level takes precedence */ @Test public void testMaxWidthPerNodeSession() throws Exception { - FixtureBuilder builder = ClusterFixture.bareBuilder().sessionOption(ExecConstants.MAX_WIDTH_PER_NODE_KEY, 2); + ClusterFixtureBuilder builder = ClusterFixture.bareBuilder().sessionOption(ExecConstants.MAX_WIDTH_PER_NODE_KEY, 2); try (ClusterFixture cluster = builder.build(); ClientFixture client = cluster.clientFixture()) { String maxWidth = client.queryBuilder().sql("SELECT val FROM sys.%s where name='planner.width.max_per_node' and optionScope = 'SESSION'", SystemTable.OPTION_VAL.getTableName()) @@ -215,7 +215,7 @@ public class TestConfigLinkage { */ @Test public void testMaxWidthPerNodeDefault() throws Exception { - FixtureBuilder builder = ClusterFixture.bareBuilder().setOptionDefault(ExecConstants.CPU_LOAD_AVERAGE_KEY, 0.70); + ClusterFixtureBuilder builder = ClusterFixture.bareBuilder().setOptionDefault(ExecConstants.CPU_LOAD_AVERAGE_KEY, 0.70); try (ClusterFixture cluster = builder.build(); ClientFixture client = cluster.clientFixture()) { long maxWidth = ExecConstants.MAX_WIDTH_PER_NODE.computeMaxWidth(0.70, 0); @@ -228,7 +228,7 @@ public class TestConfigLinkage { /* Test if the scope is set during BOOT time and scope is actually BOOT */ @Test public void testScope() throws Exception { - FixtureBuilder builder = ClusterFixture.bareBuilder().setOptionDefault(ExecConstants.SLICE_TARGET, 100000); + ClusterFixtureBuilder builder = ClusterFixture.bareBuilder().setOptionDefault(ExecConstants.SLICE_TARGET, 100000); try (ClusterFixture cluster = builder.build(); ClientFixture client = cluster.clientFixture()) { String scope = client.queryBuilder() @@ -241,7 +241,7 @@ public class TestConfigLinkage { /* Test if the option is set at SYSTEM scope and the scope is actually SYSTEM */ @Test public void testScopeSystem() throws Exception { - FixtureBuilder builder = ClusterFixture.bareBuilder().systemOption(ExecConstants.SLICE_TARGET, 10000); + ClusterFixtureBuilder builder = ClusterFixture.bareBuilder().systemOption(ExecConstants.SLICE_TARGET, 10000); try (ClusterFixture cluster = builder.build(); ClientFixture client = cluster.clientFixture()) { String scope = client.queryBuilder() @@ -254,7 +254,7 @@ public class TestConfigLinkage { /* Test if the option is set at SESSION scope and the scope is actually SESSION */ @Test public void testScopeSession() throws Exception { - FixtureBuilder builder = ClusterFixture.bareBuilder().sessionOption(ExecConstants.SLICE_TARGET, 100000); + ClusterFixtureBuilder builder = ClusterFixture.bareBuilder().sessionOption(ExecConstants.SLICE_TARGET, 100000); try (ClusterFixture cluster = builder.build(); ClientFixture client = cluster.clientFixture()) { String scope = client.queryBuilder() @@ -267,7 +267,7 @@ public class TestConfigLinkage { /* Test if the option is altered at SYSTEM scope and the scope is actually SYSTEM */ @Test public void testScopeAlterSystem() throws Exception { - FixtureBuilder builder = ClusterFixture.bareBuilder(); + ClusterFixtureBuilder builder = ClusterFixture.bareBuilder(); try (ClusterFixture cluster = builder.build(); ClientFixture client = cluster.clientFixture()) { client.queryBuilder().sql("ALTER SYSTEM set `planner.slice_target`= 10000").run(); @@ -281,7 +281,7 @@ public class TestConfigLinkage { /* Test if the option is altered at SESSION scope and the scope is actually SESSION */ @Test public void testScopeAlterSession() throws Exception { - FixtureBuilder builder = ClusterFixture.bareBuilder(); + ClusterFixtureBuilder builder = ClusterFixture.bareBuilder(); try (ClusterFixture cluster = builder.build(); ClientFixture client = cluster.clientFixture()) { client.queryBuilder().sql("ALTER SESSION set `planner.slice_target`= 10000").run(); @@ -296,7 +296,7 @@ public class TestConfigLinkage { public void testAlterInternalSystemOption() throws Exception { OptionDefinition optionDefinition = createMockPropOptionDefinition(); - FixtureBuilder builder = ClusterFixture.builder(). + ClusterFixtureBuilder builder = ClusterFixture.builder(). configProperty(ExecConstants.bootDefaultFor(MOCK_PROPERTY), "a"). putDefinition(optionDefinition); diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/package-info.java b/exec/java-exec/src/test/java/org/apache/drill/test/package-info.java index 9f62478d8..dac71ec79 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/test/package-info.java +++ b/exec/java-exec/src/test/java/org/apache/drill/test/package-info.java @@ -20,7 +20,7 @@ * tests and ad-hoc tests created while developing features. Key components * include: * <ul> - * <li>{@link FixtureBuilder}: Builder pattern to create an embedded Drillbit, + * <li>{@link ClusterFixtureBuilder}: Builder pattern to create an embedded Drillbit, * or cluster of Drillbits, using a specified set of configuration, session * and system options.</li> * <li>{@link ClusterFixture}: The cluster created by the builder.</li> |