diff options
| author | Paul Rogers <progers@maprtech.com> | 2017-12-20 23:59:06 -0800 |
|---|---|---|
| committer | Aman Sinha <asinha@maprtech.com> | 2018-01-24 10:06:03 -0800 |
| commit | e791ed62b1c91c39676c4adef438c689fd84fd4b (patch) | |
| tree | 7cb37fdcf890f7cbb253708db0cf4a5e4491d9ed /exec/vector | |
| parent | d803f0c2188c679de3dacf10741005b217425a33 (diff) | |
DRILL-6049: Misc. hygiene and code cleanup changes
close apache/drill#1085
Diffstat (limited to 'exec/vector')
22 files changed, 867 insertions, 412 deletions
diff --git a/exec/vector/src/main/codegen/templates/FixedValueVectors.java b/exec/vector/src/main/codegen/templates/FixedValueVectors.java index 1f6a00875..79beb52e0 100644 --- a/exec/vector/src/main/codegen/templates/FixedValueVectors.java +++ b/exec/vector/src/main/codegen/templates/FixedValueVectors.java @@ -106,7 +106,7 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F } @Override - public int getValueCapacity(){ + public int getValueCapacity() { return data.capacity() / VALUE_WIDTH; } @@ -129,7 +129,7 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F @Override public void allocateNew() { - if (!allocateNewSafe()){ + if (!allocateNewSafe()) { throw new OutOfMemoryException("Failure while allocating buffer."); } } @@ -264,12 +264,12 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F } @Override - public TransferPair getTransferPair(BufferAllocator allocator){ + public TransferPair getTransferPair(BufferAllocator allocator) { return new TransferImpl(getField(), allocator); } @Override - public TransferPair getTransferPair(String ref, BufferAllocator allocator){ + public TransferPair getTransferPair(String ref, BufferAllocator allocator) { return new TransferImpl(getField().withPath(ref), allocator); } @@ -278,7 +278,7 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F return new TransferImpl((${minor.class}Vector) to); } - public void transferTo(${minor.class}Vector target){ + public void transferTo(${minor.class}Vector target) { target.clear(); target.data = data.transferOwnership(target.allocator).buffer; target.data.writerIndex(data.writerIndex()); @@ -298,10 +298,10 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F return valueCount * ${type.width}; } - private class TransferImpl implements TransferPair{ + private class TransferImpl implements TransferPair { private ${minor.class}Vector to; - public TransferImpl(MaterializedField field, BufferAllocator allocator){ + public TransferImpl(MaterializedField field, BufferAllocator allocator) { to = new ${minor.class}Vector(field, allocator); } @@ -310,12 +310,12 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F } @Override - public ${minor.class}Vector getTo(){ + public ${minor.class}Vector getTo() { return to; } @Override - public void transfer(){ + public void transfer() { transferTo(to); } @@ -330,7 +330,7 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F } } - public void copyFrom(int fromIndex, int thisIndex, ${minor.class}Vector from){ + public void copyFrom(int fromIndex, int thisIndex, ${minor.class}Vector from) { <#if (type.width > 8)> from.data.getBytes(fromIndex * VALUE_WIDTH, data, thisIndex * VALUE_WIDTH, VALUE_WIDTH); <#else> <#-- type.width <= 8 --> @@ -340,7 +340,7 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F </#if> <#-- type.width --> } - public void copyFromSafe(int fromIndex, int thisIndex, ${minor.class}Vector from){ + public void copyFromSafe(int fromIndex, int thisIndex, ${minor.class}Vector from) { while(thisIndex >= getValueCapacity()) { reAlloc(); } @@ -376,24 +376,24 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F } @Override - public boolean isNull(int index){ + public boolean isNull(int index) { return false; } - <#if (type.width > 8)> + public ${minor.javaType!type.javaType} get(int index) { return data.slice(index * VALUE_WIDTH, VALUE_WIDTH); } - <#if (minor.class == "Interval")> - public void get(int index, ${minor.class}Holder holder){ + + public void get(int index, ${minor.class}Holder holder) { final int offsetIndex = index * VALUE_WIDTH; holder.months = data.getInt(offsetIndex); holder.days = data.getInt(offsetIndex + ${minor.daysOffset}); holder.milliseconds = data.getInt(offsetIndex + ${minor.millisecondsOffset}); } - public void get(int index, Nullable${minor.class}Holder holder){ + public void get(int index, Nullable${minor.class}Holder holder) { final int offsetIndex = index * VALUE_WIDTH; holder.isSet = 1; holder.months = data.getInt(offsetIndex); @@ -407,52 +407,25 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F final int months = data.getInt(offsetIndex); final int days = data.getInt(offsetIndex + ${minor.daysOffset}); final int millis = data.getInt(offsetIndex + ${minor.millisecondsOffset}); - final Period p = new Period(); - return p.plusMonths(months).plusDays(days).plusMillis(millis); + return DateUtilities.fromInterval(months, days, millis); } public StringBuilder getAsStringBuilder(int index) { - final int offsetIndex = index * VALUE_WIDTH; - - int months = data.getInt(offsetIndex); - final int days = data.getInt(offsetIndex + ${minor.daysOffset}); - int millis = data.getInt(offsetIndex + ${minor.millisecondsOffset}); - - final int years = (months / org.apache.drill.exec.expr.fn.impl.DateUtility.yearsToMonths); - months = (months % org.apache.drill.exec.expr.fn.impl.DateUtility.yearsToMonths); - - final int hours = millis / (org.apache.drill.exec.expr.fn.impl.DateUtility.hoursToMillis); - millis = millis % (org.apache.drill.exec.expr.fn.impl.DateUtility.hoursToMillis); - - final int minutes = millis / (org.apache.drill.exec.expr.fn.impl.DateUtility.minutesToMillis); - millis = millis % (org.apache.drill.exec.expr.fn.impl.DateUtility.minutesToMillis); - - final long seconds = millis / (org.apache.drill.exec.expr.fn.impl.DateUtility.secondsToMillis); - millis = millis % (org.apache.drill.exec.expr.fn.impl.DateUtility.secondsToMillis); - - final String yearString = (Math.abs(years) == 1) ? " year " : " years "; - final String monthString = (Math.abs(months) == 1) ? " month " : " months "; - final String dayString = (Math.abs(days) == 1) ? " day " : " days "; - - return(new StringBuilder(). - append(years).append(yearString). - append(months).append(monthString). - append(days).append(dayString). - append(hours).append(":"). - append(minutes).append(":"). - append(seconds).append("."). - append(millis)); + final int months = data.getInt(offsetIndex); + final int days = data.getInt(offsetIndex + ${minor.daysOffset}); + final int millis = data.getInt(offsetIndex + ${minor.millisecondsOffset}); + return DateUtilities.intervalStringBuilder(months, days, millis); } - <#elseif (minor.class == "IntervalDay")> - public void get(int index, ${minor.class}Holder holder){ + + public void get(int index, ${minor.class}Holder holder) { final int offsetIndex = index * VALUE_WIDTH; holder.days = data.getInt(offsetIndex); holder.milliseconds = data.getInt(offsetIndex + ${minor.millisecondsOffset}); } - public void get(int index, Nullable${minor.class}Holder holder){ + public void get(int index, Nullable${minor.class}Holder holder) { final int offsetIndex = index * VALUE_WIDTH; holder.isSet = 1; holder.days = data.getInt(offsetIndex); @@ -462,38 +435,19 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F @Override public ${friendlyType} getObject(int index) { final int offsetIndex = index * VALUE_WIDTH; + final int days = data.getInt(offsetIndex); final int millis = data.getInt(offsetIndex + ${minor.millisecondsOffset}); - final int days = data.getInt(offsetIndex); - final Period p = new Period(); - return p.plusDays(days).plusMillis(millis); + return DateUtilities.fromIntervalDay(days, millis); } public StringBuilder getAsStringBuilder(int index) { final int offsetIndex = index * VALUE_WIDTH; - - int millis = data.getInt(offsetIndex + ${minor.millisecondsOffset}); - final int days = data.getInt(offsetIndex); - - final int hours = millis / (org.apache.drill.exec.expr.fn.impl.DateUtility.hoursToMillis); - millis = millis % (org.apache.drill.exec.expr.fn.impl.DateUtility.hoursToMillis); - - final int minutes = millis / (org.apache.drill.exec.expr.fn.impl.DateUtility.minutesToMillis); - millis = millis % (org.apache.drill.exec.expr.fn.impl.DateUtility.minutesToMillis); - - final int seconds = millis / (org.apache.drill.exec.expr.fn.impl.DateUtility.secondsToMillis); - millis = millis % (org.apache.drill.exec.expr.fn.impl.DateUtility.secondsToMillis); - - final String dayString = (Math.abs(days) == 1) ? " day " : " days "; - - return(new StringBuilder(). - append(days).append(dayString). - append(hours).append(":"). - append(minutes).append(":"). - append(seconds).append("."). - append(millis)); + final int days = data.getInt(offsetIndex); + final int millis = data.getInt(offsetIndex + ${minor.millisecondsOffset}); + return DateUtilities.intervalDayStringBuilder(days, millis); } - <#elseif minor.class == "Decimal28Sparse" || minor.class == "Decimal38Sparse" || minor.class == "Decimal28Dense" || minor.class == "Decimal38Dense"> + public void get(int index, ${minor.class}Holder holder) { holder.start = index * VALUE_WIDTH; holder.buffer = data; @@ -515,17 +469,19 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F // Get the BigDecimal object return DecimalUtility.getBigDecimalFromSparse(data, index * VALUE_WIDTH, ${minor.nDecimalDigits}, getField().getScale()); <#else> - return DecimalUtility.getBigDecimalFromDense(data, index * VALUE_WIDTH, ${minor.nDecimalDigits}, getField().getScale(), ${minor.maxPrecisionDigits}, VALUE_WIDTH); + return DecimalUtility.getBigDecimalFromDense(data, index * VALUE_WIDTH, + ${minor.nDecimalDigits}, getField().getScale(), + ${minor.maxPrecisionDigits}, VALUE_WIDTH); </#if> } - <#else> - public void get(int index, ${minor.class}Holder holder){ + + public void get(int index, ${minor.class}Holder holder) { holder.buffer = data; holder.start = index * VALUE_WIDTH; } - public void get(int index, Nullable${minor.class}Holder holder){ + public void get(int index, Nullable${minor.class}Holder holder) { holder.isSet = 1; holder.buffer = data; holder.start = index * VALUE_WIDTH; @@ -535,76 +491,61 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F public ${friendlyType} getObject(int index) { return data.slice(index * VALUE_WIDTH, VALUE_WIDTH) } - </#if> <#else> <#-- type.width <= 8 --> + public ${minor.javaType!type.javaType} get(int index) { return data.get${(minor.javaType!type.javaType)?cap_first}(index * VALUE_WIDTH); } - <#if type.width == 4> + public long getTwoAsLong(int index) { return data.getLong(index * VALUE_WIDTH); } - </#if> <#if minor.class == "Date"> + @Override public ${friendlyType} getObject(int index) { org.joda.time.DateTime date = new org.joda.time.DateTime(get(index), org.joda.time.DateTimeZone.UTC); date = date.withZoneRetainFields(org.joda.time.DateTimeZone.getDefault()); return date; } - <#elseif minor.class == "TimeStamp"> + @Override public ${friendlyType} getObject(int index) { org.joda.time.DateTime date = new org.joda.time.DateTime(get(index), org.joda.time.DateTimeZone.UTC); date = date.withZoneRetainFields(org.joda.time.DateTimeZone.getDefault()); return date; } - <#elseif minor.class == "IntervalYear"> + @Override public ${friendlyType} getObject(int index) { - final int value = get(index); - final int years = (value / org.apache.drill.exec.expr.fn.impl.DateUtility.yearsToMonths); - final int months = (value % org.apache.drill.exec.expr.fn.impl.DateUtility.yearsToMonths); - final Period p = new Period(); - return p.plusYears(years).plusMonths(months); + return DateUtilities.fromIntervalYear(get(index)); } public StringBuilder getAsStringBuilder(int index) { - - int months = data.getInt(index); - - final int years = (months / org.apache.drill.exec.expr.fn.impl.DateUtility.yearsToMonths); - months = (months % org.apache.drill.exec.expr.fn.impl.DateUtility.yearsToMonths); - - final String yearString = (Math.abs(years) == 1) ? " year " : " years "; - final String monthString = (Math.abs(months) == 1) ? " month " : " months "; - - return(new StringBuilder(). - append(years).append(yearString). - append(months).append(monthString)); + return DateUtilities.intervalYearStringBuilder(data.getInt(index)); } - <#elseif minor.class == "Time"> + @Override public DateTime getObject(int index) { org.joda.time.DateTime time = new org.joda.time.DateTime(get(index), org.joda.time.DateTimeZone.UTC); time = time.withZoneRetainFields(org.joda.time.DateTimeZone.getDefault()); return time; } - <#elseif minor.class == "Decimal9" || minor.class == "Decimal18"> + @Override public ${friendlyType} getObject(int index) { final BigInteger value = BigInteger.valueOf(((${type.boxedType})get(index)).${type.javaType}Value()); return new BigDecimal(value, getField().getScale()); } - <#else> + @Override public ${friendlyType} getObject(int index) { return get(index); @@ -613,9 +554,9 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F public ${minor.javaType!type.javaType} getPrimitiveObject(int index) { return get(index); } - </#if> - public void get(int index, ${minor.class}Holder holder){ + + public void get(int index, ${minor.class}Holder holder) { <#if minor.class.startsWith("Decimal")> holder.scale = getField().getScale(); holder.precision = getField().getPrecision(); @@ -624,7 +565,7 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F holder.value = data.get${(minor.javaType!type.javaType)?cap_first}(index * VALUE_WIDTH); } - public void get(int index, Nullable${minor.class}Holder holder){ + public void get(int index, Nullable${minor.class}Holder holder) { holder.isSet = 1; holder.value = data.get${(minor.javaType!type.javaType)?cap_first}(index * VALUE_WIDTH); } @@ -658,8 +599,8 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F * @param value * value to set */ - <#if (type.width > 8)> + public void set(int index, <#if (type.width > 4)>${minor.javaType!type.javaType}<#else>int</#if> value) { data.setBytes(index * VALUE_WIDTH, value, 0, VALUE_WIDTH); } @@ -670,8 +611,8 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F } data.setBytes(index * VALUE_WIDTH, value, 0, VALUE_WIDTH); } - <#if minor.class == "Interval"> + public void set(int index, int months, int days, int milliseconds) { final int offsetIndex = index * VALUE_WIDTH; data.setInt(offsetIndex, months); @@ -701,8 +642,8 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F public void setSafe(int index, Nullable${minor.class}Holder holder) { setSafe(index, holder.months, holder.days, holder.milliseconds); } - <#elseif minor.class == "IntervalDay"> + public void set(int index, int days, int milliseconds) { final int offsetIndex = index * VALUE_WIDTH; data.setInt(offsetIndex, days); @@ -728,11 +669,11 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F set(index, holder.days, holder.milliseconds); } - public void setSafe(int index, Nullable${minor.class}Holder holder){ + public void setSafe(int index, Nullable${minor.class}Holder holder) { setSafe(index, holder.days, holder.milliseconds); } - <#elseif minor.class == "Decimal28Sparse" || minor.class == "Decimal38Sparse" || minor.class == "Decimal28Dense" || minor.class == "Decimal38Dense"> + public void setSafe(int index, int start, DrillBuf buffer) { while(index >= getValueCapacity()) { reAlloc(); @@ -755,8 +696,8 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F public void setSafe(int index, Nullable${minor.class}Holder holder) { setSafe(index, holder.start, holder.buffer); } - <#if minor.class == "Decimal28Sparse" || minor.class == "Decimal38Sparse"> + public void set(int index, BigDecimal value) { DecimalUtility.getSparseFromBigDecimal(value, data, index * VALUE_WIDTH, field.getScale(), field.getPrecision(), ${minor.nDecimalDigits}); @@ -768,13 +709,13 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F } set(index, value); } - </#if> - public void set(int index, int start, DrillBuf buffer){ + + public void set(int index, int start, DrillBuf buffer) { data.setBytes(index * VALUE_WIDTH, buffer, start, VALUE_WIDTH); } - </#if> + @Override public void generateTestData(int count) { setValueCount(count); @@ -782,13 +723,13 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F final int valueCount = getAccessor().getValueCount(); for(int i = 0; i < valueCount; i++, even = !even) { final byte b = even ? Byte.MIN_VALUE : Byte.MAX_VALUE; - for(int w = 0; w < VALUE_WIDTH; w++){ + for(int w = 0; w < VALUE_WIDTH; w++) { data.setByte(i + w, b); } } } - <#else> <#-- type.width <= 8 --> + public void set(int index, <#if (type.width >= 4)>${minor.javaType!type.javaType}<#else>int</#if> value) { data.set${(minor.javaType!type.javaType)?cap_first}(index * VALUE_WIDTH, value); } @@ -855,8 +796,8 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F } } } - </#if> <#-- type.width --> + @Override public void setValueCount(int valueCount) { final int currentValueCapacity = getValueCapacity(); diff --git a/exec/vector/src/main/codegen/templates/ListWriters.java b/exec/vector/src/main/codegen/templates/ListWriters.java index 16d41ecf1..f10cfc4c8 100644 --- a/exec/vector/src/main/codegen/templates/ListWriters.java +++ b/exec/vector/src/main/codegen/templates/ListWriters.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -43,7 +43,10 @@ package org.apache.drill.exec.vector.complex.impl; public class ${mode}ListWriter extends AbstractFieldWriter { private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(${mode}ListWriter.class); - static enum Mode { INIT, IN_MAP, IN_LIST <#list vv.types as type><#list type.minor as minor>, IN_${minor.class?upper_case}</#list></#list> } + enum Mode { + INIT, IN_MAP, IN_LIST + <#list vv.types as type><#list type.minor as minor>, + IN_${minor.class?upper_case}</#list></#list> } private final String name; protected final ${containerClass} container; @@ -69,7 +72,6 @@ public class ${mode}ListWriter extends AbstractFieldWriter { if(writer != null) { writer.allocate(); } - <#if mode == "Repeated"> container.allocateNew(); </#if> @@ -97,12 +99,14 @@ public class ${mode}ListWriter extends AbstractFieldWriter { } public void setValueCount(int count){ - if(innerVector != null) innerVector.getMutator().setValueCount(count); + if (innerVector != null) { + innerVector.getMutator().setValueCount(count); + } } @Override public MapWriter map() { - switch(mode) { + switch (mode) { case INIT: int vectorCount = container.size(); final RepeatedMapVector vector = container.addOrGet(name, RepeatedMapVector.TYPE, RepeatedMapVector.class); @@ -116,21 +120,23 @@ public class ${mode}ListWriter extends AbstractFieldWriter { return writer; case IN_MAP: return writer; + default: + throw UserException + .unsupportedError() + .message(getUnsupportedErrorMsg("MAP", mode.name())) + .build(logger); } - - throw UserException.unsupportedError().message(getUnsupportedErrorMsg("MAP", mode.name())).build(logger); - } @Override public ListWriter list() { - switch(mode) { + switch (mode) { case INIT: final int vectorCount = container.size(); final RepeatedListVector vector = container.addOrGet(name, RepeatedListVector.TYPE, RepeatedListVector.class); innerVector = vector; writer = new RepeatedListWriter(null, vector, this); - if(vectorCount != container.size()) { + if (vectorCount != container.size()) { writer.allocate(); } writer.setPosition(${index}); @@ -138,10 +144,12 @@ public class ${mode}ListWriter extends AbstractFieldWriter { return writer; case IN_LIST: return writer; + default: + throw UserException + .unsupportedError() + .message(getUnsupportedErrorMsg("LIST", mode.name())) + .build(logger); } - - throw UserException.unsupportedError().message(getUnsupportedErrorMsg("LIST", mode.name())).build(logger); - } <#list vv.types as type><#list type.minor as minor> @@ -149,12 +157,11 @@ public class ${mode}ListWriter extends AbstractFieldWriter { <#assign upperName = minor.class?upper_case /> <#assign capName = minor.class?cap_first /> <#if lowerName == "int" ><#assign lowerName = "integer" /></#if> - private static final MajorType ${upperName}_TYPE = Types.repeated(MinorType.${upperName}); @Override public ${capName}Writer ${lowerName}() { - switch(mode) { + switch (mode) { case INIT: final int vectorCount = container.size(); final Repeated${capName}Vector vector = container.addOrGet(name, ${upperName}_TYPE, Repeated${capName}Vector.class); @@ -168,19 +175,22 @@ public class ${mode}ListWriter extends AbstractFieldWriter { return writer; case IN_${upperName}: return writer; + default: + throw UserException + .unsupportedError() + .message(getUnsupportedErrorMsg("${upperName}", mode.name())) + .build(logger); } - - throw UserException.unsupportedError().message(getUnsupportedErrorMsg("${upperName}", mode.name())).build(logger); - } + </#list></#list> - + @Override public MaterializedField getField() { return container.getField(); } - <#if mode == "Repeated"> - + + @Override public void startList() { final RepeatedListVector list = (RepeatedListVector) container; final RepeatedListVector.RepeatedMutator mutator = list.getMutator(); @@ -202,11 +212,13 @@ public class ${mode}ListWriter extends AbstractFieldWriter { } } + @Override public void endList() { // noop, we initialize state at start rather than end. } <#else> + @Override public void setPosition(int index) { super.setPosition(index); if(writer != null) { @@ -214,10 +226,12 @@ public class ${mode}ListWriter extends AbstractFieldWriter { } } + @Override public void startList() { // noop } + @Override public void endList() { // noop } diff --git a/exec/vector/src/main/codegen/templates/NullableValueVectors.java b/exec/vector/src/main/codegen/templates/NullableValueVectors.java index fdb0200b5..93f8e7b58 100644 --- a/exec/vector/src/main/codegen/templates/NullableValueVectors.java +++ b/exec/vector/src/main/codegen/templates/NullableValueVectors.java @@ -15,7 +15,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +import org.apache.drill.common.types.TypeProtos.DataMode; import org.apache.drill.exec.memory.AllocationManager.BufferLedger; +import org.apache.drill.exec.record.MaterializedField; import org.apache.drill.exec.util.DecimalUtility; import org.apache.drill.exec.vector.BaseDataValueVector; import org.apache.drill.exec.vector.NullableVectorDefinitionSetter; @@ -49,12 +51,9 @@ package org.apache.drill.exec.vector; */ public final class ${className} extends BaseDataValueVector implements <#if type.major == "VarLen">VariableWidth<#else>FixedWidth</#if>Vector, NullableVector { - private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(${className}.class); private final FieldReader reader = new Nullable${minor.class}ReaderImpl(Nullable${minor.class}Vector.this); - private final MaterializedField bitsField = MaterializedField.create("$bits$", Types.required(MinorType.UINT1)); - /** * Set value flag. Meaning: * <ul> @@ -67,13 +66,26 @@ public final class ${className} extends BaseDataValueVector implements <#if type */ private final UInt1Vector bits = new UInt1Vector(bitsField, allocator); - private final ${valuesName} values = new ${minor.class}Vector(field, allocator); + + private final ${valuesName} values; private final Mutator mutator = new Mutator(); - private final Accessor accessor = new Accessor(); + private final Accessor accessor; public ${className}(MaterializedField field, BufferAllocator allocator) { super(field, allocator); + + // The values vector has its own name, and has the same type and attributes + // as the nullable vector. This ensures that + // things like scale and precision are preserved in the values vector. + + values = new ${minor.class}Vector( + MaterializedField.create(VALUES_VECTOR_NAME, field.getType()), + allocator); + + field.addChild(bits.getField()); + field.addChild(values.getField()); + accessor = new Accessor(); } @Override @@ -128,6 +140,11 @@ public final class ${className} extends BaseDataValueVector implements <#if type } @Override + public int getAllocatedSize(){ + return bits.getAllocatedSize() + values.getAllocatedSize(); + } + + @Override public DrillBuf getBuffer() { return values.getBuffer(); } @@ -138,6 +155,13 @@ public final class ${className} extends BaseDataValueVector implements <#if type @Override public UInt1Vector getBitsVector() { return bits; } + <#if type.major == "VarLen"> + @Override + public UInt4Vector getOffsetVector() { + return ((VariableWidthVector) values).getOffsetVector(); + } + + </#if> @Override public void setInitialCapacity(int numRecords) { bits.setInitialCapacity(numRecords); diff --git a/exec/vector/src/main/codegen/templates/UnionListWriter.java b/exec/vector/src/main/codegen/templates/UnionListWriter.java index c676769f3..81d5f9c2e 100644 --- a/exec/vector/src/main/codegen/templates/UnionListWriter.java +++ b/exec/vector/src/main/codegen/templates/UnionListWriter.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -32,15 +32,12 @@ package org.apache.drill.exec.vector.complex.impl; * This class is generated using freemarker and the ${.template_name} template. */ -@SuppressWarnings("unused") public class UnionListWriter extends AbstractFieldWriter { private ListVector vector; private UInt4Vector offsets; private PromotableWriter writer; private boolean inMap = false; - private String mapName; - private int lastIndex = 0; public UnionListWriter(ListVector vector) { super(null); @@ -74,14 +71,10 @@ public class UnionListWriter extends AbstractFieldWriter { } @Override - public void close() throws Exception { - - } - + public void close() throws Exception { } <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first /> <#assign fields = minor.fields!type.fields /> <#assign uncappedName = name?uncap_first/> - <#if !minor.class?starts_with("Decimal")> @Override @@ -92,16 +85,13 @@ public class UnionListWriter extends AbstractFieldWriter { @Override public ${name}Writer <#if uncappedName == "int">integer<#else>${uncappedName}</#if>(String name) { assert inMap; - mapName = name; final int nextOffset = offsets.getAccessor().get(idx() + 1); vector.getMutator().setNotNull(idx()); writer.setPosition(nextOffset); ${name}Writer ${uncappedName}Writer = writer.<#if uncappedName == "int">integer<#else>${uncappedName}</#if>(name); return ${uncappedName}Writer; } - </#if> - </#list></#list> @Override @@ -140,9 +130,7 @@ public class UnionListWriter extends AbstractFieldWriter { } @Override - public void endList() { - - } + public void endList() { } @Override public void start() { @@ -161,11 +149,9 @@ public class UnionListWriter extends AbstractFieldWriter { offsets.getMutator().setSafe(idx() + 1, nextOffset + 1); } } - <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first /> <#assign fields = minor.fields!type.fields /> <#assign uncappedName = name?uncap_first/> - <#if !minor.class?starts_with("Decimal")> @Override @@ -177,9 +163,6 @@ public class UnionListWriter extends AbstractFieldWriter { writer.write${name}(<#list fields as field>${field.name}<#if field_has_next>, </#if></#list>); offsets.getMutator().setSafe(idx() + 1, nextOffset + 1); } - </#if> - </#list></#list> - } diff --git a/exec/vector/src/main/codegen/templates/UnionVector.java b/exec/vector/src/main/codegen/templates/UnionVector.java index a46779de0..248b0107c 100644 --- a/exec/vector/src/main/codegen/templates/UnionVector.java +++ b/exec/vector/src/main/codegen/templates/UnionVector.java @@ -32,9 +32,15 @@ import java.util.Iterator; import java.util.Set; import org.apache.drill.exec.vector.complex.impl.ComplexCopier; + +import com.google.common.base.Preconditions; + import org.apache.drill.exec.util.CallBack; import org.apache.drill.exec.expr.BasicTypeHelper; import org.apache.drill.exec.memory.AllocationManager.BufferLedger; +import org.apache.drill.exec.record.MaterializedField; + +import com.google.common.annotations.VisibleForTesting; /* * This class is generated using freemarker and the ${.template_name} template. @@ -43,43 +49,91 @@ import org.apache.drill.exec.memory.AllocationManager.BufferLedger; /** - * A vector which can hold values of different types. It does so by using a MapVector which contains a vector for each - * primitive type that is stored. MapVector is used in order to take advantage of its serialization/deserialization methods, - * as well as the addOrGet method. + * A vector which can hold values of different types. It does so by using a + * MapVector which contains a vector for each primitive type that is stored. + * MapVector is used in order to take advantage of its + * serialization/deserialization methods, as well as the addOrGet method. * - * For performance reasons, UnionVector stores a cached reference to each subtype vector, to avoid having to do the map lookup - * each time the vector is accessed. + * For performance reasons, UnionVector stores a cached reference to each + * subtype vector, to avoid having to do the map lookup each time the vector is + * accessed. */ public class UnionVector implements ValueVector { + public static final int NULL_MARKER = 0; + public static final String TYPE_VECTOR_NAME = "types"; + public static final String INTERNAL_MAP_NAME = "internal"; + + private static final MajorType MAJOR_TYPES[] = new MajorType[MinorType.values().length]; + + static { + MAJOR_TYPES[MinorType.MAP.ordinal()] = Types.optional(MinorType.MAP); + MAJOR_TYPES[MinorType.LIST.ordinal()] = Types.optional(MinorType.LIST); + <#list vv.types as type> + <#list type.minor as minor> + <#assign name = minor.class?cap_first /> + <#assign fields = minor.fields!type.fields /> + <#assign uncappedName = name?uncap_first/> + <#if !minor.class?starts_with("Decimal")> + MAJOR_TYPES[MinorType.${name?upper_case}.ordinal()] = Types.optional(MinorType.${name?upper_case}); + </#if> + </#list> + </#list> + } + private MaterializedField field; private BufferAllocator allocator; private Accessor accessor = new Accessor(); private Mutator mutator = new Mutator(); private int valueCount; + /** + * Map which holds one vector for each subtype, along with a vector that indicates + * types and the null state. There appears to be no reason other than convenience + * for using a map. Future implementations may wish to store vectors directly in + * the union vector, but must then implement the required vector serialization/ + * deserialization and other functionality. + */ + private MapVector internalMap; + + /** + * Cached type vector. The vector's permament location is in the + * internal map, it is cached for performance. Call + * {@link #getTypeVector()} to get the cached copy, or to refresh + * the cache from the internal map if not set. + */ + private UInt1Vector typeVector; - private MapVector mapVector; - private ListVector listVector; + /** + * Set of cached vectors that duplicate vectors store in the + * internal map. Used to avoid a name lookup on every access. + * The cache is populated as vectors are added. But, after the + * union is sent over the wire, the map is populated, but the + * array is not. It will be repopulated upon first access to + * the deserialized vectors. + */ + + private ValueVector cachedSubtypes[] = new ValueVector[MinorType.values().length]; private FieldReader reader; - private NullableBitVector bit; - - private int singleType = 0; - private ValueVector singleVector; - private MajorType majorType; private final CallBack callBack; public UnionVector(MaterializedField field, BufferAllocator allocator, CallBack callBack) { + + // The metadata may start off listing subtypes for which vectors + // do not actually exist. It appears that the semantics are to list + // the subtypes that *could* appear. For example, in a sort we may + // have two types: one batch has type A, the other type B, but the + // batches must list both A and B as subtypes. + this.field = field.clone(); this.allocator = allocator; - this.internalMap = new MapVector("internal", allocator, callBack); - this.typeVector = internalMap.addOrGet("types", Types.required(MinorType.UINT1), UInt1Vector.class); + this.internalMap = new MapVector(INTERNAL_MAP_NAME, allocator, callBack); + this.typeVector = internalMap.addOrGet(TYPE_VECTOR_NAME, Types.required(MinorType.UINT1), UInt1Vector.class); this.field.addChild(internalMap.getField().clone()); - this.majorType = field.getType(); this.callBack = callBack; } @@ -87,94 +141,180 @@ public class UnionVector implements ValueVector { public BufferAllocator getAllocator() { return allocator; } - + public List<MinorType> getSubTypes() { - return majorType.getSubTypeList(); - } - + return field.getType().getSubTypeList(); + } + + @SuppressWarnings("unchecked") + public <T extends ValueVector> T subtype(MinorType type) { + return (T) cachedSubtypes[type.ordinal()]; + } + + + /** + * Add an externally-created subtype vector. The vector must represent a type that + * does not yet exist in the union, and must be of OPTIONAL mode. Does not call + * the callback since the client (presumably) knows that it is adding the type. + * The caller must also allocate the buffer for the vector. + * + * @param vector subtype vector to add + */ + + public void addType(ValueVector vector) { + MinorType type = vector.getField().getType().getMinorType(); + assert subtype(type) == null; + assert vector.getField().getType().getMode() == DataMode.OPTIONAL; + assert vector.getField().getName().equals(type.name().toLowerCase()); + cachedSubtypes[type.ordinal()] = vector; + internalMap.putChild(type.name(), vector); + addSubType(type); + } + public void addSubType(MinorType type) { - if (majorType.getSubTypeList().contains(type)) { + if (field.getType().getSubTypeList().contains(type)) { return; } - majorType = MajorType.newBuilder(this.majorType).addSubType(type).build(); - field = MaterializedField.create(field.getName(), majorType); + field.replaceType( + MajorType.newBuilder(field.getType()).addSubType(type).build()); if (callBack != null) { callBack.doWork(); } } - private static final MajorType MAP_TYPE = Types.optional(MinorType.MAP); + /** + * "Classic" way to add a subtype when working directly with a union vector. + * Creates the vector, adds it to the internal structures and creates a + * new buffer of the default size. + * + * @param type the type to add + * @param vectorClass class of the vector to create + * @return typed form of the new value vector + */ + + private <T extends ValueVector> T classicAddType(MinorType type, Class<? extends ValueVector> vectorClass) { + int vectorCount = internalMap.size(); + @SuppressWarnings("unchecked") + T vector = (T) internalMap.addOrGet(type.name().toLowerCase(), MAJOR_TYPES[type.ordinal()], vectorClass); + cachedSubtypes[type.ordinal()] = vector; + if (internalMap.size() > vectorCount) { + vector.allocateNew(); + addSubType(type); + if (callBack != null) { + callBack.doWork(); + } + } + return vector; + } public MapVector getMap() { + MapVector mapVector = subtype(MinorType.MAP); if (mapVector == null) { - int vectorCount = internalMap.size(); - mapVector = internalMap.addOrGet("map", MAP_TYPE, MapVector.class); - addSubType(MinorType.MAP); - if (internalMap.size() > vectorCount) { - mapVector.allocateNew(); - } + mapVector = classicAddType(MinorType.MAP, MapVector.class); } return mapVector; } - <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first /> - <#assign fields = minor.fields!type.fields /> - <#assign uncappedName = name?uncap_first/> - <#if !minor.class?starts_with("Decimal")> - private Nullable${name}Vector ${uncappedName}Vector; - private static final MajorType ${name?upper_case}_TYPE = Types.optional(MinorType.${name?upper_case}); + public ListVector getList() { + ListVector listVector = subtype(MinorType.LIST); + if (listVector == null) { + listVector = classicAddType(MinorType.LIST, ListVector.class); + } + return listVector; + } + <#-- Generating a method per type is probably overkill. However, existing code + depends on these methods, so didn't want to remove them. Over time, a + generic, parameterized addOrGet(MinorType type) would be more compact. + Would need a function to map from minor type to vector class, which + can be generated here or in TypeHelper. --> + <#list vv.types as type> + <#list type.minor as minor> + <#assign name = minor.class?cap_first /> + <#assign fields = minor.fields!type.fields /> + <#assign uncappedName = name?uncap_first/> + <#if !minor.class?starts_with("Decimal")> public Nullable${name}Vector get${name}Vector() { - if (${uncappedName}Vector == null) { - int vectorCount = internalMap.size(); - ${uncappedName}Vector = internalMap.addOrGet("${uncappedName}", ${name?upper_case}_TYPE, Nullable${name}Vector.class); - addSubType(MinorType.${name?upper_case}); - if (internalMap.size() > vectorCount) { - ${uncappedName}Vector.allocateNew(); - } + Nullable${name}Vector vector = subtype(MinorType.${name?upper_case}); + if (vector == null) { + vector = classicAddType(MinorType.${name?upper_case}, Nullable${name}Vector.class); } - return ${uncappedName}Vector; + return vector; } - </#if> - </#list></#list> - - private static final MajorType LIST_TYPE = Types.optional(MinorType.LIST); - - public ListVector getList() { - if (listVector == null) { - int vectorCount = internalMap.size(); - listVector = internalMap.addOrGet("list", LIST_TYPE, ListVector.class); - addSubType(MinorType.LIST); - if (internalMap.size() > vectorCount) { - listVector.allocateNew(); - } + </#if> + </#list> + </#list> + + /** + * Add or get a type member given the type. + * + * @param type the type of the vector to retrieve + * @return the (potentially newly created) vector that backs the given type + */ + + public ValueVector getMember(MinorType type) { + switch (type) { + case MAP: + return getMap(); + case LIST: + return getList(); + <#-- This awkard switch statement and call to type-specific method logic + can be generalized as described above. --> + <#list vv.types as type> + <#list type.minor as minor> + <#assign name = minor.class?cap_first /> + <#assign fields = minor.fields!type.fields /> + <#assign uncappedName = name?uncap_first/> + <#if !minor.class?starts_with("Decimal")> + case ${name?upper_case}: + return get${name}Vector(); + </#if> + </#list> + </#list> + default: + throw new UnsupportedOperationException(type.toString()); } - return listVector; + } + + @SuppressWarnings("unchecked") + public <T extends ValueVector> T member(MinorType type) { + return (T) getMember(type); } public int getTypeValue(int index) { - return typeVector.getAccessor().get(index); + return getTypeVector().getAccessor().get(index); } public UInt1Vector getTypeVector() { + if (typeVector == null) { + typeVector = (UInt1Vector) internalMap.getChild(TYPE_VECTOR_NAME); + } return typeVector; } + + @VisibleForTesting + public MapVector getTypeMap() { + return internalMap; + } @Override public void allocateNew() throws OutOfMemoryException { internalMap.allocateNew(); - if (typeVector != null) { - typeVector.zeroVector(); - } + getTypeVector().zeroVector(); + } + + public void allocateNew(int rowCount) throws OutOfMemoryException { + // The map vector does not have a form that takes a row count, + // but it should. + internalMap.allocateNew(); + getTypeVector().zeroVector(); } @Override public boolean allocateNewSafe() { boolean safe = internalMap.allocateNewSafe(); if (safe) { - if (typeVector != null) { - typeVector.zeroVector(); - } + getTypeVector().zeroVector(); } return safe; } @@ -184,7 +324,7 @@ public class UnionVector implements ValueVector { @Override public int getValueCapacity() { - return Math.min(typeVector.getValueCapacity(), internalMap.getValueCapacity()); + return Math.min(getTypeVector().getValueCapacity(), internalMap.getValueCapacity()); } @Override @@ -200,12 +340,7 @@ public class UnionVector implements ValueVector { @Override public void collectLedgers(Set<BufferLedger> ledgers) { - // Most vectors are held inside the internal map. - internalMap.collectLedgers(ledgers); - if (bit != null) { - bit.collectLedgers(ledgers); - } } @Override @@ -231,7 +366,6 @@ public class UnionVector implements ValueVector { public void transferTo(UnionVector target) { internalMap.makeTransferPair(target.internalMap).transfer(); target.valueCount = valueCount; - target.majorType = majorType; } public void copyFrom(int inIndex, int outIndex, UnionVector from) { @@ -249,16 +383,48 @@ public class UnionVector implements ValueVector { copyFromSafe(fromIndex, toIndex, (UnionVector) from); } + /** + * Add a vector that matches the argument. Transfer the buffer from the argument + * to the new vector. + * + * @param v the vector to clone and add + * @return the cloned vector that now holds the data from the argument + */ + public ValueVector addVector(ValueVector v) { String name = v.getField().getType().getMinorType().name().toLowerCase(); MajorType type = v.getField().getType(); + MinorType minorType = type.getMinorType(); Preconditions.checkState(internalMap.getChild(name) == null, String.format("%s vector already exists", name)); - final ValueVector newVector = internalMap.addOrGet(name, type, BasicTypeHelper.getValueVectorClass(type.getMinorType(), type.getMode())); + final ValueVector newVector = internalMap.addOrGet(name, type, BasicTypeHelper.getValueVectorClass(minorType, type.getMode())); v.makeTransferPair(newVector).transfer(); internalMap.putChild(name, newVector); - addSubType(v.getField().getType().getMinorType()); + cachedSubtypes[minorType.ordinal()] = newVector; + addSubType(minorType); return newVector; } + + // Called from SchemaUtil + + public ValueVector setFirstType(ValueVector v, int newValueCount) { + + // We can't check that this really is the first subtype since + // the subtypes can be declared before vectors are added. + + Preconditions.checkState(accessor.getValueCount() == 0); + final ValueVector vv = addVector(v); + MinorType type = v.getField().getType().getMinorType(); + ValueVector.Accessor vAccessor = vv.getAccessor(); + for (int i = 0; i < newValueCount; i++) { + if (! vAccessor.isNull(i)) { + mutator.setType(i, type); + } else { + mutator.setNull(i); + } + } + mutator.setValueCount(newValueCount); + return vv; + } @Override public void toNullable(ValueVector nullableVector) { @@ -267,7 +433,7 @@ public class UnionVector implements ValueVector { private class TransferImpl implements TransferPair { - UnionVector to; + private final UnionVector to; public TransferImpl(MaterializedField field, BufferAllocator allocator) { to = new UnionVector(field, allocator, null); @@ -319,13 +485,12 @@ public class UnionVector implements ValueVector { @Override public UserBitShared.SerializedField getMetadata() { - SerializedField.Builder b = getField() // - .getAsBuilder() // - .setBufferLength(getBufferSize()) // - .setValueCount(valueCount); - - b.addChild(internalMap.getMetadata()); - return b.build(); + return getField() + .getAsBuilder() + .setBufferLength(getBufferSize()) + .setValueCount(valueCount) + .addChild(internalMap.getMetadata()) + .build(); } @Override @@ -366,18 +531,16 @@ public class UnionVector implements ValueVector { @Override public Iterator<ValueVector> iterator() { - List<ValueVector> vectors = Lists.newArrayList(internalMap.iterator()); - vectors.add(typeVector); - return vectors.iterator(); + return internalMap.iterator(); } public class Accessor extends BaseValueVector.BaseAccessor { @Override public Object getObject(int index) { - int type = typeVector.getAccessor().get(index); + int type = getTypeVector().getAccessor().get(index); switch (type) { - case 0: + case NULL_MARKER: return null; <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first /> <#assign fields = minor.fields!type.fields /> @@ -386,7 +549,6 @@ public class UnionVector implements ValueVector { case MinorType.${name?upper_case}_VALUE: return get${name}Vector().getAccessor().getObject(index); </#if> - </#list></#list> case MinorType.MAP_VALUE: return getMap().getAccessor().getObject(index); @@ -412,7 +574,12 @@ public class UnionVector implements ValueVector { @Override public boolean isNull(int index) { - return typeVector.getAccessor().get(index) == 0; + + // Note that type code == 0 is used to indicate a null. + // This corresponds to the LATE type, not the NULL type. + // This is presumably an artifact of an earlier implementation... + + return getTypeVector().getAccessor().get(index) == NULL_MARKER; } public int isSet(int index) { @@ -449,14 +616,12 @@ public class UnionVector implements ValueVector { break; </#if> </#list></#list> - case MAP: { + case MAP: ComplexCopier.copy(reader, writer); break; - } - case LIST: { + case LIST: ComplexCopier.copy(reader, writer); break; - } default: throw new UnsupportedOperationException(); } @@ -475,7 +640,11 @@ public class UnionVector implements ValueVector { </#list></#list> public void setType(int index, MinorType type) { - typeVector.getMutator().setSafe(index, type.getNumber()); + getTypeVector().getMutator().setSafe(index, type.getNumber()); + } + + public void setNull(int index) { + getTypeVector().getMutator().setSafe(index, NULL_MARKER); } @Override diff --git a/exec/vector/src/main/codegen/templates/UnionWriter.java b/exec/vector/src/main/codegen/templates/UnionWriter.java index 7a123b4e7..58cc4558d 100644 --- a/exec/vector/src/main/codegen/templates/UnionWriter.java +++ b/exec/vector/src/main/codegen/templates/UnionWriter.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -29,10 +29,11 @@ package org.apache.drill.exec.vector.complex.impl; /* * This class is generated using freemarker and the ${.template_name} template. */ -@SuppressWarnings("unused") + public class UnionWriter extends AbstractFieldWriter implements FieldWriter { - UnionVector data; + // Accessed by UnionReader + protected UnionVector data; private MapWriter mapWriter; private UnionListWriter listWriter; private List<BaseWriter> writers = Lists.newArrayList(); @@ -59,7 +60,6 @@ public class UnionWriter extends AbstractFieldWriter implements FieldWriter { } } - @Override public void start() { data.getMutator().setType(idx(), MinorType.MAP); @@ -145,11 +145,9 @@ public class UnionWriter extends AbstractFieldWriter implements FieldWriter { get${name}Writer().write${name}(<#list fields as field>${field.name}<#if field_has_next>, </#if></#list>); } </#if> - </#list></#list> - public void writeNull() { - } + public void writeNull() { } @Override public MapWriter map() { diff --git a/exec/vector/src/main/codegen/templates/VariableLengthVectors.java b/exec/vector/src/main/codegen/templates/VariableLengthVectors.java index a29194aa4..87dbe95a0 100644 --- a/exec/vector/src/main/codegen/templates/VariableLengthVectors.java +++ b/exec/vector/src/main/codegen/templates/VariableLengthVectors.java @@ -50,17 +50,14 @@ package org.apache.drill.exec.vector; * variable, this width is used as a guess for certain calculations.</li> * <li>The equivalent Java primitive is '${minor.javaType!type.javaType}'<li> * </ul> - * NB: this class is automatically generated from ${.template_name} and ValueVectorTypes.tdd using FreeMarker. + * NB: this class is automatically generated from <tt>${.template_name}</tt> + * and <tt>ValueVectorTypes.tdd</tt> using FreeMarker. */ public final class ${minor.class}Vector extends BaseDataValueVector implements VariableWidthVector { - private static final int DEFAULT_RECORD_BYTE_COUNT = 8; private static final int INITIAL_BYTE_COUNT = Math.min(INITIAL_VALUE_ALLOCATION * DEFAULT_RECORD_BYTE_COUNT, MAX_BUFFER_SIZE); - private static final int MIN_BYTE_COUNT = 4096; - public final static String OFFSETS_VECTOR_NAME = "$offsets$"; - - private final MaterializedField offsetsField = MaterializedField.create(OFFSETS_VECTOR_NAME, Types.required(MinorType.UINT4)); + private final UInt${type.width}Vector offsetVector = new UInt${type.width}Vector(offsetsField, allocator); private final FieldReader reader = new ${minor.class}ReaderImpl(${minor.class}Vector.this); @@ -175,6 +172,7 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V return offsetVector.getBuffer().memoryAddress(); } + @Override public UInt${type.width}Vector getOffsetVector() { return offsetVector; } diff --git a/exec/vector/src/main/java/org/apache/drill/exec/expr/fn/impl/DateUtility.java b/exec/vector/src/main/java/org/apache/drill/exec/expr/fn/impl/DateUtility.java index 992ae0361..67594fe5c 100644 --- a/exec/vector/src/main/java/org/apache/drill/exec/expr/fn/impl/DateUtility.java +++ b/exec/vector/src/main/java/org/apache/drill/exec/expr/fn/impl/DateUtility.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -18,7 +18,6 @@ package org.apache.drill.exec.expr.fn.impl; -import org.joda.time.Period; import org.joda.time.format.DateTimeFormat; import org.joda.time.format.DateTimeFormatter; import org.joda.time.format.DateTimeFormatterBuilder; @@ -26,18 +25,23 @@ import org.joda.time.format.DateTimeParser; import com.carrotsearch.hppc.ObjectIntHashMap; -// Utility class for Date, DateTime, TimeStamp, Interval data types -public class DateUtility { +/** + * Utility class for Date, DateTime, TimeStamp, Interval data types. + * <p> + * WARNING: This class is excluded from the JDBC driver. If vectors refer + * to this code, they will fail when called from JDBC. + */ +public class DateUtility { - /* We have a hashmap that stores the timezone as the key and an index as the value - * While storing the timezone in value vectors, holders we only use this index. As we - * reconstruct the timestamp, we use this index to index through the array timezoneList - * and get the corresponding timezone and pass it to joda-time - */ + /* We have a hashmap that stores the timezone as the key and an index as the value + * While storing the timezone in value vectors, holders we only use this index. As we + * reconstruct the timestamp, we use this index to index through the array timezoneList + * and get the corresponding timezone and pass it to joda-time + */ public static ObjectIntHashMap<String> timezoneMap = new ObjectIntHashMap<String>(); - public static String[] timezoneList = {"Africa/Abidjan", + public static String[] timezoneList = { "Africa/Abidjan", "Africa/Accra", "Africa/Addis_Ababa", "Africa/Algiers", @@ -612,71 +616,53 @@ public class DateUtility { "WET", "Zulu"}; - static { - for (int i = 0; i < timezoneList.length; i++) { - timezoneMap.put(timezoneList[i], i); - } + static { + for (int i = 0; i < timezoneList.length; i++) { + timezoneMap.put(timezoneList[i], i); } + } - public static final DateTimeFormatter formatDate = DateTimeFormat.forPattern("yyyy-MM-dd"); - public static final DateTimeFormatter formatTimeStamp = DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss.SSS"); - public static final DateTimeFormatter formatTimeStampTZ = DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss.SSS ZZZ"); - public static final DateTimeFormatter formatTime = DateTimeFormat.forPattern("HH:mm:ss.SSS"); + public static final DateTimeFormatter formatDate = DateTimeFormat.forPattern("yyyy-MM-dd"); + public static final DateTimeFormatter formatTimeStamp = DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss.SSS"); + public static final DateTimeFormatter formatTimeStampTZ = DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss.SSS ZZZ"); + public static final DateTimeFormatter formatTime = DateTimeFormat.forPattern("HH:mm:ss.SSS"); - public static DateTimeFormatter dateTimeTZFormat = null; - public static DateTimeFormatter timeFormat = null; + public static DateTimeFormatter dateTimeTZFormat = null; + public static DateTimeFormatter timeFormat = null; - public static final int yearsToMonths = 12; - public static final int hoursToMillis = 60 * 60 * 1000; - public static final int minutesToMillis = 60 * 1000; - public static final int secondsToMillis = 1000; - public static final int monthToStandardDays = 30; - public static final long monthsToMillis = 2592000000L; // 30 * 24 * 60 * 60 * 1000 - public static final int daysToStandardMillis = 24 * 60 * 60 * 1000; public static int getIndex(String timezone) { - return timezoneMap.get(timezone); - } + return timezoneMap.get(timezone); + } - public static String getTimeZone(int index) { - return timezoneList[index]; - } - - // Function returns the date time formatter used to parse date strings - public static DateTimeFormatter getDateTimeFormatter() { + public static String getTimeZone(int index) { + return timezoneList[index]; + } - if (dateTimeTZFormat == null) { - DateTimeFormatter dateFormatter = DateTimeFormat.forPattern("yyyy-MM-dd"); - DateTimeParser optionalTime = DateTimeFormat.forPattern(" HH:mm:ss").getParser(); - DateTimeParser optionalSec = DateTimeFormat.forPattern(".SSS").getParser(); - DateTimeParser optionalZone = DateTimeFormat.forPattern(" ZZZ").getParser(); + // Returns the date time formatter used to parse date strings + public static DateTimeFormatter getDateTimeFormatter() { - dateTimeTZFormat = new DateTimeFormatterBuilder().append(dateFormatter).appendOptional(optionalTime).appendOptional(optionalSec).appendOptional(optionalZone).toFormatter(); - } + if (dateTimeTZFormat == null) { + DateTimeFormatter dateFormatter = DateTimeFormat.forPattern("yyyy-MM-dd"); + DateTimeParser optionalTime = DateTimeFormat.forPattern(" HH:mm:ss").getParser(); + DateTimeParser optionalSec = DateTimeFormat.forPattern(".SSS").getParser(); + DateTimeParser optionalZone = DateTimeFormat.forPattern(" ZZZ").getParser(); - return dateTimeTZFormat; + dateTimeTZFormat = new DateTimeFormatterBuilder().append(dateFormatter).appendOptional(optionalTime).appendOptional(optionalSec).appendOptional(optionalZone).toFormatter(); } - // Function returns time formatter used to parse time strings - public static DateTimeFormatter getTimeFormatter() { - if (timeFormat == null) { - DateTimeFormatter timeFormatter = DateTimeFormat.forPattern("HH:mm:ss"); - DateTimeParser optionalSec = DateTimeFormat.forPattern(".SSS").getParser(); - timeFormat = new DateTimeFormatterBuilder().append(timeFormatter).appendOptional(optionalSec).toFormatter(); - } - return timeFormat; - } - - public static int monthsFromPeriod(Period period){ - return (period.getYears() * yearsToMonths) + period.getMonths(); - } + return dateTimeTZFormat; + } - public static int millisFromPeriod(final Period period){ - return (period.getHours() * hoursToMillis) + - (period.getMinutes() * minutesToMillis) + - (period.getSeconds() * secondsToMillis) + - (period.getMillis()); + // Returns time formatter used to parse time strings + public static DateTimeFormatter getTimeFormatter() { + if (timeFormat == null) { + DateTimeFormatter timeFormatter = DateTimeFormat.forPattern("HH:mm:ss"); + DateTimeParser optionalSec = DateTimeFormat.forPattern(".SSS").getParser(); + timeFormat = new DateTimeFormatterBuilder().append(timeFormatter).appendOptional(optionalSec).toFormatter(); } + return timeFormat; + } } diff --git a/exec/vector/src/main/java/org/apache/drill/exec/record/MaterializedField.java b/exec/vector/src/main/java/org/apache/drill/exec/record/MaterializedField.java index b4b23c7ed..fa4d2767e 100644 --- a/exec/vector/src/main/java/org/apache/drill/exec/record/MaterializedField.java +++ b/exec/vector/src/main/java/org/apache/drill/exec/record/MaterializedField.java @@ -26,6 +26,7 @@ import java.util.Objects; import org.apache.drill.common.types.TypeProtos.DataMode; import org.apache.drill.common.types.TypeProtos.MajorType; import org.apache.drill.common.types.TypeProtos.MinorType; +import org.apache.drill.common.types.Types; import org.apache.drill.exec.expr.BasicTypeHelper; import org.apache.drill.exec.proto.UserBitShared.NamePart; import org.apache.drill.exec.proto.UserBitShared.SerializedField; @@ -38,7 +39,7 @@ import org.apache.drill.exec.proto.UserBitShared.SerializedField; public class MaterializedField { private final String name; - private final MajorType type; + private MajorType type; // use an ordered set as existing code relies on order (e,g. parquet writer) private final LinkedHashSet<MaterializedField> children; @@ -87,13 +88,47 @@ public class MaterializedField { children.add(field); } + public void removeChild(MaterializedField field) { + children.remove(field); + } + + /** + * Replace the type with a new one that has the same minor type + * and mode, but with perhaps different details. + * <p> + * The type is immutable. But, it contains subtypes, used or lists + * and unions. To add a subtype, we must create a whole new major type. + * <p> + * It appears that the <tt>MaterializedField</tt> class was also meant + * to be immutable. But, it holds the children for a map, and contains + * methods to add children. So, it is not immutable. + * <p> + * This method allows evolving a list or union without the need to create + * a new <tt>MaterializedField</tt>. Doing so is problematic for nested + * maps because the map (or list, or union) holds onto the + * <tt>MaterializedField</tt>'s of its children. There is no way for + * an inner map to reach out and change the child of its parent. + * <p> + * By allowing the non-critical metadata to change, we preserve the + * child relationships as a list or union evolves. + * @param type + */ + + public void replaceType(MajorType newType) { + assert type.getMinorType() == newType.getMinorType(); + assert type.getMode() == newType.getMode(); + type = newType; + } + @Override public MaterializedField clone() { return withPathAndType(name, getType()); } public MaterializedField cloneEmpty() { - return create(name, type); + return create(name, type.toBuilder() + .clearSubType() + .build()); } public MaterializedField withType(MajorType type) { @@ -213,16 +248,77 @@ public class MaterializedField { // But, unset fields are equivalent to 0. Can't use the protobuf-provided // isEquals(), that treats set and unset fields as different. + if (! Types.isEquivalent(type, other.type)) { + return false; + } + + // Compare children -- but only for maps, not the internal children + // for Varchar, repeated or nullable types. + + if (type.getMinorType() != MinorType.MAP) { + return true; + } + + if (children == null || other.children == null) { + return children == other.children; + } + if (children.size() != other.children.size()) { + return false; + } + + // Maps are name-based, not position. But, for our + // purposes, we insist on identical ordering. + + Iterator<MaterializedField> thisIter = children.iterator(); + Iterator<MaterializedField> otherIter = other.children.iterator(); + while (thisIter.hasNext()) { + MaterializedField thisChild = thisIter.next(); + MaterializedField otherChild = otherIter.next(); + if (! thisChild.isEquivalent(otherChild)) { + return false; + } + } + return true; + } + + /** + * Determine if the present column schema can be promoted to the + * given schema. Promotion is possible if the schemas are + * equivalent, or if required mode is promoted to nullable, or + * if scale or precision can be increased. + * + * @param other the field to which this one is to be promoted + * @return true if promotion is possible, false otherwise + */ + + public boolean isPromotableTo(MaterializedField other, boolean allowModeChange) { + if (! name.equalsIgnoreCase(other.name)) { + return false; + } + + // Requires full type equality, including fields such as precision and scale. + // But, unset fields are equivalent to 0. Can't use the protobuf-provided + // isEquals(), that treats set and unset fields as different. + if (type.getMinorType() != other.type.getMinorType()) { return false; } if (type.getMode() != other.type.getMode()) { - return false; + + // Modes differ, but type can be promoted from required to + // nullable + + if (! allowModeChange) { + return false; + } + if (! (type.getMode() == DataMode.REQUIRED && other.type.getMode() == DataMode.OPTIONAL)) { + return false; + } } - if (type.getScale() != other.type.getScale()) { + if (type.getScale() > other.type.getScale()) { return false; } - if (type.getPrecision() != other.type.getPrecision()) { + if (type.getPrecision() > other.type.getPrecision()) { return false; } @@ -233,7 +329,7 @@ public class MaterializedField { return true; } - if (children == null || other.children == null) { + if (children == null || other.children == null) { return children == other.children; } if (children.size() != other.children.size()) { @@ -248,7 +344,7 @@ public class MaterializedField { while (thisIter.hasNext()) { MaterializedField thisChild = thisIter.next(); MaterializedField otherChild = otherIter.next(); - if (! thisChild.isEquivalent(otherChild)) { + if (! thisChild.isPromotableTo(otherChild, allowModeChange)) { return false; } } @@ -269,30 +365,45 @@ public class MaterializedField { @Override public String toString() { final int maxLen = 10; - String childString = children != null && !children.isEmpty() ? toString(children, maxLen) : ""; StringBuilder builder = new StringBuilder(); builder - .append(name) - .append("(") - .append(type.getMinorType().name()); + .append("[`") + .append(name) + .append("` (") + .append(type.getMinorType().name()); if (type.hasPrecision()) { builder.append("("); builder.append(type.getPrecision()); if (type.hasScale()) { - builder.append(","); + builder.append(", "); builder.append(type.getScale()); } builder.append(")"); } builder - .append(":") - .append(type.getMode().name()) - .append(")") - .append(childString); + .append(":") + .append(type.getMode().name()) + .append(")"); + + if (type.getSubTypeCount() > 0) { + builder + .append(", subtypes=(") + .append(type.getSubTypeList().toString()) + .append(")"); + } - return builder.toString(); + if (children != null && ! children.isEmpty()) { + builder + .append(", children=(") + .append(toString(children, maxLen)) + .append(")"); + } + + return builder + .append("]") + .toString(); } /** @@ -307,7 +418,6 @@ public class MaterializedField { private String toString(Collection<?> collection, int maxLen) { StringBuilder builder = new StringBuilder(); - builder.append(" ["); int i = 0; for (Iterator<?> iterator = collection.iterator(); iterator.hasNext() && i < maxLen; i++) { if (i > 0){ @@ -315,7 +425,6 @@ public class MaterializedField { } builder.append(iterator.next()); } - builder.append("]"); return builder.toString(); } } diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/BaseDataValueVector.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/BaseDataValueVector.java index 4391e8ce6..b27d7baec 100644 --- a/exec/vector/src/main/java/org/apache/drill/exec/vector/BaseDataValueVector.java +++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/BaseDataValueVector.java @@ -107,6 +107,9 @@ public abstract class BaseDataValueVector extends BaseValueVector { @Override public void exchange(ValueVector other) { + + // Exchange the data buffers + BaseDataValueVector target = (BaseDataValueVector) other; DrillBuf temp = data; data = target.data; diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/BaseValueVector.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/BaseValueVector.java index 63f452831..864d8e437 100644 --- a/exec/vector/src/main/java/org/apache/drill/exec/vector/BaseValueVector.java +++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/BaseValueVector.java @@ -17,20 +17,20 @@ */ package org.apache.drill.exec.vector; -import io.netty.buffer.DrillBuf; - import java.util.Collections; import java.util.Iterator; -import com.google.common.base.Preconditions; -import com.google.common.collect.Iterators; - import org.apache.drill.exec.memory.BufferAllocator; import org.apache.drill.exec.proto.UserBitShared.SerializedField; import org.apache.drill.exec.record.MaterializedField; import org.apache.drill.exec.record.TransferPair; +import com.google.common.base.Preconditions; + +import io.netty.buffer.DrillBuf; + public abstract class BaseValueVector implements ValueVector { + /** * Physical maximum allocation. This is the value prior to Drill 1.11. * This size causes memory fragmentation. Please use diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/DateUtilities.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/DateUtilities.java new file mode 100644 index 000000000..4ea460b28 --- /dev/null +++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/DateUtilities.java @@ -0,0 +1,191 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.drill.exec.vector; + +import org.joda.time.Period; + +/** + * Utility class for Date, DateTime, TimeStamp, Interval data types. + * <p> + * WARNING: This class is included from the JDBC driver. There is another, similar + * class called <tt>org.apache.drill.exec.expr.fn.impl.DateUtility</tt>. If vectors refer + * to that class, they will fail when called from JDBC. So, place code here if + * it is needed by JDBC, in the other class if only needed by the Drill engine. + * (This is a very poor design, but it is what it is.) + */ + +public class DateUtilities { + + public static final int yearsToMonths = 12; + public static final int hoursToMillis = 60 * 60 * 1000; + public static final int minutesToMillis = 60 * 1000; + public static final int secondsToMillis = 1000; + public static final int monthToStandardDays = 30; + public static final long monthsToMillis = 2592000000L; // 30 * 24 * 60 * 60 * 1000 + public static final int daysToStandardMillis = 24 * 60 * 60 * 1000; + + public static int monthsFromPeriod(Period period){ + return (period.getYears() * yearsToMonths) + period.getMonths(); + } + + public static int periodToMillis(final Period period){ + return (period.getHours() * hoursToMillis) + + (period.getMinutes() * minutesToMillis) + + (period.getSeconds() * secondsToMillis) + + (period.getMillis()); + } + + public static int toMonths(int years, int months) { + return years * yearsToMonths + months; + } + + public static int periodToMonths(Period value) { + return value.getYears() * yearsToMonths + value.getMonths(); + } + + public static Period fromIntervalYear(int value) { + final int years = (value / yearsToMonths); + final int months = (value % yearsToMonths); + return new Period() + .plusYears(years) + .plusMonths(months); + } + + public static StringBuilder intervalYearStringBuilder(int months) { + final int years = months / yearsToMonths; + months %= yearsToMonths; + + return new StringBuilder() + .append(years) + .append(pluralify("year", years)) + .append(" ") + .append(months) + .append(pluralify("month", months)); + } + + public static StringBuilder intervalYearStringBuilder(Period value) { + return intervalYearStringBuilder( + value.getYears() * 12 + value.getMonths()); + } + + public static String pluralify(String term, int value) { + term = (Math.abs(value) == 1) ? term : term + "s"; + return " " + term; + } + + public static Period fromIntervalDay(int days, int millis) { + return new Period() + .plusDays(days) + .plusMillis(millis); + } + + public static StringBuilder intervalDayStringBuilder(int days, int millis) { + + final int hours = millis / (hoursToMillis); + millis %= (hoursToMillis); + + final int minutes = millis / (minutesToMillis); + millis %= (minutesToMillis); + + final int seconds = millis / (secondsToMillis); + millis %= (secondsToMillis); + + StringBuilder buf = new StringBuilder() + .append(days) + .append(pluralify("day", days)) + .append(" ") + .append(hours) + .append(":") + .append(asTwoDigits(minutes)) + .append(":") + .append(asTwoDigits(seconds)); + if (millis != 0) { + buf.append(".") + .append(millis); + } + return buf; + } + + public static StringBuilder intervalDayStringBuilder(Period value) { + return intervalDayStringBuilder( + value.getDays(), + periodToMillis(value)); + } + + public static Period fromInterval(int months, int days, int millis) { + return new Period() + .plusMonths(months) + .plusDays(days) + .plusMillis(millis); + } + + public static String asTwoDigits(int value) { + return String.format("%02d", value); + } + + public static StringBuilder intervalStringBuilder(int months, int days, int millis) { + + final int years = months / yearsToMonths; + months %= yearsToMonths; + + final int hours = millis / hoursToMillis; + millis %= hoursToMillis; + + final int minutes = millis / minutesToMillis; + millis %= minutesToMillis; + + final int seconds = millis / secondsToMillis; + millis %= secondsToMillis; + + StringBuilder buf = new StringBuilder() + .append(years) + .append(pluralify("year", years)) + .append(" ") + .append(months) + .append(pluralify("month", months)) + .append(" ") + .append(days) + .append(pluralify("day", days)) + .append(" ") + .append(hours) + .append(":") + .append(asTwoDigits(minutes)) + .append(":") + .append(asTwoDigits(seconds)); + if (millis != 0) { + buf.append(".") + .append(millis); + } + return buf; + } + + public static StringBuilder intervalStringBuilder(Period value) { + return intervalStringBuilder( + value.getYears() * 12 + value.getMonths(), + value.getDays(), + periodToMillis(value)); + } + + public static int timeToMillis(int hours, int minutes, int seconds, int millis) { + return ((hours * 60 + + minutes) * 60 + + seconds) * 1000 + + millis; + } +} diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/NullableVector.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/NullableVector.java index 51b5e0c48..80b732ac4 100644 --- a/exec/vector/src/main/java/org/apache/drill/exec/vector/NullableVector.java +++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/NullableVector.java @@ -17,7 +17,13 @@ */ package org.apache.drill.exec.vector; -public interface NullableVector extends ValueVector{ +import org.apache.drill.common.types.TypeProtos.MinorType; +import org.apache.drill.common.types.Types; +import org.apache.drill.exec.record.MaterializedField; + +public interface NullableVector extends ValueVector { + + MaterializedField bitsField = MaterializedField.create(BITS_VECTOR_NAME, Types.required(MinorType.UINT1)); ValueVector getBitsVector(); ValueVector getValuesVector(); diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/ValueVector.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/ValueVector.java index bc0680388..44a467e60 100644 --- a/exec/vector/src/main/java/org/apache/drill/exec/vector/ValueVector.java +++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/ValueVector.java @@ -20,7 +20,6 @@ package org.apache.drill.exec.vector; import java.io.Closeable; import java.util.Set; -import com.google.common.annotations.VisibleForTesting; import io.netty.buffer.DrillBuf; import org.apache.drill.exec.exception.OutOfMemoryException; @@ -86,6 +85,12 @@ public interface ValueVector extends Closeable, Iterable<ValueVector> { int MAX_ROW_COUNT = Character.MAX_VALUE + 1; + // Commonly-used internal vector names + + String BITS_VECTOR_NAME = "$bits$"; + String OFFSETS_VECTOR_NAME = "$offsets$"; + String VALUES_VECTOR_NAME = "$values$"; + /** * Allocate new buffers. ValueVector implements logic to determine how much to allocate. * @throws OutOfMemoryException Thrown if no memory can be allocated. diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/VariableWidthVector.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/VariableWidthVector.java index f5373d060..1e7741741 100644 --- a/exec/vector/src/main/java/org/apache/drill/exec/vector/VariableWidthVector.java +++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/VariableWidthVector.java @@ -17,8 +17,24 @@ */ package org.apache.drill.exec.vector; +import org.apache.drill.common.types.TypeProtos.MinorType; +import org.apache.drill.common.types.Types; +import org.apache.drill.exec.record.MaterializedField; + public interface VariableWidthVector extends ValueVector { + int DEFAULT_RECORD_BYTE_COUNT = 8; + int MIN_BYTE_COUNT = 4096; + MaterializedField offsetsField = MaterializedField.create(OFFSETS_VECTOR_NAME, Types.required(MinorType.UINT4)); + + interface VariableWidthAccessor extends Accessor { + int getValueLength(int index); + } + + interface VariableWidthMutator extends Mutator { + void setValueLengthSafe(int index, int length); + } + /** * Allocate a new memory space for this vector. Must be called prior to using the ValueVector. * @@ -39,13 +55,7 @@ public interface VariableWidthVector extends ValueVector { @Override VariableWidthAccessor getAccessor(); - interface VariableWidthAccessor extends Accessor { - int getValueLength(int index); - } - int getCurrentSizeInBytes(); - interface VariableWidthMutator extends Mutator { - void setValueLengthSafe(int index, int length); - } + UInt4Vector getOffsetVector(); } diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/AbstractMapVector.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/AbstractMapVector.java index 5ac28c54f..5515b7ad2 100644 --- a/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/AbstractMapVector.java +++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/AbstractMapVector.java @@ -36,7 +36,7 @@ import org.apache.drill.exec.vector.ValueVector; import com.google.common.base.Preconditions; import com.google.common.collect.Lists; -/* +/** * Base class for MapVectors. Currently used by RepeatedMapVector and MapVector */ public abstract class AbstractMapVector extends AbstractContainerVector { @@ -47,14 +47,14 @@ public abstract class AbstractMapVector extends AbstractContainerVector { protected AbstractMapVector(MaterializedField field, BufferAllocator allocator, CallBack callBack) { super(field.clone(), allocator, callBack); - MaterializedField clonedField = field.clone(); // create the hierarchy of the child vectors based on the materialized field - for (MaterializedField child : clonedField.getChildren()) { - if (!child.equals(BaseRepeatedValueVector.OFFSETS_FIELD)) { - final String fieldName = child.getName(); - final ValueVector v = BasicTypeHelper.getNewVector(child, allocator, callBack); - putVector(fieldName, v); + for (MaterializedField child : field.getChildren()) { + if (child.getName().equals(BaseRepeatedValueVector.OFFSETS_FIELD.getName())) { + continue; } + final String fieldName = child.getName(); + final ValueVector v = BasicTypeHelper.getNewVector(child, allocator, callBack); + putVector(fieldName, v); } } @@ -77,13 +77,13 @@ public abstract class AbstractMapVector extends AbstractContainerVector { boolean success = false; try { for (final ValueVector v : vectors.values()) { - if (!v.allocateNewSafe()) { + if (! v.allocateNewSafe()) { return false; } } success = true; } finally { - if (!success) { + if (! success) { clear(); } } @@ -145,7 +145,7 @@ public abstract class AbstractMapVector extends AbstractContainerVector { private boolean nullFilled(ValueVector vector) { for (int r = 0; r < vector.getAccessor().getValueCount(); r++) { - if (!vector.getAccessor().isNull(r)) { + if (! vector.getAccessor().isNull(r)) { return false; } } @@ -304,4 +304,17 @@ public abstract class AbstractMapVector extends AbstractContainerVector { } return count; } + + @Override + public void exchange(ValueVector other) { + AbstractMapVector otherMap = (AbstractMapVector) other; + if (vectors.size() != otherMap.vectors.size()) { + throw new IllegalStateException("Maps have different column counts"); + } + for (int i = 0; i < vectors.size(); i++) { + assert vectors.getByOrdinal(i).getField().isEquivalent( + otherMap.vectors.getByOrdinal(i).getField()); + vectors.getByOrdinal(i).exchange(otherMap.vectors.getByOrdinal(i)); + } + } } diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/BaseRepeatedValueVector.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/BaseRepeatedValueVector.java index 8472f80ec..4b0c1b57c 100644 --- a/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/BaseRepeatedValueVector.java +++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/BaseRepeatedValueVector.java @@ -85,7 +85,6 @@ public abstract class BaseRepeatedValueVector extends BaseValueVector implements return success; } - @Override public UInt4Vector getOffsetVector() { return offsets; } diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java index 4a501b871..9a7e8475c 100644 --- a/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java +++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java @@ -49,7 +49,6 @@ import com.google.common.collect.Ordering; import com.google.common.primitives.Ints; public class MapVector extends AbstractMapVector { - //private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(MapVector.class); public final static MajorType TYPE = Types.required(MinorType.MAP); @@ -58,11 +57,11 @@ public class MapVector extends AbstractMapVector { private final Mutator mutator = new Mutator(); private int valueCount; - public MapVector(String path, BufferAllocator allocator, CallBack callBack){ + public MapVector(String path, BufferAllocator allocator, CallBack callBack) { this(MaterializedField.create(path, TYPE), allocator, callBack); } - public MapVector(MaterializedField field, BufferAllocator allocator, CallBack callBack){ + public MapVector(MaterializedField field, BufferAllocator allocator, CallBack callBack) { super(field, allocator, callBack); } @@ -73,14 +72,14 @@ public class MapVector extends AbstractMapVector { transient private MapSingleCopier ephPair2; public void copyFromSafe(int fromIndex, int thisIndex, MapVector from) { - if(ephPair == null || ephPair.from != from) { + if (ephPair == null || ephPair.from != from) { ephPair = (MapTransferPair) from.makeTransferPair(this); } ephPair.copyValueSafe(fromIndex, thisIndex); } public void copyFromSafe(int fromSubIndex, int thisIndex, RepeatedMapVector from) { - if(ephPair2 == null || ephPair2.from != from) { + if (ephPair2 == null || ephPair2.from != from) { ephPair2 = from.makeSingularCopier(this); } ephPair2.copySafe(fromSubIndex, thisIndex); @@ -143,9 +142,6 @@ public class MapVector extends AbstractMapVector { @Override public DrillBuf[] getBuffers(boolean clear) { - //int expectedSize = getBufferSize(); - //int actualSize = super.getBufferSize(); - //Preconditions.checkArgument(expectedSize == actualSize); return super.getBuffers(clear); } @@ -294,9 +290,9 @@ public class MapVector extends AbstractMapVector { @Override public SerializedField getMetadata() { - SerializedField.Builder b = getField() // - .getAsBuilder() // - .setBufferLength(getBufferSize()) // + SerializedField.Builder b = getField() + .getAsBuilder() + .setBufferLength(getBufferSize()) .setValueCount(valueCount); @@ -311,13 +307,6 @@ public class MapVector extends AbstractMapVector { return mutator; } - @Override - public void exchange(ValueVector other) { - // Exchange is used for look-ahead writers, but writers manage - // map member vectors directly. - throw new UnsupportedOperationException("Exchange() not supported for maps"); - } - public class Accessor extends BaseValueVector.BaseAccessor { @Override @@ -357,6 +346,14 @@ public class MapVector extends AbstractMapVector { return getChildByOrdinal(id); } + /** + * Set the value count for the map without setting the counts for the contained + * vectors. Use this only when the values of the contained vectors are set + * elsewhere in the code. + * + * @param valueCount number of items in the map + */ + public void setMapValueCount(int valueCount) { this.valueCount = valueCount; } @@ -402,4 +399,13 @@ public class MapVector extends AbstractMapVector { public void toNullable(ValueVector nullableVector) { throw new UnsupportedOperationException(); } + + @Override + public void exchange(ValueVector other) { + super.exchange(other); + MapVector otherMap = (MapVector) other; + int temp = otherMap.valueCount; + otherMap.valueCount = valueCount; + valueCount = temp; + } } diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java index 57f1a679b..270f973e4 100644 --- a/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java +++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java @@ -64,9 +64,7 @@ public class RepeatedMapVector extends AbstractMapVector private final EmptyValuePopulator emptyPopulator; public RepeatedMapVector(MaterializedField field, BufferAllocator allocator, CallBack callBack) { - super(field, allocator, callBack); - this.offsets = new UInt4Vector(BaseRepeatedValueVector.OFFSETS_FIELD, allocator); - this.emptyPopulator = new EmptyValuePopulator(offsets); + this(field, new UInt4Vector(BaseRepeatedValueVector.OFFSETS_FIELD, allocator), callBack); } public RepeatedMapVector(MaterializedField field, UInt4Vector offsets, CallBack callBack) { @@ -150,7 +148,7 @@ public class RepeatedMapVector extends AbstractMapVector } long bufferSize = offsets.getBufferSizeFor(valueCount); - for (final ValueVector v : (Iterable<ValueVector>) this) { + for (final ValueVector v : this) { bufferSize += v.getBufferSizeFor(valueCount); } @@ -424,9 +422,8 @@ public class RepeatedMapVector extends AbstractMapVector @Override public void exchange(ValueVector other) { - // Exchange is used for look-ahead writers, but writers manage - // map member vectors directly. - throw new UnsupportedOperationException("Exchange() not supported for maps"); + super.exchange(other); + offsets.exchange(((RepeatedMapVector) other).offsets); } @Override @@ -459,13 +456,13 @@ public class RepeatedMapVector extends AbstractMapVector assert bufOffset == buffer.writerIndex(); } - @Override public SerializedField getMetadata() { - SerializedField.Builder builder = getField() // - .getAsBuilder() // - .setBufferLength(getBufferSize()) // - // while we don't need to actually read this on load, we need it to make sure we don't skip deserialization of this vector + SerializedField.Builder builder = getField() + .getAsBuilder() + .setBufferLength(getBufferSize()) + // while we don't need to actually read this on load, we need it to + // make sure we don't skip deserialization of this vector .setValueCount(accessor.getValueCount()); builder.addChild(offsets.getMetadata()); for (final ValueVector child : getChildren()) { diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/RepeatedValueVector.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/RepeatedValueVector.java index 0fba29265..4bcfba6a7 100644 --- a/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/RepeatedValueVector.java +++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/RepeatedValueVector.java @@ -35,9 +35,8 @@ public interface RepeatedValueVector extends ValueVector, ContainerVectorLike { /** * Returns the underlying offset vector or null if none exists. - * - * TODO(DRILL-2995): eliminate exposing low-level interfaces. */ + UInt4Vector getOffsetVector(); /** diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/StateTool.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/StateTool.java index f5ed3a019..9a736d35c 100644 --- a/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/StateTool.java +++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/StateTool.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -22,13 +22,15 @@ import java.util.Arrays; public class StateTool { static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(StateTool.class); + @SuppressWarnings("unchecked") public static <T extends Enum<?>> void check(T currentState, T... expectedStates) { for (T s : expectedStates) { if (s == currentState) { return; } } - throw new IllegalArgumentException(String.format("Expected to be in one of these states %s but was actuall in state %s", Arrays.toString(expectedStates), currentState)); + throw new IllegalArgumentException( + String.format("Expected to be in one of these states %s but was actually in state %s", + Arrays.toString(expectedStates), currentState)); } - } diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/impl/PromotableWriter.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/impl/PromotableWriter.java index 10ac551ff..28e90b942 100644 --- a/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/impl/PromotableWriter.java +++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/impl/PromotableWriter.java @@ -120,6 +120,7 @@ public class PromotableWriter extends AbstractPromotableFieldWriter { } } + @Override protected FieldWriter getWriter(MinorType type) { if (state == State.UNION) { return writer; @@ -144,6 +145,7 @@ public class PromotableWriter extends AbstractPromotableFieldWriter { return writer.isEmptyMap(); } + @Override protected FieldWriter getWriter() { return getWriter(type); } |