diff options
author | Volodymyr Vysotskyi <vvovyk@gmail.com> | 2018-05-16 21:34:56 +0300 |
---|---|---|
committer | Arina Ielchiieva <arina.yelchiyeva@gmail.com> | 2018-05-18 15:42:19 +0300 |
commit | bdbd93fc726224ec45d06f09434c7406e444242e (patch) | |
tree | a7830bd242f29824d789833d91b63e1c73801e9a /exec/vector | |
parent | 5dd8a6f60e006c2dc707f241b7619634e4e82bbd (diff) |
DRILL-6421: Refactor DecimalUtility and CoreDecimalUtility classes
closes #1267
Diffstat (limited to 'exec/vector')
4 files changed, 119 insertions, 577 deletions
diff --git a/exec/vector/src/main/codegen/templates/ColumnAccessors.java b/exec/vector/src/main/codegen/templates/ColumnAccessors.java index accf1b28d..15349a28e 100644 --- a/exec/vector/src/main/codegen/templates/ColumnAccessors.java +++ b/exec/vector/src/main/codegen/templates/ColumnAccessors.java @@ -178,13 +178,11 @@ public class ColumnAccessors { <#elseif drillType == "Decimal9"> return DecimalUtility.getBigDecimalFromPrimitiveTypes( buf.getInt(${getOffset}), - type.getScale(), - type.getPrecision()); + type.getScale()); <#elseif drillType == "Decimal18"> return DecimalUtility.getBigDecimalFromPrimitiveTypes( buf.getLong(${getOffset}), - type.getScale(), - type.getPrecision()); + type.getScale()); <#elseif drillType == "IntervalYear"> return DateUtilities.fromIntervalYear( buf.getInt(${getOffset})); @@ -296,21 +294,21 @@ public class ColumnAccessors { <#elseif drillType == "Decimal9"> drillBuf.setInt(${putOffset}, DecimalUtility.getDecimal9FromBigDecimal(value, - type.getScale(), type.getPrecision())); + type.getScale())); <#elseif drillType == "Decimal18"> drillBuf.setLong(${putOffset}, DecimalUtility.getDecimal18FromBigDecimal(value, - type.getScale(), type.getPrecision())); + type.getScale())); <#elseif drillType == "Decimal38Sparse"> <#-- Hard to optimize this case. Just use the available tools. --> DecimalUtility.getSparseFromBigDecimal(value, drillBuf, ${putOffset}, - type.getScale(), type.getPrecision(), 6); + type.getScale(), 6); <#elseif drillType == "Decimal28Sparse"> <#-- Hard to optimize this case. Just use the available tools. --> DecimalUtility.getSparseFromBigDecimal(value, drillBuf, ${putOffset}, - type.getScale(), type.getPrecision(), 5); + type.getScale(), 5); <#elseif drillType == "IntervalYear"> drillBuf.setInt(${putOffset}, value.getYears() * 12 + value.getMonths()); diff --git a/exec/vector/src/main/codegen/templates/FixedValueVectors.java b/exec/vector/src/main/codegen/templates/FixedValueVectors.java index 4a98c2608..eb2c93d5f 100644 --- a/exec/vector/src/main/codegen/templates/FixedValueVectors.java +++ b/exec/vector/src/main/codegen/templates/FixedValueVectors.java @@ -699,7 +699,7 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F public void set(int index, BigDecimal value) { DecimalUtility.getSparseFromBigDecimal(value, data, index * VALUE_WIDTH, - field.getScale(), field.getPrecision(), ${minor.nDecimalDigits}); + field.getScale(), ${minor.nDecimalDigits}); } public void setSafe(int index, BigDecimal value) { diff --git a/exec/vector/src/main/java/org/apache/drill/exec/util/DecimalUtility.java b/exec/vector/src/main/java/org/apache/drill/exec/util/DecimalUtility.java index a451b974f..f0a06d85b 100644 --- a/exec/vector/src/main/java/org/apache/drill/exec/util/DecimalUtility.java +++ b/exec/vector/src/main/java/org/apache/drill/exec/util/DecimalUtility.java @@ -25,142 +25,40 @@ import io.netty.buffer.UnpooledByteBufAllocator; import java.math.BigDecimal; import java.math.BigInteger; import java.math.RoundingMode; -import java.nio.ByteBuffer; -import java.util.Arrays; import org.apache.drill.common.types.TypeProtos; -import org.apache.drill.common.util.CoreDecimalUtility; -import org.apache.drill.exec.expr.fn.impl.ByteFunctionHelpers; -import org.apache.drill.exec.expr.holders.Decimal38SparseHolder; -public class DecimalUtility extends CoreDecimalUtility { +@SuppressWarnings("WeakerAccess") +public class DecimalUtility { public final static int MAX_DIGITS = 9; public final static int MAX_DIGITS_INT = 10; public final static int MAX_DIGITS_BIGINT = 19; public final static int DIGITS_BASE = 1000000000; - public final static int DIGITS_MAX = 999999999; - public final static int INTEGER_SIZE = (Integer.SIZE/8); - - public final static String[] decimalToString = {"", - "0", - "00", - "000", - "0000", - "00000", - "000000", - "0000000", - "00000000", - "000000000"}; - - public final static long[] scale_long_constants = { - 1, - 10, - 100, - 1000, - 10000, - 100000, - 1000000, - 10000000, - 100000000, - 1000000000, - 10000000000l, - 100000000000l, - 1000000000000l, - 10000000000000l, - 100000000000000l, - 1000000000000000l, - 10000000000000000l, - 100000000000000000l, - 1000000000000000000l}; - - /* - * Simple function that returns the static precomputed - * power of ten, instead of using Math.pow - */ - public static long getPowerOfTen(int power) { - assert power >= 0 && power < scale_long_constants.length; - return scale_long_constants[(power)]; - } - - /* - * Math.pow returns a double and while multiplying with large digits - * in the decimal data type we encounter noise. So instead of multiplying - * with Math.pow we use the static constants to perform the multiplication - */ - public static long adjustScaleMultiply(long input, int factor) { - int index = Math.abs(factor); - assert index >= 0 && index < scale_long_constants.length; - if (factor >= 0) { - return input * scale_long_constants[index]; - } else { - return input / scale_long_constants[index]; - } - } - - public static long adjustScaleDivide(long input, int factor) { - int index = Math.abs(factor); - assert index >= 0 && index < scale_long_constants.length; - if (factor >= 0) { - return input / scale_long_constants[index]; - } else { - return input * scale_long_constants[index]; - } - } + public final static int INTEGER_SIZE = Integer.SIZE / 8; - /* Given the number of actual digits this function returns the + /** + * Given the number of actual digits this function returns the * number of indexes it will occupy in the array of integers * which are stored in base 1 billion */ public static int roundUp(int ndigits) { - return (ndigits + MAX_DIGITS - 1)/MAX_DIGITS; + return (ndigits + MAX_DIGITS - 1) / MAX_DIGITS; } - /** Returns a string representation of the given integer - * If the length of the given integer is less than the - * passed length, this function will prepend zeroes to the string + /** + * Create a BigDecimal object using the data in the DrillBuf. + * This function assumes that data is provided in a sparse format. */ - public static StringBuilder toStringWithZeroes(int number, int desiredLength) { - String value = ((Integer) number).toString(); - int length = value.length(); - - StringBuilder str = new StringBuilder(); - str.append(decimalToString[desiredLength - length]); - str.append(value); - - return str; - } - - public static StringBuilder toStringWithZeroes(long number, int desiredLength) { - String value = ((Long) number).toString(); - int length = value.length(); - - StringBuilder str = new StringBuilder(); - - // Desired length can be > MAX_DIGITS - int zeroesLength = desiredLength - length; - while (zeroesLength > MAX_DIGITS) { - str.append(decimalToString[MAX_DIGITS]); - zeroesLength -= MAX_DIGITS; - } - str.append(decimalToString[zeroesLength]); - str.append(value); - - return str; - } - - public static BigDecimal getBigDecimalFromIntermediate(ByteBuf data, int startIndex, int nDecimalDigits, int scale) { - - // In the intermediate representation we don't pad the scale with zeroes, so set truncate = false - return getBigDecimalFromDrillBuf(data, startIndex, nDecimalDigits, scale, false); - } - public static BigDecimal getBigDecimalFromSparse(DrillBuf data, int startIndex, int nDecimalDigits, int scale) { - // In the sparse representation we pad the scale with zeroes for ease of arithmetic, need to truncate return getBigDecimalFromDrillBuf(data, startIndex, nDecimalDigits, scale, true); } + /** + * Create a BigDecimal object using the data in the DrillBuf. + * This function assumes that data is provided in format supported by {@link BigInteger}. + */ public static BigDecimal getBigDecimalFromDrillBuf(DrillBuf bytebuf, int start, int length, int scale) { byte[] value = new byte[length]; bytebuf.getBytes(start, value, 0, length); @@ -168,14 +66,8 @@ public class DecimalUtility extends CoreDecimalUtility { return new BigDecimal(unscaledValue, scale); } - public static BigDecimal getBigDecimalFromByteBuffer(ByteBuffer bytebuf, int start, int length, int scale) { - byte[] value = new byte[length]; - bytebuf.get(value); - BigInteger unscaledValue = new BigInteger(value); - return new BigDecimal(unscaledValue, scale); - } - - /** Create a BigDecimal object using the data in the DrillBuf. + /** + * Create a BigDecimal object using the data in the DrillBuf. * This function assumes that data is provided in a non-dense format * It works on both sparse and intermediate representations. */ @@ -183,48 +75,47 @@ public class DecimalUtility extends CoreDecimalUtility { boolean truncateScale) { // For sparse decimal type we have padded zeroes at the end, strip them while converting to BigDecimal. - int actualDigits; + int actualDigits = scale % MAX_DIGITS; // Initialize the BigDecimal, first digit in the DrillBuf has the sign so mask it out - BigInteger decimalDigits = BigInteger.valueOf((data.getInt(startIndex)) & 0x7FFFFFFF); + BigInteger decimalDigits = BigInteger.valueOf(data.getInt(startIndex) & 0x7FFFFFFF); BigInteger base = BigInteger.valueOf(DIGITS_BASE); for (int i = 1; i < nDecimalDigits; i++) { - - BigInteger temp = BigInteger.valueOf(data.getInt(startIndex + (i * INTEGER_SIZE))); - decimalDigits = decimalDigits.multiply(base); - decimalDigits = decimalDigits.add(temp); + BigInteger temp = BigInteger.valueOf(data.getInt(startIndex + i * INTEGER_SIZE)); + decimalDigits = decimalDigits.multiply(base); + decimalDigits = decimalDigits.add(temp); } // Truncate any additional padding we might have added - if (truncateScale == true && scale > 0 && (actualDigits = scale % MAX_DIGITS) != 0) { - BigInteger truncate = BigInteger.valueOf((int)Math.pow(10, (MAX_DIGITS - actualDigits))); - decimalDigits = decimalDigits.divide(truncate); + if (truncateScale && scale > 0 && actualDigits != 0) { + BigInteger truncate = BigInteger.valueOf((int) Math.pow(10, MAX_DIGITS - actualDigits)); + decimalDigits = decimalDigits.divide(truncate); } // set the sign if ((data.getInt(startIndex) & 0x80000000) != 0) { - decimalDigits = decimalDigits.negate(); + decimalDigits = decimalDigits.negate(); } - BigDecimal decimal = new BigDecimal(decimalDigits, scale); - - return decimal; + return new BigDecimal(decimalDigits, scale); } - /* This function returns a BigDecimal object from the dense decimal representation. + /** + * Returns a BigDecimal object from the dense decimal representation. * First step is to convert the dense representation into an intermediate representation * and then invoke getBigDecimalFromDrillBuf() to get the BigDecimal object */ - public static BigDecimal getBigDecimalFromDense(DrillBuf data, int startIndex, int nDecimalDigits, int scale, int maxPrecision, int width) { + public static BigDecimal getBigDecimalFromDense(DrillBuf data, int startIndex, int nDecimalDigits, + int scale, int maxPrecision, int width) { /* This method converts the dense representation to * an intermediate representation. The intermediate * representation has one more integer than the dense * representation. */ - byte[] intermediateBytes = new byte[((nDecimalDigits + 1) * INTEGER_SIZE)]; + byte[] intermediateBytes = new byte[(nDecimalDigits + 1) * INTEGER_SIZE]; // Start storing from the least significant byte of the first integer int intermediateIndex = 3; @@ -236,80 +127,72 @@ public class DecimalUtility extends CoreDecimalUtility { int shiftOrder; byte shiftBits; - // TODO: Some of the logic here is common with casting from Dense to Sparse types, factor out common code if (maxPrecision == 38) { - maskIndex = 0; - shiftOrder = 6; - shiftBits = 0x00; - intermediateBytes[intermediateIndex++] = (byte) (data.getByte(startIndex) & 0x7F); + maskIndex = 0; + shiftOrder = 6; + shiftBits = 0x00; + intermediateBytes[intermediateIndex++] = (byte) (data.getByte(startIndex) & 0x7F); } else if (maxPrecision == 28) { - maskIndex = 1; - shiftOrder = 4; - shiftBits = (byte) ((data.getByte(startIndex) & 0x03) << shiftOrder); - intermediateBytes[intermediateIndex++] = (byte) (((data.getByte(startIndex) & 0x3C) & 0xFF) >>> 2); + maskIndex = 1; + shiftOrder = 4; + shiftBits = (byte) ((data.getByte(startIndex) & 0x03) << shiftOrder); + intermediateBytes[intermediateIndex++] = (byte) (((data.getByte(startIndex) & 0x3C) & 0xFF) >>> 2); } else { - throw new UnsupportedOperationException("Dense types with max precision 38 and 28 are only supported"); + throw new UnsupportedOperationException("Dense types with max precision 38 and 28 are only supported"); } int inputIndex = 1; boolean sign = false; if ((data.getByte(startIndex) & 0x80) != 0) { - sign = true; + sign = true; } while (inputIndex < width) { - intermediateBytes[intermediateIndex] = (byte) ((shiftBits) | (((data.getByte(startIndex + inputIndex) & reverseMask[maskIndex]) & 0xFF) >>> (8 - shiftOrder))); - shiftBits = (byte) ((data.getByte(startIndex + inputIndex) & mask[maskIndex]) << shiftOrder); inputIndex++; intermediateIndex++; if (((inputIndex - 1) % INTEGER_SIZE) == 0) { - shiftBits = (byte) ((shiftBits & 0xFF) >>> 2); - maskIndex++; - shiftOrder -= 2; + shiftBits = (byte) ((shiftBits & 0xFF) >>> 2); + maskIndex++; + shiftOrder -= 2; } - } /* copy the last byte */ intermediateBytes[intermediateIndex] = shiftBits; - if (sign == true) { - intermediateBytes[0] = (byte) (intermediateBytes[0] | 0x80); + if (sign) { + intermediateBytes[0] = (byte) (intermediateBytes[0] | 0x80); } final ByteBuf intermediate = UnpooledByteBufAllocator.DEFAULT.buffer(intermediateBytes.length); try { - intermediate.setBytes(0, intermediateBytes); + intermediate.setBytes(0, intermediateBytes); - BigDecimal ret = getBigDecimalFromIntermediate(intermediate, 0, nDecimalDigits + 1, scale); - return ret; + // In the intermediate representation we don't pad the scale with zeroes, so set truncate = false + return getBigDecimalFromDrillBuf(intermediate, 0, nDecimalDigits + 1, scale, false); } finally { intermediate.release(); } - } /** * Function converts the BigDecimal and stores it in out internal sparse representation */ - public static void getSparseFromBigDecimal(BigDecimal input, ByteBuf data, int startIndex, int scale, int precision, - int nDecimalDigits) { + public static void getSparseFromBigDecimal(BigDecimal input, ByteBuf data, int startIndex, int scale, int nDecimalDigits) { // Initialize the buffer - for (int i = 0; i < nDecimalDigits; i++) { - data.setInt(startIndex + (i * INTEGER_SIZE), 0); - } + data.setZero(startIndex, nDecimalDigits * INTEGER_SIZE); boolean sign = false; if (input.signum() == -1) { - // negative input - sign = true; - input = input.abs(); + // negative input + sign = true; + input = input.abs(); } // Truncate the input as per the scale provided @@ -320,25 +203,25 @@ public class DecimalUtility extends CoreDecimalUtility { int destIndex = nDecimalDigits - roundUp(scale) - 1; - // we use base 1 billion integer digits for out integernal representation + // we use base 1 billion integer digits for out internal representation BigDecimal base = new BigDecimal(DIGITS_BASE); while (integerPart.compareTo(BigDecimal.ZERO) == 1) { - // store the modulo as the integer value - data.setInt(startIndex + (destIndex * INTEGER_SIZE), (integerPart.remainder(base)).intValue()); - destIndex--; - // Divide by base 1 billion - integerPart = (integerPart.divide(base)).setScale(0, BigDecimal.ROUND_DOWN); + // store the modulo as the integer value + data.setInt(startIndex + destIndex * INTEGER_SIZE, integerPart.remainder(base).intValue()); + destIndex--; + // Divide by base 1 billion + integerPart = integerPart.divide(base, BigDecimal.ROUND_DOWN).setScale(0, BigDecimal.ROUND_DOWN); } /* Sparse representation contains padding of additional zeroes * so each digit contains MAX_DIGITS for ease of arithmetic */ - int actualDigits; - if ((actualDigits = (scale % MAX_DIGITS)) != 0) { - // Pad additional zeroes - scale = scale + (MAX_DIGITS - actualDigits); - input = input.setScale(scale, BigDecimal.ROUND_DOWN); + int actualDigits = scale % MAX_DIGITS; + if (actualDigits != 0) { + // Pad additional zeroes + scale = scale + MAX_DIGITS - actualDigits; + input = input.setScale(scale, BigDecimal.ROUND_DOWN); } //separate out the fractional part @@ -347,15 +230,15 @@ public class DecimalUtility extends CoreDecimalUtility { destIndex = nDecimalDigits - 1; while (scale > 0) { - // Get next set of MAX_DIGITS (9) store it in the DrillBuf - fractionalPart = fractionalPart.movePointLeft(MAX_DIGITS); - BigDecimal temp = fractionalPart.remainder(BigDecimal.ONE); + // Get next set of MAX_DIGITS (9) store it in the DrillBuf + fractionalPart = fractionalPart.movePointLeft(MAX_DIGITS); + BigDecimal temp = fractionalPart.remainder(BigDecimal.ONE); - data.setInt(startIndex + (destIndex * INTEGER_SIZE), (temp.unscaledValue().intValue())); - destIndex--; + data.setInt(startIndex + destIndex * INTEGER_SIZE, temp.unscaledValue().intValue()); + destIndex--; - fractionalPart = fractionalPart.setScale(0, BigDecimal.ROUND_DOWN); - scale -= MAX_DIGITS; + fractionalPart = fractionalPart.setScale(0, BigDecimal.ROUND_DOWN); + scale -= MAX_DIGITS; } // Set the negative sign @@ -365,93 +248,55 @@ public class DecimalUtility extends CoreDecimalUtility { } /** - * Converts from an input BigDecimal into varying width "VarDecimal" representation. - * The object that manages the "data" is assumed to already have the proper scale set, - * matching that of input.scale(). - * @param input input decimal number to be stored - * @param data destination buffer to store the byte array representation of input - * @param startIndex starting index in data to hold the bytes - * @return startIndex + length of bytes stored (i.e., the next startIndex in the data buffer) + * Returns unsigned long value taken from specified {@link BigDecimal} input with specified scale + * + * @param input {@link BigDecimal} with desired value + * @param scale scale of the value + * @return long value taken from specified {@link BigDecimal} */ - public static int getVarDecimalFromBigDecimal(BigDecimal input, ByteBuf data, int startIndex) { - byte[] bytes = input.unscaledValue().toByteArray(); - int len = bytes.length; - data.setBytes(startIndex, bytes); - return startIndex + len; - } - - public static long getDecimal18FromBigDecimal(BigDecimal input, int scale, int precision) { + public static long getDecimal18FromBigDecimal(BigDecimal input, int scale) { // Truncate or pad to set the input to the correct scale input = input.setScale(scale, BigDecimal.ROUND_HALF_UP); return input.unscaledValue().longValue(); } - public static int getDecimal9FromBigDecimal(BigDecimal input, int scale, int precision) { + /** + * Returns unsigned int value taken from specified {@link BigDecimal} input with specified scale. + * + * @param input {@link BigDecimal} with desired value + * @param scale scale of the value + * @return int value taken from specified {@link BigDecimal} + */ + public static int getDecimal9FromBigDecimal(BigDecimal input, int scale) { // Truncate or pad to set the input to the correct scale input = input.setScale(scale, BigDecimal.ROUND_HALF_UP); return input.unscaledValue().intValue(); } - public static BigDecimal getBigDecimalFromPrimitiveTypes(int input, int scale, int precision) { + /** + * Returns {@link BigDecimal} value created from specified integer value with specified scale. + * + * @param input integer value to use for creating of {@link BigDecimal} + * @param scale scale for resulting {@link BigDecimal} + * @return {@link BigDecimal} value + */ + public static BigDecimal getBigDecimalFromPrimitiveTypes(int input, int scale) { return BigDecimal.valueOf(input, scale); } - public static BigDecimal getBigDecimalFromPrimitiveTypes(long input, int scale, int precision) { + /** + * Returns {@link BigDecimal} value created from specified long value with specified scale. + * + * @param input long value to use for creating of {@link BigDecimal} + * @param scale scale for resulting {@link BigDecimal} + * @return {@link BigDecimal} value + */ + public static BigDecimal getBigDecimalFromPrimitiveTypes(long input, int scale) { return BigDecimal.valueOf(input, scale); } - public static int compareDenseBytes(DrillBuf left, int leftStart, boolean leftSign, DrillBuf right, int rightStart, boolean rightSign, int width) { - - int invert = 1; - - /* If signs are different then simply look at the - * sign of the two inputs and determine which is greater - */ - if (leftSign != rightSign) { - - return((leftSign == true) ? -1 : 1); - } else if(leftSign == true) { - /* Both inputs are negative, at the end we will - * have to invert the comparison - */ - invert = -1; - } - - int cmp = 0; - - for (int i = 0; i < width; i++) { - byte leftByte = left.getByte(leftStart + i); - byte rightByte = right.getByte(rightStart + i); - // Unsigned byte comparison - if ((leftByte & 0xFF) > (rightByte & 0xFF)) { - cmp = 1; - break; - } else if ((leftByte & 0xFF) < (rightByte & 0xFF)) { - cmp = -1; - break; - } - } - cmp *= invert; // invert the comparison if both were negative values - - return cmp; - } - - public static int getIntegerFromSparseBuffer(DrillBuf buffer, int start, int index) { - int value = buffer.getInt(start + (index * 4)); - - if (index == 0) { - /* the first byte contains sign bit, return value without it */ - value = (value & 0x7FFFFFFF); - } - return value; - } - - public static void setInteger(DrillBuf buffer, int start, int index, int value) { - buffer.setInt(start + (index * 4), value); - } - /** * Compares two VarDecimal values, still stored in their respective Drill buffers * @@ -497,314 +342,6 @@ public class DecimalUtility extends CoreDecimalUtility { return bdLeft.compareTo(bdRight); } - public static int compareSparseBytes(DrillBuf left, int leftStart, boolean leftSign, int leftScale, int leftPrecision, DrillBuf right, int rightStart, boolean rightSign, int rightPrecision, int rightScale, int width, int nDecimalDigits, boolean absCompare) { - - int invert = 1; - - if (absCompare == false) { - if (leftSign != rightSign) { - return (leftSign == true) ? -1 : 1; - } - - // Both values are negative invert the outcome of the comparison - if (leftSign == true) { - invert = -1; - } - } - - int cmp = compareSparseBytesInner(left, leftStart, leftSign, leftScale, leftPrecision, right, rightStart, rightSign, rightPrecision, rightScale, width, nDecimalDigits); - return cmp * invert; - } - - public static int compareSparseBytesInner(DrillBuf left, int leftStart, boolean leftSign, int leftScale, int leftPrecision, DrillBuf right, int rightStart, boolean rightSign, int rightPrecision, int rightScale, int width, int nDecimalDigits) { - /* compute the number of integer digits in each decimal */ - int leftInt = leftPrecision - leftScale; - int rightInt = rightPrecision - rightScale; - - /* compute the number of indexes required for storing integer digits */ - int leftIntRoundedUp = org.apache.drill.exec.util.DecimalUtility.roundUp(leftInt); - int rightIntRoundedUp = org.apache.drill.exec.util.DecimalUtility.roundUp(rightInt); - - /* compute number of indexes required for storing scale */ - int leftScaleRoundedUp = org.apache.drill.exec.util.DecimalUtility.roundUp(leftScale); - int rightScaleRoundedUp = org.apache.drill.exec.util.DecimalUtility.roundUp(rightScale); - - /* compute index of the most significant integer digits */ - int leftIndex1 = nDecimalDigits - leftScaleRoundedUp - leftIntRoundedUp; - int rightIndex1 = nDecimalDigits - rightScaleRoundedUp - rightIntRoundedUp; - - int leftStopIndex = nDecimalDigits - leftScaleRoundedUp; - int rightStopIndex = nDecimalDigits - rightScaleRoundedUp; - - /* Discard the zeroes in the integer part */ - while (leftIndex1 < leftStopIndex) { - if (getIntegerFromSparseBuffer(left, leftStart, leftIndex1) != 0) { - break; - } - - /* Digit in this location is zero, decrement the actual number - * of integer digits - */ - leftIntRoundedUp--; - leftIndex1++; - } - - /* If we reached the stop index then the number of integers is zero */ - if (leftIndex1 == leftStopIndex) { - leftIntRoundedUp = 0; - } - - while (rightIndex1 < rightStopIndex) { - if (getIntegerFromSparseBuffer(right, rightStart, rightIndex1) != 0) { - break; - } - - /* Digit in this location is zero, decrement the actual number - * of integer digits - */ - rightIntRoundedUp--; - rightIndex1++; - } - - if (rightIndex1 == rightStopIndex) { - rightIntRoundedUp = 0; - } - - /* We have the accurate number of non-zero integer digits, - * if the number of integer digits are different then we can determine - * which decimal is larger and needn't go down to comparing individual values - */ - if (leftIntRoundedUp > rightIntRoundedUp) { - return 1; - } - else if (rightIntRoundedUp > leftIntRoundedUp) { - return -1; - } - - /* The number of integer digits are the same, set the each index - * to the first non-zero integer and compare each digit - */ - leftIndex1 = nDecimalDigits - leftScaleRoundedUp - leftIntRoundedUp; - rightIndex1 = nDecimalDigits - rightScaleRoundedUp - rightIntRoundedUp; - - while (leftIndex1 < leftStopIndex && rightIndex1 < rightStopIndex) { - if (getIntegerFromSparseBuffer(left, leftStart, leftIndex1) > getIntegerFromSparseBuffer(right, rightStart, rightIndex1)) { - return 1; - } - else if (getIntegerFromSparseBuffer(right, rightStart, rightIndex1) > getIntegerFromSparseBuffer(left, leftStart, leftIndex1)) { - return -1; - } - - leftIndex1++; - rightIndex1++; - } - - /* The integer part of both the decimal's are equal, now compare - * each individual fractional part. Set the index to be at the - * beginning of the fractional part - */ - leftIndex1 = leftStopIndex; - rightIndex1 = rightStopIndex; - - /* Stop indexes will be the end of the array */ - leftStopIndex = nDecimalDigits; - rightStopIndex = nDecimalDigits; - - /* compare the two fractional parts of the decimal */ - while (leftIndex1 < leftStopIndex && rightIndex1 < rightStopIndex) { - if (getIntegerFromSparseBuffer(left, leftStart, leftIndex1) > getIntegerFromSparseBuffer(right, rightStart, rightIndex1)) { - return 1; - } - else if (getIntegerFromSparseBuffer(right, rightStart, rightIndex1) > getIntegerFromSparseBuffer(left, leftStart, leftIndex1)) { - return -1; - } - - leftIndex1++; - rightIndex1++; - } - - /* Till now the fractional part of the decimals are equal, check - * if one of the decimal has fractional part that is remaining - * and is non-zero - */ - while (leftIndex1 < leftStopIndex) { - if (getIntegerFromSparseBuffer(left, leftStart, leftIndex1) != 0) { - return 1; - } - leftIndex1++; - } - - while(rightIndex1 < rightStopIndex) { - if (getIntegerFromSparseBuffer(right, rightStart, rightIndex1) != 0) { - return -1; - } - rightIndex1++; - } - - /* Both decimal values are equal */ - return 0; - } - - public static BigDecimal getBigDecimalFromByteArray(byte[] bytes, int start, int length, int scale) { - byte[] value = Arrays.copyOfRange(bytes, start, start + length); - BigInteger unscaledValue = new BigInteger(value); - return new BigDecimal(unscaledValue, scale); - } - - public static void roundDecimal(DrillBuf result, int start, int nDecimalDigits, int desiredScale, int currentScale) { - int newScaleRoundedUp = org.apache.drill.exec.util.DecimalUtility.roundUp(desiredScale); - int origScaleRoundedUp = org.apache.drill.exec.util.DecimalUtility.roundUp(currentScale); - - if (desiredScale < currentScale) { - - boolean roundUp = false; - - //Extract the first digit to be truncated to check if we need to round up - int truncatedScaleIndex = desiredScale + 1; - if (truncatedScaleIndex <= currentScale) { - int extractDigitIndex = nDecimalDigits - origScaleRoundedUp -1; - extractDigitIndex += org.apache.drill.exec.util.DecimalUtility.roundUp(truncatedScaleIndex); - int extractDigit = getIntegerFromSparseBuffer(result, start, extractDigitIndex); - int temp = org.apache.drill.exec.util.DecimalUtility.MAX_DIGITS - (truncatedScaleIndex % org.apache.drill.exec.util.DecimalUtility.MAX_DIGITS); - if (temp != 0) { - extractDigit = extractDigit / (int) (Math.pow(10, temp)); - } - if ((extractDigit % 10) > 4) { - roundUp = true; - } - } - - // Get the source index beyond which we will truncate - int srcIntIndex = nDecimalDigits - origScaleRoundedUp - 1; - int srcIndex = srcIntIndex + newScaleRoundedUp; - - // Truncate the remaining fractional part, move the integer part - int destIndex = nDecimalDigits - 1; - if (srcIndex != destIndex) { - while (srcIndex >= 0) { - setInteger(result, start, destIndex--, getIntegerFromSparseBuffer(result, start, srcIndex--)); - } - - // Set the remaining portion of the decimal to be zeroes - while (destIndex >= 0) { - setInteger(result, start, destIndex--, 0); - } - srcIndex = nDecimalDigits - 1; - } - - // We truncated the decimal digit. Now we need to truncate within the base 1 billion fractional digit - int truncateFactor = org.apache.drill.exec.util.DecimalUtility.MAX_DIGITS - (desiredScale % org.apache.drill.exec.util.DecimalUtility.MAX_DIGITS); - if (truncateFactor != org.apache.drill.exec.util.DecimalUtility.MAX_DIGITS) { - truncateFactor = (int) Math.pow(10, truncateFactor); - int fractionalDigits = getIntegerFromSparseBuffer(result, start, nDecimalDigits - 1); - fractionalDigits /= truncateFactor; - setInteger(result, start, nDecimalDigits - 1, fractionalDigits * truncateFactor); - } - - // Finally round up the digit if needed - if (roundUp == true) { - srcIndex = nDecimalDigits - 1; - int carry; - if (truncateFactor != org.apache.drill.exec.util.DecimalUtility.MAX_DIGITS) { - carry = truncateFactor; - } else { - carry = 1; - } - - while (srcIndex >= 0) { - int value = getIntegerFromSparseBuffer(result, start, srcIndex); - value += carry; - - if (value >= org.apache.drill.exec.util.DecimalUtility.DIGITS_BASE) { - setInteger(result, start, srcIndex--, value % org.apache.drill.exec.util.DecimalUtility.DIGITS_BASE); - carry = value / org.apache.drill.exec.util.DecimalUtility.DIGITS_BASE; - } else { - setInteger(result, start, srcIndex--, value); - carry = 0; - break; - } - } - } - } else if (desiredScale > currentScale) { - // Add fractional digits to the decimal - - // Check if we need to shift the decimal digits to the left - if (newScaleRoundedUp > origScaleRoundedUp) { - int srcIndex = 0; - int destIndex = newScaleRoundedUp - origScaleRoundedUp; - - // Check while extending scale, we are not overwriting integer part - while (srcIndex < destIndex) { - if (getIntegerFromSparseBuffer(result, start, srcIndex++) != 0) { - throw new org.apache.drill.common.exceptions.DrillRuntimeException("Truncate resulting in loss of integer part, reduce scale specified"); - } - } - - srcIndex = 0; - while (destIndex < nDecimalDigits) { - setInteger(result, start, srcIndex++, getIntegerFromSparseBuffer(result, start, destIndex++)); - } - - // Clear the remaining part - while (srcIndex < nDecimalDigits) { - setInteger(result, start, srcIndex++, 0); - } - } - } - } - - public static int getFirstFractionalDigit(int decimal, int scale) { - if (scale == 0) { - return 0; - } - int temp = (int) adjustScaleDivide(decimal, scale - 1); - return Math.abs(temp % 10); - } - - public static int getFirstFractionalDigit(long decimal, int scale) { - if (scale == 0) { - return 0; - } - long temp = adjustScaleDivide(decimal, scale - 1); - return (int) (Math.abs(temp % 10)); - } - - public static int getFirstFractionalDigit(DrillBuf data, int scale, int start, int nDecimalDigits) { - if (scale == 0) { - return 0; - } - - int index = nDecimalDigits - roundUp(scale); - return (int) (adjustScaleDivide(data.getInt(start + (index * INTEGER_SIZE)), MAX_DIGITS - 1)); - } - - public static int compareSparseSamePrecScale(DrillBuf left, int lStart, byte[] right, int length) { - // check the sign first - boolean lSign = (left.getInt(lStart) & 0x80000000) != 0; - boolean rSign = ByteFunctionHelpers.getSign(right); - int cmp = 0; - - if (lSign != rSign) { - return (lSign == false) ? 1 : -1; - } - - // invert the comparison if we are comparing negative numbers - int invert = (lSign == true) ? -1 : 1; - - // compare byte by byte - int n = 0; - while (n < length/4) { - int leftInt = Decimal38SparseHolder.getInteger(n, lStart, left); - int rightInt = ByteFunctionHelpers.getInteger(right, n); - if (leftInt != rightInt) { - cmp = (leftInt - rightInt ) > 0 ? 1 : -1; - break; - } - n++; - } - return cmp * invert; - } - /** * Returns max length of byte array, required to store value with specified precision. * @@ -817,6 +354,13 @@ public class DecimalUtility extends CoreDecimalUtility { return 0; } if (precision < 300) { // normal case, use exact heuristic formula + // Math.pow(10, precision) - 1 returns max integer value for the specified precision, example 999 for precision 3 + // Math.log(Math.pow(10, precision) - 1) / Math.log(2) is just log with base 2 to calculate size + // in bits for max integer value mentioned above + // Math.log(Math.pow(10, precision) - 1) / Math.log(2) + 1 in this expression was added 1 to the count of bits to + // take into account case with negative value + // size in bits was divided by size of byte to calculate bytes count required to store value + // with the specified precision return (int) Math.ceil((Math.log(Math.pow(10, precision) - 1) / Math.log(2) + 1) / Byte.SIZE); } else { // for values greater than 304 Math.pow(10, precision) returns Infinity, therefore 1 is neglected in Math.log() diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/ValueHolderHelper.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/ValueHolderHelper.java index d1cc3b4a1..73027aaed 100644 --- a/exec/vector/src/main/java/org/apache/drill/exec/vector/ValueHolderHelper.java +++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/ValueHolderHelper.java @@ -127,7 +127,7 @@ public class ValueHolderHelper { byte[] b = s.getBytes(Charsets.UTF_8); vch.start = 0; vch.end = b.length; - vch.buffer = a.buffer(b.length); // + vch.buffer = a.buffer(b.length); vch.buffer.setBytes(0, b); return vch; } @@ -180,7 +180,7 @@ public class ValueHolderHelper { dch.start = 0; dch.buffer = buf.reallocIfNeeded(5 * DecimalUtility.INTEGER_SIZE); DecimalUtility - .getSparseFromBigDecimal(bigDecimal, dch.buffer, dch.start, dch.scale, dch.precision, dch.nDecimalDigits); + .getSparseFromBigDecimal(bigDecimal, dch.buffer, dch.start, dch.scale, Decimal28SparseHolder.nDecimalDigits); return dch; } @@ -195,9 +195,9 @@ public class ValueHolderHelper { dch.precision = bigDecimal.precision(); Decimal38SparseHolder.setSign(bigDecimal.signum() == -1, dch.start, dch.buffer); dch.start = 0; - dch.buffer = buf.reallocIfNeeded(dch.maxPrecision * DecimalUtility.INTEGER_SIZE); + dch.buffer = buf.reallocIfNeeded(Decimal38SparseHolder.maxPrecision * DecimalUtility.INTEGER_SIZE); DecimalUtility - .getSparseFromBigDecimal(bigDecimal, dch.buffer, dch.start, dch.scale, dch.precision, dch.nDecimalDigits); + .getSparseFromBigDecimal(bigDecimal, dch.buffer, dch.start, dch.scale, Decimal38SparseHolder.nDecimalDigits); return dch; } |