8026943: SQE test jce/Global/Cipher/SameBuffer failed

Summary: Always use different input/output buffers when calling FeedbackCipher objects
Reviewed-by: mullan

3 files changed, 51 insertions, 44 deletions

diff --git a/src/share/classes/com/sun/crypto/provider/CipherBlockChaining.java b/src/share/classes/com/sun/crypto/provider/CipherBlockChaining.java
index c32aa7fea..51d95c12e 100644
--- a/src/share/classes/com/sun/crypto/provider/CipherBlockChaining.java
+++ b/src/share/classes/com/sun/crypto/provider/CipherBlockChaining.java
@@ -186,29 +186,15 @@ class CipherBlockChaining extends FeedbackCipher  {
                 byte[] plain, int plainOffset)
     {
         int i;
-        byte[] cipherOrig=null;
         int endIndex = cipherOffset + cipherLen;
 
-        if (cipher==plain && (cipherOffset >= plainOffset)
-            && ((cipherOffset - plainOffset) < blockSize)) {
-            // Save the original ciphertext blocks, so they can be
-            // stored in the feedback register "r".
-            // This is necessary because in this constellation, a
-            // ciphertext block (or parts of it) will be overridden by
-            // the plaintext result.
-            cipherOrig = cipher.clone();
-        }
         for (; cipherOffset < endIndex;
              cipherOffset += blockSize, plainOffset += blockSize) {
             embeddedCipher.decryptBlock(cipher, cipherOffset, k, 0);
             for (i = 0; i < blockSize; i++) {
                 plain[i+plainOffset] = (byte)(k[i] ^ r[i]);
             }
-            if (cipherOrig==null) {
-                System.arraycopy(cipher, cipherOffset, r, 0, blockSize);
-            } else {
-                System.arraycopy(cipherOrig, cipherOffset, r, 0, blockSize);
-            }
+            System.arraycopy(cipher, cipherOffset, r, 0, blockSize);
         }
         return cipherLen;
     }
diff --git a/src/share/classes/com/sun/crypto/provider/CipherCore.java b/src/share/classes/com/sun/crypto/provider/CipherCore.java
index dab1b4d6f..8d54633ad 100644
--- a/src/share/classes/com/sun/crypto/provider/CipherCore.java
+++ b/src/share/classes/com/sun/crypto/provider/CipherCore.java
@@ -732,8 +732,12 @@ final class CipherCore {
                     System.arraycopy(buffer, len, buffer, 0, buffered);
                 }
             } else { // len > buffered
-                if (buffered == 0) {
+                if ((input != output) && (buffered == 0)) {
                     // all to-be-processed data are from 'input'
+                    // however, note that if 'input' and 'output' are the same,
+                    // then they can't be passed directly to the underlying cipher
+                    // engine operations as data may be overwritten before they
+                    // are read.
                     if (decrypting) {
                         outLen = cipher.decrypt(input, inputOffset, len, output, outputOffset);
                     } else {
@@ -744,12 +748,16 @@ final class CipherCore {
                 } else {
                     // assemble the data using both 'buffer' and 'input'
                     byte[] in = new byte[len];
-                    System.arraycopy(buffer, 0, in, 0, buffered);
                     int inConsumed = len - buffered;
-                    System.arraycopy(input, inputOffset, in, buffered, inConsumed);
-                    buffered = 0;
-                    inputOffset += inConsumed;
-                    inputLen -= inConsumed;
+                    if (buffered != 0) {
+                        System.arraycopy(buffer, 0, in, 0, buffered);
+                        buffered = 0;
+                    }
+                    if (inConsumed != 0) {
+                        System.arraycopy(input, inputOffset, in, len - inConsumed, inConsumed);
+                        inputOffset += inConsumed;
+                        inputLen -= inConsumed;
+                    }
                     if (decrypting) {
                         outLen = cipher.decrypt(in, 0, len, output, outputOffset);
                     } else {
@@ -907,11 +915,18 @@ final class CipherCore {
                  " when decrypting with padded cipher");
         }
 
-        // prepare the final input avoiding copying if possible
+        /*
+         * prepare the final input, assemble a new buffer if any
+         * of the following is true:
+         *  - 'input' and 'output' are the same buffer
+         *  - there are internally buffered bytes
+         *  - doing encryption and padding is needed
+         */
         byte[] finalBuf = input;
         int finalOffset = inputOffset;
         int finalBufLen = inputLen;
-        if ((buffered != 0) || (!decrypting && padding != null)) {
+        if ((input == output) || (buffered != 0) ||
+            (!decrypting && padding != null)) {
             if (decrypting || padding == null) {
                 paddingLen = 0;
             }
diff --git a/src/share/classes/com/sun/crypto/provider/DESedeWrapCipher.java b/src/share/classes/com/sun/crypto/provider/DESedeWrapCipher.java
index d4e27528d..06ef9e2b6 100644
--- a/src/share/classes/com/sun/crypto/provider/DESedeWrapCipher.java
+++ b/src/share/classes/com/sun/crypto/provider/DESedeWrapCipher.java
@@ -50,6 +50,9 @@ public final class DESedeWrapCipher extends CipherSpi {
         (byte) 0x79, (byte) 0xe8, (byte) 0x21, (byte) 0x05
     };
 
+    private static final int CHECKSUM_LEN = 8;
+    private static final int IV_LEN = 8;
+
     /*
      * internal cipher object which does the real work.
      */
@@ -135,7 +138,7 @@ public final class DESedeWrapCipher extends CipherSpi {
         // can only return an upper-limit if not initialized yet.
         int result = 0;
         if (decrypting) {
-            result = inputLen - 16;
+            result = inputLen - 16; // CHECKSUM_LEN + IV_LEN;
         } else {
             result = inputLen + 16;
         }
@@ -215,7 +218,7 @@ public final class DESedeWrapCipher extends CipherSpi {
         if (opmode == Cipher.WRAP_MODE) {
             decrypting = false;
             if (params == null) {
-                iv = new byte[8];
+                iv = new byte[IV_LEN];
                 if (random == null) {
                     random = SunJCE.getRandom();
                 }
@@ -449,14 +452,15 @@ public final class DESedeWrapCipher extends CipherSpi {
         }
 
         byte[] cks = getChecksum(keyVal);
-        byte[] out = new byte[iv.length + keyVal.length + cks.length];
-
-        System.arraycopy(keyVal, 0, out, iv.length, keyVal.length);
-        System.arraycopy(cks, 0, out, iv.length+keyVal.length, cks.length);
-        cipher.encrypt(out, iv.length, keyVal.length+cks.length,
-                       out, iv.length);
+        byte[] in = new byte[keyVal.length + CHECKSUM_LEN];
+        System.arraycopy(keyVal, 0, in, 0, keyVal.length);
+        System.arraycopy(cks, 0, in, keyVal.length, CHECKSUM_LEN);
 
+        byte[] out = new byte[iv.length + in.length];
         System.arraycopy(iv, 0, out, 0, iv.length);
+
+        cipher.encrypt(in, 0, in.length, out, iv.length);
+
         // reverse the array content
         for (int i = 0; i < out.length/2; i++) {
             byte temp = out[i];
@@ -470,7 +474,8 @@ public final class DESedeWrapCipher extends CipherSpi {
             // should never happen
             throw new RuntimeException("Internal cipher key is corrupted");
         }
-        cipher.encrypt(out, 0, out.length, out, 0);
+        byte[] out2 = new byte[out.length];
+        cipher.encrypt(out, 0, out.length, out2, 0);
 
         // restore cipher state to prior to this call
         try {
@@ -480,7 +485,7 @@ public final class DESedeWrapCipher extends CipherSpi {
             // should never happen
             throw new RuntimeException("Internal cipher key is corrupted");
         }
-        return out;
+        return out2;
     }
 
     /**
@@ -520,25 +525,26 @@ public final class DESedeWrapCipher extends CipherSpi {
             buffer[i] = buffer[buffer.length-1-i];
             buffer[buffer.length-1-i] = temp;
         }
-        iv = new byte[IV2.length];
+        iv = new byte[IV_LEN];
         System.arraycopy(buffer, 0, iv, 0, iv.length);
         cipher.init(true, cipherKey.getAlgorithm(), cipherKey.getEncoded(),
                     iv);
-        cipher.decrypt(buffer, iv.length, buffer.length-iv.length,
-                       buffer, iv.length);
-        int origLen = buffer.length - iv.length - 8;
-        byte[] cks = getChecksum(buffer, iv.length, origLen);
-        int offset = iv.length + origLen;
-        for (int i = 0; i < cks.length; i++) {
-            if (buffer[offset + i] != cks[i]) {
+        byte[] buffer2 = new byte[buffer.length - iv.length];
+        cipher.decrypt(buffer, iv.length, buffer2.length,
+                       buffer2, 0);
+        int keyValLen = buffer2.length - CHECKSUM_LEN;
+        byte[] cks = getChecksum(buffer2, 0, keyValLen);
+        int offset = keyValLen;
+        for (int i = 0; i < CHECKSUM_LEN; i++) {
+            if (buffer2[offset + i] != cks[i]) {
                 throw new InvalidKeyException("Checksum comparison failed");
             }
         }
         // restore cipher state to prior to this call
         cipher.init(decrypting, cipherKey.getAlgorithm(),
                     cipherKey.getEncoded(), IV2);
-        byte[] out = new byte[origLen];
-        System.arraycopy(buffer, iv.length, out, 0, out.length);
+        byte[] out = new byte[keyValLen];
+        System.arraycopy(buffer2, 0, out, 0, keyValLen);
         return ConstructKeys.constructKey(out, wrappedKeyAlgorithm,
                                           wrappedKeyType);
     }
@@ -554,7 +560,7 @@ public final class DESedeWrapCipher extends CipherSpi {
             throw new RuntimeException("SHA1 message digest not available");
         }
         md.update(in, offset, len);
-        byte[] cks = new byte[8];
+        byte[] cks = new byte[CHECKSUM_LEN];
         System.arraycopy(md.digest(), 0, cks, 0, cks.length);
         return cks;
     }