From c3e73e76a90b1e790e0bb7bb36135be9232f58de Mon Sep 17 00:00:00 2001 From: "Gustavo F. Padovan" Date: Thu, 26 May 2011 13:29:33 +1000 Subject: crypto: ghash-intel - Fix set but not used in ghash_async_setkey() Signed-off-by: Gustavo F. Padovan Signed-off-by: Herbert Xu --- arch/x86/crypto/ghash-clmulni-intel_glue.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/x86/crypto/ghash-clmulni-intel_glue.c b/arch/x86/crypto/ghash-clmulni-intel_glue.c index 7a6e68e4f74..976aa64d9a2 100644 --- a/arch/x86/crypto/ghash-clmulni-intel_glue.c +++ b/arch/x86/crypto/ghash-clmulni-intel_glue.c @@ -245,7 +245,7 @@ static int ghash_async_setkey(struct crypto_ahash *tfm, const u8 *key, crypto_ahash_set_flags(tfm, crypto_ahash_get_flags(child) & CRYPTO_TFM_RES_MASK); - return 0; + return err; } static int ghash_async_init_tfm(struct crypto_tfm *tfm) -- cgit v1.2.3 From 5165e5b112c4fb3a9258d5f6246cbbd44108c181 Mon Sep 17 00:00:00 2001 From: Jarod Wilson Date: Tue, 31 May 2011 15:23:57 +1000 Subject: crypto: testmgr - add xts-aes-256 self-test FIPS compliance requires a known-answer self-test for all approved cipher and mode combinations, for all valid key sizes. Presently, there are only self-tests for xts-aes-128. This adds a 256-bit one, pulled from the same reference document, which should satisfy the requirement. Signed-off-by: Jarod Wilson Signed-off-by: Herbert Xu --- crypto/testmgr.h | 293 ++++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 291 insertions(+), 2 deletions(-) diff --git a/crypto/testmgr.h b/crypto/testmgr.h index 27e60619538..27adc92842b 100644 --- a/crypto/testmgr.h +++ b/crypto/testmgr.h @@ -2976,8 +2976,8 @@ static struct cipher_testvec cast6_dec_tv_template[] = { #define AES_CBC_DEC_TEST_VECTORS 4 #define AES_LRW_ENC_TEST_VECTORS 8 #define AES_LRW_DEC_TEST_VECTORS 8 -#define AES_XTS_ENC_TEST_VECTORS 4 -#define AES_XTS_DEC_TEST_VECTORS 4 +#define AES_XTS_ENC_TEST_VECTORS 5 +#define AES_XTS_DEC_TEST_VECTORS 5 #define AES_CTR_ENC_TEST_VECTORS 3 #define AES_CTR_DEC_TEST_VECTORS 3 #define AES_OFB_ENC_TEST_VECTORS 1 @@ -3926,6 +3926,150 @@ static struct cipher_testvec aes_xts_enc_tv_template[] = { "\x0a\x28\x2d\xf9\x20\x14\x7b\xea" "\xbe\x42\x1e\xe5\x31\x9d\x05\x68", .rlen = 512, + }, { /* XTS-AES 10, XTS-AES-256, data unit 512 bytes */ + .key = "\x27\x18\x28\x18\x28\x45\x90\x45" + "\x23\x53\x60\x28\x74\x71\x35\x26" + "\x62\x49\x77\x57\x24\x70\x93\x69" + "\x99\x59\x57\x49\x66\x96\x76\x27" + "\x31\x41\x59\x26\x53\x58\x97\x93" + "\x23\x84\x62\x64\x33\x83\x27\x95" + "\x02\x88\x41\x97\x16\x93\x99\x37" + "\x51\x05\x82\x09\x74\x94\x45\x92", + .klen = 64, + .iv = "\xff\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00", + "\x00\x00\x00\x00\x00\x00\x00\x00", + "\x00\x00\x00\x00\x00\x00\x00\x00", + .input = "\x00\x01\x02\x03\x04\x05\x06\x07" + "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f" + "\x10\x11\x12\x13\x14\x15\x16\x17" + "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f" + "\x20\x21\x22\x23\x24\x25\x26\x27" + "\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f" + "\x30\x31\x32\x33\x34\x35\x36\x37" + "\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f" + "\x40\x41\x42\x43\x44\x45\x46\x47" + "\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f" + "\x50\x51\x52\x53\x54\x55\x56\x57" + "\x58\x59\x5a\x5b\x5c\x5d\x5e\x5f" + "\x60\x61\x62\x63\x64\x65\x66\x67" + "\x68\x69\x6a\x6b\x6c\x6d\x6e\x6f" + "\x70\x71\x72\x73\x74\x75\x76\x77" + "\x78\x79\x7a\x7b\x7c\x7d\x7e\x7f" + "\x80\x81\x82\x83\x84\x85\x86\x87" + "\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f" + "\x90\x91\x92\x93\x94\x95\x96\x97" + "\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f" + "\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7" + "\xa8\xa9\xaa\xab\xac\xad\xae\xaf" + "\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7" + "\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf" + "\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7" + "\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf" + "\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7" + "\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf" + "\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7" + "\xe8\xe9\xea\xeb\xec\xed\xee\xef" + "\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7" + "\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff" + "\x00\x01\x02\x03\x04\x05\x06\x07" + "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f" + "\x10\x11\x12\x13\x14\x15\x16\x17" + "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f" + "\x20\x21\x22\x23\x24\x25\x26\x27" + "\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f" + "\x30\x31\x32\x33\x34\x35\x36\x37" + "\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f" + "\x40\x41\x42\x43\x44\x45\x46\x47" + "\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f" + "\x50\x51\x52\x53\x54\x55\x56\x57" + "\x58\x59\x5a\x5b\x5c\x5d\x5e\x5f" + "\x60\x61\x62\x63\x64\x65\x66\x67" + "\x68\x69\x6a\x6b\x6c\x6d\x6e\x6f" + "\x70\x71\x72\x73\x74\x75\x76\x77" + "\x78\x79\x7a\x7b\x7c\x7d\x7e\x7f" + "\x80\x81\x82\x83\x84\x85\x86\x87" + "\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f" + "\x90\x91\x92\x93\x94\x95\x96\x97" + "\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f" + "\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7" + "\xa8\xa9\xaa\xab\xac\xad\xae\xaf" + "\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7" + "\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf" + "\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7" + "\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf" + "\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7" + "\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf" + "\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7" + "\xe8\xe9\xea\xeb\xec\xed\xee\xef" + "\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7" + "\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff", + .ilen = 512, + .result = "\x1c\x3b\x3a\x10\x2f\x77\x03\x86" + "\xe4\x83\x6c\x99\xe3\x70\xcf\x9b" + "\xea\x00\x80\x3f\x5e\x48\x23\x57" + "\xa4\xae\x12\xd4\x14\xa3\xe6\x3b" + "\x5d\x31\xe2\x76\xf8\xfe\x4a\x8d" + "\x66\xb3\x17\xf9\xac\x68\x3f\x44" + "\x68\x0a\x86\xac\x35\xad\xfc\x33" + "\x45\xbe\xfe\xcb\x4b\xb1\x88\xfd" + "\x57\x76\x92\x6c\x49\xa3\x09\x5e" + "\xb1\x08\xfd\x10\x98\xba\xec\x70" + "\xaa\xa6\x69\x99\xa7\x2a\x82\xf2" + "\x7d\x84\x8b\x21\xd4\xa7\x41\xb0" + "\xc5\xcd\x4d\x5f\xff\x9d\xac\x89" + "\xae\xba\x12\x29\x61\xd0\x3a\x75" + "\x71\x23\xe9\x87\x0f\x8a\xcf\x10" + "\x00\x02\x08\x87\x89\x14\x29\xca" + "\x2a\x3e\x7a\x7d\x7d\xf7\xb1\x03" + "\x55\x16\x5c\x8b\x9a\x6d\x0a\x7d" + "\xe8\xb0\x62\xc4\x50\x0d\xc4\xcd" + "\x12\x0c\x0f\x74\x18\xda\xe3\xd0" + "\xb5\x78\x1c\x34\x80\x3f\xa7\x54" + "\x21\xc7\x90\xdf\xe1\xde\x18\x34" + "\xf2\x80\xd7\x66\x7b\x32\x7f\x6c" + "\x8c\xd7\x55\x7e\x12\xac\x3a\x0f" + "\x93\xec\x05\xc5\x2e\x04\x93\xef" + "\x31\xa1\x2d\x3d\x92\x60\xf7\x9a" + "\x28\x9d\x6a\x37\x9b\xc7\x0c\x50" + "\x84\x14\x73\xd1\xa8\xcc\x81\xec" + "\x58\x3e\x96\x45\xe0\x7b\x8d\x96" + "\x70\x65\x5b\xa5\xbb\xcf\xec\xc6" + "\xdc\x39\x66\x38\x0a\xd8\xfe\xcb" + "\x17\xb6\xba\x02\x46\x9a\x02\x0a" + "\x84\xe1\x8e\x8f\x84\x25\x20\x70" + "\xc1\x3e\x9f\x1f\x28\x9b\xe5\x4f" + "\xbc\x48\x14\x57\x77\x8f\x61\x60" + "\x15\xe1\x32\x7a\x02\xb1\x40\xf1" + "\x50\x5e\xb3\x09\x32\x6d\x68\x37" + "\x8f\x83\x74\x59\x5c\x84\x9d\x84" + "\xf4\xc3\x33\xec\x44\x23\x88\x51" + "\x43\xcb\x47\xbd\x71\xc5\xed\xae" + "\x9b\xe6\x9a\x2f\xfe\xce\xb1\xbe" + "\xc9\xde\x24\x4f\xbe\x15\x99\x2b" + "\x11\xb7\x7c\x04\x0f\x12\xbd\x8f" + "\x6a\x97\x5a\x44\xa0\xf9\x0c\x29" + "\xa9\xab\xc3\xd4\xd8\x93\x92\x72" + "\x84\xc5\x87\x54\xcc\xe2\x94\x52" + "\x9f\x86\x14\xdc\xd2\xab\xa9\x91" + "\x92\x5f\xed\xc4\xae\x74\xff\xac" + "\x6e\x33\x3b\x93\xeb\x4a\xff\x04" + "\x79\xda\x9a\x41\x0e\x44\x50\xe0" + "\xdd\x7a\xe4\xc6\xe2\x91\x09\x00" + "\x57\x5d\xa4\x01\xfc\x07\x05\x9f" + "\x64\x5e\x8b\x7e\x9b\xfd\xef\x33" + "\x94\x30\x54\xff\x84\x01\x14\x93" + "\xc2\x7b\x34\x29\xea\xed\xb4\xed" + "\x53\x76\x44\x1a\x77\xed\x43\x85" + "\x1a\xd7\x7f\x16\xf5\x41\xdf\xd2" + "\x69\xd5\x0d\x6a\x5f\x14\xfb\x0a" + "\xab\x1c\xbb\x4c\x15\x50\xbe\x97" + "\xf7\xab\x40\x66\x19\x3c\x4c\xaa" + "\x77\x3d\xad\x38\x01\x4b\xd2\x09" + "\x2f\xa7\x55\xc8\x24\xbb\x5e\x54" + "\xc4\xf3\x6f\xfd\xa9\xfc\xea\x70" + "\xb9\xc6\xe6\x93\xe1\x48\xc1\x51", + .rlen = 512, } }; @@ -4123,6 +4267,151 @@ static struct cipher_testvec aes_xts_dec_tv_template[] = { "\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7" "\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff", .rlen = 512, + }, { /* XTS-AES 10, XTS-AES-256, data unit 512 bytes */ + .key = "\x27\x18\x28\x18\x28\x45\x90\x45" + "\x23\x53\x60\x28\x74\x71\x35\x26" + "\x62\x49\x77\x57\x24\x70\x93\x69" + "\x99\x59\x57\x49\x66\x96\x76\x27" + "\x31\x41\x59\x26\x53\x58\x97\x93" + "\x23\x84\x62\x64\x33\x83\x27\x95" + "\x02\x88\x41\x97\x16\x93\x99\x37" + "\x51\x05\x82\x09\x74\x94\x45\x92", + .klen = 64, + .iv = "\xff\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00", + "\x00\x00\x00\x00\x00\x00\x00\x00", + "\x00\x00\x00\x00\x00\x00\x00\x00", + .input = "\x1c\x3b\x3a\x10\x2f\x77\x03\x86" + "\xe4\x83\x6c\x99\xe3\x70\xcf\x9b" + "\xea\x00\x80\x3f\x5e\x48\x23\x57" + "\xa4\xae\x12\xd4\x14\xa3\xe6\x3b" + "\x5d\x31\xe2\x76\xf8\xfe\x4a\x8d" + "\x66\xb3\x17\xf9\xac\x68\x3f\x44" + "\x68\x0a\x86\xac\x35\xad\xfc\x33" + "\x45\xbe\xfe\xcb\x4b\xb1\x88\xfd" + "\x57\x76\x92\x6c\x49\xa3\x09\x5e" + "\xb1\x08\xfd\x10\x98\xba\xec\x70" + "\xaa\xa6\x69\x99\xa7\x2a\x82\xf2" + "\x7d\x84\x8b\x21\xd4\xa7\x41\xb0" + "\xc5\xcd\x4d\x5f\xff\x9d\xac\x89" + "\xae\xba\x12\x29\x61\xd0\x3a\x75" + "\x71\x23\xe9\x87\x0f\x8a\xcf\x10" + "\x00\x02\x08\x87\x89\x14\x29\xca" + "\x2a\x3e\x7a\x7d\x7d\xf7\xb1\x03" + "\x55\x16\x5c\x8b\x9a\x6d\x0a\x7d" + "\xe8\xb0\x62\xc4\x50\x0d\xc4\xcd" + "\x12\x0c\x0f\x74\x18\xda\xe3\xd0" + "\xb5\x78\x1c\x34\x80\x3f\xa7\x54" + "\x21\xc7\x90\xdf\xe1\xde\x18\x34" + "\xf2\x80\xd7\x66\x7b\x32\x7f\x6c" + "\x8c\xd7\x55\x7e\x12\xac\x3a\x0f" + "\x93\xec\x05\xc5\x2e\x04\x93\xef" + "\x31\xa1\x2d\x3d\x92\x60\xf7\x9a" + "\x28\x9d\x6a\x37\x9b\xc7\x0c\x50" + "\x84\x14\x73\xd1\xa8\xcc\x81\xec" + "\x58\x3e\x96\x45\xe0\x7b\x8d\x96" + "\x70\x65\x5b\xa5\xbb\xcf\xec\xc6" + "\xdc\x39\x66\x38\x0a\xd8\xfe\xcb" + "\x17\xb6\xba\x02\x46\x9a\x02\x0a" + "\x84\xe1\x8e\x8f\x84\x25\x20\x70" + "\xc1\x3e\x9f\x1f\x28\x9b\xe5\x4f" + "\xbc\x48\x14\x57\x77\x8f\x61\x60" + "\x15\xe1\x32\x7a\x02\xb1\x40\xf1" + "\x50\x5e\xb3\x09\x32\x6d\x68\x37" + "\x8f\x83\x74\x59\x5c\x84\x9d\x84" + "\xf4\xc3\x33\xec\x44\x23\x88\x51" + "\x43\xcb\x47\xbd\x71\xc5\xed\xae" + "\x9b\xe6\x9a\x2f\xfe\xce\xb1\xbe" + "\xc9\xde\x24\x4f\xbe\x15\x99\x2b" + "\x11\xb7\x7c\x04\x0f\x12\xbd\x8f" + "\x6a\x97\x5a\x44\xa0\xf9\x0c\x29" + "\xa9\xab\xc3\xd4\xd8\x93\x92\x72" + "\x84\xc5\x87\x54\xcc\xe2\x94\x52" + "\x9f\x86\x14\xdc\xd2\xab\xa9\x91" + "\x92\x5f\xed\xc4\xae\x74\xff\xac" + "\x6e\x33\x3b\x93\xeb\x4a\xff\x04" + "\x79\xda\x9a\x41\x0e\x44\x50\xe0" + "\xdd\x7a\xe4\xc6\xe2\x91\x09\x00" + "\x57\x5d\xa4\x01\xfc\x07\x05\x9f" + "\x64\x5e\x8b\x7e\x9b\xfd\xef\x33" + "\x94\x30\x54\xff\x84\x01\x14\x93" + "\xc2\x7b\x34\x29\xea\xed\xb4\xed" + "\x53\x76\x44\x1a\x77\xed\x43\x85" + "\x1a\xd7\x7f\x16\xf5\x41\xdf\xd2" + "\x69\xd5\x0d\x6a\x5f\x14\xfb\x0a" + "\xab\x1c\xbb\x4c\x15\x50\xbe\x97" + "\xf7\xab\x40\x66\x19\x3c\x4c\xaa" + "\x77\x3d\xad\x38\x01\x4b\xd2\x09" + "\x2f\xa7\x55\xc8\x24\xbb\x5e\x54" + "\xc4\xf3\x6f\xfd\xa9\xfc\xea\x70" + "\xb9\xc6\xe6\x93\xe1\x48\xc1\x51", + .ilen = 512, + .result = "\x00\x01\x02\x03\x04\x05\x06\x07" + "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f" + "\x10\x11\x12\x13\x14\x15\x16\x17" + "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f" + "\x20\x21\x22\x23\x24\x25\x26\x27" + "\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f" + "\x30\x31\x32\x33\x34\x35\x36\x37" + "\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f" + "\x40\x41\x42\x43\x44\x45\x46\x47" + "\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f" + "\x50\x51\x52\x53\x54\x55\x56\x57" + "\x58\x59\x5a\x5b\x5c\x5d\x5e\x5f" + "\x60\x61\x62\x63\x64\x65\x66\x67" + "\x68\x69\x6a\x6b\x6c\x6d\x6e\x6f" + "\x70\x71\x72\x73\x74\x75\x76\x77" + "\x78\x79\x7a\x7b\x7c\x7d\x7e\x7f" + "\x80\x81\x82\x83\x84\x85\x86\x87" + "\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f" + "\x90\x91\x92\x93\x94\x95\x96\x97" + "\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f" + "\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7" + "\xa8\xa9\xaa\xab\xac\xad\xae\xaf" + "\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7" + "\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf" + "\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7" + "\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf" + "\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7" + "\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf" + "\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7" + "\xe8\xe9\xea\xeb\xec\xed\xee\xef" + "\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7" + "\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff" + "\x00\x01\x02\x03\x04\x05\x06\x07" + "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f" + "\x10\x11\x12\x13\x14\x15\x16\x17" + "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f" + "\x20\x21\x22\x23\x24\x25\x26\x27" + "\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f" + "\x30\x31\x32\x33\x34\x35\x36\x37" + "\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f" + "\x40\x41\x42\x43\x44\x45\x46\x47" + "\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f" + "\x50\x51\x52\x53\x54\x55\x56\x57" + "\x58\x59\x5a\x5b\x5c\x5d\x5e\x5f" + "\x60\x61\x62\x63\x64\x65\x66\x67" + "\x68\x69\x6a\x6b\x6c\x6d\x6e\x6f" + "\x70\x71\x72\x73\x74\x75\x76\x77" + "\x78\x79\x7a\x7b\x7c\x7d\x7e\x7f" + "\x80\x81\x82\x83\x84\x85\x86\x87" + "\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f" + "\x90\x91\x92\x93\x94\x95\x96\x97" + "\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f" + "\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7" + "\xa8\xa9\xaa\xab\xac\xad\xae\xaf" + "\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7" + "\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf" + "\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7" + "\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf" + "\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7" + "\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf" + "\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7" + "\xe8\xe9\xea\xeb\xec\xed\xee\xef" + "\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7" + "\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff", + .rlen = 512, + } }; -- cgit v1.2.3 From 8af00860c913d5e814f3c10232a0f42e2a964b99 Mon Sep 17 00:00:00 2001 From: Richard Weinberger Date: Wed, 8 Jun 2011 20:56:29 +0800 Subject: crypto: UML build fixes CRYPTO_GHASH_CLMUL_NI_INTEL and CRYPTO_AES_NI_INTEL cannot be used on UML. Commit 3e02e5cb and 54b6a1b enabled them by accident. Signed-off-by: Richard Weinberger Signed-off-by: Herbert Xu --- crypto/Kconfig | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crypto/Kconfig b/crypto/Kconfig index 87b22ca9c22..2af81552d65 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -458,7 +458,7 @@ config CRYPTO_WP512 config CRYPTO_GHASH_CLMUL_NI_INTEL tristate "GHASH digest algorithm (CLMUL-NI accelerated)" - depends on (X86 || UML_X86) && 64BIT + depends on X86 && 64BIT select CRYPTO_SHASH select CRYPTO_CRYPTD help @@ -533,7 +533,7 @@ config CRYPTO_AES_X86_64 config CRYPTO_AES_NI_INTEL tristate "AES cipher algorithms (AES-NI)" - depends on (X86 || UML_X86) + depends on X86 select CRYPTO_AES_X86_64 if 64BIT select CRYPTO_AES_586 if !64BIT select CRYPTO_CRYPTD -- cgit v1.2.3 From 6ba1a31ef146f25a457968c00c8c10bbe1a3ccfe Mon Sep 17 00:00:00 2001 From: Julia Lawall Date: Wed, 8 Jun 2011 20:59:38 +0800 Subject: hwrng: omap - add missing clk_put Convert a return to a jump to an existing label that calls clk_put. A simplified version of the semantic match that finds this problem is as follows: (http://coccinelle.lip6.fr/) // @r exists@ expression e1,e2; statement S; @@ e1 = clk_get@p1(...); ... when != e1 = e2 when != clk_put(e1) when any if (...) { ... when != clk_put(e1) when != if (...) { ... clk_put(e1) ... } * return@p3 ...; } else S // Signed-off-by: Julia Lawall Signed-off-by: Matt Mackall Signed-off-by: Herbert Xu --- drivers/char/hw_random/omap-rng.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/drivers/char/hw_random/omap-rng.c b/drivers/char/hw_random/omap-rng.c index 2cc755a6430..b757fac3cd1 100644 --- a/drivers/char/hw_random/omap-rng.c +++ b/drivers/char/hw_random/omap-rng.c @@ -113,8 +113,10 @@ static int __devinit omap_rng_probe(struct platform_device *pdev) res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - if (!res) - return -ENOENT; + if (!res) { + ret = -ENOENT; + goto err_region; + } if (!request_mem_region(res->start, resource_size(res), pdev->name)) { ret = -EBUSY; -- cgit v1.2.3 From 2725ae17591bf771608fe62c54f7b2badcf7492f Mon Sep 17 00:00:00 2001 From: Julia Lawall Date: Wed, 8 Jun 2011 21:00:55 +0800 Subject: hwrng: nomadik - add missing clk_put Jump to the end of the function for the clk_disable and clk_put rather than returning directly. A simplified version of the semantic match that finds this problem is as follows: (http://coccinelle.lip6.fr/) // @r exists@ expression e1,e2; statement S; @@ e1 = clk_get@p1(...); ... when != e1 = e2 when != clk_put(e1) when any if (...) { ... when != clk_put(e1) when != if (...) { ... clk_put(e1) ... } * return@p3 ...; } else S // Signed-off-by: Julia Lawall Signed-off-by: Matt Mackall Signed-off-by: Herbert Xu --- drivers/char/hw_random/nomadik-rng.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/char/hw_random/nomadik-rng.c b/drivers/char/hw_random/nomadik-rng.c index dd1d143eb8e..52e08ca3ccd 100644 --- a/drivers/char/hw_random/nomadik-rng.c +++ b/drivers/char/hw_random/nomadik-rng.c @@ -55,7 +55,7 @@ static int nmk_rng_probe(struct amba_device *dev, const struct amba_id *id) ret = amba_request_regions(dev, dev->dev.init_name); if (ret) - return ret; + goto out_clk; ret = -ENOMEM; base = ioremap(dev->res.start, resource_size(&dev->res)); if (!base) @@ -70,6 +70,7 @@ out_unmap: iounmap(base); out_release: amba_release_regions(dev); +out_clk: clk_disable(rng_clk); clk_put(rng_clk); return ret; -- cgit v1.2.3 From 033590fc9d38bb82b904439daada78f930c070db Mon Sep 17 00:00:00 2001 From: H Hartley Sweeten Date: Wed, 8 Jun 2011 21:05:51 +0800 Subject: hwrng: timeriomem - use resource_size Signed-off-by: H Hartley Sweeten Acked-by: Matt Mackall Signed-off-by: Herbert Xu --- drivers/char/hw_random/timeriomem-rng.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/drivers/char/hw_random/timeriomem-rng.c b/drivers/char/hw_random/timeriomem-rng.c index a94e930575f..a8428e6f64a 100644 --- a/drivers/char/hw_random/timeriomem-rng.c +++ b/drivers/char/hw_random/timeriomem-rng.c @@ -100,8 +100,7 @@ static int __devinit timeriomem_rng_probe(struct platform_device *pdev) timeriomem_rng_data = pdev->dev.platform_data; - timeriomem_rng_data->address = ioremap(res->start, - res->end - res->start + 1); + timeriomem_rng_data->address = ioremap(res->start, resource_size(res)); if (!timeriomem_rng_data->address) return -EIO; -- cgit v1.2.3 From 0e87b159b31e551fe5447dcafc5769c05dc5e404 Mon Sep 17 00:00:00 2001 From: Dmitry Kasatkin Date: Thu, 2 Jun 2011 21:10:03 +0300 Subject: crypto: omap-sham - remove extra reference Request pointer is already available in the function. Signed-off-by: Dmitry Kasatkin Signed-off-by: Herbert Xu --- drivers/crypto/omap-sham.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/crypto/omap-sham.c b/drivers/crypto/omap-sham.c index ba8f1ea84c5..8a45fb71325 100644 --- a/drivers/crypto/omap-sham.c +++ b/drivers/crypto/omap-sham.c @@ -639,7 +639,7 @@ static void omap_sham_finish_req(struct ahash_request *req, int err) struct omap_sham_dev *dd = ctx->dd; if (!err) { - omap_sham_copy_hash(ctx->dd->req, 1); + omap_sham_copy_hash(req, 1); if (ctx->flags & FLAGS_FINAL) err = omap_sham_finish(req); } else { -- cgit v1.2.3 From c3304721ed666804395fc340f1aa347b18f1dda0 Mon Sep 17 00:00:00 2001 From: Dmitry Kasatkin Date: Thu, 2 Jun 2011 21:10:04 +0300 Subject: crypto: omap-sham - remove unused code Signed-off-by: Dmitry Kasatkin Signed-off-by: Herbert Xu --- drivers/crypto/omap-sham.c | 5 ----- 1 file changed, 5 deletions(-) diff --git a/drivers/crypto/omap-sham.c b/drivers/crypto/omap-sham.c index 8a45fb71325..ac12a608d50 100644 --- a/drivers/crypto/omap-sham.c +++ b/drivers/crypto/omap-sham.c @@ -658,7 +658,6 @@ static int omap_sham_handle_queue(struct omap_sham_dev *dd, { struct crypto_async_request *async_req, *backlog; struct omap_sham_reqctx *ctx; - struct ahash_request *prev_req; unsigned long flags; int err = 0, ret = 0; @@ -682,16 +681,12 @@ static int omap_sham_handle_queue(struct omap_sham_dev *dd, backlog->complete(backlog, -EINPROGRESS); req = ahash_request_cast(async_req); - - prev_req = dd->req; dd->req = req; - ctx = ahash_request_ctx(req); dev_dbg(dd->dev, "handling new req, op: %lu, nbytes: %d\n", ctx->op, req->nbytes); - err = omap_sham_hw_init(dd); if (err) goto err1; -- cgit v1.2.3 From ea1fd2246f3c3c6b739529db2a547fa080cf09a3 Mon Sep 17 00:00:00 2001 From: Dmitry Kasatkin Date: Thu, 2 Jun 2011 21:10:05 +0300 Subject: crypto: omap-sham - replace flags bit mask with bit number Flags mask cannot be used with atomic bit operations. This patch changes masks to bit numbers. Atomic bit operations will be used by following patches. Signed-off-by: Dmitry Kasatkin Signed-off-by: Herbert Xu --- drivers/crypto/omap-sham.c | 102 +++++++++++++++++++++++---------------------- 1 file changed, 52 insertions(+), 50 deletions(-) diff --git a/drivers/crypto/omap-sham.c b/drivers/crypto/omap-sham.c index ac12a608d50..64698adec0c 100644 --- a/drivers/crypto/omap-sham.c +++ b/drivers/crypto/omap-sham.c @@ -72,17 +72,19 @@ #define DEFAULT_TIMEOUT_INTERVAL HZ -#define FLAGS_FINUP 0x0002 -#define FLAGS_FINAL 0x0004 -#define FLAGS_SG 0x0008 -#define FLAGS_SHA1 0x0010 -#define FLAGS_DMA_ACTIVE 0x0020 -#define FLAGS_OUTPUT_READY 0x0040 -#define FLAGS_INIT 0x0100 -#define FLAGS_CPU 0x0200 -#define FLAGS_HMAC 0x0400 -#define FLAGS_ERROR 0x0800 -#define FLAGS_BUSY 0x1000 +/* mostly device flags */ +#define FLAGS_BUSY 0 +#define FLAGS_FINAL 1 +#define FLAGS_DMA_ACTIVE 2 +#define FLAGS_OUTPUT_READY 3 +#define FLAGS_INIT 4 +#define FLAGS_CPU 5 +/* context flags */ +#define FLAGS_FINUP 16 +#define FLAGS_SG 17 +#define FLAGS_SHA1 18 +#define FLAGS_HMAC 19 +#define FLAGS_ERROR 20 #define OP_UPDATE 1 #define OP_FINAL 2 @@ -223,7 +225,7 @@ static void omap_sham_copy_ready_hash(struct ahash_request *req) if (!hash) return; - if (likely(ctx->flags & FLAGS_SHA1)) { + if (likely(ctx->flags & BIT(FLAGS_SHA1))) { /* SHA1 results are in big endian */ for (i = 0; i < SHA1_DIGEST_SIZE / sizeof(u32); i++) hash[i] = be32_to_cpu(in[i]); @@ -238,7 +240,7 @@ static int omap_sham_hw_init(struct omap_sham_dev *dd) { clk_enable(dd->iclk); - if (!(dd->flags & FLAGS_INIT)) { + if (!(dd->flags & BIT(FLAGS_INIT))) { omap_sham_write_mask(dd, SHA_REG_MASK, SHA_REG_MASK_SOFTRESET, SHA_REG_MASK_SOFTRESET); @@ -246,7 +248,7 @@ static int omap_sham_hw_init(struct omap_sham_dev *dd) SHA_REG_SYSSTATUS_RESETDONE)) return -ETIMEDOUT; - dd->flags |= FLAGS_INIT; + dd->flags |= BIT(FLAGS_INIT); dd->err = 0; } @@ -269,7 +271,7 @@ static void omap_sham_write_ctrl(struct omap_sham_dev *dd, size_t length, * Setting ALGO_CONST only for the first iteration * and CLOSE_HASH only for the last one. */ - if (ctx->flags & FLAGS_SHA1) + if (ctx->flags & BIT(FLAGS_SHA1)) val |= SHA_REG_CTRL_ALGO; if (!ctx->digcnt) val |= SHA_REG_CTRL_ALGO_CONST; @@ -301,7 +303,7 @@ static int omap_sham_xmit_cpu(struct omap_sham_dev *dd, const u8 *buf, return -ETIMEDOUT; if (final) - ctx->flags |= FLAGS_FINAL; /* catch last interrupt */ + ctx->flags |= BIT(FLAGS_FINAL); /* catch last interrupt */ len32 = DIV_ROUND_UP(length, sizeof(u32)); @@ -334,9 +336,9 @@ static int omap_sham_xmit_dma(struct omap_sham_dev *dd, dma_addr_t dma_addr, ctx->digcnt += length; if (final) - ctx->flags |= FLAGS_FINAL; /* catch last interrupt */ + ctx->flags |= BIT(FLAGS_FINAL); /* catch last interrupt */ - dd->flags |= FLAGS_DMA_ACTIVE; + dd->flags |= BIT(FLAGS_DMA_ACTIVE); omap_start_dma(dd->dma_lch); @@ -392,7 +394,7 @@ static int omap_sham_xmit_dma_map(struct omap_sham_dev *dd, return -EINVAL; } - ctx->flags &= ~FLAGS_SG; + ctx->flags &= ~BIT(FLAGS_SG); /* next call does not fail... so no unmap in the case of error */ return omap_sham_xmit_dma(dd, ctx->dma_addr, length, final); @@ -406,7 +408,7 @@ static int omap_sham_update_dma_slow(struct omap_sham_dev *dd) omap_sham_append_sg(ctx); - final = (ctx->flags & FLAGS_FINUP) && !ctx->total; + final = (ctx->flags & BIT(FLAGS_FINUP)) && !ctx->total; dev_dbg(dd->dev, "slow: bufcnt: %u, digcnt: %d, final: %d\n", ctx->bufcnt, ctx->digcnt, final); @@ -452,7 +454,7 @@ static int omap_sham_update_dma_start(struct omap_sham_dev *dd) length = min(ctx->total, sg->length); if (sg_is_last(sg)) { - if (!(ctx->flags & FLAGS_FINUP)) { + if (!(ctx->flags & BIT(FLAGS_FINUP))) { /* not last sg must be SHA1_MD5_BLOCK_SIZE aligned */ tail = length & (SHA1_MD5_BLOCK_SIZE - 1); /* without finup() we need one block to close hash */ @@ -467,12 +469,12 @@ static int omap_sham_update_dma_start(struct omap_sham_dev *dd) return -EINVAL; } - ctx->flags |= FLAGS_SG; + ctx->flags |= BIT(FLAGS_SG); ctx->total -= length; ctx->offset = length; /* offset where to start slow */ - final = (ctx->flags & FLAGS_FINUP) && !ctx->total; + final = (ctx->flags & BIT(FLAGS_FINUP)) && !ctx->total; /* next call does not fail... so no unmap in the case of error */ return omap_sham_xmit_dma(dd, sg_dma_address(ctx->sg), length, final); @@ -495,7 +497,7 @@ static int omap_sham_update_dma_stop(struct omap_sham_dev *dd) struct omap_sham_reqctx *ctx = ahash_request_ctx(dd->req); omap_stop_dma(dd->dma_lch); - if (ctx->flags & FLAGS_SG) { + if (ctx->flags & BIT(FLAGS_SG)) { dma_unmap_sg(dd->dev, ctx->sg, 1, DMA_TO_DEVICE); if (ctx->sg->length == ctx->offset) { ctx->sg = sg_next(ctx->sg); @@ -537,18 +539,18 @@ static int omap_sham_init(struct ahash_request *req) crypto_ahash_digestsize(tfm)); if (crypto_ahash_digestsize(tfm) == SHA1_DIGEST_SIZE) - ctx->flags |= FLAGS_SHA1; + ctx->flags |= BIT(FLAGS_SHA1); ctx->bufcnt = 0; ctx->digcnt = 0; ctx->buflen = BUFLEN; - if (tctx->flags & FLAGS_HMAC) { + if (tctx->flags & BIT(FLAGS_HMAC)) { struct omap_sham_hmac_ctx *bctx = tctx->base; memcpy(ctx->buffer, bctx->ipad, SHA1_MD5_BLOCK_SIZE); ctx->bufcnt = SHA1_MD5_BLOCK_SIZE; - ctx->flags |= FLAGS_HMAC; + ctx->flags |= BIT(FLAGS_HMAC); } return 0; @@ -562,9 +564,9 @@ static int omap_sham_update_req(struct omap_sham_dev *dd) int err; dev_dbg(dd->dev, "update_req: total: %u, digcnt: %d, finup: %d\n", - ctx->total, ctx->digcnt, (ctx->flags & FLAGS_FINUP) != 0); + ctx->total, ctx->digcnt, (ctx->flags & BIT(FLAGS_FINUP)) != 0); - if (ctx->flags & FLAGS_CPU) + if (ctx->flags & BIT(FLAGS_CPU)) err = omap_sham_update_cpu(dd); else err = omap_sham_update_dma_start(dd); @@ -624,7 +626,7 @@ static int omap_sham_finish(struct ahash_request *req) if (ctx->digcnt) { omap_sham_copy_ready_hash(req); - if (ctx->flags & FLAGS_HMAC) + if (ctx->flags & BIT(FLAGS_HMAC)) err = omap_sham_finish_hmac(req); } @@ -640,14 +642,14 @@ static void omap_sham_finish_req(struct ahash_request *req, int err) if (!err) { omap_sham_copy_hash(req, 1); - if (ctx->flags & FLAGS_FINAL) + if (ctx->flags & BIT(FLAGS_FINAL)) err = omap_sham_finish(req); } else { - ctx->flags |= FLAGS_ERROR; + ctx->flags |= BIT(FLAGS_ERROR); } clk_disable(dd->iclk); - dd->flags &= ~FLAGS_BUSY; + dd->flags &= ~BIT(FLAGS_BUSY); if (req->base.complete) req->base.complete(&req->base, err); @@ -664,14 +666,14 @@ static int omap_sham_handle_queue(struct omap_sham_dev *dd, spin_lock_irqsave(&dd->lock, flags); if (req) ret = ahash_enqueue_request(&dd->queue, req); - if (dd->flags & FLAGS_BUSY) { + if (dd->flags & BIT(FLAGS_BUSY)) { spin_unlock_irqrestore(&dd->lock, flags); return ret; } backlog = crypto_get_backlog(&dd->queue); async_req = crypto_dequeue_request(&dd->queue); if (async_req) - dd->flags |= FLAGS_BUSY; + dd->flags |= BIT(FLAGS_BUSY); spin_unlock_irqrestore(&dd->lock, flags); if (!async_req) @@ -707,7 +709,7 @@ static int omap_sham_handle_queue(struct omap_sham_dev *dd, if (ctx->op == OP_UPDATE) { err = omap_sham_update_req(dd); - if (err != -EINPROGRESS && (ctx->flags & FLAGS_FINUP)) + if (err != -EINPROGRESS && (ctx->flags & BIT(FLAGS_FINUP))) /* no final() after finup() */ err = omap_sham_final_req(dd); } else if (ctx->op == OP_FINAL) { @@ -747,7 +749,7 @@ static int omap_sham_update(struct ahash_request *req) ctx->sg = req->src; ctx->offset = 0; - if (ctx->flags & FLAGS_FINUP) { + if (ctx->flags & BIT(FLAGS_FINUP)) { if ((ctx->digcnt + ctx->bufcnt + ctx->total) < 9) { /* * OMAP HW accel works only with buffers >= 9 @@ -760,7 +762,7 @@ static int omap_sham_update(struct ahash_request *req) /* * faster to use CPU for short transfers */ - ctx->flags |= FLAGS_CPU; + ctx->flags |= BIT(FLAGS_CPU); } } else if (ctx->bufcnt + ctx->total < ctx->buflen) { omap_sham_append_sg(ctx); @@ -797,9 +799,9 @@ static int omap_sham_final(struct ahash_request *req) { struct omap_sham_reqctx *ctx = ahash_request_ctx(req); - ctx->flags |= FLAGS_FINUP; + ctx->flags |= BIT(FLAGS_FINUP); - if (ctx->flags & FLAGS_ERROR) + if (ctx->flags & BIT(FLAGS_ERROR)) return 0; /* uncompleted hash is not needed */ /* OMAP HW accel works only with buffers >= 9 */ @@ -818,7 +820,7 @@ static int omap_sham_finup(struct ahash_request *req) struct omap_sham_reqctx *ctx = ahash_request_ctx(req); int err1, err2; - ctx->flags |= FLAGS_FINUP; + ctx->flags |= BIT(FLAGS_FINUP); err1 = omap_sham_update(req); if (err1 == -EINPROGRESS || err1 == -EBUSY) @@ -890,7 +892,7 @@ static int omap_sham_cra_init_alg(struct crypto_tfm *tfm, const char *alg_base) if (alg_base) { struct omap_sham_hmac_ctx *bctx = tctx->base; - tctx->flags |= FLAGS_HMAC; + tctx->flags |= BIT(FLAGS_HMAC); bctx->shash = crypto_alloc_shash(alg_base, 0, CRYPTO_ALG_NEED_FALLBACK); if (IS_ERR(bctx->shash)) { @@ -927,7 +929,7 @@ static void omap_sham_cra_exit(struct crypto_tfm *tfm) crypto_free_shash(tctx->fallback); tctx->fallback = NULL; - if (tctx->flags & FLAGS_HMAC) { + if (tctx->flags & BIT(FLAGS_HMAC)) { struct omap_sham_hmac_ctx *bctx = tctx->base; crypto_free_shash(bctx->shash); } @@ -1035,13 +1037,13 @@ static void omap_sham_done_task(unsigned long data) struct omap_sham_reqctx *ctx = ahash_request_ctx(req); int ready = 0, err = 0; - if (ctx->flags & FLAGS_OUTPUT_READY) { - ctx->flags &= ~FLAGS_OUTPUT_READY; + if (ctx->flags & BIT(FLAGS_OUTPUT_READY)) { + ctx->flags &= ~BIT(FLAGS_OUTPUT_READY); ready = 1; } - if (dd->flags & FLAGS_DMA_ACTIVE) { - dd->flags &= ~FLAGS_DMA_ACTIVE; + if (dd->flags & BIT(FLAGS_DMA_ACTIVE)) { + dd->flags &= ~BIT(FLAGS_DMA_ACTIVE); omap_sham_update_dma_stop(dd); if (!dd->err) err = omap_sham_update_dma_start(dd); @@ -1075,7 +1077,7 @@ static irqreturn_t omap_sham_irq(int irq, void *dev_id) return IRQ_HANDLED; } - if (unlikely(ctx->flags & FLAGS_FINAL)) + if (unlikely(ctx->flags & BIT(FLAGS_FINAL))) /* final -> allow device to go to power-saving mode */ omap_sham_write_mask(dd, SHA_REG_CTRL, 0, SHA_REG_CTRL_LENGTH); @@ -1083,7 +1085,7 @@ static irqreturn_t omap_sham_irq(int irq, void *dev_id) SHA_REG_CTRL_OUTPUT_READY); omap_sham_read(dd, SHA_REG_CTRL); - ctx->flags |= FLAGS_OUTPUT_READY; + ctx->flags |= BIT(FLAGS_OUTPUT_READY); dd->err = 0; tasklet_schedule(&dd->done_task); @@ -1097,7 +1099,7 @@ static void omap_sham_dma_callback(int lch, u16 ch_status, void *data) if (ch_status != OMAP_DMA_BLOCK_IRQ) { pr_err("omap-sham DMA error status: 0x%hx\n", ch_status); dd->err = -EIO; - dd->flags &= ~FLAGS_INIT; /* request to re-initialize */ + dd->flags &= ~BIT(FLAGS_INIT); /* request to re-initialize */ } tasklet_schedule(&dd->done_task); -- cgit v1.2.3 From a929cbeef7d8e5c3603a6f43b977c03c922aada3 Mon Sep 17 00:00:00 2001 From: Dmitry Kasatkin Date: Thu, 2 Jun 2011 21:10:06 +0300 Subject: crypto: omap-sham - replace flags operation with atomic bit operations Some flags are changed in interrupt handlers and verified in the tasklet. There might be a race condition when tasklet is interrupted or another cpu/core will run IRQ handler and tasklet in parallel. Atomic bitops functions are now used instead of bitmask operations. Signed-off-by: Dmitry Kasatkin Signed-off-by: Herbert Xu --- drivers/crypto/omap-sham.c | 29 +++++++++++++---------------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/drivers/crypto/omap-sham.c b/drivers/crypto/omap-sham.c index 64698adec0c..208404e20ef 100644 --- a/drivers/crypto/omap-sham.c +++ b/drivers/crypto/omap-sham.c @@ -240,7 +240,7 @@ static int omap_sham_hw_init(struct omap_sham_dev *dd) { clk_enable(dd->iclk); - if (!(dd->flags & BIT(FLAGS_INIT))) { + if (!test_bit(FLAGS_INIT, &dd->flags)) { omap_sham_write_mask(dd, SHA_REG_MASK, SHA_REG_MASK_SOFTRESET, SHA_REG_MASK_SOFTRESET); @@ -248,7 +248,7 @@ static int omap_sham_hw_init(struct omap_sham_dev *dd) SHA_REG_SYSSTATUS_RESETDONE)) return -ETIMEDOUT; - dd->flags |= BIT(FLAGS_INIT); + set_bit(FLAGS_INIT, &dd->flags); dd->err = 0; } @@ -303,7 +303,7 @@ static int omap_sham_xmit_cpu(struct omap_sham_dev *dd, const u8 *buf, return -ETIMEDOUT; if (final) - ctx->flags |= BIT(FLAGS_FINAL); /* catch last interrupt */ + set_bit(FLAGS_FINAL, &ctx->flags); /* catch last interrupt */ len32 = DIV_ROUND_UP(length, sizeof(u32)); @@ -336,9 +336,9 @@ static int omap_sham_xmit_dma(struct omap_sham_dev *dd, dma_addr_t dma_addr, ctx->digcnt += length; if (final) - ctx->flags |= BIT(FLAGS_FINAL); /* catch last interrupt */ + set_bit(FLAGS_FINAL, &ctx->flags); /* catch last interrupt */ - dd->flags |= BIT(FLAGS_DMA_ACTIVE); + set_bit(FLAGS_DMA_ACTIVE, &dd->flags); omap_start_dma(dd->dma_lch); @@ -642,7 +642,7 @@ static void omap_sham_finish_req(struct ahash_request *req, int err) if (!err) { omap_sham_copy_hash(req, 1); - if (ctx->flags & BIT(FLAGS_FINAL)) + if (test_bit(FLAGS_FINAL, &ctx->flags)) err = omap_sham_finish(req); } else { ctx->flags |= BIT(FLAGS_ERROR); @@ -666,14 +666,14 @@ static int omap_sham_handle_queue(struct omap_sham_dev *dd, spin_lock_irqsave(&dd->lock, flags); if (req) ret = ahash_enqueue_request(&dd->queue, req); - if (dd->flags & BIT(FLAGS_BUSY)) { + if (test_bit(FLAGS_BUSY, &dd->flags)) { spin_unlock_irqrestore(&dd->lock, flags); return ret; } backlog = crypto_get_backlog(&dd->queue); async_req = crypto_dequeue_request(&dd->queue); if (async_req) - dd->flags |= BIT(FLAGS_BUSY); + set_bit(FLAGS_BUSY, &dd->flags); spin_unlock_irqrestore(&dd->lock, flags); if (!async_req) @@ -1037,13 +1037,10 @@ static void omap_sham_done_task(unsigned long data) struct omap_sham_reqctx *ctx = ahash_request_ctx(req); int ready = 0, err = 0; - if (ctx->flags & BIT(FLAGS_OUTPUT_READY)) { - ctx->flags &= ~BIT(FLAGS_OUTPUT_READY); + if (test_and_clear_bit(FLAGS_OUTPUT_READY, &ctx->flags)) ready = 1; - } - if (dd->flags & BIT(FLAGS_DMA_ACTIVE)) { - dd->flags &= ~BIT(FLAGS_DMA_ACTIVE); + if (test_and_clear_bit(FLAGS_DMA_ACTIVE, &dd->flags)) { omap_sham_update_dma_stop(dd); if (!dd->err) err = omap_sham_update_dma_start(dd); @@ -1077,7 +1074,7 @@ static irqreturn_t omap_sham_irq(int irq, void *dev_id) return IRQ_HANDLED; } - if (unlikely(ctx->flags & BIT(FLAGS_FINAL))) + if (unlikely(test_bit(FLAGS_FINAL, &ctx->flags))) /* final -> allow device to go to power-saving mode */ omap_sham_write_mask(dd, SHA_REG_CTRL, 0, SHA_REG_CTRL_LENGTH); @@ -1085,7 +1082,7 @@ static irqreturn_t omap_sham_irq(int irq, void *dev_id) SHA_REG_CTRL_OUTPUT_READY); omap_sham_read(dd, SHA_REG_CTRL); - ctx->flags |= BIT(FLAGS_OUTPUT_READY); + set_bit(FLAGS_OUTPUT_READY, &ctx->flags); dd->err = 0; tasklet_schedule(&dd->done_task); @@ -1099,7 +1096,7 @@ static void omap_sham_dma_callback(int lch, u16 ch_status, void *data) if (ch_status != OMAP_DMA_BLOCK_IRQ) { pr_err("omap-sham DMA error status: 0x%hx\n", ch_status); dd->err = -EIO; - dd->flags &= ~BIT(FLAGS_INIT); /* request to re-initialize */ + clear_bit(FLAGS_INIT, &dd->flags);/* request to re-initialize */ } tasklet_schedule(&dd->done_task); -- cgit v1.2.3 From ed3ea9a82b4c63e591ceaa065b2e04a3fe946265 Mon Sep 17 00:00:00 2001 From: Dmitry Kasatkin Date: Thu, 2 Jun 2011 21:10:07 +0300 Subject: crypto: omap-sham - move some flags to device context Couple of context flags have been moved to device flags. IRQ and tasklet handlers does not need to access request context anymore. Signed-off-by: Dmitry Kasatkin Signed-off-by: Herbert Xu --- drivers/crypto/omap-sham.c | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/drivers/crypto/omap-sham.c b/drivers/crypto/omap-sham.c index 208404e20ef..b959dc6dbbd 100644 --- a/drivers/crypto/omap-sham.c +++ b/drivers/crypto/omap-sham.c @@ -303,7 +303,7 @@ static int omap_sham_xmit_cpu(struct omap_sham_dev *dd, const u8 *buf, return -ETIMEDOUT; if (final) - set_bit(FLAGS_FINAL, &ctx->flags); /* catch last interrupt */ + set_bit(FLAGS_FINAL, &dd->flags); /* catch last interrupt */ len32 = DIV_ROUND_UP(length, sizeof(u32)); @@ -336,7 +336,7 @@ static int omap_sham_xmit_dma(struct omap_sham_dev *dd, dma_addr_t dma_addr, ctx->digcnt += length; if (final) - set_bit(FLAGS_FINAL, &ctx->flags); /* catch last interrupt */ + set_bit(FLAGS_FINAL, &dd->flags); /* catch last interrupt */ set_bit(FLAGS_DMA_ACTIVE, &dd->flags); @@ -642,7 +642,7 @@ static void omap_sham_finish_req(struct ahash_request *req, int err) if (!err) { omap_sham_copy_hash(req, 1); - if (test_bit(FLAGS_FINAL, &ctx->flags)) + if (test_bit(FLAGS_FINAL, &dd->flags)) err = omap_sham_finish(req); } else { ctx->flags |= BIT(FLAGS_ERROR); @@ -1034,10 +1034,9 @@ static void omap_sham_done_task(unsigned long data) { struct omap_sham_dev *dd = (struct omap_sham_dev *)data; struct ahash_request *req = dd->req; - struct omap_sham_reqctx *ctx = ahash_request_ctx(req); int ready = 0, err = 0; - if (test_and_clear_bit(FLAGS_OUTPUT_READY, &ctx->flags)) + if (test_and_clear_bit(FLAGS_OUTPUT_READY, &dd->flags)) ready = 1; if (test_and_clear_bit(FLAGS_DMA_ACTIVE, &dd->flags)) { @@ -1067,14 +1066,8 @@ static void omap_sham_queue_task(unsigned long data) static irqreturn_t omap_sham_irq(int irq, void *dev_id) { struct omap_sham_dev *dd = dev_id; - struct omap_sham_reqctx *ctx = ahash_request_ctx(dd->req); - - if (!ctx) { - dev_err(dd->dev, "unknown interrupt.\n"); - return IRQ_HANDLED; - } - if (unlikely(test_bit(FLAGS_FINAL, &ctx->flags))) + if (unlikely(test_bit(FLAGS_FINAL, &dd->flags))) /* final -> allow device to go to power-saving mode */ omap_sham_write_mask(dd, SHA_REG_CTRL, 0, SHA_REG_CTRL_LENGTH); @@ -1082,7 +1075,7 @@ static irqreturn_t omap_sham_irq(int irq, void *dev_id) SHA_REG_CTRL_OUTPUT_READY); omap_sham_read(dd, SHA_REG_CTRL); - set_bit(FLAGS_OUTPUT_READY, &ctx->flags); + set_bit(FLAGS_OUTPUT_READY, &dd->flags); dd->err = 0; tasklet_schedule(&dd->done_task); -- cgit v1.2.3 From 171cb9a2807588630fed401f3e0b8ae22d91595f Mon Sep 17 00:00:00 2001 From: Dmitry Kasatkin Date: Thu, 2 Jun 2011 21:10:08 +0300 Subject: crypto: omap-sham - remove unnecessary local variable Signed-off-by: Dmitry Kasatkin Signed-off-by: Herbert Xu --- drivers/crypto/omap-sham.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/drivers/crypto/omap-sham.c b/drivers/crypto/omap-sham.c index b959dc6dbbd..84e5890c57e 100644 --- a/drivers/crypto/omap-sham.c +++ b/drivers/crypto/omap-sham.c @@ -1033,7 +1033,6 @@ static struct ahash_alg algs[] = { static void omap_sham_done_task(unsigned long data) { struct omap_sham_dev *dd = (struct omap_sham_dev *)data; - struct ahash_request *req = dd->req; int ready = 0, err = 0; if (test_and_clear_bit(FLAGS_OUTPUT_READY, &dd->flags)) @@ -1050,7 +1049,7 @@ static void omap_sham_done_task(unsigned long data) if (err != -EINPROGRESS && (ready || err)) { dev_dbg(dd->dev, "update done: err: %d\n", err); /* finish curent request */ - omap_sham_finish_req(req, err); + omap_sham_finish_req(dd->req, err); /* start new request */ omap_sham_handle_queue(dd, NULL); } -- cgit v1.2.3 From 6cb3ffe100ef4c6d5be0bfe4f8ac6d4227bb91bd Mon Sep 17 00:00:00 2001 From: Dmitry Kasatkin Date: Thu, 2 Jun 2011 21:10:09 +0300 Subject: crypto: omap-sham - remove dedicated queue handling tasklet Calling omap_sham_handle_queue from "done" tasklet should be done after irq scheduled tasklet completes. Having additional tasklet does not solve that issue because it might be execute before. So queue handling tasklet has been removed and functionality integrated into single tasklet. Signed-off-by: Dmitry Kasatkin Signed-off-by: Herbert Xu --- drivers/crypto/omap-sham.c | 24 +++++++++--------------- 1 file changed, 9 insertions(+), 15 deletions(-) diff --git a/drivers/crypto/omap-sham.c b/drivers/crypto/omap-sham.c index 84e5890c57e..24de4ace978 100644 --- a/drivers/crypto/omap-sham.c +++ b/drivers/crypto/omap-sham.c @@ -146,7 +146,6 @@ struct omap_sham_dev { int dma; int dma_lch; struct tasklet_struct done_task; - struct tasklet_struct queue_task; unsigned long flags; struct crypto_queue queue; @@ -653,6 +652,9 @@ static void omap_sham_finish_req(struct ahash_request *req, int err) if (req->base.complete) req->base.complete(&req->base, err); + + /* handle new request */ + tasklet_schedule(&dd->done_task); } static int omap_sham_handle_queue(struct omap_sham_dev *dd, @@ -716,11 +718,9 @@ static int omap_sham_handle_queue(struct omap_sham_dev *dd, err = omap_sham_final_req(dd); } err1: - if (err != -EINPROGRESS) { + if (err != -EINPROGRESS) /* done_task will not finish it, so do it here */ omap_sham_finish_req(req, err); - tasklet_schedule(&dd->queue_task); - } dev_dbg(dd->dev, "exit, err: %d\n", err); @@ -1035,6 +1035,11 @@ static void omap_sham_done_task(unsigned long data) struct omap_sham_dev *dd = (struct omap_sham_dev *)data; int ready = 0, err = 0; + if (!test_bit(FLAGS_BUSY, &dd->flags)) { + omap_sham_handle_queue(dd, NULL); + return; + } + if (test_and_clear_bit(FLAGS_OUTPUT_READY, &dd->flags)) ready = 1; @@ -1050,18 +1055,9 @@ static void omap_sham_done_task(unsigned long data) dev_dbg(dd->dev, "update done: err: %d\n", err); /* finish curent request */ omap_sham_finish_req(dd->req, err); - /* start new request */ - omap_sham_handle_queue(dd, NULL); } } -static void omap_sham_queue_task(unsigned long data) -{ - struct omap_sham_dev *dd = (struct omap_sham_dev *)data; - - omap_sham_handle_queue(dd, NULL); -} - static irqreturn_t omap_sham_irq(int irq, void *dev_id) { struct omap_sham_dev *dd = dev_id; @@ -1137,7 +1133,6 @@ static int __devinit omap_sham_probe(struct platform_device *pdev) INIT_LIST_HEAD(&dd->list); spin_lock_init(&dd->lock); tasklet_init(&dd->done_task, omap_sham_done_task, (unsigned long)dd); - tasklet_init(&dd->queue_task, omap_sham_queue_task, (unsigned long)dd); crypto_init_queue(&dd->queue, OMAP_SHAM_QUEUE_LENGTH); dd->irq = -1; @@ -1246,7 +1241,6 @@ static int __devexit omap_sham_remove(struct platform_device *pdev) for (i = 0; i < ARRAY_SIZE(algs); i++) crypto_unregister_ahash(&algs[i]); tasklet_kill(&dd->done_task); - tasklet_kill(&dd->queue_task); iounmap(dd->io_base); clk_put(dd->iclk); omap_sham_dma_cleanup(dd); -- cgit v1.2.3 From 6c63db821d4663c92cd326f6adb8f3d627971a53 Mon Sep 17 00:00:00 2001 From: Dmitry Kasatkin Date: Thu, 2 Jun 2011 21:10:10 +0300 Subject: crypto: omap-sham - irq and dma handling changes It could be a situation, that tasklet is executed twice because of certain delay between dma callback and irq handler execution. In that case, second tasklet execution could actually corrupt the data of the new started dma transactions. This patch improves tasklet logic and prevents above described cases. Signed-off-by: Dmitry Kasatkin Signed-off-by: Herbert Xu --- drivers/crypto/omap-sham.c | 40 +++++++++++++++++++++++++++------------- 1 file changed, 27 insertions(+), 13 deletions(-) diff --git a/drivers/crypto/omap-sham.c b/drivers/crypto/omap-sham.c index 24de4ace978..a8de7b890d4 100644 --- a/drivers/crypto/omap-sham.c +++ b/drivers/crypto/omap-sham.c @@ -79,6 +79,7 @@ #define FLAGS_OUTPUT_READY 3 #define FLAGS_INIT 4 #define FLAGS_CPU 5 +#define FLAGS_DMA_READY 6 /* context flags */ #define FLAGS_FINUP 16 #define FLAGS_SG 17 @@ -304,6 +305,8 @@ static int omap_sham_xmit_cpu(struct omap_sham_dev *dd, const u8 *buf, if (final) set_bit(FLAGS_FINAL, &dd->flags); /* catch last interrupt */ + set_bit(FLAGS_CPU, &dd->flags); + len32 = DIV_ROUND_UP(length, sizeof(u32)); for (count = 0; count < len32; count++) @@ -1033,29 +1036,39 @@ static struct ahash_alg algs[] = { static void omap_sham_done_task(unsigned long data) { struct omap_sham_dev *dd = (struct omap_sham_dev *)data; - int ready = 0, err = 0; + int err = 0; if (!test_bit(FLAGS_BUSY, &dd->flags)) { omap_sham_handle_queue(dd, NULL); return; } - if (test_and_clear_bit(FLAGS_OUTPUT_READY, &dd->flags)) - ready = 1; - - if (test_and_clear_bit(FLAGS_DMA_ACTIVE, &dd->flags)) { - omap_sham_update_dma_stop(dd); - if (!dd->err) + if (test_bit(FLAGS_CPU, &dd->flags)) { + if (test_and_clear_bit(FLAGS_OUTPUT_READY, &dd->flags)) + goto finish; + } else if (test_bit(FLAGS_DMA_READY, &dd->flags)) { + if (test_and_clear_bit(FLAGS_DMA_ACTIVE, &dd->flags)) { + omap_sham_update_dma_stop(dd); + if (dd->err) { + err = dd->err; + goto finish; + } + } + if (test_and_clear_bit(FLAGS_OUTPUT_READY, &dd->flags)) { + /* hash or semi-hash ready */ + clear_bit(FLAGS_DMA_READY, &dd->flags); err = omap_sham_update_dma_start(dd); + if (err != -EINPROGRESS) + goto finish; + } } - err = dd->err ? : err; + return; - if (err != -EINPROGRESS && (ready || err)) { - dev_dbg(dd->dev, "update done: err: %d\n", err); - /* finish curent request */ - omap_sham_finish_req(dd->req, err); - } +finish: + dev_dbg(dd->dev, "update done: err: %d\n", err); + /* finish curent request */ + omap_sham_finish_req(dd->req, err); } static irqreturn_t omap_sham_irq(int irq, void *dev_id) @@ -1087,6 +1100,7 @@ static void omap_sham_dma_callback(int lch, u16 ch_status, void *data) clear_bit(FLAGS_INIT, &dd->flags);/* request to re-initialize */ } + set_bit(FLAGS_DMA_READY, &dd->flags); tasklet_schedule(&dd->done_task); } -- cgit v1.2.3 From ed635cb85efbeebc974c1c477ddc971f5dcc962e Mon Sep 17 00:00:00 2001 From: Dmitry Kasatkin Date: Thu, 2 Jun 2011 21:10:11 +0300 Subject: crypto: omap-sham - irq handler must not clear error code Signed-off-by: Dmitry Kasatkin Signed-off-by: Herbert Xu --- drivers/crypto/omap-sham.c | 1 - 1 file changed, 1 deletion(-) diff --git a/drivers/crypto/omap-sham.c b/drivers/crypto/omap-sham.c index a8de7b890d4..7ca707536f8 100644 --- a/drivers/crypto/omap-sham.c +++ b/drivers/crypto/omap-sham.c @@ -1084,7 +1084,6 @@ static irqreturn_t omap_sham_irq(int irq, void *dev_id) omap_sham_read(dd, SHA_REG_CTRL); set_bit(FLAGS_OUTPUT_READY, &dd->flags); - dd->err = 0; tasklet_schedule(&dd->done_task); return IRQ_HANDLED; -- cgit v1.2.3 From 0efd4d8a6add75d6008a95fb628d376fb7cfcbe2 Mon Sep 17 00:00:00 2001 From: Dmitry Kasatkin Date: Thu, 2 Jun 2011 21:10:12 +0300 Subject: crypto: omap-sham - clear device flags when finishing request Signed-off-by: Dmitry Kasatkin Signed-off-by: Herbert Xu --- drivers/crypto/omap-sham.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/crypto/omap-sham.c b/drivers/crypto/omap-sham.c index 7ca707536f8..804c16bc2d6 100644 --- a/drivers/crypto/omap-sham.c +++ b/drivers/crypto/omap-sham.c @@ -650,8 +650,10 @@ static void omap_sham_finish_req(struct ahash_request *req, int err) ctx->flags |= BIT(FLAGS_ERROR); } + /* atomic operation is not needed here */ + dd->flags &= ~(BIT(FLAGS_BUSY) | BIT(FLAGS_FINAL) | BIT(FLAGS_CPU) | + BIT(FLAGS_DMA_READY) | BIT(FLAGS_OUTPUT_READY)); clk_disable(dd->iclk); - dd->flags &= ~BIT(FLAGS_BUSY); if (req->base.complete) req->base.complete(&req->base, err); -- cgit v1.2.3 From cd3f1d545c45489c88fbc71a82559664b8bf0bb2 Mon Sep 17 00:00:00 2001 From: Dmitry Kasatkin Date: Thu, 2 Jun 2011 21:10:13 +0300 Subject: crypto: omap-sham - do not schedule tasklet if there is no active requests Signed-off-by: Dmitry Kasatkin Signed-off-by: Herbert Xu --- drivers/crypto/omap-sham.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/drivers/crypto/omap-sham.c b/drivers/crypto/omap-sham.c index 804c16bc2d6..6399a8f1938 100644 --- a/drivers/crypto/omap-sham.c +++ b/drivers/crypto/omap-sham.c @@ -1085,6 +1085,11 @@ static irqreturn_t omap_sham_irq(int irq, void *dev_id) SHA_REG_CTRL_OUTPUT_READY); omap_sham_read(dd, SHA_REG_CTRL); + if (!test_bit(FLAGS_BUSY, &dd->flags)) { + dev_warn(dd->dev, "Interrupt when no active requests.\n"); + return IRQ_HANDLED; + } + set_bit(FLAGS_OUTPUT_READY, &dd->flags); tasklet_schedule(&dd->done_task); -- cgit v1.2.3 From fae366401bfcfe18b9260b9057f506c38c171b7c Mon Sep 17 00:00:00 2001 From: Mati Vait Date: Wed, 8 Jun 2011 21:23:40 +0800 Subject: crypto: crc32c - Fixed coding style issue Removed coding style issue: space before tabs. Signed-off-by: Mati Vait Signed-off-by: Herbert Xu --- crypto/crc32c.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/crypto/crc32c.c b/crypto/crc32c.c index de9e55c2979..3f9ad280105 100644 --- a/crypto/crc32c.c +++ b/crypto/crc32c.c @@ -224,11 +224,11 @@ static int crc32c_cra_init(struct crypto_tfm *tfm) static struct shash_alg alg = { .digestsize = CHKSUM_DIGEST_SIZE, .setkey = chksum_setkey, - .init = chksum_init, - .update = chksum_update, - .final = chksum_final, - .finup = chksum_finup, - .digest = chksum_digest, + .init = chksum_init, + .update = chksum_update, + .final = chksum_final, + .finup = chksum_finup, + .digest = chksum_digest, .descsize = sizeof(struct chksum_desc_ctx), .base = { .cra_name = "crc32c", -- cgit v1.2.3 From cfa2b54eca28103d07e5334aa06b78321fbe1d9a Mon Sep 17 00:00:00 2001 From: Mati Vait Date: Wed, 8 Jun 2011 21:26:00 +0800 Subject: crypto: arc4 - Fixed coding style issues Fixed coding style issues: unnecessary spaces, parentheses on wrong lines. Signed-off-by: Mati Vait Signed-off-by: Herbert Xu --- crypto/arc4.c | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/crypto/arc4.c b/crypto/arc4.c index 8be47e13a9e..0d12a96da1d 100644 --- a/crypto/arc4.c +++ b/crypto/arc4.c @@ -1,4 +1,4 @@ -/* +/* * Cryptographic API * * ARC4 Cipher Algorithm @@ -33,16 +33,15 @@ static int arc4_set_key(struct crypto_tfm *tfm, const u8 *in_key, ctx->x = 1; ctx->y = 0; - for(i = 0; i < 256; i++) + for (i = 0; i < 256; i++) ctx->S[i] = i; - for(i = 0; i < 256; i++) - { + for (i = 0; i < 256; i++) { u8 a = ctx->S[i]; j = (j + in_key[k] + a) & 0xff; ctx->S[i] = ctx->S[j]; ctx->S[j] = a; - if(++k >= key_len) + if (++k >= key_len) k = 0; } @@ -80,9 +79,9 @@ static struct crypto_alg arc4_alg = { .cra_u = { .cipher = { .cia_min_keysize = ARC4_MIN_KEY_SIZE, .cia_max_keysize = ARC4_MAX_KEY_SIZE, - .cia_setkey = arc4_set_key, - .cia_encrypt = arc4_crypt, - .cia_decrypt = arc4_crypt } } + .cia_setkey = arc4_set_key, + .cia_encrypt = arc4_crypt, + .cia_decrypt = arc4_crypt } } }; static int __init arc4_init(void) -- cgit v1.2.3 From 23457bc9566be64f33afdc159aadd42712673e0b Mon Sep 17 00:00:00 2001 From: Kim Phillips Date: Sun, 5 Jun 2011 16:42:54 -0500 Subject: crypto: caam - fix build warning when DEBUG_FS not configured drivers/crypto/caam/ctrl.c: In function 'caam_probe': drivers/crypto/caam/ctrl.c:55:23: warning: unused variable 'perfmon' Signed-off-by: Kim Phillips Signed-off-by: Herbert Xu --- drivers/crypto/caam/ctrl.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/crypto/caam/ctrl.c b/drivers/crypto/caam/ctrl.c index 9009713a3c2..fc2d9ed2247 100644 --- a/drivers/crypto/caam/ctrl.c +++ b/drivers/crypto/caam/ctrl.c @@ -52,9 +52,11 @@ static int caam_probe(struct platform_device *pdev) struct caam_ctrl __iomem *ctrl; struct caam_full __iomem *topregs; struct caam_drv_private *ctrlpriv; - struct caam_perfmon *perfmon; struct caam_deco **deco; u32 deconum; +#ifdef CONFIG_DEBUG_FS + struct caam_perfmon *perfmon; +#endif ctrlpriv = kzalloc(sizeof(struct caam_drv_private), GFP_KERNEL); if (!ctrlpriv) -- cgit v1.2.3 From c5697462ae94693764e468b701c616bbbd6f951c Mon Sep 17 00:00:00 2001 From: Josh Boyer Date: Mon, 27 Jun 2011 15:39:07 +0800 Subject: crypto: crypto4xx - Perform read/modify/write on device control register The Security function on the AMCC SoCs has multiple engines within a single MMIO range. The crypto driver currently enables the 3DES functionality by doing a blind write to the device control register. This can unintentionally disable other functions like the PKA or TRNG when the driver is loaded. Perform a read/modify/write to enable the 3DES function instead. Signed-off-by: Josh Boyer Signed-off-by: Herbert Xu --- drivers/crypto/amcc/crypto4xx_core.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/drivers/crypto/amcc/crypto4xx_core.c b/drivers/crypto/amcc/crypto4xx_core.c index 18912521a7a..1d103f997dc 100644 --- a/drivers/crypto/amcc/crypto4xx_core.c +++ b/drivers/crypto/amcc/crypto4xx_core.c @@ -51,6 +51,7 @@ static void crypto4xx_hw_init(struct crypto4xx_device *dev) union ce_io_threshold io_threshold; u32 rand_num; union ce_pe_dma_cfg pe_dma_cfg; + u32 device_ctrl; writel(PPC4XX_BYTE_ORDER, dev->ce_base + CRYPTO4XX_BYTE_ORDER_CFG); /* setup pe dma, include reset sg, pdr and pe, then release reset */ @@ -84,7 +85,9 @@ static void crypto4xx_hw_init(struct crypto4xx_device *dev) writel(ring_size.w, dev->ce_base + CRYPTO4XX_RING_SIZE); ring_ctrl.w = 0; writel(ring_ctrl.w, dev->ce_base + CRYPTO4XX_RING_CTRL); - writel(PPC4XX_DC_3DES_EN, dev->ce_base + CRYPTO4XX_DEVICE_CTRL); + device_ctrl = readl(dev->ce_base + CRYPTO4XX_DEVICE_CTRL); + device_ctrl |= PPC4XX_DC_3DES_EN; + writel(device_ctrl, dev->ce_base + CRYPTO4XX_DEVICE_CTRL); writel(dev->gdr_pa, dev->ce_base + CRYPTO4XX_GATH_RING_BASE); writel(dev->sdr_pa, dev->ce_base + CRYPTO4XX_SCAT_RING_BASE); part_ring_size.w = 0; -- cgit v1.2.3 From 52527cf3f95d437b02f54437d5c3d7fda06474d1 Mon Sep 17 00:00:00 2001 From: Josh Boyer Date: Mon, 27 Jun 2011 15:34:54 +0800 Subject: hwrng: ppc4xx - add support for ppc4xx TRNG Various PowerPC 4xx SoCs contain a TRNG embedded in the Security function. This adds a device driver for that TRNG. Signed-off-by: Josh Boyer Acked-by: Matt Mackall Signed-off-by: Herbert Xu --- drivers/char/hw_random/Kconfig | 12 +++ drivers/char/hw_random/Makefile | 1 + drivers/char/hw_random/ppc4xx-rng.c | 156 ++++++++++++++++++++++++++++++++++++ 3 files changed, 169 insertions(+) create mode 100644 drivers/char/hw_random/ppc4xx-rng.c diff --git a/drivers/char/hw_random/Kconfig b/drivers/char/hw_random/Kconfig index a60043b3e40..1d2ebc7a494 100644 --- a/drivers/char/hw_random/Kconfig +++ b/drivers/char/hw_random/Kconfig @@ -210,3 +210,15 @@ config HW_RANDOM_PICOXCELL module will be called picoxcell-rng. If unsure, say Y. + +config HW_RANDOM_PPC4XX + tristate "PowerPC 4xx generic true random number generator support" + depends on HW_RANDOM && PPC && 4xx + ---help--- + This driver provides the kernel-side support for the TRNG hardware + found in the security function of some PowerPC 4xx SoCs. + + To compile this driver as a module, choose M here: the + module will be called ppc4xx-rng. + + If unsure, say N. diff --git a/drivers/char/hw_random/Makefile b/drivers/char/hw_random/Makefile index 3db4eb8b19c..c88f244c8a7 100644 --- a/drivers/char/hw_random/Makefile +++ b/drivers/char/hw_random/Makefile @@ -20,3 +20,4 @@ obj-$(CONFIG_HW_RANDOM_MXC_RNGA) += mxc-rnga.o obj-$(CONFIG_HW_RANDOM_OCTEON) += octeon-rng.o obj-$(CONFIG_HW_RANDOM_NOMADIK) += nomadik-rng.o obj-$(CONFIG_HW_RANDOM_PICOXCELL) += picoxcell-rng.o +obj-$(CONFIG_HW_RANDOM_PPC4XX) += ppc4xx-rng.o diff --git a/drivers/char/hw_random/ppc4xx-rng.c b/drivers/char/hw_random/ppc4xx-rng.c new file mode 100644 index 00000000000..b8afa6a4ff6 --- /dev/null +++ b/drivers/char/hw_random/ppc4xx-rng.c @@ -0,0 +1,156 @@ +/* + * Generic PowerPC 44x RNG driver + * + * Copyright 2011 IBM Corporation + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the + * Free Software Foundation; version 2 of the License. + */ + +#include +#include +#include +#include +#include +#include +#include + +#define PPC4XX_TRNG_DEV_CTRL 0x60080 + +#define PPC4XX_TRNGE 0x00020000 +#define PPC4XX_TRNG_CTRL 0x0008 +#define PPC4XX_TRNG_CTRL_DALM 0x20 +#define PPC4XX_TRNG_STAT 0x0004 +#define PPC4XX_TRNG_STAT_B 0x1 +#define PPC4XX_TRNG_DATA 0x0000 + +#define MODULE_NAME "ppc4xx_rng" + +static int ppc4xx_rng_data_present(struct hwrng *rng, int wait) +{ + void __iomem *rng_regs = (void __iomem *) rng->priv; + int busy, i, present = 0; + + for (i = 0; i < 20; i++) { + busy = (in_le32(rng_regs + PPC4XX_TRNG_STAT) & PPC4XX_TRNG_STAT_B); + if (!busy || !wait) { + present = 1; + break; + } + udelay(10); + } + return present; +} + +static int ppc4xx_rng_data_read(struct hwrng *rng, u32 *data) +{ + void __iomem *rng_regs = (void __iomem *) rng->priv; + *data = in_le32(rng_regs + PPC4XX_TRNG_DATA); + return 4; +} + +static int ppc4xx_rng_enable(int enable) +{ + struct device_node *ctrl; + void __iomem *ctrl_reg; + int err = 0; + u32 val; + + /* Find the main crypto device node and map it to turn the TRNG on */ + ctrl = of_find_compatible_node(NULL, NULL, "amcc,ppc4xx-crypto"); + if (!ctrl) + return -ENODEV; + + ctrl_reg = of_iomap(ctrl, 0); + if (!ctrl_reg) { + err = -ENODEV; + goto out; + } + + val = in_le32(ctrl_reg + PPC4XX_TRNG_DEV_CTRL); + + if (enable) + val |= PPC4XX_TRNGE; + else + val = val & ~PPC4XX_TRNGE; + + out_le32(ctrl_reg + PPC4XX_TRNG_DEV_CTRL, val); + iounmap(ctrl_reg); + +out: + of_node_put(ctrl); + + return err; +} + +static struct hwrng ppc4xx_rng = { + .name = MODULE_NAME, + .data_present = ppc4xx_rng_data_present, + .data_read = ppc4xx_rng_data_read, +}; + +static int __devinit ppc4xx_rng_probe(struct platform_device *dev) +{ + void __iomem *rng_regs; + int err = 0; + + rng_regs = of_iomap(dev->dev.of_node, 0); + if (!rng_regs) + return -ENODEV; + + err = ppc4xx_rng_enable(1); + if (err) + return err; + + out_le32(rng_regs + PPC4XX_TRNG_CTRL, PPC4XX_TRNG_CTRL_DALM); + ppc4xx_rng.priv = (unsigned long) rng_regs; + + err = hwrng_register(&ppc4xx_rng); + + return err; +} + +static int __devexit ppc4xx_rng_remove(struct platform_device *dev) +{ + void __iomem *rng_regs = (void __iomem *) ppc4xx_rng.priv; + + hwrng_unregister(&ppc4xx_rng); + ppc4xx_rng_enable(0); + iounmap(rng_regs); + + return 0; +} + +static struct of_device_id ppc4xx_rng_match[] = { + { .compatible = "ppc4xx-rng", }, + { .compatible = "amcc,ppc460ex-rng", }, + { .compatible = "amcc,ppc440epx-rng", }, + {}, +}; + +static struct platform_driver ppc4xx_rng_driver = { + .driver = { + .name = MODULE_NAME, + .owner = THIS_MODULE, + .of_match_table = ppc4xx_rng_match, + }, + .probe = ppc4xx_rng_probe, + .remove = ppc4xx_rng_remove, +}; + +static int __init ppc4xx_rng_init(void) +{ + return platform_driver_register(&ppc4xx_rng_driver); +} +module_init(ppc4xx_rng_init); + +static void __exit ppc4xx_rng_exit(void) +{ + platform_driver_unregister(&ppc4xx_rng_driver); +} +module_exit(ppc4xx_rng_exit); + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Josh Boyer "); +MODULE_DESCRIPTION("HW RNG driver for PPC 4xx processors"); -- cgit v1.2.3 From 36ca239b245dbdec3b0c94976e956226bdd4108c Mon Sep 17 00:00:00 2001 From: Mandeep Singh Baines Date: Mon, 27 Jun 2011 15:41:56 +0800 Subject: crypto: sha1_generic - use SHA1_BLOCK_SIZE Modify sha1_update to use SHA1_BLOCK_SIZE. Signed-off-by: Mandeep Singh Baines Signed-off-by: Herbert Xu --- crypto/sha1_generic.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/crypto/sha1_generic.c b/crypto/sha1_generic.c index 0416091bf45..00ae60eb925 100644 --- a/crypto/sha1_generic.c +++ b/crypto/sha1_generic.c @@ -43,25 +43,26 @@ static int sha1_update(struct shash_desc *desc, const u8 *data, unsigned int partial, done; const u8 *src; - partial = sctx->count & 0x3f; + partial = sctx->count % SHA1_BLOCK_SIZE; sctx->count += len; done = 0; src = data; - if ((partial + len) > 63) { + if ((partial + len) >= SHA1_BLOCK_SIZE) { u32 temp[SHA_WORKSPACE_WORDS]; if (partial) { done = -partial; - memcpy(sctx->buffer + partial, data, done + 64); + memcpy(sctx->buffer + partial, data, + done + SHA1_BLOCK_SIZE); src = sctx->buffer; } do { sha_transform(sctx->state, src, temp); - done += 64; + done += SHA1_BLOCK_SIZE; src = data + done; - } while (done + 63 < len); + } while (done + SHA1_BLOCK_SIZE <= len); memset(temp, 0, sizeof(temp)); partial = 0; -- cgit v1.2.3 From 269230e7c52623cda279010c8032a6f5b59caaaf Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Mon, 27 Jun 2011 15:45:19 +0800 Subject: crypto: algif_hash - Handle initial af_alg_make_sg error correctly When the first call to af_alg_make_sg fails, we may return garbage instead of the real error. This patch fixes it by setting the error if "copied" is zero. Based on a patch by Jesper Juhl. Reported-by: Jesper Juhl Signed-off-by: Herbert Xu --- crypto/algif_hash.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/crypto/algif_hash.c b/crypto/algif_hash.c index 62122a1a2f7..ef5356cd280 100644 --- a/crypto/algif_hash.c +++ b/crypto/algif_hash.c @@ -68,8 +68,10 @@ static int hash_sendmsg(struct kiocb *unused, struct socket *sock, int newlen; newlen = af_alg_make_sg(&ctx->sgl, from, len, 0); - if (newlen < 0) + if (newlen < 0) { + err = copied ? 0 : newlen; goto unlock; + } ahash_request_set_crypt(&ctx->req, ctx->sgl.sg, NULL, newlen); -- cgit v1.2.3 From e3b4f515c43553a9950017d8d052541ccb8081fa Mon Sep 17 00:00:00 2001 From: Jan Glauber Date: Mon, 4 Jul 2011 20:06:01 +0800 Subject: crypto: s390 - support hardware accelerated SHA-224 On recent s390 machines hardware acceleration is available for SHA-256. SHA-224 is based on SHA-256 so it can also be accelerated by hardware. Do this by adding the proper algorithm description and initialization. Signed-off-by: Jan Glauber Signed-off-by: Herbert Xu --- arch/s390/crypto/sha256_s390.c | 66 +++++++++++++++++++++++++++++++++++------- 1 file changed, 55 insertions(+), 11 deletions(-) diff --git a/arch/s390/crypto/sha256_s390.c b/arch/s390/crypto/sha256_s390.c index 5ed8d64fc2e..0317a3547cb 100644 --- a/arch/s390/crypto/sha256_s390.c +++ b/arch/s390/crypto/sha256_s390.c @@ -1,15 +1,12 @@ /* * Cryptographic API. * - * s390 implementation of the SHA256 Secure Hash Algorithm. + * s390 implementation of the SHA256 and SHA224 Secure Hash Algorithm. * * s390 Version: - * Copyright IBM Corp. 2005,2007 + * Copyright IBM Corp. 2005,2011 * Author(s): Jan Glauber (jang@de.ibm.com) * - * Derived from "crypto/sha256_generic.c" - * and "arch/s390/crypto/sha1_s390.c" - * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the Free * Software Foundation; either version 2 of the License, or (at your option) @@ -65,7 +62,7 @@ static int sha256_import(struct shash_desc *desc, const void *in) return 0; } -static struct shash_alg alg = { +static struct shash_alg sha256_alg = { .digestsize = SHA256_DIGEST_SIZE, .init = sha256_init, .update = s390_sha_update, @@ -84,22 +81,69 @@ static struct shash_alg alg = { } }; -static int sha256_s390_init(void) +static int sha224_init(struct shash_desc *desc) { + struct s390_sha_ctx *sctx = shash_desc_ctx(desc); + + sctx->state[0] = SHA224_H0; + sctx->state[1] = SHA224_H1; + sctx->state[2] = SHA224_H2; + sctx->state[3] = SHA224_H3; + sctx->state[4] = SHA224_H4; + sctx->state[5] = SHA224_H5; + sctx->state[6] = SHA224_H6; + sctx->state[7] = SHA224_H7; + sctx->count = 0; + sctx->func = KIMD_SHA_256; + + return 0; +} + +static struct shash_alg sha224_alg = { + .digestsize = SHA224_DIGEST_SIZE, + .init = sha224_init, + .update = s390_sha_update, + .final = s390_sha_final, + .export = sha256_export, + .import = sha256_import, + .descsize = sizeof(struct s390_sha_ctx), + .statesize = sizeof(struct sha256_state), + .base = { + .cra_name = "sha224", + .cra_driver_name= "sha224-s390", + .cra_priority = CRYPT_S390_PRIORITY, + .cra_flags = CRYPTO_ALG_TYPE_SHASH, + .cra_blocksize = SHA224_BLOCK_SIZE, + .cra_module = THIS_MODULE, + } +}; + +static int __init sha256_s390_init(void) +{ + int ret; + if (!crypt_s390_func_available(KIMD_SHA_256, CRYPT_S390_MSA)) return -EOPNOTSUPP; - - return crypto_register_shash(&alg); + ret = crypto_register_shash(&sha256_alg); + if (ret < 0) + goto out; + ret = crypto_register_shash(&sha224_alg); + if (ret < 0) + crypto_unregister_shash(&sha256_alg); +out: + return ret; } static void __exit sha256_s390_fini(void) { - crypto_unregister_shash(&alg); + crypto_unregister_shash(&sha224_alg); + crypto_unregister_shash(&sha256_alg); } module_init(sha256_s390_init); module_exit(sha256_s390_fini); MODULE_ALIAS("sha256"); +MODULE_ALIAS("sha224"); MODULE_LICENSE("GPL"); -MODULE_DESCRIPTION("SHA256 Secure Hash Algorithm"); +MODULE_DESCRIPTION("SHA256 and SHA224 Secure Hash Algorithm"); -- cgit v1.2.3 From 625426633d7786f26a33123a9d12bec476bcc3cd Mon Sep 17 00:00:00 2001 From: Mathias Krause Date: Fri, 8 Jul 2011 17:21:21 +0800 Subject: crypto: gf128mul - fix call to memset() In gf128mul_lle() and gf128mul_bbe() r isn't completely initialized with zero because the size argument passed to memset() is the size of the pointer, not the structure it points to. Luckily there are no in-kernel users of those functions so the ABI change implied by this fix should break no existing code. Based on a patch by the PaX Team. Signed-off-by: Mathias Krause Cc: PaX Team Acked-by: David S. Miller Signed-off-by: Herbert Xu --- crypto/gf128mul.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crypto/gf128mul.c b/crypto/gf128mul.c index df35e4ccd07..5276607c72d 100644 --- a/crypto/gf128mul.c +++ b/crypto/gf128mul.c @@ -182,7 +182,7 @@ void gf128mul_lle(be128 *r, const be128 *b) for (i = 0; i < 7; ++i) gf128mul_x_lle(&p[i + 1], &p[i]); - memset(r, 0, sizeof(r)); + memset(r, 0, sizeof(*r)); for (i = 0;;) { u8 ch = ((u8 *)b)[15 - i]; @@ -220,7 +220,7 @@ void gf128mul_bbe(be128 *r, const be128 *b) for (i = 0; i < 7; ++i) gf128mul_x_bbe(&p[i + 1], &p[i]); - memset(r, 0, sizeof(r)); + memset(r, 0, sizeof(*r)); for (i = 0;;) { u8 ch = ((u8 *)b)[i]; -- cgit v1.2.3 From 5228f0f79e983c2b39c202c75af901ceb0003fc1 Mon Sep 17 00:00:00 2001 From: Kim Phillips Date: Fri, 15 Jul 2011 11:21:38 +0800 Subject: crypto: talitos - ensure request ordering within a single tfm Assign single target channel per tfm in talitos_cra_init instead of performing channel scheduling dynamically during the encryption request. This changes the talitos_submit interface to accept a new channel number argument. Without this, rapid bursts of misc. sized requests could make it possible for IPsec packets to be encrypted out-of-order, which would result in packet drops due to sequence numbers falling outside the anti-reply window on a peer gateway. Signed-off-by: Kim Phillips Signed-off-by: Herbert Xu --- drivers/crypto/talitos.c | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/drivers/crypto/talitos.c b/drivers/crypto/talitos.c index 854e2632f9a..b8ca58394f0 100644 --- a/drivers/crypto/talitos.c +++ b/drivers/crypto/talitos.c @@ -1,7 +1,7 @@ /* * talitos - Freescale Integrated Security Engine (SEC) device driver * - * Copyright (c) 2008-2010 Freescale Semiconductor, Inc. + * Copyright (c) 2008-2011 Freescale Semiconductor, Inc. * * Scatterlist Crypto API glue code copied from files with the following: * Copyright (c) 2006-2007 Herbert Xu @@ -282,6 +282,7 @@ static int init_device(struct device *dev) /** * talitos_submit - submits a descriptor to the device for processing * @dev: the SEC device to be used + * @ch: the SEC device channel to be used * @desc: the descriptor to be processed by the device * @callback: whom to call when processing is complete * @context: a handle for use by caller (optional) @@ -290,7 +291,7 @@ static int init_device(struct device *dev) * callback must check err and feedback in descriptor header * for device processing status. */ -static int talitos_submit(struct device *dev, struct talitos_desc *desc, +static int talitos_submit(struct device *dev, int ch, struct talitos_desc *desc, void (*callback)(struct device *dev, struct talitos_desc *desc, void *context, int error), @@ -298,15 +299,12 @@ static int talitos_submit(struct device *dev, struct talitos_desc *desc, { struct talitos_private *priv = dev_get_drvdata(dev); struct talitos_request *request; - unsigned long flags, ch; + unsigned long flags; int head; /* select done notification */ desc->hdr |= DESC_HDR_DONE_NOTIFY; - /* emulate SEC's round-robin channel fifo polling scheme */ - ch = atomic_inc_return(&priv->last_chan) & (priv->num_channels - 1); - spin_lock_irqsave(&priv->chan[ch].head_lock, flags); if (!atomic_inc_not_zero(&priv->chan[ch].submit_count)) { @@ -706,6 +704,7 @@ static void talitos_unregister_rng(struct device *dev) struct talitos_ctx { struct device *dev; + int ch; __be32 desc_hdr_template; u8 key[TALITOS_MAX_KEY_SIZE]; u8 iv[TALITOS_MAX_IV_LENGTH]; @@ -1117,7 +1116,7 @@ static int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq, map_single_talitos_ptr(dev, &desc->ptr[6], ivsize, ctx->iv, 0, DMA_FROM_DEVICE); - ret = talitos_submit(dev, desc, callback, areq); + ret = talitos_submit(dev, ctx->ch, desc, callback, areq); if (ret != -EINPROGRESS) { ipsec_esp_unmap(dev, edesc, areq); kfree(edesc); @@ -1524,7 +1523,7 @@ static int common_nonsnoop(struct talitos_edesc *edesc, to_talitos_ptr(&desc->ptr[6], 0); desc->ptr[6].j_extent = 0; - ret = talitos_submit(dev, desc, callback, areq); + ret = talitos_submit(dev, ctx->ch, desc, callback, areq); if (ret != -EINPROGRESS) { common_nonsnoop_unmap(dev, edesc, areq); kfree(edesc); @@ -1703,7 +1702,7 @@ static int common_nonsnoop_hash(struct talitos_edesc *edesc, /* last DWORD empty */ desc->ptr[6] = zero_entry; - ret = talitos_submit(dev, desc, callback, areq); + ret = talitos_submit(dev, ctx->ch, desc, callback, areq); if (ret != -EINPROGRESS) { common_nonsnoop_hash_unmap(dev, edesc, areq); kfree(edesc); @@ -2244,6 +2243,7 @@ static int talitos_cra_init(struct crypto_tfm *tfm) struct crypto_alg *alg = tfm->__crt_alg; struct talitos_crypto_alg *talitos_alg; struct talitos_ctx *ctx = crypto_tfm_ctx(tfm); + struct talitos_private *priv; if ((alg->cra_flags & CRYPTO_ALG_TYPE_MASK) == CRYPTO_ALG_TYPE_AHASH) talitos_alg = container_of(__crypto_ahash_alg(alg), @@ -2256,6 +2256,11 @@ static int talitos_cra_init(struct crypto_tfm *tfm) /* update context with ptr to dev */ ctx->dev = talitos_alg->dev; + /* assign SEC channel to tfm in round-robin fashion */ + priv = dev_get_drvdata(ctx->dev); + ctx->ch = atomic_inc_return(&priv->last_chan) & + (priv->num_channels - 1); + /* copy descriptor header template value */ ctx->desc_hdr_template = talitos_alg->algt.desc_hdr_template; -- cgit v1.2.3 From 602dba5a81790375f364e484cbc439f17157752e Mon Sep 17 00:00:00 2001 From: Kim Phillips Date: Fri, 15 Jul 2011 11:21:39 +0800 Subject: crypto: talitos - don't set done notification in hot path IRQ done notification is always set. Remove its explicit assignment from the hot path by including it in the descriptor header template assignment in talitos_cra_init. Signed-off-by: Kim Phillips Signed-off-by: Herbert Xu --- drivers/crypto/talitos.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/crypto/talitos.c b/drivers/crypto/talitos.c index b8ca58394f0..bd9e2ca0728 100644 --- a/drivers/crypto/talitos.c +++ b/drivers/crypto/talitos.c @@ -302,9 +302,6 @@ static int talitos_submit(struct device *dev, int ch, struct talitos_desc *desc, unsigned long flags; int head; - /* select done notification */ - desc->hdr |= DESC_HDR_DONE_NOTIFY; - spin_lock_irqsave(&priv->chan[ch].head_lock, flags); if (!atomic_inc_not_zero(&priv->chan[ch].submit_count)) { @@ -2264,6 +2261,9 @@ static int talitos_cra_init(struct crypto_tfm *tfm) /* copy descriptor header template value */ ctx->desc_hdr_template = talitos_alg->algt.desc_hdr_template; + /* select done notification */ + ctx->desc_hdr_template |= DESC_HDR_DONE_NOTIFY; + return 0; } -- cgit v1.2.3 From febec54286623a540cbe483a5d471ff6576a6e2e Mon Sep 17 00:00:00 2001 From: Kim Phillips Date: Fri, 15 Jul 2011 11:21:39 +0800 Subject: crypto: talitos - remove unused giv from ablkcipher methods Signed-off-by: Kim Phillips Signed-off-by: Herbert Xu --- drivers/crypto/talitos.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/drivers/crypto/talitos.c b/drivers/crypto/talitos.c index bd9e2ca0728..521244e08fb 100644 --- a/drivers/crypto/talitos.c +++ b/drivers/crypto/talitos.c @@ -1429,7 +1429,6 @@ static void ablkcipher_done(struct device *dev, static int common_nonsnoop(struct talitos_edesc *edesc, struct ablkcipher_request *areq, - u8 *giv, void (*callback) (struct device *dev, struct talitos_desc *desc, void *context, int error)) @@ -1449,7 +1448,7 @@ static int common_nonsnoop(struct talitos_edesc *edesc, /* cipher iv */ ivsize = crypto_ablkcipher_ivsize(cipher); - map_single_talitos_ptr(dev, &desc->ptr[1], ivsize, giv ?: areq->info, 0, + map_single_talitos_ptr(dev, &desc->ptr[1], ivsize, areq->info, 0, DMA_TO_DEVICE); /* cipher key */ @@ -1552,7 +1551,7 @@ static int ablkcipher_encrypt(struct ablkcipher_request *areq) /* set encrypt */ edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_MODE0_ENCRYPT; - return common_nonsnoop(edesc, areq, NULL, ablkcipher_done); + return common_nonsnoop(edesc, areq, ablkcipher_done); } static int ablkcipher_decrypt(struct ablkcipher_request *areq) @@ -1568,7 +1567,7 @@ static int ablkcipher_decrypt(struct ablkcipher_request *areq) edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_DIR_INBOUND; - return common_nonsnoop(edesc, areq, NULL, ablkcipher_done); + return common_nonsnoop(edesc, areq, ablkcipher_done); } static void common_nonsnoop_hash_unmap(struct device *dev, -- cgit v1.2.3 From 1c2b4abb19285368808f303ad61e17b172f1a108 Mon Sep 17 00:00:00 2001 From: Kim Phillips Date: Fri, 15 Jul 2011 11:21:40 +0800 Subject: crypto: talitos - don't bad_key in ablkcipher setkey crypto/ablkcipher.c's setkey() has already checked against the min, max key sizes before it calls here, and all max_keysize assignments in the algorithm template array do not exceed TALITOS_MAX_KEY_SIZE. Signed-off-by: Kim Phillips Signed-off-by: Herbert Xu --- drivers/crypto/talitos.c | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/drivers/crypto/talitos.c b/drivers/crypto/talitos.c index 521244e08fb..8a0bb417aa1 100644 --- a/drivers/crypto/talitos.c +++ b/drivers/crypto/talitos.c @@ -1378,22 +1378,11 @@ static int ablkcipher_setkey(struct crypto_ablkcipher *cipher, const u8 *key, unsigned int keylen) { struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher); - struct ablkcipher_alg *alg = crypto_ablkcipher_alg(cipher); - - if (keylen > TALITOS_MAX_KEY_SIZE) - goto badkey; - - if (keylen < alg->min_keysize || keylen > alg->max_keysize) - goto badkey; memcpy(&ctx->key, key, keylen); ctx->keylen = keylen; return 0; - -badkey: - crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN); - return -EINVAL; } static void common_nonsnoop_unmap(struct device *dev, -- cgit v1.2.3 From 0e47930020081b07047842be3e4552edb2395352 Mon Sep 17 00:00:00 2001 From: Yuan Kang Date: Fri, 15 Jul 2011 11:21:41 +0800 Subject: crypto: caam - shorter names "aead_authenc" and "ipsec_esp" changed to "aead," except for function "ipsec_esp," which is changed to "init_aead_job." Variable name of aead_request structures changed to "req" and name of aead_givcrypt_request structure changed to "areq" Signed-off-by: Yuan Kang Signed-off-by: Kim Phillips Signed-off-by: Herbert Xu --- drivers/crypto/caam/caamalg.c | 274 +++++++++++++++++++++--------------------- 1 file changed, 137 insertions(+), 137 deletions(-) diff --git a/drivers/crypto/caam/caamalg.c b/drivers/crypto/caam/caamalg.c index 676d957c22b..4786a204633 100644 --- a/drivers/crypto/caam/caamalg.c +++ b/drivers/crypto/caam/caamalg.c @@ -94,7 +94,7 @@ struct caam_ctx { unsigned int authsize; }; -static int aead_authenc_setauthsize(struct crypto_aead *authenc, +static int aead_setauthsize(struct crypto_aead *authenc, unsigned int authsize) { struct caam_ctx *ctx = crypto_aead_ctx(authenc); @@ -286,7 +286,7 @@ static int build_sh_desc_ipsec(struct caam_ctx *ctx) return 0; } -static int aead_authenc_setkey(struct crypto_aead *aead, +static int aead_setkey(struct crypto_aead *aead, const u8 *key, unsigned int keylen) { /* Sizes for MDHA pads (*not* keys): MD5, SHA1, 224, 256, 384, 512 */ @@ -379,7 +379,7 @@ struct link_tbl_entry { }; /* - * ipsec_esp_edesc - s/w-extended ipsec_esp descriptor + * aead_edesc - s/w-extended ipsec_esp descriptor * @src_nents: number of segments in input scatterlist * @dst_nents: number of segments in output scatterlist * @assoc_nents: number of segments in associated data (SPI+Seq) scatterlist @@ -388,7 +388,7 @@ struct link_tbl_entry { * @link_tbl_dma: bus physical mapped address of h/w link table * @hw_desc: the h/w job descriptor followed by any referenced link tables */ -struct ipsec_esp_edesc { +struct aead_edesc { int assoc_nents; int src_nents; int dst_nents; @@ -398,19 +398,19 @@ struct ipsec_esp_edesc { u32 hw_desc[0]; }; -static void ipsec_esp_unmap(struct device *dev, - struct ipsec_esp_edesc *edesc, - struct aead_request *areq) +static void aead_unmap(struct device *dev, + struct aead_edesc *edesc, + struct aead_request *req) { - dma_unmap_sg(dev, areq->assoc, edesc->assoc_nents, DMA_TO_DEVICE); + dma_unmap_sg(dev, req->assoc, edesc->assoc_nents, DMA_TO_DEVICE); - if (unlikely(areq->dst != areq->src)) { - dma_unmap_sg(dev, areq->src, edesc->src_nents, + if (unlikely(req->dst != req->src)) { + dma_unmap_sg(dev, req->src, edesc->src_nents, DMA_TO_DEVICE); - dma_unmap_sg(dev, areq->dst, edesc->dst_nents, + dma_unmap_sg(dev, req->dst, edesc->dst_nents, DMA_FROM_DEVICE); } else { - dma_unmap_sg(dev, areq->src, edesc->src_nents, + dma_unmap_sg(dev, req->src, edesc->src_nents, DMA_BIDIRECTIONAL); } @@ -423,20 +423,20 @@ static void ipsec_esp_unmap(struct device *dev, /* * ipsec_esp descriptor callbacks */ -static void ipsec_esp_encrypt_done(struct device *jrdev, u32 *desc, u32 err, +static void aead_encrypt_done(struct device *jrdev, u32 *desc, u32 err, void *context) { - struct aead_request *areq = context; - struct ipsec_esp_edesc *edesc; + struct aead_request *req = context; + struct aead_edesc *edesc; #ifdef DEBUG - struct crypto_aead *aead = crypto_aead_reqtfm(areq); + struct crypto_aead *aead = crypto_aead_reqtfm(req); int ivsize = crypto_aead_ivsize(aead); struct caam_ctx *ctx = crypto_aead_ctx(aead); dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); #endif - edesc = (struct ipsec_esp_edesc *)((char *)desc - - offsetof(struct ipsec_esp_edesc, hw_desc)); + edesc = (struct aead_edesc *)((char *)desc - + offsetof(struct aead_edesc, hw_desc)); if (err) { char tmp[CAAM_ERROR_STR_MAX]; @@ -444,39 +444,39 @@ static void ipsec_esp_encrypt_done(struct device *jrdev, u32 *desc, u32 err, dev_err(jrdev, "%08x: %s\n", err, caam_jr_strstatus(tmp, err)); } - ipsec_esp_unmap(jrdev, edesc, areq); + aead_unmap(jrdev, edesc, req); #ifdef DEBUG print_hex_dump(KERN_ERR, "assoc @"xstr(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(areq->assoc), - areq->assoclen , 1); + DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->assoc), + req->assoclen , 1); print_hex_dump(KERN_ERR, "dstiv @"xstr(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(areq->src) - ivsize, + DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src) - ivsize, edesc->src_nents ? 100 : ivsize, 1); print_hex_dump(KERN_ERR, "dst @"xstr(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(areq->src), - edesc->src_nents ? 100 : areq->cryptlen + + DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src), + edesc->src_nents ? 100 : req->cryptlen + ctx->authsize + 4, 1); #endif kfree(edesc); - aead_request_complete(areq, err); + aead_request_complete(req, err); } -static void ipsec_esp_decrypt_done(struct device *jrdev, u32 *desc, u32 err, +static void aead_decrypt_done(struct device *jrdev, u32 *desc, u32 err, void *context) { - struct aead_request *areq = context; - struct ipsec_esp_edesc *edesc; + struct aead_request *req = context; + struct aead_edesc *edesc; #ifdef DEBUG - struct crypto_aead *aead = crypto_aead_reqtfm(areq); + struct crypto_aead *aead = crypto_aead_reqtfm(req); struct caam_ctx *ctx = crypto_aead_ctx(aead); dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); #endif - edesc = (struct ipsec_esp_edesc *)((char *)desc - - offsetof(struct ipsec_esp_edesc, hw_desc)); + edesc = (struct aead_edesc *)((char *)desc - + offsetof(struct aead_edesc, hw_desc)); if (err) { char tmp[CAAM_ERROR_STR_MAX]; @@ -484,7 +484,7 @@ static void ipsec_esp_decrypt_done(struct device *jrdev, u32 *desc, u32 err, dev_err(jrdev, "%08x: %s\n", err, caam_jr_strstatus(tmp, err)); } - ipsec_esp_unmap(jrdev, edesc, areq); + aead_unmap(jrdev, edesc, req); /* * verify hw auth check passed else return -EBADMSG @@ -495,12 +495,12 @@ static void ipsec_esp_decrypt_done(struct device *jrdev, u32 *desc, u32 err, #ifdef DEBUG print_hex_dump(KERN_ERR, "iphdrout@"xstr(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, - ((char *)sg_virt(areq->assoc) - sizeof(struct iphdr)), - sizeof(struct iphdr) + areq->assoclen + - ((areq->cryptlen > 1500) ? 1500 : areq->cryptlen) + + ((char *)sg_virt(req->assoc) - sizeof(struct iphdr)), + sizeof(struct iphdr) + req->assoclen + + ((req->cryptlen > 1500) ? 1500 : req->cryptlen) + ctx->authsize + 36, 1); if (!err && edesc->link_tbl_bytes) { - struct scatterlist *sg = sg_last(areq->src, edesc->src_nents); + struct scatterlist *sg = sg_last(req->src, edesc->src_nents); print_hex_dump(KERN_ERR, "sglastout@"xstr(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(sg), sg->length + ctx->authsize + 16, 1); @@ -508,7 +508,7 @@ static void ipsec_esp_decrypt_done(struct device *jrdev, u32 *desc, u32 err, #endif kfree(edesc); - aead_request_complete(areq, err); + aead_request_complete(req, err); } /* @@ -537,12 +537,12 @@ static void sg_to_link_tbl(struct scatterlist *sg, int sg_count, /* * fill in and submit ipsec_esp job descriptor */ -static int ipsec_esp(struct ipsec_esp_edesc *edesc, struct aead_request *areq, +static int init_aead_job(struct aead_edesc *edesc, struct aead_request *req, u32 encrypt, void (*callback) (struct device *dev, u32 *desc, u32 err, void *context)) { - struct crypto_aead *aead = crypto_aead_reqtfm(areq); + struct crypto_aead *aead = crypto_aead_reqtfm(req); struct caam_ctx *ctx = crypto_aead_ctx(aead); struct device *jrdev = ctx->jrdev; u32 *desc = edesc->hw_desc, options; @@ -554,27 +554,27 @@ static int ipsec_esp(struct ipsec_esp_edesc *edesc, struct aead_request *areq, u32 *sh_desc = ctx->sh_desc; debug("assoclen %d cryptlen %d authsize %d\n", - areq->assoclen, areq->cryptlen, authsize); + req->assoclen, req->cryptlen, authsize); print_hex_dump(KERN_ERR, "assoc @"xstr(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(areq->assoc), - areq->assoclen , 1); + DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->assoc), + req->assoclen , 1); print_hex_dump(KERN_ERR, "presciv@"xstr(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(areq->src) - ivsize, + DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src) - ivsize, edesc->src_nents ? 100 : ivsize, 1); print_hex_dump(KERN_ERR, "src @"xstr(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(areq->src), - edesc->src_nents ? 100 : areq->cryptlen + authsize, 1); + DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src), + edesc->src_nents ? 100 : req->cryptlen + authsize, 1); print_hex_dump(KERN_ERR, "shrdesc@"xstr(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, sh_desc, desc_bytes(sh_desc), 1); #endif - assoc_sg_count = dma_map_sg(jrdev, areq->assoc, edesc->assoc_nents ?: 1, + assoc_sg_count = dma_map_sg(jrdev, req->assoc, edesc->assoc_nents ?: 1, DMA_TO_DEVICE); - if (areq->src == areq->dst) - sg_count = dma_map_sg(jrdev, areq->src, edesc->src_nents ? : 1, + if (req->src == req->dst) + sg_count = dma_map_sg(jrdev, req->src, edesc->src_nents ? : 1, DMA_BIDIRECTIONAL); else - sg_count = dma_map_sg(jrdev, areq->src, edesc->src_nents ? : 1, + sg_count = dma_map_sg(jrdev, req->src, edesc->src_nents ? : 1, DMA_TO_DEVICE); /* start auth operation */ @@ -584,14 +584,14 @@ static int ipsec_esp(struct ipsec_esp_edesc *edesc, struct aead_request *areq, /* Load FIFO with data for Class 2 CHA */ options = FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG; if (!edesc->assoc_nents) { - ptr = sg_dma_address(areq->assoc); + ptr = sg_dma_address(req->assoc); } else { - sg_to_link_tbl(areq->assoc, edesc->assoc_nents, + sg_to_link_tbl(req->assoc, edesc->assoc_nents, edesc->link_tbl, 0); ptr = edesc->link_tbl_dma; options |= LDST_SGF; } - append_fifo_load(desc, ptr, areq->assoclen, options); + append_fifo_load(desc, ptr, req->assoclen, options); /* copy iv from cipher/class1 input context to class2 infifo */ append_move(desc, MOVE_SRC_CLASS1CTX | MOVE_DEST_CLASS2INFIFO | ivsize); @@ -621,31 +621,31 @@ static int ipsec_esp(struct ipsec_esp_edesc *edesc, struct aead_request *areq, /* load payload & instruct to class2 to snoop class 1 if encrypting */ options = 0; if (!edesc->src_nents) { - src_dma = sg_dma_address(areq->src); + src_dma = sg_dma_address(req->src); } else { - sg_to_link_tbl(areq->src, edesc->src_nents, edesc->link_tbl + + sg_to_link_tbl(req->src, edesc->src_nents, edesc->link_tbl + edesc->assoc_nents, 0); src_dma = edesc->link_tbl_dma + edesc->assoc_nents * sizeof(struct link_tbl_entry); options |= LDST_SGF; } - append_seq_in_ptr(desc, src_dma, areq->cryptlen + authsize, options); - append_seq_fifo_load(desc, areq->cryptlen, FIFOLD_CLASS_BOTH | + append_seq_in_ptr(desc, src_dma, req->cryptlen + authsize, options); + append_seq_fifo_load(desc, req->cryptlen, FIFOLD_CLASS_BOTH | FIFOLD_TYPE_LASTBOTH | (encrypt ? FIFOLD_TYPE_MSG1OUT2 : FIFOLD_TYPE_MSG)); /* specify destination */ - if (areq->src == areq->dst) { + if (req->src == req->dst) { dst_dma = src_dma; } else { - sg_count = dma_map_sg(jrdev, areq->dst, edesc->dst_nents ? : 1, + sg_count = dma_map_sg(jrdev, req->dst, edesc->dst_nents ? : 1, DMA_FROM_DEVICE); if (!edesc->dst_nents) { - dst_dma = sg_dma_address(areq->dst); + dst_dma = sg_dma_address(req->dst); options = 0; } else { - sg_to_link_tbl(areq->dst, edesc->dst_nents, + sg_to_link_tbl(req->dst, edesc->dst_nents, edesc->link_tbl + edesc->assoc_nents + edesc->src_nents, 0); dst_dma = edesc->link_tbl_dma + (edesc->assoc_nents + @@ -654,8 +654,8 @@ static int ipsec_esp(struct ipsec_esp_edesc *edesc, struct aead_request *areq, options = LDST_SGF; } } - append_seq_out_ptr(desc, dst_dma, areq->cryptlen + authsize, options); - append_seq_fifo_store(desc, areq->cryptlen, FIFOST_TYPE_MESSAGE_DATA); + append_seq_out_ptr(desc, dst_dma, req->cryptlen + authsize, options); + append_seq_fifo_store(desc, req->cryptlen, FIFOST_TYPE_MESSAGE_DATA); /* ICV */ if (encrypt) @@ -674,11 +674,11 @@ static int ipsec_esp(struct ipsec_esp_edesc *edesc, struct aead_request *areq, edesc->link_tbl_bytes, 1); #endif - ret = caam_jr_enqueue(jrdev, desc, callback, areq); + ret = caam_jr_enqueue(jrdev, desc, callback, req); if (!ret) ret = -EINPROGRESS; else { - ipsec_esp_unmap(jrdev, edesc, areq); + aead_unmap(jrdev, edesc, req); kfree(edesc); } @@ -708,30 +708,30 @@ static int sg_count(struct scatterlist *sg_list, int nbytes, int *chained) /* * allocate and map the ipsec_esp extended descriptor */ -static struct ipsec_esp_edesc *ipsec_esp_edesc_alloc(struct aead_request *areq, +static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, int desc_bytes) { - struct crypto_aead *aead = crypto_aead_reqtfm(areq); + struct crypto_aead *aead = crypto_aead_reqtfm(req); struct caam_ctx *ctx = crypto_aead_ctx(aead); struct device *jrdev = ctx->jrdev; - gfp_t flags = areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL : + gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL : GFP_ATOMIC; int assoc_nents, src_nents, dst_nents = 0, chained, link_tbl_bytes; - struct ipsec_esp_edesc *edesc; + struct aead_edesc *edesc; - assoc_nents = sg_count(areq->assoc, areq->assoclen, &chained); + assoc_nents = sg_count(req->assoc, req->assoclen, &chained); BUG_ON(chained); if (likely(assoc_nents == 1)) assoc_nents = 0; - src_nents = sg_count(areq->src, areq->cryptlen + ctx->authsize, + src_nents = sg_count(req->src, req->cryptlen + ctx->authsize, &chained); BUG_ON(chained); if (src_nents == 1) src_nents = 0; - if (unlikely(areq->dst != areq->src)) { - dst_nents = sg_count(areq->dst, areq->cryptlen + ctx->authsize, + if (unlikely(req->dst != req->src)) { + dst_nents = sg_count(req->dst, req->cryptlen + ctx->authsize, &chained); BUG_ON(chained); if (dst_nents == 1) @@ -743,7 +743,7 @@ static struct ipsec_esp_edesc *ipsec_esp_edesc_alloc(struct aead_request *areq, debug("link_tbl_bytes %d\n", link_tbl_bytes); /* allocate space for base edesc and hw desc commands, link tables */ - edesc = kmalloc(sizeof(struct ipsec_esp_edesc) + desc_bytes + + edesc = kmalloc(sizeof(struct aead_edesc) + desc_bytes + link_tbl_bytes, GFP_DMA | flags); if (!edesc) { dev_err(jrdev, "could not allocate extended descriptor\n"); @@ -753,7 +753,7 @@ static struct ipsec_esp_edesc *ipsec_esp_edesc_alloc(struct aead_request *areq, edesc->assoc_nents = assoc_nents; edesc->src_nents = src_nents; edesc->dst_nents = dst_nents; - edesc->link_tbl = (void *)edesc + sizeof(struct ipsec_esp_edesc) + + edesc->link_tbl = (void *)edesc + sizeof(struct aead_edesc) + desc_bytes; edesc->link_tbl_dma = dma_map_single(jrdev, edesc->link_tbl, link_tbl_bytes, DMA_TO_DEVICE); @@ -762,10 +762,10 @@ static struct ipsec_esp_edesc *ipsec_esp_edesc_alloc(struct aead_request *areq, return edesc; } -static int aead_authenc_encrypt(struct aead_request *areq) +static int aead_encrypt(struct aead_request *req) { - struct ipsec_esp_edesc *edesc; - struct crypto_aead *aead = crypto_aead_reqtfm(areq); + struct aead_edesc *edesc; + struct crypto_aead *aead = crypto_aead_reqtfm(req); struct caam_ctx *ctx = crypto_aead_ctx(aead); struct device *jrdev = ctx->jrdev; int ivsize = crypto_aead_ivsize(aead); @@ -773,7 +773,7 @@ static int aead_authenc_encrypt(struct aead_request *areq) dma_addr_t iv_dma; /* allocate extended descriptor */ - edesc = ipsec_esp_edesc_alloc(areq, DESC_AEAD_ENCRYPT_TEXT_LEN * + edesc = aead_edesc_alloc(req, DESC_AEAD_ENCRYPT_TEXT_LEN * CAAM_CMD_SZ); if (IS_ERR(edesc)) return PTR_ERR(edesc); @@ -784,29 +784,29 @@ static int aead_authenc_encrypt(struct aead_request *areq) init_job_desc_shared(desc, ctx->shared_desc_phys, desc_len(ctx->sh_desc), HDR_SHARE_DEFER); - iv_dma = dma_map_single(jrdev, areq->iv, ivsize, DMA_TO_DEVICE); + iv_dma = dma_map_single(jrdev, req->iv, ivsize, DMA_TO_DEVICE); /* check dma error */ append_load(desc, iv_dma, ivsize, LDST_CLASS_1_CCB | LDST_SRCDST_BYTE_CONTEXT); - return ipsec_esp(edesc, areq, OP_ALG_ENCRYPT, ipsec_esp_encrypt_done); + return init_aead_job(edesc, req, OP_ALG_ENCRYPT, aead_encrypt_done); } -static int aead_authenc_decrypt(struct aead_request *req) +static int aead_decrypt(struct aead_request *req) { struct crypto_aead *aead = crypto_aead_reqtfm(req); int ivsize = crypto_aead_ivsize(aead); struct caam_ctx *ctx = crypto_aead_ctx(aead); struct device *jrdev = ctx->jrdev; - struct ipsec_esp_edesc *edesc; + struct aead_edesc *edesc; u32 *desc; dma_addr_t iv_dma; req->cryptlen -= ctx->authsize; /* allocate extended descriptor */ - edesc = ipsec_esp_edesc_alloc(req, DESC_AEAD_DECRYPT_TEXT_LEN * + edesc = aead_edesc_alloc(req, DESC_AEAD_DECRYPT_TEXT_LEN * CAAM_CMD_SZ); if (IS_ERR(edesc)) return PTR_ERR(edesc); @@ -823,26 +823,26 @@ static int aead_authenc_decrypt(struct aead_request *req) append_load(desc, iv_dma, ivsize, LDST_CLASS_1_CCB | LDST_SRCDST_BYTE_CONTEXT); - return ipsec_esp(edesc, req, !OP_ALG_ENCRYPT, ipsec_esp_decrypt_done); + return init_aead_job(edesc, req, !OP_ALG_ENCRYPT, aead_decrypt_done); } -static int aead_authenc_givencrypt(struct aead_givcrypt_request *req) +static int aead_givencrypt(struct aead_givcrypt_request *areq) { - struct aead_request *areq = &req->areq; - struct ipsec_esp_edesc *edesc; - struct crypto_aead *aead = crypto_aead_reqtfm(areq); + struct aead_request *req = &areq->areq; + struct aead_edesc *edesc; + struct crypto_aead *aead = crypto_aead_reqtfm(req); struct caam_ctx *ctx = crypto_aead_ctx(aead); struct device *jrdev = ctx->jrdev; int ivsize = crypto_aead_ivsize(aead); dma_addr_t iv_dma; u32 *desc; - iv_dma = dma_map_single(jrdev, req->giv, ivsize, DMA_FROM_DEVICE); + iv_dma = dma_map_single(jrdev, areq->giv, ivsize, DMA_FROM_DEVICE); - debug("%s: giv %p\n", __func__, req->giv); + debug("%s: giv %p\n", __func__, areq->giv); /* allocate extended descriptor */ - edesc = ipsec_esp_edesc_alloc(areq, DESC_AEAD_GIVENCRYPT_TEXT_LEN * + edesc = aead_edesc_alloc(req, DESC_AEAD_GIVENCRYPT_TEXT_LEN * CAAM_CMD_SZ); if (IS_ERR(edesc)) return PTR_ERR(edesc); @@ -881,7 +881,7 @@ static int aead_authenc_givencrypt(struct aead_givcrypt_request *req) append_fifo_store(desc, iv_dma, ivsize, FIFOST_TYPE_MESSAGE_DATA); - return ipsec_esp(edesc, areq, OP_ALG_ENCRYPT, ipsec_esp_encrypt_done); + return init_aead_job(edesc, req, OP_ALG_ENCRYPT, aead_encrypt_done); } struct caam_alg_template { @@ -901,11 +901,11 @@ static struct caam_alg_template driver_algs[] = { .driver_name = "authenc-hmac-sha1-cbc-aes-caam", .blocksize = AES_BLOCK_SIZE, .aead = { - .setkey = aead_authenc_setkey, - .setauthsize = aead_authenc_setauthsize, - .encrypt = aead_authenc_encrypt, - .decrypt = aead_authenc_decrypt, - .givencrypt = aead_authenc_givencrypt, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .givencrypt = aead_givencrypt, .geniv = "", .ivsize = AES_BLOCK_SIZE, .maxauthsize = SHA1_DIGEST_SIZE, @@ -919,11 +919,11 @@ static struct caam_alg_template driver_algs[] = { .driver_name = "authenc-hmac-sha256-cbc-aes-caam", .blocksize = AES_BLOCK_SIZE, .aead = { - .setkey = aead_authenc_setkey, - .setauthsize = aead_authenc_setauthsize, - .encrypt = aead_authenc_encrypt, - .decrypt = aead_authenc_decrypt, - .givencrypt = aead_authenc_givencrypt, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .givencrypt = aead_givencrypt, .geniv = "", .ivsize = AES_BLOCK_SIZE, .maxauthsize = SHA256_DIGEST_SIZE, @@ -938,11 +938,11 @@ static struct caam_alg_template driver_algs[] = { .driver_name = "authenc-hmac-sha512-cbc-aes-caam", .blocksize = AES_BLOCK_SIZE, .aead = { - .setkey = aead_authenc_setkey, - .setauthsize = aead_authenc_setauthsize, - .encrypt = aead_authenc_encrypt, - .decrypt = aead_authenc_decrypt, - .givencrypt = aead_authenc_givencrypt, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .givencrypt = aead_givencrypt, .geniv = "", .ivsize = AES_BLOCK_SIZE, .maxauthsize = SHA512_DIGEST_SIZE, @@ -957,11 +957,11 @@ static struct caam_alg_template driver_algs[] = { .driver_name = "authenc-hmac-sha1-cbc-des3_ede-caam", .blocksize = DES3_EDE_BLOCK_SIZE, .aead = { - .setkey = aead_authenc_setkey, - .setauthsize = aead_authenc_setauthsize, - .encrypt = aead_authenc_encrypt, - .decrypt = aead_authenc_decrypt, - .givencrypt = aead_authenc_givencrypt, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .givencrypt = aead_givencrypt, .geniv = "", .ivsize = DES3_EDE_BLOCK_SIZE, .maxauthsize = SHA1_DIGEST_SIZE, @@ -975,11 +975,11 @@ static struct caam_alg_template driver_algs[] = { .driver_name = "authenc-hmac-sha256-cbc-des3_ede-caam", .blocksize = DES3_EDE_BLOCK_SIZE, .aead = { - .setkey = aead_authenc_setkey, - .setauthsize = aead_authenc_setauthsize, - .encrypt = aead_authenc_encrypt, - .decrypt = aead_authenc_decrypt, - .givencrypt = aead_authenc_givencrypt, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .givencrypt = aead_givencrypt, .geniv = "", .ivsize = DES3_EDE_BLOCK_SIZE, .maxauthsize = SHA256_DIGEST_SIZE, @@ -994,11 +994,11 @@ static struct caam_alg_template driver_algs[] = { .driver_name = "authenc-hmac-sha512-cbc-des3_ede-caam", .blocksize = DES3_EDE_BLOCK_SIZE, .aead = { - .setkey = aead_authenc_setkey, - .setauthsize = aead_authenc_setauthsize, - .encrypt = aead_authenc_encrypt, - .decrypt = aead_authenc_decrypt, - .givencrypt = aead_authenc_givencrypt, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .givencrypt = aead_givencrypt, .geniv = "", .ivsize = DES3_EDE_BLOCK_SIZE, .maxauthsize = SHA512_DIGEST_SIZE, @@ -1013,11 +1013,11 @@ static struct caam_alg_template driver_algs[] = { .driver_name = "authenc-hmac-sha1-cbc-des-caam", .blocksize = DES_BLOCK_SIZE, .aead = { - .setkey = aead_authenc_setkey, - .setauthsize = aead_authenc_setauthsize, - .encrypt = aead_authenc_encrypt, - .decrypt = aead_authenc_decrypt, - .givencrypt = aead_authenc_givencrypt, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .givencrypt = aead_givencrypt, .geniv = "", .ivsize = DES_BLOCK_SIZE, .maxauthsize = SHA1_DIGEST_SIZE, @@ -1031,11 +1031,11 @@ static struct caam_alg_template driver_algs[] = { .driver_name = "authenc-hmac-sha256-cbc-des-caam", .blocksize = DES_BLOCK_SIZE, .aead = { - .setkey = aead_authenc_setkey, - .setauthsize = aead_authenc_setauthsize, - .encrypt = aead_authenc_encrypt, - .decrypt = aead_authenc_decrypt, - .givencrypt = aead_authenc_givencrypt, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .givencrypt = aead_givencrypt, .geniv = "", .ivsize = DES_BLOCK_SIZE, .maxauthsize = SHA256_DIGEST_SIZE, @@ -1050,11 +1050,11 @@ static struct caam_alg_template driver_algs[] = { .driver_name = "authenc-hmac-sha512-cbc-des-caam", .blocksize = DES_BLOCK_SIZE, .aead = { - .setkey = aead_authenc_setkey, - .setauthsize = aead_authenc_setauthsize, - .encrypt = aead_authenc_encrypt, - .decrypt = aead_authenc_decrypt, - .givencrypt = aead_authenc_givencrypt, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .givencrypt = aead_givencrypt, .geniv = "", .ivsize = DES_BLOCK_SIZE, .maxauthsize = SHA512_DIGEST_SIZE, -- cgit v1.2.3 From 885e9e2fd3f009de56dd265f4ecd8740e9ad5aaa Mon Sep 17 00:00:00 2001 From: Yuan Kang Date: Fri, 15 Jul 2011 11:21:41 +0800 Subject: crypto: caam - structure renaming caam_ctx.key_phys to key_dma caam_alg_template supports multiple algorithm types listed in union, which requires cases for different types in function caam_alg_alloc Signed-off-by: Yuan Kang Signed-off-by: Kim Phillips Signed-off-by: Herbert Xu --- drivers/crypto/caam/caamalg.c | 64 +++++++++++++++++++++++++++++-------------- 1 file changed, 43 insertions(+), 21 deletions(-) diff --git a/drivers/crypto/caam/caamalg.c b/drivers/crypto/caam/caamalg.c index 4786a204633..403b293509b 100644 --- a/drivers/crypto/caam/caamalg.c +++ b/drivers/crypto/caam/caamalg.c @@ -87,7 +87,7 @@ struct caam_ctx { u32 class2_alg_type; u32 alg_op; u8 *key; - dma_addr_t key_phys; + dma_addr_t key_dma; unsigned int enckeylen; unsigned int split_key_len; unsigned int split_key_pad_len; @@ -263,9 +263,9 @@ static int build_sh_desc_ipsec(struct caam_ctx *ctx) ctx->split_key_pad_len, ctx->enckeylen, ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG); } else { - append_key(sh_desc, ctx->key_phys, ctx->split_key_len, CLASS_2 | + append_key(sh_desc, ctx->key_dma, ctx->split_key_len, CLASS_2 | KEY_DEST_MDHA_SPLIT | KEY_ENC); - append_key(sh_desc, ctx->key_phys + ctx->split_key_pad_len, + append_key(sh_desc, ctx->key_dma + ctx->split_key_pad_len, ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG); } @@ -342,9 +342,9 @@ static int aead_setkey(struct crypto_aead *aead, /* postpend encryption key to auth split key */ memcpy(ctx->key + ctx->split_key_pad_len, key + authkeylen, enckeylen); - ctx->key_phys = dma_map_single(jrdev, ctx->key, ctx->split_key_pad_len + + ctx->key_dma = dma_map_single(jrdev, ctx->key, ctx->split_key_pad_len + enckeylen, DMA_TO_DEVICE); - if (dma_mapping_error(jrdev, ctx->key_phys)) { + if (dma_mapping_error(jrdev, ctx->key_dma)) { dev_err(jrdev, "unable to map key i/o memory\n"); kfree(ctx->key); return -ENOMEM; @@ -359,7 +359,7 @@ static int aead_setkey(struct crypto_aead *aead, ret = build_sh_desc_ipsec(ctx); if (ret) { - dma_unmap_single(jrdev, ctx->key_phys, ctx->split_key_pad_len + + dma_unmap_single(jrdev, ctx->key_dma, ctx->split_key_pad_len + enckeylen, DMA_TO_DEVICE); kfree(ctx->key); } @@ -884,11 +884,20 @@ static int aead_givencrypt(struct aead_givcrypt_request *areq) return init_aead_job(edesc, req, OP_ALG_ENCRYPT, aead_encrypt_done); } +#define template_aead template_u.aead struct caam_alg_template { char name[CRYPTO_MAX_ALG_NAME]; char driver_name[CRYPTO_MAX_ALG_NAME]; unsigned int blocksize; - struct aead_alg aead; + u32 type; + union { + struct ablkcipher_alg ablkcipher; + struct aead_alg aead; + struct blkcipher_alg blkcipher; + struct cipher_alg cipher; + struct compress_alg compress; + struct rng_alg rng; + } template_u; u32 class1_alg_type; u32 class2_alg_type; u32 alg_op; @@ -900,7 +909,8 @@ static struct caam_alg_template driver_algs[] = { .name = "authenc(hmac(sha1),cbc(aes))", .driver_name = "authenc-hmac-sha1-cbc-aes-caam", .blocksize = AES_BLOCK_SIZE, - .aead = { + .type = CRYPTO_ALG_TYPE_AEAD, + .template_aead = { .setkey = aead_setkey, .setauthsize = aead_setauthsize, .encrypt = aead_encrypt, @@ -918,7 +928,8 @@ static struct caam_alg_template driver_algs[] = { .name = "authenc(hmac(sha256),cbc(aes))", .driver_name = "authenc-hmac-sha256-cbc-aes-caam", .blocksize = AES_BLOCK_SIZE, - .aead = { + .type = CRYPTO_ALG_TYPE_AEAD, + .template_aead = { .setkey = aead_setkey, .setauthsize = aead_setauthsize, .encrypt = aead_encrypt, @@ -937,7 +948,8 @@ static struct caam_alg_template driver_algs[] = { .name = "authenc(hmac(sha512),cbc(aes))", .driver_name = "authenc-hmac-sha512-cbc-aes-caam", .blocksize = AES_BLOCK_SIZE, - .aead = { + .type = CRYPTO_ALG_TYPE_AEAD, + .template_aead = { .setkey = aead_setkey, .setauthsize = aead_setauthsize, .encrypt = aead_encrypt, @@ -956,7 +968,8 @@ static struct caam_alg_template driver_algs[] = { .name = "authenc(hmac(sha1),cbc(des3_ede))", .driver_name = "authenc-hmac-sha1-cbc-des3_ede-caam", .blocksize = DES3_EDE_BLOCK_SIZE, - .aead = { + .type = CRYPTO_ALG_TYPE_AEAD, + .template_aead = { .setkey = aead_setkey, .setauthsize = aead_setauthsize, .encrypt = aead_encrypt, @@ -974,7 +987,8 @@ static struct caam_alg_template driver_algs[] = { .name = "authenc(hmac(sha256),cbc(des3_ede))", .driver_name = "authenc-hmac-sha256-cbc-des3_ede-caam", .blocksize = DES3_EDE_BLOCK_SIZE, - .aead = { + .type = CRYPTO_ALG_TYPE_AEAD, + .template_aead = { .setkey = aead_setkey, .setauthsize = aead_setauthsize, .encrypt = aead_encrypt, @@ -993,7 +1007,8 @@ static struct caam_alg_template driver_algs[] = { .name = "authenc(hmac(sha512),cbc(des3_ede))", .driver_name = "authenc-hmac-sha512-cbc-des3_ede-caam", .blocksize = DES3_EDE_BLOCK_SIZE, - .aead = { + .type = CRYPTO_ALG_TYPE_AEAD, + .template_aead = { .setkey = aead_setkey, .setauthsize = aead_setauthsize, .encrypt = aead_encrypt, @@ -1012,7 +1027,8 @@ static struct caam_alg_template driver_algs[] = { .name = "authenc(hmac(sha1),cbc(des))", .driver_name = "authenc-hmac-sha1-cbc-des-caam", .blocksize = DES_BLOCK_SIZE, - .aead = { + .type = CRYPTO_ALG_TYPE_AEAD, + .template_aead = { .setkey = aead_setkey, .setauthsize = aead_setauthsize, .encrypt = aead_encrypt, @@ -1030,7 +1046,8 @@ static struct caam_alg_template driver_algs[] = { .name = "authenc(hmac(sha256),cbc(des))", .driver_name = "authenc-hmac-sha256-cbc-des-caam", .blocksize = DES_BLOCK_SIZE, - .aead = { + .type = CRYPTO_ALG_TYPE_AEAD, + .template_aead = { .setkey = aead_setkey, .setauthsize = aead_setauthsize, .encrypt = aead_encrypt, @@ -1049,7 +1066,8 @@ static struct caam_alg_template driver_algs[] = { .name = "authenc(hmac(sha512),cbc(des))", .driver_name = "authenc-hmac-sha512-cbc-des-caam", .blocksize = DES_BLOCK_SIZE, - .aead = { + .type = CRYPTO_ALG_TYPE_AEAD, + .template_aead = { .setkey = aead_setkey, .setauthsize = aead_setauthsize, .encrypt = aead_encrypt, @@ -1107,8 +1125,8 @@ static void caam_cra_exit(struct crypto_tfm *tfm) desc_bytes(ctx->sh_desc), DMA_TO_DEVICE); kfree(ctx->sh_desc); - if (!dma_mapping_error(ctx->jrdev, ctx->key_phys)) - dma_unmap_single(ctx->jrdev, ctx->key_phys, + if (!dma_mapping_error(ctx->jrdev, ctx->key_dma)) + dma_unmap_single(ctx->jrdev, ctx->key_dma, ctx->split_key_pad_len + ctx->enckeylen, DMA_TO_DEVICE); kfree(ctx->key); @@ -1175,12 +1193,16 @@ static struct caam_crypto_alg *caam_alg_alloc(struct device *ctrldev, alg->cra_init = caam_cra_init; alg->cra_exit = caam_cra_exit; alg->cra_priority = CAAM_CRA_PRIORITY; - alg->cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC; alg->cra_blocksize = template->blocksize; alg->cra_alignmask = 0; - alg->cra_type = &crypto_aead_type; alg->cra_ctxsize = sizeof(struct caam_ctx); - alg->cra_u.aead = template->aead; + alg->cra_flags = CRYPTO_ALG_ASYNC | template->type; + switch (template->type) { + case CRYPTO_ALG_TYPE_AEAD: + alg->cra_type = &crypto_aead_type; + alg->cra_aead = template->template_aead; + break; + } t_alg->class1_alg_type = template->class1_alg_type; t_alg->class2_alg_type = template->class2_alg_type; -- cgit v1.2.3 From 1acebad3d8db8d5220b3010c2eb160c625434cf2 Mon Sep 17 00:00:00 2001 From: Yuan Kang Date: Fri, 15 Jul 2011 11:21:42 +0800 Subject: crypto: caam - faster aead implementation Job descriptors only contain header and seq pointers. Other commands are stored in separate shared descriptors for encrypt, decrypt and givencrypt, stored as arrays in caam_ctx. This requires additional macros to create math commands to calculate assoclen and cryptlen. Signed-off-by: Yuan Kang Signed-off-by: Kim Phillips Signed-off-by: Herbert Xu --- drivers/crypto/caam/caamalg.c | 1104 ++++++++++++++++++++++++++----------- drivers/crypto/caam/desc_constr.h | 58 +- 2 files changed, 832 insertions(+), 330 deletions(-) diff --git a/drivers/crypto/caam/caamalg.c b/drivers/crypto/caam/caamalg.c index 403b293509b..ed7d59d168a 100644 --- a/drivers/crypto/caam/caamalg.c +++ b/drivers/crypto/caam/caamalg.c @@ -62,10 +62,16 @@ #define CAAM_MAX_IV_LENGTH 16 /* length of descriptors text */ -#define DESC_AEAD_SHARED_TEXT_LEN 4 -#define DESC_AEAD_ENCRYPT_TEXT_LEN 21 -#define DESC_AEAD_DECRYPT_TEXT_LEN 24 -#define DESC_AEAD_GIVENCRYPT_TEXT_LEN 27 +#define DESC_JOB_IO_LEN (CAAM_CMD_SZ * 3 + CAAM_PTR_SZ * 3) + +#define DESC_AEAD_BASE (4 * CAAM_CMD_SZ) +#define DESC_AEAD_ENC_LEN (DESC_AEAD_BASE + 16 * CAAM_CMD_SZ) +#define DESC_AEAD_DEC_LEN (DESC_AEAD_BASE + 21 * CAAM_CMD_SZ) +#define DESC_AEAD_GIVENC_LEN (DESC_AEAD_ENC_LEN + 7 * CAAM_CMD_SZ) + +#define DESC_MAX_USED_BYTES (DESC_AEAD_GIVENC_LEN + \ + CAAM_MAX_KEY_SIZE) +#define DESC_MAX_USED_LEN (DESC_MAX_USED_BYTES / CAAM_CMD_SZ) #ifdef DEBUG /* for print_hex_dumps with line references */ @@ -76,17 +82,77 @@ #define debug(format, arg...) #endif +/* Set DK bit in class 1 operation if shared */ +static inline void append_dec_op1(u32 *desc, u32 type) +{ + u32 *jump_cmd, *uncond_jump_cmd; + + jump_cmd = append_jump(desc, JUMP_TEST_ALL | JUMP_COND_SHRD); + append_operation(desc, type | OP_ALG_AS_INITFINAL | + OP_ALG_DECRYPT); + uncond_jump_cmd = append_jump(desc, JUMP_TEST_ALL); + set_jump_tgt_here(desc, jump_cmd); + append_operation(desc, type | OP_ALG_AS_INITFINAL | + OP_ALG_DECRYPT | OP_ALG_AAI_DK); + set_jump_tgt_here(desc, uncond_jump_cmd); +} + +/* + * Wait for completion of class 1 key loading before allowing + * error propagation + */ +static inline void append_dec_shr_done(u32 *desc) +{ + u32 *jump_cmd; + + jump_cmd = append_jump(desc, JUMP_CLASS_CLASS1 | JUMP_TEST_ALL); + set_jump_tgt_here(desc, jump_cmd); + append_cmd(desc, SET_OK_PROP_ERRORS | CMD_LOAD); +} + +/* + * For aead functions, read payload and write payload, + * both of which are specified in req->src and req->dst + */ +static inline void aead_append_src_dst(u32 *desc, u32 msg_type) +{ + append_seq_fifo_load(desc, 0, FIFOLD_CLASS_BOTH | + KEY_VLF | msg_type | FIFOLD_TYPE_LASTBOTH); + append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | KEY_VLF); +} + +/* + * For aead encrypt and decrypt, read iv for both classes + */ +static inline void aead_append_ld_iv(u32 *desc, int ivsize) +{ + append_cmd(desc, CMD_SEQ_LOAD | LDST_SRCDST_BYTE_CONTEXT | + LDST_CLASS_1_CCB | ivsize); + append_move(desc, MOVE_SRC_CLASS1CTX | MOVE_DEST_CLASS2INFIFO | ivsize); +} + +/* + * If all data, including src (with assoc and iv) or dst (with iv only) are + * contiguous + */ +#define GIV_SRC_CONTIG 1 +#define GIV_DST_CONTIG (1 << 1) + /* * per-session context */ struct caam_ctx { struct device *jrdev; - u32 *sh_desc; - dma_addr_t shared_desc_phys; + u32 sh_desc_enc[DESC_MAX_USED_LEN]; + u32 sh_desc_dec[DESC_MAX_USED_LEN]; + u32 sh_desc_givenc[DESC_MAX_USED_LEN]; + dma_addr_t sh_desc_enc_dma; + dma_addr_t sh_desc_dec_dma; + dma_addr_t sh_desc_givenc_dma; u32 class1_alg_type; u32 class2_alg_type; u32 alg_op; - u8 *key; + u8 key[CAAM_MAX_KEY_SIZE]; dma_addr_t key_dma; unsigned int enckeylen; unsigned int split_key_len; @@ -94,12 +160,275 @@ struct caam_ctx { unsigned int authsize; }; +static void append_key_aead(u32 *desc, struct caam_ctx *ctx, + int keys_fit_inline) +{ + if (keys_fit_inline) { + append_key_as_imm(desc, ctx->key, ctx->split_key_pad_len, + ctx->split_key_len, CLASS_2 | + KEY_DEST_MDHA_SPLIT | KEY_ENC); + append_key_as_imm(desc, (void *)ctx->key + + ctx->split_key_pad_len, ctx->enckeylen, + ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG); + } else { + append_key(desc, ctx->key_dma, ctx->split_key_len, CLASS_2 | + KEY_DEST_MDHA_SPLIT | KEY_ENC); + append_key(desc, ctx->key_dma + ctx->split_key_pad_len, + ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG); + } +} + +static void init_sh_desc_key_aead(u32 *desc, struct caam_ctx *ctx, + int keys_fit_inline) +{ + u32 *key_jump_cmd; + + init_sh_desc(desc, HDR_SHARE_WAIT); + + /* Skip if already shared */ + key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | + JUMP_COND_SHRD); + + append_key_aead(desc, ctx, keys_fit_inline); + + set_jump_tgt_here(desc, key_jump_cmd); + + /* Propagate errors from shared to job descriptor */ + append_cmd(desc, SET_OK_PROP_ERRORS | CMD_LOAD); +} + +static int aead_set_sh_desc(struct crypto_aead *aead) +{ + struct aead_tfm *tfm = &aead->base.crt_aead; + struct caam_ctx *ctx = crypto_aead_ctx(aead); + struct device *jrdev = ctx->jrdev; + bool keys_fit_inline = 0; + u32 *key_jump_cmd, *jump_cmd; + u32 geniv, moveiv; + u32 *desc; + + if (!ctx->enckeylen || !ctx->authsize) + return 0; + + /* + * Job Descriptor and Shared Descriptors + * must all fit into the 64-word Descriptor h/w Buffer + */ + if (DESC_AEAD_ENC_LEN + DESC_JOB_IO_LEN + + ctx->split_key_pad_len + ctx->enckeylen <= + CAAM_DESC_BYTES_MAX) + keys_fit_inline = 1; + + /* aead_encrypt shared descriptor */ + desc = ctx->sh_desc_enc; + + init_sh_desc_key_aead(desc, ctx, keys_fit_inline); + + /* Class 2 operation */ + append_operation(desc, ctx->class2_alg_type | + OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT); + + /* cryptlen = seqoutlen - authsize */ + append_math_sub_imm_u32(desc, REG3, SEQOUTLEN, IMM, ctx->authsize); + + /* assoclen + cryptlen = seqinlen - ivsize */ + append_math_sub_imm_u32(desc, REG2, SEQINLEN, IMM, tfm->ivsize); + + /* assoclen + cryptlen = (assoclen + cryptlen) - cryptlen */ + append_math_sub(desc, VARSEQINLEN, REG2, REG3, CAAM_CMD_SZ); + + /* read assoc before reading payload */ + append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG | + KEY_VLF); + aead_append_ld_iv(desc, tfm->ivsize); + + /* Class 1 operation */ + append_operation(desc, ctx->class1_alg_type | + OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT); + + /* Read and write cryptlen bytes */ + append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ); + append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); + aead_append_src_dst(desc, FIFOLD_TYPE_MSG1OUT2); + + /* Write ICV */ + append_seq_store(desc, ctx->authsize, LDST_CLASS_2_CCB | + LDST_SRCDST_BYTE_CONTEXT); + + ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc, + desc_bytes(desc), + DMA_TO_DEVICE); + if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) { + dev_err(jrdev, "unable to map shared descriptor\n"); + return -ENOMEM; + } +#ifdef DEBUG + print_hex_dump(KERN_ERR, "aead enc shdesc@"xstr(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, + desc_bytes(desc), 1); +#endif + + /* + * Job Descriptor and Shared Descriptors + * must all fit into the 64-word Descriptor h/w Buffer + */ + if (DESC_AEAD_DEC_LEN + DESC_JOB_IO_LEN + + ctx->split_key_pad_len + ctx->enckeylen <= + CAAM_DESC_BYTES_MAX) + keys_fit_inline = 1; + + desc = ctx->sh_desc_dec; + + /* aead_decrypt shared descriptor */ + init_sh_desc(desc, HDR_SHARE_WAIT); + + /* Skip if already shared */ + key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | + JUMP_COND_SHRD); + + append_key_aead(desc, ctx, keys_fit_inline); + + /* Only propagate error immediately if shared */ + jump_cmd = append_jump(desc, JUMP_TEST_ALL); + set_jump_tgt_here(desc, key_jump_cmd); + append_cmd(desc, SET_OK_PROP_ERRORS | CMD_LOAD); + set_jump_tgt_here(desc, jump_cmd); + + /* Class 2 operation */ + append_operation(desc, ctx->class2_alg_type | + OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT | OP_ALG_ICV_ON); + + /* assoclen + cryptlen = seqinlen - ivsize */ + append_math_sub_imm_u32(desc, REG3, SEQINLEN, IMM, + ctx->authsize + tfm->ivsize) + /* assoclen = (assoclen + cryptlen) - cryptlen */ + append_math_sub(desc, REG2, SEQOUTLEN, REG0, CAAM_CMD_SZ); + append_math_sub(desc, VARSEQINLEN, REG3, REG2, CAAM_CMD_SZ); + + /* read assoc before reading payload */ + append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG | + KEY_VLF); + + aead_append_ld_iv(desc, tfm->ivsize); + + append_dec_op1(desc, ctx->class1_alg_type); + + /* Read and write cryptlen bytes */ + append_math_add(desc, VARSEQINLEN, ZERO, REG2, CAAM_CMD_SZ); + append_math_add(desc, VARSEQOUTLEN, ZERO, REG2, CAAM_CMD_SZ); + aead_append_src_dst(desc, FIFOLD_TYPE_MSG); + + /* Load ICV */ + append_seq_fifo_load(desc, ctx->authsize, FIFOLD_CLASS_CLASS2 | + FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_ICV); + append_dec_shr_done(desc); + + ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc, + desc_bytes(desc), + DMA_TO_DEVICE); + if (dma_mapping_error(jrdev, ctx->sh_desc_dec_dma)) { + dev_err(jrdev, "unable to map shared descriptor\n"); + return -ENOMEM; + } +#ifdef DEBUG + print_hex_dump(KERN_ERR, "aead dec shdesc@"xstr(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, + desc_bytes(desc), 1); +#endif + + /* + * Job Descriptor and Shared Descriptors + * must all fit into the 64-word Descriptor h/w Buffer + */ + if (DESC_AEAD_GIVENC_LEN + DESC_JOB_IO_LEN + + ctx->split_key_pad_len + ctx->enckeylen <= + CAAM_DESC_BYTES_MAX) + keys_fit_inline = 1; + + /* aead_givencrypt shared descriptor */ + desc = ctx->sh_desc_givenc; + + init_sh_desc_key_aead(desc, ctx, keys_fit_inline); + + /* Generate IV */ + geniv = NFIFOENTRY_STYPE_PAD | NFIFOENTRY_DEST_DECO | + NFIFOENTRY_DTYPE_MSG | NFIFOENTRY_LC1 | + NFIFOENTRY_PTYPE_RND | (tfm->ivsize << NFIFOENTRY_DLEN_SHIFT); + append_load_imm_u32(desc, geniv, LDST_CLASS_IND_CCB | + LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM); + append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO); + append_move(desc, MOVE_SRC_INFIFO | + MOVE_DEST_CLASS1CTX | (tfm->ivsize << MOVE_LEN_SHIFT)); + append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO); + + /* Copy IV to class 1 context */ + append_move(desc, MOVE_SRC_CLASS1CTX | + MOVE_DEST_OUTFIFO | (tfm->ivsize << MOVE_LEN_SHIFT)); + + /* Return to encryption */ + append_operation(desc, ctx->class2_alg_type | + OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT); + + /* ivsize + cryptlen = seqoutlen - authsize */ + append_math_sub_imm_u32(desc, REG3, SEQOUTLEN, IMM, ctx->authsize); + + /* assoclen = seqinlen - (ivsize + cryptlen) */ + append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG3, CAAM_CMD_SZ); + + /* read assoc before reading payload */ + append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG | + KEY_VLF); + + /* Copy iv from class 1 ctx to class 2 fifo*/ + moveiv = NFIFOENTRY_STYPE_OFIFO | NFIFOENTRY_DEST_CLASS2 | + NFIFOENTRY_DTYPE_MSG | (tfm->ivsize << NFIFOENTRY_DLEN_SHIFT); + append_load_imm_u32(desc, moveiv, LDST_CLASS_IND_CCB | + LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM); + append_load_imm_u32(desc, tfm->ivsize, LDST_CLASS_2_CCB | + LDST_SRCDST_WORD_DATASZ_REG | LDST_IMM); + + /* Class 1 operation */ + append_operation(desc, ctx->class1_alg_type | + OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT); + + /* Will write ivsize + cryptlen */ + append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ); + + /* Not need to reload iv */ + append_seq_fifo_load(desc, tfm->ivsize, + FIFOLD_CLASS_SKIP); + + /* Will read cryptlen */ + append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ); + aead_append_src_dst(desc, FIFOLD_TYPE_MSG1OUT2); + + /* Write ICV */ + append_seq_store(desc, ctx->authsize, LDST_CLASS_2_CCB | + LDST_SRCDST_BYTE_CONTEXT); + + ctx->sh_desc_givenc_dma = dma_map_single(jrdev, desc, + desc_bytes(desc), + DMA_TO_DEVICE); + if (dma_mapping_error(jrdev, ctx->sh_desc_givenc_dma)) { + dev_err(jrdev, "unable to map shared descriptor\n"); + return -ENOMEM; + } +#ifdef DEBUG + print_hex_dump(KERN_ERR, "aead givenc shdesc@"xstr(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, + desc_bytes(desc), 1); +#endif + + return 0; +} + static int aead_setauthsize(struct crypto_aead *authenc, unsigned int authsize) { struct caam_ctx *ctx = crypto_aead_ctx(authenc); ctx->authsize = authsize; + aead_set_sh_desc(authenc); return 0; } @@ -117,6 +446,7 @@ static void split_key_done(struct device *dev, u32 *desc, u32 err, #ifdef DEBUG dev_err(dev, "%s %d: err 0x%x\n", __func__, __LINE__, err); #endif + if (err) { char tmp[CAAM_ERROR_STR_MAX]; @@ -220,72 +550,6 @@ static u32 gen_split_key(struct caam_ctx *ctx, const u8 *key_in, u32 authkeylen) return ret; } -static int build_sh_desc_ipsec(struct caam_ctx *ctx) -{ - struct device *jrdev = ctx->jrdev; - u32 *sh_desc; - u32 *jump_cmd; - bool keys_fit_inline = 0; - - /* - * largest Job Descriptor and its Shared Descriptor - * must both fit into the 64-word Descriptor h/w Buffer - */ - if ((DESC_AEAD_GIVENCRYPT_TEXT_LEN + - DESC_AEAD_SHARED_TEXT_LEN) * CAAM_CMD_SZ + - ctx->split_key_pad_len + ctx->enckeylen <= CAAM_DESC_BYTES_MAX) - keys_fit_inline = 1; - - /* build shared descriptor for this session */ - sh_desc = kmalloc(CAAM_CMD_SZ * DESC_AEAD_SHARED_TEXT_LEN + - (keys_fit_inline ? - ctx->split_key_pad_len + ctx->enckeylen : - CAAM_PTR_SZ * 2), GFP_DMA | GFP_KERNEL); - if (!sh_desc) { - dev_err(jrdev, "could not allocate shared descriptor\n"); - return -ENOMEM; - } - - init_sh_desc(sh_desc, HDR_SAVECTX | HDR_SHARE_SERIAL); - - jump_cmd = append_jump(sh_desc, CLASS_BOTH | JUMP_TEST_ALL | - JUMP_COND_SHRD | JUMP_COND_SELF); - - /* - * process keys, starting with class 2/authentication. - */ - if (keys_fit_inline) { - append_key_as_imm(sh_desc, ctx->key, ctx->split_key_pad_len, - ctx->split_key_len, - CLASS_2 | KEY_DEST_MDHA_SPLIT | KEY_ENC); - - append_key_as_imm(sh_desc, (void *)ctx->key + - ctx->split_key_pad_len, ctx->enckeylen, - ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG); - } else { - append_key(sh_desc, ctx->key_dma, ctx->split_key_len, CLASS_2 | - KEY_DEST_MDHA_SPLIT | KEY_ENC); - append_key(sh_desc, ctx->key_dma + ctx->split_key_pad_len, - ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG); - } - - /* update jump cmd now that we are at the jump target */ - set_jump_tgt_here(sh_desc, jump_cmd); - - ctx->shared_desc_phys = dma_map_single(jrdev, sh_desc, - desc_bytes(sh_desc), - DMA_TO_DEVICE); - if (dma_mapping_error(jrdev, ctx->shared_desc_phys)) { - dev_err(jrdev, "unable to map shared descriptor\n"); - kfree(sh_desc); - return -ENOMEM; - } - - ctx->sh_desc = sh_desc; - - return 0; -} - static int aead_setkey(struct crypto_aead *aead, const u8 *key, unsigned int keylen) { @@ -326,16 +590,9 @@ static int aead_setkey(struct crypto_aead *aead, print_hex_dump(KERN_ERR, "key in @"xstr(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); #endif - ctx->key = kmalloc(ctx->split_key_pad_len + enckeylen, - GFP_KERNEL | GFP_DMA); - if (!ctx->key) { - dev_err(jrdev, "could not allocate key output memory\n"); - return -ENOMEM; - } ret = gen_split_key(ctx, key, authkeylen); if (ret) { - kfree(ctx->key); goto badkey; } @@ -346,7 +603,6 @@ static int aead_setkey(struct crypto_aead *aead, enckeylen, DMA_TO_DEVICE); if (dma_mapping_error(jrdev, ctx->key_dma)) { dev_err(jrdev, "unable to map key i/o memory\n"); - kfree(ctx->key); return -ENOMEM; } #ifdef DEBUG @@ -357,11 +613,10 @@ static int aead_setkey(struct crypto_aead *aead, ctx->enckeylen = enckeylen; - ret = build_sh_desc_ipsec(ctx); + ret = aead_set_sh_desc(aead); if (ret) { dma_unmap_single(jrdev, ctx->key_dma, ctx->split_key_pad_len + enckeylen, DMA_TO_DEVICE); - kfree(ctx->key); } return ret; @@ -379,10 +634,11 @@ struct link_tbl_entry { }; /* - * aead_edesc - s/w-extended ipsec_esp descriptor + * aead_edesc - s/w-extended aead descriptor + * @assoc_nents: number of segments in associated data (SPI+Seq) scatterlist * @src_nents: number of segments in input scatterlist * @dst_nents: number of segments in output scatterlist - * @assoc_nents: number of segments in associated data (SPI+Seq) scatterlist + * @iv_dma: dma address of iv for checking continuity and link table * @desc: h/w descriptor (variable length; must not exceed MAX_CAAM_DESCSIZE) * @link_tbl_bytes: length of dma mapped link_tbl space * @link_tbl_dma: bus physical mapped address of h/w link table @@ -392,37 +648,47 @@ struct aead_edesc { int assoc_nents; int src_nents; int dst_nents; + dma_addr_t iv_dma; int link_tbl_bytes; dma_addr_t link_tbl_dma; struct link_tbl_entry *link_tbl; u32 hw_desc[0]; }; -static void aead_unmap(struct device *dev, - struct aead_edesc *edesc, - struct aead_request *req) +static void caam_unmap(struct device *dev, struct scatterlist *src, + struct scatterlist *dst, int src_nents, int dst_nents, + dma_addr_t iv_dma, int ivsize, dma_addr_t link_tbl_dma, + int link_tbl_bytes) { - dma_unmap_sg(dev, req->assoc, edesc->assoc_nents, DMA_TO_DEVICE); - - if (unlikely(req->dst != req->src)) { - dma_unmap_sg(dev, req->src, edesc->src_nents, - DMA_TO_DEVICE); - dma_unmap_sg(dev, req->dst, edesc->dst_nents, - DMA_FROM_DEVICE); + if (unlikely(dst != src)) { + dma_unmap_sg(dev, src, src_nents, DMA_TO_DEVICE); + dma_unmap_sg(dev, dst, dst_nents, DMA_FROM_DEVICE); } else { - dma_unmap_sg(dev, req->src, edesc->src_nents, - DMA_BIDIRECTIONAL); + dma_unmap_sg(dev, src, src_nents, DMA_BIDIRECTIONAL); } - if (edesc->link_tbl_bytes) - dma_unmap_single(dev, edesc->link_tbl_dma, - edesc->link_tbl_bytes, + if (iv_dma) + dma_unmap_single(dev, iv_dma, ivsize, DMA_TO_DEVICE); + if (link_tbl_bytes) + dma_unmap_single(dev, link_tbl_dma, link_tbl_bytes, DMA_TO_DEVICE); } -/* - * ipsec_esp descriptor callbacks - */ +static void aead_unmap(struct device *dev, + struct aead_edesc *edesc, + struct aead_request *req) +{ + struct crypto_aead *aead = crypto_aead_reqtfm(req); + int ivsize = crypto_aead_ivsize(aead); + + dma_unmap_sg(dev, req->assoc, edesc->assoc_nents, DMA_TO_DEVICE); + + caam_unmap(dev, req->src, req->dst, + edesc->src_nents, edesc->dst_nents, + edesc->iv_dma, ivsize, edesc->link_tbl_dma, + edesc->link_tbl_bytes); +} + static void aead_encrypt_done(struct device *jrdev, u32 *desc, u32 err, void *context) { @@ -430,11 +696,12 @@ static void aead_encrypt_done(struct device *jrdev, u32 *desc, u32 err, struct aead_edesc *edesc; #ifdef DEBUG struct crypto_aead *aead = crypto_aead_reqtfm(req); - int ivsize = crypto_aead_ivsize(aead); struct caam_ctx *ctx = crypto_aead_ctx(aead); + int ivsize = crypto_aead_ivsize(aead); dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); #endif + edesc = (struct aead_edesc *)((char *)desc - offsetof(struct aead_edesc, hw_desc)); @@ -472,12 +739,23 @@ static void aead_decrypt_done(struct device *jrdev, u32 *desc, u32 err, #ifdef DEBUG struct crypto_aead *aead = crypto_aead_reqtfm(req); struct caam_ctx *ctx = crypto_aead_ctx(aead); + int ivsize = crypto_aead_ivsize(aead); dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); #endif + edesc = (struct aead_edesc *)((char *)desc - offsetof(struct aead_edesc, hw_desc)); +#ifdef DEBUG + print_hex_dump(KERN_ERR, "dstiv @"xstr(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, req->iv, + ivsize, 1); + print_hex_dump(KERN_ERR, "dst @"xstr(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->dst), + req->cryptlen, 1); +#endif + if (err) { char tmp[CAAM_ERROR_STR_MAX]; @@ -506,241 +784,271 @@ static void aead_decrypt_done(struct device *jrdev, u32 *desc, u32 err, sg->length + ctx->authsize + 16, 1); } #endif + kfree(edesc); aead_request_complete(req, err); } +static void sg_to_link_tbl_one(struct link_tbl_entry *link_tbl_ptr, + dma_addr_t dma, u32 len, u32 offset) +{ + link_tbl_ptr->ptr = dma; + link_tbl_ptr->len = len; + link_tbl_ptr->reserved = 0; + link_tbl_ptr->buf_pool_id = 0; + link_tbl_ptr->offset = offset; +#ifdef DEBUG + print_hex_dump(KERN_ERR, "link_tbl_ptr@"xstr(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, link_tbl_ptr, + sizeof(struct link_tbl_entry), 1); +#endif +} + /* * convert scatterlist to h/w link table format - * scatterlist must have been previously dma mapped + * but does not have final bit; instead, returns last entry */ -static void sg_to_link_tbl(struct scatterlist *sg, int sg_count, - struct link_tbl_entry *link_tbl_ptr, u32 offset) +static struct link_tbl_entry *sg_to_link_tbl(struct scatterlist *sg, + int sg_count, struct link_tbl_entry + *link_tbl_ptr, u32 offset) { while (sg_count) { - link_tbl_ptr->ptr = sg_dma_address(sg); - link_tbl_ptr->len = sg_dma_len(sg); - link_tbl_ptr->reserved = 0; - link_tbl_ptr->buf_pool_id = 0; - link_tbl_ptr->offset = offset; + sg_to_link_tbl_one(link_tbl_ptr, sg_dma_address(sg), + sg_dma_len(sg), offset); link_tbl_ptr++; sg = sg_next(sg); sg_count--; } + return link_tbl_ptr - 1; +} - /* set Final bit (marks end of link table) */ - link_tbl_ptr--; +/* + * convert scatterlist to h/w link table format + * scatterlist must have been previously dma mapped + */ +static void sg_to_link_tbl_last(struct scatterlist *sg, int sg_count, + struct link_tbl_entry *link_tbl_ptr, u32 offset) +{ + link_tbl_ptr = sg_to_link_tbl(sg, sg_count, link_tbl_ptr, offset); link_tbl_ptr->len |= 0x40000000; } /* - * fill in and submit ipsec_esp job descriptor + * Fill in aead job descriptor */ -static int init_aead_job(struct aead_edesc *edesc, struct aead_request *req, - u32 encrypt, - void (*callback) (struct device *dev, u32 *desc, - u32 err, void *context)) +static void init_aead_job(u32 *sh_desc, dma_addr_t ptr, + struct aead_edesc *edesc, + struct aead_request *req, + bool all_contig, bool encrypt) { struct crypto_aead *aead = crypto_aead_reqtfm(req); struct caam_ctx *ctx = crypto_aead_ctx(aead); - struct device *jrdev = ctx->jrdev; - u32 *desc = edesc->hw_desc, options; - int ret, sg_count, assoc_sg_count; int ivsize = crypto_aead_ivsize(aead); int authsize = ctx->authsize; - dma_addr_t ptr, dst_dma, src_dma; -#ifdef DEBUG - u32 *sh_desc = ctx->sh_desc; + u32 *desc = edesc->hw_desc; + u32 out_options = 0, in_options; + dma_addr_t dst_dma, src_dma; + int len, link_tbl_index = 0; +#ifdef DEBUG debug("assoclen %d cryptlen %d authsize %d\n", req->assoclen, req->cryptlen, authsize); print_hex_dump(KERN_ERR, "assoc @"xstr(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->assoc), req->assoclen , 1); print_hex_dump(KERN_ERR, "presciv@"xstr(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src) - ivsize, + DUMP_PREFIX_ADDRESS, 16, 4, req->iv, edesc->src_nents ? 100 : ivsize, 1); print_hex_dump(KERN_ERR, "src @"xstr(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src), - edesc->src_nents ? 100 : req->cryptlen + authsize, 1); + edesc->src_nents ? 100 : req->cryptlen, 1); print_hex_dump(KERN_ERR, "shrdesc@"xstr(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, sh_desc, desc_bytes(sh_desc), 1); #endif - assoc_sg_count = dma_map_sg(jrdev, req->assoc, edesc->assoc_nents ?: 1, - DMA_TO_DEVICE); - if (req->src == req->dst) - sg_count = dma_map_sg(jrdev, req->src, edesc->src_nents ? : 1, - DMA_BIDIRECTIONAL); - else - sg_count = dma_map_sg(jrdev, req->src, edesc->src_nents ? : 1, - DMA_TO_DEVICE); - /* start auth operation */ - append_operation(desc, ctx->class2_alg_type | OP_ALG_AS_INITFINAL | - (encrypt ? : OP_ALG_ICV_ON)); + len = desc_len(sh_desc); + init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE); - /* Load FIFO with data for Class 2 CHA */ - options = FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG; - if (!edesc->assoc_nents) { - ptr = sg_dma_address(req->assoc); + if (all_contig) { + src_dma = sg_dma_address(req->assoc); + in_options = 0; } else { - sg_to_link_tbl(req->assoc, edesc->assoc_nents, - edesc->link_tbl, 0); - ptr = edesc->link_tbl_dma; - options |= LDST_SGF; + src_dma = edesc->link_tbl_dma; + link_tbl_index += (edesc->assoc_nents ? : 1) + 1 + + (edesc->src_nents ? : 1); + in_options = LDST_SGF; } - append_fifo_load(desc, ptr, req->assoclen, options); - - /* copy iv from cipher/class1 input context to class2 infifo */ - append_move(desc, MOVE_SRC_CLASS1CTX | MOVE_DEST_CLASS2INFIFO | ivsize); - - if (!encrypt) { - u32 *jump_cmd, *uncond_jump_cmd; - - /* JUMP if shared */ - jump_cmd = append_jump(desc, JUMP_TEST_ALL | JUMP_COND_SHRD); - - /* start class 1 (cipher) operation, non-shared version */ - append_operation(desc, ctx->class1_alg_type | - OP_ALG_AS_INITFINAL); - - uncond_jump_cmd = append_jump(desc, 0); - - set_jump_tgt_here(desc, jump_cmd); - - /* start class 1 (cipher) operation, shared version */ - append_operation(desc, ctx->class1_alg_type | - OP_ALG_AS_INITFINAL | OP_ALG_AAI_DK); - set_jump_tgt_here(desc, uncond_jump_cmd); - } else - append_operation(desc, ctx->class1_alg_type | - OP_ALG_AS_INITFINAL | encrypt); + if (encrypt) + append_seq_in_ptr(desc, src_dma, req->assoclen + ivsize + + req->cryptlen - authsize, in_options); + else + append_seq_in_ptr(desc, src_dma, req->assoclen + ivsize + + req->cryptlen, in_options); - /* load payload & instruct to class2 to snoop class 1 if encrypting */ - options = 0; - if (!edesc->src_nents) { - src_dma = sg_dma_address(req->src); - } else { - sg_to_link_tbl(req->src, edesc->src_nents, edesc->link_tbl + - edesc->assoc_nents, 0); - src_dma = edesc->link_tbl_dma + edesc->assoc_nents * - sizeof(struct link_tbl_entry); - options |= LDST_SGF; - } - append_seq_in_ptr(desc, src_dma, req->cryptlen + authsize, options); - append_seq_fifo_load(desc, req->cryptlen, FIFOLD_CLASS_BOTH | - FIFOLD_TYPE_LASTBOTH | - (encrypt ? FIFOLD_TYPE_MSG1OUT2 - : FIFOLD_TYPE_MSG)); - - /* specify destination */ - if (req->src == req->dst) { - dst_dma = src_dma; + if (likely(req->src == req->dst)) { + if (all_contig) { + dst_dma = sg_dma_address(req->src); + } else { + dst_dma = src_dma + sizeof(struct link_tbl_entry) * + ((edesc->assoc_nents ? : 1) + 1); + out_options = LDST_SGF; + } } else { - sg_count = dma_map_sg(jrdev, req->dst, edesc->dst_nents ? : 1, - DMA_FROM_DEVICE); if (!edesc->dst_nents) { dst_dma = sg_dma_address(req->dst); - options = 0; } else { - sg_to_link_tbl(req->dst, edesc->dst_nents, - edesc->link_tbl + edesc->assoc_nents + - edesc->src_nents, 0); - dst_dma = edesc->link_tbl_dma + (edesc->assoc_nents + - edesc->src_nents) * + dst_dma = edesc->link_tbl_dma + + link_tbl_index * sizeof(struct link_tbl_entry); - options = LDST_SGF; + out_options = LDST_SGF; } } - append_seq_out_ptr(desc, dst_dma, req->cryptlen + authsize, options); - append_seq_fifo_store(desc, req->cryptlen, FIFOST_TYPE_MESSAGE_DATA); - - /* ICV */ if (encrypt) - append_seq_store(desc, authsize, LDST_CLASS_2_CCB | - LDST_SRCDST_BYTE_CONTEXT); + append_seq_out_ptr(desc, dst_dma, req->cryptlen, out_options); else - append_seq_fifo_load(desc, authsize, FIFOLD_CLASS_CLASS2 | - FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_ICV); + append_seq_out_ptr(desc, dst_dma, req->cryptlen - authsize, + out_options); +} + +/* + * Fill in aead givencrypt job descriptor + */ +static void init_aead_giv_job(u32 *sh_desc, dma_addr_t ptr, + struct aead_edesc *edesc, + struct aead_request *req, + int contig) +{ + struct crypto_aead *aead = crypto_aead_reqtfm(req); + struct caam_ctx *ctx = crypto_aead_ctx(aead); + int ivsize = crypto_aead_ivsize(aead); + int authsize = ctx->authsize; + u32 *desc = edesc->hw_desc; + u32 out_options = 0, in_options; + dma_addr_t dst_dma, src_dma; + int len, link_tbl_index = 0; #ifdef DEBUG - debug("job_desc_len %d\n", desc_len(desc)); - print_hex_dump(KERN_ERR, "jobdesc@"xstr(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc) , 1); - print_hex_dump(KERN_ERR, "jdlinkt@"xstr(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, edesc->link_tbl, - edesc->link_tbl_bytes, 1); + debug("assoclen %d cryptlen %d authsize %d\n", + req->assoclen, req->cryptlen, authsize); + print_hex_dump(KERN_ERR, "assoc @"xstr(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->assoc), + req->assoclen , 1); + print_hex_dump(KERN_ERR, "presciv@"xstr(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, req->iv, ivsize, 1); + print_hex_dump(KERN_ERR, "src @"xstr(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src), + edesc->src_nents > 1 ? 100 : req->cryptlen, 1); + print_hex_dump(KERN_ERR, "shrdesc@"xstr(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, sh_desc, + desc_bytes(sh_desc), 1); #endif - ret = caam_jr_enqueue(jrdev, desc, callback, req); - if (!ret) - ret = -EINPROGRESS; - else { - aead_unmap(jrdev, edesc, req); - kfree(edesc); + len = desc_len(sh_desc); + init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE); + + if (contig & GIV_SRC_CONTIG) { + src_dma = sg_dma_address(req->assoc); + in_options = 0; + } else { + src_dma = edesc->link_tbl_dma; + link_tbl_index += edesc->assoc_nents + 1 + edesc->src_nents; + in_options = LDST_SGF; } + append_seq_in_ptr(desc, src_dma, req->assoclen + ivsize + + req->cryptlen - authsize, in_options); - return ret; + if (contig & GIV_DST_CONTIG) { + dst_dma = edesc->iv_dma; + } else { + if (likely(req->src == req->dst)) { + dst_dma = src_dma + sizeof(struct link_tbl_entry) * + edesc->assoc_nents; + out_options = LDST_SGF; + } else { + dst_dma = edesc->link_tbl_dma + + link_tbl_index * + sizeof(struct link_tbl_entry); + out_options = LDST_SGF; + } + } + + append_seq_out_ptr(desc, dst_dma, ivsize + req->cryptlen, out_options); } /* * derive number of elements in scatterlist */ -static int sg_count(struct scatterlist *sg_list, int nbytes, int *chained) +static int sg_count(struct scatterlist *sg_list, int nbytes) { struct scatterlist *sg = sg_list; int sg_nents = 0; - *chained = 0; while (nbytes > 0) { sg_nents++; nbytes -= sg->length; if (!sg_is_last(sg) && (sg + 1)->length == 0) - *chained = 1; + BUG(); /* Not support chaining */ sg = scatterwalk_sg_next(sg); } + if (likely(sg_nents == 1)) + return 0; + return sg_nents; } /* - * allocate and map the ipsec_esp extended descriptor + * allocate and map the aead extended descriptor */ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, - int desc_bytes) + int desc_bytes, bool *all_contig_ptr) { struct crypto_aead *aead = crypto_aead_reqtfm(req); struct caam_ctx *ctx = crypto_aead_ctx(aead); struct device *jrdev = ctx->jrdev; - gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL : - GFP_ATOMIC; - int assoc_nents, src_nents, dst_nents = 0, chained, link_tbl_bytes; + gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG | + CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC; + int assoc_nents, src_nents, dst_nents = 0; struct aead_edesc *edesc; + dma_addr_t iv_dma = 0; + int sgc; + bool all_contig = true; + int ivsize = crypto_aead_ivsize(aead); + int link_tbl_index, link_tbl_len = 0, link_tbl_bytes; - assoc_nents = sg_count(req->assoc, req->assoclen, &chained); - BUG_ON(chained); - if (likely(assoc_nents == 1)) - assoc_nents = 0; - - src_nents = sg_count(req->src, req->cryptlen + ctx->authsize, - &chained); - BUG_ON(chained); - if (src_nents == 1) - src_nents = 0; - - if (unlikely(req->dst != req->src)) { - dst_nents = sg_count(req->dst, req->cryptlen + ctx->authsize, - &chained); - BUG_ON(chained); - if (dst_nents == 1) - dst_nents = 0; + assoc_nents = sg_count(req->assoc, req->assoclen); + src_nents = sg_count(req->src, req->cryptlen); + + if (unlikely(req->dst != req->src)) + dst_nents = sg_count(req->dst, req->cryptlen); + + sgc = dma_map_sg(jrdev, req->assoc, assoc_nents ? : 1, + DMA_BIDIRECTIONAL); + if (likely(req->src == req->dst)) { + sgc = dma_map_sg(jrdev, req->src, src_nents ? : 1, + DMA_BIDIRECTIONAL); + } else { + sgc = dma_map_sg(jrdev, req->src, src_nents ? : 1, + DMA_TO_DEVICE); + sgc = dma_map_sg(jrdev, req->dst, dst_nents ? : 1, + DMA_FROM_DEVICE); + } + + /* Check if data are contiguous */ + iv_dma = dma_map_single(jrdev, req->iv, ivsize, DMA_TO_DEVICE); + if (assoc_nents || sg_dma_address(req->assoc) + req->assoclen != + iv_dma || src_nents || iv_dma + ivsize != + sg_dma_address(req->src)) { + all_contig = false; + assoc_nents = assoc_nents ? : 1; + src_nents = src_nents ? : 1; + link_tbl_len = assoc_nents + 1 + src_nents; } + link_tbl_len += dst_nents; - link_tbl_bytes = (assoc_nents + src_nents + dst_nents) * - sizeof(struct link_tbl_entry); - debug("link_tbl_bytes %d\n", link_tbl_bytes); + link_tbl_bytes = link_tbl_len * sizeof(struct link_tbl_entry); /* allocate space for base edesc and hw desc commands, link tables */ edesc = kmalloc(sizeof(struct aead_edesc) + desc_bytes + @@ -753,11 +1061,34 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, edesc->assoc_nents = assoc_nents; edesc->src_nents = src_nents; edesc->dst_nents = dst_nents; + edesc->iv_dma = iv_dma; + edesc->link_tbl_bytes = link_tbl_bytes; edesc->link_tbl = (void *)edesc + sizeof(struct aead_edesc) + desc_bytes; edesc->link_tbl_dma = dma_map_single(jrdev, edesc->link_tbl, link_tbl_bytes, DMA_TO_DEVICE); - edesc->link_tbl_bytes = link_tbl_bytes; + *all_contig_ptr = all_contig; + + link_tbl_index = 0; + if (!all_contig) { + sg_to_link_tbl(req->assoc, + (assoc_nents ? : 1), + edesc->link_tbl + + link_tbl_index, 0); + link_tbl_index += assoc_nents ? : 1; + sg_to_link_tbl_one(edesc->link_tbl + link_tbl_index, + iv_dma, ivsize, 0); + link_tbl_index += 1; + sg_to_link_tbl_last(req->src, + (src_nents ? : 1), + edesc->link_tbl + + link_tbl_index, 0); + link_tbl_index += src_nents ? : 1; + } + if (dst_nents) { + sg_to_link_tbl_last(req->dst, dst_nents, + edesc->link_tbl + link_tbl_index, 0); + } return edesc; } @@ -768,62 +1099,185 @@ static int aead_encrypt(struct aead_request *req) struct crypto_aead *aead = crypto_aead_reqtfm(req); struct caam_ctx *ctx = crypto_aead_ctx(aead); struct device *jrdev = ctx->jrdev; - int ivsize = crypto_aead_ivsize(aead); + bool all_contig; u32 *desc; - dma_addr_t iv_dma; + int ret = 0; + + req->cryptlen += ctx->authsize; /* allocate extended descriptor */ - edesc = aead_edesc_alloc(req, DESC_AEAD_ENCRYPT_TEXT_LEN * - CAAM_CMD_SZ); + edesc = aead_edesc_alloc(req, DESC_JOB_IO_LEN * + CAAM_CMD_SZ, &all_contig); if (IS_ERR(edesc)) return PTR_ERR(edesc); - desc = edesc->hw_desc; - - /* insert shared descriptor pointer */ - init_job_desc_shared(desc, ctx->shared_desc_phys, - desc_len(ctx->sh_desc), HDR_SHARE_DEFER); - - iv_dma = dma_map_single(jrdev, req->iv, ivsize, DMA_TO_DEVICE); - /* check dma error */ + /* Create and submit job descriptor */ + init_aead_job(ctx->sh_desc_enc, ctx->sh_desc_enc_dma, edesc, req, + all_contig, true); +#ifdef DEBUG + print_hex_dump(KERN_ERR, "aead jobdesc@"xstr(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc, + desc_bytes(edesc->hw_desc), 1); +#endif - append_load(desc, iv_dma, ivsize, - LDST_CLASS_1_CCB | LDST_SRCDST_BYTE_CONTEXT); + desc = edesc->hw_desc; + ret = caam_jr_enqueue(jrdev, desc, aead_encrypt_done, req); + if (!ret) { + ret = -EINPROGRESS; + } else { + aead_unmap(jrdev, edesc, req); + kfree(edesc); + } - return init_aead_job(edesc, req, OP_ALG_ENCRYPT, aead_encrypt_done); + return ret; } static int aead_decrypt(struct aead_request *req) { + struct aead_edesc *edesc; struct crypto_aead *aead = crypto_aead_reqtfm(req); - int ivsize = crypto_aead_ivsize(aead); struct caam_ctx *ctx = crypto_aead_ctx(aead); struct device *jrdev = ctx->jrdev; - struct aead_edesc *edesc; + bool all_contig; u32 *desc; - dma_addr_t iv_dma; - - req->cryptlen -= ctx->authsize; + int ret = 0; /* allocate extended descriptor */ - edesc = aead_edesc_alloc(req, DESC_AEAD_DECRYPT_TEXT_LEN * - CAAM_CMD_SZ); + edesc = aead_edesc_alloc(req, DESC_JOB_IO_LEN * + CAAM_CMD_SZ, &all_contig); if (IS_ERR(edesc)) return PTR_ERR(edesc); +#ifdef DEBUG + print_hex_dump(KERN_ERR, "dec src@"xstr(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src), + req->cryptlen, 1); +#endif + + /* Create and submit job descriptor*/ + init_aead_job(ctx->sh_desc_dec, + ctx->sh_desc_dec_dma, edesc, req, all_contig, false); +#ifdef DEBUG + print_hex_dump(KERN_ERR, "aead jobdesc@"xstr(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc, + desc_bytes(edesc->hw_desc), 1); +#endif + desc = edesc->hw_desc; + ret = caam_jr_enqueue(jrdev, desc, aead_decrypt_done, req); + if (!ret) { + ret = -EINPROGRESS; + } else { + aead_unmap(jrdev, edesc, req); + kfree(edesc); + } - /* insert shared descriptor pointer */ - init_job_desc_shared(desc, ctx->shared_desc_phys, - desc_len(ctx->sh_desc), HDR_SHARE_DEFER); + return ret; +} - iv_dma = dma_map_single(jrdev, req->iv, ivsize, DMA_TO_DEVICE); - /* check dma error */ +/* + * allocate and map the aead extended descriptor for aead givencrypt + */ +static struct aead_edesc *aead_giv_edesc_alloc(struct aead_givcrypt_request + *greq, int desc_bytes, + u32 *contig_ptr) +{ + struct aead_request *req = &greq->areq; + struct crypto_aead *aead = crypto_aead_reqtfm(req); + struct caam_ctx *ctx = crypto_aead_ctx(aead); + struct device *jrdev = ctx->jrdev; + gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG | + CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC; + int assoc_nents, src_nents, dst_nents = 0; + struct aead_edesc *edesc; + dma_addr_t iv_dma = 0; + int sgc; + u32 contig = GIV_SRC_CONTIG | GIV_DST_CONTIG; + int ivsize = crypto_aead_ivsize(aead); + int link_tbl_index, link_tbl_len = 0, link_tbl_bytes; - append_load(desc, iv_dma, ivsize, - LDST_CLASS_1_CCB | LDST_SRCDST_BYTE_CONTEXT); + assoc_nents = sg_count(req->assoc, req->assoclen); + src_nents = sg_count(req->src, req->cryptlen); - return init_aead_job(edesc, req, !OP_ALG_ENCRYPT, aead_decrypt_done); + if (unlikely(req->dst != req->src)) + dst_nents = sg_count(req->dst, req->cryptlen); + + sgc = dma_map_sg(jrdev, req->assoc, assoc_nents ? : 1, + DMA_BIDIRECTIONAL); + if (likely(req->src == req->dst)) { + sgc = dma_map_sg(jrdev, req->src, src_nents ? : 1, + DMA_BIDIRECTIONAL); + } else { + sgc = dma_map_sg(jrdev, req->src, src_nents ? : 1, + DMA_TO_DEVICE); + sgc = dma_map_sg(jrdev, req->dst, dst_nents ? : 1, + DMA_FROM_DEVICE); + } + + /* Check if data are contiguous */ + iv_dma = dma_map_single(jrdev, greq->giv, ivsize, DMA_TO_DEVICE); + if (assoc_nents || sg_dma_address(req->assoc) + req->assoclen != + iv_dma || src_nents || iv_dma + ivsize != sg_dma_address(req->src)) + contig &= ~GIV_SRC_CONTIG; + if (dst_nents || iv_dma + ivsize != sg_dma_address(req->dst)) + contig &= ~GIV_DST_CONTIG; + if (unlikely(req->src != req->dst)) { + dst_nents = dst_nents ? : 1; + link_tbl_len += 1; + } + if (!(contig & GIV_SRC_CONTIG)) { + assoc_nents = assoc_nents ? : 1; + src_nents = src_nents ? : 1; + link_tbl_len += assoc_nents + 1 + src_nents; + if (likely(req->src == req->dst)) + contig &= ~GIV_DST_CONTIG; + } + link_tbl_len += dst_nents; + + link_tbl_bytes = link_tbl_len * sizeof(struct link_tbl_entry); + + /* allocate space for base edesc and hw desc commands, link tables */ + edesc = kmalloc(sizeof(struct aead_edesc) + desc_bytes + + link_tbl_bytes, GFP_DMA | flags); + if (!edesc) { + dev_err(jrdev, "could not allocate extended descriptor\n"); + return ERR_PTR(-ENOMEM); + } + + edesc->assoc_nents = assoc_nents; + edesc->src_nents = src_nents; + edesc->dst_nents = dst_nents; + edesc->iv_dma = iv_dma; + edesc->link_tbl_bytes = link_tbl_bytes; + edesc->link_tbl = (void *)edesc + sizeof(struct aead_edesc) + + desc_bytes; + edesc->link_tbl_dma = dma_map_single(jrdev, edesc->link_tbl, + link_tbl_bytes, DMA_TO_DEVICE); + *contig_ptr = contig; + + link_tbl_index = 0; + if (!(contig & GIV_SRC_CONTIG)) { + sg_to_link_tbl(req->assoc, assoc_nents, + edesc->link_tbl + + link_tbl_index, 0); + link_tbl_index += assoc_nents; + sg_to_link_tbl_one(edesc->link_tbl + link_tbl_index, + iv_dma, ivsize, 0); + link_tbl_index += 1; + sg_to_link_tbl_last(req->src, src_nents, + edesc->link_tbl + + link_tbl_index, 0); + link_tbl_index += src_nents; + } + if (unlikely(req->src != req->dst && !(contig & GIV_DST_CONTIG))) { + sg_to_link_tbl_one(edesc->link_tbl + link_tbl_index, + iv_dma, ivsize, 0); + link_tbl_index += 1; + sg_to_link_tbl_last(req->dst, dst_nents, + edesc->link_tbl + link_tbl_index, 0); + } + + return edesc; } static int aead_givencrypt(struct aead_givcrypt_request *areq) @@ -833,55 +1287,44 @@ static int aead_givencrypt(struct aead_givcrypt_request *areq) struct crypto_aead *aead = crypto_aead_reqtfm(req); struct caam_ctx *ctx = crypto_aead_ctx(aead); struct device *jrdev = ctx->jrdev; - int ivsize = crypto_aead_ivsize(aead); - dma_addr_t iv_dma; + u32 contig; u32 *desc; + int ret = 0; - iv_dma = dma_map_single(jrdev, areq->giv, ivsize, DMA_FROM_DEVICE); - - debug("%s: giv %p\n", __func__, areq->giv); + req->cryptlen += ctx->authsize; /* allocate extended descriptor */ - edesc = aead_edesc_alloc(req, DESC_AEAD_GIVENCRYPT_TEXT_LEN * - CAAM_CMD_SZ); + edesc = aead_giv_edesc_alloc(areq, DESC_JOB_IO_LEN * + CAAM_CMD_SZ, &contig); + if (IS_ERR(edesc)) return PTR_ERR(edesc); - desc = edesc->hw_desc; - - /* insert shared descriptor pointer */ - init_job_desc_shared(desc, ctx->shared_desc_phys, - desc_len(ctx->sh_desc), HDR_SHARE_DEFER); - - /* - * LOAD IMM Info FIFO - * to DECO, Last, Padding, Random, Message, 16 bytes - */ - append_load_imm_u32(desc, NFIFOENTRY_DEST_DECO | NFIFOENTRY_LC1 | - NFIFOENTRY_STYPE_PAD | NFIFOENTRY_DTYPE_MSG | - NFIFOENTRY_PTYPE_RND | ivsize, - LDST_SRCDST_WORD_INFO_FIFO); - - /* - * disable info fifo entries since the above serves as the entry - * this way, the MOVE command won't generate an entry. - * Note that this isn't required in more recent versions of - * SEC as a MOVE that doesn't do info FIFO entries is available. - */ - append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO); - - /* MOVE DECO Alignment -> C1 Context 16 bytes */ - append_move(desc, MOVE_SRC_INFIFO | MOVE_DEST_CLASS1CTX | ivsize); - - /* re-enable info fifo entries */ - append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO); +#ifdef DEBUG + print_hex_dump(KERN_ERR, "giv src@"xstr(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src), + req->cryptlen, 1); +#endif - /* MOVE C1 Context -> OFIFO 16 bytes */ - append_move(desc, MOVE_SRC_CLASS1CTX | MOVE_DEST_OUTFIFO | ivsize); + /* Create and submit job descriptor*/ + init_aead_giv_job(ctx->sh_desc_givenc, + ctx->sh_desc_givenc_dma, edesc, req, contig); +#ifdef DEBUG + print_hex_dump(KERN_ERR, "aead jobdesc@"xstr(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc, + desc_bytes(edesc->hw_desc), 1); +#endif - append_fifo_store(desc, iv_dma, ivsize, FIFOST_TYPE_MESSAGE_DATA); + desc = edesc->hw_desc; + ret = caam_jr_enqueue(jrdev, desc, aead_encrypt_done, req); + if (!ret) { + ret = -EINPROGRESS; + } else { + aead_unmap(jrdev, edesc, req); + kfree(edesc); + } - return init_aead_job(edesc, req, OP_ALG_ENCRYPT, aead_encrypt_done); + return ret; } #define template_aead template_u.aead @@ -1120,16 +1563,19 @@ static void caam_cra_exit(struct crypto_tfm *tfm) { struct caam_ctx *ctx = crypto_tfm_ctx(tfm); - if (!dma_mapping_error(ctx->jrdev, ctx->shared_desc_phys)) - dma_unmap_single(ctx->jrdev, ctx->shared_desc_phys, - desc_bytes(ctx->sh_desc), DMA_TO_DEVICE); - kfree(ctx->sh_desc); - - if (!dma_mapping_error(ctx->jrdev, ctx->key_dma)) - dma_unmap_single(ctx->jrdev, ctx->key_dma, - ctx->split_key_pad_len + ctx->enckeylen, + if (ctx->sh_desc_enc_dma && + !dma_mapping_error(ctx->jrdev, ctx->sh_desc_enc_dma)) + dma_unmap_single(ctx->jrdev, ctx->sh_desc_enc_dma, + desc_bytes(ctx->sh_desc_enc), DMA_TO_DEVICE); + if (ctx->sh_desc_dec_dma && + !dma_mapping_error(ctx->jrdev, ctx->sh_desc_dec_dma)) + dma_unmap_single(ctx->jrdev, ctx->sh_desc_dec_dma, + desc_bytes(ctx->sh_desc_dec), DMA_TO_DEVICE); + if (ctx->sh_desc_givenc_dma && + !dma_mapping_error(ctx->jrdev, ctx->sh_desc_givenc_dma)) + dma_unmap_single(ctx->jrdev, ctx->sh_desc_givenc_dma, + desc_bytes(ctx->sh_desc_givenc), DMA_TO_DEVICE); - kfree(ctx->key); } static void __exit caam_algapi_exit(void) diff --git a/drivers/crypto/caam/desc_constr.h b/drivers/crypto/caam/desc_constr.h index 46915800c26..0991323cf3f 100644 --- a/drivers/crypto/caam/desc_constr.h +++ b/drivers/crypto/caam/desc_constr.h @@ -9,7 +9,7 @@ #define IMMEDIATE (1 << 23) #define CAAM_CMD_SZ sizeof(u32) #define CAAM_PTR_SZ sizeof(dma_addr_t) -#define CAAM_DESC_BYTES_MAX (CAAM_CMD_SZ * 64) +#define CAAM_DESC_BYTES_MAX (CAAM_CMD_SZ * MAX_CAAM_DESCSIZE) #ifdef DEBUG #define PRINT_POS do { printk(KERN_DEBUG "%02d: %s\n", desc_len(desc),\ @@ -18,6 +18,9 @@ #define PRINT_POS #endif +#define SET_OK_PROP_ERRORS (IMMEDIATE | LDST_CLASS_DECO | \ + LDST_SRCDST_WORD_DECOCTRL | \ + (LDOFF_CHG_SHARE_OK_PROP << LDST_OFFSET_SHIFT)) #define DISABLE_AUTO_INFO_FIFO (IMMEDIATE | LDST_CLASS_DECO | \ LDST_SRCDST_WORD_DECOCTRL | \ (LDOFF_DISABLE_AUTO_NFIFO << LDST_OFFSET_SHIFT)) @@ -203,3 +206,56 @@ static inline void append_##cmd##_imm_##type(u32 *desc, type immediate, \ append_cmd(desc, immediate); \ } APPEND_CMD_RAW_IMM(load, LOAD, u32); + +/* + * Append math command. Only the last part of destination and source need to + * be specified + */ +#define APPEND_MATH(op, desc, dest, src_0, src_1, len) \ +append_cmd(desc, CMD_MATH | MATH_FUN_##op | MATH_DEST_##dest | \ + MATH_SRC0_##src_0 | MATH_SRC1_##src_1 | (u32) (len & MATH_LEN_MASK)); + +#define append_math_add(desc, dest, src0, src1, len) \ + APPEND_MATH(ADD, desc, dest, src0, src1, len) +#define append_math_sub(desc, dest, src0, src1, len) \ + APPEND_MATH(SUB, desc, dest, src0, src1, len) +#define append_math_add_c(desc, dest, src0, src1, len) \ + APPEND_MATH(ADDC, desc, dest, src0, src1, len) +#define append_math_sub_b(desc, dest, src0, src1, len) \ + APPEND_MATH(SUBB, desc, dest, src0, src1, len) +#define append_math_and(desc, dest, src0, src1, len) \ + APPEND_MATH(AND, desc, dest, src0, src1, len) +#define append_math_or(desc, dest, src0, src1, len) \ + APPEND_MATH(OR, desc, dest, src0, src1, len) +#define append_math_xor(desc, dest, src0, src1, len) \ + APPEND_MATH(XOR, desc, dest, src0, src1, len) +#define append_math_lshift(desc, dest, src0, src1, len) \ + APPEND_MATH(LSHIFT, desc, dest, src0, src1, len) +#define append_math_rshift(desc, dest, src0, src1, len) \ + APPEND_MATH(RSHIFT, desc, dest, src0, src1, len) + +/* Exactly one source is IMM. Data is passed in as u32 value */ +#define APPEND_MATH_IMM_u32(op, desc, dest, src_0, src_1, data) \ +do { \ + APPEND_MATH(op, desc, dest, src_0, src_1, CAAM_CMD_SZ); \ + append_cmd(desc, data); \ +} while (0); + +#define append_math_add_imm_u32(desc, dest, src0, src1, data) \ + APPEND_MATH_IMM_u32(ADD, desc, dest, src0, src1, data) +#define append_math_sub_imm_u32(desc, dest, src0, src1, data) \ + APPEND_MATH_IMM_u32(SUB, desc, dest, src0, src1, data) +#define append_math_add_c_imm_u32(desc, dest, src0, src1, data) \ + APPEND_MATH_IMM_u32(ADDC, desc, dest, src0, src1, data) +#define append_math_sub_b_imm_u32(desc, dest, src0, src1, data) \ + APPEND_MATH_IMM_u32(SUBB, desc, dest, src0, src1, data) +#define append_math_and_imm_u32(desc, dest, src0, src1, data) \ + APPEND_MATH_IMM_u32(AND, desc, dest, src0, src1, data) +#define append_math_or_imm_u32(desc, dest, src0, src1, data) \ + APPEND_MATH_IMM_u32(OR, desc, dest, src0, src1, data) +#define append_math_xor_imm_u32(desc, dest, src0, src1, data) \ + APPEND_MATH_IMM_u32(XOR, desc, dest, src0, src1, data) +#define append_math_lshift_imm_u32(desc, dest, src0, src1, data) \ + APPEND_MATH_IMM_u32(LSHIFT, desc, dest, src0, src1, data) +#define append_math_rshift_imm_u32(desc, dest, src0, src1, data) \ + APPEND_MATH_IMM_u32(RSHIFT, desc, dest, src0, src1, data) -- cgit v1.2.3 From acdca31dba86c4f426460aa000d13930a00549b7 Mon Sep 17 00:00:00 2001 From: Yuan Kang Date: Fri, 15 Jul 2011 11:21:42 +0800 Subject: crypto: caam - ablkcipher support caam now supports encrypt and decrypt for aes, des and 3des Signed-off-by: Yuan Kang Signed-off-by: Kim Phillips Signed-off-by: Herbert Xu --- drivers/crypto/caam/caamalg.c | 510 ++++++++++++++++++++++++++++++++++++++++++ drivers/crypto/caam/compat.h | 1 + 2 files changed, 511 insertions(+) diff --git a/drivers/crypto/caam/caamalg.c b/drivers/crypto/caam/caamalg.c index ed7d59d168a..4159265b453 100644 --- a/drivers/crypto/caam/caamalg.c +++ b/drivers/crypto/caam/caamalg.c @@ -69,6 +69,12 @@ #define DESC_AEAD_DEC_LEN (DESC_AEAD_BASE + 21 * CAAM_CMD_SZ) #define DESC_AEAD_GIVENC_LEN (DESC_AEAD_ENC_LEN + 7 * CAAM_CMD_SZ) +#define DESC_ABLKCIPHER_BASE (3 * CAAM_CMD_SZ) +#define DESC_ABLKCIPHER_ENC_LEN (DESC_ABLKCIPHER_BASE + \ + 20 * CAAM_CMD_SZ) +#define DESC_ABLKCIPHER_DEC_LEN (DESC_ABLKCIPHER_BASE + \ + 15 * CAAM_CMD_SZ) + #define DESC_MAX_USED_BYTES (DESC_AEAD_GIVENC_LEN + \ CAAM_MAX_KEY_SIZE) #define DESC_MAX_USED_LEN (DESC_MAX_USED_BYTES / CAAM_CMD_SZ) @@ -131,6 +137,19 @@ static inline void aead_append_ld_iv(u32 *desc, int ivsize) append_move(desc, MOVE_SRC_CLASS1CTX | MOVE_DEST_CLASS2INFIFO | ivsize); } +/* + * For ablkcipher encrypt and decrypt, read from req->src and + * write to req->dst + */ +static inline void ablkcipher_append_src_dst(u32 *desc) +{ + append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ); \ + append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ); \ + append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | \ + KEY_VLF | FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1); \ + append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | KEY_VLF); \ +} + /* * If all data, including src (with assoc and iv) or dst (with iv only) are * contiguous @@ -625,6 +644,119 @@ badkey: return -EINVAL; } +static int ablkcipher_setkey(struct crypto_ablkcipher *ablkcipher, + const u8 *key, unsigned int keylen) +{ + struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher); + struct ablkcipher_tfm *tfm = &ablkcipher->base.crt_ablkcipher; + struct device *jrdev = ctx->jrdev; + int ret = 0; + u32 *key_jump_cmd, *jump_cmd; + u32 *desc; + +#ifdef DEBUG + print_hex_dump(KERN_ERR, "key in @"xstr(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); +#endif + + memcpy(ctx->key, key, keylen); + ctx->key_dma = dma_map_single(jrdev, ctx->key, keylen, + DMA_TO_DEVICE); + if (dma_mapping_error(jrdev, ctx->key_dma)) { + dev_err(jrdev, "unable to map key i/o memory\n"); + return -ENOMEM; + } + ctx->enckeylen = keylen; + + /* ablkcipher_encrypt shared descriptor */ + desc = ctx->sh_desc_enc; + init_sh_desc(desc, HDR_SHARE_WAIT); + /* Skip if already shared */ + key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | + JUMP_COND_SHRD); + + /* Load class1 key only */ + append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen, + ctx->enckeylen, CLASS_1 | + KEY_DEST_CLASS_REG); + + set_jump_tgt_here(desc, key_jump_cmd); + + /* Propagate errors from shared to job descriptor */ + append_cmd(desc, SET_OK_PROP_ERRORS | CMD_LOAD); + + /* Load iv */ + append_cmd(desc, CMD_SEQ_LOAD | LDST_SRCDST_BYTE_CONTEXT | + LDST_CLASS_1_CCB | tfm->ivsize); + + /* Load operation */ + append_operation(desc, ctx->class1_alg_type | + OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT); + + /* Perform operation */ + ablkcipher_append_src_dst(desc); + + ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc, + desc_bytes(desc), + DMA_TO_DEVICE); + if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) { + dev_err(jrdev, "unable to map shared descriptor\n"); + return -ENOMEM; + } +#ifdef DEBUG + print_hex_dump(KERN_ERR, "ablkcipher enc shdesc@"xstr(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, + desc_bytes(desc), 1); +#endif + /* ablkcipher_decrypt shared descriptor */ + desc = ctx->sh_desc_dec; + + init_sh_desc(desc, HDR_SHARE_WAIT); + /* Skip if already shared */ + key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | + JUMP_COND_SHRD); + + /* Load class1 key only */ + append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen, + ctx->enckeylen, CLASS_1 | + KEY_DEST_CLASS_REG); + + /* For aead, only propagate error immediately if shared */ + jump_cmd = append_jump(desc, JUMP_TEST_ALL); + set_jump_tgt_here(desc, key_jump_cmd); + append_cmd(desc, SET_OK_PROP_ERRORS | CMD_LOAD); + set_jump_tgt_here(desc, jump_cmd); + + /* load IV */ + append_cmd(desc, CMD_SEQ_LOAD | LDST_SRCDST_BYTE_CONTEXT | + LDST_CLASS_1_CCB | tfm->ivsize); + + /* Choose operation */ + append_dec_op1(desc, ctx->class1_alg_type); + + /* Perform operation */ + ablkcipher_append_src_dst(desc); + + /* Wait for key to load before allowing propagating error */ + append_dec_shr_done(desc); + + ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc, + desc_bytes(desc), + DMA_TO_DEVICE); + if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) { + dev_err(jrdev, "unable to map shared descriptor\n"); + return -ENOMEM; + } + +#ifdef DEBUG + print_hex_dump(KERN_ERR, "ablkcipher dec shdesc@"xstr(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, + desc_bytes(desc), 1); +#endif + + return ret; +} + struct link_tbl_entry { u64 ptr; u32 len; @@ -655,6 +787,26 @@ struct aead_edesc { u32 hw_desc[0]; }; +/* + * ablkcipher_edesc - s/w-extended ablkcipher descriptor + * @src_nents: number of segments in input scatterlist + * @dst_nents: number of segments in output scatterlist + * @iv_dma: dma address of iv for checking continuity and link table + * @desc: h/w descriptor (variable length; must not exceed MAX_CAAM_DESCSIZE) + * @link_tbl_bytes: length of dma mapped link_tbl space + * @link_tbl_dma: bus physical mapped address of h/w link table + * @hw_desc: the h/w job descriptor followed by any referenced link tables + */ +struct ablkcipher_edesc { + int src_nents; + int dst_nents; + dma_addr_t iv_dma; + int link_tbl_bytes; + dma_addr_t link_tbl_dma; + struct link_tbl_entry *link_tbl; + u32 hw_desc[0]; +}; + static void caam_unmap(struct device *dev, struct scatterlist *src, struct scatterlist *dst, int src_nents, int dst_nents, dma_addr_t iv_dma, int ivsize, dma_addr_t link_tbl_dma, @@ -689,6 +841,19 @@ static void aead_unmap(struct device *dev, edesc->link_tbl_bytes); } +static void ablkcipher_unmap(struct device *dev, + struct ablkcipher_edesc *edesc, + struct ablkcipher_request *req) +{ + struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); + int ivsize = crypto_ablkcipher_ivsize(ablkcipher); + + caam_unmap(dev, req->src, req->dst, + edesc->src_nents, edesc->dst_nents, + edesc->iv_dma, ivsize, edesc->link_tbl_dma, + edesc->link_tbl_bytes); +} + static void aead_encrypt_done(struct device *jrdev, u32 *desc, u32 err, void *context) { @@ -790,6 +955,77 @@ static void aead_decrypt_done(struct device *jrdev, u32 *desc, u32 err, aead_request_complete(req, err); } +static void ablkcipher_encrypt_done(struct device *jrdev, u32 *desc, u32 err, + void *context) +{ + struct ablkcipher_request *req = context; + struct ablkcipher_edesc *edesc; +#ifdef DEBUG + struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); + int ivsize = crypto_ablkcipher_ivsize(ablkcipher); + + dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); +#endif + + edesc = (struct ablkcipher_edesc *)((char *)desc - + offsetof(struct ablkcipher_edesc, hw_desc)); + + if (err) { + char tmp[CAAM_ERROR_STR_MAX]; + + dev_err(jrdev, "%08x: %s\n", err, caam_jr_strstatus(tmp, err)); + } + +#ifdef DEBUG + print_hex_dump(KERN_ERR, "dstiv @"xstr(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, req->info, + edesc->src_nents > 1 ? 100 : ivsize, 1); + print_hex_dump(KERN_ERR, "dst @"xstr(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src), + edesc->dst_nents > 1 ? 100 : req->nbytes, 1); +#endif + + ablkcipher_unmap(jrdev, edesc, req); + kfree(edesc); + + ablkcipher_request_complete(req, err); +} + +static void ablkcipher_decrypt_done(struct device *jrdev, u32 *desc, u32 err, + void *context) +{ + struct ablkcipher_request *req = context; + struct ablkcipher_edesc *edesc; +#ifdef DEBUG + struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); + int ivsize = crypto_ablkcipher_ivsize(ablkcipher); + + dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); +#endif + + edesc = (struct ablkcipher_edesc *)((char *)desc - + offsetof(struct ablkcipher_edesc, hw_desc)); + if (err) { + char tmp[CAAM_ERROR_STR_MAX]; + + dev_err(jrdev, "%08x: %s\n", err, caam_jr_strstatus(tmp, err)); + } + +#ifdef DEBUG + print_hex_dump(KERN_ERR, "dstiv @"xstr(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, req->info, + ivsize, 1); + print_hex_dump(KERN_ERR, "dst @"xstr(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src), + edesc->dst_nents > 1 ? 100 : req->nbytes, 1); +#endif + + ablkcipher_unmap(jrdev, edesc, req); + kfree(edesc); + + ablkcipher_request_complete(req, err); +} + static void sg_to_link_tbl_one(struct link_tbl_entry *link_tbl_ptr, dma_addr_t dma, u32 len, u32 offset) { @@ -977,6 +1213,63 @@ static void init_aead_giv_job(u32 *sh_desc, dma_addr_t ptr, append_seq_out_ptr(desc, dst_dma, ivsize + req->cryptlen, out_options); } +/* + * Fill in ablkcipher job descriptor + */ +static void init_ablkcipher_job(u32 *sh_desc, dma_addr_t ptr, + struct ablkcipher_edesc *edesc, + struct ablkcipher_request *req, + bool iv_contig) +{ + struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); + int ivsize = crypto_ablkcipher_ivsize(ablkcipher); + u32 *desc = edesc->hw_desc; + u32 out_options = 0, in_options; + dma_addr_t dst_dma, src_dma; + int len, link_tbl_index = 0; + +#ifdef DEBUG + print_hex_dump(KERN_ERR, "presciv@"xstr(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, req->info, + ivsize, 1); + print_hex_dump(KERN_ERR, "src @"xstr(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src), + edesc->src_nents ? 100 : req->nbytes, 1); +#endif + + len = desc_len(sh_desc); + init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE); + + if (iv_contig) { + src_dma = edesc->iv_dma; + in_options = 0; + } else { + src_dma = edesc->link_tbl_dma; + link_tbl_index += (iv_contig ? 0 : 1) + edesc->src_nents; + in_options = LDST_SGF; + } + append_seq_in_ptr(desc, src_dma, req->nbytes + ivsize, in_options); + + if (likely(req->src == req->dst)) { + if (!edesc->src_nents && iv_contig) { + dst_dma = sg_dma_address(req->src); + } else { + dst_dma = edesc->link_tbl_dma + + sizeof(struct link_tbl_entry); + out_options = LDST_SGF; + } + } else { + if (!edesc->dst_nents) { + dst_dma = sg_dma_address(req->dst); + } else { + dst_dma = edesc->link_tbl_dma + + link_tbl_index * sizeof(struct link_tbl_entry); + out_options = LDST_SGF; + } + } + append_seq_out_ptr(desc, dst_dma, req->nbytes, out_options); +} + /* * derive number of elements in scatterlist */ @@ -1327,7 +1620,171 @@ static int aead_givencrypt(struct aead_givcrypt_request *areq) return ret; } +/* + * allocate and map the ablkcipher extended descriptor for ablkcipher + */ +static struct ablkcipher_edesc *ablkcipher_edesc_alloc(struct ablkcipher_request + *req, int desc_bytes, + bool *iv_contig_out) +{ + struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); + struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher); + struct device *jrdev = ctx->jrdev; + gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG | + CRYPTO_TFM_REQ_MAY_SLEEP)) ? + GFP_KERNEL : GFP_ATOMIC; + int src_nents, dst_nents = 0, link_tbl_bytes; + struct ablkcipher_edesc *edesc; + dma_addr_t iv_dma = 0; + bool iv_contig = false; + int sgc; + int ivsize = crypto_ablkcipher_ivsize(ablkcipher); + int link_tbl_index; + + src_nents = sg_count(req->src, req->nbytes); + + if (unlikely(req->dst != req->src)) + dst_nents = sg_count(req->dst, req->nbytes); + + if (likely(req->src == req->dst)) { + sgc = dma_map_sg(jrdev, req->src, src_nents ? : 1, + DMA_BIDIRECTIONAL); + } else { + sgc = dma_map_sg(jrdev, req->src, src_nents ? : 1, + DMA_TO_DEVICE); + sgc = dma_map_sg(jrdev, req->dst, dst_nents ? : 1, + DMA_FROM_DEVICE); + } + + /* + * Check if iv can be contiguous with source and destination. + * If so, include it. If not, create scatterlist. + */ + iv_dma = dma_map_single(jrdev, req->info, ivsize, DMA_TO_DEVICE); + if (!src_nents && iv_dma + ivsize == sg_dma_address(req->src)) + iv_contig = true; + else + src_nents = src_nents ? : 1; + link_tbl_bytes = ((iv_contig ? 0 : 1) + src_nents + dst_nents) * + sizeof(struct link_tbl_entry); + + /* allocate space for base edesc and hw desc commands, link tables */ + edesc = kmalloc(sizeof(struct ablkcipher_edesc) + desc_bytes + + link_tbl_bytes, GFP_DMA | flags); + if (!edesc) { + dev_err(jrdev, "could not allocate extended descriptor\n"); + return ERR_PTR(-ENOMEM); + } + + edesc->src_nents = src_nents; + edesc->dst_nents = dst_nents; + edesc->link_tbl_bytes = link_tbl_bytes; + edesc->link_tbl = (void *)edesc + sizeof(struct ablkcipher_edesc) + + desc_bytes; + + link_tbl_index = 0; + if (!iv_contig) { + sg_to_link_tbl_one(edesc->link_tbl, iv_dma, ivsize, 0); + sg_to_link_tbl_last(req->src, src_nents, + edesc->link_tbl + 1, 0); + link_tbl_index += 1 + src_nents; + } + + if (unlikely(dst_nents)) { + sg_to_link_tbl_last(req->dst, dst_nents, + edesc->link_tbl + link_tbl_index, 0); + } + + edesc->link_tbl_dma = dma_map_single(jrdev, edesc->link_tbl, + link_tbl_bytes, DMA_TO_DEVICE); + edesc->iv_dma = iv_dma; + +#ifdef DEBUG + print_hex_dump(KERN_ERR, "ablkcipher link_tbl@"xstr(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, edesc->link_tbl, + link_tbl_bytes, 1); +#endif + + *iv_contig_out = iv_contig; + return edesc; +} + +static int ablkcipher_encrypt(struct ablkcipher_request *req) +{ + struct ablkcipher_edesc *edesc; + struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); + struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher); + struct device *jrdev = ctx->jrdev; + bool iv_contig; + u32 *desc; + int ret = 0; + + /* allocate extended descriptor */ + edesc = ablkcipher_edesc_alloc(req, DESC_JOB_IO_LEN * + CAAM_CMD_SZ, &iv_contig); + if (IS_ERR(edesc)) + return PTR_ERR(edesc); + + /* Create and submit job descriptor*/ + init_ablkcipher_job(ctx->sh_desc_enc, + ctx->sh_desc_enc_dma, edesc, req, iv_contig); +#ifdef DEBUG + print_hex_dump(KERN_ERR, "ablkcipher jobdesc@"xstr(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc, + desc_bytes(edesc->hw_desc), 1); +#endif + desc = edesc->hw_desc; + ret = caam_jr_enqueue(jrdev, desc, ablkcipher_encrypt_done, req); + + if (!ret) { + ret = -EINPROGRESS; + } else { + ablkcipher_unmap(jrdev, edesc, req); + kfree(edesc); + } + + return ret; +} + +static int ablkcipher_decrypt(struct ablkcipher_request *req) +{ + struct ablkcipher_edesc *edesc; + struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); + struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher); + struct device *jrdev = ctx->jrdev; + bool iv_contig; + u32 *desc; + int ret = 0; + + /* allocate extended descriptor */ + edesc = ablkcipher_edesc_alloc(req, DESC_JOB_IO_LEN * + CAAM_CMD_SZ, &iv_contig); + if (IS_ERR(edesc)) + return PTR_ERR(edesc); + + /* Create and submit job descriptor*/ + init_ablkcipher_job(ctx->sh_desc_dec, + ctx->sh_desc_dec_dma, edesc, req, iv_contig); + desc = edesc->hw_desc; +#ifdef DEBUG + print_hex_dump(KERN_ERR, "ablkcipher jobdesc@"xstr(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc, + desc_bytes(edesc->hw_desc), 1); +#endif + + ret = caam_jr_enqueue(jrdev, desc, ablkcipher_decrypt_done, req); + if (!ret) { + ret = -EINPROGRESS; + } else { + ablkcipher_unmap(jrdev, edesc, req); + kfree(edesc); + } + + return ret; +} + #define template_aead template_u.aead +#define template_ablkcipher template_u.ablkcipher struct caam_alg_template { char name[CRYPTO_MAX_ALG_NAME]; char driver_name[CRYPTO_MAX_ALG_NAME]; @@ -1525,6 +1982,55 @@ static struct caam_alg_template driver_algs[] = { OP_ALG_AAI_HMAC_PRECOMP, .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC, }, + /* ablkcipher descriptor */ + { + .name = "cbc(aes)", + .driver_name = "cbc-aes-caam", + .blocksize = AES_BLOCK_SIZE, + .type = CRYPTO_ALG_TYPE_ABLKCIPHER, + .template_ablkcipher = { + .setkey = ablkcipher_setkey, + .encrypt = ablkcipher_encrypt, + .decrypt = ablkcipher_decrypt, + .geniv = "eseqiv", + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + }, + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, + }, + { + .name = "cbc(des3_ede)", + .driver_name = "cbc-3des-caam", + .blocksize = DES3_EDE_BLOCK_SIZE, + .type = CRYPTO_ALG_TYPE_ABLKCIPHER, + .template_ablkcipher = { + .setkey = ablkcipher_setkey, + .encrypt = ablkcipher_encrypt, + .decrypt = ablkcipher_decrypt, + .geniv = "eseqiv", + .min_keysize = DES3_EDE_KEY_SIZE, + .max_keysize = DES3_EDE_KEY_SIZE, + .ivsize = DES3_EDE_BLOCK_SIZE, + }, + .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, + }, + { + .name = "cbc(des)", + .driver_name = "cbc-des-caam", + .blocksize = DES_BLOCK_SIZE, + .type = CRYPTO_ALG_TYPE_ABLKCIPHER, + .template_ablkcipher = { + .setkey = ablkcipher_setkey, + .encrypt = ablkcipher_encrypt, + .decrypt = ablkcipher_decrypt, + .geniv = "eseqiv", + .min_keysize = DES_KEY_SIZE, + .max_keysize = DES_KEY_SIZE, + .ivsize = DES_BLOCK_SIZE, + }, + .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, + } }; struct caam_crypto_alg { @@ -1644,6 +2150,10 @@ static struct caam_crypto_alg *caam_alg_alloc(struct device *ctrldev, alg->cra_ctxsize = sizeof(struct caam_ctx); alg->cra_flags = CRYPTO_ALG_ASYNC | template->type; switch (template->type) { + case CRYPTO_ALG_TYPE_ABLKCIPHER: + alg->cra_type = &crypto_ablkcipher_type; + alg->cra_ablkcipher = template->template_ablkcipher; + break; case CRYPTO_ALG_TYPE_AEAD: alg->cra_type = &crypto_aead_type; alg->cra_aead = template->template_aead; diff --git a/drivers/crypto/caam/compat.h b/drivers/crypto/caam/compat.h index 950450346f7..d38f2afaa96 100644 --- a/drivers/crypto/caam/compat.h +++ b/drivers/crypto/caam/compat.h @@ -31,5 +31,6 @@ #include #include #include +#include #endif /* !defined(CAAM_COMPAT_H) */ -- cgit v1.2.3