From e98c15a9d2ee9d1dd21e3d6c3d9829581d8328c0 Mon Sep 17 00:00:00 2001 From: Dmitry Eremin-Solenikov Date: Thu, 14 Dec 2017 04:22:12 +0300 Subject: linux-gen: crypto: significant speedup of all operations Per idea of Janne Peltonen, do not allocate/free crypto contexts for each operation, providing significant speed increase. Each thread on startup allocates hmac+cipher contexts pair for each crypto session. Then they are initialized on demand, when session is first executed using this thread. Signed-off-by: Dmitry Eremin-Solenikov Reviewed-by: Janne Peltonen Reviewed-by: Bill Fischofer Signed-off-by: Maxim Uvarov --- platform/linux-generic/odp_crypto.c | 319 ++++++++++++++++++++++++------------ 1 file changed, 217 insertions(+), 102 deletions(-) (limited to 'platform/linux-generic/odp_crypto.c') diff --git a/platform/linux-generic/odp_crypto.c b/platform/linux-generic/odp_crypto.c index 3b110c191..8ae484fd8 100644 --- a/platform/linux-generic/odp_crypto.c +++ b/platform/linux-generic/odp_crypto.c @@ -9,7 +9,6 @@ #include #include #include -#include #include #include #include @@ -98,6 +97,7 @@ typedef odp_crypto_alg_err_t (*crypto_func_t)(odp_packet_t pkt, const odp_crypto_packet_op_param_t *param, odp_crypto_generic_session_t *session); +typedef void (*crypto_init_func_t)(odp_crypto_generic_session_t *session); /** * Per crypto session data structure @@ -117,6 +117,7 @@ struct odp_crypto_generic_session_t { const EVP_CIPHER *evp_cipher; crypto_func_t func; + crypto_init_func_t init; } cipher; struct { @@ -128,7 +129,10 @@ struct odp_crypto_generic_session_t { const EVP_CIPHER *evp_cipher; }; crypto_func_t func; + crypto_init_func_t init; } auth; + + unsigned idx; }; typedef struct odp_crypto_global_s odp_crypto_global_t; @@ -137,15 +141,42 @@ struct odp_crypto_global_s { odp_spinlock_t lock; odp_crypto_generic_session_t *free; odp_crypto_generic_session_t sessions[MAX_SESSIONS]; + + /* These flags are cleared at alloc_session() */ + uint8_t ctx_valid[ODP_THREAD_COUNT_MAX][MAX_SESSIONS]; + odp_ticketlock_t openssl_lock[0]; }; static odp_crypto_global_t *global; +typedef struct crypto_local_t { + HMAC_CTX *hmac_ctx[MAX_SESSIONS]; + EVP_CIPHER_CTX *cipher_ctx[MAX_SESSIONS]; + EVP_CIPHER_CTX *mac_cipher_ctx[MAX_SESSIONS]; + uint8_t *ctx_valid; +} crypto_local_t; + +static __thread crypto_local_t local; + +typedef enum { KIND_HMAC, KIND_CIPHER, KIND_MAC_CIPHER } crypto_kind_t; + +static inline void crypto_init(odp_crypto_generic_session_t *session) +{ + if (local.ctx_valid[session->idx]) + return; + + session->cipher.init(session); + session->auth.init(session); + + local.ctx_valid[session->idx] = 1; +} + static odp_crypto_generic_session_t *alloc_session(void) { odp_crypto_generic_session_t *session = NULL; + unsigned i; odp_spinlock_lock(&global->lock); session = global->free; @@ -155,6 +186,11 @@ odp_crypto_generic_session_t *alloc_session(void) } odp_spinlock_unlock(&global->lock); + session->idx = session - global->sessions; + + for (i = 0; i < ODP_THREAD_COUNT_MAX; i++) + global->ctx_valid[i][session->idx] = 0; + return session; } @@ -175,24 +211,57 @@ null_crypto_routine(odp_packet_t pkt ODP_UNUSED, return ODP_CRYPTO_ALG_ERR_NONE; } -static -void packet_hmac_calculate(HMAC_CTX *ctx, - odp_packet_t pkt, - const odp_crypto_packet_op_param_t *param, - odp_crypto_generic_session_t *session, - uint8_t *hash) +static void +null_crypto_init_routine(odp_crypto_generic_session_t *session ODP_UNUSED) { - uint32_t offset = param->auth_range.offset; - uint32_t len = param->auth_range.length; + return; +} - ODP_ASSERT(offset + len <= odp_packet_len(pkt)); +/* Mimic new OpenSSL 1.1.y API */ +#if OPENSSL_VERSION_NUMBER < 0x10100000L +static HMAC_CTX *HMAC_CTX_new(void) +{ + HMAC_CTX *ctx = malloc(sizeof(*ctx)); + + HMAC_CTX_init(ctx); + return ctx; +} + +static void HMAC_CTX_free(HMAC_CTX *ctx) +{ + HMAC_CTX_cleanup(ctx); + free(ctx); +} +#endif + +static void +auth_init(odp_crypto_generic_session_t *session) +{ + HMAC_CTX *ctx = local.hmac_ctx[session->idx]; HMAC_Init_ex(ctx, session->auth.key, session->auth.key_length, session->auth.evp_md, NULL); +} + +static +void packet_hmac(odp_packet_t pkt, + const odp_crypto_packet_op_param_t *param, + odp_crypto_generic_session_t *session, + uint8_t *hash) +{ + HMAC_CTX *ctx = local.hmac_ctx[session->idx]; + uint32_t offset = param->auth_range.offset; + uint32_t len = param->auth_range.length; + ODP_ASSERT(offset + len <= odp_packet_len(pkt)); + + /* Reinitialize HMAC calculation without resetting the key */ + HMAC_Init_ex(ctx, NULL, 0, NULL, NULL); + + /* Hash it */ while (len > 0) { uint32_t seglen = 0; /* GCC */ void *mapaddr = odp_packet_offset(pkt, offset, &seglen, NULL); @@ -206,36 +275,6 @@ void packet_hmac_calculate(HMAC_CTX *ctx, HMAC_Final(ctx, hash, NULL); } -#if OPENSSL_VERSION_NUMBER < 0x10100000L -static -void packet_hmac(odp_packet_t pkt, - const odp_crypto_packet_op_param_t *param, - odp_crypto_generic_session_t *session, - uint8_t *hash) -{ - HMAC_CTX ctx; - - /* Hash it */ - HMAC_CTX_init(&ctx); - packet_hmac_calculate(&ctx, pkt, param, session, hash); - HMAC_CTX_cleanup(&ctx); -} -#else -static -void packet_hmac(odp_packet_t pkt, - const odp_crypto_packet_op_param_t *param, - odp_crypto_generic_session_t *session, - uint8_t *hash) -{ - HMAC_CTX *ctx; - - /* Hash it */ - ctx = HMAC_CTX_new(); - packet_hmac_calculate(ctx, pkt, param, session, hash); - HMAC_CTX_free(ctx); -} -#endif - static odp_crypto_alg_err_t auth_gen(odp_packet_t pkt, const odp_crypto_packet_op_param_t *param, @@ -425,12 +464,22 @@ int internal_decrypt(EVP_CIPHER_CTX *ctx, return ret; } +static void +cipher_encrypt_init(odp_crypto_generic_session_t *session) +{ + EVP_CIPHER_CTX *ctx = local.cipher_ctx[session->idx]; + + EVP_EncryptInit_ex(ctx, session->cipher.evp_cipher, NULL, + session->cipher.key_data, NULL); + EVP_CIPHER_CTX_set_padding(ctx, 0); +} + static odp_crypto_alg_err_t cipher_encrypt(odp_packet_t pkt, const odp_crypto_packet_op_param_t *param, odp_crypto_generic_session_t *session) { - EVP_CIPHER_CTX *ctx; + EVP_CIPHER_CTX *ctx = local.cipher_ctx[session->idx]; void *iv_ptr; int ret; @@ -441,27 +490,30 @@ odp_crypto_alg_err_t cipher_encrypt(odp_packet_t pkt, else return ODP_CRYPTO_ALG_ERR_IV_INVALID; - /* Encrypt it */ - ctx = EVP_CIPHER_CTX_new(); - EVP_EncryptInit_ex(ctx, session->cipher.evp_cipher, NULL, - session->cipher.key_data, NULL); EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv_ptr); - EVP_CIPHER_CTX_set_padding(ctx, 0); ret = internal_encrypt(ctx, pkt, param); - EVP_CIPHER_CTX_free(ctx); - return ret <= 0 ? ODP_CRYPTO_ALG_ERR_DATA_SIZE : ODP_CRYPTO_ALG_ERR_NONE; } +static void +cipher_decrypt_init(odp_crypto_generic_session_t *session) +{ + EVP_CIPHER_CTX *ctx = local.cipher_ctx[session->idx]; + + EVP_DecryptInit_ex(ctx, session->cipher.evp_cipher, NULL, + session->cipher.key_data, NULL); + EVP_CIPHER_CTX_set_padding(ctx, 0); +} + static odp_crypto_alg_err_t cipher_decrypt(odp_packet_t pkt, const odp_crypto_packet_op_param_t *param, odp_crypto_generic_session_t *session) { - EVP_CIPHER_CTX *ctx; + EVP_CIPHER_CTX *ctx = local.cipher_ctx[session->idx]; void *iv_ptr; int ret; @@ -472,17 +524,10 @@ odp_crypto_alg_err_t cipher_decrypt(odp_packet_t pkt, else return ODP_CRYPTO_ALG_ERR_IV_INVALID; - /* Decrypt it */ - ctx = EVP_CIPHER_CTX_new(); - EVP_DecryptInit_ex(ctx, session->cipher.evp_cipher, NULL, - session->cipher.key_data, NULL); EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv_ptr); - EVP_CIPHER_CTX_set_padding(ctx, 0); ret = internal_decrypt(ctx, pkt, param); - EVP_CIPHER_CTX_free(ctx); - return ret <= 0 ? ODP_CRYPTO_ALG_ERR_DATA_SIZE : ODP_CRYPTO_ALG_ERR_NONE; } @@ -506,20 +551,35 @@ static int process_cipher_param(odp_crypto_generic_session_t *session, session->p.cipher_key.length); /* Set function */ - if (ODP_CRYPTO_OP_ENCODE == session->p.op) + if (ODP_CRYPTO_OP_ENCODE == session->p.op) { session->cipher.func = cipher_encrypt; - else + session->cipher.init = cipher_encrypt_init; + } else { session->cipher.func = cipher_decrypt; + session->cipher.init = cipher_decrypt_init; + } return 0; } +static void +aes_gcm_encrypt_init(odp_crypto_generic_session_t *session) +{ + EVP_CIPHER_CTX *ctx = local.cipher_ctx[session->idx]; + + EVP_EncryptInit_ex(ctx, session->cipher.evp_cipher, NULL, + session->cipher.key_data, NULL); + EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_IVLEN, + session->p.iv.length, NULL); + EVP_CIPHER_CTX_set_padding(ctx, 0); +} + static odp_crypto_alg_err_t aes_gcm_encrypt(odp_packet_t pkt, const odp_crypto_packet_op_param_t *param, odp_crypto_generic_session_t *session) { - EVP_CIPHER_CTX *ctx; + EVP_CIPHER_CTX *ctx = local.cipher_ctx[session->idx]; const uint8_t *aad_head = param->aad_ptr; uint32_t aad_len = session->p.auth_aad_len; void *iv_ptr; @@ -534,14 +594,7 @@ odp_crypto_alg_err_t aes_gcm_encrypt(odp_packet_t pkt, else return ODP_CRYPTO_ALG_ERR_IV_INVALID; - /* Encrypt it */ - ctx = EVP_CIPHER_CTX_new(); - EVP_EncryptInit_ex(ctx, session->cipher.evp_cipher, NULL, - session->cipher.key_data, NULL); - EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_IVLEN, - session->p.iv.length, NULL); EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv_ptr); - EVP_CIPHER_CTX_set_padding(ctx, 0); /* Authenticate header data (if any) without encrypting them */ if (aad_len > 0) @@ -555,18 +608,28 @@ odp_crypto_alg_err_t aes_gcm_encrypt(odp_packet_t pkt, odp_packet_copy_from_mem(pkt, param->hash_result_offset, session->p.auth_digest_len, block); - EVP_CIPHER_CTX_free(ctx); - return ret <= 0 ? ODP_CRYPTO_ALG_ERR_DATA_SIZE : ODP_CRYPTO_ALG_ERR_NONE; } +static void +aes_gcm_decrypt_init(odp_crypto_generic_session_t *session) +{ + EVP_CIPHER_CTX *ctx = local.cipher_ctx[session->idx]; + + EVP_DecryptInit_ex(ctx, session->cipher.evp_cipher, NULL, + session->cipher.key_data, NULL); + EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_IVLEN, + session->p.iv.length, NULL); + EVP_CIPHER_CTX_set_padding(ctx, 0); +} + static odp_crypto_alg_err_t aes_gcm_decrypt(odp_packet_t pkt, const odp_crypto_packet_op_param_t *param, odp_crypto_generic_session_t *session) { - EVP_CIPHER_CTX *ctx; + EVP_CIPHER_CTX *ctx = local.cipher_ctx[session->idx]; const uint8_t *aad_head = param->aad_ptr; uint32_t aad_len = session->p.auth_aad_len; int dummy_len = 0; @@ -581,14 +644,7 @@ odp_crypto_alg_err_t aes_gcm_decrypt(odp_packet_t pkt, else return ODP_CRYPTO_ALG_ERR_IV_INVALID; - /* Decrypt it */ - ctx = EVP_CIPHER_CTX_new(); - EVP_DecryptInit_ex(ctx, session->cipher.evp_cipher, NULL, - session->cipher.key_data, NULL); - EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_IVLEN, - session->p.iv.length, NULL); EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv_ptr); - EVP_CIPHER_CTX_set_padding(ctx, 0); odp_packet_copy_to_mem(pkt, param->hash_result_offset, session->p.auth_digest_len, block); @@ -602,8 +658,6 @@ odp_crypto_alg_err_t aes_gcm_decrypt(odp_packet_t pkt, ret = internal_decrypt(ctx, pkt, param); - EVP_CIPHER_CTX_free(ctx); - return ret <= 0 ? ODP_CRYPTO_ALG_ERR_ICV_CHECK : ODP_CRYPTO_ALG_ERR_NONE; } @@ -622,20 +676,35 @@ static int process_aes_gcm_param(odp_crypto_generic_session_t *session, session->cipher.evp_cipher = cipher; /* Set function */ - if (ODP_CRYPTO_OP_ENCODE == session->p.op) + if (ODP_CRYPTO_OP_ENCODE == session->p.op) { session->cipher.func = aes_gcm_encrypt; - else + session->cipher.init = aes_gcm_encrypt_init; + } else { session->cipher.func = aes_gcm_decrypt; + session->cipher.init = aes_gcm_decrypt_init; + } return 0; } +static void +aes_gmac_gen_init(odp_crypto_generic_session_t *session) +{ + EVP_CIPHER_CTX *ctx = local.mac_cipher_ctx[session->idx]; + + EVP_EncryptInit_ex(ctx, session->auth.evp_cipher, NULL, + session->auth.key, NULL); + EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_IVLEN, + session->p.iv.length, NULL); + EVP_CIPHER_CTX_set_padding(ctx, 0); +} + static odp_crypto_alg_err_t aes_gmac_gen(odp_packet_t pkt, const odp_crypto_packet_op_param_t *param, odp_crypto_generic_session_t *session) { - EVP_CIPHER_CTX *ctx; + EVP_CIPHER_CTX *ctx = local.mac_cipher_ctx[session->idx]; void *iv_ptr; uint8_t block[EVP_MAX_MD_SIZE]; int ret; @@ -647,14 +716,7 @@ odp_crypto_alg_err_t aes_gmac_gen(odp_packet_t pkt, else return ODP_CRYPTO_ALG_ERR_IV_INVALID; - /* Encrypt it */ - ctx = EVP_CIPHER_CTX_new(); - EVP_EncryptInit_ex(ctx, session->auth.evp_cipher, NULL, - session->auth.key, NULL); - EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_IVLEN, - session->p.iv.length, NULL); EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv_ptr); - EVP_CIPHER_CTX_set_padding(ctx, 0); ret = internal_aad(ctx, pkt, param); @@ -663,18 +725,28 @@ odp_crypto_alg_err_t aes_gmac_gen(odp_packet_t pkt, odp_packet_copy_from_mem(pkt, param->hash_result_offset, session->p.auth_digest_len, block); - EVP_CIPHER_CTX_free(ctx); - return ret <= 0 ? ODP_CRYPTO_ALG_ERR_DATA_SIZE : ODP_CRYPTO_ALG_ERR_NONE; } +static void +aes_gmac_check_init(odp_crypto_generic_session_t *session) +{ + EVP_CIPHER_CTX *ctx = local.mac_cipher_ctx[session->idx]; + + EVP_DecryptInit_ex(ctx, session->auth.evp_cipher, NULL, + session->auth.key, NULL); + EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_IVLEN, + session->p.iv.length, NULL); + EVP_CIPHER_CTX_set_padding(ctx, 0); +} + static odp_crypto_alg_err_t aes_gmac_check(odp_packet_t pkt, const odp_crypto_packet_op_param_t *param, odp_crypto_generic_session_t *session) { - EVP_CIPHER_CTX *ctx; + EVP_CIPHER_CTX *ctx = local.mac_cipher_ctx[session->idx]; void *iv_ptr; uint8_t block[EVP_MAX_MD_SIZE]; int ret; @@ -686,14 +758,7 @@ odp_crypto_alg_err_t aes_gmac_check(odp_packet_t pkt, else return ODP_CRYPTO_ALG_ERR_IV_INVALID; - /* Decrypt it */ - ctx = EVP_CIPHER_CTX_new(); - EVP_DecryptInit_ex(ctx, session->auth.evp_cipher, NULL, - session->auth.key, NULL); - EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_IVLEN, - session->p.iv.length, NULL); EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv_ptr); - EVP_CIPHER_CTX_set_padding(ctx, 0); odp_packet_copy_to_mem(pkt, param->hash_result_offset, session->p.auth_digest_len, block); @@ -704,8 +769,6 @@ odp_crypto_alg_err_t aes_gmac_check(odp_packet_t pkt, ret = internal_aad(ctx, pkt, param); - EVP_CIPHER_CTX_free(ctx); - return ret <= 0 ? ODP_CRYPTO_ALG_ERR_ICV_CHECK : ODP_CRYPTO_ALG_ERR_NONE; } @@ -724,10 +787,13 @@ static int process_aes_gmac_param(odp_crypto_generic_session_t *session, session->auth.evp_cipher = cipher; /* Set function */ - if (ODP_CRYPTO_OP_ENCODE == session->p.op) + if (ODP_CRYPTO_OP_ENCODE == session->p.op) { session->auth.func = aes_gmac_gen; - else + session->auth.init = aes_gmac_gen_init; + } else { session->auth.func = aes_gmac_check; + session->auth.init = aes_gmac_check_init; + } return 0; } @@ -741,6 +807,7 @@ static int process_auth_param(odp_crypto_generic_session_t *session, session->auth.func = auth_gen; else session->auth.func = auth_check; + session->auth.init = auth_init; session->auth.evp_md = evp_md; @@ -923,6 +990,7 @@ odp_crypto_session_create(odp_crypto_session_param_t *param, switch (param->cipher_alg) { case ODP_CIPHER_ALG_NULL: session->cipher.func = null_crypto_routine; + session->cipher.init = null_crypto_init_routine; rc = 0; break; case ODP_CIPHER_ALG_3DES_CBC: @@ -998,6 +1066,7 @@ odp_crypto_session_create(odp_crypto_session_param_t *param, switch (param->auth_alg) { case ODP_AUTH_ALG_NULL: session->auth.func = null_crypto_routine; + session->auth.init = null_crypto_init_routine; rc = 0; break; #if ODP_DEPRECATED_API @@ -1037,6 +1106,7 @@ odp_crypto_session_create(odp_crypto_session_param_t *param, * cipher at the same time */ if (param->cipher_alg == ODP_CIPHER_ALG_AES_GCM || aes_gcm) { session->auth.func = null_crypto_routine; + session->auth.init = null_crypto_init_routine; rc = 0; } else { rc = -1; @@ -1216,6 +1286,49 @@ int odp_crypto_term_global(void) return rc; } +int _odp_crypto_init_local(void) +{ + unsigned i; + int id; + + memset(&local, 0, sizeof(local)); + + for (i = 0; i < MAX_SESSIONS; i++) { + local.hmac_ctx[i] = HMAC_CTX_new(); + local.cipher_ctx[i] = EVP_CIPHER_CTX_new(); + local.mac_cipher_ctx[i] = EVP_CIPHER_CTX_new(); + + if (local.hmac_ctx[i] == NULL || + local.cipher_ctx[i] == NULL || + local.mac_cipher_ctx[i] == NULL) { + _odp_crypto_term_local(); + return -1; + } + } + + id = odp_thread_id(); + local.ctx_valid = global->ctx_valid[id]; + /* No need to clear flags here, alloc_session did the job for us */ + + return 0; +} + +int _odp_crypto_term_local(void) +{ + unsigned i; + + for (i = 0; i < MAX_SESSIONS; i++) { + if (local.hmac_ctx[i] != NULL) + HMAC_CTX_free(local.hmac_ctx[i]); + if (local.cipher_ctx[i] != NULL) + EVP_CIPHER_CTX_free(local.cipher_ctx[i]); + if (local.mac_cipher_ctx[i] != NULL) + EVP_CIPHER_CTX_free(local.mac_cipher_ctx[i]); + } + + return 0; +} + odp_random_kind_t odp_random_max_kind(void) { return ODP_RANDOM_CRYPTO; @@ -1385,6 +1498,8 @@ int odp_crypto_int(odp_packet_t pkt_in, pkt_in = ODP_PACKET_INVALID; } + crypto_init(session); + /* Invoke the functions */ if (session->do_cipher_first) { rc_cipher = session->cipher.func(out_pkt, param, session); -- cgit v1.2.3