1 files changed, 152 insertions, 113 deletions

diff --git a/platform/linux-generic/arch/aarch64/odp_crypto_armv8.c b/platform/linux-generic/arch/aarch64/odp_crypto_armv8.c
index d719c6cfa..4d46846ce 100644
--- a/platform/linux-generic/arch/aarch64/odp_crypto_armv8.c
+++ b/platform/linux-generic/arch/aarch64/odp_crypto_armv8.c
@@ -1,5 +1,6 @@
 /* Copyright (c) 2014-2018, Linaro Limited
  * Copyright (c) 2021, ARM Limited
+ * Copyright (c) 2022, Nokia
  * All rights reserved.
  *
  * SPDX-License-Identifier:     BSD-3-Clause
@@ -37,9 +38,9 @@
 
 /*
  * ARM crypto library may read up to 15 bytes past the end of input
- * data and AAD.
+ * data and AAD and write up to 15 bytes past the end of output data.
  */
-#define OOB_READ_LEN 15
+#define OOB_WRITE_LEN 16 /* rounded up to 16 bytes for efficiency */
 
 /*
  * Data buffer size must be a multiple of 16, because the ARM crypto
@@ -80,9 +81,11 @@ static const odp_crypto_cipher_capability_t cipher_capa_aes_gcm[] = {
 static const odp_crypto_auth_capability_t auth_capa_null[] = {
 {.digest_len = 0, .key_len = 0, .aad_len = {.min = 0, .max = 0, .inc = 0} } };
 
+#define AES_GCM_TAG_LEN 16
+
 #ifdef __ARM_FEATURE_AES
 static const odp_crypto_auth_capability_t auth_capa_aes_gcm[] = {
-{.digest_len = 16, .key_len = 0, .aad_len = {.min = 8, .max = 12, .inc = 4} } };
+{.digest_len = AES_GCM_TAG_LEN, .key_len = 0, .aad_len = {.min = 8, .max = 12, .inc = 4} } };
 #endif
 
 /** Forward declaration of session structure */
@@ -92,9 +95,9 @@ typedef struct odp_crypto_generic_session_t odp_crypto_generic_session_t;
  * Algorithm handler function prototype
  */
 typedef
-odp_crypto_alg_err_t (*crypto_func_t)(odp_packet_t pkt,
-				      const odp_crypto_packet_op_param_t *param,
-				      odp_crypto_generic_session_t *session);
+odp_bool_t (*crypto_func_t)(odp_packet_t pkt,
+			    const odp_crypto_packet_op_param_t *param,
+			    odp_crypto_generic_session_t *session);
 
 /**
  * Per crypto session data structure
@@ -105,15 +108,12 @@ struct odp_crypto_generic_session_t {
 	/* Session creation parameters */
 	odp_crypto_session_param_t p;
 
-	odp_bool_t do_cipher_first;
-
 	struct {
 #if ODP_DEPRECATED_API
 		/* Copy of session IV data */
 		uint8_t iv_data[ARM_CRYPTO_MAX_IV_LENGTH];
 #endif
 		uint8_t key_data[ARM_CRYPTO_MAX_CIPHER_KEY_LENGTH];
-		crypto_func_t func;
 	} cipher;
 
 	struct {
@@ -121,11 +121,10 @@ struct odp_crypto_generic_session_t {
 #if ODP_DEPRECATED_API
 		uint8_t  iv_data[ARM_CRYPTO_MAX_IV_LENGTH];
 #endif
-		crypto_func_t func;
 	} auth;
 
+	crypto_func_t func;
 	unsigned int idx;
-
 	armv8_cipher_constants_t cc;
 };
 
@@ -141,7 +140,6 @@ static odp_crypto_global_t *global;
 
 typedef struct crypto_local_t {
 	uint8_t buffer[ARM_CRYPTO_MAX_DATA_LENGTH];
-	uint8_t digest[ARM_CRYPTO_MAX_DIGEST_LENGTH];
 } crypto_local_t;
 
 static __thread crypto_local_t local;
@@ -176,18 +174,60 @@ void free_session(odp_crypto_generic_session_t *session)
 	odp_spinlock_unlock(&global->lock);
 }
 
-static odp_crypto_alg_err_t
+static
+odp_crypto_packet_result_t *get_op_result_from_packet(odp_packet_t pkt)
+{
+	odp_packet_hdr_t *hdr = packet_hdr(pkt);
+
+	return &hdr->crypto_op_result;
+}
+
+static inline void set_crypto_op_result(odp_packet_t pkt,
+					odp_crypto_alg_err_t cipher_err,
+					odp_crypto_alg_err_t auth_err)
+{
+	odp_crypto_packet_result_t *op_result;
+
+	op_result = get_op_result_from_packet(pkt);
+	op_result->cipher_status.alg_err = cipher_err;
+	op_result->cipher_status.hw_err = ODP_CRYPTO_HW_ERR_NONE;
+	op_result->auth_status.alg_err = auth_err;
+	op_result->auth_status.hw_err = ODP_CRYPTO_HW_ERR_NONE;
+	op_result->ok = (cipher_err == ODP_CRYPTO_ALG_ERR_NONE &&
+			 auth_err   == ODP_CRYPTO_ALG_ERR_NONE);
+}
+
+static inline void set_crypto_op_result_ok(odp_packet_t pkt)
+{
+	set_crypto_op_result(pkt,
+			     ODP_CRYPTO_ALG_ERR_NONE,
+			     ODP_CRYPTO_ALG_ERR_NONE);
+}
+
+static odp_bool_t
 null_crypto_routine(odp_packet_t pkt ODP_UNUSED,
 		    const odp_crypto_packet_op_param_t *param ODP_UNUSED,
 		    odp_crypto_generic_session_t *session ODP_UNUSED)
 {
-	return ODP_CRYPTO_ALG_ERR_NONE;
+	set_crypto_op_result_ok(pkt);
+	return true;
+}
+
+static inline void copy_aad(uint8_t *dst, uint8_t *src, uint32_t len)
+{
+	ODP_ASSERT(len == 8 || len == 12);
+
+	/* Use constant length memcpy for better optimization result */
+	if (len == 8)
+		memcpy(dst, src, 8);
+	else
+		memcpy(dst, src, 12);
 }
 
 static
-odp_crypto_alg_err_t aes_gcm_encrypt(odp_packet_t pkt,
-				     const odp_crypto_packet_op_param_t *param,
-				     odp_crypto_generic_session_t *session)
+odp_bool_t aes_gcm_encrypt(odp_packet_t pkt,
+			   const odp_crypto_packet_op_param_t *param,
+			   odp_crypto_generic_session_t *session)
 {
 	armv8_cipher_state_t cs = {
 		.counter = {
@@ -200,12 +240,14 @@ odp_crypto_alg_err_t aes_gcm_encrypt(odp_packet_t pkt,
 	uint64_t aad_bit_length = session->p.auth_aad_len * 8;
 	uint32_t in_pos = param->cipher_range.offset;
 	uint32_t in_len = param->cipher_range.length;
-	int ret = 0;
+	odp_bool_t continuous_data;
+	uint16_t saved_tail[OOB_WRITE_LEN];
+	uint8_t tag[AES_GCM_TAG_LEN];
+	int rc;
 
 	/* Fail early if cipher_range is too large */
-	if (in_len > ARM_CRYPTO_MAX_DATA_LENGTH) {
+	if (odp_unlikely(in_len > ARM_CRYPTO_MAX_DATA_LENGTH)) {
 		ODP_DBG("ARM Crypto: Packet size too large for requested operation\n");
-		ret = -1;
 		goto err;
 	}
 
@@ -223,9 +265,9 @@ odp_crypto_alg_err_t aes_gcm_encrypt(odp_packet_t pkt,
 
 	cs.constants = &session->cc;
 
-	if (armv8_aes_gcm_set_counter(iv_ptr, iv_bit_length, &cs) != 0) {
+	rc = armv8_aes_gcm_set_counter(iv_ptr, iv_bit_length, &cs);
+	if (odp_unlikely(rc)) {
 		ODP_DBG("ARM Crypto: Failure while setting nonce\n");
-		ret = -1;
 		goto err;
 	}
 
@@ -233,46 +275,60 @@ odp_crypto_alg_err_t aes_gcm_encrypt(odp_packet_t pkt,
 	 * read it in 16 byte chunks. */
 	uint8_t aad[ARM_CRYPTO_MAX_AAD_LENGTH];
 
-	memcpy(aad, param->aad_ptr, session->p.auth_aad_len);
+	copy_aad(aad, param->aad_ptr, session->p.auth_aad_len);
 
 	uint32_t seg_len = 0;
 	uint8_t *data = odp_packet_offset(pkt, in_pos, &seg_len, NULL);
 
 	if (odp_unlikely(odp_packet_is_segmented(pkt)) ||
-	    odp_unlikely(odp_packet_tailroom(pkt) < OOB_READ_LEN)) {
-		/* Packet is segmented or it may not be safe to read beyond
-		 * the end of packet data. Copy the cipher range to a
+	    odp_unlikely(odp_packet_tailroom(pkt) < OOB_WRITE_LEN)) {
+		/* Packet is segmented or it may not be safe to read and write
+		 * beyond the end of packet data. Copy the cipher range to a
 		 * contiguous buffer. */
 		odp_packet_copy_to_mem(pkt, in_pos, in_len, local.buffer);
 
 		data = local.buffer;
+		continuous_data = false;
+	} else {
+		/* Save data that might get overwritten */
+		memcpy(saved_tail, data + in_len, OOB_WRITE_LEN);
+		continuous_data = true;
 	}
 
-	if (armv8_enc_aes_gcm_from_state(&cs,
-					 aad, aad_bit_length,
-					 data, plaintext_bit_length,
-					 local.buffer,
-					 local.digest) != 0) {
+	rc = armv8_enc_aes_gcm_from_state(&cs,
+					  aad, aad_bit_length,
+					  data, plaintext_bit_length,
+					  data,
+					  tag);
+	if (odp_unlikely(rc)) {
 		ODP_DBG("ARM Crypto: AES GCM Encoding failed\n");
-		ret = -1;
 		goto err;
 	}
 
-	odp_packet_copy_from_mem(pkt, param->cipher_range.offset,
-				 param->cipher_range.length, local.buffer);
+	if (odp_likely(continuous_data)) {
+		memcpy(data + in_len, saved_tail, OOB_WRITE_LEN);
+		memcpy(data - in_pos + param->hash_result_offset,
+		       tag, AES_GCM_TAG_LEN);
+	} else {
+		odp_packet_copy_from_mem(pkt, in_pos, in_len, data);
+		odp_packet_copy_from_mem(pkt, param->hash_result_offset,
+					 AES_GCM_TAG_LEN, tag);
+	}
 
-	odp_packet_copy_from_mem(pkt, param->hash_result_offset,
-				 session->p.auth_digest_len, local.digest);
+	set_crypto_op_result_ok(pkt);
+	return true;
 
 err:
-	return ret < 0 ? ODP_CRYPTO_ALG_ERR_DATA_SIZE :
-			 ODP_CRYPTO_ALG_ERR_NONE;
+	set_crypto_op_result(pkt,
+			     ODP_CRYPTO_ALG_ERR_DATA_SIZE,
+			     ODP_CRYPTO_ALG_ERR_NONE);
+	return false;
 }
 
 static
-odp_crypto_alg_err_t aes_gcm_decrypt(odp_packet_t pkt,
-				     const odp_crypto_packet_op_param_t *param,
-				     odp_crypto_generic_session_t *session)
+odp_bool_t aes_gcm_decrypt(odp_packet_t pkt,
+			   const odp_crypto_packet_op_param_t *param,
+			   odp_crypto_generic_session_t *session)
 {
 	armv8_cipher_state_t cs = {
 		.counter = {
@@ -280,18 +336,19 @@ odp_crypto_alg_err_t aes_gcm_decrypt(odp_packet_t pkt,
 		}
 	};
 	uint8_t *iv_ptr;
-	uint8_t tag[16];
+	uint8_t tag[AES_GCM_TAG_LEN];
 	uint64_t iv_bit_length = session->p.cipher_iv_len * 8;
 	uint64_t plaintext_bit_length = param->cipher_range.length * 8;
 	uint64_t aad_bit_length = session->p.auth_aad_len * 8;
 	uint32_t in_pos = param->cipher_range.offset;
 	uint32_t in_len = param->cipher_range.length;
-	int ret = 0;
+	odp_bool_t continuous_data;
+	uint16_t saved_tail[OOB_WRITE_LEN];
+	int rc;
 
 	/* Fail early if cipher_range is too large */
-	if (in_len > ARM_CRYPTO_MAX_DATA_LENGTH) {
+	if (odp_unlikely(in_len > ARM_CRYPTO_MAX_DATA_LENGTH)) {
 		ODP_DBG("ARM Crypto: Packet size too large for requested operation\n");
-		ret = -1;
 		goto err;
 	}
 
@@ -309,51 +366,63 @@ odp_crypto_alg_err_t aes_gcm_decrypt(odp_packet_t pkt,
 
 	cs.constants = &session->cc;
 
-	if (armv8_aes_gcm_set_counter(iv_ptr, iv_bit_length, &cs) != 0) {
+	rc = armv8_aes_gcm_set_counter(iv_ptr, iv_bit_length, &cs);
+	if (odp_unlikely(rc)) {
 		ODP_DBG("ARM Crypto: Failure while setting nonce\n");
-		ret = -1;
 		goto err;
 	}
 
-	/* Copy current hash bytes to a buffer and clear it */
-	odp_packet_copy_to_mem(pkt, param->hash_result_offset,
-			       session->p.auth_digest_len, tag);
-
 	/* Copy AAD in a stack to make sure that the ARM crypto library can
 	 * read it in 16 byte chunks. */
 	uint8_t aad[ARM_CRYPTO_MAX_AAD_LENGTH];
 
-	memcpy(aad, param->aad_ptr, session->p.auth_aad_len);
+	copy_aad(aad, param->aad_ptr, session->p.auth_aad_len);
 
 	uint32_t seg_len = 0;
 	uint8_t *data = odp_packet_offset(pkt, in_pos, &seg_len, NULL);
 
 	if (odp_unlikely(odp_packet_is_segmented(pkt)) ||
-	    odp_unlikely(odp_packet_tailroom(pkt) < OOB_READ_LEN)) {
-		/* Packet is segmented or it may not be safe to read beyond
-		 * the end of packet data. Copy the cipher range to a
+	    odp_unlikely(odp_packet_tailroom(pkt) < OOB_WRITE_LEN)) {
+		/* Packet is segmented or it may not be safe to read and write
+		 * beyond the end of packet data. Copy the cipher range to a
 		 * contiguous buffer. */
 		odp_packet_copy_to_mem(pkt, in_pos, in_len, local.buffer);
-
 		data = local.buffer;
+		/* Copy tag from the packet to a buffer */
+		odp_packet_copy_to_mem(pkt, param->hash_result_offset,
+				       AES_GCM_TAG_LEN, tag);
+		continuous_data = false;
+	} else {
+		/* Save data that might get overwritten */
+		memcpy(saved_tail, data + in_len, OOB_WRITE_LEN);
+		/* Copy tag from the packet to a buffer */
+		memcpy(tag, data - in_pos + param->hash_result_offset, AES_GCM_TAG_LEN);
+		continuous_data = true;
 	}
 
-	if (armv8_dec_aes_gcm_from_state(&cs,
-					 aad, aad_bit_length,
-					 data, plaintext_bit_length,
-					 tag,
-					 local.buffer) != 0) {
+	rc = armv8_dec_aes_gcm_from_state(&cs,
+					  aad, aad_bit_length,
+					  data, plaintext_bit_length,
+					  tag,
+					  data);
+	if (odp_unlikely(rc)) {
 		ODP_DBG("ARM Crypto: AES GCM Decoding failed\n");
-		ret = -1;
 		goto err;
 	}
 
-	odp_packet_copy_from_mem(pkt, param->cipher_range.offset,
-				 param->cipher_range.length, local.buffer);
+	if (odp_likely(continuous_data))
+		memcpy(data + in_len, saved_tail, OOB_WRITE_LEN);
+	else
+		odp_packet_copy_from_mem(pkt, in_pos, in_len, data);
+
+	set_crypto_op_result_ok(pkt);
+	return true;
 
 err:
-	return ret < 0 ? ODP_CRYPTO_ALG_ERR_ICV_CHECK :
-			 ODP_CRYPTO_ALG_ERR_NONE;
+	set_crypto_op_result(pkt,
+			     ODP_CRYPTO_ALG_ERR_NONE,
+			     ODP_CRYPTO_ALG_ERR_ICV_CHECK);
+	return false;
 }
 
 static int process_aes_gcm_param(odp_crypto_generic_session_t *session)
@@ -376,9 +445,9 @@ static int process_aes_gcm_param(odp_crypto_generic_session_t *session)
 
 	/* Set function */
 	if (ODP_CRYPTO_OP_ENCODE == session->p.op)
-		session->cipher.func = aes_gcm_encrypt;
+		session->func = aes_gcm_encrypt;
 	else
-		session->cipher.func = aes_gcm_decrypt;
+		session->func = aes_gcm_decrypt;
 
 	return 0;
 }
@@ -520,16 +589,10 @@ odp_crypto_session_create(const odp_crypto_session_param_t *param,
 		       session->p.auth_iv.length);
 #endif
 
-	/* Derive order */
-	if (ODP_CRYPTO_OP_ENCODE == param->op)
-		session->do_cipher_first = param->auth_cipher_text;
-	else
-		session->do_cipher_first = !param->auth_cipher_text;
-
 	/* Process based on cipher */
 	switch (param->cipher_alg) {
 	case ODP_CIPHER_ALG_NULL:
-		session->cipher.func = null_crypto_routine;
+		session->func = null_crypto_routine;
 		rc = 0;
 		break;
 	case ODP_CIPHER_ALG_AES_GCM:
@@ -584,14 +647,15 @@ odp_crypto_session_create(const odp_crypto_session_param_t *param,
 	/* Process based on auth */
 	switch (param->auth_alg) {
 	case ODP_AUTH_ALG_NULL:
-		session->auth.func = null_crypto_routine;
-		rc = 0;
+		if (param->cipher_alg == ODP_CIPHER_ALG_NULL)
+			rc = 0;
+		else
+			rc = -1;
 		break;
 	case ODP_AUTH_ALG_AES_GCM:
 		/* AES-GCM requires to do both auth and
 		 * cipher at the same time */
 		if (param->cipher_alg == ODP_CIPHER_ALG_AES_GCM) {
-			session->auth.func = null_crypto_routine;
 			rc = 0;
 		} else {
 			rc = -1;
@@ -630,9 +694,7 @@ int odp_crypto_session_destroy(odp_crypto_session_t session)
 	return 0;
 }
 
-/*
- * Shim function around packet operation, can be used by other implementations.
- */
+#if ODP_DEPRECATED_API
 int
 odp_crypto_operation(odp_crypto_op_param_t *param,
 		     odp_bool_t *posted,
@@ -687,6 +749,7 @@ odp_crypto_operation(odp_crypto_op_param_t *param,
 
 	return 0;
 }
+#endif
 
 int _odp_crypto_init_global(void)
 {
@@ -767,6 +830,7 @@ int _odp_crypto_term_local(void)
 	return 0;
 }
 
+#if ODP_DEPRECATED_API
 odp_crypto_compl_t odp_crypto_compl_from_event(odp_event_t ev)
 {
 	/* This check not mandated by the API specification */
@@ -803,6 +867,7 @@ uint64_t odp_crypto_compl_to_u64(odp_crypto_compl_t hdl)
 {
 	return _odp_pri(hdl);
 }
+#endif /* ODP_DEPRECATED_API */
 
 void odp_crypto_session_param_init(odp_crypto_session_param_t *param)
 {
@@ -828,14 +893,6 @@ odp_event_t odp_crypto_packet_to_event(odp_packet_t pkt)
 	return odp_packet_to_event(pkt);
 }
 
-static
-odp_crypto_packet_result_t *get_op_result_from_packet(odp_packet_t pkt)
-{
-	odp_packet_hdr_t *hdr = packet_hdr(pkt);
-
-	return &hdr->crypto_op_result;
-}
-
 int odp_crypto_result(odp_crypto_packet_result_t *result,
 		      odp_packet_t packet)
 {
@@ -856,18 +913,16 @@ int crypto_int(odp_packet_t pkt_in,
 	       odp_packet_t *pkt_out,
 	       const odp_crypto_packet_op_param_t *param)
 {
-	odp_crypto_alg_err_t rc_cipher = ODP_CRYPTO_ALG_ERR_NONE;
-	odp_crypto_alg_err_t rc_auth = ODP_CRYPTO_ALG_ERR_NONE;
 	odp_crypto_generic_session_t *session;
 	odp_bool_t allocated = false;
 	odp_packet_t out_pkt = *pkt_out;
-	odp_crypto_packet_result_t *op_result;
 	odp_packet_hdr_t *pkt_hdr;
+	odp_bool_t ok;
 
 	session = (odp_crypto_generic_session_t *)(intptr_t)param->session;
 
 	/* Resolve output buffer */
-	if (ODP_PACKET_INVALID == out_pkt &&
+	if (odp_unlikely(ODP_PACKET_INVALID == out_pkt) &&
 	    ODP_POOL_INVALID != session->p.output_pool) {
 		out_pkt = odp_packet_alloc(session->p.output_pool,
 					   odp_packet_len(pkt_in));
@@ -879,7 +934,7 @@ int crypto_int(odp_packet_t pkt_in,
 		return -1;
 	}
 
-	if (pkt_in != out_pkt) {
+	if (odp_unlikely(pkt_in != out_pkt)) {
 		int ret;
 		int md_copy;
 
@@ -903,28 +958,12 @@ int crypto_int(odp_packet_t pkt_in,
 		pkt_in = ODP_PACKET_INVALID;
 	}
 
-	/* Invoke the functions */
-	if (session->do_cipher_first) {
-		rc_cipher = session->cipher.func(out_pkt, param, session);
-		rc_auth = session->auth.func(out_pkt, param, session);
-	} else {
-		rc_auth = session->auth.func(out_pkt, param, session);
-		rc_cipher = session->cipher.func(out_pkt, param, session);
-	}
+	/* Invoke the crypto function */
+	ok = session->func(out_pkt, param, session);
 
-	/* Fill in result */
 	packet_subtype_set(out_pkt, ODP_EVENT_PACKET_CRYPTO);
-	op_result = get_op_result_from_packet(out_pkt);
-	op_result->cipher_status.alg_err = rc_cipher;
-	op_result->cipher_status.hw_err = ODP_CRYPTO_HW_ERR_NONE;
-	op_result->auth_status.alg_err = rc_auth;
-	op_result->auth_status.hw_err = ODP_CRYPTO_HW_ERR_NONE;
-	op_result->ok =
-		(rc_cipher == ODP_CRYPTO_ALG_ERR_NONE) &&
-		(rc_auth == ODP_CRYPTO_ALG_ERR_NONE);
-
 	pkt_hdr = packet_hdr(out_pkt);
-	pkt_hdr->p.flags.crypto_err = !op_result->ok;
+	pkt_hdr->p.flags.crypto_err = !ok;
 
 	/* Synchronous, simply return results */
 	*pkt_out = out_pkt;