diff options
author | Malvika Gupta <Malvika.Gupta@arm.com> | 2022-10-12 00:15:37 +0000 |
---|---|---|
committer | Matias Elo <matias.elo@nokia.com> | 2022-12-13 10:58:42 +0200 |
commit | 2dde0be2817a0833c447f46083b4fd278863bcdb (patch) | |
tree | 5af629fd70fb910f5a3996a7a6397382fc093680 | |
parent | 7c69ea604ecaaa2e953ef9bad6cc6365b6c5ce64 (diff) |
linux-gen: crypto: add support for Arm-optimzed IPSec multi-buffer library
- Enable detection of the Arm-optimized IPSec Multi-buffer library.
- Integrate ZUC128-EEA3 and ZUC128-EIA3 algorithms.
- Integrate ZUC256-EEA3 and ZUC256-EIA3 algorithms.
Signed-off-by: Malvika Gupta <Malvika.Gupta@arm.com>
Signed-off-by: Tianyu Li <tianyu.li@arm.com>
Reviewed-by: Ruifeng Wang <Ruifeng.Wang@arm.com>
Reviewed-by: Matias Elo <matias.elo@nokia.com>
Reviewed-by: Janne Peltonen <janne.peltonen@nokia.com>
-rw-r--r-- | DEPENDENCIES | 60 | ||||
-rw-r--r-- | include/odp/autoheader_internal.h.in | 3 | ||||
-rw-r--r-- | platform/linux-generic/Makefile.am | 6 | ||||
-rw-r--r-- | platform/linux-generic/libodp-linux.pc.in | 2 | ||||
-rw-r--r-- | platform/linux-generic/m4/configure.m4 | 3 | ||||
-rw-r--r-- | platform/linux-generic/m4/odp_crypto.m4 | 12 | ||||
-rw-r--r-- | platform/linux-generic/m4/odp_ipsec_mb.m4 | 19 | ||||
-rw-r--r-- | platform/linux-generic/odp_crypto_ipsecmb.c | 964 |
8 files changed, 1049 insertions, 20 deletions
diff --git a/DEPENDENCIES b/DEPENDENCIES index 0a1bea32b..34a6a0aa6 100644 --- a/DEPENDENCIES +++ b/DEPENDENCIES @@ -124,11 +124,39 @@ Prerequisites for building the OpenDataPlane (ODP) API $ ./configure --with-crypto=armv8crypto $ make -3.4 Netmap packet I/O support (optional) +3.4 Multi-buffer Crypto for IPsec Library (optional) + + Multi-Buffer Crypto for IPsec Library is a set of functions that + implement authentication and encryption processing for IPsec, these functions + take advantage of SIMD instructions to improve performance. + + Note ODP assumes that IPSec MB library is compiled with SAFE_PARAM enabled + (enabled by default), otherwise crypto operations with too long cipher/auth + ranges will have undefined behaviour. + +3.4.1 Building Multi-buffer Crypto for IPSec Library for Arm + + # Checkout and build Arm code + $ git clone https://git.gitlab.arm.com/arm-reference-solutions/ipsec-mb.git + $ cd ipsec-mb/ + $ git checkout SECLIB-IPSEC-2022.05.25 + $ make + $ sudo make install + + For additional instructions, refer to README.md in crypto library repository. + +3.4.2 Building ODP with Multi-buffer IPSec Library + $ ./bootstrap + + # Compile and build ODP with Multi-buffer IPSec library + $ ./configure --with-crypto=ipsecmb + $ make + +3.5 Netmap packet I/O support (optional) Netmap accelerated ODP packet I/O. -3.4.1 Building netmap kernel modules +3.5.1 Building netmap kernel modules ODP works at least with the latest release version of netmap, which is currently v13.0. However, if possible one should try to use the latest netmap @@ -172,14 +200,14 @@ Prerequisites for building the OpenDataPlane (ODP) API $ ./configure --kernel-sources=<path_to_kernel_src> $ make -3.4.2 Building ODP +3.5.2 Building ODP $ cd <odp_dir> $ ./bootstrap $ ./configure --with-netmap-path=<netmap_dir> $ make -3.4.3 Inserting netmap kernel modules +3.5.3 Inserting netmap kernel modules In order to use netmap I/O you need to insert at least the core netmap kernel module. @@ -197,13 +225,13 @@ Prerequisites for building the OpenDataPlane (ODP) API To restore the original drivers you should be able to use modprobe. -3.4.4 Running ODP with netmap I/O +3.5.4 Running ODP with netmap I/O ODP applications will use netmap for packet I/O by default as long as the netmap kernel module is loaded. If socket I/O is desired instead, it can be activated by setting the environment variable ODP_PKTIO_DISABLE_NETMAP. -3.5 DPDK packet I/O support (optional) +3.6 DPDK packet I/O support (optional) Use DPDK for ODP packet I/O. Currently supported DPDK versions are v19.11, v20.11 (recommended), v21.11. @@ -212,7 +240,7 @@ Prerequisites for building the OpenDataPlane (ODP) API https://github.com/OpenDataPlane/odp-dpdk.git for a full DPDK based ODP implementation. -3.5.1 DPDK pktio requirements +3.6.1 DPDK pktio requirements DPDK pktio adds a dependency to NUMA library. # Debian/Ubuntu @@ -221,11 +249,11 @@ Prerequisites for building the OpenDataPlane (ODP) API # CentOS/RedHat/Fedora $ sudo yum install numactl-devel -3.5.2 Native DPDK install +3.6.2 Native DPDK install # Debian/Ubuntu starting from 20.04 $ sudo apt-get install dpdk-dev -3.5.3 Build DPDK v19.11 from source +3.6.3 Build DPDK v19.11 from source $ git clone https://dpdk.org/git/dpdk-stable --branch 19.11 --depth 1 ./<dpdk-dir> # Make and edit DPDK configuration @@ -243,7 +271,7 @@ Prerequisites for building the OpenDataPlane (ODP) API # Configure ODP $ ./configure --with-dpdk-path=<dpdk-dir> -3.5.4 Build DPDK v20.11 and onwards from source +3.6.4 Build DPDK v20.11 and onwards from source $ git clone https://dpdk.org/git/dpdk-stable --branch <version, e.g. 21.11> --depth 1 ./<dpdk-dir> # Prepare the build directory @@ -263,7 +291,7 @@ Prerequisites for building the OpenDataPlane (ODP) API # Or, if DPDK was not installed to the default location, set PKG_CONFIG_PATH: $ PKG_CONFIG_PATH=<dpdk-dir>/install/lib/x86_64-linux-gnu/pkgconfig ./configure --enable-dpdk -3.5.5 Setup system +3.6.5 Setup system # Load DPDK modules $ sudo modprobe uio @@ -275,7 +303,7 @@ Prerequisites for building the OpenDataPlane (ODP) API 512 x 2MB huge pages. All this can be done with the DPDK setup script (<dpdk-dir>/usertools/dpdk-setup.sh). -3.5.6 Running ODP with DPDK pktio +3.6.6 Running ODP with DPDK pktio ODP applications will try use DPDK for packet I/O by default. If some other I/O type is desired instead, DPDK I/O can be disabled by setting the @@ -291,7 +319,7 @@ Prerequisites for building the OpenDataPlane (ODP) API 1024MB of memory: $ sudo ODP_PKTIO_DPDK_PARAMS="-m 1024" ./test/performance/odp_l2fwd -i 0 -c 1 -3.6 AF_XDP socket based packet I/O support (optional) +3.7 AF_XDP socket based packet I/O support (optional) Use AF_XDP socket for packet I/O. At the moment, only zero-copy variant is supported, requiring a kernel version 5.4 or higher. Additionally, if packet @@ -308,7 +336,7 @@ Prerequisites for building the OpenDataPlane (ODP) API Note that, currently, AF_XDP socket packet I/O cannot be instantiated if DPDK zero-copy is enabled. -3.6.1 AF_XDP socket packet I/O requirements +3.7.1 AF_XDP socket packet I/O requirements AF_XDP socket packet I/O implementation requires libxdp and libbpf libraries. They can be fetched from XDP-project in GitHub: @@ -330,14 +358,14 @@ Prerequisites for building the OpenDataPlane (ODP) API $ cd <path to built libbpf> $ make install -3.6.2 Build ODP with AF_XDP socket packet I/O support +3.7.2 Build ODP with AF_XDP socket packet I/O support After building and installing libxdp and libbpf, ODP can be configured to be built with AF_XDP support (pass PKG_CONFIG_PATH if needed). $ ./configure --enable-xdp -3.6.3 Running ODP with AF_XDP socket packet I/O +3.7.3 Running ODP with AF_XDP socket packet I/O AF_XDP socket packet I/Os bind to TRX-combined queues. Based on the packet prosessing needs, NIC(s) of the environment should be configured diff --git a/include/odp/autoheader_internal.h.in b/include/odp/autoheader_internal.h.in index 33d9f280f..a42b34ad0 100644 --- a/include/odp/autoheader_internal.h.in +++ b/include/odp/autoheader_internal.h.in @@ -35,4 +35,7 @@ /* Define to 1 to enable XDP support */ #undef _ODP_PKTIO_XDP +/* Define to 1 to enable IPSec MB crypto support */ +#undef _ODP_IPSECMB + #endif diff --git a/platform/linux-generic/Makefile.am b/platform/linux-generic/Makefile.am index e762148aa..3310e9603 100644 --- a/platform/linux-generic/Makefile.am +++ b/platform/linux-generic/Makefile.am @@ -278,10 +278,15 @@ if WITH_ARMV8_CRYPTO __LIB__libodp_linux_la_SOURCES += \ arch/aarch64/odp_crypto_armv8.c else +if WITH_IPSECMB_CRYPTO +__LIB__libodp_linux_la_SOURCES += \ + odp_crypto_ipsecmb.c +else __LIB__libodp_linux_la_SOURCES += \ odp_crypto_null.c endif endif +endif if ODP_ABI_COMPAT __LIB__libodp_linux_la_SOURCES += \ odp_atomic_api.c \ @@ -438,6 +443,7 @@ __LIB__libodp_linux_la_LIBADD += $(DPDK_LIBS_LIBODP) __LIB__libodp_linux_la_LIBADD += $(PTHREAD_LIBS) __LIB__libodp_linux_la_LIBADD += $(TIMER_LIBS) __LIB__libodp_linux_la_LIBADD += $(LIBXDP_LIBS) +__LIB__libodp_linux_la_LIBADD += $(IPSEC_MB_LIBS) if ODP_PKTIO_PCAP __LIB__libodp_linux_la_LIBADD += $(PCAP_LIBS) diff --git a/platform/linux-generic/libodp-linux.pc.in b/platform/linux-generic/libodp-linux.pc.in index f9a339fb8..05ba5b9d6 100644 --- a/platform/linux-generic/libodp-linux.pc.in +++ b/platform/linux-generic/libodp-linux.pc.in @@ -8,5 +8,5 @@ Description: The ODP packet processing engine Version: @PKGCONFIG_VERSION@ Requires.private: libconfig@AARCH64CRYPTO_PKG@ Libs: -L${libdir} -l@ODP_LIB_NAME@ @ATOMIC_LIBS_NON_ABI_COMPAT@ -Libs.private: @OPENSSL_STATIC_LIBS@ @DPDK_LIBS@ @PCAP_LIBS@ @PTHREAD_LIBS@ @TIMER_LIBS@ @LIBXDP_LIBS@ -lpthread @ATOMIC_LIBS_ABI_COMPAT@ +Libs.private: @OPENSSL_STATIC_LIBS@ @DPDK_LIBS@ @PCAP_LIBS@ @PTHREAD_LIBS@ @TIMER_LIBS@ @LIBXDP_LIBS@ -lpthread @ATOMIC_LIBS_ABI_COMPAT@ @IPSEC_MB_LIBS@ Cflags: -I${includedir} diff --git a/platform/linux-generic/m4/configure.m4 b/platform/linux-generic/m4/configure.m4 index 4f3365ea6..61d57634f 100644 --- a/platform/linux-generic/m4/configure.m4 +++ b/platform/linux-generic/m4/configure.m4 @@ -25,13 +25,14 @@ AM_CONDITIONAL([ODP_PKTIO_PCAP], [test x$have_pcap = xyes]) m4_include([platform/linux-generic/m4/odp_libconfig.m4]) m4_include([platform/linux-generic/m4/odp_openssl.m4]) m4_include([platform/linux-generic/m4/odp_crypto.m4]) +m4_include([platform/linux-generic/m4/odp_ipsec_mb.m4]) m4_include([platform/linux-generic/m4/odp_pcapng.m4]) m4_include([platform/linux-generic/m4/odp_netmap.m4]) m4_include([platform/linux-generic/m4/odp_dpdk.m4]) m4_include([platform/linux-generic/m4/odp_xdp.m4]) ODP_SCHEDULER -AS_VAR_APPEND([PLAT_DEP_LIBS], ["${ATOMIC_LIBS} ${AARCH64CRYPTO_LIBS} ${LIBCONFIG_LIBS} ${OPENSSL_LIBS} ${DPDK_LIBS_LT} ${LIBCLI_LIBS} ${LIBXDP_LIBS}"]) +AS_VAR_APPEND([PLAT_DEP_LIBS], ["${ATOMIC_LIBS} ${AARCH64CRYPTO_LIBS} ${LIBCONFIG_LIBS} ${OPENSSL_LIBS} ${IPSEC_MB_LIBS} ${DPDK_LIBS_LT} ${LIBCLI_LIBS} ${LIBXDP_LIBS}"]) # Add text to the end of configure with platform specific settings. # Make sure it's aligned same as other lines in configure.ac. diff --git a/platform/linux-generic/m4/odp_crypto.m4 b/platform/linux-generic/m4/odp_crypto.m4 index 9bb99f7dd..1cec6edb4 100644 --- a/platform/linux-generic/m4/odp_crypto.m4 +++ b/platform/linux-generic/m4/odp_crypto.m4 @@ -3,7 +3,7 @@ # Select default crypto implementation AC_ARG_WITH([crypto], [AS_HELP_STRING([--with-crypto], - [Choose crypto implementation (openssl/armv8crypto/null)] + [Choose crypto implementation (openssl/armv8crypto/ipsecmb/null)] [[default=openssl] (linux-generic)])], [], [with_crypto=openssl]) @@ -14,7 +14,7 @@ AS_IF([test "x$with_crypto" = "xyes"], [with_crypto=openssl]) AS_IF([test "x$with_crypto" = "xno"], [with_crypto=null]) AS_IF([test "x$with_crypto" = "xopenssl" -a "x$with_openssl" = "xno"], [with_crypto=null]) -AS_IF([test "x$with_crypto" != "xopenssl" -a "x$with_crypto" != "xarmv8crypto" -a "x$with_crypto" != "xnull"], +AS_IF([test "x$with_crypto" != "xopenssl" -a "x$with_crypto" != "xarmv8crypto" -a "x$with_crypto" != "xipsecmb" -a "x$with_crypto" != "xnull"], [AC_MSG_ERROR([Invalid crypto implementation name])]) ########################################################################## @@ -31,11 +31,19 @@ AS_IF([test "x$with_crypto" == "xarmv8crypto"], [PKG_CHECK_MODULES([AARCH64CRYPTO], [libAArch64crypto]) AARCH64CRYPTO_PKG=", libAArch64crypto" AC_SUBST([AARCH64CRYPTO_PKG])]) + AC_CONFIG_COMMANDS_PRE([dnl AM_CONDITIONAL([WITH_ARMV8_CRYPTO], [test "x$with_crypto" == "xarmv8crypto"]) ]) ########################################################################## +# Multi-buffer IPSec library implementation +########################################################################## +AC_CONFIG_COMMANDS_PRE([dnl +AM_CONDITIONAL([WITH_IPSECMB_CRYPTO], [test "x$with_crypto" == "xipsecmb"]) +]) + +########################################################################## # Null implementation ########################################################################## AS_IF([test "x$with_crypto" == "xnull"], diff --git a/platform/linux-generic/m4/odp_ipsec_mb.m4 b/platform/linux-generic/m4/odp_ipsec_mb.m4 new file mode 100644 index 000000000..3268d94c0 --- /dev/null +++ b/platform/linux-generic/m4/odp_ipsec_mb.m4 @@ -0,0 +1,19 @@ +######################################################################### +# Check for libIPSec_MB availability +######################################################################### +ipsecmb_support=no +AC_CHECK_HEADERS([ipsec-mb.h], + [AC_CHECK_LIB([IPSec_MB], [init_mb_mgr_auto], [ipsecmb_support=yes], + [ipsecmb_support=no])], + [ipsecmb_support=no]) + +AS_IF([test "x$with_crypto" = "xipsecmb" -a "x$ipsecmb_support" = "xno"], + [AC_MSG_ERROR([IPSec MB library not found on this platform])]) + +if test "x$with_crypto" = "xipsecmb"; then + IPSEC_MB_LIBS="-lIPSec_MB" +else + IPSEC_MB_LIBS="" +fi + +AC_SUBST([IPSEC_MB_LIBS]) diff --git a/platform/linux-generic/odp_crypto_ipsecmb.c b/platform/linux-generic/odp_crypto_ipsecmb.c new file mode 100644 index 000000000..d291a7067 --- /dev/null +++ b/platform/linux-generic/odp_crypto_ipsecmb.c @@ -0,0 +1,964 @@ +/* Copyright (c) 2014-2018, Linaro Limited + * Copyright (c) 2021, ARM Limited + * Copyright (c) 2022, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <odp_posix_extensions.h> +#include <odp/autoheader_internal.h> + +#include <odp/api/crypto.h> +#include <odp/api/spinlock.h> +#include <odp/api/debug.h> +#include <odp/api/align.h> +#include <odp/api/shared_memory.h> +#include <odp/api/hints.h> + +#include <odp/api/plat/event_inlines.h> +#include <odp/api/plat/packet_inlines.h> +#include <odp/api/plat/queue_inlines.h> +#include <odp/api/plat/thread_inlines.h> + +#include <odp_debug_internal.h> +#include <odp_global_data.h> +#include <odp_init_internal.h> +#include <odp_packet_internal.h> + +#include <ipsec-mb.h> + +#define MAX_SESSIONS 4000 +/* Length in bytes */ +#define IPSEC_MB_CRYPTO_MAX_CIPHER_KEY_LENGTH 32 +#define IPSEC_MB_CRYPTO_MAX_AUTH_KEY_LENGTH 32 +#define IPSEC_MB_CRYPTO_MAX_IV_LENGTH 32 +#define IPSEC_MB_CRYPTO_MAX_DATA_LENGTH 65536 +#define ZUC_DIGEST_LENGTH 4 + +#define ODP_CRYPTO_IPSEC_MB_SHM_NAME "_odp_crypto_ipsecmb" +/* + * Cipher algorithm capabilities + * + * Keep sorted: first by key length, then by IV length + */ +static const odp_crypto_cipher_capability_t cipher_capa_null[] = { +{.key_len = 0, .iv_len = 0} }; + +static const odp_crypto_cipher_capability_t cipher_capa_zuc_eea3[] = { +{.key_len = 16, .iv_len = 16}, +{.key_len = 32, .iv_len = 25} }; + +/* + * Authentication algorithm capabilities + * + * Keep sorted: first by digest length, then by key length + */ +static const odp_crypto_auth_capability_t auth_capa_null[] = { +{.digest_len = 0, .key_len = 0, .aad_len = {.min = 0, .max = 0, .inc = 0} } }; + +static const odp_crypto_auth_capability_t auth_capa_zuc_eia3[] = { +{.digest_len = 4, .key_len = 16, .aad_len = {.min = 0, .max = 0, .inc = 0}, + .iv_len = 16}, +{.digest_len = 4, .key_len = 32, .aad_len = {.min = 0, .max = 0, .inc = 0}, + .iv_len = 25} }; + +/** Forward declaration of session structure */ +typedef struct odp_crypto_generic_session_t odp_crypto_generic_session_t; + +/** + * Algorithm handler function prototype + */ +typedef odp_crypto_alg_err_t (*crypto_func_t)(odp_packet_t pkt, + const odp_crypto_packet_op_param_t *param, + odp_crypto_generic_session_t *session); + +/** + * Per crypto session data structure + */ +struct odp_crypto_generic_session_t { + odp_crypto_generic_session_t *next; + + /* Session creation parameters */ + odp_crypto_session_param_t p; + + odp_bool_t do_cipher_first; + + struct { +#if ODP_DEPRECATED_API + /* Copy of session IV data */ + uint8_t iv_data[IPSEC_MB_CRYPTO_MAX_IV_LENGTH]; +#endif + uint8_t key_data[IPSEC_MB_CRYPTO_MAX_CIPHER_KEY_LENGTH]; + crypto_func_t func; + } cipher; + + struct { + uint8_t key[IPSEC_MB_CRYPTO_MAX_AUTH_KEY_LENGTH]; +#if ODP_DEPRECATED_API + uint8_t iv_data[IPSEC_MB_CRYPTO_MAX_IV_LENGTH]; +#endif + crypto_func_t func; + } auth; + + unsigned int idx; +}; + +typedef struct odp_crypto_global_s odp_crypto_global_t; + +struct odp_crypto_global_s { + odp_spinlock_t lock; + odp_crypto_generic_session_t *free; + odp_crypto_generic_session_t sessions[MAX_SESSIONS]; +}; + +static odp_crypto_global_t *global; + +typedef struct crypto_local_t { + uint8_t buffer[IPSEC_MB_CRYPTO_MAX_DATA_LENGTH]; + IMB_MGR *mb_mgr; +} crypto_local_t; + +static __thread crypto_local_t local; + +static +odp_crypto_generic_session_t *alloc_session(void) +{ + odp_crypto_generic_session_t *session = NULL; + + odp_spinlock_lock(&global->lock); + session = global->free; + if (session) { + global->free = session->next; + session->next = NULL; + } + odp_spinlock_unlock(&global->lock); + + if (!session) + return NULL; + + session->idx = session - global->sessions; + + return session; +} + +static +void free_session(odp_crypto_generic_session_t *session) +{ + odp_spinlock_lock(&global->lock); + session->next = global->free; + global->free = session; + odp_spinlock_unlock(&global->lock); +} + +static +odp_crypto_packet_result_t *get_op_result_from_packet(odp_packet_t pkt) +{ + odp_packet_hdr_t *hdr = packet_hdr(pkt); + + return &hdr->crypto_op_result; +} + +static odp_crypto_alg_err_t +null_crypto_routine(odp_packet_t pkt ODP_UNUSED, + const odp_crypto_packet_op_param_t *param ODP_UNUSED, + odp_crypto_generic_session_t *session ODP_UNUSED) +{ + return ODP_CRYPTO_ALG_ERR_NONE; +} + +static +odp_crypto_alg_err_t zuc_eea3_cipher_op(odp_packet_t pkt, + const odp_crypto_packet_op_param_t *param, + odp_crypto_generic_session_t *session) +{ + IMB_MGR *mb_mgr = local.mb_mgr; + uint8_t *iv_ptr; + uint32_t in_pos = param->cipher_range.offset; + uint32_t in_len = param->cipher_range.length; + +#if ODP_DEPRECATED_API + if (param->cipher_iv_ptr) + iv_ptr = param->cipher_iv_ptr; + else if (session->p.cipher_iv.data) + iv_ptr = session->cipher.iv_data; + else + return ODP_CRYPTO_ALG_ERR_IV_INVALID; +#else + iv_ptr = param->cipher_iv_ptr; + _ODP_ASSERT(session->p.cipher_iv_len == 0 || iv_ptr != NULL); +#endif + + uint32_t seg_len = 0; + uint8_t *data = odp_packet_offset(pkt, in_pos, &seg_len, NULL); + + if (odp_unlikely(seg_len < in_len)) { + if (odp_unlikely(in_len > IPSEC_MB_CRYPTO_MAX_DATA_LENGTH)) + return ODP_CRYPTO_ALG_ERR_DATA_SIZE; + + /* Packet is segmented within the cipher range. Copy the cipher + * range to a contiguous buffer. */ + odp_packet_copy_to_mem(pkt, in_pos, in_len, local.buffer); + + data = local.buffer; + } + + if (session->p.cipher_key.length == 16) { + /* ZUC128 EEA3 */ + IMB_ZUC_EEA3_1_BUFFER(mb_mgr, session->cipher.key_data, + iv_ptr, + data, + data, + in_len); + } else { + /* Only 16 and 32 byte keys are supported + * ZUC256 EEA3 */ + IMB_ZUC256_EEA3_1_BUFFER(mb_mgr, session->cipher.key_data, + iv_ptr, + data, + data, + in_len); + } + if (odp_unlikely(imb_get_errno(mb_mgr) != 0)) + return ODP_CRYPTO_ALG_ERR_DATA_SIZE; + + if (odp_unlikely(seg_len < in_len)) + odp_packet_copy_from_mem(pkt, in_pos, in_len, data); + + return ODP_CRYPTO_ALG_ERR_NONE; +} + +static int process_zuc_eea3_param(odp_crypto_generic_session_t *session) +{ + if (!((16 == session->p.cipher_key.length && + 16 == session->p.cipher_iv_len) || + (32 == session->p.cipher_key.length && + 25 == session->p.cipher_iv_len))) + return -1; + + memcpy(session->cipher.key_data, session->p.cipher_key.data, + session->p.cipher_key.length); + + session->cipher.func = zuc_eea3_cipher_op; + + return 0; +} + +static +odp_crypto_alg_err_t auth_zuc_eia3_gen(odp_packet_t pkt, + const odp_crypto_packet_op_param_t *param, + odp_crypto_generic_session_t *session) +{ + IMB_MGR *mb_mgr = local.mb_mgr; + uint8_t *iv_ptr; + uint32_t in_pos = param->auth_range.offset; + uint32_t in_len = param->auth_range.length; + uint32_t auth_tag; + +#if ODP_DEPRECATED_API + if (param->auth_iv_ptr) + iv_ptr = param->auth_iv_ptr; + else if (session->p.auth_iv.data) + iv_ptr = session->auth.iv_data; + else + return ODP_CRYPTO_ALG_ERR_IV_INVALID; +#else + iv_ptr = param->auth_iv_ptr; + _ODP_ASSERT(session->p.auth_iv_len == 0 || iv_ptr != NULL); +#endif + + uint32_t seg_len = 0; + uint8_t *data = odp_packet_offset(pkt, in_pos, &seg_len, NULL); + + if (odp_unlikely(seg_len < in_len)) { + if (odp_unlikely(in_len > IPSEC_MB_CRYPTO_MAX_DATA_LENGTH)) + return ODP_CRYPTO_ALG_ERR_DATA_SIZE; + + /* Packet is segmented within the auth range. Copy the auth + * range to a contiguous buffer. */ + odp_packet_copy_to_mem(pkt, in_pos, in_len, local.buffer); + + data = local.buffer; + } + + if (session->p.auth_key.length == 16) { + /* ZUC128 EIA3 */ + IMB_ZUC_EIA3_1_BUFFER(mb_mgr, session->auth.key, + iv_ptr, + data, + param->auth_range.length * 8, + &auth_tag); + } else { + /* Only 16 and 32 byte keys are supported + * ZUC256 EIA3 */ + IMB_ZUC256_EIA3_1_BUFFER(mb_mgr, session->auth.key, + iv_ptr, + data, + param->auth_range.length * 8, + &auth_tag); + } + if (odp_unlikely(imb_get_errno(mb_mgr) != 0)) + return ODP_CRYPTO_ALG_ERR_DATA_SIZE; + + /* Copy to the output location */ + odp_packet_copy_from_mem(pkt, param->hash_result_offset, + session->p.auth_digest_len, + &auth_tag); + + return ODP_CRYPTO_ALG_ERR_NONE; +} + +static +odp_crypto_alg_err_t auth_zuc_eia3_check(odp_packet_t pkt, + const odp_crypto_packet_op_param_t *param, + odp_crypto_generic_session_t *session) +{ + IMB_MGR *mb_mgr = local.mb_mgr; + uint8_t *iv_ptr; + uint32_t in_pos = param->auth_range.offset; + uint32_t in_len = param->auth_range.length; + uint32_t bytes = ZUC_DIGEST_LENGTH; + uint32_t hash_in; + uint32_t hash_out; + + /* Copy current value out and clear it before authentication */ + odp_packet_copy_to_mem(pkt, param->hash_result_offset, + bytes, &hash_in); + + if (odp_unlikely(session->p.hash_result_in_auth_range)) + _odp_packet_set_data(pkt, param->hash_result_offset, 0, bytes); + +#if ODP_DEPRECATED_API + if (param->auth_iv_ptr) + iv_ptr = param->auth_iv_ptr; + else if (session->p.auth_iv.data) + iv_ptr = session->auth.iv_data; + else + return ODP_CRYPTO_ALG_ERR_IV_INVALID; +#else + iv_ptr = param->auth_iv_ptr; + _ODP_ASSERT(session->p.auth_iv_len == 0 || iv_ptr != NULL); +#endif + + uint32_t seg_len = 0; + uint8_t *data = odp_packet_offset(pkt, in_pos, &seg_len, NULL); + + if (odp_unlikely(seg_len < in_len)) { + if (odp_unlikely(in_len > IPSEC_MB_CRYPTO_MAX_DATA_LENGTH)) + return ODP_CRYPTO_ALG_ERR_DATA_SIZE; + + /* Packet is segmented within the auth range. Copy the auth + * range to a contiguous buffer. */ + odp_packet_copy_to_mem(pkt, in_pos, in_len, local.buffer); + + data = local.buffer; + } + + if (session->p.auth_key.length == 16) { + /* ZUC128 EIA3 */ + IMB_ZUC_EIA3_1_BUFFER(mb_mgr, session->auth.key, + iv_ptr, + data, + param->auth_range.length * 8, + &hash_out); + } else { + /* Only 16 and 32 byte keys are supported + * ZUC256 EIA3 */ + IMB_ZUC256_EIA3_1_BUFFER(mb_mgr, session->auth.key, + iv_ptr, + data, + param->auth_range.length * 8, + &hash_out); + } + if (odp_unlikely(imb_get_errno(mb_mgr) != 0)) + return ODP_CRYPTO_ALG_ERR_DATA_SIZE; + + /* Verify match */ + if (hash_in != hash_out) + return ODP_CRYPTO_ALG_ERR_ICV_CHECK; + + return ODP_CRYPTO_ALG_ERR_NONE; +} + +static int process_auth_zuc_eia3_param(odp_crypto_generic_session_t *session) +{ + if (!((16 == session->p.auth_key.length && + 16 == session->p.auth_iv_len) || + (32 == session->p.auth_key.length && + 25 == session->p.auth_iv_len))) + return -1; + + if (ODP_CRYPTO_OP_ENCODE == session->p.op) + session->auth.func = auth_zuc_eia3_gen; + else + session->auth.func = auth_zuc_eia3_check; + + if (session->p.auth_digest_len != ZUC_DIGEST_LENGTH) + return -1; + + memcpy(session->auth.key, session->p.auth_key.data, + session->p.auth_key.length); + + return 0; +} + +int odp_crypto_capability(odp_crypto_capability_t *capa) +{ + if (NULL == capa) + return -1; + + memset(capa, 0, sizeof(odp_crypto_capability_t)); + + capa->sync_mode = ODP_SUPPORT_PREFERRED; + capa->async_mode = ODP_SUPPORT_YES; + capa->queue_type_plain = 1; + capa->queue_type_sched = 1; + + capa->ciphers.bit.null = 1; + capa->auths.bit.null = 1; + + capa->ciphers.bit.zuc_eea3 = 1; + capa->auths.bit.zuc_eia3 = 1; + + capa->max_sessions = MAX_SESSIONS; + + return 0; +} + +int odp_crypto_cipher_capability(odp_cipher_alg_t cipher, + odp_crypto_cipher_capability_t dst[], + int num_copy) +{ + const odp_crypto_cipher_capability_t *src; + int num; + int size = sizeof(odp_crypto_cipher_capability_t); + + switch (cipher) { + case ODP_CIPHER_ALG_NULL: + src = cipher_capa_null; + num = sizeof(cipher_capa_null) / size; + break; + case ODP_CIPHER_ALG_ZUC_EEA3: + src = cipher_capa_zuc_eea3; + num = sizeof(cipher_capa_zuc_eea3) / size; + break; + default: + return -1; + } + + if (num < num_copy) + num_copy = num; + + memcpy(dst, src, num_copy * size); + + return num; +} + +int odp_crypto_auth_capability(odp_auth_alg_t auth, + odp_crypto_auth_capability_t dst[], int num_copy) +{ + const odp_crypto_auth_capability_t *src; + int num; + int size = sizeof(odp_crypto_auth_capability_t); + + switch (auth) { + case ODP_AUTH_ALG_NULL: + src = auth_capa_null; + num = sizeof(auth_capa_null) / size; + break; + case ODP_AUTH_ALG_ZUC_EIA3: + src = auth_capa_zuc_eia3; + num = sizeof(auth_capa_zuc_eia3) / size; + break; + default: + return -1; + } + + if (num < num_copy) + num_copy = num; + + memcpy(dst, src, num_copy * size); + + return num; +} + +int +odp_crypto_session_create(const odp_crypto_session_param_t *param, + odp_crypto_session_t *session_out, + odp_crypto_ses_create_err_t *status) +{ + int rc = 0; + odp_crypto_generic_session_t *session; + + if (odp_global_ro.disable.crypto) { + _ODP_ERR("Crypto is disabled\n"); + /* Dummy output to avoid compiler warning about uninitialized + * variables */ + *status = ODP_CRYPTO_SES_ERR_ENOMEM; + *session_out = ODP_CRYPTO_SESSION_INVALID; + return -1; + } + + session = alloc_session(); + if (NULL == session) { + *status = ODP_CRYPTO_SES_ERR_ENOMEM; + goto err; + } + + session->p = *param; + + if (session->p.cipher_iv_len > IPSEC_MB_CRYPTO_MAX_IV_LENGTH) { + _ODP_DBG("Maximum IV length exceeded\n"); + *status = ODP_CRYPTO_SES_ERR_CIPHER; + goto err; + } + + if (session->p.auth_iv_len > IPSEC_MB_CRYPTO_MAX_IV_LENGTH) { + _ODP_DBG("Maximum auth IV length exceeded\n"); + *status = ODP_CRYPTO_SES_ERR_CIPHER; + goto err; + } + +#if ODP_DEPRECATED_API + /* Copy IV data */ + if (session->p.cipher_iv.data) + memcpy(session->cipher.iv_data, session->p.cipher_iv.data, + session->p.cipher_iv.length); + + if (session->p.auth_iv.data) + memcpy(session->auth.iv_data, session->p.auth_iv.data, + session->p.auth_iv.length); +#endif + + /* Derive order */ + if (ODP_CRYPTO_OP_ENCODE == param->op) + session->do_cipher_first = param->auth_cipher_text; + else + session->do_cipher_first = !param->auth_cipher_text; + + /* Process based on cipher */ + switch (param->cipher_alg) { + case ODP_CIPHER_ALG_NULL: + session->cipher.func = null_crypto_routine; + rc = 0; + break; + case ODP_CIPHER_ALG_ZUC_EEA3: + rc = process_zuc_eea3_param(session); + break; + default: + rc = -1; + } + + if (rc) { + *status = ODP_CRYPTO_SES_ERR_CIPHER; + goto err; + } + + /* Process based on auth */ + switch (param->auth_alg) { + case ODP_AUTH_ALG_NULL: + session->auth.func = null_crypto_routine; + rc = 0; + break; + case ODP_AUTH_ALG_ZUC_EIA3: + rc = process_auth_zuc_eia3_param(session); + break; + default: + rc = -1; + } + + if (rc) { + *status = ODP_CRYPTO_SES_ERR_AUTH; + goto err; + } + + *session_out = (intptr_t)session; + *status = ODP_CRYPTO_SES_ERR_NONE; + return 0; + +err: + /* error status should be set at this moment */ + if (session != NULL) + free_session(session); + *session_out = ODP_CRYPTO_SESSION_INVALID; + return -1; +} + +int odp_crypto_session_destroy(odp_crypto_session_t session) +{ + odp_crypto_generic_session_t *generic; + + generic = (odp_crypto_generic_session_t *)(intptr_t)session; + memset(generic, 0, sizeof(*generic)); + free_session(generic); + return 0; +} + +#if ODP_DEPRECATED_API +int +odp_crypto_operation(odp_crypto_op_param_t *param, + odp_bool_t *posted, + odp_crypto_op_result_t *result) +{ + odp_crypto_packet_op_param_t packet_param; + odp_packet_t out_pkt = param->out_pkt; + odp_crypto_packet_result_t packet_result; + odp_crypto_op_result_t local_result; + int rc; + + packet_param.session = param->session; + packet_param.cipher_iv_ptr = param->cipher_iv_ptr; + packet_param.auth_iv_ptr = param->auth_iv_ptr; + packet_param.hash_result_offset = param->hash_result_offset; + packet_param.aad_ptr = param->aad_ptr; + packet_param.cipher_range = param->cipher_range; + packet_param.auth_range = param->auth_range; + + rc = odp_crypto_op(¶m->pkt, &out_pkt, &packet_param, 1); + if (rc <= 0) + return -1; + + rc = odp_crypto_result(&packet_result, out_pkt); + if (rc < 0) { + /* + * We cannot fail since odp_crypto_op() has already processed + * the packet. Let's indicate error in the result instead. + */ + packet_result.ok = false; + } + + /* Indicate to caller operation was sync */ + *posted = 0; + + packet_subtype_set(out_pkt, ODP_EVENT_PACKET_BASIC); + + /* Fill in result */ + local_result.ctx = param->ctx; + local_result.pkt = out_pkt; + local_result.cipher_status = packet_result.cipher_status; + local_result.auth_status = packet_result.auth_status; + local_result.ok = packet_result.ok; + + /* + * Be bug-to-bug compatible. Return output packet also through params. + */ + param->out_pkt = out_pkt; + + *result = local_result; + + return 0; +} +#endif + +int _odp_crypto_init_global(void) +{ + size_t mem_size; + odp_shm_t shm; + int idx; + + if (odp_global_ro.disable.crypto) { + _ODP_PRINT("\nODP crypto is DISABLED\n"); + return 0; + } + + /* Calculate the memory size we need */ + mem_size = sizeof(odp_crypto_global_t); + + /* Allocate our globally shared memory */ + shm = odp_shm_reserve(ODP_CRYPTO_IPSEC_MB_SHM_NAME, mem_size, + ODP_CACHE_LINE_SIZE, + 0); + if (ODP_SHM_INVALID == shm) { + _ODP_ERR("unable to allocate crypto pool\n"); + return -1; + } + + global = odp_shm_addr(shm); + + /* Clear it out */ + memset(global, 0, mem_size); + + /* Initialize free list and lock */ + for (idx = 0; idx < MAX_SESSIONS; idx++) { + global->sessions[idx].next = global->free; + global->free = &global->sessions[idx]; + } + odp_spinlock_init(&global->lock); + + return 0; +} + +int _odp_crypto_term_global(void) +{ + int rc = 0; + int ret; + int count = 0; + odp_crypto_generic_session_t *session; + + if (odp_global_ro.disable.crypto) + return 0; + + for (session = global->free; session != NULL; session = session->next) + count++; + if (count != MAX_SESSIONS) { + _ODP_ERR("crypto sessions still active\n"); + rc = -1; + } + + ret = odp_shm_free(odp_shm_lookup(ODP_CRYPTO_IPSEC_MB_SHM_NAME)); + if (ret < 0) { + _ODP_ERR("shm free failed for %s\n", ODP_CRYPTO_IPSEC_MB_SHM_NAME); + rc = -1; + } + + return rc; +} + +int _odp_crypto_init_local(void) +{ + uint64_t flags = 0; + + if (odp_global_ro.disable.crypto) + return 0; + + memset(&local, 0, sizeof(local)); + + local.mb_mgr = alloc_mb_mgr(flags); + if (local.mb_mgr == NULL) + return -1; + + init_mb_mgr_auto(local.mb_mgr, NULL); + + return 0; +} + +int _odp_crypto_term_local(void) +{ + if (odp_global_ro.disable.crypto) + return 0; + + free_mb_mgr(local.mb_mgr); + return 0; +} + +#if ODP_DEPRECATED_API +odp_crypto_compl_t odp_crypto_compl_from_event(odp_event_t ev) +{ + /* This check not mandated by the API specification */ + if (odp_event_type(ev) != ODP_EVENT_CRYPTO_COMPL) + _ODP_ABORT("Event not a crypto completion"); + return (odp_crypto_compl_t)ev; +} + +odp_event_t odp_crypto_compl_to_event(odp_crypto_compl_t completion_event) +{ + return (odp_event_t)completion_event; +} + +void +odp_crypto_compl_result(odp_crypto_compl_t completion_event, + odp_crypto_op_result_t *result) +{ + (void)completion_event; + (void)result; + + /* We won't get such events anyway, so there can be no result */ + _ODP_ASSERT(0); +} + +void +odp_crypto_compl_free(odp_crypto_compl_t completion_event) +{ + odp_event_t ev = odp_crypto_compl_to_event(completion_event); + + odp_buffer_free(odp_buffer_from_event(ev)); +} + +uint64_t odp_crypto_compl_to_u64(odp_crypto_compl_t hdl) +{ + return _odp_pri(hdl); +} +#endif /* ODP_DEPRECATED_API */ + +void odp_crypto_session_param_init(odp_crypto_session_param_t *param) +{ + memset(param, 0, sizeof(odp_crypto_session_param_t)); +} + +uint64_t odp_crypto_session_to_u64(odp_crypto_session_t hdl) +{ + return (uint64_t)hdl; +} + +odp_packet_t odp_crypto_packet_from_event(odp_event_t ev) +{ + /* This check not mandated by the API specification */ + _ODP_ASSERT(odp_event_type(ev) == ODP_EVENT_PACKET); + _ODP_ASSERT(odp_event_subtype(ev) == ODP_EVENT_PACKET_CRYPTO); + + return odp_packet_from_event(ev); +} + +odp_event_t odp_crypto_packet_to_event(odp_packet_t pkt) +{ + return odp_packet_to_event(pkt); +} + +int odp_crypto_result(odp_crypto_packet_result_t *result, + odp_packet_t packet) +{ + odp_crypto_packet_result_t *op_result; + + _ODP_ASSERT(odp_event_subtype(odp_packet_to_event(packet)) == + ODP_EVENT_PACKET_CRYPTO); + + op_result = get_op_result_from_packet(packet); + + memcpy(result, op_result, sizeof(*result)); + + return 0; +} + +static int copy_data_and_metadata(odp_packet_t dst, odp_packet_t src) +{ + int md_copy; + int rc; + + md_copy = _odp_packet_copy_md_possible(odp_packet_pool(dst), + odp_packet_pool(src)); + if (odp_unlikely(md_copy < 0)) { + _ODP_ERR("Unable to copy packet metadata\n"); + return -1; + } + + rc = odp_packet_copy_from_pkt(dst, 0, src, 0, odp_packet_len(src)); + if (odp_unlikely(rc < 0)) { + _ODP_ERR("Unable to copy packet data\n"); + return -1; + } + + _odp_packet_copy_md(packet_hdr(dst), packet_hdr(src), md_copy); + return 0; +} + +static odp_packet_t get_output_packet(const odp_crypto_generic_session_t *session, + odp_packet_t pkt_in, + odp_packet_t pkt_out) +{ + int rc; + + if (odp_likely(pkt_in == pkt_out)) + return pkt_out; + + if (pkt_out == ODP_PACKET_INVALID) { + odp_pool_t pool = session->p.output_pool; + + _ODP_ASSERT(pool != ODP_POOL_INVALID); + if (pool == odp_packet_pool(pkt_in)) { + pkt_out = pkt_in; + } else { + pkt_out = odp_packet_copy(pkt_in, pool); + if (odp_likely(pkt_out != ODP_PACKET_INVALID)) + odp_packet_free(pkt_in); + } + return pkt_out; + } + rc = copy_data_and_metadata(pkt_out, pkt_in); + if (odp_unlikely(rc < 0)) + return ODP_PACKET_INVALID; + + odp_packet_free(pkt_in); + return pkt_out; +} + +static +int crypto_int(odp_packet_t pkt_in, + odp_packet_t *pkt_out, + const odp_crypto_packet_op_param_t *param) +{ + odp_crypto_alg_err_t rc_cipher = ODP_CRYPTO_ALG_ERR_NONE; + odp_crypto_alg_err_t rc_auth = ODP_CRYPTO_ALG_ERR_NONE; + odp_crypto_generic_session_t *session; + odp_packet_t out_pkt; + odp_crypto_packet_result_t *op_result; + + session = (odp_crypto_generic_session_t *)(intptr_t)param->session; + + out_pkt = get_output_packet(session, pkt_in, *pkt_out); + if (odp_unlikely(out_pkt == ODP_PACKET_INVALID)) + return -1; + + /* Invoke the crypto function */ + if (session->do_cipher_first) { + rc_cipher = session->cipher.func(out_pkt, param, session); + rc_auth = session->auth.func(out_pkt, param, session); + } else { + rc_auth = session->auth.func(out_pkt, param, session); + rc_cipher = session->cipher.func(out_pkt, param, session); + } + + packet_subtype_set(out_pkt, ODP_EVENT_PACKET_CRYPTO); + op_result = get_op_result_from_packet(out_pkt); + op_result->cipher_status.alg_err = rc_cipher; + op_result->cipher_status.hw_err = ODP_CRYPTO_HW_ERR_NONE; + op_result->auth_status.alg_err = rc_auth; + op_result->auth_status.hw_err = ODP_CRYPTO_HW_ERR_NONE; + op_result->ok = + (rc_cipher == ODP_CRYPTO_ALG_ERR_NONE) && + (rc_auth == ODP_CRYPTO_ALG_ERR_NONE); + + /* Synchronous, simply return results */ + *pkt_out = out_pkt; + + return 0; +} + +int odp_crypto_op(const odp_packet_t pkt_in[], + odp_packet_t pkt_out[], + const odp_crypto_packet_op_param_t param[], + int num_pkt) +{ + int i, rc; + odp_crypto_generic_session_t *session; + + for (i = 0; i < num_pkt; i++) { + session = (odp_crypto_generic_session_t *)(intptr_t)param[i].session; + _ODP_ASSERT(ODP_CRYPTO_SYNC == session->p.op_mode); + + rc = crypto_int(pkt_in[i], &pkt_out[i], ¶m[i]); + if (rc < 0) + break; + } + + return i; +} + +int odp_crypto_op_enq(const odp_packet_t pkt_in[], + const odp_packet_t pkt_out[], + const odp_crypto_packet_op_param_t param[], + int num_pkt) +{ + odp_packet_t pkt; + odp_event_t event; + odp_crypto_generic_session_t *session; + int i, rc; + + for (i = 0; i < num_pkt; i++) { + session = (odp_crypto_generic_session_t *)(intptr_t)param[i].session; + _ODP_ASSERT(ODP_CRYPTO_ASYNC == session->p.op_mode); + _ODP_ASSERT(ODP_QUEUE_INVALID != session->p.compl_queue); + + pkt = pkt_out[i]; + rc = crypto_int(pkt_in[i], &pkt, ¶m[i]); + if (rc < 0) + break; + + event = odp_packet_to_event(pkt); + if (odp_queue_enq(session->p.compl_queue, event)) { + odp_event_free(event); + break; + } + } + + return i; +} |