diff options
Diffstat (limited to 'platform')
35 files changed, 759 insertions, 556 deletions
diff --git a/platform/linux-dpdk/Makefile.am b/platform/linux-dpdk/Makefile.am index 2f257044e..789fbced6 100644 --- a/platform/linux-dpdk/Makefile.am +++ b/platform/linux-dpdk/Makefile.am @@ -144,7 +144,7 @@ noinst_HEADERS = \ ${top_srcdir}/platform/linux-generic/include/odp_sysinfo_internal.h \ include/odp_shm_internal.h \ include/odp_thread_internal.h \ - ${top_srcdir}/platform/linux-generic/include/odp_timer_internal.h \ + include/odp_timer_internal.h \ ${top_srcdir}/platform/linux-generic/include/odp_timer_wheel_internal.h \ ${top_srcdir}/platform/linux-generic/include/odp_traffic_mngr_internal.h \ include/odp_event_vector_internal.h \ @@ -292,7 +292,7 @@ noinst_HEADERS += arch/aarch64/odp_atomic.h \ arch/aarch64/cpu_flags.h \ arch/aarch64/odp_cpu_idling.h \ arch/aarch64/odp_llsc.h \ - arch/default/odp_random.h + arch/aarch64/odp_random.h endif if ARCH_IS_DEFAULT __LIB__libodp_dpdk_la_SOURCES += arch/default/odp_atomic.c \ @@ -315,26 +315,6 @@ noinst_HEADERS += arch/default/odp_atomic.h \ arch/default/odp_cpu_idling.h \ arch/default/odp_random.h endif -if ARCH_IS_MIPS64 -__LIB__libodp_dpdk_la_SOURCES += arch/default/odp_atomic.c \ - arch/default/odp_global_time.c \ - arch/default/odp_hash_crc32.c \ - arch/default/odp_random.c \ - arch/mips64/odp_sysinfo_parse.c -odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_time.h \ - arch/default/odp/api/abi/hash_crc32.h -if !ODP_ABI_COMPAT -odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ - arch/default/odp/api/abi/atomic_inlines.h \ - arch/default/odp/api/abi/cpu_generic.h \ - arch/mips64/odp/api/abi/cpu_inlines.h \ - arch/mips64/odp/api/abi/cpu.h -endif -noinst_HEADERS += arch/default/odp_atomic.h \ - arch/default/odp_cpu.h \ - arch/default/odp_cpu_idling.h \ - arch/default/odp_random.h -endif if ARCH_IS_POWERPC __LIB__libodp_dpdk_la_SOURCES += arch/default/odp_atomic.c \ arch/default/odp_cpu_cycles.c \ diff --git a/platform/linux-dpdk/arch/aarch64/odp_random.h b/platform/linux-dpdk/arch/aarch64/odp_random.h new file mode 120000 index 000000000..892ad529c --- /dev/null +++ b/platform/linux-dpdk/arch/aarch64/odp_random.h @@ -0,0 +1 @@ +../../../linux-generic/arch/aarch64/odp_random.h
\ No newline at end of file diff --git a/platform/linux-dpdk/arch/mips64/odp/api/abi/cpu.h b/platform/linux-dpdk/arch/mips64/odp/api/abi/cpu.h deleted file mode 100644 index 6644a1ed3..000000000 --- a/platform/linux-dpdk/arch/mips64/odp/api/abi/cpu.h +++ /dev/null @@ -1,26 +0,0 @@ -/* Copyright (c) 2016-2018, Linaro Limited - * Copyright (c) 2021, Nokia - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -#ifndef ODP_API_ABI_CPU_H_ -#define ODP_API_ABI_CPU_H_ - -#ifdef __cplusplus -extern "C" { -#endif - -#include <rte_common.h> - -#define ODP_CACHE_LINE_SIZE RTE_CACHE_LINE_SIZE - -/* Inlined functions for non-ABI compat mode */ -#include <odp/api/plat/cpu_inlines.h> - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/platform/linux-dpdk/arch/mips64/odp/api/abi/cpu_inlines.h b/platform/linux-dpdk/arch/mips64/odp/api/abi/cpu_inlines.h deleted file mode 120000 index 873e0681f..000000000 --- a/platform/linux-dpdk/arch/mips64/odp/api/abi/cpu_inlines.h +++ /dev/null @@ -1 +0,0 @@ -../../../../../../linux-generic/arch/mips64/odp/api/abi/cpu_inlines.h
\ No newline at end of file diff --git a/platform/linux-dpdk/arch/mips64/odp_sysinfo_parse.c b/platform/linux-dpdk/arch/mips64/odp_sysinfo_parse.c deleted file mode 120000 index 794300475..000000000 --- a/platform/linux-dpdk/arch/mips64/odp_sysinfo_parse.c +++ /dev/null @@ -1 +0,0 @@ -../../../linux-generic/arch/mips64/odp_sysinfo_parse.c
\ No newline at end of file diff --git a/platform/linux-dpdk/include/odp_event_internal.h b/platform/linux-dpdk/include/odp_event_internal.h index 9221def98..b3ce6c795 100644 --- a/platform/linux-dpdk/include/odp_event_internal.h +++ b/platform/linux-dpdk/include/odp_event_internal.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2021, Nokia +/* Copyright (c) 2021-2022, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -54,15 +54,9 @@ typedef struct _odp_event_hdr_t { /* --- Mostly read only data --- */ - /* User pointer */ - const void *user_ptr; - /* Pool pointer */ void *pool_ptr; - /* User area pointer */ - void *uarea_addr; - } _odp_event_hdr_t; static inline odp_event_t _odp_event_from_hdr(_odp_event_hdr_t *hdr) diff --git a/platform/linux-dpdk/include/odp_packet_internal.h b/platform/linux-dpdk/include/odp_packet_internal.h index a3e806f54..bda9b6c98 100644 --- a/platform/linux-dpdk/include/odp_packet_internal.h +++ b/platform/linux-dpdk/include/odp_packet_internal.h @@ -1,5 +1,5 @@ /* Copyright (c) 2014-2018, Linaro Limited - * Copyright (c) 2021, Nokia + * Copyright (c) 2021-2022, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -126,6 +126,12 @@ typedef struct odp_packet_hdr_t { /* Classifier destination queue */ odp_queue_t dst_queue; + /* User area pointer */ + void *uarea_addr; + + /* User context pointer */ + const void *user_ptr; + /* Classifier mark */ uint16_t cls_mark; diff --git a/platform/linux-dpdk/include/odp_timer_internal.h b/platform/linux-dpdk/include/odp_timer_internal.h new file mode 100644 index 000000000..2e8487689 --- /dev/null +++ b/platform/linux-dpdk/include/odp_timer_internal.h @@ -0,0 +1,54 @@ +/* Copyright (c) 2014-2018, Linaro Limited + * Copyright (c) 2021-2022, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +/** + * @file + * + * ODP timeout descriptor - implementation internal + */ + +#ifndef ODP_TIMER_INTERNAL_H_ +#define ODP_TIMER_INTERNAL_H_ + +#include <odp/api/align.h> +#include <odp/api/debug.h> +#include <odp/api/timer.h> + +#include <odp_event_internal.h> +#include <odp_global_data.h> +#include <odp_pool_internal.h> + +/** + * Internal Timeout header + */ +typedef struct ODP_ALIGNED_CACHE odp_timeout_hdr_t { + /* Common event header */ + _odp_event_hdr_t event_hdr; + + /* Requested expiration time */ + uint64_t expiration; + + /* User ptr inherited from parent timer */ + const void *user_ptr; + + /* Parent timer */ + odp_timer_t timer; + +} odp_timeout_hdr_t; + +/* A larger decrement value should be used after receiving events compared to + * an 'empty' call. */ +void _odp_timer_run_inline(int dec); + +/* Static inline wrapper to minimize modification of schedulers. */ +static inline void timer_run(int dec) +{ + if (odp_global_rw->inline_timers) + _odp_timer_run_inline(dec); +} + +#endif diff --git a/platform/linux-dpdk/odp_crypto.c b/platform/linux-dpdk/odp_crypto.c index 02c06be0e..818993acc 100644 --- a/platform/linux-dpdk/odp_crypto.c +++ b/platform/linux-dpdk/odp_crypto.c @@ -1,5 +1,5 @@ /* Copyright (c) 2017-2018, Linaro Limited - * Copyright (c) 2018-2021, Nokia + * Copyright (c) 2018-2022, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -70,8 +70,10 @@ typedef struct crypto_session_entry_s { } flags; uint16_t cdev_nb_qpairs; uint8_t cdev_id; +#if ODP_DEPRECATED_API uint8_t cipher_iv_data[MAX_IV_LENGTH]; uint8_t auth_iv_data[MAX_IV_LENGTH]; +#endif } crypto_session_entry_t; typedef struct crypto_global_s { @@ -1332,7 +1334,7 @@ static int crypto_fill_cipher_xform(struct rte_crypto_sym_xform *cipher_xform, "cipher key")) return -1; cipher_xform->cipher.iv.offset = IV_OFFSET; - cipher_xform->cipher.iv.length = param->cipher_iv.length; + cipher_xform->cipher.iv.length = param->cipher_iv_len; /* Derive order */ if (ODP_CRYPTO_OP_ENCODE == param->op) @@ -1365,7 +1367,7 @@ static int crypto_fill_auth_xform(struct rte_crypto_sym_xform *auth_xform, return -1; auth_xform->auth.iv.offset = IV_OFFSET + MAX_IV_LENGTH; - auth_xform->auth.iv.length = param->auth_iv.length; + auth_xform->auth.iv.length = param->auth_iv_len; if (ODP_CRYPTO_OP_ENCODE == param->op) auth_xform->auth.op = RTE_CRYPTO_AUTH_OP_GENERATE; @@ -1391,7 +1393,7 @@ static int crypto_fill_aead_xform(struct rte_crypto_sym_xform *aead_xform, return -1; aead_xform->aead.iv.offset = IV_OFFSET; - aead_xform->aead.iv.length = param->cipher_iv.length; + aead_xform->aead.iv.length = param->cipher_iv_len; aead_xform->aead.aad_length = param->auth_aad_len; if (aead_xform->aead.aad_length > PACKET_AAD_MAX) { @@ -1439,14 +1441,14 @@ int odp_crypto_session_create(const odp_crypto_session_param_t *param, ODP_ERR("Crypto is disabled\n"); /* Dummy output to avoid compiler warning about uninitialized * variables */ - *status = ODP_CRYPTO_SES_CREATE_ERR_ENOMEM; + *status = ODP_CRYPTO_SES_ERR_ENOMEM; *session_out = ODP_CRYPTO_SESSION_INVALID; return -1; } if (rte_cryptodev_count() == 0) { ODP_ERR("No crypto devices available\n"); - *status = ODP_CRYPTO_SES_CREATE_ERR_ENOMEM; + *status = ODP_CRYPTO_SES_ERR_ENOMEM; goto err; } @@ -1454,7 +1456,7 @@ int odp_crypto_session_create(const odp_crypto_session_param_t *param, session = alloc_session(); if (session == NULL) { ODP_ERR("Failed to allocate a session session"); - *status = ODP_CRYPTO_SES_CREATE_ERR_ENOMEM; + *status = ODP_CRYPTO_SES_ERR_ENOMEM; goto err; } @@ -1481,7 +1483,7 @@ int odp_crypto_session_create(const odp_crypto_session_param_t *param, if (cipher_is_aead(param->cipher_alg)) { if (crypto_fill_aead_xform(&cipher_xform, &session->p) < 0) { - *status = ODP_CRYPTO_SES_CREATE_ERR_INV_CIPHER; + *status = ODP_CRYPTO_SES_ERR_CIPHER; goto err; } @@ -1493,12 +1495,12 @@ int odp_crypto_session_create(const odp_crypto_session_param_t *param, odp_bool_t do_cipher_first; if (crypto_fill_cipher_xform(&cipher_xform, &session->p) < 0) { - *status = ODP_CRYPTO_SES_CREATE_ERR_INV_CIPHER; + *status = ODP_CRYPTO_SES_ERR_CIPHER; goto err; } if (crypto_fill_auth_xform(&auth_xform, &session->p) < 0) { - *status = ODP_CRYPTO_SES_CREATE_ERR_INV_AUTH; + *status = ODP_CRYPTO_SES_ERR_AUTH; goto err; } @@ -1534,7 +1536,7 @@ int odp_crypto_session_create(const odp_crypto_session_param_t *param, } if (rc) { ODP_ERR("Couldn't find a crypto device"); - *status = ODP_CRYPTO_SES_CREATE_ERR_ENOMEM; + *status = ODP_CRYPTO_SES_ERR_ENOMEM; goto err; } @@ -1544,7 +1546,7 @@ int odp_crypto_session_create(const odp_crypto_session_param_t *param, /* Setup session */ rte_session = rte_cryptodev_sym_session_create(sess_mp); if (rte_session == NULL) { - *status = ODP_CRYPTO_SES_CREATE_ERR_ENOMEM; + *status = ODP_CRYPTO_SES_ERR_ENOMEM; goto err; } @@ -1552,7 +1554,7 @@ int odp_crypto_session_create(const odp_crypto_session_param_t *param, first_xform, sess_mp) < 0) { /* remove the crypto_session_entry_t */ rte_cryptodev_sym_session_free(rte_session); - *status = ODP_CRYPTO_SES_CREATE_ERR_ENOMEM; + *status = ODP_CRYPTO_SES_ERR_ENOMEM; goto err; } @@ -1567,6 +1569,7 @@ out_null: session->cdev_id = cdev_id; session->cipher_xform = cipher_xform; session->auth_xform = auth_xform; +#if ODP_DEPRECATED_API if (param->cipher_iv.data) memcpy(session->cipher_iv_data, param->cipher_iv.data, @@ -1575,10 +1578,10 @@ out_null: memcpy(session->auth_iv_data, param->auth_iv.data, param->auth_iv.length); - +#endif /* We're happy */ *session_out = (intptr_t)session; - *status = ODP_CRYPTO_SES_CREATE_ERR_NONE; + *status = ODP_CRYPTO_SES_ERR_NONE; return 0; @@ -1738,12 +1741,14 @@ static uint8_t *crypto_prepare_digest(crypto_session_entry_t *session, uint8_t *data; odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - if (verify) + if (verify) { odp_packet_copy_to_mem(pkt, param->hash_result_offset, session->p.auth_digest_len, pkt_hdr->crypto_digest_buf); - _odp_packet_set_data(pkt, param->hash_result_offset, 0, - session->p.auth_digest_len); + if (odp_unlikely(session->p.hash_result_in_auth_range)) + _odp_packet_set_data(pkt, param->hash_result_offset, 0, + session->p.auth_digest_len); + } data = pkt_hdr->crypto_digest_buf; mb = &pkt_hdr->event_hdr.mb; *phys_addr = @@ -1756,9 +1761,7 @@ static uint8_t *crypto_prepare_digest(crypto_session_entry_t *session, static void crypto_fill_aead_param(crypto_session_entry_t *session, odp_packet_t pkt, const odp_crypto_packet_op_param_t *param, - struct rte_crypto_op *op, - odp_crypto_alg_err_t *rc_cipher, - odp_crypto_alg_err_t *rc_auth ODP_UNUSED) + struct rte_crypto_op *op) { odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); struct rte_crypto_sym_xform *aead_xform; @@ -1792,6 +1795,7 @@ static void crypto_fill_aead_param(crypto_session_entry_t *session, iv_ptr++; } +#if ODP_DEPRECATED_API if (param->cipher_iv_ptr) memcpy(iv_ptr, param->cipher_iv_ptr, @@ -1800,8 +1804,12 @@ static void crypto_fill_aead_param(crypto_session_entry_t *session, memcpy(iv_ptr, session->cipher_iv_data, aead_xform->aead.iv.length); - else if (aead_xform->aead.iv.length != 0) - *rc_cipher = ODP_CRYPTO_ALG_ERR_IV_INVALID; + else + ODP_ASSERT(aead_xform->aead.iv.length == 0); +#else + ODP_ASSERT(aead_xform->aead.iv.length == 0 || param->cipher_iv_ptr != NULL); + memcpy(iv_ptr, param->cipher_iv_ptr, aead_xform->aead.iv.length); +#endif op->sym->aead.data.offset = param->cipher_range.offset; op->sym->aead.data.length = param->cipher_range.length; @@ -1810,12 +1818,11 @@ static void crypto_fill_aead_param(crypto_session_entry_t *session, static void crypto_fill_sym_param(crypto_session_entry_t *session, odp_packet_t pkt, const odp_crypto_packet_op_param_t *param, - struct rte_crypto_op *op, - odp_crypto_alg_err_t *rc_cipher, - odp_crypto_alg_err_t *rc_auth) + struct rte_crypto_op *op) { struct rte_crypto_sym_xform *cipher_xform; struct rte_crypto_sym_xform *auth_xform; + uint8_t *iv_ptr; cipher_xform = &session->cipher_xform; auth_xform = &session->auth_xform; @@ -1831,43 +1838,47 @@ static void crypto_fill_sym_param(crypto_session_entry_t *session, &op->sym->auth.digest.phys_addr); } +#if ODP_DEPRECATED_API if (param->cipher_iv_ptr) { - uint8_t *iv_ptr; - iv_ptr = rte_crypto_op_ctod_offset(op, uint8_t *, IV_OFFSET); memcpy(iv_ptr, param->cipher_iv_ptr, cipher_xform->cipher.iv.length); } else if (session->p.cipher_iv.data) { - uint8_t *iv_ptr; - iv_ptr = rte_crypto_op_ctod_offset(op, uint8_t *, IV_OFFSET); memcpy(iv_ptr, session->cipher_iv_data, cipher_xform->cipher.iv.length); - } else if (cipher_xform->cipher.iv.length != 0) { - *rc_cipher = ODP_CRYPTO_ALG_ERR_IV_INVALID; + } else { + ODP_ASSERT(cipher_xform->cipher.iv.length == 0); } if (param->auth_iv_ptr) { - uint8_t *iv_ptr; - iv_ptr = rte_crypto_op_ctod_offset(op, uint8_t *, IV_OFFSET + MAX_IV_LENGTH); memcpy(iv_ptr, param->auth_iv_ptr, auth_xform->auth.iv.length); } else if (session->p.auth_iv.data) { - uint8_t *iv_ptr; - iv_ptr = rte_crypto_op_ctod_offset(op, uint8_t *, IV_OFFSET + MAX_IV_LENGTH); memcpy(iv_ptr, session->auth_iv_data, auth_xform->auth.iv.length); - } else if (auth_xform->auth.iv.length != 0) { - *rc_auth = ODP_CRYPTO_ALG_ERR_IV_INVALID; + } else { + ODP_ASSERT(auth_xform->auth.iv.length == 0); } +#else + ODP_ASSERT(cipher_xform->cipher.iv.length == 0 || param->cipher_iv_ptr != NULL); + ODP_ASSERT(auth_xform->auth.iv.length == 0 || param->auth_iv_ptr != NULL); + iv_ptr = rte_crypto_op_ctod_offset(op, uint8_t *, IV_OFFSET); + memcpy(iv_ptr, param->cipher_iv_ptr, cipher_xform->cipher.iv.length); + + if (odp_unlikely(auth_xform->auth.iv.length > 0)) { + iv_ptr = rte_crypto_op_ctod_offset(op, uint8_t *, IV_OFFSET + MAX_IV_LENGTH); + memcpy(iv_ptr, param->auth_iv_ptr, auth_xform->auth.iv.length); + } +#endif op->sym->cipher.data.offset = param->cipher_range.offset; op->sym->cipher.data.length = param->cipher_range.length; @@ -1986,11 +1997,9 @@ int odp_crypto_int(odp_packet_t pkt_in, goto out; if (cipher_is_aead(session->p.cipher_alg)) - crypto_fill_aead_param(session, out_pkt, param, op, - &rc_cipher, &rc_auth); + crypto_fill_aead_param(session, out_pkt, param, op); else - crypto_fill_sym_param(session, out_pkt, param, op, - &rc_cipher, &rc_auth); + crypto_fill_sym_param(session, out_pkt, param, op); if (odp_likely(rc_cipher == ODP_CRYPTO_ALG_ERR_NONE && rc_auth == ODP_CRYPTO_ALG_ERR_NONE)) { @@ -2073,7 +2082,8 @@ int odp_crypto_int(odp_packet_t pkt_in, result_ok = false; } - if (session->p.auth_digest_len != 0 && + if (session->p.op == ODP_CRYPTO_OP_ENCODE && + session->p.auth_digest_len != 0 && op->status == RTE_CRYPTO_OP_STATUS_SUCCESS) { odp_packet_hdr_t *pkt_hdr = packet_hdr(out_pkt); diff --git a/platform/linux-dpdk/odp_packet.c b/platform/linux-dpdk/odp_packet.c index 339d2d8fc..88ba446f8 100644 --- a/platform/linux-dpdk/odp_packet.c +++ b/platform/linux-dpdk/odp_packet.c @@ -44,7 +44,7 @@ const _odp_packet_inline_offset_t _odp_packet_inline ODP_ALIGNED_CACHE = { .mb = offsetof(odp_packet_hdr_t, event_hdr.mb), .pool = offsetof(odp_packet_hdr_t, event_hdr.pool_ptr), .input = offsetof(odp_packet_hdr_t, input), - .user_ptr = offsetof(odp_packet_hdr_t, event_hdr.user_ptr), + .user_ptr = offsetof(odp_packet_hdr_t, user_ptr), .l2_offset = offsetof(odp_packet_hdr_t, p.l2_offset), .l3_offset = offsetof(odp_packet_hdr_t, p.l3_offset), .l4_offset = offsetof(odp_packet_hdr_t, p.l4_offset), @@ -57,7 +57,7 @@ const _odp_packet_inline_offset_t _odp_packet_inline ODP_ALIGNED_CACHE = { .pkt_len = offsetof(odp_packet_hdr_t, event_hdr.mb.pkt_len), .seg_len = offsetof(odp_packet_hdr_t, event_hdr.mb.data_len), .nb_segs = offsetof(odp_packet_hdr_t, event_hdr.mb.nb_segs), - .user_area = offsetof(odp_packet_hdr_t, event_hdr.uarea_addr), + .user_area = offsetof(odp_packet_hdr_t, uarea_addr), .rss = offsetof(odp_packet_hdr_t, event_hdr.mb.hash.rss), .ol_flags = offsetof(odp_packet_hdr_t, event_hdr.mb.ol_flags), .rss_flag = PKT_RX_RSS_HASH @@ -593,7 +593,7 @@ void odp_packet_user_ptr_set(odp_packet_t pkt, const void *ptr) return; } - pkt_hdr->event_hdr.user_ptr = ptr; + pkt_hdr->user_ptr = ptr; pkt_hdr->p.flags.user_ptr_set = 1; } @@ -1227,8 +1227,8 @@ int _odp_packet_copy_md_to_packet(odp_packet_t srcpkt, odp_packet_t dstpkt) dsthdr->dst_queue = srchdr->dst_queue; dsthdr->cos = srchdr->cos; dsthdr->cls_mark = srchdr->cls_mark; + dsthdr->user_ptr = srchdr->user_ptr; - dsthdr->event_hdr.user_ptr = srchdr->event_hdr.user_ptr; dsthdr->event_hdr.mb.port = srchdr->event_hdr.mb.port; dsthdr->event_hdr.mb.ol_flags = srchdr->event_hdr.mb.ol_flags; dsthdr->event_hdr.mb.packet_type = srchdr->event_hdr.mb.packet_type; diff --git a/platform/linux-dpdk/odp_pool.c b/platform/linux-dpdk/odp_pool.c index 318e0071a..dfe7c1737 100644 --- a/platform/linux-dpdk/odp_pool.c +++ b/platform/linux-dpdk/odp_pool.c @@ -1,5 +1,5 @@ /* Copyright (c) 2013-2018, Linaro Limited - * Copyright (c) 2019-2021, Nokia + * Copyright (c) 2019-2022, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -59,6 +59,9 @@ /* Define a practical limit for contiguous memory allocations */ #define MAX_SIZE (10 * 1024 * 1024) +/* Maximum packet user area size */ +#define MAX_UAREA_SIZE 2048 + /* The pool table ptr - resides in shared memory */ pool_global_t *_odp_pool_glb; @@ -269,7 +272,7 @@ int odp_pool_capability(odp_pool_capability_t *capa) capa->pkt.max_segs_per_pkt = PKT_MAX_SEGS; capa->pkt.min_seg_len = CONFIG_PACKET_SEG_LEN_MIN; capa->pkt.max_seg_len = CONFIG_PACKET_MAX_SEG_LEN; - capa->pkt.max_uarea_size = MAX_SIZE; + capa->pkt.max_uarea_size = MAX_UAREA_SIZE; capa->pkt.min_cache_size = 0; capa->pkt.max_cache_size = RTE_MEMPOOL_CACHE_MAX_SIZE; capa->pkt.stats.all = supported_stats.all; @@ -362,8 +365,14 @@ odp_dpdk_mbuf_ctor(struct rte_mempool *mp, event_hdr->pool_ptr = mb_ctor_arg->pool; event_hdr->type = mb_ctor_arg->type; event_hdr->event_type = mb_ctor_arg->event_type; - event_hdr->uarea_addr = mb_ctor_arg->pool->uarea_base_addr + - i * mb_ctor_arg->pool->uarea_size; + + /* Initialize packet metadata */ + if (mb_ctor_arg->type == ODP_POOL_PACKET) { + odp_packet_hdr_t *pkt_hdr = (odp_packet_hdr_t *)raw_mbuf; + + pkt_hdr->uarea_addr = mb_ctor_arg->pool->uarea_base_addr + + i * mb_ctor_arg->pool->uarea_size; + } /* Initialize event vector metadata */ if (mb_ctor_arg->type == ODP_POOL_VECTOR) { @@ -1152,7 +1161,7 @@ int odp_pool_ext_capability(odp_pool_type_t type, capa->pkt.max_headroom = RTE_PKTMBUF_HEADROOM; capa->pkt.max_headroom_size = RTE_PKTMBUF_HEADROOM; capa->pkt.max_segs_per_pkt = PKT_MAX_SEGS; - capa->pkt.max_uarea_size = MAX_SIZE; + capa->pkt.max_uarea_size = MAX_UAREA_SIZE; return 0; } diff --git a/platform/linux-dpdk/odp_shared_memory.c b/platform/linux-dpdk/odp_shared_memory.c index c18c9acca..f3a217341 100644 --- a/platform/linux-dpdk/odp_shared_memory.c +++ b/platform/linux-dpdk/odp_shared_memory.c @@ -1,5 +1,5 @@ /* Copyright (c) 2017-2018, Linaro Limited - * Copyright (c) 2021, Nokia + * Copyright (c) 2021-2022, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -8,6 +8,7 @@ #include <odp_posix_extensions.h> #include <odp/api/debug.h> +#include <odp/api/deprecated.h> #include <odp/api/plat/strong_types.h> #include <odp/api/shared_memory.h> #include <odp/api/spinlock.h> @@ -28,7 +29,13 @@ #include <rte_memzone.h> /* Supported ODP_SHM_* flags */ -#define SUPPORTED_SHM_FLAGS (ODP_SHM_SW_ONLY | ODP_SHM_EXPORT | ODP_SHM_HP) +#if ODP_DEPRECATED_API + #define DEPRECATED_SHM_FLAGS (ODP_SHM_SW_ONLY) +#else + #define DEPRECATED_SHM_FLAGS 0 +#endif + +#define SUPPORTED_SHM_FLAGS (ODP_SHM_EXPORT | ODP_SHM_HP | DEPRECATED_SHM_FLAGS) #define SHM_MAX_ALIGN (0x80000000) #define SHM_BLOCK_NAME "%" PRIu64 "-%d-%s" diff --git a/platform/linux-dpdk/odp_system_info.c b/platform/linux-dpdk/odp_system_info.c index 1529cd8ea..c506dbc1e 100644 --- a/platform/linux-dpdk/odp_system_info.c +++ b/platform/linux-dpdk/odp_system_info.c @@ -398,7 +398,14 @@ uint64_t odp_sys_huge_page_size(void) static int pagesz_compare(const void *pagesz1, const void *pagesz2) { - return (*(const uint64_t *)pagesz1 - *(const uint64_t *)pagesz2); + const uint64_t val1 = *(const uint64_t *)pagesz1; + const uint64_t val2 = *(const uint64_t *)pagesz2; + + if (val1 < val2) + return -1; + if (val1 > val2) + return 1; + return 0; } int odp_sys_huge_page_size_all(uint64_t size[], int num) diff --git a/platform/linux-generic/Makefile.am b/platform/linux-generic/Makefile.am index 64245801e..07f1f462b 100644 --- a/platform/linux-generic/Makefile.am +++ b/platform/linux-generic/Makefile.am @@ -321,7 +321,7 @@ noinst_HEADERS += arch/aarch64/odp_atomic.h \ arch/aarch64/cpu_flags.h \ arch/aarch64/odp_cpu_idling.h \ arch/aarch64/odp_llsc.h \ - arch/default/odp_random.h + arch/aarch64/odp_random.h endif if ARCH_IS_DEFAULT __LIB__libodp_linux_la_SOURCES += arch/default/odp_atomic.c \ @@ -344,26 +344,6 @@ noinst_HEADERS += arch/default/odp_atomic.h \ arch/default/odp_cpu_idling.h \ arch/default/odp_random.h endif -if ARCH_IS_MIPS64 -__LIB__libodp_linux_la_SOURCES += arch/default/odp_atomic.c \ - arch/default/odp_global_time.c \ - arch/default/odp_hash_crc32.c \ - arch/default/odp_random.c \ - arch/mips64/odp_sysinfo_parse.c -odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_time.h \ - arch/default/odp/api/abi/hash_crc32.h -if !ODP_ABI_COMPAT -odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ - arch/default/odp/api/abi/atomic_inlines.h \ - arch/default/odp/api/abi/cpu_generic.h \ - arch/mips64/odp/api/abi/cpu_inlines.h \ - arch/mips64/odp/api/abi/cpu.h -endif -noinst_HEADERS += arch/default/odp_atomic.h \ - arch/default/odp_cpu.h \ - arch/default/odp_cpu_idling.h \ - arch/default/odp_random.h -endif if ARCH_IS_POWERPC __LIB__libodp_linux_la_SOURCES += arch/default/odp_atomic.c \ arch/default/odp_cpu_cycles.c \ diff --git a/platform/linux-generic/arch/aarch64/odp_random.h b/platform/linux-generic/arch/aarch64/odp_random.h new file mode 100644 index 000000000..023e6c455 --- /dev/null +++ b/platform/linux-generic/arch/aarch64/odp_random.h @@ -0,0 +1,166 @@ +/* Copyright (c) 2021, ARM Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_AARCH64_RANDOM_H_ +#define ODP_AARCH64_RANDOM_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <odp/api/spec/random.h> +#include <odp/autoheader_internal.h> + +#include <stdint.h> + +odp_random_kind_t _odp_random_max_kind_generic(void); +int32_t _odp_random_true_data_generic(uint8_t *buf, uint32_t len); +int32_t _odp_random_crypto_data_generic(uint8_t *buf, uint32_t len); + +#ifdef __ARM_FEATURE_RNG + +#if __ARM_FEATURE_UNALIGNED != 1 +#error This implementation requires unaligned access +#endif + +static inline int _odp_random_max_kind(void) +{ + return ODP_RANDOM_TRUE; +} + +static inline int _odp_rndr(uint64_t *v) +{ + int pass; + + /* Return a 64-bit random number which is reseeded from the True Random + * Number source. If the hardware returns a genuine random number, + * PSTATE.NZCV is set to 0b0000. The NZCV condition flag is checked via + * the CSET instruction. If the hardware cannot return a genuine random + * number in a reasonable period of time, PSTATE.NZCV is set to 0b0100 + * and the data value returned is 0. */ + __asm__ volatile("mrs %0, s3_3_c2_c4_0\n\t" + "cset %w[pass], ne" + : "=r" (*v), [pass] "=r" (pass) + : + : "cc"); + + return pass; +} + +static inline int _odp_rndrrs(uint64_t *v) +{ + int pass; + + /* Return a 64-bit random number which is reseeded from the True Random + * Number source immediately before the read of the random number. + * If the hardware returns a genuine random number, PSTATE.NZCV is + * set to 0b0000. The NZCV condition flag is checked via the CSET + * instruction. If the hardware cannot return a genuine random number + * in a reasonable period of time, PSTATE.NZCV is set to 0b0100 and the + * data value returned is 0. */ + __asm__ volatile("mrs %0, s3_3_c2_c4_1\n\t" + "cset %w[pass], ne" + : "=r" (*v), [pass] "=r" (pass) + : + : "cc"); + + return pass; +} + +static inline int32_t _odp_random_crypto_data(uint8_t *buf, uint32_t len) +{ + uint64_t temp; + + for (uint32_t i = 0; i < len / 8; i++) { + while (!_odp_rndr(&temp)) + ; + + *(uint64_t *)(uintptr_t)buf = temp; + buf += 8; + } + + if (len & 7) { + while (!_odp_rndr(&temp)) + ; + + if (len & 4) { + *(uint32_t *)(uintptr_t)buf = temp & 0xffffffff; + temp >>= 32; + buf += 4; + } + + if (len & 2) { + *(uint16_t *)(uintptr_t)buf = temp & 0xffff; + temp >>= 16; + buf += 2; + } + + if (len & 1) + *buf = temp & 0xff; + } + + return len; +} + +static inline int32_t _odp_random_true_data(uint8_t *buf, uint32_t len) +{ + uint64_t temp; + + for (uint32_t i = 0; i < len / 8; i++) { + while (!_odp_rndrrs(&temp)) + ; + + *(uint64_t *)(uintptr_t)buf = temp; + buf += 8; + } + + if (len & 7) { + while (!_odp_rndrrs(&temp)) + ; + + if (len & 4) { + *(uint32_t *)(uintptr_t)buf = temp & 0xffffffff; + temp >>= 32; + buf += 4; + } + + if (len & 2) { + *(uint16_t *)(uintptr_t)buf = temp & 0xffff; + temp >>= 16; + buf += 2; + } + + if (len & 1) + *buf = temp & 0xff; + } + + return len; +} + +#else + +static inline int _odp_random_max_kind(void) +{ + return _odp_random_max_kind_generic(); +} + +static inline int32_t _odp_random_crypto_data(uint8_t *buf, uint32_t len) +{ + return _odp_random_crypto_data_generic(buf, len); +} + +static inline int32_t _odp_random_true_data(uint8_t *buf, uint32_t len) +{ + return _odp_random_true_data_generic(buf, len); +} + +#endif + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/arch/mips64/odp/api/abi/cpu.h b/platform/linux-generic/arch/mips64/odp/api/abi/cpu.h deleted file mode 100644 index a6ce523d0..000000000 --- a/platform/linux-generic/arch/mips64/odp/api/abi/cpu.h +++ /dev/null @@ -1,27 +0,0 @@ -/* Copyright (c) 2016-2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -#ifndef ODP_API_ABI_CPU_H_ -#define ODP_API_ABI_CPU_H_ - -#ifdef __cplusplus -extern "C" { -#endif - -#if defined __OCTEON__ -#define ODP_CACHE_LINE_SIZE 128 -#else -#error Please add support for your arch in cpu_arch.h -#endif - -/* Inlined functions for non-ABI compat mode */ -#include <odp/api/plat/cpu_inlines.h> - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/platform/linux-generic/arch/mips64/odp/api/abi/cpu_inlines.h b/platform/linux-generic/arch/mips64/odp/api/abi/cpu_inlines.h deleted file mode 100644 index d3a424432..000000000 --- a/platform/linux-generic/arch/mips64/odp/api/abi/cpu_inlines.h +++ /dev/null @@ -1,56 +0,0 @@ -/* Copyright (c) 2016-2018, Linaro Limited - * Copyright (c) 2021, Nokia - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -#ifndef ODP_ARCH_CPU_INLINES_H_ -#define ODP_ARCH_CPU_INLINES_H_ - -#ifdef __cplusplus -extern "C" { -#endif - -#include <stdint.h> - -static inline void _odp_cpu_pause(void) -{ - __asm__ __volatile__ ("nop"); - __asm__ __volatile__ ("nop"); - __asm__ __volatile__ ("nop"); - __asm__ __volatile__ ("nop"); -} - -uint64_t _odp_cpu_cycles(void) -{ - #define CVMX_TMP_STR(x) CVMX_TMP_STR2(x) - #define CVMX_TMP_STR2(x) #x - uint64_t cycle; - - __asm__ __volatile__ ("rdhwr %[rt],$" CVMX_TMP_STR(31) : - [rt] "=d" (cycle) : : "memory"); - - return cycle; -} - -uint64_t _odp_cpu_cycles_max(void) -{ - return UINT64_MAX; -} - -uint64_t _odp_cpu_cycles_resolution(void) -{ - return 1; -} - -int _odp_cpu_cycles_init_global(void) -{ - return 0; -} - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/platform/linux-generic/arch/mips64/odp_sysinfo_parse.c b/platform/linux-generic/arch/mips64/odp_sysinfo_parse.c deleted file mode 100644 index 897637516..000000000 --- a/platform/linux-generic/arch/mips64/odp_sysinfo_parse.c +++ /dev/null @@ -1,73 +0,0 @@ -/* Copyright (c) 2016-2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -#include <odp_global_data.h> -#include <odp_sysinfo_internal.h> -#include <string.h> - -int _odp_cpuinfo_parser(FILE *file, system_info_t *sysinfo) -{ - char str[1024]; - char *pos; - double mhz = 0.0; - uint64_t hz; - int model = 0; - int count = 2; - int id = 0; - - sysinfo->cpu_arch = ODP_CPU_ARCH_MIPS; - sysinfo->cpu_isa_sw.mips = ODP_CPU_ARCH_MIPS_UNKNOWN; - sysinfo->cpu_isa_hw.mips = ODP_CPU_ARCH_MIPS_UNKNOWN; - - strcpy(sysinfo->cpu_arch_str, "mips64"); - while (fgets(str, sizeof(str), file) != NULL && id < CONFIG_NUM_CPU_IDS) { - if (!mhz) { - pos = strstr(str, "BogoMIPS"); - - if (pos) - if (sscanf(pos, "BogoMIPS : %lf", &mhz) == 1) { - /* bogomips seems to be 2x freq */ - hz = (uint64_t)(mhz * 1000000.0 / 2.0); - sysinfo->cpu_hz_max[id] = hz; - count--; - } - } - - if (!model) { - pos = strstr(str, "cpu model"); - - if (pos) { - int len; - - pos = strchr(str, ':'); - strncpy(sysinfo->model_str[id], pos + 2, - MODEL_STR_SIZE - 1); - len = strlen(sysinfo->model_str[id]); - sysinfo->model_str[id][len - 1] = 0; - model = 1; - count--; - } - } - - if (count == 0) { - mhz = 0.0; - model = 0; - count = 2; - id++; - } - } - - return 0; -} - -void _odp_sys_info_print_arch(void) -{ -} - -uint64_t odp_cpu_arch_hz_current(int id ODP_UNUSED) -{ - return odp_global_ro.system_info.default_cpu_hz; -} diff --git a/platform/linux-generic/include/odp_event_internal.h b/platform/linux-generic/include/odp_event_internal.h index fa7e5f354..92f201b01 100644 --- a/platform/linux-generic/include/odp_event_internal.h +++ b/platform/linux-generic/include/odp_event_internal.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2021, Nokia +/* Copyright (c) 2021-2022, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -17,21 +17,21 @@ extern "C" { #endif -#include <odp/api/atomic.h> #include <odp/api/debug.h> #include <odp/api/event.h> #include <odp/api/std_types.h> #include <odp_config_internal.h> -typedef union buffer_index_t { +/* Combined pool and event index */ +typedef union _odp_event_index_t { uint32_t u32; struct { uint32_t pool :8; - uint32_t buffer :24; + uint32_t event :24; }; -} buffer_index_t; +} _odp_event_index_t; /* Check that pool index fit into bit field */ ODP_STATIC_ASSERT(ODP_CONFIG_POOLS <= (0xFF + 1), "TOO_MANY_POOLS"); @@ -51,19 +51,12 @@ typedef struct _odp_event_hdr_t { void *pool_ptr; /* --- Mostly read only data --- */ - const void *user_ptr; /* Initial buffer tail pointer */ uint8_t *buf_end; - /* User area pointer */ - void *uarea_addr; - - /* Combined pool and buffer index */ - buffer_index_t index; - - /* Reference count */ - odp_atomic_u32_t ref_cnt; + /* Combined pool and event index */ + _odp_event_index_t index; /* Pool type */ int8_t type; diff --git a/platform/linux-generic/include/odp_packet_internal.h b/platform/linux-generic/include/odp_packet_internal.h index 7c9b7735e..a230c3b76 100644 --- a/platform/linux-generic/include/odp_packet_internal.h +++ b/platform/linux-generic/include/odp_packet_internal.h @@ -1,5 +1,5 @@ /* Copyright (c) 2013-2018, Linaro Limited - * Copyright (c) 2019-2021, Nokia + * Copyright (c) 2019-2022, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -19,6 +19,7 @@ extern "C" { #endif #include <odp/api/align.h> +#include <odp/api/atomic.h> #include <odp/api/debug.h> #include <odp/api/packet.h> #include <odp/api/plat/packet_inline_types.h> @@ -117,9 +118,18 @@ typedef struct ODP_ALIGNED_CACHE odp_packet_hdr_t { /* Classifier destination queue */ odp_queue_t dst_queue; + /* Reference count */ + odp_atomic_u32_t ref_cnt; + /* Flow hash value */ uint32_t flow_hash; + /* User area pointer */ + void *uarea_addr; + + /* User context pointer */ + const void *user_ptr; + /* Classifier mark */ uint16_t cls_mark; diff --git a/platform/linux-generic/include/odp_pool_internal.h b/platform/linux-generic/include/odp_pool_internal.h index c9bae7142..4c9f9a9ce 100644 --- a/platform/linux-generic/include/odp_pool_internal.h +++ b/platform/linux-generic/include/odp_pool_internal.h @@ -1,5 +1,5 @@ /* Copyright (c) 2013-2018, Linaro Limited - * Copyright (c) 2019-2021, Nokia + * Copyright (c) 2019-2022, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -159,16 +159,16 @@ static inline _odp_event_hdr_t *event_hdr_from_index(pool_t *pool, static inline _odp_event_hdr_t *_odp_event_hdr_from_index_u32(uint32_t u32) { - buffer_index_t index; - uint32_t pool_idx, buffer_idx; + _odp_event_index_t index; + uint32_t pool_idx, event_idx; pool_t *pool; index.u32 = u32; pool_idx = index.pool; - buffer_idx = index.buffer; + event_idx = index.event; pool = pool_entry(pool_idx); - return event_hdr_from_index(pool, buffer_idx); + return event_hdr_from_index(pool, event_idx); } odp_event_t _odp_event_alloc(pool_t *pool); diff --git a/platform/linux-generic/include/odp_timer_internal.h b/platform/linux-generic/include/odp_timer_internal.h index 2a7173d29..b83591446 100644 --- a/platform/linux-generic/include/odp_timer_internal.h +++ b/platform/linux-generic/include/odp_timer_internal.h @@ -1,5 +1,5 @@ /* Copyright (c) 2014-2018, Linaro Limited - * Copyright (c) 2021, Nokia + * Copyright (c) 2021-2022, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -40,6 +40,9 @@ typedef struct ODP_ALIGNED_CACHE odp_timeout_hdr_t { } odp_timeout_hdr_t; +ODP_STATIC_ASSERT(sizeof(odp_timeout_hdr_t) <= ODP_CACHE_LINE_SIZE, + "TIMEOUT_HDR_SIZE_ERROR"); + /* A larger decrement value should be used after receiving events compared to * an 'empty' call. */ void _odp_timer_run_inline(int dec); diff --git a/platform/linux-generic/include/odp_traffic_mngr_internal.h b/platform/linux-generic/include/odp_traffic_mngr_internal.h index a54847319..2b5e47637 100644 --- a/platform/linux-generic/include/odp_traffic_mngr_internal.h +++ b/platform/linux-generic/include/odp_traffic_mngr_internal.h @@ -286,7 +286,6 @@ struct tm_queue_obj_s { uint8_t blocked_cnt; odp_bool_t ordered_enqueue; tm_status_t status; - odp_queue_t queue; /* Statistics for odp_tm_queue_stats_t */ struct { odp_atomic_u64_t discards; diff --git a/platform/linux-generic/odp_buffer.c b/platform/linux-generic/odp_buffer.c index 0e606017f..f9f67a2e4 100644 --- a/platform/linux-generic/odp_buffer.c +++ b/platform/linux-generic/odp_buffer.c @@ -1,5 +1,5 @@ /* Copyright (c) 2013-2018, Linaro Limited - * Copyright (c) 2019, Nokia + * Copyright (c) 2019-2022, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -51,7 +51,7 @@ void odp_buffer_print(odp_buffer_t buf) len += snprintf(&str[len], n - len, "Buffer\n------\n"); len += snprintf(&str[len], n - len, " pool index %u\n", hdr->event_hdr.index.pool); - len += snprintf(&str[len], n - len, " buffer index %u\n", hdr->event_hdr.index.buffer); + len += snprintf(&str[len], n - len, " buffer index %u\n", hdr->event_hdr.index.event); len += snprintf(&str[len], n - len, " addr %p\n", (void *)hdr->event_hdr.base_data); len += snprintf(&str[len], n - len, " size %u\n", odp_buffer_size(buf)); diff --git a/platform/linux-generic/odp_crypto_openssl.c b/platform/linux-generic/odp_crypto_openssl.c index 6e702acab..19b57f502 100644 --- a/platform/linux-generic/odp_crypto_openssl.c +++ b/platform/linux-generic/odp_crypto_openssl.c @@ -1,5 +1,5 @@ /* Copyright (c) 2014-2018, Linaro Limited - * Copyright (c) 2021, Nokia + * Copyright (c) 2021-2022, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -212,8 +212,10 @@ struct odp_crypto_generic_session_t { odp_bool_t do_cipher_first; struct { +#if ODP_DEPRECATED_API /* Copy of session IV data */ uint8_t iv_data[EVP_MAX_IV_LENGTH]; +#endif uint8_t key_data[EVP_MAX_KEY_LENGTH]; const EVP_CIPHER *evp_cipher; @@ -223,7 +225,9 @@ struct odp_crypto_generic_session_t { struct { uint8_t key[EVP_MAX_KEY_LENGTH]; +#if ODP_DEPRECATED_API uint8_t iv_data[EVP_MAX_IV_LENGTH]; +#endif union { const EVP_MD *evp_md; const EVP_CIPHER *evp_cipher; @@ -509,8 +513,8 @@ auth_xcbcmac_check(odp_packet_t pkt, odp_packet_copy_to_mem(pkt, param->hash_result_offset, bytes, hash_in); - _odp_packet_set_data(pkt, param->hash_result_offset, - 0, bytes); + if (odp_unlikely(session->p.hash_result_in_auth_range)) + _odp_packet_set_data(pkt, param->hash_result_offset, 0, bytes); /* Hash it */ packet_aes_xcbc_mac(pkt, param, session, hash_out); @@ -591,8 +595,8 @@ odp_crypto_alg_err_t auth_hmac_check(odp_packet_t pkt, odp_packet_copy_to_mem(pkt, param->hash_result_offset, bytes, hash_in); - _odp_packet_set_data(pkt, param->hash_result_offset, - 0, bytes); + if (odp_unlikely(session->p.hash_result_in_auth_range)) + _odp_packet_set_data(pkt, param->hash_result_offset, 0, bytes); /* Hash it */ packet_hmac(pkt, param, session, hash_out); @@ -678,8 +682,8 @@ odp_crypto_alg_err_t auth_cmac_check(odp_packet_t pkt, odp_packet_copy_to_mem(pkt, param->hash_result_offset, bytes, hash_in); - _odp_packet_set_data(pkt, param->hash_result_offset, - 0, bytes); + if (odp_unlikely(session->p.hash_result_in_auth_range)) + _odp_packet_set_data(pkt, param->hash_result_offset, 0, bytes); /* Hash it */ packet_cmac(pkt, param, session, hash_out); @@ -704,19 +708,24 @@ int packet_cmac_eia2(odp_packet_t pkt, uint32_t len = (param->auth_range.length + 7) / 8; size_t outlen; +#if ODP_DEPRECATED_API if (param->auth_iv_ptr) iv_ptr = param->auth_iv_ptr; else if (session->p.auth_iv.data) iv_ptr = session->auth.iv_data; else return ODP_CRYPTO_ALG_ERR_IV_INVALID; +#else + iv_ptr = param->auth_iv_ptr; + ODP_ASSERT(session->p.auth_iv_len == 0 || iv_ptr != NULL); +#endif ODP_ASSERT(offset + len <= odp_packet_len(pkt)); /* Reinitialize CMAC calculation without resetting the key */ CMAC_Init(ctx, NULL, 0, NULL, NULL); - CMAC_Update(ctx, iv_ptr, session->p.auth_iv.length); + CMAC_Update(ctx, iv_ptr, session->p.auth_iv_len); while (len > 0) { uint32_t seglen = 0; /* GCC */ @@ -772,8 +781,8 @@ odp_crypto_alg_err_t auth_cmac_eia2_check(odp_packet_t pkt, odp_packet_copy_to_mem(pkt, param->hash_result_offset, bytes, hash_in); - _odp_packet_set_data(pkt, param->hash_result_offset, - 0, bytes); + if (odp_unlikely(session->p.hash_result_in_auth_range)) + _odp_packet_set_data(pkt, param->hash_result_offset, 0, bytes); /* Hash it */ ret = packet_cmac_eia2(pkt, param, session, hash_out); @@ -851,8 +860,8 @@ odp_crypto_alg_err_t auth_digest_check(odp_packet_t pkt, odp_packet_copy_to_mem(pkt, param->hash_result_offset, bytes, hash_in); - _odp_packet_set_data(pkt, param->hash_result_offset, - 0, bytes); + if (odp_unlikely(session->p.hash_result_in_auth_range)) + _odp_packet_set_data(pkt, param->hash_result_offset, 0, bytes); /* Hash it */ packet_digest(pkt, param, session, hash_out); @@ -1060,12 +1069,17 @@ odp_crypto_alg_err_t cipher_encrypt(odp_packet_t pkt, void *iv_ptr; int ret; +#if ODP_DEPRECATED_API if (param->cipher_iv_ptr) iv_ptr = param->cipher_iv_ptr; else if (session->p.cipher_iv.data) iv_ptr = session->cipher.iv_data; else return ODP_CRYPTO_ALG_ERR_IV_INVALID; +#else + iv_ptr = param->cipher_iv_ptr; + ODP_ASSERT(session->p.cipher_iv_len == 0 || iv_ptr != NULL); +#endif EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv_ptr); @@ -1094,12 +1108,17 @@ odp_crypto_alg_err_t cipher_decrypt(odp_packet_t pkt, void *iv_ptr; int ret; +#if ODP_DEPRECATED_API if (param->cipher_iv_ptr) iv_ptr = param->cipher_iv_ptr; else if (session->p.cipher_iv.data) iv_ptr = session->cipher.iv_data; else return ODP_CRYPTO_ALG_ERR_IV_INVALID; +#else + iv_ptr = param->cipher_iv_ptr; + ODP_ASSERT(session->p.cipher_iv_len == 0 || iv_ptr != NULL); +#endif EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv_ptr); @@ -1119,7 +1138,7 @@ static int process_cipher_param(odp_crypto_generic_session_t *session, /* Verify IV len is correct */ if ((uint32_t)EVP_CIPHER_iv_length(cipher) != - session->p.cipher_iv.length) + session->p.cipher_iv_len) return -1; session->cipher.evp_cipher = cipher; @@ -1157,13 +1176,17 @@ odp_crypto_alg_err_t cipher_encrypt_bits(odp_packet_t pkt, /* Range offset is in bits in bit mode but must be divisible by 8. */ offset = param->cipher_range.offset / 8; +#if ODP_DEPRECATED_API if (param->cipher_iv_ptr) iv_ptr = param->cipher_iv_ptr; else if (session->p.cipher_iv.data) iv_ptr = session->cipher.iv_data; else return ODP_CRYPTO_ALG_ERR_IV_INVALID; - +#else + iv_ptr = param->cipher_iv_ptr; + ODP_ASSERT(session->p.cipher_iv_len == 0 || iv_ptr != NULL); +#endif EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv_ptr); odp_packet_copy_to_mem(pkt, offset, in_len, data); @@ -1197,13 +1220,17 @@ odp_crypto_alg_err_t cipher_decrypt_bits(odp_packet_t pkt, /* Range offset is in bits in bit mode but must be divisible by 8. */ offset = param->cipher_range.offset / 8; +#if ODP_DEPRECATED_API if (param->cipher_iv_ptr) iv_ptr = param->cipher_iv_ptr; else if (session->p.cipher_iv.data) iv_ptr = session->cipher.iv_data; else return ODP_CRYPTO_ALG_ERR_IV_INVALID; - +#else + iv_ptr = param->cipher_iv_ptr; + ODP_ASSERT(session->p.cipher_iv_len == 0 || iv_ptr != NULL); +#endif EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv_ptr); odp_packet_copy_to_mem(pkt, offset, in_len, data); @@ -1229,7 +1256,7 @@ static int process_cipher_param_bits(odp_crypto_generic_session_t *session, /* Verify IV len is correct */ if ((uint32_t)EVP_CIPHER_iv_length(cipher) != - session->p.cipher_iv.length) + session->p.cipher_iv_len) return -1; session->cipher.evp_cipher = cipher; @@ -1257,7 +1284,7 @@ aes_gcm_encrypt_init(odp_crypto_generic_session_t *session) EVP_EncryptInit_ex(ctx, session->cipher.evp_cipher, NULL, session->cipher.key_data, NULL); EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_IVLEN, - session->p.cipher_iv.length, NULL); + session->p.cipher_iv_len, NULL); EVP_CIPHER_CTX_set_padding(ctx, 0); } @@ -1274,12 +1301,17 @@ odp_crypto_alg_err_t aes_gcm_encrypt(odp_packet_t pkt, uint8_t block[EVP_MAX_MD_SIZE]; int ret; +#if ODP_DEPRECATED_API if (param->cipher_iv_ptr) iv_ptr = param->cipher_iv_ptr; else if (session->p.cipher_iv.data) iv_ptr = session->cipher.iv_data; else return ODP_CRYPTO_ALG_ERR_IV_INVALID; +#else + iv_ptr = param->cipher_iv_ptr; + ODP_ASSERT(session->p.cipher_iv_len == 0 || iv_ptr != NULL); +#endif EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv_ptr); @@ -1307,7 +1339,7 @@ aes_gcm_decrypt_init(odp_crypto_generic_session_t *session) EVP_DecryptInit_ex(ctx, session->cipher.evp_cipher, NULL, session->cipher.key_data, NULL); EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_IVLEN, - session->p.cipher_iv.length, NULL); + session->p.cipher_iv_len, NULL); EVP_CIPHER_CTX_set_padding(ctx, 0); } @@ -1324,12 +1356,17 @@ odp_crypto_alg_err_t aes_gcm_decrypt(odp_packet_t pkt, uint8_t block[EVP_MAX_MD_SIZE]; int ret; +#if ODP_DEPRECATED_API if (param->cipher_iv_ptr) iv_ptr = param->cipher_iv_ptr; else if (session->p.cipher_iv.data) iv_ptr = session->cipher.iv_data; else return ODP_CRYPTO_ALG_ERR_IV_INVALID; +#else + iv_ptr = param->cipher_iv_ptr; + ODP_ASSERT(session->p.cipher_iv_len == 0 || iv_ptr != NULL); +#endif EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv_ptr); @@ -1358,7 +1395,7 @@ static int process_aes_gcm_param(odp_crypto_generic_session_t *session, return -1; /* Verify IV len is correct */ - if (12 != session->p.cipher_iv.length) + if (12 != session->p.cipher_iv_len) return -1; memcpy(session->cipher.key_data, session->p.cipher_key.data, @@ -1386,7 +1423,7 @@ aes_gmac_gen_init(odp_crypto_generic_session_t *session) EVP_EncryptInit_ex(ctx, session->auth.evp_cipher, NULL, session->auth.key, NULL); EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_IVLEN, - session->p.auth_iv.length, NULL); + session->p.auth_iv_len, NULL); EVP_CIPHER_CTX_set_padding(ctx, 0); } @@ -1400,12 +1437,17 @@ odp_crypto_alg_err_t aes_gmac_gen(odp_packet_t pkt, uint8_t block[EVP_MAX_MD_SIZE]; int ret; +#if ODP_DEPRECATED_API if (param->auth_iv_ptr) iv_ptr = param->auth_iv_ptr; else if (session->p.auth_iv.data) iv_ptr = session->auth.iv_data; else return ODP_CRYPTO_ALG_ERR_IV_INVALID; +#else + iv_ptr = param->auth_iv_ptr; + ODP_ASSERT(session->p.auth_iv_len == 0 || iv_ptr != NULL); +#endif EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv_ptr); @@ -1428,7 +1470,7 @@ aes_gmac_check_init(odp_crypto_generic_session_t *session) EVP_DecryptInit_ex(ctx, session->auth.evp_cipher, NULL, session->auth.key, NULL); EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_IVLEN, - session->p.auth_iv.length, NULL); + session->p.auth_iv_len, NULL); EVP_CIPHER_CTX_set_padding(ctx, 0); } @@ -1442,12 +1484,17 @@ odp_crypto_alg_err_t aes_gmac_check(odp_packet_t pkt, uint8_t block[EVP_MAX_MD_SIZE]; int ret; +#if ODP_DEPRECATED_API if (param->auth_iv_ptr) iv_ptr = param->auth_iv_ptr; else if (session->p.auth_iv.data) iv_ptr = session->auth.iv_data; else return ODP_CRYPTO_ALG_ERR_IV_INVALID; +#else + iv_ptr = param->auth_iv_ptr; + ODP_ASSERT(session->p.auth_iv_len == 0 || iv_ptr != NULL); +#endif EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv_ptr); @@ -1455,8 +1502,9 @@ odp_crypto_alg_err_t aes_gmac_check(odp_packet_t pkt, session->p.auth_digest_len, block); EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_TAG, session->p.auth_digest_len, block); - _odp_packet_set_data(pkt, param->hash_result_offset, - 0, session->p.auth_digest_len); + if (odp_unlikely(session->p.hash_result_in_auth_range)) + _odp_packet_set_data(pkt, param->hash_result_offset, + 0, session->p.auth_digest_len); ret = internal_aad(ctx, pkt, param, false); @@ -1473,7 +1521,7 @@ static int process_aes_gmac_param(odp_crypto_generic_session_t *session, return -1; /* Verify IV len is correct */ - if (12 != session->p.auth_iv.length) + if (12 != session->p.auth_iv_len) return -1; memcpy(session->auth.key, session->p.auth_key.data, @@ -1501,7 +1549,7 @@ aes_ccm_encrypt_init(odp_crypto_generic_session_t *session) EVP_EncryptInit_ex(ctx, session->cipher.evp_cipher, NULL, NULL, NULL); EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_CCM_SET_IVLEN, - session->p.cipher_iv.length, NULL); + session->p.cipher_iv_len, NULL); EVP_CIPHER_CTX_set_padding(ctx, 0); } @@ -1521,12 +1569,17 @@ odp_crypto_alg_err_t aes_ccm_encrypt(odp_packet_t pkt, uint8_t block[EVP_MAX_MD_SIZE]; int ret; +#if ODP_DEPRECATED_API if (param->cipher_iv_ptr) iv_ptr = param->cipher_iv_ptr; else if (session->p.cipher_iv.data) iv_ptr = session->cipher.iv_data; else return ODP_CRYPTO_ALG_ERR_IV_INVALID; +#else + iv_ptr = param->cipher_iv_ptr; + ODP_ASSERT(session->p.cipher_iv_len == 0 || iv_ptr != NULL); +#endif EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_CCM_SET_TAG, session->p.auth_digest_len, NULL); @@ -1568,7 +1621,7 @@ aes_ccm_decrypt_init(odp_crypto_generic_session_t *session) EVP_DecryptInit_ex(ctx, session->cipher.evp_cipher, NULL, session->cipher.key_data, NULL); EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_IVLEN, - session->p.cipher_iv.length, NULL); + session->p.cipher_iv_len, NULL); EVP_CIPHER_CTX_set_padding(ctx, 0); } @@ -1588,12 +1641,17 @@ odp_crypto_alg_err_t aes_ccm_decrypt(odp_packet_t pkt, uint8_t block[EVP_MAX_MD_SIZE]; int ret; +#if ODP_DEPRECATED_API if (param->cipher_iv_ptr) iv_ptr = param->cipher_iv_ptr; else if (session->p.cipher_iv.data) iv_ptr = session->cipher.iv_data; else return ODP_CRYPTO_ALG_ERR_IV_INVALID; +#else + iv_ptr = param->cipher_iv_ptr; + ODP_ASSERT(session->p.cipher_iv_len == 0 || iv_ptr != NULL); +#endif odp_packet_copy_to_mem(pkt, param->hash_result_offset, session->p.auth_digest_len, block); @@ -1633,8 +1691,8 @@ static int process_aes_ccm_param(odp_crypto_generic_session_t *session, return -1; /* Verify IV len is correct */ - if (11 != session->p.cipher_iv.length && - 13 != session->p.cipher_iv.length) + if (11 != session->p.cipher_iv_len && + 13 != session->p.cipher_iv_len) return -1; memcpy(session->cipher.key_data, session->p.cipher_key.data, @@ -1667,12 +1725,17 @@ odp_crypto_alg_err_t xts_encrypt(odp_packet_t pkt, uint8_t data[in_len]; int ret; +#if ODP_DEPRECATED_API if (param->cipher_iv_ptr) iv_ptr = param->cipher_iv_ptr; else if (session->p.cipher_iv.data) iv_ptr = session->cipher.iv_data; else return ODP_CRYPTO_ALG_ERR_IV_INVALID; +#else + iv_ptr = param->cipher_iv_ptr; + ODP_ASSERT(session->p.cipher_iv_len == 0 || iv_ptr != NULL); +#endif EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv_ptr); @@ -1704,12 +1767,17 @@ odp_crypto_alg_err_t xts_decrypt(odp_packet_t pkt, uint8_t data[in_len]; int ret; +#if ODP_DEPRECATED_API if (param->cipher_iv_ptr) iv_ptr = param->cipher_iv_ptr; else if (session->p.cipher_iv.data) iv_ptr = session->cipher.iv_data; else return ODP_CRYPTO_ALG_ERR_IV_INVALID; +#else + iv_ptr = param->cipher_iv_ptr; + ODP_ASSERT(session->p.cipher_iv_len == 0 || iv_ptr != NULL); +#endif EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv_ptr); @@ -1738,7 +1806,7 @@ static int process_xts_param(odp_crypto_generic_session_t *session, /* Verify IV len is correct */ if ((uint32_t)EVP_CIPHER_iv_length(cipher) != - session->p.cipher_iv.length) + session->p.cipher_iv_len) return -1; session->cipher.evp_cipher = cipher; @@ -1762,7 +1830,7 @@ static int process_auth_hmac_param(odp_crypto_generic_session_t *session, const EVP_MD *evp_md) { /* Verify IV len is correct */ - if (0 != session->p.auth_iv.length) + if (0 != session->p.auth_iv_len) return -1; /* Set function */ @@ -1793,7 +1861,7 @@ static int process_auth_cmac_param(odp_crypto_generic_session_t *session, session->p.auth_key.length) return -1; - if (0 != session->p.auth_iv.length) + if (0 != session->p.auth_iv_len) return -1; /* Set function */ @@ -1826,7 +1894,7 @@ static int process_auth_cmac_eia2_param(odp_crypto_generic_session_t *session, return -1; /* Verify IV len is correct */ - if (8 != session->p.auth_iv.length) + if (8 != session->p.auth_iv_len) return -1; /* Set function */ @@ -2125,7 +2193,7 @@ odp_crypto_session_create(const odp_crypto_session_param_t *param, ODP_ERR("Crypto is disabled\n"); /* Dummy output to avoid compiler warning about uninitialized * variables */ - *status = ODP_CRYPTO_SES_CREATE_ERR_ENOMEM; + *status = ODP_CRYPTO_SES_ERR_ENOMEM; *session_out = ODP_CRYPTO_SESSION_INVALID; return -1; } @@ -2133,25 +2201,26 @@ odp_crypto_session_create(const odp_crypto_session_param_t *param, /* Allocate memory for this session */ session = alloc_session(); if (NULL == session) { - *status = ODP_CRYPTO_SES_CREATE_ERR_ENOMEM; + *status = ODP_CRYPTO_SES_ERR_ENOMEM; goto err; } /* Copy parameters */ session->p = *param; - if (session->p.cipher_iv.length > EVP_MAX_IV_LENGTH) { + if (session->p.cipher_iv_len > EVP_MAX_IV_LENGTH) { ODP_DBG("Maximum IV length exceeded\n"); - *status = ODP_CRYPTO_SES_CREATE_ERR_INV_CIPHER; + *status = ODP_CRYPTO_SES_ERR_CIPHER; goto err; } - if (session->p.auth_iv.length > EVP_MAX_IV_LENGTH) { + if (session->p.auth_iv_len > EVP_MAX_IV_LENGTH) { ODP_DBG("Maximum auth IV length exceeded\n"); - *status = ODP_CRYPTO_SES_CREATE_ERR_INV_CIPHER; + *status = ODP_CRYPTO_SES_ERR_CIPHER; goto err; } +#if ODP_DEPRECATED_API /* Copy IV data */ if (session->p.cipher_iv.data) memcpy(session->cipher.iv_data, session->p.cipher_iv.data, @@ -2160,6 +2229,7 @@ odp_crypto_session_create(const odp_crypto_session_param_t *param, if (session->p.auth_iv.data) memcpy(session->auth.iv_data, session->p.auth_iv.data, session->p.auth_iv.length); +#endif /* Derive order */ if (ODP_CRYPTO_OP_ENCODE == param->op) @@ -2303,7 +2373,7 @@ odp_crypto_session_create(const odp_crypto_session_param_t *param, /* Check result */ if (rc) { - *status = ODP_CRYPTO_SES_CREATE_ERR_INV_CIPHER; + *status = ODP_CRYPTO_SES_ERR_CIPHER; goto err; } @@ -2446,13 +2516,13 @@ odp_crypto_session_create(const odp_crypto_session_param_t *param, /* Check result */ if (rc) { - *status = ODP_CRYPTO_SES_CREATE_ERR_INV_AUTH; + *status = ODP_CRYPTO_SES_ERR_AUTH; goto err; } /* We're happy */ *session_out = (intptr_t)session; - *status = ODP_CRYPTO_SES_CREATE_ERR_NONE; + *status = ODP_CRYPTO_SES_ERR_NONE; return 0; err: diff --git a/platform/linux-generic/odp_ipsec_sad.c b/platform/linux-generic/odp_ipsec_sad.c index 50f2bb360..23da4b2d1 100644 --- a/platform/linux-generic/odp_ipsec_sad.c +++ b/platform/linux-generic/odp_ipsec_sad.c @@ -624,6 +624,8 @@ odp_ipsec_sa_t odp_ipsec_sa_create(const odp_ipsec_sa_param_t *param) ODP_CRYPTO_OP_DECODE : ODP_CRYPTO_OP_ENCODE; crypto_param.auth_cipher_text = 1; + if (param->proto == ODP_IPSEC_AH) + crypto_param.hash_result_in_auth_range = 1; crypto_param.op_mode = ODP_CRYPTO_SYNC; crypto_param.compl_queue = ODP_QUEUE_INVALID; @@ -634,7 +636,7 @@ odp_ipsec_sa_t odp_ipsec_sa_create(const odp_ipsec_sa_param_t *param) crypto_param.auth_alg = param->crypto.auth_alg; crypto_param.auth_key = param->crypto.auth_key; - crypto_param.cipher_iv.length = + crypto_param.cipher_iv_len = _odp_ipsec_cipher_iv_len(crypto_param.cipher_alg); crypto_param.auth_digest_len = @@ -644,7 +646,7 @@ odp_ipsec_sa_t odp_ipsec_sa_create(const odp_ipsec_sa_param_t *param) param->crypto.icv_len != crypto_param.auth_digest_len) goto error; - if ((uint32_t)-1 == crypto_param.cipher_iv.length || + if ((uint32_t)-1 == crypto_param.cipher_iv_len || (uint32_t)-1 == crypto_param.auth_digest_len) goto error; @@ -729,7 +731,7 @@ odp_ipsec_sa_t odp_ipsec_sa_create(const odp_ipsec_sa_param_t *param) ipsec_sa->use_counter_iv = 1; ipsec_sa->esp_iv_len = 8; ipsec_sa->esp_pad_mask = esp_block_len_to_mask(1); - crypto_param.auth_iv.length = 12; + crypto_param.auth_iv_len = 12; ipsec_sa->salt_length = 4; salt_param = ¶m->crypto.auth_key_extra; break; diff --git a/platform/linux-generic/odp_packet.c b/platform/linux-generic/odp_packet.c index a15508ca2..2ad86ee99 100644 --- a/platform/linux-generic/odp_packet.c +++ b/platform/linux-generic/odp_packet.c @@ -1,5 +1,5 @@ /* Copyright (c) 2013-2018, Linaro Limited - * Copyright (c) 2019-2021, Nokia + * Copyright (c) 2019-2022, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -50,8 +50,8 @@ const _odp_packet_inline_offset_t _odp_packet_inline ODP_ALIGNED_CACHE = { .pool = offsetof(odp_packet_hdr_t, event_hdr.pool_ptr), .input = offsetof(odp_packet_hdr_t, input), .seg_count = offsetof(odp_packet_hdr_t, seg_count), - .user_ptr = offsetof(odp_packet_hdr_t, event_hdr.user_ptr), - .user_area = offsetof(odp_packet_hdr_t, event_hdr.uarea_addr), + .user_ptr = offsetof(odp_packet_hdr_t, user_ptr), + .user_area = offsetof(odp_packet_hdr_t, uarea_addr), .l2_offset = offsetof(odp_packet_hdr_t, p.l2_offset), .l3_offset = offsetof(odp_packet_hdr_t, p.l3_offset), .l4_offset = offsetof(odp_packet_hdr_t, p.l4_offset), @@ -193,9 +193,8 @@ static inline void packet_seg_copy_md(odp_packet_hdr_t *dst, if (src->p.flags.payload_off) dst->payload_offset = src->payload_offset; - /* event header side packet metadata */ - dst->event_hdr.user_ptr = src->event_hdr.user_ptr; - dst->event_hdr.uarea_addr = src->event_hdr.uarea_addr; + dst->user_ptr = src->user_ptr; + dst->uarea_addr = src->uarea_addr; /* segmentation data is not copied: * seg_next @@ -325,7 +324,7 @@ static inline void link_segments(odp_packet_hdr_t *pkt_hdr[], int num) uint32_t prev_ref; odp_atomic_u32_t *ref_cnt; - ref_cnt = &pkt_hdr[cur]->event_hdr.ref_cnt; + ref_cnt = &pkt_hdr[cur]->ref_cnt; prev_ref = odp_atomic_fetch_inc_u32(ref_cnt); ODP_ASSERT(prev_ref == 0); @@ -361,7 +360,7 @@ static inline void init_segments(odp_packet_hdr_t *pkt_hdr[], int num) if (ODP_DEBUG == 1) { uint32_t prev_ref = - odp_atomic_fetch_inc_u32(&hdr->event_hdr.ref_cnt); + odp_atomic_fetch_inc_u32(&hdr->ref_cnt); ODP_ASSERT(prev_ref == 0); } @@ -474,25 +473,25 @@ static inline odp_packet_hdr_t *add_segments(odp_packet_hdr_t *pkt_hdr, return pkt_hdr; } -static inline void segment_ref_inc(_odp_event_hdr_t *event_hdr) +static inline void segment_ref_inc(odp_packet_hdr_t *seg_hdr) { - uint32_t ref_cnt = odp_atomic_load_u32(&event_hdr->ref_cnt); + uint32_t ref_cnt = odp_atomic_load_u32(&seg_hdr->ref_cnt); /* First count increment after alloc */ if (odp_likely(ref_cnt == 0)) - odp_atomic_store_u32(&event_hdr->ref_cnt, 2); + odp_atomic_store_u32(&seg_hdr->ref_cnt, 2); else - odp_atomic_inc_u32(&event_hdr->ref_cnt); + odp_atomic_inc_u32(&seg_hdr->ref_cnt); } -static inline uint32_t segment_ref_dec(_odp_event_hdr_t *event_hdr) +static inline uint32_t segment_ref_dec(odp_packet_hdr_t *seg_hdr) { - return odp_atomic_fetch_dec_u32(&event_hdr->ref_cnt); + return odp_atomic_fetch_dec_u32(&seg_hdr->ref_cnt); } -static inline uint32_t segment_ref(_odp_event_hdr_t *event_hdr) +static inline uint32_t segment_ref(odp_packet_hdr_t *seg_hdr) { - return odp_atomic_load_u32(&event_hdr->ref_cnt); + return odp_atomic_load_u32(&seg_hdr->ref_cnt); } static inline int is_multi_ref(uint32_t ref_cnt) @@ -500,7 +499,7 @@ static inline int is_multi_ref(uint32_t ref_cnt) return (ref_cnt > 1); } -static inline void packet_free_multi(_odp_event_hdr_t *hdr[], int num) +static inline void packet_free_multi(odp_packet_hdr_t *hdr[], int num) { int i; uint32_t ref_cnt; @@ -527,21 +526,21 @@ static inline void packet_free_multi(_odp_event_hdr_t *hdr[], int num) num -= num_ref; if (odp_likely(num)) - _odp_event_free_multi(hdr, num); + _odp_event_free_multi((_odp_event_hdr_t **)(uintptr_t)hdr, num); } static inline void free_all_segments(odp_packet_hdr_t *pkt_hdr, int num) { int i; - _odp_event_hdr_t *event_hdr[num]; + odp_packet_hdr_t *pkt_hdrs[num]; odp_packet_hdr_t *seg_hdr = pkt_hdr; for (i = 0; i < num; i++) { - event_hdr[i] = &seg_hdr->event_hdr; + pkt_hdrs[i] = seg_hdr; seg_hdr = seg_hdr->seg_next; } - packet_free_multi(event_hdr, num); + packet_free_multi(pkt_hdrs, num); } static inline odp_packet_hdr_t *free_segments(odp_packet_hdr_t *pkt_hdr, @@ -553,14 +552,14 @@ static inline odp_packet_hdr_t *free_segments(odp_packet_hdr_t *pkt_hdr, int num_remain = pkt_hdr->seg_count - num; odp_packet_hdr_t *hdr = pkt_hdr; odp_packet_hdr_t *last_hdr = packet_last_seg(pkt_hdr); - _odp_event_hdr_t *event_hdr[num]; + odp_packet_hdr_t *pkt_hdrs[num]; if (head) { odp_packet_hdr_t *new_hdr; for (i = 0; i < num; i++) { seg_hdr = packet_seg_step(&hdr); - event_hdr[i] = &seg_hdr->event_hdr; + pkt_hdrs[i] = seg_hdr; } /* The first remaining header is the new packet descriptor. @@ -584,7 +583,7 @@ static inline odp_packet_hdr_t *free_segments(odp_packet_hdr_t *pkt_hdr, pkt_hdr = new_hdr; - packet_free_multi(event_hdr, num); + packet_free_multi(pkt_hdrs, num); } else { /* Free last 'num' bufs. * First, find the last remaining header. */ @@ -595,10 +594,10 @@ static inline odp_packet_hdr_t *free_segments(odp_packet_hdr_t *pkt_hdr, for (i = 0; i < num; i++) { seg_hdr = packet_seg_step(&hdr); - event_hdr[i] = &seg_hdr->event_hdr; + pkt_hdrs[i] = seg_hdr; } - packet_free_multi(event_hdr, num); + packet_free_multi(pkt_hdrs, num); /* Head segment remains, no need to copy or update majority * of the metadata. */ @@ -731,20 +730,17 @@ void odp_packet_free(odp_packet_t pkt) odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); int num_seg = pkt_hdr->seg_count; - ODP_ASSERT(segment_ref(&pkt_hdr->event_hdr) > 0); + ODP_ASSERT(segment_ref(pkt_hdr) > 0); - if (odp_likely(num_seg == 1)) { - _odp_event_hdr_t *event_hdr = &pkt_hdr->event_hdr; - - packet_free_multi(&event_hdr, 1); - } else { + if (odp_likely(num_seg == 1)) + packet_free_multi(&pkt_hdr, 1); + else free_all_segments(pkt_hdr, num_seg); - } } void odp_packet_free_multi(const odp_packet_t pkt[], int num) { - _odp_event_hdr_t *event_hdr[num]; + odp_packet_hdr_t *pkt_hdrs[num]; int i; int num_freed = 0; @@ -752,7 +748,7 @@ void odp_packet_free_multi(const odp_packet_t pkt[], int num) odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt[i]); int num_seg = pkt_hdr->seg_count; - ODP_ASSERT(segment_ref(&pkt_hdr->event_hdr) > 0); + ODP_ASSERT(segment_ref(pkt_hdr) > 0); if (odp_unlikely(num_seg > 1)) { free_all_segments(pkt_hdr, num_seg); @@ -760,11 +756,11 @@ void odp_packet_free_multi(const odp_packet_t pkt[], int num) continue; } - event_hdr[i - num_freed] = &pkt_hdr->event_hdr; + pkt_hdrs[i - num_freed] = pkt_hdr; } if (odp_likely(num - num_freed)) - packet_free_multi(event_hdr, num - num_freed); + packet_free_multi(pkt_hdrs, num - num_freed); } void odp_packet_free_sp(const odp_packet_t pkt[], int num) @@ -1076,7 +1072,7 @@ void odp_packet_user_ptr_set(odp_packet_t pkt, const void *ptr) return; } - pkt_hdr->event_hdr.user_ptr = ptr; + pkt_hdr->user_ptr = ptr; pkt_hdr->p.flags.user_ptr_set = 1; } @@ -1577,7 +1573,7 @@ void odp_packet_print(odp_packet_t pkt) len += snprintf(&str[len], n - len, "Packet\n------\n"); len += snprintf(&str[len], n - len, " pool index %u\n", hdr->event_hdr.index.pool); - len += snprintf(&str[len], n - len, " buf index %u\n", hdr->event_hdr.index.buffer); + len += snprintf(&str[len], n - len, " buf index %u\n", hdr->event_hdr.index.event); len += snprintf(&str[len], n - len, " ev subtype %i\n", hdr->subtype); len += snprintf(&str[len], n - len, " input_flags 0x%" PRIx64 "\n", hdr->p.input_flags.all); @@ -1614,7 +1610,6 @@ void odp_packet_print(odp_packet_t pkt) for (int seg_idx = 0; seg != ODP_PACKET_SEG_INVALID; seg_idx++) { odp_packet_hdr_t *seg_hdr = packet_seg_to_hdr(seg); - _odp_event_hdr_t *event_hdr = &seg_hdr->event_hdr; char seg_str[max_len]; int str_len; @@ -1624,7 +1619,7 @@ void odp_packet_print(odp_packet_t pkt) seg_idx, odp_packet_seg_data_len(pkt, seg), odp_packet_seg_data(pkt, seg), - segment_ref(event_hdr)); + segment_ref(seg_hdr)); /* Prevent print buffer overflow */ if (n - len - str_len < 10) { @@ -1657,7 +1652,7 @@ void odp_packet_print_data(odp_packet_t pkt, uint32_t offset, " pool index %" PRIu32 "\n", pool->pool_idx); len += snprintf(&str[len], n - len, " buf index %" PRIu32 "\n", - hdr->event_hdr.index.buffer); + hdr->event_hdr.index.event); len += snprintf(&str[len], n - len, " seg_count %" PRIu16 "\n", hdr->seg_count); len += snprintf(&str[len], n - len, @@ -1753,10 +1748,9 @@ int _odp_packet_copy_md_to_packet(odp_packet_t srcpkt, odp_packet_t dstpkt) dsthdr->dst_queue = srchdr->dst_queue; dsthdr->cos = srchdr->cos; dsthdr->cls_mark = srchdr->cls_mark; - dsthdr->event_hdr.user_ptr = srchdr->event_hdr.user_ptr; - if (dsthdr->event_hdr.uarea_addr != NULL && - srchdr->event_hdr.uarea_addr != NULL) { - memcpy(dsthdr->event_hdr.uarea_addr, srchdr->event_hdr.uarea_addr, + dsthdr->user_ptr = srchdr->user_ptr; + if (dsthdr->uarea_addr != NULL && srchdr->uarea_addr != NULL) { + memcpy(dsthdr->uarea_addr, srchdr->uarea_addr, dst_uarea_size <= src_uarea_size ? dst_uarea_size : src_uarea_size); } @@ -2778,7 +2772,7 @@ odp_packet_t odp_packet_ref_static(odp_packet_t pkt) odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); while (pkt_hdr != NULL) { - segment_ref_inc(&pkt_hdr->event_hdr); + segment_ref_inc(pkt_hdr); pkt_hdr = pkt_hdr->seg_next; } @@ -2834,14 +2828,11 @@ odp_packet_t odp_packet_ref_pkt(odp_packet_t pkt, uint32_t offset, int odp_packet_has_ref(odp_packet_t pkt) { - _odp_event_hdr_t *event_hdr; odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); uint32_t ref_cnt; while (pkt_hdr != NULL) { - event_hdr = &pkt_hdr->event_hdr; - - ref_cnt = segment_ref(event_hdr); + ref_cnt = segment_ref(pkt_hdr); if (is_multi_ref(ref_cnt)) return 1; diff --git a/platform/linux-generic/odp_pool.c b/platform/linux-generic/odp_pool.c index ca696fc96..cca2d55f0 100644 --- a/platform/linux-generic/odp_pool.c +++ b/platform/linux-generic/odp_pool.c @@ -1,5 +1,5 @@ /* Copyright (c) 2013-2018, Linaro Limited - * Copyright (c) 2019-2021, Nokia + * Copyright (c) 2019-2022, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -47,6 +47,9 @@ /* Define a practical limit for contiguous memory allocations */ #define MAX_SIZE (10 * 1024 * 1024) +/* Maximum packet user area size */ +#define MAX_UAREA_SIZE 2048 + ODP_STATIC_ASSERT(CONFIG_PACKET_SEG_LEN_MIN >= 256, "ODP Segment size must be a minimum of 256 bytes"); @@ -419,7 +422,7 @@ static pool_t *reserve_pool(uint32_t shmflags, uint8_t pool_ext, uint32_t num) return NULL; } -static void init_event_hdr(pool_t *pool, _odp_event_hdr_t *event_hdr, uint32_t buf_index, +static void init_event_hdr(pool_t *pool, _odp_event_hdr_t *event_hdr, uint32_t event_index, uint32_t hdr_len, uint8_t *data_ptr, void *uarea) { odp_pool_type_t type = pool->type; @@ -429,12 +432,10 @@ static void init_event_hdr(pool_t *pool, _odp_event_hdr_t *event_hdr, uint32_t b /* Initialize common event metadata */ event_hdr->index.u32 = 0; event_hdr->index.pool = pool->pool_idx; - event_hdr->index.buffer = buf_index; + event_hdr->index.event = event_index; event_hdr->type = type; event_hdr->event_type = type; event_hdr->pool_ptr = pool; - event_hdr->uarea_addr = uarea; - odp_atomic_init_u32(&event_hdr->ref_cnt, 0); /* Store base values for fast init */ if (type == ODP_POOL_BUFFER || type == ODP_POOL_PACKET) { @@ -446,10 +447,14 @@ static void init_event_hdr(pool_t *pool, _odp_event_hdr_t *event_hdr, uint32_t b if (type == ODP_POOL_PACKET) { odp_packet_hdr_t *pkt_hdr = (void *)event_hdr; + pkt_hdr->user_ptr = NULL; + pkt_hdr->uarea_addr = uarea; pkt_hdr->seg_data = data_ptr; pkt_hdr->seg_len = pool->seg_len; pkt_hdr->seg_count = 1; pkt_hdr->seg_next = NULL; + + odp_atomic_init_u32(&pkt_hdr->ref_cnt, 0); } /* Initialize event vector metadata */ @@ -1388,7 +1393,7 @@ int odp_pool_capability(odp_pool_capability_t *capa) capa->pkt.max_segs_per_pkt = PKT_MAX_SEGS; capa->pkt.min_seg_len = CONFIG_PACKET_SEG_LEN_MIN; capa->pkt.max_seg_len = max_seg_len; - capa->pkt.max_uarea_size = MAX_SIZE; + capa->pkt.max_uarea_size = MAX_UAREA_SIZE; capa->pkt.min_cache_size = 0; capa->pkt.max_cache_size = CONFIG_POOL_CACHE_MAX_SIZE; capa->pkt.stats.all = supported_stats.all; @@ -1634,7 +1639,7 @@ int _odp_event_is_valid(odp_event_t event) if (pool != event_hdr->pool_ptr) return 0; - if (event_hdr->index.buffer >= (pool->num + pool->skipped_blocks)) + if (event_hdr->index.event >= (pool->num + pool->skipped_blocks)) return 0; return 1; @@ -1682,7 +1687,7 @@ int odp_pool_ext_capability(odp_pool_type_t type, odp_pool_ext_capability_t *cap capa->pkt.max_headroom = CONFIG_PACKET_HEADROOM; capa->pkt.max_headroom_size = CONFIG_PACKET_HEADROOM; capa->pkt.max_segs_per_pkt = PKT_MAX_SEGS; - capa->pkt.max_uarea_size = MAX_SIZE; + capa->pkt.max_uarea_size = MAX_UAREA_SIZE; return 0; } diff --git a/platform/linux-generic/odp_random_std.c b/platform/linux-generic/odp_random_std.c index 3afd049f4..50cd773f2 100644 --- a/platform/linux-generic/odp_random_std.c +++ b/platform/linux-generic/odp_random_std.c @@ -4,52 +4,88 @@ * SPDX-License-Identifier: BSD-3-Clause */ -#include <odp_posix_extensions.h> -#include <stdint.h> -#include <stdlib.h> #include <odp/api/byteorder.h> #include <odp/api/cpu.h> #include <odp/api/debug.h> #include <odp_init_internal.h> #include <odp_random_std_internal.h> +#include <odp_cpu.h> +#include <stdint.h> #include <time.h> -/* Assume at least two rand bytes are available and RAND_MAX is power of two - 1 */ -ODP_STATIC_ASSERT(RAND_MAX >= UINT16_MAX, "RAND_MAX too small"); -ODP_STATIC_ASSERT((RAND_MAX & (RAND_MAX + 1ULL)) == 0, "RAND_MAX not power of two - 1"); +/* + * Xorshift64*, adapted from [1], and modified to return only the high 32 bits. + * + * [1] An experimental exploration of Marsaglia's xorshift generators, scrambled + * Sebastiano Vigna, July 2016. + * http://vigna.di.unimi.it/ftp/papers/xorshift.pdf + */ +static inline uint32_t xorshift64s32(uint64_t *x) +{ + /* The variable x should be initialized to a nonzero seed. [1] */ + if (!*x) + /* + * 2^64 / phi. As far away as possible from any small integer + * fractions, which the caller might be likely to use for the + * next seed after 0. + */ + *x = 11400714819323198485ull; -static int32_t _random_data(uint8_t *buf, uint32_t len, uint32_t *seed) + *x ^= *x >> 12; /* a */ + *x ^= *x << 25; /* b */ + *x ^= *x >> 27; /* c */ + return (*x * 2685821657736338717ull) >> 32; +} + +static int32_t _random_data(uint8_t *buf, uint32_t len, uint64_t *seed) { - union { - uint32_t rand_word; - uint8_t rand_byte[4]; - } u; - uint32_t i = 0, j, k; - - while (i < len) { - u.rand_word = rand_r(seed); - - /* Use two least significant bytes */ - j = ODP_LITTLE_ENDIAN ? 0 : 2; - for (k = 0; k < 2 && i < len; i++, j++, k++) - *buf++ = u.rand_byte[j]; + const uint32_t ret = len; + + if (!_ODP_UNALIGNED && ((uintptr_t)buf & 3) && len) { + uint32_t r = xorshift64s32(seed); + + if ((uintptr_t)buf & 1) { + *(uint8_t *)(uintptr_t)buf = r & 0xff; + r >>= 8; + buf += 1; + len -= 1; + } + + if (((uintptr_t)buf & 2) && len >= 2) { + *(uint16_t *)(uintptr_t)buf = r & 0xffff; + buf += 2; + len -= 2; + } + } + + for (uint32_t i = 0; i < len / 4; i++) { + *(uint32_t *)(uintptr_t)buf = xorshift64s32(seed); + buf += 4; } - return len; + if (len & 3) { + uint32_t r = xorshift64s32(seed); + + if (len & 2) { + *(uint16_t *)(uintptr_t)buf = r & 0xffff; + r >>= 16; + buf += 2; + } + + if (len & 1) + *(uint8_t *)(uintptr_t)buf = r & 0xff; + } + + return ret; } int32_t _odp_random_std_test_data(uint8_t *buf, uint32_t len, uint64_t *seed) { - uint32_t seed32 = (*seed) & 0xffffffff; - - _random_data(buf, len, &seed32); - - *seed = seed32; - return len; + return _random_data(buf, len, seed); } -static __thread uint32_t this_seed; +static __thread uint64_t this_seed; int32_t _odp_random_std_data(uint8_t *buf, uint32_t len) { @@ -59,7 +95,7 @@ int32_t _odp_random_std_data(uint8_t *buf, uint32_t len) int _odp_random_std_init_local(void) { this_seed = time(NULL); - this_seed ^= odp_cpu_id() << 16; + this_seed ^= (uint64_t)odp_cpu_id() << 32; return 0; } diff --git a/platform/linux-generic/odp_schedule_basic.c b/platform/linux-generic/odp_schedule_basic.c index 0761def57..3b22a5bb6 100644 --- a/platform/linux-generic/odp_schedule_basic.c +++ b/platform/linux-generic/odp_schedule_basic.c @@ -137,23 +137,24 @@ typedef union { ODP_STATIC_ASSERT(sizeof(lock_called_t) == sizeof(uint32_t), "Lock_called_values_do_not_fit_in_uint32"); +/* Shuffled values from 0 to 127 */ static uint8_t sched_random_u8[] = { - 0x64, 0xe3, 0x64, 0x0a, 0x0a, 0x5b, 0x7e, 0xd7, - 0x43, 0xb7, 0x90, 0x71, 0x76, 0x17, 0x8e, 0x3f, - 0x17, 0x60, 0x7e, 0xfd, 0x99, 0xe3, 0xab, 0x06, - 0x77, 0xf9, 0x45, 0x17, 0x2f, 0x81, 0x9e, 0x7b, - 0x20, 0x1b, 0x36, 0x75, 0x69, 0xc5, 0x69, 0x27, - 0x7a, 0xf6, 0x3f, 0x63, 0x2c, 0x3f, 0x1b, 0xeb, - 0x12, 0xe1, 0x6f, 0xd4, 0xd9, 0x14, 0x97, 0xa6, - 0x2a, 0xe5, 0xb0, 0x45, 0x27, 0xa6, 0x48, 0xbc, - 0x2b, 0xec, 0xd8, 0xda, 0x55, 0xef, 0x15, 0xce, - 0xf8, 0xc2, 0x1e, 0xc8, 0x16, 0x6c, 0xf0, 0x4f, - 0x1a, 0xc7, 0x50, 0x9e, 0x0b, 0xa5, 0xe9, 0xf3, - 0x28, 0x79, 0x2e, 0x18, 0xb0, 0xb4, 0xac, 0xce, - 0x67, 0x04, 0x52, 0x98, 0xce, 0x8c, 0x05, 0x87, - 0xab, 0xc8, 0x94, 0x7e, 0x46, 0x63, 0x60, 0x8d, - 0x3d, 0x8f, 0x14, 0x85, 0x1e, 0x92, 0xd2, 0x40, - 0x2d, 0x42, 0xfe, 0xf1, 0xc2, 0xb6, 0x03, 0x43 + 0x5B, 0x56, 0x21, 0x28, 0x77, 0x2C, 0x7E, 0x10, + 0x29, 0x73, 0x39, 0x74, 0x60, 0x2B, 0x2D, 0x3E, + 0x6C, 0x4C, 0x1B, 0x79, 0x14, 0x76, 0x7B, 0x5A, + 0x4F, 0x3B, 0x0B, 0x16, 0x66, 0x0D, 0x05, 0x27, + 0x3F, 0x7F, 0x67, 0x3C, 0x41, 0x6F, 0x4E, 0x7A, + 0x04, 0x26, 0x11, 0x7C, 0x43, 0x38, 0x30, 0x2A, + 0x03, 0x22, 0x17, 0x75, 0x08, 0x71, 0x6D, 0x6B, + 0x0A, 0x4B, 0x52, 0x1D, 0x63, 0x59, 0x1C, 0x50, + 0x15, 0x1A, 0x64, 0x42, 0x47, 0x62, 0x1F, 0x37, + 0x46, 0x5D, 0x19, 0x35, 0x78, 0x68, 0x57, 0x7D, + 0x3A, 0x31, 0x4A, 0x45, 0x09, 0x49, 0x00, 0x01, + 0x65, 0x13, 0x48, 0x70, 0x5E, 0x69, 0x36, 0x58, + 0x1E, 0x5C, 0x23, 0x12, 0x18, 0x25, 0x55, 0x32, + 0x33, 0x61, 0x2F, 0x02, 0x06, 0x53, 0x24, 0x6E, + 0x2E, 0x5F, 0x54, 0x6A, 0x20, 0x07, 0x0F, 0x51, + 0x3D, 0x34, 0x44, 0x0C, 0x4D, 0x40, 0x72, 0x0E }; ODP_STATIC_ASSERT(sizeof(sched_random_u8) == RANDOM_TBL_SIZE, "Bad_random_table_size"); diff --git a/platform/linux-generic/odp_shared_memory.c b/platform/linux-generic/odp_shared_memory.c index 424d32ccf..74e5c9864 100644 --- a/platform/linux-generic/odp_shared_memory.c +++ b/platform/linux-generic/odp_shared_memory.c @@ -8,6 +8,7 @@ #include <odp_config_internal.h> #include <odp_debug_internal.h> #include <odp/api/debug.h> +#include <odp/api/deprecated.h> #include <odp/api/std_types.h> #include <odp/api/shared_memory.h> #include <odp/api/plat/strong_types.h> @@ -17,8 +18,14 @@ #include <string.h> /* Supported ODP_SHM_* flags */ -#define SUPPORTED_SHM_FLAGS (ODP_SHM_SW_ONLY | ODP_SHM_PROC | ODP_SHM_SINGLE_VA | ODP_SHM_EXPORT | \ - ODP_SHM_HP | ODP_SHM_NO_HP) +#if ODP_DEPRECATED_API + #define DEPRECATED_SHM_FLAGS (ODP_SHM_SW_ONLY) +#else + #define DEPRECATED_SHM_FLAGS 0 +#endif + +#define SUPPORTED_SHM_FLAGS (ODP_SHM_PROC | ODP_SHM_SINGLE_VA | ODP_SHM_EXPORT | \ + ODP_SHM_HP | ODP_SHM_NO_HP | DEPRECATED_SHM_FLAGS) static inline uint32_t from_handle(odp_shm_t shm) { diff --git a/platform/linux-generic/odp_system_info.c b/platform/linux-generic/odp_system_info.c index 1e4f775e1..5665a3ece 100644 --- a/platform/linux-generic/odp_system_info.c +++ b/platform/linux-generic/odp_system_info.c @@ -471,7 +471,14 @@ uint64_t odp_sys_huge_page_size(void) static int pagesz_compare(const void *pagesz1, const void *pagesz2) { - return (*(const uint64_t *)pagesz1 - *(const uint64_t *)pagesz2); + const uint64_t val1 = *(const uint64_t *)pagesz1; + const uint64_t val2 = *(const uint64_t *)pagesz2; + + if (val1 < val2) + return -1; + if (val1 > val2) + return 1; + return 0; } int odp_sys_huge_page_size_all(uint64_t size[], int num) diff --git a/platform/linux-generic/odp_timer.c b/platform/linux-generic/odp_timer.c index cd98fcc7d..e6bcbbace 100644 --- a/platform/linux-generic/odp_timer.c +++ b/platform/linux-generic/odp_timer.c @@ -1,5 +1,5 @@ /* Copyright (c) 2013-2018, Linaro Limited - * Copyright (c) 2019-2021, Nokia + * Copyright (c) 2019-2022, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -71,6 +71,9 @@ /* Max inline timer resolution */ #define MAX_INLINE_RES_NS 500 +/* Timer pool may be reused after this period */ +#define TIMER_POOL_REUSE_NS ODP_TIME_SEC_IN_NS + /* Mutual exclusion in the absence of CAS16 */ #ifndef ODP_ATOMIC_U128 #define NUM_LOCKS 1024 @@ -129,7 +132,6 @@ typedef struct timer_pool_s { uint32_t tp_idx;/* Index into timer_pool array */ odp_timer_pool_param_t param; char name[ODP_TIMER_POOL_NAME_LEN]; - odp_shm_t shm; timer_t timerid; int notify_overrun; int owner; @@ -157,6 +159,8 @@ typedef struct timer_global_t { uint64_t poll_interval_nsec; int num_timer_pools; uint8_t timer_pool_used[MAX_TIMER_POOLS]; + odp_time_t destroy_time[MAX_TIMER_POOLS]; + odp_shm_t tp_shm[MAX_TIMER_POOLS]; timer_pool_t *timer_pool[MAX_TIMER_POOLS]; #ifndef ODP_ATOMIC_U128 /* Multiple locks per cache line! */ @@ -285,11 +289,17 @@ static odp_timer_pool_t timer_pool_new(const char *name, size_t sz0, sz1, sz2; uint64_t tp_size; uint64_t res_ns, nsec_per_scan; + odp_shm_t shm; + timer_pool_t *tp; + odp_time_t diff, time; + odp_time_t max_diff = ODP_TIME_NULL; uint32_t flags = 0; if (odp_global_ro.shm_single_va) flags |= ODP_SHM_SINGLE_VA; + time = odp_time_global(); + odp_ticketlock_lock(&timer_global->lock); if (timer_global->num_timer_pools >= MAX_TIMER_POOLS) { @@ -298,31 +308,59 @@ static odp_timer_pool_t timer_pool_new(const char *name, return ODP_TIMER_POOL_INVALID; } + /* Find timer pool that has not been used for a while, or is used least recently. + * This ensures that inline scan of an old timer pool has completed and its memory + * can be freed. */ + tp_idx = -1; for (i = 0; i < MAX_TIMER_POOLS; i++) { if (timer_global->timer_pool_used[i] == 0) { - timer_global->timer_pool_used[i] = 1; - break; + diff = odp_time_diff(time, timer_global->destroy_time[i]); + + if (odp_time_to_ns(diff) > TIMER_POOL_REUSE_NS) { + tp_idx = i; + break; + } + + if (odp_time_cmp(diff, max_diff) > 0) { + max_diff = diff; + tp_idx = i; + } } } - tp_idx = i; + if (tp_idx < 0) { + odp_ticketlock_unlock(&timer_global->lock); + ODP_DBG("Did not find free timer pool\n"); + return ODP_TIMER_POOL_INVALID; + } + + shm = timer_global->tp_shm[tp_idx]; + timer_global->timer_pool_used[tp_idx] = 1; timer_global->num_timer_pools++; odp_ticketlock_unlock(&timer_global->lock); + /* Free memory of previously destroyed timer pool */ + if (shm != ODP_SHM_INVALID) { + if (odp_shm_free(shm)) { + ODP_ERR("Failed to free shared memory: tp_idx %i\n", tp_idx); + goto error; + } + } + sz0 = ROUNDUP_CACHE_LINE(sizeof(timer_pool_t)); sz1 = ROUNDUP_CACHE_LINE(sizeof(tick_buf_t) * param->num_timers); - sz2 = ROUNDUP_CACHE_LINE(sizeof(_odp_timer_t) * - param->num_timers); + sz2 = ROUNDUP_CACHE_LINE(sizeof(_odp_timer_t) * param->num_timers); tp_size = sz0 + sz1 + sz2; - odp_shm_t shm = odp_shm_reserve(name, tp_size, ODP_CACHE_LINE_SIZE, - flags); - if (odp_unlikely(shm == ODP_SHM_INVALID)) - ODP_ABORT("%s: timer pool shm-alloc(%zuKB) failed\n", - name, (sz0 + sz1 + sz2) / 1024); - timer_pool_t *tp = (timer_pool_t *)odp_shm_addr(shm); + shm = odp_shm_reserve(name, tp_size, ODP_CACHE_LINE_SIZE, flags); + if (odp_unlikely(shm == ODP_SHM_INVALID)) { + ODP_ERR("Timer pool shm reserve failed %" PRIu64 "kB\n", tp_size / 1024); + goto error; + } + + tp = (timer_pool_t *)odp_shm_addr(shm); memset(tp, 0, tp_size); if (param->res_ns) @@ -346,7 +384,7 @@ static odp_timer_pool_t timer_pool_new(const char *name, strncpy(tp->name, name, ODP_TIMER_POOL_NAME_LEN - 1); tp->name[ODP_TIMER_POOL_NAME_LEN - 1] = 0; } - tp->shm = shm; + tp->param = *param; tp->param.res_ns = res_ns; tp->min_rel_tck = odp_timer_ns_to_tick(timer_pool_to_hdl(tp), @@ -382,8 +420,11 @@ static odp_timer_pool_t timer_pool_new(const char *name, tp->start_time = odp_time_global(); odp_ticketlock_lock(&timer_global->lock); + /* Inline timer scan may find the timer pool after this */ + odp_mb_release(); timer_global->timer_pool[tp_idx] = tp; + timer_global->tp_shm[tp_idx] = shm; if (timer_global->num_timer_pools == 1) odp_global_rw->inline_timers = timer_global->use_inline_timers; @@ -395,20 +436,25 @@ static odp_timer_pool_t timer_pool_new(const char *name, odp_time_global_from_ns(nsec_per_scan); } + /* Update the highest index for inline timer scan */ + if (tp_idx > timer_global->highest_tp_idx) + timer_global->highest_tp_idx = tp_idx; + odp_ticketlock_unlock(&timer_global->lock); - if (!odp_global_rw->inline_timers) { - if (tp->param.clk_src == ODP_CLOCK_DEFAULT) - itimer_init(tp); - } else { - /* Update the highest index for inline timer scan */ - odp_ticketlock_lock(&timer_global->lock); - if (tp_idx > timer_global->highest_tp_idx) - timer_global->highest_tp_idx = tp_idx; - odp_ticketlock_unlock(&timer_global->lock); - } + if (!odp_global_rw->inline_timers) + itimer_init(tp); return timer_pool_to_hdl(tp); + +error: + odp_ticketlock_lock(&timer_global->lock); + timer_global->tp_shm[tp_idx] = shm; + timer_global->timer_pool_used[tp_idx] = 0; + timer_global->num_timer_pools--; + odp_ticketlock_unlock(&timer_global->lock); + + return ODP_TIMER_POOL_INVALID; } static void block_sigalarm(void) @@ -433,16 +479,14 @@ static void stop_timer_thread(timer_pool_t *tp) static void odp_timer_pool_del(timer_pool_t *tp) { - int rc, highest; - odp_shm_t shm; + int highest; + uint32_t tp_idx = tp->tp_idx; odp_spinlock_lock(&tp->lock); if (!odp_global_rw->inline_timers) { /* Stop POSIX itimer signals */ - if (tp->param.clk_src == ODP_CLOCK_DEFAULT) - itimer_fini(tp); - + itimer_fini(tp); stop_timer_thread(tp); } @@ -456,10 +500,10 @@ static void odp_timer_pool_del(timer_pool_t *tp) odp_spinlock_unlock(&tp->lock); odp_ticketlock_lock(&timer_global->lock); - shm = tp->shm; - timer_global->timer_pool[tp->tp_idx] = NULL; - timer_global->timer_pool_used[tp->tp_idx] = 0; + timer_global->timer_pool[tp_idx] = NULL; + timer_global->timer_pool_used[tp_idx] = 0; timer_global->num_timer_pools--; + timer_global->destroy_time[tp_idx] = odp_time_global(); highest = -1; @@ -477,11 +521,6 @@ static void odp_timer_pool_del(timer_pool_t *tp) timer_global->highest_tp_idx = highest; odp_ticketlock_unlock(&timer_global->lock); - - rc = odp_shm_free(shm); - - if (rc != 0) - ODP_ABORT("Failed to free shared memory (%d)\n", rc); } static inline odp_timer_t timer_alloc(timer_pool_t *tp, odp_queue_t queue, const void *user_ptr) @@ -943,10 +982,10 @@ static inline void timer_pool_scan_inline(int num, odp_time_t now) if (odp_atomic_cas_u64(&tp->cur_tick, &old_tick, new_tick)) { if (tp->notify_overrun && diff > 1) { if (old_tick == 0) { - ODP_ERR("Timer pool (%s) missed %" PRIi64 " scans in start up\n", + ODP_DBG("Timer pool (%s) missed %" PRIi64 " scans in start up\n", tp->name, diff - 1); } else { - ODP_ERR("Timer pool (%s) resolution too high: missed %" PRIi64 " scans\n", + ODP_DBG("Timer pool (%s) resolution too high: %" PRIi64 " scans missed\n", tp->name, diff - 1); tp->notify_overrun = 0; } @@ -1002,7 +1041,7 @@ static inline void timer_run_posix(timer_pool_t *tp) if (tp->notify_overrun) { overrun = timer_getoverrun(tp->timerid); if (overrun) { - ODP_ERR("\n\t%d ticks overrun on timer pool \"%s\", timer resolution too high\n", + ODP_DBG("\n\t%d ticks overrun on timer pool \"%s\", timer resolution too high\n", overrun, tp->name); tp->notify_overrun = 0; } @@ -1278,6 +1317,11 @@ odp_timer_pool_t odp_timer_pool_create(const char *name, return ODP_TIMER_POOL_INVALID; } + if (param->clk_src != ODP_CLOCK_DEFAULT) { + ODP_ERR("Only ODP_CLOCK_DEFAULT supported. Requested %i.\n", param->clk_src); + return ODP_TIMER_POOL_INVALID; + } + if ((param->res_ns && param->res_hz) || (param->res_ns == 0 && param->res_hz == 0)) { _odp_errno = EINVAL; @@ -1596,7 +1640,9 @@ void odp_timeout_print(odp_timeout_t tmo) int _odp_timer_init_global(const odp_init_t *params) { odp_shm_t shm; + odp_time_t time; const char *conf_str; + uint32_t i; int val = 0; if (params && params->not_used.feat.timer) { @@ -1621,9 +1667,13 @@ int _odp_timer_init_global(const odp_init_t *params) timer_global->highest_res_ns = MAX_INLINE_RES_NS; timer_global->highest_tp_idx = -1; -#ifndef ODP_ATOMIC_U128 - uint32_t i; + time = odp_time_global(); + for (i = 0; i < MAX_TIMER_POOLS; i++) { + timer_global->destroy_time[i] = time; + timer_global->tp_shm[i] = ODP_SHM_INVALID; + } +#ifndef ODP_ATOMIC_U128 for (i = 0; i < NUM_LOCKS; i++) _odp_atomic_flag_clear(&timer_global->locks[i]); #else @@ -1676,8 +1726,24 @@ error: int _odp_timer_term_global(void) { - if (timer_global && odp_shm_free(timer_global->shm)) { - ODP_ERR("Shm free failed for odp_timer\n"); + odp_shm_t shm; + int i; + + if (timer_global == NULL) + return 0; + + for (i = 0; i < MAX_TIMER_POOLS; i++) { + shm = timer_global->tp_shm[i]; + if (shm != ODP_SHM_INVALID) { + if (odp_shm_free(shm)) { + ODP_ERR("Shm free failed for timer pool %i\n", i); + return -1; + } + } + } + + if (odp_shm_free(timer_global->shm)) { + ODP_ERR("Shm free failed for timer_global\n"); return -1; } diff --git a/platform/linux-generic/odp_traffic_mngr.c b/platform/linux-generic/odp_traffic_mngr.c index e9e6822a1..119be7fba 100644 --- a/platform/linux-generic/odp_traffic_mngr.c +++ b/platform/linux-generic/odp_traffic_mngr.c @@ -1,6 +1,7 @@ /* Copyright 2015 EZchip Semiconductor Ltd. All Rights Reserved. * * Copyright (c) 2015-2018, Linaro Limited + * Copyright (c) 2022, Marvell * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -49,6 +50,10 @@ static const pkt_desc_t EMPTY_PKT_DESC = { .word = 0 }; #define MAX_PRIORITIES ODP_TM_MAX_PRIORITIES #define NUM_SHAPER_COLORS ODP_NUM_SHAPER_COLORS +/* Shaper BW limits in bits/sec */ +#define TM_MIN_SHAPER_BW 8000ULL +#define TM_MAX_SHAPER_BW (100ULL * 1000ULL * 1000ULL * 1000ULL) + static const tm_prop_t basic_prop_tbl[MAX_PRIORITIES][NUM_SHAPER_COLORS] = { [0] = { [ODP_TM_SHAPER_GREEN] = { 0, DECR_BOTH }, @@ -167,24 +172,6 @@ static inline tm_node_obj_t *tm_nobj_from_index(uint32_t node_id) return &tm_glb->node_obj.obj[node_id]; } -static int queue_tm_reenq(odp_queue_t queue, _odp_event_hdr_t *event_hdr) -{ - odp_tm_queue_t tm_queue = MAKE_ODP_TM_QUEUE(odp_queue_context(queue)); - odp_packet_t pkt = packet_from_event_hdr(event_hdr); - - return odp_tm_enq(tm_queue, pkt); -} - -static int queue_tm_reenq_multi(odp_queue_t queue, _odp_event_hdr_t *event[], - int num) -{ - (void)queue; - (void)event; - (void)num; - ODP_ABORT("Invalid call to queue_tm_reenq_multi()\n"); - return 0; -} - static tm_queue_obj_t *get_tm_queue_obj(tm_system_t *tm_system, pkt_desc_t *pkt_desc) { @@ -632,7 +619,7 @@ static void tm_shaper_params_cvt_to(const odp_tm_shaper_params_t *shaper_params, commit_burst = (int64_t)shaper_params->commit_burst; peak_rate = tm_bps_to_rate(shaper_params->peak_rate); - if ((shaper_params->peak_rate == 0) || (peak_rate == 0)) { + if ((!shaper_params->dual_rate) || (peak_rate == 0)) { peak_rate = 0; max_peak_time_delta = 0; peak_burst = 0; @@ -829,7 +816,7 @@ static void update_shaper_elapsed_time(tm_system_t *tm_system, shaper_obj->commit_cnt = (int64_t)MIN(max_commit, commit + commit_inc); - if (shaper_params->peak_rate != 0) { + if (shaper_params->dual_rate) { peak = shaper_obj->peak_cnt; max_peak = shaper_params->max_peak; if (shaper_params->max_peak_time_delta <= time_delta) @@ -860,7 +847,7 @@ static uint64_t time_till_not_red(tm_shaper_params_t *shaper_params, min_time_delay = MAX(shaper_obj->shaper_params->min_time_delta, UINT64_C(256)); commit_delay = MAX(commit_delay, min_time_delay); - if (shaper_params->peak_rate == 0) + if (!shaper_params->dual_rate) return commit_delay; peak_delay = 0; @@ -1064,7 +1051,7 @@ static odp_bool_t rm_pkt_from_shaper(tm_system_t *tm_system, (shaper_action == DECR_COMMIT)) shaper_obj->commit_cnt -= tkn_count; - if (shaper_params->peak_rate != 0) + if (shaper_params->dual_rate) if ((shaper_action == DECR_BOTH) || (shaper_action == DECR_PEAK)) shaper_obj->peak_cnt -= tkn_count; @@ -1096,7 +1083,7 @@ static odp_bool_t run_shaper(tm_system_t *tm_system, if (shaper_params->enabled) { if (0 < shaper_obj->commit_cnt) shaper_color = ODP_TM_SHAPER_GREEN; - else if (shaper_params->peak_rate == 0) + else if (!shaper_params->dual_rate) shaper_color = ODP_TM_SHAPER_RED; else if (shaper_obj->peak_cnt <= 0) shaper_color = ODP_TM_SHAPER_RED; @@ -2608,6 +2595,10 @@ static int tm_capabilities(odp_tm_capabilities_t capabilities[], per_level_cap->max_priority = ODP_TM_MAX_PRIORITIES - 1; per_level_cap->min_weight = ODP_TM_MIN_SCHED_WEIGHT; per_level_cap->max_weight = ODP_TM_MAX_SCHED_WEIGHT; + per_level_cap->min_burst = 0; + per_level_cap->max_burst = UINT32_MAX; + per_level_cap->min_rate = TM_MIN_SHAPER_BW; + per_level_cap->max_rate = TM_MAX_SHAPER_BW; per_level_cap->tm_node_shaper_supported = true; per_level_cap->tm_node_wred_supported = true; @@ -2730,6 +2721,10 @@ static void tm_system_capabilities_set(odp_tm_capabilities_t *cap_ptr, per_level_cap->max_priority = max_priority; per_level_cap->min_weight = min_weight; per_level_cap->max_weight = max_weight; + per_level_cap->min_burst = 0; + per_level_cap->max_burst = UINT32_MAX; + per_level_cap->min_rate = TM_MIN_SHAPER_BW; + per_level_cap->max_rate = TM_MAX_SHAPER_BW; per_level_cap->tm_node_shaper_supported = shaper_supported; per_level_cap->tm_node_wred_supported = wred_supported; @@ -3923,6 +3918,7 @@ int odp_tm_node_shaper_config(odp_tm_node_t tm_node, { tm_node_obj_t *tm_node_obj; tm_system_t *tm_system; + odp_bool_t sync_needed; tm_node_obj = GET_TM_NODE_OBJ(tm_node); if (!tm_node_obj) @@ -3933,8 +3929,13 @@ int odp_tm_node_shaper_config(odp_tm_node_t tm_node, return -1; odp_ticketlock_lock(&tm_glb->profile_lock); + sync_needed = tm_glb->main_loop_running; + if (sync_needed) + signal_request(); tm_shaper_config_set(tm_system, shaper_profile, &tm_node_obj->shaper_obj); + if (sync_needed) + signal_request_done(); odp_ticketlock_unlock(&tm_glb->profile_lock); return 0; } @@ -4052,7 +4053,6 @@ odp_tm_queue_t odp_tm_queue_create(odp_tm_t odp_tm, _odp_int_pkt_queue_t _odp_int_pkt_queue; tm_queue_obj_t *queue_obj; odp_tm_queue_t odp_tm_queue = ODP_TM_INVALID; - odp_queue_t queue; odp_tm_wred_t wred_profile; tm_system_t *tm_system; uint32_t color; @@ -4090,25 +4090,6 @@ odp_tm_queue_t odp_tm_queue_create(odp_tm_t odp_tm, odp_atomic_init_u64(&queue_obj->stats.errors, 0); odp_atomic_init_u64(&queue_obj->stats.packets, 0); - queue = odp_queue_create(NULL, NULL); - if (queue == ODP_QUEUE_INVALID) { - odp_tm_queue = ODP_TM_INVALID; - continue; - } - - queue_obj->queue = queue; - if (odp_queue_context_set(queue, queue_obj, sizeof(tm_queue_obj_t))) { - ODP_ERR("Queue context set failed\n"); - if (odp_queue_destroy(queue)) - ODP_ERR("Queue destroy failed\n"); - - odp_tm_queue = ODP_TM_INVALID; - break; - } - - _odp_queue_fn->set_enq_deq_fn(queue, queue_tm_reenq, - queue_tm_reenq_multi, NULL, NULL); - tm_system->queue_num_tbl[queue_obj->queue_num - 1] = queue_obj; odp_ticketlock_lock(&tm_system->tm_system_lock); @@ -4169,8 +4150,6 @@ int odp_tm_queue_destroy(odp_tm_queue_t tm_queue) odp_ticketlock_lock(&tm_system->tm_system_lock); tm_system->queue_num_tbl[tm_queue_obj->queue_num - 1] = NULL; - odp_queue_destroy(tm_queue_obj->queue); - odp_ticketlock_lock(&tm_glb->queue_obj.lock); _odp_pkt_queue_destroy(tm_system->_odp_int_queue_pool, tm_queue_obj->_odp_int_pkt_queue); @@ -4485,6 +4464,7 @@ int odp_tm_enq(odp_tm_queue_t tm_queue, odp_packet_t pkt) { tm_queue_obj_t *tm_queue_obj; tm_system_t *tm_system; + int rc; tm_queue_obj = GET_TM_QUEUE_OBJ(tm_queue); if (!tm_queue_obj) @@ -4497,7 +4477,10 @@ int odp_tm_enq(odp_tm_queue_t tm_queue, odp_packet_t pkt) if (odp_atomic_load_u64(&tm_system->destroying)) return -1; - return tm_enqueue(tm_system, tm_queue_obj, pkt); + rc = tm_enqueue(tm_system, tm_queue_obj, pkt); + if (rc < 0) + return rc; + return 0; } int odp_tm_enq_with_cnt(odp_tm_queue_t tm_queue, odp_packet_t pkt) diff --git a/platform/linux-generic/pktio/socket.c b/platform/linux-generic/pktio/socket.c index 291ee543f..0bd7a21c4 100644 --- a/platform/linux-generic/pktio/socket.c +++ b/platform/linux-generic/pktio/socket.c @@ -608,7 +608,7 @@ static int sock_init_global(void) disable_pktio = 1; } else { ODP_PRINT("PKTIO: initialized socket mmsg," - "use export ODP_PKTIO_DISABLE_SOCKET_MMSG=1 to disable.\n"); + " use export ODP_PKTIO_DISABLE_SOCKET_MMSG=1 to disable.\n"); } return 0; } |