diff options
author | Matias Elo <matias.elo@nokia.com> | 2023-08-17 15:39:46 +0300 |
---|---|---|
committer | GitHub <noreply@github.com> | 2023-08-17 15:39:46 +0300 |
commit | 5d505fde6d4327b27446389699abf3a593c02a17 (patch) | |
tree | 937164d89e7b8679a42f0e3705e19243baaf0147 | |
parent | de97121a2e3afa072f7c51a0570f4b3bed0236c2 (diff) | |
parent | 7a200d709b64e743c002dd9737cd71f54ccc54ac (diff) |
Merge ODP v1.42.0.0v1.42.0.0_DPDK_22.11
Merge ODP linux-generic v1.42.0.0 into linux-dpdk.
102 files changed, 2724 insertions, 1560 deletions
diff --git a/.github/workflows/ci-pipeline-arm64.yml b/.github/workflows/ci-pipeline-arm64.yml index b2a7ad042..84f52c1da 100644 --- a/.github/workflows/ci-pipeline-arm64.yml +++ b/.github/workflows/ci-pipeline-arm64.yml @@ -80,7 +80,7 @@ jobs: strategy: fail-fast: false matrix: - cc_ver: [10, 11, 12] + cc_ver: [10, 11, 12, 13] conf: ['', '--enable-abi-compat'] steps: - uses: OpenDataPlane/action-clean-up@main @@ -91,25 +91,6 @@ jobs: if: ${{ failure() }} run: find . -name config.log -exec cat {} \; - Build_gcc_u23: - if: ${{ github.repository == 'OpenDataPlane/odp-dpdk' }} - runs-on: [self-hosted, ARM64] - env: - OS: ubuntu_23.04 - strategy: - fail-fast: false - matrix: - cc_ver: [13] - conf: ['', '--enable-abi-compat'] - steps: - - uses: OpenDataPlane/action-clean-up@main - - uses: actions/checkout@v3 - - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="gcc-${{matrix.cc_ver}}" -e CXX="g++-${{matrix.cc_ver}}" - -e CONF="${{matrix.conf}}" $CONTAINER_NAMESPACE/odp-ci-${OS}-${ARCH}-native /odp/scripts/ci/build_${ARCH}.sh - - name: Failure log - if: ${{ failure() }} - run: find . -name config.log -exec cat {} \; - Build_out-of-tree: if: ${{ github.repository == 'OpenDataPlane/odp-dpdk' }} runs-on: [self-hosted, ARM64] diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index 87b6d11bf..280e9653e 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -210,7 +210,7 @@ jobs: strategy: fail-fast: false matrix: - cc_ver: [10, 11, 12] + cc_ver: [10, 11, 12, 13] conf: ['', '--enable-abi-compat'] steps: - uses: actions/checkout@v3 @@ -220,23 +220,6 @@ jobs: if: ${{ failure() }} run: find . -name config.log -exec cat {} \; - Build_gcc_u23: - runs-on: ubuntu-20.04 - env: - OS: ubuntu_23.04 - strategy: - fail-fast: false - matrix: - cc_ver: [13] - conf: ['', '--enable-abi-compat'] - steps: - - uses: actions/checkout@v3 - - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="gcc-${{matrix.cc_ver}}" -e CXX="g++-${{matrix.cc_ver}}" - -e CONF="${{matrix.conf}}" $CONTAINER_NAMESPACE/odp-ci-${OS}-${ARCH} /odp/scripts/ci/build_${ARCH}.sh - - name: Failure log - if: ${{ failure() }} - run: find . -name config.log -exec cat {} \; - Build_out-of-tree: runs-on: ubuntu-20.04 steps: @@ -1,3 +1,69 @@ +== OpenDataPlane (1.42.0.0) + +=== Backward incompatible API changes +==== Crypto +* Add `odp_crypto_session_param_t.cipher_range_in_bits` and +`odp_crypto_session_param_t.auth_range_in_bits` session parameters that control +whether cipher/auth range is given in bits or bytes. +* Change `odp_crypto_cipher_capability_t.bit_mode` cipher/auth capability flag +to indicate whether bit mode is supported in addition to byte mode that is +always supported. + +=== Backward compatible API changes +==== Barrier +* Add memory barriers (`odp_mb_sync()`, `odp_mb_sync_load()`, +`odp_mb_sync_store()`) which ensure that load/store instructions started before +the barrier are complete, before load/store instructions after the barrier are +started. + +==== Crypto +* Allow non-zero-length cipher and auth ranges +(`odp_crypto_packet_op_param_t.cipher_range/auth_range`) for the null cipher and +auth algorithms in the OOP operation type. + +==== DMA +* Add new `odp_dma_pool_capability_t.uarea_persistence` pool capability to +signal if implementation is able to maintain the content of pool user areas +across frees and allocations. +* Add new `odp_dma_pool_param_t.uarea_init` pool parameters `init_fn` and +`args` that can be used to initialize event user areas of a pool at pool +creation time. +* Move `odp_dma_seg_t` structure fields so that `addr/packet` field is directly +followed by `len` and `offset` fields. After this change all commonly used +fields are located in the first 16 bytes of the structure. + +==== Event +* Add new `odp_event_types_multi()` function for reading event types and +subtypes (optional) from all given events. +* Add new `odp_event_user_area_and_flag()` function for reading both event +user area pointer and user flag value. + +==== Init +* Add new `ODP_LOG_WARN` log level to `odp_log_level_t`. + +==== Packet +* Change `odp_packet_user_flag()` and `odp_packet_vector_user_flag()` +documentations to specify that the return values are positive if user flag is +set. + +==== Pool +* Add new `uarea_persistence` pool capability to signal if implementation is +able to maintain the content of pool user areas across frees and allocations. +* Add new `uarea_init` pool parameters `init_fn` and `args` that can be used to +initialize event user areas of a pool at pool creation time. + +==== Timer +* Add new `odp_timeout_from_event_multi()` function for converting multiple +events of type `ODP_EVENT_TIMEOUT` to timeout handles. +* Clarify `odp_timer_alloc()` and `odp_timer_restart()` documentation. + +=== Implementation +==== Time +* Refactor time codebase to enable more optimized architecture specific +implementations. x86 and aarch64 architectures are assumed to always have +support for HW time and fallback to POSIX time functions has been removed for +improved performance. + == OpenDataPlane (1.41.1.0) === Backward compatible API changes @@ -1,21 +1,20 @@ -Copyright (c) 2013-2018, Linaro Limited -All rights reserved. - -SPDX-License-Identifier: BSD-3-Clause +SPDX-License-Identifier: BSD-3-Clause +Copyright (c) 2013-2019 Linaro Limited +Copyright (c) 2019-2023 OpenFastPath Foundation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: -Redistributions of source code must retain the above copyright notice, this +1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. -Redistributions in binary form must reproduce the above copyright notice, this -list of conditions and the following disclaimer in the documentation and/or +2. Redistributions in binary form must reproduce the above copyright notice, +this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. -Neither the name of Linaro Limited nor the names of its contributors may be -used to endorse or promote products derived from this software without specific -prior written permission. +3. Neither the name of the copyright holder nor the names of its contributors +may be used to endorse or promote products derived from this software without +specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED diff --git a/Makefile.inc b/Makefile.inc index e3c100e96..26588e812 100644 --- a/Makefile.inc +++ b/Makefile.inc @@ -14,6 +14,7 @@ ODP_INCLUDES += \ -I$(top_srcdir)/platform/@with_platform@/include \ -I$(top_srcdir)/platform/@with_platform@/arch/@ARCH_DIR@ \ -I$(top_srcdir)/platform/@with_platform@/arch/default \ + -I$(top_srcdir)/platform/@with_platform@/arch/common \ -I$(top_srcdir)/platform/@with_platform@/include-abi endif @@ -1,7 +1,6 @@ -Copyright (c) 2013-2019, Linaro Limited -All rights reserved. - -SPDX-License-Identifier: BSD-3-Clause +SPDX-License-Identifier: BSD-3-Clause +Copyright (c) 2013-2019 Linaro Limited +Copyright (c) 2019-2023 OpenFastPath Foundation OpenDataPlane (ODP) project web page: https://www.opendataplane.org @@ -40,8 +39,10 @@ How to build: make Licensing: - ODP project uses BSD-3-Clause license as the default license. See LICENSE - file for details. + The default license for ODP project is BSD-3-Clause (see LICENSE file). + SPDX short-form identifiers (https://spdx.dev/ids/) are used in project + source files to identify the used license. The SPDX license list can be + found from https://spdx.org/licenses/. Mailing list: odp@lists.opendataplane.org diff --git a/configure.ac b/configure.ac index c397338ae..b0e16930e 100644 --- a/configure.ac +++ b/configure.ac @@ -3,8 +3,8 @@ AC_PREREQ([2.5]) # ODP API version ########################################################################## m4_define([odp_version_generation], [1]) -m4_define([odp_version_major], [41]) -m4_define([odp_version_minor], [1]) +m4_define([odp_version_major], [42]) +m4_define([odp_version_minor], [0]) m4_define([odp_version_patch], [0]) m4_define([odp_version_api], diff --git a/include/Makefile.am b/include/Makefile.am index 81d0253e5..ea3f17add 100644 --- a/include/Makefile.am +++ b/include/Makefile.am @@ -159,6 +159,7 @@ odpapiabidefaultinclude_HEADERS = \ odp/api/abi-default/crypto.h \ odp/api/abi-default/crypto_types.h \ odp/api/abi-default/debug.h \ + odp/api/abi-default/dma.h \ odp/api/abi-default/dma_types.h \ odp/api/abi-default/errno.h \ odp/api/abi-default/event.h \ @@ -220,6 +221,7 @@ odpapiabiarchinclude_HEADERS = \ odp/arch/arm32-linux/odp/api/abi/crypto.h \ odp/arch/arm32-linux/odp/api/abi/crypto_types.h \ odp/arch/arm32-linux/odp/api/abi/debug.h \ + odp/arch/arm32-linux/odp/api/abi/dma.h \ odp/arch/arm32-linux/odp/api/abi/dma_types.h \ odp/arch/arm32-linux/odp/api/abi/errno.h \ odp/arch/arm32-linux/odp/api/abi/event.h \ @@ -277,6 +279,7 @@ odpapiabiarchinclude_HEADERS = \ odp/arch/arm64-linux/odp/api/abi/crypto.h \ odp/arch/arm64-linux/odp/api/abi/crypto_types.h \ odp/arch/arm64-linux/odp/api/abi/debug.h \ + odp/arch/arm64-linux/odp/api/abi/dma.h \ odp/arch/arm64-linux/odp/api/abi/dma_types.h \ odp/arch/arm64-linux/odp/api/abi/errno.h \ odp/arch/arm64-linux/odp/api/abi/event.h \ @@ -334,6 +337,7 @@ odpapiabiarchinclude_HEADERS = \ odp/arch/default-linux/odp/api/abi/crypto.h \ odp/arch/default-linux/odp/api/abi/crypto_types.h \ odp/arch/default-linux/odp/api/abi/debug.h \ + odp/arch/default-linux/odp/api/abi/dma.h \ odp/arch/default-linux/odp/api/abi/dma_types.h \ odp/arch/default-linux/odp/api/abi/errno.h \ odp/arch/default-linux/odp/api/abi/event.h \ @@ -391,6 +395,7 @@ odpapiabiarchinclude_HEADERS = \ odp/arch/power64-linux/odp/api/abi/crypto.h \ odp/arch/power64-linux/odp/api/abi/crypto_types.h \ odp/arch/power64-linux/odp/api/abi/debug.h \ + odp/arch/power64-linux/odp/api/abi/dma.h \ odp/arch/power64-linux/odp/api/abi/dma_types.h \ odp/arch/power64-linux/odp/api/abi/errno.h \ odp/arch/power64-linux/odp/api/abi/event.h \ @@ -448,6 +453,7 @@ odpapiabiarchinclude_HEADERS = \ odp/arch/x86_32-linux/odp/api/abi/crypto.h \ odp/arch/x86_32-linux/odp/api/abi/crypto_types.h \ odp/arch/x86_32-linux/odp/api/abi/debug.h \ + odp/arch/x86_32-linux/odp/api/abi/dma.h \ odp/arch/x86_32-linux/odp/api/abi/dma_types.h \ odp/arch/x86_32-linux/odp/api/abi/errno.h \ odp/arch/x86_32-linux/odp/api/abi/event.h \ @@ -505,6 +511,7 @@ odpapiabiarchinclude_HEADERS = \ odp/arch/x86_64-linux/odp/api/abi/crypto.h \ odp/arch/x86_64-linux/odp/api/abi/crypto_types.h \ odp/arch/x86_64-linux/odp/api/abi/debug.h \ + odp/arch/x86_64-linux/odp/api/abi/dma.h \ odp/arch/x86_64-linux/odp/api/abi/dma_types.h \ odp/arch/x86_64-linux/odp/api/abi/errno.h \ odp/arch/x86_64-linux/odp/api/abi/event.h \ diff --git a/include/odp/api/abi-default/dma.h b/include/odp/api/abi-default/dma.h new file mode 100644 index 000000000..e7e0ad970 --- /dev/null +++ b/include/odp/api/abi-default/dma.h @@ -0,0 +1,20 @@ +/* Copyright (c) 2023, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_ABI_DMA_H_ +#define ODP_ABI_DMA_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +/* Empty header required due to the inline functions */ + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/include/odp/api/dma.h b/include/odp/api/dma.h index 4720f57c9..10e00a506 100644 --- a/include/odp/api/dma.h +++ b/include/odp/api/dma.h @@ -17,6 +17,8 @@ extern "C" { #endif +#include <odp/api/abi/dma.h> + #include <odp/api/spec/dma.h> #ifdef __cplusplus diff --git a/include/odp/api/spec/crypto.h b/include/odp/api/spec/crypto.h index 1c477336e..453eb9eeb 100644 --- a/include/odp/api/spec/crypto.h +++ b/include/odp/api/spec/crypto.h @@ -255,9 +255,7 @@ int odp_crypto_result(odp_crypto_packet_result_t *result, * input packet to the output packet and then the crypto operation * was applied to the output packet. * - * Crypto range and auth range of null cipher and auth algorithms are - * ignored, i.e. not copied in the output packet. Auth range of (AEAD) - * algorithms that ignore auth range is not copied. + * Auth range of (AEAD) algorithms that ignore auth range is not copied. * * The offset of the crypto range and auth range in the output packet is * the same as in the input packet, adjusted by dst_offset_shift operation diff --git a/include/odp/api/spec/crypto_types.h b/include/odp/api/spec/crypto_types.h index 46d9ae3f7..6aa3f38a6 100644 --- a/include/odp/api/spec/crypto_types.h +++ b/include/odp/api/spec/crypto_types.h @@ -1,5 +1,5 @@ /* Copyright (c) 2014-2018, Linaro Limited - * Copyright (c) 2021-2022, Nokia + * Copyright (c) 2021-2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -509,7 +509,7 @@ typedef struct odp_crypto_key { */ typedef enum odp_crypto_op_type_t { /** - * Input packet data and metadata are copied in the output packet + * Input packet data and metadata are copied to the output packet * and then processed. Output packet is allocated by the caller * or by ODP. * @@ -518,7 +518,7 @@ typedef enum odp_crypto_op_type_t { ODP_CRYPTO_OP_TYPE_LEGACY, /** - * Input packet data and metadata are copied in the output packet + * Input packet data and metadata are copied to the output packet * and then processed. Output packet is allocated by ODP. */ ODP_CRYPTO_OP_TYPE_BASIC, @@ -560,6 +560,30 @@ typedef struct odp_crypto_session_param_t { */ odp_crypto_op_type_t op_type; + /** Cipher range unit + * + * When this flag is true, cipher range offset and length are in bits. + * Otherwise the offset and length are in bytes. + * + * If cipher capabilities do not include bit_mode, setting this to + * true causes a session creation failure. + * + * The default value is false. + */ + odp_bool_t cipher_range_in_bits; + + /** Auth range unit + * + * When this flag is true, auth range offset and length are in bits. + * Otherwise the offset and length are in bytes. + * + * If auth capabilities do not include bit_mode, setting this to + * true causes a session creation failure. + * + * The default value is false. + */ + odp_bool_t auth_range_in_bits; + /** Authenticate cipher vs. plain text * * Controls ordering of authentication and cipher operations, @@ -725,23 +749,41 @@ typedef struct odp_crypto_packet_op_param_t { /** Data range to be ciphered. * + * The range is given in bits or bytes as configured at session + * creation. + * * Ignored by the null cipher with operation types other than - * ODP_CRYPTO_OP_TYPE_OOP. Must be set to zero range (zero offset - * and zero length) with the null cipher used with the out-of-place - * operation type. + * ODP_CRYPTO_OP_TYPE_OOP. + * + * With the OOP operation type the cipher range is copied to the + * output packet even with the null cipher. Non-zero-length ranges + * are not necessarily supported with the null cipher and the OOP + * operation type. If the requested range is not supported, the + * crypto operation will fail. The failure is indicated through + * odp_crypto_result() or through a negative return value of + * odp_crypto_op()/odp_crypto_op_enq(). **/ odp_packet_data_range_t cipher_range; /** Data range to be authenticated * + * The range is given in bits or bytes as configured at session + * creation. + * * The value is ignored with authenticated encryption algorithms, * such as AES-GCM, which authenticate data in the cipher range * and the AAD. * * Ignored by the null auth algorithm with operation types other than - * ODP_CRYPTO_OP_TYPE_OOP. Must be set to zero range (zero offset - * and zero length) with the null cipher used with the out-of-place - * operation type. + * ODP_CRYPTO_OP_TYPE_OOP. + * + * With the OOP operation type the auth range is copied to the + * output packet even with the null auth algorithm. Non-zero-length + * ranges are not necessarily supported with the null algorithm and + * the OOP operation type. If the requested range is not supported, + * the crypto operation will fail. The failure is indicated through + * odp_crypto_result() or through a negative return value of + * odp_crypto_op()/odp_crypto_op_enq(). * * As a special case AES-GMAC uses this field instead of aad_ptr * for the data bytes to be authenticated. @@ -937,22 +979,23 @@ typedef struct odp_crypto_cipher_capability_t { /** IV length in bytes */ uint32_t iv_len; - /** Cipher is operating in bitwise mode + /** Cipher supports bit mode * - * This cipher works on series of bits, rather than sequences of bytes: - * cipher_range in odp_crypto_op_param_t and - * odp_crypto_packet_op_param_t will use bits, rather than bytes. + * This cipher can work on a range of bits in addition to a range of + * bytes. When this capability is not present, only byte ranges are + * supported. The unit of cipher range is selected at session creation + * through the cipher_range_in_bits session parameter. * - * Note: data buffer MUST start on the byte boundary, using offset - * which is not divisible by 8 is unsupported and will result in - * unspecified behaviour. + * Note: In bit mode the cipher range must start on a byte boundary. + * Using an offset which is not divisible by 8 will result in + * undefined behaviour. * - * Note2: If the data length is not a multiple of 8, the remaining - * bits of the data in the last byte of the input/output will be the - * most significant bits, i.e. the most significant bit is considered - * to be the first bit of a byte for the purpose of input and output - * data range. The output bits that fall out of the output range are - * undefined. + * Note2: If the range length in bit mode is not a multiple of 8, + * the remaining bits of the data in the last byte of the input/output + * will be the most significant bits, i.e. the most significant bit is + * considered to be the first bit of a byte for the purpose of input + * and output data range. The output bits that fall out of the output + * range are undefined. */ odp_bool_t bit_mode; @@ -984,22 +1027,23 @@ typedef struct odp_crypto_auth_capability_t { uint32_t inc; } aad_len; - /** Auth is operating in bitstring mode + /** Auth algorithm supports bit mode * - * This auth works on series of bits, rather than sequences of bytes: - * auth_range in odp_crypto_op_param_t and - * odp_crypto_packet_op_param_t will use bits, rather than bytes. + * This auth algorithm can work on a range of bits in addition to + * a range of bytes. When this capability is not present, only byte + * ranges are supported. The unit of auth range is selected at session + * creation through the auth_range_in_bits session parameter. * - * Note: data buffer MUST start on the byte boundary, using offset - * which is not divisible by 8 is unsupported and will result in - * unpredictable behaviour. + * Note: In bit mode the auth range must start on a byte boundary. + * Using an offset which is not divisible by 8 will result in + * undefined behaviour. * - * Note2: If the data length is not a multiple of 8, the remaining - * bits of the data in the last byte of the input/output will be the - * most significant bits, i.e. the most significant bit is considered - * to be the first bit of a byte for the purpose of input and output - * data range. The output bits that fall out of the output range are - * undefined. + * Note2: If the range length in bit mode is not a multiple of 8, + * the remaining bits of the data in the last byte of the input/output + * will be the most significant bits, i.e. the most significant bit is + * considered to be the first bit of a byte for the purpose of input + * and output data range. The output bits that fall out of the output + * range are undefined. */ odp_bool_t bit_mode; diff --git a/include/odp/api/spec/dma_types.h b/include/odp/api/spec/dma_types.h index 26350e998..5e18faab2 100644 --- a/include/odp/api/spec/dma_types.h +++ b/include/odp/api/spec/dma_types.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2021-2022, Nokia +/* Copyright (c) 2021-2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -80,6 +80,12 @@ typedef struct odp_dma_pool_capability_t { /** Maximum user area size in bytes */ uint32_t max_uarea_size; + /** Pool user area persistence + * + * See buf.uarea_persistence of odp_pool_capability_t for details + * (odp_pool_capability_t::uarea_persistence). */ + odp_bool_t uarea_persistence; + /** Minimum size of thread local cache */ uint32_t min_cache_size; @@ -104,6 +110,19 @@ typedef struct odp_dma_pool_param_t { */ uint32_t uarea_size; + /** Parameters for user area initialization */ + struct { + /** See uarea_init.init_fn of odp_pool_param_t for details + * (odp_pool_param_t::init_fn). Function is called during + * odp_dma_pool_create(). */ + void (*init_fn)(void *uarea, uint32_t size, void *args, uint32_t index); + + /** See uarea_init.args of odp_pool_param_t for details + * (odp_pool_param_t::args). */ + void *args; + + } uarea_init; + /** Maximum number of events cached locally per thread * * See odp_pool_param_t::cache_size documentation for details. Valid values range from @@ -365,41 +384,39 @@ typedef struct odp_dma_param_t { * DMA segment */ typedef struct odp_dma_seg_t { - /** Segment start */ + /** Segment start address or packet handle */ union { /** Segment start address in memory * - * Defines segment start when data format is ODP_DMA_FORMAT_ADDR. Ignored with + * Defines segment start when data format is #ODP_DMA_FORMAT_ADDR. Ignored with * other data formats. */ void *addr; - /** Segment start as an offset into a packet */ - struct { - /** Packet handle - * - * Defines the packet when data format is ODP_DMA_FORMAT_PACKET. Ignored - * with other data formats. */ - odp_packet_t packet; + /** Packet handle + * + * Defines the packet when data format is #ODP_DMA_FORMAT_PACKET. Ignored + * with other data formats. */ + odp_packet_t packet; - /** Segment start offset into the packet - * - * Defines segment start when data format is ODP_DMA_FORMAT_PACKET. - * The offset is calculated from odp_packet_data() position, and the value - * must not exceed odp_packet_len(). - */ - uint32_t offset; - }; }; /** Segment length in bytes * * Defines segment length with all data formats. The maximum value is defined by - * max_seg_len capability. When data format is ODP_DMA_FORMAT_PACKET, the value must not + * max_seg_len capability. When data format is #ODP_DMA_FORMAT_PACKET, the value must not * exceed odp_packet_len() - 'offset'. */ uint32_t len; + /** Segment start offset into the packet + * + * Defines segment start within the packet data. The offset is calculated from + * odp_packet_data() position, and the value must not exceed odp_packet_len(). + * Ignored when data format is other than #ODP_DMA_FORMAT_PACKET. + */ + uint32_t offset; + /** Segment hints * * Depending on the implementation, setting these hints may improve performance. diff --git a/include/odp/api/spec/event.h b/include/odp/api/spec/event.h index 9dc614329..65b5dd7ff 100644 --- a/include/odp/api/spec/event.h +++ b/include/odp/api/spec/event.h @@ -64,6 +64,22 @@ odp_event_type_t odp_event_types(odp_event_t event, odp_event_subtype_t *subtype); /** + * Event types and subtypes of multiple events + * + * Outputs the event types and subtypes (optional) of all given events into the + * output arrays. Application can pass NULL as 'subtype' parameter if subtype + * values are not required. The types are written in the same order as input + * events. + * + * @param event Array of event handles + * @param[out] type Event type array for output + * @param[out] subtype Event subtype array for output, or NULL + * @param num Number of events, event types, and optionally subtypes + */ +void odp_event_types_multi(const odp_event_t event[], odp_event_type_t type[], + odp_event_subtype_t subtype[], int num); + +/** * Event type of multiple events * * Returns the number of first events in the array which have the same event @@ -94,6 +110,28 @@ int odp_event_type_multi(const odp_event_t event[], int num, void *odp_event_user_area(odp_event_t event); /** + * Event user area and flag + * + * Returns pointer to the user area and outputs value of user flag associated + * with the event. The user area maps to the user area of underlying event type + * (e.g. odp_packet_user_area() for packets). If the event does not have user + * area, NULL is returned. + * + * The user flag maps to the user flag value of underlying event type (e.g. + * odp_packet_user_flag() for packets). If the event does not have user flag, a + * negative value is outputted. + * + * @param event Event handle + * @param[out] flag User flag value pointer for output. >0 if user flag is + * set, 0 if flags is not set, or <0 if event does not have + * user flag. + * + * @return Pointer to the user area of the event + * @retval NULL The event does not have user area + */ +void *odp_event_user_area_and_flag(odp_event_t event, int *flag); + +/** * Filter and convert packet events * * Checks event type of all input events, converts all packet events and outputs diff --git a/include/odp/api/spec/init.h b/include/odp/api/spec/init.h index c4f64d4fd..371222267 100644 --- a/include/odp/api/spec/init.h +++ b/include/odp/api/spec/init.h @@ -51,6 +51,9 @@ typedef enum { /** Debug */ ODP_LOG_DBG, + /** Warning */ + ODP_LOG_WARN, + /** Error */ ODP_LOG_ERR, diff --git a/include/odp/api/spec/packet.h b/include/odp/api/spec/packet.h index 9f41edf1a..267bf819c 100644 --- a/include/odp/api/spec/packet.h +++ b/include/odp/api/spec/packet.h @@ -1502,7 +1502,7 @@ uint32_t odp_packet_user_area_size(odp_packet_t pkt); * @param pkt Packet handle * * @retval 0 User flag is clear - * @retval !0 User flag is set + * @retval >0 User flag is set */ int odp_packet_user_flag(odp_packet_t pkt); @@ -2306,7 +2306,7 @@ void *odp_packet_vector_user_area(odp_packet_vector_t pktv); * @param pktv Packet vector handle * * @retval 0 User flag is clear - * @retval !0 User flag is set + * @retval >0 User flag is set */ int odp_packet_vector_user_flag(odp_packet_vector_t pktv); diff --git a/include/odp/api/spec/pool_types.h b/include/odp/api/spec/pool_types.h index cf1051a07..81974abc6 100644 --- a/include/odp/api/spec/pool_types.h +++ b/include/odp/api/spec/pool_types.h @@ -209,6 +209,19 @@ typedef struct odp_pool_capability_t { /** Maximum user area size in bytes */ uint32_t max_uarea_size; + /** Pool user area persistence + * + * When supported, implementation does not overwrite buffer user area + * content at any point of buffer lifetime nor after freeing a buffer + * back into pool. + * + * 0: User area content is maintained throughout regular buffer usage + * after allocation, but may be modified after free (default) + * 1: User area content is maintained throughout regular buffer usage + * and additionally also after buffer is freed into the pool (between + * buffer free and allocation) */ + odp_bool_t uarea_persistence; + /** Minimum size of thread local cache */ uint32_t min_cache_size; @@ -288,6 +301,11 @@ typedef struct odp_pool_capability_t { /** Maximum user area size in bytes */ uint32_t max_uarea_size; + /** Pool user area persistence + * + * See buf.uarea_persistence for details. */ + odp_bool_t uarea_persistence; + /** Maximum number of subparameters * * Maximum number of packet pool subparameters. Valid range is @@ -318,6 +336,11 @@ typedef struct odp_pool_capability_t { /** Maximum user area size in bytes */ uint32_t max_uarea_size; + /** Pool user area persistence + * + * See buf.uarea_persistence for details. */ + odp_bool_t uarea_persistence; + /** Minimum size of thread local cache */ uint32_t min_cache_size; @@ -345,6 +368,11 @@ typedef struct odp_pool_capability_t { /** Maximum user area size in bytes */ uint32_t max_uarea_size; + /** Pool user area persistence + * + * See buf.uarea_persistence for details. */ + odp_bool_t uarea_persistence; + /** Minimum size of thread local cache */ uint32_t min_cache_size; @@ -581,6 +609,30 @@ typedef struct odp_pool_param_t { uint32_t cache_size; } vector; + /** Parameters for user area initialization */ + struct { + /** User area initialization function + * + * Application defined user area initialization function to be + * called for each event of the pool during odp_pool_create(). Ignored + * if user area persistence is not supported + * (odp_pool_capability_t::uarea_persistence) or pool will not have any + * user area. The default value is NULL. + * + * @param uarea Pointer to the user area of an event + * @param size User area size + * @param args Pointer to application defined arguments + * @param index Index of the event (0..num events in pool - 1), not + * necessarily in order + */ + void (*init_fn)(void *uarea, uint32_t size, void *args, uint32_t index); + + /** Pointer to application defined arguments to be passed to every call + * of init_fn. The default value is NULL. + */ + void *args; + } uarea_init; + /** * Configure statistics counters * @@ -713,6 +765,12 @@ typedef struct odp_pool_ext_capability_t { /** Maximum user area size in bytes */ uint32_t max_uarea_size; + /** Pool user area persistence + * + * See buf.uarea_persistence of odp_pool_capability_t for details + * (odp_pool_capability_t::uarea_persistence). */ + odp_bool_t uarea_persistence; + } pkt; } odp_pool_ext_capability_t; @@ -724,6 +782,23 @@ typedef struct odp_pool_ext_param_t { /** Pool type */ odp_pool_type_t type; + /** Parameters for user area initialization */ + struct { + /** See uarea_init.init_fn of odp_pool_param_t for details + * (odp_pool_param_t::init_fn). However, note that with external memory + * pools, this function is called during memory population and not during + * pool creation (odp_pool_ext_populate()). Depending on the implementation, + * the function may be called each time pool is being populated with + * odp_pool_ext_populate() or during the last population call + * (odp_pool_ext_populate() with #ODP_POOL_POPULATE_DONE). */ + void (*init_fn)(void *uarea, uint32_t size, void *args, uint32_t index); + + /** See uarea_init.args of odp_pool_param_t for details + * (odp_pool_param_t::args). */ + void *args; + + } uarea_init; + /** Maximum thread local cache size for the pool * * Valid value range is from min_cache_size to max_cache_size capability. diff --git a/include/odp/api/spec/sync.h b/include/odp/api/spec/sync.h index 0b75ffaee..18272af88 100644 --- a/include/odp/api/spec/sync.h +++ b/include/odp/api/spec/sync.h @@ -1,7 +1,6 @@ -/* Copyright (c) 2013-2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2013-2018 Linaro Limited + * Copyright (c) 2023 Nokia */ /** @@ -23,15 +22,24 @@ extern "C" { * @details * <b> Memory barriers </b> * - * Memory barriers enforce ordering of memory load and store operations - * specified before and after the barrier. These barriers may affect both - * compiler optimizations and CPU out-of-order execution. All ODP - * synchronization mechanisms (e.g. execution barriers, locks, queues, etc ) - * include all necessary memory barriers, so these calls are not needed when - * using those. Also ODP atomic operations have memory ordered versions. These - * explicit barriers may be needed when thread synchronization is based on - * a non-ODP defined mechanism. Depending on the HW platform, heavy usage of - * memory barriers may cause significant performance degradation. + * A memory barrier enforces the order between memory accesses (loads and/or stores) + * specified (in program order) before the barrier with those specified after the barrier. + * A barrier may affect both compiler optimizations and CPU out-of-order execution. Depending + * on the used HW platform and barrier types, heavy usage of barriers may cause significant + * performance degradation. + * + * An application may use these memory barrier functions e.g. to build a synchronization + * mechanism between its threads in shared memory, or when it accesses memory mapped registers + * of a device. + * + * An application does not need to use these memory barriers when using other ODP APIs for thread + * synchronization (execution barriers, spinlocks, etc.), or when exchanging data through ODP API + * mechanisms (queues, stashes, etc.). Those ODP calls include necessary (acquire and release) + * memory barriers to maintain coherency between data producers and consumers. + * + * Some ODP atomic operations include a memory barrier - see for example odp_atomic_load_acq_u32() + * or odp_atomic_store_rel_u32(). Application may use also those (non-relaxed) atomic operations + * to enforce memory ordering while using atomic variables. * * @{ */ @@ -80,6 +88,39 @@ void odp_mb_acquire(void); void odp_mb_full(void); /** + * Memory barrier for load and store synchronization + * + * This memory barrier ensures that all memory accesses (loads and stores) specified before the + * barrier (in program order) are complete prior to any memory access specified after the barrier + * begins execution. + * + * This is a stronger barrier than odp_mb_full(), as in addition to visibility order also memory + * access completion is ensured. The barrier may be useful e.g. when synchronizing loads and stores + * into memory mapped registers of a device. + */ +void odp_mb_sync(void); + +/** + * Memory barrier for load synchronization + * + * This memory barrier ensures that all memory loads specified before the barrier (in program + * order) are complete prior to any memory load specified after the barrier begins execution. + * + * The barrier may be useful e.g. when synchronizing loads from memory mapped registers of a device. + */ +void odp_mb_sync_load(void); + +/** + * Memory synchronization barrier for stores + * + * This memory barrier ensures that all memory stores specified before the barrier (in program + * order) are complete prior to any memory store specified after the barrier begins execution. + * + * The barrier may be useful e.g. when synchronizing stores to memory mapped registers of a device. + */ +void odp_mb_sync_store(void); + +/** * @} */ diff --git a/include/odp/api/spec/timer.h b/include/odp/api/spec/timer.h index bbb439692..a6ad6f2b3 100644 --- a/include/odp/api/spec/timer.h +++ b/include/odp/api/spec/timer.h @@ -1,5 +1,5 @@ /* Copyright (c) 2013-2018, Linaro Limited - * Copyright (c) 2019-2022, Nokia + * Copyright (c) 2019-2023, Nokia * * All rights reserved. * @@ -227,13 +227,20 @@ int odp_timer_pool_info(odp_timer_pool_t timer_pool, /** * Allocate a timer * - * Create a timer (allocating all necessary resources e.g. timeout event) from - * the timer pool. The user_ptr is copied to timeouts and can be retrieved - * using the odp_timeout_user_ptr() call. + * Allocates a timer from the timer pool. Depending on timer type, the allocated timer is started + * with either odp_timer_start() or odp_timer_periodic_start() call. A timer may be reused multiple + * times before freeing it back into the timer pool. + * + * When timer expires, the timeout event defined in timer start parameters (see + * odp_timer_start_t::tmo_ev or odp_timer_periodic_start_t::tmo_ev) is sent into the provided + * destination queue. + * + * The provided user pointer value is copied into timeout events when the event type is + * ODP_EVENT_TIMEOUT. The value can be retrieved from an event with odp_timeout_user_ptr() call. * * @param timer_pool Timer pool - * @param queue Destination queue for timeout notifications - * @param user_ptr User defined pointer or NULL to be copied to timeouts + * @param queue Destination queue for timeout events + * @param user_ptr User defined pointer value or NULL * * @return Timer handle on success * @retval ODP_TIMER_INVALID on failure @@ -286,9 +293,12 @@ int odp_timer_start(odp_timer_t timer, const odp_timer_start_t *start_param); * Restart a timer * * A successful restart call updates the expiration time of an active timer. The timeout event - * is not changed. The timer is not modified when a failure is returned. The call returns - * ODP_TIMER_FAIL if the timer has expired already, or is so close to expire that it cannot be - * restarted anymore. + * is not changed. + * + * The timer is not modified when a failure is returned. The call returns #ODP_TIMER_FAIL if + * the timer has expired already, or is so close to expire that it cannot be restarted anymore. + * A failure is returned also when the new expiration time is too near to the current time + * (#ODP_TIMER_TOO_NEAR) or too far from the current time (#ODP_TIMER_TOO_FAR). * * The new expiration time is passed the same way as with odp_timer_start() call. * @@ -478,6 +488,17 @@ int odp_timer_cancel(odp_timer_t timer, odp_event_t *tmo_ev); odp_timeout_t odp_timeout_from_event(odp_event_t ev); /** + * Convert multiple timeout events to timeout handles + * + * All events must be of type ODP_EVENT_TIMEOUT. + * + * @param[out] tmo Timeout handle array for output + * @param ev Array of event handles to convert + * @param num Number of timeouts and events + */ +void odp_timeout_from_event_multi(odp_timeout_t tmo[], const odp_event_t ev[], int num); + +/** * Convert timeout handle to event handle * * @param tmo Timeout handle diff --git a/include/odp/arch/arm32-linux/odp/api/abi/dma.h b/include/odp/arch/arm32-linux/odp/api/abi/dma.h new file mode 100644 index 000000000..f4656c4cf --- /dev/null +++ b/include/odp/arch/arm32-linux/odp/api/abi/dma.h @@ -0,0 +1,7 @@ +/* Copyright (c) 2023, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <odp/api/abi-default/dma.h> diff --git a/include/odp/arch/arm64-linux/odp/api/abi/dma.h b/include/odp/arch/arm64-linux/odp/api/abi/dma.h new file mode 100644 index 000000000..f4656c4cf --- /dev/null +++ b/include/odp/arch/arm64-linux/odp/api/abi/dma.h @@ -0,0 +1,7 @@ +/* Copyright (c) 2023, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <odp/api/abi-default/dma.h> diff --git a/include/odp/arch/default-linux/odp/api/abi/dma.h b/include/odp/arch/default-linux/odp/api/abi/dma.h new file mode 100644 index 000000000..f4656c4cf --- /dev/null +++ b/include/odp/arch/default-linux/odp/api/abi/dma.h @@ -0,0 +1,7 @@ +/* Copyright (c) 2023, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <odp/api/abi-default/dma.h> diff --git a/include/odp/arch/power64-linux/odp/api/abi/dma.h b/include/odp/arch/power64-linux/odp/api/abi/dma.h new file mode 100644 index 000000000..f4656c4cf --- /dev/null +++ b/include/odp/arch/power64-linux/odp/api/abi/dma.h @@ -0,0 +1,7 @@ +/* Copyright (c) 2023, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <odp/api/abi-default/dma.h> diff --git a/include/odp/arch/x86_32-linux/odp/api/abi/dma.h b/include/odp/arch/x86_32-linux/odp/api/abi/dma.h new file mode 100644 index 000000000..f4656c4cf --- /dev/null +++ b/include/odp/arch/x86_32-linux/odp/api/abi/dma.h @@ -0,0 +1,7 @@ +/* Copyright (c) 2023, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <odp/api/abi-default/dma.h> diff --git a/include/odp/arch/x86_64-linux/odp/api/abi/dma.h b/include/odp/arch/x86_64-linux/odp/api/abi/dma.h new file mode 100644 index 000000000..f4656c4cf --- /dev/null +++ b/include/odp/arch/x86_64-linux/odp/api/abi/dma.h @@ -0,0 +1,7 @@ +/* Copyright (c) 2023, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <odp/api/abi-default/dma.h> diff --git a/platform/linux-dpdk/Makefile.am b/platform/linux-dpdk/Makefile.am index 252b34fb2..ad608c4e1 100644 --- a/platform/linux-dpdk/Makefile.am +++ b/platform/linux-dpdk/Makefile.am @@ -33,6 +33,7 @@ odpapiplatinclude_HEADERS = \ include/odp/api/plat/byteorder_inlines.h \ include/odp/api/plat/cpu_inlines.h \ include/odp/api/plat/crypto_inlines.h \ + include/odp/api/plat/dma_inlines.h \ include/odp/api/plat/debug_inlines.h \ include/odp/api/plat/event_inlines.h \ include/odp/api/plat/event_inline_types.h \ @@ -78,6 +79,7 @@ odpapiabiarchinclude_HEADERS += \ include-abi/odp/api/abi/crypto.h \ include-abi/odp/api/abi/crypto_types.h \ include-abi/odp/api/abi/debug.h \ + include-abi/odp/api/abi/dma.h \ include-abi/odp/api/abi/dma_types.h \ include-abi/odp/api/abi/errno.h \ include-abi/odp/api/abi/event.h \ @@ -259,6 +261,7 @@ __LIB__libodp_dpdk_la_SOURCES += \ ../linux-generic/odp_byteorder_api.c \ ../linux-generic/odp_cpu_api.c \ ../linux-generic/odp_crypto_api.c \ + ../linux-generic/odp_dma_api.c \ ../linux-generic/odp_event_api.c \ ../linux-generic/odp_hash_api.c \ ../linux-generic/odp_ipsec_api.c \ @@ -283,18 +286,17 @@ endif if ARCH_IS_ARM __LIB__libodp_dpdk_la_SOURCES += arch/default/odp_atomic.c \ arch/default/odp_cpu_cycles.c \ - arch/default/odp_global_time.c \ arch/default/odp_hash_crc32.c \ arch/default/odp_random.c \ arch/arm/odp_sysinfo_parse.c -odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_time.h \ - arch/default/odp/api/abi/hash_crc32.h +odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/hash_crc32.h if !ODP_ABI_COMPAT odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ arch/default/odp/api/abi/atomic_inlines.h \ arch/default/odp/api/abi/cpu_generic.h \ arch/arm/odp/api/abi/cpu_inlines.h \ - arch/arm/odp/api/abi/cpu.h + arch/arm/odp/api/abi/cpu.h \ + arch/default/odp/api/abi/sync_inlines.h endif noinst_HEADERS += arch/arm/odp_atomic.h \ arch/arm/odp_cpu.h \ @@ -310,19 +312,18 @@ if ARCH_IS_AARCH64 __LIB__libodp_dpdk_la_SOURCES += arch/aarch64/odp_atomic.c \ arch/default/odp_cpu_cycles.c \ arch/aarch64/cpu_flags.c \ - arch/aarch64/odp_global_time.c \ arch/default/odp_hash_crc32.c \ arch/default/odp_random.c \ arch/aarch64/odp_sysinfo_parse.c -odpapiabiarchinclude_HEADERS += arch/aarch64/odp/api/abi/cpu_time.h \ - arch/aarch64/odp/api/abi/hash_crc32.h +odpapiabiarchinclude_HEADERS += arch/aarch64/odp/api/abi/hash_crc32.h if !ODP_ABI_COMPAT odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ arch/aarch64/odp/api/abi/atomic_inlines.h \ arch/aarch64/odp/api/abi/atomic.h \ arch/default/odp/api/abi/cpu_generic.h \ arch/aarch64/odp/api/abi/cpu_inlines.h \ - arch/aarch64/odp/api/abi/cpu.h + arch/aarch64/odp/api/abi/cpu.h \ + arch/aarch64/odp/api/abi/sync_inlines.h endif noinst_HEADERS += arch/aarch64/odp_atomic.h \ arch/aarch64/odp_cpu.h \ @@ -334,18 +335,17 @@ endif if ARCH_IS_DEFAULT __LIB__libodp_dpdk_la_SOURCES += arch/default/odp_atomic.c \ arch/default/odp_cpu_cycles.c \ - arch/default/odp_global_time.c \ arch/default/odp_hash_crc32.c \ arch/default/odp_random.c \ arch/default/odp_sysinfo_parse.c -odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_time.h \ - arch/default/odp/api/abi/hash_crc32.h +odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/hash_crc32.h if !ODP_ABI_COMPAT odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ arch/default/odp/api/abi/atomic_inlines.h \ arch/default/odp/api/abi/cpu_generic.h \ arch/default/odp/api/abi/cpu_inlines.h \ - arch/default/odp/api/abi/cpu.h + arch/default/odp/api/abi/cpu.h \ + arch/default/odp/api/abi/sync_inlines.h endif noinst_HEADERS += arch/default/odp_atomic.h \ arch/default/odp_cpu.h \ @@ -355,18 +355,17 @@ endif if ARCH_IS_POWERPC __LIB__libodp_dpdk_la_SOURCES += arch/default/odp_atomic.c \ arch/default/odp_cpu_cycles.c \ - arch/default/odp_global_time.c \ arch/default/odp_hash_crc32.c \ arch/default/odp_random.c \ arch/powerpc/odp_sysinfo_parse.c -odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_time.h \ - arch/default/odp/api/abi/hash_crc32.h +odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/hash_crc32.h if !ODP_ABI_COMPAT odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ arch/default/odp/api/abi/atomic_inlines.h \ arch/default/odp/api/abi/cpu_generic.h \ arch/default/odp/api/abi/cpu_inlines.h \ - arch/powerpc/odp/api/abi/cpu.h + arch/powerpc/odp/api/abi/cpu.h \ + arch/default/odp/api/abi/sync_inlines.h endif noinst_HEADERS += arch/default/odp_atomic.h \ arch/default/odp_cpu.h \ @@ -377,18 +376,18 @@ if ARCH_IS_X86 __LIB__libodp_dpdk_la_SOURCES += arch/default/odp_atomic.c \ arch/x86/cpu_flags.c \ arch/x86/odp_cpu_cycles.c \ - arch/x86/odp_global_time.c \ arch/default/odp_hash_crc32.c \ arch/default/odp_random.c \ arch/x86/odp_sysinfo_parse.c odpapiabiarchinclude_HEADERS += arch/x86/odp/api/abi/cpu_rdtsc.h \ - arch/x86/odp/api/abi/cpu_time.h \ - arch/x86/odp/api/abi/hash_crc32.h + arch/x86/odp/api/abi/hash_crc32.h \ + arch/x86/odp/api/abi/time_cpu.h if !ODP_ABI_COMPAT odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ arch/default/odp/api/abi/atomic_inlines.h \ arch/x86/odp/api/abi/cpu_inlines.h \ - arch/x86/odp/api/abi/cpu.h + arch/x86/odp/api/abi/cpu.h \ + arch/x86/odp/api/abi/sync_inlines.h endif noinst_HEADERS += arch/x86/cpu_flags.h \ arch/x86/odp_cpu.h \ diff --git a/platform/linux-dpdk/arch/aarch64/odp/api/abi/cpu_time.h b/platform/linux-dpdk/arch/aarch64/odp/api/abi/cpu_time.h deleted file mode 120000 index 52719241f..000000000 --- a/platform/linux-dpdk/arch/aarch64/odp/api/abi/cpu_time.h +++ /dev/null @@ -1 +0,0 @@ -../../../../../../linux-generic/arch/aarch64/odp/api/abi/cpu_time.h
\ No newline at end of file diff --git a/platform/linux-dpdk/arch/aarch64/odp/api/abi/sync_inlines.h b/platform/linux-dpdk/arch/aarch64/odp/api/abi/sync_inlines.h new file mode 120000 index 000000000..1281f2376 --- /dev/null +++ b/platform/linux-dpdk/arch/aarch64/odp/api/abi/sync_inlines.h @@ -0,0 +1 @@ +../../../../../../linux-generic/arch/aarch64/odp/api/abi/sync_inlines.h
\ No newline at end of file diff --git a/platform/linux-dpdk/arch/aarch64/odp_global_time.c b/platform/linux-dpdk/arch/aarch64/odp_global_time.c deleted file mode 120000 index 1b7fa3c24..000000000 --- a/platform/linux-dpdk/arch/aarch64/odp_global_time.c +++ /dev/null @@ -1 +0,0 @@ -../../../linux-generic/arch/aarch64/odp_global_time.c
\ No newline at end of file diff --git a/platform/linux-dpdk/arch/default/odp/api/abi/cpu_time.h b/platform/linux-dpdk/arch/default/odp/api/abi/cpu_time.h deleted file mode 120000 index 941d6ccca..000000000 --- a/platform/linux-dpdk/arch/default/odp/api/abi/cpu_time.h +++ /dev/null @@ -1 +0,0 @@ -../../../../../../linux-generic/arch/default/odp/api/abi/cpu_time.h
\ No newline at end of file diff --git a/platform/linux-dpdk/arch/default/odp/api/abi/sync_inlines.h b/platform/linux-dpdk/arch/default/odp/api/abi/sync_inlines.h new file mode 120000 index 000000000..328ff07d6 --- /dev/null +++ b/platform/linux-dpdk/arch/default/odp/api/abi/sync_inlines.h @@ -0,0 +1 @@ +../../../../../../linux-generic/arch/default/odp/api/abi/sync_inlines.h
\ No newline at end of file diff --git a/platform/linux-dpdk/arch/default/odp_global_time.c b/platform/linux-dpdk/arch/default/odp_global_time.c deleted file mode 120000 index 3f54b1478..000000000 --- a/platform/linux-dpdk/arch/default/odp_global_time.c +++ /dev/null @@ -1 +0,0 @@ -../../../linux-generic/arch/default/odp_global_time.c
\ No newline at end of file diff --git a/platform/linux-dpdk/arch/x86/odp/api/abi/cpu_time.h b/platform/linux-dpdk/arch/x86/odp/api/abi/cpu_time.h deleted file mode 120000 index 2fb280b8c..000000000 --- a/platform/linux-dpdk/arch/x86/odp/api/abi/cpu_time.h +++ /dev/null @@ -1 +0,0 @@ -../../../../../../linux-generic/arch/x86/odp/api/abi/cpu_time.h
\ No newline at end of file diff --git a/platform/linux-dpdk/arch/x86/odp/api/abi/sync_inlines.h b/platform/linux-dpdk/arch/x86/odp/api/abi/sync_inlines.h new file mode 120000 index 000000000..d5dba2679 --- /dev/null +++ b/platform/linux-dpdk/arch/x86/odp/api/abi/sync_inlines.h @@ -0,0 +1 @@ +../../../../../../linux-generic/arch/x86/odp/api/abi/sync_inlines.h
\ No newline at end of file diff --git a/platform/linux-dpdk/arch/x86/odp/api/abi/time_cpu.h b/platform/linux-dpdk/arch/x86/odp/api/abi/time_cpu.h new file mode 100644 index 000000000..cc313cff1 --- /dev/null +++ b/platform/linux-dpdk/arch/x86/odp/api/abi/time_cpu.h @@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2023 Nokia + */ + +#ifndef ODP_ARCH_TIME_CPU_H_ +#define ODP_ARCH_TIME_CPU_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +int _odp_time_cpu_global_freq_is_const(void); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-dpdk/arch/x86/odp_global_time.c b/platform/linux-dpdk/arch/x86/odp_global_time.c deleted file mode 120000 index 33a50e23e..000000000 --- a/platform/linux-dpdk/arch/x86/odp_global_time.c +++ /dev/null @@ -1 +0,0 @@ -../../../linux-generic/arch/x86/odp_global_time.c
\ No newline at end of file diff --git a/platform/linux-dpdk/include-abi/odp/api/abi/dma.h b/platform/linux-dpdk/include-abi/odp/api/abi/dma.h new file mode 120000 index 000000000..112a7b5a7 --- /dev/null +++ b/platform/linux-dpdk/include-abi/odp/api/abi/dma.h @@ -0,0 +1 @@ +../../../../../linux-generic/include-abi/odp/api/abi/dma.h
\ No newline at end of file diff --git a/platform/linux-dpdk/include/odp/api/plat/dma_inlines.h b/platform/linux-dpdk/include/odp/api/plat/dma_inlines.h new file mode 120000 index 000000000..5b60f374e --- /dev/null +++ b/platform/linux-dpdk/include/odp/api/plat/dma_inlines.h @@ -0,0 +1 @@ +../../../../../linux-generic/include/odp/api/plat/dma_inlines.h
\ No newline at end of file diff --git a/platform/linux-dpdk/include/odp/api/plat/time_inlines.h b/platform/linux-dpdk/include/odp/api/plat/time_inlines.h index 4b1320300..8e4eec59c 100644 --- a/platform/linux-dpdk/include/odp/api/plat/time_inlines.h +++ b/platform/linux-dpdk/include/odp/api/plat/time_inlines.h @@ -12,125 +12,116 @@ #include <odp/api/hints.h> #include <odp/api/time_types.h> -#include <odp/api/abi/cpu_time.h> +#include <rte_config.h> +#include <rte_atomic.h> +#include <rte_cycles.h> #include <stdint.h> /** @cond _ODP_HIDE_FROM_DOXYGEN_ */ -#define _ODP_TIMESPEC_SIZE 16 -#define _ODP_TIME_GIGA_HZ 1000000000ULL - -typedef odp_time_t (*time_cur_fn)(void); -typedef odp_time_t (*time_cur_strict_fn)(void); -typedef uint64_t (*time_res_fn)(void); - -typedef struct time_handler_ { - time_cur_fn time_cur; - time_cur_strict_fn time_cur_strict; - time_res_fn time_res; - -} time_handler_t; - typedef struct _odp_time_global_t { - /* Storage space for struct timespec. Posix headers are not included - * here to avoid application exposure. */ - uint8_t timespec[_ODP_TIMESPEC_SIZE] ODP_ALIGNED(_ODP_TIMESPEC_SIZE); - - int use_hw; - uint64_t hw_start; - uint64_t hw_freq_hz; - /* DPDK specific */ - time_handler_t handler; + uint64_t start_cycles; + uint64_t freq_hz; } _odp_time_global_t; extern _odp_time_global_t _odp_time_glob; -odp_time_t _odp_timespec_cur(void); - -static inline odp_time_t _odp_time_cur_gen(void) +static inline odp_time_t _odp_time_cur(void) { - if (_odp_time_glob.use_hw) { - odp_time_t time; + odp_time_t time; - time.count = _odp_cpu_global_time() - _odp_time_glob.hw_start; - return time; - } + time.u64 = rte_get_timer_cycles() - _odp_time_glob.start_cycles; - return _odp_timespec_cur(); + return time; } -static inline odp_time_t _odp_time_cur_gen_strict(void) +static inline odp_time_t _odp_time_cur_strict(void) { - if (_odp_time_glob.use_hw) { - odp_time_t time; - - time.count = _odp_cpu_global_time_strict() - _odp_time_glob.hw_start; - return time; - } + odp_time_t time; - return _odp_timespec_cur(); -} + rte_mb(); + time.u64 = rte_get_timer_cycles() - _odp_time_glob.start_cycles; -static inline odp_time_t _odp_time_cur(void) -{ - return _odp_time_glob.handler.time_cur(); -} - -static inline odp_time_t _odp_time_cur_strict(void) -{ - return _odp_time_glob.handler.time_cur_strict(); + return time; } -static inline uint64_t _odp_time_hw_to_ns(odp_time_t time) +static inline uint64_t _odp_time_to_ns(odp_time_t time) { uint64_t nsec; - uint64_t freq_hz = _odp_time_glob.hw_freq_hz; uint64_t count = time.count; uint64_t sec = 0; + const uint64_t freq_hz = _odp_time_glob.freq_hz; + const uint64_t giga_hz = 1000000000; if (count >= freq_hz) { sec = count / freq_hz; count = count - sec * freq_hz; } - nsec = (_ODP_TIME_GIGA_HZ * count) / freq_hz; + nsec = (giga_hz * count) / freq_hz; - return (sec * _ODP_TIME_GIGA_HZ) + nsec; + return (sec * giga_hz) + nsec; } -static inline uint64_t _odp_time_convert_to_ns(odp_time_t time) +static inline odp_time_t _odp_time_from_ns(uint64_t ns) { - if (_odp_time_glob.use_hw) - return _odp_time_hw_to_ns(time); + odp_time_t time; + uint64_t count; + uint64_t sec = 0; + const uint64_t freq_hz = _odp_time_glob.freq_hz; + + if (ns >= ODP_TIME_SEC_IN_NS) { + sec = ns / ODP_TIME_SEC_IN_NS; + ns = ns - sec * ODP_TIME_SEC_IN_NS; + } - return time.nsec; + count = sec * freq_hz; + count += (ns * freq_hz) / ODP_TIME_SEC_IN_NS; + + time.count = count; + + return time; } #ifndef _ODP_NO_INLINE /* Inline functions by default */ #define _ODP_INLINE static inline - #define odp_time_local __odp_time_local - #define odp_time_global __odp_time_global - #define odp_time_to_ns __odp_time_to_ns - #define odp_time_local_ns __odp_time_local_ns - #define odp_time_global_ns __odp_time_global_ns - - #define odp_time_local_strict __odp_time_local_strict - #define odp_time_global_strict __odp_time_global_strict - #define odp_time_local_strict_ns __odp_time_local_strict_ns - #define odp_time_global_strict_ns __odp_time_global_strict_ns - - #define odp_time_cmp __odp_time_cmp - #define odp_time_diff __odp_time_diff - #define odp_time_diff_ns __odp_time_diff_ns - #define odp_time_sum __odp_time_sum + #define odp_time_local_res __odp_time_local_res + #define odp_time_global_res __odp_time_global_res + #define odp_time_local __odp_time_local + #define odp_time_global __odp_time_global + #define odp_time_local_strict __odp_time_local_strict + #define odp_time_global_strict __odp_time_global_strict + #define odp_time_local_ns __odp_time_local_ns + #define odp_time_global_ns __odp_time_global_ns + #define odp_time_local_from_ns __odp_time_local_from_ns + #define odp_time_global_from_ns __odp_time_global_from_ns + #define odp_time_local_strict_ns __odp_time_local_strict_ns + #define odp_time_global_strict_ns __odp_time_global_strict_ns + #define odp_time_to_ns __odp_time_to_ns + #define odp_time_cmp __odp_time_cmp + #define odp_time_diff __odp_time_diff + #define odp_time_diff_ns __odp_time_diff_ns + #define odp_time_sum __odp_time_sum + #define odp_time_wait_ns __odp_time_wait_ns + #define odp_time_wait_until __odp_time_wait_until #else #define _ODP_INLINE #endif +_ODP_INLINE uint64_t odp_time_local_res(void) +{ + return _odp_time_glob.freq_hz; +} + +_ODP_INLINE uint64_t odp_time_global_res(void) +{ + return _odp_time_glob.freq_hz; +} + _ODP_INLINE odp_time_t odp_time_local(void) { return _odp_time_cur(); @@ -153,27 +144,37 @@ _ODP_INLINE odp_time_t odp_time_global_strict(void) _ODP_INLINE uint64_t odp_time_local_ns(void) { - return _odp_time_convert_to_ns(_odp_time_cur()); + return _odp_time_to_ns(_odp_time_cur()); } _ODP_INLINE uint64_t odp_time_global_ns(void) { - return _odp_time_convert_to_ns(_odp_time_cur()); + return _odp_time_to_ns(_odp_time_cur()); +} + +_ODP_INLINE odp_time_t odp_time_local_from_ns(uint64_t ns) +{ + return _odp_time_from_ns(ns); +} + +_ODP_INLINE odp_time_t odp_time_global_from_ns(uint64_t ns) +{ + return _odp_time_from_ns(ns); } _ODP_INLINE uint64_t odp_time_local_strict_ns(void) { - return _odp_time_convert_to_ns(_odp_time_cur_strict()); + return _odp_time_to_ns(_odp_time_cur_strict()); } _ODP_INLINE uint64_t odp_time_global_strict_ns(void) { - return _odp_time_convert_to_ns(_odp_time_cur_strict()); + return _odp_time_to_ns(_odp_time_cur_strict()); } _ODP_INLINE uint64_t odp_time_to_ns(odp_time_t time) { - return _odp_time_convert_to_ns(time); + return _odp_time_to_ns(time); } _ODP_INLINE int odp_time_cmp(odp_time_t t2, odp_time_t t1) @@ -214,6 +215,29 @@ _ODP_INLINE odp_time_t odp_time_sum(odp_time_t t1, odp_time_t t2) return time; } +static inline void _odp_time_wait_until(odp_time_t time) +{ + odp_time_t cur; + + do { + cur = _odp_time_cur(); + } while (odp_time_cmp(time, cur) > 0); +} + +_ODP_INLINE void odp_time_wait_ns(uint64_t ns) +{ + const odp_time_t cur = _odp_time_cur(); + const odp_time_t wait = _odp_time_from_ns(ns); + const odp_time_t end_time = odp_time_sum(cur, wait); + + _odp_time_wait_until(end_time); +} + +_ODP_INLINE void odp_time_wait_until(odp_time_t time) +{ + _odp_time_wait_until(time); +} + /** @endcond */ #endif diff --git a/platform/linux-dpdk/include/odp/api/plat/timer_inlines.h b/platform/linux-dpdk/include/odp/api/plat/timer_inlines.h index 357d4df06..66327acdb 100644 --- a/platform/linux-dpdk/include/odp/api/plat/timer_inlines.h +++ b/platform/linux-dpdk/include/odp/api/plat/timer_inlines.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2022, Nokia +/* Copyright (c) 2022-2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -25,6 +25,7 @@ #define odp_timeout_user_ptr __odp_timeout_user_ptr #define odp_timeout_user_area __odp_timeout_user_area #define odp_timeout_from_event __odp_timeout_from_event + #define odp_timeout_from_event_multi __odp_timeout_from_event_multi #define odp_timeout_to_event __odp_timeout_to_event #else #define _ODP_INLINE @@ -57,6 +58,15 @@ _ODP_INLINE odp_timeout_t odp_timeout_from_event(odp_event_t ev) return (odp_timeout_t)ev; } +_ODP_INLINE void odp_timeout_from_event_multi(odp_timeout_t tmo[], const odp_event_t ev[], int num) +{ + for (int i = 0; i < num; i++) { + _ODP_ASSERT(odp_event_type(ev[i]) == ODP_EVENT_TIMEOUT); + + tmo[i] = (odp_timeout_t)ev[i]; + } +} + _ODP_INLINE odp_event_t odp_timeout_to_event(odp_timeout_t tmo) { return (odp_event_t)tmo; diff --git a/platform/linux-dpdk/include/odp_pool_internal.h b/platform/linux-dpdk/include/odp_pool_internal.h index b82324a77..8512806f7 100644 --- a/platform/linux-dpdk/include/odp_pool_internal.h +++ b/platform/linux-dpdk/include/odp_pool_internal.h @@ -83,6 +83,7 @@ typedef struct ODP_ALIGNED_CACHE { odp_pool_ext_param_t ext_param; odp_shm_t uarea_shm; uint64_t uarea_shm_size; + uint32_t param_uarea_size; uint32_t uarea_size; uint32_t trailer_size; /* Endmark size */ uint8_t *uarea_base_addr; diff --git a/platform/linux-dpdk/odp_crypto.c b/platform/linux-dpdk/odp_crypto.c index 5761787f8..3269e3075 100644 --- a/platform/linux-dpdk/odp_crypto.c +++ b/platform/linux-dpdk/odp_crypto.c @@ -1431,6 +1431,16 @@ int odp_crypto_session_create(const odp_crypto_session_param_t *param, return -1; } + if (param->cipher_range_in_bits) { + *status = ODP_CRYPTO_SES_ERR_CIPHER; + *session_out = ODP_CRYPTO_SESSION_INVALID; + return -1; + } + if (param->auth_range_in_bits) { + *status = ODP_CRYPTO_SES_ERR_AUTH; + *session_out = ODP_CRYPTO_SESSION_INVALID; + return -1; + } if (param->auth_alg == ODP_AUTH_ALG_AES_GMAC && param->cipher_alg != ODP_CIPHER_ALG_NULL) { *status = ODP_CRYPTO_SES_ERR_ALG_COMBO; diff --git a/platform/linux-dpdk/odp_init.c b/platform/linux-dpdk/odp_init.c index c7ae1ea47..7e6aaab55 100644 --- a/platform/linux-dpdk/odp_init.c +++ b/platform/linux-dpdk/odp_init.c @@ -35,8 +35,8 @@ enum init_stage { LIBCONFIG_INIT, CPUMASK_INIT, CPU_CYCLES_INIT, - TIME_INIT, SYSINFO_INIT, + TIME_INIT, ISHM_INIT, FDSERVER_INIT, GLOBAL_RW_DATA_INIT, @@ -473,16 +473,16 @@ static int term_global(enum init_stage stage) } /* Fall through */ - case SYSINFO_INIT: - if (_odp_system_info_term()) { - _ODP_ERR("ODP system info term failed.\n"); + case TIME_INIT: + if (_odp_time_term_global()) { + _ODP_ERR("ODP time term failed.\n"); rc = -1; } /* Fall through */ - case TIME_INIT: - if (_odp_time_term_global()) { - _ODP_ERR("ODP time term failed.\n"); + case SYSINFO_INIT: + if (_odp_system_info_term()) { + _ODP_ERR("ODP system info term failed.\n"); rc = -1; } /* Fall through */ @@ -558,18 +558,18 @@ int odp_init_global(odp_instance_t *instance, return -1; } - if (_odp_time_init_global()) { - _ODP_ERR("ODP time init failed.\n"); - goto init_failed; - } - stage = TIME_INIT; - if (_odp_system_info_init()) { _ODP_ERR("ODP system_info init failed.\n"); goto init_failed; } stage = SYSINFO_INIT; + if (_odp_time_init_global()) { + _ODP_ERR("ODP time init failed.\n"); + goto init_failed; + } + stage = TIME_INIT; + if (_odp_shm_init_global(params)) { _ODP_ERR("ODP shm init failed.\n"); goto init_failed; diff --git a/platform/linux-dpdk/odp_pool.c b/platform/linux-dpdk/odp_pool.c index 83d75dd6c..fd27cd79e 100644 --- a/platform/linux-dpdk/odp_pool.c +++ b/platform/linux-dpdk/odp_pool.c @@ -263,6 +263,7 @@ int odp_pool_capability(odp_pool_capability_t *capa) capa->buf.max_size = MAX_SIZE; capa->buf.max_num = CONFIG_POOL_MAX_NUM; capa->buf.max_uarea_size = MAX_UAREA_SIZE; + capa->buf.uarea_persistence = true; capa->buf.min_cache_size = 0; capa->buf.max_cache_size = RTE_MEMPOOL_CACHE_MAX_SIZE; capa->buf.stats.all = supported_stats.all; @@ -279,6 +280,7 @@ int odp_pool_capability(odp_pool_capability_t *capa) capa->pkt.min_seg_len = CONFIG_PACKET_SEG_LEN_MIN; capa->pkt.max_seg_len = CONFIG_PACKET_MAX_SEG_LEN; capa->pkt.max_uarea_size = MAX_UAREA_SIZE; + capa->pkt.uarea_persistence = true; capa->pkt.min_cache_size = 0; capa->pkt.max_cache_size = RTE_MEMPOOL_CACHE_MAX_SIZE; capa->pkt.stats.all = supported_stats.all; @@ -287,6 +289,7 @@ int odp_pool_capability(odp_pool_capability_t *capa) capa->tmo.max_pools = max_pools; capa->tmo.max_num = CONFIG_POOL_MAX_NUM; capa->tmo.max_uarea_size = MAX_UAREA_SIZE; + capa->tmo.uarea_persistence = true; capa->tmo.min_cache_size = 0; capa->tmo.max_cache_size = RTE_MEMPOOL_CACHE_MAX_SIZE; capa->tmo.stats.all = supported_stats.all; @@ -295,6 +298,7 @@ int odp_pool_capability(odp_pool_capability_t *capa) capa->vector.max_pools = max_pools; capa->vector.max_num = CONFIG_POOL_MAX_NUM; capa->vector.max_uarea_size = MAX_UAREA_SIZE; + capa->vector.uarea_persistence = true; capa->vector.max_size = CONFIG_PACKET_VECTOR_MAX_SIZE; capa->vector.min_cache_size = 0; capa->vector.max_cache_size = RTE_MEMPOOL_CACHE_MAX_SIZE; @@ -550,6 +554,7 @@ static int reserve_uarea(pool_t *pool, uint32_t uarea_size, uint32_t num_pkt) pool->uarea_shm = ODP_SHM_INVALID; if (uarea_size == 0) { + pool->param_uarea_size = 0; pool->uarea_size = 0; pool->uarea_shm_size = 0; return 0; @@ -559,6 +564,7 @@ static int reserve_uarea(pool_t *pool, uint32_t uarea_size, uint32_t num_pkt) pool->pool_idx, pool->name); uarea_name[ODP_SHM_NAME_LEN - 1] = 0; + pool->param_uarea_size = uarea_size; pool->uarea_size = _ODP_ROUNDUP_CACHE_LINE(uarea_size); pool->uarea_shm_size = num_pkt * (uint64_t)pool->uarea_size; @@ -605,7 +611,8 @@ static void init_obj_priv_data(struct rte_mempool *mp ODP_UNUSED, void *arg, voi struct priv_data_t *priv_data = arg; struct rte_mbuf *mb = mbuf; _odp_event_hdr_t *event_hdr = (_odp_event_hdr_t *)mbuf; - void *uarea = priv_data->pool->uarea_base_addr + i * priv_data->pool->uarea_size; + pool_t *pool = priv_data->pool; + void *uarea = pool->uarea_base_addr + i * pool->uarea_size; void **obj_uarea; if (priv_data->type != ODP_POOL_PACKET) @@ -613,7 +620,7 @@ static void init_obj_priv_data(struct rte_mempool *mp ODP_UNUSED, void *arg, voi mb->data_off = 0; event_hdr->hdr.index = i; - event_hdr->hdr.pool = _odp_pool_handle(priv_data->pool); + event_hdr->hdr.pool = _odp_pool_handle(pool); event_hdr->hdr.type = priv_data->type; event_hdr->hdr.event_type = priv_data->event_type; @@ -636,6 +643,10 @@ static void init_obj_priv_data(struct rte_mempool *mp ODP_UNUSED, void *arg, voi *obj_uarea = uarea; + if (uarea && pool->params.uarea_init.init_fn) + pool->params.uarea_init.init_fn(uarea, pool->param_uarea_size, + pool->params.uarea_init.args, i); + if (priv_data->type == ODP_POOL_BUFFER || priv_data->type == ODP_POOL_PACKET) { mb->buf_len -= _ODP_EV_ENDMARK_SIZE; _odp_event_endmark_set(_odp_event_from_mbuf(mb)); @@ -751,13 +762,15 @@ odp_pool_t _odp_pool_create(const char *name, const odp_pool_param_t *params, return ODP_POOL_INVALID; } - rte_mempool_obj_iter(mp, init_obj_priv_data, &priv_data); pool->rte_mempool = mp; pool->seg_len = seg_size; pool->type_2 = type_2; pool->type = params->type; pool->params = *params; pool->trailer_size = trailer; + + rte_mempool_obj_iter(mp, init_obj_priv_data, &priv_data); + UNLOCK(&pool->lock); pool_hdl = _odp_pool_handle(pool); @@ -1097,6 +1110,7 @@ int odp_pool_ext_capability(odp_pool_type_t type, capa->pkt.max_headroom_size = RTE_PKTMBUF_HEADROOM; capa->pkt.max_segs_per_pkt = PKT_MAX_SEGS; capa->pkt.max_uarea_size = MAX_UAREA_SIZE; + capa->pkt.uarea_persistence = true; return 0; } @@ -1297,10 +1311,11 @@ static void init_ext_obj(struct rte_mempool *mp, void *arg, void *mbuf, unsigned { struct mbuf_ctor_arg *mb_ctor_arg = arg; struct rte_mbuf *mb = mbuf; - void *uarea = mb_ctor_arg->pool->uarea_base_addr + i * mb_ctor_arg->pool->uarea_size; + pool_t *pool = mb_ctor_arg->pool; + void *uarea = pool->uarea_base_addr + i * pool->uarea_size; _odp_event_hdr_t *event_hdr = (_odp_event_hdr_t *)mbuf; void **obj_uarea; - odp_pool_ext_param_t *p = &mb_ctor_arg->pool->ext_param; + odp_pool_ext_param_t *p = &pool->ext_param; uint32_t app_hdr_offset = sizeof(odp_packet_hdr_t); uint32_t app_hdr_size = p->pkt.app_header_size; uint32_t buf_size = p->pkt.buf_size; @@ -1333,7 +1348,7 @@ static void init_ext_obj(struct rte_mempool *mp, void *arg, void *mbuf, unsigned mb->next = NULL; /* Save index, might be useful for debugging purposes */ event_hdr->hdr.index = i; - event_hdr->hdr.pool = _odp_pool_handle(mb_ctor_arg->pool); + event_hdr->hdr.pool = _odp_pool_handle(pool); event_hdr->hdr.type = mb_ctor_arg->type; event_hdr->hdr.event_type = mb_ctor_arg->event_type; @@ -1356,6 +1371,10 @@ static void init_ext_obj(struct rte_mempool *mp, void *arg, void *mbuf, unsigned *obj_uarea = uarea; + if (uarea && pool->ext_param.uarea_init.init_fn) + pool->ext_param.uarea_init.init_fn(uarea, pool->param_uarea_size, + pool->ext_param.uarea_init.args, i); + if (mb_ctor_arg->type == ODP_POOL_BUFFER || mb_ctor_arg->type == ODP_POOL_PACKET) { mb->buf_len -= _ODP_EV_ENDMARK_SIZE; _odp_event_endmark_set(_odp_event_from_mbuf(mb)); diff --git a/platform/linux-dpdk/odp_system_info.c b/platform/linux-dpdk/odp_system_info.c index c76820e5d..886f7f216 100644 --- a/platform/linux-dpdk/odp_system_info.c +++ b/platform/linux-dpdk/odp_system_info.c @@ -59,8 +59,8 @@ static int read_cache_line_size(void) file = fopen(CACHE_LNSZ_FILE, "rt"); if (file == NULL) { /* File not found */ - _ODP_PRINT("WARN: unable to read host CPU cache line size. " - "Using ODP_CACHE_LINE_SIZE instead.\n"); + _ODP_WARN("Unable to read host CPU cache line size. " + "Using ODP_CACHE_LINE_SIZE instead.\n"); return ODP_CACHE_LINE_SIZE; } @@ -225,7 +225,7 @@ static int system_cache_line(system_info_t *sysinfo) sysinfo->cache_line_size = ret; if (ret != ODP_CACHE_LINE_SIZE) - _ODP_PRINT("WARN: host CPU cache line size and ODP_CACHE_LINE_SIZE don't match.\n"); + _ODP_WARN("Host CPU cache line size and ODP_CACHE_LINE_SIZE don't match.\n"); return 0; } diff --git a/platform/linux-dpdk/odp_time.c b/platform/linux-dpdk/odp_time.c index fbbc9358d..2fc25eb96 100644 --- a/platform/linux-dpdk/odp_time.c +++ b/platform/linux-dpdk/odp_time.c @@ -1,302 +1,43 @@ /* Copyright (c) 2013-2018, Linaro Limited - * Copyright (c) 2021-2022, Nokia + * Copyright (c) 2021-2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause */ -#include <odp_posix_extensions.h> - -#include <time.h> -#include <string.h> -#include <inttypes.h> +#include <odp/api/plat/time_inlines.h> -#include <odp/api/time.h> -#include <odp/api/hints.h> #include <odp_debug_internal.h> #include <odp_init_internal.h> -#include <odp/api/plat/time_inlines.h> #include <rte_config.h> -#include <rte_atomic.h> #include <rte_cycles.h> -ODP_STATIC_ASSERT(_ODP_TIMESPEC_SIZE >= (sizeof(struct timespec)), - "_ODP_TIMESPEC_SIZE too small"); +#include <string.h> #include <odp/visibility_begin.h> _odp_time_global_t _odp_time_glob; -/* - * Posix timespec based functions - */ - -static inline uint64_t time_spec_diff_nsec(struct timespec *t2, - struct timespec *t1) -{ - struct timespec diff; - uint64_t nsec; - - diff.tv_sec = t2->tv_sec - t1->tv_sec; - diff.tv_nsec = t2->tv_nsec - t1->tv_nsec; - - if (diff.tv_nsec < 0) { - diff.tv_nsec += ODP_TIME_SEC_IN_NS; - diff.tv_sec -= 1; - } - - nsec = (diff.tv_sec * ODP_TIME_SEC_IN_NS) + diff.tv_nsec; - - return nsec; -} - -odp_time_t _odp_timespec_cur(void) -{ - int ret; - odp_time_t time; - struct timespec sys_time; - struct timespec *start_time; - - start_time = (struct timespec *)(uintptr_t)&_odp_time_glob.timespec; - - ret = clock_gettime(CLOCK_MONOTONIC_RAW, &sys_time); - if (odp_unlikely(ret != 0)) - _ODP_ABORT("clock_gettime failed\n"); - - time.nsec = time_spec_diff_nsec(&sys_time, start_time); - - return time; -} - #include <odp/visibility_end.h> -static inline uint64_t time_spec_res(void) -{ - int ret; - struct timespec tres; - - ret = clock_getres(CLOCK_MONOTONIC_RAW, &tres); - if (odp_unlikely(ret != 0)) - _ODP_ABORT("clock_getres failed\n"); - - return ODP_TIME_SEC_IN_NS / (uint64_t)tres.tv_nsec; -} - -static inline odp_time_t time_spec_from_ns(uint64_t ns) -{ - odp_time_t time; - - time.nsec = ns; - - return time; -} - -/* - * HW time counter based functions - */ - -static inline uint64_t time_hw_res(void) -{ - return _odp_time_glob.hw_freq_hz; -} - -static inline odp_time_t time_hw_from_ns(uint64_t ns) -{ - odp_time_t time; - uint64_t count; - uint64_t freq_hz = _odp_time_glob.hw_freq_hz; - uint64_t sec = 0; - - if (ns >= ODP_TIME_SEC_IN_NS) { - sec = ns / ODP_TIME_SEC_IN_NS; - ns = ns - sec * ODP_TIME_SEC_IN_NS; - } - - count = sec * freq_hz; - count += (ns * freq_hz) / ODP_TIME_SEC_IN_NS; - - time.count = count; - - return time; -} - -/* - * Common functions - */ - -static inline uint64_t time_res(void) -{ - if (_odp_time_glob.use_hw) - return time_hw_res(); - - return time_spec_res(); -} - -static inline odp_time_t time_from_ns(uint64_t ns) -{ - if (_odp_time_glob.use_hw) - return time_hw_from_ns(ns); - - return time_spec_from_ns(ns); -} - -static inline odp_time_t time_cur_dpdk(void) -{ - odp_time_t time; - - time.u64 = rte_get_timer_cycles() - _odp_time_glob.hw_start; - - return time; -} - -static inline odp_time_t time_cur_dpdk_strict(void) -{ - odp_time_t time; - - rte_mb(); - time.u64 = rte_get_timer_cycles() - _odp_time_glob.hw_start; - - return time; -} - -static inline uint64_t time_res_dpdk(void) -{ - return rte_get_timer_hz(); -} - -static inline void time_wait_until(odp_time_t time) -{ - odp_time_t cur; - - do { - cur = _odp_time_cur(); - } while (odp_time_cmp(time, cur) > 0); -} - -odp_time_t odp_time_local_from_ns(uint64_t ns) -{ - return time_from_ns(ns); -} - -odp_time_t odp_time_global_from_ns(uint64_t ns) -{ - return time_from_ns(ns); -} - -uint64_t odp_time_local_res(void) -{ - return _odp_time_glob.handler.time_res(); -} - -uint64_t odp_time_global_res(void) -{ - return _odp_time_glob.handler.time_res(); -} - -void odp_time_wait_ns(uint64_t ns) -{ - odp_time_t cur = _odp_time_cur(); - odp_time_t wait = time_from_ns(ns); - odp_time_t end_time = odp_time_sum(cur, wait); - - time_wait_until(end_time); -} - -void odp_time_wait_until(odp_time_t time) -{ - time_wait_until(time); -} - -static odp_bool_t is_invariant_tsc_supported(void) -{ - FILE *file; - char *line = NULL; - size_t len = 0; - odp_bool_t nonstop_tsc = false; - odp_bool_t constant_tsc = false; - odp_bool_t ret = false; - - file = fopen("/proc/cpuinfo", "rt"); - while (getline(&line, &len, file) != -1) { - if (strstr(line, "flags") != NULL) { - if (strstr(line, "constant_tsc") != NULL) - constant_tsc = true; - if (strstr(line, "nonstop_tsc") != NULL) - nonstop_tsc = true; - - if (constant_tsc && nonstop_tsc) - ret = true; - else - ret = false; - - free(line); - fclose(file); - return ret; - } - } - free(line); - fclose(file); - return false; -} - -static inline odp_bool_t is_dpdk_timer_cycles_support(void) +int _odp_time_init_global(void) { - if (is_invariant_tsc_supported() == true) - return true; + memset(&_odp_time_glob, 0, sizeof(_odp_time_global_t)); #ifdef RTE_LIBEAL_USE_HPET - if (rte_eal_hpet_init(1) == 0) - return true; + if (rte_eal_hpet_init(1) != 0) + _ODP_WARN("HPET init failed. Using TSC time.\n"); #endif - return false; -} - -int _odp_time_init_global(void) -{ - struct timespec *timespec; - int ret = 0; - _odp_time_global_t *global = &_odp_time_glob; - - memset(global, 0, sizeof(_odp_time_global_t)); - - if (is_dpdk_timer_cycles_support()) { - _odp_time_glob.handler.time_cur = time_cur_dpdk; - _odp_time_glob.handler.time_cur_strict = time_cur_dpdk_strict; - _odp_time_glob.handler.time_res = time_res_dpdk; - _odp_time_glob.hw_freq_hz = time_res_dpdk(); - _odp_time_glob.use_hw = 1; - _odp_time_glob.hw_start = rte_get_timer_cycles(); - if (_odp_time_glob.hw_start == 0) - return -1; - else - return 0; - } - _odp_time_glob.handler.time_cur = _odp_time_cur_gen; - _odp_time_glob.handler.time_cur_strict = _odp_time_cur_gen_strict; - _odp_time_glob.handler.time_res = time_res; - - if (_odp_cpu_has_global_time()) { - global->use_hw = 1; - global->hw_freq_hz = _odp_cpu_global_time_freq(); - - if (global->hw_freq_hz == 0) - return -1; - - _ODP_PRINT("HW time counter freq: %" PRIu64 " hz\n\n", global->hw_freq_hz); - - global->hw_start = _odp_cpu_global_time(); - return 0; + _odp_time_glob.freq_hz = rte_get_timer_hz(); + _odp_time_glob.start_cycles = rte_get_timer_cycles(); + if (_odp_time_glob.start_cycles == 0) { + _ODP_ERR("Initializing start cycles failed.\n"); + return -1; } - timespec = (struct timespec *)(uintptr_t)global->timespec; - timespec->tv_sec = 0; - timespec->tv_nsec = 0; - - ret = clock_gettime(CLOCK_MONOTONIC_RAW, timespec); - - return ret; + return 0; } int _odp_time_term_global(void) diff --git a/platform/linux-generic/Makefile.am b/platform/linux-generic/Makefile.am index 42760f579..7afdbf620 100644 --- a/platform/linux-generic/Makefile.am +++ b/platform/linux-generic/Makefile.am @@ -10,6 +10,7 @@ AM_CPPFLAGS += -I$(top_builddir)/platform/$(with_platform)/include AM_CPPFLAGS += -I$(top_srcdir)/platform/$(with_platform)/arch AM_CPPFLAGS += -I$(top_srcdir)/platform/$(with_platform)/arch/@ARCH_DIR@ AM_CPPFLAGS += -I$(top_srcdir)/platform/$(with_platform)/arch/default +AM_CPPFLAGS += -I$(top_srcdir)/platform/$(with_platform)/arch/common AM_CPPFLAGS += $(OPENSSL_CPPFLAGS) @@ -34,6 +35,7 @@ odpapiplatinclude_HEADERS = \ include/odp/api/plat/byteorder_inlines.h \ include/odp/api/plat/cpu_inlines.h \ include/odp/api/plat/crypto_inlines.h \ + include/odp/api/plat/dma_inlines.h \ include/odp/api/plat/debug_inlines.h \ include/odp/api/plat/event_inlines.h \ include/odp/api/plat/event_inline_types.h \ @@ -79,6 +81,7 @@ odpapiabiarchinclude_HEADERS += \ include-abi/odp/api/abi/crypto.h \ include-abi/odp/api/abi/crypto_types.h \ include-abi/odp/api/abi/debug.h \ + include-abi/odp/api/abi/dma.h \ include-abi/odp/api/abi/dma_types.h \ include-abi/odp/api/abi/errno.h \ include-abi/odp/api/abi/event.h \ @@ -255,7 +258,6 @@ __LIB__libodp_linux_la_SOURCES = \ odp_pcapng.c \ odp_thread.c \ odp_thrmask.c \ - odp_time.c \ odp_timer.c \ odp_timer_wheel.c \ odp_traffic_mngr.c \ @@ -301,6 +303,7 @@ __LIB__libodp_linux_la_SOURCES += \ odp_byteorder_api.c \ odp_cpu_api.c \ odp_crypto_api.c \ + odp_dma_api.c \ odp_event_api.c \ odp_hash_api.c \ odp_ipsec_api.c \ @@ -325,18 +328,19 @@ endif if ARCH_IS_ARM __LIB__libodp_linux_la_SOURCES += arch/default/odp_atomic.c \ arch/default/odp_cpu_cycles.c \ - arch/default/odp_global_time.c \ arch/default/odp_hash_crc32.c \ arch/default/odp_random.c \ - arch/arm/odp_sysinfo_parse.c -odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_time.h \ - arch/default/odp/api/abi/hash_crc32.h + arch/arm/odp_sysinfo_parse.c \ + arch/default/odp_time.c +odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/hash_crc32.h if !ODP_ABI_COMPAT odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ arch/default/odp/api/abi/atomic_inlines.h \ arch/default/odp/api/abi/cpu_generic.h \ arch/arm/odp/api/abi/cpu_inlines.h \ - arch/arm/odp/api/abi/cpu.h + arch/arm/odp/api/abi/cpu.h \ + arch/default/odp/api/abi/sync_inlines.h \ + arch/default/odp/api/abi/time_inlines.h endif noinst_HEADERS += arch/arm/odp_atomic.h \ arch/arm/odp_cpu.h \ @@ -351,19 +355,22 @@ if ARCH_IS_AARCH64 __LIB__libodp_linux_la_SOURCES += arch/aarch64/odp_atomic.c \ arch/default/odp_cpu_cycles.c \ arch/aarch64/cpu_flags.c \ - arch/aarch64/odp_global_time.c \ arch/default/odp_hash_crc32.c \ arch/default/odp_random.c \ - arch/aarch64/odp_sysinfo_parse.c -odpapiabiarchinclude_HEADERS += arch/aarch64/odp/api/abi/cpu_time.h \ - arch/aarch64/odp/api/abi/hash_crc32.h + arch/aarch64/odp_sysinfo_parse.c \ + arch/common/odp_time_cpu.c +odpapiabiarchinclude_HEADERS += arch/aarch64/odp/api/abi/hash_crc32.h \ + arch/aarch64/odp/api/abi/time_cpu.h if !ODP_ABI_COMPAT odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ arch/aarch64/odp/api/abi/atomic_inlines.h \ arch/aarch64/odp/api/abi/atomic.h \ arch/default/odp/api/abi/cpu_generic.h \ arch/aarch64/odp/api/abi/cpu_inlines.h \ - arch/aarch64/odp/api/abi/cpu.h + arch/aarch64/odp/api/abi/cpu.h \ + arch/aarch64/odp/api/abi/sync_inlines.h \ + arch/common/odp/api/abi/time_cpu_inlines.h \ + arch/aarch64/odp/api/abi/time_inlines.h endif noinst_HEADERS += arch/aarch64/odp_atomic.h \ arch/aarch64/odp_cpu.h \ @@ -375,18 +382,19 @@ endif if ARCH_IS_DEFAULT __LIB__libodp_linux_la_SOURCES += arch/default/odp_atomic.c \ arch/default/odp_cpu_cycles.c \ - arch/default/odp_global_time.c \ arch/default/odp_hash_crc32.c \ arch/default/odp_random.c \ - arch/default/odp_sysinfo_parse.c -odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_time.h \ - arch/default/odp/api/abi/hash_crc32.h + arch/default/odp_sysinfo_parse.c \ + arch/default/odp_time.c +odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/hash_crc32.h if !ODP_ABI_COMPAT odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ arch/default/odp/api/abi/atomic_inlines.h \ arch/default/odp/api/abi/cpu_generic.h \ arch/default/odp/api/abi/cpu_inlines.h \ - arch/default/odp/api/abi/cpu.h + arch/default/odp/api/abi/cpu.h \ + arch/default/odp/api/abi/sync_inlines.h \ + arch/default/odp/api/abi/time_inlines.h endif noinst_HEADERS += arch/default/odp_atomic.h \ arch/default/odp_cpu.h \ @@ -396,18 +404,19 @@ endif if ARCH_IS_POWERPC __LIB__libodp_linux_la_SOURCES += arch/default/odp_atomic.c \ arch/default/odp_cpu_cycles.c \ - arch/default/odp_global_time.c \ arch/default/odp_hash_crc32.c \ arch/default/odp_random.c \ - arch/powerpc/odp_sysinfo_parse.c -odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_time.h \ - arch/default/odp/api/abi/hash_crc32.h + arch/powerpc/odp_sysinfo_parse.c \ + arch/default/odp_time.c +odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/hash_crc32.h if !ODP_ABI_COMPAT odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ arch/default/odp/api/abi/atomic_inlines.h \ arch/default/odp/api/abi/cpu_generic.h \ arch/default/odp/api/abi/cpu_inlines.h \ - arch/powerpc/odp/api/abi/cpu.h + arch/powerpc/odp/api/abi/cpu.h \ + arch/default/odp/api/abi/sync_inlines.h \ + arch/default/odp/api/abi/time_inlines.h endif noinst_HEADERS += arch/default/odp_atomic.h \ arch/default/odp_cpu.h \ @@ -418,18 +427,22 @@ if ARCH_IS_X86 __LIB__libodp_linux_la_SOURCES += arch/default/odp_atomic.c \ arch/x86/cpu_flags.c \ arch/x86/odp_cpu_cycles.c \ - arch/x86/odp_global_time.c \ arch/default/odp_hash_crc32.c \ arch/default/odp_random.c \ - arch/x86/odp_sysinfo_parse.c + arch/x86/odp_sysinfo_parse.c \ + arch/x86/odp_time_cpu.c \ + arch/common/odp_time_cpu.c odpapiabiarchinclude_HEADERS += arch/x86/odp/api/abi/cpu_rdtsc.h \ - arch/x86/odp/api/abi/cpu_time.h \ - arch/x86/odp/api/abi/hash_crc32.h + arch/x86/odp/api/abi/hash_crc32.h \ + arch/x86/odp/api/abi/time_cpu.h if !ODP_ABI_COMPAT odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ arch/default/odp/api/abi/atomic_inlines.h \ arch/x86/odp/api/abi/cpu_inlines.h \ - arch/x86/odp/api/abi/cpu.h + arch/x86/odp/api/abi/cpu.h \ + arch/x86/odp/api/abi/sync_inlines.h \ + arch/common/odp/api/abi/time_cpu_inlines.h \ + arch/x86/odp/api/abi/time_inlines.h endif noinst_HEADERS += arch/x86/cpu_flags.h \ arch/x86/odp_cpu.h \ diff --git a/platform/linux-generic/arch/aarch64/odp/api/abi/sync_inlines.h b/platform/linux-generic/arch/aarch64/odp/api/abi/sync_inlines.h new file mode 100644 index 000000000..3d42e7dd8 --- /dev/null +++ b/platform/linux-generic/arch/aarch64/odp/api/abi/sync_inlines.h @@ -0,0 +1,31 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2023 Nokia + */ + +#ifndef ODP_ARCH_SYNC_INLINES_H_ +#define ODP_ARCH_SYNC_INLINES_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +static inline void _odp_mb_sync(void) +{ + __asm__ volatile("dsb sy" ::: "memory"); +} + +static inline void _odp_mb_sync_load(void) +{ + __asm__ volatile("dsb ld" ::: "memory"); +} + +static inline void _odp_mb_sync_store(void) +{ + __asm__ volatile("dsb st" ::: "memory"); +} + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/arch/aarch64/odp/api/abi/cpu_time.h b/platform/linux-generic/arch/aarch64/odp/api/abi/time_cpu.h index 781ee683c..aba2799c7 100644 --- a/platform/linux-generic/arch/aarch64/odp/api/abi/cpu_time.h +++ b/platform/linux-generic/arch/aarch64/odp/api/abi/time_cpu.h @@ -4,8 +4,8 @@ * SPDX-License-Identifier: BSD-3-Clause */ -#ifndef ODP_API_ABI_CPU_TIME_H_ -#define ODP_API_ABI_CPU_TIME_H_ +#ifndef ODP_API_ABI_TIME_CPU_H_ +#define ODP_API_ABI_TIME_CPU_H_ #ifdef __cplusplus extern "C" { @@ -13,7 +13,7 @@ extern "C" { #include <stdint.h> -static inline uint64_t _odp_cpu_global_time(void) +static inline uint64_t _odp_time_cpu_global(void) { uint64_t cntvct; @@ -22,7 +22,7 @@ static inline uint64_t _odp_cpu_global_time(void) return cntvct; } -static inline uint64_t _odp_cpu_global_time_strict(void) +static inline uint64_t _odp_time_cpu_global_strict(void) { uint64_t cntvct; @@ -32,7 +32,7 @@ static inline uint64_t _odp_cpu_global_time_strict(void) return cntvct; } -static inline uint64_t _odp_cpu_global_time_freq(void) +static inline uint64_t _odp_time_cpu_global_freq(void) { uint64_t cntfrq; @@ -41,7 +41,10 @@ static inline uint64_t _odp_cpu_global_time_freq(void) return cntfrq; } -int _odp_cpu_has_global_time(void); +static inline int _odp_time_cpu_global_freq_is_const(void) +{ + return 1; +} #ifdef __cplusplus } diff --git a/platform/linux-generic/arch/aarch64/odp/api/abi/time_inlines.h b/platform/linux-generic/arch/aarch64/odp/api/abi/time_inlines.h new file mode 100644 index 000000000..331d1996f --- /dev/null +++ b/platform/linux-generic/arch/aarch64/odp/api/abi/time_inlines.h @@ -0,0 +1,7 @@ +/* Copyright (c) 2023, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <odp/api/abi/time_cpu_inlines.h> diff --git a/platform/linux-generic/arch/aarch64/odp_global_time.c b/platform/linux-generic/arch/aarch64/odp_global_time.c deleted file mode 100644 index 53561b00c..000000000 --- a/platform/linux-generic/arch/aarch64/odp_global_time.c +++ /dev/null @@ -1,25 +0,0 @@ -/* Copyright (c) 2015-2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -#include <odp/api/abi/cpu_time.h> - -int _odp_cpu_has_global_time(void) -{ - uint64_t hz = _odp_cpu_global_time_freq(); - - /* - * The system counter portion of the architected timer must - * provide a uniform view of system time to all processing - * elements in the system. This should hold true even for - * heterogeneous SoCs. - * - * Determine whether the system has 'global time' by checking - * whether a read of the architected timer frequency sys reg - * returns a sane value. Sane is considered to be within - * 1MHz and 6GHz (1us and .1667ns period). - */ - return hz >= 1000000 && hz <= 6000000000; -} diff --git a/platform/linux-generic/arch/aarch64/odp_sysinfo_parse.c b/platform/linux-generic/arch/aarch64/odp_sysinfo_parse.c index 25e3e4fc6..352bee9e9 100644 --- a/platform/linux-generic/arch/aarch64/odp_sysinfo_parse.c +++ b/platform/linux-generic/arch/aarch64/odp_sysinfo_parse.c @@ -336,9 +336,8 @@ int _odp_cpuinfo_parser(FILE *file, system_info_t *sysinfo) if (sysinfo->cpu_hz_max[id] == 0) { uint64_t hz = sysinfo->default_cpu_hz_max; - _ODP_PRINT("WARN: cpu[%i] uses default max " - "frequency of %" PRIu64 " Hz from " - "config file\n", id, hz); + _ODP_WARN("CPU[%i] uses default max frequency of %" PRIu64 " " + "Hz from config file\n", id, hz); sysinfo->cpu_hz_max[id] = hz; } diff --git a/platform/linux-generic/arch/common/odp/api/abi/time_cpu_inlines.h b/platform/linux-generic/arch/common/odp/api/abi/time_cpu_inlines.h new file mode 100644 index 000000000..c154c5f1a --- /dev/null +++ b/platform/linux-generic/arch/common/odp/api/abi/time_cpu_inlines.h @@ -0,0 +1,93 @@ +/* Copyright (c) 2013-2018, Linaro Limited + * Copyright (c) 2020-2023, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_ARCH_TIME_CPU_INLINES_H_ +#define ODP_ARCH_TIME_CPU_INLINES_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <odp/api/time_types.h> + +#include <odp/api/abi/time_cpu.h> + +#include <stdint.h> + +#define _ODP_TIME_GIGA_HZ 1000000000ULL + +typedef struct _odp_time_global_t { + uint64_t start_time; + uint64_t freq_hz; + +} _odp_time_global_t; + +extern _odp_time_global_t _odp_time_glob; + +static inline odp_time_t _odp_time_cur(void) +{ + odp_time_t time; + + time.count = _odp_time_cpu_global() - _odp_time_glob.start_time; + return time; +} + +static inline odp_time_t _odp_time_cur_strict(void) +{ + odp_time_t time; + + time.count = _odp_time_cpu_global_strict() - _odp_time_glob.start_time; + return time; +} + +static inline uint64_t _odp_time_to_ns(odp_time_t time) +{ + uint64_t nsec; + uint64_t freq_hz = _odp_time_glob.freq_hz; + uint64_t count = time.count; + uint64_t sec = 0; + + if (count >= freq_hz) { + sec = count / freq_hz; + count = count - sec * freq_hz; + } + + nsec = (_ODP_TIME_GIGA_HZ * count) / freq_hz; + + return (sec * _ODP_TIME_GIGA_HZ) + nsec; +} + +static inline odp_time_t _odp_time_from_ns(uint64_t ns) +{ + odp_time_t time; + uint64_t count; + uint64_t freq_hz = _odp_time_glob.freq_hz; + uint64_t sec = 0; + + if (ns >= ODP_TIME_SEC_IN_NS) { + sec = ns / ODP_TIME_SEC_IN_NS; + ns = ns - sec * ODP_TIME_SEC_IN_NS; + } + + count = sec * freq_hz; + count += (ns * freq_hz) / ODP_TIME_SEC_IN_NS; + + time.count = count; + + return time; +} + +static inline uint64_t _odp_time_res(void) +{ + return _odp_time_glob.freq_hz; +} + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/arch/common/odp_time_cpu.c b/platform/linux-generic/arch/common/odp_time_cpu.c new file mode 100644 index 000000000..bbfe82e21 --- /dev/null +++ b/platform/linux-generic/arch/common/odp_time_cpu.c @@ -0,0 +1,48 @@ +/* Copyright (c) 2013-2018, Linaro Limited + * Copyright (c) 2020-2023, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <odp/api/time_types.h> + +#include <odp/api/abi/time_cpu.h> +#include <odp/api/abi/time_cpu_inlines.h> + +#include <odp_debug_internal.h> +#include <odp_init_internal.h> + +#include <inttypes.h> +#include <stdint.h> +#include <string.h> + +#include <odp/visibility_begin.h> + +_odp_time_global_t _odp_time_glob; + +#include <odp/visibility_end.h> + +int _odp_time_init_global(void) +{ + _odp_time_global_t *global = &_odp_time_glob; + + memset(global, 0, sizeof(_odp_time_global_t)); + + if (!_odp_time_cpu_global_freq_is_const()) + return -1; + + global->freq_hz = _odp_time_cpu_global_freq(); + if (global->freq_hz == 0) + return -1; + + _ODP_PRINT("HW time counter freq: %" PRIu64 " hz\n\n", global->freq_hz); + + global->start_time = _odp_time_cpu_global(); + return 0; +} + +int _odp_time_term_global(void) +{ + return 0; +} diff --git a/platform/linux-generic/arch/default/odp/api/abi/cpu_time.h b/platform/linux-generic/arch/default/odp/api/abi/cpu_time.h deleted file mode 100644 index 24e1c7d33..000000000 --- a/platform/linux-generic/arch/default/odp/api/abi/cpu_time.h +++ /dev/null @@ -1,25 +0,0 @@ -/* Copyright (c) 2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -#ifndef ODP_ARCH_CPU_TIME_H_ -#define ODP_ARCH_CPU_TIME_H_ - -#ifdef __cplusplus -extern "C" { -#endif - -#include <stdint.h> - -int _odp_cpu_has_global_time(void); -uint64_t _odp_cpu_global_time(void); -uint64_t _odp_cpu_global_time_strict(void); -uint64_t _odp_cpu_global_time_freq(void); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/platform/linux-generic/arch/default/odp/api/abi/sync_inlines.h b/platform/linux-generic/arch/default/odp/api/abi/sync_inlines.h new file mode 100644 index 000000000..bfbb3039f --- /dev/null +++ b/platform/linux-generic/arch/default/odp/api/abi/sync_inlines.h @@ -0,0 +1,31 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2023 Nokia + */ + +#ifndef ODP_ARCH_SYNC_INLINES_H_ +#define ODP_ARCH_SYNC_INLINES_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +static inline void _odp_mb_sync(void) +{ + __sync_synchronize(); +} + +static inline void _odp_mb_sync_load(void) +{ + __sync_synchronize(); +} + +static inline void _odp_mb_sync_store(void) +{ + __sync_synchronize(); +} + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/arch/default/odp/api/abi/time_inlines.h b/platform/linux-generic/arch/default/odp/api/abi/time_inlines.h new file mode 100644 index 000000000..b38e52dac --- /dev/null +++ b/platform/linux-generic/arch/default/odp/api/abi/time_inlines.h @@ -0,0 +1,45 @@ +/* Copyright (c) 2013-2018, Linaro Limited + * Copyright (c) 2020-2023, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_ARCH_TIME_INLINES_H_ +#define ODP_ARCH_TIME_INLINES_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <odp/api/time_types.h> + +#include <stdint.h> + +odp_time_t _odp_time_cur(void); +uint64_t _odp_time_res(void); + +static inline odp_time_t _odp_time_cur_strict(void) +{ + return _odp_time_cur(); +} + +static inline uint64_t _odp_time_to_ns(odp_time_t time) +{ + return time.nsec; +} + +static inline odp_time_t _odp_time_from_ns(uint64_t ns) +{ + odp_time_t time; + + time.nsec = ns; + + return time; +} + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/arch/default/odp_global_time.c b/platform/linux-generic/arch/default/odp_global_time.c deleted file mode 100644 index ee835413f..000000000 --- a/platform/linux-generic/arch/default/odp_global_time.c +++ /dev/null @@ -1,31 +0,0 @@ -/* Copyright (c) 2015-2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -#include <odp/api/abi/cpu_time.h> - -#include <odp/visibility_begin.h> - -uint64_t _odp_cpu_global_time(void) -{ - return 0; -} - -uint64_t _odp_cpu_global_time_strict(void) -{ - return 0; -} - -#include <odp/visibility_end.h> - -int _odp_cpu_has_global_time(void) -{ - return 0; -} - -uint64_t _odp_cpu_global_time_freq(void) -{ - return 0; -} diff --git a/platform/linux-generic/arch/default/odp_time.c b/platform/linux-generic/arch/default/odp_time.c new file mode 100644 index 000000000..919a3ba68 --- /dev/null +++ b/platform/linux-generic/arch/default/odp_time.c @@ -0,0 +1,102 @@ +/* Copyright (c) 2013-2018, Linaro Limited + * Copyright (c) 2020-2023, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <odp_posix_extensions.h> + +#include <odp/api/align.h> +#include <odp/api/hints.h> +#include <odp/api/time_types.h> + +#include <odp/api/abi/time_inlines.h> + +#include <odp_debug_internal.h> +#include <odp_init_internal.h> + +#include <stdint.h> +#include <string.h> +#include <time.h> + +typedef struct _odp_time_global_t { + struct timespec start_time; + +} _odp_time_global_t; + +_odp_time_global_t _odp_time_glob; + +static inline uint64_t time_diff_nsec(struct timespec *t2, struct timespec *t1) +{ + struct timespec diff; + uint64_t nsec; + + diff.tv_sec = t2->tv_sec - t1->tv_sec; + diff.tv_nsec = t2->tv_nsec - t1->tv_nsec; + + if (diff.tv_nsec < 0) { + diff.tv_nsec += ODP_TIME_SEC_IN_NS; + diff.tv_sec -= 1; + } + + nsec = (diff.tv_sec * ODP_TIME_SEC_IN_NS) + diff.tv_nsec; + + return nsec; +} + +#include <odp/visibility_begin.h> + +odp_time_t _odp_time_cur(void) +{ + int ret; + odp_time_t time; + struct timespec sys_time; + struct timespec *start_time = &_odp_time_glob.start_time; + + ret = clock_gettime(CLOCK_MONOTONIC_RAW, &sys_time); + if (odp_unlikely(ret != 0)) + _ODP_ABORT("clock_gettime() failed\n"); + + time.nsec = time_diff_nsec(&sys_time, start_time); + + return time; +} + +uint64_t _odp_time_res(void) +{ + int ret; + struct timespec tres; + + ret = clock_getres(CLOCK_MONOTONIC_RAW, &tres); + if (odp_unlikely(ret != 0)) + _ODP_ABORT("clock_getres() failed\n"); + + return ODP_TIME_SEC_IN_NS / (uint64_t)tres.tv_nsec; +} + +#include <odp/visibility_end.h> + +int _odp_time_init_global(void) +{ + struct timespec *start_time; + int ret = 0; + _odp_time_global_t *global = &_odp_time_glob; + + memset(global, 0, sizeof(_odp_time_global_t)); + + start_time = &global->start_time; + start_time->tv_sec = 0; + start_time->tv_nsec = 0; + + ret = clock_gettime(CLOCK_MONOTONIC_RAW, start_time); + if (ret) + _ODP_ERR("clock_gettime() failed: %d\n", ret); + + return ret; +} + +int _odp_time_term_global(void) +{ + return 0; +} diff --git a/platform/linux-generic/arch/x86/cpu_flags.c b/platform/linux-generic/arch/x86/cpu_flags.c index 036645dbc..9211df002 100644 --- a/platform/linux-generic/arch/x86/cpu_flags.c +++ b/platform/linux-generic/arch/x86/cpu_flags.c @@ -1,45 +1,23 @@ /* Copyright (c) 2017-2018, Linaro Limited + * Copyright (c) 2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause */ -/*- - * BSD LICENSE - * - * Copyright(c) 2010-2015 Intel Corporation. All rights reserved. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2010-2015 Intel Corporation */ #include "cpu_flags.h" + +#include <odp/api/abi/time_cpu.h> + #include <odp_debug_internal.h> -#include <odp/api/abi/cpu_time.h> +#include <odp_global_data.h> + +#include <cpuid.h> +#include <errno.h> #include <stdio.h> #include <stdint.h> @@ -74,6 +52,7 @@ enum rte_cpu_flag_t { RTE_CPUFLAG_AVX, /**< AVX */ RTE_CPUFLAG_F16C, /**< F16C */ RTE_CPUFLAG_RDRAND, /**< RDRAND */ + RTE_CPUFLAG_HYPERVISOR, /**< Running in a VM */ /* (EAX 01h) EDX features */ RTE_CPUFLAG_FPU, /**< FPU */ @@ -130,6 +109,7 @@ enum rte_cpu_flag_t { RTE_CPUFLAG_INVPCID, /**< INVPCID */ RTE_CPUFLAG_RTM, /**< Transactional memory */ RTE_CPUFLAG_AVX512F, /**< AVX512F */ + RTE_CPUFLAG_RDSEED, /**< RDSEED instruction */ /* (EAX 80000001h) ECX features */ RTE_CPUFLAG_LAHF_SAHF, /**< LAHF_SAHF */ @@ -145,8 +125,29 @@ enum rte_cpu_flag_t { /* (EAX 80000007h) EDX features */ RTE_CPUFLAG_INVTSC, /**< INVTSC */ + RTE_CPUFLAG_AVX512DQ, /**< AVX512 Doubleword and Quadword */ + RTE_CPUFLAG_AVX512IFMA, /**< AVX512 Integer Fused Multiply-Add */ + RTE_CPUFLAG_AVX512CD, /**< AVX512 Conflict Detection*/ + RTE_CPUFLAG_AVX512BW, /**< AVX512 Byte and Word */ + RTE_CPUFLAG_AVX512VL, /**< AVX512 Vector Length */ + RTE_CPUFLAG_AVX512VBMI, /**< AVX512 Vector Bit Manipulation */ + RTE_CPUFLAG_AVX512VBMI2, /**< AVX512 Vector Bit Manipulation 2 */ + RTE_CPUFLAG_GFNI, /**< Galois Field New Instructions */ + RTE_CPUFLAG_VAES, /**< Vector AES */ + RTE_CPUFLAG_VPCLMULQDQ, /**< Vector Carry-less Multiply */ + RTE_CPUFLAG_AVX512VNNI, + /**< AVX512 Vector Neural Network Instructions */ + RTE_CPUFLAG_AVX512BITALG, /**< AVX512 Bit Algorithms */ + RTE_CPUFLAG_AVX512VPOPCNTDQ, /**< AVX512 Vector Popcount */ + RTE_CPUFLAG_CLDEMOTE, /**< Cache Line Demote */ + RTE_CPUFLAG_MOVDIRI, /**< Direct Store Instructions */ + RTE_CPUFLAG_MOVDIR64B, /**< Direct Store Instructions 64B */ + RTE_CPUFLAG_AVX512VP2INTERSECT, /**< AVX512 Two Register Intersection */ + + RTE_CPUFLAG_WAITPKG, /**< UMONITOR/UMWAIT/TPAUSE */ + /* The last item */ - RTE_CPUFLAG_NUMFLAGS, /**< This should always be the last! */ + RTE_CPUFLAG_NUMFLAGS, /**< This should always be the last! */ }; enum cpu_register_t { @@ -203,6 +204,7 @@ static const struct feature_entry cpu_feature_table[] = { FEAT_DEF(AVX, 0x00000001, 0, RTE_REG_ECX, 28) FEAT_DEF(F16C, 0x00000001, 0, RTE_REG_ECX, 29) FEAT_DEF(RDRAND, 0x00000001, 0, RTE_REG_ECX, 30) + FEAT_DEF(HYPERVISOR, 0x00000001, 0, RTE_REG_ECX, 31) FEAT_DEF(FPU, 0x00000001, 0, RTE_REG_EDX, 0) FEAT_DEF(VME, 0x00000001, 0, RTE_REG_EDX, 1) @@ -246,15 +248,36 @@ static const struct feature_entry cpu_feature_table[] = { FEAT_DEF(ENERGY_EFF, 0x00000006, 0, RTE_REG_ECX, 3) FEAT_DEF(FSGSBASE, 0x00000007, 0, RTE_REG_EBX, 0) - FEAT_DEF(BMI1, 0x00000007, 0, RTE_REG_EBX, 2) + FEAT_DEF(BMI1, 0x00000007, 0, RTE_REG_EBX, 3) FEAT_DEF(HLE, 0x00000007, 0, RTE_REG_EBX, 4) FEAT_DEF(AVX2, 0x00000007, 0, RTE_REG_EBX, 5) - FEAT_DEF(SMEP, 0x00000007, 0, RTE_REG_EBX, 6) - FEAT_DEF(BMI2, 0x00000007, 0, RTE_REG_EBX, 7) - FEAT_DEF(ERMS, 0x00000007, 0, RTE_REG_EBX, 8) + FEAT_DEF(SMEP, 0x00000007, 0, RTE_REG_EBX, 7) + FEAT_DEF(BMI2, 0x00000007, 0, RTE_REG_EBX, 8) + FEAT_DEF(ERMS, 0x00000007, 0, RTE_REG_EBX, 9) FEAT_DEF(INVPCID, 0x00000007, 0, RTE_REG_EBX, 10) FEAT_DEF(RTM, 0x00000007, 0, RTE_REG_EBX, 11) FEAT_DEF(AVX512F, 0x00000007, 0, RTE_REG_EBX, 16) + FEAT_DEF(AVX512DQ, 0x00000007, 0, RTE_REG_EBX, 17) + FEAT_DEF(RDSEED, 0x00000007, 0, RTE_REG_EBX, 18) + FEAT_DEF(AVX512IFMA, 0x00000007, 0, RTE_REG_EBX, 21) + FEAT_DEF(AVX512CD, 0x00000007, 0, RTE_REG_EBX, 28) + FEAT_DEF(AVX512BW, 0x00000007, 0, RTE_REG_EBX, 30) + FEAT_DEF(AVX512VL, 0x00000007, 0, RTE_REG_EBX, 31) + + FEAT_DEF(AVX512VBMI, 0x00000007, 0, RTE_REG_ECX, 1) + FEAT_DEF(WAITPKG, 0x00000007, 0, RTE_REG_ECX, 5) + FEAT_DEF(AVX512VBMI2, 0x00000007, 0, RTE_REG_ECX, 6) + FEAT_DEF(GFNI, 0x00000007, 0, RTE_REG_ECX, 8) + FEAT_DEF(VAES, 0x00000007, 0, RTE_REG_ECX, 9) + FEAT_DEF(VPCLMULQDQ, 0x00000007, 0, RTE_REG_ECX, 10) + FEAT_DEF(AVX512VNNI, 0x00000007, 0, RTE_REG_ECX, 11) + FEAT_DEF(AVX512BITALG, 0x00000007, 0, RTE_REG_ECX, 12) + FEAT_DEF(AVX512VPOPCNTDQ, 0x00000007, 0, RTE_REG_ECX, 14) + FEAT_DEF(CLDEMOTE, 0x00000007, 0, RTE_REG_ECX, 25) + FEAT_DEF(MOVDIRI, 0x00000007, 0, RTE_REG_ECX, 27) + FEAT_DEF(MOVDIR64B, 0x00000007, 0, RTE_REG_ECX, 28) + + FEAT_DEF(AVX512VP2INTERSECT, 0x00000007, 0, RTE_REG_EDX, 8) FEAT_DEF(LAHF_SAHF, 0x80000001, 0, RTE_REG_ECX, 0) FEAT_DEF(LZCNT, 0x80000001, 0, RTE_REG_ECX, 4) @@ -268,55 +291,30 @@ static const struct feature_entry cpu_feature_table[] = { FEAT_DEF(INVTSC, 0x80000007, 0, RTE_REG_EDX, 8) }; -/* - * Execute CPUID instruction and get contents of a specific register - * - * This function, when compiled with GCC, will generate architecture-neutral - * code, as per GCC manual. - */ -static void cpu_get_features(uint32_t leaf, uint32_t subleaf, - cpuid_registers_t out) -{ -#if defined(__i386__) && defined(__PIC__) - /* %ebx is a forbidden register if we compile with -fPIC or -fPIE */ - __asm__ __volatile__("movl %%ebx,%0 ; cpuid ; xchgl %%ebx,%0" - : "=r" (out[RTE_REG_EBX]), - "=a" (out[RTE_REG_EAX]), - "=c" (out[RTE_REG_ECX]), - "=d" (out[RTE_REG_EDX]) - : "a" (leaf), "c" (subleaf)); -#else - __asm__ __volatile__("cpuid" - : "=a" (out[RTE_REG_EAX]), - "=b" (out[RTE_REG_EBX]), - "=c" (out[RTE_REG_ECX]), - "=d" (out[RTE_REG_EDX]) - : "a" (leaf), "c" (subleaf)); -#endif -} - static int cpu_get_flag_enabled(enum rte_cpu_flag_t feature) { const struct feature_entry *feat; cpuid_registers_t regs; + unsigned int maxleaf; if (feature >= RTE_CPUFLAG_NUMFLAGS) /* Flag does not match anything in the feature tables */ - return -1; + return -ENOENT; feat = &cpu_feature_table[feature]; if (!feat->leaf) /* This entry in the table wasn't filled out! */ - return -1; + return -EFAULT; + + maxleaf = __get_cpuid_max(feat->leaf & 0x80000000, NULL); - cpu_get_features(feat->leaf & 0xffff0000, 0, regs); - if (((regs[RTE_REG_EAX] ^ feat->leaf) & 0xffff0000) || - regs[RTE_REG_EAX] < feat->leaf) + if (maxleaf < feat->leaf) return 0; - /* get the cpuid leaf containing the desired feature */ - cpu_get_features(feat->leaf, feat->subleaf, regs); + __cpuid_count(feat->leaf, feat->subleaf, + regs[RTE_REG_EAX], regs[RTE_REG_EBX], + regs[RTE_REG_ECX], regs[RTE_REG_EDX]); /* check if the feature is enabled */ return (regs[feat->reg] >> feat->bit) & 1; @@ -359,12 +357,16 @@ void _odp_cpu_flags_print_all(void) _ODP_PRINT("%s", str); } -int _odp_cpu_has_global_time(void) +int _odp_time_cpu_global_freq_is_const(void) { - if (cpu_get_flag_enabled(RTE_CPUFLAG_INVTSC) > 0) + if (odp_global_ro.system_info.cpu_constant_tsc || + cpu_get_flag_enabled(RTE_CPUFLAG_INVTSC) > 0) return 1; - return 0; + _ODP_ERR("WARN: assuming constant TSC based on CPU arch, but could not confirm from CPU " + "flags\n"); + + return 1; } int _odp_cpu_flags_has_rdtsc(void) diff --git a/platform/linux-generic/arch/x86/odp/api/abi/sync_inlines.h b/platform/linux-generic/arch/x86/odp/api/abi/sync_inlines.h new file mode 100644 index 000000000..bebe6b571 --- /dev/null +++ b/platform/linux-generic/arch/x86/odp/api/abi/sync_inlines.h @@ -0,0 +1,31 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2023 Nokia + */ + +#ifndef ODP_ARCH_SYNC_INLINES_H_ +#define ODP_ARCH_SYNC_INLINES_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +static inline void _odp_mb_sync(void) +{ + __asm__ volatile("mfence" ::: "memory"); +} + +static inline void _odp_mb_sync_load(void) +{ + __asm__ volatile("lfence" ::: "memory"); +} + +static inline void _odp_mb_sync_store(void) +{ + __asm__ volatile("sfence" ::: "memory"); +} + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/arch/x86/odp/api/abi/cpu_time.h b/platform/linux-generic/arch/x86/odp/api/abi/time_cpu.h index c74c4d606..baf79ad3f 100644 --- a/platform/linux-generic/arch/x86/odp/api/abi/cpu_time.h +++ b/platform/linux-generic/arch/x86/odp/api/abi/time_cpu.h @@ -4,8 +4,8 @@ * SPDX-License-Identifier: BSD-3-Clause */ -#ifndef ODP_ARCH_CPU_TIME_H_ -#define ODP_ARCH_CPU_TIME_H_ +#ifndef ODP_ARCH_TIME_CPU_H_ +#define ODP_ARCH_TIME_CPU_H_ #ifdef __cplusplus extern "C" { @@ -14,19 +14,19 @@ extern "C" { #include <stdint.h> #include <odp/api/abi/cpu_rdtsc.h> -static inline uint64_t _odp_cpu_global_time(void) +static inline uint64_t _odp_time_cpu_global(void) { return _odp_cpu_rdtsc(); } -static inline uint64_t _odp_cpu_global_time_strict(void) +static inline uint64_t _odp_time_cpu_global_strict(void) { __atomic_thread_fence(__ATOMIC_SEQ_CST); return _odp_cpu_rdtsc(); } -int _odp_cpu_has_global_time(void); -uint64_t _odp_cpu_global_time_freq(void); +int _odp_time_cpu_global_freq_is_const(void); +uint64_t _odp_time_cpu_global_freq(void); #ifdef __cplusplus } diff --git a/platform/linux-generic/arch/x86/odp/api/abi/time_inlines.h b/platform/linux-generic/arch/x86/odp/api/abi/time_inlines.h new file mode 100644 index 000000000..331d1996f --- /dev/null +++ b/platform/linux-generic/arch/x86/odp/api/abi/time_inlines.h @@ -0,0 +1,7 @@ +/* Copyright (c) 2023, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <odp/api/abi/time_cpu_inlines.h> diff --git a/platform/linux-generic/arch/x86/odp_sysinfo_parse.c b/platform/linux-generic/arch/x86/odp_sysinfo_parse.c index c74c52045..3cbdb2037 100644 --- a/platform/linux-generic/arch/x86/odp_sysinfo_parse.c +++ b/platform/linux-generic/arch/x86/odp_sysinfo_parse.c @@ -1,4 +1,5 @@ /* Copyright (c) 2016-2018, Linaro Limited + * Copyright (c) 2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -30,6 +31,11 @@ int _odp_cpuinfo_parser(FILE *file, system_info_t *sysinfo) strcpy(sysinfo->cpu_arch_str, "x86"); while (fgets(str, sizeof(str), file) != NULL && id < CONFIG_NUM_CPU_IDS) { + if (strstr(str, "flags") && strstr(str, "constant_tsc")) { + sysinfo->cpu_constant_tsc = 1; + continue; + } + pos = strstr(str, "model name"); if (pos) { freq_set = false; diff --git a/platform/linux-generic/arch/x86/odp_global_time.c b/platform/linux-generic/arch/x86/odp_time_cpu.c index 00e3a1ac1..aa00ac04e 100644 --- a/platform/linux-generic/arch/x86/odp_global_time.c +++ b/platform/linux-generic/arch/x86/odp_time_cpu.c @@ -6,17 +6,18 @@ #include <odp_posix_extensions.h> -#include <time.h> - #include <odp/api/hints.h> +#include <odp/api/time_types.h> + +#include <odp/api/abi/time_cpu.h> + #include <odp_debug_internal.h> -#include <odp/api/abi/cpu_time.h> -#define SEC_IN_NS 1000000000ULL +#include <time.h> /* Measure TSC frequency. Frequency information registers are defined for x86, * but those are often not enumerated. */ -uint64_t _odp_cpu_global_time_freq(void) +uint64_t _odp_time_cpu_global_freq(void) { struct timespec sleep, ts1, ts2; uint64_t t1, t2, ts_nsec, cycles, hz; @@ -29,35 +30,35 @@ uint64_t _odp_cpu_global_time_freq(void) sleep.tv_sec = 0; if (warm_up) - sleep.tv_nsec = SEC_IN_NS / 1000; + sleep.tv_nsec = ODP_TIME_SEC_IN_NS / 1000; else - sleep.tv_nsec = SEC_IN_NS / 4; + sleep.tv_nsec = ODP_TIME_SEC_IN_NS / 4; if (clock_gettime(CLOCK_MONOTONIC_RAW, &ts1)) { - _ODP_DBG("clock_gettime failed\n"); + _ODP_ERR("clock_gettime() failed\n"); return 0; } - t1 = _odp_cpu_global_time(); + t1 = _odp_time_cpu_global(); if (nanosleep(&sleep, NULL) < 0) { - _ODP_DBG("nanosleep failed\n"); + _ODP_ERR("nanosleep() failed\n"); return 0; } if (clock_gettime(CLOCK_MONOTONIC_RAW, &ts2)) { - _ODP_DBG("clock_gettime failed\n"); + _ODP_ERR("clock_gettime() failed\n"); return 0; } - t2 = _odp_cpu_global_time(); + t2 = _odp_time_cpu_global(); - ts_nsec = (ts2.tv_sec - ts1.tv_sec) * SEC_IN_NS; + ts_nsec = (ts2.tv_sec - ts1.tv_sec) * ODP_TIME_SEC_IN_NS; ts_nsec += ts2.tv_nsec - ts1.tv_nsec; cycles = t2 - t1; - hz = (cycles * SEC_IN_NS) / ts_nsec; + hz = (cycles * ODP_TIME_SEC_IN_NS) / ts_nsec; if (warm_up) warm_up = 0; diff --git a/platform/linux-generic/include-abi/odp/api/abi/dma.h b/platform/linux-generic/include-abi/odp/api/abi/dma.h new file mode 100644 index 000000000..60798143c --- /dev/null +++ b/platform/linux-generic/include-abi/odp/api/abi/dma.h @@ -0,0 +1,27 @@ +/* Copyright (c) 2023, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +/** + * @file + * + * ODP DMA + */ + +#ifndef ODP_API_ABI_DMA_H_ +#define ODP_API_ABI_DMA_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +/* Inlined functions for non-ABI compat mode */ +#include <odp/api/plat/dma_inlines.h> + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/include/odp/api/plat/debug_inlines.h b/platform/linux-generic/include/odp/api/plat/debug_inlines.h index 41af3dca4..0755b1fda 100644 --- a/platform/linux-generic/include/odp/api/plat/debug_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/debug_inlines.h @@ -1,5 +1,5 @@ /* Copyright (c) 2014-2018, Linaro Limited - * Copyright (c) 2020-2022, Nokia + * Copyright (c) 2020-2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -50,9 +50,9 @@ extern odp_abort_func_t _odp_abort_fn; /** * ODP LOG macro. */ -#define _ODP_LOG(level, fmt, ...) \ - _ODP_LOG_FN(level, "%s:%d:%s(): " fmt, __FILE__, \ - __LINE__, __func__, ##__VA_ARGS__) +#define _ODP_LOG(level, prefix, fmt, ...) \ + _ODP_LOG_FN(level, "%s: %s:%d:%s(): " fmt, prefix, \ + __FILE__, __LINE__, __func__, ##__VA_ARGS__) /** * Runtime assertion-macro - aborts if 'cond' is false. @@ -70,17 +70,27 @@ extern odp_abort_func_t _odp_abort_fn; do { \ if (ODP_DEBUG_PRINT == 1) \ __extension__ ({ \ - _ODP_LOG(ODP_LOG_DBG, ##__VA_ARGS__); \ + _ODP_LOG(ODP_LOG_DBG, "DBG", ##__VA_ARGS__); \ }); \ } while (0) /** + * Log warning message. + */ +#define _ODP_WARN(...) \ + do { \ + __extension__ ({ \ + _ODP_LOG(ODP_LOG_WARN, "WARN", ##__VA_ARGS__); \ + }); \ + } while (0) + +/** * Log error message. */ #define _ODP_ERR(...) \ do { \ __extension__ ({ \ - _ODP_LOG(ODP_LOG_ERR, ##__VA_ARGS__); \ + _ODP_LOG(ODP_LOG_ERR, "ERR", ##__VA_ARGS__); \ }); \ } while (0) @@ -91,7 +101,7 @@ extern odp_abort_func_t _odp_abort_fn; #define _ODP_ABORT(...) \ do { \ __extension__ ({ \ - _ODP_LOG(ODP_LOG_ABORT, ##__VA_ARGS__); \ + _ODP_LOG(ODP_LOG_ABORT, "ABORT", ##__VA_ARGS__); \ }); \ _odp_abort_fn(); \ } while (0) diff --git a/platform/linux-generic/include/odp/api/plat/dma_inlines.h b/platform/linux-generic/include/odp/api/plat/dma_inlines.h new file mode 100644 index 000000000..84b5fef5b --- /dev/null +++ b/platform/linux-generic/include/odp/api/plat/dma_inlines.h @@ -0,0 +1,135 @@ +/* Copyright (c) 2023, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_PLAT_DMA_INLINES_H_ +#define ODP_PLAT_DMA_INLINES_H_ + +#include <odp/api/buffer.h> +#include <odp/api/dma_types.h> +#include <odp/api/event_types.h> +#include <odp/api/hints.h> +#include <odp/api/pool_types.h> +#include <odp/api/queue_types.h> + +#include <odp/api/plat/debug_inlines.h> +#include <odp/api/plat/event_inline_types.h> + +#include <stdint.h> +#include <string.h> + +/** @cond _ODP_HIDE_FROM_DOXYGEN_ */ + +#ifndef _ODP_NO_INLINE + /* Inline functions by default */ + #define _ODP_INLINE static inline + #define odp_dma_compl_from_event __odp_dma_compl_from_event + #define odp_dma_compl_to_event __odp_dma_compl_to_event + #define odp_dma_compl_user_area __odp_dma_compl_user_area + #define odp_dma_compl_result __odp_dma_compl_result + #define odp_dma_transfer_param_init __odp_dma_transfer_param_init + #define odp_dma_compl_param_init __odp_dma_compl_param_init + #define odp_dma_compl_alloc __odp_dma_compl_alloc + #define odp_dma_compl_free __odp_dma_compl_free +#else + #define _ODP_INLINE +#endif + +_ODP_INLINE odp_dma_compl_t odp_dma_compl_from_event(odp_event_t ev) +{ + _ODP_ASSERT(_odp_event_hdr_field(ev, int8_t, event_type) == ODP_EVENT_DMA_COMPL); + + return (odp_dma_compl_t)(uintptr_t)ev; +} + +_ODP_INLINE odp_event_t odp_dma_compl_to_event(odp_dma_compl_t dma_compl) +{ + return (odp_event_t)(uintptr_t)dma_compl; +} + +_ODP_INLINE void *odp_dma_compl_user_area(odp_dma_compl_t dma_compl) +{ + return odp_buffer_user_area((odp_buffer_t)(uintptr_t)dma_compl); +} + +_ODP_INLINE int odp_dma_compl_result(odp_dma_compl_t dma_compl, odp_dma_result_t *result_out) +{ + odp_dma_result_t *result; + odp_buffer_t buf = (odp_buffer_t)(uintptr_t)dma_compl; + + if (odp_unlikely(dma_compl == ODP_DMA_COMPL_INVALID)) { + _ODP_ERR("Bad DMA compl handle\n"); + return -1; + } + + result = (odp_dma_result_t *)odp_buffer_addr(buf); + + if (result_out) + *result_out = *result; + + return result->success ? 0 : -1; +} + +_ODP_INLINE void odp_dma_transfer_param_init(odp_dma_transfer_param_t *trs_param) +{ + memset(trs_param, 0, sizeof(odp_dma_transfer_param_t)); + + trs_param->src_format = ODP_DMA_FORMAT_ADDR; + trs_param->dst_format = ODP_DMA_FORMAT_ADDR; + trs_param->num_src = 1; + trs_param->num_dst = 1; +} + +_ODP_INLINE void odp_dma_compl_param_init(odp_dma_compl_param_t *compl_param) +{ + memset(compl_param, 0, sizeof(odp_dma_compl_param_t)); + + compl_param->queue = ODP_QUEUE_INVALID; + compl_param->event = ODP_EVENT_INVALID; + compl_param->transfer_id = ODP_DMA_TRANSFER_ID_INVALID; +} + +_ODP_INLINE odp_dma_compl_t odp_dma_compl_alloc(odp_pool_t pool) +{ + odp_buffer_t buf; + odp_event_t ev; + odp_dma_result_t *result; + int8_t *ev_type; + + buf = odp_buffer_alloc(pool); + if (odp_unlikely(buf == ODP_BUFFER_INVALID)) + return ODP_DMA_COMPL_INVALID; + + result = (odp_dma_result_t *)odp_buffer_addr(buf); + memset(result, 0, sizeof(odp_dma_result_t)); + + ev = odp_buffer_to_event(buf); + ev_type = _odp_event_hdr_ptr(ev, int8_t, event_type); + *ev_type = ODP_EVENT_DMA_COMPL; + + return (odp_dma_compl_t)(uintptr_t)buf; +} + +_ODP_INLINE void odp_dma_compl_free(odp_dma_compl_t dma_compl) +{ + int8_t *ev_type; + odp_event_t ev; + odp_buffer_t buf = (odp_buffer_t)(uintptr_t)dma_compl; + + if (odp_unlikely(dma_compl == ODP_DMA_COMPL_INVALID)) { + _ODP_ERR("Bad DMA compl handle\n"); + return; + } + + ev = odp_buffer_to_event(buf); + ev_type = _odp_event_hdr_ptr(ev, int8_t, event_type); + *ev_type = ODP_EVENT_BUFFER; + + odp_buffer_free(buf); +} + +/** @endcond */ + +#endif diff --git a/platform/linux-generic/include/odp/api/plat/event_inlines.h b/platform/linux-generic/include/odp/api/plat/event_inlines.h index 4e3368ff0..2e7c7db5e 100644 --- a/platform/linux-generic/include/odp/api/plat/event_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/event_inlines.h @@ -9,12 +9,12 @@ #define ODP_PLAT_EVENT_INLINES_H_ #include <odp/api/buffer_types.h> -#include <odp/api/dma.h> #include <odp/api/event_types.h> #include <odp/api/packet_types.h> #include <odp/api/timer_types.h> #include <odp/api/plat/buffer_inline_types.h> +#include <odp/api/plat/debug_inlines.h> #include <odp/api/plat/event_inline_types.h> #include <odp/api/plat/event_vector_inline_types.h> #include <odp/api/plat/packet_inline_types.h> @@ -28,8 +28,10 @@ #define odp_event_type __odp_event_type #define odp_event_type_multi __odp_event_type_multi #define odp_event_user_area __odp_event_user_area + #define odp_event_user_area_and_flag __odp_event_user_area_and_flag #define odp_event_subtype __odp_event_subtype #define odp_event_types __odp_event_types + #define odp_event_types_multi __odp_event_types_multi #define odp_event_flow_id __odp_event_flow_id #define odp_event_flow_id_set __odp_event_flow_id_set #else @@ -72,6 +74,7 @@ _ODP_INLINE void *odp_event_user_area(odp_event_t event) switch (type) { case ODP_EVENT_BUFFER: + case ODP_EVENT_DMA_COMPL: return _odp_buffer_get((odp_buffer_t)event, void *, uarea_addr); case ODP_EVENT_PACKET: return _odp_pkt_get((odp_packet_t)event, void *, user_area); @@ -79,9 +82,47 @@ _ODP_INLINE void *odp_event_user_area(odp_event_t event) return _odp_event_vect_get((odp_packet_vector_t)event, void *, uarea_addr); case ODP_EVENT_TIMEOUT: return _odp_timeout_hdr_field((odp_timeout_t)event, void *, uarea_addr); + default: + return NULL; + } +} + +_ODP_INLINE void *odp_event_user_area_and_flag(odp_event_t event, int *flag) +{ + const odp_event_type_t type = __odp_event_type_get(event); + + _ODP_ASSERT(flag != NULL); + + switch (type) { + case ODP_EVENT_BUFFER: case ODP_EVENT_DMA_COMPL: - return odp_dma_compl_user_area((odp_dma_compl_t)event); + *flag = -1; + return _odp_buffer_get((odp_buffer_t)event, void *, uarea_addr); + case ODP_EVENT_PACKET: + { + _odp_packet_flags_t pkt_flags; + odp_packet_t pkt = (odp_packet_t)event; + + pkt_flags.all_flags = _odp_pkt_get(pkt, uint32_t, flags); + *flag = pkt_flags.user_flag; + + return _odp_pkt_get(pkt, void *, user_area); + } + case ODP_EVENT_PACKET_VECTOR: + { + _odp_event_vector_flags_t pktv_flags; + odp_packet_vector_t pktv = (odp_packet_vector_t)event; + + pktv_flags.all_flags = _odp_event_vect_get(pktv, uint32_t, flags); + *flag = pktv_flags.user_flag; + + return _odp_event_vect_get(pktv, void *, uarea_addr); + } + case ODP_EVENT_TIMEOUT: + *flag = -1; + return _odp_timeout_hdr_field((odp_timeout_t)event, void *, uarea_addr); default: + *flag = -1; return NULL; } } @@ -106,6 +147,22 @@ _ODP_INLINE odp_event_type_t odp_event_types(odp_event_t event, return event_type; } +_ODP_INLINE void odp_event_types_multi(const odp_event_t event[], odp_event_type_t type[], + odp_event_subtype_t subtype[], int num) +{ + for (int i = 0; i < num; i++) + type[i] = __odp_event_type_get(event[i]); + + if (subtype == NULL) + return; + + for (int i = 0; i < num; i++) { + subtype[i] = (type[i] == ODP_EVENT_PACKET) ? + (odp_event_subtype_t)_odp_pkt_get((odp_packet_t)event[i], int8_t, + subtype) : ODP_EVENT_NO_SUBTYPE; + } +} + _ODP_INLINE uint32_t odp_event_flow_id(odp_event_t event) { return _odp_event_hdr_field(event, uint8_t, flow_id); diff --git a/platform/linux-generic/include/odp/api/plat/sync_inlines.h b/platform/linux-generic/include/odp/api/plat/sync_inlines.h index b6a96188c..b3a88b629 100644 --- a/platform/linux-generic/include/odp/api/plat/sync_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/sync_inlines.h @@ -1,7 +1,6 @@ -/* Copyright (c) 2016-2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2016-2018 Linaro Limited + * Copyright (c) 2023 Nokia */ /** @@ -13,6 +12,8 @@ #ifndef ODP_PLAT_SYNC_INLINE_H_ #define ODP_PLAT_SYNC_INLINE_H_ +#include <odp/api/abi/sync_inlines.h> + #ifdef __cplusplus extern "C" { #endif @@ -25,6 +26,9 @@ extern "C" { #define odp_mb_release __odp_mb_release #define odp_mb_acquire __odp_mb_acquire #define odp_mb_full __odp_mb_full + #define odp_mb_sync __odp_mb_sync + #define odp_mb_sync_load __odp_mb_sync_load + #define odp_mb_sync_store __odp_mb_sync_store #else #define _ODP_INLINE #endif @@ -44,6 +48,21 @@ _ODP_INLINE void odp_mb_full(void) __atomic_thread_fence(__ATOMIC_SEQ_CST); } +_ODP_INLINE void odp_mb_sync(void) +{ + _odp_mb_sync(); +} + +_ODP_INLINE void odp_mb_sync_load(void) +{ + _odp_mb_sync_load(); +} + +_ODP_INLINE void odp_mb_sync_store(void) +{ + _odp_mb_sync_store(); +} + /** @endcond */ #ifdef __cplusplus diff --git a/platform/linux-generic/include/odp/api/plat/time_inlines.h b/platform/linux-generic/include/odp/api/plat/time_inlines.h index f8f4bee89..35a35c72e 100644 --- a/platform/linux-generic/include/odp/api/plat/time_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/time_inlines.h @@ -1,5 +1,5 @@ /* Copyright (c) 2018, Linaro Limited - * Copyright (c) 2020-2022, Nokia + * Copyright (c) 2020-2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -12,79 +12,12 @@ #include <odp/api/hints.h> #include <odp/api/time_types.h> -#include <odp/api/abi/cpu_time.h> +#include <odp/api/abi/time_inlines.h> #include <stdint.h> /** @cond _ODP_HIDE_FROM_DOXYGEN_ */ -#define _ODP_TIMESPEC_SIZE 16 -#define _ODP_TIME_GIGA_HZ 1000000000ULL - -typedef struct _odp_time_global_t { - /* Storage space for struct timespec. Posix headers are not included - * here to avoid application exposure. */ - uint8_t timespec[_ODP_TIMESPEC_SIZE] ODP_ALIGNED(_ODP_TIMESPEC_SIZE); - - int use_hw; - uint64_t hw_start; - uint64_t hw_freq_hz; - -} _odp_time_global_t; - -extern _odp_time_global_t _odp_time_glob; - -odp_time_t _odp_timespec_cur(void); - -static inline odp_time_t _odp_time_cur(void) -{ - if (_odp_time_glob.use_hw) { - odp_time_t time; - - time.count = _odp_cpu_global_time() - _odp_time_glob.hw_start; - return time; - } - - return _odp_timespec_cur(); -} - -static inline odp_time_t _odp_time_cur_strict(void) -{ - if (_odp_time_glob.use_hw) { - odp_time_t time; - - time.count = _odp_cpu_global_time_strict() - _odp_time_glob.hw_start; - return time; - } - - return _odp_timespec_cur(); -} - -static inline uint64_t _odp_time_hw_to_ns(odp_time_t time) -{ - uint64_t nsec; - uint64_t freq_hz = _odp_time_glob.hw_freq_hz; - uint64_t count = time.count; - uint64_t sec = 0; - - if (count >= freq_hz) { - sec = count / freq_hz; - count = count - sec * freq_hz; - } - - nsec = (_ODP_TIME_GIGA_HZ * count) / freq_hz; - - return (sec * _ODP_TIME_GIGA_HZ) + nsec; -} - -static inline uint64_t _odp_time_convert_to_ns(odp_time_t time) -{ - if (_odp_time_glob.use_hw) - return _odp_time_hw_to_ns(time); - - return time.nsec; -} - #ifndef _ODP_NO_INLINE /* Inline functions by default */ #define _ODP_INLINE static inline @@ -104,6 +37,14 @@ static inline uint64_t _odp_time_convert_to_ns(odp_time_t time) #define odp_time_diff_ns __odp_time_diff_ns #define odp_time_sum __odp_time_sum + #define odp_time_local_from_ns __odp_time_local_from_ns + #define odp_time_global_from_ns __odp_time_global_from_ns + + #define odp_time_local_res __odp_time_local_res + #define odp_time_global_res __odp_time_global_res + + #define odp_time_wait_ns __odp_time_wait_ns + #define odp_time_wait_until __odp_time_wait_until #else #define _ODP_INLINE #endif @@ -130,27 +71,27 @@ _ODP_INLINE odp_time_t odp_time_global_strict(void) _ODP_INLINE uint64_t odp_time_local_ns(void) { - return _odp_time_convert_to_ns(_odp_time_cur()); + return _odp_time_to_ns(_odp_time_cur()); } _ODP_INLINE uint64_t odp_time_global_ns(void) { - return _odp_time_convert_to_ns(_odp_time_cur()); + return _odp_time_to_ns(_odp_time_cur()); } _ODP_INLINE uint64_t odp_time_local_strict_ns(void) { - return _odp_time_convert_to_ns(_odp_time_cur_strict()); + return _odp_time_to_ns(_odp_time_cur_strict()); } _ODP_INLINE uint64_t odp_time_global_strict_ns(void) { - return _odp_time_convert_to_ns(_odp_time_cur_strict()); + return _odp_time_to_ns(_odp_time_cur_strict()); } _ODP_INLINE uint64_t odp_time_to_ns(odp_time_t time) { - return _odp_time_convert_to_ns(time); + return _odp_time_to_ns(time); } _ODP_INLINE int odp_time_cmp(odp_time_t t2, odp_time_t t1) @@ -191,6 +132,44 @@ _ODP_INLINE odp_time_t odp_time_sum(odp_time_t t1, odp_time_t t2) return time; } +_ODP_INLINE odp_time_t odp_time_local_from_ns(uint64_t ns) +{ + return _odp_time_from_ns(ns); +} + +_ODP_INLINE odp_time_t odp_time_global_from_ns(uint64_t ns) +{ + return _odp_time_from_ns(ns); +} + +_ODP_INLINE uint64_t odp_time_local_res(void) +{ + return _odp_time_res(); +} + +_ODP_INLINE uint64_t odp_time_global_res(void) +{ + return _odp_time_res(); +} + +_ODP_INLINE void odp_time_wait_until(odp_time_t time) +{ + odp_time_t cur; + + do { + cur = _odp_time_cur(); + } while (odp_time_cmp(time, cur) > 0); +} + +_ODP_INLINE void odp_time_wait_ns(uint64_t ns) +{ + odp_time_t cur = _odp_time_cur(); + odp_time_t wait = _odp_time_from_ns(ns); + odp_time_t end_time = odp_time_sum(cur, wait); + + odp_time_wait_until(end_time); +} + /** @endcond */ #endif diff --git a/platform/linux-generic/include/odp/api/plat/timer_inlines.h b/platform/linux-generic/include/odp/api/plat/timer_inlines.h index 648459c78..9ba0287e0 100644 --- a/platform/linux-generic/include/odp/api/plat/timer_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/timer_inlines.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2022, Nokia +/* Copyright (c) 2022-2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -27,6 +27,7 @@ #define odp_timer_tick_to_ns __odp_timer_tick_to_ns #define odp_timer_ns_to_tick __odp_timer_ns_to_tick #define odp_timeout_from_event __odp_timeout_from_event + #define odp_timeout_from_event_multi __odp_timeout_from_event_multi #define odp_timeout_to_event __odp_timeout_to_event #else #define _ODP_INLINE @@ -75,6 +76,15 @@ _ODP_INLINE odp_timeout_t odp_timeout_from_event(odp_event_t ev) return (odp_timeout_t)ev; } +_ODP_INLINE void odp_timeout_from_event_multi(odp_timeout_t tmo[], const odp_event_t ev[], int num) +{ + for (int i = 0; i < num; i++) { + _ODP_ASSERT(odp_event_type(ev[i]) == ODP_EVENT_TIMEOUT); + + tmo[i] = odp_timeout_from_event(ev[i]); + } +} + _ODP_INLINE odp_event_t odp_timeout_to_event(odp_timeout_t tmo) { return (odp_event_t)tmo; diff --git a/platform/linux-generic/include/odp_debug_internal.h b/platform/linux-generic/include/odp_debug_internal.h index 4c1fa8bbb..d1fc0d0ba 100644 --- a/platform/linux-generic/include/odp_debug_internal.h +++ b/platform/linux-generic/include/odp_debug_internal.h @@ -55,7 +55,7 @@ extern "C" { do { \ if (ODP_DEBUG_PRINT == 1 && CONFIG_DEBUG_LEVEL >= (level)) \ __extension__ ({ \ - _ODP_LOG(ODP_LOG_DBG, ##__VA_ARGS__); \ + _ODP_LOG(ODP_LOG_DBG, "DBG", ##__VA_ARGS__); \ }); \ } while (0) diff --git a/platform/linux-generic/include/odp_global_data.h b/platform/linux-generic/include/odp_global_data.h index d4cc9cda4..67b7572ef 100644 --- a/platform/linux-generic/include/odp_global_data.h +++ b/platform/linux-generic/include/odp_global_data.h @@ -1,4 +1,5 @@ /* Copyright (c) 2013-2018, Linaro Limited + * Copyright (c) 2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -34,7 +35,8 @@ typedef struct { uint64_t default_cpu_hz; uint64_t page_size; int cache_line_size; - odp_bool_t cpu_hz_static; + uint8_t cpu_hz_static; + uint8_t cpu_constant_tsc; odp_cpu_arch_t cpu_arch; odp_cpu_arch_isa_t cpu_isa_sw; odp_cpu_arch_isa_t cpu_isa_hw; diff --git a/platform/linux-generic/include/odp_sysinfo_internal.h b/platform/linux-generic/include/odp_sysinfo_internal.h index 0fef1aa32..c14cf78d9 100644 --- a/platform/linux-generic/include/odp_sysinfo_internal.h +++ b/platform/linux-generic/include/odp_sysinfo_internal.h @@ -27,9 +27,9 @@ static inline int _odp_dummy_cpuinfo(system_info_t *sysinfo) sysinfo->cpu_arch = ODP_CPU_ARCH_UNKNOWN; - _ODP_DBG("Warning: use dummy values for freq and model string\n"); + _ODP_WARN("Use dummy values for freq and model string\n"); for (i = 0; i < CONFIG_NUM_CPU_IDS; i++) { - _ODP_PRINT("WARN: cpu[%i] uses default max frequency of " + _ODP_WARN("CPU[%i] uses default max frequency of " "%" PRIu64 " Hz from config file\n", i, cpu_hz_max); sysinfo->cpu_hz_max[i] = cpu_hz_max; strcpy(sysinfo->model_str[i], "UNKNOWN"); diff --git a/platform/linux-generic/odp_crypto_openssl.c b/platform/linux-generic/odp_crypto_openssl.c index d4c7a3f1a..26c9ce7b7 100644 --- a/platform/linux-generic/odp_crypto_openssl.c +++ b/platform/linux-generic/odp_crypto_openssl.c @@ -55,7 +55,6 @@ * Keep sorted: first by key length, then by IV length */ static const odp_crypto_cipher_capability_t cipher_capa_null[] = { -{.key_len = 0, .iv_len = 0}, {.key_len = 0, .iv_len = 0, .bit_mode = 1} }; static const odp_crypto_cipher_capability_t cipher_capa_trides_cbc[] = { @@ -115,9 +114,7 @@ static const odp_crypto_cipher_capability_t cipher_capa_aes_eea2[] = { * Keep sorted: first by digest length, then by key length */ static const odp_crypto_auth_capability_t auth_capa_null[] = { -{.digest_len = 0, .key_len = 0, .aad_len = {.min = 0, .max = 0, .inc = 0}, - .bit_mode = 1}, -{.digest_len = 0, .key_len = 0, .aad_len = {.min = 0, .max = 0, .inc = 0} } }; +{.digest_len = 0, .key_len = 0, .aad_len = {.min = 0, .max = 0, .inc = 0}, .bit_mode = 1} }; static const odp_crypto_auth_capability_t auth_capa_md5_hmac[] = { {.digest_len = 12, .key_len = 16, .aad_len = {.min = 0, .max = 0, .inc = 0} }, @@ -217,8 +214,8 @@ struct odp_crypto_generic_session_t { odp_crypto_session_param_t p; odp_bool_t do_cipher_first; - uint8_t cipher_bit_mode : 1; - uint8_t cipher_range_used : 1; + uint8_t cipher_range_in_bits : 1; + uint8_t auth_range_in_bits : 1; uint8_t auth_range_used : 1; struct { @@ -1122,70 +1119,72 @@ static int process_cipher_param(odp_crypto_generic_session_t *session, return 0; } -static -odp_crypto_alg_err_t cipher_encrypt_bits(odp_packet_t pkt, - const odp_crypto_packet_op_param_t - *param, - odp_crypto_generic_session_t *session) +static odp_crypto_alg_err_t cipher_encrypt_bytes(odp_packet_t pkt, + const odp_crypto_packet_op_param_t *param, + odp_crypto_generic_session_t *session) { EVP_CIPHER_CTX *ctx = local.cipher_ctx[session->idx]; int dummy_len = 0; int cipher_len; - uint32_t in_len = (param->cipher_range.length + 7) / 8; + uint32_t in_len = param->cipher_range.length; + uint32_t offset = param->cipher_range.offset; uint8_t data[in_len]; int ret; - uint32_t offset; - - /* Range offset is in bits in bit mode but must be divisible by 8. */ - offset = param->cipher_range.offset / 8; EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, param->cipher_iv_ptr); - odp_packet_copy_to_mem(pkt, offset, in_len, data); - EVP_EncryptUpdate(ctx, data, &cipher_len, data, in_len); - ret = EVP_EncryptFinal_ex(ctx, data + cipher_len, &dummy_len); cipher_len += dummy_len; - odp_packet_copy_from_mem(pkt, offset, in_len, data); - return ret <= 0 ? ODP_CRYPTO_ALG_ERR_DATA_SIZE : ODP_CRYPTO_ALG_ERR_NONE; } -static -odp_crypto_alg_err_t cipher_decrypt_bits(odp_packet_t pkt, - const odp_crypto_packet_op_param_t - *param, - odp_crypto_generic_session_t *session) +static odp_crypto_alg_err_t cipher_decrypt_bytes(odp_packet_t pkt, + const odp_crypto_packet_op_param_t *param, + odp_crypto_generic_session_t *session) { EVP_CIPHER_CTX *ctx = local.cipher_ctx[session->idx]; int dummy_len = 0; int cipher_len; - uint32_t in_len = (param->cipher_range.length + 7) / 8; + uint32_t in_len = param->cipher_range.length; + uint32_t offset = param->cipher_range.offset; uint8_t data[in_len]; int ret; - uint32_t offset; - - /* Range offset is in bits in bit mode but must be divisible by 8. */ - offset = param->cipher_range.offset / 8; EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, param->cipher_iv_ptr); - odp_packet_copy_to_mem(pkt, offset, in_len, data); - EVP_DecryptUpdate(ctx, data, &cipher_len, data, in_len); - ret = EVP_DecryptFinal_ex(ctx, data + cipher_len, &dummy_len); cipher_len += dummy_len; - odp_packet_copy_from_mem(pkt, offset, in_len, data); - return ret <= 0 ? ODP_CRYPTO_ALG_ERR_DATA_SIZE : ODP_CRYPTO_ALG_ERR_NONE; } +static odp_crypto_alg_err_t cipher_encrypt_bits(odp_packet_t pkt, + const odp_crypto_packet_op_param_t *param, + odp_crypto_generic_session_t *session) +{ + odp_crypto_packet_op_param_t new_param = *param; + + new_param.cipher_range.offset /= 8; + new_param.cipher_range.length = (new_param.cipher_range.length + 7) / 8; + return cipher_encrypt_bytes(pkt, &new_param, session); +} + +static odp_crypto_alg_err_t cipher_decrypt_bits(odp_packet_t pkt, + const odp_crypto_packet_op_param_t *param, + odp_crypto_generic_session_t *session) +{ + odp_crypto_packet_op_param_t new_param = *param; + + new_param.cipher_range.offset /= 8; + new_param.cipher_range.length = (new_param.cipher_range.length + 7) / 8; + return cipher_decrypt_bytes(pkt, &new_param, session); +} + static int process_cipher_param_bits(odp_crypto_generic_session_t *session, const EVP_CIPHER *cipher) { @@ -1199,7 +1198,6 @@ static int process_cipher_param_bits(odp_crypto_generic_session_t *session, session->p.cipher_iv_len) return -1; - session->cipher_bit_mode = 1; session->cipher.evp_cipher = cipher; memcpy(session->cipher.key_data, session->p.cipher_key.data, @@ -1207,11 +1205,14 @@ static int process_cipher_param_bits(odp_crypto_generic_session_t *session, /* Set function */ if (ODP_CRYPTO_OP_ENCODE == session->p.op) { - session->cipher.func = cipher_encrypt_bits; session->cipher.init = cipher_encrypt_init; + session->cipher.func = session->cipher_range_in_bits ? cipher_encrypt_bits + : cipher_encrypt_bytes; + } else { - session->cipher.func = cipher_decrypt_bits; session->cipher.init = cipher_decrypt_init; + session->cipher.func = session->cipher_range_in_bits ? cipher_decrypt_bits + : cipher_decrypt_bytes; } return 0; @@ -1813,7 +1814,7 @@ int odp_crypto_capability(odp_crypto_capability_t *capa) #if _ODP_HAVE_CHACHA20_POLY1305 capa->ciphers.bit.chacha20_poly1305 = 1; #endif - capa->ciphers.bit.aes_eea2 = 1; + capa->ciphers.bit.aes_eea2 = 1; capa->auths.bit.null = 1; capa->auths.bit.md5_hmac = 1; @@ -2022,6 +2023,8 @@ odp_crypto_session_create(const odp_crypto_session_param_t *param, { int rc; odp_crypto_generic_session_t *session; + int cipher_bit_mode_supported = 0; + int auth_bit_mode_supported = 0; if (odp_global_ro.disable.crypto) { _ODP_ERR("Crypto is disabled\n"); @@ -2052,9 +2055,9 @@ odp_crypto_session_create(const odp_crypto_session_param_t *param, /* Copy parameters */ session->p = *param; - session->cipher_bit_mode = 0; + session->cipher_range_in_bits = !!param->cipher_range_in_bits; + session->auth_range_in_bits = !!param->auth_range_in_bits; session->auth_range_used = 1; - session->cipher_range_used = 1; if (session->p.cipher_iv_len > EVP_MAX_IV_LENGTH) { _ODP_DBG("Maximum IV length exceeded\n"); @@ -2079,7 +2082,7 @@ odp_crypto_session_create(const odp_crypto_session_param_t *param, case ODP_CIPHER_ALG_NULL: session->cipher.func = null_crypto_routine; session->cipher.init = null_crypto_init_routine; - session->cipher_range_used = 0; + cipher_bit_mode_supported = 1; rc = 0; break; case ODP_CIPHER_ALG_3DES_CBC: @@ -2184,11 +2187,15 @@ odp_crypto_session_create(const odp_crypto_session_param_t *param, EVP_aes_128_ctr()); else rc = -1; + cipher_bit_mode_supported = 1; break; default: rc = -1; } + if (session->cipher_range_in_bits && !cipher_bit_mode_supported) + rc = -1; + /* Check result */ if (rc) { *status = ODP_CRYPTO_SES_ERR_CIPHER; @@ -2200,7 +2207,7 @@ odp_crypto_session_create(const odp_crypto_session_param_t *param, case ODP_AUTH_ALG_NULL: session->auth.func = null_crypto_routine; session->auth.init = null_crypto_init_routine; - session->auth_range_used = 0; + auth_bit_mode_supported = 1; rc = 0; break; case ODP_AUTH_ALG_MD5_HMAC: @@ -2314,6 +2321,9 @@ odp_crypto_session_create(const odp_crypto_session_param_t *param, rc = -1; } + if (session->auth_range_in_bits && !auth_bit_mode_supported) + rc = -1; + /* Check result */ if (rc) { *status = ODP_CRYPTO_SES_ERR_AUTH; @@ -2632,12 +2642,16 @@ static void copy_ranges(odp_packet_t dst, int32_t shift = param->dst_offset_shift; int rc; - if (session->cipher_bit_mode) { + if (session->cipher_range_in_bits) { c_range.offset /= 8; c_range.length = (c_range.length + 7) / 8; } + if (session->auth_range_in_bits) { + a_range.offset /= 8; + a_range.length = (a_range.length + 7) / 8; + } - if (session->cipher_range_used) { + if (c_range.length > 0) { rc = odp_packet_copy_from_pkt(dst, c_range.offset + shift, src, c_range.offset, c_range.length); @@ -2646,7 +2660,7 @@ static void copy_ranges(odp_packet_t dst, return; } } - if (session->auth_range_used) { + if (session->auth_range_used && a_range.length > 0) { rc = odp_packet_copy_from_pkt(dst, a_range.offset + shift, src, a_range.offset, a_range.length); @@ -2663,12 +2677,13 @@ static int crypto_int_oop_encode(odp_packet_t pkt_in, const odp_crypto_packet_op_param_t *param) { odp_crypto_packet_op_param_t new_param = *param; - const uint32_t scale = session->cipher_bit_mode ? 8 : 1; + const uint32_t c_scale = session->cipher_range_in_bits ? 8 : 1; + const uint32_t a_scale = session->auth_range_in_bits ? 8 : 1; copy_ranges(*pkt_out, pkt_in, session, param); - new_param.cipher_range.offset += param->dst_offset_shift * scale; - new_param.auth_range.offset += param->dst_offset_shift; + new_param.cipher_range.offset += param->dst_offset_shift * c_scale; + new_param.auth_range.offset += param->dst_offset_shift * a_scale; return crypto_int(*pkt_out, pkt_out, &new_param); } diff --git a/platform/linux-generic/odp_dma.c b/platform/linux-generic/odp_dma.c index fa45b3b02..8ac18260c 100644 --- a/platform/linux-generic/odp_dma.c +++ b/platform/linux-generic/odp_dma.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2021-2022, Nokia +/* Copyright (c) 2021-2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -99,11 +99,12 @@ int odp_dma_capability(odp_dma_capability_t *capa) capa->queue_type_sched = 1; capa->queue_type_plain = 1; - capa->pool.max_pools = _odp_dma_glb->pool_capa.buf.max_pools; - capa->pool.max_num = _odp_dma_glb->pool_capa.buf.max_num; - capa->pool.max_uarea_size = _odp_dma_glb->pool_capa.buf.max_uarea_size; - capa->pool.min_cache_size = _odp_dma_glb->pool_capa.buf.min_cache_size; - capa->pool.max_cache_size = _odp_dma_glb->pool_capa.buf.max_cache_size; + capa->pool.max_pools = _odp_dma_glb->pool_capa.buf.max_pools; + capa->pool.max_num = _odp_dma_glb->pool_capa.buf.max_num; + capa->pool.max_uarea_size = _odp_dma_glb->pool_capa.buf.max_uarea_size; + capa->pool.uarea_persistence = _odp_dma_glb->pool_capa.buf.uarea_persistence; + capa->pool.min_cache_size = _odp_dma_glb->pool_capa.buf.min_cache_size; + capa->pool.max_cache_size = _odp_dma_glb->pool_capa.buf.max_cache_size; return 0; } @@ -321,16 +322,6 @@ odp_dma_t odp_dma_lookup(const char *name) return ODP_DMA_INVALID; } -void odp_dma_transfer_param_init(odp_dma_transfer_param_t *trs_param) -{ - memset(trs_param, 0, sizeof(odp_dma_transfer_param_t)); - - trs_param->src_format = ODP_DMA_FORMAT_ADDR; - trs_param->dst_format = ODP_DMA_FORMAT_ADDR; - trs_param->num_src = 1; - trs_param->num_dst = 1; -} - static uint32_t transfer_len(const odp_dma_transfer_param_t *trs_param) { uint32_t i; @@ -568,14 +559,6 @@ int odp_dma_transfer_multi(odp_dma_t dma, const odp_dma_transfer_param_t *trs_pa return i; } -void odp_dma_compl_param_init(odp_dma_compl_param_t *compl_param) -{ - memset(compl_param, 0, sizeof(odp_dma_compl_param_t)); - compl_param->queue = ODP_QUEUE_INVALID; - compl_param->event = ODP_EVENT_INVALID; - compl_param->transfer_id = ODP_DMA_TRANSFER_ID_INVALID; -} - odp_dma_transfer_id_t odp_dma_transfer_id_alloc(odp_dma_t dma) { int32_t num; @@ -756,83 +739,19 @@ odp_pool_t odp_dma_pool_create(const char *name, const odp_dma_pool_param_t *dma } odp_pool_param_init(&pool_param); - pool_param.type = ODP_POOL_BUFFER; - pool_param.buf.num = num; - pool_param.buf.uarea_size = uarea_size; - pool_param.buf.cache_size = cache_size; - pool_param.buf.size = sizeof(odp_dma_result_t); + pool_param.type = ODP_POOL_BUFFER; + pool_param.uarea_init.init_fn = dma_pool_param->uarea_init.init_fn; + pool_param.uarea_init.args = dma_pool_param->uarea_init.args; + pool_param.buf.num = num; + pool_param.buf.uarea_size = uarea_size; + pool_param.buf.cache_size = cache_size; + pool_param.buf.size = sizeof(odp_dma_result_t); pool = _odp_pool_create(name, &pool_param, ODP_POOL_DMA_COMPL); return pool; } -odp_dma_compl_t odp_dma_compl_alloc(odp_pool_t pool) -{ - odp_buffer_t buf; - odp_event_t ev; - odp_dma_result_t *result; - - buf = odp_buffer_alloc(pool); - - if (odp_unlikely(buf == ODP_BUFFER_INVALID)) - return ODP_DMA_COMPL_INVALID; - - result = odp_buffer_addr(buf); - memset(result, 0, sizeof(odp_dma_result_t)); - - ev = odp_buffer_to_event(buf); - _odp_event_type_set(ev, ODP_EVENT_DMA_COMPL); - - return (odp_dma_compl_t)(uintptr_t)buf; -} - -void odp_dma_compl_free(odp_dma_compl_t dma_compl) -{ - odp_event_t ev; - odp_buffer_t buf = (odp_buffer_t)(uintptr_t)dma_compl; - - if (odp_unlikely(dma_compl == ODP_DMA_COMPL_INVALID)) { - _ODP_ERR("Bad DMA compl handle\n"); - return; - } - - ev = odp_buffer_to_event(buf); - _odp_event_type_set(ev, ODP_EVENT_BUFFER); - - odp_buffer_free(buf); -} - -odp_dma_compl_t odp_dma_compl_from_event(odp_event_t ev) -{ - _ODP_ASSERT(odp_event_type(ev) == ODP_EVENT_DMA_COMPL); - - return (odp_dma_compl_t)(uintptr_t)ev; -} - -odp_event_t odp_dma_compl_to_event(odp_dma_compl_t dma_compl) -{ - return (odp_event_t)(uintptr_t)dma_compl; -} - -int odp_dma_compl_result(odp_dma_compl_t dma_compl, odp_dma_result_t *result_out) -{ - odp_dma_result_t *result; - odp_buffer_t buf = (odp_buffer_t)(uintptr_t)dma_compl; - - if (odp_unlikely(dma_compl == ODP_DMA_COMPL_INVALID)) { - _ODP_ERR("Bad DMA compl handle\n"); - return -1; - } - - result = odp_buffer_addr(buf); - - if (result_out) - *result_out = *result; - - return result->success ? 0 : -1; -} - uint64_t odp_dma_to_u64(odp_dma_t dma) { return (uint64_t)(uintptr_t)dma; @@ -843,11 +762,6 @@ uint64_t odp_dma_compl_to_u64(odp_dma_compl_t dma_compl) return (uint64_t)(uintptr_t)dma_compl; } -void *odp_dma_compl_user_area(odp_dma_compl_t dma_compl) -{ - return odp_buffer_user_area((odp_buffer_t)(uintptr_t)dma_compl); -} - void odp_dma_print(odp_dma_t dma) { dma_session_t *session = dma_session_from_handle(dma); diff --git a/platform/linux-generic/odp_dma_api.c b/platform/linux-generic/odp_dma_api.c new file mode 100644 index 000000000..1e1d5d91f --- /dev/null +++ b/platform/linux-generic/odp_dma_api.c @@ -0,0 +1,11 @@ +/* Copyright (c) 2023, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <odp/api/dma.h> + +/* Non-inlined functions for ABI compat mode */ +#define _ODP_NO_INLINE +#include <odp/api/plat/dma_inlines.h> diff --git a/platform/linux-generic/odp_init.c b/platform/linux-generic/odp_init.c index f1aae95bb..e6ea8bc0c 100644 --- a/platform/linux-generic/odp_init.c +++ b/platform/linux-generic/odp_init.c @@ -27,8 +27,8 @@ enum init_stage { LIBCONFIG_INIT, CPUMASK_INIT, CPU_CYCLES_INIT, - TIME_INIT, SYSINFO_INIT, + TIME_INIT, ISHM_INIT, FDSERVER_INIT, GLOBAL_RW_DATA_INIT, @@ -292,16 +292,16 @@ static int term_global(enum init_stage stage) } /* Fall through */ - case SYSINFO_INIT: - if (_odp_system_info_term()) { - _ODP_ERR("ODP system info term failed.\n"); + case TIME_INIT: + if (_odp_time_term_global()) { + _ODP_ERR("ODP time term failed.\n"); rc = -1; } /* Fall through */ - case TIME_INIT: - if (_odp_time_term_global()) { - _ODP_ERR("ODP time term failed.\n"); + case SYSINFO_INIT: + if (_odp_system_info_term()) { + _ODP_ERR("ODP system info term failed.\n"); rc = -1; } /* Fall through */ @@ -372,18 +372,18 @@ int odp_init_global(odp_instance_t *instance, } stage = CPU_CYCLES_INIT; - if (_odp_time_init_global()) { - _ODP_ERR("ODP time init failed.\n"); - goto init_failed; - } - stage = TIME_INIT; - if (_odp_system_info_init()) { _ODP_ERR("ODP system_info init failed.\n"); goto init_failed; } stage = SYSINFO_INIT; + if (_odp_time_init_global()) { + _ODP_ERR("ODP time init failed.\n"); + goto init_failed; + } + stage = TIME_INIT; + if (_odp_ishm_init_global(params)) { _ODP_ERR("ODP ishm init failed.\n"); goto init_failed; diff --git a/platform/linux-generic/odp_ishm.c b/platform/linux-generic/odp_ishm.c index 926e20a8a..3ef1894bc 100644 --- a/platform/linux-generic/odp_ishm.c +++ b/platform/linux-generic/odp_ishm.c @@ -1160,9 +1160,8 @@ int _odp_ishm_reserve(const char *name, uint64_t size, int fd, if (addr == NULL) { if (!huge_error_printed) { - _ODP_ERR("No huge pages, fall back to " - "normal pages. Check: " - "/proc/sys/vm/nr_hugepages.\n"); + _ODP_WARN("No huge pages, fall back to normal pages. " + "Check: /proc/sys/vm/nr_hugepages.\n"); huge_error_printed = 1; } } else { @@ -1274,8 +1273,8 @@ static void *reserve_single_va(uint64_t size, int *fd_out) } } if (fd < 0) - _ODP_ERR("No huge pages, fall back to normal pages. " - "Check: /proc/sys/vm/nr_hugepages.\n"); + _ODP_WARN("No huge pages, fall back to normal pages. " + "Check: /proc/sys/vm/nr_hugepages.\n"); ishm_tbl->single_va_huge = true; } diff --git a/platform/linux-generic/odp_pool.c b/platform/linux-generic/odp_pool.c index 672b92c02..e7b2398de 100644 --- a/platform/linux-generic/odp_pool.c +++ b/platform/linux-generic/odp_pool.c @@ -604,6 +604,14 @@ static void init_buffers(pool_t *pool) ring_ptr_enq(ring, mask, event_hdr); } pool->skipped_blocks = skipped_blocks; + + if (pool->uarea_size && pool->params.uarea_init.init_fn) { + for (uint32_t i = 0; i < pool->num; i++) { + uarea = &pool->uarea_base_addr[i * pool->uarea_size]; + pool->params.uarea_init.init_fn(uarea, pool->param_uarea_size, + pool->params.uarea_init.args, i); + } + } } static bool shm_is_from_huge_pages(odp_shm_t shm) @@ -1479,6 +1487,7 @@ int odp_pool_capability(odp_pool_capability_t *capa) capa->buf.max_size = MAX_SIZE; capa->buf.max_num = CONFIG_POOL_MAX_NUM; capa->buf.max_uarea_size = MAX_UAREA_SIZE; + capa->buf.uarea_persistence = true; capa->buf.min_cache_size = 0; capa->buf.max_cache_size = CONFIG_POOL_CACHE_MAX_SIZE; capa->buf.stats.all = supported_stats.all; @@ -1495,6 +1504,7 @@ int odp_pool_capability(odp_pool_capability_t *capa) capa->pkt.min_seg_len = CONFIG_PACKET_SEG_LEN_MIN; capa->pkt.max_seg_len = max_seg_len; capa->pkt.max_uarea_size = MAX_UAREA_SIZE; + capa->pkt.uarea_persistence = true; capa->pkt.min_cache_size = 0; capa->pkt.max_cache_size = CONFIG_POOL_CACHE_MAX_SIZE; capa->pkt.stats.all = supported_stats.all; @@ -1503,6 +1513,7 @@ int odp_pool_capability(odp_pool_capability_t *capa) capa->tmo.max_pools = max_pools; capa->tmo.max_num = CONFIG_POOL_MAX_NUM; capa->tmo.max_uarea_size = MAX_UAREA_SIZE; + capa->tmo.uarea_persistence = true; capa->tmo.min_cache_size = 0; capa->tmo.max_cache_size = CONFIG_POOL_CACHE_MAX_SIZE; capa->tmo.stats.all = supported_stats.all; @@ -1512,6 +1523,7 @@ int odp_pool_capability(odp_pool_capability_t *capa) capa->vector.max_num = CONFIG_POOL_MAX_NUM; capa->vector.max_size = CONFIG_PACKET_VECTOR_MAX_SIZE; capa->vector.max_uarea_size = MAX_UAREA_SIZE; + capa->vector.uarea_persistence = true; capa->vector.min_cache_size = 0; capa->vector.max_cache_size = CONFIG_POOL_CACHE_MAX_SIZE; capa->vector.stats.all = supported_stats.all; @@ -1869,6 +1881,7 @@ int odp_pool_ext_capability(odp_pool_type_t type, odp_pool_ext_capability_t *cap capa->pkt.max_headroom_size = CONFIG_PACKET_HEADROOM; capa->pkt.max_segs_per_pkt = PKT_MAX_SEGS; capa->pkt.max_uarea_size = MAX_UAREA_SIZE; + capa->pkt.uarea_persistence = true; return 0; } @@ -2096,8 +2109,18 @@ int odp_pool_ext_populate(odp_pool_t pool_hdl, void *buf[], uint32_t buf_size, u pool->base_addr = min_addr; pool->max_addr = max_addr; - if (flags & ODP_POOL_POPULATE_DONE) + if (flags & ODP_POOL_POPULATE_DONE) { pool->max_addr = max_addr + buf_size - 1; + if (pool->uarea_size && pool->ext_param.uarea_init.init_fn) { + for (i = 0; i < pool->num_populated; i++) { + uarea = &pool->uarea_base_addr[i * pool->uarea_size]; + pool->ext_param.uarea_init.init_fn(uarea, pool->param_uarea_size, + pool->ext_param.uarea_init.args, + i); + } + } + } + return 0; } diff --git a/platform/linux-generic/odp_system_info.c b/platform/linux-generic/odp_system_info.c index 4f1d92357..7864a3d17 100644 --- a/platform/linux-generic/odp_system_info.c +++ b/platform/linux-generic/odp_system_info.c @@ -55,7 +55,7 @@ static int read_cache_line_size(void) file = fopen(CACHE_LNSZ_FILE, "rt"); if (file == NULL) { /* File not found */ - _ODP_PRINT("WARN: unable to read host CPU cache line size. " + _ODP_WARN("Unable to read host CPU cache line size. " "Using ODP_CACHE_LINE_SIZE instead.\n"); return ODP_CACHE_LINE_SIZE; } @@ -299,7 +299,7 @@ static int system_cache_line(system_info_t *sysinfo) sysinfo->cache_line_size = ret; if (ret != ODP_CACHE_LINE_SIZE) - _ODP_PRINT("WARN: host CPU cache line size and ODP_CACHE_LINE_SIZE don't match.\n"); + _ODP_WARN("Host CPU cache line size and ODP_CACHE_LINE_SIZE don't match.\n"); return 0; } diff --git a/platform/linux-generic/odp_time.c b/platform/linux-generic/odp_time.c deleted file mode 100644 index 3498008f1..000000000 --- a/platform/linux-generic/odp_time.c +++ /dev/null @@ -1,215 +0,0 @@ -/* Copyright (c) 2013-2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -#include <odp_posix_extensions.h> - -#include <time.h> -#include <string.h> -#include <inttypes.h> - -#include <odp/api/time.h> -#include <odp/api/hints.h> -#include <odp_debug_internal.h> -#include <odp_init_internal.h> -#include <odp/api/plat/time_inlines.h> - -ODP_STATIC_ASSERT(_ODP_TIMESPEC_SIZE >= (sizeof(struct timespec)), - "_ODP_TIMESPEC_SIZE too small"); - -#include <odp/visibility_begin.h> - -_odp_time_global_t _odp_time_glob; - -/* - * Posix timespec based functions - */ - -static inline uint64_t time_spec_diff_nsec(struct timespec *t2, - struct timespec *t1) -{ - struct timespec diff; - uint64_t nsec; - - diff.tv_sec = t2->tv_sec - t1->tv_sec; - diff.tv_nsec = t2->tv_nsec - t1->tv_nsec; - - if (diff.tv_nsec < 0) { - diff.tv_nsec += ODP_TIME_SEC_IN_NS; - diff.tv_sec -= 1; - } - - nsec = (diff.tv_sec * ODP_TIME_SEC_IN_NS) + diff.tv_nsec; - - return nsec; -} - -odp_time_t _odp_timespec_cur(void) -{ - int ret; - odp_time_t time; - struct timespec sys_time; - struct timespec *start_time; - - start_time = (struct timespec *)(uintptr_t)&_odp_time_glob.timespec; - - ret = clock_gettime(CLOCK_MONOTONIC_RAW, &sys_time); - if (odp_unlikely(ret != 0)) - _ODP_ABORT("clock_gettime failed\n"); - - time.nsec = time_spec_diff_nsec(&sys_time, start_time); - - return time; -} - -#include <odp/visibility_end.h> - -static inline uint64_t time_spec_res(void) -{ - int ret; - struct timespec tres; - - ret = clock_getres(CLOCK_MONOTONIC_RAW, &tres); - if (odp_unlikely(ret != 0)) - _ODP_ABORT("clock_getres failed\n"); - - return ODP_TIME_SEC_IN_NS / (uint64_t)tres.tv_nsec; -} - -static inline odp_time_t time_spec_from_ns(uint64_t ns) -{ - odp_time_t time; - - time.nsec = ns; - - return time; -} - -/* - * HW time counter based functions - */ - -static inline uint64_t time_hw_res(void) -{ - return _odp_time_glob.hw_freq_hz; -} - -static inline odp_time_t time_hw_from_ns(uint64_t ns) -{ - odp_time_t time; - uint64_t count; - uint64_t freq_hz = _odp_time_glob.hw_freq_hz; - uint64_t sec = 0; - - if (ns >= ODP_TIME_SEC_IN_NS) { - sec = ns / ODP_TIME_SEC_IN_NS; - ns = ns - sec * ODP_TIME_SEC_IN_NS; - } - - count = sec * freq_hz; - count += (ns * freq_hz) / ODP_TIME_SEC_IN_NS; - - time.count = count; - - return time; -} - -/* - * Common functions - */ - -static inline uint64_t time_res(void) -{ - if (_odp_time_glob.use_hw) - return time_hw_res(); - - return time_spec_res(); -} - -static inline odp_time_t time_from_ns(uint64_t ns) -{ - if (_odp_time_glob.use_hw) - return time_hw_from_ns(ns); - - return time_spec_from_ns(ns); -} - -static inline void time_wait_until(odp_time_t time) -{ - odp_time_t cur; - - do { - cur = _odp_time_cur(); - } while (odp_time_cmp(time, cur) > 0); -} - -odp_time_t odp_time_local_from_ns(uint64_t ns) -{ - return time_from_ns(ns); -} - -odp_time_t odp_time_global_from_ns(uint64_t ns) -{ - return time_from_ns(ns); -} - -uint64_t odp_time_local_res(void) -{ - return time_res(); -} - -uint64_t odp_time_global_res(void) -{ - return time_res(); -} - -void odp_time_wait_ns(uint64_t ns) -{ - odp_time_t cur = _odp_time_cur(); - odp_time_t wait = time_from_ns(ns); - odp_time_t end_time = odp_time_sum(cur, wait); - - time_wait_until(end_time); -} - -void odp_time_wait_until(odp_time_t time) -{ - time_wait_until(time); -} - -int _odp_time_init_global(void) -{ - struct timespec *timespec; - int ret = 0; - _odp_time_global_t *global = &_odp_time_glob; - - memset(global, 0, sizeof(_odp_time_global_t)); - - if (_odp_cpu_has_global_time()) { - global->use_hw = 1; - global->hw_freq_hz = _odp_cpu_global_time_freq(); - - if (global->hw_freq_hz == 0) - return -1; - - _ODP_PRINT("HW time counter freq: %" PRIu64 " hz\n\n", global->hw_freq_hz); - - global->hw_start = _odp_cpu_global_time(); - return 0; - } - - timespec = (struct timespec *)(uintptr_t)global->timespec; - timespec->tv_sec = 0; - timespec->tv_nsec = 0; - - ret = clock_gettime(CLOCK_MONOTONIC_RAW, timespec); - - return ret; -} - -int _odp_time_term_global(void) -{ - return 0; -} diff --git a/platform/linux-generic/odp_weak.c b/platform/linux-generic/odp_weak.c index 9e1c9da6a..747886d4e 100644 --- a/platform/linux-generic/odp_weak.c +++ b/platform/linux-generic/odp_weak.c @@ -20,6 +20,7 @@ int odp_override_log(odp_log_level_t level, const char *fmt, ...) switch (level) { case ODP_LOG_ERR: case ODP_LOG_UNIMPLEMENTED: + case ODP_LOG_WARN: case ODP_LOG_ABORT: logfd = stderr; break; diff --git a/platform/linux-generic/pktio/socket_xdp.c b/platform/linux-generic/pktio/socket_xdp.c index 867483f76..599942657 100644 --- a/platform/linux-generic/pktio/socket_xdp.c +++ b/platform/linux-generic/pktio/socket_xdp.c @@ -309,8 +309,7 @@ static int sock_xdp_open(odp_pktio_t pktio, pktio_entry_t *pktio_entry, const ch if (!get_nic_queue_count(priv->helper_sock, devname, &priv->q_num_conf.drv_channels) || !get_nic_rss_indir_count(priv->helper_sock, devname, &priv->q_num_conf.drv_num_rss)) - _ODP_PRINT("Warning: Unable to query NIC queue count/RSS, manual cleanup" - " required\n"); + _ODP_WARN("Unable to query NIC queue count/RSS, manual cleanup required\n"); priv->is_shadow_q = is_shadow_q_driver(priv->helper_sock, pktio_entry->name); parse_options(priv->umem_info); @@ -543,8 +542,7 @@ static int sock_xdp_start(pktio_entry_t *pktio_entry) channels.combined = priv->q_num_conf.num_qs; if (!set_nic_queue_count(priv->helper_sock, pktio_entry->name, &channels)) - _ODP_PRINT("Warning: Unable to configure NIC queue count, manual configuration" - " required\n"); + _ODP_WARN("Unable to configure NIC queue count, manual configuration required\n"); if (priv->q_num_conf.num_in_conf_qs > 0U && !priv->is_shadow_q) { indir->indir_size = priv->q_num_conf.drv_num_rss; @@ -553,8 +551,7 @@ static int sock_xdp_start(pktio_entry_t *pktio_entry) indir->rss_config[i] = (i % priv->q_num_conf.num_in_conf_qs); if (!set_nic_rss_indir(priv->helper_sock, pktio_entry->name, indir)) - _ODP_PRINT("Warning: Unable to configure NIC RSS, manual configuration" - " required\n"); + _ODP_WARN("Unable to configure NIC RSS, manual configuration required\n"); } if (!create_sockets(priv, pktio_entry->name)) diff --git a/test/performance/Makefile.am b/test/performance/Makefile.am index 67d57590a..6e20ec07c 100644 --- a/test/performance/Makefile.am +++ b/test/performance/Makefile.am @@ -84,6 +84,7 @@ odp_timer_perf_SOURCES = odp_timer_perf.c if LIBCONFIG odp_ipsecfwd_SOURCES = odp_ipsecfwd.c +AM_CFLAGS += $(LIBCONFIG_CFLAGS) endif # l2fwd test depends on generator example diff --git a/test/performance/odp_crypto.c b/test/performance/odp_crypto.c index 52af6d2fc..fed3ebad8 100644 --- a/test/performance/odp_crypto.c +++ b/test/performance/odp_crypto.c @@ -1,5 +1,5 @@ /* Copyright (c) 2015-2018, Linaro Limited - * Copyright (c) 2022, Nokia + * Copyright (c) 2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -1170,6 +1170,12 @@ static int run_measure_one_config(test_run_arg_t *arg) rc = 1; } +#if ODP_VERSION_API >= ODP_VERSION_API_NUM(1, 42, 0) + /* Bit mode ciphers can now be used in byte mode. */ + config->cipher_in_bit_mode = 0; + config->auth_in_bit_mode = 0; +#endif + if (rc == 0) rc = create_session_from_config(&session, config, cargs); if (rc) { diff --git a/test/performance/odp_stress.c b/test/performance/odp_stress.c index 15b44c113..d5e3142f6 100644 --- a/test/performance/odp_stress.c +++ b/test/performance/odp_stress.c @@ -213,7 +213,7 @@ static int worker_thread(void *arg) odp_timer_start_t start_param; odp_time_t t1, t2, max_time; odp_time_t work_t1, work_t2; - uint8_t *src, *dst; + uint8_t *src = NULL, *dst = NULL; thread_arg_t *thread_arg = arg; int worker_idx = thread_arg->worker_idx; test_global_t *global = thread_arg->global; diff --git a/test/validation/api/barrier/barrier.c b/test/validation/api/barrier/barrier.c index 610b5db70..e4fba770f 100644 --- a/test/validation/api/barrier/barrier.c +++ b/test/validation/api/barrier/barrier.c @@ -283,6 +283,9 @@ static void barrier_test_memory_barrier(void) volatile int b = 0; volatile int c = 0; volatile int d = 0; + volatile int e = 0; + volatile int f = 0; + volatile int g = 0; /* Call all memory barriers to verify that those are implemented */ a = 1; @@ -292,9 +295,15 @@ static void barrier_test_memory_barrier(void) c = 1; odp_mb_full(); d = 1; + odp_mb_sync(); + e = 1; + odp_mb_sync_load(); + f = 1; + odp_mb_sync_store(); + g = 1; /* Avoid "variable set but not used" warning */ - temp_result = a + b + c + d; + temp_result = a + b + c + d + e + f + g; } static int barrier_init(odp_instance_t *inst) diff --git a/test/validation/api/buffer/buffer.c b/test/validation/api/buffer/buffer.c index ef26ea6e0..1d0cc9d01 100644 --- a/test/validation/api/buffer/buffer.c +++ b/test/validation/api/buffer/buffer.c @@ -537,6 +537,7 @@ static void buffer_test_user_area(void) for (i = 0; i < num; i++) { odp_event_t ev; + int flag; buffer[i] = odp_buffer_alloc(pool); @@ -550,6 +551,8 @@ static void buffer_test_user_area(void) ev = odp_buffer_to_event(buffer[i]); CU_ASSERT(odp_event_user_area(ev) == addr); + CU_ASSERT(odp_event_user_area_and_flag(ev, &flag) == addr); + CU_ASSERT(flag < 0); prev = addr; memset(addr, 0, size); diff --git a/test/validation/api/crypto/crypto_op_test.c b/test/validation/api/crypto/crypto_op_test.c index 4ad333488..1d883f6b8 100644 --- a/test/validation/api/crypto/crypto_op_test.c +++ b/test/validation/api/crypto/crypto_op_test.c @@ -178,19 +178,13 @@ static void prepare_crypto_ranges(const crypto_op_test_param_t *param, odp_packet_data_range_t *cipher_range, odp_packet_data_range_t *auth_range) { - odp_packet_data_range_t zero_range = {.offset = 0, .length = 0}; - uint32_t c_scale = param->is_bit_mode_cipher ? 8 : 1; - uint32_t a_scale = param->is_bit_mode_auth ? 8 : 1; + uint32_t c_scale = param->session.cipher_range_in_bits ? 8 : 1; + uint32_t a_scale = param->session.auth_range_in_bits ? 8 : 1; *cipher_range = param->cipher_range; *auth_range = param->auth_range; cipher_range->offset += c_scale * param->header_len; auth_range->offset += a_scale * param->header_len; - - if (param->ref->cipher == ODP_CIPHER_ALG_NULL) - *cipher_range = zero_range; - if (param->ref->auth == ODP_AUTH_ALG_NULL) - *auth_range = zero_range; } static int prepare_input_packet(const crypto_op_test_param_t *param, @@ -217,7 +211,7 @@ static int prepare_input_packet(const crypto_op_test_param_t *param, write_header_and_trailer(pkt, param->header_len, param->trailer_len); - if (param->op == ODP_CRYPTO_OP_ENCODE) { + if (param->session.op == ODP_CRYPTO_OP_ENCODE) { odp_packet_copy_from_mem(pkt, param->header_len, reflength, ref->plaintext); } else { @@ -322,7 +316,7 @@ static void prepare_ignore_info(const crypto_op_test_param_t *param, * Leftover bits in the last byte of the cipher range of bit mode * ciphers have undefined values. */ - if (param->is_bit_mode_cipher && + if (param->session.cipher_range_in_bits && param->ref->cipher != ODP_CIPHER_ALG_NULL) { uint8_t leftover_bits = ref_length_in_bits(param->ref) % 8; @@ -338,10 +332,10 @@ static void prepare_ignore_info(const crypto_op_test_param_t *param, * undefined values. */ if (param->ref->auth != ODP_AUTH_ALG_NULL && - param->op == ODP_CRYPTO_OP_DECODE) { + param->session.op == ODP_CRYPTO_OP_DECODE) { uint32_t offs = param->digest_offset; - if (param->op_type != ODP_CRYPTO_OP_TYPE_OOP || + if (param->session.op_type != ODP_CRYPTO_OP_TYPE_OOP || is_in_range(offs, cipher_offset, cipher_len) || is_in_range(offs, auth_offset, auth_len)) { add_ignored_range(ignore, @@ -351,11 +345,11 @@ static void prepare_ignore_info(const crypto_op_test_param_t *param, } /* Decrypted bytes are undefined if authentication fails. */ - if (param->op == ODP_CRYPTO_OP_DECODE && + if (param->session.op == ODP_CRYPTO_OP_DECODE && param->wrong_digest) { add_ignored_range(ignore, cipher_offset + shift, cipher_len); /* In OOP case, auth range may not get copied */ - if (param->op_type == ODP_CRYPTO_OP_TYPE_OOP) + if (param->session.op_type == ODP_CRYPTO_OP_TYPE_OOP) add_ignored_range(ignore, auth_offset + shift, auth_len); } } @@ -372,30 +366,29 @@ static void prepare_expected_data(const crypto_op_test_param_t *param, uint32_t cipher_len = cipher_range->length; uint32_t auth_offset = auth_range->offset; uint32_t auth_len = auth_range->length; - const int32_t shift = param->op_type == ODP_CRYPTO_OP_TYPE_OOP ? param->oop_shift : 0; - const odp_packet_t base_pkt = param->op_type == ODP_CRYPTO_OP_TYPE_OOP ? pkt_out : pkt_in; + const int32_t shift = param->session.op_type == ODP_CRYPTO_OP_TYPE_OOP ? param->oop_shift + : 0; + const odp_packet_t base_pkt = param->session.op_type == ODP_CRYPTO_OP_TYPE_OOP ? pkt_out + : pkt_in; int rc; uint32_t cipher_offset_in_ref = param->cipher_range.offset; - if (param->op == ODP_CRYPTO_OP_ENCODE) + if (param->session.op == ODP_CRYPTO_OP_ENCODE) digest_offset += shift; - if (param->is_bit_mode_cipher) { + if (param->session.cipher_range_in_bits) { cipher_offset_in_ref /= 8; cipher_offset /= 8; cipher_len = (cipher_len + 7) / 8; } - if (param->is_bit_mode_auth) { + if (param->session.auth_range_in_bits) { auth_offset /= 8; auth_len = (auth_len + 7) / 8; } - if (param->ref->cipher == ODP_CIPHER_ALG_NULL) - cipher_len = 0; - if (param->ref->auth == ODP_AUTH_ALG_NULL || - param->ref->auth == ODP_AUTH_ALG_AES_GCM || + if (param->ref->auth == ODP_AUTH_ALG_AES_GCM || param->ref->auth == ODP_AUTH_ALG_AES_CCM || param->ref->auth == ODP_AUTH_ALG_CHACHA20_POLY1305) { - /* auth range is ignored with null and AEAD algorithms */ + /* auth range is ignored with AEAD algorithms */ auth_len = 0; } @@ -405,14 +398,14 @@ static void prepare_expected_data(const crypto_op_test_param_t *param, rc = odp_packet_copy_to_mem(base_pkt, 0, ex->len, ex->data); CU_ASSERT(rc == 0); - if (param->op_type == ODP_CRYPTO_OP_TYPE_OOP && auth_len > 0) { + if (param->session.op_type == ODP_CRYPTO_OP_TYPE_OOP && auth_len > 0) { /* copy auth range from input packet */ rc = odp_packet_copy_to_mem(pkt_in, auth_offset, auth_len, ex->data + auth_offset + shift); CU_ASSERT(rc == 0); } - if (param->op == ODP_CRYPTO_OP_ENCODE) { + if (param->session.op == ODP_CRYPTO_OP_ENCODE) { /* copy hash first */ memcpy(ex->data + digest_offset, param->ref->digest, @@ -472,6 +465,23 @@ static void check_output_packet_data(odp_packet_t pkt, expected_t *ex) } } +static int is_digest_in_cipher_range(const crypto_op_test_param_t *param, + const odp_crypto_packet_op_param_t *op_params) +{ + /* + * Do not use op_params.hash_result_offset here as it refers to + * the output packet which (in the OOP case) might be shifted + * relative to the input packet. + */ + uint32_t d_offset = param->digest_offset; + + if (param->session.cipher_range_in_bits) + d_offset *= 8; + + return d_offset >= op_params->cipher_range.offset && + d_offset < op_params->cipher_range.offset + op_params->cipher_range.length; +} + void test_crypto_op(const crypto_op_test_param_t *param) { odp_bool_t ok = false; @@ -481,13 +491,14 @@ void test_crypto_op(const crypto_op_test_param_t *param) test_packet_md_t md_in, md_out, md_out_orig; expected_t expected; odp_crypto_packet_op_param_t op_params = { - .session = param->session, + .session = param->session.session, .cipher_iv_ptr = param->ref->cipher_iv, .auth_iv_ptr = param->ref->auth_iv, .hash_result_offset = param->digest_offset, .aad_ptr = param->ref->aad, .dst_offset_shift = param->oop_shift, }; + odp_bool_t failure_allowed = false; /* * Test detection of wrong digest value in input packet @@ -495,26 +506,34 @@ void test_crypto_op(const crypto_op_test_param_t *param) */ if (param->wrong_digest && (param->ref->auth == ODP_AUTH_ALG_NULL || - param->op == ODP_CRYPTO_OP_ENCODE)) + param->session.op == ODP_CRYPTO_OP_ENCODE)) return; prepare_crypto_ranges(param, &op_params.cipher_range, &op_params.auth_range); if (prepare_input_packet(param, &pkt)) return; - if (param->op_type == ODP_CRYPTO_OP_TYPE_OOP) { + if (param->session.op_type == ODP_CRYPTO_OP_TYPE_OOP) { prepare_oop_output_packet(param, &pkt_out, odp_packet_len(pkt)); pkt_copy = odp_packet_copy(pkt, suite_context.pool); CU_ASSERT_FATAL(pkt_copy != ODP_PACKET_INVALID); test_packet_get_md(pkt_out, &md_out_orig); + + /* Non-zero-length ranges do not have to be supported. */ + if ((param->ref->cipher == ODP_CIPHER_ALG_NULL && + op_params.cipher_range.length != 0)) + failure_allowed = true; + if ((param->ref->auth == ODP_AUTH_ALG_NULL && + op_params.auth_range.length != 0)) + failure_allowed = true; } prepare_expected_data(param, &op_params.cipher_range, &op_params.auth_range, pkt, pkt_out, &expected); - if (param->op_type == ODP_CRYPTO_OP_TYPE_OOP && - param->op == ODP_CRYPTO_OP_ENCODE) { + if (param->session.op_type == ODP_CRYPTO_OP_TYPE_OOP && + param->session.op == ODP_CRYPTO_OP_ENCODE) { /* * In this type of sessions digest offset is an offset to the output * packet, so apply the shift. @@ -525,12 +544,12 @@ void test_crypto_op(const crypto_op_test_param_t *param) test_packet_set_md(pkt); test_packet_get_md(pkt, &md_in); - if (crypto_op(pkt, &pkt_out, &ok, &op_params, param->op_type)) + if (crypto_op(pkt, &pkt_out, &ok, &op_params, param->session.op_type)) return; test_packet_get_md(pkt_out, &md_out); - if (param->op_type == ODP_CRYPTO_OP_TYPE_OOP) { + if (param->session.op_type == ODP_CRYPTO_OP_TYPE_OOP) { test_packet_md_t md; /* check that input packet has not changed */ @@ -548,17 +567,18 @@ void test_crypto_op(const crypto_op_test_param_t *param) if (param->ref->cipher != ODP_CIPHER_ALG_NULL && param->ref->auth != ODP_AUTH_ALG_NULL && - param->digest_offset >= op_params.cipher_range.offset && - param->digest_offset < op_params.cipher_range.offset + op_params.cipher_range.length) { + is_digest_in_cipher_range(param, &op_params)) { /* * Not all implementations support digest offset in cipher * range, so allow crypto op failure without further checks * in this case. */ - if (!ok) - goto out; + failure_allowed = true; } + if (!ok && failure_allowed) + goto out; + if (param->wrong_digest) { CU_ASSERT(!ok); } else { diff --git a/test/validation/api/crypto/crypto_op_test.h b/test/validation/api/crypto/crypto_op_test.h index 4b2f66c47..4f5957820 100644 --- a/test/validation/api/crypto/crypto_op_test.h +++ b/test/validation/api/crypto/crypto_op_test.h @@ -12,17 +12,21 @@ #include <stdint.h> #include "test_vectors.h" -typedef struct crypto_op_test_param_t { +typedef struct crypto_session_t { odp_crypto_session_t session; odp_crypto_op_t op; odp_crypto_op_type_t op_type; + odp_bool_t cipher_range_in_bits; + odp_bool_t auth_range_in_bits; +} crypto_session_t; + +typedef struct crypto_op_test_param_t { + crypto_session_t session; int32_t oop_shift; crypto_test_reference_t *ref; odp_packet_data_range_t cipher_range; odp_packet_data_range_t auth_range; uint32_t digest_offset; - odp_bool_t is_bit_mode_cipher; - odp_bool_t is_bit_mode_auth; odp_bool_t adjust_segmentation; odp_bool_t wrong_digest; uint32_t first_seg_len; diff --git a/test/validation/api/crypto/odp_crypto_test_inp.c b/test/validation/api/crypto/odp_crypto_test_inp.c index d7e2bd16b..0cc01b8e4 100644 --- a/test/validation/api/crypto/odp_crypto_test_inp.c +++ b/test/validation/api/crypto/odp_crypto_test_inp.c @@ -36,6 +36,8 @@ static void test_defaults(uint8_t fill) CU_ASSERT_EQUAL(param.op, ODP_CRYPTO_OP_ENCODE); CU_ASSERT_EQUAL(param.op_type, ODP_CRYPTO_OP_TYPE_LEGACY); + CU_ASSERT_EQUAL(param.cipher_range_in_bits, false); + CU_ASSERT_EQUAL(param.auth_range_in_bits, false); CU_ASSERT_EQUAL(param.auth_cipher_text, false); CU_ASSERT_EQUAL(param.op_mode, ODP_CRYPTO_SYNC); CU_ASSERT_EQUAL(param.cipher_alg, ODP_CIPHER_ALG_NULL); @@ -53,15 +55,10 @@ static void test_default_values(void) static void print_alg_test_param(const crypto_op_test_param_t *p) { - const char *cipher_mode = p->is_bit_mode_cipher ? "bit" : "byte"; + const char *cipher_mode = p->session.cipher_range_in_bits ? "bit" : "byte"; + const char *auth_mode = p->session.auth_range_in_bits ? "bit" : "byte"; - - - - - const char *auth_mode = p->is_bit_mode_auth ? "bit" : "byte"; - - switch (p->op_type) { + switch (p->session.op_type) { case ODP_CRYPTO_OP_TYPE_LEGACY: printf("legacy "); break; @@ -72,7 +69,7 @@ static void print_alg_test_param(const crypto_op_test_param_t *p) printf("out-of-place "); break; } - printf("%s\n", p->op == ODP_CRYPTO_OP_ENCODE ? "encode" : "decode"); + printf("%s\n", p->session.op == ODP_CRYPTO_OP_ENCODE ? "encode" : "decode"); printf("cipher: %s, %s mode\n", cipher_alg_name(p->ref->cipher), cipher_mode); printf(" key length: %d, iv length: %d\n", @@ -93,7 +90,7 @@ static void print_alg_test_param(const crypto_op_test_param_t *p) printf("header length: %d, trailer length: %d\n", p->header_len, p->trailer_len); if (p->adjust_segmentation) printf("segmentation adjusted, first_seg_len: %d\n", p->first_seg_len); - if (p->op_type == ODP_CRYPTO_OP_TYPE_OOP) + if (p->session.op_type == ODP_CRYPTO_OP_TYPE_OOP) printf("oop_shift: %d\n", p->oop_shift); } @@ -122,7 +119,7 @@ static void alg_test_op(crypto_op_test_param_t *param) for (uint32_t n = 0; n < ARRAY_SIZE(oop_shifts); n++) { if (oop_shifts[n] != 0 && - param->op_type != ODP_CRYPTO_OP_TYPE_OOP) + param->session.op_type != ODP_CRYPTO_OP_TYPE_OOP) continue; if ((int32_t)param->header_len + oop_shifts[n] < 0) continue; @@ -150,13 +147,11 @@ typedef enum { AUTH_PLAINTEXT } alg_order_t; -static odp_crypto_session_t session_create(odp_crypto_op_t op, - odp_crypto_op_type_t op_type, - alg_order_t order, - crypto_test_reference_t *ref, - hash_test_mode_t hash_mode) +static int session_create(crypto_session_t *session, + alg_order_t order, + crypto_test_reference_t *ref, + hash_test_mode_t hash_mode) { - odp_crypto_session_t session = ODP_CRYPTO_SESSION_INVALID; int rc; odp_crypto_ses_create_err_t status; odp_crypto_session_param_t ses_params; @@ -176,8 +171,10 @@ static odp_crypto_session_t session_create(odp_crypto_op_t op, /* Create a crypto session */ odp_crypto_session_param_init(&ses_params); - ses_params.op = op; - ses_params.op_type = op_type; + ses_params.op = session->op; + ses_params.op_type = session->op_type; + ses_params.cipher_range_in_bits = session->cipher_range_in_bits; + ses_params.auth_range_in_bits = session->auth_range_in_bits; ses_params.auth_cipher_text = (order == AUTH_CIPHERTEXT); ses_params.op_mode = suite_context.op_mode; ses_params.cipher_alg = ref->cipher; @@ -191,7 +188,7 @@ static odp_crypto_session_t session_create(odp_crypto_op_t op, ses_params.auth_digest_len = ref->digest_length; ses_params.auth_aad_len = ref->aad_length; ses_params.hash_result_in_auth_range = (hash_mode == HASH_OVERLAP); - rc = odp_crypto_session_create(&ses_params, &session, &status); + rc = odp_crypto_session_create(&ses_params, &session->session, &status); if (rc < 0 && status == ODP_CRYPTO_SES_ERR_ALG_COMBO) { if (!combo_warning_shown) { @@ -200,7 +197,7 @@ static odp_crypto_session_t session_create(odp_crypto_op_t op, cipher_alg_name(ref->cipher), auth_alg_name(ref->auth)); } - return ODP_CRYPTO_SESSION_INVALID; + return -1; } /* @@ -214,21 +211,21 @@ static odp_crypto_session_t session_create(odp_crypto_op_t op, cipher_alg_name(ref->cipher), auth_alg_name(ref->auth), ses_params.auth_cipher_text); - return ODP_CRYPTO_SESSION_INVALID; + return -1; } /* For now, allow out-of-place sessions not to be supported. */ if (rc < 0 && status == ODP_CRYPTO_SES_ERR_PARAMS && - op_type == ODP_CRYPTO_OP_TYPE_OOP) { + ses_params.op_type == ODP_CRYPTO_OP_TYPE_OOP) { if (!oop_warning_shown) printf("\n Skipping ODP_CRYPTO_OP_TYPE_OOP tests\n"); oop_warning_shown = 1; - return ODP_CRYPTO_SESSION_INVALID; + return -1; } CU_ASSERT_FATAL(!rc); CU_ASSERT(status == ODP_CRYPTO_SES_ERR_NONE); - CU_ASSERT(odp_crypto_session_to_u64(session) != + CU_ASSERT(odp_crypto_session_to_u64(session->session) != odp_crypto_session_to_u64(ODP_CRYPTO_SESSION_INVALID)); /* @@ -239,7 +236,7 @@ static odp_crypto_session_t session_create(odp_crypto_op_t op, memset(auth_key_data, 0, sizeof(auth_key_data)); memset(&ses_params, 0, sizeof(ses_params)); - return session; + return 0; } static void alg_test_ses(odp_crypto_op_t op, @@ -249,35 +246,32 @@ static void alg_test_ses(odp_crypto_op_t op, odp_packet_data_range_t cipher_range, odp_packet_data_range_t auth_range, uint32_t digest_offset, - odp_bool_t is_bit_mode_cipher, - odp_bool_t is_bit_mode_auth) + odp_bool_t cipher_range_in_bits, + odp_bool_t auth_range_in_bits) { unsigned int initial_num_failures = CU_get_number_of_failures(); const uint32_t reflength = ref_length_in_bytes(ref); + const uint32_t auth_scale = auth_range_in_bits ? 8 : 1; hash_test_mode_t hash_mode = HASH_NO_OVERLAP; - odp_crypto_session_t session; int rc; uint32_t seg_len; uint32_t max_shift; crypto_op_test_param_t test_param; - if (digest_offset >= auth_range.offset && - digest_offset < auth_range.offset + auth_range.length) + if (digest_offset * auth_scale >= auth_range.offset && + digest_offset * auth_scale < auth_range.offset + auth_range.length) hash_mode = HASH_OVERLAP; - session = session_create(op, op_type, order, ref, hash_mode); - if (session == ODP_CRYPTO_SESSION_INVALID) - return; - memset(&test_param, 0, sizeof(test_param)); - test_param.session = session; - test_param.op = op; - test_param.op_type = op_type; + test_param.session.op = op; + test_param.session.op_type = op_type; + test_param.session.cipher_range_in_bits = cipher_range_in_bits; + test_param.session.auth_range_in_bits = auth_range_in_bits; + if (session_create(&test_param.session, order, ref, hash_mode)) + return; test_param.ref = ref; test_param.cipher_range = cipher_range; test_param.auth_range = auth_range; - test_param.is_bit_mode_cipher = is_bit_mode_cipher; - test_param.is_bit_mode_auth = is_bit_mode_auth; test_param.digest_offset = digest_offset; alg_test_op(&test_param); @@ -319,18 +313,18 @@ static void alg_test_ses(odp_crypto_op_t op, alg_test_op(&test_param); } - rc = odp_crypto_session_destroy(session); + rc = odp_crypto_session_destroy(test_param.session.session); CU_ASSERT(!rc); } -static void alg_test(odp_crypto_op_t op, - alg_order_t order, - crypto_test_reference_t *ref, - odp_packet_data_range_t cipher_range, - odp_packet_data_range_t auth_range, - uint32_t digest_offset, - odp_bool_t is_bit_mode_cipher, - odp_bool_t is_bit_mode_auth) +static void alg_test_op_types(odp_crypto_op_t op, + alg_order_t order, + crypto_test_reference_t *ref, + odp_packet_data_range_t cipher_range, + odp_packet_data_range_t auth_range, + uint32_t digest_offset, + odp_bool_t cipher_range_in_bits, + odp_bool_t auth_range_in_bits) { odp_crypto_op_type_t op_types[] = { ODP_CRYPTO_OP_TYPE_LEGACY, @@ -346,8 +340,47 @@ static void alg_test(odp_crypto_op_t op, cipher_range, auth_range, digest_offset, - is_bit_mode_cipher, - is_bit_mode_auth); + cipher_range_in_bits, + auth_range_in_bits); + } +} + +static void alg_test(odp_crypto_op_t op, + alg_order_t order, + crypto_test_reference_t *ref, + odp_packet_data_range_t cipher_bit_range, + odp_packet_data_range_t auth_bit_range, + uint32_t digest_offset, + odp_bool_t is_bit_mode_cipher, + odp_bool_t is_bit_mode_auth) +{ + odp_packet_data_range_t cipher_range; + odp_packet_data_range_t auth_range; + + for (int cr_in_bits = 0; cr_in_bits <= 1; cr_in_bits++) { + if (!cr_in_bits && cipher_bit_range.length % 8 != 0) + continue; + if (cr_in_bits && !is_bit_mode_cipher) + continue; + for (int ar_in_bits = 0; ar_in_bits <= 1; ar_in_bits++) { + if (!ar_in_bits && auth_bit_range.length % 8 != 0) + continue; + if (ar_in_bits && !is_bit_mode_auth) + continue; + + cipher_range = cipher_bit_range; + auth_range = auth_bit_range; + if (!cr_in_bits) { + cipher_range.offset /= 8; + cipher_range.length /= 8; + } + if (!ar_in_bits) { + auth_range.offset /= 8; + auth_range.length /= 8; + } + alg_test_op_types(op, order, ref, cipher_range, auth_range, + digest_offset, cr_in_bits, ar_in_bits); + } } } @@ -404,8 +437,8 @@ static void check_alg(odp_crypto_op_t op, odp_bool_t is_bit_mode_cipher = false; odp_bool_t is_bit_mode_auth = false; uint32_t digest_offs = ref_length_in_bytes(&ref[idx]); - odp_packet_data_range_t cipher_range = {.offset = 0}; - odp_packet_data_range_t auth_range = {.offset = 0}; + odp_packet_data_range_t cipher_bit_range = {.offset = 0}; + odp_packet_data_range_t auth_bit_range = {.offset = 0}; if (ref_length_in_bits(&ref[idx]) % 8 != 0) bit_mode_needed = true; @@ -465,18 +498,14 @@ static void check_alg(odp_crypto_op_t op, continue; } - cipher_range.length = is_bit_mode_cipher ? - ref_length_in_bits(&ref[idx]) : - ref_length_in_bytes(&ref[idx]); - auth_range.length = is_bit_mode_auth ? - ref_length_in_bits(&ref[idx]) : - ref_length_in_bytes(&ref[idx]); + cipher_bit_range.length = ref_length_in_bits(&ref[idx]); + auth_bit_range.length = ref_length_in_bits(&ref[idx]); alg_test(op, AUTH_PLAINTEXT, &ref[idx], - cipher_range, auth_range, digest_offs, + cipher_bit_range, auth_bit_range, digest_offs, is_bit_mode_cipher, is_bit_mode_auth); alg_test(op, AUTH_CIPHERTEXT, &ref[idx], - cipher_range, auth_range, digest_offs, + cipher_bit_range, auth_bit_range, digest_offs, is_bit_mode_cipher, is_bit_mode_auth); cipher_tested[cipher_idx] = true; @@ -537,7 +566,7 @@ static int create_hash_test_reference(odp_auth_alg_t auth, uint32_t digest_offset, uint8_t digest_fill) { - odp_crypto_session_t session; + crypto_session_t session; int rc; odp_packet_t pkt; odp_bool_t ok; @@ -570,33 +599,33 @@ static int create_hash_test_reference(odp_auth_alg_t auth, rc = odp_packet_copy_from_mem(pkt, 0, auth_bytes, ref->plaintext); CU_ASSERT(rc == 0); - session = session_create(ODP_CRYPTO_OP_ENCODE, - ODP_CRYPTO_OP_TYPE_LEGACY, - AUTH_PLAINTEXT, ref, HASH_NO_OVERLAP); - if (session == ODP_CRYPTO_SESSION_INVALID) + session.op = ODP_CRYPTO_OP_ENCODE; + session.op_type = ODP_CRYPTO_OP_TYPE_LEGACY; + session.cipher_range_in_bits = false; + session.auth_range_in_bits = false; + if (session_create(&session, AUTH_PLAINTEXT, ref, HASH_NO_OVERLAP)) return -1; odp_crypto_packet_op_param_t op_params = { - .session = session, + .session = session.session, .cipher_iv_ptr = ref->cipher_iv, .auth_iv_ptr = ref->auth_iv, .hash_result_offset = enc_digest_offset, .aad_ptr = ref->aad, .cipher_range = {.offset = 0, .length = 0}, - .auth_range = { .offset = 0, - .length = capa->bit_mode ? auth_bytes * 8 : auth_bytes }, + .auth_range = { .offset = 0, .length = auth_bytes }, .dst_offset_shift = 0, }; rc = crypto_op(pkt, &pkt, &ok, &op_params, ODP_CRYPTO_OP_TYPE_LEGACY); CU_ASSERT(rc == 0); if (rc) { - (void)odp_crypto_session_destroy(session); + (void)odp_crypto_session_destroy(session.session); return -1; } CU_ASSERT(ok); - rc = odp_crypto_session_destroy(session); + rc = odp_crypto_session_destroy(session.session); CU_ASSERT(rc == 0); /* copy the processed packet to the ciphertext packet in ref */ @@ -624,8 +653,8 @@ static void test_auth_hash_in_auth_range(odp_auth_alg_t auth, { static crypto_test_reference_t ref = {.length = 0}; uint32_t digest_offset = 13; - const odp_packet_data_range_t cipher_range = {.offset = 0, .length = 0}; - odp_packet_data_range_t auth_range; + const odp_packet_data_range_t cipher_bit_range = {.offset = 0, .length = 0}; + odp_packet_data_range_t auth_bit_range; if (!full_test && capa->digest_len % 4 != 0) return; @@ -637,10 +666,8 @@ static void test_auth_hash_in_auth_range(odp_auth_alg_t auth, if (create_hash_test_reference(auth, capa, &ref, digest_offset, 0)) return; - auth_range.offset = 0; - auth_range.length = capa->bit_mode ? - ref_length_in_bits(&ref) : - ref_length_in_bytes(&ref); + auth_bit_range.offset = 0; + auth_bit_range.length = ref_length_in_bits(&ref); /* * Decode the ciphertext packet. @@ -652,7 +679,7 @@ static void test_auth_hash_in_auth_range(odp_auth_alg_t auth, alg_test(ODP_CRYPTO_OP_DECODE, order, &ref, - cipher_range, auth_range, + cipher_bit_range, auth_bit_range, digest_offset, false, capa->bit_mode); @@ -664,10 +691,8 @@ static void test_auth_hash_in_auth_range(odp_auth_alg_t auth, if (create_hash_test_reference(auth, capa, &ref, digest_offset, 1)) return; - auth_range.offset = 0; - auth_range.length = capa->bit_mode ? - ref_length_in_bits(&ref) : - ref_length_in_bytes(&ref); + auth_bit_range.offset = 0; + auth_bit_range.length = ref_length_in_bits(&ref); /* * Encode the plaintext packet. @@ -679,7 +704,7 @@ static void test_auth_hash_in_auth_range(odp_auth_alg_t auth, alg_test(ODP_CRYPTO_OP_ENCODE, order, &ref, - cipher_range, auth_range, + cipher_bit_range, auth_bit_range, digest_offset, false, capa->bit_mode); @@ -689,6 +714,7 @@ static void test_auth_hash_in_auth_range(odp_auth_alg_t auth, * Cipher algorithms that are not AEAD algorithms */ static odp_cipher_alg_t cipher_algs[] = { + ODP_CIPHER_ALG_NULL, ODP_CIPHER_ALG_DES, ODP_CIPHER_ALG_3DES_CBC, ODP_CIPHER_ALG_3DES_ECB, @@ -704,10 +730,11 @@ static odp_cipher_alg_t cipher_algs[] = { }; /* - * Authentication algorithms and hashes that use auth_range + * Authentication algorithms and hashes that may use auth_range * parameter. AEAD algorithms are excluded. */ static odp_auth_alg_t auth_algs[] = { + ODP_AUTH_ALG_NULL, ODP_AUTH_ALG_MD5_HMAC, ODP_AUTH_ALG_SHA1_HMAC, ODP_AUTH_ALG_SHA224_HMAC, @@ -763,7 +790,7 @@ static int crypto_encode_ref(crypto_test_reference_t *ref, odp_packet_data_range_t zero_range = {.offset = 0, .length = 0}; odp_packet_t pkt; int rc; - odp_crypto_session_t session; + crypto_session_t session; odp_bool_t ok; pkt = odp_packet_alloc(suite_context.pool, ref->length); @@ -772,13 +799,11 @@ static int crypto_encode_ref(crypto_test_reference_t *ref, rc = odp_packet_copy_from_mem(pkt, 0, ref->length, ref->plaintext); CU_ASSERT(rc == 0); - session = session_create(ODP_CRYPTO_OP_ENCODE, - ODP_CRYPTO_OP_TYPE_LEGACY, - AUTH_PLAINTEXT, - ref, - HASH_OVERLAP); - - if (session == ODP_CRYPTO_SESSION_INVALID) { + session.op = ODP_CRYPTO_OP_ENCODE; + session.op_type = ODP_CRYPTO_OP_TYPE_LEGACY; + session.cipher_range_in_bits = false; + session.auth_range_in_bits = false; + if (session_create(&session, AUTH_PLAINTEXT, ref, HASH_OVERLAP)) { odp_packet_free(pkt); return 1; } @@ -793,7 +818,7 @@ static int crypto_encode_ref(crypto_test_reference_t *ref, CU_ASSERT_FATAL(hash_result_offset + ref->digest_length <= ref->length); odp_crypto_packet_op_param_t op_params = { - .session = session, + .session = session.session, .cipher_iv_ptr = ref->cipher_iv, .auth_iv_ptr = ref->auth_iv, .hash_result_offset = hash_result_offset, @@ -805,12 +830,12 @@ static int crypto_encode_ref(crypto_test_reference_t *ref, rc = crypto_op(pkt, &pkt, &ok, &op_params, ODP_CRYPTO_OP_TYPE_LEGACY); CU_ASSERT(rc == 0); if (rc) { - (void)odp_crypto_session_destroy(session); + (void)odp_crypto_session_destroy(session.session); return -1; } CU_ASSERT(ok); - rc = odp_crypto_session_destroy(session); + rc = odp_crypto_session_destroy(session.session); CU_ASSERT(rc == 0); rc = odp_packet_copy_to_mem(pkt, 0, ref->length, ref->ciphertext); @@ -834,8 +859,8 @@ typedef struct crypto_suite_t { */ static int create_combined_ref(const crypto_suite_t *suite, crypto_test_reference_t *ref, - odp_packet_data_range_t *cipher_range, - odp_packet_data_range_t *auth_range, + const odp_packet_data_range_t *cipher_range, + const odp_packet_data_range_t *auth_range, uint32_t digest_offset) { uint32_t total_len; @@ -861,15 +886,6 @@ static int create_combined_ref(const crypto_suite_t *suite, ref->is_length_in_bits = false; ref->length = total_len; - if (suite->cipher_capa->bit_mode) { - cipher_range->offset *= 8; - cipher_range->length *= 8; - } - if (suite->auth_capa->bit_mode) { - auth_range->offset *= 8; - auth_range->length *= 8; - } - if (ref->auth_key_length > MAX_KEY_LEN || ref->auth_iv_length > MAX_IV_LEN || total_len > MAX_DATA_LEN || @@ -1189,6 +1205,11 @@ static void test_combo(const crypto_suite_t *suite, if (rc) return; + cipher_range.offset *= 8; + cipher_range.length *= 8; + auth_range.offset *= 8; + auth_range.length *= 8; + alg_test(ODP_CRYPTO_OP_ENCODE, suite->order, &ref, diff --git a/test/validation/api/dma/dma.c b/test/validation/api/dma/dma.c index 6c8eeb00c..05771e3cb 100644 --- a/test/validation/api/dma/dma.c +++ b/test/validation/api/dma/dma.c @@ -19,6 +19,8 @@ #define MULTI 1 #define RESULT 1 #define USER_DATA 0xdeadbeef +#define ELEM_NUM 10 +#define UAREA 0xaa #define MIN(a, b) (a < b ? a : b) @@ -39,6 +41,11 @@ typedef struct global_t { } global_t; +typedef struct { + uint32_t count; + uint8_t mark[ELEM_NUM]; +} uarea_init_t; + static global_t global; static int dma_suite_init(void) @@ -234,6 +241,8 @@ static void test_dma_param(uint8_t fill) memset(&dma_pool_param, fill, sizeof(dma_pool_param)); odp_dma_pool_param_init(&dma_pool_param); + CU_ASSERT(dma_pool_param.uarea_init.init_fn == NULL); + CU_ASSERT(dma_pool_param.uarea_init.args == NULL); CU_ASSERT(dma_pool_param.uarea_size == 0); CU_ASSERT(dma_pool_param.cache_size <= global.dma_capa.pool.max_cache_size); CU_ASSERT(dma_pool_param.cache_size >= global.dma_capa.pool.min_cache_size); @@ -417,7 +426,7 @@ static void test_dma_compl_pool_max_pools(void) static void test_dma_compl_user_area(void) { odp_dma_pool_param_t dma_pool_param; - uint32_t num = MIN(10, global.dma_capa.pool.max_num), + uint32_t num = MIN(ELEM_NUM, global.dma_capa.pool.max_num), size = global.dma_capa.pool.max_uarea_size, i; odp_pool_t pool; odp_dma_compl_t compl_evs[num]; @@ -432,6 +441,7 @@ static void test_dma_compl_user_area(void) for (i = 0; i < num; i++) { odp_event_t ev; + int flag; compl_evs[i] = odp_dma_compl_alloc(pool); @@ -445,6 +455,8 @@ static void test_dma_compl_user_area(void) ev = odp_dma_compl_to_event(compl_evs[i]); CU_ASSERT(odp_event_user_area(ev) == addr); + CU_ASSERT(odp_event_user_area_and_flag(ev, &flag) == addr); + CU_ASSERT(flag < 0); prev = addr; memset(addr, 0, size); @@ -458,6 +470,56 @@ static void test_dma_compl_user_area(void) CU_ASSERT(odp_pool_destroy(pool) == 0); } +static void init_event_uarea(void *uarea, uint32_t size, void *args, uint32_t index) +{ + uarea_init_t *data = args; + + data->count++; + data->mark[index] = 1; + memset(uarea, UAREA, size); +} + +static void test_dma_compl_user_area_init(void) +{ + odp_dma_pool_param_t dma_pool_param; + uint32_t num = MIN(ELEM_NUM, global.dma_capa.pool.max_num), i; + odp_pool_t pool; + uarea_init_t data; + odp_dma_compl_t compl_evs[num]; + uint8_t *uarea; + + memset(&data, 0, sizeof(uarea_init_t)); + odp_dma_pool_param_init(&dma_pool_param); + dma_pool_param.uarea_init.init_fn = init_event_uarea; + dma_pool_param.uarea_init.args = &data; + dma_pool_param.num = num; + dma_pool_param.uarea_size = 1; + pool = odp_dma_pool_create(NULL, &dma_pool_param); + + CU_ASSERT_FATAL(pool != ODP_POOL_INVALID); + CU_ASSERT(data.count == num); + + for (i = 0; i < num; i++) { + CU_ASSERT(data.mark[i] == 1); + + compl_evs[i] = odp_dma_compl_alloc(pool); + + CU_ASSERT(compl_evs[i] != ODP_DMA_COMPL_INVALID); + + if (compl_evs[i] == ODP_DMA_COMPL_INVALID) + break; + + uarea = odp_dma_compl_user_area(compl_evs[i]); + + CU_ASSERT(*uarea == UAREA); + } + + for (uint32_t j = 0; j < i; j++) + odp_dma_compl_free(compl_evs[j]); + + odp_pool_destroy(pool); +} + static void init_source(uint8_t *src, uint32_t len) { uint32_t i; @@ -1171,6 +1233,17 @@ static int check_event_user_area(void) return ODP_TEST_INACTIVE; } +static int check_event_user_area_init(void) +{ + if (global.disabled) + return ODP_TEST_INACTIVE; + + if (global.dma_capa.pool.max_uarea_size > 0 && global.dma_capa.pool.uarea_persistence) + return ODP_TEST_ACTIVE; + + return ODP_TEST_INACTIVE; +} + static int check_scheduled(void) { if (global.disabled) @@ -1568,6 +1641,7 @@ odp_testinfo_t dma_suite[] = { ODP_TEST_INFO_CONDITIONAL(test_dma_compl_pool_same_name, check_event), ODP_TEST_INFO_CONDITIONAL(test_dma_compl_pool_max_pools, check_event), ODP_TEST_INFO_CONDITIONAL(test_dma_compl_user_area, check_event_user_area), + ODP_TEST_INFO_CONDITIONAL(test_dma_compl_user_area_init, check_event_user_area_init), ODP_TEST_INFO_CONDITIONAL(test_dma_addr_to_addr_sync, check_sync), ODP_TEST_INFO_CONDITIONAL(test_dma_addr_to_addr_sync_mtrs, check_sync), ODP_TEST_INFO_CONDITIONAL(test_dma_addr_to_addr_sync_mseg, check_sync), diff --git a/test/validation/api/event/event.c b/test/validation/api/event/event.c index 1d6422f96..f6ad86365 100644 --- a/test/validation/api/event/event.c +++ b/test/validation/api/event/event.c @@ -1,4 +1,5 @@ /* Copyright (c) 2017-2018, Linaro Limited + * Copyright (c) 2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -349,6 +350,50 @@ static void event_test_type_multi(void) CU_ASSERT(odp_pool_destroy(pkt_pool) == 0); } +static void event_test_types_multi(void) +{ + odp_pool_t buf_pool, pkt_pool; + odp_event_t buf_event[NUM_TYPE_TEST]; + odp_event_t pkt_event[NUM_TYPE_TEST]; + odp_event_t event[2 * NUM_TYPE_TEST]; + odp_event_type_t event_types[2 * NUM_TYPE_TEST]; + odp_event_subtype_t event_subtypes[2 * NUM_TYPE_TEST]; + int i; + + type_test_init(&buf_pool, &pkt_pool, buf_event, pkt_event, event); + + /* Only buffers */ + odp_event_types_multi(buf_event, event_types, event_subtypes, NUM_TYPE_TEST); + for (i = 0; i < NUM_TYPE_TEST; i++) { + CU_ASSERT(event_types[i] == ODP_EVENT_BUFFER); + CU_ASSERT(event_subtypes[i] == ODP_EVENT_NO_SUBTYPE); + } + + /* Only packets */ + odp_event_types_multi(pkt_event, event_types, event_subtypes, NUM_TYPE_TEST); + for (i = 0; i < NUM_TYPE_TEST; i++) { + CU_ASSERT(event_types[i] == ODP_EVENT_PACKET); + CU_ASSERT(event_subtypes[i] == ODP_EVENT_PACKET_BASIC); + } + + /* Mixed events: B P BB PP BBB PPP */ + odp_event_types_multi(event, event_types, NULL, 2 * NUM_TYPE_TEST); + for (i = 0; i < 2 * NUM_TYPE_TEST; i++) { + if (i == 0 || i == 2 || i == 3 || i == 6 || i == 7 || i == 8) { + /* CU_ASSERT requires extra brackets */ + CU_ASSERT(event_types[i] == ODP_EVENT_BUFFER); + } else { + CU_ASSERT(event_types[i] == ODP_EVENT_PACKET); + } + } + + odp_event_free_multi(buf_event, NUM_TYPE_TEST); + odp_event_free_multi(pkt_event, NUM_TYPE_TEST); + + CU_ASSERT(odp_pool_destroy(buf_pool) == 0); + CU_ASSERT(odp_pool_destroy(pkt_pool) == 0); +} + static void event_test_filter_packet(void) { odp_pool_t buf_pool, pkt_pool; @@ -400,6 +445,7 @@ odp_testinfo_t event_suite[] = { ODP_TEST_INFO(event_test_free_multi), ODP_TEST_INFO(event_test_free_multi_mixed), ODP_TEST_INFO(event_test_type_multi), + ODP_TEST_INFO(event_test_types_multi), ODP_TEST_INFO(event_test_filter_packet), ODP_TEST_INFO(event_test_is_valid), ODP_TEST_INFO_NULL, diff --git a/test/validation/api/ipsec/ipsec.c b/test/validation/api/ipsec/ipsec.c index ea2685666..87688838b 100644 --- a/test/validation/api/ipsec/ipsec.c +++ b/test/validation/api/ipsec/ipsec.c @@ -447,6 +447,7 @@ static void ipsec_status_event_handle(odp_event_t ev_status, odp_ipsec_sa_t sa, enum ipsec_test_sa_expiry sa_expiry) { + int flag; odp_ipsec_status_t status = { .id = 0, .sa = ODP_IPSEC_SA_INVALID, @@ -460,6 +461,8 @@ static void ipsec_status_event_handle(odp_event_t ev_status, /* No user area for IPsec status events */ CU_ASSERT(odp_event_user_area(ev_status) == NULL); + CU_ASSERT(odp_event_user_area_and_flag(ev_status, &flag) == NULL); + CU_ASSERT(flag < 0); CU_ASSERT_EQUAL(0, odp_ipsec_status(&status, ev_status)); CU_ASSERT_EQUAL(ODP_IPSEC_STATUS_WARN, status.id); diff --git a/test/validation/api/packet/packet.c b/test/validation/api/packet/packet.c index a7631d47c..43530220e 100644 --- a/test/validation/api/packet/packet.c +++ b/test/validation/api/packet/packet.c @@ -3262,6 +3262,7 @@ static void packet_vector_test_user_area(void) for (i = 0; i < num; i++) { odp_event_t ev; + int flag; pktv[i] = odp_packet_vector_alloc(pool); @@ -3275,6 +3276,8 @@ static void packet_vector_test_user_area(void) ev = odp_packet_vector_to_event(pktv[i]); CU_ASSERT(odp_event_user_area(ev) == addr); + CU_ASSERT(odp_event_user_area_and_flag(ev, &flag) == addr); + CU_ASSERT(flag == 0); prev = addr; memset(addr, 0, size); @@ -3448,6 +3451,7 @@ static void packet_test_user_area(void) odp_packet_t pkt; odp_pool_t pool; odp_event_t ev; + int flag; memcpy(¶m, &default_param, sizeof(odp_pool_param_t)); @@ -3465,6 +3469,8 @@ static void packet_test_user_area(void) } ev = odp_packet_to_event(pkt); CU_ASSERT(odp_event_user_area(ev) == odp_packet_user_area(pkt)); + CU_ASSERT(odp_event_user_area_and_flag(ev, &flag) == odp_packet_user_area(pkt)); + CU_ASSERT(flag == 0); odp_packet_free(pkt); CU_ASSERT(odp_pool_destroy(pool) == 0); @@ -3480,6 +3486,8 @@ static void packet_test_user_area(void) CU_ASSERT_FATAL(odp_packet_user_area(pkt) != NULL); ev = odp_packet_to_event(pkt); CU_ASSERT(odp_event_user_area(ev) == odp_packet_user_area(pkt)); + CU_ASSERT(odp_event_user_area_and_flag(ev, &flag) == odp_packet_user_area(pkt)); + CU_ASSERT(flag == 0); CU_ASSERT(odp_packet_user_area_size(pkt) >= 1); *(char *)odp_packet_user_area(pkt) = 0; CU_ASSERT_FATAL(odp_packet_is_valid(pkt) == 1); @@ -3491,9 +3499,12 @@ static void packet_test_user_area(void) CU_ASSERT_FATAL(pool != ODP_POOL_INVALID); pkt = odp_packet_alloc(pool, param.pkt.len); CU_ASSERT_FATAL(pkt != ODP_PACKET_INVALID); + odp_packet_user_flag_set(pkt, 1); CU_ASSERT_FATAL(odp_packet_user_area(pkt) != NULL); ev = odp_packet_to_event(pkt); CU_ASSERT(odp_event_user_area(ev) == odp_packet_user_area(pkt)); + CU_ASSERT(odp_event_user_area_and_flag(ev, &flag) == odp_packet_user_area(pkt)); + CU_ASSERT(flag > 0); CU_ASSERT(odp_packet_user_area_size(pkt) == param.pkt.uarea_size); memset(odp_packet_user_area(pkt), 0, param.pkt.uarea_size); CU_ASSERT_FATAL(odp_packet_is_valid(pkt) == 1); diff --git a/test/validation/api/pktio/pktio.c b/test/validation/api/pktio/pktio.c index f412a01c4..fa752c0a2 100644 --- a/test/validation/api/pktio/pktio.c +++ b/test/validation/api/pktio/pktio.c @@ -3513,7 +3513,7 @@ static void pktio_test_pktout_compl_event(bool use_plain_queue) pktio_info_t pktio_rx_info; odp_pktio_config_t config; odp_queue_param_t qparam; - int ret, i, num_rx = 0; + int flag, ret, i, num_rx = 0; odp_event_t ev; uint64_t wait; @@ -3647,6 +3647,8 @@ static void pktio_test_pktout_compl_event(bool use_plain_queue) /* No user area for TX completion events */ CU_ASSERT(odp_event_user_area(ev) == NULL); + CU_ASSERT(odp_event_user_area_and_flag(ev, &flag) == NULL); + CU_ASSERT(flag < 0); /* Alternatively call event free / compl free */ if (i % 2) @@ -3685,6 +3687,8 @@ static void pktio_test_pktout_compl_event(bool use_plain_queue) /* No user area for TX completion events */ CU_ASSERT(odp_event_user_area(ev) == NULL); + CU_ASSERT(odp_event_user_area_and_flag(ev, &flag) == NULL); + CU_ASSERT(flag < 0); /* Check that sequence number is found */ CU_ASSERT(j < TX_BATCH_LEN); diff --git a/test/validation/api/pool/pool.c b/test/validation/api/pool/pool.c index b335f194c..5176bb96f 100644 --- a/test/validation/api/pool/pool.c +++ b/test/validation/api/pool/pool.c @@ -20,8 +20,11 @@ #define VEC_LEN 32 #define PKT_LEN 400 #define PKT_NUM 500 +#define ELEM_NUM 10 +#define ELEM_SIZE 128 #define CACHE_SIZE 32 #define MAX_NUM_DEFAULT (10 * 1024 * 1024) +#define UAREA 0xaa #define EXT_NUM_BUF 10 #define EXT_BUF_SIZE 2048 @@ -31,6 +34,8 @@ #define EXT_HEADROOM 16 #define MAGIC_U8 0x7a +#define MIN(a, b) (((a) < (b)) ? (a) : (b)) + typedef struct { odp_barrier_t init_barrier; odp_atomic_u32_t index; @@ -38,6 +43,11 @@ typedef struct { odp_pool_t pool; } global_shared_mem_t; +typedef struct { + uint32_t count; + uint8_t mark[ELEM_NUM]; +} uarea_init_t; + static global_shared_mem_t *global_mem; static odp_pool_capability_t global_pool_capa; @@ -51,6 +61,9 @@ static void test_param_init(uint8_t fill) memset(¶m, fill, sizeof(param)); odp_pool_param_init(¶m); + CU_ASSERT(param.uarea_init.init_fn == NULL); + CU_ASSERT(param.uarea_init.args == NULL); + CU_ASSERT(param.buf.uarea_size == 0); CU_ASSERT(param.buf.cache_size >= global_pool_capa.buf.min_cache_size && param.buf.cache_size <= global_pool_capa.buf.max_cache_size); @@ -146,6 +159,218 @@ static void pool_test_create_destroy_vector(void) pool_create_destroy(¶m); } +static int pool_check_buffer_uarea_init(void) +{ + if (global_pool_capa.buf.max_uarea_size == 0 || !global_pool_capa.buf.uarea_persistence) + return ODP_TEST_INACTIVE; + + return ODP_TEST_ACTIVE; +} + +static int pool_check_packet_uarea_init(void) +{ + if (global_pool_capa.pkt.max_uarea_size == 0 || !global_pool_capa.pkt.uarea_persistence) + return ODP_TEST_INACTIVE; + + return ODP_TEST_ACTIVE; +} + +static int pool_check_vector_uarea_init(void) +{ + if (global_pool_capa.vector.max_uarea_size == 0 || + !global_pool_capa.vector.uarea_persistence) + return ODP_TEST_INACTIVE; + + return ODP_TEST_ACTIVE; +} + +static int pool_check_timeout_uarea_init(void) +{ + if (global_pool_capa.tmo.max_uarea_size == 0 || !global_pool_capa.tmo.uarea_persistence) + return ODP_TEST_INACTIVE; + + return ODP_TEST_ACTIVE; +} + +static void init_event_uarea(void *uarea, uint32_t size, void *args, uint32_t index) +{ + uarea_init_t *data = args; + + data->count++; + data->mark[index] = 1; + memset(uarea, UAREA, size); +} + +static void pool_test_buffer_uarea_init(void) +{ + odp_pool_param_t param; + uint32_t num = MIN(global_pool_capa.buf.max_num, ELEM_NUM), + size = MIN(global_pool_capa.buf.max_size, ELEM_SIZE), i; + odp_pool_t pool; + uarea_init_t data; + odp_buffer_t bufs[num]; + uint8_t *uarea; + + memset(&data, 0, sizeof(uarea_init_t)); + odp_pool_param_init(¶m); + param.type = ODP_POOL_BUFFER; + param.uarea_init.init_fn = init_event_uarea; + param.uarea_init.args = &data; + param.buf.num = num; + param.buf.size = size; + param.buf.uarea_size = 1; + pool = odp_pool_create(NULL, ¶m); + + CU_ASSERT_FATAL(pool != ODP_POOL_INVALID); + CU_ASSERT(data.count == num); + + for (i = 0; i < num; i++) { + CU_ASSERT(data.mark[i] == 1); + + bufs[i] = odp_buffer_alloc(pool); + + CU_ASSERT(bufs[i] != ODP_BUFFER_INVALID); + + if (bufs[i] == ODP_BUFFER_INVALID) + break; + + uarea = odp_buffer_user_area(bufs[i]); + + CU_ASSERT(*uarea == UAREA); + } + + odp_buffer_free_multi(bufs, i); + odp_pool_destroy(pool); +} + +static void pool_test_packet_uarea_init(void) +{ + odp_pool_param_t param; + uint32_t num = MIN(global_pool_capa.pkt.max_num, ELEM_NUM), + size = MIN(global_pool_capa.pkt.max_len, ELEM_SIZE), i; + odp_pool_t pool; + uarea_init_t data; + odp_packet_t pkts[num]; + uint8_t *uarea; + + memset(&data, 0, sizeof(uarea_init_t)); + odp_pool_param_init(¶m); + param.type = ODP_POOL_PACKET; + param.uarea_init.init_fn = init_event_uarea; + param.uarea_init.args = &data; + param.pkt.num = num; + param.pkt.len = size; + param.pkt.uarea_size = 1; + pool = odp_pool_create(NULL, ¶m); + + CU_ASSERT_FATAL(pool != ODP_POOL_INVALID); + CU_ASSERT(data.count == num); + + for (i = 0; i < num; i++) { + CU_ASSERT(data.mark[i] == 1); + + pkts[i] = odp_packet_alloc(pool, ELEM_SIZE); + + CU_ASSERT(pkts[i] != ODP_PACKET_INVALID); + + if (pkts[i] == ODP_PACKET_INVALID) + break; + + uarea = odp_packet_user_area(pkts[i]); + + CU_ASSERT(*uarea == UAREA); + } + + odp_packet_free_multi(pkts, i); + odp_pool_destroy(pool); +} + +static void pool_test_vector_uarea_init(void) +{ + odp_pool_param_t param; + uint32_t num = MIN(global_pool_capa.vector.max_num, ELEM_NUM), + size = MIN(global_pool_capa.vector.max_size, ELEM_NUM), i; + odp_pool_t pool; + uarea_init_t data; + odp_packet_vector_t vecs[num]; + uint8_t *uarea; + + memset(&data, 0, sizeof(uarea_init_t)); + odp_pool_param_init(¶m); + param.type = ODP_POOL_VECTOR; + param.uarea_init.init_fn = init_event_uarea; + param.uarea_init.args = &data; + param.vector.num = num; + param.vector.max_size = size; + param.vector.uarea_size = 1; + pool = odp_pool_create(NULL, ¶m); + + CU_ASSERT_FATAL(pool != ODP_POOL_INVALID); + CU_ASSERT(data.count == num); + + for (i = 0; i < num; i++) { + CU_ASSERT(data.mark[i] == 1); + + vecs[i] = odp_packet_vector_alloc(pool); + + CU_ASSERT(vecs[i] != ODP_PACKET_VECTOR_INVALID); + + if (vecs[i] == ODP_PACKET_VECTOR_INVALID) + break; + + uarea = odp_packet_vector_user_area(vecs[i]); + + CU_ASSERT(*uarea == UAREA); + } + + for (uint32_t j = 0; j < i; j++) + odp_packet_vector_free(vecs[j]); + + odp_pool_destroy(pool); +} + +static void pool_test_timeout_uarea_init(void) +{ + odp_pool_param_t param; + uint32_t num = MIN(global_pool_capa.tmo.max_num, ELEM_NUM), i; + odp_pool_t pool; + uarea_init_t data; + odp_timeout_t tmos[num]; + uint8_t *uarea; + + memset(&data, 0, sizeof(uarea_init_t)); + odp_pool_param_init(¶m); + param.type = ODP_POOL_TIMEOUT; + param.uarea_init.init_fn = init_event_uarea; + param.uarea_init.args = &data; + param.tmo.num = num; + param.tmo.uarea_size = 1; + pool = odp_pool_create(NULL, ¶m); + + CU_ASSERT_FATAL(pool != ODP_POOL_INVALID); + CU_ASSERT(data.count == num); + + for (i = 0; i < num; i++) { + CU_ASSERT(data.mark[i] == 1); + + tmos[i] = odp_timeout_alloc(pool); + + CU_ASSERT(tmos[i] != ODP_TIMEOUT_INVALID); + + if (tmos[i] == ODP_TIMEOUT_INVALID) + break; + + uarea = odp_timeout_user_area(tmos[i]); + + CU_ASSERT(*uarea == UAREA); + } + + for (uint32_t j = 0; j < i; j++) + odp_timeout_free(tmos[j]); + + odp_pool_destroy(pool); +} + static void pool_test_lookup_info_print(void) { odp_pool_t pool; @@ -1505,6 +1730,8 @@ static void test_ext_param_init(uint8_t fill) odp_pool_ext_param_init(ODP_POOL_PACKET, ¶m); CU_ASSERT(param.type == ODP_POOL_PACKET); + CU_ASSERT(param.uarea_init.init_fn == NULL); + CU_ASSERT(param.uarea_init.args == NULL); CU_ASSERT(param.cache_size >= global_pool_ext_capa.min_cache_size && param.cache_size <= global_pool_ext_capa.max_cache_size); CU_ASSERT(param.stats.all == 0); @@ -1791,6 +2018,57 @@ static void test_packet_pool_ext_alloc(void) packet_pool_ext_alloc(PKT_LEN_NORMAL); } +static void test_packet_pool_ext_uarea_init(void) +{ + odp_pool_ext_capability_t capa; + odp_pool_ext_param_t param; + uint32_t num = ELEM_NUM, i; + odp_pool_t pool; + uarea_init_t data; + odp_shm_t shm; + uint8_t *uarea; + + CU_ASSERT_FATAL(odp_pool_ext_capability(ODP_POOL_PACKET, &capa) == 0); + + memset(&data, 0, sizeof(uarea_init_t)); + pool_ext_init_packet_pool_param(¶m); + param.uarea_init.init_fn = init_event_uarea; + param.uarea_init.args = &data; + num = MIN(num, param.pkt.num_buf); + param.pkt.num_buf = num; + param.pkt.uarea_size = 1; + pool = odp_pool_ext_create(NULL, ¶m); + + CU_ASSERT_FATAL(pool != ODP_POOL_INVALID); + + void *buf[num]; + odp_packet_t pkts[num]; + + shm = populate_pool(pool, &capa, buf, num, param.pkt.buf_size); + + CU_ASSERT_FATAL(shm != ODP_SHM_INVALID); + CU_ASSERT(data.count == num); + + for (i = 0; i < num; i++) { + CU_ASSERT(data.mark[i] == 1); + + pkts[i] = odp_packet_alloc(pool, (param.pkt.buf_size - param.pkt.headroom) / 2); + + CU_ASSERT(pkts[i] != ODP_PACKET_INVALID); + + if (pkts[i] == ODP_PACKET_INVALID) + break; + + uarea = odp_packet_user_area(pkts[i]); + + CU_ASSERT(*uarea == UAREA); + } + + odp_packet_free_multi(pkts, i); + odp_pool_destroy(pool); + odp_shm_free(shm); +} + static void test_packet_pool_ext_alloc_max(void) { packet_pool_ext_alloc(PKT_LEN_MAX); @@ -1961,6 +2239,15 @@ static int check_pool_ext_support(void) return ODP_TEST_ACTIVE; } +static int check_pool_ext_uarea_init_support(void) +{ + if (global_pool_ext_capa.max_pools == 0 || !global_pool_ext_capa.pkt.uarea_persistence || + global_pool_ext_capa.pkt.max_uarea_size == 0) + return ODP_TEST_INACTIVE; + + return ODP_TEST_ACTIVE; +} + static int check_pool_ext_segment_support(void) { if (global_pool_ext_capa.max_pools == 0 || global_pool_ext_capa.pkt.max_segs_per_pkt < 2) @@ -1975,6 +2262,10 @@ odp_testinfo_t pool_suite[] = { ODP_TEST_INFO(pool_test_create_destroy_packet), ODP_TEST_INFO(pool_test_create_destroy_timeout), ODP_TEST_INFO(pool_test_create_destroy_vector), + ODP_TEST_INFO_CONDITIONAL(pool_test_buffer_uarea_init, pool_check_buffer_uarea_init), + ODP_TEST_INFO_CONDITIONAL(pool_test_packet_uarea_init, pool_check_packet_uarea_init), + ODP_TEST_INFO_CONDITIONAL(pool_test_vector_uarea_init, pool_check_vector_uarea_init), + ODP_TEST_INFO_CONDITIONAL(pool_test_timeout_uarea_init, pool_check_timeout_uarea_init), ODP_TEST_INFO(pool_test_lookup_info_print), ODP_TEST_INFO(pool_test_same_name_buf), ODP_TEST_INFO(pool_test_same_name_pkt), @@ -2022,6 +2313,8 @@ odp_testinfo_t pool_ext_suite[] = { ODP_TEST_INFO_CONDITIONAL(test_packet_pool_ext_info, check_pool_ext_support), ODP_TEST_INFO_CONDITIONAL(test_packet_pool_ext_populate, check_pool_ext_support), ODP_TEST_INFO_CONDITIONAL(test_packet_pool_ext_alloc, check_pool_ext_support), + ODP_TEST_INFO_CONDITIONAL(test_packet_pool_ext_uarea_init, + check_pool_ext_uarea_init_support), ODP_TEST_INFO_CONDITIONAL(test_packet_pool_ext_alloc_max, check_pool_ext_support), ODP_TEST_INFO_CONDITIONAL(test_packet_pool_ext_alloc_seg, check_pool_ext_segment_support), ODP_TEST_INFO_CONDITIONAL(test_packet_pool_ext_disassemble, check_pool_ext_segment_support), diff --git a/test/validation/api/shmem/shmem.c b/test/validation/api/shmem/shmem.c index 86d070a4a..257e0214e 100644 --- a/test/validation/api/shmem/shmem.c +++ b/test/validation/api/shmem/shmem.c @@ -99,7 +99,7 @@ static int run_test_basic_thread(void *arg ODP_UNUSED) odp_shm_print_all(); CU_ASSERT(TEST_SHARE_FOO == shared_test_data->foo); CU_ASSERT(TEST_SHARE_BAR == shared_test_data->bar); - CU_ASSERT(0 == odp_shm_info(shm, &info)); + CU_ASSERT_FATAL(0 == odp_shm_info(shm, &info)); CU_ASSERT(0 == strcmp(MEM_NAME, info.name)); CU_ASSERT(0 == info.flags); CU_ASSERT(shared_test_data == info.addr); diff --git a/test/validation/api/time/time.c b/test/validation/api/time/time.c index 22189ce03..7664ec542 100644 --- a/test/validation/api/time/time.c +++ b/test/validation/api/time/time.c @@ -216,58 +216,157 @@ static void time_test_monotony(void) { volatile uint64_t count = 0; odp_time_t l_t1, l_t2, l_t3; + odp_time_t ls_t1, ls_t2, ls_t3; odp_time_t g_t1, g_t2, g_t3; - uint64_t lns_t1, lns_t2, lns_t3; - uint64_t gns_t1, gns_t2, gns_t3; + odp_time_t gs_t1, gs_t2, gs_t3; + uint64_t l_ns1, l_ns2, l_ns3; + uint64_t ls_ns1, ls_ns2, ls_ns3; + uint64_t g_ns1, g_ns2, g_ns3; + uint64_t gs_ns1, gs_ns2, gs_ns3; uint64_t ns1, ns2, ns3; + uint64_t s_ns1, s_ns2, s_ns3; - l_t1 = odp_time_local(); - g_t1 = odp_time_global(); - lns_t1 = odp_time_local_ns(); - gns_t1 = odp_time_global_ns(); + l_t1 = odp_time_local(); + ls_t1 = odp_time_local_strict(); + l_ns1 = odp_time_local_ns(); + ls_ns1 = odp_time_local_strict_ns(); + + g_t1 = odp_time_global(); + gs_t1 = odp_time_global_strict(); + g_ns1 = odp_time_global_ns(); + gs_ns1 = odp_time_global_strict_ns(); while (count < BUSY_LOOP_CNT) { count++; }; - l_t2 = odp_time_local(); - g_t2 = odp_time_global(); - lns_t2 = odp_time_local_ns(); - gns_t2 = odp_time_global_ns(); + l_t2 = odp_time_local(); + ls_t2 = odp_time_local_strict(); + l_ns2 = odp_time_local_ns(); + ls_ns2 = odp_time_local_strict_ns(); + + g_t2 = odp_time_global(); + gs_t2 = odp_time_global_strict(); + g_ns2 = odp_time_global_ns(); + gs_ns2 = odp_time_global_strict_ns(); count = 0; while (count < BUSY_LOOP_CNT) { count++; }; - l_t3 = odp_time_local(); - g_t3 = odp_time_global(); - lns_t3 = odp_time_local_ns(); - gns_t3 = odp_time_global_ns(); + l_t3 = odp_time_local(); + ls_t3 = odp_time_local_strict(); + l_ns3 = odp_time_local_ns(); + ls_ns3 = odp_time_local_strict_ns(); + + g_t3 = odp_time_global(); + gs_t3 = odp_time_global_strict(); + g_ns3 = odp_time_global_ns(); + gs_ns3 = odp_time_global_strict_ns(); + + /* Local time tests + * ---------------- */ ns1 = odp_time_to_ns(l_t1); ns2 = odp_time_to_ns(l_t2); ns3 = odp_time_to_ns(l_t3); - /* Local time assertions */ + s_ns1 = odp_time_to_ns(ls_t1); + s_ns2 = odp_time_to_ns(ls_t2); + s_ns3 = odp_time_to_ns(ls_t3); + + /* Time counting starts from zero. Assuming that the ODP instance has run + * less than 10 minutes before running this test case. */ + CU_ASSERT(ns1 < 10 * ODP_TIME_MIN_IN_NS); + CU_ASSERT(s_ns1 < 10 * ODP_TIME_MIN_IN_NS); + CU_ASSERT(l_ns1 < 10 * ODP_TIME_MIN_IN_NS); + CU_ASSERT(ls_ns1 < 10 * ODP_TIME_MIN_IN_NS); + + /* Time stamp */ CU_ASSERT(ns2 > ns1); CU_ASSERT(ns3 > ns2); + /* Strict time stamp */ + CU_ASSERT(s_ns2 > s_ns1); + CU_ASSERT(s_ns3 > s_ns2); + + /* Nsec time */ + CU_ASSERT(l_ns2 > l_ns1); + CU_ASSERT(l_ns3 > l_ns2); + + /* Strict nsec time */ + CU_ASSERT(ls_ns2 > ls_ns1); + CU_ASSERT(ls_ns3 > ls_ns2); + + /* Strict time stamp order is maintained */ + CU_ASSERT(ls_ns1 >= s_ns1); + CU_ASSERT(ls_ns2 >= s_ns2); + CU_ASSERT(ls_ns3 >= s_ns3); + + /* Time in nanoseconds have the same time base. Allow less than 100 msec error + * between time stamp converted to nsec and nsec time. */ + CU_ASSERT((ls_ns1 - s_ns1) < (100 * ODP_TIME_MSEC_IN_NS)); + CU_ASSERT((ls_ns2 - s_ns2) < (100 * ODP_TIME_MSEC_IN_NS)); + CU_ASSERT((ls_ns3 - s_ns3) < (100 * ODP_TIME_MSEC_IN_NS)); + + /* Global time tests + * ----------------- */ + ns1 = odp_time_to_ns(g_t1); ns2 = odp_time_to_ns(g_t2); ns3 = odp_time_to_ns(g_t3); - /* Global time assertions */ + s_ns1 = odp_time_to_ns(gs_t1); + s_ns2 = odp_time_to_ns(gs_t2); + s_ns3 = odp_time_to_ns(gs_t3); + + /* Time counting starts from zero. Assuming that the ODP instance has run + * less than 10 minutes before running this test case. */ + CU_ASSERT(ns1 < 10 * ODP_TIME_MIN_IN_NS); + CU_ASSERT(s_ns1 < 10 * ODP_TIME_MIN_IN_NS); + CU_ASSERT(g_ns1 < 10 * ODP_TIME_MIN_IN_NS); + CU_ASSERT(gs_ns1 < 10 * ODP_TIME_MIN_IN_NS); + + /* Time stamp */ CU_ASSERT(ns2 > ns1); CU_ASSERT(ns3 > ns2); - /* Local time in nsec */ - CU_ASSERT(lns_t2 > lns_t1); - CU_ASSERT(lns_t3 > lns_t2); - - /* Global time in nsec */ - CU_ASSERT(gns_t2 > gns_t1); - CU_ASSERT(gns_t3 > gns_t2); + /* Strict time stamp */ + CU_ASSERT(s_ns2 > s_ns1); + CU_ASSERT(s_ns3 > s_ns2); + + /* Nsec time */ + CU_ASSERT(g_ns2 > g_ns1); + CU_ASSERT(g_ns3 > g_ns2); + + /* Strict nsec time */ + CU_ASSERT(gs_ns2 > gs_ns1); + CU_ASSERT(gs_ns3 > gs_ns2); + + /* Strict time stamp order is maintained */ + CU_ASSERT(gs_ns1 >= s_ns1); + CU_ASSERT(gs_ns2 >= s_ns2); + CU_ASSERT(gs_ns3 >= s_ns3); + + /* Time in nanoseconds have the same time base. Allow less than 100 msec error + * between time stamp converted to nsec and nsec time. */ + CU_ASSERT((gs_ns1 - s_ns1) < (100 * ODP_TIME_MSEC_IN_NS)); + CU_ASSERT((gs_ns2 - s_ns2) < (100 * ODP_TIME_MSEC_IN_NS)); + CU_ASSERT((gs_ns3 - s_ns3) < (100 * ODP_TIME_MSEC_IN_NS)); + + /* Tight error margin cannot be used due to possible OS interrupts during the test. + * Record all time stamp values into the log to help debugging their relative order and + * accuracy. */ + printf("\n Time stamp values in nsec:\n"); + printf(" odp_time_local(): %" PRIu64 "\n", odp_time_to_ns(l_t1)); + printf(" odp_time_local_strict(): %" PRIu64 "\n", odp_time_to_ns(ls_t1)); + printf(" odp_time_local_ns(): %" PRIu64 "\n", l_ns1); + printf(" odp_time_local_strict_ns(): %" PRIu64 "\n", ls_ns1); + printf(" odp_time_global(): %" PRIu64 "\n", odp_time_to_ns(g_t1)); + printf(" odp_time_global_strict(): %" PRIu64 "\n", odp_time_to_ns(gs_t1)); + printf(" odp_time_global_ns(): %" PRIu64 "\n", g_ns1); + printf(" odp_time_global_strict_ns(): %" PRIu64 "\n\n", gs_ns1); } static void time_test_cmp(time_cb time_cur, time_from_ns_cb time_from_ns) diff --git a/test/validation/api/timer/timer.c b/test/validation/api/timer/timer.c index dad99ab41..1927156ba 100644 --- a/test/validation/api/timer/timer.c +++ b/test/validation/api/timer/timer.c @@ -453,6 +453,50 @@ static void timer_test_timeout_pool_alloc(void) CU_ASSERT(odp_pool_destroy(pool) == 0); } +static void timer_test_timeout_from_event(void) +{ + odp_pool_t pool; + odp_pool_param_t param; + uint32_t i; + const uint32_t num = 10; + uint32_t num_alloc = 0; + odp_timeout_t tmo_tbl[num]; + odp_timeout_t tmo2_tbl[num]; + odp_event_t ev_tbl[num]; + + odp_pool_param_init(¶m); + param.type = ODP_POOL_TIMEOUT; + param.tmo.num = num; + + pool = odp_pool_create("test_timeout_from_event", ¶m); + CU_ASSERT_FATAL(pool != ODP_POOL_INVALID); + + for (i = 0; i < num; i++) { + tmo_tbl[i] = odp_timeout_alloc(pool); + if (tmo_tbl[i] == ODP_TIMEOUT_INVALID) + break; + ev_tbl[i] = odp_timeout_to_event(tmo_tbl[i]); + num_alloc++; + } + + CU_ASSERT(num_alloc == num); + + for (i = 0; i < num_alloc; i++) { + odp_timeout_t tmo = odp_timeout_from_event(ev_tbl[i]); + + CU_ASSERT(odp_timeout_to_u64(tmo) == odp_timeout_to_u64(tmo_tbl[i])); + } + + odp_timeout_from_event_multi(tmo2_tbl, ev_tbl, num_alloc); + for (i = 0; i < num_alloc; i++) + CU_ASSERT(odp_timeout_to_u64(tmo2_tbl[i]) == odp_timeout_to_u64(tmo_tbl[i])); + + for (i = 0; i < num_alloc; i++) + odp_timeout_free(tmo_tbl[i]); + + CU_ASSERT_FATAL(odp_pool_destroy(pool) == 0); +} + static void timer_test_timeout_pool_free(void) { odp_pool_t pool; @@ -518,6 +562,7 @@ static void timer_test_timeout_user_area(void) for (i = 0; i < num; i++) { odp_event_t ev; + int flag; tmo[i] = odp_timeout_alloc(pool); @@ -532,6 +577,8 @@ static void timer_test_timeout_user_area(void) ev = odp_timeout_to_event(tmo[i]); CU_ASSERT(odp_event_user_area(ev) == addr); + CU_ASSERT(odp_event_user_area_and_flag(ev, &flag) == addr); + CU_ASSERT(flag < 0); prev = addr; memset(addr, 0, size); @@ -1323,7 +1370,7 @@ static void timer_pool_tick_info(void) } static void timer_test_event_type(odp_queue_type_t queue_type, - odp_event_type_t event_type) + odp_event_type_t event_type, int rounds) { odp_pool_t pool; odp_pool_param_t pool_param; @@ -1401,79 +1448,83 @@ static void timer_test_event_type(odp_queue_type_t queue_type, ODPH_DBG(" user_ptr %p\n\n", (const void *)user_ctx); for (i = 0; i < num; i++) { - if (event_type == ODP_EVENT_BUFFER) { - buf = odp_buffer_alloc(pool); - ev = odp_buffer_to_event(buf); - } else if (event_type == ODP_EVENT_PACKET) { - pkt = odp_packet_alloc(pool, 10); - ev = odp_packet_to_event(pkt); - } else { - tmo = odp_timeout_alloc(pool); - ev = odp_timeout_to_event(tmo); - } - - CU_ASSERT(ev != ODP_EVENT_INVALID); - timer[i] = odp_timer_alloc(timer_pool, queue, user_ctx); CU_ASSERT_FATAL(timer[i] != ODP_TIMER_INVALID); - - start_param.tick_type = ODP_TIMER_TICK_REL; - start_param.tick = (i + 1) * period_tick; - start_param.tmo_ev = ev; - - ret = odp_timer_start(timer[i], &start_param); - - if (ret == ODP_TIMER_TOO_NEAR) - ODPH_DBG("Timer set failed. Too near %i.\n", i); - else if (ret == ODP_TIMER_TOO_FAR) - ODPH_DBG("Timer set failed. Too far %i.\n", i); - else if (ret == ODP_TIMER_FAIL) - ODPH_DBG("Timer set failed %i\n", i); - - CU_ASSERT(ret == ODP_TIMER_SUCCESS); } - if (test_print) { - printf("\n"); - odp_timer_pool_print(timer_pool); - odp_timer_print(timer[0]); - } + for (int round = 0; round < rounds; round++) { + for (i = 0; i < num; i++) { + if (event_type == ODP_EVENT_BUFFER) { + buf = odp_buffer_alloc(pool); + ev = odp_buffer_to_event(buf); + } else if (event_type == ODP_EVENT_PACKET) { + pkt = odp_packet_alloc(pool, 10); + ev = odp_packet_to_event(pkt); + } else { + tmo = odp_timeout_alloc(pool); + ev = odp_timeout_to_event(tmo); + } - ev = ODP_EVENT_INVALID; - num_tmo = 0; - t1 = odp_time_local(); + CU_ASSERT(ev != ODP_EVENT_INVALID); - /* Wait for timers. Make sure that scheduler context is not held when - * exiting the loop. */ - do { - if (queue_type == ODP_QUEUE_TYPE_SCHED) - ev = odp_schedule(NULL, ODP_SCHED_NO_WAIT); - else - ev = odp_queue_deq(queue); + start_param.tick_type = ODP_TIMER_TICK_REL; + start_param.tick = (i + 1) * period_tick; + start_param.tmo_ev = ev; - if (ev == ODP_EVENT_INVALID) { - t2 = odp_time_local(); - if (odp_time_diff_ns(t2, t1) > (10 * duration_ns)) - break; + ret = odp_timer_start(timer[i], &start_param); - continue; - } + if (ret == ODP_TIMER_TOO_NEAR) + ODPH_DBG("Timer set failed. Too near %i.\n", i); + else if (ret == ODP_TIMER_TOO_FAR) + ODPH_DBG("Timer set failed. Too far %i.\n", i); + else if (ret == ODP_TIMER_FAIL) + ODPH_DBG("Timer set failed %i\n", i); - CU_ASSERT(odp_event_type(ev) == event_type); + CU_ASSERT(ret == ODP_TIMER_SUCCESS); + } if (test_print) { - test_print = 0; - tmo = odp_timeout_from_event(ev); - odp_timeout_print(tmo); printf("\n"); + odp_timer_pool_print(timer_pool); + odp_timer_print(timer[0]); } - odp_event_free(ev); - num_tmo++; + ev = ODP_EVENT_INVALID; + num_tmo = 0; + t1 = odp_time_local(); - } while (num_tmo < num || ev != ODP_EVENT_INVALID); + /* Wait for timers. Make sure that scheduler context is not held when + * exiting the loop. */ + do { + if (queue_type == ODP_QUEUE_TYPE_SCHED) + ev = odp_schedule(NULL, ODP_SCHED_NO_WAIT); + else + ev = odp_queue_deq(queue); - CU_ASSERT(num_tmo == num); + if (ev == ODP_EVENT_INVALID) { + t2 = odp_time_local(); + if (odp_time_diff_ns(t2, t1) > (10 * duration_ns)) + break; + + continue; + } + + CU_ASSERT(odp_event_type(ev) == event_type); + + if (test_print) { + test_print = 0; + tmo = odp_timeout_from_event(ev); + odp_timeout_print(tmo); + printf("\n"); + } + + odp_event_free(ev); + num_tmo++; + + } while (num_tmo < num || ev != ODP_EVENT_INVALID); + + CU_ASSERT(num_tmo == num); + } for (i = 0; i < num; i++) CU_ASSERT(odp_timer_free(timer[i]) == ODP_EVENT_INVALID); @@ -1485,32 +1536,47 @@ static void timer_test_event_type(odp_queue_type_t queue_type, static void timer_test_tmo_event_plain(void) { - timer_test_event_type(ODP_QUEUE_TYPE_PLAIN, ODP_EVENT_TIMEOUT); + timer_test_event_type(ODP_QUEUE_TYPE_PLAIN, ODP_EVENT_TIMEOUT, 1); } static void timer_test_tmo_event_sched(void) { - timer_test_event_type(ODP_QUEUE_TYPE_SCHED, ODP_EVENT_TIMEOUT); + timer_test_event_type(ODP_QUEUE_TYPE_SCHED, ODP_EVENT_TIMEOUT, 1); } static void timer_test_buf_event_plain(void) { - timer_test_event_type(ODP_QUEUE_TYPE_PLAIN, ODP_EVENT_BUFFER); + timer_test_event_type(ODP_QUEUE_TYPE_PLAIN, ODP_EVENT_BUFFER, 1); } static void timer_test_buf_event_sched(void) { - timer_test_event_type(ODP_QUEUE_TYPE_SCHED, ODP_EVENT_BUFFER); + timer_test_event_type(ODP_QUEUE_TYPE_SCHED, ODP_EVENT_BUFFER, 1); } static void timer_test_pkt_event_plain(void) { - timer_test_event_type(ODP_QUEUE_TYPE_PLAIN, ODP_EVENT_PACKET); + timer_test_event_type(ODP_QUEUE_TYPE_PLAIN, ODP_EVENT_PACKET, 1); } static void timer_test_pkt_event_sched(void) { - timer_test_event_type(ODP_QUEUE_TYPE_SCHED, ODP_EVENT_PACKET); + timer_test_event_type(ODP_QUEUE_TYPE_SCHED, ODP_EVENT_PACKET, 1); +} + +static void timer_test_tmo_event_reuse(void) +{ + timer_test_event_type(ODP_QUEUE_TYPE_SCHED, ODP_EVENT_TIMEOUT, 2); +} + +static void timer_test_buf_event_reuse(void) +{ + timer_test_event_type(ODP_QUEUE_TYPE_SCHED, ODP_EVENT_BUFFER, 2); +} + +static void timer_test_pkt_event_reuse(void) +{ + timer_test_event_type(ODP_QUEUE_TYPE_SCHED, ODP_EVENT_PACKET, 2); } static void timer_test_queue_type(odp_queue_type_t queue_type, int priv, int exp_relax) @@ -2701,7 +2767,7 @@ static void timer_test_periodic_capa(void) } } -static void timer_test_periodic(odp_queue_type_t queue_type, int use_first) +static void timer_test_periodic(odp_queue_type_t queue_type, int use_first, int rounds) { odp_timer_capability_t timer_capa; odp_timer_periodic_capability_t periodic_capa; @@ -2720,8 +2786,8 @@ static void timer_test_periodic(odp_queue_type_t queue_type, int use_first) double freq, freq_out, min_freq, max_freq; int ret; const char *user_ctx = "User context"; - int num_tmo = 0; - int done = 0; + int num_tmo; + int done; const int num = 200; /* Test frequency: 1x 100Hz, or 1x min/max_base_freq */ const uint64_t multiplier = 1; @@ -2829,127 +2895,135 @@ static void timer_test_periodic(odp_queue_type_t queue_type, int use_first) queue = odp_queue_create("timeout_queue", &queue_param); CU_ASSERT_FATAL(queue != ODP_QUEUE_INVALID); - tmo = odp_timeout_alloc(pool); - ev = odp_timeout_to_event(tmo); - CU_ASSERT_FATAL(ev != ODP_EVENT_INVALID); - timer = odp_timer_alloc(timer_pool, queue, user_ctx); CU_ASSERT_FATAL(timer != ODP_TIMER_INVALID); memset(&start_param, 0, sizeof(odp_timer_periodic_start_t)); - cur_tick = odp_timer_current_tick(timer_pool); offset_ns = period_ns / 2; - tick = cur_tick + odp_timer_ns_to_tick(timer_pool, offset_ns); if (use_first) { /* First tick moves timer to start before the first period */ duration_ns -= (period_ns - offset_ns); - start_param.first_tick = tick; } - start_param.freq_multiplier = multiplier; - start_param.tmo_ev = ev; + for (int round = 0; round < rounds; round++) { + num_tmo = 0; + done = 0; - ODPH_DBG("Periodic timer start:\n"); - ODPH_DBG(" Current tick: %" PRIu64 "\n", cur_tick); - ODPH_DBG(" First tick: %" PRIu64 "\n", start_param.first_tick); - ODPH_DBG(" Multiplier: %" PRIu64 "\n", start_param.freq_multiplier); - ODPH_DBG(" Period: %" PRIu64 " nsec\n", period_ns); - ODPH_DBG("Expected duration: %" PRIu64 " nsec\n", duration_ns); + tmo = odp_timeout_alloc(pool); + ev = odp_timeout_to_event(tmo); + CU_ASSERT_FATAL(ev != ODP_EVENT_INVALID); - ret = odp_timer_periodic_start(timer, &start_param); + cur_tick = odp_timer_current_tick(timer_pool); + tick = cur_tick + odp_timer_ns_to_tick(timer_pool, offset_ns); - if (ret == ODP_TIMER_TOO_NEAR) - ODPH_ERR("First tick too near\n"); - else if (ret == ODP_TIMER_TOO_FAR) - ODPH_ERR("First tick too far\n"); - else if (ret == ODP_TIMER_FAIL) - ODPH_ERR("Periodic timer start failed\n"); + if (use_first) + start_param.first_tick = tick; - CU_ASSERT_FATAL(ret == ODP_TIMER_SUCCESS); + start_param.freq_multiplier = multiplier; + start_param.tmo_ev = ev; - t1 = odp_time_local(); + ODPH_DBG("Periodic timer start:\n"); + ODPH_DBG(" Current tick: %" PRIu64 "\n", cur_tick); + ODPH_DBG(" First tick: %" PRIu64 "\n", start_param.first_tick); + ODPH_DBG(" Multiplier: %" PRIu64 "\n", start_param.freq_multiplier); + ODPH_DBG(" Period: %" PRIu64 " nsec\n", period_ns); + ODPH_DBG("Expected duration: %" PRIu64 " nsec\n", duration_ns); - /* Wait for timeouts. Make sure that scheduler context is not held when - * exiting the loop. */ - while (1) { - if (queue_type == ODP_QUEUE_TYPE_SCHED) - ev = odp_schedule(NULL, ODP_SCHED_NO_WAIT); - else - ev = odp_queue_deq(queue); + ret = odp_timer_periodic_start(timer, &start_param); - if (ev == ODP_EVENT_INVALID) { - t2 = odp_time_local(); - diff_ns = odp_time_diff_ns(t2, t1); - if (diff_ns > (10 * duration_ns)) - break; + if (ret == ODP_TIMER_TOO_NEAR) + ODPH_ERR("First tick too near\n"); + else if (ret == ODP_TIMER_TOO_FAR) + ODPH_ERR("First tick too far\n"); + else if (ret == ODP_TIMER_FAIL) + ODPH_ERR("Periodic timer start failed\n"); - if (num_tmo >= num) - break; + CU_ASSERT_FATAL(ret == ODP_TIMER_SUCCESS); - continue; - } + t1 = odp_time_local(); - CU_ASSERT(odp_event_type(ev) == ODP_EVENT_TIMEOUT); + /* Wait for timeouts. Make sure that scheduler context is not held when + * exiting the loop. */ + while (1) { + if (queue_type == ODP_QUEUE_TYPE_SCHED) + ev = odp_schedule(NULL, ODP_SCHED_NO_WAIT); + else + ev = odp_queue_deq(queue); - if (odp_event_type(ev) != ODP_EVENT_TIMEOUT) { - odp_event_free(ev); - continue; - } + if (ev == ODP_EVENT_INVALID) { + t2 = odp_time_local(); + diff_ns = odp_time_diff_ns(t2, t1); + if (diff_ns > (10 * duration_ns)) + break; - CU_ASSERT(odp_timer_periodic_ack(timer, ev) == 0); - num_tmo++; - } + if (num_tmo >= num) + break; - CU_ASSERT(num_tmo == num); + continue; + } - /* Allow +-30% error on test duration */ - CU_ASSERT((diff_ns > 0.7 * duration_ns) && (diff_ns < 1.3 * duration_ns)); + CU_ASSERT(odp_event_type(ev) == ODP_EVENT_TIMEOUT); - /* Stop periodic timer */ - ret = odp_timer_periodic_cancel(timer); - CU_ASSERT_FATAL(ret == 0); + if (odp_event_type(ev) != ODP_EVENT_TIMEOUT) { + odp_event_free(ev); + continue; + } - ODPH_DBG("Measured duration: %" PRIu64 " nsec\n", diff_ns); + CU_ASSERT(odp_timer_periodic_ack(timer, ev) == 0); + num_tmo++; + } - t1 = odp_time_local(); - while (1) { - if (queue_type == ODP_QUEUE_TYPE_SCHED) - ev = odp_schedule(NULL, ODP_SCHED_NO_WAIT); - else - ev = odp_queue_deq(queue); + CU_ASSERT(num_tmo == num); - if (ev == ODP_EVENT_INVALID) { - t2 = odp_time_local(); - diff_ns = odp_time_diff_ns(t2, t1); - if (diff_ns > (10 * duration_ns)) - break; + /* Allow +-30% error on test duration */ + CU_ASSERT((diff_ns > 0.7 * duration_ns) && (diff_ns < 1.3 * duration_ns)); - if (done) - break; + /* Stop periodic timer */ + ret = odp_timer_periodic_cancel(timer); + CU_ASSERT_FATAL(ret == 0); - continue; - } + ODPH_DBG("Measured duration: %" PRIu64 " nsec\n", diff_ns); - CU_ASSERT(odp_event_type(ev) == ODP_EVENT_TIMEOUT); + t1 = odp_time_local(); + while (1) { + if (queue_type == ODP_QUEUE_TYPE_SCHED) + ev = odp_schedule(NULL, ODP_SCHED_NO_WAIT); + else + ev = odp_queue_deq(queue); - if (odp_event_type(ev) != ODP_EVENT_TIMEOUT) { - odp_event_free(ev); - continue; - } + if (ev == ODP_EVENT_INVALID) { + t2 = odp_time_local(); + diff_ns = odp_time_diff_ns(t2, t1); + if (diff_ns > (10 * duration_ns)) + break; + + if (done) + break; - ret = odp_timer_periodic_ack(timer, ev); - CU_ASSERT(ret == 1 || ret == 2); + continue; + } - if (ret == 2) { - odp_event_free(ev); - done = 1; + CU_ASSERT(odp_event_type(ev) == ODP_EVENT_TIMEOUT); + + if (odp_event_type(ev) != ODP_EVENT_TIMEOUT) { + odp_event_free(ev); + continue; + } + + ret = odp_timer_periodic_ack(timer, ev); + CU_ASSERT(ret == 1 || ret == 2); + + if (ret == 2) { + odp_event_free(ev); + done = 1; + } } - } - /* Check that ack() returned 2 on the last event */ - CU_ASSERT(done); - CU_ASSERT(ret == 2); + /* Check that ack() returned 2 on the last event */ + CU_ASSERT(done); + CU_ASSERT(ret == 2); + } CU_ASSERT(odp_timer_free(timer) == ODP_EVENT_INVALID); odp_timer_pool_destroy(timer_pool); @@ -2959,28 +3033,34 @@ static void timer_test_periodic(odp_queue_type_t queue_type, int use_first) static void timer_test_periodic_sched(void) { - timer_test_periodic(ODP_QUEUE_TYPE_SCHED, 0); + timer_test_periodic(ODP_QUEUE_TYPE_SCHED, 0, 1); } static void timer_test_periodic_plain(void) { - timer_test_periodic(ODP_QUEUE_TYPE_PLAIN, 0); + timer_test_periodic(ODP_QUEUE_TYPE_PLAIN, 0, 1); } static void timer_test_periodic_sched_first(void) { - timer_test_periodic(ODP_QUEUE_TYPE_SCHED, FIRST_TICK); + timer_test_periodic(ODP_QUEUE_TYPE_SCHED, FIRST_TICK, 1); } static void timer_test_periodic_plain_first(void) { - timer_test_periodic(ODP_QUEUE_TYPE_PLAIN, FIRST_TICK); + timer_test_periodic(ODP_QUEUE_TYPE_PLAIN, FIRST_TICK, 1); +} + +static void timer_test_periodic_reuse(void) +{ + timer_test_periodic(ODP_QUEUE_TYPE_SCHED, 0, 2); } odp_testinfo_t timer_suite[] = { ODP_TEST_INFO(timer_test_capa), ODP_TEST_INFO(timer_test_param_init), ODP_TEST_INFO(timer_test_timeout_pool_alloc), + ODP_TEST_INFO(timer_test_timeout_from_event), ODP_TEST_INFO(timer_test_timeout_pool_free), ODP_TEST_INFO(timer_test_timeout_user_area), ODP_TEST_INFO(timer_pool_create_destroy), @@ -3019,6 +3099,12 @@ odp_testinfo_t timer_suite[] = { check_plain_queue_support), ODP_TEST_INFO_CONDITIONAL(timer_test_pkt_event_sched, check_sched_queue_support), + ODP_TEST_INFO_CONDITIONAL(timer_test_tmo_event_reuse, + check_sched_queue_support), + ODP_TEST_INFO_CONDITIONAL(timer_test_buf_event_reuse, + check_sched_queue_support), + ODP_TEST_INFO_CONDITIONAL(timer_test_pkt_event_reuse, + check_sched_queue_support), ODP_TEST_INFO(timer_test_cancel), ODP_TEST_INFO_CONDITIONAL(timer_test_max_res_min_tmo_plain, check_plain_queue_support), @@ -3062,6 +3148,8 @@ odp_testinfo_t timer_suite[] = { check_periodic_plain_support), ODP_TEST_INFO_CONDITIONAL(timer_test_periodic_plain_first, check_periodic_plain_support), + ODP_TEST_INFO_CONDITIONAL(timer_test_periodic_reuse, + check_periodic_sched_support), ODP_TEST_INFO_NULL, }; |