diff options
author | Matias Elo <matias.elo@nokia.com> | 2023-02-24 09:39:16 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2023-02-24 09:39:16 +0200 |
commit | 0f2f3271f64b25d7ce101b7a5c82b7fc17ba2d32 (patch) | |
tree | 5d4df86a452c52b1421b8039835ec793ac4019c7 | |
parent | 282188e33695bb6863e289df61e0dfbd87ca205c (diff) | |
parent | c7c99cfd5635da8c0afc7f3803ce6e1c82c3465b (diff) |
Merge ODP v1.40.0.0v1.40.0.0_DPDK_19.11
Merge ODP linux-generic v1.40.0.0 into linux-dpdk.
159 files changed, 6981 insertions, 2710 deletions
diff --git a/.github/workflows/ci-pipeline-arm64.yml b/.github/workflows/ci-pipeline-arm64.yml index b4fb1e271..3d2f106a8 100644 --- a/.github/workflows/ci-pipeline-arm64.yml +++ b/.github/workflows/ci-pipeline-arm64.yml @@ -129,7 +129,8 @@ jobs: cc: [gcc, clang] conf: ['', '--enable-abi-compat', '--enable-deprecated --enable-helper-deprecated --enable-debug=full', '--enable-dpdk-zero-copy --disable-static-applications', - '--disable-host-optimization', '--disable-host-optimization --enable-abi-compat', + '--disable-host-optimization --enable-event-validation=warn', + '--disable-host-optimization --enable-abi-compat', '--without-openssl --without-pcap'] steps: - uses: AutoModality/action-clean@v1.1.0 @@ -185,6 +186,18 @@ jobs: if: ${{ failure() }} run: find . -name "*.trs" | xargs grep -l '^.test-result. FAIL' | while read trs ; do echo FAILURE detected at $trs; cat ${trs%%.trs}.log ; done + Run_stash_config: + if: ${{ github.repository == 'OpenDataPlane/odp-dpdk' }} + runs-on: [self-hosted, ARM64] + steps: + - uses: AutoModality/action-clean@v1.1.0 + - uses: actions/checkout@v3 + - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${CC}" -e ARCH="${ARCH}" + -e CONF="${CONF}" -e ODP_CONFIG_FILE=/odp/platform/linux-dpdk/test/stash-custom.conf $CONTAINER_NAMESPACE/odp-ci-${OS}-${ARCH}-native /odp/scripts/ci/check.sh + - name: Failure log + if: ${{ failure() }} + run: find . -name "*.trs" | xargs grep -l '^.test-result. FAIL' | while read trs ; do echo FAILURE detected at $trs; cat ${trs%%.trs}.log ; done + Run_scheduler_sp: if: ${{ github.repository == 'OpenDataPlane/odp-dpdk' }} runs-on: [self-hosted, ARM64] diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index 425ee17b6..dff61a187 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -277,7 +277,8 @@ jobs: cc: [gcc, clang] conf: ['', '--enable-abi-compat', '--enable-deprecated --enable-helper-deprecated --enable-debug=full', '--disable-static-applications', - '--disable-host-optimization', '--disable-host-optimization --enable-abi-compat', + '--disable-host-optimization --enable-event-validation=warn', + '--disable-host-optimization --enable-abi-compat', '--without-openssl --without-pcap'] steps: - uses: actions/checkout@v3 @@ -312,6 +313,16 @@ jobs: if: ${{ failure() }} run: find . -name "*.trs" | xargs grep -l '^.test-result. FAIL' | while read trs ; do echo FAILURE detected at $trs; cat ${trs%%.trs}.log ; done + Run_stash_config: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v3 + - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${CC}" -e ARCH="${ARCH}" + -e CONF="${CONF}" -e ODP_CONFIG_FILE=/odp/platform/linux-dpdk/test/stash-custom.conf $CONTAINER_NAMESPACE/odp-ci-${OS}-${ARCH} /odp/scripts/ci/check.sh + - name: Failure log + if: ${{ failure() }} + run: find . -name "*.trs" | xargs grep -l '^.test-result. FAIL' | while read trs ; do echo FAILURE detected at $trs; cat ${trs%%.trs}.log ; done + Run_scheduler_sp: runs-on: ubuntu-20.04 steps: @@ -1,3 +1,71 @@ +== OpenDataPlane (1.40.0.0) + +=== Backward incompatible API changes +==== Packet +* Specify which packet metadata flags cannot be set simultaneously using +`odp_packet_has_XX_set()` functions. + +==== Timer +* Use `ODP_DEPRECATE()` macro for `odp_timer_set_abs()` and +`odp_timer_set_rel()` functions. Previously, the deprecation was only mentioned +in the function documentation. +* Deprecate old timer pool clock sources `ODP_CLOCK_CPU` and `ODP_CLOCK_EXT`, +which have been replaced by `ODP_CLOCK_DEFAULT` and `ODP_CLOCK_SRC_1`. +* Deprecate old timer set return values `ODP_TIMER_TOOEARLY`, +`ODP_TIMER_TOOLATE`, and `ODP_TIMER_NOEVENT`, which have been replaced by +`ODP_TIMER_TOO_NEAR`, `ODP_TIMER_TOO_FAR`, and `ODP_TIMER_FAIL`. + +=== Backward compatible API changes +==== Crypto +* Add new operation type session parameter +(`odp_crypto_session_param_t.op_type`) that controls how crypto operations +interpret their parameters and handle output packets. Defaults to backward +compatible `ODP_CRYPTO_OP_TYPE_LEGACY` mode. +* Add `ODP_CRYPTO_OP_TYPE_BASIC` operation type with simplified interface +compared to `ODP_CRYPTO_OP_TYPE_LEGACY`. +* Add `ODP_CRYPTO_OP_TYPE_OOP` operation type that writes the output of the +crypto operation into a caller provided output packet and does not consume the +input packet. +* Clarify that `odp_crypto_op()` copies all packet data and metadata from the +input packet to the output packet in `ODP_CRYPTO_OP_TYPE_LEGACY` and +`ODP_CRYPTO_OP_TYPE_BASIC` modes. +* Require that `odp_crypto_result()` is called before packet data of +asynchronously processed packets can be assumed to be valid in non-legacy modes. +* Fix EIA2 IV length in API documentation. EIA2 uses 64-bit IV, not 128-bit as +previously mentioned in the API text. + +==== Packet +* Clarify `odp_packet_has_vlan()` and `odp_packet_has_vlan_qinq()` +specifications. + +=== Remove deprecated APIs +==== Crypto +* Remove deprecated per-session IV configuration. + +==== Packet IO +* Remove deprecated `in_unknown_protos` field from `odp_pktio_stats_t`. +* Remove deprecated `odp_pktin_ts_res()` function. +* Remove deprecated `odp_pktin_ts_from_ns()` function. + +==== Shared Memory +* Remove deprecated `ODP_SHM_SW_ONLY` define. + +==== Traffic Manager +* Remove deprecated `odp_tm_capabilities()` function. +* Remove deprecated `commit_bps` field from `odp_tm_shaper_params_t`. +* Remove deprecated `peak_bps` field from `odp_tm_shaper_params_t`. + +=== Implementation +==== Crypto +* Add Multi-Buffer Crypto for IPsec library (Arm optimized) based crypto +implementation. See `DEPENDENCIES` for additional information. + +==== Debug +* Add support for runtime event validation (buffer endmark checking). Event +validation can be enabled during configure with +`--enable-event-validation [warn/abort]` or with `--enabled-debug=full`. See +`README` for additional information. + == OpenDataPlane (1.39.0.0) === Backward incompatible API changes diff --git a/DEPENDENCIES b/DEPENDENCIES index 0a1bea32b..816639bee 100644 --- a/DEPENDENCIES +++ b/DEPENDENCIES @@ -124,11 +124,39 @@ Prerequisites for building the OpenDataPlane (ODP) API $ ./configure --with-crypto=armv8crypto $ make -3.4 Netmap packet I/O support (optional) +3.4 Multi-buffer Crypto for IPsec Library (optional) + + Multi-Buffer Crypto for IPsec Library is a set of functions that + implement authentication and encryption processing for IPsec, these functions + take advantage of SIMD instructions to improve performance. + + Note ODP assumes that IPSec MB library is compiled with SAFE_PARAM enabled + (enabled by default), otherwise crypto operations with too long cipher/auth + ranges will have undefined behaviour. + +3.4.1 Building Multi-buffer Crypto for IPSec Library for Arm + + # Checkout and build Arm code + $ git clone https://git.gitlab.arm.com/arm-reference-solutions/ipsec-mb.git + $ cd ipsec-mb/ + $ git checkout SECLIB-IPSEC-2022.12.13 + $ make + $ sudo make install + + For additional instructions, refer to README.md in crypto library repository. + +3.4.2 Building ODP with Multi-buffer IPSec Library + $ ./bootstrap + + # Compile and build ODP with Multi-buffer IPSec library + $ ./configure --with-crypto=ipsecmb + $ make + +3.5 Netmap packet I/O support (optional) Netmap accelerated ODP packet I/O. -3.4.1 Building netmap kernel modules +3.5.1 Building netmap kernel modules ODP works at least with the latest release version of netmap, which is currently v13.0. However, if possible one should try to use the latest netmap @@ -172,14 +200,14 @@ Prerequisites for building the OpenDataPlane (ODP) API $ ./configure --kernel-sources=<path_to_kernel_src> $ make -3.4.2 Building ODP +3.5.2 Building ODP $ cd <odp_dir> $ ./bootstrap $ ./configure --with-netmap-path=<netmap_dir> $ make -3.4.3 Inserting netmap kernel modules +3.5.3 Inserting netmap kernel modules In order to use netmap I/O you need to insert at least the core netmap kernel module. @@ -197,13 +225,13 @@ Prerequisites for building the OpenDataPlane (ODP) API To restore the original drivers you should be able to use modprobe. -3.4.4 Running ODP with netmap I/O +3.5.4 Running ODP with netmap I/O ODP applications will use netmap for packet I/O by default as long as the netmap kernel module is loaded. If socket I/O is desired instead, it can be activated by setting the environment variable ODP_PKTIO_DISABLE_NETMAP. -3.5 DPDK packet I/O support (optional) +3.6 DPDK packet I/O support (optional) Use DPDK for ODP packet I/O. Currently supported DPDK versions are v19.11, v20.11 (recommended), v21.11. @@ -212,7 +240,7 @@ Prerequisites for building the OpenDataPlane (ODP) API https://github.com/OpenDataPlane/odp-dpdk.git for a full DPDK based ODP implementation. -3.5.1 DPDK pktio requirements +3.6.1 DPDK pktio requirements DPDK pktio adds a dependency to NUMA library. # Debian/Ubuntu @@ -221,11 +249,11 @@ Prerequisites for building the OpenDataPlane (ODP) API # CentOS/RedHat/Fedora $ sudo yum install numactl-devel -3.5.2 Native DPDK install +3.6.2 Native DPDK install # Debian/Ubuntu starting from 20.04 $ sudo apt-get install dpdk-dev -3.5.3 Build DPDK v19.11 from source +3.6.3 Build DPDK v19.11 from source $ git clone https://dpdk.org/git/dpdk-stable --branch 19.11 --depth 1 ./<dpdk-dir> # Make and edit DPDK configuration @@ -243,7 +271,7 @@ Prerequisites for building the OpenDataPlane (ODP) API # Configure ODP $ ./configure --with-dpdk-path=<dpdk-dir> -3.5.4 Build DPDK v20.11 and onwards from source +3.6.4 Build DPDK v20.11 and onwards from source $ git clone https://dpdk.org/git/dpdk-stable --branch <version, e.g. 21.11> --depth 1 ./<dpdk-dir> # Prepare the build directory @@ -263,7 +291,7 @@ Prerequisites for building the OpenDataPlane (ODP) API # Or, if DPDK was not installed to the default location, set PKG_CONFIG_PATH: $ PKG_CONFIG_PATH=<dpdk-dir>/install/lib/x86_64-linux-gnu/pkgconfig ./configure --enable-dpdk -3.5.5 Setup system +3.6.5 Setup system # Load DPDK modules $ sudo modprobe uio @@ -275,7 +303,7 @@ Prerequisites for building the OpenDataPlane (ODP) API 512 x 2MB huge pages. All this can be done with the DPDK setup script (<dpdk-dir>/usertools/dpdk-setup.sh). -3.5.6 Running ODP with DPDK pktio +3.6.6 Running ODP with DPDK pktio ODP applications will try use DPDK for packet I/O by default. If some other I/O type is desired instead, DPDK I/O can be disabled by setting the @@ -291,7 +319,7 @@ Prerequisites for building the OpenDataPlane (ODP) API 1024MB of memory: $ sudo ODP_PKTIO_DPDK_PARAMS="-m 1024" ./test/performance/odp_l2fwd -i 0 -c 1 -3.6 AF_XDP socket based packet I/O support (optional) +3.7 AF_XDP socket based packet I/O support (optional) Use AF_XDP socket for packet I/O. At the moment, only zero-copy variant is supported, requiring a kernel version 5.4 or higher. Additionally, if packet @@ -308,7 +336,7 @@ Prerequisites for building the OpenDataPlane (ODP) API Note that, currently, AF_XDP socket packet I/O cannot be instantiated if DPDK zero-copy is enabled. -3.6.1 AF_XDP socket packet I/O requirements +3.7.1 AF_XDP socket packet I/O requirements AF_XDP socket packet I/O implementation requires libxdp and libbpf libraries. They can be fetched from XDP-project in GitHub: @@ -330,14 +358,14 @@ Prerequisites for building the OpenDataPlane (ODP) API $ cd <path to built libbpf> $ make install -3.6.2 Build ODP with AF_XDP socket packet I/O support +3.7.2 Build ODP with AF_XDP socket packet I/O support After building and installing libxdp and libbpf, ODP can be configured to be built with AF_XDP support (pass PKG_CONFIG_PATH if needed). $ ./configure --enable-xdp -3.6.3 Running ODP with AF_XDP socket packet I/O +3.7.3 Running ODP with AF_XDP socket packet I/O AF_XDP socket packet I/Os bind to TRX-combined queues. Based on the packet prosessing needs, NIC(s) of the environment should be configured diff --git a/config/odp-linux-dpdk.conf b/config/odp-linux-dpdk.conf index d28f728f2..8fe298e5b 100644 --- a/config/odp-linux-dpdk.conf +++ b/config/odp-linux-dpdk.conf @@ -16,7 +16,7 @@ # Mandatory fields odp_implementation = "linux-dpdk" -config_file_version = "0.1.18" +config_file_version = "0.1.21" # System options system: { @@ -192,6 +192,15 @@ sched_basic: { worker = 1 control = 1 } + + # Ordered queue reorder stash size + # + # Number of events each thread can stash internally before having to + # wait for the right order context. Reorder stash can improve + # performance if threads process events in bursts. If 'order_stash_size' + # > 0, events may be dropped by the implementation if the target queue + # is full. To prevent this set 'order_stash_size' to 0. + order_stash_size = 512 } sched_eventdev: { @@ -209,6 +218,23 @@ sched_eventdev: { num_ports = 0 } +stash: { + # Maximum number of stashes + max_num = 512 + + # Maximum number of objects in a stash + # + # The value may be rounded up by the implementation. For optimal memory + # usage set value to a power of two - 1. + max_num_obj = 4095 + + # Strict size + # + # When set to 0, application can attempt to store more handles into a + # stash than it specified in the creation parameters. + strict_size = 1 +} + timer: { # Inline timer poll interval # diff --git a/config/odp-linux-generic.conf b/config/odp-linux-generic.conf index f8accd07f..df5328d1f 100644 --- a/config/odp-linux-generic.conf +++ b/config/odp-linux-generic.conf @@ -16,7 +16,7 @@ # Mandatory fields odp_implementation = "linux-generic" -config_file_version = "0.1.22" +config_file_version = "0.1.25" # System options system: { @@ -234,6 +234,32 @@ sched_basic: { worker = 1 control = 1 } + + # Ordered queue reorder stash size + # + # Number of events each thread can stash internally before having to + # wait for the right order context. Reorder stash can improve + # performance if threads process events in bursts. If 'order_stash_size' + # > 0, events may be dropped by the implementation if the target queue + # is full. To prevent this set 'order_stash_size' to 0. + order_stash_size = 512 +} + +stash: { + # Maximum number of stashes + max_num = 512 + + # Maximum number of objects in a stash + # + # The value may be rounded up by the implementation. For optimal memory + # usage set value to a power of two - 1. + max_num_obj = 4095 + + # Strict size + # + # When set to 0, application can attempt to store more handles into a + # stash than it specified in the creation parameters. + strict_size = 1 } timer: { diff --git a/configure.ac b/configure.ac index 9b3863cef..19b23e4b5 100644 --- a/configure.ac +++ b/configure.ac @@ -3,7 +3,7 @@ AC_PREREQ([2.5]) # ODP API version ########################################################################## m4_define([odp_version_generation], [1]) -m4_define([odp_version_major], [39]) +m4_define([odp_version_major], [40]) m4_define([odp_version_minor], [0]) m4_define([odp_version_patch], [0]) diff --git a/doc/images/timeout_fsm.gv b/doc/images/timeout_fsm.gv index dab24d0d6..179434bce 100644 --- a/doc/images/timeout_fsm.gv +++ b/doc/images/timeout_fsm.gv @@ -8,17 +8,13 @@ digraph timer_state_machine { TO_Unalloc -> TO_Alloc [label="odp_timeout_alloc()"]; TO_Alloc -> TO_Unalloc [label="odp_timeout_free()"]; TO_Alloc -> TO_Pending [fontcolor=green, - label="odp_timer_set_abs()"]; - TO_Alloc -> TO_Pending [fontcolor=green, - label="odp_timer_set_rel()"]; + label="odp_timer_start()"]; TO_Pending -> TO_Alloc [fontcolor=green, label="odp_timer_cancel()"]; TO_Pending -> TO_Enqueued [fontcolor=green, label="timer expires"]; TO_Enqueued -> TO_Delivered [label="odp_schedule()"]; TO_Delivered -> TO_Pending [fontcolor=green, - label="odp_timer_set_abs()"]; - TO_Delivered -> TO_Pending [fontcolor=green, - label="odp_timer_set_rel()"]; + label="odp_timer_start()"]; TO_Delivered -> TO_Delivered [label="odp_timeout_from_event()"]; TO_Delivered -> TO_Delivered [label="odp_timeout_timer()"]; TO_Delivered -> TO_Unalloc diff --git a/doc/images/timer_fsm.gv b/doc/images/timer_fsm.gv index 819d0f979..eb1df8ae9 100644 --- a/doc/images/timer_fsm.gv +++ b/doc/images/timer_fsm.gv @@ -3,17 +3,12 @@ digraph timer_state_machine { node [fontsize=28]; edge [fontsize=28]; node [shape=doublecircle]; Timer_Unalloc; - node [shape=circle]; Timer_Alloc Timer_Set Timer_Expired + node [shape=circle]; Timer_Alloc Timer_Active Timer_Expired Timer_Unalloc -> Timer_Alloc [label="odp_timer_alloc()"]; Timer_Alloc -> Timer_Unalloc [label="odp_timer_free()"]; - Timer_Alloc -> Timer_Set [fontcolor=green,label="odp_timer_set_abs()"]; - Timer_Alloc -> Timer_Set [fontcolor=green,label="odp_timer_set_rel()"]; - Timer_Set -> Timer_Alloc [fontcolor=green,label="odp_timer_cancel()"]; - Timer_Set -> Timer_Expired [fontcolor=green,label="timer expires"]; + Timer_Alloc -> Timer_Active [fontcolor=green,label="odp_timer_start()"]; + Timer_Active -> Timer_Alloc [fontcolor=green,label="odp_timer_cancel()"]; + Timer_Active -> Timer_Expired [fontcolor=green,label="timer expires"]; Timer_Expired -> Timer_Unalloc [label="odp_timer_free()"]; - Timer_Expired -> Timer_Set [fontcolor=green, - label="odp_timer_set_abs()"]; - Timer_Expired -> Timer_Set [fontcolor=green, - label="odp_timer_set_rel()"]; - + Timer_Expired -> Timer_Active [fontcolor=green, label="odp_timer_start()"]; } diff --git a/doc/users-guide/users-guide-timer.adoc b/doc/users-guide/users-guide-timer.adoc index 4a8bad74f..e854b5f2d 100644 --- a/doc/users-guide/users-guide-timer.adoc +++ b/doc/users-guide/users-guide-timer.adoc @@ -46,11 +46,11 @@ as an input parameter to enable the pool-specific conversion ratios to be used. Associated with each timer pool is a free running tick counter that can be -sampled at any time via the `odp_timer_current_tick()` API. Timers can be set -to an absolute future tick value via `odp_timer_set_abs()` or to a future tick -value relative to the current tick via `odp_timer_set_rel()`. Implementations -may impose minimum and maximum future values supported by a given timer pool -and timer set operations will fail if the requested value is outside of the +sampled at any time via the `odp_timer_current_tick()` API. Timers are started +with `odp_timer_start()` and the expiration time can be an absolute future tick +value or a future tick value relative to the current tick. Implementations may +impose minimum and maximum future values supported by a given timer pool and +timer start operations will fail if the requested value is outside of the supported range. Before a set timer expires, it can be canceled via the `odp_timer_cancel()` diff --git a/example/debug/odp_debug.c b/example/debug/odp_debug.c index 6b4ea977f..6683e46b4 100644 --- a/example/debug/odp_debug.c +++ b/example/debug/odp_debug.c @@ -372,6 +372,7 @@ static int timer_debug(void) odp_timer_capability_t timer_capa; odp_timer_pool_t timer_pool; odp_timer_pool_param_t timer_param; + odp_timer_start_t start_param; odp_timer_t timer; odp_queue_t queue; odp_queue_param_t queue_param; @@ -452,8 +453,13 @@ static int timer_debug(void) odp_timeout_print(timeout); event = odp_timeout_to_event(timeout); - if (odp_timer_set_rel(timer, tick, &event) != ODP_TIMER_SUCCESS) - ODPH_ERR("Timer set failed.\n"); + + start_param.tick_type = ODP_TIMER_TICK_REL; + start_param.tick = tick; + start_param.tmo_ev = event; + + if (odp_timer_start(timer, &start_param) != ODP_TIMER_SUCCESS) + ODPH_ERR("Timer start failed.\n"); printf("\n"); odp_timer_print(timer); diff --git a/example/sysinfo/odp_sysinfo.c b/example/sysinfo/odp_sysinfo.c index 523f70496..093b2f1bf 100644 --- a/example/sysinfo/odp_sysinfo.c +++ b/example/sysinfo/odp_sysinfo.c @@ -134,12 +134,26 @@ static const char *cipher_alg_name(odp_cipher_alg_t cipher) return "aes_cbc"; case ODP_CIPHER_ALG_AES_CTR: return "aes_ctr"; + case ODP_CIPHER_ALG_AES_ECB: + return "aes_ecb"; + case ODP_CIPHER_ALG_AES_CFB128: + return "aes_cfb128"; + case ODP_CIPHER_ALG_AES_XTS: + return "aes_xts"; case ODP_CIPHER_ALG_AES_GCM: return "aes_gcm"; case ODP_CIPHER_ALG_AES_CCM: return "aes_ccm"; case ODP_CIPHER_ALG_CHACHA20_POLY1305: return "chacha20_poly1305"; + case ODP_CIPHER_ALG_KASUMI_F8: + return "kasumi_f8"; + case ODP_CIPHER_ALG_SNOW3G_UEA2: + return "snow3g_uea2"; + case ODP_CIPHER_ALG_AES_EEA2: + return "aes_eea2"; + case ODP_CIPHER_ALG_ZUC_EEA3: + return "zuc_eea3"; default: return "Unknown"; } @@ -154,14 +168,14 @@ static const char *auth_alg_name(odp_auth_alg_t auth) return "md5_hmac"; case ODP_AUTH_ALG_SHA1_HMAC: return "sha1_hmac"; + case ODP_AUTH_ALG_SHA224_HMAC: + return "sha224_hmac"; case ODP_AUTH_ALG_SHA256_HMAC: return "sha256_hmac"; case ODP_AUTH_ALG_SHA384_HMAC: return "sha384_hmac"; case ODP_AUTH_ALG_SHA512_HMAC: return "sha512_hmac"; - case ODP_AUTH_ALG_AES_XCBC_MAC: - return "aes_xcbc_mac"; case ODP_AUTH_ALG_AES_GCM: return "aes_gcm"; case ODP_AUTH_ALG_AES_GMAC: @@ -170,8 +184,30 @@ static const char *auth_alg_name(odp_auth_alg_t auth) return "aes_ccm"; case ODP_AUTH_ALG_AES_CMAC: return "aes_cmac"; + case ODP_AUTH_ALG_AES_XCBC_MAC: + return "aes_xcbc_mac"; case ODP_AUTH_ALG_CHACHA20_POLY1305: return "chacha20_poly1305"; + case ODP_AUTH_ALG_KASUMI_F9: + return "kasumi_f9"; + case ODP_AUTH_ALG_SNOW3G_UIA2: + return "snow3g_uia2"; + case ODP_AUTH_ALG_AES_EIA2: + return "aes_eia2"; + case ODP_AUTH_ALG_ZUC_EIA3: + return "zuc_eia3"; + case ODP_AUTH_ALG_MD5: + return "md5"; + case ODP_AUTH_ALG_SHA1: + return "sha1"; + case ODP_AUTH_ALG_SHA224: + return "sha224"; + case ODP_AUTH_ALG_SHA256: + return "sha256"; + case ODP_AUTH_ALG_SHA384: + return "sha384"; + case ODP_AUTH_ALG_SHA512: + return "sha512"; default: return "Unknown"; } diff --git a/example/timer/odp_timer_accuracy.c b/example/timer/odp_timer_accuracy.c index 84031b9e9..0952320bb 100644 --- a/example/timer/odp_timer_accuracy.c +++ b/example/timer/odp_timer_accuracy.c @@ -233,7 +233,7 @@ static int start_timers(test_global_t *test_global) odp_timer_t timer; odp_queue_t queue; odp_queue_param_t queue_param; - uint64_t tick, start_tick; + uint64_t start_tick; uint64_t period_ns, res_ns, res_hz, start_ns, nsec, offset_ns; uint64_t max_res_ns, max_res_hz; odp_event_t event; @@ -452,17 +452,17 @@ static int start_timers(test_global_t *test_global) for (i = 0; i < num_tmo; i++) { for (j = 0; j < burst; j++) { - nsec = offset_ns + (i * period_ns) + (j * burst_gap); - tick = start_tick + odp_timer_ns_to_tick(timer_pool, - nsec); - timer_ctx_t *ctx = &test_global->timer_ctx[idx]; + odp_timer_start_t start_param; - timer = ctx->timer; - event = ctx->event; + nsec = offset_ns + (i * period_ns) + (j * burst_gap); ctx->nsec = start_ns + nsec; - ret = odp_timer_set_abs(timer, tick, &event); + start_param.tick_type = ODP_TIMER_TICK_ABS; + start_param.tick = start_tick + odp_timer_ns_to_tick(timer_pool, nsec); + start_param.tmo_ev = ctx->event; + + ret = odp_timer_start(ctx->timer, &start_param); if (ret != ODP_TIMER_SUCCESS) { printf("Timer[%" PRIu64 "] set failed: %i\n", @@ -647,6 +647,7 @@ static void run_test(test_global_t *test_global) odp_timer_pool_t tp = test_global->timer_pool; unsigned int retries = test_global->opt.early_retry; uint64_t start_ns = test_global->start_ns; + odp_timer_start_t start_param; tim = ctx->timer; @@ -659,18 +660,20 @@ static void run_test(test_global_t *test_global) nsec = ctx->nsec - start_ns; tick = test_global->start_tick + odp_timer_ns_to_tick(tp, nsec); - - ret = odp_timer_set_abs(tim, tick, &ev); + start_param.tick_type = ODP_TIMER_TICK_ABS; } else { /* Relative time */ tick = test_global->period_tick; time = odp_time_local(); time_ns = odp_time_to_ns(time); ctx->nsec = time_ns + period_ns; - - ret = odp_timer_set_rel(tim, tick, &ev); + start_param.tick_type = ODP_TIMER_TICK_REL; } + start_param.tmo_ev = ev; + start_param.tick = tick; + + ret = odp_timer_start(tim, &start_param); if (ret == ODP_TIMER_TOO_NEAR) stat->num_too_near++; else diff --git a/example/timer/odp_timer_simple.c b/example/timer/odp_timer_simple.c index f44977476..4e3120653 100644 --- a/example/timer/odp_timer_simple.c +++ b/example/timer/odp_timer_simple.c @@ -129,13 +129,19 @@ int main(int argc ODP_UNUSED, char *argv[] ODP_UNUSED) for (i = 0; i < 5; i++) { odp_time_t time; + odp_timer_start_t start_param; /* Program timeout action on current tick + period */ tick = odp_timer_current_tick(timer_pool); - rc = odp_timer_set_abs(tim, tick + period, &ev); + + start_param.tick_type = ODP_TIMER_TICK_ABS; + start_param.tick = tick + period; + start_param.tmo_ev = ev; + + rc = odp_timer_start(tim, &start_param); /* Too early or too late timeout requested */ if (odp_unlikely(rc != ODP_TIMER_SUCCESS)) - ODPH_ABORT("odp_timer_set_abs() failed: %d\n", rc); + ODPH_ABORT("odp_timer_start() failed: %d\n", rc); /* Wait for 2 seconds for timeout action to be generated */ ev = odp_schedule(&queue, sched_tmo); diff --git a/example/timer/odp_timer_test.c b/example/timer/odp_timer_test.c index 277bbf6ae..f298c9fe8 100644 --- a/example/timer/odp_timer_test.c +++ b/example/timer/odp_timer_test.c @@ -121,14 +121,19 @@ static void test_abs_timeouts(int thr, test_globals_t *gbls) int wait = 0; odp_event_t ev; odp_timer_set_t rc; + odp_timer_start_t start_param; if (ttp) { tick += period; - rc = odp_timer_set_abs(ttp->tim, tick, &ttp->ev); + + start_param.tick_type = ODP_TIMER_TICK_ABS; + start_param.tick = tick; + start_param.tmo_ev = ttp->ev; + + rc = odp_timer_start(ttp->tim, &start_param); if (odp_unlikely(rc != ODP_TIMER_SUCCESS)) { /* Too early or too late timeout requested */ - ODPH_ABORT("odp_timer_set_abs() failed: %s\n", - timerset2str(rc)); + ODPH_ABORT("odp_timer_start() failed: %s\n", timerset2str(rc)); } } diff --git a/include/Makefile.am b/include/Makefile.am index 49ccf552e..cc2f17ae2 100644 --- a/include/Makefile.am +++ b/include/Makefile.am @@ -13,6 +13,7 @@ odpapiinclude_HEADERS = \ odp/api/atomic.h \ odp/api/barrier.h \ odp/api/buffer.h \ + odp/api/buffer_types.h \ odp/api/byteorder.h \ odp/api/chksum.h \ odp/api/classification.h \ @@ -20,6 +21,7 @@ odpapiinclude_HEADERS = \ odp/api/cpu.h \ odp/api/cpumask.h \ odp/api/crypto.h \ + odp/api/crypto_types.h \ odp/api/debug.h \ odp/api/deprecated.h \ odp/api/dma.h \ @@ -77,6 +79,7 @@ odpapispecinclude_HEADERS = \ odp/api/spec/atomic.h \ odp/api/spec/barrier.h \ odp/api/spec/buffer.h \ + odp/api/spec/buffer_types.h \ odp/api/spec/byteorder.h \ odp/api/spec/chksum.h \ odp/api/spec/classification.h \ @@ -84,6 +87,7 @@ odpapispecinclude_HEADERS = \ odp/api/spec/cpu.h \ odp/api/spec/cpumask.h \ odp/api/spec/crypto.h \ + odp/api/spec/crypto_types.h \ odp/api/spec/debug.h \ odp/api/spec/dma.h \ odp/api/spec/dma_types.h \ @@ -144,12 +148,14 @@ odpapiabidefaultinclude_HEADERS = \ odp/api/abi-default/atomic.h \ odp/api/abi-default/barrier.h \ odp/api/abi-default/buffer.h \ + odp/api/abi-default/buffer_types.h \ odp/api/abi-default/byteorder.h \ odp/api/abi-default/classification.h \ odp/api/abi-default/comp.h \ odp/api/abi-default/cpu.h \ odp/api/abi-default/cpumask.h \ odp/api/abi-default/crypto.h \ + odp/api/abi-default/crypto_types.h \ odp/api/abi-default/debug.h \ odp/api/abi-default/dma_types.h \ odp/api/abi-default/errno.h \ @@ -202,12 +208,14 @@ odpapiabiarchinclude_HEADERS = \ odp/arch/arm32-linux/odp/api/abi/atomic.h \ odp/arch/arm32-linux/odp/api/abi/barrier.h \ odp/arch/arm32-linux/odp/api/abi/buffer.h \ + odp/arch/arm32-linux/odp/api/abi/buffer_types.h \ odp/arch/arm32-linux/odp/api/abi/byteorder.h \ odp/arch/arm32-linux/odp/api/abi/classification.h \ odp/arch/arm32-linux/odp/api/abi/comp.h \ odp/arch/arm32-linux/odp/api/abi/cpu.h \ odp/arch/arm32-linux/odp/api/abi/cpumask.h \ odp/arch/arm32-linux/odp/api/abi/crypto.h \ + odp/arch/arm32-linux/odp/api/abi/crypto_types.h \ odp/arch/arm32-linux/odp/api/abi/debug.h \ odp/arch/arm32-linux/odp/api/abi/dma_types.h \ odp/arch/arm32-linux/odp/api/abi/errno.h \ @@ -256,12 +264,14 @@ odpapiabiarchinclude_HEADERS = \ odp/arch/arm64-linux/odp/api/abi/atomic.h \ odp/arch/arm64-linux/odp/api/abi/barrier.h \ odp/arch/arm64-linux/odp/api/abi/buffer.h \ + odp/arch/arm64-linux/odp/api/abi/buffer_types.h \ odp/arch/arm64-linux/odp/api/abi/byteorder.h \ odp/arch/arm64-linux/odp/api/abi/classification.h \ odp/arch/arm64-linux/odp/api/abi/comp.h \ odp/arch/arm64-linux/odp/api/abi/cpu.h \ odp/arch/arm64-linux/odp/api/abi/cpumask.h \ odp/arch/arm64-linux/odp/api/abi/crypto.h \ + odp/arch/arm64-linux/odp/api/abi/crypto_types.h \ odp/arch/arm64-linux/odp/api/abi/debug.h \ odp/arch/arm64-linux/odp/api/abi/dma_types.h \ odp/arch/arm64-linux/odp/api/abi/errno.h \ @@ -310,12 +320,14 @@ odpapiabiarchinclude_HEADERS = \ odp/arch/default-linux/odp/api/abi/atomic.h \ odp/arch/default-linux/odp/api/abi/barrier.h \ odp/arch/default-linux/odp/api/abi/buffer.h \ + odp/arch/default-linux/odp/api/abi/buffer_types.h \ odp/arch/default-linux/odp/api/abi/byteorder.h \ odp/arch/default-linux/odp/api/abi/classification.h \ odp/arch/default-linux/odp/api/abi/comp.h \ odp/arch/default-linux/odp/api/abi/cpu.h \ odp/arch/default-linux/odp/api/abi/cpumask.h \ odp/arch/default-linux/odp/api/abi/crypto.h \ + odp/arch/default-linux/odp/api/abi/crypto_types.h \ odp/arch/default-linux/odp/api/abi/debug.h \ odp/arch/default-linux/odp/api/abi/dma_types.h \ odp/arch/default-linux/odp/api/abi/errno.h \ @@ -364,12 +376,14 @@ odpapiabiarchinclude_HEADERS = \ odp/arch/power64-linux/odp/api/abi/atomic.h \ odp/arch/power64-linux/odp/api/abi/barrier.h \ odp/arch/power64-linux/odp/api/abi/buffer.h \ + odp/arch/power64-linux/odp/api/abi/buffer_types.h \ odp/arch/power64-linux/odp/api/abi/byteorder.h \ odp/arch/power64-linux/odp/api/abi/classification.h \ odp/arch/power64-linux/odp/api/abi/comp.h \ odp/arch/power64-linux/odp/api/abi/cpu.h \ odp/arch/power64-linux/odp/api/abi/cpumask.h \ odp/arch/power64-linux/odp/api/abi/crypto.h \ + odp/arch/power64-linux/odp/api/abi/crypto_types.h \ odp/arch/power64-linux/odp/api/abi/debug.h \ odp/arch/power64-linux/odp/api/abi/dma_types.h \ odp/arch/power64-linux/odp/api/abi/errno.h \ @@ -418,12 +432,14 @@ odpapiabiarchinclude_HEADERS = \ odp/arch/x86_32-linux/odp/api/abi/atomic.h \ odp/arch/x86_32-linux/odp/api/abi/barrier.h \ odp/arch/x86_32-linux/odp/api/abi/buffer.h \ + odp/arch/x86_32-linux/odp/api/abi/buffer_types.h \ odp/arch/x86_32-linux/odp/api/abi/byteorder.h \ odp/arch/x86_32-linux/odp/api/abi/classification.h \ odp/arch/x86_32-linux/odp/api/abi/comp.h \ odp/arch/x86_32-linux/odp/api/abi/cpu.h \ odp/arch/x86_32-linux/odp/api/abi/cpumask.h \ odp/arch/x86_32-linux/odp/api/abi/crypto.h \ + odp/arch/x86_32-linux/odp/api/abi/crypto_types.h \ odp/arch/x86_32-linux/odp/api/abi/debug.h \ odp/arch/x86_32-linux/odp/api/abi/dma_types.h \ odp/arch/x86_32-linux/odp/api/abi/errno.h \ @@ -472,12 +488,14 @@ odpapiabiarchinclude_HEADERS = \ odp/arch/x86_64-linux/odp/api/abi/atomic.h \ odp/arch/x86_64-linux/odp/api/abi/barrier.h \ odp/arch/x86_64-linux/odp/api/abi/buffer.h \ + odp/arch/x86_64-linux/odp/api/abi/buffer_types.h \ odp/arch/x86_64-linux/odp/api/abi/byteorder.h \ odp/arch/x86_64-linux/odp/api/abi/classification.h \ odp/arch/x86_64-linux/odp/api/abi/comp.h \ odp/arch/x86_64-linux/odp/api/abi/cpu.h \ odp/arch/x86_64-linux/odp/api/abi/cpumask.h \ odp/arch/x86_64-linux/odp/api/abi/crypto.h \ + odp/arch/x86_64-linux/odp/api/abi/crypto_types.h \ odp/arch/x86_64-linux/odp/api/abi/debug.h \ odp/arch/x86_64-linux/odp/api/abi/dma_types.h \ odp/arch/x86_64-linux/odp/api/abi/errno.h \ diff --git a/include/odp/api/abi-default/buffer.h b/include/odp/api/abi-default/buffer.h index 2bc27bba7..59b81186b 100644 --- a/include/odp/api/abi-default/buffer.h +++ b/include/odp/api/abi-default/buffer.h @@ -11,20 +11,7 @@ extern "C" { #endif -/** @internal Dummy type for strong typing */ -typedef struct { char dummy; /**< @internal Dummy */ } _odp_abi_buffer_t; - -/** @ingroup odp_buffer - * @{ - */ - -typedef _odp_abi_buffer_t *odp_buffer_t; - -#define ODP_BUFFER_INVALID ((odp_buffer_t)0) - -/** - * @} - */ +/* Empty header required due to the inline functions */ #ifdef __cplusplus } diff --git a/include/odp/api/abi-default/buffer_types.h b/include/odp/api/abi-default/buffer_types.h new file mode 100644 index 000000000..f01466151 --- /dev/null +++ b/include/odp/api/abi-default/buffer_types.h @@ -0,0 +1,34 @@ +/* Copyright (c) 2017-2018, Linaro Limited + * Copyright (c) 2023, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_ABI_BUFFER_TYPES_H_ +#define ODP_ABI_BUFFER_TYPES_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +/** @internal Dummy type for strong typing */ +typedef struct { char dummy; /**< @internal Dummy */ } _odp_abi_buffer_t; + +/** @ingroup odp_buffer + * @{ + */ + +typedef _odp_abi_buffer_t *odp_buffer_t; + +#define ODP_BUFFER_INVALID ((odp_buffer_t)0) + +/** + * @} + */ + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/include/odp/api/abi-default/crypto.h b/include/odp/api/abi-default/crypto.h index e7c0a651c..0e9e8506a 100644 --- a/include/odp/api/abi-default/crypto.h +++ b/include/odp/api/abi-default/crypto.h @@ -1,4 +1,5 @@ /* Copyright (c) 2017-2018, Linaro Limited + * Copyright (c) 2022, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -11,23 +12,7 @@ extern "C" { #endif -#include <stdint.h> - -/** @internal Dummy type for strong typing */ -typedef struct { char dummy; /**< @internal Dummy */ } _odp_abi_crypto_compl_t; - -/** @ingroup odp_crypto - * @{ - */ - -#define ODP_CRYPTO_SESSION_INVALID (0xffffffffffffffffULL) - -typedef uint64_t odp_crypto_session_t; -typedef _odp_abi_crypto_compl_t *odp_crypto_compl_t; - -/** - * @} - */ +/* Empty header required due to the inline functions */ #ifdef __cplusplus } diff --git a/include/odp/api/abi-default/crypto_types.h b/include/odp/api/abi-default/crypto_types.h new file mode 100644 index 000000000..cc9155bfa --- /dev/null +++ b/include/odp/api/abi-default/crypto_types.h @@ -0,0 +1,37 @@ +/* Copyright (c) 2017-2018, Linaro Limited + * Copyright (c) 2022, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_ABI_CRYPTO_TYPES_H_ +#define ODP_ABI_CRYPTO_TYPES_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <stdint.h> + +/** @internal Dummy type for strong typing */ +typedef struct { char dummy; /**< @internal Dummy */ } _odp_abi_crypto_compl_t; + +/** @ingroup odp_crypto + * @{ + */ + +#define ODP_CRYPTO_SESSION_INVALID (0xffffffffffffffffULL) + +typedef uint64_t odp_crypto_session_t; +typedef _odp_abi_crypto_compl_t *odp_crypto_compl_t; + +/** + * @} + */ + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/include/odp/api/buffer_types.h b/include/odp/api/buffer_types.h new file mode 100644 index 000000000..767aae560 --- /dev/null +++ b/include/odp/api/buffer_types.h @@ -0,0 +1,28 @@ +/* Copyright (c) 2023, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +/** + * @file + * + * ODP buffer types + */ + +#ifndef ODP_API_BUFFER_TYPES_H_ +#define ODP_API_BUFFER_TYPES_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <odp/api/abi/buffer_types.h> + +#include <odp/api/spec/buffer_types.h> + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/include/odp/api/crypto.h b/include/odp/api/crypto.h index ec7c8370b..336304553 100644 --- a/include/odp/api/crypto.h +++ b/include/odp/api/crypto.h @@ -17,19 +17,7 @@ extern "C" { #endif -#include <odp/api/std_types.h> -#include <odp/api/abi/packet_types.h> #include <odp/api/abi/crypto.h> -#include <odp/api/abi/buffer.h> -#include <odp/api/abi/queue_types.h> - -/** @ingroup odp_crypto - * @{ - */ - -/** - * @} - */ #include <odp/api/spec/crypto.h> diff --git a/include/odp/api/crypto_types.h b/include/odp/api/crypto_types.h new file mode 100644 index 000000000..225ff7df0 --- /dev/null +++ b/include/odp/api/crypto_types.h @@ -0,0 +1,28 @@ +/* Copyright (c) 2022, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +/** + * @file + * + * ODP crypto + */ + +#ifndef ODP_API_CRYPTO_TYPES_H_ +#define ODP_API_CRYPTO_TYPES_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <odp/api/abi/crypto_types.h> + +#include <odp/api/spec/crypto_types.h> + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/include/odp/api/spec/buffer.h b/include/odp/api/spec/buffer.h index cfb85df17..dd3749afe 100644 --- a/include/odp/api/spec/buffer.h +++ b/include/odp/api/spec/buffer.h @@ -1,5 +1,5 @@ /* Copyright (c) 2013-2018, Linaro Limited - * Copyright (c) 2022, Nokia + * Copyright (c) 2022-2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -8,7 +8,7 @@ /** * @file * - * ODP buffer descriptor + * ODP buffer */ #ifndef ODP_API_SPEC_BUFFER_H_ @@ -19,6 +19,7 @@ extern "C" { #endif +#include <odp/api/buffer_types.h> #include <odp/api/event_types.h> #include <odp/api/pool_types.h> #include <odp/api/std_types.h> @@ -29,16 +30,6 @@ extern "C" { */ /** - * @typedef odp_buffer_t - * ODP buffer - */ - -/** - * @def ODP_BUFFER_INVALID - * Invalid buffer - */ - -/** * Get buffer handle from event * * Converts an ODP_EVENT_BUFFER type event to a buffer. diff --git a/include/odp/api/spec/buffer_types.h b/include/odp/api/spec/buffer_types.h new file mode 100644 index 000000000..7a7bc770f --- /dev/null +++ b/include/odp/api/spec/buffer_types.h @@ -0,0 +1,45 @@ +/* Copyright (c) 2013-2018, Linaro Limited + * Copyright (c) 2023, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +/** + * @file + * + * ODP buffer types + */ + +#ifndef ODP_API_SPEC_BUFFER_TYPES_H_ +#define ODP_API_SPEC_BUFFER_TYPES_H_ +#include <odp/visibility_begin.h> + +#ifdef __cplusplus +extern "C" { +#endif + +/** @addtogroup odp_buffer + * @{ + */ + +/** + * @typedef odp_buffer_t + * ODP buffer + */ + +/** + * @def ODP_BUFFER_INVALID + * Invalid buffer + */ + +/** + * @} + */ + +#ifdef __cplusplus +} +#endif + +#include <odp/visibility_end.h> +#endif diff --git a/include/odp/api/spec/crypto.h b/include/odp/api/spec/crypto.h index a79a05ad3..4292b78c1 100644 --- a/include/odp/api/spec/crypto.h +++ b/include/odp/api/spec/crypto.h @@ -1,5 +1,5 @@ /* Copyright (c) 2014-2018, Linaro Limited - * Copyright (c) 2021-2022, Nokia + * Copyright (c) 2021-2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -15,9 +15,9 @@ #define ODP_API_SPEC_CRYPTO_H_ #include <odp/visibility_begin.h> +#include <odp/api/crypto_types.h> #include <odp/api/deprecated.h> #include <odp/api/packet_types.h> -#include <odp/api/pool_types.h> #include <odp/api/std_types.h> #ifdef __cplusplus @@ -30,1062 +30,6 @@ extern "C" { */ /** - * @def ODP_CRYPTO_SESSION_INVALID - * Invalid session handle - */ - -/** - * @typedef odp_crypto_session_t - * Crypto API opaque session handle - */ - -/** - * @typedef odp_crypto_compl_t -* @deprecated Crypto API completion event (platform dependent). -*/ - -/** - * Crypto API operation mode - */ -typedef enum { - /** Synchronous, return results immediately */ - ODP_CRYPTO_SYNC, - /** Asynchronous, return results via posted event */ - ODP_CRYPTO_ASYNC, -} odp_crypto_op_mode_t; - -/** - * Crypto API operation type - */ -typedef enum { - /** Encrypt and/or compute authentication ICV */ - ODP_CRYPTO_OP_ENCODE, - /** Decrypt and/or verify authentication ICV */ - ODP_CRYPTO_OP_DECODE, -} odp_crypto_op_t; - -/** - * Crypto API cipher algorithm - */ -typedef enum { - /** No cipher algorithm specified */ - ODP_CIPHER_ALG_NULL, - - /** DES */ - ODP_CIPHER_ALG_DES, - - /** Triple DES with cipher block chaining */ - ODP_CIPHER_ALG_3DES_CBC, - - /** Triple DES with Electronic Codebook */ - ODP_CIPHER_ALG_3DES_ECB, - - /** AES with cipher block chaining */ - ODP_CIPHER_ALG_AES_CBC, - - /** AES with counter mode */ - ODP_CIPHER_ALG_AES_CTR, - - /** AES with electronic codebook */ - ODP_CIPHER_ALG_AES_ECB, - - /** AES with 128-bit cipher feedback */ - ODP_CIPHER_ALG_AES_CFB128, - - /** AES with XEX-based tweaked-codebook mode with ciphertext stealing - * (XTS) */ - ODP_CIPHER_ALG_AES_XTS, - - /** AES-GCM - * - * AES in Galois/Counter Mode (GCM) algorithm. GCM provides both - * authentication and ciphering of data (authenticated encryption) - * in the same operation. Hence this algorithm must be paired always - * with ODP_AUTH_ALG_AES_GCM authentication. - */ - ODP_CIPHER_ALG_AES_GCM, - - /** AES-CCM - * - * AES in Counter with CBC-MAC (CCM) mode algorithm. CCM provides both - * authentication and ciphering of data (authenticated encryption) - * in the same operation. Hence this algorithm must be paired always - * with ODP_AUTH_ALG_AES_CCM authentication. - */ - ODP_CIPHER_ALG_AES_CCM, - - /** ChaCha20-Poly1305 - * - * ChaCha20 with Poly1305 provide both authentication and ciphering of - * data (authenticated encryption) in the same operation. Hence this - * algorithm must be paired always with ODP_AUTH_ALG_CHACHA20_POLY1305 - * authentication. - */ - ODP_CIPHER_ALG_CHACHA20_POLY1305, - - /** Confidentiality F8 algorithm (UEA1) - * - * KASUMI-based F8 algorithm (also known as UEA1). - * - * IV should be formatted according to the 3GPP TS 35.201: - * COUNT || BEARER || DIRECTION || 0...0 - */ - ODP_CIPHER_ALG_KASUMI_F8, - - /** Confidentiality UEA2 algorithm (128-EEA1) - * - * SNOW 3G-based UEA2 algorithm (also known as 128-EEA1). - * - * IV (128 bit) should be formatted according to the ETSI/SAGE - * UEA2 & UIA2 specification: - * COUNT || BEARER || DIRECTION || 0...0 || - * COUNT || BEARER || DIRECTION || 0...0 || - */ - ODP_CIPHER_ALG_SNOW3G_UEA2, - - /** Confidentiality 128-EEA2 algorithm - * - * AES-CTR-based 128-EEA2 algorithm. - * - * IV (128 bit) should be formatted according to the ETSI/SAGE - * 128-EA2 & 128-EIA2 specification: - * COUNT || BEARER || - * DIRECTION || 0....0 - */ - ODP_CIPHER_ALG_AES_EEA2, - - /** ZUC based confidentiality algorithm - * - * 128-EEA3/128-NEA3 algorithm when key length is 128 bits. - * - * IV (128 bit) should be formatted according to the ETSI/SAGE - * 128-EEA3 & 128-EIA3 specification: - * COUNT || BEARER || DIRECTION || 0...0 || - * COUNT || BEARER || DIRECTION || 0...0 || - * - * 256-bit key length support is experimental and subject to - * change. The following variants may be supported: - * - * - ZUC-256 with 25 byte IV (of which 184 bits are variable) - * as specified in "The ZUC-256 Stream Cipher". - * - ZUC-256 with 16 byte IV as specified in - * "An Addendum to the ZUC-256 Stream Cipher", - * https://eprint.iacr.org/2021/1439 - */ - ODP_CIPHER_ALG_ZUC_EEA3, - -} odp_cipher_alg_t; - -/** - * Crypto API authentication algorithm - */ -typedef enum { - /** No authentication algorithm specified */ - ODP_AUTH_ALG_NULL, - - /** HMAC-MD5 - * - * MD5 algorithm in HMAC mode - */ - ODP_AUTH_ALG_MD5_HMAC, - - /** HMAC-SHA-1 - * - * SHA-1 algorithm in HMAC mode - */ - ODP_AUTH_ALG_SHA1_HMAC, - - /** HMAC-SHA-224 - * - * SHA-224 algorithm in HMAC mode - */ - ODP_AUTH_ALG_SHA224_HMAC, - - /** HMAC-SHA-256 - * - * SHA-256 algorithm in HMAC mode - */ - ODP_AUTH_ALG_SHA256_HMAC, - - /** HMAC-SHA-384 - * - * SHA-384 algorithm in HMAC mode - */ - ODP_AUTH_ALG_SHA384_HMAC, - - /** HMAC-SHA-512 - * - * SHA-512 algorithm in HMAC mode - */ - ODP_AUTH_ALG_SHA512_HMAC, - - /** AES-GCM - * - * AES in Galois/Counter Mode (GCM) algorithm. GCM provides both - * authentication and ciphering of data (authenticated encryption) - * in the same operation. Hence this algorithm must be paired always - * with ODP_CIPHER_ALG_AES_GCM cipher. - */ - ODP_AUTH_ALG_AES_GCM, - - /** AES-GMAC - * - * AES Galois Message Authentication Code (GMAC) algorithm. AES-GMAC - * is based on AES-GCM operation, but provides authentication only. - * Hence this algorithm can be paired only with ODP_CIPHER_ALG_NULL - * cipher. - * - * NIST and RFC specifications of GMAC refer to all data to be - * authenticated as AAD. In ODP the data to be authenticated, i.e. - * AAD, is ODP packet data and specified using the auth_range - * parameter. The aad_length and aad_ptr parameters, which would - * require the data to be contiguous in memory, are ignored with - * AES-GMAC. - * - * GMAC needs an initialization vector, which can be passed via - * session (auth_iv) or packet (auth_iv_ptr) level parameters. - */ - ODP_AUTH_ALG_AES_GMAC, - - /** AES-CCM - * - * AES in Counter with CBC-MAC (CCM) mode algorithm. CCM provides both - * authentication and ciphering of data (authenticated encryption) - * in the same operation. Hence this algorithm must be paired always - * with ODP_CIPHER_ALG_AES_CCM cipher. - */ - ODP_AUTH_ALG_AES_CCM, - - /** AES-CMAC - * - * AES Cipher-based Message Authentication Code (CMAC) algorithm. CMAC - * is a keyed hash function that is based on a symmetric key block - * cipher, such as the AES. - */ - ODP_AUTH_ALG_AES_CMAC, - - /** AES-XCBC-MAC - * - * AES CBC MAC for arbitrary-length messages (XCBC-MAC). - * - */ - ODP_AUTH_ALG_AES_XCBC_MAC, - - /** ChaCha20-Poly1305 AEAD - * - * ChaCha20 with Poly1305 provide both authentication and ciphering of - * data (authenticated encryption) in the same operation. Hence this - * algorithm must be paired always with - * ODP_CIPHER_ALG_CHACHA20_POLY1305 cipher. - */ - ODP_AUTH_ALG_CHACHA20_POLY1305, - - /** Integrity F9 algorithm (UIA1) - * - * KASUMI-based F9 algorithm (also known as UIA1). - * - * IV (9 bytes) is a concatenation of COUNT (32b), FRESH (32b) and - * DIRECTION (LSB-aligned, 1b). - * IV (8 bytes) is a concatenation of COUNT (32b) and FRESH (32b) - * DIRECTION (1b) and padding should come at the end of message. - */ - ODP_AUTH_ALG_KASUMI_F9, - - /** Integrity UIA2 algorithm (128-EIA1) - * - * SNOW 3G-based UIA2 algorithm (also known as 128-EIA1). - * IV (128 bit) should be formatted according to the ETSI/SAGE - * UEA2 & UIA2 specification: - * COUNT || FRESH || - * DIRECTION XOR COUNT0 || COUNT1 .. COUNT31 || - * FRESH0 .. FRESH15 || FRESH16 XOR DIRECTION || FRESH17 .. FRESH31 - */ - ODP_AUTH_ALG_SNOW3G_UIA2, - - /** Integrity 128-EIA2 algorithm - * - * AES_CMAC-based 128-EIA2 algorithm. - * - * IV (128 bit) should be formatted according to the ETSI/SAGE - * 128-EA2 & 128-EIA2 specification: - * COUNT || BEARER || - * DIRECTION || 0....0 - */ - ODP_AUTH_ALG_AES_EIA2, - - /** ZUC-based integrity algorithm. - * - * 128-EIA3/128-NIA3 algorithm when key length is 128 bits. - * - * IV (128 bit) should be formatted according to the ETSI/SAGE - * 128-EA3 & 128-EIA2 specification: - * COUNT || BEARER || - * DIRECTION XOR COUNT0 || COUNT1 .. COUNT31 || - * BEARER || 0...0 || DIRECTION || 0...0 - * - * 256-bit key length support is experimental and subject to - * change. The following variants may be supported: - * - * - ZUC-256 with 25 byte IV (of which 184 bits are variable) and - * 32/64/128 bit MAC as specified in "The ZUC-256 Stream Cipher". - * - ZUC-256 with 16 byte IV and 32/64/128 bit MAC as specified in - * "An Addendum to the ZUC-256 Stream Cipher", - * https://eprint.iacr.org/2021/1439 - */ - ODP_AUTH_ALG_ZUC_EIA3, - - /** MD5 algorithm */ - ODP_AUTH_ALG_MD5, - - /** SHA1 algorithm */ - ODP_AUTH_ALG_SHA1, - - /** 224 bit SHA2 algorithm */ - ODP_AUTH_ALG_SHA224, - - /** 256 bit SHA2 algorithm */ - ODP_AUTH_ALG_SHA256, - - /** 384 bit SHA2 algorithm */ - ODP_AUTH_ALG_SHA384, - - /** 512 bit SHA2 algorithm */ - ODP_AUTH_ALG_SHA512, - -} odp_auth_alg_t; - -/** - * Cipher algorithms in a bit field structure - */ -typedef union odp_crypto_cipher_algos_t { - /** Cipher algorithms */ - struct { - /** ODP_CIPHER_ALG_NULL */ - uint32_t null : 1; - - /** ODP_CIPHER_ALG_DES */ - uint32_t des : 1; - - /** ODP_CIPHER_ALG_3DES_CBC */ - uint32_t trides_cbc : 1; - - /** ODP_CIPHER_ALG_3DES_ECB */ - uint32_t trides_ecb : 1; - - /** ODP_CIPHER_ALG_AES_CBC */ - uint32_t aes_cbc : 1; - - /** ODP_CIPHER_ALG_AES_CTR */ - uint32_t aes_ctr : 1; - - /** ODP_CIPHER_ALG_AES_ECB */ - uint32_t aes_ecb : 1; - - /** ODP_CIPHER_ALG_AES_CFB128 */ - uint32_t aes_cfb128 : 1; - - /** ODP_CIPHER_ALG_AES_XTS */ - uint32_t aes_xts : 1; - - /** ODP_CIPHER_ALG_AES_GCM */ - uint32_t aes_gcm : 1; - - /** ODP_CIPHER_ALG_AES_CCM */ - uint32_t aes_ccm : 1; - - /** ODP_CIPHER_ALG_CHACHA20_POLY1305 */ - uint32_t chacha20_poly1305 : 1; - - /** ODP_CIPHER_ALG_KASUMI_F8 */ - uint32_t kasumi_f8 : 1; - - /** ODP_CIPHER_ALG_SNOW3G_UEA2 */ - uint32_t snow3g_uea2 : 1; - - /** ODP_CIPHER_ALG_AES_EEA2 */ - uint32_t aes_eea2 : 1; - - /** ODP_CIPHER_ALG_ZUC_EEA3 */ - uint32_t zuc_eea3 : 1; - - } bit; - - /** All bits of the bit field structure - * - * This field can be used to set/clear all flags, or bitwise - * operations over the entire structure. */ - uint32_t all_bits; -} odp_crypto_cipher_algos_t; - -/** - * Authentication algorithms in a bit field structure - */ -typedef union odp_crypto_auth_algos_t { - /** Authentication algorithms */ - struct { - /** ODP_AUTH_ALG_NULL */ - uint32_t null : 1; - - /** ODP_AUTH_ALG_MD5_HMAC */ - uint32_t md5_hmac : 1; - - /** ODP_AUTH_ALG_SHA1_HMAC */ - uint32_t sha1_hmac : 1; - - /** ODP_AUTH_ALG_SHA224_HMAC */ - uint32_t sha224_hmac : 1; - - /** ODP_AUTH_ALG_SHA256_HMAC */ - uint32_t sha256_hmac : 1; - - /** ODP_AUTH_ALG_SHA384_HMAC */ - uint32_t sha384_hmac : 1; - - /** ODP_AUTH_ALG_SHA512_HMAC */ - uint32_t sha512_hmac : 1; - - /** ODP_AUTH_ALG_AES_GCM */ - uint32_t aes_gcm : 1; - - /** ODP_AUTH_ALG_AES_GMAC*/ - uint32_t aes_gmac : 1; - - /** ODP_AUTH_ALG_AES_CCM */ - uint32_t aes_ccm : 1; - - /** ODP_AUTH_ALG_AES_CMAC*/ - uint32_t aes_cmac : 1; - - /** ODP_AUTH_ALG_AES_XCBC_MAC*/ - uint32_t aes_xcbc_mac : 1; - - /** ODP_AUTH_ALG_CHACHA20_POLY1305 */ - uint32_t chacha20_poly1305 : 1; - - /** ODP_AUTH_ALG_KASUMI_F9 */ - uint32_t kasumi_f9 : 1; - - /** ODP_AUTH_ALG_SNOW3G_UIA2 */ - uint32_t snow3g_uia2 : 1; - - /** ODP_AUTH_ALG_AES_EIA2 */ - uint32_t aes_eia2 : 1; - - /** ODP_AUTH_ALG_ZUC_EIA3 */ - uint32_t zuc_eia3 : 1; - - /** ODP_AUTH_ALG_MD5 */ - uint32_t md5 : 1; - - /** ODP_AUTH_ALG_SHA1 */ - uint32_t sha1 : 1; - - /** ODP_AUTH_ALG_SHA224 */ - uint32_t sha224 : 1; - - /** ODP_AUTH_ALG_SHA256 */ - uint32_t sha256 : 1; - - /** ODP_AUTH_ALG_SHA384 */ - uint32_t sha384 : 1; - - /** ODP_AUTH_ALG_SHA512 */ - uint32_t sha512 : 1; - - } bit; - - /** All bits of the bit field structure - * - * This field can be used to set/clear all flags, or bitwise - * operations over the entire structure. */ - uint32_t all_bits; -} odp_crypto_auth_algos_t; - -/** - * Crypto API key structure - */ -typedef struct odp_crypto_key { - /** Key data */ - uint8_t *data; - - /** Key length in bytes */ - uint32_t length; - -} odp_crypto_key_t; - -/** - * Crypto API IV structure - * - * @deprecated Use per-packet IV in crypto operation parameters - */ -typedef struct odp_crypto_iv { - /** IV data - * - * Ignored when length is zero. Null value indicates that an - * IV will be provided for each packet through the crypto - * operation parameters. In that case the per-operation - * IV parameter must always point to a valid IV. - * - * Default value is NULL. - */ - uint8_t *data; - - /** IV length in bytes. Default value is zero. */ - uint32_t length; - -} ODP_DEPRECATE(odp_crypto_iv_t); - -/** - * Crypto API session creation parameters - */ -typedef struct odp_crypto_session_param_t { - /** Encode vs. decode operation - * - * The default value is ODP_CRYPTO_OP_ENCODE. - */ - odp_crypto_op_t op; - - /** Authenticate cipher vs. plain text - * - * Controls ordering of authentication and cipher operations, - * and is relative to the operation (encode vs decode). When encoding, - * TRUE indicates the authentication operation should be performed - * after the cipher operation else before. When decoding, TRUE - * indicates the reverse order of operation. - * - * The value is ignored with authenticated encryption algorithms - * such as AES-GCM. - * - * true: Authenticate cipher text - * false: Authenticate plain text - * - * The default value is false. - */ - odp_bool_t auth_cipher_text; - - /** Hash result location may overlap authentication range - * - * This flag indicates that the hash result location may (but is - * not required to) overlap authentication range. Setting this - * flag may reduce performance. - * - * Default value is false. - */ - odp_bool_t hash_result_in_auth_range; - - /** Preferred sync vs. async for odp_crypto_operation() - * - * The default value is ODP_CRYPTO_SYNC. - * - * @deprecated Used only with deprecated odp_crypto_operation() - */ - odp_crypto_op_mode_t ODP_DEPRECATE(pref_mode); - - /** Operation mode when using packet interface: sync or async - * - * The default value is ODP_CRYPTO_SYNC. - */ - odp_crypto_op_mode_t op_mode; - - /** Cipher algorithm - * - * Select cipher algorithm to be used. ODP_CIPHER_ALG_NULL indicates - * that ciphering is disabled. Use odp_crypto_capability() for - * supported algorithms. Note that some algorithms restrict choice of - * the pairing authentication algorithm. When ciphering is enabled - * cipher key and IV need to be set. The default value is - * ODP_CIPHER_ALG_NULL. - */ - odp_cipher_alg_t cipher_alg; - - /** Cipher key - * - * Use odp_crypto_cipher_capa() for supported key and IV lengths. - */ - odp_crypto_key_t cipher_key; - - /** Cipher Initialization Vector (IV) - * - * Unless using the deprecated API, this specifies the length of - * the IV only. The actual IV must then be provided in per-packet - * parameters of crypto operations. - */ - union { -#if ODP_DEPRECATED_API - /** @deprecated Cipher IV */ - odp_crypto_iv_t ODP_DEPRECATE(cipher_iv); -#endif - /** Cipher IV length */ - struct { -#if ODP_DEPRECATED_API - /** @cond - * Unused padding field - */ - uint8_t *dummy_padding_0; - /** @endcond */ -#endif - /** Length of cipher initialization vector. - * Default value is zero. - */ - uint32_t cipher_iv_len; - }; - }; - - /** Authentication algorithm - * - * Select authentication algorithm to be used. ODP_AUTH_ALG_NULL - * indicates that authentication is disabled. Use - * odp_crypto_capability() for supported algorithms. Note that some - * algorithms restrict choice of the pairing cipher algorithm. When - * single algorithm provides both ciphering and authentication - * (i.e. Authenticated Encryption), authentication side key - * (auth_key) and IV (auth_iv) are ignored, and cipher side values are - * used instead. These algorithms ignore authentication side key - * and IV: ODP_AUTH_ALG_AES_GCM, ODP_AUTH_ALG_AES_CCM and - * ODP_AUTH_ALG_CHACHA20_POLY1305. Otherwise, all authentication side - * parameters must be set when authentication is enabled. The default - * value is ODP_AUTH_ALG_NULL. - */ - odp_auth_alg_t auth_alg; - - /** Authentication key - * - * Use odp_crypto_auth_capability() for supported key lengths. - */ - odp_crypto_key_t auth_key; - - /** Authentication Initialization Vector (IV) - * - * Unless using the deprecated API, this specifies the length of - * the IV only. The actual IV must then be provided in per-packet - * parameters of crypto operations. - */ - union { -#if ODP_DEPRECATED_API - /** @deprecated Authentication IV */ - odp_crypto_iv_t ODP_DEPRECATE(auth_iv); -#endif - /** Authentication IV length */ - struct { -#if ODP_DEPRECATED_API - /** @cond - * Unused padding field - */ - uint8_t *dummy_padding_1; - /** @endcond */ -#endif - /** Length of authentication initialization vector. - * Default value is zero. - */ - uint32_t auth_iv_len; - }; - }; - - /** Authentication digest length in bytes - * - * Use odp_crypto_auth_capability() for supported digest lengths. - */ - uint32_t auth_digest_len; - - /** Additional Authenticated Data (AAD) length in bytes - * - * AAD length is constant for all operations (packets) of the session. - * Set to zero when AAD is not used. Use odp_crypto_auth_capability() - * for supported AAD lengths. The default value is zero. - */ - uint32_t auth_aad_len; - - /** Async mode completion event queue - * - * The completion queue is used to return completions from - * odp_crypto_op_enq() (and the deprecated odp_crypto_operation()) - * to the application. - */ - odp_queue_t compl_queue; - - /** Output pool - * - * When the output packet is not specified during the call to - * crypto operation, the output packet will be allocated - * from this pool. - */ - odp_pool_t output_pool; - -} odp_crypto_session_param_t; - -/** - * Crypto API per packet operation parameters - * - * @deprecated Use odp_crypto_packet_op_param_t instead. - */ -typedef struct odp_crypto_op_param_t { - /** Session handle from creation */ - odp_crypto_session_t session; - - /** User context */ - void *ctx; - - /** Input packet - * - * Specifies the input packet for the crypto operation. When the - * 'out_pkt' variable is set to ODP_PACKET_INVALID (indicating a new - * packet should be allocated for the resulting packet). - */ - odp_packet_t pkt; - - /** Output packet - * - * Both "in place" (the original packet 'pkt' is modified) and - * "copy" (the packet is replicated to a new packet which contains - * the modified data) modes are supported. The "in place" mode of - * operation is indicated by setting 'out_pkt' equal to 'pkt'. - * For the copy mode of operation, setting 'out_pkt' to a valid packet - * value indicates the caller wishes to specify the destination packet. - * Setting 'out_pkt' to ODP_PACKET_INVALID indicates the caller wishes - * the destination packet be allocated from the output pool specified - * during session creation. - */ - odp_packet_t out_pkt; - - /** IV pointer for cipher */ - uint8_t *cipher_iv_ptr; - - /** Authentication IV pointer */ - uint8_t *auth_iv_ptr; - - /** Offset from start of packet for hash result - * - * In case of decode sessions, the expected hash will be read from - * this offset and compared with the calculated hash. After the - * operation the hash bytes will have undefined values. - * - * In case of encode sessions the calculated hash will be stored in - * this offset. - * - * If the hash_result_in_auth_range session parameter is true, - * the hash result location may overlap auth_range. In that case - * the result location will be zeroed in decode sessions before - * hash calculation. Zeroing is not done in encode sessions. - */ - uint32_t hash_result_offset; - - /** Pointer to AAD. AAD length is defined by 'auth_aad_len' - * session parameter. - */ - uint8_t *aad_ptr; - - /** Data range to be ciphered */ - odp_packet_data_range_t cipher_range; - - /** Data range to be authenticated - * - * The value is ignored with authenticated encryption algorithms, - * such as AES-GCM, which authenticate data in the cipher range - * and the AAD. - * - * As a special case AES-GMAC uses this field instead of aad_ptr - * for the data bytes to be authenticated. - */ - odp_packet_data_range_t auth_range; - -} ODP_DEPRECATE(odp_crypto_op_param_t); - -/** - * Crypto packet API per packet operation parameters - */ -typedef struct odp_crypto_packet_op_param_t { - /** Session handle from creation */ - odp_crypto_session_t session; - - /** IV pointer for cipher */ - uint8_t *cipher_iv_ptr; - - /** IV pointer for authentication */ - uint8_t *auth_iv_ptr; - - /** Offset from start of packet for hash result - * - * In case of decode sessions, the expected hash will be read from - * this offset and compared with the calculated hash. After the - * operation the hash bytes will have undefined values. - * - * In case of encode sessions the calculated hash will be stored in - * this offset. - * - * If the hash_result_in_auth_range session parameter is true, - * the hash result location may overlap auth_range. In that case the - * result location will be zeroed in decode sessions before hash - * calculation. Zeroing is not done in encode sessions. - */ - uint32_t hash_result_offset; - - /** Pointer to AAD. AAD length is defined by 'auth_aad_len' - * session parameter. - */ - uint8_t *aad_ptr; - - /** Data range to apply cipher */ - odp_packet_data_range_t cipher_range; - - /** Data range to authenticate */ - odp_packet_data_range_t auth_range; - -} odp_crypto_packet_op_param_t; - -/** - * Crypto API session creation return code - */ -typedef enum { - /** Session created */ - ODP_CRYPTO_SES_ERR_NONE, - /** Creation failed, no resources */ - ODP_CRYPTO_SES_ERR_ENOMEM, - /** Creation failed, bad cipher params */ - ODP_CRYPTO_SES_ERR_CIPHER, - /** Creation failed, bad auth params */ - ODP_CRYPTO_SES_ERR_AUTH, - - /** Unsupported combination of algorithms - * - * The combination of cipher and auth algorithms with their - * specific parameters is not supported even if the algorithms - * appear in capabilities and are supported in combination with - * other algorithms or other algorithm specific parameters. - */ - ODP_CRYPTO_SES_ERR_ALG_COMBO, - - /** Unsupported order of cipher and auth - * - * The requested mutual order of ciphering and authentication - * is not supported with the chosen individual cipher and - * authentication algorithms. - */ - ODP_CRYPTO_SES_ERR_ALG_ORDER, - - /** Unsupported combination of session creation parameters - * - * The combination of provided session creation parameters is not - * supported. This error can occur when there are limitations that - * are not expressible through crypto capabilities or other error - * status values. - */ - ODP_CRYPTO_SES_ERR_PARAMS, -} odp_crypto_ses_create_err_t; - -/** This synonym for backward compatibility will be deprecated later */ -#define ODP_CRYPTO_SES_CREATE_ERR_NONE ODP_CRYPTO_SES_ERR_NONE -/** This synonym for backward compatibility will be deprecated later */ -#define ODP_CRYPTO_SES_CREATE_ERR_ENOMEM ODP_CRYPTO_SES_ERR_ENOMEM -/** This synonym for backward compatibility will be deprecated later */ -#define ODP_CRYPTO_SES_CREATE_ERR_INV_CIPHER ODP_CRYPTO_SES_ERR_CIPHER -/** This synonym for backward compatibility will be deprecated later */ -#define ODP_CRYPTO_SES_CREATE_ERR_INV_AUTH ODP_CRYPTO_SES_ERR_AUTH - -/** - * Crypto API algorithm return code - */ -typedef enum { - /** Algorithm successful */ - ODP_CRYPTO_ALG_ERR_NONE, - /** Invalid data block size */ - ODP_CRYPTO_ALG_ERR_DATA_SIZE, - /** Key size invalid for algorithm */ - ODP_CRYPTO_ALG_ERR_KEY_SIZE, - /** Computed ICV value mismatch */ - ODP_CRYPTO_ALG_ERR_ICV_CHECK, - /** IV value not specified */ - ODP_CRYPTO_ALG_ERR_IV_INVALID, -} odp_crypto_alg_err_t; - -/** - * Crypto API hardware centric return code - */ -typedef enum { - /** Operation completed successfully */ - ODP_CRYPTO_HW_ERR_NONE, - /** Error detected during DMA of data */ - ODP_CRYPTO_HW_ERR_DMA, - /** Operation failed due to pool depletion */ - ODP_CRYPTO_HW_ERR_BP_DEPLETED, -} odp_crypto_hw_err_t; - -/** - * Cryto API per packet operation completion status - */ -typedef struct odp_crypto_op_status { - /** Algorithm specific return code */ - odp_crypto_alg_err_t alg_err; - - /** Hardware specific return code */ - odp_crypto_hw_err_t hw_err; - -} odp_crypto_op_status_t; - -/** - * Crypto API operation result - * - * @deprecated Use odp_crypto_packet_result_t instead. - */ -typedef struct odp_crypto_op_result { - /** Request completed successfully */ - odp_bool_t ok; - - /** User context from request */ - void *ctx; - - /** Output packet */ - odp_packet_t pkt; - - /** Cipher status */ - odp_crypto_op_status_t cipher_status; - - /** Authentication status */ - odp_crypto_op_status_t auth_status; - -} ODP_DEPRECATE(odp_crypto_op_result_t); - -/** - * Crypto packet API operation result - */ -typedef struct odp_crypto_packet_result_t { - /** Request completed successfully */ - odp_bool_t ok; - - /** Cipher status */ - odp_crypto_op_status_t cipher_status; - - /** Authentication status */ - odp_crypto_op_status_t auth_status; - -} odp_crypto_packet_result_t; - -/** - * Crypto capabilities - */ -typedef struct odp_crypto_capability_t { - /** Maximum number of crypto sessions */ - uint32_t max_sessions; - - /** Supported packet operation in SYNC mode */ - odp_support_t sync_mode; - - /** Supported packet operation in ASYNC mode */ - odp_support_t async_mode; - - /** Supported cipher algorithms */ - odp_crypto_cipher_algos_t ciphers; - - /** Cipher algorithms implemented with HW offload */ - odp_crypto_cipher_algos_t hw_ciphers; - - /** Supported authentication algorithms */ - odp_crypto_auth_algos_t auths; - - /** Authentication algorithms implemented with HW offload */ - odp_crypto_auth_algos_t hw_auths; - - /** - * Scheduled crypto completion queue support - * - * This defines whether scheduled queues are supported as crypto - * compl_queue. - * 0: Scheduled queues are not supported as crypto completion queues - * 1: Scheduled queues are supported as crypto completion queues - * @see odp_crypto_session_param_t - */ - odp_bool_t queue_type_sched; - - /** - * Plain crypto completion queue support - * - * This defines whether plain queues are supported as crypto - * compl_queue. - * 0: Plain queues are not supported as crypto completion queues - * 1: Plain queues are supported as crypto completion queues - * @see odp_crypto_session_param_t - */ - odp_bool_t queue_type_plain; -} odp_crypto_capability_t; - -/** - * Cipher algorithm capabilities - */ -typedef struct odp_crypto_cipher_capability_t { - /** Key length in bytes */ - uint32_t key_len; - - /** IV length in bytes */ - uint32_t iv_len; - - /** Cipher is operating in bitwise mode - * - * This cipher works on series of bits, rather than sequences of bytes: - * cipher_range in odp_crypto_op_param_t and - * odp_crypto_packet_op_param_t will use bits, rather than bytes. - * - * Note: data buffer MUST start on the byte boundary, using offset - * which is not divisible by 8 is unsupported and will result in - * unspecified behaviour. - * - * Note2: If the data length is not a multiple of 8, the remaining - * bits of the data in the last byte of the input/output will be the - * most significant bits, i.e. the most significant bit is considered - * to be the first bit of a byte for the purpose of input and output - * data range. The output bits that fall out of the output range are - * undefined. - */ - odp_bool_t bit_mode; - -} odp_crypto_cipher_capability_t; - -/** - * Authentication algorithm capabilities - */ -typedef struct odp_crypto_auth_capability_t { - /** Digest length in bytes */ - uint32_t digest_len; - - /** Key length in bytes */ - uint32_t key_len; - - /** IV length in bytes */ - uint32_t iv_len; - - /** Additional Authenticated Data (AAD) lengths */ - struct { - /** Minimum AAD length in bytes */ - uint32_t min; - - /** Maximum AAD length in bytes */ - uint32_t max; - - /** Increment of supported lengths between min and max - * (in bytes) */ - uint32_t inc; - } aad_len; - - /** Auth is operating in bitstring mode - * - * This auth works on series of bits, rather than sequences of bytes: - * auth_range in odp_crypto_op_param_t and - * odp_crypto_packet_op_param_t will use bits, rather than bytes. - * - * Note: data buffer MUST start on the byte boundary, using offset - * which is not divisible by 8 is unsupported and will result in - * unpredictable behaviour. - * - * Note2: If the data length is not a multiple of 8, the remaining - * bits of the data in the last byte of the input/output will be the - * most significant bits, i.e. the most significant bit is considered - * to be the first bit of a byte for the purpose of input and output - * data range. The output bits that fall out of the output range are - * undefined. - */ - odp_bool_t bit_mode; - -} odp_crypto_auth_capability_t; - -/** * Query crypto capabilities * * Outputs crypto capabilities on success. @@ -1212,6 +156,9 @@ void odp_crypto_compl_free(odp_crypto_compl_t completion_event); * * @deprecated Use odp_crypto_op() or odp_crypto_op_enq() instead. * + * This function may be called only for sessions configured with + * the ODP_CRYPTO_OP_TYPE_LEGACY operation type. + * * Performs the cryptographic operations specified during session creation * on the packet. If the operation is performed synchronously, "posted" * will return FALSE and the result of the operation is immediately available. @@ -1310,15 +257,15 @@ odp_packet_t odp_crypto_packet_from_event(odp_event_t ev); odp_event_t odp_crypto_packet_to_event(odp_packet_t pkt); /** - * Get crypto operation results from an crypto processed packet + * Get crypto operation results from a crypto processed packet * * Successful crypto operations of all types (SYNC and ASYNC) produce packets * which contain crypto result metadata. This function copies the operation - * results from an crypto processed packet. Event subtype of this kind of + * results from a crypto processed packet. Event subtype of this kind of * packet is ODP_EVENT_PACKET_CRYPTO. Results are undefined if a non-crypto * processed packet is passed as input. * - * @param packet An crypto processed packet (ODP_EVENT_PACKET_CRYPTO) + * @param packet A crypto processed packet (ODP_EVENT_PACKET_CRYPTO) * @param[out] result Pointer to operation result for output * * @retval 0 On success @@ -1331,16 +278,77 @@ int odp_crypto_result(odp_crypto_packet_result_t *result, * Crypto packet operation * * Performs the SYNC cryptographic operations specified during session creation - * on the packets. Caller should initialize pkt_out either with desired output - * packet handles or with ODP_PACKET_INVALID to make ODP allocate new packets - * from provided pool. All arrays should be of num_pkt size. + * on the packets. All arrays should be of num_pkt size. + * + * Use of the pkt_out parameter depends on the configured crypto operation + * type as described below. + * + * ODP_CRYPTO_OP_TYPE_LEGACY: + * + * Caller should initialize each element of pkt_out either with the desired + * output packet handle or with ODP_PACKET_INVALID to make ODP allocate a new + * packet from provided pool. + * + * All packet data and metadata are copied from the input packet to the output + * packet before the requested crypto operation is performed to the output + * packet. If an output packet is given to the operation, it must be at least + * as long as the input packet and, in encode operations, long enough for the + * hash result to be fully inside the packet data. Memory layout of the output + * packet may change during the crypto operation. If the output packet is + * longer than needed, it is not truncated and the extra data bytes retain + * their content. + * + * It is ok to pass the same packet handle as both the input packet and the + * output packet for the same crypto operation. In that case the input packet + * is consumed but returned as the output packet (with possibly different + * memory layout). + * + * ODP_CRYPTO_OP_TYPE_BASIC: + * + * ODP allocates the output packet from the pool from which the input + * packet was allocated. The processed input packet is consumed. All + * packet data and metadata are copied from the input packet to the output + * packet before the requested crypto operation is applied to the output + * packet. Memory layout (including packet data pointers, head and tail room, + * segmentation) of the output packet may differ from that of the input + * packet. + * + * The value of pktout[n] is ignored as pktout[n] is used purely as an + * output parameter that returns the handle of the newly allocated packet. + * + * ODP_CRYPTO_OP_TYPE_OOP: + * + * Writes the output bytes of the crypto operation in a caller provided + * output packet passed through pkt_out[n]. Input packets are not consumed + * nor modified. Memory layout (including packet data pointers, head and + * tail room, segmentation) of the output packet may change during the + * operation. + * + * Crypto output is the processed crypto_range, auth_range and + * MAC/digest (in encode sessions) of the input packet. The operation + * behaves as if crypto range and auth range were first copied from the + * input packet to the output packet and then the crypto operation + * was applied to the output packet. + * + * Crypto range and auth range of null cipher and auth algorithms are + * ignored, i.e. not copied in the output packet. Auth range of (AEAD) + * algorithms that ignore auth range is not copied. + * + * The offset of the crypto range and auth range in the output packet is + * the same as in the input packet, adjusted by dst_offset_shift operation + * parameter. + * + * pkt_out[n] must be a valid handle to a packet that is long enough to + * contain the shifted crypto range, auth range and, in encode sessions, + * the MAC/digest result. pkt_out[n] must not be the same as any input + * packet or any other output packet. * * @param pkt_in Packets to be processed - * @param[in,out] pkt_out Packet handle array specifyint resulting packets + * @param[in,out] pkt_out Packet handle array for resulting packets * @param param Operation parameters array * @param num_pkt Number of packets to be processed * - * @return Number of input packets consumed (0 ... num_pkt) + * @return Number of input packets processed (0 ... num_pkt) * @retval <0 on failure */ int odp_crypto_op(const odp_packet_t pkt_in[], @@ -1351,14 +359,24 @@ int odp_crypto_op(const odp_packet_t pkt_in[], /** * Crypto packet operation * - * Performs the ASYNC cryptographic operations specified during session creation - * on the packets. Caller should initialize pkt_out either with desired output - * packet handles or with ODP_PACKET_INVALID to make ODP allocate new packets - * from provided pool. All arrays should be of num_pkt size. Resulting packets - * are returned through events. + * Performs the ASYNC cryptographic operations specified during session + * creation on the packets. Behaves otherwise like odp_crypto_op() but + * returns output packets through events. + * + * With operation types other than ODP_CRYPTO_OP_TYPE_LEGACY, packet + * data of processed packets may not be valid before odp_crypto_result() + * has been called. + * + * With ODP_CRYPTO_OP_TYPE_OOP, an enqueued input packet is consumed but + * returned back unmodified after the crypto operation is complete. The + * caller may not access the input packet until getting the handle back + * through odp_crypto_result(). + * + * All arrays should be of num_pkt size, except that pkt_out parameter + * is ignored when the crypto operation type is ODP_CRYPTO_OP_TYPE_BASIC. * * @param pkt_in Packets to be processed - * @param pkt_out Packet handle array specifying resulting packets + * @param pkt_out Packet handle array for resulting packets * @param param Operation parameters array * @param num_pkt Number of packets to be processed * diff --git a/include/odp/api/spec/crypto_types.h b/include/odp/api/spec/crypto_types.h new file mode 100644 index 000000000..574f56445 --- /dev/null +++ b/include/odp/api/spec/crypto_types.h @@ -0,0 +1,1102 @@ +/* Copyright (c) 2014-2018, Linaro Limited + * Copyright (c) 2021-2022, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +/** + * @file + * + * ODP crypto types */ + +#ifndef ODP_API_SPEC_CRYPTO_TYPES_H_ +#define ODP_API_SPEC_CRYPTO_TYPES_H_ +#include <odp/visibility_begin.h> + +#include <odp/api/deprecated.h> +#include <odp/api/packet_types.h> +#include <odp/api/pool_types.h> +#include <odp/api/std_types.h> + +#ifdef __cplusplus +extern "C" { +#endif + +/** @addtogroup odp_crypto + * @{ + */ + +/** + * @def ODP_CRYPTO_SESSION_INVALID + * Invalid session handle + */ + +/** + * @typedef odp_crypto_session_t + * Crypto API opaque session handle + */ + +/** + * @typedef odp_crypto_compl_t +* @deprecated Crypto API completion event (platform dependent). +*/ + +/** + * Crypto API operation mode + */ +typedef enum { + /** Synchronous, return results immediately */ + ODP_CRYPTO_SYNC, + /** Asynchronous, return results via posted event */ + ODP_CRYPTO_ASYNC, +} odp_crypto_op_mode_t; + +/** + * Crypto API operation + */ +typedef enum { + /** Encrypt and/or compute authentication ICV */ + ODP_CRYPTO_OP_ENCODE, + /** Decrypt and/or verify authentication ICV */ + ODP_CRYPTO_OP_DECODE, +} odp_crypto_op_t; + +/** + * Crypto API cipher algorithm + */ +typedef enum { + /** No cipher algorithm specified */ + ODP_CIPHER_ALG_NULL, + + /** DES */ + ODP_CIPHER_ALG_DES, + + /** Triple DES with cipher block chaining */ + ODP_CIPHER_ALG_3DES_CBC, + + /** Triple DES with Electronic Codebook */ + ODP_CIPHER_ALG_3DES_ECB, + + /** AES with cipher block chaining */ + ODP_CIPHER_ALG_AES_CBC, + + /** AES with counter mode */ + ODP_CIPHER_ALG_AES_CTR, + + /** AES with electronic codebook */ + ODP_CIPHER_ALG_AES_ECB, + + /** AES with 128-bit cipher feedback */ + ODP_CIPHER_ALG_AES_CFB128, + + /** AES with XEX-based tweaked-codebook mode with ciphertext stealing + * (XTS) */ + ODP_CIPHER_ALG_AES_XTS, + + /** AES-GCM + * + * AES in Galois/Counter Mode (GCM) algorithm. GCM provides both + * authentication and ciphering of data (authenticated encryption) + * in the same operation. Hence this algorithm must be paired always + * with ODP_AUTH_ALG_AES_GCM authentication. + */ + ODP_CIPHER_ALG_AES_GCM, + + /** AES-CCM + * + * AES in Counter with CBC-MAC (CCM) mode algorithm. CCM provides both + * authentication and ciphering of data (authenticated encryption) + * in the same operation. Hence this algorithm must be paired always + * with ODP_AUTH_ALG_AES_CCM authentication. + */ + ODP_CIPHER_ALG_AES_CCM, + + /** ChaCha20-Poly1305 + * + * ChaCha20 with Poly1305 provide both authentication and ciphering of + * data (authenticated encryption) in the same operation. Hence this + * algorithm must be paired always with ODP_AUTH_ALG_CHACHA20_POLY1305 + * authentication. + */ + ODP_CIPHER_ALG_CHACHA20_POLY1305, + + /** Confidentiality F8 algorithm (UEA1) + * + * KASUMI-based F8 algorithm (also known as UEA1). + * + * IV should be formatted according to the 3GPP TS 35.201: + * COUNT || BEARER || DIRECTION || 0...0 + */ + ODP_CIPHER_ALG_KASUMI_F8, + + /** Confidentiality UEA2 algorithm (128-EEA1) + * + * SNOW 3G-based UEA2 algorithm (also known as 128-EEA1). + * + * IV (128 bits) should be formatted according to the ETSI/SAGE + * UEA2 & UIA2 specification: + * COUNT || BEARER || DIRECTION || 0...0 || + * COUNT || BEARER || DIRECTION || 0...0 || + */ + ODP_CIPHER_ALG_SNOW3G_UEA2, + + /** Confidentiality 128-EEA2 algorithm + * + * AES-CTR-based 128-EEA2 algorithm. + * + * IV (128 bits) should be formatted according to the ETSI/SAGE + * 128-EAA2 & 128-EIA2 specification: + * COUNT || BEARER || DIRECTION || 0....0 + */ + ODP_CIPHER_ALG_AES_EEA2, + + /** ZUC based confidentiality algorithm + * + * 128-EEA3/128-NEA3 algorithm when key length is 128 bits. + * + * IV (128 bits) should be formatted according to the ETSI/SAGE + * 128-EEA3 & 128-EIA3 specification: + * COUNT || BEARER || DIRECTION || 0...0 || + * COUNT || BEARER || DIRECTION || 0...0 || + * + * 256-bit key length support is experimental and subject to + * change. The following variants may be supported: + * + * - ZUC-256 with 25 byte IV (of which 184 bits are variable) + * as specified in "The ZUC-256 Stream Cipher". + * - ZUC-256 with 16 byte IV as specified in + * "An Addendum to the ZUC-256 Stream Cipher", + * https://eprint.iacr.org/2021/1439 + */ + ODP_CIPHER_ALG_ZUC_EEA3, + +} odp_cipher_alg_t; + +/** + * Crypto API authentication algorithm + */ +typedef enum { + /** No authentication algorithm specified */ + ODP_AUTH_ALG_NULL, + + /** HMAC-MD5 + * + * MD5 algorithm in HMAC mode + */ + ODP_AUTH_ALG_MD5_HMAC, + + /** HMAC-SHA-1 + * + * SHA-1 algorithm in HMAC mode + */ + ODP_AUTH_ALG_SHA1_HMAC, + + /** HMAC-SHA-224 + * + * SHA-224 algorithm in HMAC mode + */ + ODP_AUTH_ALG_SHA224_HMAC, + + /** HMAC-SHA-256 + * + * SHA-256 algorithm in HMAC mode + */ + ODP_AUTH_ALG_SHA256_HMAC, + + /** HMAC-SHA-384 + * + * SHA-384 algorithm in HMAC mode + */ + ODP_AUTH_ALG_SHA384_HMAC, + + /** HMAC-SHA-512 + * + * SHA-512 algorithm in HMAC mode + */ + ODP_AUTH_ALG_SHA512_HMAC, + + /** AES-GCM + * + * AES in Galois/Counter Mode (GCM) algorithm. GCM provides both + * authentication and ciphering of data (authenticated encryption) + * in the same operation. Hence this algorithm must be paired always + * with ODP_CIPHER_ALG_AES_GCM cipher. + */ + ODP_AUTH_ALG_AES_GCM, + + /** AES-GMAC + * + * AES Galois Message Authentication Code (GMAC) algorithm. AES-GMAC + * is based on AES-GCM operation, but provides authentication only. + * Hence this algorithm can be paired only with ODP_CIPHER_ALG_NULL + * cipher. + * + * NIST and RFC specifications of GMAC refer to all data to be + * authenticated as AAD. In ODP the data to be authenticated, i.e. + * AAD, is ODP packet data and specified using the auth_range + * parameter. The aad_length and aad_ptr parameters, which would + * require the data to be contiguous in memory, are ignored with + * AES-GMAC. + * + * GMAC needs an initialization vector, which must be passed via + * operation parameters (auth_iv_ptr). + */ + ODP_AUTH_ALG_AES_GMAC, + + /** AES-CCM + * + * AES in Counter with CBC-MAC (CCM) mode algorithm. CCM provides both + * authentication and ciphering of data (authenticated encryption) + * in the same operation. Hence this algorithm must be paired always + * with ODP_CIPHER_ALG_AES_CCM cipher. + */ + ODP_AUTH_ALG_AES_CCM, + + /** AES-CMAC + * + * AES Cipher-based Message Authentication Code (CMAC) algorithm. CMAC + * is a keyed hash function that is based on a symmetric key block + * cipher, such as the AES. + */ + ODP_AUTH_ALG_AES_CMAC, + + /** AES-XCBC-MAC + * + * AES CBC MAC for arbitrary-length messages (XCBC-MAC). + * + */ + ODP_AUTH_ALG_AES_XCBC_MAC, + + /** ChaCha20-Poly1305 AEAD + * + * ChaCha20 with Poly1305 provide both authentication and ciphering of + * data (authenticated encryption) in the same operation. Hence this + * algorithm must be paired always with + * ODP_CIPHER_ALG_CHACHA20_POLY1305 cipher. + */ + ODP_AUTH_ALG_CHACHA20_POLY1305, + + /** Integrity F9 algorithm (UIA1) + * + * KASUMI-based F9 algorithm (also known as UIA1). + * + * IV (9 bytes) is a concatenation of COUNT (32b), FRESH (32b) and + * DIRECTION (LSB-aligned, 1b). + * IV (8 bytes) is a concatenation of COUNT (32b) and FRESH (32b) + * DIRECTION (1b) and padding should come at the end of message. + */ + ODP_AUTH_ALG_KASUMI_F9, + + /** Integrity UIA2 algorithm (128-EIA1) + * + * SNOW 3G-based UIA2 algorithm (also known as 128-EIA1). + * IV (128 bits) should be formatted according to the ETSI/SAGE + * UEA2 & UIA2 specification: + * COUNT || FRESH || + * DIRECTION XOR COUNT0 || COUNT1 .. COUNT31 || + * FRESH0 .. FRESH15 || FRESH16 XOR DIRECTION || FRESH17 .. FRESH31 + */ + ODP_AUTH_ALG_SNOW3G_UIA2, + + /** Integrity 128-EIA2 algorithm + * + * AES_CMAC-based 128-EIA2 algorithm. + * + * IV (64 bits) should be formatted according to the ETSI/SAGE + * 128-EEA2 & 128-EIA2 specification: + * COUNT || BEARER || DIRECTION || 0....0 + */ + ODP_AUTH_ALG_AES_EIA2, + + /** ZUC-based integrity algorithm. + * + * 128-EIA3/128-NIA3 algorithm when key length is 128 bits. + * + * IV (128 bits) should be formatted according to the ETSI/SAGE + * 128-EEA3 & 128-EIA2 specification: + * COUNT || BEARER || + * DIRECTION XOR COUNT0 || COUNT1 .. COUNT31 || + * BEARER || 0...0 || DIRECTION || 0...0 + * + * 256-bit key length support is experimental and subject to + * change. The following variants may be supported: + * + * - ZUC-256 with 25 byte IV (of which 184 bits are variable) and + * 32/64/128 bit MAC as specified in "The ZUC-256 Stream Cipher". + * - ZUC-256 with 16 byte IV and 32/64/128 bit MAC as specified in + * "An Addendum to the ZUC-256 Stream Cipher", + * https://eprint.iacr.org/2021/1439 + */ + ODP_AUTH_ALG_ZUC_EIA3, + + /** MD5 algorithm */ + ODP_AUTH_ALG_MD5, + + /** SHA1 algorithm */ + ODP_AUTH_ALG_SHA1, + + /** 224 bit SHA2 algorithm */ + ODP_AUTH_ALG_SHA224, + + /** 256 bit SHA2 algorithm */ + ODP_AUTH_ALG_SHA256, + + /** 384 bit SHA2 algorithm */ + ODP_AUTH_ALG_SHA384, + + /** 512 bit SHA2 algorithm */ + ODP_AUTH_ALG_SHA512, + +} odp_auth_alg_t; + +/** + * Cipher algorithms in a bit field structure + */ +typedef union odp_crypto_cipher_algos_t { + /** Cipher algorithms */ + struct { + /** ODP_CIPHER_ALG_NULL */ + uint32_t null : 1; + + /** ODP_CIPHER_ALG_DES */ + uint32_t des : 1; + + /** ODP_CIPHER_ALG_3DES_CBC */ + uint32_t trides_cbc : 1; + + /** ODP_CIPHER_ALG_3DES_ECB */ + uint32_t trides_ecb : 1; + + /** ODP_CIPHER_ALG_AES_CBC */ + uint32_t aes_cbc : 1; + + /** ODP_CIPHER_ALG_AES_CTR */ + uint32_t aes_ctr : 1; + + /** ODP_CIPHER_ALG_AES_ECB */ + uint32_t aes_ecb : 1; + + /** ODP_CIPHER_ALG_AES_CFB128 */ + uint32_t aes_cfb128 : 1; + + /** ODP_CIPHER_ALG_AES_XTS */ + uint32_t aes_xts : 1; + + /** ODP_CIPHER_ALG_AES_GCM */ + uint32_t aes_gcm : 1; + + /** ODP_CIPHER_ALG_AES_CCM */ + uint32_t aes_ccm : 1; + + /** ODP_CIPHER_ALG_CHACHA20_POLY1305 */ + uint32_t chacha20_poly1305 : 1; + + /** ODP_CIPHER_ALG_KASUMI_F8 */ + uint32_t kasumi_f8 : 1; + + /** ODP_CIPHER_ALG_SNOW3G_UEA2 */ + uint32_t snow3g_uea2 : 1; + + /** ODP_CIPHER_ALG_AES_EEA2 */ + uint32_t aes_eea2 : 1; + + /** ODP_CIPHER_ALG_ZUC_EEA3 */ + uint32_t zuc_eea3 : 1; + + } bit; + + /** All bits of the bit field structure + * + * This field can be used to set/clear all flags, or bitwise + * operations over the entire structure. */ + uint32_t all_bits; +} odp_crypto_cipher_algos_t; + +/** + * Authentication algorithms in a bit field structure + */ +typedef union odp_crypto_auth_algos_t { + /** Authentication algorithms */ + struct { + /** ODP_AUTH_ALG_NULL */ + uint32_t null : 1; + + /** ODP_AUTH_ALG_MD5_HMAC */ + uint32_t md5_hmac : 1; + + /** ODP_AUTH_ALG_SHA1_HMAC */ + uint32_t sha1_hmac : 1; + + /** ODP_AUTH_ALG_SHA224_HMAC */ + uint32_t sha224_hmac : 1; + + /** ODP_AUTH_ALG_SHA256_HMAC */ + uint32_t sha256_hmac : 1; + + /** ODP_AUTH_ALG_SHA384_HMAC */ + uint32_t sha384_hmac : 1; + + /** ODP_AUTH_ALG_SHA512_HMAC */ + uint32_t sha512_hmac : 1; + + /** ODP_AUTH_ALG_AES_GCM */ + uint32_t aes_gcm : 1; + + /** ODP_AUTH_ALG_AES_GMAC*/ + uint32_t aes_gmac : 1; + + /** ODP_AUTH_ALG_AES_CCM */ + uint32_t aes_ccm : 1; + + /** ODP_AUTH_ALG_AES_CMAC*/ + uint32_t aes_cmac : 1; + + /** ODP_AUTH_ALG_AES_XCBC_MAC*/ + uint32_t aes_xcbc_mac : 1; + + /** ODP_AUTH_ALG_CHACHA20_POLY1305 */ + uint32_t chacha20_poly1305 : 1; + + /** ODP_AUTH_ALG_KASUMI_F9 */ + uint32_t kasumi_f9 : 1; + + /** ODP_AUTH_ALG_SNOW3G_UIA2 */ + uint32_t snow3g_uia2 : 1; + + /** ODP_AUTH_ALG_AES_EIA2 */ + uint32_t aes_eia2 : 1; + + /** ODP_AUTH_ALG_ZUC_EIA3 */ + uint32_t zuc_eia3 : 1; + + /** ODP_AUTH_ALG_MD5 */ + uint32_t md5 : 1; + + /** ODP_AUTH_ALG_SHA1 */ + uint32_t sha1 : 1; + + /** ODP_AUTH_ALG_SHA224 */ + uint32_t sha224 : 1; + + /** ODP_AUTH_ALG_SHA256 */ + uint32_t sha256 : 1; + + /** ODP_AUTH_ALG_SHA384 */ + uint32_t sha384 : 1; + + /** ODP_AUTH_ALG_SHA512 */ + uint32_t sha512 : 1; + + } bit; + + /** All bits of the bit field structure + * + * This field can be used to set/clear all flags, or bitwise + * operations over the entire structure. */ + uint32_t all_bits; +} odp_crypto_auth_algos_t; + +/** + * Crypto API key structure + */ +typedef struct odp_crypto_key { + /** Key data */ + uint8_t *data; + + /** Key length in bytes */ + uint32_t length; + +} odp_crypto_key_t; + +/** + * Type of odp_crypto_op()/odp_crypto_op_enq() calls. + */ +typedef enum odp_crypto_op_type_t { + /** + * Input packet data and metadata are copied in the output packet + * and then processed. Output packet is allocated by the caller + * or by ODP. odp_crypto_op(), odp_crypto_op_enq() and + * odp_crypto_operation() can be used. + * + * This is the default value but will be deprecated in the future. + */ + ODP_CRYPTO_OP_TYPE_LEGACY, + + /** + * Input packet data and metadata are copied in the output packet + * and then processed. Output packet is allocated by ODP. + * odp_crypto_op() and odp_crypto_op_enq() can be used. + */ + ODP_CRYPTO_OP_TYPE_BASIC, + + /** + * Out-of-place crypto operation. Output packet is provided by + * the caller and the input packet is not consumed nor modified. + * + * Output of the crypto operation is written in the caller provided + * output packet without affecting other data and metadata of the + * output packet. Memory layout of the output packet may change + * during the operation. + * + * Crypto output is the processed crypto_range, auth_range and + * MAC/digest (in encode sessions) of the input packet. + * + * odp_crypto_op() and odp_crypto_op_enq() can be used. + */ + ODP_CRYPTO_OP_TYPE_OOP, +} odp_crypto_op_type_t; + +/** + * Crypto API session creation parameters + */ +typedef struct odp_crypto_session_param_t { + /** Encode vs. decode operation + * + * The default value is ODP_CRYPTO_OP_ENCODE. + */ + odp_crypto_op_t op; + + /** Crypto operation type + * + * This field defines how the crypto operation functions are + * to be called and what they return. In particular, this field + * specifies the interpretation of the output packet parameter, + * how output packets are allocated and what data and metadata + * they contain. + * + * The default value is ODP_CRYPTO_OP_TYPE_LEGACY. + */ + odp_crypto_op_type_t op_type; + + /** Authenticate cipher vs. plain text + * + * Controls ordering of authentication and cipher operations, + * and is relative to the operation (encode vs decode). When encoding, + * TRUE indicates the authentication operation should be performed + * after the cipher operation else before. When decoding, TRUE + * indicates the reverse order of operation. + * + * The value is ignored with authenticated encryption algorithms + * such as AES-GCM. + * + * true: Authenticate cipher text + * false: Authenticate plain text + * + * The default value is false. + */ + odp_bool_t auth_cipher_text; + + /** Hash result location may overlap authentication range + * + * This flag indicates that the hash result location may (but is + * not required to) overlap authentication range. Setting this + * flag may reduce performance. + * + * Default value is false. + */ + odp_bool_t hash_result_in_auth_range; + + /** Preferred sync vs. async for odp_crypto_operation() + * + * The default value is ODP_CRYPTO_SYNC. + * + * @deprecated Used only with deprecated odp_crypto_operation() + */ + odp_crypto_op_mode_t ODP_DEPRECATE(pref_mode); + + /** Operation mode when using packet interface: sync or async + * + * The default value is ODP_CRYPTO_SYNC. + */ + odp_crypto_op_mode_t op_mode; + + /** Cipher algorithm + * + * Select cipher algorithm to be used. ODP_CIPHER_ALG_NULL indicates + * that ciphering is disabled. Use odp_crypto_capability() for + * supported algorithms. Note that some algorithms restrict choice of + * the pairing authentication algorithm. When ciphering is enabled + * cipher key and IV need to be set. The default value is + * ODP_CIPHER_ALG_NULL. + */ + odp_cipher_alg_t cipher_alg; + + /** Cipher key + * + * Use odp_crypto_cipher_capa() for supported key and IV lengths. + */ + odp_crypto_key_t cipher_key; + + /** Cipher IV length. The default value is zero. */ + uint32_t cipher_iv_len; + + /** Authentication algorithm + * + * Select authentication algorithm to be used. ODP_AUTH_ALG_NULL + * indicates that authentication is disabled. Use + * odp_crypto_capability() for supported algorithms. Note that some + * algorithms restrict choice of the pairing cipher algorithm. When + * single algorithm provides both ciphering and authentication + * (i.e. Authenticated Encryption), authentication side key + * (auth_key) and IV (auth_iv) are ignored, and cipher side values are + * used instead. These algorithms ignore authentication side key + * and IV: ODP_AUTH_ALG_AES_GCM, ODP_AUTH_ALG_AES_CCM and + * ODP_AUTH_ALG_CHACHA20_POLY1305. Otherwise, all authentication side + * parameters must be set when authentication is enabled. The default + * value is ODP_AUTH_ALG_NULL. + */ + odp_auth_alg_t auth_alg; + + /** Authentication key + * + * Use odp_crypto_auth_capability() for supported key lengths. + */ + odp_crypto_key_t auth_key; + + /** Authentication IV length. The default value is zero. */ + uint32_t auth_iv_len; + + /** Authentication digest length in bytes + * + * Use odp_crypto_auth_capability() for supported digest lengths. + */ + uint32_t auth_digest_len; + + /** Additional Authenticated Data (AAD) length in bytes + * + * AAD length is constant for all operations (packets) of the session. + * Set to zero when AAD is not used. Use odp_crypto_auth_capability() + * for supported AAD lengths. The default value is zero. + */ + uint32_t auth_aad_len; + + /** Async mode completion event queue + * + * The completion queue is used to return completions from + * odp_crypto_op_enq() (and the deprecated odp_crypto_operation()) + * to the application. + */ + odp_queue_t compl_queue; + + /** Output pool + * + * When the output packet is not specified during the call to + * crypto operation in the legacy operation type, the output + * packet will be allocated from this pool. + * + * In ODP_CRYPTO_OP_TYPE_BASIC and ODP_CRYPTO_OP_TYPE_OOP + * operation types this must be set to ODP_POOL_INVALID. + */ + odp_pool_t output_pool; + +} odp_crypto_session_param_t; + +/** + * Crypto API per packet operation parameters + * + * @deprecated Use odp_crypto_packet_op_param_t instead. + */ +typedef struct odp_crypto_op_param_t { + /** Session handle from creation */ + odp_crypto_session_t session; + + /** User context */ + void *ctx; + + /** Input packet + * + * Specifies the input packet for the crypto operation. When the + * 'out_pkt' variable is set to ODP_PACKET_INVALID (indicating a new + * packet should be allocated for the resulting packet). + */ + odp_packet_t pkt; + + /** Output packet + * + * Both "in place" (the original packet 'pkt' is modified) and + * "copy" (the packet is replicated to a new packet which contains + * the modified data) modes are supported. The "in place" mode of + * operation is indicated by setting 'out_pkt' equal to 'pkt'. + * For the copy mode of operation, setting 'out_pkt' to a valid packet + * value indicates the caller wishes to specify the destination packet. + * Setting 'out_pkt' to ODP_PACKET_INVALID indicates the caller wishes + * the destination packet be allocated from the output pool specified + * during session creation. + */ + odp_packet_t out_pkt; + + /** IV pointer for cipher */ + uint8_t *cipher_iv_ptr; + + /** Authentication IV pointer */ + uint8_t *auth_iv_ptr; + + /** Offset from start of packet for hash result + * + * In case of decode sessions, the expected hash will be read from + * this offset and compared with the calculated hash. After the + * operation the hash bytes will have undefined values. + * + * In case of encode sessions the calculated hash will be stored in + * this offset. + * + * If the hash_result_in_auth_range session parameter is true, + * the hash result location may overlap auth_range. In that case + * the result location will be zeroed in decode sessions before + * hash calculation. Zeroing is not done in encode sessions. + */ + uint32_t hash_result_offset; + + /** Pointer to AAD. AAD length is defined by 'auth_aad_len' + * session parameter. + */ + uint8_t *aad_ptr; + + /** Data range to be ciphered */ + odp_packet_data_range_t cipher_range; + + /** Data range to be authenticated + * + * The value is ignored with authenticated encryption algorithms, + * such as AES-GCM, which authenticate data in the cipher range + * and the AAD. + * + * As a special case AES-GMAC uses this field instead of aad_ptr + * for the data bytes to be authenticated. + */ + odp_packet_data_range_t auth_range; + +} ODP_DEPRECATE(odp_crypto_op_param_t); + +/** + * Crypto packet API per packet operation parameters + */ +typedef struct odp_crypto_packet_op_param_t { + /** Session handle from creation */ + odp_crypto_session_t session; + + /** IV pointer for cipher */ + uint8_t *cipher_iv_ptr; + + /** IV pointer for authentication */ + uint8_t *auth_iv_ptr; + + /** Offset from start of packet for hash result + * + * In case of decode sessions, the expected hash will be read from + * this offset from the input packet and compared with the calculated + * hash. After the operation the hash bytes will have undefined + * values except with out-of-place sessions (ODP_CRYPTO_OP_TYPE_OOP + * operation type). + * + * With out-of-place decode sessions the input packet is not modified + * but if the hash location overlaps the cipher range or the auth + * range, then the corresponding location in the output packet will + * have undefined content. + * + * In case of encode sessions the calculated hash will be stored in + * this offset in the output packet. + * + * If the hash_result_in_auth_range session parameter is true, + * the hash result location may overlap auth_range. In that case the + * result location will be treated as containing zero bytes for the + * purpose of hash calculation in decode sessions. + */ + uint32_t hash_result_offset; + + /** Pointer to AAD. AAD length is defined by 'auth_aad_len' + * session parameter. + */ + uint8_t *aad_ptr; + + /** Data range to apply cipher */ + odp_packet_data_range_t cipher_range; + + /** Data range to authenticate */ + odp_packet_data_range_t auth_range; + + /** Shift of the output offsets with ODP_CRYPTO_OP_TYPE_OOP + * + * The processed crypto range and auth range of the input packet + * will be written in the output packet at the offset specified + * in the ranges (i.e. the same as in the input packet), shifted + * by this many bytes. This allows directing the output to + * a different packet offset than the offset of the input data. + * + * This is ignored if the crypto operation type is not + * ODP_CRYPTO_OP_TYPE_OOP. + */ + int32_t dst_offset_shift; + +} odp_crypto_packet_op_param_t; + +/** + * Crypto API session creation return code + */ +typedef enum { + /** Session created */ + ODP_CRYPTO_SES_ERR_NONE, + /** Creation failed, no resources */ + ODP_CRYPTO_SES_ERR_ENOMEM, + /** Creation failed, bad cipher params */ + ODP_CRYPTO_SES_ERR_CIPHER, + /** Creation failed, bad auth params */ + ODP_CRYPTO_SES_ERR_AUTH, + + /** Unsupported combination of algorithms + * + * The combination of cipher and auth algorithms with their + * specific parameters is not supported even if the algorithms + * appear in capabilities and are supported in combination with + * other algorithms or other algorithm specific parameters. + */ + ODP_CRYPTO_SES_ERR_ALG_COMBO, + + /** Unsupported order of cipher and auth + * + * The requested mutual order of ciphering and authentication + * is not supported with the chosen individual cipher and + * authentication algorithms. + */ + ODP_CRYPTO_SES_ERR_ALG_ORDER, + + /** Unsupported combination of session creation parameters + * + * The combination of provided session creation parameters is not + * supported. This error can occur when there are limitations that + * are not expressible through crypto capabilities or other error + * status values. + */ + ODP_CRYPTO_SES_ERR_PARAMS, +} odp_crypto_ses_create_err_t; + +/** This synonym for backward compatibility will be deprecated later */ +#define ODP_CRYPTO_SES_CREATE_ERR_NONE ODP_CRYPTO_SES_ERR_NONE +/** This synonym for backward compatibility will be deprecated later */ +#define ODP_CRYPTO_SES_CREATE_ERR_ENOMEM ODP_CRYPTO_SES_ERR_ENOMEM +/** This synonym for backward compatibility will be deprecated later */ +#define ODP_CRYPTO_SES_CREATE_ERR_INV_CIPHER ODP_CRYPTO_SES_ERR_CIPHER +/** This synonym for backward compatibility will be deprecated later */ +#define ODP_CRYPTO_SES_CREATE_ERR_INV_AUTH ODP_CRYPTO_SES_ERR_AUTH + +/** + * Crypto API algorithm return code + */ +typedef enum { + /** Algorithm successful */ + ODP_CRYPTO_ALG_ERR_NONE, + /** Invalid data block size */ + ODP_CRYPTO_ALG_ERR_DATA_SIZE, + /** Key size invalid for algorithm */ + ODP_CRYPTO_ALG_ERR_KEY_SIZE, + /** Computed ICV value mismatch */ + ODP_CRYPTO_ALG_ERR_ICV_CHECK, + /** IV value not specified */ + ODP_CRYPTO_ALG_ERR_IV_INVALID, +} odp_crypto_alg_err_t; + +/** + * Crypto API hardware centric return code + */ +typedef enum { + /** Operation completed successfully */ + ODP_CRYPTO_HW_ERR_NONE, + /** Error detected during DMA of data */ + ODP_CRYPTO_HW_ERR_DMA, + /** Operation failed due to pool depletion */ + ODP_CRYPTO_HW_ERR_BP_DEPLETED, +} odp_crypto_hw_err_t; + +/** + * Cryto API per packet operation completion status + */ +typedef struct odp_crypto_op_status { + /** Algorithm specific return code */ + odp_crypto_alg_err_t alg_err; + + /** Hardware specific return code */ + odp_crypto_hw_err_t hw_err; + +} odp_crypto_op_status_t; + +/** + * Crypto API operation result + * + * @deprecated Use odp_crypto_packet_result_t instead. + */ +typedef struct odp_crypto_op_result { + /** Request completed successfully */ + odp_bool_t ok; + + /** User context from request */ + void *ctx; + + /** Output packet */ + odp_packet_t pkt; + + /** Cipher status */ + odp_crypto_op_status_t cipher_status; + + /** Authentication status */ + odp_crypto_op_status_t auth_status; + +} ODP_DEPRECATE(odp_crypto_op_result_t); + +/** + * Crypto packet API operation result + */ +typedef struct odp_crypto_packet_result_t { + /** Request completed successfully */ + odp_bool_t ok; + + /** Input packet passed to odp_crypo_op_enq() when the operation + * type of the session is ODP_CRYPTO_OP_TYPE_OOP. In other cases + * this field does not have a valid value. + */ + odp_packet_t pkt_in; + + /** Cipher status */ + odp_crypto_op_status_t cipher_status; + + /** Authentication status */ + odp_crypto_op_status_t auth_status; + +} odp_crypto_packet_result_t; + +/** + * Crypto capabilities + */ +typedef struct odp_crypto_capability_t { + /** Maximum number of crypto sessions */ + uint32_t max_sessions; + + /** Supported packet operation in SYNC mode */ + odp_support_t sync_mode; + + /** Supported packet operation in ASYNC mode */ + odp_support_t async_mode; + + /** Supported cipher algorithms */ + odp_crypto_cipher_algos_t ciphers; + + /** Cipher algorithms implemented with HW offload */ + odp_crypto_cipher_algos_t hw_ciphers; + + /** Supported authentication algorithms */ + odp_crypto_auth_algos_t auths; + + /** Authentication algorithms implemented with HW offload */ + odp_crypto_auth_algos_t hw_auths; + + /** + * Scheduled crypto completion queue support + * + * This defines whether scheduled queues are supported as crypto + * compl_queue. + * 0: Scheduled queues are not supported as crypto completion queues + * 1: Scheduled queues are supported as crypto completion queues + * @see odp_crypto_session_param_t + */ + odp_bool_t queue_type_sched; + + /** + * Plain crypto completion queue support + * + * This defines whether plain queues are supported as crypto + * compl_queue. + * 0: Plain queues are not supported as crypto completion queues + * 1: Plain queues are supported as crypto completion queues + * @see odp_crypto_session_param_t + */ + odp_bool_t queue_type_plain; +} odp_crypto_capability_t; + +/** + * Cipher algorithm capabilities + */ +typedef struct odp_crypto_cipher_capability_t { + /** Key length in bytes */ + uint32_t key_len; + + /** IV length in bytes */ + uint32_t iv_len; + + /** Cipher is operating in bitwise mode + * + * This cipher works on series of bits, rather than sequences of bytes: + * cipher_range in odp_crypto_op_param_t and + * odp_crypto_packet_op_param_t will use bits, rather than bytes. + * + * Note: data buffer MUST start on the byte boundary, using offset + * which is not divisible by 8 is unsupported and will result in + * unspecified behaviour. + * + * Note2: If the data length is not a multiple of 8, the remaining + * bits of the data in the last byte of the input/output will be the + * most significant bits, i.e. the most significant bit is considered + * to be the first bit of a byte for the purpose of input and output + * data range. The output bits that fall out of the output range are + * undefined. + */ + odp_bool_t bit_mode; + +} odp_crypto_cipher_capability_t; + +/** + * Authentication algorithm capabilities + */ +typedef struct odp_crypto_auth_capability_t { + /** Digest length in bytes */ + uint32_t digest_len; + + /** Key length in bytes */ + uint32_t key_len; + + /** IV length in bytes */ + uint32_t iv_len; + + /** Additional Authenticated Data (AAD) lengths */ + struct { + /** Minimum AAD length in bytes */ + uint32_t min; + + /** Maximum AAD length in bytes */ + uint32_t max; + + /** Increment of supported lengths between min and max + * (in bytes) */ + uint32_t inc; + } aad_len; + + /** Auth is operating in bitstring mode + * + * This auth works on series of bits, rather than sequences of bytes: + * auth_range in odp_crypto_op_param_t and + * odp_crypto_packet_op_param_t will use bits, rather than bytes. + * + * Note: data buffer MUST start on the byte boundary, using offset + * which is not divisible by 8 is unsupported and will result in + * unpredictable behaviour. + * + * Note2: If the data length is not a multiple of 8, the remaining + * bits of the data in the last byte of the input/output will be the + * most significant bits, i.e. the most significant bit is considered + * to be the first bit of a byte for the purpose of input and output + * data range. The output bits that fall out of the output range are + * undefined. + */ + odp_bool_t bit_mode; + +} odp_crypto_auth_capability_t; + +/** + * @} + */ + +#ifdef __cplusplus +} +#endif + +#include <odp/visibility_end.h> +#endif diff --git a/include/odp/api/spec/ipsec.h b/include/odp/api/spec/ipsec.h index b091961cc..603bf3aef 100644 --- a/include/odp/api/spec/ipsec.h +++ b/include/odp/api/spec/ipsec.h @@ -19,7 +19,7 @@ extern "C" { #endif -#include <odp/api/crypto.h> +#include <odp/api/crypto_types.h> #include <odp/api/event_types.h> #include <odp/api/ipsec_types.h> #include <odp/api/packet_types.h> diff --git a/include/odp/api/spec/ipsec_types.h b/include/odp/api/spec/ipsec_types.h index b74d26d4f..b34fb5982 100644 --- a/include/odp/api/spec/ipsec_types.h +++ b/include/odp/api/spec/ipsec_types.h @@ -20,7 +20,7 @@ extern "C" { #endif #include <odp/api/classification.h> -#include <odp/api/crypto.h> +#include <odp/api/crypto_types.h> #include <odp/api/packet_io_types.h> #include <odp/api/protocols.h> #include <odp/api/std_types.h> diff --git a/include/odp/api/spec/packet_flags.h b/include/odp/api/spec/packet_flags.h index 98bf2b7bf..c62e50885 100644 --- a/include/odp/api/spec/packet_flags.h +++ b/include/odp/api/spec/packet_flags.h @@ -1,4 +1,5 @@ /* Copyright (c) 2014-2018, Linaro Limited + * Copyright (c) 2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -22,7 +23,13 @@ extern "C" { #include <odp/api/packet_types.h> /** @addtogroup odp_packet - * Operations on packet metadata flags. + * @par Operations on packet metadata flags + * + * If user sets multiple conflicting packet metadata flags + * using odp_packet_has_XX_set() functions, only the last set flag value is + * guaranteed to hold. The values of other conflicting flags are implementation + * specific. The conflicting flag combinations are defined in function + * documentations. * @{ */ @@ -172,6 +179,8 @@ int odp_packet_has_jumbo(odp_packet_t pkt); /** * Check for VLAN * + * Check if packet contains normal or QinQ VLAN header. + * * @param pkt Packet handle * * @retval non-zero Packet contains a VLAN header @@ -182,6 +191,8 @@ int odp_packet_has_vlan(odp_packet_t pkt); /** * Check for VLAN QinQ (stacked VLAN) * + * Check if packet contains QinQ VLAN header. + * * @param pkt Packet handle * * @retval non-zero Packet contains a VLAN QinQ header @@ -401,6 +412,9 @@ void odp_packet_has_jumbo_set(odp_packet_t pkt, int val); /** * Set flag for VLAN * + * Set when packet contains normal VLAN header. Only one VLAN flag + * (VLAN/VLAN QinQ) can be set simultaneously. + * * @param pkt Packet handle * @param val Value */ @@ -409,6 +423,9 @@ void odp_packet_has_vlan_set(odp_packet_t pkt, int val); /** * Set flag for VLAN QinQ (stacked VLAN) * + * Set when packet contains QinQ VLAN header. Only one VLAN flag + * (VLAN/VLAN QinQ) can be set simultaneously. + * * @param pkt Packet handle * @param val Value */ @@ -417,6 +434,8 @@ void odp_packet_has_vlan_qinq_set(odp_packet_t pkt, int val); /** * Set flag for ARP * + * Only one of ARP/IPv4/IPv6 flags can be set simultaneously. + * * @param pkt Packet handle * @param val Value */ @@ -425,6 +444,8 @@ void odp_packet_has_arp_set(odp_packet_t pkt, int val); /** * Set flag for IPv4 * + * Only one of ARP/IPv4/IPv6 flags can be set simultaneously. + * * @param pkt Packet handle * @param val Value */ @@ -433,6 +454,8 @@ void odp_packet_has_ipv4_set(odp_packet_t pkt, int val); /** * Set flag for IPv6 * + * Only one of ARP/IPv4/IPv6 flags can be set simultaneously. + * * @param pkt Packet handle * @param val Value */ @@ -441,6 +464,8 @@ void odp_packet_has_ipv6_set(odp_packet_t pkt, int val); /** * Set flag for IP broadcast address * + * Only one of IP broadcast/multicast flags can be set simultaneously. + * * @param pkt Packet handle * @param val Value */ @@ -449,6 +474,8 @@ void odp_packet_has_ip_bcast_set(odp_packet_t pkt, int val); /** * Set flag for IP multicast address * + * Only one of IP broadcast/multicast flags can be set simultaneously. + * * @param pkt Packet handle * @param val Value */ @@ -481,6 +508,8 @@ void odp_packet_has_ipsec_set(odp_packet_t pkt, int val); /** * Set flag for UDP * + * Only one of TCP/UDP/SCTP/ICMP flags can be set simultaneously. + * * @param pkt Packet handle * @param val Value */ @@ -489,6 +518,8 @@ void odp_packet_has_udp_set(odp_packet_t pkt, int val); /** * Set flag for TCP * + * Only one of TCP/UDP/SCTP/ICMP flags can be set simultaneously. + * * @param pkt Packet handle * @param val Value */ @@ -497,6 +528,8 @@ void odp_packet_has_tcp_set(odp_packet_t pkt, int val); /** * Set flag for SCTP * + * Only one of TCP/UDP/SCTP/ICMP flags can be set simultaneously. + * * @param pkt Packet handle * @param val Value */ @@ -505,6 +538,8 @@ void odp_packet_has_sctp_set(odp_packet_t pkt, int val); /** * Set flag for ICMP * + * Only one of TCP/UDP/SCTP/ICMP flags can be set simultaneously. + * * @param pkt Packet handle * @param val Value */ diff --git a/include/odp/api/spec/packet_io.h b/include/odp/api/spec/packet_io.h index e8f0cd0b9..9a6117cee 100644 --- a/include/odp/api/spec/packet_io.h +++ b/include/odp/api/spec/packet_io.h @@ -20,7 +20,6 @@ extern "C" { #endif #include <odp/api/classification.h> -#include <odp/api/deprecated.h> #include <odp/api/packet_types.h> #include <odp/api/packet_io_stats.h> #include <odp/api/packet_io_types.h> @@ -833,38 +832,6 @@ int odp_pktio_info(odp_pktio_t pktio, odp_pktio_info_t *info); int odp_pktio_link_info(odp_pktio_t pktio, odp_pktio_link_info_t *info); /** - * Packet input timestamp resolution in hertz - * - * @deprecated Use odp_pktio_ts_res() instead, which returns resolution for - * both packet input and output timestamps. - * - * This is the resolution of packet input timestamps. Returns zero on a failure - * or when timestamping is disabled. - * - * @param pktio Packet IO handle - * - * @return Packet input timestamp resolution in hertz - * @retval 0 on failure - */ -uint64_t ODP_DEPRECATE(odp_pktin_ts_res)(odp_pktio_t pktio); - -/** - * Convert nanoseconds to packet input time - * - * @deprecated Use odp_pktio_ts_from_ns() instead, which can be used with both - * packet input and output timestamps. - * - * Packet input time source is used for timestamping incoming packets. - * This function is used convert nanosecond time to packet input timestamp time. - * - * @param pktio Packet IO handle - * @param ns Time in nanoseconds - * - * @return Packet input timestamp - */ -odp_time_t ODP_DEPRECATE(odp_pktin_ts_from_ns)(odp_pktio_t pktio, uint64_t ns); - -/** * Packet IO timestamp resolution in hertz * * This is the resolution of packet input and output timestamps using a packet diff --git a/include/odp/api/spec/packet_io_stats.h b/include/odp/api/spec/packet_io_stats.h index c6b782d5a..156a3a95e 100644 --- a/include/odp/api/spec/packet_io_stats.h +++ b/include/odp/api/spec/packet_io_stats.h @@ -19,7 +19,6 @@ extern "C" { #endif -#include <odp/api/deprecated.h> #include <odp/api/queue_types.h> /** @addtogroup odp_packet_io @@ -68,20 +67,6 @@ typedef struct odp_pktio_stats_t { * configuration, packets with errors may be dropped or not. */ uint64_t in_errors; - /** - * For packet-oriented interfaces, the number of packets received via - * the interface which were discarded because of an unknown or - * unsupported protocol. For character-oriented or fixed-length - * interfaces that support protocol multiplexing the number of - * transmission units received via the interface which were discarded - * because of an unknown or unsupported protocol. For any interface - * that does not support protocol multiplexing, this counter will always - * be 0. See ifInUnknownProtos in RFC 2863, RFC 3635. - * - * @deprecated This counter has been deprecated. - */ - uint64_t ODP_DEPRECATE(in_unknown_protos); - /** Number of octets in successfully transmitted packets. In case of * Ethernet, packet size includes MAC header. */ uint64_t out_octets; diff --git a/include/odp/api/spec/packet_io_types.h b/include/odp/api/spec/packet_io_types.h index fe86f6f12..a46668c30 100644 --- a/include/odp/api/spec/packet_io_types.h +++ b/include/odp/api/spec/packet_io_types.h @@ -19,7 +19,6 @@ extern "C" { #endif -#include <odp/api/deprecated.h> #include <odp/api/packet_types.h> #include <odp/api/packet_io_stats.h> #include <odp/api/pool_types.h> diff --git a/include/odp/api/spec/shared_memory.h b/include/odp/api/spec/shared_memory.h index 36b38782f..eebf288c9 100644 --- a/include/odp/api/spec/shared_memory.h +++ b/include/odp/api/spec/shared_memory.h @@ -51,19 +51,6 @@ extern "C" { */ /** - * Application SW only, no HW access - * - * @deprecated When set, application will not share the reserved memory with HW - * accelerators. However, leaving this flag to zero does not guarantee that - * the reserved memory can be accessed from HW, and thus usage of this flag is - * considered deprecated. If HW accessible memory is required, set - * ODP_SHM_HW_ACCESS instead. - * - * This flag must not be combined with ODP_SHM_HW_ACCESS. - */ -#define ODP_SHM_SW_ONLY 0x1 - -/** * Share with external processes */ #define ODP_SHM_PROC 0x2 diff --git a/include/odp/api/spec/timer.h b/include/odp/api/spec/timer.h index edfa423e7..0e1f07f09 100644 --- a/include/odp/api/spec/timer.h +++ b/include/odp/api/spec/timer.h @@ -20,6 +20,7 @@ extern "C" { #endif +#include <odp/api/deprecated.h> #include <odp/api/timer_types.h> #include <odp/api/event_types.h> #include <odp/api/pool_types.h> @@ -387,7 +388,7 @@ int odp_timer_periodic_cancel(odp_timer_t timer); * * @deprecated Use odp_timer_start() or odp_timer_restart() instead */ -int odp_timer_set_abs(odp_timer_t timer, uint64_t abs_tick, odp_event_t *tmo_ev); +int ODP_DEPRECATE(odp_timer_set_abs)(odp_timer_t timer, uint64_t abs_tick, odp_event_t *tmo_ev); /** * Set (or reset) a timer with relative expiration time @@ -416,7 +417,7 @@ int odp_timer_set_abs(odp_timer_t timer, uint64_t abs_tick, odp_event_t *tmo_ev) * * @deprecated Use odp_timer_start() or odp_timer_restart() instead */ -int odp_timer_set_rel(odp_timer_t timer, uint64_t rel_tick, odp_event_t *tmo_ev); +int ODP_DEPRECATE(odp_timer_set_rel)(odp_timer_t timer, uint64_t rel_tick, odp_event_t *tmo_ev); /** * Cancel a timer diff --git a/include/odp/api/spec/timer_types.h b/include/odp/api/spec/timer_types.h index 754c831b2..d61992fba 100644 --- a/include/odp/api/spec/timer_types.h +++ b/include/odp/api/spec/timer_types.h @@ -20,8 +20,9 @@ extern "C" { #endif -#include <odp/api/std_types.h> +#include <odp/api/deprecated.h> #include <odp/api/event_types.h> +#include <odp/api/std_types.h> /** @defgroup odp_timer ODP TIMER * Timer generating timeout events. @@ -245,13 +246,21 @@ typedef enum { /** The default clock source */ #define ODP_CLOCK_DEFAULT ODP_CLOCK_SRC_0 -/** For backwards compatibility, ODP_CLOCK_CPU is synonym of ODP_CLOCK_DEFAULT. - * This will be deprecated in the future. */ +#if ODP_DEPRECATED_API +/** + * For backwards compatibility, ODP_CLOCK_CPU is synonym of ODP_CLOCK_DEFAULT. + * + * @deprecated Use #ODP_CLOCK_DEFAULT instead. + */ #define ODP_CLOCK_CPU ODP_CLOCK_DEFAULT -/** For backwards compatibility, ODP_CLOCK_EXT is synonym of ODP_CLOCK_SRC_1. - * This will be deprecated in the future. */ +/** + * For backwards compatibility, ODP_CLOCK_EXT is synonym of ODP_CLOCK_SRC_1. + * + * @deprecated Use #ODP_CLOCK_SRC_1 instead. + */ #define ODP_CLOCK_EXT ODP_CLOCK_SRC_1 +#endif /** * Timer expiration mode @@ -514,17 +523,28 @@ typedef enum { } odp_timer_set_t; -/** For backwards compatibility, ODP_TIMER_TOOEARLY is synonym of ODP_TIMER_TOO_NEAR. - * This will be deprecated in the future. */ +#if ODP_DEPRECATED_API +/** + * For backwards compatibility, ODP_TIMER_TOOEARLY is synonym of ODP_TIMER_TOO_NEAR. + * + * @deprecated Use #ODP_TIMER_TOO_NEAR instead. + */ #define ODP_TIMER_TOOEARLY ODP_TIMER_TOO_NEAR -/** For backwards compatibility, ODP_TIMER_TOOLATE is synonym of ODP_TIMER_TOO_FAR. - * This will be deprecated in the future. */ +/** + * For backwards compatibility, ODP_TIMER_TOOLATE is synonym of ODP_TIMER_TOO_FAR. + * + * @deprecated Use #ODP_TIMER_TOO_FAR instead. + */ #define ODP_TIMER_TOOLATE ODP_TIMER_TOO_FAR -/** For backwards compatibility, ODP_TIMER_NOEVENT is synonym of ODP_TIMER_FAIL. - * This will be deprecated in the future. */ +/** + * For backwards compatibility, ODP_TIMER_NOEVENT is synonym of ODP_TIMER_FAIL. + * + * @deprecated Use #ODP_TIMER_FAIL instead. + */ #define ODP_TIMER_NOEVENT ODP_TIMER_FAIL +#endif /** * Timer tick information diff --git a/include/odp/api/spec/traffic_mngr.h b/include/odp/api/spec/traffic_mngr.h index f634cfb44..9bd735a7a 100644 --- a/include/odp/api/spec/traffic_mngr.h +++ b/include/odp/api/spec/traffic_mngr.h @@ -745,39 +745,6 @@ void odp_tm_requirements_init(odp_tm_requirements_t *requirements); */ void odp_tm_egress_init(odp_tm_egress_t *egress); -/** Query All TM Capabilities - * - * @deprecated Use odp_tm_egress_capabilities() instead that also additionally - * takes egress as input to provide capabilities specific to a given egress. - * - * This function returns the set of TM capabilities that are common for all - * egresses. The reason that this returns a SET of capabilities and not just - * one, is because it is expected that many HW based implementations may have - * one set of limits for the HW and also support a SW TM implementation with a - * (presumably larger) different set of limits. There are also cases where - * there could be more than one SW implementation (one supporting say tens of - * thousands of tm_queues and a variant supporting tens of millions of - * tm_queues). It returns capabilities that are valid for all egresses. - * The caller passes in an array of odp_tm_capabilities_t records and the - * maximum number of such records to output. If number of such records - * implementation supports is larger than caller requested number, then - * only caller requested number of records are written and return value is - * max number of records implementation supports. - * Caller then may again call with larger number of records to be returned. - * - * @param[out] capabilities An array of odp_tm_capabilities_t records to - * be filled in. - * @param capabilities_size The number of odp_tm_capabilities_t records - * in the capabilities array. - * @return Returns < 0 upon failure. Returns N > 0, - * where N is the maximum number of different - * odp_tm_capabilities_t records that the - * implementations supports. *NOTE* that this - * number can be > capabilities_size! - */ -int ODP_DEPRECATE(odp_tm_capabilities)(odp_tm_capabilities_t capabilities[], - uint32_t capabilities_size); - /** Query TM Capabilities specific to an egress * * The function returns the set of TM limits supported by this implementation @@ -1094,22 +1061,14 @@ typedef struct { * for this integer is in bits per second when packet_mode is * not TRUE while packets per second when packet mode is TRUE. */ - union { - /** @deprecated Use commit_rate instead */ - uint64_t ODP_DEPRECATE(commit_bps); - uint64_t commit_rate; /**< Commit information rate */ - }; + uint64_t commit_rate; /** The peak information rate for this shaper profile. The units for * this integer is in bits per second when packet_mode is * not TRUE while in packets per second when packet mode is TRUE. * This field is ignored when dual_rate is FALSE. */ - union { - /** @deprecated Use peak_rate instead */ - uint64_t ODP_DEPRECATE(peak_bps); - uint64_t peak_rate; /**< Peak information rate */ - }; + uint64_t peak_rate; /** The commit burst tolerance for this shaper profile. The units for * this field is bits when packet_mode is not TRUE and packets when diff --git a/include/odp/arch/arm32-linux/odp/api/abi/buffer_types.h b/include/odp/arch/arm32-linux/odp/api/abi/buffer_types.h new file mode 100644 index 000000000..8dd78321f --- /dev/null +++ b/include/odp/arch/arm32-linux/odp/api/abi/buffer_types.h @@ -0,0 +1,7 @@ +/* Copyright (c) 2023, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <odp/api/abi-default/buffer_types.h> diff --git a/include/odp/arch/arm32-linux/odp/api/abi/crypto_types.h b/include/odp/arch/arm32-linux/odp/api/abi/crypto_types.h new file mode 100644 index 000000000..ddadec1c1 --- /dev/null +++ b/include/odp/arch/arm32-linux/odp/api/abi/crypto_types.h @@ -0,0 +1,7 @@ +/* Copyright (c) 2022, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <odp/api/abi-default/crypto_types.h> diff --git a/include/odp/arch/arm64-linux/odp/api/abi/buffer_types.h b/include/odp/arch/arm64-linux/odp/api/abi/buffer_types.h new file mode 100644 index 000000000..8dd78321f --- /dev/null +++ b/include/odp/arch/arm64-linux/odp/api/abi/buffer_types.h @@ -0,0 +1,7 @@ +/* Copyright (c) 2023, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <odp/api/abi-default/buffer_types.h> diff --git a/include/odp/arch/arm64-linux/odp/api/abi/crypto_types.h b/include/odp/arch/arm64-linux/odp/api/abi/crypto_types.h new file mode 100644 index 000000000..ddadec1c1 --- /dev/null +++ b/include/odp/arch/arm64-linux/odp/api/abi/crypto_types.h @@ -0,0 +1,7 @@ +/* Copyright (c) 2022, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <odp/api/abi-default/crypto_types.h> diff --git a/include/odp/arch/default-linux/odp/api/abi/buffer_types.h b/include/odp/arch/default-linux/odp/api/abi/buffer_types.h new file mode 100644 index 000000000..8dd78321f --- /dev/null +++ b/include/odp/arch/default-linux/odp/api/abi/buffer_types.h @@ -0,0 +1,7 @@ +/* Copyright (c) 2023, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <odp/api/abi-default/buffer_types.h> diff --git a/include/odp/arch/default-linux/odp/api/abi/crypto_types.h b/include/odp/arch/default-linux/odp/api/abi/crypto_types.h new file mode 100644 index 000000000..ddadec1c1 --- /dev/null +++ b/include/odp/arch/default-linux/odp/api/abi/crypto_types.h @@ -0,0 +1,7 @@ +/* Copyright (c) 2022, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <odp/api/abi-default/crypto_types.h> diff --git a/include/odp/arch/power64-linux/odp/api/abi/buffer_types.h b/include/odp/arch/power64-linux/odp/api/abi/buffer_types.h new file mode 100644 index 000000000..8dd78321f --- /dev/null +++ b/include/odp/arch/power64-linux/odp/api/abi/buffer_types.h @@ -0,0 +1,7 @@ +/* Copyright (c) 2023, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <odp/api/abi-default/buffer_types.h> diff --git a/include/odp/arch/power64-linux/odp/api/abi/crypto_types.h b/include/odp/arch/power64-linux/odp/api/abi/crypto_types.h new file mode 100644 index 000000000..ddadec1c1 --- /dev/null +++ b/include/odp/arch/power64-linux/odp/api/abi/crypto_types.h @@ -0,0 +1,7 @@ +/* Copyright (c) 2022, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <odp/api/abi-default/crypto_types.h> diff --git a/include/odp/arch/x86_32-linux/odp/api/abi/buffer_types.h b/include/odp/arch/x86_32-linux/odp/api/abi/buffer_types.h new file mode 100644 index 000000000..8dd78321f --- /dev/null +++ b/include/odp/arch/x86_32-linux/odp/api/abi/buffer_types.h @@ -0,0 +1,7 @@ +/* Copyright (c) 2023, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <odp/api/abi-default/buffer_types.h> diff --git a/include/odp/arch/x86_32-linux/odp/api/abi/crypto_types.h b/include/odp/arch/x86_32-linux/odp/api/abi/crypto_types.h new file mode 100644 index 000000000..ddadec1c1 --- /dev/null +++ b/include/odp/arch/x86_32-linux/odp/api/abi/crypto_types.h @@ -0,0 +1,7 @@ +/* Copyright (c) 2022, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <odp/api/abi-default/crypto_types.h> diff --git a/include/odp/arch/x86_64-linux/odp/api/abi/buffer_types.h b/include/odp/arch/x86_64-linux/odp/api/abi/buffer_types.h new file mode 100644 index 000000000..8dd78321f --- /dev/null +++ b/include/odp/arch/x86_64-linux/odp/api/abi/buffer_types.h @@ -0,0 +1,7 @@ +/* Copyright (c) 2023, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <odp/api/abi-default/buffer_types.h> diff --git a/include/odp/arch/x86_64-linux/odp/api/abi/crypto_types.h b/include/odp/arch/x86_64-linux/odp/api/abi/crypto_types.h new file mode 100644 index 000000000..ddadec1c1 --- /dev/null +++ b/include/odp/arch/x86_64-linux/odp/api/abi/crypto_types.h @@ -0,0 +1,7 @@ +/* Copyright (c) 2022, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <odp/api/abi-default/crypto_types.h> diff --git a/include/odp/autoheader_external.h.in b/include/odp/autoheader_external.h.in index dd60fa0b2..74662b2e0 100644 --- a/include/odp/autoheader_external.h.in +++ b/include/odp/autoheader_external.h.in @@ -11,4 +11,7 @@ /* Define cache line size */ #undef _ODP_CACHE_LINE_SIZE +/* Define to 1 or 2 to enable event validation */ +#undef _ODP_EVENT_VALIDATION + #endif diff --git a/include/odp/autoheader_internal.h.in b/include/odp/autoheader_internal.h.in index 33d9f280f..a42b34ad0 100644 --- a/include/odp/autoheader_internal.h.in +++ b/include/odp/autoheader_internal.h.in @@ -35,4 +35,7 @@ /* Define to 1 to enable XDP support */ #undef _ODP_PKTIO_XDP +/* Define to 1 to enable IPSec MB crypto support */ +#undef _ODP_IPSECMB + #endif diff --git a/platform/linux-dpdk/Makefile.am b/platform/linux-dpdk/Makefile.am index 3bd599bcd..76e205212 100644 --- a/platform/linux-dpdk/Makefile.am +++ b/platform/linux-dpdk/Makefile.am @@ -32,9 +32,11 @@ odpapiplatinclude_HEADERS = \ include/odp/api/plat/buffer_inline_types.h \ include/odp/api/plat/byteorder_inlines.h \ include/odp/api/plat/cpu_inlines.h \ + include/odp/api/plat/crypto_inlines.h \ include/odp/api/plat/debug_inlines.h \ include/odp/api/plat/event_inlines.h \ include/odp/api/plat/event_inline_types.h \ + include/odp/api/plat/event_validation_external.h \ include/odp/api/plat/event_vector_inline_types.h \ include/odp/api/plat/hash_inlines.h \ include/odp/api/plat/ipsec_inlines.h \ @@ -68,11 +70,13 @@ odpapiabiarchinclude_HEADERS += \ include-abi/odp/api/abi/atomic.h \ include-abi/odp/api/abi/barrier.h \ include-abi/odp/api/abi/buffer.h \ + include-abi/odp/api/abi/buffer_types.h \ include-abi/odp/api/abi/byteorder.h \ include-abi/odp/api/abi/classification.h \ include-abi/odp/api/abi/comp.h \ include-abi/odp/api/abi/cpumask.h \ include-abi/odp/api/abi/crypto.h \ + include-abi/odp/api/abi/crypto_types.h \ include-abi/odp/api/abi/debug.h \ include-abi/odp/api/abi/dma_types.h \ include-abi/odp/api/abi/errno.h \ @@ -141,6 +145,7 @@ noinst_HEADERS = \ ${top_srcdir}/platform/linux-generic/include/odp_print_internal.h \ include/odp_errno_define.h \ include/odp_event_internal.h \ + ${top_srcdir}/platform/linux-generic/include/odp_event_validation_internal.h \ ${top_srcdir}/platform/linux-generic/include/odp_packet_dpdk.h \ ${top_srcdir}/platform/linux-generic/include/odp_pcapng.h \ ${top_srcdir}/platform/linux-generic/include/odp_pkt_queue_internal.h \ @@ -154,6 +159,8 @@ noinst_HEADERS = \ ${top_srcdir}/platform/linux-generic/include/odp_ring_common.h \ ${top_srcdir}/platform/linux-generic/include/odp_ring_internal.h \ ${top_srcdir}/platform/linux-generic/include/odp_ring_mpmc_internal.h \ + ${top_srcdir}/platform/linux-generic/include/odp_ring_mpmc_u32_internal.h \ + ${top_srcdir}/platform/linux-generic/include/odp_ring_mpmc_u64_internal.h \ ${top_srcdir}/platform/linux-generic/include/odp_ring_ptr_internal.h \ include/odp_ptr_ring_mpmc_internal.h \ include/odp_ptr_ring_spsc_internal.h \ @@ -197,6 +204,7 @@ __LIB__libodp_dpdk_la_SOURCES = \ odp_crypto.c \ odp_errno.c \ odp_event.c \ + ../linux-generic/odp_event_validation.c \ ../linux-generic/odp_hash_crc_gen.c \ odp_init.c \ ../linux-generic/odp_impl.c \ @@ -249,6 +257,7 @@ __LIB__libodp_dpdk_la_SOURCES += \ ../linux-generic/odp_buffer_api.c \ ../linux-generic/odp_byteorder_api.c \ ../linux-generic/odp_cpu_api.c \ + ../linux-generic/odp_crypto_api.c \ ../linux-generic/odp_event_api.c \ ../linux-generic/odp_hash_api.c \ ../linux-generic/odp_ipsec_api.c \ diff --git a/platform/linux-dpdk/include-abi/odp/api/abi/buffer_types.h b/platform/linux-dpdk/include-abi/odp/api/abi/buffer_types.h new file mode 120000 index 000000000..49e88bd2e --- /dev/null +++ b/platform/linux-dpdk/include-abi/odp/api/abi/buffer_types.h @@ -0,0 +1 @@ +../../../../../linux-generic/include-abi/odp/api/abi/buffer_types.h
\ No newline at end of file diff --git a/platform/linux-dpdk/include-abi/odp/api/abi/crypto_types.h b/platform/linux-dpdk/include-abi/odp/api/abi/crypto_types.h new file mode 120000 index 000000000..ce973bb2d --- /dev/null +++ b/platform/linux-dpdk/include-abi/odp/api/abi/crypto_types.h @@ -0,0 +1 @@ +../../../../../linux-generic/include-abi/odp/api/abi/crypto_types.h
\ No newline at end of file diff --git a/platform/linux-dpdk/include/odp/api/plat/buffer_inlines.h b/platform/linux-dpdk/include/odp/api/plat/buffer_inlines.h index 6265914b4..396d78559 100644 --- a/platform/linux-dpdk/include/odp/api/plat/buffer_inlines.h +++ b/platform/linux-dpdk/include/odp/api/plat/buffer_inlines.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2019-2022, Nokia +/* Copyright (c) 2019-2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -7,15 +7,15 @@ #ifndef ODP_PLAT_BUFFER_INLINES_H_ #define ODP_PLAT_BUFFER_INLINES_H_ +#include <odp/api/buffer_types.h> #include <odp/api/event.h> #include <odp/api/hints.h> #include <odp/api/pool_types.h> -#include <odp/api/abi/buffer.h> - #include <odp/api/plat/buffer_inline_types.h> #include <odp/api/plat/debug_inlines.h> #include <odp/api/plat/event_inline_types.h> +#include <odp/api/plat/event_validation_external.h> #include <rte_mbuf.h> #include <rte_mempool.h> @@ -95,6 +95,8 @@ _ODP_INLINE void odp_buffer_free(odp_buffer_t buf) { struct rte_mbuf *mbuf = (struct rte_mbuf *)buf; + _odp_buffer_validate(buf, _ODP_EV_BUFFER_FREE); + rte_mempool_put(mbuf->pool, mbuf); } @@ -107,6 +109,8 @@ _ODP_INLINE void odp_buffer_free_multi(const odp_buffer_t buf[], int num) if (odp_unlikely(num <= 0)) return; + _odp_buffer_validate_multi(buf, num, _ODP_EV_BUFFER_FREE_MULTI); + mbuf_tbl[0] = (struct rte_mbuf *)buf[0]; mp_pending = mbuf_tbl[0]->pool; num_pending = 1; @@ -134,6 +138,9 @@ _ODP_INLINE int odp_buffer_is_valid(odp_buffer_t buf) if (odp_event_type(odp_buffer_to_event(buf)) != ODP_EVENT_BUFFER) return 0; + if (odp_unlikely(_odp_buffer_validate(buf, _ODP_EV_BUFFER_IS_VALID))) + return 0; + return 1; } diff --git a/platform/linux-dpdk/include/odp/api/plat/crypto_inlines.h b/platform/linux-dpdk/include/odp/api/plat/crypto_inlines.h new file mode 120000 index 000000000..0c1935696 --- /dev/null +++ b/platform/linux-dpdk/include/odp/api/plat/crypto_inlines.h @@ -0,0 +1 @@ +../../../../../linux-generic/include/odp/api/plat/crypto_inlines.h
\ No newline at end of file diff --git a/platform/linux-dpdk/include/odp/api/plat/event_validation_external.h b/platform/linux-dpdk/include/odp/api/plat/event_validation_external.h new file mode 120000 index 000000000..001662d8b --- /dev/null +++ b/platform/linux-dpdk/include/odp/api/plat/event_validation_external.h @@ -0,0 +1 @@ +../../../../../linux-generic/include/odp/api/plat/event_validation_external.h
\ No newline at end of file diff --git a/platform/linux-dpdk/include/odp/api/plat/packet_inline_types.h b/platform/linux-dpdk/include/odp/api/plat/packet_inline_types.h index 9a72a70ef..c6ede5d3a 100644 --- a/platform/linux-dpdk/include/odp/api/plat/packet_inline_types.h +++ b/platform/linux-dpdk/include/odp/api/plat/packet_inline_types.h @@ -1,5 +1,5 @@ /* Copyright (c) 2018, Linaro Limited - * Copyright (c) 2022, Nokia + * Copyright (c) 2022-2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -44,6 +44,7 @@ typedef struct _odp_packet_inline_offset_t { uint16_t subtype; uint16_t cls_mark; uint16_t ipsec_ctx; + uint16_t crypto_op; uint16_t buf_addr; uint16_t data; uint16_t pkt_len; diff --git a/platform/linux-dpdk/include/odp/api/plat/packet_inlines.h b/platform/linux-dpdk/include/odp/api/plat/packet_inlines.h index 4347afb88..4ae53c7b3 100644 --- a/platform/linux-dpdk/include/odp/api/plat/packet_inlines.h +++ b/platform/linux-dpdk/include/odp/api/plat/packet_inlines.h @@ -1,5 +1,5 @@ /* Copyright (c) 2016-2018, Linaro Limited - * Copyright (c) 2019-2022, Nokia + * Copyright (c) 2019-2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -25,6 +25,7 @@ extern "C" { #include <odp/api/time.h> #include <odp/api/plat/debug_inlines.h> +#include <odp/api/plat/event_validation_external.h> #include <odp/api/plat/packet_io_inlines.h> #include <odp/api/plat/packet_inline_types.h> #include <odp/api/plat/pool_inline_types.h> @@ -663,17 +664,23 @@ _ODP_INLINE odp_event_t odp_packet_tx_compl_to_event(odp_packet_tx_compl_t tx_co _ODP_INLINE void odp_packet_free(odp_packet_t pkt) { + _odp_packet_validate(pkt, _ODP_EV_PACKET_FREE); + rte_pktmbuf_free((struct rte_mbuf *)pkt); } _ODP_INLINE void odp_packet_free_multi(const odp_packet_t pkt[], int num) { + _odp_packet_validate_multi(pkt, num, _ODP_EV_PACKET_FREE_MULTI); + rte_pktmbuf_free_bulk((struct rte_mbuf **)(uintptr_t)pkt, (unsigned int)num); } _ODP_INLINE void odp_packet_free_sp(const odp_packet_t pkt[], int num) { - odp_packet_free_multi(pkt, num); + _odp_packet_validate_multi(pkt, num, _ODP_EV_PACKET_FREE_SP); + + rte_pktmbuf_free_bulk((struct rte_mbuf **)(uintptr_t)pkt, (unsigned int)num); } _ODP_INLINE void *odp_packet_seg_data(odp_packet_t pkt ODP_UNUSED, diff --git a/platform/linux-dpdk/include/odp_config_internal.h b/platform/linux-dpdk/include/odp_config_internal.h index 18754184e..e104b9d24 100644 --- a/platform/linux-dpdk/include/odp_config_internal.h +++ b/platform/linux-dpdk/include/odp_config_internal.h @@ -1,5 +1,5 @@ /* Copyright (c) 2016-2018, Linaro Limited - * Copyright (c) 2020-2021, Nokia + * Copyright (c) 2020-2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -70,7 +70,7 @@ extern "C" { /* * Maximum number of stashes */ -#define CONFIG_MAX_STASHES 128 +#define CONFIG_MAX_STASHES 2048 /* * Maximum number of packet IO resources @@ -133,10 +133,10 @@ extern "C" { /* * Number of shared memory blocks reserved for implementation internal use. * - * Each stash and packet pool requires one SHM block, and 20 blocks are - * reserved for per ODP module global data. + * Each packet pool requires one SHM block and 20 blocks are reserved for + * ODP module global data. */ -#define CONFIG_INTERNAL_SHM_BLOCKS (CONFIG_MAX_STASHES + ODP_CONFIG_POOLS + 20) +#define CONFIG_INTERNAL_SHM_BLOCKS (ODP_CONFIG_POOLS + 20) /* * Maximum number of shared memory blocks. diff --git a/platform/linux-dpdk/include/odp_event_internal.h b/platform/linux-dpdk/include/odp_event_internal.h index 320f42146..5ccf5a3ed 100644 --- a/platform/linux-dpdk/include/odp_event_internal.h +++ b/platform/linux-dpdk/include/odp_event_internal.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2021-2022, Nokia +/* Copyright (c) 2021-2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -80,6 +80,13 @@ static inline void _odp_event_type_set(odp_event_t event, int ev) _odp_event_hdr(event)->event_type = ev; } +static inline uint64_t *_odp_event_endmark_get_ptr(odp_event_t event) +{ + struct rte_mbuf *mbuf = _odp_event_to_mbuf(event); + + return (uint64_t *)((uint8_t *)mbuf->buf_addr + mbuf->buf_len); +} + #ifdef __cplusplus } #endif diff --git a/platform/linux-dpdk/include/odp_packet_io_internal.h b/platform/linux-dpdk/include/odp_packet_io_internal.h index 78b1f343a..2efa52b67 100644 --- a/platform/linux-dpdk/include/odp_packet_io_internal.h +++ b/platform/linux-dpdk/include/odp_packet_io_internal.h @@ -1,5 +1,5 @@ /* Copyright (c) 2013-2018, Linaro Limited - * Copyright (c) 2019-2022, Nokia + * Copyright (c) 2019-2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -53,11 +53,7 @@ ODP_STATIC_ASSERT(PKTIO_LSO_PROFILES < UINT8_MAX, "PKTIO_LSO_PROFILES_ERROR"); /* Forward declaration */ struct pktio_if_ops; -#if ODP_CACHE_LINE_SIZE == 128 -#define PKTIO_PRIVATE_SIZE 1536 -#else -#define PKTIO_PRIVATE_SIZE 1344 -#endif +#define PKTIO_PRIVATE_SIZE 9216 typedef struct ODP_ALIGNED_CACHE { const struct pktio_if_ops *ops; /**< Implementation specific methods */ diff --git a/platform/linux-dpdk/include/odp_pool_internal.h b/platform/linux-dpdk/include/odp_pool_internal.h index a5ffcf517..b82324a77 100644 --- a/platform/linux-dpdk/include/odp_pool_internal.h +++ b/platform/linux-dpdk/include/odp_pool_internal.h @@ -1,5 +1,5 @@ /* Copyright (c) 2013-2018, Linaro Limited - * Copyright (c) 2021-2022, Nokia + * Copyright (c) 2021-2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -72,7 +72,7 @@ typedef struct ODP_ALIGNED_CACHE { /* Everything under this mark is memset() to zero on pool create */ uint8_t memset_mark; struct rte_mempool *rte_mempool; - uint32_t seg_len; + uint32_t seg_len; /* Initial packet segment length (excludes endmark) */ uint32_t ext_head_offset; uint32_t num; uint32_t num_populated; @@ -84,6 +84,7 @@ typedef struct ODP_ALIGNED_CACHE { odp_shm_t uarea_shm; uint64_t uarea_shm_size; uint32_t uarea_size; + uint32_t trailer_size; /* Endmark size */ uint8_t *uarea_base_addr; char name[ODP_POOL_NAME_LEN]; diff --git a/platform/linux-dpdk/m4/configure.m4 b/platform/linux-dpdk/m4/configure.m4 index 90d55b6f6..70295400f 100644 --- a/platform/linux-dpdk/m4/configure.m4 +++ b/platform/linux-dpdk/m4/configure.m4 @@ -5,11 +5,13 @@ ODP_VISIBILITY ODP_ATOMIC m4_include([platform/linux-dpdk/m4/odp_cpu.m4]) +m4_include([platform/linux-dpdk/m4/odp_event_validation.m4]) m4_include([platform/linux-dpdk/m4/odp_libconfig.m4]) m4_include([platform/linux-dpdk/m4/odp_openssl.m4]) m4_include([platform/linux-dpdk/m4/odp_pcapng.m4]) m4_include([platform/linux-dpdk/m4/odp_scheduler.m4]) +ODP_EVENT_VALIDATION ODP_PTHREAD ODP_SCHEDULER ODP_TIMER @@ -68,6 +70,7 @@ AS_VAR_APPEND([PLAT_DEP_LIBS], ["${ATOMIC_LIBS} ${LIBCONFIG_LIBS} ${OPENSSL_LIBS # Add text to the end of configure with platform specific settings. # Make sure it's aligned same as other lines in configure.ac. AS_VAR_APPEND([PLAT_CFG_TEXT], [" + event_validation: ${enable_event_validation} openssl: ${with_openssl} openssl_rand: ${openssl_rand} pcap: ${have_pmd_pcap} diff --git a/platform/linux-dpdk/m4/odp_event_validation.m4 b/platform/linux-dpdk/m4/odp_event_validation.m4 new file mode 120000 index 000000000..0d457c6ff --- /dev/null +++ b/platform/linux-dpdk/m4/odp_event_validation.m4 @@ -0,0 +1 @@ +../../linux-generic/m4/odp_event_validation.m4
\ No newline at end of file diff --git a/platform/linux-dpdk/m4/odp_libconfig.m4 b/platform/linux-dpdk/m4/odp_libconfig.m4 index 84d331b8a..1d56f0a5c 100644 --- a/platform/linux-dpdk/m4/odp_libconfig.m4 +++ b/platform/linux-dpdk/m4/odp_libconfig.m4 @@ -3,7 +3,7 @@ ########################################################################## m4_define([_odp_config_version_generation], [0]) m4_define([_odp_config_version_major], [1]) -m4_define([_odp_config_version_minor], [18]) +m4_define([_odp_config_version_minor], [21]) m4_define([_odp_config_version], [_odp_config_version_generation._odp_config_version_major._odp_config_version_minor]) diff --git a/platform/linux-dpdk/odp_crypto.c b/platform/linux-dpdk/odp_crypto.c index 076de1b0e..713419a14 100644 --- a/platform/linux-dpdk/odp_crypto.c +++ b/platform/linux-dpdk/odp_crypto.c @@ -1,5 +1,5 @@ /* Copyright (c) 2017-2018, Linaro Limited - * Copyright (c) 2018-2022, Nokia + * Copyright (c) 2018-2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -83,10 +83,7 @@ typedef struct crypto_session_entry_s { unsigned int chained_bufs_ok:1; } flags; uint8_t cdev_id; -#if ODP_DEPRECATED_API - uint8_t cipher_iv_data[MAX_IV_LENGTH]; - uint8_t auth_iv_data[MAX_IV_LENGTH]; -#endif + } crypto_session_entry_t; typedef struct crypto_global_s { @@ -1395,6 +1392,13 @@ int odp_crypto_session_create(const odp_crypto_session_param_t *param, return -1; } + /* ODP_CRYPTO_OP_TYPE_OOP not supported */ + if (param->op_type == ODP_CRYPTO_OP_TYPE_OOP) { + *status = ODP_CRYPTO_SES_ERR_PARAMS; + *session_out = ODP_CRYPTO_SESSION_INVALID; + return -1; + } + if (rte_cryptodev_count() == 0) { _ODP_ERR("No crypto devices available\n"); *status = ODP_CRYPTO_SES_ERR_ENOMEM; @@ -1496,16 +1500,7 @@ int odp_crypto_session_create(const odp_crypto_session_param_t *param, out_null: session->rte_session = rte_session; session->cdev_id = cdev_id; -#if ODP_DEPRECATED_API - if (param->cipher_iv.data) - memcpy(session->cipher_iv_data, - param->cipher_iv.data, - param->cipher_iv.length); - if (param->auth_iv.data) - memcpy(session->auth_iv_data, - param->auth_iv.data, - param->auth_iv.length); -#endif + /* We're happy */ *session_out = (intptr_t)session; *status = ODP_CRYPTO_SES_ERR_NONE; @@ -1623,42 +1618,6 @@ uint64_t odp_crypto_session_to_u64(odp_crypto_session_t hdl) return (uint64_t)hdl; } -odp_packet_t odp_crypto_packet_from_event(odp_event_t ev) -{ - /* This check not mandated by the API specification */ - _ODP_ASSERT(odp_event_type(ev) == ODP_EVENT_PACKET); - _ODP_ASSERT(odp_event_subtype(ev) == ODP_EVENT_PACKET_CRYPTO); - - return odp_packet_from_event(ev); -} - -odp_event_t odp_crypto_packet_to_event(odp_packet_t pkt) -{ - return odp_packet_to_event(pkt); -} - -static -odp_crypto_packet_result_t *get_op_result_from_packet(odp_packet_t pkt) -{ - odp_packet_hdr_t *hdr = packet_hdr(pkt); - - return &hdr->crypto_op_result; -} - -int odp_crypto_result(odp_crypto_packet_result_t *result, - odp_packet_t packet) -{ - odp_crypto_packet_result_t *op_result; - - _ODP_ASSERT(odp_event_subtype(odp_packet_to_event(packet)) == ODP_EVENT_PACKET_CRYPTO); - - op_result = get_op_result_from_packet(packet); - - memcpy(result, op_result, sizeof(*result)); - - return 0; -} - static uint8_t *crypto_prepare_digest(const crypto_session_entry_t *session, odp_packet_t pkt, const odp_crypto_packet_op_param_t *param, @@ -1718,17 +1677,8 @@ static void crypto_fill_aead_param(const crypto_session_entry_t *session, iv_ptr++; } -#if ODP_DEPRECATED_API - if (param->cipher_iv_ptr) - memcpy(iv_ptr, param->cipher_iv_ptr, iv_len); - else if (session->p.cipher_iv.data) - memcpy(iv_ptr, session->cipher_iv_data, iv_len); - else - _ODP_ASSERT(iv_len == 0); -#else _ODP_ASSERT(iv_len == 0 || param->cipher_iv_ptr != NULL); memcpy(iv_ptr, param->cipher_iv_ptr, iv_len); -#endif op->sym->aead.data.offset = param->cipher_range.offset; op->sym->aead.data.length = param->cipher_range.length; @@ -1752,29 +1702,6 @@ static void crypto_fill_sym_param(const crypto_session_entry_t *session, &op->sym->auth.digest.phys_addr); } -#if ODP_DEPRECATED_API - if (param->cipher_iv_ptr) { - iv_ptr = rte_crypto_op_ctod_offset(op, uint8_t *, IV_OFFSET); - memcpy(iv_ptr, param->cipher_iv_ptr, cipher_iv_len); - } else if (session->p.cipher_iv.data) { - iv_ptr = rte_crypto_op_ctod_offset(op, uint8_t *, IV_OFFSET); - memcpy(iv_ptr, session->cipher_iv_data, cipher_iv_len); - } else { - _ODP_ASSERT(cipher_iv_len == 0); - } - - if (param->auth_iv_ptr) { - iv_ptr = rte_crypto_op_ctod_offset(op, uint8_t *, - IV_OFFSET + MAX_IV_LENGTH); - memcpy(iv_ptr, param->auth_iv_ptr, auth_iv_len); - } else if (session->p.auth_iv.data) { - iv_ptr = rte_crypto_op_ctod_offset(op, uint8_t *, - IV_OFFSET + MAX_IV_LENGTH); - memcpy(iv_ptr, session->auth_iv_data, auth_iv_len); - } else { - _ODP_ASSERT(auth_iv_len == 0); - } -#else _ODP_ASSERT(cipher_iv_len == 0 || param->cipher_iv_ptr != NULL); _ODP_ASSERT(auth_iv_len == 0 || param->auth_iv_ptr != NULL); iv_ptr = rte_crypto_op_ctod_offset(op, uint8_t *, IV_OFFSET); @@ -1784,7 +1711,6 @@ static void crypto_fill_sym_param(const crypto_session_entry_t *session, iv_ptr = rte_crypto_op_ctod_offset(op, uint8_t *, IV_OFFSET + MAX_IV_LENGTH); memcpy(iv_ptr, param->auth_iv_ptr, auth_iv_len); } -#endif op->sym->cipher.data.offset = param->cipher_range.offset; op->sym->cipher.data.length = param->cipher_range.length; @@ -1865,6 +1791,9 @@ static odp_packet_t get_output_packet(const crypto_session_entry_t *session, { int rc; + if (odp_likely(session->p.op_type == ODP_CRYPTO_OP_TYPE_BASIC)) + return pkt_in; + if (odp_likely(pkt_in == pkt_out)) return pkt_out; @@ -2145,7 +2074,7 @@ static void op_finish(crypto_op_t *op) /* Fill in result */ packet_subtype_set(pkt, ODP_EVENT_PACKET_CRYPTO); - op_result = get_op_result_from_packet(pkt); + op_result = &packet_hdr(pkt)->crypto_op_result; op_result->cipher_status.alg_err = rc_cipher; op_result->cipher_status.hw_err = ODP_CRYPTO_HW_ERR_NONE; op_result->auth_status.alg_err = rc_auth; @@ -2188,6 +2117,10 @@ int odp_crypto_operation(odp_crypto_op_param_t *param, odp_crypto_op_result_t local_result; int rc; + if (((crypto_session_entry_t *)(intptr_t)param->session)->p.op_type != + ODP_CRYPTO_OP_TYPE_LEGACY) + return -1; + packet_param.session = param->session; packet_param.cipher_iv_ptr = param->cipher_iv_ptr; packet_param.auth_iv_ptr = param->auth_iv_ptr; @@ -2268,7 +2201,8 @@ int odp_crypto_op_enq(const odp_packet_t pkt_in[], _ODP_ASSERT(ODP_CRYPTO_ASYNC == session->p.op_mode); _ODP_ASSERT(ODP_QUEUE_INVALID != session->p.compl_queue); - out_pkts[i] = pkt_out[i]; + if (session->p.op_type != ODP_CRYPTO_OP_TYPE_BASIC) + out_pkts[i] = pkt_out[i]; } num_pkt = odp_crypto_int(pkt_in, out_pkts, param, num_pkt); diff --git a/platform/linux-dpdk/odp_event.c b/platform/linux-dpdk/odp_event.c index 24853f83e..625b6eefe 100644 --- a/platform/linux-dpdk/odp_event.c +++ b/platform/linux-dpdk/odp_event.c @@ -1,5 +1,5 @@ /* Copyright (c) 2015-2018, Linaro Limited - * Copyright (c) 2020-2022, Nokia + * Copyright (c) 2020-2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -18,6 +18,7 @@ #include <odp_debug_internal.h> #include <odp_packet_internal.h> #include <odp_event_internal.h> +#include <odp_event_validation_internal.h> #include <odp_event_vector_internal.h> /* Inlined API functions */ @@ -42,13 +43,15 @@ _odp_event_inline_offset ODP_ALIGNED_CACHE = { #include <odp/visibility_end.h> -void odp_event_free(odp_event_t event) +static inline void event_free(odp_event_t event, _odp_ev_id_t id) { switch (odp_event_type(event)) { case ODP_EVENT_BUFFER: + _odp_buffer_validate(odp_buffer_from_event(event), id); odp_buffer_free(odp_buffer_from_event(event)); break; case ODP_EVENT_PACKET: + _odp_packet_validate(odp_packet_from_event(event), id); odp_packet_free(odp_packet_from_event(event)); break; case ODP_EVENT_PACKET_VECTOR: @@ -76,17 +79,21 @@ void odp_event_free(odp_event_t event) } } -void odp_event_free_multi(const odp_event_t event[], int num) +void odp_event_free(odp_event_t event) { - int i; + event_free(event, _ODP_EV_EVENT_FREE); +} - for (i = 0; i < num; i++) - odp_event_free(event[i]); +void odp_event_free_multi(const odp_event_t event[], int num) +{ + for (int i = 0; i < num; i++) + event_free(event[i], _ODP_EV_EVENT_FREE_MULTI); } void odp_event_free_sp(const odp_event_t event[], int num) { - odp_event_free_multi(event, num); + for (int i = 0; i < num; i++) + event_free(event[i], _ODP_EV_EVENT_FREE_SP); } uint64_t odp_event_to_u64(odp_event_t hdl) @@ -104,9 +111,9 @@ int odp_event_is_valid(odp_event_t event) switch (odp_event_type(event)) { case ODP_EVENT_BUFFER: - /* Fall through */ + return !_odp_buffer_validate(odp_buffer_from_event(event), _ODP_EV_EVENT_IS_VALID); case ODP_EVENT_PACKET: - /* Fall through */ + return !_odp_packet_validate(odp_packet_from_event(event), _ODP_EV_EVENT_IS_VALID); case ODP_EVENT_TIMEOUT: /* Fall through */ #if ODP_DEPRECATED_API diff --git a/platform/linux-dpdk/odp_init.c b/platform/linux-dpdk/odp_init.c index 329252f9c..b144835ea 100644 --- a/platform/linux-dpdk/odp_init.c +++ b/platform/linux-dpdk/odp_init.c @@ -1,5 +1,5 @@ /* Copyright (c) 2013-2018, Linaro Limited - * Copyright (c) 2019-2021, Nokia + * Copyright (c) 2019-2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -41,6 +41,7 @@ enum init_stage { HASH_INIT, THREAD_INIT, POOL_INIT, + EVENT_VALIDATION_INIT, STASH_INIT, QUEUE_INIT, SCHED_INIT, @@ -426,6 +427,13 @@ static int term_global(enum init_stage stage) } /* Fall through */ + case EVENT_VALIDATION_INIT: + if (_odp_event_validation_term_global()) { + _ODP_ERR("ODP event validation term failed.\n"); + rc = -1; + } + /* Fall through */ + case POOL_INIT: if (_odp_pool_term_global()) { _ODP_ERR("ODP buffer pool term failed.\n"); @@ -590,6 +598,12 @@ int odp_init_global(odp_instance_t *instance, } stage = POOL_INIT; + if (_odp_event_validation_init_global()) { + _ODP_ERR("ODP event validation init failed.\n"); + goto init_failed; + } + stage = EVENT_VALIDATION_INIT; + if (_odp_stash_init_global()) { _ODP_ERR("ODP stash init failed.\n"); goto init_failed; diff --git a/platform/linux-dpdk/odp_packet.c b/platform/linux-dpdk/odp_packet.c index 16b940f2f..9b8755ce9 100644 --- a/platform/linux-dpdk/odp_packet.c +++ b/platform/linux-dpdk/odp_packet.c @@ -1,5 +1,5 @@ /* Copyright (c) 2013-2018, Linaro Limited - * Copyright (c) 2019-2022, Nokia + * Copyright (c) 2019-2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -24,6 +24,7 @@ #include <odp_chksum_internal.h> #include <odp_debug_internal.h> #include <odp_event_internal.h> +#include <odp_event_validation_internal.h> #include <odp_macros_internal.h> #include <odp_packet_internal.h> #include <odp_packet_io_internal.h> @@ -66,6 +67,7 @@ const _odp_packet_inline_offset_t _odp_packet_inline ODP_ALIGNED_CACHE = { .subtype = offsetof(odp_packet_hdr_t, subtype), .cls_mark = offsetof(odp_packet_hdr_t, cls_mark), .ipsec_ctx = offsetof(odp_packet_hdr_t, ipsec_ctx), + .crypto_op = offsetof(odp_packet_hdr_t, crypto_op_result), .buf_addr = offsetof(odp_packet_hdr_t, event_hdr.mb.buf_addr), .data = offsetof(odp_packet_hdr_t, event_hdr.mb.data_off), .pkt_len = offsetof(odp_packet_hdr_t, event_hdr.mb.pkt_len), @@ -1144,6 +1146,9 @@ int odp_packet_is_valid(odp_packet_t pkt) if (odp_event_type(ev) != ODP_EVENT_PACKET) return 0; + if (odp_unlikely(_odp_packet_validate(pkt, _ODP_EV_PACKET_IS_VALID))) + return 0; + switch (odp_event_subtype(ev)) { case ODP_EVENT_PACKET_BASIC: /* Fall through */ diff --git a/platform/linux-dpdk/odp_packet_flags.c b/platform/linux-dpdk/odp_packet_flags.c index 728df157a..345ac8488 100644 --- a/platform/linux-dpdk/odp_packet_flags.c +++ b/platform/linux-dpdk/odp_packet_flags.c @@ -72,10 +72,12 @@ void odp_packet_has_jumbo_set(odp_packet_t pkt, int val) void odp_packet_has_vlan_set(odp_packet_t pkt, int val) { setflag(pkt, input_flags.vlan, val); + setflag(pkt, input_flags.vlan_qinq, 0); } void odp_packet_has_vlan_qinq_set(odp_packet_t pkt, int val) { + setflag(pkt, input_flags.vlan, val); setflag(pkt, input_flags.vlan_qinq, val); } diff --git a/platform/linux-dpdk/odp_pool.c b/platform/linux-dpdk/odp_pool.c index b0df7997a..d36f197c0 100644 --- a/platform/linux-dpdk/odp_pool.c +++ b/platform/linux-dpdk/odp_pool.c @@ -1,5 +1,5 @@ /* Copyright (c) 2013-2018, Linaro Limited - * Copyright (c) 2019-2022, Nokia + * Copyright (c) 2019-2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -16,6 +16,8 @@ #include <odp_buffer_internal.h> #include <odp_config_internal.h> #include <odp_debug_internal.h> +#include <odp_event_internal.h> +#include <odp_event_validation_internal.h> #include <odp_event_vector_internal.h> #include <odp_init_internal.h> #include <odp_libconfig_internal.h> @@ -380,6 +382,12 @@ odp_dpdk_mbuf_ctor(struct rte_mempool *mp, pkt_hdr->uarea_addr = uarea; } + /* Initialize data endmark */ + if (mb_ctor_arg->type == ODP_POOL_BUFFER || mb_ctor_arg->type == ODP_POOL_PACKET) { + mb->buf_len -= _ODP_EV_ENDMARK_SIZE; + _odp_event_endmark_set(_odp_event_from_mbuf(mb)); + } + /* Initialize event vector metadata */ if (mb_ctor_arg->type == ODP_POOL_VECTOR) { odp_event_vector_hdr_t *vect_hdr = (void *)raw_mbuf; @@ -679,6 +687,7 @@ odp_pool_t _odp_pool_create(const char *name, const odp_pool_param_t *params, uint32_t buf_align, blk_size, headroom, tailroom, min_seg_len; uint32_t max_len, min_align; uint32_t uarea_size = 0; + uint32_t trailer_size = 0; int8_t event_type; char pool_name[ODP_POOL_NAME_LEN]; char rte_name[RTE_MEMPOOL_NAMESIZE]; @@ -709,7 +718,8 @@ odp_pool_t _odp_pool_create(const char *name, const odp_pool_param_t *params, switch (type) { case ODP_POOL_BUFFER: buf_align = params->buf.align; - blk_size = params->buf.size; + trailer_size = _ODP_EV_ENDMARK_SIZE; + blk_size = params->buf.size + trailer_size; cache_size = params->buf.cache_size; uarea_size = params->buf.uarea_size; @@ -734,6 +744,7 @@ odp_pool_t _odp_pool_create(const char *name, const odp_pool_param_t *params, case ODP_POOL_PACKET: headroom = RTE_PKTMBUF_HEADROOM; tailroom = CONFIG_PACKET_TAILROOM; + trailer_size = _ODP_EV_ENDMARK_SIZE; min_seg_len = CONFIG_PACKET_SEG_LEN_MIN; min_align = ODP_CONFIG_BUFFER_ALIGN_MIN; cache_size = params->pkt.cache_size; @@ -743,16 +754,16 @@ odp_pool_t _odp_pool_create(const char *name, const odp_pool_param_t *params, blk_size = params->pkt.seg_len; if (params->pkt.len > blk_size) blk_size = params->pkt.len; - /* Make sure at least one max len packet fits in the - * pool. - */ + + /* Make sure at least one max len packet fits in the pool */ max_len = 0; if (params->pkt.max_len != 0) max_len = params->pkt.max_len; if ((max_len + blk_size) / blk_size > params->pkt.num) blk_size = (max_len + params->pkt.num) / params->pkt.num; - blk_size = _ODP_ROUNDUP_ALIGN(headroom + blk_size + tailroom, min_align); + blk_size = _ODP_ROUNDUP_ALIGN(headroom + blk_size + tailroom + trailer_size, + min_align); /* Segment size minus headroom might be rounded down by the driver (e.g. * ixgbe) to the nearest multiple of 1024. Round it up here to make sure the @@ -827,7 +838,7 @@ odp_pool_t _odp_pool_create(const char *name, const odp_pool_param_t *params, mp = rte_pktmbuf_pool_create(rte_name, num, cache_size, priv_size, data_room_size, rte_socket_id()); - pool->seg_len = data_room_size; + pool->seg_len = data_room_size - trailer_size; } else { unsigned int priv_size; @@ -855,6 +866,7 @@ odp_pool_t _odp_pool_create(const char *name, const odp_pool_param_t *params, pool->type = type; pool->type_2 = type_2; pool->params = *params; + pool->trailer_size = trailer_size; if (reserve_uarea(pool, uarea_size, num)) { _ODP_ERR("User area SHM reserve failed\n"); @@ -1157,7 +1169,7 @@ int odp_pool_ext_capability(odp_pool_type_t type, capa->pkt.max_num_buf = _odp_pool_glb->config.pkt_max_num; capa->pkt.max_buf_size = MAX_SIZE; capa->pkt.odp_header_size = SIZEOF_OBJHDR + sizeof(odp_packet_hdr_t); - capa->pkt.odp_trailer_size = 0; + capa->pkt.odp_trailer_size = _ODP_EV_ENDMARK_SIZE; capa->pkt.min_mem_align = ODP_CACHE_LINE_SIZE; capa->pkt.min_buf_align = ODP_CACHE_LINE_SIZE; capa->pkt.min_head_align = EXT_MIN_HEAD_ALIGN; @@ -1326,12 +1338,13 @@ odp_pool_t odp_pool_ext_create(const char *name, pool->ext_param = *params; pool->ext_head_offset = hdr_size; + pool->trailer_size = _ODP_EV_ENDMARK_SIZE; pool->num = num; pool->num_populated = 0; pool->params.pkt.uarea_size = params->pkt.uarea_size; pool->params.type = params->type; pool->pool_ext = 1; - pool->seg_len = blk_size; + pool->seg_len = blk_size - pool->trailer_size; pool->type = params->type; strcpy(pool->name, pool_name); @@ -1449,7 +1462,7 @@ int odp_pool_ext_populate(odp_pool_t pool_hdl, void *buf[], uint32_t buf_size, mb_ctor_arg.seg_buf_offset = sizeof(odp_packet_hdr_t) + params->pkt.app_header_size; - mb_ctor_arg.seg_buf_size = pool->seg_len; + mb_ctor_arg.seg_buf_size = pool->seg_len + pool->trailer_size; mb_ctor_arg.type = params->type; mb_ctor_arg.event_type = pool->type; mb_ctor_arg.pool = pool; diff --git a/platform/linux-dpdk/odp_queue_basic.c b/platform/linux-dpdk/odp_queue_basic.c index a83164e2f..ebe86ef1b 100644 --- a/platform/linux-dpdk/odp_queue_basic.c +++ b/platform/linux-dpdk/odp_queue_basic.c @@ -1,43 +1,46 @@ /* Copyright (c) 2013-2018, Linaro Limited - * Copyright (c) 2021, Nokia + * Copyright (c) 2021-2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause */ -#include <odp/api/queue.h> -#include <odp_queue_basic_internal.h> -#include <odp_queue_if.h> -#include <odp/api/std_types.h> #include <odp/api/align.h> -#include <odp_pool_internal.h> -#include <odp_init_internal.h> -#include <odp/api/shared_memory.h> -#include <odp/api/schedule.h> -#include <odp_schedule_if.h> -#include <odp_config_internal.h> -#include <odp_packet_io_internal.h> -#include <odp_debug_internal.h> -#include <odp_macros_internal.h> #include <odp/api/hints.h> +#include <odp/api/packet_io.h> +#include <odp/api/queue.h> +#include <odp/api/schedule.h> +#include <odp/api/shared_memory.h> +#include <odp/api/std_types.h> #include <odp/api/sync.h> -#include <odp/api/plat/sync_inlines.h> +#include <odp/api/ticketlock.h> #include <odp/api/traffic_mngr.h> -#include <odp_libconfig_internal.h> -#include <odp_timer_internal.h> + #include <odp/api/plat/queue_inline_types.h> +#include <odp/api/plat/sync_inlines.h> +#include <odp/api/plat/ticketlock_inlines.h> + +#include <odp_config_internal.h> +#include <odp_debug_internal.h> +#include <odp_event_internal.h> #include <odp_global_data.h> +#include <odp_init_internal.h> +#include <odp_libconfig_internal.h> +#include <odp_macros_internal.h> +#include <odp_packet_io_internal.h> +#include <odp_pool_internal.h> #include <odp_queue_basic_internal.h> -#include <odp_event_internal.h> +#include <odp_queue_if.h> +#include <odp_schedule_if.h> +#include <odp_timer_internal.h> + +#include <inttypes.h> +#include <string.h> -#include <odp/api/plat/ticketlock_inlines.h> #define LOCK(queue_ptr) odp_ticketlock_lock(&((queue_ptr)->lock)) #define UNLOCK(queue_ptr) odp_ticketlock_unlock(&((queue_ptr)->lock)) #define LOCK_INIT(queue_ptr) odp_ticketlock_init(&((queue_ptr)->lock)) -#include <string.h> -#include <inttypes.h> - #define MIN_QUEUE_SIZE (32 - 1) #define MAX_QUEUE_SIZE ((1 * 1024 * 1024) - 1) diff --git a/platform/linux-dpdk/odp_shared_memory.c b/platform/linux-dpdk/odp_shared_memory.c index 470b78eb5..871a22ed7 100644 --- a/platform/linux-dpdk/odp_shared_memory.c +++ b/platform/linux-dpdk/odp_shared_memory.c @@ -1,5 +1,5 @@ /* Copyright (c) 2017-2018, Linaro Limited - * Copyright (c) 2021-2022, Nokia + * Copyright (c) 2021-2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -8,7 +8,6 @@ #include <odp_posix_extensions.h> #include <odp/api/debug.h> -#include <odp/api/deprecated.h> #include <odp/api/shared_memory.h> #include <odp/api/spinlock.h> @@ -32,14 +31,7 @@ #include <rte_memzone.h> /* Supported ODP_SHM_* flags */ -#if ODP_DEPRECATED_API - #define DEPRECATED_SHM_FLAGS (ODP_SHM_SW_ONLY) -#else - #define DEPRECATED_SHM_FLAGS 0 -#endif - -#define SUPPORTED_SHM_FLAGS (ODP_SHM_EXPORT | ODP_SHM_HP | \ - ODP_SHM_SINGLE_VA | DEPRECATED_SHM_FLAGS) +#define SUPPORTED_SHM_FLAGS (ODP_SHM_EXPORT | ODP_SHM_HP | ODP_SHM_SINGLE_VA) #define SHM_MAX_ALIGN (0x80000000) #define SHM_BLOCK_NAME "%" PRIu64 "-%d-%s" @@ -299,8 +291,7 @@ odp_shm_t odp_shm_reserve(const char *name, uint64_t size, uint64_t align, block->type = SHM_TYPE_LOCAL; block->size = size; - /* Note: ODP_SHM_SW_ONLY/ODP_SHM_PROC/ODP_SHM_SINGLE_VA flags are - * currently ignored. */ + /* Note: ODP_SHM_PROC/ODP_SHM_SINGLE_VA flags are currently ignored. */ shm_zone(mz)->flags = flags; odp_spinlock_unlock(&shm_tbl->lock); diff --git a/platform/linux-dpdk/odp_timer.c b/platform/linux-dpdk/odp_timer.c index 5296a9d3c..1021a96cd 100644 --- a/platform/linux-dpdk/odp_timer.c +++ b/platform/linux-dpdk/odp_timer.c @@ -1,5 +1,5 @@ /* Copyright (c) 2018, Linaro Limited - * Copyright (c) 2019-2022, Nokia + * Copyright (c) 2019-2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -7,6 +7,7 @@ #include <odp_posix_extensions.h> +#include <odp/api/deprecated.h> #include <odp/api/queue.h> #include <odp/api/shared_memory.h> #include <odp/api/std.h> @@ -1015,14 +1016,14 @@ retry: return ODP_TIMER_SUCCESS; } -int odp_timer_set_abs(odp_timer_t timer_hdl, uint64_t abs_tick, - odp_event_t *tmo_ev) +int ODP_DEPRECATE(odp_timer_set_abs)(odp_timer_t timer_hdl, uint64_t abs_tick, + odp_event_t *tmo_ev) { return timer_set(timer_hdl, abs_tick, tmo_ev, 1); } -int odp_timer_set_rel(odp_timer_t timer_hdl, uint64_t rel_tick, - odp_event_t *tmo_ev) +int ODP_DEPRECATE(odp_timer_set_rel)(odp_timer_t timer_hdl, uint64_t rel_tick, + odp_event_t *tmo_ev) { return timer_set(timer_hdl, rel_tick, tmo_ev, 0); } diff --git a/platform/linux-dpdk/test/alternate-timer.conf b/platform/linux-dpdk/test/alternate-timer.conf index 0b326f259..d976d05fe 100644 --- a/platform/linux-dpdk/test/alternate-timer.conf +++ b/platform/linux-dpdk/test/alternate-timer.conf @@ -1,6 +1,6 @@ # Mandatory fields odp_implementation = "linux-dpdk" -config_file_version = "0.1.18" +config_file_version = "0.1.21" timer: { # Enable alternate DPDK timer implementation diff --git a/platform/linux-dpdk/test/crypto.conf b/platform/linux-dpdk/test/crypto.conf index f3d642963..1fcaf658e 100644 --- a/platform/linux-dpdk/test/crypto.conf +++ b/platform/linux-dpdk/test/crypto.conf @@ -1,6 +1,6 @@ # Mandatory fields odp_implementation = "linux-dpdk" -config_file_version = "0.1.18" +config_file_version = "0.1.21" system: { # One crypto queue pair is required per thread for lockless operation diff --git a/platform/linux-dpdk/test/process-mode.conf b/platform/linux-dpdk/test/process-mode.conf index b95f50ea2..7a98cef10 100644 --- a/platform/linux-dpdk/test/process-mode.conf +++ b/platform/linux-dpdk/test/process-mode.conf @@ -1,6 +1,6 @@ # Mandatory fields odp_implementation = "linux-dpdk" -config_file_version = "0.1.18" +config_file_version = "0.1.21" dpdk: { process_mode_memory_mb = 1024 diff --git a/platform/linux-dpdk/test/sched-basic.conf b/platform/linux-dpdk/test/sched-basic.conf index 181136c33..e3d9dd304 100644 --- a/platform/linux-dpdk/test/sched-basic.conf +++ b/platform/linux-dpdk/test/sched-basic.conf @@ -1,9 +1,10 @@ # Mandatory fields odp_implementation = "linux-dpdk" -config_file_version = "0.1.18" +config_file_version = "0.1.21" # Test scheduler with an odd spread value and without dynamic load balance sched_basic: { prio_spread = 3 load_balance = 0 + order_stash_size = 0 } diff --git a/platform/linux-dpdk/test/stash-custom.conf b/platform/linux-dpdk/test/stash-custom.conf new file mode 100644 index 000000000..5e3ea5cdf --- /dev/null +++ b/platform/linux-dpdk/test/stash-custom.conf @@ -0,0 +1,8 @@ +# Mandatory fields +odp_implementation = "linux-dpdk" +config_file_version = "0.1.21" + +# Test overflow safe stash variant +stash: { + strict_size = 0 +} diff --git a/platform/linux-generic/Makefile.am b/platform/linux-generic/Makefile.am index e762148aa..d49a2138b 100644 --- a/platform/linux-generic/Makefile.am +++ b/platform/linux-generic/Makefile.am @@ -34,9 +34,11 @@ odpapiplatinclude_HEADERS = \ include/odp/api/plat/buffer_inline_types.h \ include/odp/api/plat/byteorder_inlines.h \ include/odp/api/plat/cpu_inlines.h \ + include/odp/api/plat/crypto_inlines.h \ include/odp/api/plat/debug_inlines.h \ include/odp/api/plat/event_inlines.h \ include/odp/api/plat/event_inline_types.h \ + include/odp/api/plat/event_validation_external.h \ include/odp/api/plat/event_vector_inline_types.h \ include/odp/api/plat/hash_inlines.h \ include/odp/api/plat/ipsec_inlines.h \ @@ -70,11 +72,13 @@ odpapiabiarchinclude_HEADERS += \ include-abi/odp/api/abi/atomic.h \ include-abi/odp/api/abi/barrier.h \ include-abi/odp/api/abi/buffer.h \ + include-abi/odp/api/abi/buffer_types.h \ include-abi/odp/api/abi/byteorder.h \ include-abi/odp/api/abi/classification.h \ include-abi/odp/api/abi/comp.h \ include-abi/odp/api/abi/cpumask.h \ include-abi/odp/api/abi/crypto.h \ + include-abi/odp/api/abi/crypto_types.h \ include-abi/odp/api/abi/debug.h \ include-abi/odp/api/abi/dma_types.h \ include-abi/odp/api/abi/errno.h \ @@ -129,6 +133,7 @@ noinst_HEADERS = \ include/odp_debug_internal.h \ include/odp_errno_define.h \ include/odp_event_internal.h \ + include/odp_event_validation_internal.h \ include/odp_fdserver_internal.h \ include/odp_forward_typedefs_internal.h \ include/odp_global_data.h \ @@ -164,6 +169,8 @@ noinst_HEADERS = \ include/odp_ring_common.h \ include/odp_ring_internal.h \ include/odp_ring_mpmc_internal.h \ + include/odp_ring_mpmc_u32_internal.h \ + include/odp_ring_mpmc_u64_internal.h \ include/odp_ring_ptr_internal.h \ include/odp_ring_spsc_internal.h \ include/odp_ring_st_internal.h \ @@ -205,6 +212,7 @@ __LIB__libodp_linux_la_SOURCES = \ odp_dma.c \ odp_errno.c \ odp_event.c \ + odp_event_validation.c \ odp_fdserver.c \ odp_hash_crc_gen.c \ odp_impl.c \ @@ -278,16 +286,22 @@ if WITH_ARMV8_CRYPTO __LIB__libodp_linux_la_SOURCES += \ arch/aarch64/odp_crypto_armv8.c else +if WITH_IPSECMB_CRYPTO +__LIB__libodp_linux_la_SOURCES += \ + odp_crypto_ipsecmb.c +else __LIB__libodp_linux_la_SOURCES += \ odp_crypto_null.c endif endif +endif if ODP_ABI_COMPAT __LIB__libodp_linux_la_SOURCES += \ odp_atomic_api.c \ odp_buffer_api.c \ odp_byteorder_api.c \ odp_cpu_api.c \ + odp_crypto_api.c \ odp_event_api.c \ odp_hash_api.c \ odp_ipsec_api.c \ @@ -438,6 +452,7 @@ __LIB__libodp_linux_la_LIBADD += $(DPDK_LIBS_LIBODP) __LIB__libodp_linux_la_LIBADD += $(PTHREAD_LIBS) __LIB__libodp_linux_la_LIBADD += $(TIMER_LIBS) __LIB__libodp_linux_la_LIBADD += $(LIBXDP_LIBS) +__LIB__libodp_linux_la_LIBADD += $(IPSEC_MB_LIBS) if ODP_PKTIO_PCAP __LIB__libodp_linux_la_LIBADD += $(PCAP_LIBS) diff --git a/platform/linux-generic/README b/platform/linux-generic/README index 8f41d1d45..138e6040c 100644 --- a/platform/linux-generic/README +++ b/platform/linux-generic/README @@ -1,5 +1,5 @@ Copyright (c) 2014-2018, Linaro Limited -Copyright (c) 2019, Nokia +Copyright (c) 2019-2023, Nokia All rights reserved. SPDX-License-Identifier: BSD-3-Clause @@ -52,7 +52,29 @@ SPDX-License-Identifier: BSD-3-Clause Note that there may be issues with the quality or security of rdrand and rdseed. [2] -6. References +6. Event validation + ODP linux-generic implementation supports additional fast path event + validity checks which are disabled by default to minimize overhead. These + checks can be enabled with --enable-event-validation [abort/warn] or + --enabled-debug=full configuration options. + + Event validation adds additional endmark data to ODP buffers and packets, + which is used to detect data writes outside allowed areas. Endmarks are + checked by the implementation each time application calls one the following + API functions: + - odp_buffer_free() / odp_buffer_free_multi() + - odp_buffer_is_valid() + - odp_event_free() / odp_event_free_multi() / odp_event_free_sp() + - odp_event_is_valid() + - odp_packet_free() / odp_packet_free_multi() / odp_packet_free_sp() + - odp_packet_is_valid() + - odp_queue_enq() / odp_queue_enq_multi() + + Event validation can function in two modes: abort (default) and warn. In + abort mode the application is terminated immediately if an event validity + check fails. In warn mode only an error log message is printed. + +7. References [1] Intel Digital Random Number Generator (DRNG) Software Implementation Guide. John P Mechalas, 17 October 2018. https://www.intel.com/content/www/us/en/developer/articles/guide/intel-digital-random-number-generator-drng-software-implementation-guide.html diff --git a/platform/linux-generic/arch/aarch64/odp_crypto_armv8.c b/platform/linux-generic/arch/aarch64/odp_crypto_armv8.c index 11fadc971..67ae6a389 100644 --- a/platform/linux-generic/arch/aarch64/odp_crypto_armv8.c +++ b/platform/linux-generic/arch/aarch64/odp_crypto_armv8.c @@ -1,6 +1,6 @@ /* Copyright (c) 2014-2018, Linaro Limited * Copyright (c) 2021, ARM Limited - * Copyright (c) 2022, Nokia + * Copyright (c) 2022-2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -38,6 +38,10 @@ #define ARM_CRYPTO_MAX_DATA_LENGTH 65536 #define ARM_CRYPTO_MAX_DIGEST_LENGTH 16 +#define AES_GCM_IV_LEN 12 +ODP_STATIC_ASSERT(AES_GCM_IV_LEN <= ARM_CRYPTO_MAX_IV_LENGTH, + "AES_GCM_IV_LEN exceeds ARM_CRYPTO_MAX_IV_LENGTH"); + /* * ARM crypto library may read up to 15 bytes past the end of input * data and AAD and write up to 15 bytes past the end of output data. @@ -70,9 +74,9 @@ static const odp_crypto_cipher_capability_t cipher_capa_null[] = { #ifdef __ARM_FEATURE_AES static const odp_crypto_cipher_capability_t cipher_capa_aes_gcm[] = { -{.key_len = 16, .iv_len = 12}, -{.key_len = 24, .iv_len = 12}, -{.key_len = 32, .iv_len = 12} }; +{.key_len = 16, .iv_len = AES_GCM_IV_LEN}, +{.key_len = 24, .iv_len = AES_GCM_IV_LEN}, +{.key_len = 32, .iv_len = AES_GCM_IV_LEN} }; #endif /* @@ -111,18 +115,11 @@ struct odp_crypto_generic_session_t { odp_crypto_session_param_t p; struct { -#if ODP_DEPRECATED_API - /* Copy of session IV data */ - uint8_t iv_data[ARM_CRYPTO_MAX_IV_LENGTH]; -#endif uint8_t key_data[ARM_CRYPTO_MAX_CIPHER_KEY_LENGTH]; } cipher; struct { uint8_t key[ARM_CRYPTO_MAX_AUTH_KEY_LENGTH]; -#if ODP_DEPRECATED_API - uint8_t iv_data[ARM_CRYPTO_MAX_IV_LENGTH]; -#endif } auth; crypto_func_t func; @@ -176,21 +173,13 @@ void free_session(odp_crypto_generic_session_t *session) odp_spinlock_unlock(&global->lock); } -static -odp_crypto_packet_result_t *get_op_result_from_packet(odp_packet_t pkt) -{ - odp_packet_hdr_t *hdr = packet_hdr(pkt); - - return &hdr->crypto_op_result; -} - static inline void set_crypto_op_result(odp_packet_t pkt, odp_crypto_alg_err_t cipher_err, odp_crypto_alg_err_t auth_err) { odp_crypto_packet_result_t *op_result; - op_result = get_op_result_from_packet(pkt); + op_result = &packet_hdr(pkt)->crypto_op_result; op_result->cipher_status.alg_err = cipher_err; op_result->cipher_status.hw_err = ODP_CRYPTO_HW_ERR_NONE; op_result->auth_status.alg_err = auth_err; @@ -235,8 +224,8 @@ void aes_gcm_encrypt(odp_packet_t pkt, .d = {0, 0} } }; - uint8_t *iv_ptr; - uint64_t iv_bit_length = session->p.cipher_iv_len * 8; + uint8_t iv_data[ARM_CRYPTO_MAX_IV_LENGTH]; + uint64_t iv_bit_length = AES_GCM_IV_LEN * 8; uint64_t plaintext_bit_length = param->cipher_range.length * 8; uint64_t aad_bit_length = session->p.auth_aad_len * 8; uint32_t in_pos = param->cipher_range.offset; @@ -252,21 +241,13 @@ void aes_gcm_encrypt(odp_packet_t pkt, goto err; } -#if ODP_DEPRECATED_API - if (param->cipher_iv_ptr) - iv_ptr = param->cipher_iv_ptr; - else if (session->p.cipher_iv.data) - iv_ptr = session->cipher.iv_data; - else - goto err; -#else - iv_ptr = param->cipher_iv_ptr; - _ODP_ASSERT(session->p.cipher_iv_len == 0 || iv_ptr != NULL); -#endif + /* The crypto lib may read 16 bytes. Copy to a big enough buffer */ + _ODP_ASSERT(param->cipher_iv_ptr != NULL); + memcpy(iv_data, param->cipher_iv_ptr, AES_GCM_IV_LEN); cs.constants = &session->cc; - rc = armv8_aes_gcm_set_counter(iv_ptr, iv_bit_length, &cs); + rc = armv8_aes_gcm_set_counter(iv_data, iv_bit_length, &cs); if (odp_unlikely(rc)) { _ODP_DBG("ARM Crypto: Failure while setting nonce\n"); goto err; @@ -335,9 +316,9 @@ void aes_gcm_decrypt(odp_packet_t pkt, .d = {0, 0} } }; - uint8_t *iv_ptr; + uint8_t iv_data[ARM_CRYPTO_MAX_IV_LENGTH]; uint8_t tag[AES_GCM_TAG_LEN]; - uint64_t iv_bit_length = session->p.cipher_iv_len * 8; + uint64_t iv_bit_length = AES_GCM_IV_LEN * 8; uint64_t plaintext_bit_length = param->cipher_range.length * 8; uint64_t aad_bit_length = session->p.auth_aad_len * 8; uint32_t in_pos = param->cipher_range.offset; @@ -352,21 +333,13 @@ void aes_gcm_decrypt(odp_packet_t pkt, goto err; } -#if ODP_DEPRECATED_API - if (param->cipher_iv_ptr) - iv_ptr = param->cipher_iv_ptr; - else if (session->p.cipher_iv.data) - iv_ptr = session->cipher.iv_data; - else - goto err; -#else - iv_ptr = param->cipher_iv_ptr; - _ODP_ASSERT(session->p.cipher_iv_len == 0 || iv_ptr != NULL); -#endif + /* The crypto lib may read 16 bytes. Copy to a big enough buffer */ + _ODP_ASSERT(param->cipher_iv_ptr != NULL); + memcpy(iv_data, param->cipher_iv_ptr, AES_GCM_IV_LEN); cs.constants = &session->cc; - rc = armv8_aes_gcm_set_counter(iv_ptr, iv_bit_length, &cs); + rc = armv8_aes_gcm_set_counter(iv_data, iv_bit_length, &cs); if (odp_unlikely(rc)) { _ODP_DBG("ARM Crypto: Failure while setting nonce\n"); goto err; @@ -433,7 +406,7 @@ static int process_aes_gcm_param(odp_crypto_generic_session_t *session) return -1; /* Verify IV len is correct */ - if (12 != session->p.cipher_iv_len) + if (session->p.cipher_iv_len != AES_GCM_IV_LEN) return -1; if (ARM_CRYPTO_MAX_CIPHER_KEY_LENGTH < session->p.cipher_key.length) @@ -555,6 +528,12 @@ odp_crypto_session_create(const odp_crypto_session_param_t *param, return -1; } + if (param->op_type == ODP_CRYPTO_OP_TYPE_OOP) { + *status = ODP_CRYPTO_SES_ERR_PARAMS; + *session_out = ODP_CRYPTO_SESSION_INVALID; + return -1; + } + /* Allocate memory for this session */ session = alloc_session(); if (NULL == session) { @@ -577,17 +556,6 @@ odp_crypto_session_create(const odp_crypto_session_param_t *param, goto err; } -#if ODP_DEPRECATED_API - /* Copy IV data */ - if (session->p.cipher_iv.data) - memcpy(session->cipher.iv_data, session->p.cipher_iv.data, - session->p.cipher_iv.length); - - if (session->p.auth_iv.data) - memcpy(session->auth.iv_data, session->p.auth_iv.data, - session->p.auth_iv.length); -#endif - /* Process based on cipher */ switch (param->cipher_alg) { case ODP_CIPHER_ALG_NULL: @@ -705,6 +673,10 @@ odp_crypto_operation(odp_crypto_op_param_t *param, odp_crypto_op_result_t local_result; int rc; + if (((odp_crypto_generic_session_t *)(intptr_t)param->session)->p.op_type != + ODP_CRYPTO_OP_TYPE_LEGACY) + return -1; + packet_param.session = param->session; packet_param.cipher_iv_ptr = param->cipher_iv_ptr; packet_param.auth_iv_ptr = param->auth_iv_ptr; @@ -877,35 +849,6 @@ uint64_t odp_crypto_session_to_u64(odp_crypto_session_t hdl) return (uint64_t)hdl; } -odp_packet_t odp_crypto_packet_from_event(odp_event_t ev) -{ - /* This check not mandated by the API specification */ - _ODP_ASSERT(odp_event_type(ev) == ODP_EVENT_PACKET); - _ODP_ASSERT(odp_event_subtype(ev) == ODP_EVENT_PACKET_CRYPTO); - - return odp_packet_from_event(ev); -} - -odp_event_t odp_crypto_packet_to_event(odp_packet_t pkt) -{ - return odp_packet_to_event(pkt); -} - -int odp_crypto_result(odp_crypto_packet_result_t *result, - odp_packet_t packet) -{ - odp_crypto_packet_result_t *op_result; - - _ODP_ASSERT(odp_event_subtype(odp_packet_to_event(packet)) == - ODP_EVENT_PACKET_CRYPTO); - - op_result = get_op_result_from_packet(packet); - - memcpy(result, op_result, sizeof(*result)); - - return 0; -} - static int copy_data_and_metadata(odp_packet_t dst, odp_packet_t src) { int md_copy; @@ -934,6 +877,9 @@ static odp_packet_t get_output_packet(const odp_crypto_generic_session_t *sessio { int rc; + if (odp_likely(session->p.op_type == ODP_CRYPTO_OP_TYPE_BASIC)) + return pkt_in; + if (odp_likely(pkt_in == pkt_out)) return pkt_out; @@ -1018,7 +964,9 @@ int odp_crypto_op_enq(const odp_packet_t pkt_in[], _ODP_ASSERT(ODP_CRYPTO_ASYNC == session->p.op_mode); _ODP_ASSERT(ODP_QUEUE_INVALID != session->p.compl_queue); - pkt = pkt_out[i]; + if (session->p.op_type != ODP_CRYPTO_OP_TYPE_BASIC) + pkt = pkt_out[i]; + rc = crypto_int(pkt_in[i], &pkt, ¶m[i]); if (rc < 0) break; diff --git a/platform/linux-generic/include-abi/odp/api/abi/buffer.h b/platform/linux-generic/include-abi/odp/api/abi/buffer.h index 8239e15da..a6309fe39 100644 --- a/platform/linux-generic/include-abi/odp/api/abi/buffer.h +++ b/platform/linux-generic/include-abi/odp/api/abi/buffer.h @@ -1,4 +1,5 @@ /* Copyright (c) 2015-2018, Linaro Limited + * Copyright (c) 2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -7,7 +8,7 @@ /** * @file * - * ODP buffer descriptor + * ODP buffer */ #ifndef ODP_API_ABI_BUFFER_H_ @@ -17,24 +18,9 @@ extern "C" { #endif -#include <odp/api/std_types.h> -#include <odp/api/plat/strong_types.h> - -/** @ingroup odp_buffer - * @{ - */ - -typedef ODP_HANDLE_T(odp_buffer_t); - -#define ODP_BUFFER_INVALID _odp_cast_scalar(odp_buffer_t, 0) - -/* Inlined functions for non-ABI compat mode */ +/* Inlined API functions */ #include <odp/api/plat/buffer_inlines.h> -/** - * @} - */ - #ifdef __cplusplus } #endif diff --git a/platform/linux-generic/include-abi/odp/api/abi/buffer_types.h b/platform/linux-generic/include-abi/odp/api/abi/buffer_types.h new file mode 100644 index 000000000..1d54bab07 --- /dev/null +++ b/platform/linux-generic/include-abi/odp/api/abi/buffer_types.h @@ -0,0 +1,40 @@ +/* Copyright (c) 2015-2018, Linaro Limited + * Copyright (c) 2023, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +/** + * @file + * + * ODP buffer types + */ + +#ifndef ODP_API_ABI_BUFFER_TYPES_H_ +#define ODP_API_ABI_BUFFER_TYPES_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <odp/api/std_types.h> +#include <odp/api/plat/strong_types.h> + +/** @ingroup odp_buffer + * @{ + */ + +typedef ODP_HANDLE_T(odp_buffer_t); + +#define ODP_BUFFER_INVALID _odp_cast_scalar(odp_buffer_t, 0) + +/** + * @} + */ + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/include-abi/odp/api/abi/crypto.h b/platform/linux-generic/include-abi/odp/api/abi/crypto.h index b57667e76..bef725c28 100644 --- a/platform/linux-generic/include-abi/odp/api/abi/crypto.h +++ b/platform/linux-generic/include-abi/odp/api/abi/crypto.h @@ -1,4 +1,5 @@ /* Copyright (c) 2015-2018, Linaro Limited + * Copyright (c) 2022, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -18,18 +19,8 @@ extern "C" { #endif -/** @ingroup odp_crypto - * @{ - */ - -#define ODP_CRYPTO_SESSION_INVALID (0xffffffffffffffffULL) - -typedef uint64_t odp_crypto_session_t; -typedef ODP_HANDLE_T(odp_crypto_compl_t); - -/** - * @} - */ +/* Inlined API functions */ +#include <odp/api/plat/crypto_inlines.h> #ifdef __cplusplus } diff --git a/platform/linux-generic/include-abi/odp/api/abi/crypto_types.h b/platform/linux-generic/include-abi/odp/api/abi/crypto_types.h new file mode 100644 index 000000000..a5cb43c5d --- /dev/null +++ b/platform/linux-generic/include-abi/odp/api/abi/crypto_types.h @@ -0,0 +1,42 @@ +/* Copyright (c) 2015-2018, Linaro Limited + * Copyright (c) 2022, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +/** + * @file + * + * ODP crypto + */ + +#ifndef ODP_API_ABI_CRYPTO_TYPES_H_ +#define ODP_API_ABI_CRYPTO_TYPES_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <odp/api/std_types.h> + +#include <odp/api/plat/strong_types.h> + +/** @ingroup odp_crypto + * @{ + */ + +#define ODP_CRYPTO_SESSION_INVALID (0xffffffffffffffffULL) + +typedef uint64_t odp_crypto_session_t; +typedef ODP_HANDLE_T(odp_crypto_compl_t); + +/** + * @} + */ + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/include/odp/api/plat/buffer_inlines.h b/platform/linux-generic/include/odp/api/plat/buffer_inlines.h index 34d4b5675..75ef36cf3 100644 --- a/platform/linux-generic/include/odp/api/plat/buffer_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/buffer_inlines.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2019-2022, Nokia +/* Copyright (c) 2019-2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -7,11 +7,10 @@ #ifndef ODP_PLAT_BUFFER_INLINES_H_ #define ODP_PLAT_BUFFER_INLINES_H_ +#include <odp/api/buffer_types.h> #include <odp/api/event.h> #include <odp/api/pool_types.h> -#include <odp/api/abi/buffer.h> - #include <odp/api/plat/buffer_inline_types.h> #include <odp/api/plat/debug_inlines.h> #include <odp/api/plat/event_inline_types.h> diff --git a/platform/linux-generic/include/odp/api/plat/crypto_inlines.h b/platform/linux-generic/include/odp/api/plat/crypto_inlines.h new file mode 100644 index 000000000..8e98d8580 --- /dev/null +++ b/platform/linux-generic/include/odp/api/plat/crypto_inlines.h @@ -0,0 +1,65 @@ +/* Copyright (c) 2022, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_PLAT_CRYPTO_INLINES_H_ +#define ODP_PLAT_CRYPTO_INLINES_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <odp/api/crypto_types.h> +#include <odp/api/event.h> +#include <odp/api/packet.h> + +#include <odp/api/plat/debug_inlines.h> +#include <odp/api/plat/packet_inline_types.h> + +/** @cond _ODP_HIDE_FROM_DOXYGEN_ */ + +#ifndef _ODP_NO_INLINE + /* Inline functions by default */ + #define _ODP_INLINE static inline + #define odp_crypto_packet_from_event __odp_crypto_packet_from_event + #define odp_crypto_packet_to_event __odp_crypto_packet_to_event + #define odp_crypto_result __odp_crypto_result +#else + #define _ODP_INLINE +#endif + +_ODP_INLINE odp_packet_t odp_crypto_packet_from_event(odp_event_t ev) +{ + _ODP_ASSERT(odp_event_type(ev) == ODP_EVENT_PACKET); + _ODP_ASSERT(odp_event_subtype(ev) == ODP_EVENT_PACKET_CRYPTO); + + return odp_packet_from_event(ev); +} + +_ODP_INLINE odp_event_t odp_crypto_packet_to_event(odp_packet_t pkt) +{ + return odp_packet_to_event(pkt); +} + +_ODP_INLINE int odp_crypto_result(odp_crypto_packet_result_t *result, odp_packet_t pkt) +{ + odp_crypto_packet_result_t *op_result; + + _ODP_ASSERT(odp_packet_subtype(pkt) == ODP_EVENT_PACKET_CRYPTO); + + op_result = _odp_pkt_get_ptr(pkt, odp_crypto_packet_result_t, crypto_op); + + *result = *op_result; + + return 0; +} + +/** @endcond */ + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/include/odp/api/plat/event_validation_external.h b/platform/linux-generic/include/odp/api/plat/event_validation_external.h new file mode 100644 index 000000000..7f5c0364f --- /dev/null +++ b/platform/linux-generic/include/odp/api/plat/event_validation_external.h @@ -0,0 +1,111 @@ +/* Copyright (c) 2023, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +/** + * @file + * + * ODP event validation + * + * @warning These definitions are not part of ODP API, they are for + * implementation internal use only. + */ + +#ifndef ODP_EVENT_VALIDATION_EXTERNAL_H_ +#define ODP_EVENT_VALIDATION_EXTERNAL_H_ + +#include <odp/autoheader_external.h> + +#include <odp/api/buffer_types.h> +#include <odp/api/event_types.h> +#include <odp/api/hints.h> +#include <odp/api/packet_types.h> + +/** @cond _ODP_HIDE_FROM_DOXYGEN_ */ + +#ifdef __cplusplus +extern "C" { +#endif + +/** Enumerations for identifying ODP API functions */ +typedef enum { + _ODP_EV_BUFFER_FREE = 0, + _ODP_EV_BUFFER_FREE_MULTI, + _ODP_EV_BUFFER_IS_VALID, + _ODP_EV_EVENT_FREE, + _ODP_EV_EVENT_FREE_MULTI, + _ODP_EV_EVENT_FREE_SP, + _ODP_EV_EVENT_IS_VALID, + _ODP_EV_PACKET_FREE, + _ODP_EV_PACKET_FREE_MULTI, + _ODP_EV_PACKET_FREE_SP, + _ODP_EV_PACKET_IS_VALID, + _ODP_EV_QUEUE_ENQ, + _ODP_EV_QUEUE_ENQ_MULTI, + _ODP_EV_MAX +} _odp_ev_id_t; + +/* Implementation internal event validation functions */ +#if _ODP_EVENT_VALIDATION + +int _odp_event_validate(odp_event_t event, _odp_ev_id_t id); + +int _odp_event_validate_multi(const odp_event_t event[], int num, _odp_ev_id_t id); + +int _odp_buffer_validate(odp_buffer_t buf, _odp_ev_id_t ev_id); + +int _odp_buffer_validate_multi(const odp_buffer_t buf[], int num, _odp_ev_id_t ev_id); + +int _odp_packet_validate(odp_packet_t pkt, _odp_ev_id_t ev_id); + +int _odp_packet_validate_multi(const odp_packet_t pkt[], int num, _odp_ev_id_t ev_id); + +#else + +static inline int _odp_event_validate(odp_event_t event ODP_UNUSED, _odp_ev_id_t ev_id ODP_UNUSED) +{ + return 0; +} + +static inline int _odp_event_validate_multi(const odp_event_t event[] ODP_UNUSED, + int num ODP_UNUSED, + _odp_ev_id_t ev_id ODP_UNUSED) +{ + return 0; +} + +static inline int _odp_buffer_validate(odp_buffer_t buf ODP_UNUSED, _odp_ev_id_t ev_id ODP_UNUSED) +{ + return 0; +} + +static inline int _odp_buffer_validate_multi(const odp_buffer_t buf[] ODP_UNUSED, + int num ODP_UNUSED, + _odp_ev_id_t ev_id ODP_UNUSED) +{ + return 0; +} + +static inline int _odp_packet_validate(odp_packet_t pkt ODP_UNUSED, _odp_ev_id_t ev_id ODP_UNUSED) +{ + return 0; +} + +static inline int _odp_packet_validate_multi(const odp_packet_t pkt[] ODP_UNUSED, + int num ODP_UNUSED, + _odp_ev_id_t ev_id ODP_UNUSED) +{ + return 0; +} + +#endif /* _ODP_EVENT_VALIDATION */ + +#ifdef __cplusplus +} +#endif + +/** @endcond */ + +#endif diff --git a/platform/linux-generic/include/odp/api/plat/packet_inline_types.h b/platform/linux-generic/include/odp/api/plat/packet_inline_types.h index ae03457f9..6773b73ad 100644 --- a/platform/linux-generic/include/odp/api/plat/packet_inline_types.h +++ b/platform/linux-generic/include/odp/api/plat/packet_inline_types.h @@ -53,6 +53,7 @@ typedef struct _odp_packet_inline_offset_t { uint16_t subtype; uint16_t cls_mark; uint16_t ipsec_ctx; + uint16_t crypto_op; } _odp_packet_inline_offset_t; diff --git a/platform/linux-generic/include/odp/api/plat/packet_inlines.h b/platform/linux-generic/include/odp/api/plat/packet_inlines.h index 01d47d837..93e95e21c 100644 --- a/platform/linux-generic/include/odp/api/plat/packet_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/packet_inlines.h @@ -632,7 +632,8 @@ _ODP_INLINE uint32_t odp_packet_buf_size(odp_packet_buf_t pkt_buf) odp_pool_t pool = _odp_pkt_get(pkt_buf, odp_pool_t, pool); return _odp_pool_get(pool, uint32_t, ext_pkt_buf_size) - - _odp_pool_get(pool, uint32_t, ext_head_offset); + _odp_pool_get(pool, uint32_t, ext_head_offset) - + _odp_pool_get(pool, uint32_t, trailer_size); } _ODP_INLINE void *odp_packet_buf_head(odp_packet_buf_t pkt_buf) diff --git a/platform/linux-generic/include/odp/api/plat/pool_inline_types.h b/platform/linux-generic/include/odp/api/plat/pool_inline_types.h index 02f59f982..fbff7eda7 100644 --- a/platform/linux-generic/include/odp/api/plat/pool_inline_types.h +++ b/platform/linux-generic/include/odp/api/plat/pool_inline_types.h @@ -30,6 +30,7 @@ typedef struct _odp_pool_inline_offset_t { uint16_t index; uint16_t seg_len; uint16_t uarea_size; + uint16_t trailer_size; uint16_t ext_head_offset; uint16_t ext_pkt_buf_size; diff --git a/platform/linux-generic/include/odp/api/plat/queue_inlines.h b/platform/linux-generic/include/odp/api/plat/queue_inlines.h index 22673a887..609c0c9e4 100644 --- a/platform/linux-generic/include/odp/api/plat/queue_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/queue_inlines.h @@ -1,4 +1,5 @@ /* Copyright (c) 2018, Linaro Limited + * Copyright (c) 2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -7,6 +8,9 @@ #ifndef ODP_PLAT_QUEUE_INLINES_H_ #define ODP_PLAT_QUEUE_INLINES_H_ +#include <odp/api/hints.h> + +#include <odp/api/plat/event_validation_external.h> #include <odp/api/plat/queue_inline_types.h> /** @cond _ODP_HIDE_FROM_DOXYGEN_ */ @@ -37,12 +41,18 @@ _ODP_INLINE void *odp_queue_context(odp_queue_t handle) _ODP_INLINE int odp_queue_enq(odp_queue_t queue, odp_event_t ev) { + if (odp_unlikely(_odp_event_validate(ev, _ODP_EV_QUEUE_ENQ))) + return -1; + return _odp_queue_api->queue_enq(queue, ev); } _ODP_INLINE int odp_queue_enq_multi(odp_queue_t queue, const odp_event_t events[], int num) { + if (odp_unlikely(_odp_event_validate_multi(events, num, _ODP_EV_QUEUE_ENQ_MULTI))) + return -1; + return _odp_queue_api->queue_enq_multi(queue, events, num); } diff --git a/platform/linux-generic/include/odp_config_internal.h b/platform/linux-generic/include/odp_config_internal.h index d3d09abf4..e4f8d6d6d 100644 --- a/platform/linux-generic/include/odp_config_internal.h +++ b/platform/linux-generic/include/odp_config_internal.h @@ -1,5 +1,5 @@ /* Copyright (c) 2016-2018, Linaro Limited - * Copyright (c) 2019-2021, Nokia + * Copyright (c) 2019-2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -66,7 +66,7 @@ extern "C" { /* * Maximum number of stashes */ -#define CONFIG_MAX_STASHES 128 +#define CONFIG_MAX_STASHES 2048 /* * Maximum number of packet IO resources @@ -134,10 +134,10 @@ extern "C" { /* * Number of shared memory blocks reserved for implementation internal use. * - * Each stash requires one SHM block, each pool requires three blocks (buffers, - * ring, user area), and 20 blocks are reserved for per ODP module global data. + * Each pool requires three blocks (buffers, ring, user area), and 20 blocks + * are reserved for per ODP module global data. */ -#define CONFIG_INTERNAL_SHM_BLOCKS (CONFIG_MAX_STASHES + (ODP_CONFIG_POOLS * 3) + 20) +#define CONFIG_INTERNAL_SHM_BLOCKS ((ODP_CONFIG_POOLS * 3) + 20) /* * Maximum number of shared memory blocks. diff --git a/platform/linux-generic/include/odp_event_internal.h b/platform/linux-generic/include/odp_event_internal.h index 5a29e926e..4bc28d708 100644 --- a/platform/linux-generic/include/odp_event_internal.h +++ b/platform/linux-generic/include/odp_event_internal.h @@ -53,7 +53,7 @@ typedef struct _odp_event_hdr_t { /* --- Mostly read only data --- */ - /* Initial buffer tail pointer */ + /* Initial buffer tail pointer and endmark location (if enabled) */ uint8_t *buf_end; /* Combined pool and event index */ @@ -85,6 +85,11 @@ static inline void _odp_event_type_set(odp_event_t event, int ev) _odp_event_hdr(event)->event_type = ev; } +static inline uint64_t *_odp_event_endmark_get_ptr(odp_event_t event) +{ + return (uint64_t *)(uintptr_t)_odp_event_hdr(event)->buf_end; +} + #ifdef __cplusplus } #endif diff --git a/platform/linux-generic/include/odp_event_validation_internal.h b/platform/linux-generic/include/odp_event_validation_internal.h new file mode 100644 index 000000000..f4ac16f31 --- /dev/null +++ b/platform/linux-generic/include/odp_event_validation_internal.h @@ -0,0 +1,52 @@ +/* Copyright (c) 2023, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_EVENT_VALIDATION_INTERNAL_H_ +#define ODP_EVENT_VALIDATION_INTERNAL_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <odp/autoheader_external.h> + +#include <odp/api/event.h> +#include <odp/api/hints.h> + +#include <odp/api/plat/event_validation_external.h> + +#include <odp_event_internal.h> + +#include <stdint.h> + +#if _ODP_EVENT_VALIDATION + +#define _ODP_EV_ENDMARK_VAL 0xDEADBEEFDEADBEEF +#define _ODP_EV_ENDMARK_SIZE (sizeof(uint64_t)) + +static inline void _odp_event_endmark_set(odp_event_t event) +{ + uint64_t *endmark_ptr; + + endmark_ptr = _odp_event_endmark_get_ptr(event); + *endmark_ptr = _ODP_EV_ENDMARK_VAL; +} + +#else + +#define _ODP_EV_ENDMARK_VAL 0 +#define _ODP_EV_ENDMARK_SIZE 0 + +static inline void _odp_event_endmark_set(odp_event_t event ODP_UNUSED) +{ +} + +#endif + +#ifdef __cplusplus +} +#endif +#endif diff --git a/platform/linux-generic/include/odp_init_internal.h b/platform/linux-generic/include/odp_init_internal.h index 2a1039854..24e8346ad 100644 --- a/platform/linux-generic/include/odp_init_internal.h +++ b/platform/linux-generic/include/odp_init_internal.h @@ -33,6 +33,9 @@ int _odp_pool_init_local(void); int _odp_pool_term_global(void); int _odp_pool_term_local(void); +int _odp_event_validation_init_global(void); +int _odp_event_validation_term_global(void); + int _odp_queue_init_global(void); int _odp_queue_term_global(void); diff --git a/platform/linux-generic/include/odp_packet_io_internal.h b/platform/linux-generic/include/odp_packet_io_internal.h index 954602959..187a3a76f 100644 --- a/platform/linux-generic/include/odp_packet_io_internal.h +++ b/platform/linux-generic/include/odp_packet_io_internal.h @@ -1,5 +1,5 @@ /* Copyright (c) 2013-2018, Linaro Limited - * Copyright (c) 2019-2022, Nokia + * Copyright (c) 2019-2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -68,12 +68,8 @@ struct pktio_if_ops; #define PKTIO_PRIVATE_SIZE 33792 #elif defined(_ODP_PKTIO_XDP) #define PKTIO_PRIVATE_SIZE 29696 -#elif defined(_ODP_PKTIO_DPDK) && ODP_CACHE_LINE_SIZE == 128 -#define PKTIO_PRIVATE_SIZE 4160 -#elif defined(_ODP_PKTIO_DPDK) -#define PKTIO_PRIVATE_SIZE 3968 #else -#define PKTIO_PRIVATE_SIZE 384 +#define PKTIO_PRIVATE_SIZE 9216 #endif typedef struct ODP_ALIGNED_CACHE { diff --git a/platform/linux-generic/include/odp_pool_internal.h b/platform/linux-generic/include/odp_pool_internal.h index 1c5b51c3d..c8d2168f3 100644 --- a/platform/linux-generic/include/odp_pool_internal.h +++ b/platform/linux-generic/include/odp_pool_internal.h @@ -87,6 +87,7 @@ typedef struct pool_t { uint32_t block_size; uint32_t block_offset; uint32_t num_populated; + uint32_t trailer_size; uint8_t *base_addr; uint8_t *max_addr; uint8_t *uarea_base_addr; diff --git a/platform/linux-generic/include/odp_queue_basic_internal.h b/platform/linux-generic/include/odp_queue_basic_internal.h index 830f50a9d..3cdcf8600 100644 --- a/platform/linux-generic/include/odp_queue_basic_internal.h +++ b/platform/linux-generic/include/odp_queue_basic_internal.h @@ -1,4 +1,5 @@ /* Copyright (c) 2013-2018, Linaro Limited + * Copyright (c) 2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -23,7 +24,7 @@ extern "C" { #include <odp/api/ticketlock.h> #include <odp_config_internal.h> #include <odp_macros_internal.h> -#include <odp_ring_mpmc_internal.h> +#include <odp_ring_mpmc_u32_internal.h> #include <odp_ring_st_internal.h> #include <odp_ring_spsc_internal.h> #include <odp_queue_lf.h> @@ -47,7 +48,7 @@ typedef struct ODP_ALIGNED_CACHE queue_entry_s { odp_queue_type_t type; /* MPMC ring (2 cache lines). */ - ring_mpmc_t ring_mpmc; + ring_mpmc_u32_t ring_mpmc; odp_ticketlock_t lock; union { diff --git a/platform/linux-generic/include/odp_ring_mpmc_internal.h b/platform/linux-generic/include/odp_ring_mpmc_internal.h index 6ed4dd4d1..e35179267 100644 --- a/platform/linux-generic/include/odp_ring_mpmc_internal.h +++ b/platform/linux-generic/include/odp_ring_mpmc_internal.h @@ -1,4 +1,5 @@ /* Copyright (c) 2018, Linaro Limited + * Copyright (c) 2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -19,6 +20,8 @@ extern "C" { #include <odp/api/plat/atomic_inlines.h> #include <odp/api/plat/cpu_inlines.h> +#include <odp_ring_common.h> + /* Ring of uint32_t data * * Ring stores head and tail counters. Ring indexes are formed from these @@ -34,14 +37,22 @@ extern "C" { * r_tail r_head w_tail w_head * */ -typedef struct { + +struct ring_mpmc_common { odp_atomic_u32_t r_head ODP_ALIGNED_CACHE; odp_atomic_u32_t r_tail; odp_atomic_u32_t w_head ODP_ALIGNED_CACHE; odp_atomic_u32_t w_tail; +}; + +typedef struct ODP_ALIGNED_CACHE { + struct ring_mpmc_common r; +} ring_mpmc_u32_t; -} ring_mpmc_t; +typedef struct ODP_ALIGNED_CACHE { + struct ring_mpmc_common r; +} ring_mpmc_u64_t; static inline int ring_mpmc_cas_u32(odp_atomic_u32_t *atom, uint32_t *old_val, uint32_t new_val) @@ -52,21 +63,63 @@ static inline int ring_mpmc_cas_u32(odp_atomic_u32_t *atom, __ATOMIC_RELAXED); } +#endif /* End of include guards */ + +#undef _ring_mpmc_gen_t +#undef _ring_mpmc_data_t +#undef _RING_MPMC_INIT +#undef _RING_MPMC_DEQ_MULTI +#undef _RING_MPMC_ENQ_MULTI +#undef _RING_MPMC_DEQ_BATCH +#undef _RING_MPMC_ENQ_BATCH +#undef _RING_MPMC_IS_EMPTY +#undef _RING_MPMC_LEN + +/* This header should NOT be included directly. There are no include guards for + * the following types and function definitions! */ +#ifndef _ODP_RING_TYPE +#error Include type specific (u32/u64) ring header instead of this common file. +#endif + +#if _ODP_RING_TYPE == _ODP_RING_TYPE_U32 + #define _ring_mpmc_gen_t ring_mpmc_u32_t + #define _ring_mpmc_data_t uint32_t + + #define _RING_MPMC_INIT ring_mpmc_u32_init + #define _RING_MPMC_DEQ_MULTI ring_mpmc_u32_deq_multi + #define _RING_MPMC_ENQ_MULTI ring_mpmc_u32_enq_multi + #define _RING_MPMC_DEQ_BATCH ring_mpmc_u32_deq_batch + #define _RING_MPMC_ENQ_BATCH ring_mpmc_u32_enq_batch + #define _RING_MPMC_IS_EMPTY ring_mpmc_u32_is_empty + #define _RING_MPMC_LEN ring_mpmc_u32_len +#elif _ODP_RING_TYPE == _ODP_RING_TYPE_U64 + #define _ring_mpmc_gen_t ring_mpmc_u64_t + #define _ring_mpmc_data_t uint64_t + + #define _RING_MPMC_INIT ring_mpmc_u64_init + #define _RING_MPMC_DEQ_MULTI ring_mpmc_u64_deq_multi + #define _RING_MPMC_ENQ_MULTI ring_mpmc_u64_enq_multi + #define _RING_MPMC_DEQ_BATCH ring_mpmc_u64_deq_batch + #define _RING_MPMC_ENQ_BATCH ring_mpmc_u64_enq_batch + #define _RING_MPMC_IS_EMPTY ring_mpmc_u64_is_empty + #define _RING_MPMC_LEN ring_mpmc_u64_len +#endif + /* Initialize ring */ -static inline void ring_mpmc_init(ring_mpmc_t *ring) +static inline void _RING_MPMC_INIT(_ring_mpmc_gen_t *ring) { - odp_atomic_init_u32(&ring->w_head, 0); - odp_atomic_init_u32(&ring->w_tail, 0); - odp_atomic_init_u32(&ring->r_head, 0); - odp_atomic_init_u32(&ring->r_tail, 0); + odp_atomic_init_u32(&ring->r.w_head, 0); + odp_atomic_init_u32(&ring->r.w_tail, 0); + odp_atomic_init_u32(&ring->r.r_head, 0); + odp_atomic_init_u32(&ring->r.r_tail, 0); } -/* Dequeue data from the ring head. Num is smaller than ring size. */ -static inline uint32_t ring_mpmc_deq_multi(ring_mpmc_t *ring, - uint32_t *ring_data, - uint32_t ring_mask, - uint32_t data[], - uint32_t num) +/* Dequeue data from the ring head */ +static inline uint32_t _RING_MPMC_DEQ_MULTI(_ring_mpmc_gen_t *ring, + _ring_mpmc_data_t *ring_data, + uint32_t ring_mask, + _ring_mpmc_data_t data[], + uint32_t num) { uint32_t old_head, new_head, w_tail, num_data, i; @@ -75,9 +128,9 @@ static inline uint32_t ring_mpmc_deq_multi(ring_mpmc_t *ring, * When CAS operation succeeds, this thread owns data between old * and new r_head. */ do { - old_head = odp_atomic_load_acq_u32(&ring->r_head); + old_head = odp_atomic_load_acq_u32(&ring->r.r_head); odp_prefetch(&ring_data[(old_head + 1) & ring_mask]); - w_tail = odp_atomic_load_acq_u32(&ring->w_tail); + w_tail = odp_atomic_load_acq_u32(&ring->r.w_tail); num_data = w_tail - old_head; /* Ring is empty */ @@ -90,7 +143,49 @@ static inline uint32_t ring_mpmc_deq_multi(ring_mpmc_t *ring, new_head = old_head + num; - } while (odp_unlikely(ring_mpmc_cas_u32(&ring->r_head, &old_head, + } while (odp_unlikely(ring_mpmc_cas_u32(&ring->r.r_head, &old_head, + new_head) == 0)); + + /* Read data. This will not move above load acquire of r_head. */ + for (i = 0; i < num; i++) + data[i] = ring_data[(old_head + 1 + i) & ring_mask]; + + /* Wait until other readers have updated the tail */ + while (odp_unlikely(odp_atomic_load_u32(&ring->r.r_tail) != old_head)) + odp_cpu_pause(); + + /* Release the new reader tail, writers acquire it. */ + odp_atomic_store_rel_u32(&ring->r.r_tail, new_head); + + return num; +} + +/* Dequeue num or 0 data from the ring head */ +static inline uint32_t _RING_MPMC_DEQ_BATCH(_ring_mpmc_gen_t *ring, + _ring_mpmc_data_t *ring_data, + uint32_t ring_mask, + _ring_mpmc_data_t data[], + uint32_t num) +{ + uint32_t old_head, new_head, w_tail, num_data, i; + + /* Load acquires ensure that w_tail load happens after r_head load, + * and thus r_head value is always behind or equal to w_tail value. + * When CAS operation succeeds, this thread owns data between old + * and new r_head. */ + do { + old_head = odp_atomic_load_acq_u32(&ring->r.r_head); + odp_prefetch(&ring_data[(old_head + 1) & ring_mask]); + w_tail = odp_atomic_load_acq_u32(&ring->r.w_tail); + num_data = w_tail - old_head; + + /* Not enough data available */ + if (num_data < num) + return 0; + + new_head = old_head + num; + + } while (odp_unlikely(ring_mpmc_cas_u32(&ring->r.r_head, &old_head, new_head) == 0)); /* Read data. This will not move above load acquire of r_head. */ @@ -98,21 +193,21 @@ static inline uint32_t ring_mpmc_deq_multi(ring_mpmc_t *ring, data[i] = ring_data[(old_head + 1 + i) & ring_mask]; /* Wait until other readers have updated the tail */ - while (odp_unlikely(odp_atomic_load_u32(&ring->r_tail) != old_head)) + while (odp_unlikely(odp_atomic_load_u32(&ring->r.r_tail) != old_head)) odp_cpu_pause(); /* Release the new reader tail, writers acquire it. */ - odp_atomic_store_rel_u32(&ring->r_tail, new_head); + odp_atomic_store_rel_u32(&ring->r.r_tail, new_head); return num; } -/* Enqueue multiple data into the ring tail. Num is smaller than ring size. */ -static inline uint32_t ring_mpmc_enq_multi(ring_mpmc_t *ring, - uint32_t *ring_data, - uint32_t ring_mask, - const uint32_t data[], - uint32_t num) +/* Enqueue multiple data into the ring tail */ +static inline uint32_t _RING_MPMC_ENQ_MULTI(_ring_mpmc_gen_t *ring, + _ring_mpmc_data_t *ring_data, + uint32_t ring_mask, + const _ring_mpmc_data_t data[], + uint32_t num) { uint32_t old_head, new_head, r_tail, num_free, i; uint32_t size = ring_mask + 1; @@ -122,8 +217,8 @@ static inline uint32_t ring_mpmc_enq_multi(ring_mpmc_t *ring, * When CAS operation succeeds, this thread owns data between old * and new w_head. */ do { - r_tail = odp_atomic_load_acq_u32(&ring->r_tail); - old_head = odp_atomic_load_acq_u32(&ring->w_head); + r_tail = odp_atomic_load_acq_u32(&ring->r.r_tail); + old_head = odp_atomic_load_acq_u32(&ring->r.w_head); num_free = size - (old_head - r_tail); @@ -137,7 +232,50 @@ static inline uint32_t ring_mpmc_enq_multi(ring_mpmc_t *ring, new_head = old_head + num; - } while (odp_unlikely(ring_mpmc_cas_u32(&ring->w_head, &old_head, + } while (odp_unlikely(ring_mpmc_cas_u32(&ring->r.w_head, &old_head, + new_head) == 0)); + + /* Write data. This will not move above load acquire of w_head. */ + for (i = 0; i < num; i++) + ring_data[(old_head + 1 + i) & ring_mask] = data[i]; + + /* Wait until other writers have updated the tail */ + while (odp_unlikely(odp_atomic_load_u32(&ring->r.w_tail) != old_head)) + odp_cpu_pause(); + + /* Release the new writer tail, readers acquire it. */ + odp_atomic_store_rel_u32(&ring->r.w_tail, new_head); + + return num; +} + +/* Enqueue num or 0 data into the ring tail */ +static inline uint32_t _RING_MPMC_ENQ_BATCH(_ring_mpmc_gen_t *ring, + _ring_mpmc_data_t *ring_data, + uint32_t ring_mask, + const _ring_mpmc_data_t data[], + uint32_t num) +{ + uint32_t old_head, new_head, r_tail, num_free, i; + uint32_t size = ring_mask + 1; + + /* Load acquires ensure that w_head load happens after r_tail load, + * and thus r_tail value is always behind or equal to w_head value. + * When CAS operation succeeds, this thread owns data between old + * and new w_head. */ + do { + r_tail = odp_atomic_load_acq_u32(&ring->r.r_tail); + old_head = odp_atomic_load_acq_u32(&ring->r.w_head); + + num_free = size - (old_head - r_tail); + + /* Not enough free space available */ + if (num_free < num) + return 0; + + new_head = old_head + num; + + } while (odp_unlikely(ring_mpmc_cas_u32(&ring->r.w_head, &old_head, new_head) == 0)); /* Write data. This will not move above load acquire of w_head. */ @@ -145,29 +283,29 @@ static inline uint32_t ring_mpmc_enq_multi(ring_mpmc_t *ring, ring_data[(old_head + 1 + i) & ring_mask] = data[i]; /* Wait until other writers have updated the tail */ - while (odp_unlikely(odp_atomic_load_u32(&ring->w_tail) != old_head)) + while (odp_unlikely(odp_atomic_load_u32(&ring->r.w_tail) != old_head)) odp_cpu_pause(); /* Release the new writer tail, readers acquire it. */ - odp_atomic_store_rel_u32(&ring->w_tail, new_head); + odp_atomic_store_rel_u32(&ring->r.w_tail, new_head); return num; } /* Check if ring is empty */ -static inline int ring_mpmc_is_empty(ring_mpmc_t *ring) +static inline int _RING_MPMC_IS_EMPTY(_ring_mpmc_gen_t *ring) { - uint32_t head = odp_atomic_load_u32(&ring->r_head); - uint32_t tail = odp_atomic_load_u32(&ring->w_tail); + uint32_t head = odp_atomic_load_u32(&ring->r.r_head); + uint32_t tail = odp_atomic_load_u32(&ring->r.w_tail); return head == tail; } /* Return current ring length */ -static inline uint32_t ring_mpmc_length(ring_mpmc_t *ring) +static inline uint32_t _RING_MPMC_LEN(_ring_mpmc_gen_t *ring) { - uint32_t head = odp_atomic_load_u32(&ring->r_head); - uint32_t tail = odp_atomic_load_u32(&ring->w_tail); + uint32_t head = odp_atomic_load_u32(&ring->r.r_head); + uint32_t tail = odp_atomic_load_u32(&ring->r.w_tail); return tail - head; } @@ -175,5 +313,3 @@ static inline uint32_t ring_mpmc_length(ring_mpmc_t *ring) #ifdef __cplusplus } #endif - -#endif diff --git a/platform/linux-generic/include/odp_ring_mpmc_u32_internal.h b/platform/linux-generic/include/odp_ring_mpmc_u32_internal.h new file mode 100644 index 000000000..4699b5b47 --- /dev/null +++ b/platform/linux-generic/include/odp_ring_mpmc_u32_internal.h @@ -0,0 +1,25 @@ +/* Copyright (c) 2023, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_RING_MPMC_U32_INTERNAL_H_ +#define ODP_RING_MPMC_U32_INTERNAL_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <odp_ring_common.h> + +#undef _ODP_RING_TYPE +#define _ODP_RING_TYPE _ODP_RING_TYPE_U32 + +#include <odp_ring_mpmc_internal.h> + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/include/odp_ring_mpmc_u64_internal.h b/platform/linux-generic/include/odp_ring_mpmc_u64_internal.h new file mode 100644 index 000000000..e7bf31a94 --- /dev/null +++ b/platform/linux-generic/include/odp_ring_mpmc_u64_internal.h @@ -0,0 +1,25 @@ +/* Copyright (c) 2023, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_RING_MPMC_U64_INTERNAL_H_ +#define ODP_RING_MPMC_U64_INTERNAL_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <odp_ring_common.h> + +#undef _ODP_RING_TYPE +#define _ODP_RING_TYPE _ODP_RING_TYPE_U64 + +#include <odp_ring_mpmc_internal.h> + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/libodp-linux.pc.in b/platform/linux-generic/libodp-linux.pc.in index f9a339fb8..05ba5b9d6 100644 --- a/platform/linux-generic/libodp-linux.pc.in +++ b/platform/linux-generic/libodp-linux.pc.in @@ -8,5 +8,5 @@ Description: The ODP packet processing engine Version: @PKGCONFIG_VERSION@ Requires.private: libconfig@AARCH64CRYPTO_PKG@ Libs: -L${libdir} -l@ODP_LIB_NAME@ @ATOMIC_LIBS_NON_ABI_COMPAT@ -Libs.private: @OPENSSL_STATIC_LIBS@ @DPDK_LIBS@ @PCAP_LIBS@ @PTHREAD_LIBS@ @TIMER_LIBS@ @LIBXDP_LIBS@ -lpthread @ATOMIC_LIBS_ABI_COMPAT@ +Libs.private: @OPENSSL_STATIC_LIBS@ @DPDK_LIBS@ @PCAP_LIBS@ @PTHREAD_LIBS@ @TIMER_LIBS@ @LIBXDP_LIBS@ -lpthread @ATOMIC_LIBS_ABI_COMPAT@ @IPSEC_MB_LIBS@ Cflags: -I${includedir} diff --git a/platform/linux-generic/m4/configure.m4 b/platform/linux-generic/m4/configure.m4 index 70a393f56..515c85239 100644 --- a/platform/linux-generic/m4/configure.m4 +++ b/platform/linux-generic/m4/configure.m4 @@ -7,6 +7,7 @@ ODP_ATOMIC ODP_PTHREAD ODP_TIMER m4_include([platform/linux-generic/m4/odp_cpu.m4]) +m4_include([platform/linux-generic/m4/odp_event_validation.m4]) m4_include([platform/linux-generic/m4/odp_pcap.m4]) m4_include([platform/linux-generic/m4/odp_scheduler.m4]) @@ -23,17 +24,20 @@ AM_CONDITIONAL([ODP_PKTIO_PCAP], [test x$have_pcap = xyes]) m4_include([platform/linux-generic/m4/odp_libconfig.m4]) m4_include([platform/linux-generic/m4/odp_openssl.m4]) m4_include([platform/linux-generic/m4/odp_crypto.m4]) +m4_include([platform/linux-generic/m4/odp_ipsec_mb.m4]) m4_include([platform/linux-generic/m4/odp_pcapng.m4]) m4_include([platform/linux-generic/m4/odp_netmap.m4]) m4_include([platform/linux-generic/m4/odp_dpdk.m4]) m4_include([platform/linux-generic/m4/odp_xdp.m4]) +ODP_EVENT_VALIDATION ODP_SCHEDULER -AS_VAR_APPEND([PLAT_DEP_LIBS], ["${ATOMIC_LIBS} ${AARCH64CRYPTO_LIBS} ${LIBCONFIG_LIBS} ${OPENSSL_LIBS} ${DPDK_LIBS_LT} ${LIBCLI_LIBS} ${LIBXDP_LIBS}"]) +AS_VAR_APPEND([PLAT_DEP_LIBS], ["${ATOMIC_LIBS} ${AARCH64CRYPTO_LIBS} ${LIBCONFIG_LIBS} ${OPENSSL_LIBS} ${IPSEC_MB_LIBS} ${DPDK_LIBS_LT} ${LIBCLI_LIBS} ${LIBXDP_LIBS}"]) # Add text to the end of configure with platform specific settings. # Make sure it's aligned same as other lines in configure.ac. AS_VAR_APPEND([PLAT_CFG_TEXT], [" + event_validation: ${enable_event_validation} openssl: ${with_openssl} openssl_rand: ${openssl_rand} crypto: ${with_crypto} diff --git a/platform/linux-generic/m4/odp_crypto.m4 b/platform/linux-generic/m4/odp_crypto.m4 index 9bb99f7dd..1cec6edb4 100644 --- a/platform/linux-generic/m4/odp_crypto.m4 +++ b/platform/linux-generic/m4/odp_crypto.m4 @@ -3,7 +3,7 @@ # Select default crypto implementation AC_ARG_WITH([crypto], [AS_HELP_STRING([--with-crypto], - [Choose crypto implementation (openssl/armv8crypto/null)] + [Choose crypto implementation (openssl/armv8crypto/ipsecmb/null)] [[default=openssl] (linux-generic)])], [], [with_crypto=openssl]) @@ -14,7 +14,7 @@ AS_IF([test "x$with_crypto" = "xyes"], [with_crypto=openssl]) AS_IF([test "x$with_crypto" = "xno"], [with_crypto=null]) AS_IF([test "x$with_crypto" = "xopenssl" -a "x$with_openssl" = "xno"], [with_crypto=null]) -AS_IF([test "x$with_crypto" != "xopenssl" -a "x$with_crypto" != "xarmv8crypto" -a "x$with_crypto" != "xnull"], +AS_IF([test "x$with_crypto" != "xopenssl" -a "x$with_crypto" != "xarmv8crypto" -a "x$with_crypto" != "xipsecmb" -a "x$with_crypto" != "xnull"], [AC_MSG_ERROR([Invalid crypto implementation name])]) ########################################################################## @@ -31,11 +31,19 @@ AS_IF([test "x$with_crypto" == "xarmv8crypto"], [PKG_CHECK_MODULES([AARCH64CRYPTO], [libAArch64crypto]) AARCH64CRYPTO_PKG=", libAArch64crypto" AC_SUBST([AARCH64CRYPTO_PKG])]) + AC_CONFIG_COMMANDS_PRE([dnl AM_CONDITIONAL([WITH_ARMV8_CRYPTO], [test "x$with_crypto" == "xarmv8crypto"]) ]) ########################################################################## +# Multi-buffer IPSec library implementation +########################################################################## +AC_CONFIG_COMMANDS_PRE([dnl +AM_CONDITIONAL([WITH_IPSECMB_CRYPTO], [test "x$with_crypto" == "xipsecmb"]) +]) + +########################################################################## # Null implementation ########################################################################## AS_IF([test "x$with_crypto" == "xnull"], diff --git a/platform/linux-generic/m4/odp_event_validation.m4 b/platform/linux-generic/m4/odp_event_validation.m4 new file mode 100644 index 000000000..08bb8902e --- /dev/null +++ b/platform/linux-generic/m4/odp_event_validation.m4 @@ -0,0 +1,23 @@ +# ODP_EVENT_VALIDATION +# -------------------- +# Select event validation level +AC_DEFUN([ODP_EVENT_VALIDATION], [dnl +AC_ARG_ENABLE([event-validation], + [AS_HELP_STRING([--enable-event-validation], + [enable event validation (warn/abort) + [default=disabled] (linux-generic)])], + [], [AS_IF([test "x$enable_debug" = "xfull"], + [enable_event_validation=yes], [enable_event_validation=no])]) + +# Default to abort mode if validation is enabled +AS_IF([test "x$enable_event_validation" = "xyes"], + [enable_event_validation="abort"]) + +validation_level=0 +AS_IF([test "x$enable_event_validation" = "xwarn"], [validation_level=1]) +AS_IF([test "x$enable_event_validation" = "xyes" -o "x$enable_event_validation" = "xabort"], + [validation_level=2]) + +AC_DEFINE_UNQUOTED([_ODP_EVENT_VALIDATION], [$validation_level], + [Define to 1 or 2 to enable event validation]) +]) # ODP_EVENT_VALIDATION diff --git a/platform/linux-generic/m4/odp_ipsec_mb.m4 b/platform/linux-generic/m4/odp_ipsec_mb.m4 new file mode 100644 index 000000000..3268d94c0 --- /dev/null +++ b/platform/linux-generic/m4/odp_ipsec_mb.m4 @@ -0,0 +1,19 @@ +######################################################################### +# Check for libIPSec_MB availability +######################################################################### +ipsecmb_support=no +AC_CHECK_HEADERS([ipsec-mb.h], + [AC_CHECK_LIB([IPSec_MB], [init_mb_mgr_auto], [ipsecmb_support=yes], + [ipsecmb_support=no])], + [ipsecmb_support=no]) + +AS_IF([test "x$with_crypto" = "xipsecmb" -a "x$ipsecmb_support" = "xno"], + [AC_MSG_ERROR([IPSec MB library not found on this platform])]) + +if test "x$with_crypto" = "xipsecmb"; then + IPSEC_MB_LIBS="-lIPSec_MB" +else + IPSEC_MB_LIBS="" +fi + +AC_SUBST([IPSEC_MB_LIBS]) diff --git a/platform/linux-generic/m4/odp_libconfig.m4 b/platform/linux-generic/m4/odp_libconfig.m4 index 03dbc929d..0d268c935 100644 --- a/platform/linux-generic/m4/odp_libconfig.m4 +++ b/platform/linux-generic/m4/odp_libconfig.m4 @@ -3,7 +3,7 @@ ########################################################################## m4_define([_odp_config_version_generation], [0]) m4_define([_odp_config_version_major], [1]) -m4_define([_odp_config_version_minor], [22]) +m4_define([_odp_config_version_minor], [25]) m4_define([_odp_config_version], [_odp_config_version_generation._odp_config_version_major._odp_config_version_minor]) diff --git a/platform/linux-generic/odp_crypto_api.c b/platform/linux-generic/odp_crypto_api.c new file mode 100644 index 000000000..646472e2e --- /dev/null +++ b/platform/linux-generic/odp_crypto_api.c @@ -0,0 +1,11 @@ +/* Copyright (c) 2022, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <odp/api/crypto.h> + +/* Non-inlined versions of API functions */ +#define _ODP_NO_INLINE +#include <odp/api/plat/crypto_inlines.h> diff --git a/platform/linux-generic/odp_crypto_ipsecmb.c b/platform/linux-generic/odp_crypto_ipsecmb.c new file mode 100644 index 000000000..a7ce6077a --- /dev/null +++ b/platform/linux-generic/odp_crypto_ipsecmb.c @@ -0,0 +1,881 @@ +/* Copyright (c) 2014-2018, Linaro Limited + * Copyright (c) 2021, ARM Limited + * Copyright (c) 2022-2023, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <odp_posix_extensions.h> +#include <odp/autoheader_internal.h> + +#include <odp/api/crypto.h> +#include <odp/api/spinlock.h> +#include <odp/api/debug.h> +#include <odp/api/align.h> +#include <odp/api/shared_memory.h> +#include <odp/api/hints.h> + +#include <odp/api/plat/event_inlines.h> +#include <odp/api/plat/packet_inlines.h> +#include <odp/api/plat/queue_inlines.h> +#include <odp/api/plat/thread_inlines.h> + +#include <odp_debug_internal.h> +#include <odp_global_data.h> +#include <odp_init_internal.h> +#include <odp_packet_internal.h> + +#include <ipsec-mb.h> + +#define MAX_SESSIONS 4000 +/* Length in bytes */ +#define IPSEC_MB_CRYPTO_MAX_CIPHER_KEY_LENGTH 32 +#define IPSEC_MB_CRYPTO_MAX_AUTH_KEY_LENGTH 32 +#define IPSEC_MB_CRYPTO_MAX_DATA_LENGTH 65536 +#define ZUC_DIGEST_LENGTH 4 + +#define ODP_CRYPTO_IPSEC_MB_SHM_NAME "_odp_crypto_ipsecmb" +/* + * Cipher algorithm capabilities + * + * Keep sorted: first by key length, then by IV length + */ +static const odp_crypto_cipher_capability_t cipher_capa_null[] = { +{.key_len = 0, .iv_len = 0} }; + +static const odp_crypto_cipher_capability_t cipher_capa_zuc_eea3[] = { +{.key_len = 16, .iv_len = 16}, +{.key_len = 32, .iv_len = 25} }; + +/* + * Authentication algorithm capabilities + * + * Keep sorted: first by digest length, then by key length + */ +static const odp_crypto_auth_capability_t auth_capa_null[] = { +{.digest_len = 0, .key_len = 0, .aad_len = {.min = 0, .max = 0, .inc = 0} } }; + +static const odp_crypto_auth_capability_t auth_capa_zuc_eia3[] = { +{.digest_len = 4, .key_len = 16, .aad_len = {.min = 0, .max = 0, .inc = 0}, + .iv_len = 16}, +{.digest_len = 4, .key_len = 32, .aad_len = {.min = 0, .max = 0, .inc = 0}, + .iv_len = 25} }; + +/** Forward declaration of session structure */ +typedef struct odp_crypto_generic_session_t odp_crypto_generic_session_t; + +/** + * Algorithm handler function prototype + */ +typedef odp_crypto_alg_err_t (*crypto_func_t)(odp_packet_t pkt, + const odp_crypto_packet_op_param_t *param, + odp_crypto_generic_session_t *session); + +/** + * Per crypto session data structure + */ +struct odp_crypto_generic_session_t { + odp_crypto_generic_session_t *next; + + /* Session creation parameters */ + odp_crypto_session_param_t p; + + odp_bool_t do_cipher_first; + + struct { + uint8_t key_data[IPSEC_MB_CRYPTO_MAX_CIPHER_KEY_LENGTH]; + crypto_func_t func; + } cipher; + + struct { + uint8_t key[IPSEC_MB_CRYPTO_MAX_AUTH_KEY_LENGTH]; + crypto_func_t func; + } auth; + + unsigned int idx; +}; + +typedef struct odp_crypto_global_s odp_crypto_global_t; + +struct odp_crypto_global_s { + odp_spinlock_t lock; + odp_crypto_generic_session_t *free; + odp_crypto_generic_session_t sessions[MAX_SESSIONS]; +}; + +static odp_crypto_global_t *global; + +typedef struct crypto_local_t { + uint8_t buffer[IPSEC_MB_CRYPTO_MAX_DATA_LENGTH]; + IMB_MGR *mb_mgr; +} crypto_local_t; + +static __thread crypto_local_t local; + +static +odp_crypto_generic_session_t *alloc_session(void) +{ + odp_crypto_generic_session_t *session = NULL; + + odp_spinlock_lock(&global->lock); + session = global->free; + if (session) { + global->free = session->next; + session->next = NULL; + } + odp_spinlock_unlock(&global->lock); + + if (!session) + return NULL; + + session->idx = session - global->sessions; + + return session; +} + +static +void free_session(odp_crypto_generic_session_t *session) +{ + odp_spinlock_lock(&global->lock); + session->next = global->free; + global->free = session; + odp_spinlock_unlock(&global->lock); +} + +static odp_crypto_alg_err_t +null_crypto_routine(odp_packet_t pkt ODP_UNUSED, + const odp_crypto_packet_op_param_t *param ODP_UNUSED, + odp_crypto_generic_session_t *session ODP_UNUSED) +{ + return ODP_CRYPTO_ALG_ERR_NONE; +} + +static +odp_crypto_alg_err_t zuc_eea3_cipher_op(odp_packet_t pkt, + const odp_crypto_packet_op_param_t *param, + odp_crypto_generic_session_t *session) +{ + IMB_MGR *mb_mgr = local.mb_mgr; + uint8_t *iv_ptr = param->cipher_iv_ptr; + uint32_t in_pos = param->cipher_range.offset; + uint32_t in_len = param->cipher_range.length; + + _ODP_ASSERT(iv_ptr != NULL); + + uint32_t seg_len = 0; + uint8_t *data = odp_packet_offset(pkt, in_pos, &seg_len, NULL); + + if (odp_unlikely(seg_len < in_len)) { + if (odp_unlikely(in_len > IPSEC_MB_CRYPTO_MAX_DATA_LENGTH)) + return ODP_CRYPTO_ALG_ERR_DATA_SIZE; + + /* Packet is segmented within the cipher range. Copy the cipher + * range to a contiguous buffer. */ + odp_packet_copy_to_mem(pkt, in_pos, in_len, local.buffer); + + data = local.buffer; + } + + if (session->p.cipher_key.length == 16) { + /* ZUC128 EEA3 */ + IMB_ZUC_EEA3_1_BUFFER(mb_mgr, session->cipher.key_data, + iv_ptr, + data, + data, + in_len); + } else { + /* Only 16 and 32 byte keys are supported + * ZUC256 EEA3 */ + IMB_ZUC256_EEA3_1_BUFFER(mb_mgr, session->cipher.key_data, + iv_ptr, + data, + data, + in_len); + } + if (odp_unlikely(imb_get_errno(mb_mgr) != 0)) + return ODP_CRYPTO_ALG_ERR_DATA_SIZE; + + if (odp_unlikely(seg_len < in_len)) + odp_packet_copy_from_mem(pkt, in_pos, in_len, data); + + return ODP_CRYPTO_ALG_ERR_NONE; +} + +static int process_zuc_eea3_param(odp_crypto_generic_session_t *session) +{ + if (!((16 == session->p.cipher_key.length && + 16 == session->p.cipher_iv_len) || + (32 == session->p.cipher_key.length && + 25 == session->p.cipher_iv_len))) + return -1; + + memcpy(session->cipher.key_data, session->p.cipher_key.data, + session->p.cipher_key.length); + + session->cipher.func = zuc_eea3_cipher_op; + + return 0; +} + +static +odp_crypto_alg_err_t auth_zuc_eia3_gen(odp_packet_t pkt, + const odp_crypto_packet_op_param_t *param, + odp_crypto_generic_session_t *session) +{ + IMB_MGR *mb_mgr = local.mb_mgr; + uint8_t *iv_ptr = param->auth_iv_ptr; + uint32_t in_pos = param->auth_range.offset; + uint32_t in_len = param->auth_range.length; + uint32_t auth_tag; + + _ODP_ASSERT(iv_ptr != NULL); + + uint32_t seg_len = 0; + uint8_t *data = odp_packet_offset(pkt, in_pos, &seg_len, NULL); + + if (odp_unlikely(seg_len < in_len)) { + if (odp_unlikely(in_len > IPSEC_MB_CRYPTO_MAX_DATA_LENGTH)) + return ODP_CRYPTO_ALG_ERR_DATA_SIZE; + + /* Packet is segmented within the auth range. Copy the auth + * range to a contiguous buffer. */ + odp_packet_copy_to_mem(pkt, in_pos, in_len, local.buffer); + + data = local.buffer; + } + + if (session->p.auth_key.length == 16) { + /* ZUC128 EIA3 */ + IMB_ZUC_EIA3_1_BUFFER(mb_mgr, session->auth.key, + iv_ptr, + data, + param->auth_range.length * 8, + &auth_tag); + } else { + /* Only 16 and 32 byte keys are supported + * ZUC256 EIA3 */ + IMB_ZUC256_EIA3_1_BUFFER(mb_mgr, session->auth.key, + iv_ptr, + data, + param->auth_range.length * 8, + &auth_tag); + } + if (odp_unlikely(imb_get_errno(mb_mgr) != 0)) + return ODP_CRYPTO_ALG_ERR_DATA_SIZE; + + /* Copy to the output location */ + odp_packet_copy_from_mem(pkt, param->hash_result_offset, + session->p.auth_digest_len, + &auth_tag); + + return ODP_CRYPTO_ALG_ERR_NONE; +} + +static +odp_crypto_alg_err_t auth_zuc_eia3_check(odp_packet_t pkt, + const odp_crypto_packet_op_param_t *param, + odp_crypto_generic_session_t *session) +{ + IMB_MGR *mb_mgr = local.mb_mgr; + uint8_t *iv_ptr = param->auth_iv_ptr; + uint32_t in_pos = param->auth_range.offset; + uint32_t in_len = param->auth_range.length; + uint32_t bytes = ZUC_DIGEST_LENGTH; + uint32_t hash_in; + uint32_t hash_out; + + /* Copy current value out and clear it before authentication */ + odp_packet_copy_to_mem(pkt, param->hash_result_offset, + bytes, &hash_in); + + if (odp_unlikely(session->p.hash_result_in_auth_range)) + _odp_packet_set_data(pkt, param->hash_result_offset, 0, bytes); + + _ODP_ASSERT(iv_ptr != NULL); + + uint32_t seg_len = 0; + uint8_t *data = odp_packet_offset(pkt, in_pos, &seg_len, NULL); + + if (odp_unlikely(seg_len < in_len)) { + if (odp_unlikely(in_len > IPSEC_MB_CRYPTO_MAX_DATA_LENGTH)) + return ODP_CRYPTO_ALG_ERR_DATA_SIZE; + + /* Packet is segmented within the auth range. Copy the auth + * range to a contiguous buffer. */ + odp_packet_copy_to_mem(pkt, in_pos, in_len, local.buffer); + + data = local.buffer; + } + + if (session->p.auth_key.length == 16) { + /* ZUC128 EIA3 */ + IMB_ZUC_EIA3_1_BUFFER(mb_mgr, session->auth.key, + iv_ptr, + data, + param->auth_range.length * 8, + &hash_out); + } else { + /* Only 16 and 32 byte keys are supported + * ZUC256 EIA3 */ + IMB_ZUC256_EIA3_1_BUFFER(mb_mgr, session->auth.key, + iv_ptr, + data, + param->auth_range.length * 8, + &hash_out); + } + if (odp_unlikely(imb_get_errno(mb_mgr) != 0)) + return ODP_CRYPTO_ALG_ERR_DATA_SIZE; + + /* Verify match */ + if (hash_in != hash_out) + return ODP_CRYPTO_ALG_ERR_ICV_CHECK; + + return ODP_CRYPTO_ALG_ERR_NONE; +} + +static int process_auth_zuc_eia3_param(odp_crypto_generic_session_t *session) +{ + if (!((16 == session->p.auth_key.length && + 16 == session->p.auth_iv_len) || + (32 == session->p.auth_key.length && + 25 == session->p.auth_iv_len))) + return -1; + + if (ODP_CRYPTO_OP_ENCODE == session->p.op) + session->auth.func = auth_zuc_eia3_gen; + else + session->auth.func = auth_zuc_eia3_check; + + if (session->p.auth_digest_len != ZUC_DIGEST_LENGTH) + return -1; + + memcpy(session->auth.key, session->p.auth_key.data, + session->p.auth_key.length); + + return 0; +} + +int odp_crypto_capability(odp_crypto_capability_t *capa) +{ + if (NULL == capa) + return -1; + + memset(capa, 0, sizeof(odp_crypto_capability_t)); + + capa->sync_mode = ODP_SUPPORT_PREFERRED; + capa->async_mode = ODP_SUPPORT_YES; + capa->queue_type_plain = 1; + capa->queue_type_sched = 1; + + capa->ciphers.bit.null = 1; + capa->auths.bit.null = 1; + + capa->ciphers.bit.zuc_eea3 = 1; + capa->auths.bit.zuc_eia3 = 1; + + capa->max_sessions = MAX_SESSIONS; + + return 0; +} + +int odp_crypto_cipher_capability(odp_cipher_alg_t cipher, + odp_crypto_cipher_capability_t dst[], + int num_copy) +{ + const odp_crypto_cipher_capability_t *src; + int num; + int size = sizeof(odp_crypto_cipher_capability_t); + + switch (cipher) { + case ODP_CIPHER_ALG_NULL: + src = cipher_capa_null; + num = sizeof(cipher_capa_null) / size; + break; + case ODP_CIPHER_ALG_ZUC_EEA3: + src = cipher_capa_zuc_eea3; + num = sizeof(cipher_capa_zuc_eea3) / size; + break; + default: + return -1; + } + + if (num < num_copy) + num_copy = num; + + memcpy(dst, src, num_copy * size); + + return num; +} + +int odp_crypto_auth_capability(odp_auth_alg_t auth, + odp_crypto_auth_capability_t dst[], int num_copy) +{ + const odp_crypto_auth_capability_t *src; + int num; + int size = sizeof(odp_crypto_auth_capability_t); + + switch (auth) { + case ODP_AUTH_ALG_NULL: + src = auth_capa_null; + num = sizeof(auth_capa_null) / size; + break; + case ODP_AUTH_ALG_ZUC_EIA3: + src = auth_capa_zuc_eia3; + num = sizeof(auth_capa_zuc_eia3) / size; + break; + default: + return -1; + } + + if (num < num_copy) + num_copy = num; + + memcpy(dst, src, num_copy * size); + + return num; +} + +int +odp_crypto_session_create(const odp_crypto_session_param_t *param, + odp_crypto_session_t *session_out, + odp_crypto_ses_create_err_t *status) +{ + int rc = 0; + odp_crypto_generic_session_t *session; + + if (odp_global_ro.disable.crypto) { + _ODP_ERR("Crypto is disabled\n"); + /* Dummy output to avoid compiler warning about uninitialized + * variables */ + *status = ODP_CRYPTO_SES_ERR_ENOMEM; + *session_out = ODP_CRYPTO_SESSION_INVALID; + return -1; + } + + if (param->op_type == ODP_CRYPTO_OP_TYPE_OOP) { + *status = ODP_CRYPTO_SES_ERR_PARAMS; + *session_out = ODP_CRYPTO_SESSION_INVALID; + return -1; + } + + session = alloc_session(); + if (NULL == session) { + *status = ODP_CRYPTO_SES_ERR_ENOMEM; + goto err; + } + + session->p = *param; + + /* Derive order */ + if (ODP_CRYPTO_OP_ENCODE == param->op) + session->do_cipher_first = param->auth_cipher_text; + else + session->do_cipher_first = !param->auth_cipher_text; + + /* Process based on cipher */ + switch (param->cipher_alg) { + case ODP_CIPHER_ALG_NULL: + session->cipher.func = null_crypto_routine; + rc = 0; + break; + case ODP_CIPHER_ALG_ZUC_EEA3: + rc = process_zuc_eea3_param(session); + break; + default: + rc = -1; + } + + if (rc) { + *status = ODP_CRYPTO_SES_ERR_CIPHER; + goto err; + } + + /* Process based on auth */ + switch (param->auth_alg) { + case ODP_AUTH_ALG_NULL: + session->auth.func = null_crypto_routine; + rc = 0; + break; + case ODP_AUTH_ALG_ZUC_EIA3: + rc = process_auth_zuc_eia3_param(session); + break; + default: + rc = -1; + } + + if (rc) { + *status = ODP_CRYPTO_SES_ERR_AUTH; + goto err; + } + + *session_out = (intptr_t)session; + *status = ODP_CRYPTO_SES_ERR_NONE; + return 0; + +err: + /* error status should be set at this moment */ + if (session != NULL) + free_session(session); + *session_out = ODP_CRYPTO_SESSION_INVALID; + return -1; +} + +int odp_crypto_session_destroy(odp_crypto_session_t session) +{ + odp_crypto_generic_session_t *generic; + + generic = (odp_crypto_generic_session_t *)(intptr_t)session; + memset(generic, 0, sizeof(*generic)); + free_session(generic); + return 0; +} + +#if ODP_DEPRECATED_API +int +odp_crypto_operation(odp_crypto_op_param_t *param, + odp_bool_t *posted, + odp_crypto_op_result_t *result) +{ + odp_crypto_packet_op_param_t packet_param; + odp_packet_t out_pkt = param->out_pkt; + odp_crypto_packet_result_t packet_result; + odp_crypto_op_result_t local_result; + int rc; + + if (((odp_crypto_generic_session_t *)(intptr_t)param->session)->p.op_type != + ODP_CRYPTO_OP_TYPE_LEGACY) + return -1; + + packet_param.session = param->session; + packet_param.cipher_iv_ptr = param->cipher_iv_ptr; + packet_param.auth_iv_ptr = param->auth_iv_ptr; + packet_param.hash_result_offset = param->hash_result_offset; + packet_param.aad_ptr = param->aad_ptr; + packet_param.cipher_range = param->cipher_range; + packet_param.auth_range = param->auth_range; + + rc = odp_crypto_op(¶m->pkt, &out_pkt, &packet_param, 1); + if (rc <= 0) + return -1; + + rc = odp_crypto_result(&packet_result, out_pkt); + if (rc < 0) { + /* + * We cannot fail since odp_crypto_op() has already processed + * the packet. Let's indicate error in the result instead. + */ + packet_result.ok = false; + } + + /* Indicate to caller operation was sync */ + *posted = 0; + + packet_subtype_set(out_pkt, ODP_EVENT_PACKET_BASIC); + + /* Fill in result */ + local_result.ctx = param->ctx; + local_result.pkt = out_pkt; + local_result.cipher_status = packet_result.cipher_status; + local_result.auth_status = packet_result.auth_status; + local_result.ok = packet_result.ok; + + /* + * Be bug-to-bug compatible. Return output packet also through params. + */ + param->out_pkt = out_pkt; + + *result = local_result; + + return 0; +} +#endif + +int _odp_crypto_init_global(void) +{ + size_t mem_size; + odp_shm_t shm; + int idx; + + if (odp_global_ro.disable.crypto) { + _ODP_PRINT("\nODP crypto is DISABLED\n"); + return 0; + } + + /* Calculate the memory size we need */ + mem_size = sizeof(odp_crypto_global_t); + + /* Allocate our globally shared memory */ + shm = odp_shm_reserve(ODP_CRYPTO_IPSEC_MB_SHM_NAME, mem_size, + ODP_CACHE_LINE_SIZE, + 0); + if (ODP_SHM_INVALID == shm) { + _ODP_ERR("unable to allocate crypto pool\n"); + return -1; + } + + global = odp_shm_addr(shm); + + /* Clear it out */ + memset(global, 0, mem_size); + + /* Initialize free list and lock */ + for (idx = 0; idx < MAX_SESSIONS; idx++) { + global->sessions[idx].next = global->free; + global->free = &global->sessions[idx]; + } + odp_spinlock_init(&global->lock); + + return 0; +} + +int _odp_crypto_term_global(void) +{ + int rc = 0; + int ret; + int count = 0; + odp_crypto_generic_session_t *session; + + if (odp_global_ro.disable.crypto) + return 0; + + for (session = global->free; session != NULL; session = session->next) + count++; + if (count != MAX_SESSIONS) { + _ODP_ERR("crypto sessions still active\n"); + rc = -1; + } + + ret = odp_shm_free(odp_shm_lookup(ODP_CRYPTO_IPSEC_MB_SHM_NAME)); + if (ret < 0) { + _ODP_ERR("shm free failed for %s\n", ODP_CRYPTO_IPSEC_MB_SHM_NAME); + rc = -1; + } + + return rc; +} + +int _odp_crypto_init_local(void) +{ + uint64_t flags = 0; + + if (odp_global_ro.disable.crypto) + return 0; + + memset(&local, 0, sizeof(local)); + + local.mb_mgr = alloc_mb_mgr(flags); + if (local.mb_mgr == NULL) + return -1; + + init_mb_mgr_auto(local.mb_mgr, NULL); + + return 0; +} + +int _odp_crypto_term_local(void) +{ + if (odp_global_ro.disable.crypto) + return 0; + + free_mb_mgr(local.mb_mgr); + return 0; +} + +#if ODP_DEPRECATED_API +odp_crypto_compl_t odp_crypto_compl_from_event(odp_event_t ev) +{ + /* This check not mandated by the API specification */ + if (odp_event_type(ev) != ODP_EVENT_CRYPTO_COMPL) + _ODP_ABORT("Event not a crypto completion"); + return (odp_crypto_compl_t)ev; +} + +odp_event_t odp_crypto_compl_to_event(odp_crypto_compl_t completion_event) +{ + return (odp_event_t)completion_event; +} + +void +odp_crypto_compl_result(odp_crypto_compl_t completion_event, + odp_crypto_op_result_t *result) +{ + (void)completion_event; + (void)result; + + /* We won't get such events anyway, so there can be no result */ + _ODP_ASSERT(0); +} + +void +odp_crypto_compl_free(odp_crypto_compl_t completion_event) +{ + odp_event_t ev = odp_crypto_compl_to_event(completion_event); + + odp_buffer_free(odp_buffer_from_event(ev)); +} + +uint64_t odp_crypto_compl_to_u64(odp_crypto_compl_t hdl) +{ + return _odp_pri(hdl); +} +#endif /* ODP_DEPRECATED_API */ + +void odp_crypto_session_param_init(odp_crypto_session_param_t *param) +{ + memset(param, 0, sizeof(odp_crypto_session_param_t)); +} + +uint64_t odp_crypto_session_to_u64(odp_crypto_session_t hdl) +{ + return (uint64_t)hdl; +} + +static int copy_data_and_metadata(odp_packet_t dst, odp_packet_t src) +{ + int md_copy; + int rc; + + md_copy = _odp_packet_copy_md_possible(odp_packet_pool(dst), + odp_packet_pool(src)); + if (odp_unlikely(md_copy < 0)) { + _ODP_ERR("Unable to copy packet metadata\n"); + return -1; + } + + rc = odp_packet_copy_from_pkt(dst, 0, src, 0, odp_packet_len(src)); + if (odp_unlikely(rc < 0)) { + _ODP_ERR("Unable to copy packet data\n"); + return -1; + } + + _odp_packet_copy_md(packet_hdr(dst), packet_hdr(src), md_copy); + return 0; +} + +static odp_packet_t get_output_packet(const odp_crypto_generic_session_t *session, + odp_packet_t pkt_in, + odp_packet_t pkt_out) +{ + int rc; + + if (odp_likely(session->p.op_type == ODP_CRYPTO_OP_TYPE_BASIC)) + return pkt_in; + + if (odp_likely(pkt_in == pkt_out)) + return pkt_out; + + if (pkt_out == ODP_PACKET_INVALID) { + odp_pool_t pool = session->p.output_pool; + + _ODP_ASSERT(pool != ODP_POOL_INVALID); + if (pool == odp_packet_pool(pkt_in)) { + pkt_out = pkt_in; + } else { + pkt_out = odp_packet_copy(pkt_in, pool); + if (odp_likely(pkt_out != ODP_PACKET_INVALID)) + odp_packet_free(pkt_in); + } + return pkt_out; + } + rc = copy_data_and_metadata(pkt_out, pkt_in); + if (odp_unlikely(rc < 0)) + return ODP_PACKET_INVALID; + + odp_packet_free(pkt_in); + return pkt_out; +} + +static +int crypto_int(odp_packet_t pkt_in, + odp_packet_t *pkt_out, + const odp_crypto_packet_op_param_t *param) +{ + odp_crypto_alg_err_t rc_cipher = ODP_CRYPTO_ALG_ERR_NONE; + odp_crypto_alg_err_t rc_auth = ODP_CRYPTO_ALG_ERR_NONE; + odp_crypto_generic_session_t *session; + odp_packet_t out_pkt; + odp_crypto_packet_result_t *op_result; + + session = (odp_crypto_generic_session_t *)(intptr_t)param->session; + + out_pkt = get_output_packet(session, pkt_in, *pkt_out); + if (odp_unlikely(out_pkt == ODP_PACKET_INVALID)) + return -1; + + /* Invoke the crypto function */ + if (session->do_cipher_first) { + rc_cipher = session->cipher.func(out_pkt, param, session); + rc_auth = session->auth.func(out_pkt, param, session); + } else { + rc_auth = session->auth.func(out_pkt, param, session); + rc_cipher = session->cipher.func(out_pkt, param, session); + } + + packet_subtype_set(out_pkt, ODP_EVENT_PACKET_CRYPTO); + op_result = &packet_hdr(out_pkt)->crypto_op_result; + op_result->cipher_status.alg_err = rc_cipher; + op_result->cipher_status.hw_err = ODP_CRYPTO_HW_ERR_NONE; + op_result->auth_status.alg_err = rc_auth; + op_result->auth_status.hw_err = ODP_CRYPTO_HW_ERR_NONE; + op_result->ok = + (rc_cipher == ODP_CRYPTO_ALG_ERR_NONE) && + (rc_auth == ODP_CRYPTO_ALG_ERR_NONE); + + /* Synchronous, simply return results */ + *pkt_out = out_pkt; + + return 0; +} + +int odp_crypto_op(const odp_packet_t pkt_in[], + odp_packet_t pkt_out[], + const odp_crypto_packet_op_param_t param[], + int num_pkt) +{ + int i, rc; + odp_crypto_generic_session_t *session; + + for (i = 0; i < num_pkt; i++) { + session = (odp_crypto_generic_session_t *)(intptr_t)param[i].session; + _ODP_ASSERT(ODP_CRYPTO_SYNC == session->p.op_mode); + + rc = crypto_int(pkt_in[i], &pkt_out[i], ¶m[i]); + if (rc < 0) + break; + } + + return i; +} + +int odp_crypto_op_enq(const odp_packet_t pkt_in[], + const odp_packet_t pkt_out[], + const odp_crypto_packet_op_param_t param[], + int num_pkt) +{ + odp_packet_t pkt; + odp_event_t event; + odp_crypto_generic_session_t *session; + int i, rc; + + for (i = 0; i < num_pkt; i++) { + session = (odp_crypto_generic_session_t *)(intptr_t)param[i].session; + _ODP_ASSERT(ODP_CRYPTO_ASYNC == session->p.op_mode); + _ODP_ASSERT(ODP_QUEUE_INVALID != session->p.compl_queue); + + if (session->p.op_type != ODP_CRYPTO_OP_TYPE_BASIC) + pkt = pkt_out[i]; + + rc = crypto_int(pkt_in[i], &pkt, ¶m[i]); + if (rc < 0) + break; + + event = odp_packet_to_event(pkt); + if (odp_queue_enq(session->p.compl_queue, event)) { + odp_event_free(event); + break; + } + } + + return i; +} diff --git a/platform/linux-generic/odp_crypto_null.c b/platform/linux-generic/odp_crypto_null.c index b9d319861..3ca27ca00 100644 --- a/platform/linux-generic/odp_crypto_null.c +++ b/platform/linux-generic/odp_crypto_null.c @@ -1,5 +1,5 @@ /* Copyright (c) 2014-2018, Linaro Limited - * Copyright (c) 2021-2022, Nokia + * Copyright (c) 2021-2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -201,7 +201,13 @@ odp_crypto_session_create(const odp_crypto_session_param_t *param, _ODP_ERR("Crypto is disabled\n"); /* Dummy output to avoid compiler warning about uninitialized * variables */ - *status = ODP_CRYPTO_SES_CREATE_ERR_ENOMEM; + *status = ODP_CRYPTO_SES_ERR_ENOMEM; + *session_out = ODP_CRYPTO_SESSION_INVALID; + return -1; + } + + if (param->op_type == ODP_CRYPTO_OP_TYPE_OOP) { + *status = ODP_CRYPTO_SES_ERR_PARAMS; *session_out = ODP_CRYPTO_SESSION_INVALID; return -1; } @@ -209,7 +215,7 @@ odp_crypto_session_create(const odp_crypto_session_param_t *param, /* Allocate memory for this session */ session = alloc_session(); if (NULL == session) { - *status = ODP_CRYPTO_SES_CREATE_ERR_ENOMEM; + *status = ODP_CRYPTO_SES_ERR_ENOMEM; goto err; } @@ -227,7 +233,7 @@ odp_crypto_session_create(const odp_crypto_session_param_t *param, /* Check result */ if (rc) { - *status = ODP_CRYPTO_SES_CREATE_ERR_INV_CIPHER; + *status = ODP_CRYPTO_SES_ERR_CIPHER; goto err; } @@ -242,13 +248,13 @@ odp_crypto_session_create(const odp_crypto_session_param_t *param, /* Check result */ if (rc) { - *status = ODP_CRYPTO_SES_CREATE_ERR_INV_AUTH; + *status = ODP_CRYPTO_SES_ERR_AUTH; goto err; } /* We're happy */ *session_out = (intptr_t)session; - *status = ODP_CRYPTO_SES_CREATE_ERR_NONE; + *status = ODP_CRYPTO_SES_ERR_NONE; return 0; err: @@ -281,6 +287,10 @@ odp_crypto_operation(odp_crypto_op_param_t *param, odp_crypto_op_result_t local_result; int rc; + if (((odp_crypto_generic_session_t *)(intptr_t)param->session)->p.op_type != + ODP_CRYPTO_OP_TYPE_LEGACY) + return -1; + packet_param.session = param->session; packet_param.cipher_iv_ptr = param->cipher_iv_ptr; packet_param.auth_iv_ptr = param->auth_iv_ptr; @@ -449,43 +459,6 @@ uint64_t odp_crypto_session_to_u64(odp_crypto_session_t hdl) return (uint64_t)hdl; } -odp_packet_t odp_crypto_packet_from_event(odp_event_t ev) -{ - /* This check not mandated by the API specification */ - _ODP_ASSERT(odp_event_type(ev) == ODP_EVENT_PACKET); - _ODP_ASSERT(odp_event_subtype(ev) == ODP_EVENT_PACKET_CRYPTO); - - return odp_packet_from_event(ev); -} - -odp_event_t odp_crypto_packet_to_event(odp_packet_t pkt) -{ - return odp_packet_to_event(pkt); -} - -static -odp_crypto_packet_result_t *get_op_result_from_packet(odp_packet_t pkt) -{ - odp_packet_hdr_t *hdr = packet_hdr(pkt); - - return &hdr->crypto_op_result; -} - -int odp_crypto_result(odp_crypto_packet_result_t *result, - odp_packet_t packet) -{ - odp_crypto_packet_result_t *op_result; - - _ODP_ASSERT(odp_event_subtype(odp_packet_to_event(packet)) == - ODP_EVENT_PACKET_CRYPTO); - - op_result = get_op_result_from_packet(packet); - - memcpy(result, op_result, sizeof(*result)); - - return 0; -} - static int copy_data_and_metadata(odp_packet_t dst, odp_packet_t src) { int md_copy; @@ -514,6 +487,9 @@ static odp_packet_t get_output_packet(const odp_crypto_generic_session_t *sessio { int rc; + if (odp_likely(session->p.op_type == ODP_CRYPTO_OP_TYPE_BASIC)) + return pkt_in; + if (odp_likely(pkt_in == pkt_out)) return pkt_out; @@ -555,7 +531,7 @@ int crypto_int(odp_packet_t pkt_in, /* Fill in result */ packet_subtype_set(out_pkt, ODP_EVENT_PACKET_CRYPTO); - op_result = get_op_result_from_packet(out_pkt); + op_result = &packet_hdr(out_pkt)->crypto_op_result; op_result->cipher_status.alg_err = ODP_CRYPTO_ALG_ERR_NONE; op_result->cipher_status.hw_err = ODP_CRYPTO_HW_ERR_NONE; op_result->auth_status.alg_err = ODP_CRYPTO_ALG_ERR_NONE; @@ -603,7 +579,9 @@ int odp_crypto_op_enq(const odp_packet_t pkt_in[], _ODP_ASSERT(ODP_CRYPTO_ASYNC == session->p.op_mode); _ODP_ASSERT(ODP_QUEUE_INVALID != session->p.compl_queue); - pkt = pkt_out[i]; + if (session->p.op_type != ODP_CRYPTO_OP_TYPE_BASIC) + pkt = pkt_out[i]; + rc = crypto_int(pkt_in[i], &pkt, ¶m[i]); if (rc < 0) break; diff --git a/platform/linux-generic/odp_crypto_openssl.c b/platform/linux-generic/odp_crypto_openssl.c index 0f637850f..19925fc09 100644 --- a/platform/linux-generic/odp_crypto_openssl.c +++ b/platform/linux-generic/odp_crypto_openssl.c @@ -1,5 +1,5 @@ /* Copyright (c) 2014-2018, Linaro Limited - * Copyright (c) 2021-2022, Nokia + * Copyright (c) 2021-2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -217,14 +217,12 @@ struct odp_crypto_generic_session_t { odp_crypto_session_param_t p; odp_bool_t do_cipher_first; + uint8_t cipher_bit_mode : 1; + uint8_t cipher_range_used : 1; + uint8_t auth_range_used : 1; struct { -#if ODP_DEPRECATED_API - /* Copy of session IV data */ - uint8_t iv_data[EVP_MAX_IV_LENGTH]; -#endif uint8_t key_data[EVP_MAX_KEY_LENGTH]; - const EVP_CIPHER *evp_cipher; crypto_func_t func; crypto_init_func_t init; @@ -232,9 +230,6 @@ struct odp_crypto_generic_session_t { struct { uint8_t key[EVP_MAX_KEY_LENGTH]; -#if ODP_DEPRECATED_API - uint8_t iv_data[EVP_MAX_IV_LENGTH]; -#endif union { const EVP_MD *evp_md; const EVP_CIPHER *evp_cipher; @@ -710,23 +705,11 @@ int packet_cmac_eia2(odp_packet_t pkt, uint8_t *hash) { CMAC_CTX *ctx = local.cmac_ctx[session->idx]; - void *iv_ptr; + void *iv_ptr = param->auth_iv_ptr; uint32_t offset = param->auth_range.offset; uint32_t len = param->auth_range.length; size_t outlen; -#if ODP_DEPRECATED_API - if (param->auth_iv_ptr) - iv_ptr = param->auth_iv_ptr; - else if (session->p.auth_iv.data) - iv_ptr = session->auth.iv_data; - else - return ODP_CRYPTO_ALG_ERR_IV_INVALID; -#else - iv_ptr = param->auth_iv_ptr; - _ODP_ASSERT(session->p.auth_iv_len == 0 || iv_ptr != NULL); -#endif - _ODP_ASSERT(offset + len <= odp_packet_len(pkt)); /* Reinitialize CMAC calculation without resetting the key */ @@ -1073,22 +1056,9 @@ odp_crypto_alg_err_t cipher_encrypt(odp_packet_t pkt, odp_crypto_generic_session_t *session) { EVP_CIPHER_CTX *ctx = local.cipher_ctx[session->idx]; - void *iv_ptr; int ret; -#if ODP_DEPRECATED_API - if (param->cipher_iv_ptr) - iv_ptr = param->cipher_iv_ptr; - else if (session->p.cipher_iv.data) - iv_ptr = session->cipher.iv_data; - else - return ODP_CRYPTO_ALG_ERR_IV_INVALID; -#else - iv_ptr = param->cipher_iv_ptr; - _ODP_ASSERT(session->p.cipher_iv_len == 0 || iv_ptr != NULL); -#endif - - EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv_ptr); + EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, param->cipher_iv_ptr); ret = internal_encrypt(ctx, pkt, param); @@ -1112,22 +1082,9 @@ odp_crypto_alg_err_t cipher_decrypt(odp_packet_t pkt, odp_crypto_generic_session_t *session) { EVP_CIPHER_CTX *ctx = local.cipher_ctx[session->idx]; - void *iv_ptr; int ret; -#if ODP_DEPRECATED_API - if (param->cipher_iv_ptr) - iv_ptr = param->cipher_iv_ptr; - else if (session->p.cipher_iv.data) - iv_ptr = session->cipher.iv_data; - else - return ODP_CRYPTO_ALG_ERR_IV_INVALID; -#else - iv_ptr = param->cipher_iv_ptr; - _ODP_ASSERT(session->p.cipher_iv_len == 0 || iv_ptr != NULL); -#endif - - EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv_ptr); + EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, param->cipher_iv_ptr); ret = internal_decrypt(ctx, pkt, param); @@ -1172,7 +1129,6 @@ odp_crypto_alg_err_t cipher_encrypt_bits(odp_packet_t pkt, odp_crypto_generic_session_t *session) { EVP_CIPHER_CTX *ctx = local.cipher_ctx[session->idx]; - void *iv_ptr; int dummy_len = 0; int cipher_len; uint32_t in_len = (param->cipher_range.length + 7) / 8; @@ -1183,18 +1139,7 @@ odp_crypto_alg_err_t cipher_encrypt_bits(odp_packet_t pkt, /* Range offset is in bits in bit mode but must be divisible by 8. */ offset = param->cipher_range.offset / 8; -#if ODP_DEPRECATED_API - if (param->cipher_iv_ptr) - iv_ptr = param->cipher_iv_ptr; - else if (session->p.cipher_iv.data) - iv_ptr = session->cipher.iv_data; - else - return ODP_CRYPTO_ALG_ERR_IV_INVALID; -#else - iv_ptr = param->cipher_iv_ptr; - _ODP_ASSERT(session->p.cipher_iv_len == 0 || iv_ptr != NULL); -#endif - EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv_ptr); + EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, param->cipher_iv_ptr); odp_packet_copy_to_mem(pkt, offset, in_len, data); @@ -1216,7 +1161,6 @@ odp_crypto_alg_err_t cipher_decrypt_bits(odp_packet_t pkt, odp_crypto_generic_session_t *session) { EVP_CIPHER_CTX *ctx = local.cipher_ctx[session->idx]; - void *iv_ptr; int dummy_len = 0; int cipher_len; uint32_t in_len = (param->cipher_range.length + 7) / 8; @@ -1227,18 +1171,7 @@ odp_crypto_alg_err_t cipher_decrypt_bits(odp_packet_t pkt, /* Range offset is in bits in bit mode but must be divisible by 8. */ offset = param->cipher_range.offset / 8; -#if ODP_DEPRECATED_API - if (param->cipher_iv_ptr) - iv_ptr = param->cipher_iv_ptr; - else if (session->p.cipher_iv.data) - iv_ptr = session->cipher.iv_data; - else - return ODP_CRYPTO_ALG_ERR_IV_INVALID; -#else - iv_ptr = param->cipher_iv_ptr; - _ODP_ASSERT(session->p.cipher_iv_len == 0 || iv_ptr != NULL); -#endif - EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv_ptr); + EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, param->cipher_iv_ptr); odp_packet_copy_to_mem(pkt, offset, in_len, data); @@ -1266,6 +1199,7 @@ static int process_cipher_param_bits(odp_crypto_generic_session_t *session, session->p.cipher_iv_len) return -1; + session->cipher_bit_mode = 1; session->cipher.evp_cipher = cipher; memcpy(session->cipher.key_data, session->p.cipher_key.data, @@ -1303,24 +1237,11 @@ odp_crypto_alg_err_t aes_gcm_encrypt(odp_packet_t pkt, EVP_CIPHER_CTX *ctx = local.cipher_ctx[session->idx]; const uint8_t *aad_head = param->aad_ptr; uint32_t aad_len = session->p.auth_aad_len; - void *iv_ptr; int dummy_len = 0; uint8_t block[EVP_MAX_MD_SIZE]; int ret; -#if ODP_DEPRECATED_API - if (param->cipher_iv_ptr) - iv_ptr = param->cipher_iv_ptr; - else if (session->p.cipher_iv.data) - iv_ptr = session->cipher.iv_data; - else - return ODP_CRYPTO_ALG_ERR_IV_INVALID; -#else - iv_ptr = param->cipher_iv_ptr; - _ODP_ASSERT(session->p.cipher_iv_len == 0 || iv_ptr != NULL); -#endif - - EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv_ptr); + EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, param->cipher_iv_ptr); /* Authenticate header data (if any) without encrypting them */ if (aad_len > 0) @@ -1359,23 +1280,10 @@ odp_crypto_alg_err_t aes_gcm_decrypt(odp_packet_t pkt, const uint8_t *aad_head = param->aad_ptr; uint32_t aad_len = session->p.auth_aad_len; int dummy_len = 0; - void *iv_ptr; uint8_t block[EVP_MAX_MD_SIZE]; int ret; -#if ODP_DEPRECATED_API - if (param->cipher_iv_ptr) - iv_ptr = param->cipher_iv_ptr; - else if (session->p.cipher_iv.data) - iv_ptr = session->cipher.iv_data; - else - return ODP_CRYPTO_ALG_ERR_IV_INVALID; -#else - iv_ptr = param->cipher_iv_ptr; - _ODP_ASSERT(session->p.cipher_iv_len == 0 || iv_ptr != NULL); -#endif - - EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv_ptr); + EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, param->cipher_iv_ptr); odp_packet_copy_to_mem(pkt, param->hash_result_offset, session->p.auth_digest_len, block); @@ -1440,23 +1348,10 @@ odp_crypto_alg_err_t aes_gmac_gen(odp_packet_t pkt, odp_crypto_generic_session_t *session) { EVP_CIPHER_CTX *ctx = local.mac_cipher_ctx[session->idx]; - void *iv_ptr; uint8_t block[EVP_MAX_MD_SIZE]; int ret; -#if ODP_DEPRECATED_API - if (param->auth_iv_ptr) - iv_ptr = param->auth_iv_ptr; - else if (session->p.auth_iv.data) - iv_ptr = session->auth.iv_data; - else - return ODP_CRYPTO_ALG_ERR_IV_INVALID; -#else - iv_ptr = param->auth_iv_ptr; - _ODP_ASSERT(session->p.auth_iv_len == 0 || iv_ptr != NULL); -#endif - - EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv_ptr); + EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, param->auth_iv_ptr); ret = internal_aad(ctx, pkt, param, true); @@ -1487,23 +1382,10 @@ odp_crypto_alg_err_t aes_gmac_check(odp_packet_t pkt, odp_crypto_generic_session_t *session) { EVP_CIPHER_CTX *ctx = local.mac_cipher_ctx[session->idx]; - void *iv_ptr; uint8_t block[EVP_MAX_MD_SIZE]; int ret; -#if ODP_DEPRECATED_API - if (param->auth_iv_ptr) - iv_ptr = param->auth_iv_ptr; - else if (session->p.auth_iv.data) - iv_ptr = session->auth.iv_data; - else - return ODP_CRYPTO_ALG_ERR_IV_INVALID; -#else - iv_ptr = param->auth_iv_ptr; - _ODP_ASSERT(session->p.auth_iv_len == 0 || iv_ptr != NULL); -#endif - - EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv_ptr); + EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, param->auth_iv_ptr); odp_packet_copy_to_mem(pkt, param->hash_result_offset, session->p.auth_digest_len, block); @@ -1568,7 +1450,6 @@ odp_crypto_alg_err_t aes_ccm_encrypt(odp_packet_t pkt, EVP_CIPHER_CTX *ctx = local.cipher_ctx[session->idx]; const uint8_t *aad_head = param->aad_ptr; uint32_t aad_len = session->p.auth_aad_len; - void *iv_ptr; int dummy_len = 0; int cipher_len; uint32_t in_len = param->cipher_range.length; @@ -1576,21 +1457,9 @@ odp_crypto_alg_err_t aes_ccm_encrypt(odp_packet_t pkt, uint8_t block[EVP_MAX_MD_SIZE]; int ret; -#if ODP_DEPRECATED_API - if (param->cipher_iv_ptr) - iv_ptr = param->cipher_iv_ptr; - else if (session->p.cipher_iv.data) - iv_ptr = session->cipher.iv_data; - else - return ODP_CRYPTO_ALG_ERR_IV_INVALID; -#else - iv_ptr = param->cipher_iv_ptr; - _ODP_ASSERT(session->p.cipher_iv_len == 0 || iv_ptr != NULL); -#endif - EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_CCM_SET_TAG, session->p.auth_digest_len, NULL); - EVP_EncryptInit_ex(ctx, NULL, NULL, session->cipher.key_data, iv_ptr); + EVP_EncryptInit_ex(ctx, NULL, NULL, session->cipher.key_data, param->cipher_iv_ptr); /* Set len */ EVP_EncryptUpdate(ctx, NULL, &dummy_len, NULL, in_len); @@ -1640,7 +1509,6 @@ odp_crypto_alg_err_t aes_ccm_decrypt(odp_packet_t pkt, EVP_CIPHER_CTX *ctx = local.cipher_ctx[session->idx]; const uint8_t *aad_head = param->aad_ptr; uint32_t aad_len = session->p.auth_aad_len; - void *iv_ptr; int dummy_len = 0; int cipher_len; uint32_t in_len = param->cipher_range.length; @@ -1648,23 +1516,11 @@ odp_crypto_alg_err_t aes_ccm_decrypt(odp_packet_t pkt, uint8_t block[EVP_MAX_MD_SIZE]; int ret; -#if ODP_DEPRECATED_API - if (param->cipher_iv_ptr) - iv_ptr = param->cipher_iv_ptr; - else if (session->p.cipher_iv.data) - iv_ptr = session->cipher.iv_data; - else - return ODP_CRYPTO_ALG_ERR_IV_INVALID; -#else - iv_ptr = param->cipher_iv_ptr; - _ODP_ASSERT(session->p.cipher_iv_len == 0 || iv_ptr != NULL); -#endif - odp_packet_copy_to_mem(pkt, param->hash_result_offset, session->p.auth_digest_len, block); EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_CCM_SET_TAG, session->p.auth_digest_len, block); - EVP_DecryptInit_ex(ctx, NULL, NULL, session->cipher.key_data, iv_ptr); + EVP_DecryptInit_ex(ctx, NULL, NULL, session->cipher.key_data, param->cipher_iv_ptr); /* Set len */ EVP_DecryptUpdate(ctx, NULL, &dummy_len, NULL, in_len); @@ -1725,26 +1581,13 @@ odp_crypto_alg_err_t xts_encrypt(odp_packet_t pkt, odp_crypto_generic_session_t *session) { EVP_CIPHER_CTX *ctx = local.cipher_ctx[session->idx]; - void *iv_ptr; int dummy_len = 0; int cipher_len; uint32_t in_len = param->cipher_range.length; uint8_t data[in_len]; int ret; -#if ODP_DEPRECATED_API - if (param->cipher_iv_ptr) - iv_ptr = param->cipher_iv_ptr; - else if (session->p.cipher_iv.data) - iv_ptr = session->cipher.iv_data; - else - return ODP_CRYPTO_ALG_ERR_IV_INVALID; -#else - iv_ptr = param->cipher_iv_ptr; - _ODP_ASSERT(session->p.cipher_iv_len == 0 || iv_ptr != NULL); -#endif - - EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv_ptr); + EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, param->cipher_iv_ptr); odp_packet_copy_to_mem(pkt, param->cipher_range.offset, in_len, data); @@ -1767,26 +1610,13 @@ odp_crypto_alg_err_t xts_decrypt(odp_packet_t pkt, odp_crypto_generic_session_t *session) { EVP_CIPHER_CTX *ctx = local.cipher_ctx[session->idx]; - void *iv_ptr; int dummy_len = 0; int cipher_len; uint32_t in_len = param->cipher_range.length; uint8_t data[in_len]; int ret; -#if ODP_DEPRECATED_API - if (param->cipher_iv_ptr) - iv_ptr = param->cipher_iv_ptr; - else if (session->p.cipher_iv.data) - iv_ptr = session->cipher.iv_data; - else - return ODP_CRYPTO_ALG_ERR_IV_INVALID; -#else - iv_ptr = param->cipher_iv_ptr; - _ODP_ASSERT(session->p.cipher_iv_len == 0 || iv_ptr != NULL); -#endif - - EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv_ptr); + EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, param->cipher_iv_ptr); odp_packet_copy_to_mem(pkt, param->cipher_range.offset, in_len, data); @@ -2222,6 +2052,10 @@ odp_crypto_session_create(const odp_crypto_session_param_t *param, /* Copy parameters */ session->p = *param; + session->cipher_bit_mode = 0; + session->auth_range_used = 1; + session->cipher_range_used = 1; + if (session->p.cipher_iv_len > EVP_MAX_IV_LENGTH) { _ODP_DBG("Maximum IV length exceeded\n"); *status = ODP_CRYPTO_SES_ERR_CIPHER; @@ -2234,17 +2068,6 @@ odp_crypto_session_create(const odp_crypto_session_param_t *param, goto err; } -#if ODP_DEPRECATED_API - /* Copy IV data */ - if (session->p.cipher_iv.data) - memcpy(session->cipher.iv_data, session->p.cipher_iv.data, - session->p.cipher_iv.length); - - if (session->p.auth_iv.data) - memcpy(session->auth.iv_data, session->p.auth_iv.data, - session->p.auth_iv.length); -#endif - /* Derive order */ if (ODP_CRYPTO_OP_ENCODE == param->op) session->do_cipher_first = param->auth_cipher_text; @@ -2256,6 +2079,7 @@ odp_crypto_session_create(const odp_crypto_session_param_t *param, case ODP_CIPHER_ALG_NULL: session->cipher.func = null_crypto_routine; session->cipher.init = null_crypto_init_routine; + session->cipher_range_used = 0; rc = 0; break; case ODP_CIPHER_ALG_3DES_CBC: @@ -2376,6 +2200,7 @@ odp_crypto_session_create(const odp_crypto_session_param_t *param, case ODP_AUTH_ALG_NULL: session->auth.func = null_crypto_routine; session->auth.init = null_crypto_init_routine; + session->auth_range_used = 0; rc = 0; break; case ODP_AUTH_ALG_MD5_HMAC: @@ -2409,6 +2234,7 @@ odp_crypto_session_create(const odp_crypto_session_param_t *param, } else { rc = -1; } + session->auth_range_used = 0; break; case ODP_AUTH_ALG_AES_GMAC: if (param->auth_key.length == 16) @@ -2430,6 +2256,7 @@ odp_crypto_session_create(const odp_crypto_session_param_t *param, } else { rc = -1; } + session->auth_range_used = 0; break; case ODP_AUTH_ALG_AES_CMAC: if (param->auth_key.length == 16) @@ -2455,6 +2282,7 @@ odp_crypto_session_create(const odp_crypto_session_param_t *param, } else { rc = -1; } + session->auth_range_used = 0; break; #endif case ODP_AUTH_ALG_AES_EIA2: @@ -2530,6 +2358,10 @@ odp_crypto_operation(odp_crypto_op_param_t *param, odp_crypto_op_result_t local_result; int rc; + if (((odp_crypto_generic_session_t *)(intptr_t)param->session)->p.op_type != + ODP_CRYPTO_OP_TYPE_LEGACY) + return -1; + packet_param.session = param->session; packet_param.cipher_iv_ptr = param->cipher_iv_ptr; packet_param.auth_iv_ptr = param->auth_iv_ptr; @@ -2778,43 +2610,6 @@ uint64_t odp_crypto_session_to_u64(odp_crypto_session_t hdl) return (uint64_t)hdl; } -odp_packet_t odp_crypto_packet_from_event(odp_event_t ev) -{ - /* This check not mandated by the API specification */ - _ODP_ASSERT(odp_event_type(ev) == ODP_EVENT_PACKET); - _ODP_ASSERT(odp_event_subtype(ev) == ODP_EVENT_PACKET_CRYPTO); - - return odp_packet_from_event(ev); -} - -odp_event_t odp_crypto_packet_to_event(odp_packet_t pkt) -{ - return odp_packet_to_event(pkt); -} - -static -odp_crypto_packet_result_t *get_op_result_from_packet(odp_packet_t pkt) -{ - odp_packet_hdr_t *hdr = packet_hdr(pkt); - - return &hdr->crypto_op_result; -} - -int odp_crypto_result(odp_crypto_packet_result_t *result, - odp_packet_t packet) -{ - odp_crypto_packet_result_t *op_result; - - _ODP_ASSERT(odp_event_subtype(odp_packet_to_event(packet)) == - ODP_EVENT_PACKET_CRYPTO); - - op_result = get_op_result_from_packet(packet); - - memcpy(result, op_result, sizeof(*result)); - - return 0; -} - static int copy_data_and_metadata(odp_packet_t dst, odp_packet_t src) { int md_copy; @@ -2843,6 +2638,9 @@ static odp_packet_t get_output_packet(const odp_crypto_generic_session_t *sessio { int rc; + if (odp_likely(session->p.op_type == ODP_CRYPTO_OP_TYPE_BASIC)) + return pkt_in; + if (odp_likely(pkt_in == pkt_out)) return pkt_out; @@ -2894,6 +2692,8 @@ int crypto_int(odp_packet_t pkt_in, goto out; } } + _ODP_ASSERT(session->p.cipher_iv_len == 0 || param->cipher_iv_ptr != NULL); + _ODP_ASSERT(session->p.auth_iv_len == 0 || param->auth_iv_ptr != NULL); crypto_init(session); @@ -2909,7 +2709,7 @@ int crypto_int(odp_packet_t pkt_in, out: /* Fill in result */ packet_subtype_set(out_pkt, ODP_EVENT_PACKET_CRYPTO); - op_result = get_op_result_from_packet(out_pkt); + op_result = &packet_hdr(out_pkt)->crypto_op_result; op_result->cipher_status.alg_err = rc_cipher; op_result->cipher_status.hw_err = ODP_CRYPTO_HW_ERR_NONE; op_result->auth_status.alg_err = rc_auth; @@ -2924,6 +2724,113 @@ out: return 0; } +/* + * Copy cipher range and auth range from src to dst, + * with shifting by dst_offset_shift. + */ +static void copy_ranges(odp_packet_t dst, + odp_packet_t src, + const odp_crypto_generic_session_t *session, + const odp_crypto_packet_op_param_t *param) +{ + odp_packet_data_range_t c_range = param->cipher_range; + odp_packet_data_range_t a_range = param->auth_range; + int32_t shift = param->dst_offset_shift; + int rc; + + if (session->cipher_bit_mode) { + c_range.offset /= 8; + c_range.length = (c_range.length + 7) / 8; + } + + if (session->cipher_range_used) { + rc = odp_packet_copy_from_pkt(dst, c_range.offset + shift, + src, c_range.offset, + c_range.length); + if (rc) { + _ODP_ERR("cipher range copying failed\n"); + return; + } + } + if (session->auth_range_used) { + rc = odp_packet_copy_from_pkt(dst, a_range.offset + shift, + src, a_range.offset, + a_range.length); + if (rc) { + _ODP_ERR("auth range copying failed\n"); + return; + } + } +} + +static int crypto_int_oop_encode(odp_packet_t pkt_in, + odp_packet_t *pkt_out, + const odp_crypto_generic_session_t *session, + const odp_crypto_packet_op_param_t *param) +{ + odp_crypto_packet_op_param_t new_param = *param; + const uint32_t scale = session->cipher_bit_mode ? 8 : 1; + + copy_ranges(*pkt_out, pkt_in, session, param); + + new_param.cipher_range.offset += param->dst_offset_shift * scale; + new_param.auth_range.offset += param->dst_offset_shift; + + return crypto_int(*pkt_out, pkt_out, &new_param); +} + +static int crypto_int_oop_decode(odp_packet_t pkt_in, + odp_packet_t *pkt_out, + const odp_crypto_generic_session_t *session, + const odp_crypto_packet_op_param_t *param) +{ + odp_packet_t copy; + int rc; + + copy = odp_packet_copy(pkt_in, odp_packet_pool(pkt_in)); + if (copy == ODP_PACKET_INVALID) + return -1; + + rc = crypto_int(copy, ©, param); + if (rc < 0) { + odp_packet_free(copy); + return rc; + } + + copy_ranges(*pkt_out, copy, session, param); + + packet_subtype_set(*pkt_out, ODP_EVENT_PACKET_CRYPTO); + packet_hdr(*pkt_out)->crypto_op_result = packet_hdr(copy)->crypto_op_result; + odp_packet_free(copy); + + return 0; +} + +/* + * Slow out-of-place operation implemented using copying and in-place operation + */ +static int crypto_int_oop(odp_packet_t pkt_in, + odp_packet_t *pkt_out, + const odp_crypto_packet_op_param_t *param) +{ + odp_crypto_generic_session_t *session; + int rc; + + session = (odp_crypto_generic_session_t *)(intptr_t)param->session; + + if (session->p.op == ODP_CRYPTO_OP_ENCODE) + rc = crypto_int_oop_encode(pkt_in, pkt_out, session, param); + else + rc = crypto_int_oop_decode(pkt_in, pkt_out, session, param); + if (rc) + return rc; + + if (session->p.op_mode == ODP_CRYPTO_ASYNC) + packet_hdr(*pkt_out)->crypto_op_result.pkt_in = pkt_in; + + return 0; +} + int odp_crypto_op(const odp_packet_t pkt_in[], odp_packet_t pkt_out[], const odp_crypto_packet_op_param_t param[], @@ -2936,7 +2843,10 @@ int odp_crypto_op(const odp_packet_t pkt_in[], session = (odp_crypto_generic_session_t *)(intptr_t)param[i].session; _ODP_ASSERT(ODP_CRYPTO_SYNC == session->p.op_mode); - rc = crypto_int(pkt_in[i], &pkt_out[i], ¶m[i]); + if (odp_unlikely(session->p.op_type == ODP_CRYPTO_OP_TYPE_OOP)) + rc = crypto_int_oop(pkt_in[i], &pkt_out[i], ¶m[i]); + else + rc = crypto_int(pkt_in[i], &pkt_out[i], ¶m[i]); if (rc < 0) break; } @@ -2959,8 +2869,13 @@ int odp_crypto_op_enq(const odp_packet_t pkt_in[], _ODP_ASSERT(ODP_CRYPTO_ASYNC == session->p.op_mode); _ODP_ASSERT(ODP_QUEUE_INVALID != session->p.compl_queue); - pkt = pkt_out[i]; - rc = crypto_int(pkt_in[i], &pkt, ¶m[i]); + if (session->p.op_type != ODP_CRYPTO_OP_TYPE_BASIC) + pkt = pkt_out[i]; + + if (odp_unlikely(session->p.op_type == ODP_CRYPTO_OP_TYPE_OOP)) + rc = crypto_int_oop(pkt_in[i], &pkt, ¶m[i]); + else + rc = crypto_int(pkt_in[i], &pkt, ¶m[i]); if (rc < 0) break; diff --git a/platform/linux-generic/odp_event.c b/platform/linux-generic/odp_event.c index c4e0f2c9d..edf77e2dc 100644 --- a/platform/linux-generic/odp_event.c +++ b/platform/linux-generic/odp_event.c @@ -1,5 +1,5 @@ /* Copyright (c) 2015-2018, Linaro Limited - * Copyright (c) 2020-2022, Nokia + * Copyright (c) 2020-2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -18,6 +18,7 @@ #include <odp_debug_internal.h> #include <odp_packet_internal.h> #include <odp_event_internal.h> +#include <odp_event_validation_internal.h> #include <odp_event_vector_internal.h> /* Inlined API functions */ @@ -41,13 +42,15 @@ _odp_event_inline_offset ODP_ALIGNED_CACHE = { #include <odp/visibility_end.h> -void odp_event_free(odp_event_t event) +static inline void event_free(odp_event_t event, _odp_ev_id_t id) { switch (odp_event_type(event)) { case ODP_EVENT_BUFFER: + _odp_buffer_validate(odp_buffer_from_event(event), id); odp_buffer_free(odp_buffer_from_event(event)); break; case ODP_EVENT_PACKET: + _odp_packet_validate(odp_packet_from_event(event), id); odp_packet_free(odp_packet_from_event(event)); break; case ODP_EVENT_PACKET_VECTOR: @@ -75,17 +78,21 @@ void odp_event_free(odp_event_t event) } } -void odp_event_free_multi(const odp_event_t event[], int num) +void odp_event_free(odp_event_t event) { - int i; + event_free(event, _ODP_EV_EVENT_FREE); +} - for (i = 0; i < num; i++) - odp_event_free(event[i]); +void odp_event_free_multi(const odp_event_t event[], int num) +{ + for (int i = 0; i < num; i++) + event_free(event[i], _ODP_EV_EVENT_FREE_MULTI); } void odp_event_free_sp(const odp_event_t event[], int num) { - odp_event_free_multi(event, num); + for (int i = 0; i < num; i++) + event_free(event[i], _ODP_EV_EVENT_FREE_SP); } uint64_t odp_event_to_u64(odp_event_t hdl) @@ -103,9 +110,9 @@ int odp_event_is_valid(odp_event_t event) switch (odp_event_type(event)) { case ODP_EVENT_BUFFER: - /* Fall through */ + return !_odp_buffer_validate(odp_buffer_from_event(event), _ODP_EV_EVENT_IS_VALID); case ODP_EVENT_PACKET: - /* Fall through */ + return !_odp_packet_validate(odp_packet_from_event(event), _ODP_EV_EVENT_IS_VALID); case ODP_EVENT_TIMEOUT: /* Fall through */ #if ODP_DEPRECATED_API diff --git a/platform/linux-generic/odp_event_validation.c b/platform/linux-generic/odp_event_validation.c new file mode 100644 index 000000000..c2d430f1a --- /dev/null +++ b/platform/linux-generic/odp_event_validation.c @@ -0,0 +1,260 @@ +/* Copyright (c) 2023, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <odp/api/atomic.h> +#include <odp/api/buffer.h> +#include <odp/api/debug.h> +#include <odp/api/event.h> +#include <odp/api/hints.h> +#include <odp/api/packet.h> +#include <odp/api/shared_memory.h> + +#include <odp_buffer_internal.h> +#include <odp_debug_internal.h> +#include <odp_event_internal.h> +#include <odp_event_validation_internal.h> +#include <odp_global_data.h> +#include <odp_init_internal.h> +#include <odp_libconfig_internal.h> +#include <odp_macros_internal.h> +#include <odp_print_internal.h> + +#include <inttypes.h> +#include <string.h> + +#define EVENT_VALIDATION_NONE 0 +#define EVENT_VALIDATION_WARN 1 +#define EVENT_VALIDATION_ABORT 2 + +#define EVENT_DATA_PRINT_MAX_LEN 128 + +typedef struct { + odp_atomic_u64_t err_count[_ODP_EV_MAX]; + odp_shm_t shm; + +} event_validation_global_t; + +typedef struct { + const char *str; +} _odp_ev_info_t; + +static event_validation_global_t *_odp_ev_glb; + +#if _ODP_EVENT_VALIDATION + +/* Table for mapping function IDs to API function names */ +static const _odp_ev_info_t ev_info_tbl[] = { + [_ODP_EV_BUFFER_FREE] = {.str = "odp_buffer_free()"}, + [_ODP_EV_BUFFER_FREE_MULTI] = {.str = "odp_buffer_free_multi()"}, + [_ODP_EV_BUFFER_IS_VALID] = {.str = "odp_buffer_is_valid()"}, + [_ODP_EV_EVENT_FREE] = {.str = "odp_event_free()"}, + [_ODP_EV_EVENT_FREE_MULTI] = {.str = "odp_event_free_multi()"}, + [_ODP_EV_EVENT_FREE_SP] = {.str = "odp_event_free()_sp"}, + [_ODP_EV_EVENT_IS_VALID] = {.str = "odp_event_is_valid()"}, + [_ODP_EV_PACKET_FREE] = {.str = "odp_packet_free()"}, + [_ODP_EV_PACKET_FREE_MULTI] = {.str = "odp_packet_free_multi()"}, + [_ODP_EV_PACKET_FREE_SP] = {.str = "odp_packet_free_sp()"}, + [_ODP_EV_PACKET_IS_VALID] = {.str = "odp_packet_is_valid()"}, + [_ODP_EV_QUEUE_ENQ] = {.str = "odp_queue_enq()"}, + [_ODP_EV_QUEUE_ENQ_MULTI] = {.str = "odp_queue_enq_multi()"} +}; + +ODP_STATIC_ASSERT(_ODP_ARRAY_SIZE(ev_info_tbl) == _ODP_EV_MAX, "ev_info_tbl missing entries"); + +static void print_event_data(odp_event_t event, odp_event_type_t type) +{ + const char *type_str; + const uint32_t bytes_per_row = 16; + uint32_t byte_len; + int num_rows, max_len, n; + int len = 0; + uint8_t *data; + + if (type == ODP_EVENT_PACKET) { + odp_packet_t pkt = odp_packet_from_event(event); + + data = odp_packet_data(pkt); + byte_len = odp_packet_seg_len(pkt); + type_str = "Packet"; + } else { + odp_buffer_t buf = odp_buffer_from_event(event); + + data = odp_buffer_addr(buf); + byte_len = odp_buffer_size(buf); + type_str = "Buffer"; + } + + if (byte_len > EVENT_DATA_PRINT_MAX_LEN) + byte_len = EVENT_DATA_PRINT_MAX_LEN; + + num_rows = (byte_len + bytes_per_row - 1) / bytes_per_row; + max_len = 256 + (3 * byte_len) + (3 * num_rows); + n = max_len - 1; + + char str[max_len]; + + len += _odp_snprint(&str[len], n - len, "%s %p data %p:\n", type_str, event, data); + while (byte_len) { + uint32_t row_len = byte_len > bytes_per_row ? bytes_per_row : byte_len; + + len += _odp_snprint(&str[len], n - len, " "); + + for (uint32_t i = 0; i < row_len; i++) + len += _odp_snprint(&str[len], n - len, " %02x", data[i]); + + len += _odp_snprint(&str[len], n - len, "\n"); + + byte_len -= row_len; + data += row_len; + } + + _ODP_PRINT("%s\n", str); +} + +static inline int validate_event_endmark(odp_event_t event, _odp_ev_id_t id, odp_event_type_t type) +{ + uint64_t err_count; + uint64_t *endmark_ptr = _odp_event_endmark_get_ptr(event); + + if (odp_likely(*endmark_ptr == _ODP_EV_ENDMARK_VAL)) + return 0; + + err_count = odp_atomic_fetch_inc_u64(&_odp_ev_glb->err_count[id]) + 1; + + _ODP_ERR("Event %p endmark mismatch in %s: endmark=0x%" PRIx64 " (expected 0x%" PRIx64 ") " + "err_count=%" PRIu64 "\n", event, ev_info_tbl[id].str, *endmark_ptr, + _ODP_EV_ENDMARK_VAL, err_count); + + print_event_data(event, type); + + if (_ODP_EVENT_VALIDATION == EVENT_VALIDATION_ABORT) + _ODP_ABORT("Abort due to event %p endmark mismatch\n", event); + + /* Fix endmark value */ + _odp_event_endmark_set(event); + + return -1; +} + +static inline int buffer_validate(odp_buffer_t buf, _odp_ev_id_t id) +{ + return validate_event_endmark(odp_buffer_to_event(buf), id, ODP_EVENT_BUFFER); +} + +static inline int packet_validate(odp_packet_t pkt, _odp_ev_id_t id) +{ + return validate_event_endmark(odp_packet_to_event(pkt), id, ODP_EVENT_PACKET); +} + +static inline int event_validate(odp_event_t event, int id) +{ + if (odp_event_type(event) == ODP_EVENT_BUFFER) + return buffer_validate(odp_buffer_from_event(event), id); + if (odp_event_type(event) == ODP_EVENT_PACKET) + return packet_validate(odp_packet_from_event(event), id); + return 0; +} + +/* Enable usage from API inline files */ +#include <odp/visibility_begin.h> + +int _odp_buffer_validate(odp_buffer_t buf, _odp_ev_id_t id) +{ + return buffer_validate(buf, id); +} + +int _odp_buffer_validate_multi(const odp_buffer_t buf[], int num, + _odp_ev_id_t id) +{ + for (int i = 0; i < num; i++) { + if (odp_unlikely(buffer_validate(buf[i], id))) + return -1; + } + return 0; +} + +int _odp_packet_validate(odp_packet_t pkt, _odp_ev_id_t id) +{ + return packet_validate(pkt, id); +} + +int _odp_packet_validate_multi(const odp_packet_t pkt[], int num, + _odp_ev_id_t id) +{ + for (int i = 0; i < num; i++) { + if (odp_unlikely(packet_validate(pkt[i], id))) + return -1; + } + return 0; +} + +int _odp_event_validate(odp_event_t event, _odp_ev_id_t id) +{ + return event_validate(event, id); +} + +int _odp_event_validate_multi(const odp_event_t event[], int num, + _odp_ev_id_t id) +{ + for (int i = 0; i < num; i++) { + if (odp_unlikely(event_validate(event[i], id))) + return -1; + } + return 0; +} + +#include <odp/visibility_end.h> + +#endif /* _ODP_EVENT_VALIDATION */ + +int _odp_event_validation_init_global(void) +{ + odp_shm_t shm; + + _ODP_PRINT("\nEvent validation mode: %s\n\n", + _ODP_EVENT_VALIDATION == EVENT_VALIDATION_NONE ? "none" : + _ODP_EVENT_VALIDATION == EVENT_VALIDATION_WARN ? "warn" : "abort"); + + if (_ODP_EVENT_VALIDATION == EVENT_VALIDATION_NONE) + return 0; + + shm = odp_shm_reserve("_odp_event_validation_global", + sizeof(event_validation_global_t), + ODP_CACHE_LINE_SIZE, ODP_SHM_EXPORT); + if (shm == ODP_SHM_INVALID) + return -1; + + _odp_ev_glb = odp_shm_addr(shm); + if (_odp_ev_glb == NULL) + return -1; + + memset(_odp_ev_glb, 0, sizeof(event_validation_global_t)); + _odp_ev_glb->shm = shm; + + for (int i = 0; i < _ODP_EV_MAX; i++) + odp_atomic_init_u64(&_odp_ev_glb->err_count[i], 0); + + return 0; +} + +int _odp_event_validation_term_global(void) +{ + int ret; + + if (_ODP_EVENT_VALIDATION == EVENT_VALIDATION_NONE) + return 0; + + if (_odp_ev_glb == NULL) + return 0; + + ret = odp_shm_free(_odp_ev_glb->shm); + if (ret) { + _ODP_ERR("SHM free failed: %d\n", ret); + return -1; + } + + return 0; +} diff --git a/platform/linux-generic/odp_init.c b/platform/linux-generic/odp_init.c index 284b3e566..bd27641aa 100644 --- a/platform/linux-generic/odp_init.c +++ b/platform/linux-generic/odp_init.c @@ -1,5 +1,5 @@ /* Copyright (c) 2013-2018, Linaro Limited - * Copyright (c) 2021, Nokia + * Copyright (c) 2021-2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -35,6 +35,7 @@ enum init_stage { HASH_INIT, THREAD_INIT, POOL_INIT, + EVENT_VALIDATION_INIT, STASH_INIT, QUEUE_INIT, SCHED_INIT, @@ -242,6 +243,13 @@ static int term_global(enum init_stage stage) } /* Fall through */ + case EVENT_VALIDATION_INIT: + if (_odp_event_validation_term_global()) { + _ODP_ERR("ODP event validation term failed.\n"); + rc = -1; + } + /* Fall through */ + case POOL_INIT: if (_odp_pool_term_global()) { _ODP_ERR("ODP buffer pool term failed.\n"); @@ -412,6 +420,12 @@ int odp_init_global(odp_instance_t *instance, } stage = POOL_INIT; + if (_odp_event_validation_init_global()) { + _ODP_ERR("ODP event validation init failed.\n"); + goto init_failed; + } + stage = EVENT_VALIDATION_INIT; + if (_odp_stash_init_global()) { _ODP_ERR("ODP stash init failed.\n"); goto init_failed; diff --git a/platform/linux-generic/odp_ipsec_sad.c b/platform/linux-generic/odp_ipsec_sad.c index 1b3a90e6a..4cf265694 100644 --- a/platform/linux-generic/odp_ipsec_sad.c +++ b/platform/linux-generic/odp_ipsec_sad.c @@ -1,11 +1,12 @@ /* Copyright (c) 2017-2018, Linaro Limited - * Copyright (c) 2018-2022, Nokia + * Copyright (c) 2018-2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause */ #include <odp/api/atomic.h> +#include <odp/api/crypto.h> #include <odp/api/ipsec.h> #include <odp/api/random.h> #include <odp/api/shared_memory.h> @@ -18,7 +19,7 @@ #include <odp_debug_internal.h> #include <odp_ipsec_internal.h> #include <odp_macros_internal.h> -#include <odp_ring_mpmc_internal.h> +#include <odp_ring_mpmc_u32_internal.h> #include <odp_global_data.h> #include <string.h> @@ -102,7 +103,7 @@ typedef struct ODP_ALIGNED_CACHE ipsec_thread_local_s { typedef struct ipsec_sa_table_t { ipsec_sa_t ipsec_sa[CONFIG_IPSEC_MAX_NUM_SA]; struct ODP_ALIGNED_CACHE { - ring_mpmc_t ipv4_id_ring; + ring_mpmc_u32_t ipv4_id_ring; uint32_t ipv4_id_data[IPV4_ID_RING_SIZE] ODP_ALIGNED_CACHE; } hot; struct { @@ -192,7 +193,7 @@ int _odp_ipsec_sad_init_global(void) ipsec_sa_tbl->shm = shm; ipsec_sa_tbl->max_num_sa = max_num_sa; - ring_mpmc_init(&ipsec_sa_tbl->hot.ipv4_id_ring); + ring_mpmc_u32_init(&ipsec_sa_tbl->hot.ipv4_id_ring); for (i = 0; i < thread_count_max; i++) { /* * Make the current ID block fully used, forcing allocation @@ -210,11 +211,11 @@ int _odp_ipsec_sad_init_global(void) for (i = 0; i < IPV4_ID_RING_SIZE - 1; i++) { uint32_t data = i * IPV4_ID_BLOCK_SIZE; - ring_mpmc_enq_multi(&ipsec_sa_tbl->hot.ipv4_id_ring, - ipsec_sa_tbl->hot.ipv4_id_data, - IPV4_ID_RING_MASK, - &data, - 1); + ring_mpmc_u32_enq_multi(&ipsec_sa_tbl->hot.ipv4_id_ring, + ipsec_sa_tbl->hot.ipv4_id_data, + IPV4_ID_RING_MASK, + &data, + 1); } for (i = 0; i < ipsec_sa_tbl->max_num_sa; i++) { @@ -1118,17 +1119,17 @@ uint16_t _odp_ipsec_sa_alloc_ipv4_id(ipsec_sa_t *ipsec_sa) tl->first_ipv4_id + IPV4_ID_BLOCK_SIZE)) { /* Return used ID block to the ring */ data = tl->first_ipv4_id; - ring_mpmc_enq_multi(&ipsec_sa_tbl->hot.ipv4_id_ring, - ipsec_sa_tbl->hot.ipv4_id_data, - IPV4_ID_RING_MASK, - &data, - 1); + ring_mpmc_u32_enq_multi(&ipsec_sa_tbl->hot.ipv4_id_ring, + ipsec_sa_tbl->hot.ipv4_id_data, + IPV4_ID_RING_MASK, + &data, + 1); /* Get new ID block */ - ring_mpmc_deq_multi(&ipsec_sa_tbl->hot.ipv4_id_ring, - ipsec_sa_tbl->hot.ipv4_id_data, - IPV4_ID_RING_MASK, - &data, - 1); + ring_mpmc_u32_deq_multi(&ipsec_sa_tbl->hot.ipv4_id_ring, + ipsec_sa_tbl->hot.ipv4_id_data, + IPV4_ID_RING_MASK, + &data, + 1); tl->first_ipv4_id = data; tl->next_ipv4_id = data; } diff --git a/platform/linux-generic/odp_packet.c b/platform/linux-generic/odp_packet.c index 212e48de0..639a74e0c 100644 --- a/platform/linux-generic/odp_packet.c +++ b/platform/linux-generic/odp_packet.c @@ -1,5 +1,5 @@ /* Copyright (c) 2013-2018, Linaro Limited - * Copyright (c) 2019-2022, Nokia + * Copyright (c) 2019-2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -21,6 +21,7 @@ #include <odp_debug_internal.h> #include <odp_errno_define.h> #include <odp_event_internal.h> +#include <odp_event_validation_internal.h> #include <odp_macros_internal.h> #include <odp_packet_internal.h> #include <odp_packet_io_internal.h> @@ -69,6 +70,7 @@ const _odp_packet_inline_offset_t _odp_packet_inline ODP_ALIGNED_CACHE = { .subtype = offsetof(odp_packet_hdr_t, subtype), .cls_mark = offsetof(odp_packet_hdr_t, cls_mark), .ipsec_ctx = offsetof(odp_packet_hdr_t, ipsec_ctx), + .crypto_op = offsetof(odp_packet_hdr_t, crypto_op_result), }; #include <odp/visibility_end.h> @@ -695,6 +697,8 @@ void odp_packet_free(odp_packet_t pkt) odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); int num_seg = pkt_hdr->seg_count; + _odp_packet_validate(pkt, _ODP_EV_PACKET_FREE); + _ODP_ASSERT(segment_ref(pkt_hdr) > 0); if (odp_likely(num_seg == 1)) @@ -703,12 +707,14 @@ void odp_packet_free(odp_packet_t pkt) free_all_segments(pkt_hdr, num_seg); } -void odp_packet_free_multi(const odp_packet_t pkt[], int num) +static inline void packet_free_multi_ev(const odp_packet_t pkt[], int num, _odp_ev_id_t id) { odp_packet_hdr_t *pkt_hdrs[num]; int i; int num_freed = 0; + _odp_packet_validate_multi(pkt, num, id); + for (i = 0; i < num; i++) { odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt[i]); int num_seg = pkt_hdr->seg_count; @@ -728,9 +734,14 @@ void odp_packet_free_multi(const odp_packet_t pkt[], int num) packet_free_multi(pkt_hdrs, num - num_freed); } +void odp_packet_free_multi(const odp_packet_t pkt[], int num) +{ + packet_free_multi_ev(pkt, num, _ODP_EV_PACKET_FREE_MULTI); +} + void odp_packet_free_sp(const odp_packet_t pkt[], int num) { - odp_packet_free_multi(pkt, num); + packet_free_multi_ev(pkt, num, _ODP_EV_PACKET_FREE_SP); } int odp_packet_reset(odp_packet_t pkt, uint32_t len) @@ -1587,6 +1598,9 @@ int odp_packet_is_valid(odp_packet_t pkt) if (odp_event_type(ev) != ODP_EVENT_PACKET) return 0; + if (odp_unlikely(_odp_packet_validate(pkt, _ODP_EV_PACKET_IS_VALID))) + return 0; + switch (odp_event_subtype(ev)) { case ODP_EVENT_PACKET_BASIC: /* Fall through */ @@ -2343,6 +2357,7 @@ odp_packet_t odp_packet_reassemble(odp_pool_t pool_hdl, odp_packet_buf_t pkt_buf tailroom = pool->ext_param.pkt.buf_size - sizeof(odp_packet_hdr_t); tailroom -= pool->ext_param.pkt.app_header_size; tailroom -= odp_packet_buf_data_len(pkt_buf[num - 1]); + tailroom -= pool->trailer_size; pkt_hdr->seg_count = num; pkt_hdr->frame_len = data_len; diff --git a/platform/linux-generic/odp_packet_flags.c b/platform/linux-generic/odp_packet_flags.c index ef542a2cd..777da12ae 100644 --- a/platform/linux-generic/odp_packet_flags.c +++ b/platform/linux-generic/odp_packet_flags.c @@ -72,10 +72,12 @@ void odp_packet_has_jumbo_set(odp_packet_t pkt, int val) void odp_packet_has_vlan_set(odp_packet_t pkt, int val) { setflag(pkt, input_flags.vlan, val); + setflag(pkt, input_flags.vlan_qinq, 0); } void odp_packet_has_vlan_qinq_set(odp_packet_t pkt, int val) { + setflag(pkt, input_flags.vlan, val); setflag(pkt, input_flags.vlan_qinq, val); } diff --git a/platform/linux-generic/odp_packet_io.c b/platform/linux-generic/odp_packet_io.c index 22b6bc916..5a6c0f460 100644 --- a/platform/linux-generic/odp_packet_io.c +++ b/platform/linux-generic/odp_packet_io.c @@ -1613,16 +1613,6 @@ odp_time_t odp_pktio_time(odp_pktio_t hdl, odp_time_t *global_ts) return ts; } -uint64_t ODP_DEPRECATE(odp_pktin_ts_res)(odp_pktio_t hdl) -{ - return odp_pktio_ts_res(hdl); -} - -odp_time_t ODP_DEPRECATE(odp_pktin_ts_from_ns)(odp_pktio_t hdl, uint64_t ns) -{ - return odp_pktio_ts_from_ns(hdl, ns); -} - void odp_pktio_print(odp_pktio_t hdl) { pktio_entry_t *entry; @@ -2166,8 +2156,7 @@ void odp_pktio_extra_stats_print(odp_pktio_t pktio) _ODP_PRINT("\n"); } -int odp_pktin_queue_config(odp_pktio_t pktio, - const odp_pktin_queue_param_t *param) +int odp_pktin_queue_config(odp_pktio_t pktio, const odp_pktin_queue_param_t *param) { pktio_entry_t *entry; odp_pktin_mode_t mode; @@ -2175,7 +2164,7 @@ int odp_pktin_queue_config(odp_pktio_t pktio, uint32_t num_queues, i; int rc; odp_queue_t queue; - odp_pktin_queue_param_t default_param; + odp_pktin_queue_param_t default_param, local_param; if (param == NULL) { odp_pktin_queue_param_init(&default_param); @@ -2204,7 +2193,19 @@ int odp_pktin_queue_config(odp_pktio_t pktio, return -1; } - num_queues = param->classifier_enable ? 1 : param->num_queues; + if (param->classifier_enable) { + num_queues = 1; + + if (param->num_queues != num_queues) { + /* When classifier is enabled, ensure that only one input queue will be + * configured by driver. */ + memcpy(&local_param, param, sizeof(odp_pktin_queue_param_t)); + local_param.num_queues = num_queues; + param = &local_param; + } + } else { + num_queues = param->num_queues; + } rc = odp_pktio_capability(pktio, &capa); if (rc) { diff --git a/platform/linux-generic/odp_pool.c b/platform/linux-generic/odp_pool.c index f414b0626..59c007ee2 100644 --- a/platform/linux-generic/odp_pool.c +++ b/platform/linux-generic/odp_pool.c @@ -1,5 +1,5 @@ /* Copyright (c) 2013-2018, Linaro Limited - * Copyright (c) 2019-2022, Nokia + * Copyright (c) 2019-2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -7,6 +7,7 @@ #include <odp/api/align.h> #include <odp/api/atomic.h> +#include <odp/api/hints.h> #include <odp/api/pool.h> #include <odp/api/shared_memory.h> #include <odp/api/system_info.h> @@ -21,6 +22,8 @@ #include <odp_packet_internal.h> #include <odp_config_internal.h> #include <odp_debug_internal.h> +#include <odp_event_internal.h> +#include <odp_event_validation_internal.h> #include <odp_macros_internal.h> #include <odp_ring_ptr_internal.h> #include <odp_global_data.h> @@ -77,6 +80,7 @@ const _odp_pool_inline_offset_t _odp_pool_inline ODP_ALIGNED_CACHE = { .index = offsetof(pool_t, pool_idx), .seg_len = offsetof(pool_t, seg_len), .uarea_size = offsetof(pool_t, param_uarea_size), + .trailer_size = offsetof(pool_t, trailer_size), .ext_head_offset = offsetof(pool_t, ext_head_offset), .ext_pkt_buf_size = offsetof(pool_t, ext_param.pkt.buf_size) }; @@ -472,6 +476,7 @@ static void init_event_hdr(pool_t *pool, _odp_event_hdr_t *event_hdr, uint32_t e if (type == ODP_POOL_BUFFER || type == ODP_POOL_PACKET) { event_hdr->base_data = data_ptr; event_hdr->buf_end = data_ptr + pool->seg_len + pool->tailroom; + _odp_event_endmark_set(_odp_event_from_hdr(event_hdr)); } if (type == ODP_POOL_BUFFER) { @@ -697,7 +702,7 @@ odp_pool_t _odp_pool_create(const char *name, const odp_pool_param_t *params, uint32_t uarea_size, headroom, tailroom; odp_shm_t shm; uint32_t seg_len, align, num, hdr_size, block_size; - uint32_t max_len, cache_size; + uint32_t max_len, cache_size, trailer_size; uint32_t ring_size; odp_pool_type_t type = params->type; uint32_t shmflags = 0; @@ -743,6 +748,7 @@ odp_pool_t _odp_pool_create(const char *name, const odp_pool_param_t *params, tailroom = 0; seg_len = 0; max_len = 0; + trailer_size = 0; uarea_size = 0; cache_size = 0; @@ -752,6 +758,7 @@ odp_pool_t _odp_pool_create(const char *name, const odp_pool_param_t *params, seg_len = params->buf.size; uarea_size = params->buf.uarea_size; cache_size = params->buf.cache_size; + trailer_size = _ODP_EV_ENDMARK_SIZE; break; case ODP_POOL_PACKET: @@ -763,6 +770,7 @@ odp_pool_t _odp_pool_create(const char *name, const odp_pool_param_t *params, num = params->pkt.num; seg_len = CONFIG_PACKET_MAX_SEG_LEN; max_len = _odp_pool_glb->config.pkt_max_len; + trailer_size = _ODP_EV_ENDMARK_SIZE; if (params->pkt.len && params->pkt.len < CONFIG_PACKET_MAX_SEG_LEN) @@ -840,7 +848,7 @@ odp_pool_t _odp_pool_create(const char *name, const odp_pool_param_t *params, uint32_t adj_size; hdr_size = _ODP_ROUNDUP_CACHE_LINE(sizeof(odp_packet_hdr_t)); - block_size = hdr_size + align + headroom + seg_len + tailroom; + block_size = hdr_size + align + headroom + seg_len + tailroom + trailer_size; adj_size = block_size; if (pool->mem_src_ops && pool->mem_src_ops->adjust_size) { @@ -871,7 +879,7 @@ odp_pool_t _odp_pool_create(const char *name, const odp_pool_param_t *params, else hdr_size = _ODP_ROUNDUP_CACHE_LINE(sizeof(odp_event_vector_hdr_t)); - block_size = _ODP_ROUNDUP_CACHE_LINE(hdr_size + align_pad + seg_len); + block_size = _ODP_ROUNDUP_CACHE_LINE(hdr_size + align_pad + seg_len + trailer_size); } /* Allocate extra memory for skipping packet buffers which cross huge @@ -894,6 +902,7 @@ odp_pool_t _odp_pool_create(const char *name, const odp_pool_param_t *params, pool->align = align; pool->headroom = headroom; pool->seg_len = seg_len; + pool->trailer_size = trailer_size; pool->max_seg_len = headroom + seg_len + tailroom; pool->max_len = max_len; pool->tailroom = tailroom; @@ -1419,11 +1428,15 @@ int odp_buffer_alloc_multi(odp_pool_t pool_hdl, odp_buffer_t buf[], int num) void odp_buffer_free(odp_buffer_t buf) { + _odp_buffer_validate(buf, _ODP_EV_BUFFER_FREE); + _odp_event_free_multi((_odp_event_hdr_t **)&buf, 1); } void odp_buffer_free_multi(const odp_buffer_t buf[], int num) { + _odp_buffer_validate_multi(buf, num, _ODP_EV_BUFFER_FREE_MULTI); + _odp_event_free_multi((_odp_event_hdr_t **)(uintptr_t)buf, num); } @@ -1530,6 +1543,7 @@ void odp_pool_print(odp_pool_t pool_hdl) _ODP_PRINT(" burst size %u\n", pool->burst_size); _ODP_PRINT(" mem src %s\n", pool->mem_src_ops ? pool->mem_src_ops->name : "(none)"); + _ODP_PRINT(" event valid. %d\n", _ODP_EVENT_VALIDATION); _ODP_PRINT("\n"); } @@ -1727,6 +1741,9 @@ int odp_buffer_is_valid(odp_buffer_t buf) if (odp_event_type(odp_buffer_to_event(buf)) != ODP_EVENT_BUFFER) return 0; + if (odp_unlikely(_odp_buffer_validate(buf, _ODP_EV_BUFFER_IS_VALID))) + return 0; + return 1; } @@ -1753,7 +1770,7 @@ int odp_pool_ext_capability(odp_pool_type_t type, odp_pool_ext_capability_t *cap capa->pkt.max_num_buf = _odp_pool_glb->config.pkt_max_num; capa->pkt.max_buf_size = MAX_SIZE; capa->pkt.odp_header_size = sizeof(odp_packet_hdr_t); - capa->pkt.odp_trailer_size = 0; + capa->pkt.odp_trailer_size = _ODP_EV_ENDMARK_SIZE; capa->pkt.min_mem_align = ODP_CACHE_LINE_SIZE; capa->pkt.min_buf_align = ODP_CACHE_LINE_SIZE; capa->pkt.min_head_align = MIN_HEAD_ALIGN; @@ -1878,7 +1895,9 @@ odp_pool_t odp_pool_ext_create(const char *name, const odp_pool_ext_param_t *par pool->num = num_buf; pool->headroom = headroom; pool->tailroom = 0; - pool->seg_len = buf_size - head_offset - headroom - pool->tailroom; + pool->trailer_size = _ODP_EV_ENDMARK_SIZE; + pool->seg_len = buf_size - head_offset - headroom - pool->tailroom - + pool->trailer_size; pool->max_seg_len = headroom + pool->seg_len + pool->tailroom; pool->max_len = PKT_MAX_SEGS * pool->seg_len; pool->ext_head_offset = head_offset; diff --git a/platform/linux-generic/odp_queue_basic.c b/platform/linux-generic/odp_queue_basic.c index 360f907bf..83694f84f 100644 --- a/platform/linux-generic/odp_queue_basic.c +++ b/platform/linux-generic/odp_queue_basic.c @@ -1,43 +1,46 @@ /* Copyright (c) 2013-2018, Linaro Limited - * Copyright (c) 2021-2022, Nokia + * Copyright (c) 2021-2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause */ -#include <odp/api/queue.h> -#include <odp_queue_basic_internal.h> -#include <odp_queue_if.h> -#include <odp/api/std_types.h> #include <odp/api/align.h> -#include <odp_pool_internal.h> -#include <odp_init_internal.h> -#include <odp_timer_internal.h> -#include <odp/api/shared_memory.h> -#include <odp/api/schedule.h> -#include <odp_schedule_if.h> -#include <odp_config_internal.h> -#include <odp_packet_io_internal.h> -#include <odp_debug_internal.h> #include <odp/api/hints.h> +#include <odp/api/packet_io.h> +#include <odp/api/queue.h> +#include <odp/api/schedule.h> +#include <odp/api/shared_memory.h> +#include <odp/api/std_types.h> #include <odp/api/sync.h> -#include <odp/api/plat/sync_inlines.h> +#include <odp/api/ticketlock.h> #include <odp/api/traffic_mngr.h> -#include <odp_libconfig_internal.h> + #include <odp/api/plat/queue_inline_types.h> -#include <odp_global_data.h> -#include <odp_queue_basic_internal.h> +#include <odp/api/plat/sync_inlines.h> +#include <odp/api/plat/ticketlock_inlines.h> + +#include <odp_config_internal.h> +#include <odp_debug_internal.h> #include <odp_event_internal.h> +#include <odp_global_data.h> +#include <odp_init_internal.h> +#include <odp_libconfig_internal.h> #include <odp_macros_internal.h> +#include <odp_packet_io_internal.h> +#include <odp_pool_internal.h> +#include <odp_queue_basic_internal.h> +#include <odp_queue_if.h> +#include <odp_schedule_if.h> +#include <odp_timer_internal.h> + +#include <inttypes.h> +#include <string.h> -#include <odp/api/plat/ticketlock_inlines.h> #define LOCK(queue_ptr) odp_ticketlock_lock(&((queue_ptr)->lock)) #define UNLOCK(queue_ptr) odp_ticketlock_unlock(&((queue_ptr)->lock)) #define LOCK_INIT(queue_ptr) odp_ticketlock_init(&((queue_ptr)->lock)) -#include <string.h> -#include <inttypes.h> - #define MIN_QUEUE_SIZE 32 #define MAX_QUEUE_SIZE (1 * 1024 * 1024) @@ -405,7 +408,7 @@ static int queue_destroy(odp_queue_t handle) else if (queue->type == ODP_QUEUE_TYPE_SCHED) empty = ring_st_is_empty(&queue->ring_st); else - empty = ring_mpmc_is_empty(&queue->ring_mpmc); + empty = ring_mpmc_u32_is_empty(&queue->ring_mpmc); if (!empty) { UNLOCK(queue); @@ -494,7 +497,7 @@ static inline int _plain_queue_enq_multi(odp_queue_t handle, { queue_entry_t *queue; int ret, num_enq; - ring_mpmc_t *ring_mpmc; + ring_mpmc_u32_t *ring_mpmc; uint32_t event_idx[num]; queue = qentry_from_handle(handle); @@ -505,8 +508,8 @@ static inline int _plain_queue_enq_multi(odp_queue_t handle, event_index_from_hdr(event_idx, event_hdr, num); - num_enq = ring_mpmc_enq_multi(ring_mpmc, queue->ring_data, - queue->ring_mask, event_idx, num); + num_enq = ring_mpmc_u32_enq_multi(ring_mpmc, queue->ring_data, + queue->ring_mask, event_idx, num); return num_enq; } @@ -516,14 +519,14 @@ static inline int _plain_queue_deq_multi(odp_queue_t handle, { int num_deq; queue_entry_t *queue; - ring_mpmc_t *ring_mpmc; + ring_mpmc_u32_t *ring_mpmc; uint32_t event_idx[num]; queue = qentry_from_handle(handle); ring_mpmc = &queue->ring_mpmc; - num_deq = ring_mpmc_deq_multi(ring_mpmc, queue->ring_data, - queue->ring_mask, event_idx, num); + num_deq = ring_mpmc_u32_deq_multi(ring_mpmc, queue->ring_data, + queue->ring_mask, event_idx, num); if (num_deq == 0) return 0; @@ -751,7 +754,7 @@ static void queue_print(odp_queue_t handle) } else { _ODP_PRINT(" implementation ring_mpmc\n"); _ODP_PRINT(" length %" PRIu32 "/%" PRIu32 "\n", - ring_mpmc_length(&queue->ring_mpmc), queue->ring_mask + 1); + ring_mpmc_u32_len(&queue->ring_mpmc), queue->ring_mask + 1); } _ODP_PRINT("\n"); @@ -817,7 +820,7 @@ static void queue_print_all(void) if (_odp_sched_id == _ODP_SCHED_ID_BASIC) spr = _odp_sched_basic_get_spread(index); } else { - len = ring_mpmc_length(&queue->ring_mpmc); + len = ring_mpmc_u32_len(&queue->ring_mpmc); max_len = queue->ring_mask + 1; } @@ -1070,7 +1073,7 @@ static int queue_init(queue_entry_t *queue, const char *name, queue->ring_data = &_odp_queue_glb->ring_data[offset]; queue->ring_mask = queue_size - 1; - ring_mpmc_init(&queue->ring_mpmc); + ring_mpmc_u32_init(&queue->ring_mpmc); } else { queue->enqueue = sched_queue_enq; diff --git a/platform/linux-generic/odp_schedule_basic.c b/platform/linux-generic/odp_schedule_basic.c index 594360326..3fc9c1c17 100644 --- a/platform/linux-generic/odp_schedule_basic.c +++ b/platform/linux-generic/odp_schedule_basic.c @@ -238,13 +238,13 @@ typedef struct { struct { uint8_t burst_default[NUM_SCHED_SYNC][NUM_PRIO]; uint8_t burst_max[NUM_SCHED_SYNC][NUM_PRIO]; + uint16_t order_stash_size; uint8_t num_spread; uint8_t prefer_ratio; } config; - - uint8_t load_balance; - uint16_t max_spread; uint32_t ring_mask; + uint16_t max_spread; + uint8_t load_balance; odp_atomic_u32_t grp_epoch; odp_shm_t shm; odp_ticketlock_t mask_lock[NUM_SCHED_GRPS]; @@ -437,6 +437,20 @@ static int read_config_file(sched_global_t *sched) if (val == 0 || sched->config.num_spread == 1) sched->load_balance = 0; + str = "sched_basic.order_stash_size"; + if (!_odp_libconfig_lookup_int(str, &val)) { + _ODP_ERR("Config option '%s' not found.\n", str); + return -1; + } + + if (val > MAX_ORDERED_STASH || val < 0) { + _ODP_ERR("Bad value %s = %i [min: 0, max: %u]\n", str, val, MAX_ORDERED_STASH); + return -1; + } + + sched->config.order_stash_size = val; + _ODP_PRINT(" %s: %i\n", str, val); + /* Initialize default values for all queue types */ str = "sched_basic.burst_size_default"; if (read_burst_size_conf(sched->config.burst_default[ODP_SCHED_SYNC_ATOMIC], str, 1, @@ -1204,7 +1218,7 @@ static int schedule_ord_enq_multi(odp_queue_t dst_queue, void *event_hdr[], /* Pktout may drop packets, so the operation cannot be stashed. */ if (dst_qentry->pktout.pktio != ODP_PKTIO_INVALID || - odp_unlikely(stash_num >= MAX_ORDERED_STASH)) { + odp_unlikely(stash_num >= sched->config.order_stash_size)) { /* If the local stash is full, wait until it is our turn and * then release the stash and do enqueue directly. */ wait_for_order(src_queue); diff --git a/platform/linux-generic/odp_shared_memory.c b/platform/linux-generic/odp_shared_memory.c index a0e822a53..ef4df3a33 100644 --- a/platform/linux-generic/odp_shared_memory.c +++ b/platform/linux-generic/odp_shared_memory.c @@ -8,7 +8,6 @@ #include <odp_config_internal.h> #include <odp_debug_internal.h> #include <odp/api/debug.h> -#include <odp/api/deprecated.h> #include <odp/api/std_types.h> #include <odp/api/shared_memory.h> #include <odp/api/plat/strong_types.h> @@ -18,14 +17,8 @@ #include <string.h> /* Supported ODP_SHM_* flags */ -#if ODP_DEPRECATED_API - #define DEPRECATED_SHM_FLAGS (ODP_SHM_SW_ONLY) -#else - #define DEPRECATED_SHM_FLAGS 0 -#endif - #define SUPPORTED_SHM_FLAGS (ODP_SHM_PROC | ODP_SHM_SINGLE_VA | ODP_SHM_EXPORT | \ - ODP_SHM_HP | ODP_SHM_NO_HP | DEPRECATED_SHM_FLAGS) + ODP_SHM_HP | ODP_SHM_NO_HP) static inline uint32_t from_handle(odp_shm_t shm) { diff --git a/platform/linux-generic/odp_stash.c b/platform/linux-generic/odp_stash.c index c7d4136ab..5ff499843 100644 --- a/platform/linux-generic/odp_stash.c +++ b/platform/linux-generic/odp_stash.c @@ -1,9 +1,10 @@ -/* Copyright (c) 2020-2022, Nokia +/* Copyright (c) 2020-2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause */ +#include <odp/api/align.h> #include <odp/api/shared_memory.h> #include <odp/api/stash.h> #include <odp/api/std_types.h> @@ -15,7 +16,10 @@ #include <odp_debug_internal.h> #include <odp_global_data.h> #include <odp_init_internal.h> +#include <odp_libconfig_internal.h> #include <odp_macros_internal.h> +#include <odp_ring_mpmc_u32_internal.h> +#include <odp_ring_mpmc_u64_internal.h> #include <odp_ring_u32_internal.h> #include <odp_ring_u64_internal.h> @@ -26,18 +30,61 @@ ODP_STATIC_ASSERT(CONFIG_INTERNAL_STASHES < CONFIG_MAX_STASHES, "TOO_MANY_INTERNAL_STASHES"); -#define MAX_RING_SIZE (1024 * 1024) #define MIN_RING_SIZE 64 +enum { + STASH_FREE = 0, + STASH_RESERVED, + STASH_ACTIVE +}; + +typedef struct stash_t stash_t; + +typedef void (*ring_u32_init_fn_t)(stash_t *stash); +typedef int32_t (*ring_u32_enq_multi_fn_t)(stash_t *stash, const uint32_t val[], int32_t num); +typedef int32_t (*ring_u32_enq_batch_fn_t)(stash_t *stash, const uint32_t val[], int32_t num); +typedef int32_t (*ring_u32_deq_multi_fn_t)(stash_t *stash, uint32_t val[], int32_t num); +typedef int32_t (*ring_u32_deq_batch_fn_t)(stash_t *stash, uint32_t val[], int32_t num); +typedef int32_t (*ring_u32_len_fn_t)(stash_t *stash); + +typedef void (*ring_u64_init_fn_t)(stash_t *stash); +typedef int32_t (*ring_u64_enq_multi_fn_t)(stash_t *stash, const uint64_t val[], int32_t num); +typedef int32_t (*ring_u64_enq_batch_fn_t)(stash_t *stash, const uint64_t val[], int32_t num); +typedef int32_t (*ring_u64_deq_multi_fn_t)(stash_t *stash, uint64_t val[], int32_t num); +typedef int32_t (*ring_u64_deq_batch_fn_t)(stash_t *stash, uint64_t val[], int32_t num); +typedef int32_t (*ring_u64_len_fn_t)(stash_t *stash); + #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wpedantic" -typedef struct stash_t { - char name[ODP_STASH_NAME_LEN]; - odp_shm_t shm; - int index; +typedef struct ODP_ALIGNED_CACHE stash_t { + /* Ring functions */ + union { + struct { + ring_u32_enq_multi_fn_t enq_multi; + ring_u32_enq_batch_fn_t enq_batch; + ring_u32_deq_multi_fn_t deq_multi; + ring_u32_deq_batch_fn_t deq_batch; + ring_u32_init_fn_t init; + ring_u32_len_fn_t len; + } u32; + + struct { + ring_u64_enq_multi_fn_t enq_multi; + ring_u64_enq_batch_fn_t enq_batch; + ring_u64_deq_multi_fn_t deq_multi; + ring_u64_deq_batch_fn_t deq_batch; + ring_u64_init_fn_t init; + ring_u64_len_fn_t len; + } u64; + } ring_fn; + uint32_t ring_mask; + uint32_t ring_size; uint32_t obj_size; + char name[ODP_STASH_NAME_LEN]; + int index; + /* Ring header followed by variable sized data (object handles) */ union { struct ODP_ALIGNED_CACHE { @@ -49,6 +96,16 @@ typedef struct stash_t { ring_u64_t hdr; uint64_t data[]; } ring_u64; + + struct ODP_ALIGNED_CACHE { + ring_mpmc_u32_t hdr; + uint32_t data[]; + } ring_mpmc_u32; + + struct ODP_ALIGNED_CACHE { + ring_mpmc_u64_t hdr; + uint64_t data[]; + } ring_mpmc_u64; }; } stash_t; @@ -57,23 +114,92 @@ typedef struct stash_t { typedef struct stash_global_t { odp_ticketlock_t lock; odp_shm_t shm; - uint8_t stash_reserved[CONFIG_MAX_STASHES]; + uint32_t max_num; + uint32_t max_num_obj; + uint32_t num_internal; + uint8_t strict_size; + uint8_t stash_state[CONFIG_MAX_STASHES]; stash_t *stash[CONFIG_MAX_STASHES]; + uint8_t data[] ODP_ALIGNED_CACHE; } stash_global_t; static stash_global_t *stash_global; +static inline stash_t *stash_entry(odp_stash_t st) +{ + return (stash_t *)(uintptr_t)st; +} + +static inline odp_stash_t stash_handle(stash_t *stash) +{ + return (odp_stash_t)(uintptr_t)stash; +} + int _odp_stash_init_global(void) { odp_shm_t shm; - - if (odp_global_ro.disable.stash) { + uint32_t max_num, max_num_obj; + const char *str; + uint64_t ring_max_size, stash_max_size, stash_data_size, offset; + const uint32_t internal_stashes = odp_global_ro.disable.dma ? 0 : CONFIG_INTERNAL_STASHES; + uint8_t *stash_data; + uint8_t strict_size; + int val = 0; + + if (odp_global_ro.disable.stash && odp_global_ro.disable.dma) { _ODP_PRINT("Stash is DISABLED\n"); return 0; } - shm = odp_shm_reserve("_odp_stash_global", sizeof(stash_global_t), + _ODP_PRINT("Stash config:\n"); + + str = "stash.max_num"; + if (!_odp_libconfig_lookup_int(str, &val)) { + _ODP_ERR("Config option '%s' not found.\n", str); + return -1; + } + _ODP_PRINT(" %s: %i\n", str, val); + max_num = val; + + str = "stash.max_num_obj"; + if (!_odp_libconfig_lookup_int(str, &val)) { + _ODP_ERR("Config option '%s' not found.\n", str); + return -1; + } + _ODP_PRINT(" %s: %i\n", str, val); + max_num_obj = val; + + str = "stash.strict_size"; + if (!_odp_libconfig_lookup_int(str, &val)) { + _ODP_ERR("Config option '%s' not found.\n", str); + return -1; + } + _ODP_PRINT(" %s: %i\n", str, val); + strict_size = !!val; + + _ODP_PRINT("\n"); + + /* Reserve resources for implementation internal stashes */ + if (max_num > CONFIG_MAX_STASHES - internal_stashes) { + _ODP_ERR("Maximum supported number of stashes: %d\n", + CONFIG_MAX_STASHES - internal_stashes); + return -1; + } + max_num += internal_stashes; + + /* Must have room for minimum sized ring */ + if (max_num_obj < MIN_RING_SIZE) + max_num_obj = MIN_RING_SIZE - 1; + + /* Ring size must be larger than the number of items stored */ + ring_max_size = _ODP_ROUNDUP_POWER2_U32(max_num_obj + 1); + + stash_max_size = _ODP_ROUNDUP_CACHE_LINE(sizeof(stash_t) + + (ring_max_size * sizeof(uint64_t))); + stash_data_size = max_num * stash_max_size; + + shm = odp_shm_reserve("_odp_stash_global", sizeof(stash_global_t) + stash_data_size, ODP_CACHE_LINE_SIZE, 0); stash_global = odp_shm_addr(shm); @@ -85,8 +211,21 @@ int _odp_stash_init_global(void) memset(stash_global, 0, sizeof(stash_global_t)); stash_global->shm = shm; + stash_global->max_num = max_num; + stash_global->max_num_obj = max_num_obj; + stash_global->strict_size = strict_size; + stash_global->num_internal = internal_stashes; odp_ticketlock_init(&stash_global->lock); + /* Initialize stash pointers */ + stash_data = stash_global->data; + offset = 0; + + for (uint32_t i = 0; i < max_num; i++) { + stash_global->stash[i] = (stash_t *)(uintptr_t)(stash_data + offset); + offset += stash_max_size; + } + return 0; } @@ -108,17 +247,21 @@ int _odp_stash_term_global(void) int odp_stash_capability(odp_stash_capability_t *capa, odp_stash_type_t type) { + uint32_t max_stashes; + if (odp_global_ro.disable.stash) { _ODP_ERR("Stash is disabled\n"); return -1; } (void)type; + max_stashes = stash_global->max_num - stash_global->num_internal; + memset(capa, 0, sizeof(odp_stash_capability_t)); - capa->max_stashes_any_type = CONFIG_MAX_STASHES - CONFIG_INTERNAL_STASHES; - capa->max_stashes = CONFIG_MAX_STASHES - CONFIG_INTERNAL_STASHES; - capa->max_num_obj = MAX_RING_SIZE; + capa->max_stashes_any_type = max_stashes; + capa->max_stashes = max_stashes; + capa->max_num_obj = stash_global->max_num_obj; capa->max_obj_size = sizeof(uint64_t); capa->max_get_batch = MIN_RING_SIZE; capa->max_put_batch = MIN_RING_SIZE; @@ -137,15 +280,14 @@ void odp_stash_param_init(odp_stash_param_t *param) static int reserve_index(void) { - int i; int index = -1; odp_ticketlock_lock(&stash_global->lock); - for (i = 0; i < CONFIG_MAX_STASHES; i++) { - if (stash_global->stash_reserved[i] == 0) { + for (uint32_t i = 0; i < stash_global->max_num; i++) { + if (stash_global->stash_state[i] == STASH_FREE) { index = i; - stash_global->stash_reserved[i] = 1; + stash_global->stash_state[i] = STASH_RESERVED; break; } } @@ -159,20 +301,152 @@ static void free_index(int i) { odp_ticketlock_lock(&stash_global->lock); - stash_global->stash[i] = NULL; - stash_global->stash_reserved[i] = 0; + stash_global->stash_state[i] = STASH_FREE; odp_ticketlock_unlock(&stash_global->lock); } +static inline void strict_ring_u32_init(stash_t *stash) +{ + ring_u32_init(&stash->ring_u32.hdr); + + for (uint32_t i = 0; i < stash->ring_size; i++) + stash->ring_u32.data[i] = 0; +} + +static inline void strict_ring_u64_init(stash_t *stash) +{ + ring_u64_init(&stash->ring_u64.hdr); + + for (uint32_t i = 0; i < stash->ring_size; i++) + stash->ring_u64.data[i] = 0; +} + +static inline int32_t strict_ring_u32_enq_multi(stash_t *stash, const uint32_t val[], int32_t num) +{ + /* Success always */ + ring_u32_enq_multi(&stash->ring_u32.hdr, stash->ring_mask, (uint32_t *)(uintptr_t)val, num); + + return num; +} + +static inline int32_t strict_ring_u64_enq_multi(stash_t *stash, const uint64_t val[], int32_t num) +{ + /* Success always */ + ring_u64_enq_multi(&stash->ring_u64.hdr, stash->ring_mask, (uint64_t *)(uintptr_t)val, num); + + return num; +} + +static inline int32_t strict_ring_u32_deq_multi(stash_t *stash, uint32_t val[], int32_t num) +{ + return ring_u32_deq_multi(&stash->ring_u32.hdr, stash->ring_mask, val, num); +} + +static inline int32_t strict_ring_u64_deq_multi(stash_t *stash, uint64_t val[], int32_t num) +{ + return ring_u64_deq_multi(&stash->ring_u64.hdr, stash->ring_mask, val, num); +} + +static inline int32_t strict_ring_u32_deq_batch(stash_t *stash, uint32_t val[], int32_t num) +{ + return ring_u32_deq_batch(&stash->ring_u32.hdr, stash->ring_mask, val, num); +} + +static inline int32_t strict_ring_u64_deq_batch(stash_t *stash, uint64_t val[], int32_t num) +{ + return ring_u64_deq_batch(&stash->ring_u64.hdr, stash->ring_mask, val, num); +} + +static inline int32_t strict_ring_u32_len(stash_t *stash) +{ + return ring_u32_len(&stash->ring_u32.hdr); +} + +static inline int32_t strict_ring_u64_len(stash_t *stash) +{ + return ring_u64_len(&stash->ring_u64.hdr); +} + +static inline void mpmc_ring_u32_init(stash_t *stash) +{ + ring_mpmc_u32_init(&stash->ring_mpmc_u32.hdr); + + for (uint32_t i = 0; i < stash->ring_size; i++) + stash->ring_mpmc_u32.data[i] = 0; +} + +static inline void mpmc_ring_u64_init(stash_t *stash) +{ + ring_mpmc_u64_init(&stash->ring_mpmc_u64.hdr); + + for (uint32_t i = 0; i < stash->ring_size; i++) + stash->ring_mpmc_u64.data[i] = 0; +} + +static inline int32_t mpmc_ring_u32_enq_multi(stash_t *stash, const uint32_t val[], int32_t num) +{ + return ring_mpmc_u32_enq_multi(&stash->ring_mpmc_u32.hdr, stash->ring_mpmc_u32.data, + stash->ring_mask, val, num); +} + +static inline int32_t mpmc_ring_u64_enq_multi(stash_t *stash, const uint64_t val[], int32_t num) +{ + return ring_mpmc_u64_enq_multi(&stash->ring_mpmc_u64.hdr, stash->ring_mpmc_u64.data, + stash->ring_mask, val, num); +} + +static inline int32_t mpmc_ring_u32_enq_batch(stash_t *stash, const uint32_t val[], int32_t num) +{ + return ring_mpmc_u32_enq_batch(&stash->ring_mpmc_u32.hdr, stash->ring_mpmc_u32.data, + stash->ring_mask, val, num); +} + +static inline int32_t mpmc_ring_u64_enq_batch(stash_t *stash, const uint64_t val[], int32_t num) +{ + return ring_mpmc_u64_enq_batch(&stash->ring_mpmc_u64.hdr, stash->ring_mpmc_u64.data, + stash->ring_mask, val, num); +} + +static inline int32_t mpmc_ring_u32_deq_multi(stash_t *stash, uint32_t val[], int32_t num) +{ + return ring_mpmc_u32_deq_multi(&stash->ring_mpmc_u32.hdr, stash->ring_mpmc_u32.data, + stash->ring_mask, val, num); +} + +static inline int32_t mpmc_ring_u64_deq_multi(stash_t *stash, uint64_t val[], int32_t num) +{ + return ring_mpmc_u64_deq_multi(&stash->ring_mpmc_u64.hdr, stash->ring_mpmc_u64.data, + stash->ring_mask, val, num); +} + +static inline int32_t mpmc_ring_u32_deq_batch(stash_t *stash, uint32_t val[], int32_t num) +{ + return ring_mpmc_u32_deq_batch(&stash->ring_mpmc_u32.hdr, stash->ring_mpmc_u32.data, + stash->ring_mask, val, num); +} + +static inline int32_t mpmc_ring_u64_deq_batch(stash_t *stash, uint64_t val[], int32_t num) +{ + return ring_mpmc_u64_deq_batch(&stash->ring_mpmc_u64.hdr, stash->ring_mpmc_u64.data, + stash->ring_mask, val, num); +} + +static inline int32_t mpmc_ring_u32_len(stash_t *stash) +{ + return ring_mpmc_u32_len(&stash->ring_mpmc_u32.hdr); +} + +static inline int32_t mpmc_ring_u64_len(stash_t *stash) +{ + return ring_mpmc_u64_len(&stash->ring_mpmc_u64.hdr); +} + odp_stash_t odp_stash_create(const char *name, const odp_stash_param_t *param) { - odp_shm_t shm; stash_t *stash; - uint64_t i, ring_size, shm_size; + uint64_t ring_size; int ring_u64, index; - char shm_name[ODP_STASH_NAME_LEN + 8]; - uint32_t shm_flags = 0; if (odp_global_ro.disable.stash) { _ODP_ERR("Stash is disabled\n"); @@ -184,7 +458,7 @@ odp_stash_t odp_stash_create(const char *name, const odp_stash_param_t *param) return ODP_STASH_INVALID; } - if (param->num_obj > MAX_RING_SIZE) { + if (param->num_obj > stash_global->max_num_obj) { _ODP_ERR("Too many objects.\n"); return ODP_STASH_INVALID; } @@ -213,75 +487,71 @@ odp_stash_t odp_stash_create(const char *name, const odp_stash_param_t *param) else ring_size = _ODP_ROUNDUP_POWER2_U32(ring_size + 1); - memset(shm_name, 0, sizeof(shm_name)); - snprintf(shm_name, sizeof(shm_name) - 1, "_stash_%s", name); - - if (ring_u64) - shm_size = sizeof(stash_t) + (ring_size * sizeof(uint64_t)); - else - shm_size = sizeof(stash_t) + (ring_size * sizeof(uint32_t)); - - if (odp_global_ro.shm_single_va) - shm_flags |= ODP_SHM_SINGLE_VA; - - shm = odp_shm_reserve(shm_name, shm_size, ODP_CACHE_LINE_SIZE, shm_flags); - - if (shm == ODP_SHM_INVALID) { - _ODP_ERR("SHM reserve failed.\n"); - free_index(index); - return ODP_STASH_INVALID; - } - - stash = odp_shm_addr(shm); + stash = stash_global->stash[index]; memset(stash, 0, sizeof(stash_t)); - if (ring_u64) { - ring_u64_init(&stash->ring_u64.hdr); - - for (i = 0; i < ring_size; i++) - stash->ring_u64.data[i] = 0; + /* Set ring function pointers */ + if (stash_global->strict_size) { + if (ring_u64) { + stash->ring_fn.u64.init = strict_ring_u64_init; + stash->ring_fn.u64.enq_multi = strict_ring_u64_enq_multi; + stash->ring_fn.u64.enq_batch = strict_ring_u64_enq_multi; + stash->ring_fn.u64.deq_multi = strict_ring_u64_deq_multi; + stash->ring_fn.u64.deq_batch = strict_ring_u64_deq_batch; + stash->ring_fn.u64.len = strict_ring_u64_len; + } else { + stash->ring_fn.u32.init = strict_ring_u32_init; + stash->ring_fn.u32.enq_multi = strict_ring_u32_enq_multi; + stash->ring_fn.u32.enq_batch = strict_ring_u32_enq_multi; + stash->ring_fn.u32.deq_multi = strict_ring_u32_deq_multi; + stash->ring_fn.u32.deq_batch = strict_ring_u32_deq_batch; + stash->ring_fn.u32.len = strict_ring_u32_len; + } } else { - ring_u32_init(&stash->ring_u32.hdr); - - for (i = 0; i < ring_size; i++) - stash->ring_u32.data[i] = 0; + if (ring_u64) { + stash->ring_fn.u64.init = mpmc_ring_u64_init; + stash->ring_fn.u64.enq_multi = mpmc_ring_u64_enq_multi; + stash->ring_fn.u64.enq_batch = mpmc_ring_u64_enq_batch; + stash->ring_fn.u64.deq_multi = mpmc_ring_u64_deq_multi; + stash->ring_fn.u64.deq_batch = mpmc_ring_u64_deq_batch; + stash->ring_fn.u64.len = mpmc_ring_u64_len; + } else { + stash->ring_fn.u32.init = mpmc_ring_u32_init; + stash->ring_fn.u32.enq_multi = mpmc_ring_u32_enq_multi; + stash->ring_fn.u32.enq_batch = mpmc_ring_u32_enq_batch; + stash->ring_fn.u32.deq_multi = mpmc_ring_u32_deq_multi; + stash->ring_fn.u32.deq_batch = mpmc_ring_u32_deq_batch; + stash->ring_fn.u32.len = mpmc_ring_u32_len; + } } if (name) strcpy(stash->name, name); stash->index = index; - stash->shm = shm; stash->obj_size = param->obj_size; stash->ring_mask = ring_size - 1; + stash->ring_size = ring_size; + + if (ring_u64) + stash->ring_fn.u64.init(stash); + else + stash->ring_fn.u32.init(stash); /* This makes stash visible to lookups */ odp_ticketlock_lock(&stash_global->lock); - stash_global->stash[index] = stash; + stash_global->stash_state[index] = STASH_ACTIVE; odp_ticketlock_unlock(&stash_global->lock); - return (odp_stash_t)stash; + return stash_handle(stash); } int odp_stash_destroy(odp_stash_t st) { - stash_t *stash; - int index; - odp_shm_t shm; - if (st == ODP_STASH_INVALID) return -1; - stash = (stash_t *)(uintptr_t)st; - index = stash->index; - shm = stash->shm; - - free_index(index); - - if (odp_shm_free(shm)) { - _ODP_ERR("SHM free failed.\n"); - return -1; - } + free_index(stash_entry(st)->index); return 0; } @@ -293,7 +563,6 @@ uint64_t odp_stash_to_u64(odp_stash_t st) odp_stash_t odp_stash_lookup(const char *name) { - int i; stash_t *stash; if (name == NULL) @@ -301,12 +570,13 @@ odp_stash_t odp_stash_lookup(const char *name) odp_ticketlock_lock(&stash_global->lock); - for (i = 0; i < CONFIG_MAX_STASHES; i++) { + for (uint32_t i = 0; i < stash_global->max_num; i++) { stash = stash_global->stash[i]; - if (stash && strcmp(stash->name, name) == 0) { + if (stash_global->stash_state[i] == STASH_ACTIVE && + strcmp(stash->name, name) == 0) { odp_ticketlock_unlock(&stash_global->lock); - return (odp_stash_t)stash; + return stash_handle(stash); } } @@ -315,57 +585,51 @@ odp_stash_t odp_stash_lookup(const char *name) return ODP_STASH_INVALID; } -static inline int32_t stash_put(odp_stash_t st, const void *obj, int32_t num) +static inline int32_t stash_put(odp_stash_t st, const void *obj, int32_t num, odp_bool_t is_batch) { - stash_t *stash; + int32_t (*ring_u32_enq)(stash_t *stash, const uint32_t val[], int32_t num); + int32_t (*ring_u64_enq)(stash_t *stash, const uint64_t val[], int32_t num); + stash_t *stash = stash_entry(st); uint32_t obj_size; int32_t i; - stash = (stash_t *)(uintptr_t)st; - if (odp_unlikely(st == ODP_STASH_INVALID)) return -1; - obj_size = stash->obj_size; - - if (obj_size == sizeof(uint64_t)) { - ring_u64_t *ring_u64 = &stash->ring_u64.hdr; - - ring_u64_enq_multi(ring_u64, stash->ring_mask, - (uint64_t *)(uintptr_t)obj, num); - return num; + if (is_batch) { + ring_u32_enq = stash->ring_fn.u32.enq_batch; + ring_u64_enq = stash->ring_fn.u64.enq_batch; + } else { + ring_u32_enq = stash->ring_fn.u32.enq_multi; + ring_u64_enq = stash->ring_fn.u64.enq_multi; } - if (obj_size == sizeof(uint32_t)) { - ring_u32_t *ring_u32 = &stash->ring_u32.hdr; + obj_size = stash->obj_size; - ring_u32_enq_multi(ring_u32, stash->ring_mask, - (uint32_t *)(uintptr_t)obj, num); - return num; - } + if (obj_size == sizeof(uint64_t)) + return ring_u64_enq(stash, (uint64_t *)(uintptr_t)obj, num); + + if (obj_size == sizeof(uint32_t)) + return ring_u32_enq(stash, (uint32_t *)(uintptr_t)obj, num); if (obj_size == sizeof(uint16_t)) { const uint16_t *u16_ptr = obj; - ring_u32_t *ring_u32 = &stash->ring_u32.hdr; uint32_t u32[num]; for (i = 0; i < num; i++) u32[i] = u16_ptr[i]; - ring_u32_enq_multi(ring_u32, stash->ring_mask, u32, num); - return num; + return ring_u32_enq(stash, u32, num); } if (obj_size == sizeof(uint8_t)) { const uint8_t *u8_ptr = obj; - ring_u32_t *ring_u32 = &stash->ring_u32.hdr; uint32_t u32[num]; for (i = 0; i < num; i++) u32[i] = u8_ptr[i]; - ring_u32_enq_multi(ring_u32, stash->ring_mask, u32, num); - return num; + return ring_u32_enq(stash, u32, num); } return -1; @@ -373,73 +637,66 @@ static inline int32_t stash_put(odp_stash_t st, const void *obj, int32_t num) int32_t odp_stash_put(odp_stash_t st, const void *obj, int32_t num) { - return stash_put(st, obj, num); + return stash_put(st, obj, num, false); } int32_t odp_stash_put_batch(odp_stash_t st, const void *obj, int32_t num) { - /* Returns always 'num', or -1 on failure. */ - return stash_put(st, obj, num); + return stash_put(st, obj, num, true); } -static inline int32_t stash_put_u32(odp_stash_t st, const uint32_t val[], - int32_t num) +int32_t odp_stash_put_u32(odp_stash_t st, const uint32_t val[], int32_t num) { - stash_t *stash = (stash_t *)(uintptr_t)st; + stash_t *stash = stash_entry(st); if (odp_unlikely(st == ODP_STASH_INVALID)) return -1; _ODP_ASSERT(stash->obj_size == sizeof(uint32_t)); - ring_u32_enq_multi(&stash->ring_u32.hdr, stash->ring_mask, - (uint32_t *)(uintptr_t)val, num); - return num; + return stash->ring_fn.u32.enq_multi(stash, val, num); } -int32_t odp_stash_put_u32(odp_stash_t st, const uint32_t val[], int32_t num) +int32_t odp_stash_put_u32_batch(odp_stash_t st, const uint32_t val[], int32_t num) { - return stash_put_u32(st, val, num); -} + stash_t *stash = stash_entry(st); -int32_t odp_stash_put_u32_batch(odp_stash_t st, const uint32_t val[], - int32_t num) -{ - /* Returns always 'num', or -1 on failure. */ - return stash_put_u32(st, val, num); + if (odp_unlikely(st == ODP_STASH_INVALID)) + return -1; + + _ODP_ASSERT(stash->obj_size == sizeof(uint32_t)); + + return stash->ring_fn.u32.enq_batch(stash, val, num); } -static inline int32_t stash_put_u64(odp_stash_t st, const uint64_t val[], - int32_t num) +int32_t odp_stash_put_u64(odp_stash_t st, const uint64_t val[], int32_t num) { - stash_t *stash = (stash_t *)(uintptr_t)st; + stash_t *stash = stash_entry(st); if (odp_unlikely(st == ODP_STASH_INVALID)) return -1; _ODP_ASSERT(stash->obj_size == sizeof(uint64_t)); - ring_u64_enq_multi(&stash->ring_u64.hdr, stash->ring_mask, - (uint64_t *)(uintptr_t)val, num); - return num; -} - -int32_t odp_stash_put_u64(odp_stash_t st, const uint64_t val[], int32_t num) -{ - return stash_put_u64(st, val, num); + return stash->ring_fn.u64.enq_multi(stash, (uint64_t *)(uintptr_t)val, num); } int32_t odp_stash_put_u64_batch(odp_stash_t st, const uint64_t val[], int32_t num) { - /* Returns always 'num', or -1 on failure. */ - return stash_put_u64(st, val, num); + stash_t *stash = stash_entry(st); + + if (odp_unlikely(st == ODP_STASH_INVALID)) + return -1; + + _ODP_ASSERT(stash->obj_size == sizeof(uint64_t)); + + return stash->ring_fn.u64.enq_batch(stash, (uint64_t *)(uintptr_t)val, num); } -static inline int32_t stash_put_ptr(odp_stash_t st, const uintptr_t ptr[], - int32_t num) +int32_t odp_stash_put_ptr(odp_stash_t st, const uintptr_t ptr[], int32_t num) { - stash_t *stash = (stash_t *)(uintptr_t)st; + stash_t *stash = stash_entry(st); if (odp_unlikely(st == ODP_STASH_INVALID)) return -1; @@ -447,69 +704,65 @@ static inline int32_t stash_put_ptr(odp_stash_t st, const uintptr_t ptr[], _ODP_ASSERT(stash->obj_size == sizeof(uintptr_t)); if (sizeof(uintptr_t) == sizeof(uint32_t)) - ring_u32_enq_multi(&stash->ring_u32.hdr, stash->ring_mask, - (uint32_t *)(uintptr_t)ptr, num); - else if (sizeof(uintptr_t) == sizeof(uint64_t)) - ring_u64_enq_multi(&stash->ring_u64.hdr, stash->ring_mask, - (uint64_t *)(uintptr_t)ptr, num); - else - return -1; + return stash->ring_fn.u32.enq_multi(stash, (uint32_t *)(uintptr_t)ptr, num); - return num; -} + if (sizeof(uintptr_t) == sizeof(uint64_t)) + return stash->ring_fn.u64.enq_multi(stash, (uint64_t *)(uintptr_t)ptr, num); -int32_t odp_stash_put_ptr(odp_stash_t st, const uintptr_t ptr[], int32_t num) -{ - return stash_put_ptr(st, ptr, num); + return -1; } int32_t odp_stash_put_ptr_batch(odp_stash_t st, const uintptr_t ptr[], int32_t num) { - /* Returns always 'num', or -1 on failure. */ - return stash_put_ptr(st, ptr, num); + stash_t *stash = stash_entry(st); + + if (odp_unlikely(st == ODP_STASH_INVALID)) + return -1; + + _ODP_ASSERT(stash->obj_size == sizeof(uintptr_t)); + + if (sizeof(uintptr_t) == sizeof(uint32_t)) + return stash->ring_fn.u32.enq_batch(stash, (uint32_t *)(uintptr_t)ptr, num); + + if (sizeof(uintptr_t) == sizeof(uint64_t)) + return stash->ring_fn.u64.enq_batch(stash, (uint64_t *)(uintptr_t)ptr, num); + + return -1; } -static inline int32_t stash_get(odp_stash_t st, void *obj, int32_t num, odp_bool_t batch) +static inline int32_t stash_get(odp_stash_t st, void *obj, int32_t num, odp_bool_t is_batch) { - stash_t *stash; + int32_t (*ring_u32_deq)(stash_t *stash, uint32_t val[], int32_t num); + int32_t (*ring_u64_deq)(stash_t *stash, uint64_t val[], int32_t num); + stash_t *stash = stash_entry(st); uint32_t obj_size; uint32_t i, num_deq; - stash = (stash_t *)(uintptr_t)st; - if (odp_unlikely(st == ODP_STASH_INVALID)) return -1; - obj_size = stash->obj_size; - - if (obj_size == sizeof(uint64_t)) { - ring_u64_t *ring_u64 = &stash->ring_u64.hdr; - - if (batch) - return ring_u64_deq_batch(ring_u64, stash->ring_mask, obj, num); - else - return ring_u64_deq_multi(ring_u64, stash->ring_mask, obj, num); + if (is_batch) { + ring_u32_deq = stash->ring_fn.u32.deq_batch; + ring_u64_deq = stash->ring_fn.u64.deq_batch; + } else { + ring_u32_deq = stash->ring_fn.u32.deq_multi; + ring_u64_deq = stash->ring_fn.u64.deq_multi; } - if (obj_size == sizeof(uint32_t)) { - ring_u32_t *ring_u32 = &stash->ring_u32.hdr; + obj_size = stash->obj_size; + + if (obj_size == sizeof(uint64_t)) + return ring_u64_deq(stash, obj, num); - if (batch) - return ring_u32_deq_batch(ring_u32, stash->ring_mask, obj, num); - else - return ring_u32_deq_multi(ring_u32, stash->ring_mask, obj, num); - } + if (obj_size == sizeof(uint32_t)) + return ring_u32_deq(stash, obj, num); if (obj_size == sizeof(uint16_t)) { uint16_t *u16_ptr = obj; - ring_u32_t *ring_u32 = &stash->ring_u32.hdr; uint32_t u32[num]; - if (batch) - num_deq = ring_u32_deq_batch(ring_u32, stash->ring_mask, u32, num); - else - num_deq = ring_u32_deq_multi(ring_u32, stash->ring_mask, u32, num); + num_deq = ring_u32_deq(stash, u32, num); for (i = 0; i < num_deq; i++) u16_ptr[i] = u32[i]; @@ -519,13 +772,9 @@ static inline int32_t stash_get(odp_stash_t st, void *obj, int32_t num, odp_bool if (obj_size == sizeof(uint8_t)) { uint8_t *u8_ptr = obj; - ring_u32_t *ring_u32 = &stash->ring_u32.hdr; uint32_t u32[num]; - if (batch) - num_deq = ring_u32_deq_batch(ring_u32, stash->ring_mask, u32, num); - else - num_deq = ring_u32_deq_multi(ring_u32, stash->ring_mask, u32, num); + num_deq = ring_u32_deq(stash, u32, num); for (i = 0; i < num_deq; i++) u8_ptr[i] = u32[i]; @@ -538,67 +787,65 @@ static inline int32_t stash_get(odp_stash_t st, void *obj, int32_t num, odp_bool int32_t odp_stash_get(odp_stash_t st, void *obj, int32_t num) { - return stash_get(st, obj, num, 0); + return stash_get(st, obj, num, false); } int32_t odp_stash_get_batch(odp_stash_t st, void *obj, int32_t num) { - return stash_get(st, obj, num, 1); + return stash_get(st, obj, num, true); } int32_t odp_stash_get_u32(odp_stash_t st, uint32_t val[], int32_t num) { - stash_t *stash = (stash_t *)(uintptr_t)st; + stash_t *stash = stash_entry(st); if (odp_unlikely(st == ODP_STASH_INVALID)) return -1; _ODP_ASSERT(stash->obj_size == sizeof(uint32_t)); - return ring_u32_deq_multi(&stash->ring_u32.hdr, stash->ring_mask, val, - num); + return stash->ring_fn.u32.deq_multi(stash, val, num); } int32_t odp_stash_get_u32_batch(odp_stash_t st, uint32_t val[], int32_t num) { - stash_t *stash = (stash_t *)(uintptr_t)st; + stash_t *stash = stash_entry(st); if (odp_unlikely(st == ODP_STASH_INVALID)) return -1; _ODP_ASSERT(stash->obj_size == sizeof(uint32_t)); - return ring_u32_deq_batch(&stash->ring_u32.hdr, stash->ring_mask, val, num); + return stash->ring_fn.u32.deq_batch(stash, val, num); } int32_t odp_stash_get_u64(odp_stash_t st, uint64_t val[], int32_t num) { - stash_t *stash = (stash_t *)(uintptr_t)st; + stash_t *stash = stash_entry(st); if (odp_unlikely(st == ODP_STASH_INVALID)) return -1; _ODP_ASSERT(stash->obj_size == sizeof(uint64_t)); - return ring_u64_deq_multi(&stash->ring_u64.hdr, stash->ring_mask, val, - num); + return stash->ring_fn.u64.deq_multi(stash, val, num); } int32_t odp_stash_get_u64_batch(odp_stash_t st, uint64_t val[], int32_t num) { - stash_t *stash = (stash_t *)(uintptr_t)st; + stash_t *stash = stash_entry(st); if (odp_unlikely(st == ODP_STASH_INVALID)) return -1; _ODP_ASSERT(stash->obj_size == sizeof(uint64_t)); - return ring_u64_deq_batch(&stash->ring_u64.hdr, stash->ring_mask, val, num); + return stash->ring_fn.u64.deq_batch(stash, val, num); } int32_t odp_stash_get_ptr(odp_stash_t st, uintptr_t ptr[], int32_t num) { - stash_t *stash = (stash_t *)(uintptr_t)st; + stash_t *stash = stash_entry(st); if (odp_unlikely(st == ODP_STASH_INVALID)) return -1; @@ -606,19 +853,17 @@ int32_t odp_stash_get_ptr(odp_stash_t st, uintptr_t ptr[], int32_t num) _ODP_ASSERT(stash->obj_size == sizeof(uintptr_t)); if (sizeof(uintptr_t) == sizeof(uint32_t)) - return ring_u32_deq_multi(&stash->ring_u32.hdr, - stash->ring_mask, - (uint32_t *)(uintptr_t)ptr, num); - else if (sizeof(uintptr_t) == sizeof(uint64_t)) - return ring_u64_deq_multi(&stash->ring_u64.hdr, - stash->ring_mask, - (uint64_t *)(uintptr_t)ptr, num); + return stash->ring_fn.u32.deq_multi(stash, (uint32_t *)(uintptr_t)ptr, num); + + if (sizeof(uintptr_t) == sizeof(uint64_t)) + return stash->ring_fn.u64.deq_multi(stash, (uint64_t *)(uintptr_t)ptr, num); + return -1; } int32_t odp_stash_get_ptr_batch(odp_stash_t st, uintptr_t ptr[], int32_t num) { - stash_t *stash = (stash_t *)(uintptr_t)st; + stash_t *stash = stash_entry(st); if (odp_unlikely(st == ODP_STASH_INVALID)) return -1; @@ -626,11 +871,11 @@ int32_t odp_stash_get_ptr_batch(odp_stash_t st, uintptr_t ptr[], int32_t num) _ODP_ASSERT(stash->obj_size == sizeof(uintptr_t)); if (sizeof(uintptr_t) == sizeof(uint32_t)) - return ring_u32_deq_batch(&stash->ring_u32.hdr, stash->ring_mask, - (uint32_t *)(uintptr_t)ptr, num); - else if (sizeof(uintptr_t) == sizeof(uint64_t)) - return ring_u64_deq_batch(&stash->ring_u64.hdr, stash->ring_mask, - (uint64_t *)(uintptr_t)ptr, num); + return stash->ring_fn.u32.deq_batch(stash, (uint32_t *)(uintptr_t)ptr, num); + + if (sizeof(uintptr_t) == sizeof(uint64_t)) + return stash->ring_fn.u64.deq_batch(stash, (uint64_t *)(uintptr_t)ptr, num); + return -1; } @@ -644,23 +889,17 @@ int odp_stash_flush_cache(odp_stash_t st) static uint32_t stash_obj_count(stash_t *stash) { - ring_u32_t *ring_u32; uint32_t obj_size = stash->obj_size; - if (obj_size == sizeof(uint64_t)) { - ring_u64_t *ring_u64 = &stash->ring_u64.hdr; - - return ring_u64_len(ring_u64); - } - - ring_u32 = &stash->ring_u32.hdr; + if (obj_size == sizeof(uint64_t)) + return stash->ring_fn.u64.len(stash); - return ring_u32_len(ring_u32); + return stash->ring_fn.u32.len(stash); } void odp_stash_print(odp_stash_t st) { - stash_t *stash = (stash_t *)(uintptr_t)st; + stash_t *stash = stash_entry(st); if (st == ODP_STASH_INVALID) { _ODP_ERR("Bad stash handle\n"); @@ -674,13 +913,13 @@ void odp_stash_print(odp_stash_t st) _ODP_PRINT(" index %i\n", stash->index); _ODP_PRINT(" obj size %u\n", stash->obj_size); _ODP_PRINT(" obj count %u\n", stash_obj_count(stash)); - _ODP_PRINT(" ring size %u\n", stash->ring_mask + 1); + _ODP_PRINT(" ring size %u\n", stash->ring_size); _ODP_PRINT("\n"); } int odp_stash_stats(odp_stash_t st, odp_stash_stats_t *stats) { - stash_t *stash = (stash_t *)(uintptr_t)st; + stash_t *stash = stash_entry(st); if (st == ODP_STASH_INVALID) { _ODP_ERR("Bad stash handle\n"); diff --git a/platform/linux-generic/odp_timer.c b/platform/linux-generic/odp_timer.c index 05660eb7a..8597c3f4e 100644 --- a/platform/linux-generic/odp_timer.c +++ b/platform/linux-generic/odp_timer.c @@ -17,6 +17,7 @@ #include <odp/api/atomic.h> #include <odp/api/cpu.h> #include <odp/api/debug.h> +#include <odp/api/deprecated.h> #include <odp/api/event.h> #include <odp/api/hints.h> #include <odp/api/pool.h> @@ -1494,9 +1495,7 @@ odp_event_t odp_timer_free(odp_timer_t hdl) return timer_free(tp, idx); } -int odp_timer_set_abs(odp_timer_t hdl, - uint64_t abs_tck, - odp_event_t *tmo_ev) +int ODP_DEPRECATE(odp_timer_set_abs)(odp_timer_t hdl, uint64_t abs_tck, odp_event_t *tmo_ev) { timer_pool_t *tp = handle_to_tp(hdl); uint64_t cur_tick = current_nsec(tp); @@ -1512,9 +1511,7 @@ int odp_timer_set_abs(odp_timer_t hdl, return ODP_TIMER_FAIL; } -int odp_timer_set_rel(odp_timer_t hdl, - uint64_t rel_tck, - odp_event_t *tmo_ev) +int ODP_DEPRECATE(odp_timer_set_rel)(odp_timer_t hdl, uint64_t rel_tck, odp_event_t *tmo_ev) { timer_pool_t *tp = handle_to_tp(hdl); uint64_t cur_tick = current_nsec(tp); diff --git a/platform/linux-generic/odp_traffic_mngr.c b/platform/linux-generic/odp_traffic_mngr.c index 284a6f7f5..c8d8bc5c0 100644 --- a/platform/linux-generic/odp_traffic_mngr.c +++ b/platform/linux-generic/odp_traffic_mngr.c @@ -2665,12 +2665,6 @@ static int tm_capabilities(odp_tm_capabilities_t capabilities[], return 1; } -int ODP_DEPRECATE(odp_tm_capabilities)(odp_tm_capabilities_t capabilities[], - uint32_t capabilities_size) -{ - return tm_capabilities(capabilities, capabilities_size); -} - int odp_tm_egress_capabilities(odp_tm_capabilities_t *capabilities, const odp_tm_egress_t *egress) { diff --git a/platform/linux-generic/pktio/dpdk.c b/platform/linux-generic/pktio/dpdk.c index 7e9db90ce..fa40d1bde 100644 --- a/platform/linux-generic/pktio/dpdk.c +++ b/platform/linux-generic/pktio/dpdk.c @@ -49,6 +49,7 @@ #include <rte_tcp.h> #include <rte_udp.h> #include <rte_version.h> +#include <rte_vfio.h> /* NUMA is not supported on all platforms */ #ifdef _ODP_HAVE_NUMA_LIBRARY @@ -438,6 +439,15 @@ static struct rte_mempool *mbuf_pool_create(const char *name, goto fail; } + /* Map pages for DMA access to enable VFIO usage */ + for (uint64_t i = 0; i < pool_entry->shm_size; i += page_size) { + addr = pool_entry->base_addr + i; + + rte_vfio_container_dma_map(RTE_VFIO_DEFAULT_CONTAINER_FD, + (uint64_t)(uintptr_t)addr, + rte_mem_virt2iova(addr), page_size); + } + rte_mempool_obj_iter(mp, pktmbuf_init, NULL); return mp; diff --git a/platform/linux-generic/pktio/loop.c b/platform/linux-generic/pktio/loop.c index b30535f22..a69daf56b 100644 --- a/platform/linux-generic/pktio/loop.c +++ b/platform/linux-generic/pktio/loop.c @@ -1,5 +1,5 @@ /* Copyright (c) 2013-2018, Linaro Limited - * Copyright (c) 2013-2022, Nokia Solutions and Networks + * Copyright (c) 2013-2023, Nokia Solutions and Networks * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -7,11 +7,11 @@ #include <odp/api/debug.h> #include <odp/api/event.h> +#include <odp/api/hash.h> #include <odp/api/hints.h> #include <odp/api/packet.h> #include <odp/api/packet_io.h> #include <odp/api/queue.h> -#include <odp/api/ticketlock.h> #include <odp/api/time.h> #include <odp/api/plat/byteorder_inlines.h> @@ -39,6 +39,9 @@ #include <stdint.h> #include <stdlib.h> +#define MAX_QUEUES (ODP_PKTIN_MAX_QUEUES > ODP_PKTOUT_MAX_QUEUES ? \ + ODP_PKTIN_MAX_QUEUES : ODP_PKTOUT_MAX_QUEUES) + #define MAX_LOOP 16 #define LOOP_MTU_MIN 68 @@ -47,12 +50,40 @@ #define LOOP_MAX_QUEUE_SIZE 1024 typedef struct { - odp_queue_t loopq; /**< loopback queue for "loop" device */ - uint32_t pktin_queue_size; /**< input queue size */ - uint32_t pktout_queue_size; /**< output queue size */ - uint16_t mtu; /**< link MTU */ - uint8_t idx; /**< index of "loop" device */ - uint8_t queue_create; /**< create or re-create queue during start */ + odp_atomic_u64_t in_octets; + odp_atomic_u64_t in_packets; + odp_atomic_u64_t in_discards; + odp_atomic_u64_t in_errors; + odp_atomic_u64_t out_octets; + odp_atomic_u64_t out_packets; +} stats_t; + +typedef struct ODP_ALIGNED_CACHE { + /* queue handle as the "wire" */ + odp_queue_t queue; + /* queue specific statistics */ + stats_t stats; + /* config input queue size */ + uint32_t in_size; + /* config output queue size */ + uint32_t out_size; +} loop_queue_t; + +typedef struct { + /* loopback entries for "loop" device */ + loop_queue_t loopqs[MAX_QUEUES]; + /* hash config */ + odp_pktin_hash_proto_t hash; + /* config queue count */ + uint32_t num_conf_qs; + /* actual number queues */ + uint32_t num_qs; + /* link MTU */ + uint16_t mtu; + /* index of "loop" device */ + uint8_t idx; + /* create or re-create queue during start */ + uint8_t queue_create; } pkt_loop_t; ODP_STATIC_ASSERT(PKTIO_PRIVATE_SIZE >= sizeof(pkt_loop_t), @@ -66,7 +97,6 @@ static inline pkt_loop_t *pkt_priv(pktio_entry_t *pktio_entry) /* MAC address for the "loop" interface */ static const uint8_t pktio_loop_mac[] = {0x02, 0xe9, 0x34, 0x80, 0x73, 0x01}; -static int loopback_stats_reset(pktio_entry_t *pktio_entry); static int loopback_init_capability(pktio_entry_t *pktio_entry); static int loopback_open(odp_pktio_t id ODP_UNUSED, pktio_entry_t *pktio_entry, @@ -88,14 +118,20 @@ static int loopback_open(odp_pktio_t id ODP_UNUSED, pktio_entry_t *pktio_entry, } memset(pkt_loop, 0, sizeof(pkt_loop_t)); - pkt_loop->idx = idx; pkt_loop->mtu = LOOP_MTU_MAX; - pkt_loop->loopq = ODP_QUEUE_INVALID; + pkt_loop->idx = idx; pkt_loop->queue_create = 1; - - loopback_stats_reset(pktio_entry); loopback_init_capability(pktio_entry); + for (uint32_t i = 0; i < MAX_QUEUES; i++) { + odp_atomic_init_u64(&pkt_loop->loopqs[i].stats.in_octets, 0); + odp_atomic_init_u64(&pkt_loop->loopqs[i].stats.in_packets, 0); + odp_atomic_init_u64(&pkt_loop->loopqs[i].stats.in_discards, 0); + odp_atomic_init_u64(&pkt_loop->loopqs[i].stats.in_errors, 0); + odp_atomic_init_u64(&pkt_loop->loopqs[i].stats.out_octets, 0); + odp_atomic_init_u64(&pkt_loop->loopqs[i].stats.out_packets, 0); + } + return 0; } @@ -117,6 +153,18 @@ static int loopback_queue_destroy(odp_queue_t queue) return 0; } +static int loopback_queues_destroy(loop_queue_t *queues, uint32_t num_queues) +{ + int ret = 0; + + for (uint32_t i = 0; i < num_queues; i++) { + if (loopback_queue_destroy(queues[i].queue)) + ret = -1; + } + + return ret; +} + static int loopback_start(pktio_entry_t *pktio_entry) { pkt_loop_t *pkt_loop = pkt_priv(pktio_entry); @@ -127,23 +175,28 @@ static int loopback_start(pktio_entry_t *pktio_entry) if (!pkt_loop->queue_create) return 0; - /* Destroy old queue */ - if (pkt_loop->loopq != ODP_QUEUE_INVALID && loopback_queue_destroy(pkt_loop->loopq)) + /* Destroy old queues */ + if (loopback_queues_destroy(pkt_loop->loopqs, pkt_loop->num_qs)) return -1; - odp_queue_param_init(&queue_param); - queue_param.size = pkt_loop->pktin_queue_size > pkt_loop->pktout_queue_size ? - pkt_loop->pktin_queue_size : pkt_loop->pktout_queue_size; + pkt_loop->num_qs = 0; - snprintf(queue_name, sizeof(queue_name), "_odp_pktio_loopq-%" PRIu64 "", - odp_pktio_to_u64(pktio_entry->handle)); + for (uint32_t i = 0; i < pkt_loop->num_conf_qs; i++) { + odp_queue_param_init(&queue_param); + queue_param.size = _ODP_MAX(pkt_loop->loopqs[i].in_size, + pkt_loop->loopqs[i].out_size); + snprintf(queue_name, sizeof(queue_name), "_odp_pktio_loopq-%" PRIu64 "-%u", + odp_pktio_to_u64(pktio_entry->handle), i); + pkt_loop->loopqs[i].queue = odp_queue_create(queue_name, &queue_param); - pkt_loop->loopq = odp_queue_create(queue_name, &queue_param); - if (pkt_loop->loopq == ODP_QUEUE_INVALID) { - _ODP_ERR("Creating loopback pktio queue failed\n"); - return -1; + if (pkt_loop->loopqs[i].queue == ODP_QUEUE_INVALID) { + _ODP_ERR("Creating loopback pktio queue %s failed\n", queue_name); + (void)loopback_queues_destroy(pkt_loop->loopqs, i); + return -1; + } } - pkt_loop->queue_create = 0; + + pkt_loop->num_qs = pkt_loop->num_conf_qs; return 0; } @@ -153,9 +206,17 @@ static int loopback_pktin_queue_config(pktio_entry_t *pktio_entry, { pkt_loop_t *pkt_loop = pkt_priv(pktio_entry); + pkt_loop->num_conf_qs = param->num_queues; + pkt_loop->queue_create = 1; + pkt_loop->hash.all_bits = param->hash_enable ? param->hash_proto.all_bits : 0; + if (pktio_entry->param.in_mode == ODP_PKTIN_MODE_DIRECT) { - pkt_loop->pktin_queue_size = param->queue_size[0]; - pkt_loop->queue_create = 1; + for (uint32_t i = 0; i < MAX_QUEUES; i++) { + if (i < pkt_loop->num_conf_qs) + pkt_loop->loopqs[i].in_size = param->queue_size[i]; + else + pkt_loop->loopqs[i].in_size = 0; + } } return 0; @@ -166,9 +227,15 @@ static int loopback_pktout_queue_config(pktio_entry_t *pktio_entry, { pkt_loop_t *pkt_loop = pkt_priv(pktio_entry); - pkt_loop->pktout_queue_size = param->queue_size[0]; pkt_loop->queue_create = 1; + for (uint32_t i = 0; i < MAX_QUEUES; i++) { + if (i < param->num_queues) + pkt_loop->loopqs[i].out_size = param->queue_size[i]; + else + pkt_loop->loopqs[i].out_size = 0; + } + return 0; } @@ -176,18 +243,16 @@ static int loopback_close(pktio_entry_t *pktio_entry) { pkt_loop_t *pkt_loop = pkt_priv(pktio_entry); - if (pkt_loop->loopq != ODP_QUEUE_INVALID) - return loopback_queue_destroy(pkt_loop->loopq); - - return 0; + return loopback_queues_destroy(pkt_loop->loopqs, pkt_loop->num_qs); } -static int loopback_recv(pktio_entry_t *pktio_entry, int index ODP_UNUSED, - odp_packet_t pkts[], int num) +static int loopback_recv(pktio_entry_t *pktio_entry, int index, odp_packet_t pkts[], int num) { int nbr, i; + loop_queue_t *entry = &pkt_priv(pktio_entry)->loopqs[index]; + odp_queue_t queue = entry->queue; + stats_t *stats = &entry->stats; _odp_event_hdr_t *hdr_tbl[QUEUE_MULTI_MAX]; - odp_queue_t queue; odp_packet_hdr_t *pkt_hdr; odp_packet_t pkt; odp_time_t ts_val; @@ -201,9 +266,6 @@ static int loopback_recv(pktio_entry_t *pktio_entry, int index ODP_UNUSED, if (odp_unlikely(num > QUEUE_MULTI_MAX)) num = QUEUE_MULTI_MAX; - odp_ticketlock_lock(&pktio_entry->rxl); - - queue = pkt_priv(pktio_entry)->loopq; nbr = odp_queue_deq_multi(queue, (odp_event_t *)hdr_tbl, num); if (opt.bit.ts_all || opt.bit.ts_ptp) { @@ -239,7 +301,7 @@ static int loopback_recv(pktio_entry_t *pktio_entry, int index ODP_UNUSED, ret = _odp_packet_parse_common(pkt_hdr, pkt_addr, pkt_len, seg_len, layer, opt); if (ret) - odp_atomic_inc_u64(&pktio_entry->stats_extra.in_errors); + odp_atomic_inc_u64(&stats->in_errors); if (ret < 0) { odp_packet_free(pkt); @@ -252,7 +314,7 @@ static int loopback_recv(pktio_entry_t *pktio_entry, int index ODP_UNUSED, ret = _odp_cls_classify_packet(pktio_entry, pkt_addr, &new_pool, pkt_hdr); if (ret < 0) - odp_atomic_inc_u64(&pktio_entry->stats_extra.in_discards); + odp_atomic_inc_u64(&stats->in_discards); if (ret) { odp_packet_free(pkt); @@ -262,7 +324,7 @@ static int loopback_recv(pktio_entry_t *pktio_entry, int index ODP_UNUSED, if (odp_unlikely(_odp_pktio_packet_to_pool( &pkt, &pkt_hdr, new_pool))) { odp_packet_free(pkt); - odp_atomic_inc_u64(&pktio_entry->stats_extra.in_discards); + odp_atomic_inc_u64(&stats->in_discards); continue; } } @@ -285,10 +347,8 @@ static int loopback_recv(pktio_entry_t *pktio_entry, int index ODP_UNUSED, pkts[num_rx++] = pkt; } - pktio_entry->stats.in_octets += octets; - pktio_entry->stats.in_packets += packets; - - odp_ticketlock_unlock(&pktio_entry->rxl); + odp_atomic_add_u64(&stats->in_octets, octets); + odp_atomic_add_u64(&stats->in_packets, packets); return num_rx; } @@ -381,22 +441,82 @@ static inline void loopback_fix_checksums(odp_packet_t pkt, _odp_packet_sctp_chksum_insert(pkt); } -static int loopback_send(pktio_entry_t *pktio_entry, int index ODP_UNUSED, - const odp_packet_t pkt_tbl[], int num) +static inline uint8_t *add_data(uint8_t *data, void *src, uint32_t len) +{ + return (uint8_t *)memcpy(data, src, len) + len; +} + +static inline odp_queue_t get_dest_queue(const pkt_loop_t *pkt_loop, odp_packet_t pkt, int index) +{ + const odp_pktin_hash_proto_t *hash = &pkt_loop->hash; + _odp_udphdr_t udp; + _odp_tcphdr_t tcp; + _odp_ipv4hdr_t ipv4; + _odp_ipv6hdr_t ipv6; + uint32_t off; + /* Space for UDP/TCP source and destination ports and IPv4/IPv6 source and destination + * addresses. */ + uint8_t data[2 * sizeof(uint16_t) + 2 * 4 * sizeof(uint32_t)]; + uint8_t *head = data; + + if (hash->all_bits == 0) + return pkt_loop->loopqs[index % pkt_loop->num_qs].queue; + + memset(data, 0, sizeof(data)); + off = odp_packet_l4_offset(pkt); + + if (off != ODP_PACKET_OFFSET_INVALID) { + if ((hash->proto.ipv4_udp || hash->proto.ipv6_udp) && odp_packet_has_udp(pkt)) { + if (odp_packet_copy_to_mem(pkt, off, _ODP_UDPHDR_LEN, &udp) == 0) { + head = add_data(head, &udp.src_port, sizeof(udp.src_port)); + head = add_data(head, &udp.dst_port, sizeof(udp.dst_port)); + } + } else if ((hash->proto.ipv4_tcp || hash->proto.ipv6_tcp) && + odp_packet_has_tcp(pkt)) { + if (odp_packet_copy_to_mem(pkt, off, _ODP_TCPHDR_LEN, &tcp) == 0) { + head = add_data(head, &tcp.src_port, sizeof(tcp.src_port)); + head = add_data(head, &tcp.dst_port, sizeof(tcp.dst_port)); + } + } + } + + off = odp_packet_l3_offset(pkt); + + if (off != ODP_PACKET_OFFSET_INVALID) { + if (hash->proto.ipv4 && odp_packet_has_ipv4(pkt)) { + if (odp_packet_copy_to_mem(pkt, off, _ODP_IPV4HDR_LEN, &ipv4) == 0) { + head = add_data(head, &ipv4.src_addr, sizeof(ipv4.src_addr)); + head = add_data(head, &ipv4.dst_addr, sizeof(ipv4.dst_addr)); + } + } else if (hash->proto.ipv6 && odp_packet_has_ipv6(pkt)) { + if (odp_packet_copy_to_mem(pkt, off, _ODP_IPV6HDR_LEN, &ipv6) == 0) { + head = add_data(head, &ipv6.src_addr, sizeof(ipv6.src_addr)); + head = add_data(head, &ipv6.dst_addr, sizeof(ipv6.dst_addr)); + } + } + } + + return pkt_loop->loopqs[odp_hash_crc32c(data, head - data, 0) % pkt_loop->num_qs].queue; +} + +static int loopback_send(pktio_entry_t *pktio_entry, int index, const odp_packet_t pkt_tbl[], + int num) { pkt_loop_t *pkt_loop = pkt_priv(pktio_entry); - _odp_event_hdr_t *hdr_tbl[QUEUE_MULTI_MAX]; odp_queue_t queue; + stats_t *stats; int i; int ret; int nb_tx = 0; int tx_ts_idx = 0; uint8_t tx_ts_enabled = _odp_pktio_tx_ts_enabled(pktio_entry); - uint32_t bytes = 0; - uint32_t out_octets_tbl[num]; odp_pktout_config_opt_t *pktout_cfg = &pktio_entry->config.pktout; - odp_pktout_config_opt_t *pktout_capa = - &pktio_entry->capa.config.pktout; + odp_pktout_config_opt_t *pktout_capa = &pktio_entry->capa.config.pktout; + + if (pkt_loop->num_qs == 0) + return 0; + + stats = &pkt_loop->loopqs[index].stats; if (odp_unlikely(num > QUEUE_MULTI_MAX)) num = QUEUE_MULTI_MAX; @@ -411,44 +531,34 @@ static int loopback_send(pktio_entry_t *pktio_entry, int index ODP_UNUSED, } break; } - hdr_tbl[i] = packet_to_event_hdr(pkt_tbl[i]); - bytes += pkt_len; - /* Store cumulative byte counts to update 'stats.out_octets' - * correctly in case enq_multi() fails to enqueue all packets. - */ - out_octets_tbl[i] = bytes; - nb_tx++; if (tx_ts_enabled && tx_ts_idx == 0) { if (odp_unlikely(packet_hdr(pkt_tbl[i])->p.flags.ts_set)) tx_ts_idx = i + 1; } - } - for (i = 0; i < nb_tx; ++i) { packet_subtype_set(pkt_tbl[i], ODP_EVENT_PACKET_BASIC); loopback_fix_checksums(pkt_tbl[i], pktout_cfg, pktout_capa); - } + queue = get_dest_queue(pkt_loop, pkt_tbl[i], index); + ret = odp_queue_enq(queue, odp_packet_to_event(pkt_tbl[i])); - odp_ticketlock_lock(&pktio_entry->txl); + if (ret < 0) { + _ODP_DBG("queue enqueue failed %i to queue: %" PRIu64 "\n", ret, + odp_queue_to_u64(queue)); + break; + } - queue = pkt_priv(pktio_entry)->loopq; - ret = odp_queue_enq_multi(queue, (odp_event_t *)hdr_tbl, nb_tx); + nb_tx++; + odp_atomic_inc_u64(&stats->out_packets); + odp_atomic_add_u64(&stats->out_octets, pkt_len); + } - if (ret > 0) { - if (odp_unlikely(tx_ts_idx) && ret >= tx_ts_idx) + if (nb_tx > 0) { + if (odp_unlikely(tx_ts_idx) && nb_tx >= tx_ts_idx) _odp_pktio_tx_ts_set(pktio_entry); - - pktio_entry->stats.out_packets += ret; - pktio_entry->stats.out_octets += out_octets_tbl[ret - 1]; - } else { - _ODP_DBG("queue enqueue failed %i\n", ret); - ret = -1; } - odp_ticketlock_unlock(&pktio_entry->txl); - - return ret; + return nb_tx; } static uint32_t loopback_mtu_get(pktio_entry_t *pktio_entry) @@ -508,8 +618,8 @@ static int loopback_init_capability(pktio_entry_t *pktio_entry) memset(capa, 0, sizeof(odp_pktio_capability_t)); - capa->max_input_queues = 1; - capa->max_output_queues = 1; + capa->max_input_queues = ODP_PKTIN_MAX_QUEUES; + capa->max_output_queues = ODP_PKTOUT_MAX_QUEUES; capa->set_op.op.promisc_mode = 0; capa->set_op.op.maxlen = 1; @@ -566,6 +676,7 @@ static int loopback_init_capability(pktio_entry_t *pktio_entry) capa->stats.pktin_queue.counter.octets = 1; capa->stats.pktin_queue.counter.packets = 1; capa->stats.pktin_queue.counter.errors = 1; + capa->stats.pktin_queue.counter.discards = 1; capa->stats.pktout_queue.counter.octets = 1; capa->stats.pktout_queue.counter.packets = 1; return 0; @@ -582,37 +693,67 @@ static int loopback_promisc_mode_get(pktio_entry_t *pktio_entry ODP_UNUSED) return 1; } -static int loopback_stats(pktio_entry_t *pktio_entry, - odp_pktio_stats_t *stats) +static int loopback_stats(pktio_entry_t *pktio_entry, odp_pktio_stats_t *stats) { - memcpy(stats, &pktio_entry->stats, sizeof(odp_pktio_stats_t)); + pkt_loop_t *pkt_loop = pkt_priv(pktio_entry); + + memset(stats, 0, sizeof(odp_pktio_stats_t)); + + for (uint32_t i = 0; i < MAX_QUEUES; i++) { + stats_t *qs = &pkt_loop->loopqs[i].stats; + + stats->in_octets += odp_atomic_load_u64(&qs->in_octets); + stats->in_packets += odp_atomic_load_u64(&qs->in_packets); + stats->in_discards += odp_atomic_load_u64(&qs->in_discards); + stats->in_errors += odp_atomic_load_u64(&qs->in_errors); + stats->out_octets += odp_atomic_load_u64(&qs->out_octets); + stats->out_packets += odp_atomic_load_u64(&qs->out_packets); + } + return 0; } static int loopback_stats_reset(pktio_entry_t *pktio_entry) { - memset(&pktio_entry->stats, 0, sizeof(odp_pktio_stats_t)); + pkt_loop_t *pkt_loop = pkt_priv(pktio_entry); + + for (uint32_t i = 0; i < MAX_QUEUES; i++) { + stats_t *qs = &pkt_loop->loopqs[i].stats; + + odp_atomic_store_u64(&qs->in_octets, 0); + odp_atomic_store_u64(&qs->in_packets, 0); + odp_atomic_store_u64(&qs->in_discards, 0); + odp_atomic_store_u64(&qs->in_errors, 0); + odp_atomic_store_u64(&qs->out_octets, 0); + odp_atomic_store_u64(&qs->out_packets, 0); + } + return 0; } -static int loopback_pktin_stats(pktio_entry_t *pktio_entry, - uint32_t index ODP_UNUSED, +static int loopback_pktin_stats(pktio_entry_t *pktio_entry, uint32_t index, odp_pktin_queue_stats_t *pktin_stats) { + stats_t *qs = &pkt_priv(pktio_entry)->loopqs[index].stats; + memset(pktin_stats, 0, sizeof(odp_pktin_queue_stats_t)); - pktin_stats->octets = pktio_entry->stats.in_octets; - pktin_stats->packets = pktio_entry->stats.in_packets; - pktin_stats->errors = pktio_entry->stats.in_errors; + pktin_stats->octets = odp_atomic_load_u64(&qs->in_octets); + pktin_stats->packets = odp_atomic_load_u64(&qs->in_packets); + pktin_stats->discards = odp_atomic_load_u64(&qs->in_discards); + pktin_stats->errors = odp_atomic_load_u64(&qs->in_errors); + return 0; } -static int loopback_pktout_stats(pktio_entry_t *pktio_entry, - uint32_t index ODP_UNUSED, +static int loopback_pktout_stats(pktio_entry_t *pktio_entry, uint32_t index, odp_pktout_queue_stats_t *pktout_stats) { + stats_t *qs = &pkt_priv(pktio_entry)->loopqs[index].stats; + memset(pktout_stats, 0, sizeof(odp_pktout_queue_stats_t)); - pktout_stats->octets = pktio_entry->stats.out_octets; - pktout_stats->packets = pktio_entry->stats.out_packets; + pktout_stats->octets = odp_atomic_load_u64(&qs->out_octets); + pktout_stats->packets = odp_atomic_load_u64(&qs->out_packets); + return 0; } diff --git a/platform/linux-generic/pktio/stats/packet_io_stats.c b/platform/linux-generic/pktio/stats/packet_io_stats.c index ac61c0343..280aca250 100644 --- a/platform/linux-generic/pktio/stats/packet_io_stats.c +++ b/platform/linux-generic/pktio/stats/packet_io_stats.c @@ -5,7 +5,6 @@ * SPDX-License-Identifier: BSD-3-Clause */ -#include <odp/api/deprecated.h> #include <odp_packet_io_stats.h> #include <odp_ethtool_stats.h> #include <odp_sysfs_stats.h> @@ -77,10 +76,6 @@ int _odp_sock_stats_fd(pktio_entry_t *pktio_entry, pktio_entry->stats.in_discards; stats->in_errors = cur_stats.in_errors - pktio_entry->stats.in_errors; -#if ODP_DEPRECATED_API - stats->in_unknown_protos = cur_stats.in_unknown_protos - - pktio_entry->stats.in_unknown_protos; -#endif stats->out_octets = cur_stats.out_octets - pktio_entry->stats.out_octets; stats->out_packets = cur_stats.out_packets - diff --git a/platform/linux-generic/test/inline-timer.conf b/platform/linux-generic/test/inline-timer.conf index 261aa0141..44db4e337 100644 --- a/platform/linux-generic/test/inline-timer.conf +++ b/platform/linux-generic/test/inline-timer.conf @@ -1,6 +1,6 @@ # Mandatory fields odp_implementation = "linux-generic" -config_file_version = "0.1.22" +config_file_version = "0.1.25" timer: { # Enable inline timer implementation diff --git a/platform/linux-generic/test/packet_align.conf b/platform/linux-generic/test/packet_align.conf index 8d2d00e63..26491bd53 100644 --- a/platform/linux-generic/test/packet_align.conf +++ b/platform/linux-generic/test/packet_align.conf @@ -1,6 +1,6 @@ # Mandatory fields odp_implementation = "linux-generic" -config_file_version = "0.1.22" +config_file_version = "0.1.25" pool: { pkt: { diff --git a/platform/linux-generic/test/process-mode.conf b/platform/linux-generic/test/process-mode.conf index 1e0e7cc95..2277aabdf 100644 --- a/platform/linux-generic/test/process-mode.conf +++ b/platform/linux-generic/test/process-mode.conf @@ -1,6 +1,6 @@ # Mandatory fields odp_implementation = "linux-generic" -config_file_version = "0.1.22" +config_file_version = "0.1.25" # Shared memory options shm: { diff --git a/platform/linux-generic/test/sched-basic.conf b/platform/linux-generic/test/sched-basic.conf index e63ffa2f3..c9f7c79fd 100644 --- a/platform/linux-generic/test/sched-basic.conf +++ b/platform/linux-generic/test/sched-basic.conf @@ -1,9 +1,10 @@ # Mandatory fields odp_implementation = "linux-generic" -config_file_version = "0.1.22" +config_file_version = "0.1.25" # Test scheduler with an odd spread value and without dynamic load balance sched_basic: { prio_spread = 3 load_balance = 0 + order_stash_size = 0 } diff --git a/platform/linux-generic/test/stash-custom.conf b/platform/linux-generic/test/stash-custom.conf new file mode 100644 index 000000000..95af7a259 --- /dev/null +++ b/platform/linux-generic/test/stash-custom.conf @@ -0,0 +1,8 @@ +# Mandatory fields +odp_implementation = "linux-generic" +config_file_version = "0.1.25" + +# Test overflow safe stash variant +stash: { + strict_size = 0 +} diff --git a/test/Makefile.inc b/test/Makefile.inc index f5235f471..6d14f9c17 100644 --- a/test/Makefile.inc +++ b/test/Makefile.inc @@ -10,6 +10,7 @@ LIBODP = $(LIB)/libodphelper.la $(LIB)/lib$(ODP_LIB_NAME).la LIBCUNIT_COMMON = $(COMMON_DIR)/libcunit_common.la LIBCPUMASK_COMMON = $(COMMON_DIR)/libcpumask_common.la +LIBPACKET_COMMON = $(COMMON_DIR)/libpacket_common.la LIBTHRMASK_COMMON = $(COMMON_DIR)/libthrmask_common.la #in the following line, the libs using the symbols should come before diff --git a/test/common/Makefile.am b/test/common/Makefile.am index 694b442c2..72658df73 100644 --- a/test/common/Makefile.am +++ b/test/common/Makefile.am @@ -2,13 +2,19 @@ include $(top_srcdir)/test/Makefile.inc if cunit_support -noinst_LTLIBRARIES = libcunit_common.la libcpumask_common.la libthrmask_common.la +noinst_LTLIBRARIES = \ + libcunit_common.la \ + libcpumask_common.la \ + libpacket_common.la \ + libthrmask_common.la libcunit_common_la_SOURCES = odp_cunit_common.c odp_cunit_common.h libcunit_common_la_LIBADD = $(CUNIT_LIBS) libcpumask_common_la_SOURCES = mask_common.c mask_common.h +libpacket_common_la_SOURCES = packet_common.c packet_common.h + libthrmask_common_la_SOURCES = mask_common.c mask_common.h libthrmask_common_la_CFLAGS = $(AM_CFLAGS) -DTEST_THRMASK diff --git a/test/common/packet_common.c b/test/common/packet_common.c new file mode 100644 index 000000000..e0bca3147 --- /dev/null +++ b/test/common/packet_common.c @@ -0,0 +1,133 @@ +/* Copyright (c) 2023, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <packet_common.h> +#include <string.h> + +void test_packet_set_md(odp_packet_t pkt) +{ + const int binary_flag = 1; + const uint32_t offset = 1; + uint8_t *uarea = odp_packet_user_area(pkt); + uint32_t uarea_size = odp_packet_user_area_size(pkt); + + for (uint32_t n = 0; n < uarea_size; n++) + uarea[n] = n; + + /* + * Some of these flags cannot be set simultaneously, but we do not + * care here if some flags get cleared. + */ + odp_packet_has_l2_set(pkt, binary_flag); + odp_packet_has_l3_set(pkt, binary_flag); + odp_packet_has_l4_set(pkt, binary_flag); + odp_packet_has_eth_set(pkt, binary_flag); + odp_packet_has_eth_bcast_set(pkt, binary_flag); + odp_packet_has_eth_mcast_set(pkt, !binary_flag); + odp_packet_has_jumbo_set(pkt, binary_flag); + odp_packet_has_vlan_set(pkt, binary_flag); + odp_packet_has_vlan_qinq_set(pkt, binary_flag); + odp_packet_has_arp_set(pkt, binary_flag); + odp_packet_has_ipv4_set(pkt, binary_flag); + odp_packet_has_ipv6_set(pkt, !binary_flag); + odp_packet_has_ip_bcast_set(pkt, binary_flag); + odp_packet_has_ip_mcast_set(pkt, binary_flag); + odp_packet_has_ipfrag_set(pkt, binary_flag); + odp_packet_has_ipopt_set(pkt, binary_flag); + odp_packet_has_ipsec_set(pkt, binary_flag); + odp_packet_has_udp_set(pkt, binary_flag); + odp_packet_has_tcp_set(pkt, !binary_flag); + odp_packet_has_sctp_set(pkt, binary_flag); + odp_packet_has_icmp_set(pkt, binary_flag); + + odp_packet_user_ptr_set(pkt, &pkt); + odp_packet_user_flag_set(pkt, binary_flag); + (void)odp_packet_l2_offset_set(pkt, offset); + (void)odp_packet_l3_offset_set(pkt, offset); + (void)odp_packet_l4_offset_set(pkt, offset); + odp_packet_flow_hash_set(pkt, 0x12345678); + odp_packet_ts_set(pkt, odp_time_local_from_ns(ODP_TIME_SEC_IN_NS)); + odp_packet_color_set(pkt, ODP_PACKET_YELLOW); + odp_packet_drop_eligible_set(pkt, binary_flag); + odp_packet_shaper_len_adjust_set(pkt, -42); + (void)odp_packet_payload_offset_set(pkt, offset); +} + +void test_packet_get_md(odp_packet_t pkt, test_packet_md_t *md) +{ + uint8_t *uarea = odp_packet_user_area(pkt); + uint32_t uarea_size = odp_packet_user_area_size(pkt); + + memset(md, 0, sizeof(*md)); + + if (uarea) + md->user_area_chksum = odp_chksum_ones_comp16(uarea, uarea_size); + + md->has_error = !!odp_packet_has_error(pkt); + md->has_l2_error = !!odp_packet_has_l2_error(pkt); + md->has_l3_error = !!odp_packet_has_l3_error(pkt); + md->has_l4_error = !!odp_packet_has_l4_error(pkt); + md->has_l2 = !!odp_packet_has_l2(pkt); + md->has_l3 = !!odp_packet_has_l3(pkt); + md->has_l4 = !!odp_packet_has_l4(pkt); + md->has_eth = !!odp_packet_has_eth(pkt); + md->has_eth_bcast = !!odp_packet_has_eth_bcast(pkt); + md->has_eth_mcast = !!odp_packet_has_eth_mcast(pkt); + md->has_jumbo = !!odp_packet_has_jumbo(pkt); + md->has_vlan = !!odp_packet_has_vlan(pkt); + md->has_vlan_qinq = !!odp_packet_has_vlan_qinq(pkt); + md->has_arp = !!odp_packet_has_arp(pkt); + md->has_ipv4 = !!odp_packet_has_ipv4(pkt); + md->has_ipv6 = !!odp_packet_has_ipv6(pkt); + md->has_ip_bcast = !!odp_packet_has_ip_bcast(pkt); + md->has_ip_mcast = !!odp_packet_has_ip_mcast(pkt); + md->has_ipfrag = !!odp_packet_has_ipfrag(pkt); + md->has_ipopt = !!odp_packet_has_ipopt(pkt); + md->has_ipsec = !!odp_packet_has_ipsec(pkt); + md->has_udp = !!odp_packet_has_udp(pkt); + md->has_tcp = !!odp_packet_has_tcp(pkt); + md->has_sctp = !!odp_packet_has_sctp(pkt); + md->has_icmp = !!odp_packet_has_icmp(pkt); + md->has_flow_hash = !!odp_packet_has_flow_hash(pkt); + md->has_ts = !!odp_packet_has_ts(pkt); + + md->len = odp_packet_len(pkt); + md->packet_pool = odp_packet_pool(pkt); + md->packet_input = odp_packet_input(pkt); + md->user_ptr = odp_packet_user_ptr(pkt); + md->user_flag = odp_packet_user_flag(pkt); + md->l2_offset = odp_packet_l2_offset(pkt); + md->l3_offset = odp_packet_l3_offset(pkt); + md->l4_offset = odp_packet_l4_offset(pkt); + md->l2_type = odp_packet_l2_type(pkt); + md->l3_type = odp_packet_l3_type(pkt); + md->l4_type = odp_packet_l4_type(pkt); + md->l3_chksum_status = odp_packet_l3_chksum_status(pkt); + md->l4_chksum_status = odp_packet_l4_chksum_status(pkt); + md->flow_hash = md->has_flow_hash ? odp_packet_flow_hash(pkt) : 0; + md->ts = md->has_ts ? odp_packet_ts(pkt) : odp_time_global_from_ns(0); + md->color = odp_packet_color(pkt); + md->drop_eligible = !!odp_packet_drop_eligible(pkt); + md->shaper_len_adjust = odp_packet_shaper_len_adjust(pkt); + md->cls_mark = odp_packet_cls_mark(pkt); + md->has_lso_request = !!odp_packet_has_lso_request(pkt); + md->payload_offset = odp_packet_payload_offset(pkt); + md->aging_tmo = odp_packet_aging_tmo(pkt); + md->has_tx_compl_request = !!odp_packet_has_tx_compl_request(pkt); + md->proto_stats = odp_packet_proto_stats(pkt); +} + +int test_packet_is_md_equal(const test_packet_md_t *md_1, + const test_packet_md_t *md_2) +{ + /* + * With certain assumptions this should typically work. If it does + * not, we would get false negatives which we should spot as test + * failures. + */ + + return (!memcmp(md_1, md_2, sizeof(*md_1))); +} diff --git a/test/common/packet_common.h b/test/common/packet_common.h new file mode 100644 index 000000000..c7cd5e27f --- /dev/null +++ b/test/common/packet_common.h @@ -0,0 +1,67 @@ +/* Copyright (c) 2023, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <odp_api.h> + +typedef struct test_packet_md_t { + int has_error; + int has_l2_error; + int has_l3_error; + int has_l4_error; + int has_l2; + int has_l3; + int has_l4; + int has_eth; + int has_eth_bcast; + int has_eth_mcast; + int has_jumbo; + int has_vlan; + int has_vlan_qinq; + int has_arp; + int has_ipv4; + int has_ipv6; + int has_ip_bcast; + int has_ip_mcast; + int has_ipfrag; + int has_ipopt; + int has_ipsec; + int has_udp; + int has_tcp; + int has_sctp; + int has_icmp; + int has_flow_hash; + int has_ts; + uint32_t len; + odp_pool_t packet_pool; + odp_pktio_t packet_input; + void *user_ptr; + uint16_t user_area_chksum; + int user_flag; + uint32_t l2_offset; + uint32_t l3_offset; + uint32_t l4_offset; + odp_proto_l2_type_t l2_type; + odp_proto_l3_type_t l3_type; + odp_proto_l4_type_t l4_type; + odp_packet_chksum_status_t l3_chksum_status; + odp_packet_chksum_status_t l4_chksum_status; + uint32_t flow_hash; + odp_time_t ts; + odp_packet_color_t color; + odp_bool_t drop_eligible; + int8_t shaper_len_adjust; + uint64_t cls_mark; + int has_lso_request; + uint32_t payload_offset; + uint64_t aging_tmo; + int has_tx_compl_request; + odp_proto_stats_t proto_stats; +} test_packet_md_t; + +void test_packet_set_md(odp_packet_t pkt); +void test_packet_get_md(odp_packet_t pkt, test_packet_md_t *md); +int test_packet_is_md_equal(const test_packet_md_t *md_1, + const test_packet_md_t *md_2); diff --git a/test/performance/.gitignore b/test/performance/.gitignore index ec1915bba..b86699c91 100644 --- a/test/performance/.gitignore +++ b/test/performance/.gitignore @@ -3,6 +3,7 @@ odp_atomic odp_atomic_perf odp_bench_buffer +odp_bench_misc odp_bench_packet odp_cpu_bench odp_crc diff --git a/test/performance/Makefile.am b/test/performance/Makefile.am index 336a46d22..beeb652cc 100644 --- a/test/performance/Makefile.am +++ b/test/performance/Makefile.am @@ -4,6 +4,7 @@ TESTS_ENVIRONMENT += TEST_DIR=${builddir} EXECUTABLES = odp_atomic_perf \ odp_bench_buffer \ + odp_bench_misc \ odp_bench_packet \ odp_crc \ odp_lock_perf \ @@ -54,6 +55,7 @@ bin_PROGRAMS = $(EXECUTABLES) $(COMPILE_ONLY) odp_atomic_perf_SOURCES = odp_atomic_perf.c odp_bench_buffer_SOURCES = odp_bench_buffer.c +odp_bench_misc_SOURCES = odp_bench_misc.c odp_bench_packet_SOURCES = odp_bench_packet.c odp_cpu_bench_SOURCES = odp_cpu_bench.c odp_crc_SOURCES = odp_crc.c diff --git a/test/performance/odp_bench_buffer.c b/test/performance/odp_bench_buffer.c index e7029f57d..e0e7c85cd 100644 --- a/test/performance/odp_bench_buffer.c +++ b/test/performance/odp_bench_buffer.c @@ -199,7 +199,7 @@ static int run_benchmarks(void *arg) /* Skip unsupported tests */ if (args->bench[j].cond != NULL && !args->bench[j].cond()) { j++; - k = 0; + k = 1; if (i > 0) printf("[%02d] odp_%-26s: n/a\n", j, desc); continue; @@ -236,7 +236,7 @@ static int run_benchmarks(void *arg) printf("[%02d] odp_%-26s: %8.1f\n", j + 1, desc, cycles); j++; - k = 0; + k = 1; tot_cycles = 0; } } diff --git a/test/performance/odp_bench_misc.c b/test/performance/odp_bench_misc.c new file mode 100644 index 000000000..8b9e27bf2 --- /dev/null +++ b/test/performance/odp_bench_misc.c @@ -0,0 +1,1069 @@ +/* Copyright (c) 2022, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef _GNU_SOURCE +#define _GNU_SOURCE /* Needed for sigaction */ +#endif + +#include <odp_api.h> +#include <odp/helper/odph_api.h> + +#include <getopt.h> +#include <inttypes.h> +#include <signal.h> +#include <stdlib.h> +#include <unistd.h> + +/* Number of API function calls per test case */ +#define REPEAT_COUNT 1000 + +/* Default number of rounds per test case */ +#define ROUNDS 1000u + +#define BENCH_INFO(run, init, max, name) \ + {#run, run, init, max, name} + +typedef struct { + /* Measure time vs CPU cycles */ + int time; + + /* Benchmark index to run indefinitely */ + int bench_idx; + + /* Rounds per test case */ + uint32_t rounds; + +} appl_args_t; + +/* Initialize benchmark resources */ +typedef void (*bench_init_fn_t)(void); + +/* Run benchmark, returns >0 on success */ +typedef int (*bench_run_fn_t)(void); + +/* Benchmark data */ +typedef struct { + /* Default test name */ + const char *name; + + /* Test function to run */ + bench_run_fn_t run; + + /* Test init function */ + bench_init_fn_t init; + + /* Test specific limit for rounds (tuning for slow implementation) */ + uint32_t max_rounds; + + /* Override default test name */ + const char *desc; + +} bench_info_t; + +/* Global data */ +typedef struct { + appl_args_t appl; + + /* Benchmark functions */ + bench_info_t *bench; + + /* Number of benchmark functions */ + int num_bench; + + /* Break worker loop if set to 1 */ + odp_atomic_u32_t exit_thread; + + /* Test case input / output data */ + odp_time_t t1[REPEAT_COUNT]; + odp_time_t t2[REPEAT_COUNT]; + odp_time_t t3[REPEAT_COUNT]; + uint64_t a1[REPEAT_COUNT]; + uint64_t a2[REPEAT_COUNT]; + uint32_t b1[REPEAT_COUNT]; + uint32_t b2[REPEAT_COUNT]; + uint16_t c1[REPEAT_COUNT]; + uint16_t c2[REPEAT_COUNT]; + + /* Dummy result */ + uint64_t dummy; + + /* Benchmark run failed */ + int bench_failed; + + /* CPU mask as string */ + char cpumask_str[ODP_CPUMASK_STR_SIZE]; + +} gbl_args_t; + +static gbl_args_t *gbl_args; + +static void sig_handler(int signo ODP_UNUSED) +{ + if (gbl_args == NULL) + return; + odp_atomic_store_u32(&gbl_args->exit_thread, 1); +} + +static int setup_sig_handler(void) +{ + struct sigaction action; + + memset(&action, 0, sizeof(action)); + action.sa_handler = sig_handler; + + /* No additional signals blocked. By default, the signal which triggered + * the handler is blocked. */ + if (sigemptyset(&action.sa_mask)) + return -1; + + if (sigaction(SIGINT, &action, NULL)) + return -1; + + return 0; +} + +/* Run given benchmark indefinitely */ +static void run_indef(gbl_args_t *args, int idx) +{ + const char *desc; + const bench_info_t *bench = &args->bench[idx]; + + desc = bench->desc != NULL ? bench->desc : bench->name; + + printf("Running odp_%s test indefinitely\n", desc); + + while (!odp_atomic_load_u32(&gbl_args->exit_thread)) { + int ret; + + if (bench->init != NULL) + bench->init(); + + ret = bench->run(); + + if (!ret) + ODPH_ABORT("Benchmark %s failed\n", desc); + } +} + +static int run_benchmarks(void *arg) +{ + int i, j; + uint64_t c1, c2; + odp_time_t t1, t2; + gbl_args_t *args = arg; + const int meas_time = args->appl.time; + + printf("\nAverage %s per function call\n", meas_time ? "time (nsec)" : "CPU cycles"); + printf("-------------------------------------------------\n"); + + /* Run each test twice. Results from the first warm-up round are ignored. */ + for (i = 0; i < 2; i++) { + uint64_t total = 0; + uint32_t round = 1; + + for (j = 0; j < gbl_args->num_bench; round++) { + int ret; + const char *desc; + const bench_info_t *bench = &args->bench[j]; + uint32_t max_rounds = args->appl.rounds; + + if (bench->max_rounds && max_rounds > bench->max_rounds) + max_rounds = bench->max_rounds; + + /* Run selected test indefinitely */ + if (args->appl.bench_idx) { + if ((j + 1) != args->appl.bench_idx) { + j++; + continue; + } + + run_indef(args, j); + return 0; + } + + desc = bench->desc != NULL ? bench->desc : bench->name; + + if (bench->init != NULL) + bench->init(); + + if (meas_time) + t1 = odp_time_local(); + else + c1 = odp_cpu_cycles(); + + ret = bench->run(); + + if (meas_time) + t2 = odp_time_local(); + else + c2 = odp_cpu_cycles(); + + if (!ret) { + ODPH_ERR("Benchmark odp_%s failed\n", desc); + args->bench_failed = -1; + return -1; + } + + if (meas_time) + total += odp_time_diff_ns(t2, t1); + else + total += odp_cpu_cycles_diff(c2, c1); + + if (round >= max_rounds) { + double result; + + /* Each benchmark runs internally REPEAT_COUNT times. */ + result = ((double)total) / (max_rounds * REPEAT_COUNT); + + /* No print from warm-up round */ + if (i > 0) + printf("[%02d] odp_%-26s: %12.2f\n", j + 1, desc, result); + + j++; + total = 0; + round = 1; + } + } + } + + /* Print dummy result to prevent compiler to optimize it away*/ + printf("\n(dummy result: 0x%" PRIx64 ")\n", args->dummy); + + printf("\n"); + + return 0; +} + +static void init_time_global(void) +{ + int i; + odp_time_t *t1 = gbl_args->t1; + odp_time_t *t2 = gbl_args->t2; + uint64_t *a1 = gbl_args->a1; + + for (i = 0; i < REPEAT_COUNT; i++) + t1[i] = odp_time_global(); + + for (i = 0; i < REPEAT_COUNT; i++) + t2[i] = odp_time_global(); + + for (i = 0; i < REPEAT_COUNT; i++) + a1[i] = odp_time_global_ns(); +} + +static void init_time_local(void) +{ + int i; + odp_time_t *t1 = gbl_args->t1; + odp_time_t *t2 = gbl_args->t2; + uint64_t *a1 = gbl_args->a1; + + for (i = 0; i < REPEAT_COUNT; i++) + t1[i] = odp_time_local(); + + for (i = 0; i < REPEAT_COUNT; i++) + t2[i] = odp_time_local(); + + for (i = 0; i < REPEAT_COUNT; i++) + a1[i] = odp_time_local_ns(); +} + +static void init_cpu_cycles(void) +{ + int i; + uint64_t *a1 = gbl_args->a1; + uint64_t *a2 = gbl_args->a2; + + for (i = 0; i < REPEAT_COUNT; i++) + a1[i] = odp_cpu_cycles(); + + for (i = 0; i < REPEAT_COUNT; i++) + a2[i] = odp_cpu_cycles(); +} + +static int time_local(void) +{ + int i; + odp_time_t *t1 = gbl_args->t1; + + for (i = 0; i < REPEAT_COUNT; i++) + t1[i] = odp_time_local(); + + return i; +} + +static int time_local_strict(void) +{ + int i; + odp_time_t *t1 = gbl_args->t1; + + for (i = 0; i < REPEAT_COUNT; i++) + t1[i] = odp_time_local_strict(); + + return i; +} + +static int time_local_ns(void) +{ + int i; + uint64_t *a1 = gbl_args->a1; + + for (i = 0; i < REPEAT_COUNT; i++) + a1[i] = odp_time_local_ns(); + + return i; +} + +static int time_local_strict_ns(void) +{ + int i; + uint64_t *a1 = gbl_args->a1; + + for (i = 0; i < REPEAT_COUNT; i++) + a1[i] = odp_time_local_strict_ns(); + + return i; +} + +static int time_global(void) +{ + int i; + odp_time_t *t1 = gbl_args->t1; + + for (i = 0; i < REPEAT_COUNT; i++) + t1[i] = odp_time_global(); + + return i; +} + +static int time_global_strict(void) +{ + int i; + odp_time_t *t1 = gbl_args->t1; + + for (i = 0; i < REPEAT_COUNT; i++) + t1[i] = odp_time_global_strict(); + + return i; +} + +static int time_global_ns(void) +{ + int i; + uint64_t *a1 = gbl_args->a1; + + for (i = 0; i < REPEAT_COUNT; i++) + a1[i] = odp_time_global_ns(); + + return i; +} + +static int time_global_strict_ns(void) +{ + int i; + uint64_t *a1 = gbl_args->a1; + + for (i = 0; i < REPEAT_COUNT; i++) + a1[i] = odp_time_global_strict_ns(); + + return i; +} + +static int time_diff(void) +{ + int i; + odp_time_t *t1 = gbl_args->t1; + odp_time_t *t2 = gbl_args->t2; + odp_time_t *t3 = gbl_args->t3; + + for (i = 0; i < REPEAT_COUNT; i++) + t3[i] = odp_time_diff(t2[i], t1[i]); + + return i; +} + +static int time_diff_ns(void) +{ + int i; + odp_time_t *t1 = gbl_args->t1; + odp_time_t *t2 = gbl_args->t2; + uint64_t res = 0; + + for (i = 0; i < REPEAT_COUNT; i++) + res += odp_time_diff_ns(t2[i], t1[i]); + + gbl_args->dummy += res; + + return i; +} + +static int time_sum(void) +{ + int i; + odp_time_t *t1 = gbl_args->t1; + odp_time_t *t2 = gbl_args->t2; + odp_time_t *t3 = gbl_args->t3; + + for (i = 0; i < REPEAT_COUNT; i++) + t3[i] = odp_time_sum(t1[i], t2[i]); + + return i; +} + +static int time_to_ns(void) +{ + int i; + odp_time_t *t1 = gbl_args->t1; + uint64_t res = 0; + + for (i = 0; i < REPEAT_COUNT; i++) + res += odp_time_to_ns(t1[i]); + + gbl_args->dummy += res; + + return i; +} + +static int time_local_from_ns(void) +{ + int i; + odp_time_t *t1 = gbl_args->t1; + uint64_t *a1 = gbl_args->a1; + + for (i = 0; i < REPEAT_COUNT; i++) + t1[i] = odp_time_local_from_ns(a1[i]); + + return i; +} + +static int time_global_from_ns(void) +{ + int i; + odp_time_t *t1 = gbl_args->t1; + uint64_t *a1 = gbl_args->a1; + + for (i = 0; i < REPEAT_COUNT; i++) + t1[i] = odp_time_global_from_ns(a1[i]); + + return i; +} + +static int time_cmp(void) +{ + int i; + odp_time_t *t1 = gbl_args->t1; + odp_time_t *t2 = gbl_args->t2; + int res = 0; + + for (i = 0; i < REPEAT_COUNT; i++) + res += odp_time_cmp(t1[i], t2[i]); + + gbl_args->dummy += res; + + return i; +} + +static int time_local_res(void) +{ + int i; + uint64_t *a1 = gbl_args->a1; + + for (i = 0; i < REPEAT_COUNT; i++) + a1[i] = odp_time_local_res(); + + return i; +} + +static int time_global_res(void) +{ + int i; + uint64_t *a1 = gbl_args->a1; + + for (i = 0; i < REPEAT_COUNT; i++) + a1[i] = odp_time_global_res(); + + return i; +} + +static int cpu_id(void) +{ + int i; + uint64_t *a1 = gbl_args->a1; + + for (i = 0; i < REPEAT_COUNT; i++) + a1[i] = odp_cpu_id(); + + return i; +} + +static int cpu_count(void) +{ + int i; + uint64_t *a1 = gbl_args->a1; + + for (i = 0; i < REPEAT_COUNT; i++) + a1[i] = odp_cpu_count(); + + return i; +} + +static int cpu_hz(void) +{ + int i; + uint64_t *a1 = gbl_args->a1; + + for (i = 0; i < REPEAT_COUNT; i++) + a1[i] = odp_cpu_hz(); + + return i; +} + +static int cpu_hz_max(void) +{ + int i; + uint64_t *a1 = gbl_args->a1; + + for (i = 0; i < REPEAT_COUNT; i++) + a1[i] = odp_cpu_hz_max(); + + return i; +} + +static int cpu_cycles(void) +{ + int i; + uint64_t *a1 = gbl_args->a1; + + for (i = 0; i < REPEAT_COUNT; i++) + a1[i] = odp_cpu_cycles(); + + return i; +} + +static int cpu_cycles_diff(void) +{ + int i; + uint64_t *a1 = gbl_args->a1; + uint64_t *a2 = gbl_args->a2; + uint64_t res = 0; + + for (i = 0; i < REPEAT_COUNT; i++) + res += odp_cpu_cycles_diff(a2[i], a1[i]); + + gbl_args->dummy += res; + + return i; +} + +static int cpu_cycles_max(void) +{ + int i; + uint64_t *a1 = gbl_args->a1; + + for (i = 0; i < REPEAT_COUNT; i++) + a1[i] = odp_cpu_cycles_max(); + + return i; +} + +static int cpu_pause(void) +{ + int i; + + for (i = 0; i < REPEAT_COUNT; i++) + odp_cpu_pause(); + + return i; +} + +static int thread_id(void) +{ + int i; + uint64_t *a1 = gbl_args->a1; + + for (i = 0; i < REPEAT_COUNT; i++) + a1[i] = odp_thread_id(); + + return i; +} + +static int thread_count(void) +{ + int i; + uint64_t *a1 = gbl_args->a1; + + for (i = 0; i < REPEAT_COUNT; i++) + a1[i] = odp_thread_count(); + + return i; +} + +static int thread_count_max(void) +{ + int i; + uint64_t *a1 = gbl_args->a1; + + for (i = 0; i < REPEAT_COUNT; i++) + a1[i] = odp_thread_count_max(); + + return i; +} + +static int thread_type(void) +{ + int i; + uint64_t *a1 = gbl_args->a1; + + for (i = 0; i < REPEAT_COUNT; i++) + a1[i] = (int)odp_thread_type(); + + return i; +} + +static int be_to_cpu_64(void) +{ + int i; + uint64_t *a1 = gbl_args->a1; + uint64_t *a2 = gbl_args->a2; + + for (i = 0; i < REPEAT_COUNT; i++) + a2[i] = odp_be_to_cpu_64(a1[i]); + + return i; +} + +static int be_to_cpu_32(void) +{ + int i; + uint32_t *b1 = gbl_args->b1; + uint32_t *b2 = gbl_args->b2; + + for (i = 0; i < REPEAT_COUNT; i++) + b2[i] = odp_be_to_cpu_32(b1[i]); + + return i; +} + +static int be_to_cpu_16(void) +{ + int i; + uint16_t *c1 = gbl_args->c1; + uint16_t *c2 = gbl_args->c2; + + for (i = 0; i < REPEAT_COUNT; i++) + c2[i] = odp_be_to_cpu_16(c1[i]); + + return i; +} + +static int cpu_to_be_64(void) +{ + int i; + uint64_t *a1 = gbl_args->a1; + uint64_t *a2 = gbl_args->a2; + + for (i = 0; i < REPEAT_COUNT; i++) + a2[i] = odp_cpu_to_be_64(a1[i]); + + return i; +} + +static int cpu_to_be_32(void) +{ + int i; + uint32_t *b1 = gbl_args->b1; + uint32_t *b2 = gbl_args->b2; + + for (i = 0; i < REPEAT_COUNT; i++) + b2[i] = odp_cpu_to_be_32(b1[i]); + + return i; +} + +static int cpu_to_be_16(void) +{ + int i; + uint16_t *c1 = gbl_args->c1; + uint16_t *c2 = gbl_args->c2; + + for (i = 0; i < REPEAT_COUNT; i++) + c2[i] = odp_cpu_to_be_16(c1[i]); + + return i; +} + +static int le_to_cpu_64(void) +{ + int i; + uint64_t *a1 = gbl_args->a1; + uint64_t *a2 = gbl_args->a2; + + for (i = 0; i < REPEAT_COUNT; i++) + a2[i] = odp_le_to_cpu_64(a1[i]); + + return i; +} + +static int le_to_cpu_32(void) +{ + int i; + uint32_t *b1 = gbl_args->b1; + uint32_t *b2 = gbl_args->b2; + + for (i = 0; i < REPEAT_COUNT; i++) + b2[i] = odp_le_to_cpu_32(b1[i]); + + return i; +} + +static int le_to_cpu_16(void) +{ + int i; + uint16_t *c1 = gbl_args->c1; + uint16_t *c2 = gbl_args->c2; + + for (i = 0; i < REPEAT_COUNT; i++) + c2[i] = odp_le_to_cpu_16(c1[i]); + + return i; +} + +static int cpu_to_le_64(void) +{ + int i; + uint64_t *a1 = gbl_args->a1; + uint64_t *a2 = gbl_args->a2; + + for (i = 0; i < REPEAT_COUNT; i++) + a2[i] = odp_cpu_to_le_64(a1[i]); + + return i; +} + +static int cpu_to_le_32(void) +{ + int i; + uint32_t *b1 = gbl_args->b1; + uint32_t *b2 = gbl_args->b2; + + for (i = 0; i < REPEAT_COUNT; i++) + b2[i] = odp_cpu_to_le_32(b1[i]); + + return i; +} + +static int cpu_to_le_16(void) +{ + int i; + uint16_t *c1 = gbl_args->c1; + uint16_t *c2 = gbl_args->c2; + + for (i = 0; i < REPEAT_COUNT; i++) + c2[i] = odp_cpu_to_le_16(c1[i]); + + return i; +} + +static int mb_release(void) +{ + int i; + + for (i = 0; i < REPEAT_COUNT; i++) + odp_mb_release(); + + return i; +} + +static int mb_acquire(void) +{ + int i; + + for (i = 0; i < REPEAT_COUNT; i++) + odp_mb_acquire(); + + return i; +} + +static int mb_full(void) +{ + int i; + + for (i = 0; i < REPEAT_COUNT; i++) + odp_mb_full(); + + return i; +} + +bench_info_t test_suite[] = { + BENCH_INFO(time_local, NULL, 0, NULL), + BENCH_INFO(time_local_strict, NULL, 0, NULL), + BENCH_INFO(time_local_ns, NULL, 0, NULL), + BENCH_INFO(time_local_strict_ns, NULL, 0, NULL), + BENCH_INFO(time_global, NULL, 0, NULL), + BENCH_INFO(time_global_strict, NULL, 0, NULL), + BENCH_INFO(time_global_ns, NULL, 0, NULL), + BENCH_INFO(time_global_strict_ns, NULL, 0, NULL), + BENCH_INFO(time_diff, init_time_global, 0, "time_diff (global)"), + BENCH_INFO(time_diff, init_time_local, 0, "time_diff (local)"), + BENCH_INFO(time_diff_ns, init_time_global, 0, NULL), + BENCH_INFO(time_sum, init_time_global, 0, NULL), + BENCH_INFO(time_to_ns, init_time_global, 0, NULL), + BENCH_INFO(time_local_from_ns, init_time_global, 0, NULL), + BENCH_INFO(time_global_from_ns, init_time_global, 0, NULL), + BENCH_INFO(time_cmp, init_time_global, 0, NULL), + BENCH_INFO(time_local_res, NULL, 0, NULL), + BENCH_INFO(time_global_res, NULL, 0, NULL), + BENCH_INFO(cpu_id, NULL, 0, NULL), + BENCH_INFO(cpu_count, NULL, 0, NULL), + BENCH_INFO(cpu_hz, NULL, 1, NULL), + BENCH_INFO(cpu_hz_max, NULL, 0, NULL), + BENCH_INFO(cpu_cycles, NULL, 0, NULL), + BENCH_INFO(cpu_cycles_diff, init_cpu_cycles, 0, NULL), + BENCH_INFO(cpu_cycles_max, NULL, 0, NULL), + BENCH_INFO(cpu_pause, NULL, 0, NULL), + BENCH_INFO(thread_id, NULL, 0, NULL), + BENCH_INFO(thread_count, NULL, 0, NULL), + BENCH_INFO(thread_count_max, NULL, 0, NULL), + BENCH_INFO(thread_type, NULL, 0, NULL), + BENCH_INFO(be_to_cpu_64, NULL, 0, NULL), + BENCH_INFO(be_to_cpu_32, NULL, 0, NULL), + BENCH_INFO(be_to_cpu_16, NULL, 0, NULL), + BENCH_INFO(cpu_to_be_64, NULL, 0, NULL), + BENCH_INFO(cpu_to_be_32, NULL, 0, NULL), + BENCH_INFO(cpu_to_be_16, NULL, 0, NULL), + BENCH_INFO(le_to_cpu_64, NULL, 0, NULL), + BENCH_INFO(le_to_cpu_32, NULL, 0, NULL), + BENCH_INFO(le_to_cpu_16, NULL, 0, NULL), + BENCH_INFO(cpu_to_le_64, NULL, 0, NULL), + BENCH_INFO(cpu_to_le_32, NULL, 0, NULL), + BENCH_INFO(cpu_to_le_16, NULL, 0, NULL), + BENCH_INFO(mb_release, NULL, 0, NULL), + BENCH_INFO(mb_acquire, NULL, 0, NULL), + BENCH_INFO(mb_full, NULL, 0, NULL), +}; + +/* Print usage information */ +static void usage(void) +{ + printf("\n" + "ODP miscellaneous API micro benchmarks\n" + "\n" + "Options:\n" + " -t, --time <opt> Time measurement. 0: measure CPU cycles (default), 1: measure time\n" + " -i, --index <idx> Benchmark index to run indefinitely.\n" + " -r, --rounds <num> Run each test case 'num' times (default %u).\n" + " -h, --help Display help and exit.\n\n" + "\n", ROUNDS); +} + +/* Parse command line arguments */ +static int parse_args(int argc, char *argv[]) +{ + int opt; + int long_index; + appl_args_t *appl_args = &gbl_args->appl; + static const struct option longopts[] = { + {"time", required_argument, NULL, 't'}, + {"index", required_argument, NULL, 'i'}, + {"rounds", required_argument, NULL, 'r'}, + {"help", no_argument, NULL, 'h'}, + {NULL, 0, NULL, 0} + }; + + static const char *shortopts = "t:i:r:h"; + + appl_args->time = 0; /* Measure CPU cycles */ + appl_args->bench_idx = 0; /* Run all benchmarks */ + appl_args->rounds = ROUNDS; + + while (1) { + opt = getopt_long(argc, argv, shortopts, longopts, &long_index); + + if (opt == -1) + break; /* No more options */ + + switch (opt) { + case 't': + appl_args->time = atoi(optarg); + break; + case 'i': + appl_args->bench_idx = atoi(optarg); + break; + case 'r': + appl_args->rounds = atoi(optarg); + break; + case 'h': + usage(); + return 1; + default: + ODPH_ERR("Bad option. Use -h for help.\n"); + return -1; + } + } + + if (appl_args->rounds < 1) { + ODPH_ERR("Invalid test cycle repeat count: %u\n", appl_args->rounds); + return -1; + } + + if (appl_args->bench_idx < 0 || appl_args->bench_idx > gbl_args->num_bench) { + ODPH_ERR("Bad bench index %i\n", appl_args->bench_idx); + return -1; + } + + optind = 1; /* Reset 'extern optind' from the getopt lib */ + + return 0; +} + +/* Print system and application info */ +static void print_info(void) +{ + odp_sys_info_print(); + + printf("\n" + "odp_bench_misc options\n" + "----------------------\n"); + + printf("CPU mask: %s\n", gbl_args->cpumask_str); + printf("Measurement unit: %s\n", gbl_args->appl.time ? "nsec" : "CPU cycles"); + printf("Test rounds: %u\n", gbl_args->appl.rounds); + printf("\n"); +} + +int main(int argc, char *argv[]) +{ + odph_helper_options_t helper_options; + odph_thread_t worker_thread; + odph_thread_common_param_t thr_common; + odph_thread_param_t thr_param; + int cpu, i; + odp_shm_t shm; + odp_cpumask_t cpumask, default_mask; + odp_instance_t instance; + odp_init_t init_param; + int ret = 0; + + /* Let helper collect its own arguments (e.g. --odph_proc) */ + argc = odph_parse_options(argc, argv); + if (odph_options(&helper_options)) { + ODPH_ERR("Reading ODP helper options failed\n"); + exit(EXIT_FAILURE); + } + + odp_init_param_init(&init_param); + init_param.mem_model = helper_options.mem_model; + + /* Init ODP before calling anything else */ + if (odp_init_global(&instance, &init_param, NULL)) { + ODPH_ERR("Global init failed\n"); + exit(EXIT_FAILURE); + } + + /* Init this thread */ + if (odp_init_local(instance, ODP_THREAD_CONTROL)) { + ODPH_ERR("Local init failed\n"); + exit(EXIT_FAILURE); + } + + if (setup_sig_handler()) { + ODPH_ERR("Signal handler setup failed\n"); + exit(EXIT_FAILURE); + } + + /* Reserve memory for args from shared mem */ + shm = odp_shm_reserve("shm_args", sizeof(gbl_args_t), ODP_CACHE_LINE_SIZE, 0); + if (shm == ODP_SHM_INVALID) { + ODPH_ERR("Shared mem reserve failed\n"); + exit(EXIT_FAILURE); + } + + gbl_args = odp_shm_addr(shm); + if (gbl_args == NULL) { + ODPH_ERR("Shared mem alloc failed\n"); + exit(EXIT_FAILURE); + } + + memset(gbl_args, 0, sizeof(gbl_args_t)); + odp_atomic_init_u32(&gbl_args->exit_thread, 0); + + gbl_args->bench = test_suite; + gbl_args->num_bench = sizeof(test_suite) / sizeof(test_suite[0]); + + for (i = 0; i < REPEAT_COUNT; i++) { + gbl_args->t1[i] = ODP_TIME_NULL; + gbl_args->t2[i] = ODP_TIME_NULL; + gbl_args->t3[i] = ODP_TIME_NULL; + gbl_args->a1[i] = i; + gbl_args->a2[i] = i; + gbl_args->b1[i] = i; + gbl_args->b2[i] = i; + gbl_args->c1[i] = i; + gbl_args->c2[i] = i; + } + + /* Parse and store the application arguments */ + ret = parse_args(argc, argv); + if (ret) + goto exit; + + /* Get default worker cpumask */ + if (odp_cpumask_default_worker(&default_mask, 1) != 1) { + ODPH_ERR("Unable to allocate worker thread\n"); + ret = -1; + goto exit; + } + + (void)odp_cpumask_to_str(&default_mask, gbl_args->cpumask_str, + sizeof(gbl_args->cpumask_str)); + + print_info(); + + memset(&worker_thread, 0, sizeof(odph_thread_t)); + + /* Create worker thread */ + cpu = odp_cpumask_first(&default_mask); + + odp_cpumask_zero(&cpumask); + odp_cpumask_set(&cpumask, cpu); + + odph_thread_common_param_init(&thr_common); + thr_common.instance = instance; + thr_common.cpumask = &cpumask; + thr_common.share_param = 1; + + odph_thread_param_init(&thr_param); + thr_param.start = run_benchmarks; + thr_param.arg = gbl_args; + thr_param.thr_type = ODP_THREAD_WORKER; + + odph_thread_create(&worker_thread, &thr_common, &thr_param, 1); + + odph_thread_join(&worker_thread, 1); + + ret = gbl_args->bench_failed; + +exit: + if (odp_shm_free(shm)) { + ODPH_ERR("Shared mem free failed\n"); + exit(EXIT_FAILURE); + } + + if (odp_term_local()) { + ODPH_ERR("Local term failed\n"); + exit(EXIT_FAILURE); + } + + if (odp_term_global(instance)) { + ODPH_ERR("Global term failed\n"); + exit(EXIT_FAILURE); + } + + if (ret < 0) + return EXIT_FAILURE; + + return EXIT_SUCCESS; +} diff --git a/test/performance/odp_bench_packet.c b/test/performance/odp_bench_packet.c index b44e92b72..aaf725b57 100644 --- a/test/performance/odp_bench_packet.c +++ b/test/performance/odp_bench_packet.c @@ -275,7 +275,7 @@ static int run_benchmarks(void *arg) printf("odp_%-26s: %8.1f\n", desc, cycles); j++; - k = 0; + k = 1; tot_cycles = 0; } } diff --git a/test/performance/odp_crypto.c b/test/performance/odp_crypto.c index b381fde31..2f79ae644 100644 --- a/test/performance/odp_crypto.c +++ b/test/performance/odp_crypto.c @@ -810,6 +810,8 @@ run_measure_one(crypto_args_t *cargs, } packets_sent += rc; } else { + odp_crypto_packet_result_t result; + rc = odp_crypto_op(&pkt, &out_pkt, ¶ms, 1); if (rc <= 0) { @@ -820,6 +822,12 @@ run_measure_one(crypto_args_t *cargs, } packets_sent += rc; packets_received++; + if (odp_unlikely(odp_crypto_result(&result, out_pkt) != 0) || + odp_unlikely(!result.ok)) { + ODPH_ERR("Crypto operation failed\n"); + odp_packet_free(out_pkt); + return -1; + } if (cargs->debug_packets) { mem = odp_packet_data(out_pkt); print_mem("Immediately encrypted " @@ -849,8 +857,12 @@ run_measure_one(crypto_args_t *cargs, while (ev != ODP_EVENT_INVALID) { out_pkt = odp_crypto_packet_from_event(ev); - odp_crypto_result(&result, out_pkt); - + if (odp_unlikely(odp_crypto_result(&result, out_pkt) != 0) || + odp_unlikely(!result.ok)) { + ODPH_ERR("Crypto operation failed\n"); + odp_packet_free(out_pkt); + return -1; + } if (cargs->debug_packets) { mem = odp_packet_data(out_pkt); print_mem("Received encrypted packet", diff --git a/test/performance/odp_ipsec.c b/test/performance/odp_ipsec.c index c464ba0d7..50e26b1c9 100644 --- a/test/performance/odp_ipsec.c +++ b/test/performance/odp_ipsec.c @@ -862,7 +862,7 @@ run_measure_one_async(ipsec_args_t *cargs, packets_sent += rc; } else { odp_packet_t pkt_out[max_in_flight]; - uint32_t i = 0; + int i = 0; /* * Dequeue packets until we can enqueue the next burst @@ -895,7 +895,7 @@ run_measure_one_async(ipsec_args_t *cargs, } debug_packets(debug, pkt_out, i); - if (i) + if (i > 0) odp_packet_free_sp(pkt_out, i); } } diff --git a/test/performance/odp_sched_perf.c b/test/performance/odp_sched_perf.c index cd1afdf60..a97686c12 100644 --- a/test/performance/odp_sched_perf.c +++ b/test/performance/odp_sched_perf.c @@ -1375,6 +1375,9 @@ int main(int argc, char **argv) if (create_queues(global)) return -1; + if (global->test_options.verbose) + odp_shm_print_all(); + /* Start workers */ start_workers(global, instance); diff --git a/test/performance/odp_sched_pktio.c b/test/performance/odp_sched_pktio.c index 11be9d367..5cc07d010 100644 --- a/test/performance/odp_sched_pktio.c +++ b/test/performance/odp_sched_pktio.c @@ -173,7 +173,7 @@ static inline void send_packets(test_global_t *test_global, drop = num_pkt - sent; - if (odp_unlikely(drop)) + if (odp_unlikely(drop > 0)) odp_packet_free_multi(&pkt[sent], drop); if (odp_unlikely(test_global->opt.collect_stat)) { @@ -425,6 +425,7 @@ static int worker_thread_timers(void *arg) pktin_queue_context_t *queue_context; odp_timer_t timer; odp_timer_set_t ret; + odp_timer_start_t start_param; worker_arg_t *worker_arg = arg; test_global_t *test_global = worker_arg->test_global_ptr; int worker_id = worker_arg->worker_id; @@ -460,12 +461,17 @@ static int worker_thread_timers(void *arg) src_pktio = queue_context->src_pktio; src_queue = queue_context->src_queue; timer = test_global->timer.timer[src_pktio][src_queue]; + start_param.tick_type = ODP_TIMER_TICK_REL; + start_param.tick = tick; + start_param.tmo_ev = ODP_EVENT_INVALID; for (i = 0; i < num; i++) { if (odp_unlikely(odp_event_type(ev[i]) == ODP_EVENT_TIMEOUT)) { tmos++; - ret = odp_timer_set_rel(timer, tick, &ev[i]); + + start_param.tmo_ev = ev[i]; + ret = odp_timer_start(timer, &start_param); if (odp_unlikely(ret != ODP_TIMER_SUCCESS)) { /* Should never happen. Timeout event @@ -487,7 +493,7 @@ static int worker_thread_timers(void *arg) if (tmos == 0) { /* Reset timer with existing timeout event */ - ret = odp_timer_set_rel(timer, tick, NULL); + ret = odp_timer_restart(timer, &start_param); if (odp_unlikely(ret != ODP_TIMER_SUCCESS && ret != ODP_TIMER_FAIL)) { @@ -1318,10 +1324,10 @@ static int create_timers(test_global_t *test_global) static int start_timers(test_global_t *test_global) { int i, j; - odp_event_t event; odp_timeout_t timeout; odp_timer_t timer; odp_timer_set_t ret; + odp_timer_start_t start_param; uint64_t timeout_tick = test_global->timer.timeout_tick; int num_pktio = test_global->opt.num_pktio; int num_queue = test_global->opt.num_pktio_queue; @@ -1335,6 +1341,9 @@ static int start_timers(test_global_t *test_global) timeout_tick += odp_timer_ns_to_tick(test_global->timer.timer_pool, TIMEOUT_OFFSET_NS); + start_param.tick_type = ODP_TIMER_TICK_REL; + start_param.tick = timeout_tick; + for (i = 0; i < num_pktio; i++) { for (j = 0; j < num_queue; j++) { timer = test_global->timer.timer[i][j]; @@ -1345,10 +1354,9 @@ static int start_timers(test_global_t *test_global) return -1; } - event = odp_timeout_to_event(timeout); - - ret = odp_timer_set_rel(timer, timeout_tick, &event); + start_param.tmo_ev = odp_timeout_to_event(timeout); + ret = odp_timer_start(timer, &start_param); if (ret != ODP_TIMER_SUCCESS) { printf("Timer set failed\n"); return -1; diff --git a/test/performance/odp_timer_perf.c b/test/performance/odp_timer_perf.c index 998db628e..ccea30a14 100644 --- a/test/performance/odp_timer_perf.c +++ b/test/performance/odp_timer_perf.c @@ -438,6 +438,7 @@ static int set_timers(test_global_t *global) odp_event_t ev; int status; timer_ctx_t *ctx = &global->timer_ctx[i][j]; + odp_timer_start_t start_param; /* Set timers backwards, the last timer is set first */ if (j == 0) @@ -456,9 +457,11 @@ static int set_timers(test_global_t *global) tick_ns = odp_timer_ns_to_tick(tp, nsec); nsec = nsec - period_ns; - status = odp_timer_set_abs(timer, tick_cur + tick_ns, - &ev); + start_param.tick_type = ODP_TIMER_TICK_ABS; + start_param.tick = tick_cur + tick_ns; + start_param.tmo_ev = ev; + status = odp_timer_start(timer, &start_param); if (status != ODP_TIMER_SUCCESS) { ODPH_ERR("Timer set %i/%i (ret %i)\n", i, j, status); return -1; @@ -656,6 +659,7 @@ static int set_cancel_mode_worker(void *arg) odp_timer_t timer; odp_timer_pool_t tp; odp_timeout_t tmo; + odp_timer_start_t start_param; thread_arg_t *thread_arg = arg; test_global_t *global = thread_arg->global; test_options_t *test_options = &global->test_options; @@ -694,7 +698,11 @@ static int set_cancel_mode_worker(void *arg) period_tick = global->timer_pool[i].period_tick; tick = start_tick + j * period_tick; - status = odp_timer_set_rel(timer, tick, &ev); + start_param.tick_type = ODP_TIMER_TICK_REL; + start_param.tick = tick; + start_param.tmo_ev = ev; + + status = odp_timer_start(timer, &start_param); num_tmo++; num_set++; @@ -746,7 +754,11 @@ static int set_cancel_mode_worker(void *arg) if (status < 0) continue; - status = odp_timer_set_abs(timer, tick + j * period_tick, &ev); + start_param.tick_type = ODP_TIMER_TICK_ABS; + start_param.tick = tick + j * period_tick; + start_param.tmo_ev = ev; + + status = odp_timer_start(timer, &start_param); num_set++; if (status != ODP_TIMER_SUCCESS) { diff --git a/test/validation/api/classification/odp_classification_common.c b/test/validation/api/classification/odp_classification_common.c index 4c24099a5..fe7677044 100644 --- a/test/validation/api/classification/odp_classification_common.c +++ b/test/validation/api/classification/odp_classification_common.c @@ -399,10 +399,10 @@ odp_packet_t create_packet(cls_packet_info_t pkt_info) vlan_hdr->type = odp_cpu_to_be_16(ODPH_ETHTYPE_VLAN); vlan_hdr++; } else { + odp_packet_has_vlan_set(pkt, 1); ethhdr->type = odp_cpu_to_be_16(ODPH_ETHTYPE_VLAN); } /* Default vlan header */ - odp_packet_has_vlan_set(pkt, 1); vlan_hdr->tci = odp_cpu_to_be_16(0); vlan_hdr->type = odp_cpu_to_be_16(eth_type); } else { diff --git a/test/validation/api/crypto/Makefile.am b/test/validation/api/crypto/Makefile.am index cc4d49d60..e438ec8ca 100644 --- a/test/validation/api/crypto/Makefile.am +++ b/test/validation/api/crypto/Makefile.am @@ -5,3 +5,4 @@ crypto_main_SOURCES = \ odp_crypto_test_inp.c \ test_vectors.h \ test_vectors_len.h +PRELDADD += $(LIBPACKET_COMMON) diff --git a/test/validation/api/crypto/odp_crypto_test_inp.c b/test/validation/api/crypto/odp_crypto_test_inp.c index 10f1b5ee2..7d5e50c71 100644 --- a/test/validation/api/crypto/odp_crypto_test_inp.c +++ b/test/validation/api/crypto/odp_crypto_test_inp.c @@ -1,17 +1,20 @@ /* Copyright (c) 2014-2018, Linaro Limited - * Copyright (c) 2021-2022, Nokia + * Copyright (c) 2021-2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause */ +#include <string.h> #include <odp_api.h> #include <odp/helper/odph_api.h> #include <odp_cunit_common.h> +#include <packet_common.h> #include "test_vectors.h" #define PKT_POOL_NUM 64 #define PKT_POOL_LEN (1 * 1024) +#define UAREA_SIZE 8 #define ARRAY_SIZE(a) (sizeof(a) / sizeof(a[0])) @@ -35,6 +38,7 @@ static void test_defaults(uint8_t fill) odp_crypto_session_param_init(¶m); CU_ASSERT_EQUAL(param.op, ODP_CRYPTO_OP_ENCODE); + CU_ASSERT_EQUAL(param.op_type, ODP_CRYPTO_OP_TYPE_LEGACY); CU_ASSERT_EQUAL(param.auth_cipher_text, false); #if ODP_DEPRECATED_API CU_ASSERT_EQUAL(param.pref_mode, ODP_CRYPTO_SYNC); @@ -45,13 +49,6 @@ static void test_defaults(uint8_t fill) CU_ASSERT_EQUAL(param.auth_alg, ODP_AUTH_ALG_NULL); CU_ASSERT_EQUAL(param.auth_iv_len, 0); CU_ASSERT_EQUAL(param.auth_aad_len, 0); - -#if ODP_DEPRECATED_API - CU_ASSERT_EQUAL(param.cipher_iv.data, NULL); - CU_ASSERT_EQUAL(param.cipher_iv.length, 0); - CU_ASSERT_EQUAL(param.auth_iv.data, NULL); - CU_ASSERT_EQUAL(param.auth_iv.length, 0); -#endif } static void test_default_values(void) @@ -60,53 +57,6 @@ static void test_default_values(void) test_defaults(0xff); } -static int packet_cmp_mem_bits(odp_packet_t pkt, uint32_t offset, - uint8_t *s, uint32_t len) -{ - int rc = -1; - uint32_t len_bytes = ((len + 7) / 8); - uint8_t leftover_bits = len % 8; - uint8_t buf[len_bytes]; - - odp_packet_copy_to_mem(pkt, offset, len_bytes, buf); - - /* Compare till the last full byte */ - rc = memcmp(buf, s, leftover_bits ? len_bytes - 1 : len_bytes); - - if (rc == 0 && leftover_bits) { - /* Do masked comparison for the leftover bits */ - uint8_t mask = 0xff << (8 - leftover_bits); - - rc = !((mask & buf[len_bytes - 1]) == - (mask & s[len_bytes - 1])); - } - - return rc; -} - -static int packet_cmp_mem_bytes(odp_packet_t pkt, uint32_t offset, - uint8_t *s, uint32_t len) -{ - uint8_t buf[len]; - - odp_packet_copy_to_mem(pkt, offset, len, buf); - - return memcmp(buf, s, len); -} - -static int packet_cmp_mem(odp_packet_t pkt, uint32_t offset, - uint8_t *s, uint32_t len, odp_bool_t bit_mode) -{ - int rc = -1; - - if (bit_mode) - rc = packet_cmp_mem_bits(pkt, offset, s, len); - else - rc = packet_cmp_mem_bytes(pkt, offset, s, len); - - return rc; -} - static const char *auth_alg_name(odp_auth_alg_t auth) { switch (auth) { @@ -283,9 +233,12 @@ static int alg_op(odp_packet_t pkt, } #endif -static int alg_packet_op(odp_packet_t pkt, +static int alg_packet_op(odp_packet_t pkt_in, + odp_packet_t *pkt_out, odp_bool_t *ok, odp_crypto_session_t session, + odp_crypto_op_type_t op_type, + int32_t oop_shift, uint8_t *cipher_iv_ptr, uint8_t *auth_iv_ptr, odp_packet_data_range_t *cipher_range, @@ -298,7 +251,13 @@ static int alg_packet_op(odp_packet_t pkt, odp_crypto_packet_result_t result; odp_crypto_packet_op_param_t op_params; odp_event_subtype_t subtype; - odp_packet_t out_pkt = pkt; + odp_packet_t orig_pkt_out; + + if (op_type == ODP_CRYPTO_OP_TYPE_LEGACY) + *pkt_out = pkt_in; + else if (op_type == ODP_CRYPTO_OP_TYPE_BASIC) + *pkt_out = ODP_PACKET_INVALID; + orig_pkt_out = *pkt_out; /* Prepare input/output params */ memset(&op_params, 0, sizeof(op_params)); @@ -306,6 +265,7 @@ static int alg_packet_op(odp_packet_t pkt, op_params.cipher_range = *cipher_range; op_params.auth_range = *auth_range; + op_params.dst_offset_shift = oop_shift; if (cipher_iv_ptr) op_params.cipher_iv_ptr = cipher_iv_ptr; if (auth_iv_ptr) @@ -316,13 +276,18 @@ static int alg_packet_op(odp_packet_t pkt, op_params.hash_result_offset = hash_result_offset; if (suite_context.op_mode == ODP_CRYPTO_SYNC) { - rc = odp_crypto_op(&pkt, &out_pkt, &op_params, 1); + rc = odp_crypto_op(&pkt_in, pkt_out, &op_params, 1); if (rc <= 0) { CU_FAIL("Failed odp_crypto_packet_op()"); return rc; } } else { - rc = odp_crypto_op_enq(&pkt, &pkt, &op_params, 1); + odp_packet_t *out_param = pkt_out; + + if (op_type == ODP_CRYPTO_OP_TYPE_BASIC) + out_param = NULL; + + rc = odp_crypto_op_enq(&pkt_in, out_param, &op_params, 1); if (rc <= 0) { CU_FAIL("Failed odp_crypto_op_enq()"); return rc; @@ -339,33 +304,42 @@ static int alg_packet_op(odp_packet_t pkt, CU_ASSERT(ODP_EVENT_PACKET == odp_event_types(event, &subtype)); CU_ASSERT(ODP_EVENT_PACKET_CRYPTO == subtype); - pkt = odp_crypto_packet_from_event(event); + *pkt_out = odp_crypto_packet_from_event(event); } - CU_ASSERT(out_pkt == pkt); + if (op_type != ODP_CRYPTO_OP_TYPE_BASIC) + CU_ASSERT(*pkt_out == orig_pkt_out); CU_ASSERT(ODP_EVENT_PACKET == - odp_event_type(odp_packet_to_event(pkt))); + odp_event_type(odp_packet_to_event(*pkt_out))); CU_ASSERT(ODP_EVENT_PACKET_CRYPTO == - odp_event_subtype(odp_packet_to_event(pkt))); + odp_event_subtype(odp_packet_to_event(*pkt_out))); CU_ASSERT(ODP_EVENT_PACKET == - odp_event_types(odp_packet_to_event(pkt), &subtype)); + odp_event_types(odp_packet_to_event(*pkt_out), &subtype)); CU_ASSERT(ODP_EVENT_PACKET_CRYPTO == subtype); - CU_ASSERT(odp_packet_subtype(pkt) == ODP_EVENT_PACKET_CRYPTO); + CU_ASSERT(odp_packet_subtype(*pkt_out) == ODP_EVENT_PACKET_CRYPTO); - rc = odp_crypto_result(&result, pkt); + rc = odp_crypto_result(&result, *pkt_out); if (rc < 0) { CU_FAIL("Failed odp_crypto_packet_result()"); return rc; } + CU_ASSERT(rc == 0); + + if (op_type == ODP_CRYPTO_OP_TYPE_OOP && + suite_context.op_mode == ODP_CRYPTO_ASYNC) + CU_ASSERT(result.pkt_in == pkt_in); *ok = result.ok; return 0; } -static int crypto_op(odp_packet_t pkt, +static int crypto_op(odp_packet_t pkt_in, + odp_packet_t *pkt_out, odp_bool_t *ok, odp_crypto_session_t session, + odp_crypto_op_type_t op_type, + int32_t oop_shift, uint8_t *cipher_iv, uint8_t *auth_iv, odp_packet_data_range_t *cipher_range, @@ -375,23 +349,26 @@ static int crypto_op(odp_packet_t pkt, { int rc; - if (!suite_context.packet) + if (!suite_context.packet) { #if ODP_DEPRECATED_API - rc = alg_op(pkt, ok, session, + rc = alg_op(pkt_in, ok, session, cipher_iv, auth_iv, cipher_range, auth_range, aad, hash_result_offset); + *pkt_out = pkt_in; #else rc = -1; #endif - else - rc = alg_packet_op(pkt, ok, session, + } else { + rc = alg_packet_op(pkt_in, pkt_out, ok, session, + op_type, oop_shift, cipher_iv, auth_iv, cipher_range, auth_range, aad, hash_result_offset); + } if (rc < 0) - odp_packet_free(pkt); + odp_packet_free(pkt_in); return rc; } @@ -434,12 +411,8 @@ static void fill_with_pattern(uint8_t *buf, uint32_t len) buf[n] = n; } -/* - * Generate or verify header and trailer bytes - */ -static void do_header_and_trailer(odp_packet_t pkt, - uint32_t header_len, uint32_t trailer_len, - odp_bool_t check) +static void write_header_and_trailer(odp_packet_t pkt, + uint32_t header_len, uint32_t trailer_len) { uint32_t trailer_offset = odp_packet_len(pkt) - trailer_len; uint32_t max_len = header_len > trailer_len ? header_len : trailer_len; @@ -448,172 +421,470 @@ static void do_header_and_trailer(odp_packet_t pkt, fill_with_pattern(buffer, sizeof(buffer)); - if (check) { - CU_ASSERT(!packet_cmp_mem_bytes(pkt, 0, - buffer, header_len)); - CU_ASSERT(!packet_cmp_mem_bytes(pkt, trailer_offset, - buffer, trailer_len)); - } else { - rc = odp_packet_copy_from_mem(pkt, 0, - header_len, buffer); - CU_ASSERT(rc == 0); - rc = odp_packet_copy_from_mem(pkt, trailer_offset, - trailer_len, buffer); - CU_ASSERT(rc == 0); - } + rc = odp_packet_copy_from_mem(pkt, 0, header_len, buffer); + CU_ASSERT(rc == 0); + rc = odp_packet_copy_from_mem(pkt, trailer_offset, trailer_len, buffer); + CU_ASSERT(rc == 0); } -typedef enum crypto_test { - NORMAL_TEST = 0, /**< Plain execution */ - REPEAT_TEST, /**< Rerun without reinitializing the session */ - WRONG_DIGEST_TEST, /**< Check against wrong digest */ - MAX_TEST, /**< Final mark */ -} crypto_test; - typedef struct alg_test_param_t { odp_crypto_session_t session; odp_crypto_op_t op; + odp_crypto_op_type_t op_type; + int32_t oop_shift; odp_cipher_alg_t cipher_alg; odp_auth_alg_t auth_alg; crypto_test_reference_t *ref; uint32_t digest_offset; - odp_bool_t override_iv; odp_bool_t is_bit_mode_cipher; odp_bool_t is_bit_mode_auth; odp_bool_t adjust_segmentation; + odp_bool_t wrong_digest; uint32_t first_seg_len; uint32_t header_len; uint32_t trailer_len; } alg_test_param_t; -static void alg_test_execute(const alg_test_param_t *param) +static void prepare_crypto_ranges(const alg_test_param_t *param, + odp_packet_data_range_t *cipher_range, + odp_packet_data_range_t *auth_range) { - odp_bool_t ok = false; - int iteration; - crypto_test_reference_t *ref = param->ref; - uint32_t reflength = ref_length_in_bytes(ref); odp_packet_data_range_t zero_range = {.offset = 0, .length = 0}; - odp_packet_data_range_t cipher_range; - odp_packet_data_range_t auth_range; - uint8_t *cipher_iv = param->override_iv ? ref->cipher_iv : NULL; - uint8_t *auth_iv = param->override_iv ? ref->auth_iv : NULL; - cipher_range.offset = param->header_len; - cipher_range.length = reflength; - auth_range.offset = param->header_len; - auth_range.length = reflength; + cipher_range->offset = param->header_len; + cipher_range->length = ref_length_in_bytes(param->ref); + auth_range->offset = param->header_len; + auth_range->length = ref_length_in_bytes(param->ref); if (param->is_bit_mode_cipher) { - cipher_range.offset *= 8; - cipher_range.length = ref_length_in_bits(ref); + cipher_range->offset *= 8; + cipher_range->length = ref_length_in_bits(param->ref); } if (param->is_bit_mode_auth) { - auth_range.offset *= 8; - auth_range.length = ref_length_in_bits(ref); + auth_range->offset *= 8; + auth_range->length = ref_length_in_bits(param->ref); } /* * We did not check the bit mode of the null algorithms, so let's * not pass potentially invalid ranges to them. */ if (param->cipher_alg == ODP_CIPHER_ALG_NULL) - cipher_range = zero_range; + *cipher_range = zero_range; if (param->auth_alg == ODP_AUTH_ALG_NULL) - auth_range = zero_range; - for (iteration = NORMAL_TEST; iteration < MAX_TEST; iteration++) { - odp_packet_t pkt; - uint32_t digest_offset = param->digest_offset; - uint32_t pkt_len; + *auth_range = zero_range; +} - /* - * Test detection of wrong digest value in input packet - * only when decoding and using non-null auth algorithm. - */ - if (iteration == WRONG_DIGEST_TEST && - (param->auth_alg == ODP_AUTH_ALG_NULL || - param->op == ODP_CRYPTO_OP_ENCODE)) - continue; +static int prepare_input_packet(const alg_test_param_t *param, + odp_packet_t *pkt_in) +{ + crypto_test_reference_t *ref = param->ref; + uint32_t reflength = ref_length_in_bytes(ref); + odp_packet_t pkt; + uint32_t digest_offset = param->digest_offset; + uint32_t pkt_len; - pkt_len = param->header_len + reflength + param->trailer_len; - if (param->digest_offset == param->header_len + reflength) - pkt_len += ref->digest_length; + pkt_len = param->header_len + reflength + param->trailer_len; + if (param->digest_offset == param->header_len + reflength) + pkt_len += ref->digest_length; - pkt = odp_packet_alloc(suite_context.pool, pkt_len); + pkt = odp_packet_alloc(suite_context.pool, pkt_len); - CU_ASSERT(pkt != ODP_PACKET_INVALID); - if (pkt == ODP_PACKET_INVALID) - continue; + CU_ASSERT(pkt != ODP_PACKET_INVALID); + if (pkt == ODP_PACKET_INVALID) + return -1; - if (param->adjust_segmentation) - adjust_segments(&pkt, param->first_seg_len); - - do_header_and_trailer(pkt, param->header_len, param->trailer_len, false); - - if (param->op == ODP_CRYPTO_OP_ENCODE) { - odp_packet_copy_from_mem(pkt, param->header_len, - reflength, ref->plaintext); - } else { - odp_packet_copy_from_mem(pkt, param->header_len, - reflength, ref->ciphertext); - odp_packet_copy_from_mem(pkt, digest_offset, - ref->digest_length, - ref->digest); - if (iteration == WRONG_DIGEST_TEST) { - uint8_t byte = ~ref->digest[0]; - - odp_packet_copy_from_mem(pkt, digest_offset, - 1, &byte); - } - } + if (param->adjust_segmentation) + adjust_segments(&pkt, param->first_seg_len); - if (crypto_op(pkt, &ok, param->session, - cipher_iv, auth_iv, - &cipher_range, &auth_range, - ref->aad, digest_offset)) - break; + write_header_and_trailer(pkt, param->header_len, param->trailer_len); - if (iteration == WRONG_DIGEST_TEST) { - CU_ASSERT(!ok); - odp_packet_free(pkt); - continue; + if (param->op == ODP_CRYPTO_OP_ENCODE) { + odp_packet_copy_from_mem(pkt, param->header_len, + reflength, ref->plaintext); + } else { + odp_packet_copy_from_mem(pkt, param->header_len, + reflength, ref->ciphertext); + odp_packet_copy_from_mem(pkt, digest_offset, + ref->digest_length, + ref->digest); + if (param->wrong_digest) { + uint8_t byte = ~ref->digest[0]; + + odp_packet_copy_from_mem(pkt, digest_offset, 1, &byte); } + } + *pkt_in = pkt; + return 0; +} - CU_ASSERT(ok); +static void prepare_oop_output_packet(const alg_test_param_t *param, + odp_packet_t *pkt_out, + uint32_t pkt_len) +{ + uint32_t reflength = ref_length_in_bytes(param->ref); + const uint32_t oop_extra_len = 5; + uint32_t trl_len; + uint32_t hdr_len; + uint32_t oop_len; + + oop_len = pkt_len + param->oop_shift + oop_extra_len; + *pkt_out = odp_packet_alloc(suite_context.pool, oop_len); + CU_ASSERT_FATAL(*pkt_out != ODP_PACKET_INVALID); + + uint8_t buf[oop_len]; + + memset(buf, 0x55, sizeof(buf)); + odp_packet_copy_from_mem(*pkt_out, 0, sizeof(buf), buf); + + hdr_len = param->header_len + param->oop_shift; + trl_len = oop_len - hdr_len - reflength; + + write_header_and_trailer(*pkt_out, hdr_len, trl_len); + + /* have different metadata than in the input packet */ + memset(odp_packet_user_area(*pkt_out), 0xab, + odp_packet_user_area_size(*pkt_out)); +} + +static int is_packet_data_equal(odp_packet_t pkt_1, odp_packet_t pkt_2) +{ + uint32_t len = odp_packet_len(pkt_1); + uint8_t buf_1[len]; + uint8_t buf_2[len]; + + if (len != odp_packet_len(pkt_2) || + odp_packet_copy_to_mem(pkt_1, 0, len, buf_1) || + odp_packet_copy_to_mem(pkt_2, 0, len, buf_2)) + return 0; + + return !memcmp(buf_1, buf_2, len); +} + +static int is_in_range(uint32_t offs, uint32_t range_offs, uint32_t range_len) +{ + return offs >= range_offs && offs < range_offs + range_len; +} + +#define MAX_IGNORED_RANGES 3 + +/* + * Output packet parts that we ignore since they have undefined values + */ +typedef struct ignore_t { + uint32_t byte_offset; /* offset to a byte which has bits to be ignored */ + uint32_t byte_mask; /* mask of ignored bits in the byte */ + struct { + uint32_t offset; + uint32_t length; + } ranges[MAX_IGNORED_RANGES]; /* byte ranges to be ignored */ + uint32_t num_ranges; +} ignore_t; + +static void add_ignored_range(ignore_t *ign, uint32_t offs, uint32_t len) +{ + if (len == 0) + return; + CU_ASSERT_FATAL(ign->num_ranges < MAX_IGNORED_RANGES); + ign->ranges[ign->num_ranges].offset = offs; + ign->ranges[ign->num_ranges].length = len; + ign->num_ranges++; +} + +static void clear_ignored_data(const ignore_t *ign, uint8_t *data, uint32_t data_len) +{ + CU_ASSERT_FATAL(ign->byte_offset < data_len); + data[ign->byte_offset] &= ~ign->byte_mask; + + for (uint32_t n = 0; n < ign->num_ranges; n++) { + uint32_t offset = ign->ranges[n].offset; + uint32_t length = ign->ranges[n].length; + + CU_ASSERT(offset + length <= data_len); + memset(data + offset, 0, length); + } +} + +static void prepare_ignore_info(const alg_test_param_t *param, + uint32_t shift, + uint32_t cipher_offset, + uint32_t cipher_len, + uint32_t auth_offset, + uint32_t auth_len, + ignore_t *ignore) +{ + memset(ignore, 0, sizeof(*ignore)); - do_header_and_trailer(pkt, param->header_len, param->trailer_len, true); - - if (param->op == ODP_CRYPTO_OP_ENCODE) { - CU_ASSERT(!packet_cmp_mem(pkt, param->header_len, - ref->ciphertext, - ref->length, - ref->is_length_in_bits)); - CU_ASSERT(!packet_cmp_mem_bytes(pkt, digest_offset, - ref->digest, - ref->digest_length)); - } else { - /* - * Hash result in the packet is left to undefined - * values. Restore it from the plaintext packet - * to make the subsequent comparison work even - * if the hash result is within the auth_range. - */ - odp_packet_copy_from_mem(pkt, digest_offset, - ref->digest_length, - ref->plaintext + - digest_offset - param->header_len); - - CU_ASSERT(!packet_cmp_mem(pkt, param->header_len, - ref->plaintext, - ref->length, - ref->is_length_in_bits)); + /* + * Leftover bits in the last byte of the cipher range of bit mode + * ciphers have undefined values. + */ + if (param->is_bit_mode_cipher && + param->cipher_alg != ODP_CIPHER_ALG_NULL) { + uint8_t leftover_bits = param->ref->length % 8; + + ignore->byte_offset = cipher_offset + cipher_len - 1 + shift; + ignore->byte_mask = ~(0xff << (8 - leftover_bits)); + } + + /* + * In decode sessions the bytes in the hash location have + * undefined values. + */ + if (param->auth_alg != ODP_AUTH_ALG_NULL && + param->op == ODP_CRYPTO_OP_DECODE) { + uint32_t offs = param->digest_offset; + + if (param->op_type != ODP_CRYPTO_OP_TYPE_OOP || + is_in_range(offs, cipher_offset, cipher_len) || + is_in_range(offs, auth_offset, auth_len)) { + add_ignored_range(ignore, + param->digest_offset + shift, + param->ref->digest_length); } + } + + /* Decrypted bytes are undefined if authentication fails. */ + if (param->op == ODP_CRYPTO_OP_DECODE && + param->wrong_digest) { + add_ignored_range(ignore, cipher_offset + shift, cipher_len); + /* In OOP case, auth range may not get copied */ + if (param->op_type == ODP_CRYPTO_OP_TYPE_OOP) + add_ignored_range(ignore, auth_offset + shift, auth_len); + } +} + +/* Add room for bytes that are not included in ref->length */ +#define MAX_EXP_DATA_LEN (MAX_DATA_LEN + 200) + +/* + * Expected packet data + */ +typedef struct expected_t { + uint8_t data[MAX_EXP_DATA_LEN]; + uint32_t len; + ignore_t ignore; +} expected_t; + +static void prepare_expected_data(const alg_test_param_t *param, + const odp_packet_data_range_t *cipher_range, + const odp_packet_data_range_t *auth_range, + odp_packet_t pkt_in, + odp_packet_t pkt_out, + expected_t *ex) +{ + uint32_t digest_offset = param->digest_offset; + uint32_t cipher_offset = cipher_range->offset; + uint32_t cipher_len = cipher_range->length; + uint32_t auth_offset = auth_range->offset; + uint32_t auth_len = auth_range->length; + const int32_t shift = param->op_type == ODP_CRYPTO_OP_TYPE_OOP ? param->oop_shift : 0; + const odp_packet_t base_pkt = param->op_type == ODP_CRYPTO_OP_TYPE_OOP ? pkt_out : pkt_in; + int rc; + + if (param->op == ODP_CRYPTO_OP_ENCODE) + digest_offset += shift; + + if (param->is_bit_mode_cipher) { + cipher_offset /= 8; + cipher_len = (cipher_len + 7) / 8; + } + if (param->is_bit_mode_auth) { + auth_offset /= 8; + auth_len = (auth_len + 7) / 8; + } + if (param->cipher_alg == ODP_CIPHER_ALG_NULL) + cipher_len = 0; + if (param->auth_alg == ODP_AUTH_ALG_NULL || + param->auth_alg == ODP_AUTH_ALG_AES_GCM || + param->auth_alg == ODP_AUTH_ALG_AES_CCM || + param->auth_alg == ODP_AUTH_ALG_CHACHA20_POLY1305) { + /* auth range is ignored with null and AEAD algorithms */ + auth_len = 0; + } + + /* copy all data from base packet */ + ex->len = odp_packet_len(base_pkt); + CU_ASSERT_FATAL(ex->len <= sizeof(ex->data)); + rc = odp_packet_copy_to_mem(base_pkt, 0, ex->len, ex->data); + CU_ASSERT(rc == 0); + + if (param->op_type == ODP_CRYPTO_OP_TYPE_OOP && auth_len > 0) { + /* copy auth range from input packet */ + rc = odp_packet_copy_to_mem(pkt_in, auth_offset, auth_len, + ex->data + auth_offset + shift); + CU_ASSERT(rc == 0); + } + + if (param->op == ODP_CRYPTO_OP_ENCODE) { + /* copy hash first */ + memcpy(ex->data + digest_offset, + param->ref->digest, + param->ref->digest_length); + /* + * Copy ciphertext, possibly overwriting hash. + * The other order (hash overwriting some cipher + * text) does not work in any real use case anyway. + */ + memcpy(ex->data + cipher_offset + shift, + param->ref->ciphertext, + cipher_len); + } else { + memcpy(ex->data + cipher_offset + shift, + param->ref->plaintext, + cipher_len); + } + + prepare_ignore_info(param, shift, + cipher_offset, cipher_len, + auth_offset, auth_len, + &ex->ignore); +} + +static void print_data(const char *title, uint8_t *data, uint32_t len) +{ + static uint64_t limit; + + if (limit++ > 20) + return; + + printf("%s\n", title); + for (uint32_t n = 0; n < len ; n++) { + printf(" %02x", data[n]); + if ((n + 1) % 16 == 0) + printf("\n"); + } + printf("\n"); +} + +static void check_output_packet_data(odp_packet_t pkt, expected_t *ex) +{ + int rc; + uint8_t pkt_data[ex->len]; + + CU_ASSERT(odp_packet_len(pkt) == ex->len); + rc = odp_packet_copy_to_mem(pkt, 0, ex->len, pkt_data); + CU_ASSERT(rc == 0); + + clear_ignored_data(&ex->ignore, pkt_data, sizeof(pkt_data)); + clear_ignored_data(&ex->ignore, ex->data, sizeof(ex->data)); + + if (memcmp(pkt_data, ex->data, ex->len)) { + CU_FAIL("packet data does not match expected data"); + print_data("packet:", pkt_data, ex->len); + print_data("expected:", ex->data, ex->len); + } +} + +static void alg_test_execute(const alg_test_param_t *param) +{ + odp_bool_t ok = false; + odp_packet_data_range_t cipher_range; + odp_packet_data_range_t auth_range; + odp_packet_t pkt; + odp_packet_t pkt_copy = ODP_PACKET_INVALID; + odp_packet_t pkt_out = ODP_PACKET_INVALID; + uint32_t digest_offset = param->digest_offset; + test_packet_md_t md_in, md_out, md_out_orig; + expected_t expected; + + /* + * Test detection of wrong digest value in input packet + * only when decoding and using non-null auth algorithm. + */ + if (param->wrong_digest && + (param->auth_alg == ODP_AUTH_ALG_NULL || + param->op == ODP_CRYPTO_OP_ENCODE)) + return; + + prepare_crypto_ranges(param, &cipher_range, &auth_range); + if (prepare_input_packet(param, &pkt)) + return; + + if (param->op_type == ODP_CRYPTO_OP_TYPE_OOP) { + prepare_oop_output_packet(param, &pkt_out, odp_packet_len(pkt)); + + pkt_copy = odp_packet_copy(pkt, suite_context.pool); + CU_ASSERT_FATAL(pkt_copy != ODP_PACKET_INVALID); + test_packet_get_md(pkt_out, &md_out_orig); + } + + prepare_expected_data(param, &cipher_range, &auth_range, + pkt, pkt_out, &expected); + + if (param->op_type == ODP_CRYPTO_OP_TYPE_OOP && + param->op == ODP_CRYPTO_OP_ENCODE) { + /* + * In this type of sessions digest offset is an offset to the output + * packet, so apply the shift. + */ + digest_offset += param->oop_shift; + } + + test_packet_set_md(pkt); + test_packet_get_md(pkt, &md_in); + + if (crypto_op(pkt, &pkt_out, &ok, param->session, + param->op_type, param->oop_shift, + param->ref->cipher_iv, + param->ref->auth_iv, + &cipher_range, &auth_range, + param->ref->aad, digest_offset)) + return; + + /* + * API is not explicit about whether a failed crypto op + * sets the has_error packet flag or leaves it unchanged. + * Let's allow both behaviours. + */ + test_packet_get_md(pkt_out, &md_out); + if (param->wrong_digest) + md_out.has_error = 0; + + if (param->op_type == ODP_CRYPTO_OP_TYPE_OOP) { + test_packet_md_t md; + + /* check that input packet has not changed */ + CU_ASSERT(is_packet_data_equal(pkt, pkt_copy)); + odp_packet_free(pkt_copy); + test_packet_get_md(pkt, &md); + CU_ASSERT(test_packet_is_md_equal(&md, &md_in)); odp_packet_free(pkt); + + /* check that metadata of output packet has not changed */ + CU_ASSERT(test_packet_is_md_equal(&md_out, &md_out_orig)); + } else { + CU_ASSERT(test_packet_is_md_equal(&md_out, &md_in)); } + + if (param->wrong_digest) { + CU_ASSERT(!ok); + } else { + CU_ASSERT(ok); + } + check_output_packet_data(pkt_out, &expected); + odp_packet_free(pkt_out); } -typedef enum { - PACKET_IV, - OLD_PACKET_IV, - OLD_SESSION_IV, -} iv_test_mode_t; +static void alg_test_op(alg_test_param_t *param) +{ + int32_t oop_shifts[] = {0, 3, 130, -10}; + + for (uint32_t n = 0; n < ARRAY_SIZE(oop_shifts); n++) { + if (oop_shifts[n] != 0 && + param->op_type != ODP_CRYPTO_OP_TYPE_OOP) + continue; + if ((int32_t)param->header_len + oop_shifts[n] < 0) + continue; + param->oop_shift = oop_shifts[n]; + + param->wrong_digest = false; + alg_test_execute(param); + alg_test_execute(param); /* rerun with the same parameters */ + param->wrong_digest = true; + alg_test_execute(param); + } +} + +static int oop_warning_shown; typedef enum { HASH_NO_OVERLAP, @@ -621,10 +892,10 @@ typedef enum { } hash_test_mode_t; static odp_crypto_session_t session_create(odp_crypto_op_t op, + odp_crypto_op_type_t op_type, odp_cipher_alg_t cipher_alg, odp_auth_alg_t auth_alg, crypto_test_reference_t *ref, - iv_test_mode_t iv_mode, hash_test_mode_t hash_mode) { odp_crypto_session_t session = ODP_CRYPTO_SESSION_INVALID; @@ -641,23 +912,6 @@ static odp_crypto_session_t session_create(odp_crypto_op_t op, .data = auth_key_data, .length = ref->auth_key_length }; -#if ODP_DEPRECATED_API - uint8_t cipher_iv_data[ref->cipher_iv_length]; - uint8_t auth_iv_data[ref->auth_iv_length]; - odp_crypto_iv_t cipher_iv = { - .length = ref->cipher_iv_length - }; - odp_crypto_iv_t auth_iv = { - .length = ref->auth_iv_length - }; - - if (iv_mode == OLD_SESSION_IV) { - memcpy(cipher_iv_data, ref->cipher_iv, ref->cipher_iv_length); - memcpy(auth_iv_data, ref->auth_iv, ref->auth_iv_length); - cipher_iv.data = cipher_iv_data; - auth_iv.data = auth_iv_data; - } -#endif memcpy(cipher_key_data, ref->cipher_key, ref->cipher_key_length); memcpy(auth_key_data, ref->auth_key, ref->auth_key_length); @@ -665,6 +919,7 @@ static odp_crypto_session_t session_create(odp_crypto_op_t op, /* Create a crypto session */ odp_crypto_session_param_init(&ses_params); ses_params.op = op; + ses_params.op_type = op_type; ses_params.auth_cipher_text = false; ses_params.op_mode = suite_context.op_mode; #if ODP_DEPRECATED_API @@ -675,15 +930,8 @@ static odp_crypto_session_t session_create(odp_crypto_op_t op, ses_params.compl_queue = suite_context.queue; ses_params.output_pool = suite_context.pool; ses_params.cipher_key = cipher_key; - if (iv_mode == PACKET_IV) { - ses_params.cipher_iv_len = ref->cipher_iv_length; - ses_params.auth_iv_len = ref->auth_iv_length; - } else { -#if ODP_DEPRECATED_API - ses_params.cipher_iv = cipher_iv; - ses_params.auth_iv = auth_iv; -#endif - } + ses_params.cipher_iv_len = ref->cipher_iv_length; + ses_params.auth_iv_len = ref->auth_iv_length; ses_params.auth_key = auth_key; ses_params.auth_digest_len = ref->digest_length; ses_params.auth_aad_len = ref->aad_length; @@ -701,14 +949,21 @@ static odp_crypto_session_t session_create(odp_crypto_op_t op, auth_alg_name(auth_alg)); return ODP_CRYPTO_SESSION_INVALID; } + + /* For now, allow out-of-place sessions not to be supported. */ + if (rc < 0 && status == ODP_CRYPTO_SES_ERR_PARAMS && + op_type == ODP_CRYPTO_OP_TYPE_OOP) { + if (!oop_warning_shown) + printf("\n Skipping ODP_CRYPTO_OP_TYPE_OOP tests\n"); + oop_warning_shown = 1; + return ODP_CRYPTO_SESSION_INVALID; + } + /* * We do not allow ODP_CRYPTO_SES_ERR_ALG_ORDER since we do * not combine individual non-null crypto and auth algorithms * with each other in the tests. Both orders should work when * only one algorithm is used (i.e. the other one is null). - * - * We do not allow ODP_CRYPTO_SES_ERR_PARAMS until needed for - * some ODP implementation. */ CU_ASSERT_FATAL(!rc); CU_ASSERT(status == ODP_CRYPTO_SES_ERR_NONE); @@ -721,23 +976,19 @@ static odp_crypto_session_t session_create(odp_crypto_op_t op, */ memset(cipher_key_data, 0, sizeof(cipher_key_data)); memset(auth_key_data, 0, sizeof(auth_key_data)); -#if ODP_DEPRECATED_API - memset(cipher_iv_data, 0, sizeof(cipher_iv_data)); - memset(auth_iv_data, 0, sizeof(auth_iv_data)); -#endif memset(&ses_params, 0, sizeof(ses_params)); return session; } -static void alg_test(odp_crypto_op_t op, - odp_cipher_alg_t cipher_alg, - odp_auth_alg_t auth_alg, - crypto_test_reference_t *ref, - uint32_t digest_offset, - iv_test_mode_t iv_mode, - odp_bool_t is_bit_mode_cipher, - odp_bool_t is_bit_mode_auth) +static void alg_test_ses(odp_crypto_op_t op, + odp_crypto_op_type_t op_type, + odp_cipher_alg_t cipher_alg, + odp_auth_alg_t auth_alg, + crypto_test_reference_t *ref, + uint32_t digest_offset, + odp_bool_t is_bit_mode_cipher, + odp_bool_t is_bit_mode_auth) { unsigned int initial_num_failures = CU_get_number_of_failures(); const uint32_t reflength = ref_length_in_bytes(ref); @@ -749,22 +1000,22 @@ static void alg_test(odp_crypto_op_t op, uint32_t max_shift; alg_test_param_t test_param; - session = session_create(op, cipher_alg, auth_alg, ref, iv_mode, hash_mode); + session = session_create(op, op_type, cipher_alg, auth_alg, ref, hash_mode); if (session == ODP_CRYPTO_SESSION_INVALID) return; memset(&test_param, 0, sizeof(test_param)); test_param.session = session; test_param.op = op; + test_param.op_type = op_type; test_param.cipher_alg = cipher_alg; test_param.auth_alg = auth_alg; test_param.ref = ref; - test_param.override_iv = (iv_mode != OLD_SESSION_IV); test_param.is_bit_mode_cipher = is_bit_mode_cipher; test_param.is_bit_mode_auth = is_bit_mode_auth; test_param.digest_offset = digest_offset; - alg_test_execute(&test_param); + alg_test_op(&test_param); max_shift = reflength + ref->digest_length; @@ -785,19 +1036,48 @@ static void alg_test(odp_crypto_op_t op, test_param.header_len = 0; test_param.trailer_len = 0; test_param.digest_offset = digest_offset; - alg_test_execute(&test_param); + alg_test_op(&test_param); /* Test partial packet crypto with odd alignment. */ - test_param.header_len = 3; + test_param.header_len = 13; test_param.trailer_len = 32; test_param.digest_offset = test_param.header_len + digest_offset; - alg_test_execute(&test_param); + alg_test_op(&test_param); } rc = odp_crypto_session_destroy(session); CU_ASSERT(!rc); } +static void alg_test(odp_crypto_op_t op, + odp_cipher_alg_t cipher_alg, + odp_auth_alg_t auth_alg, + crypto_test_reference_t *ref, + uint32_t digest_offset, + odp_bool_t is_bit_mode_cipher, + odp_bool_t is_bit_mode_auth) +{ + odp_crypto_op_type_t op_types[] = { + ODP_CRYPTO_OP_TYPE_LEGACY, + ODP_CRYPTO_OP_TYPE_BASIC, + ODP_CRYPTO_OP_TYPE_OOP, + }; + + for (unsigned int n = 0; n < ARRAY_SIZE(op_types); n++) { + if (!suite_context.packet && + op_types[n] != ODP_CRYPTO_OP_TYPE_LEGACY) + continue; + alg_test_ses(op, + op_types[n], + cipher_alg, + auth_alg, + ref, + digest_offset, + is_bit_mode_cipher, + is_bit_mode_auth); + } +} + static odp_bool_t aad_len_ok(const odp_crypto_auth_capability_t *capa, uint32_t len) { if (len < capa->aad_len.min || len > capa->aad_len.max) @@ -843,6 +1123,8 @@ static void check_alg(odp_crypto_op_t op, memset(cipher_tested, 0, sizeof(cipher_tested)); memset(auth_tested, 0, sizeof(auth_tested)); + oop_warning_shown = 0; /* allow OOP-unsupported warning again */ + for (idx = 0; idx < count; idx++) { int cipher_idx = -1, auth_idx = -1; odp_bool_t bit_mode_needed = false; @@ -908,18 +1190,8 @@ static void check_alg(odp_crypto_op_t op, continue; } - /* test with per-packet IV */ alg_test(op, cipher_alg, auth_alg, &ref[idx], digest_offs, - PACKET_IV, is_bit_mode_cipher, is_bit_mode_auth); -#if ODP_DEPRECATED_API - /* test with per-packet IV using the old API*/ - alg_test(op, cipher_alg, auth_alg, &ref[idx], digest_offs, - OLD_PACKET_IV, is_bit_mode_cipher, is_bit_mode_auth); - - /* test with per-session IV */ - alg_test(op, cipher_alg, auth_alg, &ref[idx], digest_offs, - OLD_SESSION_IV, is_bit_mode_cipher, is_bit_mode_auth); -#endif + is_bit_mode_cipher, is_bit_mode_auth); cipher_tested[cipher_idx] = true; auth_tested[auth_idx] = true; @@ -1223,12 +1495,15 @@ static int create_hash_test_reference(odp_auth_alg_t auth, rc = odp_packet_copy_from_mem(pkt, 0, auth_bytes, ref->plaintext); CU_ASSERT(rc == 0); - session = session_create(ODP_CRYPTO_OP_ENCODE, ODP_CIPHER_ALG_NULL, - auth, ref, PACKET_IV, HASH_NO_OVERLAP); + session = session_create(ODP_CRYPTO_OP_ENCODE, + ODP_CRYPTO_OP_TYPE_LEGACY, + ODP_CIPHER_ALG_NULL, + auth, ref, HASH_NO_OVERLAP); if (session == ODP_CRYPTO_SESSION_INVALID) return -1; - rc = crypto_op(pkt, &ok, session, ref->cipher_iv, ref->auth_iv, + rc = crypto_op(pkt, &pkt, &ok, session, ODP_CRYPTO_OP_TYPE_LEGACY, 0, + ref->cipher_iv, ref->auth_iv, &cipher_range, &auth_range, ref->aad, enc_digest_offset); CU_ASSERT(rc == 0); CU_ASSERT(ok); @@ -1243,11 +1518,11 @@ static int create_hash_test_reference(odp_auth_alg_t auth, /* copy the calculated digest in the ciphertext packet in ref */ rc = odp_packet_copy_to_mem(pkt, enc_digest_offset, ref->digest_length, &ref->ciphertext[digest_offset]); + CU_ASSERT(rc == 0); /* copy the calculated digest the digest field in ref */ rc = odp_packet_copy_to_mem(pkt, enc_digest_offset, ref->digest_length, &ref->digest); - CU_ASSERT(rc == 0); odp_packet_free(pkt); @@ -1280,7 +1555,6 @@ static void test_auth_hash_in_auth_range(odp_auth_alg_t auth, auth, &ref, digest_offset, - PACKET_IV, false, capa->bit_mode); @@ -1303,7 +1577,6 @@ static void test_auth_hash_in_auth_range(odp_auth_alg_t auth, auth, &ref, digest_offset, - PACKET_IV, false, capa->bit_mode); } @@ -2502,6 +2775,15 @@ static int crypto_init(odp_instance_t *inst) params.pkt.num = PKT_POOL_NUM; params.type = ODP_POOL_PACKET; + /* + * Let's have a user area so that we can check that its + * content gets copied along with other metadata when needed. + */ + if (pool_capa.pkt.max_uarea_size >= UAREA_SIZE) + params.pkt.uarea_size = UAREA_SIZE; + else + printf("Warning: could not request packet user area\n"); + if (pool_capa.pkt.max_seg_len && PKT_POOL_LEN > pool_capa.pkt.max_seg_len) { fprintf(stderr, "Warning: small packet segment length\n"); diff --git a/test/validation/api/dma/dma.c b/test/validation/api/dma/dma.c index 8eb75b172..f3e967193 100644 --- a/test/validation/api/dma/dma.c +++ b/test/validation/api/dma/dma.c @@ -19,6 +19,7 @@ #define TRAILER 10 #define MULTI 1 #define RESULT 1 +#define USER_DATA 0xdeadbeef typedef struct global_t { odp_dma_capability_t dma_capa; @@ -385,12 +386,13 @@ static int do_transfer_async(odp_dma_t dma, const odp_dma_transfer_param_t *trs_ odp_dma_compl_param_t compl_param; odp_event_t ev; odp_dma_compl_t compl; - int i, ret, done, dummy; + int i, ret, done; + uint32_t user_data = USER_DATA; odp_dma_result_t result; odp_dma_transfer_id_t transfer_id = ODP_DMA_TRANSFER_ID_INVALID; uint64_t wait_ns = 500 * ODP_TIME_MSEC_IN_NS; uint64_t sched_wait = odp_schedule_wait_time(wait_ns); - void *user_ptr = &dummy; + void *user_ptr = &user_data; odp_dma_compl_param_init(&compl_param); compl_param.compl_mode = compl_mode; @@ -452,6 +454,7 @@ static int do_transfer_async(odp_dma_t dma, const odp_dma_transfer_param_t *trs_ CU_ASSERT(done == 1); CU_ASSERT(result.success); CU_ASSERT(result.user_ptr == user_ptr); + CU_ASSERT(user_data == USER_DATA); odp_dma_transfer_id_free(dma, transfer_id); @@ -479,6 +482,7 @@ static int do_transfer_async(odp_dma_t dma, const odp_dma_transfer_param_t *trs_ CU_ASSERT(odp_dma_compl_result(compl, &result) == 0); CU_ASSERT(result.success); CU_ASSERT(result.user_ptr == user_ptr); + CU_ASSERT(user_data == USER_DATA); /* Test also without result struct output */ CU_ASSERT(odp_dma_compl_result(compl, NULL) == 0); diff --git a/test/validation/api/packet/packet.c b/test/validation/api/packet/packet.c index 1ae973d79..c654aa055 100644 --- a/test/validation/api/packet/packet.c +++ b/test/validation/api/packet/packet.c @@ -1,5 +1,5 @@ /* Copyright (c) 2014-2018, Linaro Limited - * Copyright (c) 2019-2022, Nokia + * Copyright (c) 2019-2023, Nokia * Copyright (c) 2020, Marvell * All rights reserved. * @@ -329,7 +329,8 @@ static int packet_suite_term(void) return 0; } -static void packet_set_inflags(odp_packet_t pkt, int val) +/* Set all non-conflicting metadata flags */ +static void packet_set_inflags_common(odp_packet_t pkt, int val) { odp_packet_has_l2_set(pkt, val); odp_packet_has_l3_set(pkt, val); @@ -339,23 +340,37 @@ static void packet_set_inflags(odp_packet_t pkt, int val) odp_packet_has_eth_mcast_set(pkt, val); odp_packet_has_jumbo_set(pkt, val); odp_packet_has_vlan_set(pkt, val); - odp_packet_has_vlan_qinq_set(pkt, val); - odp_packet_has_arp_set(pkt, val); odp_packet_has_ipv4_set(pkt, val); - odp_packet_has_ipv6_set(pkt, val); odp_packet_has_ip_bcast_set(pkt, val); - odp_packet_has_ip_mcast_set(pkt, val); odp_packet_has_ipfrag_set(pkt, val); odp_packet_has_ipopt_set(pkt, val); odp_packet_has_ipsec_set(pkt, val); odp_packet_has_udp_set(pkt, val); - odp_packet_has_tcp_set(pkt, val); - odp_packet_has_sctp_set(pkt, val); - odp_packet_has_icmp_set(pkt, val); odp_packet_user_flag_set(pkt, val); } -static void packet_check_inflags(odp_packet_t pkt, int val) +/* Check all non-conflicting metadata flags */ +static void packet_check_inflags_common(odp_packet_t pkt, int val) +{ + CU_ASSERT(odp_packet_has_l2(pkt) == !!val); + CU_ASSERT(odp_packet_has_l3(pkt) == !!val); + CU_ASSERT(odp_packet_has_l4(pkt) == !!val); + CU_ASSERT(odp_packet_has_eth(pkt) == !!val); + CU_ASSERT(odp_packet_has_eth_bcast(pkt) == !!val); + CU_ASSERT(odp_packet_has_eth_mcast(pkt) == !!val); + CU_ASSERT(odp_packet_has_jumbo(pkt) == !!val); + CU_ASSERT(odp_packet_has_vlan(pkt) == !!val); + CU_ASSERT(odp_packet_has_ipv4(pkt) == !!val); + CU_ASSERT(odp_packet_has_ip_bcast(pkt) == !!val); + CU_ASSERT(odp_packet_has_ipfrag(pkt) == !!val); + CU_ASSERT(odp_packet_has_ipopt(pkt) == !!val); + CU_ASSERT(odp_packet_has_ipsec(pkt) == !!val); + CU_ASSERT(odp_packet_has_udp(pkt) == !!val); + CU_ASSERT(odp_packet_user_flag(pkt) == !!val); +} + +/* Check all metadata flags */ +static void packet_check_inflags_all(odp_packet_t pkt, int val) { CU_ASSERT(odp_packet_has_l2(pkt) == !!val); CU_ASSERT(odp_packet_has_l3(pkt) == !!val); @@ -418,7 +433,7 @@ static void packet_test_alloc_free(void) CU_ASSERT(odp_packet_user_ptr(packet) == NULL); /* Packet flags should be zero */ - packet_check_inflags(packet, 0); + packet_check_inflags_all(packet, 0); /* Pool should have only one packet */ CU_ASSERT_FATAL(odp_packet_alloc(pool, packet_len) @@ -937,10 +952,10 @@ static void packet_test_reset(void) CU_ASSERT(odp_packet_reset(pkt, len) == 0); CU_ASSERT(odp_packet_len(pkt) == len); - packet_set_inflags(pkt, 1); - packet_check_inflags(pkt, 1); + packet_set_inflags_common(pkt, 1); + packet_check_inflags_common(pkt, 1); CU_ASSERT(odp_packet_reset(pkt, len) == 0); - packet_check_inflags(pkt, 0); + packet_check_inflags_all(pkt, 0); CU_ASSERT(odp_packet_reset(pkt, len - 1) == 0); CU_ASSERT(odp_packet_len(pkt) == (len - 1)); @@ -1374,10 +1389,10 @@ static void packet_test_in_flags(void) { odp_packet_t pkt = test_packet; - packet_set_inflags(pkt, 0); - packet_check_inflags(pkt, 0); - packet_set_inflags(pkt, 1); - packet_check_inflags(pkt, 1); + packet_set_inflags_common(pkt, 0); + packet_check_inflags_common(pkt, 0); + packet_set_inflags_common(pkt, 1); + packet_check_inflags_common(pkt, 1); TEST_INFLAG(pkt, has_l2); TEST_INFLAG(pkt, has_l3); @@ -1402,8 +1417,32 @@ static void packet_test_in_flags(void) TEST_INFLAG(pkt, has_icmp); TEST_INFLAG(pkt, user_flag); - packet_set_inflags(pkt, 0); - packet_check_inflags(pkt, 0); + packet_set_inflags_common(pkt, 0); + packet_check_inflags_common(pkt, 0); +} + +static void packet_test_vlan_flags(void) +{ + odp_packet_t pkt = test_packet; + + odp_packet_reset(pkt, odp_packet_len(test_packet)); + + CU_ASSERT(!odp_packet_has_vlan(pkt)); + CU_ASSERT(!odp_packet_has_vlan_qinq(pkt)); + + odp_packet_has_vlan_qinq_set(pkt, 1); + CU_ASSERT(odp_packet_has_vlan(pkt)); + CU_ASSERT(odp_packet_has_vlan_qinq(pkt)); + + odp_packet_has_vlan_qinq_set(pkt, 0); + CU_ASSERT(!odp_packet_has_vlan(pkt)); + CU_ASSERT(!odp_packet_has_vlan_qinq(pkt)); + + odp_packet_has_vlan_set(pkt, 1); + CU_ASSERT(odp_packet_has_vlan(pkt)); + CU_ASSERT(!odp_packet_has_vlan_qinq(pkt)); + + odp_packet_reset(pkt, odp_packet_len(test_packet)); } static void packet_test_error_flags(void) @@ -1628,15 +1667,15 @@ static void packet_test_meta_data_copy(void) pkt = odp_packet_alloc(pool, packet_len); CU_ASSERT_FATAL(pkt != ODP_PACKET_INVALID); - packet_check_inflags(pkt, 0); + packet_check_inflags_all(pkt, 0); CU_ASSERT(odp_packet_input(pkt) == ODP_PKTIO_INVALID); CU_ASSERT(odp_packet_l3_offset(pkt) == ODP_PACKET_OFFSET_INVALID); CU_ASSERT(odp_packet_l4_offset(pkt) == ODP_PACKET_OFFSET_INVALID); CU_ASSERT(odp_packet_payload_offset(pkt) == ODP_PACKET_OFFSET_INVALID); - packet_set_inflags(pkt, 1); - packet_check_inflags(pkt, 1); + packet_set_inflags_common(pkt, 1); + packet_check_inflags_common(pkt, 1); odp_packet_input_set(pkt, pktio); odp_packet_user_ptr_set(pkt, (void *)(uintptr_t)0xdeadbeef); @@ -3098,17 +3137,31 @@ static void packet_vector_test_alloc_free(void) CU_ASSERT(odp_packet_vector_to_u64(pktv) != odp_packet_vector_to_u64(ODP_PACKET_VECTOR_INVALID)); - /* User flag should be initially zero */ + /* Vector size and user flag should be initially zero */ + CU_ASSERT(odp_packet_vector_size(pktv) == 0); CU_ASSERT(odp_packet_vector_user_flag(pktv) == 0); odp_packet_vector_user_flag_set(pktv, 1); CU_ASSERT(odp_packet_vector_user_flag(pktv) != 0); odp_packet_vector_user_flag_set(pktv, 0); CU_ASSERT(odp_packet_vector_user_flag(pktv) == 0); + /* Included packet should not be freed by odp_packet_vector_free() */ + pkt = odp_packet_alloc(default_pool, default_param.pkt.len); + CU_ASSERT_FATAL(pkt != ODP_PACKET_INVALID); + + CU_ASSERT(odp_packet_vector_tbl(pktv, &pkts_tbl) == 0); + pkts_tbl[0] = pkt; + odp_packet_vector_size_set(pktv, 1); + /* Free with flag still set, alloc should clear it. */ odp_packet_vector_user_flag_set(pktv, 1); odp_packet_vector_free(pktv); + + /* Check that included packet is still valid */ + CU_ASSERT(odp_packet_is_valid(pkt)); + pktv = odp_packet_vector_alloc(pool); + CU_ASSERT(odp_packet_vector_size(pktv) == 0); CU_ASSERT(odp_packet_vector_user_flag(pktv) == 0); /* Since it was only one buffer pool, more vector packets can't be @@ -3124,10 +3177,7 @@ static void packet_vector_test_alloc_free(void) CU_ASSERT_FATAL(pktv != ODP_PACKET_VECTOR_INVALID); CU_ASSERT(odp_packet_vector_size(pktv) == 0); - /* Free packet vector using odp_event_free() */ - pkt = odp_packet_alloc(default_pool, default_param.pkt.len); - CU_ASSERT_FATAL(pkt != ODP_PACKET_INVALID); - + /* Free packet vector and included packet using odp_event_free() */ CU_ASSERT(odp_packet_vector_tbl(pktv, &pkts_tbl) == 0); pkts_tbl[0] = pkt; odp_packet_vector_size_set(pktv, 1); @@ -4416,6 +4466,7 @@ odp_testinfo_t packet_suite[] = { ODP_TEST_INFO(packet_test_layer_offsets), ODP_TEST_INFO(packet_test_segment_last), ODP_TEST_INFO(packet_test_in_flags), + ODP_TEST_INFO(packet_test_vlan_flags), ODP_TEST_INFO(packet_test_error_flags), ODP_TEST_INFO(packet_test_add_rem_data), ODP_TEST_INFO(packet_test_meta_data_copy), diff --git a/test/validation/api/pool/pool.c b/test/validation/api/pool/pool.c index e8bf38c9d..cded05650 100644 --- a/test/validation/api/pool/pool.c +++ b/test/validation/api/pool/pool.c @@ -1,6 +1,6 @@ /* Copyright (c) 2014-2018, Linaro Limited * Copyright (c) 2020, Marvell - * Copyright (c) 2020-2022, Nokia + * Copyright (c) 2020-2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -326,13 +326,19 @@ static void alloc_packet_vector(uint32_t cache_size) num = 0; for (i = 0; i < max_num; i++) { - pkt_vec[num] = odp_packet_vector_alloc(pool); - CU_ASSERT(pkt_vec[num] != ODP_PACKET_VECTOR_INVALID); - CU_ASSERT(odp_packet_vector_valid(pkt_vec[num]) == 1); - CU_ASSERT(odp_event_is_valid(odp_packet_vector_to_event(pkt_vec[num])) == 1); + odp_packet_vector_t pktv = odp_packet_vector_alloc(pool); - if (pkt_vec[num] != ODP_PACKET_VECTOR_INVALID) - num++; + CU_ASSERT(pktv != ODP_PACKET_VECTOR_INVALID); + + if (pktv == ODP_PACKET_VECTOR_INVALID) + continue; + + CU_ASSERT(odp_packet_vector_valid(pktv) == 1); + CU_ASSERT(odp_event_is_valid(odp_packet_vector_to_event(pktv)) == 1); + CU_ASSERT(odp_packet_vector_size(pktv) == 0); + + pkt_vec[num] = pktv; + num++; } for (i = 0; i < num; i++) diff --git a/test/validation/api/system/system.c b/test/validation/api/system/system.c index f0cf48738..a98ebb634 100644 --- a/test/validation/api/system/system.c +++ b/test/validation/api/system/system.c @@ -135,6 +135,7 @@ static void system_test_cpu_cycles_resolution(void) int i; uint64_t res; uint64_t c2, c1, max; + uint64_t test_cycles = odp_cpu_hz() / 100; /* CPU cycles in 10 msec */ max = odp_cpu_cycles_max(); @@ -144,11 +145,16 @@ static void system_test_cpu_cycles_resolution(void) for (i = 0; i < RES_TRY_NUM; i++) { c1 = odp_cpu_cycles(); - odp_time_wait_ns(100 * ODP_TIME_MSEC_IN_NS + i); + odp_time_wait_ns(10 * ODP_TIME_MSEC_IN_NS + i); c2 = odp_cpu_cycles(); - CU_ASSERT(c1 % res == 0); - CU_ASSERT(c2 % res == 0); + /* Diff may be zero with low resolution */ + if (test_cycles && test_cycles > res) { + uint64_t diff = odp_cpu_cycles_diff(c2, c1); + + CU_ASSERT(diff >= res); + } + } } @@ -185,13 +191,11 @@ static void system_test_cpu_cycles_diff(void) tmp = c2 + (max - c1) + res; diff = odp_cpu_cycles_diff(c2, c1); CU_ASSERT(diff == tmp); - CU_ASSERT(diff % res == 0); /* no wrap, revert args */ tmp = c1 - c2; diff = odp_cpu_cycles_diff(c1, c2); CU_ASSERT(diff == tmp); - CU_ASSERT(diff % res == 0); } static void system_test_cpu_cycles_long_period(void) @@ -221,8 +225,6 @@ static void system_test_cpu_cycles_long_period(void) c2 = odp_cpu_cycles(); CU_ASSERT(c2 != c1); - CU_ASSERT(c1 % res == 0); - CU_ASSERT(c2 % res == 0); CU_ASSERT(c1 <= max && c2 <= max); if (c2 > c1) @@ -232,7 +234,6 @@ static void system_test_cpu_cycles_long_period(void) diff = odp_cpu_cycles_diff(c2, c1); CU_ASSERT(diff == tmp); - CU_ASSERT(diff % res == 0); /* wrap is detected and verified */ if (c2 < c1) diff --git a/test/validation/api/time/time.c b/test/validation/api/time/time.c index 45bfc8264..f285bf8e4 100644 --- a/test/validation/api/time/time.c +++ b/test/validation/api/time/time.c @@ -139,6 +139,7 @@ static void time_test_monotony(void) lns_t2 = odp_time_local_ns(); gns_t2 = odp_time_global_ns(); + count = 0; while (count < BUSY_LOOP_CNT) { count++; }; diff --git a/test/validation/api/timer/timer.c b/test/validation/api/timer/timer.c index 1c0a16349..0e3919b73 100644 --- a/test/validation/api/timer/timer.c +++ b/test/validation/api/timer/timer.c @@ -329,8 +329,10 @@ static void timer_test_capa(void) CU_ASSERT_FATAL(ODP_CLOCK_SRC_0 + 4 == ODP_CLOCK_SRC_4); CU_ASSERT_FATAL(ODP_CLOCK_SRC_0 + 5 == ODP_CLOCK_SRC_5); CU_ASSERT_FATAL(ODP_CLOCK_SRC_5 + 1 == ODP_CLOCK_NUM_SRC); +#if ODP_DEPRECATED_API CU_ASSERT_FATAL(ODP_CLOCK_CPU == ODP_CLOCK_DEFAULT); CU_ASSERT_FATAL(ODP_CLOCK_EXT == ODP_CLOCK_SRC_1); +#endif for (i = 0; i < ODP_CLOCK_NUM_SRC; i++) { clk_src = ODP_CLOCK_SRC_0 + i; @@ -666,6 +668,7 @@ static void timer_pool_max_res(void) odp_queue_param_t queue_param; odp_timer_pool_t tp; odp_timer_t timer; + odp_timer_start_t start_param; odp_pool_param_t pool_param; odp_pool_t pool; odp_queue_t queue; @@ -729,7 +732,11 @@ static void timer_pool_max_res(void) ev = odp_timeout_to_event(tmo); CU_ASSERT_FATAL(ev != ODP_EVENT_INVALID); - ret = odp_timer_set_rel(timer, tick, &ev); + start_param.tick_type = ODP_TIMER_TICK_REL; + start_param.tick = tick; + start_param.tmo_ev = ev; + + ret = odp_timer_start(timer, &start_param); CU_ASSERT(ret == ODP_TIMER_SUCCESS); ev = ODP_EVENT_INVALID; @@ -1145,6 +1152,7 @@ static void timer_test_event_type(odp_queue_type_t queue_type, odp_queue_param_t queue_param; odp_timer_pool_param_t timer_param; odp_timer_pool_t timer_pool; + odp_timer_start_t start_param; odp_queue_t queue; odp_timeout_t tmo; odp_buffer_t buf; @@ -1231,7 +1239,11 @@ static void timer_test_event_type(odp_queue_type_t queue_type, timer[i] = odp_timer_alloc(timer_pool, queue, user_ctx); CU_ASSERT_FATAL(timer[i] != ODP_TIMER_INVALID); - ret = odp_timer_set_rel(timer[i], (i + 1) * period_tick, &ev); + start_param.tick_type = ODP_TIMER_TICK_REL; + start_param.tick = (i + 1) * period_tick; + start_param.tmo_ev = ev; + + ret = odp_timer_start(timer[i], &start_param); if (ret == ODP_TIMER_TOO_NEAR) ODPH_DBG("Timer set failed. Too near %i.\n", i); @@ -1332,6 +1344,7 @@ static void timer_test_queue_type(odp_queue_type_t queue_type, int priv, int exp odp_queue_param_t queue_param; odp_timer_pool_param_t tparam; odp_timer_pool_t tp; + odp_timer_start_t start_param; odp_queue_t queue; odp_timer_t tim; int i, ret, num_tmo; @@ -1411,7 +1424,12 @@ static void timer_test_queue_type(odp_queue_type_t queue_type, int priv, int exp timer[i] = tim; tick = tick_base + ((i + 1) * period_tick); - ret = odp_timer_set_abs(tim, tick, &ev); + + start_param.tick_type = ODP_TIMER_TICK_ABS; + start_param.tick = tick; + start_param.tmo_ev = ev; + + ret = odp_timer_start(tim, &start_param); target_tick[i] = tick; target_nsec[i] = nsec_base + ((i + 1) * period_ns); @@ -1541,12 +1559,12 @@ static void timer_test_cancel(void) odp_queue_param_t queue_param; odp_timer_capability_t capa; odp_timer_pool_t tp; + odp_timer_start_t start_param; odp_queue_t queue; odp_timer_t tim; odp_event_t ev; odp_timeout_t tmo; odp_timer_set_t rc; - uint64_t tick; int ret; memset(&capa, 0, sizeof(capa)); @@ -1597,9 +1615,11 @@ static void timer_test_cancel(void) if (ev == ODP_EVENT_INVALID) CU_FAIL_FATAL("Failed to allocate timeout"); - tick = odp_timer_ns_to_tick(tp, tparam.max_tmo / 2); + start_param.tick_type = ODP_TIMER_TICK_REL; + start_param.tick = odp_timer_ns_to_tick(tp, tparam.max_tmo / 2); + start_param.tmo_ev = ev; - rc = odp_timer_set_rel(tim, tick, &ev); + rc = odp_timer_start(tim, &start_param); if (rc != ODP_TIMER_SUCCESS) CU_FAIL_FATAL("Failed to set timer (relative time)"); @@ -1645,6 +1665,7 @@ static void timer_test_tmo_limit(odp_queue_type_t queue_type, odp_queue_param_t queue_param; odp_timer_pool_param_t timer_param; odp_timer_pool_t timer_pool; + odp_timer_start_t start_param; odp_queue_t queue; odp_timeout_t tmo; odp_event_t ev; @@ -1731,7 +1752,12 @@ static void timer_test_tmo_limit(odp_queue_type_t queue_type, CU_ASSERT(ev != ODP_EVENT_INVALID); t1 = odp_time_local(); - ret = odp_timer_set_rel(timer[i], tmo_tick, &ev); + + start_param.tick_type = ODP_TIMER_TICK_REL; + start_param.tick = tmo_tick; + start_param.tmo_ev = ev; + + ret = odp_timer_start(timer[i], &start_param); if (ret == ODP_TIMER_TOO_NEAR) ODPH_DBG("Timer set failed. Too near %i.\n", i); @@ -1943,6 +1969,7 @@ static int worker_entrypoint(void *arg ODP_UNUSED) struct timespec ts; uint32_t nstale; odp_timer_set_t timer_rc; + odp_timer_start_t start_param; odp_timer_pool_t tp = global_mem->tp; odp_pool_t tbp = global_mem->tbp; uint32_t num_timers = global_mem->timers_per_thread; @@ -2008,7 +2035,13 @@ static int worker_entrypoint(void *arg ODP_UNUSED) (rand_r(&seed) % RANGE_MS) * 1000000ULL; tck = odp_timer_current_tick(tp) + odp_timer_ns_to_tick(tp, nsec); - timer_rc = odp_timer_set_abs(tt[i].tim, tck, &tt[i].ev); + + start_param.tick_type = ODP_TIMER_TICK_ABS; + start_param.tick = tck; + start_param.tmo_ev = tt[i].ev; + tt[i].ev = ODP_EVENT_INVALID; + + timer_rc = odp_timer_start(tt[i].tim, &start_param); if (timer_rc == ODP_TIMER_TOO_NEAR) { ODPH_ERR("Missed tick, setting timer\n"); } else if (timer_rc != ODP_TIMER_SUCCESS) { @@ -2058,23 +2091,34 @@ static int worker_entrypoint(void *arg ODP_UNUSED) odp_timer_set_t rc; uint64_t cur_tick; uint64_t tck; + int reset_timer = 0; - if (tt[i].ev != ODP_EVENT_INVALID) + if (tt[i].ev != ODP_EVENT_INVALID) { /* Timer inactive => set */ nset++; - else if (odp_timer_current_tick(tp) >= tt[i].tick) + } else if (odp_timer_current_tick(tp) >= tt[i].tick) { /* Timer just expired. */ goto sleep; - else + } else { /* Timer active => reset */ nreset++; + reset_timer = 1; + } nsec = min_tmo + THREE_POINT_THREE_MSEC + (rand_r(&seed) % RANGE_MS) * 1000000ULL; tck = odp_timer_ns_to_tick(tp, nsec); cur_tick = odp_timer_current_tick(tp); - rc = odp_timer_set_rel(tt[i].tim, tck, &tt[i].ev); + + start_param.tick_type = ODP_TIMER_TICK_REL; + start_param.tick = tck; + start_param.tmo_ev = tt[i].ev; + + if (reset_timer) + rc = odp_timer_restart(tt[i].tim, &start_param); + else + rc = odp_timer_start(tt[i].tim, &start_param); if (rc == ODP_TIMER_TOO_NEAR) { CU_FAIL("Failed to set timer: TOO NEAR"); |