aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorMatias Elo <matias.elo@nokia.com>2022-05-04 15:40:33 +0300
committerMatias Elo <matias.elo@nokia.com>2022-05-04 15:40:33 +0300
commit2f0df4cb4acdee52f436cad8476ccf662821b7b5 (patch)
tree085136f3d98d8187b17ea584c992c8117bdedd8f
parent2f869d2698349bf75ec2404d4ab7d7f3b5b4c88b (diff)
parente361227ac00851e8720871e927b015c4c0e6a895 (diff)
Merge tag 'v1.36.0.0' of https://github.com/OpenDataPlane/odp into odp-dpdk
Signed-off-by: Matias Elo <matias.elo@nokia.com>
-rw-r--r--.github/workflows/ci-pipeline-arm64.yml32
-rw-r--r--.github/workflows/ci-pipeline.yml41
-rw-r--r--CHANGELOG33
-rw-r--r--DEPENDENCIES13
-rw-r--r--configure.ac53
-rw-r--r--doc/users-guide/users-guide-crypto.adoc132
-rw-r--r--doc/users-guide/users-guide-pktio.adoc2
-rw-r--r--example/generator/odp_generator.c10
-rw-r--r--example/ipfragreass/odp_ipfragreass_reassemble.c3
-rwxr-xr-xexample/ipsec_api/odp_ipsec_api_run_esp_tun_out.sh1
-rw-r--r--example/ipsec_crypto/odp_ipsec_stream.c6
-rw-r--r--helper/include/odp/helper/cli.h2
-rw-r--r--helper/include/odp/helper/linux/process.h1
-rw-r--r--helper/include/odp/helper/linux/pthread.h1
-rw-r--r--helper/include/odp/helper/threads.h1
-rw-r--r--include/Makefile.am16
-rw-r--r--include/odp/api/abi-default/event.h38
-rw-r--r--include/odp/api/abi-default/event_types.h58
-rw-r--r--include/odp/api/abi-default/timer.h20
-rw-r--r--include/odp/api/abi-default/traffic_mngr.h5
-rw-r--r--include/odp/api/buffer.h2
-rw-r--r--include/odp/api/comp.h2
-rw-r--r--include/odp/api/dma_types.h3
-rw-r--r--include/odp/api/event_types.h28
-rw-r--r--include/odp/api/packet.h6
-rw-r--r--include/odp/api/queue.h5
-rw-r--r--include/odp/api/spec/buffer.h2
-rw-r--r--include/odp/api/spec/classification.h133
-rw-r--r--include/odp/api/spec/comp.h5
-rw-r--r--include/odp/api/spec/crypto.h39
-rw-r--r--include/odp/api/spec/dma_types.h2
-rw-r--r--include/odp/api/spec/event.h79
-rw-r--r--include/odp/api/spec/event_types.h110
-rw-r--r--include/odp/api/spec/packet.h7
-rw-r--r--include/odp/api/spec/packet_flags.h2
-rw-r--r--include/odp/api/spec/packet_io.h5
-rw-r--r--include/odp/api/spec/packet_io_types.h8
-rw-r--r--include/odp/api/spec/pool_types.h10
-rw-r--r--include/odp/api/spec/queue.h3
-rw-r--r--include/odp/api/spec/schedule.h2
-rw-r--r--include/odp/api/spec/shared_memory.h2
-rw-r--r--include/odp/api/spec/timer.h2
-rw-r--r--include/odp/api/spec/timer_types.h2
-rw-r--r--include/odp/api/spec/traffic_mngr.h249
-rw-r--r--include/odp/api/timer.h2
-rw-r--r--include/odp/arch/arm32-linux/odp/api/abi/event_types.h7
-rw-r--r--include/odp/arch/arm32-linux/odp/api/abi/timer.h7
-rw-r--r--include/odp/arch/arm64-linux/odp/api/abi/event_types.h7
-rw-r--r--include/odp/arch/arm64-linux/odp/api/abi/timer.h7
-rw-r--r--include/odp/arch/default-linux/odp/api/abi/event_types.h7
-rw-r--r--include/odp/arch/default-linux/odp/api/abi/timer.h7
-rw-r--r--include/odp/arch/power64-linux/odp/api/abi/event_types.h7
-rw-r--r--include/odp/arch/power64-linux/odp/api/abi/timer.h7
-rw-r--r--include/odp/arch/x86_32-linux/odp/api/abi/event_types.h7
-rw-r--r--include/odp/arch/x86_32-linux/odp/api/abi/timer.h7
-rw-r--r--include/odp/arch/x86_64-linux/odp/api/abi/event_types.h7
-rw-r--r--include/odp/arch/x86_64-linux/odp/api/abi/timer.h7
-rw-r--r--platform/linux-generic/Makefile.am12
-rw-r--r--platform/linux-generic/arch/aarch64/odp_atomic.h95
-rw-r--r--platform/linux-generic/arch/aarch64/odp_crypto_armv8.c265
-rw-r--r--platform/linux-generic/arch/aarch64/odp_llsc.h8
-rw-r--r--platform/linux-generic/arch/arm/odp_atomic.h3
-rw-r--r--platform/linux-generic/arch/default/odp_atomic.h14
-rw-r--r--platform/linux-generic/include-abi/odp/api/abi/event.h34
-rw-r--r--platform/linux-generic/include-abi/odp/api/abi/event_types.h61
-rw-r--r--platform/linux-generic/include-abi/odp/api/abi/timer.h8
-rw-r--r--platform/linux-generic/include/odp/api/plat/buffer_inlines.h2
-rw-r--r--platform/linux-generic/include/odp/api/plat/packet_flag_inlines.h195
-rw-r--r--platform/linux-generic/include/odp/api/plat/packet_inline_types.h7
-rw-r--r--platform/linux-generic/include/odp/api/plat/packet_inlines.h32
-rw-r--r--platform/linux-generic/include/odp/api/plat/packet_vector_inlines.h2
-rw-r--r--platform/linux-generic/include/odp/api/plat/queue_inline_types.h1
-rw-r--r--platform/linux-generic/include/odp/api/plat/time_inlines.h12
-rw-r--r--platform/linux-generic/include/odp/api/plat/timer_inline_types.h37
-rw-r--r--platform/linux-generic/include/odp/api/plat/timer_inlines.h78
-rw-r--r--platform/linux-generic/include/odp_atomic_internal.h13
-rw-r--r--platform/linux-generic/include/odp_classification_datamodel.h4
-rw-r--r--platform/linux-generic/include/odp_classification_internal.h7
-rw-r--r--platform/linux-generic/include/odp_llqueue.h3
-rw-r--r--platform/linux-generic/include/odp_macros_internal.h2
-rw-r--r--platform/linux-generic/include/odp_packet_dpdk.h14
-rw-r--r--platform/linux-generic/include/odp_packet_internal.h48
-rw-r--r--platform/linux-generic/include/odp_packet_io_internal.h18
-rw-r--r--platform/linux-generic/include/odp_parse_internal.h112
-rw-r--r--platform/linux-generic/include/odp_schedule_if.h2
-rw-r--r--platform/linux-generic/include/odp_types_internal.h24
-rw-r--r--platform/linux-generic/odp_classification.c119
-rw-r--r--platform/linux-generic/odp_crypto_null.c7
-rw-r--r--platform/linux-generic/odp_crypto_openssl.c10
-rw-r--r--platform/linux-generic/odp_dma.c1
-rw-r--r--platform/linux-generic/odp_event.c5
-rw-r--r--platform/linux-generic/odp_ipsec.c7
-rw-r--r--platform/linux-generic/odp_ipsec_sad.c5
-rw-r--r--platform/linux-generic/odp_packet.c568
-rw-r--r--platform/linux-generic/odp_packet_flags.c138
-rw-r--r--platform/linux-generic/odp_packet_io.c122
-rw-r--r--platform/linux-generic/odp_parse.c471
-rw-r--r--platform/linux-generic/odp_pool.c13
-rw-r--r--platform/linux-generic/odp_queue_lf.c17
-rw-r--r--platform/linux-generic/odp_schedule_scalable.c16
-rw-r--r--platform/linux-generic/odp_schedule_sp.c1
-rw-r--r--platform/linux-generic/odp_time.c9
-rw-r--r--platform/linux-generic/odp_timer.c127
-rw-r--r--platform/linux-generic/odp_timer_api.c11
-rw-r--r--platform/linux-generic/odp_traffic_mngr.c123
-rw-r--r--platform/linux-generic/pktio/dpdk.c254
-rw-r--r--platform/linux-generic/pktio/dpdk_parse.c28
-rw-r--r--platform/linux-generic/pktio/loop.c96
-rw-r--r--platform/linux-generic/pktio/netmap.c53
-rw-r--r--platform/linux-generic/pktio/null.c4
-rw-r--r--platform/linux-generic/pktio/pcap.c69
-rw-r--r--platform/linux-generic/pktio/pktio_common.c12
-rw-r--r--platform/linux-generic/pktio/socket.c67
-rw-r--r--platform/linux-generic/pktio/socket_mmap.c44
-rw-r--r--platform/linux-generic/pktio/tap.c42
-rwxr-xr-xscripts/ci/build_arm64.sh29
-rw-r--r--test/performance/odp_bench_packet.c6
-rw-r--r--test/performance/odp_crypto.c1
-rw-r--r--test/performance/odp_ipsec.c415
-rwxr-xr-xtest/performance/odp_ipsec_run.sh4
-rw-r--r--test/performance/odp_pktio_ordered.c2
-rw-r--r--test/performance/odp_random.c445
-rw-r--r--test/performance/odp_sched_pktio.c3
-rw-r--r--test/validation/api/classification/classification.h7
-rw-r--r--test/validation/api/classification/odp_classification_basic.c45
-rw-r--r--test/validation/api/classification/odp_classification_tests.c76
-rw-r--r--test/validation/api/classification/odp_classification_testsuites.h3
-rw-r--r--test/validation/api/crypto/odp_crypto_test_inp.c14
-rw-r--r--test/validation/api/ipsec/ipsec.c9
-rw-r--r--test/validation/api/pktio/pktio.c11
-rw-r--r--test/validation/api/traffic_mngr/traffic_mngr.c211
131 files changed, 3720 insertions, 2375 deletions
diff --git a/.github/workflows/ci-pipeline-arm64.yml b/.github/workflows/ci-pipeline-arm64.yml
index 6f32208dc..cacdb7a0d 100644
--- a/.github/workflows/ci-pipeline-arm64.yml
+++ b/.github/workflows/ci-pipeline-arm64.yml
@@ -31,6 +31,9 @@ jobs:
- uses: actions/checkout@v2
- run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${{matrix.cc}}"
-e CONF="${{matrix.conf}}" $CONTAINER_NAMESPACE/odp-ci-${OS}-${ARCH}-native /odp/scripts/ci/build_${ARCH}.sh
+ - name: Failure log
+ if: ${{ failure() }}
+ run: find . -name config.log -exec cat {} \;
Build_OS:
if: ${{ github.repository == 'OpenDataPlane/odp-dpdk' }}
@@ -39,12 +42,15 @@ jobs:
fail-fast: false
matrix:
cc: [gcc, clang]
- os: ['ubuntu_16.04', 'ubuntu_18.04']
+ os: ['ubuntu_18.04', 'rocky_linux_8']
steps:
- uses: AutoModality/action-clean@v1.1.0
- uses: actions/checkout@v2
- run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${{matrix.cc}}"
-e CONF="${CONF}" $CONTAINER_NAMESPACE/odp-ci-${{matrix.os}}-${ARCH}-native /odp/scripts/ci/build_${ARCH}.sh
+ - name: Failure log
+ if: ${{ failure() }}
+ run: find . -name config.log -exec cat {} \;
Build_gcc-10:
if: ${{ github.repository == 'OpenDataPlane/odp-dpdk' }}
@@ -60,6 +66,9 @@ jobs:
- uses: actions/checkout@v2
- run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${CC}"
-e CONF="${{matrix.conf}}" $CONTAINER_NAMESPACE/odp-ci-${OS}-${ARCH}-native /odp/scripts/ci/build_${ARCH}.sh
+ - name: Failure log
+ if: ${{ failure() }}
+ run: find . -name config.log -exec cat {} \;
Build_out-of-tree:
if: ${{ github.repository == 'OpenDataPlane/odp-dpdk' }}
@@ -69,6 +78,9 @@ jobs:
- uses: actions/checkout@v2
- run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${CC}"
-e CONF="${CONF}" $CONTAINER_NAMESPACE/odp-ci-${OS}-${ARCH}-native /odp/scripts/ci/out_of_tree.sh
+ - name: Failure log
+ if: ${{ failure() }}
+ run: find . -name config.log -exec cat {} \;
Run_distcheck:
if: ${{ github.repository == 'OpenDataPlane/odp-dpdk' }}
@@ -130,11 +142,11 @@ jobs:
strategy:
fail-fast: false
matrix:
- os: ['ubuntu_18.04']
+ os: ['ubuntu_18.04', 'ubuntu_22.04']
steps:
- uses: AutoModality/action-clean@v1.1.0
- uses: actions/checkout@v2
- - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${{matrix.cc}}" -e ARCH="${ARCH}"
+ - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${CC}" -e ARCH="${ARCH}"
-e CONF="${CONF}" $CONTAINER_NAMESPACE/odp-ci-${{matrix.os}}-${ARCH}-native /odp/scripts/ci/check.sh
- name: Failure log
if: ${{ failure() }}
@@ -177,3 +189,17 @@ jobs:
- name: Failure log
if: ${{ failure() }}
run: find . -name "*.trs" | xargs grep -l '^.test-result. FAIL' | while read trs ; do echo FAILURE detected at $trs; cat ${trs%%.trs}.log ; done
+
+ Run_dpdk-21_11:
+ if: ${{ github.repository == 'OpenDataPlane/odp' }}
+ runs-on: [self-hosted, ARM64]
+ env:
+ OS: ubuntu_20.04
+ steps:
+ - uses: AutoModality/action-clean@v1.1.0
+ - uses: actions/checkout@v2
+ - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${CC}" -e ARCH="${ARCH}"
+ -e CONF="${CONF}" $CONTAINER_NAMESPACE/odp-ci-${OS}-${ARCH}-native-dpdk_21.11 /odp/scripts/ci/check.sh
+ - name: Failure log
+ if: ${{ failure() }}
+ run: find . -name "*.trs" | xargs grep -l '^.test-result. FAIL' | while read trs ; do echo FAILURE detected at $trs; cat ${trs%%.trs}.log ; done
diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml
index 56dd65fcc..102ad0213 100644
--- a/.github/workflows/ci-pipeline.yml
+++ b/.github/workflows/ci-pipeline.yml
@@ -78,6 +78,9 @@ jobs:
- uses: actions/checkout@v2
- run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${{matrix.cc}}"
-e CONF="${{matrix.conf}}" $CONTAINER_NAMESPACE/odp-ci-${OS}-${ARCH} /odp/scripts/ci/build_${ARCH}.sh
+ - name: Failure log
+ if: ${{ failure() }}
+ run: find . -name config.log -exec cat {} \;
Build_arm64:
runs-on: ubuntu-18.04
@@ -118,6 +121,9 @@ jobs:
OS: ubuntu_20.04
run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${{matrix.compiler}}"
-e CONF="${CONF}" $CONTAINER_NAMESPACE/odp-ci-${OS}-${ARCH} /odp/scripts/ci/build_${ARCH}.sh
+ - name: Failure log
+ if: ${{ failure() }}
+ run: find . -name config.log -exec cat {} \;
Build_ppc64el:
runs-on: ubuntu-18.04
@@ -126,12 +132,14 @@ jobs:
strategy:
fail-fast: false
matrix:
- cc: [gcc, clang]
conf: ['', '--enable-abi-compat']
steps:
- uses: actions/checkout@v2
- run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${CC}"
-e CONF="${{matrix.conf}}" $CONTAINER_NAMESPACE/odp-ci-${OS}-${ARCH} /odp/scripts/ci/build_${ARCH}.sh
+ - name: Failure log
+ if: ${{ failure() }}
+ run: find . -name config.log -exec cat {} \;
Build_i386:
runs-on: ubuntu-18.04
@@ -146,6 +154,9 @@ jobs:
- uses: actions/checkout@v2
- run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${{matrix.cc}}"
-e CONF="${{matrix.conf}}" $CONTAINER_NAMESPACE/odp-ci-${OS}-${ARCH} /odp/scripts/ci/build_${ARCH}.sh
+ - name: Failure log
+ if: ${{ failure() }}
+ run: find . -name config.log -exec cat {} \;
Build_OS:
runs-on: ubuntu-18.04
@@ -153,12 +164,15 @@ jobs:
fail-fast: false
matrix:
cc: [gcc, clang]
- os: ['centos_7', 'centos_8', 'ubuntu_16.04']
+ os: ['centos_7', 'rocky_linux_8']
conf: ['--enable-abi-compat']
steps:
- uses: actions/checkout@v2
- run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${{matrix.cc}}"
-e CONF="${{matrix.conf}}" $CONTAINER_NAMESPACE/odp-ci-${{matrix.os}}-${ARCH} /odp/scripts/ci/build_${ARCH}.sh
+ - name: Failure log
+ if: ${{ failure() }}
+ run: find . -name config.log -exec cat {} \;
Build_gcc-10:
runs-on: ubuntu-18.04
@@ -173,6 +187,9 @@ jobs:
- uses: actions/checkout@v2
- run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${CC}" -e CXX=g++-10
-e CONF="${{matrix.conf}}" $CONTAINER_NAMESPACE/odp-ci-${OS}-${ARCH} /odp/scripts/ci/build_${ARCH}.sh
+ - name: Failure log
+ if: ${{ failure() }}
+ run: find . -name config.log -exec cat {} \;
Build_out-of-tree:
runs-on: ubuntu-18.04
@@ -180,6 +197,9 @@ jobs:
- uses: actions/checkout@v2
- run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${CC}"
-e CONF="${CONF}" $CONTAINER_NAMESPACE/odp-ci-${OS}-${ARCH} /odp/scripts/ci/out_of_tree.sh
+ - name: Failure log
+ if: ${{ failure() }}
+ run: find . -name config.log -exec cat {} \;
Build_linux-generic:
runs-on: ubuntu-18.04
@@ -189,6 +209,9 @@ jobs:
- uses: actions/checkout@v2
- run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${CC}" -e ODP_LIB_NAME="libodp-linux"
-e CONF="${CONF}" $CONTAINER_NAMESPACE/odp-ci-${OS}-${ARCH} /odp/scripts/ci/build_${ARCH}.sh
+ - name: Failure log
+ if: ${{ failure() }}
+ run: find . -name config.log -exec cat {} \;
Run_coverage:
runs-on: ubuntu-18.04
@@ -241,7 +264,7 @@ jobs:
fail-fast: false
matrix:
cc: [gcc, clang]
- os: ['ubuntu_20.04']
+ os: ['ubuntu_20.04', 'ubuntu_22.04']
steps:
- uses: actions/checkout@v2
- run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${{matrix.cc}}" -e ARCH="${ARCH}"
@@ -303,6 +326,18 @@ jobs:
if: ${{ failure() }}
run: find . -name "*.trs" | xargs grep -l '^.test-result. FAIL' | while read trs ; do echo FAILURE detected at $trs; cat ${trs%%.trs}.log ; done
+ Run_dpdk-21_11:
+ runs-on: ubuntu-18.04
+ env:
+ OS: ubuntu_20.04
+ steps:
+ - uses: actions/checkout@v2
+ - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${CC}" -e ARCH="${ARCH}"
+ -e CONF="${CONF}" $CONTAINER_NAMESPACE/odp-ci-${OS}-${ARCH}-dpdk_21.11 /odp/scripts/ci/check.sh
+ - name: Failure log
+ if: ${{ failure() }}
+ run: find . -name "*.trs" | xargs grep -l '^.test-result. FAIL' | while read trs ; do echo FAILURE detected at $trs; cat ${trs%%.trs}.log ; done
+
Run_crypto:
runs-on: ubuntu-18.04
env:
diff --git a/CHANGELOG b/CHANGELOG
index e5624a903..e182f0ad5 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -1,3 +1,36 @@
+== OpenDataPlane (1.36.0.0)
+
+=== Backward incompatible API changes
+
+==== Classifier
+* Add an action parameter `odp_cos_action_t` to CoS parameters
+(`odp_cls_cos_param_t`). Action may be to enqueue or drop the packet classified
+to the CoS. The old methods of creating a drop CoS have been replaced by the
+new drop action.
+
+==== Crypto
+* Deprecate `odp_crypto_operation()`, the associated data structures, and the
+completion event. Use `odp_crypto_op()` or `odp_crypto_op_enq()` instead.
+
+==== Traffic Manager
+* Split `odp_tm_capabilities_t.tm_queue_threshold` capability into byte and
+packet modes.
+* Split `odp_tm_level_capabilities_t.tm_node_threshold` capability into byte and
+packet modes.
+
+=== Backward compatible API changes
+==== Classifier
+* Add CoS specific statistics counters (`odp_cls_cos_stats_t`) and
+matching capabilities (`odp_cls_stats_capability_t`). Statistics counters can be
+read with `odp_cls_cos_stats()`.
+
+==== Common
+* Convert `unsigned int` types to `uint32_t`.
+
+==== Traffic Manager
+* Remove unused TM shaper color enum `odp_tm_shaper_color_t` and
+`ODP_NUM_SHAPER_COLORS` define.
+
== OpenDataPlane (1.35.0.0)
=== Backward incompatible API changes
diff --git a/DEPENDENCIES b/DEPENDENCIES
index f319eb36e..85a4cf242 100644
--- a/DEPENDENCIES
+++ b/DEPENDENCIES
@@ -1,9 +1,8 @@
Prerequisites for building the OpenDataPlane (ODP) API
1. Linux
- CentOS 7 (kernel v3.10) and Ubuntu 16.04 (kernel v4.4) are the oldest
- Linux distributions tested by the ODP CI. Earlier versions may or may not
- work.
+ CentOS 7 (kernel v3.10) is the oldest Linux distributions tested by the ODP
+ CI. Earlier versions may or may not work.
For CentOS/RedHat distros, configure the system to use Fedora EPEL repos and
third-party packages:
@@ -206,8 +205,8 @@ Prerequisites for building the OpenDataPlane (ODP) API
3.5 DPDK packet I/O support (optional)
- Use DPDK for ODP packet I/O. Currently supported DPDK versions are v19.11
- (recommended) and v20.11.
+ Use DPDK for ODP packet I/O. Currently supported DPDK versions are v19.11,
+ v20.11 (recommended), v21.11.
Note: only packet I/O is accelerated with DPDK. See
https://github.com/OpenDataPlane/odp-dpdk.git
@@ -244,8 +243,8 @@ Prerequisites for building the OpenDataPlane (ODP) API
# Configure ODP
$ ./configure --with-dpdk-path=<dpdk-dir>
-3.5.4 Build DPDK v20.11 from source
- $ git clone https://dpdk.org/git/dpdk-stable --branch 20.11 --depth 1 ./<dpdk-dir>
+3.5.4 Build DPDK v20.11 and onwards from source
+ $ git clone https://dpdk.org/git/dpdk-stable --branch <version, e.g. 21.11> --depth 1 ./<dpdk-dir>
# Prepare the build directory
$ cd <dpdk-dir>
diff --git a/configure.ac b/configure.ac
index 1a3904d53..9a4cce34a 100644
--- a/configure.ac
+++ b/configure.ac
@@ -3,7 +3,7 @@ AC_PREREQ([2.5])
# ODP API version
##########################################################################
m4_define([odpapi_generation_version], [1])
-m4_define([odpapi_major_version], [35])
+m4_define([odpapi_major_version], [36])
m4_define([odpapi_minor_version], [0])
m4_define([odpapi_point_version], [0])
m4_define([odpapi_version],
@@ -90,57 +90,6 @@ AM_PROG_LIBTOOL
PKG_PROG_PKG_CONFIG
-# Checks for library functions.
-dnl breaks cross-compilation and malloc(0) behaviour is not that important
-dnl AC_FUNC_MALLOC
-AC_FUNC_MMAP
-AC_CHECK_FUNCS(m4_normalize([
- bzero
- clock_gettime
- gethostbyname
- getpagesize
- gettimeofday
- malloc
- memset
- munmap
- socket
- strchr
- strerror
- strrchr
- strstr
- strtoull
-]))
-
-# Checks for header files.
-AC_HEADER_RESOLV
-AC_CHECK_HEADERS(m4_normalize([
- arpa/inet.h
- fcntl.h
- inttypes.h
- limits.h
- netdb.h
- netinet/in.h
- stddef.h
- stdint.h
- stdlib.h
- string.h
- sys/ioctl.h
- sys/socket.h
- sys/time.h
- unistd.h
-]))
-
-# Checks for typedefs, structures, and compiler characteristics.
-AC_HEADER_STDBOOL
-AC_C_INLINE
-AC_TYPE_SIZE_T
-AC_TYPE_SSIZE_T
-AC_TYPE_UINT8_T
-AC_TYPE_UINT16_T
-AC_TYPE_INT32_T
-AC_TYPE_UINT32_T
-AC_TYPE_UINT64_T
-
##########################################################################
# Default warning setup
##########################################################################
diff --git a/doc/users-guide/users-guide-crypto.adoc b/doc/users-guide/users-guide-crypto.adoc
index 6393a5e78..0f33d6548 100644
--- a/doc/users-guide/users-guide-crypto.adoc
+++ b/doc/users-guide/users-guide-crypto.adoc
@@ -13,11 +13,6 @@ ODP provides APIs for following cryptographic services:
* Random number generation
* Crypto capability inquiries
-Ciphering and authentication services are accessible via two complementary
-sets of related APIs. The original ODP crypto APIs, and a newer
-_packet-oriented_ set of crypto APIs that are designed to be consistent with
-the protocol-aware cryptographic services offered by the IPsec API set.
-
=== Crypto Sessions
To apply a cryptographic operation to a packet a session must be created. All
@@ -49,112 +44,35 @@ with an `odp_crypto_session_t`.
=== Crypto operations
After session creation, a cryptographic operation can be applied to a packet
-in one of two ways.
-
-==== Parameter-based Crypto Operations
-This is the original ODP support for cryptographic operations. The
-`odp_crypto_operation()` API takes an input `odp_crypto_op_param_t` struct
-that describes the cryptographic operation to be performed. This struct
-contains the session to use as well as the input packet the operation is to be
-performed on. The caller may either specify an output packet to receive the
-operation results or may request that the ODP implementation allocate a new
-packet to receive these results from the output pool associated with the
-`odp_crypto_session_t`. If the input packet is also used as the output packet,
-then an "in place" operation is requested.
-
-When using the `odp_crypto_operation()` API. Applications may indicate a
-preference for synchronous or asynchronous processing in the session's
-`pref_mode` parameter. However crypto operations may complete synchronously
-even if an asynchronous preference is indicated, and applications must examine
-the `posted` output parameter from `odp_crypto_operation()` to determine
-whether the operation has completed or if an `ODP_EVENT_CRYPTO_COMPL`
-notification is expected. In the case of an async operation, the `posted`
-output parameter will be set to true.
-
-The operation arguments specify for each packet the areas that are to be
-encrypted or decrypted and authenticated. The parameters include also
-pointers to cipher and authentication initialization vectors as needed,
-depending on the initialization vector lengths specified at session creation.
-
-An operation can be executed in in-place, out-of-place or new buffer mode.
-In in-place mode output packet is same as the input packet.
-In case of out-of-place mode output packet is different from input packet as
-specified by the application, while in new buffer mode implementation allocates
-a new output buffer from the session’s output pool.
-
-The application can also specify a context associated with a given operation
-that will be retained during async operation and can be retrieved via the
-completion event.
-
-Results of an asynchronous session will be posted as completion events to the
-session’s completion queue, which can be accessed directly or via the ODP
-scheduler. The completion event contains the status of the operation and the
-result. The application has the responsibility to free the completion event.
-
-Upon receipt of an `ODP_EVENT_CRYPTO_COMPL` event, the
-`odp_crypto_compl_result()` API is used to retrieve the
-`odp_crypto_op_result_t` associated with the event. This result struct in turn
-contains:
-
-* An indication of the success or failure of the crypto operation
-* The user context associated with the event
-* The output `odp_packet_t`.
-* The `odp_crypto_op_status_t` for the requested cipher operation
-* The `odp_crypto_op_status_t` for the requested authentication operation
-
-==== Packet-based Crypto Operations
-To simplify the original cryptographic operation request API, as well as to
-be more flexible and consistent with the protocol-aware APIs introduced for
-IPsec support, a newer packet-oriented set of cryptographic operation
-APIs is also provided. Applications may use either API set, but going forward
-it is expected that these newer APIs will be the focus of continued
-development.
-
-Instead of a single `odp_crypto_operation()` API, the packet-based form
-provides two APIs: `odp_crypto_op()` is the synchronous form while
-`odp_crypto_op_enq()` is the asynchronous form. To check which of these are
-supported by the ODP implementation, examine the `sync_mode` and `async_mode`
-fields in the `odp_crypto_capability_t` struct returned by the
+synchronously or asynchronously. `odp_crypto_op()` is the synchronous API
+while `odp_crypto_op_enq()` is the asynchronous API. To check which of these
+are supported by the ODP implementation, examine the `sync_mode` and
+`async_mode` fields in the `odp_crypto_capability_t` struct returned by the
`odp_crypto_capability()` API.
Both forms take an input array of packets, an optional output array of packets
to receive the results, and an array of `odp_crypto_packet_op_param_t` structs
-that describe the operation to be performed on each input packet. As with the
-original APIs, the output array may be the same packets to request in-place
-operation, or may be specified as `ODP_PACKET_INVALID` to request that ODP
-allocate output packets from the pool associated with the
-`odp_crypto_session_t` being used.
-
-The key differences between the `odp_crypto_op_param_t` used by the original
-APIs and the `odp_crypto_packet_op_param_t` used by the new APIs are:
-
-* The original API takes a single `odp_crypto_op_param_t` since it operates on
-a single packet whereas the new forms take an array of
-`odp_crypto_packet_op_param_t` structs, one for each input packet.
-
-* The `odp_crypto_packet_op_param_t` does not contain any packet information
-since the input and output packets are supplied as API parameters rather than
-being encoded in this struct.
-
-* The `odp_crypto_packet_op_param_t` does not contain a user context field.
-
-In addition, the `odp_crypto_session_t` field `op_mode` is used instead of
-the `pref_mode` field when the packet-oriented APIs are used. If the
-`op_mode` is set to `ODP_CRYPTO_SYNC` then the synchronous form of the API
-must be used and if `op_mode` is set to `ODP_CRYPTO_ASYNC` then the
-asynchronous form of the API must be used. It is an error to attempt to use
-a form of the API not properly matched to the mode of the crypto session.
-
-The output of a packet-based crypto operation is an `odp_packet_t` (one for
-each input packet) that is returned either synchronously or
-asynchronously. Asynchronous return is in the form of `ODP_EVENT_PACKET`
-events that have event subtype `ODP_EVENT_PACKET_CRYPTO`. The packet
-associated with such events is obtained via the
-`odp_crypto_packet_from_event()` API. The `odp_crypto_result()` API, in turn,
-retrieves the `odp_crypto_packet_result_t` from this `odp_packet_t` that
-contains:
-
-* An indication of whether the crypto packet operation was successful or not
+that describe the operation to be performed on each input packet. The output
+array may be the same packets to request in-place operation, or may be
+specified as `ODP_PACKET_INVALID` to request that ODP allocate output packets
+from the pool associated with the `odp_crypto_session_t` being used.
+
+The op_mode field of `odp_crypto_session_t` indicates whether asynchronous
+or synchronous operations are used with the session. If `op_mode` is set
+to `ODP_CRYPTO_SYNC` then the synchronous API must be used and if `op_mode`
+is set to `ODP_CRYPTO_ASYNC` then the asynchronous API must be used. It is
+an error to use a form of the API that does not match the mode of the crypto
+session.
+
+The output of a crypto operation is an `odp_packet_t` (one for each input
+packet) that is returned either synchronously or asynchronously. Asynchronous
+return is in the form of `ODP_EVENT_PACKET` events that have event subtype
+`ODP_EVENT_PACKET_CRYPTO`. The packet associated with such events is obtained
+via the `odp_crypto_packet_from_event()` API. The `odp_crypto_result()` API,
+in turn, retrieves the `odp_crypto_packet_result_t` from this `odp_packet_t`
+that contains:
+
+* An indication of whether the crypto operation was successful or not
* The `odp_crypto_op_status_t` for the requested cipher operation
* The `odp_crypto_op_status_t` for the requested authentication operation
diff --git a/doc/users-guide/users-guide-pktio.adoc b/doc/users-guide/users-guide-pktio.adoc
index 43df729fa..1698060e9 100644
--- a/doc/users-guide/users-guide-pktio.adoc
+++ b/doc/users-guide/users-guide-pktio.adoc
@@ -468,7 +468,7 @@ specified independently at *open* time.
.PktIO Input Modes
* `ODP_PKTIN_MODE_DIRECT`
* `ODP_PKTIN_MODE_QUEUE`
-* `ODP_OKTIN_MODE_SCHED`
+* `ODP_PKTIN_MODE_SCHED`
* `ODP_PKTIN_MODE_DISABLED`
.PktIO Output Modes
diff --git a/example/generator/odp_generator.c b/example/generator/odp_generator.c
index 7a070235b..b980ee180 100644
--- a/example/generator/odp_generator.c
+++ b/example/generator/odp_generator.c
@@ -55,9 +55,9 @@ typedef struct {
odp_pktio_t pktio;
odp_pktio_config_t config;
odp_pktout_queue_t pktout[MAX_WORKERS];
- unsigned pktout_count;
+ uint32_t pktout_count;
odp_pktin_queue_t pktin[MAX_WORKERS];
- unsigned pktin_count;
+ uint32_t pktin_count;
} interface_t;
/**
@@ -537,8 +537,8 @@ static int setup_icmp_pkt(odp_packet_t pkt,
* @warning This routine aborts if the create is unsuccessful.
*/
static int create_pktio(const char *dev, odp_pool_t pool,
- unsigned num_rx_queues,
- unsigned num_tx_queues,
+ uint32_t num_rx_queues,
+ uint32_t num_tx_queues,
interface_t *itf)
{
odp_pktio_capability_t capa;
@@ -1104,7 +1104,7 @@ int main(int argc, char *argv[])
odph_thread_t thread_tbl[MAX_WORKERS];
odp_pool_t pool;
int num_workers;
- unsigned num_rx_queues, num_tx_queues;
+ uint32_t num_rx_queues, num_tx_queues;
int i;
odp_shm_t shm;
odp_cpumask_t cpumask;
diff --git a/example/ipfragreass/odp_ipfragreass_reassemble.c b/example/ipfragreass/odp_ipfragreass_reassemble.c
index 57e0f375a..2542a3b97 100644
--- a/example/ipfragreass/odp_ipfragreass_reassemble.c
+++ b/example/ipfragreass/odp_ipfragreass_reassemble.c
@@ -183,11 +183,9 @@ static struct packet *extract_complete_packet(struct packet *tail,
struct packet *current = tail;
odph_ipv4hdr_t tail_hdr;
uint16_t final_frag_offset;
- uint16_t expected_frag_offset;
tail_hdr = *(odph_ipv4hdr_t *)odp_packet_data(tail->handle);
final_frag_offset = ipv4hdr_fragment_offset_oct(tail_hdr);
- expected_frag_offset = final_frag_offset;
while (current) {
odph_ipv4hdr_t curr_hdr;
uint16_t curr_offset_oct;
@@ -258,7 +256,6 @@ static struct packet *extract_complete_packet(struct packet *tail,
break;
}
- expected_frag_offset -= prev_oct;
current = prev;
}
diff --git a/example/ipsec_api/odp_ipsec_api_run_esp_tun_out.sh b/example/ipsec_api/odp_ipsec_api_run_esp_tun_out.sh
index a4c62e226..2bda78665 100755
--- a/example/ipsec_api/odp_ipsec_api_run_esp_tun_out.sh
+++ b/example/ipsec_api/odp_ipsec_api_run_esp_tun_out.sh
@@ -22,6 +22,7 @@ setup_interfaces
-r 192.168.222.2/32,$ROUTE_IF_OUTB,08:00:27:F5:8B:DB \
-p 192.168.111.0/24,192.168.222.0/24,out,esp \
-e 192.168.111.2,192.168.222.2,3des,201,656c8523255ccc23a66c1917aa0cf30991fce83532a4b224 \
+ -t 192.168.111.2,192.168.222.2,10.0.111.2,10.0.222.2 \
-s 192.168.111.2,192.168.222.2,$IN_IF,$OUT_IF,10,100 \
-c 2 "$@"
diff --git a/example/ipsec_crypto/odp_ipsec_stream.c b/example/ipsec_crypto/odp_ipsec_stream.c
index a24d4b4f9..0a0710a13 100644
--- a/example/ipsec_crypto/odp_ipsec_stream.c
+++ b/example/ipsec_crypto/odp_ipsec_stream.c
@@ -15,6 +15,7 @@
#include <openssl/des.h>
#include <openssl/rand.h>
#include <openssl/hmac.h>
+#include <openssl/opensslv.h>
#include <odp_api.h>
@@ -26,6 +27,11 @@
#define LOOP_DEQ_COUNT 32 /**< packets to dequeue at once */
+/* Ignore warnings about APIs deprecated in OpenSSL 3.0 */
+#if OPENSSL_VERSION_NUMBER >= 0x30000000L
+#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
+#endif
+
/**
* Stream packet header
*/
diff --git a/helper/include/odp/helper/cli.h b/helper/include/odp/helper/cli.h
index b2f88d7bc..e9351147f 100644
--- a/helper/include/odp/helper/cli.h
+++ b/helper/include/odp/helper/cli.h
@@ -21,8 +21,6 @@
extern "C" {
#endif
-#include <odp_api.h>
-#include <odp/helper/ip.h>
#include <stdint.h>
#include <stdarg.h>
diff --git a/helper/include/odp/helper/linux/process.h b/helper/include/odp/helper/linux/process.h
index 679c6c907..620db7405 100644
--- a/helper/include/odp/helper/linux/process.h
+++ b/helper/include/odp/helper/linux/process.h
@@ -18,6 +18,7 @@
#define ODPH_LINUX_PROCESS_H_
#include <odp/helper/threads.h>
+#include <odp_api.h>
#ifdef __cplusplus
extern "C" {
diff --git a/helper/include/odp/helper/linux/pthread.h b/helper/include/odp/helper/linux/pthread.h
index 33109ec65..298536893 100644
--- a/helper/include/odp/helper/linux/pthread.h
+++ b/helper/include/odp/helper/linux/pthread.h
@@ -18,6 +18,7 @@
#define ODPH_LINUX_PTHREAD_H_
#include <odp/helper/threads.h>
+#include <odp_api.h>
#ifdef __cplusplus
extern "C" {
diff --git a/helper/include/odp/helper/threads.h b/helper/include/odp/helper/threads.h
index cfde92830..53ce8cd07 100644
--- a/helper/include/odp/helper/threads.h
+++ b/helper/include/odp/helper/threads.h
@@ -24,6 +24,7 @@ extern "C" {
#endif
#include <odp/helper/deprecated.h>
+#include <odp_api.h>
#include <pthread.h>
#include <getopt.h>
diff --git a/include/Makefile.am b/include/Makefile.am
index 89dfddebc..510154753 100644
--- a/include/Makefile.am
+++ b/include/Makefile.am
@@ -26,6 +26,7 @@ odpapiinclude_HEADERS = \
odp/api/dma_types.h \
odp/api/errno.h \
odp/api/event.h \
+ odp/api/event_types.h \
odp/api/hash.h \
odp/api/hints.h \
odp/api/init.h \
@@ -86,6 +87,7 @@ odpapispecinclude_HEADERS = \
odp/api/spec/dma_types.h \
odp/api/spec/errno.h \
odp/api/spec/event.h \
+ odp/api/spec/event_types.h \
odp/api/spec/hash.h \
odp/api/spec/hints.h \
odp/api/spec/init.h \
@@ -148,6 +150,7 @@ odpapiabidefaultinclude_HEADERS = \
odp/api/abi-default/dma_types.h \
odp/api/abi-default/errno.h \
odp/api/abi-default/event.h \
+ odp/api/abi-default/event_types.h \
odp/api/abi-default/hash.h \
odp/api/abi-default/init.h \
odp/api/abi-default/ipsec.h \
@@ -178,6 +181,7 @@ odpapiabidefaultinclude_HEADERS = \
odp/api/abi-default/thrmask.h \
odp/api/abi-default/ticketlock.h \
odp/api/abi-default/time.h \
+ odp/api/abi-default/timer.h \
odp/api/abi-default/timer_types.h \
odp/api/abi-default/traffic_mngr.h \
odp/api/abi-default/version.h
@@ -202,6 +206,7 @@ odpapiabiarchinclude_HEADERS = \
odp/arch/arm32-linux/odp/api/abi/dma_types.h \
odp/arch/arm32-linux/odp/api/abi/errno.h \
odp/arch/arm32-linux/odp/api/abi/event.h \
+ odp/arch/arm32-linux/odp/api/abi/event_types.h \
odp/arch/arm32-linux/odp/api/abi/hash.h \
odp/arch/arm32-linux/odp/api/abi/init.h \
odp/arch/arm32-linux/odp/api/abi/ipsec.h \
@@ -232,6 +237,7 @@ odpapiabiarchinclude_HEADERS = \
odp/arch/arm32-linux/odp/api/abi/thrmask.h \
odp/arch/arm32-linux/odp/api/abi/ticketlock.h \
odp/arch/arm32-linux/odp/api/abi/time.h \
+ odp/arch/arm32-linux/odp/api/abi/timer.h \
odp/arch/arm32-linux/odp/api/abi/timer_types.h \
odp/arch/arm32-linux/odp/api/abi/traffic_mngr.h \
odp/arch/arm32-linux/odp/api/abi/version.h
@@ -252,6 +258,7 @@ odpapiabiarchinclude_HEADERS = \
odp/arch/arm64-linux/odp/api/abi/dma_types.h \
odp/arch/arm64-linux/odp/api/abi/errno.h \
odp/arch/arm64-linux/odp/api/abi/event.h \
+ odp/arch/arm64-linux/odp/api/abi/event_types.h \
odp/arch/arm64-linux/odp/api/abi/hash.h \
odp/arch/arm64-linux/odp/api/abi/init.h \
odp/arch/arm64-linux/odp/api/abi/ipsec.h \
@@ -282,6 +289,7 @@ odpapiabiarchinclude_HEADERS = \
odp/arch/arm64-linux/odp/api/abi/thrmask.h \
odp/arch/arm64-linux/odp/api/abi/ticketlock.h \
odp/arch/arm64-linux/odp/api/abi/time.h \
+ odp/arch/arm64-linux/odp/api/abi/timer.h \
odp/arch/arm64-linux/odp/api/abi/timer_types.h \
odp/arch/arm64-linux/odp/api/abi/traffic_mngr.h \
odp/arch/arm64-linux/odp/api/abi/version.h
@@ -302,6 +310,7 @@ odpapiabiarchinclude_HEADERS = \
odp/arch/default-linux/odp/api/abi/dma_types.h \
odp/arch/default-linux/odp/api/abi/errno.h \
odp/arch/default-linux/odp/api/abi/event.h \
+ odp/arch/default-linux/odp/api/abi/event_types.h \
odp/arch/default-linux/odp/api/abi/hash.h \
odp/arch/default-linux/odp/api/abi/init.h \
odp/arch/default-linux/odp/api/abi/ipsec.h \
@@ -332,6 +341,7 @@ odpapiabiarchinclude_HEADERS = \
odp/arch/default-linux/odp/api/abi/thrmask.h \
odp/arch/default-linux/odp/api/abi/ticketlock.h \
odp/arch/default-linux/odp/api/abi/time.h \
+ odp/arch/default-linux/odp/api/abi/timer.h \
odp/arch/default-linux/odp/api/abi/timer_types.h \
odp/arch/default-linux/odp/api/abi/traffic_mngr.h \
odp/arch/default-linux/odp/api/abi/version.h
@@ -352,6 +362,7 @@ odpapiabiarchinclude_HEADERS = \
odp/arch/power64-linux/odp/api/abi/dma_types.h \
odp/arch/power64-linux/odp/api/abi/errno.h \
odp/arch/power64-linux/odp/api/abi/event.h \
+ odp/arch/power64-linux/odp/api/abi/event_types.h \
odp/arch/power64-linux/odp/api/abi/hash.h \
odp/arch/power64-linux/odp/api/abi/init.h \
odp/arch/power64-linux/odp/api/abi/ipsec.h \
@@ -382,6 +393,7 @@ odpapiabiarchinclude_HEADERS = \
odp/arch/power64-linux/odp/api/abi/thrmask.h \
odp/arch/power64-linux/odp/api/abi/ticketlock.h \
odp/arch/power64-linux/odp/api/abi/time.h \
+ odp/arch/power64-linux/odp/api/abi/timer.h \
odp/arch/power64-linux/odp/api/abi/timer_types.h \
odp/arch/power64-linux/odp/api/abi/traffic_mngr.h \
odp/arch/power64-linux/odp/api/abi/version.h
@@ -402,6 +414,7 @@ odpapiabiarchinclude_HEADERS = \
odp/arch/x86_32-linux/odp/api/abi/dma_types.h \
odp/arch/x86_32-linux/odp/api/abi/errno.h \
odp/arch/x86_32-linux/odp/api/abi/event.h \
+ odp/arch/x86_32-linux/odp/api/abi/event_types.h \
odp/arch/x86_32-linux/odp/api/abi/hash.h \
odp/arch/x86_32-linux/odp/api/abi/init.h \
odp/arch/x86_32-linux/odp/api/abi/ipsec.h \
@@ -432,6 +445,7 @@ odpapiabiarchinclude_HEADERS = \
odp/arch/x86_32-linux/odp/api/abi/thrmask.h \
odp/arch/x86_32-linux/odp/api/abi/ticketlock.h \
odp/arch/x86_32-linux/odp/api/abi/time.h \
+ odp/arch/x86_32-linux/odp/api/abi/timer.h \
odp/arch/x86_32-linux/odp/api/abi/timer_types.h \
odp/arch/x86_32-linux/odp/api/abi/traffic_mngr.h \
odp/arch/x86_32-linux/odp/api/abi/version.h
@@ -452,6 +466,7 @@ odpapiabiarchinclude_HEADERS = \
odp/arch/x86_64-linux/odp/api/abi/dma_types.h \
odp/arch/x86_64-linux/odp/api/abi/errno.h \
odp/arch/x86_64-linux/odp/api/abi/event.h \
+ odp/arch/x86_64-linux/odp/api/abi/event_types.h \
odp/arch/x86_64-linux/odp/api/abi/hash.h \
odp/arch/x86_64-linux/odp/api/abi/init.h \
odp/arch/x86_64-linux/odp/api/abi/ipsec.h \
@@ -482,6 +497,7 @@ odpapiabiarchinclude_HEADERS = \
odp/arch/x86_64-linux/odp/api/abi/thrmask.h \
odp/arch/x86_64-linux/odp/api/abi/ticketlock.h \
odp/arch/x86_64-linux/odp/api/abi/time.h \
+ odp/arch/x86_64-linux/odp/api/abi/timer.h \
odp/arch/x86_64-linux/odp/api/abi/timer_types.h \
odp/arch/x86_64-linux/odp/api/abi/traffic_mngr.h \
odp/arch/x86_64-linux/odp/api/abi/version.h
diff --git a/include/odp/api/abi-default/event.h b/include/odp/api/abi-default/event.h
index ecedda3bd..a1563ac9b 100644
--- a/include/odp/api/abi-default/event.h
+++ b/include/odp/api/abi-default/event.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2017-2018, Linaro Limited
+/* Copyright (c) 2022, Nokia
* All rights reserved.
*
* SPDX-License-Identifier: BSD-3-Clause
@@ -11,41 +11,7 @@
extern "C" {
#endif
-#include <stdint.h>
-
-/** @internal Dummy type for strong typing */
-typedef struct { char dummy; /**< @internal Dummy */ } _odp_abi_event_t;
-
-/** @ingroup odp_event
- * @{
- */
-
-typedef _odp_abi_event_t *odp_event_t;
-
-#define ODP_EVENT_INVALID ((odp_event_t)0)
-
-typedef enum {
- ODP_EVENT_BUFFER = 1,
- ODP_EVENT_PACKET = 2,
- ODP_EVENT_TIMEOUT = 3,
- ODP_EVENT_CRYPTO_COMPL = 4,
- ODP_EVENT_IPSEC_STATUS = 5,
- ODP_EVENT_PACKET_VECTOR = 6,
- ODP_EVENT_PACKET_TX_COMPL = 7,
- ODP_EVENT_DMA_COMPL = 8,
-} odp_event_type_t;
-
-typedef enum {
- ODP_EVENT_NO_SUBTYPE = 0,
- ODP_EVENT_PACKET_BASIC = 1,
- ODP_EVENT_PACKET_CRYPTO = 2,
- ODP_EVENT_PACKET_IPSEC = 3,
- ODP_EVENT_PACKET_COMP = 4
-} odp_event_subtype_t;
-
-/**
- * @}
- */
+/* Empty header required due to the inline functions */
#ifdef __cplusplus
}
diff --git a/include/odp/api/abi-default/event_types.h b/include/odp/api/abi-default/event_types.h
new file mode 100644
index 000000000..fc685cfd9
--- /dev/null
+++ b/include/odp/api/abi-default/event_types.h
@@ -0,0 +1,58 @@
+/* Copyright (c) 2017-2018, Linaro Limited
+ * Copyright (c) 2022, Nokia
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+#ifndef ODP_ABI_EVENT_TYPES_H_
+#define ODP_ABI_EVENT_TYPES_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <stdint.h>
+#include <odp/api/deprecated.h>
+
+/** @internal Dummy type for strong typing */
+typedef struct { char dummy; /**< @internal Dummy */ } _odp_abi_event_t;
+
+/** @ingroup odp_event
+ * @{
+ */
+
+typedef _odp_abi_event_t *odp_event_t;
+
+#define ODP_EVENT_INVALID ((odp_event_t)0)
+
+typedef enum {
+ ODP_EVENT_BUFFER = 1,
+ ODP_EVENT_PACKET = 2,
+ ODP_EVENT_TIMEOUT = 3,
+#if ODP_DEPRECATED_API
+ ODP_EVENT_CRYPTO_COMPL = 4,
+#endif
+ ODP_EVENT_IPSEC_STATUS = 5,
+ ODP_EVENT_PACKET_VECTOR = 6,
+ ODP_EVENT_PACKET_TX_COMPL = 7,
+ ODP_EVENT_DMA_COMPL = 8,
+} odp_event_type_t;
+
+typedef enum {
+ ODP_EVENT_NO_SUBTYPE = 0,
+ ODP_EVENT_PACKET_BASIC = 1,
+ ODP_EVENT_PACKET_CRYPTO = 2,
+ ODP_EVENT_PACKET_IPSEC = 3,
+ ODP_EVENT_PACKET_COMP = 4
+} odp_event_subtype_t;
+
+/**
+ * @}
+ */
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/include/odp/api/abi-default/timer.h b/include/odp/api/abi-default/timer.h
new file mode 100644
index 000000000..faf690ae3
--- /dev/null
+++ b/include/odp/api/abi-default/timer.h
@@ -0,0 +1,20 @@
+/* Copyright (c) 2022, Nokia
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+#ifndef ODP_ABI_TIMER_H_
+#define ODP_ABI_TIMER_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* Empty header required due to the timer inline functions */
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/include/odp/api/abi-default/traffic_mngr.h b/include/odp/api/abi-default/traffic_mngr.h
index 06dc6c3c7..78985313d 100644
--- a/include/odp/api/abi-default/traffic_mngr.h
+++ b/include/odp/api/abi-default/traffic_mngr.h
@@ -76,11 +76,6 @@ extern "C" {
*/
#define ODP_TM_MAX_TM_NODE_FANIN (4 * 1024)
-/** The ODP_NUM_SHAPER_COLORS constant just counts the number of enumeration
- * values defined in the odp_tm_shaper_color_t type.
- */
-#define ODP_NUM_SHAPER_COLORS 3
-
/** The INVALID_PRIORITY constant is used when one needs to indicate an
* invalid priority value.
*/
diff --git a/include/odp/api/buffer.h b/include/odp/api/buffer.h
index 27622c2a3..a56cd0660 100644
--- a/include/odp/api/buffer.h
+++ b/include/odp/api/buffer.h
@@ -17,8 +17,6 @@
extern "C" {
#endif
-#include <odp/api/std_types.h>
-#include <odp/api/abi/event.h>
#include <odp/api/abi/buffer.h>
#include <odp/api/spec/buffer.h>
diff --git a/include/odp/api/comp.h b/include/odp/api/comp.h
index c3294e79e..300d12a61 100644
--- a/include/odp/api/comp.h
+++ b/include/odp/api/comp.h
@@ -18,8 +18,6 @@ extern "C" {
#endif
#include <odp/api/abi/comp.h>
-#include <odp/api/abi/event.h>
-#include <odp/api/abi/queue_types.h>
#include <odp/api/spec/comp.h>
diff --git a/include/odp/api/dma_types.h b/include/odp/api/dma_types.h
index 5a726a319..eeaa7bee8 100644
--- a/include/odp/api/dma_types.h
+++ b/include/odp/api/dma_types.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2021, Nokia
+/* Copyright (c) 2021-2022, Nokia
* All rights reserved.
*
* SPDX-License-Identifier: BSD-3-Clause
@@ -17,7 +17,6 @@
extern "C" {
#endif
-#include <odp/api/abi/event.h>
#include <odp/api/abi/dma_types.h>
#include <odp/api/spec/dma_types.h>
diff --git a/include/odp/api/event_types.h b/include/odp/api/event_types.h
new file mode 100644
index 000000000..a4d20a341
--- /dev/null
+++ b/include/odp/api/event_types.h
@@ -0,0 +1,28 @@
+/* Copyright (c) 2022, Nokia
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+/**
+ * @file
+ *
+ * ODP event API type definitions
+ */
+
+#ifndef ODP_API_EVENT_TYPES_H_
+#define ODP_API_EVENT_TYPES_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <odp/api/abi/event_types.h>
+
+#include <odp/api/spec/event_types.h>
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/include/odp/api/packet.h b/include/odp/api/packet.h
index eb75d18fa..1f04c026d 100644
--- a/include/odp/api/packet.h
+++ b/include/odp/api/packet.h
@@ -1,4 +1,5 @@
/* Copyright (c) 2013-2018, Linaro Limited
+ * Copyright (c) 2022, Nokia
* All rights reserved.
*
* SPDX-License-Identifier: BSD-3-Clause
@@ -17,12 +18,7 @@
extern "C" {
#endif
-#include <odp/api/std_types.h>
-#include <odp/api/abi/event.h>
-#include <odp/api/abi/packet_io_types.h>
-#include <odp/api/abi/packet_types.h>
#include <odp/api/abi/packet.h>
-#include <odp/api/abi/buffer.h>
#include <odp/api/spec/packet.h>
diff --git a/include/odp/api/queue.h b/include/odp/api/queue.h
index 7897fff86..666d5eaf9 100644
--- a/include/odp/api/queue.h
+++ b/include/odp/api/queue.h
@@ -1,4 +1,5 @@
/* Copyright (c) 2013-2018, Linaro Limited
+ * Copyright (c) 2022, Nokia
* All rights reserved.
*
* SPDX-License-Identifier: BSD-3-Clause
@@ -17,11 +18,7 @@
extern "C" {
#endif
-#include <odp/api/std_types.h>
-#include <odp/api/abi/event.h>
-#include <odp/api/abi/queue_types.h>
#include <odp/api/abi/queue.h>
-#include <odp/api/abi/buffer.h>
#include <odp/api/spec/queue.h>
diff --git a/include/odp/api/spec/buffer.h b/include/odp/api/spec/buffer.h
index 7945b59f9..c11a3c7b4 100644
--- a/include/odp/api/spec/buffer.h
+++ b/include/odp/api/spec/buffer.h
@@ -18,7 +18,9 @@
extern "C" {
#endif
+#include <odp/api/event_types.h>
#include <odp/api/pool_types.h>
+#include <odp/api/std_types.h>
/** @defgroup odp_buffer ODP BUFFER
* Buffer event metadata and operations.
diff --git a/include/odp/api/spec/classification.h b/include/odp/api/spec/classification.h
index b9682c7cb..1c76ea192 100644
--- a/include/odp/api/spec/classification.h
+++ b/include/odp/api/spec/classification.h
@@ -36,9 +36,7 @@ extern "C" {
/**
* @def ODP_COS_INVALID
- * This value is returned from odp_cls_cos_create() on failure,
- * May also be used as a sink class of service that
- * results in packets being discarded.
+ * This value is returned from odp_cls_cos_create() on failure.
*/
/**
@@ -179,6 +177,30 @@ typedef struct odp_bp_param_t {
} odp_bp_param_t;
/**
+ * Classifier CoS specific statistics counters
+ *
+ * Counters are incremented per packet classified to the CoS. In a CoS chain,
+ * counters are incremented in every CoS for which counters are enabled.
+ */
+typedef struct odp_cls_cos_stats_t {
+ /** Number of octets in classified packets. In case of Ethernet, packet
+ * size includes MAC header. */
+ uint64_t octets;
+
+ /** Number of classified packets, including packets dropped due to drop
+ * action. */
+ uint64_t packets;
+
+ /** Number of discarded packets due to other reasons than packet
+ * errors or drop action. */
+ uint64_t discards;
+
+ /** Number of packets with errors. */
+ uint64_t errors;
+
+} odp_cls_cos_stats_t;
+
+/**
* Classifier queue specific statistics counters
*
* Counters are incremented per packet destined to the queue per originating
@@ -206,6 +228,34 @@ typedef struct odp_cls_queue_stats_t {
* Classifier statistics capabilities
*/
typedef struct odp_cls_stats_capability_t {
+ /** CoS level capabilities */
+ struct {
+ /** Supported counters */
+ union {
+ /** Statistics counters in a bit field structure */
+ struct {
+ /** @see odp_cls_cos_stats_t::octets */
+ uint64_t octets : 1;
+
+ /** @see odp_cls_cos_stats_t::packets */
+ uint64_t packets : 1;
+
+ /** @see odp_cls_cos_stats_t::discards */
+ uint64_t discards : 1;
+
+ /** @see odp_cls_cos_stats_t::errors */
+ uint64_t errors : 1;
+
+ } counter;
+
+ /** All bits of the bit field structure
+ *
+ * This field can be used to set/clear all flags, or
+ * for bitwise operations over the entire structure. */
+ uint64_t all_counters;
+ };
+ } cos;
+
/** Queue level capabilities */
struct {
/** Supported counters */
@@ -247,17 +297,22 @@ typedef struct odp_cls_capability_t {
odp_cls_pmr_terms_t supported_terms;
/** Maximum number of PMR terms */
- unsigned int max_pmr_terms;
+ uint32_t max_pmr_terms;
/** Number of PMR terms available for use now */
- unsigned int available_pmr_terms;
+ uint32_t available_pmr_terms;
/** Maximum number of CoS supported */
- unsigned int max_cos;
+ uint32_t max_cos;
+
+ /** Maximum number of CoSes that can have statistics enabled at the same
+ * time. If this value is zero, then CoS level statistics are not
+ * supported. */
+ uint32_t max_cos_stats;
/** Maximun number of queues supported per CoS
* if the value is 1, then hashing is not supported*/
- unsigned int max_hash_queues;
+ uint32_t max_hash_queues;
/** Protocol header combination supported for Hashing */
odp_pktin_hash_proto_t hash_protocols;
@@ -315,10 +370,48 @@ typedef enum {
} odp_cos_hdr_flow_fields_t;
/**
+ * Enumeration of actions for CoS.
+ */
+typedef enum {
+ /**
+ * Enqueue packet
+ *
+ * Packets that arrive in the CoS are enqueued to a destination queue.
+ */
+ ODP_COS_ACTION_ENQUEUE,
+
+ /**
+ * Drop packet
+ *
+ * Packets that arrive in the CoS are dropped. Packets are freed into
+ * their originating pool.
+ */
+ ODP_COS_ACTION_DROP,
+} odp_cos_action_t;
+
+/**
* Class of service parameters
* Used to communicate class of service creation options
*/
typedef struct odp_cls_cos_param {
+ /** Action to take. When action is ODP_COS_ACTION_DROP, all the other
+ * parameters are ignored.
+ *
+ * The final match in the CoS chain defines the action for a packet.
+ * I.e. packet is dropped only when the CoS of the last matching rule
+ * has drop action. Actions in the previous CoSes in the chain are
+ * ignored.
+ *
+ * Default is ODP_COS_ACTION_ENQUEUE.
+ */
+ odp_cos_action_t action;
+
+ /** Enable statistics. If true, counters are incremented when packets
+ * are classified to the CoS. Default is false. @see
+ * odp_cls_cos_stats().
+ */
+ odp_bool_t stats_enable;
+
/** Number of queues to be linked to this CoS.
* If the number is greater than 1 then hashing is enabled.
* If number is equal to 1 then hashing is disabled.
@@ -391,6 +484,9 @@ int odp_cls_capability(odp_cls_capability_t *capability);
/**
* Create a class-of-service
*
+ * Depending on the action parameter, packets to the CoS are either enqueued to
+ * a destination queue, or dropped.
+ *
* The use of class-of-service name is optional. Unique names are not required.
* Use odp_cls_cos_param_init() to initialize parameters into their default
* values.
@@ -401,10 +497,6 @@ int odp_cls_capability(odp_cls_capability_t *capability);
*
* @retval Class-of-service handle
* @retval ODP_COS_INVALID on failure.
- *
- * @note ODP_QUEUE_INVALID and ODP_POOL_INVALID are valid values for queue
- * and pool associated with a class of service. When either of these values
- * is configured as INVALID packets assigned to the CoS get dropped.
*/
odp_cos_t odp_cls_cos_create(const char *name,
const odp_cls_cos_param_t *param);
@@ -542,6 +634,25 @@ int odp_cos_with_l3_qos(odp_pktio_t pktio_in,
odp_bool_t l3_preference);
/**
+ * Get statistics for a CoS
+ *
+ * The statistics counters are incremented for packets classified to the
+ * given CoS.
+ *
+ * Counters that are not supported are set to zero.
+ *
+ * It's implementation defined if odp_pktio_stats_reset() call affects these
+ * counters.
+ *
+ * @param cos CoS handle
+ * @param[out] stats Statistics structure for output
+ *
+ * @retval 0 on success
+ * @retval <0 on failure
+ */
+int odp_cls_cos_stats(odp_cos_t cos, odp_cls_cos_stats_t *stats);
+
+/**
* Get statistics for a queue assigned to a CoS
*
* The statistics counters are incremented only for packets originating from the
diff --git a/include/odp/api/spec/comp.h b/include/odp/api/spec/comp.h
index 7f64cca5d..39ecbfde4 100644
--- a/include/odp/api/spec/comp.h
+++ b/include/odp/api/spec/comp.h
@@ -14,8 +14,11 @@
#define ODP_API_SPEC_COMP_H_
#include <odp/visibility_begin.h>
+
+#include <odp/api/event_types.h>
+#include <odp/api/packet_types.h>
+#include <odp/api/queue_types.h>
#include <odp/api/std_types.h>
-#include <odp/api/packet.h>
#ifdef __cplusplus
extern "C" {
diff --git a/include/odp/api/spec/crypto.h b/include/odp/api/spec/crypto.h
index c0ff3710d..7cd69f4d2 100644
--- a/include/odp/api/spec/crypto.h
+++ b/include/odp/api/spec/crypto.h
@@ -16,6 +16,7 @@
#include <odp/visibility_begin.h>
#include <odp/api/deprecated.h>
+#include <odp/api/packet_types.h>
#include <odp/api/pool_types.h>
#include <odp/api/std_types.h>
@@ -23,8 +24,6 @@
extern "C" {
#endif
-#include <odp/api/packet.h>
-
/** @defgroup odp_crypto ODP CRYPTO
* Data ciphering and authentication.
* @{
@@ -42,7 +41,7 @@ extern "C" {
/**
* @typedef odp_crypto_compl_t
-* Crypto API completion event (platform dependent).
+* @deprecated Crypto API completion event (platform dependent).
*/
/**
@@ -559,8 +558,10 @@ typedef struct odp_crypto_session_param_t {
/** Preferred sync vs. async for odp_crypto_operation()
*
* The default value is ODP_CRYPTO_SYNC.
+ *
+ * @deprecated Used only with deprecated odp_crypto_operation()
*/
- odp_crypto_op_mode_t pref_mode;
+ odp_crypto_op_mode_t ODP_DEPRECATE(pref_mode);
/** Operation mode when using packet interface: sync or async
*
@@ -672,8 +673,8 @@ typedef struct odp_crypto_session_param_t {
/** Async mode completion event queue
*
* The completion queue is used to return completions from
- * odp_crypto_operation() or odp_crypto_op_enq() results to the
- * application.
+ * odp_crypto_op_enq() (and the deprecated odp_crypto_operation())
+ * to the application.
*/
odp_queue_t compl_queue;
@@ -689,6 +690,8 @@ typedef struct odp_crypto_session_param_t {
/**
* Crypto API per packet operation parameters
+ *
+ * @deprecated Use odp_crypto_packet_op_param_t instead.
*/
typedef struct odp_crypto_op_param_t {
/** Session handle from creation */
@@ -760,7 +763,7 @@ typedef struct odp_crypto_op_param_t {
*/
odp_packet_data_range_t auth_range;
-} odp_crypto_op_param_t;
+} ODP_DEPRECATE(odp_crypto_op_param_t);
/**
* Crypto packet API per packet operation parameters
@@ -895,6 +898,8 @@ typedef struct odp_crypto_op_status {
/**
* Crypto API operation result
+ *
+ * @deprecated Use odp_crypto_packet_result_t instead.
*/
typedef struct odp_crypto_op_result {
/** Request completed successfully */
@@ -912,7 +917,7 @@ typedef struct odp_crypto_op_result {
/** Authentication status */
odp_crypto_op_status_t auth_status;
-} odp_crypto_op_result_t;
+} ODP_DEPRECATE(odp_crypto_op_result_t);
/**
* Crypto packet API operation result
@@ -1141,9 +1146,13 @@ int odp_crypto_session_create(const odp_crypto_session_param_t *param,
*/
int odp_crypto_session_destroy(odp_crypto_session_t session);
+#if ODP_DEPRECATED_API
+
/**
* Return crypto completion handle that is associated with event
*
+ * @deprecated Used only with the deprecated odp_crypto_operation()
+ *
* Note: any invalid parameters will cause undefined behavior and may cause
* the application to abort or crash.
*
@@ -1156,6 +1165,8 @@ odp_crypto_compl_t odp_crypto_compl_from_event(odp_event_t ev);
/**
* Convert crypto completion handle to event handle
*
+ * @deprecated Used only with the deprecated odp_crypto_operation()
+ *
* @param completion_event Completion event to convert to generic event
*
* @return Event handle
@@ -1165,6 +1176,8 @@ odp_event_t odp_crypto_compl_to_event(odp_crypto_compl_t completion_event);
/**
* Release crypto completion event
*
+ * @deprecated Used only with the deprecated odp_crypto_operation()
+ *
* @param completion_event Completion event we are done accessing
*/
void odp_crypto_compl_free(odp_crypto_compl_t completion_event);
@@ -1172,6 +1185,8 @@ void odp_crypto_compl_free(odp_crypto_compl_t completion_event);
/**
* Crypto per packet operation
*
+ * @deprecated Use odp_crypto_op() or odp_crypto_op_enq() instead.
+ *
* Performs the cryptographic operations specified during session creation
* on the packet. If the operation is performed synchronously, "posted"
* will return FALSE and the result of the operation is immediately available.
@@ -1192,12 +1207,16 @@ int odp_crypto_operation(odp_crypto_op_param_t *param,
/**
* Crypto per packet operation query result from completion event
*
+ * @deprecated Used only with the deprecated odp_crypto_operation()
+ *
* @param completion_event Event containing operation results
* @param result Pointer to result structure
*/
void odp_crypto_compl_result(odp_crypto_compl_t completion_event,
odp_crypto_op_result_t *result);
+#endif /* ODP_DEPRECATED_API */
+
/**
* Get printable value for an odp_crypto_session_t
*
@@ -1211,9 +1230,12 @@ void odp_crypto_compl_result(odp_crypto_compl_t completion_event,
*/
uint64_t odp_crypto_session_to_u64(odp_crypto_session_t hdl);
+#if ODP_DEPRECATED_API
/**
* Get printable value for an odp_crypto_compl_t
*
+ * @deprecated Used only with the deprecated odp_crypto_operation()
+ *
* @param hdl odp_crypto_compl_t handle to be printed
* @return uint64_t value that can be used to print/display this
* handle
@@ -1223,6 +1245,7 @@ uint64_t odp_crypto_session_to_u64(odp_crypto_session_t hdl);
* an odp_crypto_compl_t handle.
*/
uint64_t odp_crypto_compl_to_u64(odp_crypto_compl_t hdl);
+#endif
/**
* Initialize crypto session parameters
diff --git a/include/odp/api/spec/dma_types.h b/include/odp/api/spec/dma_types.h
index d7ccc6d56..ae3d05287 100644
--- a/include/odp/api/spec/dma_types.h
+++ b/include/odp/api/spec/dma_types.h
@@ -18,7 +18,9 @@
extern "C" {
#endif
+#include <odp/api/event_types.h>
#include <odp/api/packet_types.h>
+#include <odp/api/queue_types.h>
#include <odp/api/std_types.h>
/** @defgroup odp_dma ODP DMA
diff --git a/include/odp/api/spec/event.h b/include/odp/api/spec/event.h
index dc99f35c7..c36d9cef1 100644
--- a/include/odp/api/spec/event.h
+++ b/include/odp/api/spec/event.h
@@ -1,4 +1,5 @@
/* Copyright (c) 2015-2018, Linaro Limited
+ * Copyright (c) 2022, Nokia
* All rights reserved.
*
* SPDX-License-Identifier: BSD-3-Clause
@@ -18,7 +19,8 @@
extern "C" {
#endif
-#include <odp/api/packet.h>
+#include <odp/api/event_types.h>
+#include <odp/api/packet_types.h>
/** @defgroup odp_event ODP EVENT
* Generic event metadata and operations.
@@ -26,81 +28,6 @@ extern "C" {
*/
/**
- * @typedef odp_event_t
- * ODP event
- */
-
-/**
- * @def ODP_EVENT_INVALID
- * Invalid event
- */
-
-/**
- * @typedef odp_event_type_t
- * Event type
- *
- * Event type specifies purpose and general format of an event. It can be
- * checked with odp_event_type() or odp_event_types(). Each event type has
- * functions (e.g. odp_buffer_from_event()) to convert between the generic event
- * handle (odp_event_t) and the type specific handle (e.g. odp_buffer_t).
- * Results are undefined, if conversion function of a wrong event type is used.
- * Application cannot change event type by chaining conversion functions.
- *
- * List of event types:
- * - ODP_EVENT_BUFFER
- * - Buffer event (odp_buffer_t) for simple data storage and message passing
- * - ODP_EVENT_PACKET
- * - Packet event (odp_packet_t) containing packet data and plenty of
- * packet processing related metadata
- * - ODP_EVENT_TIMEOUT
- * - Timeout event (odp_timeout_t) from a timer
- * - ODP_EVENT_CRYPTO_COMPL
- * - Crypto completion event (odp_crypto_compl_t)
- * - ODP_EVENT_IPSEC_STATUS
- * - IPSEC status update event (odp_ipsec_status_t)
- * - ODP_EVENT_PACKET_VECTOR
- * - Vector of packet events (odp_packet_t) as odp_packet_vector_t
- * - ODP_EVENT_PACKET_TX_COMPL
- * - Packet Tx completion event (odp_packet_tx_compl_t) generated as a result of a Packet Tx
- * completion.
- * - ODP_EVENT_DMA_COMPL
- * - DMA completion event (odp_dma_compl_t) indicates that a DMA transfer has finished
- */
-
-/**
- * @typedef odp_event_subtype_t
- * Event subtype
- *
- * Event subtype expands event type specification by providing more detailed
- * purpose and format of an event. It can be checked with odp_event_subtype() or
- * odp_event_types(). Each event subtype may define specific functions
- * (e.g. odp_ipsec_packet_from_event()) to convert between the generic event
- * handle (odp_event_t) and event type specific handle (e.g. odp_packet_t). When
- * subtype is known, these subtype specific functions should be preferred over
- * the event type general function (e.g. odp_packet_from_event()). Results are
- * undefined, if conversion function of a wrong event subtype is used.
- * Application cannot change event subtype by chaining conversion functions.
- *
- * List of event subtypes:
- * - ODP_EVENT_PACKET_BASIC
- * - Packet event (odp_packet_t) with basic packet metadata
- * - ODP_EVENT_PACKET_COMP
- * - Packet event (odp_packet_t) generated as a result of a compression/
- * decompression operation. It contains compression specific metadata in
- * addition to the basic packet metadata.
- * - ODP_EVENT_PACKET_CRYPTO
- * - Packet event (odp_packet_t) generated as a result of a Crypto
- * operation. It contains crypto specific metadata in addition to the
- * basic packet metadata.
- * - ODP_EVENT_PACKET_IPSEC
- * - Packet event (odp_packet_t) generated as a result of an IPsec
- * operation. It contains IPSEC specific metadata in addition to the basic
- * packet metadata.
- * - ODP_EVENT_NO_SUBTYPE
- * - An event type does not have any subtypes defined
- */
-
-/**
* Event type of an event
*
* Event type specifies purpose and general format of an event.
diff --git a/include/odp/api/spec/event_types.h b/include/odp/api/spec/event_types.h
new file mode 100644
index 000000000..529d031e7
--- /dev/null
+++ b/include/odp/api/spec/event_types.h
@@ -0,0 +1,110 @@
+/* Copyright (c) 2015-2018, Linaro Limited
+ * Copyright (c) 2022, Nokia
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+/**
+ * @file
+ *
+ * ODP event API type definitions
+ */
+
+#ifndef ODP_API_SPEC_EVENT_TYPES_H_
+#define ODP_API_SPEC_EVENT_TYPES_H_
+#include <odp/visibility_begin.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/** @addtogroup odp_event
+ * @{
+ */
+
+/**
+ * @typedef odp_event_t
+ * ODP event
+ */
+
+/**
+ * @def ODP_EVENT_INVALID
+ * Invalid event
+ */
+
+/**
+ * @typedef odp_event_type_t
+ * Event type
+ *
+ * Event type specifies purpose and general format of an event. It can be
+ * checked with odp_event_type() or odp_event_types(). Each event type has
+ * functions (e.g. odp_buffer_from_event()) to convert between the generic event
+ * handle (odp_event_t) and the type specific handle (e.g. odp_buffer_t).
+ * Results are undefined, if conversion function of a wrong event type is used.
+ * Application cannot change event type by chaining conversion functions.
+ *
+ * List of event types:
+ * - ODP_EVENT_BUFFER
+ * - Buffer event (odp_buffer_t) for simple data storage and message passing
+ * - ODP_EVENT_PACKET
+ * - Packet event (odp_packet_t) containing packet data and plenty of
+ * packet processing related metadata
+ * - ODP_EVENT_TIMEOUT
+ * - Timeout event (odp_timeout_t) from a timer
+ * - ODP_EVENT_CRYPTO_COMPL
+ * - Crypto completion event (odp_crypto_compl_t)
+ * - ODP_EVENT_IPSEC_STATUS
+ * - IPSEC status update event (odp_ipsec_status_t)
+ * - ODP_EVENT_PACKET_VECTOR
+ * - Vector of packet events (odp_packet_t) as odp_packet_vector_t
+ * - ODP_EVENT_PACKET_TX_COMPL
+ * - Packet Tx completion event (odp_packet_tx_compl_t) generated as a result of a Packet Tx
+ * completion.
+ * - ODP_EVENT_DMA_COMPL
+ * - DMA completion event (odp_dma_compl_t) indicates that a DMA transfer has finished
+ */
+
+/**
+ * @typedef odp_event_subtype_t
+ * Event subtype
+ *
+ * Event subtype expands event type specification by providing more detailed
+ * purpose and format of an event. It can be checked with odp_event_subtype() or
+ * odp_event_types(). Each event subtype may define specific functions
+ * (e.g. odp_ipsec_packet_from_event()) to convert between the generic event
+ * handle (odp_event_t) and event type specific handle (e.g. odp_packet_t). When
+ * subtype is known, these subtype specific functions should be preferred over
+ * the event type general function (e.g. odp_packet_from_event()). Results are
+ * undefined, if conversion function of a wrong event subtype is used.
+ * Application cannot change event subtype by chaining conversion functions.
+ *
+ * List of event subtypes:
+ * - ODP_EVENT_PACKET_BASIC
+ * - Packet event (odp_packet_t) with basic packet metadata
+ * - ODP_EVENT_PACKET_COMP
+ * - Packet event (odp_packet_t) generated as a result of a compression/
+ * decompression operation. It contains compression specific metadata in
+ * addition to the basic packet metadata.
+ * - ODP_EVENT_PACKET_CRYPTO
+ * - Packet event (odp_packet_t) generated as a result of a Crypto
+ * operation. It contains crypto specific metadata in addition to the
+ * basic packet metadata.
+ * - ODP_EVENT_PACKET_IPSEC
+ * - Packet event (odp_packet_t) generated as a result of an IPsec
+ * operation. It contains IPSEC specific metadata in addition to the basic
+ * packet metadata.
+ * - ODP_EVENT_NO_SUBTYPE
+ * - An event type does not have any subtypes defined
+ */
+
+/**
+ * @}
+ */
+
+#ifdef __cplusplus
+}
+#endif
+
+#include <odp/visibility_end.h>
+#endif
diff --git a/include/odp/api/spec/packet.h b/include/odp/api/spec/packet.h
index 8dc0a77bb..c745d29ad 100644
--- a/include/odp/api/spec/packet.h
+++ b/include/odp/api/spec/packet.h
@@ -19,10 +19,13 @@
extern "C" {
#endif
-#include <odp/api/proto_stats_types.h>
-#include <odp/api/time.h>
+#include <odp/api/event_types.h>
#include <odp/api/packet_types.h>
+#include <odp/api/packet_io_types.h>
#include <odp/api/pool_types.h>
+#include <odp/api/proto_stats_types.h>
+#include <odp/api/std_types.h>
+#include <odp/api/time.h>
/** @defgroup odp_packet ODP PACKET
* Packet event metadata and operations.
diff --git a/include/odp/api/spec/packet_flags.h b/include/odp/api/spec/packet_flags.h
index 7c1386cb6..98bf2b7bf 100644
--- a/include/odp/api/spec/packet_flags.h
+++ b/include/odp/api/spec/packet_flags.h
@@ -19,7 +19,7 @@ extern "C" {
#endif
#include <odp/api/std_types.h>
-#include <odp/api/packet.h>
+#include <odp/api/packet_types.h>
/** @addtogroup odp_packet
* Operations on packet metadata flags.
diff --git a/include/odp/api/spec/packet_io.h b/include/odp/api/spec/packet_io.h
index 3be45446d..4a8635d0a 100644
--- a/include/odp/api/spec/packet_io.h
+++ b/include/odp/api/spec/packet_io.h
@@ -431,9 +431,8 @@ int odp_pktin_recv_tmo(odp_pktin_queue_t queue, odp_packet_t packets[],
* @return Number of packets received
* @retval <0 on failure
*/
-int odp_pktin_recv_mq_tmo(const odp_pktin_queue_t queues[], unsigned int num_q,
- unsigned int *from, odp_packet_t packets[], int num,
- uint64_t wait);
+int odp_pktin_recv_mq_tmo(const odp_pktin_queue_t queues[], uint32_t num_q, uint32_t *from,
+ odp_packet_t packets[], int num, uint64_t wait);
/**
* Packet input wait time
diff --git a/include/odp/api/spec/packet_io_types.h b/include/odp/api/spec/packet_io_types.h
index 2d43822fe..153e188b7 100644
--- a/include/odp/api/spec/packet_io_types.h
+++ b/include/odp/api/spec/packet_io_types.h
@@ -257,7 +257,7 @@ typedef struct odp_pktin_queue_param_t {
* More than one input queues require flow hashing configured.
* The maximum value is defined by pktio capability 'max_input_queues'.
* Queue type is defined by the input mode. The default value is 1. */
- unsigned int num_queues;
+ uint32_t num_queues;
/** Queue parameters
*
@@ -304,7 +304,7 @@ typedef struct odp_pktout_queue_param_t {
/** Number of output queues to be created. The value must be between
* 1 and interface capability. The default value is 1. */
- unsigned int num_queues;
+ uint32_t num_queues;
/** Output queue size array
*
@@ -820,12 +820,12 @@ typedef struct odp_pktin_vector_capability_t {
*/
typedef struct odp_pktio_capability_t {
/** Maximum number of input queues */
- unsigned int max_input_queues;
+ uint32_t max_input_queues;
/** Maximum number of output queues
*
* Value does not exceed ODP_PKTOUT_MAX_QUEUES. */
- unsigned int max_output_queues;
+ uint32_t max_output_queues;
/** Minimum output queue size
*
diff --git a/include/odp/api/spec/pool_types.h b/include/odp/api/spec/pool_types.h
index b0d5b37c9..ef272b67d 100644
--- a/include/odp/api/spec/pool_types.h
+++ b/include/odp/api/spec/pool_types.h
@@ -127,12 +127,12 @@ typedef struct odp_pool_stats_t {
*/
typedef struct odp_pool_capability_t {
/** Maximum number of pools of any type */
- unsigned int max_pools;
+ uint32_t max_pools;
/** Buffer pool capabilities */
struct {
/** Maximum number of buffer pools */
- unsigned int max_pools;
+ uint32_t max_pools;
/** Maximum buffer data alignment in bytes */
uint32_t max_align;
@@ -162,7 +162,7 @@ typedef struct odp_pool_capability_t {
/** Packet pool capabilities */
struct {
/** Maximum number of packet pools */
- unsigned int max_pools;
+ uint32_t max_pools;
/** Maximum packet data length in bytes
*
@@ -250,7 +250,7 @@ typedef struct odp_pool_capability_t {
/** Timeout pool capabilities */
struct {
/** Maximum number of timeout pools */
- unsigned int max_pools;
+ uint32_t max_pools;
/** Maximum number of timeout events in a pool
*
@@ -271,7 +271,7 @@ typedef struct odp_pool_capability_t {
/** Vector pool capabilities */
struct {
/** Maximum number of vector pools */
- unsigned int max_pools;
+ uint32_t max_pools;
/** Maximum number of vector events in a pool
*
diff --git a/include/odp/api/spec/queue.h b/include/odp/api/spec/queue.h
index 9c677c926..65cf3d1ee 100644
--- a/include/odp/api/spec/queue.h
+++ b/include/odp/api/spec/queue.h
@@ -18,8 +18,9 @@
extern "C" {
#endif
-#include <odp/api/event.h>
+#include <odp/api/event_types.h>
#include <odp/api/queue_types.h>
+#include <odp/api/std_types.h>
/** @defgroup odp_queue ODP QUEUE
* Queues for event passing and scheduling.
diff --git a/include/odp/api/spec/schedule.h b/include/odp/api/spec/schedule.h
index 12da88f7d..524d9e2bf 100644
--- a/include/odp/api/spec/schedule.h
+++ b/include/odp/api/spec/schedule.h
@@ -19,7 +19,7 @@ extern "C" {
#endif
#include <odp/api/std_types.h>
-#include <odp/api/event.h>
+#include <odp/api/event_types.h>
#include <odp/api/queue_types.h>
#include <odp/api/schedule_types.h>
#include <odp/api/thrmask.h>
diff --git a/include/odp/api/spec/shared_memory.h b/include/odp/api/spec/shared_memory.h
index 6ba6f7fc2..d1955db26 100644
--- a/include/odp/api/spec/shared_memory.h
+++ b/include/odp/api/spec/shared_memory.h
@@ -133,7 +133,7 @@ typedef struct odp_shm_capability_t {
*
* This number of separate shared memory blocks can be
* reserved concurrently. */
- unsigned int max_blocks;
+ uint32_t max_blocks;
/** Maximum memory block size in bytes
*
diff --git a/include/odp/api/spec/timer.h b/include/odp/api/spec/timer.h
index 9df43d034..b51db7dbc 100644
--- a/include/odp/api/spec/timer.h
+++ b/include/odp/api/spec/timer.h
@@ -21,7 +21,7 @@ extern "C" {
#endif
#include <odp/api/timer_types.h>
-#include <odp/api/event.h>
+#include <odp/api/event_types.h>
#include <odp/api/pool_types.h>
#include <odp/api/queue_types.h>
diff --git a/include/odp/api/spec/timer_types.h b/include/odp/api/spec/timer_types.h
index 56c118ce3..754c831b2 100644
--- a/include/odp/api/spec/timer_types.h
+++ b/include/odp/api/spec/timer_types.h
@@ -21,7 +21,7 @@ extern "C" {
#endif
#include <odp/api/std_types.h>
-#include <odp/api/event.h>
+#include <odp/api/event_types.h>
/** @defgroup odp_timer ODP TIMER
* Timer generating timeout events.
diff --git a/include/odp/api/spec/traffic_mngr.h b/include/odp/api/spec/traffic_mngr.h
index 64f2eac83..b22887fc6 100644
--- a/include/odp/api/spec/traffic_mngr.h
+++ b/include/odp/api/spec/traffic_mngr.h
@@ -98,11 +98,6 @@ extern "C" {
*/
/**
- * @def ODP_NUM_SHAPER_COLORS
- * The number of enumeration values defined in the odp_tm_shaper_color_t type.
- */
-
-/**
* @def ODP_TM_INVALID_PRIORITY
* Used to indicate an invalid priority value.
*/
@@ -344,9 +339,19 @@ typedef struct {
* the legal range of such weights. */
odp_bool_t weights_supported;
- /** tm_node_threshold indicates that the tm_nodes at this
- * level support threshold profiles. */
- odp_bool_t tm_node_threshold;
+ /** TM node threshold profile support */
+ struct {
+ /** Threshold given as bytes */
+ uint8_t byte : 1;
+
+ /** Threshold given as packets */
+ uint8_t packet : 1;
+
+ /** Threshold given as bytes and packets simultaneously */
+ uint8_t byte_and_packet : 1;
+
+ } tm_node_threshold;
+
} odp_tm_level_capabilities_t;
/** The tm_pkt_prio_mode_t enumeration type is used to indicate different
@@ -515,11 +520,18 @@ typedef struct {
/** TM queue statistics counter capabilities */
odp_tm_queue_stats_capability_t queue_stats;
- /** tm_queue_threshold indicates support for threshold profile on a
- * TM queue. When TRUE, users can set/clear/update threshold profile
- * on a TM queue. When false none of it is supported.
- */
- odp_bool_t tm_queue_threshold;
+ /** TM queue threshold profile support */
+ struct {
+ /** Threshold given as bytes */
+ uint8_t byte : 1;
+
+ /** Threshold given as packets */
+ uint8_t packet : 1;
+
+ /** Threshold given as bytes and packets simultaneously */
+ uint8_t byte_and_packet : 1;
+
+ } tm_queue_threshold;
/** tm_queue_query_flags indicates supported types of TM queue query.
* Types of TM queue query are same as query_flags that are passed to
@@ -719,7 +731,6 @@ typedef struct {
*
* odp_tm_requirements_init() must be called to initialize any
* odp_tm_requirements_t record before it is first used or assigned to.
- * This is done to allow for vendor specific additions to this record.
*
* @param requirements A pointer to an odp_tm_requirements_t record which
* is to be initialized.
@@ -730,7 +741,6 @@ void odp_tm_requirements_init(odp_tm_requirements_t *requirements);
*
* odp_tm_egress_init() must be called to initialize any odp_tm_egress_t
* record before it is first used or assigned to.
- * This is done to allow for vendor specific additions to this record.
*
* @param egress A pointer to an odp_tm_egress_t record which
* is to be initialized.
@@ -940,9 +950,9 @@ int odp_tm_destroy(odp_tm_t tm);
* The odp_tm_vlan_marking() function allows one to configure the TM egress
* so as to have it set the one bit VLAN Drop Eligibility Indicator (DEI)
* field (but only for pkts that already carry a VLAN tag) of a pkt based upon
- * the final pkt (or shaper?) color assigned to the pkt when it reaches the
- * egress node. When drop_eligible_enabled is false, then the given color has
- * no effect on the VLAN fields. See IEEE 802.1q for more details.
+ * the final pkt color assigned to the pkt when it reaches the egress node. When
+ * drop_eligible_enabled is false, then the given color has no effect on the
+ * VLAN fields. See IEEE 802.1q for more details.
*
* Note that ALL ODP implementations are required to SUCCESSFULLY handle all
* calls to this function with drop_eligible_enabled == FALSE - i.e. must
@@ -1037,24 +1047,11 @@ int odp_tm_drop_prec_marking(odp_tm_t tm,
/* Shaper profile types and functions
* -------------------------------------------------------- */
-/** Possible values of running the shaper algorithm. ODP_TM_SHAPER_GREEN
- * means that the traffic is within the commit specification (rate and burst
- * size), ODP_TM_SHAPER_YELLOW means that the traffic is within the peak
- * specification (rate and burst size) and ODP_TM_SHAPER_RED means that the
- * traffic is exceeding both its commit and peak specifications. Note that
- * packets can also have an assigned <b> packet color</b> of ODP_PACKET_GREEN,
- * ODP_PACKET_YELLOW or ODP_PACKET_RED which has a different meaning and
- * purpose than the shaper colors.
- */
-typedef enum {
- ODP_TM_SHAPER_GREEN, ODP_TM_SHAPER_YELLOW, ODP_TM_SHAPER_RED
-} odp_tm_shaper_color_t;
-
-/** The odp_tm_shaper_params_t record type is used to supply the parameters
- * associated with a shaper profile. Since it is expected that
- * implementations might augment this record type with platform specific
- * additional fields - it is required that odp_tm_shaper_params_init() be
- * called on variables of this type before any of the fields are filled in.
+/**
+ * TM shaper parameters
+ *
+ * Use odp_tm_shaper_params_init() to initialize parameters into their default
+ * values.
*/
typedef struct {
/** The committed information rate for this shaper profile. The units
@@ -1123,12 +1120,12 @@ typedef struct {
odp_bool_t packet_mode;
} odp_tm_shaper_params_t;
-/** odp_tm_shaper_params_init() must be called to initialize any
- * odp_tm_shaper_params_t record before it is first used or assigned to.
- * The parameters are initialized to their default values.
+/**
+ * Initialize TM shaper parameters
*
- * @param params A pointer to an odp_tm_shaper_params_t record which
- * is to be initialized.
+ * Initialize an odp_tm_shaper_params_t to its default values for all fields.
+ *
+ * @param params Address of the odp_tm_shaper_params_t to be initialized
*/
void odp_tm_shaper_params_init(odp_tm_shaper_params_t *params);
@@ -1176,8 +1173,8 @@ int odp_tm_shaper_params_read(odp_tm_shaper_t shaper_profile,
/** odp_tm_shaper_params_update() "sets" the current set of values associated
* with the specified shaper profile object. In addition, this call has the
- * effect that all tm_input's and tm_nodes that are associated (attached?)
- * with this shaper profile object will be updated with the new values.
+ * effect that all tm_input's and tm_nodes that are associated with this shaper
+ * profile object will be updated with the new values.
*
* @param shaper_profile Specifies the shaper profile object whose
* values are to be set.
@@ -1215,11 +1212,11 @@ typedef enum {
ODP_TM_FRAME_BASED_WEIGHTS /**< Ignore the packet length */
} odp_tm_sched_mode_t;
-/** The odp_tm_sched_params_t record type is used to supply the parameters
- * associated with a scheduler profile. Since it is expected that
- * implementations might augment this record type with platform specific
- * additional fields - it is required that odp_tm_sched_params_init() be
- * called on variables of this type before any of the fields are filled in.
+/**
+ * TM scheduler parameters
+ *
+ * Use odp_tm_sched_params_init() to initialize parameters into their default
+ * values.
*/
typedef struct {
/** sched_modes indicates whether weighted scheduling should be used
@@ -1237,12 +1234,12 @@ typedef struct {
uint32_t sched_weights[ODP_TM_MAX_PRIORITIES];
} odp_tm_sched_params_t;
-/** odp_tm_sched_params_init() must be called to initialize any
- * odp_tm_sched_params_t record before it is first used or assigned to.
- * The parameters are initialized to their default values.
+/**
+ * Initialize TM scheduler parameters
*
- * @param params A pointer to an odp_tm_sched_params_t record which
- * is to be initialized.
+ * Initialize an odp_tm_sched_params_t to its default values for all fields.
+ *
+ * @param params Address of the odp_tm_sched_params_t to be initialized
*/
void odp_tm_sched_params_init(odp_tm_sched_params_t *params);
@@ -1289,8 +1286,8 @@ int odp_tm_sched_params_read(odp_tm_sched_t sched_profile,
/** odp_tm_sched_params_update() "sets" the current set of values associated
* with the specified scheduler profile object. In addition, this call has
- * the effect that all tm_nodes that are associated (attached?) with this
- * Scheduler profile object will be updated with the new values.
+ * the effect that all tm_nodes that are associated with this scheduler profile
+ * object will be updated with the new values.
*
* @param sched_profile Specifies the Scheduler profile object whose
* values are to be set.
@@ -1315,11 +1312,11 @@ odp_tm_sched_t odp_tm_sched_lookup(const char *name);
/* Queue Threshold Profiles - types and functions
* -------------------------------------------------------- */
-/** The odp_tm_threshold_params_t record type is used to supply the parameters
- * associated with a queue thresholds profile. Since it is expected that
- * implementations might augment this record type with platform specific
- * additional fields - it is required that odp_tm_threshold_params_init() be
- * called on variables of this type before any of the fields are filled in
+/**
+ * TM threshold parameters
+ *
+ * Use odp_tm_threshold_params_init() to initialize parameters into their
+ * default values.
*/
typedef struct {
uint64_t max_pkts; /**< max pkt cnt for this threshold profile */
@@ -1332,12 +1329,12 @@ typedef struct {
odp_bool_t enable_max_bytes;
} odp_tm_threshold_params_t;
-/** odp_tm_threshold_params_init() must be called to initialize any
- * odp_tm_threshold_params_t record before it is first used or assigned to.
- * The parameters are initialized to their default values.
+/**
+ * Initialize TM threshold parameters
*
- * @param params A pointer to an odp_tm_threshold_params_t record which
- * is to be initialized.
+ * Initialize an odp_tm_threshold_params_t to its default values for all fields.
+ *
+ * @param params Address of the odp_tm_threshold_params_t to be initialized
*/
void odp_tm_threshold_params_init(odp_tm_threshold_params_t *params);
@@ -1387,8 +1384,8 @@ int odp_tm_thresholds_params_read(odp_tm_threshold_t threshold_profile,
/** odp_tm_thresholds_params_update() "sets" the current set of values
* associated with the specified queue thresholds profile object. In addition,
* this call has the effect that all tm_input's and tm_nodes that are
- * associated (attached?) with this queue thresholds profile object will be
- * updated with the new values.
+ * associated with this queue thresholds profile object will be updated with the
+ * new values.
*
* @param threshold_profile Specifies the queue thresholds profile
* object whose values are to be set.
@@ -1413,11 +1410,11 @@ odp_tm_threshold_t odp_tm_thresholds_lookup(const char *name);
/* WRED Profiles - types and functions
* -------------------------------------------------------- */
-/** The odp_tm_wred_params_t record type is used to supply the parameters
- * associated with a Random Early Detection profile. Since it is expected that
- * implementations might augment this record type with platform specific
- * additional fields - it is required that odp_tm_wred_params_init() be called
- * on variables of this type before any of the fields are filled in.
+/**
+ * TM WRED parameters
+ *
+ * Use odp_tm_wred_params_init() to initialize parameters into their default
+ * values.
*/
typedef struct {
/** When min_threshold is set to zero then single-slope WRED is
@@ -1471,12 +1468,12 @@ typedef struct {
odp_bool_t use_byte_fullness;
} odp_tm_wred_params_t;
-/** odp_tm_wred_params_init() must be called to initialize any
- * odp_tm_wred_params_t record before it is first used or assigned to.
- * The parameters are initialized to their default values.
+/**
+ * Initialize TM WRED parameters
*
- * @param params A pointer to an odp_tm_wred_params_t record which
- * is to be initialized.
+ * Initialize an odp_tm_wred_params_t to its default values for all fields.
+ *
+ * @param params Address of the odp_tm_wred_params_t to be initialized
*/
void odp_tm_wred_params_init(odp_tm_wred_params_t *params);
@@ -1524,8 +1521,8 @@ int odp_tm_wred_params_read(odp_tm_wred_t wred_profile,
/** odp_tm_wred_params_update() "sets" the current set of values associated
* with the specified WRED profile object. In addition, this call has the
- * effect that all tm_input's and tm_nodes that are associated (attached?)
- * with this WRED profile object will be updated with the new values.
+ * effect that all tm_input's and tm_nodes that are associated with this WRED
+ * profile object will be updated with the new values.
*
* @param wred_profile Specifies the WRED profile object whose
* values are to be set.
@@ -1547,12 +1544,11 @@ int odp_tm_wred_params_update(odp_tm_wred_t wred_profile,
*/
odp_tm_wred_t odp_tm_wred_lookup(const char *name);
-/** The odp_tm_node_params_t record type is used to hold extra parameters when
- * calling the odp_tm_node_create() function. Many of these fields are
- * optional EXCEPT for max_fanin and level. Also since it is expected that
- * implementations might augment this record type with platform specific
- * additional fields - it is required that odp_tm_node_params_init() be called
- * on variables of this type before any of the fields are filled in.
+/**
+ * TM node parameters
+ *
+ * Many of these fields are optional EXCEPT for max_fanin and level. Use
+ * odp_tm_node_params_init() to initialize parameters into their default values.
*/
typedef struct {
/** The user_context field is an generic pointer that the user can
@@ -1571,9 +1567,9 @@ typedef struct {
odp_tm_shaper_t shaper_profile;
/** The threshold profile to be used in setting the max queue fullness
- * for WRED and/or tail drop? Can be ODP_TM_INVALID and can also be
- * set and changed post-creation via odp_tm_node_threshold_config().
- * The default value is ODP_TM_INVALID. */
+ * for WRED and/or tail drop. Can be ODP_TM_INVALID and can also be set
+ * and changed post-creation via odp_tm_node_threshold_config(). The
+ * default value is ODP_TM_INVALID. */
odp_tm_threshold_t threshold_profile;
/** The WRED profile(s) to be associated with this tm_node. Any or
@@ -1606,12 +1602,12 @@ typedef struct {
uint8_t priority;
} odp_tm_node_params_t;
-/** odp_tm_node_params_init() must be called to initialize any
- * odp_tm_node_params_t record before it is first used or assigned to.
- * The parameters are initialized to their default values.
+/**
+ * Initialize TM node parameters
*
- * @param params A pointer to an odp_tm_node_params_t record which
- * is to be initialized.
+ * Initialize an odp_tm_node_params_t to its default values for all fields.
+ *
+ * @param params Address of the odp_tm_node_params_t to be initialized
*/
void odp_tm_node_params_init(odp_tm_node_params_t *params);
@@ -1627,8 +1623,7 @@ void odp_tm_node_params_init(odp_tm_node_params_t *params);
* @param name Optional name that can be used later later to find this
* same odp_tm_node_t. Can be NULL, otherwise must be
* unique across all odp_tm_node objects.
- * @param params A pointer to a record holding (an extensible) set of
- * properties/attributes of this tm_node.
+ * @param params TM node parameters.
*
* @return Returns ODP_TM_INVALID upon failure, otherwise returns
* a valid odp_tm_node_t handle if successful.
@@ -1739,12 +1734,11 @@ void *odp_tm_node_context(odp_tm_node_t tm_node);
*/
int odp_tm_node_context_set(odp_tm_node_t tm_node, void *user_context);
-/** The odp_tm_queue_params_t record type is used to hold extra parameters
- * when calling the odp_tm_queue_create() function. Many of these fields are
- * optional EXCEPT for priority. Also since it is expected that
- * implementations might augment this record type with platform specific
- * additional fields - it is required that odp_tm_queue_params_init() be
- * called on variables of this type before any of the fields are filled in.
+/**
+ * TM queue parameters
+ *
+ * Use odp_tm_queue_params_init() to initialize parameters into their default
+ * values.
*/
typedef struct {
/** The user_context field is an generic pointer that the user can
@@ -1758,9 +1752,13 @@ typedef struct {
odp_tm_shaper_t shaper_profile;
/** The threshold profile to be used in setting the max queue fullness
- * for WRED and/or tail drop? Can be ODP_TM_INVALID and can also be
- * set and changed post-creation via odp_tm_queue_threshold_config().
- * The default value is ODP_TM_INVALID. */
+ * for WRED and/or tail drop. Can be ODP_TM_INVALID and can also be set
+ * and changed post-creation via odp_tm_queue_threshold_config(). The
+ * default value is ODP_TM_INVALID.
+ *
+ * One can specify the maximum queue limits either as a maximum number
+ * of packets in the queue or as a maximum number of bytes in the queue,
+ * or if both are specified, then whichever limit is hit first. */
odp_tm_threshold_t threshold_profile;
/** The WRED profile(s) to be associated with this tm_queue. Any or
@@ -1782,27 +1780,24 @@ typedef struct {
odp_bool_t ordered_enqueue;
} odp_tm_queue_params_t;
-/** odp_tm_queue_params_init() must be called to initialize any
- * odp_tm_queue_params_t record before it is first used or assigned to.
- * The parameters are initialized to their default values.
+/**
+ * Initialize TM queue parameters
*
- * @param params A pointer to an odp_tm_queue_params_t record which
- * is to be initialized.
+ * Initialize an odp_tm_queue_params_t to its default values for all fields.
+ *
+ * @param params Address of the odp_tm_queue_params_t to be initialized
*/
void odp_tm_queue_params_init(odp_tm_queue_params_t *params);
-/** Create an tm_queue object. One can specify the maximum queue limits
- * either as a maximum number of packets in the queue OR as a maximum number
- * of bytes in the queue, or if both are specified, then whichever limit is
- * hit first. Note that in the case of specifying the maximum queue memory
- * size as bytes, the system is free to instead convert this byte value into a
- * number of buffers and instead limit the queue memory usage by buffer counts
- * versus strictly using byte counts.
+/**
+ * TM queue create
+ *
+ * Create a TM queue according to the queue parameters. Use
+ * odp_tm_queue_params_init() to initialize parameters into their default values.
*
* @param tm Handle of the TM system into which this odp_tm_queue object is
* created.
- * @param params A pointer to a record holding (an extensible) set of
- * properties/attributes of this tm_queue.
+ * @param params TM queue parameters.
*
* @return Returns ODP_TM_INVALID upon failure, otherwise a valid
* odp_tm_queue_t handle.
@@ -1965,18 +1960,16 @@ int odp_tm_queue_disconnect(odp_tm_queue_t tm_queue);
/* Input API
* -------------------------------------------------------- */
-/** The odp_tm_enq() function is used to add packets to a given TM system.
- * Note that the System Metadata associated with the pkt needed by the TM
- * system is (a) a drop_eligible bit, (b) a two bit "pkt_color", (c) a 16-bit
- * pkt_len, and MAYBE? (d) a signed 8-bit shaper_len_adjust.
+/** Send packet to TM system
+ *
+ * Note that the packet metadata utilized by the TM system is (a)
+ * drop_eligible, (b) pkt_color, (c) pkt_len, and (d) shaper_len_adjust.
*
* If there is a non-zero shaper_len_adjust, then it is added to the pkt_len
* after any non-zero shaper_len_adjust that is part of the shaper profile.
*
* The pkt_color bits are a result of some earlier Metering/Marking/Policing
- * processing (typically ingress based), and should not be confused with the
- * shaper_color produced from the TM shaper entities within the tm_inputs and
- * tm_nodes.
+ * processing.
*
* @param tm_queue Specifies the tm_queue (and indirectly the TM system).
* @param pkt Handle to a packet.
@@ -2023,8 +2016,8 @@ int odp_tm_enq_multi_lso(odp_tm_queue_t tm_queue, const odp_packet_t packets[],
const odp_packet_lso_opt_t *lso_opt);
/** The odp_tm_enq_with_cnt() function behaves identically to odp_tm_enq(),
- * except that it also returns (an approximation to?) the current tm_queue
- * packet queue count.
+ * except that it also returns the current tm_queue packet queue count (may be
+ * an approximation).
*
* @param tm_queue Specifies the tm_queue (and indirectly the TM system).
* @param pkt Handle to a packet.
@@ -2194,7 +2187,7 @@ int odp_tm_queue_info(odp_tm_queue_t tm_queue, odp_tm_queue_info_t *info);
*/
#define ODP_TM_QUERY_PKT_CNT 0x01 /**< The total_pkt_cnt value */
#define ODP_TM_QUERY_BYTE_CNT 0x02 /**< The total_byte_cnt value */
-#define ODP_TM_QUERY_THRESHOLDS 0x04 /**< The thresholds??? */
+#define ODP_TM_QUERY_THRESHOLDS 0x04 /**< The threshold values */
/** The odp_tm_query_info_t record type is used to return the various counts
* as requested by functions like odp_tm_queue_query() and
diff --git a/include/odp/api/timer.h b/include/odp/api/timer.h
index 3041594f5..356f7e951 100644
--- a/include/odp/api/timer.h
+++ b/include/odp/api/timer.h
@@ -17,6 +17,8 @@
extern "C" {
#endif
+#include <odp/api/abi/timer.h>
+
#include <odp/api/spec/timer.h>
#ifdef __cplusplus
diff --git a/include/odp/arch/arm32-linux/odp/api/abi/event_types.h b/include/odp/arch/arm32-linux/odp/api/abi/event_types.h
new file mode 100644
index 000000000..a8321920c
--- /dev/null
+++ b/include/odp/arch/arm32-linux/odp/api/abi/event_types.h
@@ -0,0 +1,7 @@
+/* Copyright (c) 2022, Nokia
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+#include <odp/api/abi-default/event_types.h>
diff --git a/include/odp/arch/arm32-linux/odp/api/abi/timer.h b/include/odp/arch/arm32-linux/odp/api/abi/timer.h
new file mode 100644
index 000000000..0351468d3
--- /dev/null
+++ b/include/odp/arch/arm32-linux/odp/api/abi/timer.h
@@ -0,0 +1,7 @@
+/* Copyright (c) 2022, Nokia
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+#include <odp/api/abi-default/timer.h>
diff --git a/include/odp/arch/arm64-linux/odp/api/abi/event_types.h b/include/odp/arch/arm64-linux/odp/api/abi/event_types.h
new file mode 100644
index 000000000..a8321920c
--- /dev/null
+++ b/include/odp/arch/arm64-linux/odp/api/abi/event_types.h
@@ -0,0 +1,7 @@
+/* Copyright (c) 2022, Nokia
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+#include <odp/api/abi-default/event_types.h>
diff --git a/include/odp/arch/arm64-linux/odp/api/abi/timer.h b/include/odp/arch/arm64-linux/odp/api/abi/timer.h
new file mode 100644
index 000000000..0351468d3
--- /dev/null
+++ b/include/odp/arch/arm64-linux/odp/api/abi/timer.h
@@ -0,0 +1,7 @@
+/* Copyright (c) 2022, Nokia
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+#include <odp/api/abi-default/timer.h>
diff --git a/include/odp/arch/default-linux/odp/api/abi/event_types.h b/include/odp/arch/default-linux/odp/api/abi/event_types.h
new file mode 100644
index 000000000..a8321920c
--- /dev/null
+++ b/include/odp/arch/default-linux/odp/api/abi/event_types.h
@@ -0,0 +1,7 @@
+/* Copyright (c) 2022, Nokia
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+#include <odp/api/abi-default/event_types.h>
diff --git a/include/odp/arch/default-linux/odp/api/abi/timer.h b/include/odp/arch/default-linux/odp/api/abi/timer.h
new file mode 100644
index 000000000..0351468d3
--- /dev/null
+++ b/include/odp/arch/default-linux/odp/api/abi/timer.h
@@ -0,0 +1,7 @@
+/* Copyright (c) 2022, Nokia
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+#include <odp/api/abi-default/timer.h>
diff --git a/include/odp/arch/power64-linux/odp/api/abi/event_types.h b/include/odp/arch/power64-linux/odp/api/abi/event_types.h
new file mode 100644
index 000000000..a8321920c
--- /dev/null
+++ b/include/odp/arch/power64-linux/odp/api/abi/event_types.h
@@ -0,0 +1,7 @@
+/* Copyright (c) 2022, Nokia
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+#include <odp/api/abi-default/event_types.h>
diff --git a/include/odp/arch/power64-linux/odp/api/abi/timer.h b/include/odp/arch/power64-linux/odp/api/abi/timer.h
new file mode 100644
index 000000000..0351468d3
--- /dev/null
+++ b/include/odp/arch/power64-linux/odp/api/abi/timer.h
@@ -0,0 +1,7 @@
+/* Copyright (c) 2022, Nokia
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+#include <odp/api/abi-default/timer.h>
diff --git a/include/odp/arch/x86_32-linux/odp/api/abi/event_types.h b/include/odp/arch/x86_32-linux/odp/api/abi/event_types.h
new file mode 100644
index 000000000..a8321920c
--- /dev/null
+++ b/include/odp/arch/x86_32-linux/odp/api/abi/event_types.h
@@ -0,0 +1,7 @@
+/* Copyright (c) 2022, Nokia
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+#include <odp/api/abi-default/event_types.h>
diff --git a/include/odp/arch/x86_32-linux/odp/api/abi/timer.h b/include/odp/arch/x86_32-linux/odp/api/abi/timer.h
new file mode 100644
index 000000000..0351468d3
--- /dev/null
+++ b/include/odp/arch/x86_32-linux/odp/api/abi/timer.h
@@ -0,0 +1,7 @@
+/* Copyright (c) 2022, Nokia
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+#include <odp/api/abi-default/timer.h>
diff --git a/include/odp/arch/x86_64-linux/odp/api/abi/event_types.h b/include/odp/arch/x86_64-linux/odp/api/abi/event_types.h
new file mode 100644
index 000000000..a8321920c
--- /dev/null
+++ b/include/odp/arch/x86_64-linux/odp/api/abi/event_types.h
@@ -0,0 +1,7 @@
+/* Copyright (c) 2022, Nokia
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+#include <odp/api/abi-default/event_types.h>
diff --git a/include/odp/arch/x86_64-linux/odp/api/abi/timer.h b/include/odp/arch/x86_64-linux/odp/api/abi/timer.h
new file mode 100644
index 000000000..0351468d3
--- /dev/null
+++ b/include/odp/arch/x86_64-linux/odp/api/abi/timer.h
@@ -0,0 +1,7 @@
+/* Copyright (c) 2022, Nokia
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+#include <odp/api/abi-default/timer.h>
diff --git a/platform/linux-generic/Makefile.am b/platform/linux-generic/Makefile.am
index 47e5cf759..d4e94f677 100644
--- a/platform/linux-generic/Makefile.am
+++ b/platform/linux-generic/Makefile.am
@@ -49,7 +49,9 @@ odpapiplatinclude_HEADERS = \
include/odp/api/plat/sync_inlines.h \
include/odp/api/plat/thread_inlines.h \
include/odp/api/plat/ticketlock_inlines.h \
- include/odp/api/plat/time_inlines.h
+ include/odp/api/plat/time_inlines.h \
+ include/odp/api/plat/timer_inlines.h \
+ include/odp/api/plat/timer_inline_types.h
odpapiabiarchinclude_HEADERS += \
include-abi/odp/api/abi/align.h \
@@ -65,6 +67,7 @@ odpapiabiarchinclude_HEADERS += \
include-abi/odp/api/abi/dma_types.h \
include-abi/odp/api/abi/errno.h \
include-abi/odp/api/abi/event.h \
+ include-abi/odp/api/abi/event_types.h \
include-abi/odp/api/abi/hash.h \
include-abi/odp/api/abi/init.h \
include-abi/odp/api/abi/ipsec.h \
@@ -95,6 +98,7 @@ odpapiabiarchinclude_HEADERS += \
include-abi/odp/api/abi/thrmask.h \
include-abi/odp/api/abi/ticketlock.h \
include-abi/odp/api/abi/time.h \
+ include-abi/odp/api/abi/timer.h \
include-abi/odp/api/abi/timer_types.h \
include-abi/odp/api/abi/traffic_mngr.h \
include-abi/odp/api/abi/version.h
@@ -126,6 +130,7 @@ noinst_HEADERS = \
include/odp_packet_dpdk.h \
include/odp_packet_internal.h \
include/odp_packet_io_internal.h \
+ include/odp_parse_internal.h \
include/odp_socket_common.h \
include/odp_packet_io_stats_common.h \
include/odp_packet_io_stats.h \
@@ -160,6 +165,7 @@ noinst_HEADERS = \
include/odp_timer_internal.h \
include/odp_timer_wheel_internal.h \
include/odp_traffic_mngr_internal.h \
+ include/odp_types_internal.h \
include/odp_event_vector_internal.h \
include/protocols/eth.h \
include/protocols/ip.h \
@@ -201,6 +207,7 @@ __LIB__libodp_linux_la_SOURCES = \
odp_packet_vector.c \
odp_packet_flags.c \
odp_packet_io.c \
+ odp_parse.c \
odp_pkt_queue.c \
odp_pool.c \
odp_queue_basic.c \
@@ -279,7 +286,8 @@ __LIB__libodp_linux_la_SOURCES += \
odp_sync_api.c \
odp_thread_api.c \
odp_ticketlock_api.c \
- odp_time_api.c
+ odp_time_api.c \
+ odp_timer_api.c
endif
if ARCH_IS_ARM
diff --git a/platform/linux-generic/arch/aarch64/odp_atomic.h b/platform/linux-generic/arch/aarch64/odp_atomic.h
index 445d91e28..d3b8ea4dc 100644
--- a/platform/linux-generic/arch/aarch64/odp_atomic.h
+++ b/platform/linux-generic/arch/aarch64/odp_atomic.h
@@ -12,6 +12,7 @@
#error This file should not be included directly, please include odp_cpu.h
#endif
+#include <odp_types_internal.h>
#include <limits.h>
#ifdef CONFIG_DMBSTR
@@ -36,22 +37,18 @@ do { \
#define LL_MO(mo) (HAS_ACQ((mo)) ? __ATOMIC_ACQUIRE : __ATOMIC_RELAXED)
#define SC_MO(mo) (HAS_RLS((mo)) ? __ATOMIC_RELEASE : __ATOMIC_RELAXED)
-/* Prevent warnings about ISO C not supporting __int128 */
-#pragma GCC diagnostic push
-#pragma GCC diagnostic ignored "-Wpedantic"
-
#ifndef __ARM_FEATURE_QRDMX /* Feature only available in v8.1a and beyond */
static inline bool
-__lockfree_compare_exchange_16(register __int128 *var, __int128 *exp,
- register __int128 neu, bool weak, int mo_success,
+__lockfree_compare_exchange_16(register _odp_u128_t *var, _odp_u128_t *exp,
+ register _odp_u128_t neu, bool weak, int mo_success,
int mo_failure)
{
(void)weak; /* Always do strong CAS or we can't perform atomic read */
/* Ignore memory ordering for failure, memory order for
* success must be stronger or equal. */
(void)mo_failure;
- register __int128 old;
- register __int128 expected;
+ register _odp_u128_t old;
+ register _odp_u128_t expected;
int ll_mo = LL_MO(mo_success);
int sc_mo = SC_MO(mo_success);
@@ -66,10 +63,10 @@ __lockfree_compare_exchange_16(register __int128 *var, __int128 *exp,
return old == expected;
}
-static inline __int128 __lockfree_exchange_16(__int128 *var, __int128 neu,
- int mo)
+static inline _odp_u128_t __lockfree_exchange_16(_odp_u128_t *var,
+ _odp_u128_t neu, int mo)
{
- register __int128 old;
+ register _odp_u128_t old;
int ll_mo = LL_MO(mo);
int sc_mo = SC_MO(mo);
@@ -81,10 +78,10 @@ static inline __int128 __lockfree_exchange_16(__int128 *var, __int128 neu,
return old;
}
-static inline __int128 __lockfree_fetch_and_16(__int128 *var, __int128 mask,
- int mo)
+static inline _odp_u128_t __lockfree_fetch_and_16(_odp_u128_t *var,
+ _odp_u128_t mask, int mo)
{
- register __int128 old;
+ register _odp_u128_t old;
int ll_mo = LL_MO(mo);
int sc_mo = SC_MO(mo);
@@ -96,10 +93,10 @@ static inline __int128 __lockfree_fetch_and_16(__int128 *var, __int128 mask,
return old;
}
-static inline __int128 __lockfree_fetch_or_16(__int128 *var, __int128 mask,
- int mo)
+static inline _odp_u128_t __lockfree_fetch_or_16(_odp_u128_t *var,
+ _odp_u128_t mask, int mo)
{
- register __int128 old;
+ register _odp_u128_t old;
int ll_mo = LL_MO(mo);
int sc_mo = SC_MO(mo);
@@ -113,8 +110,8 @@ static inline __int128 __lockfree_fetch_or_16(__int128 *var, __int128 mask,
#else
-static inline __int128_t cas_u128(__int128_t *ptr, __int128_t old_val,
- __int128_t new_val, int mo)
+static inline _odp_u128_t cas_u128(_odp_u128_t *ptr, _odp_u128_t old_val,
+ _odp_u128_t new_val, int mo)
{
/* CASP instructions require that the first register number is paired */
register uint64_t old0 __asm__ ("x0");
@@ -151,18 +148,18 @@ static inline __int128_t cas_u128(__int128_t *ptr, __int128_t old_val,
abort();
}
- return ((__int128)old0) | (((__int128)old1) << 64);
+ return ((_odp_u128_t)old0) | (((_odp_u128_t)old1) << 64);
}
static inline bool
-__lockfree_compare_exchange_16(register __int128 *var, __int128 *exp,
- register __int128 neu, bool weak, int mo_success,
+__lockfree_compare_exchange_16(register _odp_u128_t *var, _odp_u128_t *exp,
+ register _odp_u128_t neu, bool weak, int mo_success,
int mo_failure)
{
(void)weak;
(void)mo_failure;
- __int128 old;
- __int128 expected;
+ _odp_u128_t old;
+ _odp_u128_t expected;
expected = *exp;
old = cas_u128(var, expected, neu, mo_success);
@@ -170,11 +167,11 @@ __lockfree_compare_exchange_16(register __int128 *var, __int128 *exp,
return old == expected;
}
-static inline __int128 __lockfree_exchange_16(__int128 *var, __int128 neu,
- int mo)
+static inline _odp_u128_t __lockfree_exchange_16(_odp_u128_t *var,
+ _odp_u128_t neu, int mo)
{
- __int128 old;
- __int128 expected;
+ _odp_u128_t old;
+ _odp_u128_t expected;
do {
expected = *var;
@@ -183,11 +180,11 @@ static inline __int128 __lockfree_exchange_16(__int128 *var, __int128 neu,
return old;
}
-static inline __int128 __lockfree_fetch_and_16(__int128 *var, __int128 mask,
- int mo)
+static inline _odp_u128_t __lockfree_fetch_and_16(_odp_u128_t *var,
+ _odp_u128_t mask, int mo)
{
- __int128 old;
- __int128 expected;
+ _odp_u128_t old;
+ _odp_u128_t expected;
do {
expected = *var;
@@ -196,11 +193,11 @@ static inline __int128 __lockfree_fetch_and_16(__int128 *var, __int128 mask,
return old;
}
-static inline __int128 __lockfree_fetch_or_16(__int128 *var, __int128 mask,
- int mo)
+static inline _odp_u128_t __lockfree_fetch_or_16(_odp_u128_t *var,
+ _odp_u128_t mask, int mo)
{
- __int128 old;
- __int128 expected;
+ _odp_u128_t old;
+ _odp_u128_t expected;
do {
expected = *var;
@@ -211,9 +208,9 @@ static inline __int128 __lockfree_fetch_or_16(__int128 *var, __int128 mask,
#endif /* __ARM_FEATURE_QRDMX */
-static inline __int128 __lockfree_load_16(__int128 *var, int mo)
+static inline _odp_u128_t __lockfree_load_16(_odp_u128_t *var, int mo)
{
- __int128 old = *var; /* Possibly torn read */
+ _odp_u128_t old = *var; /* Possibly torn read */
/* Do CAS to ensure atomicity
* Either CAS succeeds (writing back the same value)
@@ -223,19 +220,17 @@ static inline __int128 __lockfree_load_16(__int128 *var, int mo)
return old;
}
-__extension__ typedef unsigned __int128 _u128_t;
-
-static inline _u128_t lockfree_load_u128(_u128_t *atomic)
+static inline _odp_u128_t lockfree_load_u128(_odp_u128_t *atomic)
{
- return __lockfree_load_16((__int128 *)atomic, __ATOMIC_RELAXED);
+ return __lockfree_load_16((_odp_u128_t *)atomic, __ATOMIC_RELAXED);
}
-static inline int lockfree_cas_acq_rel_u128(_u128_t *atomic,
- _u128_t old_val,
- _u128_t new_val)
+static inline int lockfree_cas_acq_rel_u128(_odp_u128_t *atomic,
+ _odp_u128_t old_val,
+ _odp_u128_t new_val)
{
- return __lockfree_compare_exchange_16((__int128 *)atomic,
- (__int128 *)&old_val,
+ return __lockfree_compare_exchange_16((_odp_u128_t *)atomic,
+ (_odp_u128_t *)&old_val,
new_val,
0,
__ATOMIC_ACQ_REL,
@@ -249,7 +244,7 @@ static inline int lockfree_check_u128(void)
/** Atomic bit set operations with memory ordering */
#if defined(__SIZEOF_INT128__) && __SIZEOF_INT128__ == 16
-typedef __int128 bitset_t;
+typedef _odp_u128_t bitset_t;
#define ATOM_BITSET_SIZE (CHAR_BIT * __SIZEOF_INT128__)
#elif __GCC_ATOMIC_LLONG_LOCK_FREE == 2 && \
@@ -292,11 +287,9 @@ static inline bitset_t bitset_mask(uint32_t bit)
if (bit < 64)
return 1ULL << bit;
else
- return (unsigned __int128)(1ULL << (bit - 64)) << 64;
+ return (_odp_u128_t)(1ULL << (bit - 64)) << 64;
}
-#pragma GCC diagnostic pop
-
#else
#error Unsupported size of bit sets (ATOM_BITSET_SIZE)
#endif
diff --git a/platform/linux-generic/arch/aarch64/odp_crypto_armv8.c b/platform/linux-generic/arch/aarch64/odp_crypto_armv8.c
index d719c6cfa..4d46846ce 100644
--- a/platform/linux-generic/arch/aarch64/odp_crypto_armv8.c
+++ b/platform/linux-generic/arch/aarch64/odp_crypto_armv8.c
@@ -1,5 +1,6 @@
/* Copyright (c) 2014-2018, Linaro Limited
* Copyright (c) 2021, ARM Limited
+ * Copyright (c) 2022, Nokia
* All rights reserved.
*
* SPDX-License-Identifier: BSD-3-Clause
@@ -37,9 +38,9 @@
/*
* ARM crypto library may read up to 15 bytes past the end of input
- * data and AAD.
+ * data and AAD and write up to 15 bytes past the end of output data.
*/
-#define OOB_READ_LEN 15
+#define OOB_WRITE_LEN 16 /* rounded up to 16 bytes for efficiency */
/*
* Data buffer size must be a multiple of 16, because the ARM crypto
@@ -80,9 +81,11 @@ static const odp_crypto_cipher_capability_t cipher_capa_aes_gcm[] = {
static const odp_crypto_auth_capability_t auth_capa_null[] = {
{.digest_len = 0, .key_len = 0, .aad_len = {.min = 0, .max = 0, .inc = 0} } };
+#define AES_GCM_TAG_LEN 16
+
#ifdef __ARM_FEATURE_AES
static const odp_crypto_auth_capability_t auth_capa_aes_gcm[] = {
-{.digest_len = 16, .key_len = 0, .aad_len = {.min = 8, .max = 12, .inc = 4} } };
+{.digest_len = AES_GCM_TAG_LEN, .key_len = 0, .aad_len = {.min = 8, .max = 12, .inc = 4} } };
#endif
/** Forward declaration of session structure */
@@ -92,9 +95,9 @@ typedef struct odp_crypto_generic_session_t odp_crypto_generic_session_t;
* Algorithm handler function prototype
*/
typedef
-odp_crypto_alg_err_t (*crypto_func_t)(odp_packet_t pkt,
- const odp_crypto_packet_op_param_t *param,
- odp_crypto_generic_session_t *session);
+odp_bool_t (*crypto_func_t)(odp_packet_t pkt,
+ const odp_crypto_packet_op_param_t *param,
+ odp_crypto_generic_session_t *session);
/**
* Per crypto session data structure
@@ -105,15 +108,12 @@ struct odp_crypto_generic_session_t {
/* Session creation parameters */
odp_crypto_session_param_t p;
- odp_bool_t do_cipher_first;
-
struct {
#if ODP_DEPRECATED_API
/* Copy of session IV data */
uint8_t iv_data[ARM_CRYPTO_MAX_IV_LENGTH];
#endif
uint8_t key_data[ARM_CRYPTO_MAX_CIPHER_KEY_LENGTH];
- crypto_func_t func;
} cipher;
struct {
@@ -121,11 +121,10 @@ struct odp_crypto_generic_session_t {
#if ODP_DEPRECATED_API
uint8_t iv_data[ARM_CRYPTO_MAX_IV_LENGTH];
#endif
- crypto_func_t func;
} auth;
+ crypto_func_t func;
unsigned int idx;
-
armv8_cipher_constants_t cc;
};
@@ -141,7 +140,6 @@ static odp_crypto_global_t *global;
typedef struct crypto_local_t {
uint8_t buffer[ARM_CRYPTO_MAX_DATA_LENGTH];
- uint8_t digest[ARM_CRYPTO_MAX_DIGEST_LENGTH];
} crypto_local_t;
static __thread crypto_local_t local;
@@ -176,18 +174,60 @@ void free_session(odp_crypto_generic_session_t *session)
odp_spinlock_unlock(&global->lock);
}
-static odp_crypto_alg_err_t
+static
+odp_crypto_packet_result_t *get_op_result_from_packet(odp_packet_t pkt)
+{
+ odp_packet_hdr_t *hdr = packet_hdr(pkt);
+
+ return &hdr->crypto_op_result;
+}
+
+static inline void set_crypto_op_result(odp_packet_t pkt,
+ odp_crypto_alg_err_t cipher_err,
+ odp_crypto_alg_err_t auth_err)
+{
+ odp_crypto_packet_result_t *op_result;
+
+ op_result = get_op_result_from_packet(pkt);
+ op_result->cipher_status.alg_err = cipher_err;
+ op_result->cipher_status.hw_err = ODP_CRYPTO_HW_ERR_NONE;
+ op_result->auth_status.alg_err = auth_err;
+ op_result->auth_status.hw_err = ODP_CRYPTO_HW_ERR_NONE;
+ op_result->ok = (cipher_err == ODP_CRYPTO_ALG_ERR_NONE &&
+ auth_err == ODP_CRYPTO_ALG_ERR_NONE);
+}
+
+static inline void set_crypto_op_result_ok(odp_packet_t pkt)
+{
+ set_crypto_op_result(pkt,
+ ODP_CRYPTO_ALG_ERR_NONE,
+ ODP_CRYPTO_ALG_ERR_NONE);
+}
+
+static odp_bool_t
null_crypto_routine(odp_packet_t pkt ODP_UNUSED,
const odp_crypto_packet_op_param_t *param ODP_UNUSED,
odp_crypto_generic_session_t *session ODP_UNUSED)
{
- return ODP_CRYPTO_ALG_ERR_NONE;
+ set_crypto_op_result_ok(pkt);
+ return true;
+}
+
+static inline void copy_aad(uint8_t *dst, uint8_t *src, uint32_t len)
+{
+ ODP_ASSERT(len == 8 || len == 12);
+
+ /* Use constant length memcpy for better optimization result */
+ if (len == 8)
+ memcpy(dst, src, 8);
+ else
+ memcpy(dst, src, 12);
}
static
-odp_crypto_alg_err_t aes_gcm_encrypt(odp_packet_t pkt,
- const odp_crypto_packet_op_param_t *param,
- odp_crypto_generic_session_t *session)
+odp_bool_t aes_gcm_encrypt(odp_packet_t pkt,
+ const odp_crypto_packet_op_param_t *param,
+ odp_crypto_generic_session_t *session)
{
armv8_cipher_state_t cs = {
.counter = {
@@ -200,12 +240,14 @@ odp_crypto_alg_err_t aes_gcm_encrypt(odp_packet_t pkt,
uint64_t aad_bit_length = session->p.auth_aad_len * 8;
uint32_t in_pos = param->cipher_range.offset;
uint32_t in_len = param->cipher_range.length;
- int ret = 0;
+ odp_bool_t continuous_data;
+ uint16_t saved_tail[OOB_WRITE_LEN];
+ uint8_t tag[AES_GCM_TAG_LEN];
+ int rc;
/* Fail early if cipher_range is too large */
- if (in_len > ARM_CRYPTO_MAX_DATA_LENGTH) {
+ if (odp_unlikely(in_len > ARM_CRYPTO_MAX_DATA_LENGTH)) {
ODP_DBG("ARM Crypto: Packet size too large for requested operation\n");
- ret = -1;
goto err;
}
@@ -223,9 +265,9 @@ odp_crypto_alg_err_t aes_gcm_encrypt(odp_packet_t pkt,
cs.constants = &session->cc;
- if (armv8_aes_gcm_set_counter(iv_ptr, iv_bit_length, &cs) != 0) {
+ rc = armv8_aes_gcm_set_counter(iv_ptr, iv_bit_length, &cs);
+ if (odp_unlikely(rc)) {
ODP_DBG("ARM Crypto: Failure while setting nonce\n");
- ret = -1;
goto err;
}
@@ -233,46 +275,60 @@ odp_crypto_alg_err_t aes_gcm_encrypt(odp_packet_t pkt,
* read it in 16 byte chunks. */
uint8_t aad[ARM_CRYPTO_MAX_AAD_LENGTH];
- memcpy(aad, param->aad_ptr, session->p.auth_aad_len);
+ copy_aad(aad, param->aad_ptr, session->p.auth_aad_len);
uint32_t seg_len = 0;
uint8_t *data = odp_packet_offset(pkt, in_pos, &seg_len, NULL);
if (odp_unlikely(odp_packet_is_segmented(pkt)) ||
- odp_unlikely(odp_packet_tailroom(pkt) < OOB_READ_LEN)) {
- /* Packet is segmented or it may not be safe to read beyond
- * the end of packet data. Copy the cipher range to a
+ odp_unlikely(odp_packet_tailroom(pkt) < OOB_WRITE_LEN)) {
+ /* Packet is segmented or it may not be safe to read and write
+ * beyond the end of packet data. Copy the cipher range to a
* contiguous buffer. */
odp_packet_copy_to_mem(pkt, in_pos, in_len, local.buffer);
data = local.buffer;
+ continuous_data = false;
+ } else {
+ /* Save data that might get overwritten */
+ memcpy(saved_tail, data + in_len, OOB_WRITE_LEN);
+ continuous_data = true;
}
- if (armv8_enc_aes_gcm_from_state(&cs,
- aad, aad_bit_length,
- data, plaintext_bit_length,
- local.buffer,
- local.digest) != 0) {
+ rc = armv8_enc_aes_gcm_from_state(&cs,
+ aad, aad_bit_length,
+ data, plaintext_bit_length,
+ data,
+ tag);
+ if (odp_unlikely(rc)) {
ODP_DBG("ARM Crypto: AES GCM Encoding failed\n");
- ret = -1;
goto err;
}
- odp_packet_copy_from_mem(pkt, param->cipher_range.offset,
- param->cipher_range.length, local.buffer);
+ if (odp_likely(continuous_data)) {
+ memcpy(data + in_len, saved_tail, OOB_WRITE_LEN);
+ memcpy(data - in_pos + param->hash_result_offset,
+ tag, AES_GCM_TAG_LEN);
+ } else {
+ odp_packet_copy_from_mem(pkt, in_pos, in_len, data);
+ odp_packet_copy_from_mem(pkt, param->hash_result_offset,
+ AES_GCM_TAG_LEN, tag);
+ }
- odp_packet_copy_from_mem(pkt, param->hash_result_offset,
- session->p.auth_digest_len, local.digest);
+ set_crypto_op_result_ok(pkt);
+ return true;
err:
- return ret < 0 ? ODP_CRYPTO_ALG_ERR_DATA_SIZE :
- ODP_CRYPTO_ALG_ERR_NONE;
+ set_crypto_op_result(pkt,
+ ODP_CRYPTO_ALG_ERR_DATA_SIZE,
+ ODP_CRYPTO_ALG_ERR_NONE);
+ return false;
}
static
-odp_crypto_alg_err_t aes_gcm_decrypt(odp_packet_t pkt,
- const odp_crypto_packet_op_param_t *param,
- odp_crypto_generic_session_t *session)
+odp_bool_t aes_gcm_decrypt(odp_packet_t pkt,
+ const odp_crypto_packet_op_param_t *param,
+ odp_crypto_generic_session_t *session)
{
armv8_cipher_state_t cs = {
.counter = {
@@ -280,18 +336,19 @@ odp_crypto_alg_err_t aes_gcm_decrypt(odp_packet_t pkt,
}
};
uint8_t *iv_ptr;
- uint8_t tag[16];
+ uint8_t tag[AES_GCM_TAG_LEN];
uint64_t iv_bit_length = session->p.cipher_iv_len * 8;
uint64_t plaintext_bit_length = param->cipher_range.length * 8;
uint64_t aad_bit_length = session->p.auth_aad_len * 8;
uint32_t in_pos = param->cipher_range.offset;
uint32_t in_len = param->cipher_range.length;
- int ret = 0;
+ odp_bool_t continuous_data;
+ uint16_t saved_tail[OOB_WRITE_LEN];
+ int rc;
/* Fail early if cipher_range is too large */
- if (in_len > ARM_CRYPTO_MAX_DATA_LENGTH) {
+ if (odp_unlikely(in_len > ARM_CRYPTO_MAX_DATA_LENGTH)) {
ODP_DBG("ARM Crypto: Packet size too large for requested operation\n");
- ret = -1;
goto err;
}
@@ -309,51 +366,63 @@ odp_crypto_alg_err_t aes_gcm_decrypt(odp_packet_t pkt,
cs.constants = &session->cc;
- if (armv8_aes_gcm_set_counter(iv_ptr, iv_bit_length, &cs) != 0) {
+ rc = armv8_aes_gcm_set_counter(iv_ptr, iv_bit_length, &cs);
+ if (odp_unlikely(rc)) {
ODP_DBG("ARM Crypto: Failure while setting nonce\n");
- ret = -1;
goto err;
}
- /* Copy current hash bytes to a buffer and clear it */
- odp_packet_copy_to_mem(pkt, param->hash_result_offset,
- session->p.auth_digest_len, tag);
-
/* Copy AAD in a stack to make sure that the ARM crypto library can
* read it in 16 byte chunks. */
uint8_t aad[ARM_CRYPTO_MAX_AAD_LENGTH];
- memcpy(aad, param->aad_ptr, session->p.auth_aad_len);
+ copy_aad(aad, param->aad_ptr, session->p.auth_aad_len);
uint32_t seg_len = 0;
uint8_t *data = odp_packet_offset(pkt, in_pos, &seg_len, NULL);
if (odp_unlikely(odp_packet_is_segmented(pkt)) ||
- odp_unlikely(odp_packet_tailroom(pkt) < OOB_READ_LEN)) {
- /* Packet is segmented or it may not be safe to read beyond
- * the end of packet data. Copy the cipher range to a
+ odp_unlikely(odp_packet_tailroom(pkt) < OOB_WRITE_LEN)) {
+ /* Packet is segmented or it may not be safe to read and write
+ * beyond the end of packet data. Copy the cipher range to a
* contiguous buffer. */
odp_packet_copy_to_mem(pkt, in_pos, in_len, local.buffer);
-
data = local.buffer;
+ /* Copy tag from the packet to a buffer */
+ odp_packet_copy_to_mem(pkt, param->hash_result_offset,
+ AES_GCM_TAG_LEN, tag);
+ continuous_data = false;
+ } else {
+ /* Save data that might get overwritten */
+ memcpy(saved_tail, data + in_len, OOB_WRITE_LEN);
+ /* Copy tag from the packet to a buffer */
+ memcpy(tag, data - in_pos + param->hash_result_offset, AES_GCM_TAG_LEN);
+ continuous_data = true;
}
- if (armv8_dec_aes_gcm_from_state(&cs,
- aad, aad_bit_length,
- data, plaintext_bit_length,
- tag,
- local.buffer) != 0) {
+ rc = armv8_dec_aes_gcm_from_state(&cs,
+ aad, aad_bit_length,
+ data, plaintext_bit_length,
+ tag,
+ data);
+ if (odp_unlikely(rc)) {
ODP_DBG("ARM Crypto: AES GCM Decoding failed\n");
- ret = -1;
goto err;
}
- odp_packet_copy_from_mem(pkt, param->cipher_range.offset,
- param->cipher_range.length, local.buffer);
+ if (odp_likely(continuous_data))
+ memcpy(data + in_len, saved_tail, OOB_WRITE_LEN);
+ else
+ odp_packet_copy_from_mem(pkt, in_pos, in_len, data);
+
+ set_crypto_op_result_ok(pkt);
+ return true;
err:
- return ret < 0 ? ODP_CRYPTO_ALG_ERR_ICV_CHECK :
- ODP_CRYPTO_ALG_ERR_NONE;
+ set_crypto_op_result(pkt,
+ ODP_CRYPTO_ALG_ERR_NONE,
+ ODP_CRYPTO_ALG_ERR_ICV_CHECK);
+ return false;
}
static int process_aes_gcm_param(odp_crypto_generic_session_t *session)
@@ -376,9 +445,9 @@ static int process_aes_gcm_param(odp_crypto_generic_session_t *session)
/* Set function */
if (ODP_CRYPTO_OP_ENCODE == session->p.op)
- session->cipher.func = aes_gcm_encrypt;
+ session->func = aes_gcm_encrypt;
else
- session->cipher.func = aes_gcm_decrypt;
+ session->func = aes_gcm_decrypt;
return 0;
}
@@ -520,16 +589,10 @@ odp_crypto_session_create(const odp_crypto_session_param_t *param,
session->p.auth_iv.length);
#endif
- /* Derive order */
- if (ODP_CRYPTO_OP_ENCODE == param->op)
- session->do_cipher_first = param->auth_cipher_text;
- else
- session->do_cipher_first = !param->auth_cipher_text;
-
/* Process based on cipher */
switch (param->cipher_alg) {
case ODP_CIPHER_ALG_NULL:
- session->cipher.func = null_crypto_routine;
+ session->func = null_crypto_routine;
rc = 0;
break;
case ODP_CIPHER_ALG_AES_GCM:
@@ -584,14 +647,15 @@ odp_crypto_session_create(const odp_crypto_session_param_t *param,
/* Process based on auth */
switch (param->auth_alg) {
case ODP_AUTH_ALG_NULL:
- session->auth.func = null_crypto_routine;
- rc = 0;
+ if (param->cipher_alg == ODP_CIPHER_ALG_NULL)
+ rc = 0;
+ else
+ rc = -1;
break;
case ODP_AUTH_ALG_AES_GCM:
/* AES-GCM requires to do both auth and
* cipher at the same time */
if (param->cipher_alg == ODP_CIPHER_ALG_AES_GCM) {
- session->auth.func = null_crypto_routine;
rc = 0;
} else {
rc = -1;
@@ -630,9 +694,7 @@ int odp_crypto_session_destroy(odp_crypto_session_t session)
return 0;
}
-/*
- * Shim function around packet operation, can be used by other implementations.
- */
+#if ODP_DEPRECATED_API
int
odp_crypto_operation(odp_crypto_op_param_t *param,
odp_bool_t *posted,
@@ -687,6 +749,7 @@ odp_crypto_operation(odp_crypto_op_param_t *param,
return 0;
}
+#endif
int _odp_crypto_init_global(void)
{
@@ -767,6 +830,7 @@ int _odp_crypto_term_local(void)
return 0;
}
+#if ODP_DEPRECATED_API
odp_crypto_compl_t odp_crypto_compl_from_event(odp_event_t ev)
{
/* This check not mandated by the API specification */
@@ -803,6 +867,7 @@ uint64_t odp_crypto_compl_to_u64(odp_crypto_compl_t hdl)
{
return _odp_pri(hdl);
}
+#endif /* ODP_DEPRECATED_API */
void odp_crypto_session_param_init(odp_crypto_session_param_t *param)
{
@@ -828,14 +893,6 @@ odp_event_t odp_crypto_packet_to_event(odp_packet_t pkt)
return odp_packet_to_event(pkt);
}
-static
-odp_crypto_packet_result_t *get_op_result_from_packet(odp_packet_t pkt)
-{
- odp_packet_hdr_t *hdr = packet_hdr(pkt);
-
- return &hdr->crypto_op_result;
-}
-
int odp_crypto_result(odp_crypto_packet_result_t *result,
odp_packet_t packet)
{
@@ -856,18 +913,16 @@ int crypto_int(odp_packet_t pkt_in,
odp_packet_t *pkt_out,
const odp_crypto_packet_op_param_t *param)
{
- odp_crypto_alg_err_t rc_cipher = ODP_CRYPTO_ALG_ERR_NONE;
- odp_crypto_alg_err_t rc_auth = ODP_CRYPTO_ALG_ERR_NONE;
odp_crypto_generic_session_t *session;
odp_bool_t allocated = false;
odp_packet_t out_pkt = *pkt_out;
- odp_crypto_packet_result_t *op_result;
odp_packet_hdr_t *pkt_hdr;
+ odp_bool_t ok;
session = (odp_crypto_generic_session_t *)(intptr_t)param->session;
/* Resolve output buffer */
- if (ODP_PACKET_INVALID == out_pkt &&
+ if (odp_unlikely(ODP_PACKET_INVALID == out_pkt) &&
ODP_POOL_INVALID != session->p.output_pool) {
out_pkt = odp_packet_alloc(session->p.output_pool,
odp_packet_len(pkt_in));
@@ -879,7 +934,7 @@ int crypto_int(odp_packet_t pkt_in,
return -1;
}
- if (pkt_in != out_pkt) {
+ if (odp_unlikely(pkt_in != out_pkt)) {
int ret;
int md_copy;
@@ -903,28 +958,12 @@ int crypto_int(odp_packet_t pkt_in,
pkt_in = ODP_PACKET_INVALID;
}
- /* Invoke the functions */
- if (session->do_cipher_first) {
- rc_cipher = session->cipher.func(out_pkt, param, session);
- rc_auth = session->auth.func(out_pkt, param, session);
- } else {
- rc_auth = session->auth.func(out_pkt, param, session);
- rc_cipher = session->cipher.func(out_pkt, param, session);
- }
+ /* Invoke the crypto function */
+ ok = session->func(out_pkt, param, session);
- /* Fill in result */
packet_subtype_set(out_pkt, ODP_EVENT_PACKET_CRYPTO);
- op_result = get_op_result_from_packet(out_pkt);
- op_result->cipher_status.alg_err = rc_cipher;
- op_result->cipher_status.hw_err = ODP_CRYPTO_HW_ERR_NONE;
- op_result->auth_status.alg_err = rc_auth;
- op_result->auth_status.hw_err = ODP_CRYPTO_HW_ERR_NONE;
- op_result->ok =
- (rc_cipher == ODP_CRYPTO_ALG_ERR_NONE) &&
- (rc_auth == ODP_CRYPTO_ALG_ERR_NONE);
-
pkt_hdr = packet_hdr(out_pkt);
- pkt_hdr->p.flags.crypto_err = !op_result->ok;
+ pkt_hdr->p.flags.crypto_err = !ok;
/* Synchronous, simply return results */
*pkt_out = out_pkt;
diff --git a/platform/linux-generic/arch/aarch64/odp_llsc.h b/platform/linux-generic/arch/aarch64/odp_llsc.h
index 2561451ca..1c2610511 100644
--- a/platform/linux-generic/arch/aarch64/odp_llsc.h
+++ b/platform/linux-generic/arch/aarch64/odp_llsc.h
@@ -13,6 +13,8 @@
#error This file should not be included directly, please include odp_cpu.h
#endif
+#include <odp_types_internal.h>
+
static inline uint16_t ll8(uint8_t *var, int mm)
{
uint16_t old;
@@ -115,11 +117,11 @@ static inline uint32_t sc(uint64_t *var, uint64_t neu, int mm)
#define sc64(a, b, c) sc((a), (b), (c))
union i128 {
- __extension__ __int128 i128;
+ _odp_u128_t i128;
int64_t i64[2];
};
-__extension__ static inline __int128 lld(__int128 *var, int mm)
+static inline _odp_u128_t lld(_odp_u128_t *var, int mm)
{
union i128 old;
@@ -139,7 +141,7 @@ __extension__ static inline __int128 lld(__int128 *var, int mm)
}
/* Return 0 on success, 1 on failure */
-__extension__ static inline uint32_t scd(__int128 *var, __int128 neu, int mm)
+static inline uint32_t scd(_odp_u128_t *var, _odp_u128_t neu, int mm)
{
uint32_t ret;
diff --git a/platform/linux-generic/arch/arm/odp_atomic.h b/platform/linux-generic/arch/arm/odp_atomic.h
index bcc89ff73..e400f52d4 100644
--- a/platform/linux-generic/arch/arm/odp_atomic.h
+++ b/platform/linux-generic/arch/arm/odp_atomic.h
@@ -12,6 +12,7 @@
#error This file should not be included directly, please include odp_cpu.h
#endif
+#include <odp_types_internal.h>
#include <limits.h>
#ifdef CONFIG_DMBSTR
@@ -70,7 +71,7 @@ static inline bitset_t bitset_mask(uint32_t bit)
if (bit < 64)
return 1ULL << bit;
else
- return (unsigned __int128)(1ULL << (bit - 64)) << 64;
+ return (_odp_u128_t)(1ULL << (bit - 64)) << 64;
}
#else
diff --git a/platform/linux-generic/arch/default/odp_atomic.h b/platform/linux-generic/arch/default/odp_atomic.h
index 7650d2b52..4cfc6b4bd 100644
--- a/platform/linux-generic/arch/default/odp_atomic.h
+++ b/platform/linux-generic/arch/default/odp_atomic.h
@@ -7,18 +7,18 @@
#ifndef ODP_DEFAULT_ATOMIC_H_
#define ODP_DEFAULT_ATOMIC_H_
-#ifdef __SIZEOF_INT128__
+#include <odp_types_internal.h>
-__extension__ typedef unsigned __int128 _u128_t;
+#ifdef __SIZEOF_INT128__
-static inline _u128_t lockfree_load_u128(_u128_t *atomic)
+static inline _odp_u128_t lockfree_load_u128(_odp_u128_t *atomic)
{
return __atomic_load_n(atomic, __ATOMIC_RELAXED);
}
-static inline int lockfree_cas_acq_rel_u128(_u128_t *atomic,
- _u128_t old_val,
- _u128_t new_val)
+static inline int lockfree_cas_acq_rel_u128(_odp_u128_t *atomic,
+ _odp_u128_t old_val,
+ _odp_u128_t new_val)
{
return __atomic_compare_exchange_n(atomic, &old_val, new_val,
0 /* strong */,
@@ -76,7 +76,7 @@ static inline bitset_t bitset_mask(uint32_t bit)
if (bit < 64)
return 1ULL << bit;
else
- return (unsigned __int128)(1ULL << (bit - 64)) << 64;
+ return (_odp_u128_t)(1ULL << (bit - 64)) << 64;
}
#else
diff --git a/platform/linux-generic/include-abi/odp/api/abi/event.h b/platform/linux-generic/include-abi/odp/api/abi/event.h
index 6530ac2e4..e059f318c 100644
--- a/platform/linux-generic/include-abi/odp/api/abi/event.h
+++ b/platform/linux-generic/include-abi/odp/api/abi/event.h
@@ -1,4 +1,5 @@
/* Copyright (c) 2015-2018, Linaro Limited
+ * Copyright (c) 2022, Nokia
* All rights reserved.
*
* SPDX-License-Identifier: BSD-3-Clause
@@ -18,42 +19,9 @@
extern "C" {
#endif
-#include <odp/api/plat/strong_types.h>
-
-/** @ingroup odp_event
- * @{
- */
-
-typedef ODP_HANDLE_T(odp_event_t);
-
-#define ODP_EVENT_INVALID _odp_cast_scalar(odp_event_t, 0)
-
-typedef enum odp_event_type_t {
- ODP_EVENT_BUFFER = 1,
- ODP_EVENT_PACKET = 2,
- ODP_EVENT_TIMEOUT = 3,
- ODP_EVENT_CRYPTO_COMPL = 4,
- ODP_EVENT_IPSEC_STATUS = 5,
- ODP_EVENT_PACKET_VECTOR = 6,
- ODP_EVENT_PACKET_TX_COMPL = 7,
- ODP_EVENT_DMA_COMPL = 8,
-} odp_event_type_t;
-
-typedef enum odp_event_subtype_t {
- ODP_EVENT_NO_SUBTYPE = 0,
- ODP_EVENT_PACKET_BASIC = 1,
- ODP_EVENT_PACKET_CRYPTO = 2,
- ODP_EVENT_PACKET_IPSEC = 3,
- ODP_EVENT_PACKET_COMP = 4
-} odp_event_subtype_t;
-
/* Inlined functions for non-ABI compat mode */
#include <odp/api/plat/event_inlines.h>
-/**
- * @}
- */
-
#ifdef __cplusplus
}
#endif
diff --git a/platform/linux-generic/include-abi/odp/api/abi/event_types.h b/platform/linux-generic/include-abi/odp/api/abi/event_types.h
new file mode 100644
index 000000000..c39322330
--- /dev/null
+++ b/platform/linux-generic/include-abi/odp/api/abi/event_types.h
@@ -0,0 +1,61 @@
+/* Copyright (c) 2015-2018, Linaro Limited
+ * Copyright (c) 2022, Nokia
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+/**
+ * @file
+ *
+ * ODP event type definitions
+ */
+
+#ifndef ODP_API_ABI_EVENT_TYPES_H_
+#define ODP_API_ABI_EVENT_TYPES_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <odp/api/plat/strong_types.h>
+#include <odp/api/deprecated.h>
+
+/** @ingroup odp_event
+ * @{
+ */
+
+typedef ODP_HANDLE_T(odp_event_t);
+
+#define ODP_EVENT_INVALID _odp_cast_scalar(odp_event_t, 0)
+
+typedef enum odp_event_type_t {
+ ODP_EVENT_BUFFER = 1,
+ ODP_EVENT_PACKET = 2,
+ ODP_EVENT_TIMEOUT = 3,
+#if ODP_DEPRECATED_API
+ ODP_EVENT_CRYPTO_COMPL = 4,
+#endif
+ ODP_EVENT_IPSEC_STATUS = 5,
+ ODP_EVENT_PACKET_VECTOR = 6,
+ ODP_EVENT_PACKET_TX_COMPL = 7,
+ ODP_EVENT_DMA_COMPL = 8,
+} odp_event_type_t;
+
+typedef enum odp_event_subtype_t {
+ ODP_EVENT_NO_SUBTYPE = 0,
+ ODP_EVENT_PACKET_BASIC = 1,
+ ODP_EVENT_PACKET_CRYPTO = 2,
+ ODP_EVENT_PACKET_IPSEC = 3,
+ ODP_EVENT_PACKET_COMP = 4
+} odp_event_subtype_t;
+
+/**
+ * @}
+ */
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/platform/linux-generic/include-abi/odp/api/abi/timer.h b/platform/linux-generic/include-abi/odp/api/abi/timer.h
new file mode 100644
index 000000000..0a3b3a9cc
--- /dev/null
+++ b/platform/linux-generic/include-abi/odp/api/abi/timer.h
@@ -0,0 +1,8 @@
+/* Copyright (c) 2022, Nokia
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+/* Inlined functions for non-ABI compat mode */
+#include <odp/api/plat/timer_inlines.h>
diff --git a/platform/linux-generic/include/odp/api/plat/buffer_inlines.h b/platform/linux-generic/include/odp/api/plat/buffer_inlines.h
index 632ba0e69..9e0ae3f50 100644
--- a/platform/linux-generic/include/odp/api/plat/buffer_inlines.h
+++ b/platform/linux-generic/include/odp/api/plat/buffer_inlines.h
@@ -8,7 +8,7 @@
#define ODP_PLAT_BUFFER_INLINES_H_
#include <odp/api/abi/buffer.h>
-#include <odp/api/abi/event.h>
+#include <odp/api/abi/event_types.h>
#include <odp/api/plat/buffer_inline_types.h>
diff --git a/platform/linux-generic/include/odp/api/plat/packet_flag_inlines.h b/platform/linux-generic/include/odp/api/plat/packet_flag_inlines.h
index 6eb34a39b..04ad9e968 100644
--- a/platform/linux-generic/include/odp/api/plat/packet_flag_inlines.h
+++ b/platform/linux-generic/include/odp/api/plat/packet_flag_inlines.h
@@ -1,4 +1,5 @@
/* Copyright (c) 2017-2018, Linaro Limited
+ * Copyright (c) 2022, Nokia
* All rights reserved.
*
* SPDX-License-Identifier: BSD-3-Clause
@@ -30,11 +31,32 @@ static inline uint64_t _odp_packet_input_flags(odp_packet_t pkt)
/* Inline functions by default */
#define _ODP_INLINE static inline
#define odp_packet_has_l2 __odp_packet_has_l2
+ #define odp_packet_has_l3 __odp_packet_has_l3
+ #define odp_packet_has_l4 __odp_packet_has_l4
#define odp_packet_has_eth __odp_packet_has_eth
#define odp_packet_has_jumbo __odp_packet_has_jumbo
#define odp_packet_has_flow_hash __odp_packet_has_flow_hash
#define odp_packet_has_ts __odp_packet_has_ts
#define odp_packet_has_ipsec __odp_packet_has_ipsec
+ #define odp_packet_has_eth_bcast __odp_packet_has_eth_bcast
+ #define odp_packet_has_eth_mcast __odp_packet_has_eth_mcast
+ #define odp_packet_has_vlan __odp_packet_has_vlan
+ #define odp_packet_has_vlan_qinq __odp_packet_has_vlan_qinq
+ #define odp_packet_has_arp __odp_packet_has_arp
+ #define odp_packet_has_ipv4 __odp_packet_has_ipv4
+ #define odp_packet_has_ipv6 __odp_packet_has_ipv6
+ #define odp_packet_has_ip_bcast __odp_packet_has_ip_bcast
+ #define odp_packet_has_ip_mcast __odp_packet_has_ip_mcast
+ #define odp_packet_has_ipfrag __odp_packet_has_ipfrag
+ #define odp_packet_has_ipopt __odp_packet_has_ipopt
+ #define odp_packet_has_udp __odp_packet_has_udp
+ #define odp_packet_has_tcp __odp_packet_has_tcp
+ #define odp_packet_has_sctp __odp_packet_has_sctp
+ #define odp_packet_has_icmp __odp_packet_has_icmp
+ #define odp_packet_has_error __odp_packet_has_error
+ #define odp_packet_has_l2_error __odp_packet_has_l2_error
+ #define odp_packet_has_l3_error __odp_packet_has_l3_error
+ #define odp_packet_has_l4_error __odp_packet_has_l4_error
#else
#undef _ODP_INLINE
#define _ODP_INLINE
@@ -48,6 +70,22 @@ _ODP_INLINE int odp_packet_has_l2(odp_packet_t pkt)
return flags.l2;
}
+_ODP_INLINE int odp_packet_has_l3(odp_packet_t pkt)
+{
+ _odp_packet_input_flags_t flags;
+
+ flags.all = _odp_packet_input_flags(pkt);
+ return flags.l3;
+}
+
+_ODP_INLINE int odp_packet_has_l4(odp_packet_t pkt)
+{
+ _odp_packet_input_flags_t flags;
+
+ flags.all = _odp_packet_input_flags(pkt);
+ return flags.l4;
+}
+
_ODP_INLINE int odp_packet_has_eth(odp_packet_t pkt)
{
_odp_packet_input_flags_t flags;
@@ -88,6 +126,163 @@ _ODP_INLINE int odp_packet_has_ipsec(odp_packet_t pkt)
return flags.ipsec;
}
+_ODP_INLINE int odp_packet_has_eth_bcast(odp_packet_t pkt)
+{
+ _odp_packet_input_flags_t flags;
+
+ flags.all = _odp_packet_input_flags(pkt);
+ return flags.eth_bcast;
+}
+
+_ODP_INLINE int odp_packet_has_eth_mcast(odp_packet_t pkt)
+{
+ _odp_packet_input_flags_t flags;
+
+ flags.all = _odp_packet_input_flags(pkt);
+ return flags.eth_mcast;
+}
+
+_ODP_INLINE int odp_packet_has_vlan(odp_packet_t pkt)
+{
+ _odp_packet_input_flags_t flags;
+
+ flags.all = _odp_packet_input_flags(pkt);
+ return flags.vlan;
+}
+
+_ODP_INLINE int odp_packet_has_vlan_qinq(odp_packet_t pkt)
+{
+ _odp_packet_input_flags_t flags;
+
+ flags.all = _odp_packet_input_flags(pkt);
+ return flags.vlan_qinq;
+}
+
+_ODP_INLINE int odp_packet_has_arp(odp_packet_t pkt)
+{
+ _odp_packet_input_flags_t flags;
+
+ flags.all = _odp_packet_input_flags(pkt);
+ return flags.arp;
+}
+
+_ODP_INLINE int odp_packet_has_ipv4(odp_packet_t pkt)
+{
+ _odp_packet_input_flags_t flags;
+
+ flags.all = _odp_packet_input_flags(pkt);
+ return flags.ipv4;
+}
+
+_ODP_INLINE int odp_packet_has_ipv6(odp_packet_t pkt)
+{
+ _odp_packet_input_flags_t flags;
+
+ flags.all = _odp_packet_input_flags(pkt);
+ return flags.ipv6;
+}
+
+_ODP_INLINE int odp_packet_has_ip_bcast(odp_packet_t pkt)
+{
+ _odp_packet_input_flags_t flags;
+
+ flags.all = _odp_packet_input_flags(pkt);
+ return flags.ip_bcast;
+}
+
+_ODP_INLINE int odp_packet_has_ip_mcast(odp_packet_t pkt)
+{
+ _odp_packet_input_flags_t flags;
+
+ flags.all = _odp_packet_input_flags(pkt);
+ return flags.ip_mcast;
+}
+
+_ODP_INLINE int odp_packet_has_ipfrag(odp_packet_t pkt)
+{
+ _odp_packet_input_flags_t flags;
+
+ flags.all = _odp_packet_input_flags(pkt);
+ return flags.ipfrag;
+}
+
+_ODP_INLINE int odp_packet_has_ipopt(odp_packet_t pkt)
+{
+ _odp_packet_input_flags_t flags;
+
+ flags.all = _odp_packet_input_flags(pkt);
+ return flags.ipopt;
+}
+
+_ODP_INLINE int odp_packet_has_udp(odp_packet_t pkt)
+{
+ _odp_packet_input_flags_t flags;
+
+ flags.all = _odp_packet_input_flags(pkt);
+ return flags.udp;
+}
+
+_ODP_INLINE int odp_packet_has_tcp(odp_packet_t pkt)
+{
+ _odp_packet_input_flags_t flags;
+
+ flags.all = _odp_packet_input_flags(pkt);
+ return flags.tcp;
+}
+
+_ODP_INLINE int odp_packet_has_sctp(odp_packet_t pkt)
+{
+ _odp_packet_input_flags_t flags;
+
+ flags.all = _odp_packet_input_flags(pkt);
+ return flags.sctp;
+}
+
+_ODP_INLINE int odp_packet_has_icmp(odp_packet_t pkt)
+{
+ _odp_packet_input_flags_t flags;
+
+ flags.all = _odp_packet_input_flags(pkt);
+ return flags.icmp;
+}
+
+_ODP_INLINE int odp_packet_has_error(odp_packet_t pkt)
+{
+ _odp_packet_flags_t flags;
+
+ flags.all_flags = _odp_pkt_get(pkt, uint32_t, flags);
+ return flags.all.error != 0;
+}
+
+_ODP_INLINE int odp_packet_has_l2_error(odp_packet_t pkt)
+{
+ _odp_packet_flags_t flags;
+
+ flags.all_flags = _odp_pkt_get(pkt, uint32_t, flags);
+
+ /* L2 parsing is always done by default and hence
+ no additional check is required. */
+ return flags.snap_len_err;
+}
+
+_ODP_INLINE int odp_packet_has_l3_error(odp_packet_t pkt)
+{
+ _odp_packet_flags_t flags;
+
+ flags.all_flags = _odp_pkt_get(pkt, uint32_t, flags);
+
+ return flags.ip_err;
+}
+
+_ODP_INLINE int odp_packet_has_l4_error(odp_packet_t pkt)
+{
+ _odp_packet_flags_t flags;
+
+ flags.all_flags = _odp_pkt_get(pkt, uint32_t, flags);
+
+ return flags.tcp_err | flags.udp_err;
+}
+
/** @endcond */
#endif
diff --git a/platform/linux-generic/include/odp/api/plat/packet_inline_types.h b/platform/linux-generic/include/odp/api/plat/packet_inline_types.h
index 66451ce49..126404f2d 100644
--- a/platform/linux-generic/include/odp/api/plat/packet_inline_types.h
+++ b/platform/linux-generic/include/odp/api/plat/packet_inline_types.h
@@ -115,7 +115,7 @@ typedef union {
uint32_t all_flags;
struct {
- uint32_t reserved1: 7;
+ uint32_t reserved1: 6;
/*
* Init flags
@@ -133,6 +133,7 @@ typedef union {
uint32_t l4_chksum: 1; /* L4 chksum override */
uint32_t ts_set: 1; /* Set Tx timestamp */
uint32_t tx_compl: 1; /* Tx completion event requested */
+ uint32_t tx_aging: 1; /* Packet aging at Tx requested */
uint32_t shaper_len_adj: 8; /* Adjustment for traffic mgr */
/*
@@ -150,8 +151,8 @@ typedef union {
/* Flag groups */
struct {
- uint32_t reserved2: 7;
- uint32_t other: 17; /* All other flags */
+ uint32_t reserved2: 6;
+ uint32_t other: 18; /* All other flags */
uint32_t error: 8; /* All error flags */
} all;
diff --git a/platform/linux-generic/include/odp/api/plat/packet_inlines.h b/platform/linux-generic/include/odp/api/plat/packet_inlines.h
index fb63f207b..b72749ea8 100644
--- a/platform/linux-generic/include/odp/api/plat/packet_inlines.h
+++ b/platform/linux-generic/include/odp/api/plat/packet_inlines.h
@@ -20,7 +20,7 @@
#include <odp/api/time.h>
#include <odp/api/abi/buffer.h>
-#include <odp/api/abi/event.h>
+#include <odp/api/abi/event_types.h>
#include <odp/api/abi/packet.h>
#include <odp/api/abi/packet_io.h>
@@ -73,6 +73,9 @@
#define odp_packet_subtype __odp_packet_subtype
#define odp_packet_tx_compl_from_event __odp_packet_tx_compl_from_event
#define odp_packet_tx_compl_to_event __odp_packet_tx_compl_to_event
+ #define odp_packet_color __odp_packet_color
+ #define odp_packet_drop_eligible __odp_packet_drop_eligible
+ #define odp_packet_shaper_len_adjust __odp_packet_shaper_len_adjust
#else
#undef _ODP_INLINE
#define _ODP_INLINE
@@ -372,6 +375,33 @@ _ODP_INLINE odp_event_t odp_packet_tx_compl_to_event(odp_packet_tx_compl_t tx_co
return (odp_event_t)(uintptr_t)tx_compl;
}
+_ODP_INLINE odp_packet_color_t odp_packet_color(odp_packet_t pkt)
+{
+ _odp_packet_input_flags_t input_flags;
+
+ input_flags.all = _odp_pkt_get(pkt, uint64_t, input_flags);
+
+ return (odp_packet_color_t)input_flags.color;
+}
+
+_ODP_INLINE odp_bool_t odp_packet_drop_eligible(odp_packet_t pkt)
+{
+ _odp_packet_input_flags_t input_flags;
+
+ input_flags.all = _odp_pkt_get(pkt, uint64_t, input_flags);
+
+ return !input_flags.nodrop;
+}
+
+_ODP_INLINE int8_t odp_packet_shaper_len_adjust(odp_packet_t pkt)
+{
+ _odp_packet_flags_t flags;
+
+ flags.all_flags = _odp_pkt_get(pkt, uint32_t, flags);
+
+ return flags.shaper_len_adj;
+}
+
/** @endcond */
#endif
diff --git a/platform/linux-generic/include/odp/api/plat/packet_vector_inlines.h b/platform/linux-generic/include/odp/api/plat/packet_vector_inlines.h
index 3586aa6d5..54a009a36 100644
--- a/platform/linux-generic/include/odp/api/plat/packet_vector_inlines.h
+++ b/platform/linux-generic/include/odp/api/plat/packet_vector_inlines.h
@@ -14,7 +14,7 @@
#ifndef _ODP_PLAT_PACKET_VECTOR_INLINES_H_
#define _ODP_PLAT_PACKET_VECTOR_INLINES_H_
-#include <odp/api/abi/event.h>
+#include <odp/api/abi/event_types.h>
#include <odp/api/abi/packet_types.h>
#include <odp/api/abi/pool_types.h>
diff --git a/platform/linux-generic/include/odp/api/plat/queue_inline_types.h b/platform/linux-generic/include/odp/api/plat/queue_inline_types.h
index 5ce767a16..4eade3dea 100644
--- a/platform/linux-generic/include/odp/api/plat/queue_inline_types.h
+++ b/platform/linux-generic/include/odp/api/plat/queue_inline_types.h
@@ -12,6 +12,7 @@ extern "C" {
#endif
#include <stdint.h>
+#include <odp/api/event_types.h>
#include <odp/api/queue_types.h>
/** @cond _ODP_HIDE_FROM_DOXYGEN_ */
diff --git a/platform/linux-generic/include/odp/api/plat/time_inlines.h b/platform/linux-generic/include/odp/api/plat/time_inlines.h
index bb2913532..2ffb94c66 100644
--- a/platform/linux-generic/include/odp/api/plat/time_inlines.h
+++ b/platform/linux-generic/include/odp/api/plat/time_inlines.h
@@ -1,5 +1,5 @@
/* Copyright (c) 2018, Linaro Limited
- * Copyright (c) 2020-2021, Nokia
+ * Copyright (c) 2020-2022, Nokia
* All rights reserved.
*
* SPDX-License-Identifier: BSD-3-Clause
@@ -99,6 +99,7 @@ static inline uint64_t _odp_time_convert_to_ns(odp_time_t time)
#define odp_time_cmp __odp_time_cmp
#define odp_time_diff __odp_time_diff
+ #define odp_time_diff_ns __odp_time_diff_ns
#define odp_time_sum __odp_time_sum
#else
@@ -170,6 +171,15 @@ _ODP_INLINE odp_time_t odp_time_diff(odp_time_t t2, odp_time_t t1)
return time;
}
+_ODP_INLINE uint64_t odp_time_diff_ns(odp_time_t t2, odp_time_t t1)
+{
+ odp_time_t time;
+
+ time.u64 = t2.u64 - t1.u64;
+
+ return odp_time_to_ns(time);
+}
+
_ODP_INLINE odp_time_t odp_time_sum(odp_time_t t1, odp_time_t t2)
{
odp_time_t time;
diff --git a/platform/linux-generic/include/odp/api/plat/timer_inline_types.h b/platform/linux-generic/include/odp/api/plat/timer_inline_types.h
new file mode 100644
index 000000000..e3397c4df
--- /dev/null
+++ b/platform/linux-generic/include/odp/api/plat/timer_inline_types.h
@@ -0,0 +1,37 @@
+/* Copyright (c) 2022, Nokia
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+#ifndef ODP_PLAT_TIMER_INLINE_TYPES_H_
+#define ODP_PLAT_TIMER_INLINE_TYPES_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <stdint.h>
+
+/** @cond _ODP_HIDE_FROM_DOXYGEN_ */
+
+/* Timeout header field accessor */
+#define _odp_timeout_hdr_field(hdr, cast, field) \
+ (*(cast *)(uintptr_t)((uint8_t *)hdr + \
+ _odp_timeout_inline_offset.field))
+
+/* Timeout header field offsets for inline functions */
+typedef struct _odp_timeout_inline_offset_t {
+ uint16_t expiration;
+ uint16_t timer;
+ uint16_t user_ptr;
+
+} _odp_timeout_inline_offset_t;
+
+/** @endcond */
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/platform/linux-generic/include/odp/api/plat/timer_inlines.h b/platform/linux-generic/include/odp/api/plat/timer_inlines.h
new file mode 100644
index 000000000..406aefdf6
--- /dev/null
+++ b/platform/linux-generic/include/odp/api/plat/timer_inlines.h
@@ -0,0 +1,78 @@
+/* Copyright (c) 2022, Nokia
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+#ifndef ODP_PLAT_TIMER_INLINES_H_
+#define ODP_PLAT_TIMER_INLINES_H_
+
+#include <odp/api/event.h>
+#include <odp/api/timer_types.h>
+
+#include <odp/api/plat/timer_inline_types.h>
+
+#include <stdint.h>
+
+/** @cond _ODP_HIDE_FROM_DOXYGEN_ */
+
+extern const _odp_timeout_inline_offset_t _odp_timeout_inline_offset;
+
+#ifndef _ODP_NO_INLINE
+ /* Inline functions by default */
+ #define _ODP_INLINE static inline
+ #define odp_timeout_timer __odp_timeout_timer
+ #define odp_timeout_tick __odp_timeout_tick
+ #define odp_timeout_user_ptr __odp_timeout_user_ptr
+ #define odp_timer_tick_to_ns __odp_timer_tick_to_ns
+ #define odp_timer_ns_to_tick __odp_timer_ns_to_tick
+ #define odp_timeout_from_event __odp_timeout_from_event
+ #define odp_timeout_to_event __odp_timeout_to_event
+#else
+ #define _ODP_INLINE
+#endif
+
+_ODP_INLINE odp_timer_t odp_timeout_timer(odp_timeout_t tmo)
+{
+ return _odp_timeout_hdr_field(tmo, odp_timer_t, timer);
+}
+
+_ODP_INLINE uint64_t odp_timeout_tick(odp_timeout_t tmo)
+{
+ return _odp_timeout_hdr_field(tmo, uint64_t, expiration);
+}
+
+_ODP_INLINE void *odp_timeout_user_ptr(odp_timeout_t tmo)
+{
+ return _odp_timeout_hdr_field(tmo, void *, user_ptr);
+}
+
+_ODP_INLINE uint64_t odp_timer_tick_to_ns(odp_timer_pool_t tp, uint64_t ticks)
+{
+ (void)tp;
+
+ /* Timer ticks in API are nsec */
+ return ticks;
+}
+
+_ODP_INLINE uint64_t odp_timer_ns_to_tick(odp_timer_pool_t tp, uint64_t ns)
+{
+ (void)tp;
+
+ /* Timer ticks in API are nsec */
+ return ns;
+}
+
+_ODP_INLINE odp_timeout_t odp_timeout_from_event(odp_event_t ev)
+{
+ return (odp_timeout_t)ev;
+}
+
+_ODP_INLINE odp_event_t odp_timeout_to_event(odp_timeout_t tmo)
+{
+ return (odp_event_t)tmo;
+}
+
+/** @endcond */
+
+#endif
diff --git a/platform/linux-generic/include/odp_atomic_internal.h b/platform/linux-generic/include/odp_atomic_internal.h
index d98e0e6be..da79b3723 100644
--- a/platform/linux-generic/include/odp_atomic_internal.h
+++ b/platform/linux-generic/include/odp_atomic_internal.h
@@ -20,6 +20,7 @@
#include <odp/api/align.h>
#include <odp/api/hints.h>
#include <odp/api/atomic.h>
+#include <odp_types_internal.h>
#include <stdbool.h>
#ifdef __cplusplus
@@ -137,12 +138,10 @@ static inline void _odp_atomic_flag_clear(_odp_atomic_flag_t *flag)
#endif
#ifdef ODP_ATOMIC_U128
-/** An unsigned 128-bit (16-byte) scalar type */
-__extension__ typedef __int128 _uint128_t;
/** Atomic 128-bit type */
typedef struct ODP_ALIGNED(16) {
- _uint128_t v; /**< Actual storage for the atomic variable */
+ _odp_u128_t v; /**< Actual storage for the atomic variable */
} _odp_atomic_u128_t;
/**
@@ -154,8 +153,8 @@ typedef struct ODP_ALIGNED(16) {
* @param mmodel Memory model associated with the exchange operation
*/
static inline void _odp_atomic_u128_xchg_mm(_odp_atomic_u128_t *ptr,
- _uint128_t *val,
- _uint128_t *old,
+ _odp_u128_t *val,
+ _odp_u128_t *old,
_odp_memmodel_t mm)
{
__atomic_exchange(&ptr->v, val, old, mm);
@@ -177,8 +176,8 @@ static inline void _odp_atomic_u128_xchg_mm(_odp_atomic_u128_t *ptr,
* @retval 0 exchange failed and '*exp' updated with current value
*/
static inline int _odp_atomic_u128_cmp_xchg_mm(_odp_atomic_u128_t *ptr,
- _uint128_t *exp,
- _uint128_t *val,
+ _odp_u128_t *exp,
+ _odp_u128_t *val,
_odp_memmodel_t succ,
_odp_memmodel_t fail)
{
diff --git a/platform/linux-generic/include/odp_classification_datamodel.h b/platform/linux-generic/include/odp_classification_datamodel.h
index cc0e7f081..6b50fef68 100644
--- a/platform/linux-generic/include/odp_classification_datamodel.h
+++ b/platform/linux-generic/include/odp_classification_datamodel.h
@@ -129,6 +129,7 @@ typedef struct pmr_term_value {
Class Of Service
*/
struct cos_s {
+ odp_cos_action_t action; /* Action */
odp_queue_t queue; /* Associated Queue */
odp_pool_t pool; /* Associated Buffer pool */
odp_pktin_vector_config_t vector; /* Packet vector config */
@@ -145,10 +146,11 @@ struct cos_s {
odp_queue_param_t queue_param;
char name[ODP_COS_NAME_LEN]; /* name */
uint8_t index;
+ odp_bool_t stats_enable;
struct {
odp_atomic_u64_t discards;
odp_atomic_u64_t packets;
- } stats[CLS_COS_QUEUE_MAX];
+ } stats, queue_stats[CLS_COS_QUEUE_MAX];
};
typedef union cos_u {
diff --git a/platform/linux-generic/include/odp_classification_internal.h b/platform/linux-generic/include/odp_classification_internal.h
index bc0a12f8f..c20d24c58 100644
--- a/platform/linux-generic/include/odp_classification_internal.h
+++ b/platform/linux-generic/include/odp_classification_internal.h
@@ -67,9 +67,9 @@ static inline void _odp_cos_queue_stats_add(cos_t *cos, odp_queue_t queue,
}
if (packets)
- odp_atomic_add_u64(&cos->s.stats[queue_idx].packets, packets);
+ odp_atomic_add_u64(&cos->s.queue_stats[queue_idx].packets, packets);
if (discards)
- odp_atomic_add_u64(&cos->s.stats[queue_idx].discards, discards);
+ odp_atomic_add_u64(&cos->s.queue_stats[queue_idx].discards, discards);
}
/** Classification Internal function **/
@@ -84,8 +84,7 @@ This function calls Classifier module internal functions for a given packet and
selects destination queue and packet pool based on selected PMR and CoS.
**/
int _odp_cls_classify_packet(pktio_entry_t *entry, const uint8_t *base,
- uint16_t pkt_len, uint32_t seg_len, odp_pool_t *pool,
- odp_packet_hdr_t *pkt_hdr, odp_bool_t parse);
+ odp_pool_t *pool, odp_packet_hdr_t *pkt_hdr);
/**
Packet IO classifier init
diff --git a/platform/linux-generic/include/odp_llqueue.h b/platform/linux-generic/include/odp_llqueue.h
index e9cf9945e..68325624a 100644
--- a/platform/linux-generic/include/odp_llqueue.h
+++ b/platform/linux-generic/include/odp_llqueue.h
@@ -15,6 +15,7 @@
#include <odp_config_internal.h>
#include <odp_debug_internal.h>
+#include <odp_types_internal.h>
#include <odp_cpu.h>
#include <stdint.h>
@@ -49,7 +50,7 @@ static odp_bool_t llq_on_queue(struct llnode *node);
typedef uint64_t dintptr_t;
#endif
#if __SIZEOF_PTRDIFF_T__ == 8
-__extension__ typedef __int128 dintptr_t;
+typedef _odp_u128_t dintptr_t;
#endif
struct llnode {
diff --git a/platform/linux-generic/include/odp_macros_internal.h b/platform/linux-generic/include/odp_macros_internal.h
index 997e0fd5b..b8be7f938 100644
--- a/platform/linux-generic/include/odp_macros_internal.h
+++ b/platform/linux-generic/include/odp_macros_internal.h
@@ -35,6 +35,8 @@ extern "C" {
tmp_a > tmp_b ? tmp_a : tmp_b; \
})
+#define MAX3(a, b, c) (MAX(MAX((a), (b)), (c)))
+
#define odp_container_of(pointer, type, member) \
((type *)(void *)(((char *)pointer) - offsetof(type, member)))
diff --git a/platform/linux-generic/include/odp_packet_dpdk.h b/platform/linux-generic/include/odp_packet_dpdk.h
index 3add83f19..b326000e6 100644
--- a/platform/linux-generic/include/odp_packet_dpdk.h
+++ b/platform/linux-generic/include/odp_packet_dpdk.h
@@ -46,18 +46,4 @@ int _odp_dpdk_packet_parse_common(packet_parser_t *pkt_hdr,
uint32_t supported_ptypes,
odp_pktin_config_opt_t pktin_cfg);
-static inline int _odp_dpdk_packet_parse_layer(odp_packet_hdr_t *pkt_hdr,
- struct rte_mbuf *mbuf,
- odp_proto_layer_t layer,
- uint32_t supported_ptypes,
- odp_pktin_config_opt_t pktin_cfg)
-{
- uint32_t seg_len = pkt_hdr->seg_len;
- void *base = pkt_hdr->seg_data;
-
- return _odp_dpdk_packet_parse_common(&pkt_hdr->p, base,
- pkt_hdr->frame_len, seg_len, mbuf,
- layer, supported_ptypes,
- pktin_cfg);
-}
#endif
diff --git a/platform/linux-generic/include/odp_packet_internal.h b/platform/linux-generic/include/odp_packet_internal.h
index 5a29210dd..a8f58146a 100644
--- a/platform/linux-generic/include/odp_packet_internal.h
+++ b/platform/linux-generic/include/odp_packet_internal.h
@@ -38,15 +38,23 @@ extern "C" {
#include <stdint.h>
#include <string.h>
-/** Minimum segment length expected by _odp_packet_parse_common() */
-#define PACKET_PARSE_SEG_LEN 96
-
ODP_STATIC_ASSERT(sizeof(_odp_packet_input_flags_t) == sizeof(uint64_t),
"INPUT_FLAGS_SIZE_ERROR");
ODP_STATIC_ASSERT(sizeof(_odp_packet_flags_t) == sizeof(uint32_t),
"PACKET_FLAGS_SIZE_ERROR");
+/* Packet extra data length */
+#define PKT_EXTRA_LEN 128
+
+/* Packet extra data types */
+#define PKT_EXTRA_TYPE_DPDK 1
+
+/* Maximum number of segments per packet */
+#define PKT_MAX_SEGS 255
+
+ODP_STATIC_ASSERT(PKT_MAX_SEGS < UINT16_MAX, "PACKET_MAX_SEGS_ERROR");
+
/**
* Packet parser metadata
*/
@@ -67,17 +75,6 @@ typedef struct {
uint16_t l4_offset;
} packet_parser_t;
-/* Packet extra data length */
-#define PKT_EXTRA_LEN 128
-
-/* Packet extra data types */
-#define PKT_EXTRA_TYPE_DPDK 1
-
-/* Maximum number of segments per packet */
-#define PKT_MAX_SEGS 255
-
-ODP_STATIC_ASSERT(PKT_MAX_SEGS < UINT16_MAX, "PACKET_MAX_SEGS_ERROR");
-
/**
* Internal Packet header
*
@@ -146,6 +143,10 @@ typedef struct ODP_ALIGNED_CACHE odp_packet_hdr_t {
/* Max payload size in a LSO segment */
uint16_t lso_max_payload;
+ /* Packet aging drop timeout before enqueue. Once enqueued holds the maximum age (time of
+ * request + requested drop timeout). */
+ uint64_t tx_aging_ns;
+
/* LSO profile index */
uint8_t lso_profile_idx;
@@ -194,6 +195,11 @@ static inline odp_packet_t packet_from_event_hdr(_odp_event_hdr_t *event_hdr)
return (odp_packet_t)(uintptr_t)event_hdr;
}
+static inline uint32_t packet_first_seg_len(odp_packet_hdr_t *pkt_hdr)
+{
+ return pkt_hdr->seg_len;
+}
+
static inline odp_packet_hdr_t *packet_last_seg(odp_packet_hdr_t *hdr)
{
while (hdr->seg_next != NULL)
@@ -400,11 +406,6 @@ static inline void packet_set_len(odp_packet_hdr_t *pkt_hdr, uint32_t len)
int _odp_packet_alloc_multi(odp_pool_t pool_hdl, uint32_t len,
odp_packet_t pkt[], int max_num);
-/* Perform packet parse up to a given protocol layer */
-int _odp_packet_parse_layer(odp_packet_hdr_t *pkt_hdr,
- odp_proto_layer_t layer,
- odp_proto_chksums_t chksums);
-
/* Reset parser metadata for a new parse */
static inline void packet_parse_reset(odp_packet_hdr_t *pkt_hdr, int all)
{
@@ -454,12 +455,6 @@ static inline void packet_set_ts(odp_packet_hdr_t *pkt_hdr, odp_time_t *ts)
}
}
-int _odp_packet_parse_common(packet_parser_t *pkt_hdr, const uint8_t *ptr,
- uint32_t pkt_len, uint32_t seg_len, int layer,
- odp_proto_chksums_t chksums);
-
-int _odp_cls_parse(odp_packet_hdr_t *pkt_hdr, const uint8_t *parseptr);
-
int _odp_packet_set_data(odp_packet_t pkt, uint32_t offset,
uint8_t c, uint32_t len);
@@ -471,6 +466,9 @@ int _odp_packet_tcp_chksum_insert(odp_packet_t pkt);
int _odp_packet_udp_chksum_insert(odp_packet_t pkt);
int _odp_packet_sctp_chksum_insert(odp_packet_t pkt);
+int _odp_packet_l4_chksum(odp_packet_hdr_t *pkt_hdr,
+ odp_proto_chksums_t chksums, uint64_t l4_part_sum);
+
#ifdef __cplusplus
}
#endif
diff --git a/platform/linux-generic/include/odp_packet_io_internal.h b/platform/linux-generic/include/odp_packet_io_internal.h
index 3e9f94100..aed6de412 100644
--- a/platform/linux-generic/include/odp_packet_io_internal.h
+++ b/platform/linux-generic/include/odp_packet_io_internal.h
@@ -78,6 +78,7 @@ struct pktio_entry {
/* These two locks together lock the whole pktio device */
odp_ticketlock_t rxl; /**< RX ticketlock */
odp_ticketlock_t txl; /**< TX ticketlock */
+ odp_proto_layer_t parse_layer;
uint16_t pktin_frame_offset;
struct {
@@ -93,6 +94,8 @@ struct pktio_entry {
uint8_t tx_ts : 1;
/* Tx completion events */
uint8_t tx_compl : 1;
+ /* Packet aging */
+ uint8_t tx_aging : 1;
};
};
} enabled;
@@ -144,8 +147,8 @@ struct pktio_entry {
/* Storage for queue handles
* Multi-queue support is pktio driver specific */
- unsigned num_in_queue;
- unsigned num_out_queue;
+ uint32_t num_in_queue;
+ uint32_t num_out_queue;
struct {
odp_queue_t queue;
@@ -227,8 +230,8 @@ typedef struct pktio_if_ops {
int num);
int (*recv_tmo)(pktio_entry_t *entry, int index, odp_packet_t packets[],
int num, uint64_t wait_usecs);
- int (*recv_mq_tmo)(pktio_entry_t *entry[], int index[], int num_q,
- odp_packet_t packets[], int num, unsigned *from,
+ int (*recv_mq_tmo)(pktio_entry_t *entry[], int index[], uint32_t num_q,
+ odp_packet_t packets[], int num, uint32_t *from,
uint64_t wait_usecs);
int (*fd_set)(pktio_entry_t *entry, int index, fd_set *readfds);
int (*send)(pktio_entry_t *entry, int index,
@@ -291,6 +294,11 @@ static inline int _odp_pktio_tx_compl_enabled(const pktio_entry_t *entry)
return entry->s.enabled.tx_compl;
}
+static inline int _odp_pktio_tx_aging_enabled(pktio_entry_t *entry)
+{
+ return entry->s.enabled.tx_aging;
+}
+
static inline void _odp_pktio_tx_ts_set(pktio_entry_t *entry)
{
odp_time_t ts_val = odp_time_global();
@@ -326,7 +334,7 @@ extern const pktio_if_ops_t * const _odp_pktio_if_ops[];
* @return <0 on failure
*/
int _odp_sock_recv_mq_tmo_try_int_driven(const struct odp_pktin_queue_t queues[],
- unsigned int num_q, unsigned int *from,
+ uint32_t num_q, uint32_t *from,
odp_packet_t packets[], int num,
uint64_t usecs,
int *trial_successful);
diff --git a/platform/linux-generic/include/odp_parse_internal.h b/platform/linux-generic/include/odp_parse_internal.h
new file mode 100644
index 000000000..8aa5e118b
--- /dev/null
+++ b/platform/linux-generic/include/odp_parse_internal.h
@@ -0,0 +1,112 @@
+/* Copyright (c) 2013-2018, Linaro Limited
+ * Copyright (c) 2019-2022, Nokia
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+#ifndef ODP_PARSE_INTERNAL_H_
+#define ODP_PARSE_INTERNAL_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <odp_packet_internal.h>
+#include <odp_macros_internal.h>
+#include <protocols/eth.h>
+#include <protocols/ip.h>
+#include <protocols/sctp.h>
+#include <protocols/tcp.h>
+#include <protocols/udp.h>
+#include <odp/api/plat/packet_inline_types.h>
+#include <odp/api/hints.h>
+#include <odp/api/packet_io.h>
+#include <odp/api/packet_types.h>
+#include <stdint.h>
+
+/*
+ * In the worst case we look at the Ethernet header, 8 bytes of LLC/SNAP
+ * header and two VLAN tags in the same packet.
+ */
+#define PARSE_ETH_BYTES (sizeof(_odp_ethhdr_t) + 8 + 2 * sizeof(_odp_vlanhdr_t))
+#define PARSE_IPV4_BYTES (0xfU * 4) /* max IPv4 header length with options */
+/*
+ * Peek 2 bytes beyond IPv6 base header without length check if there are
+ * extension headers.
+ */
+#define PARSE_IPV6_BYTES (sizeof(_odp_ipv6hdr_t) + 2)
+#define PARSE_TCP_BYTES (sizeof(_odp_tcphdr_t))
+/*
+ * In the worst case we look at the UDP header and 4 bytes of the UDP
+ * payload (the non-ESP marker to distinguish IKE packets from ESP packets).
+ */
+#define PARSE_UDP_BYTES (sizeof(_odp_udphdr_t) + 4)
+#define PARSE_SCTP_BYTES (sizeof(_odp_sctphdr_t))
+
+/* _odp_packet_parse_common_l3_l4() requires up to this many bytes. */
+#define PARSE_L3_L4_BYTES (MAX(PARSE_IPV4_BYTES, PARSE_IPV6_BYTES) + \
+ MAX3(PARSE_TCP_BYTES, PARSE_UDP_BYTES, PARSE_SCTP_BYTES))
+
+/* _odp_packet_parse_common() requires up to this many bytes. */
+#define PARSE_BYTES (PARSE_ETH_BYTES + PARSE_L3_L4_BYTES)
+
+uint16_t _odp_parse_eth(packet_parser_t *prs, const uint8_t **parseptr,
+ uint32_t *offset, uint32_t frame_len);
+
+/*
+ * Parse common L3 and L4 packet headers up to given layer
+ *
+ * See _odp_packet_parse_common(). Requires up to PARSE_L3_L4_BYTES bytes of
+ * contiguous packet data.
+ */
+int _odp_packet_parse_common_l3_l4(packet_parser_t *prs,
+ const uint8_t *parseptr, uint32_t offset,
+ uint32_t frame_len, uint32_t seg_len,
+ int layer, uint16_t ethtype,
+ odp_proto_chksums_t chksums,
+ uint64_t *l4_part_sum,
+ odp_pktin_config_opt_t opt);
+
+/**
+ * Parse common packet headers up to given layer
+ *
+ * Requires up to PARSE_BYTES bytes of contiguous packet data. Also parse
+ * metadata must be already initialized.
+ *
+ * Returns 0 on success, 1 on packet errors, and -1 if the packet should be
+ * dropped.
+ */
+static inline int _odp_packet_parse_common(packet_parser_t *prs,
+ const uint8_t *ptr,
+ uint32_t frame_len, uint32_t seg_len,
+ int layer,
+ odp_proto_chksums_t chksums,
+ uint64_t *l4_part_sum,
+ odp_pktin_config_opt_t opt)
+{
+ uint32_t offset;
+ uint16_t ethtype;
+ const uint8_t *parseptr;
+
+ parseptr = ptr;
+ offset = 0;
+
+ if (odp_unlikely(layer == ODP_PROTO_LAYER_NONE))
+ return 0;
+
+ /* Assume valid L2 header, no CRC/FCS check in SW */
+ prs->l2_offset = offset;
+
+ ethtype = _odp_parse_eth(prs, &parseptr, &offset, frame_len);
+
+ return _odp_packet_parse_common_l3_l4(prs, parseptr, offset, frame_len,
+ seg_len, layer, ethtype, chksums,
+ l4_part_sum, opt);
+}
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/platform/linux-generic/include/odp_schedule_if.h b/platform/linux-generic/include/odp_schedule_if.h
index ba4ad53e8..cec9c8bb8 100644
--- a/platform/linux-generic/include/odp_schedule_if.h
+++ b/platform/linux-generic/include/odp_schedule_if.h
@@ -89,8 +89,6 @@ extern const schedule_fn_t *_odp_sched_fn;
/* Interface for the scheduler */
int _odp_sched_cb_pktin_poll(int pktio_index, int pktin_index,
_odp_event_hdr_t *hdr_tbl[], int num);
-/* evts must have at least QUEUE_MULTI_MAX elements */
-int _odp_sched_cb_pktin_poll_one(int pktio_index, int rx_queue, odp_event_t evts[]);
void _odp_sched_cb_pktio_stop_finalize(int pktio_index);
/* API functions */
diff --git a/platform/linux-generic/include/odp_types_internal.h b/platform/linux-generic/include/odp_types_internal.h
new file mode 100644
index 000000000..a97ac9cd4
--- /dev/null
+++ b/platform/linux-generic/include/odp_types_internal.h
@@ -0,0 +1,24 @@
+/* Copyright (c) 2022, Nokia
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+#ifndef ODP_TYPES_INTERNAL_H_
+#define ODP_TYPES_INTERNAL_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#ifdef __SIZEOF_INT128__
+
+__extension__ typedef unsigned __int128 _odp_u128_t;
+
+#endif
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/platform/linux-generic/odp_classification.c b/platform/linux-generic/odp_classification.c
index 3b50232d6..2fdecbc51 100644
--- a/platform/linux-generic/odp_classification.c
+++ b/platform/linux-generic/odp_classification.c
@@ -150,7 +150,7 @@ void odp_cls_pmr_param_init(odp_pmr_param_t *param)
int odp_cls_capability(odp_cls_capability_t *capability)
{
- unsigned int count = 0;
+ uint32_t count = 0;
memset(capability, 0, sizeof(odp_cls_capability_t));
@@ -161,6 +161,7 @@ int odp_cls_capability(odp_cls_capability_t *capability)
capability->max_pmr_terms = CLS_PMR_MAX_ENTRY;
capability->available_pmr_terms = count;
capability->max_cos = CLS_COS_MAX_ENTRY;
+ capability->max_cos_stats = capability->max_cos;
capability->pmr_range_supported = false;
capability->supported_terms.all_bits = 0;
capability->supported_terms.bit.len = 1;
@@ -186,6 +187,8 @@ int odp_cls_capability(odp_cls_capability_t *capability)
capability->threshold_bp.all_bits = 0;
capability->max_hash_queues = CLS_COS_QUEUE_MAX;
capability->max_mark = MAX_MARK;
+ capability->stats.cos.counter.discards = 1;
+ capability->stats.cos.counter.packets = 1;
capability->stats.queue.counter.discards = 1;
capability->stats.queue.counter.packets = 1;
@@ -220,21 +223,29 @@ static inline void _cls_queue_unwind(uint32_t tbl_index, uint32_t j)
odp_queue_destroy(queue_grp_tbl->s.queue[tbl_index + --j]);
}
-odp_cos_t odp_cls_cos_create(const char *name, const odp_cls_cos_param_t *param)
+odp_cos_t odp_cls_cos_create(const char *name, const odp_cls_cos_param_t *param_in)
{
uint32_t i, j;
odp_queue_t queue;
odp_cls_drop_t drop_policy;
cos_t *cos;
uint32_t tbl_index;
+ odp_cls_cos_param_t param = *param_in;
+
+ if (param.action == ODP_COS_ACTION_DROP) {
+ param.num_queue = 1;
+ param.queue = ODP_QUEUE_INVALID;
+ param.pool = ODP_POOL_INVALID;
+ param.vector.enable = false;
+ }
/* num_queue should not be zero */
- if (param->num_queue > CLS_COS_QUEUE_MAX || param->num_queue < 1)
+ if (param.num_queue > CLS_COS_QUEUE_MAX || param.num_queue < 1)
return ODP_COS_INVALID;
/* Validate packet vector parameters */
- if (param->vector.enable) {
- odp_pool_t pool = param->vector.pool;
+ if (param.vector.enable) {
+ odp_pool_t pool = param.vector.pool;
odp_pool_info_t pool_info;
if (pool == ODP_POOL_INVALID || odp_pool_info(pool, &pool_info)) {
@@ -245,17 +256,17 @@ odp_cos_t odp_cls_cos_create(const char *name, const odp_cls_cos_param_t *param)
ODP_ERR("wrong pool type\n");
return ODP_COS_INVALID;
}
- if (param->vector.max_size == 0) {
+ if (param.vector.max_size == 0) {
ODP_ERR("vector.max_size is zero\n");
return ODP_COS_INVALID;
}
- if (param->vector.max_size > pool_info.params.vector.max_size) {
+ if (param.vector.max_size > pool_info.params.vector.max_size) {
ODP_ERR("vector.max_size larger than pool max vector size\n");
return ODP_COS_INVALID;
}
}
- drop_policy = param->drop_policy;
+ drop_policy = param.drop_policy;
for (i = 0; i < CLS_COS_MAX_ENTRY; i++) {
cos = &cos_tbl->cos_entry[i];
@@ -274,16 +285,16 @@ odp_cos_t odp_cls_cos_create(const char *name, const odp_cls_cos_param_t *param)
cos->s.linked_cos[j] = NULL;
}
- cos->s.num_queue = param->num_queue;
+ cos->s.num_queue = param.num_queue;
- if (param->num_queue > 1) {
+ if (param.num_queue > 1) {
odp_queue_param_init(&cos->s.queue_param);
cos->s.queue_group = true;
cos->s.queue = ODP_QUEUE_INVALID;
_odp_cls_update_hash_proto(cos,
- param->hash_proto);
+ param.hash_proto);
tbl_index = i * CLS_COS_QUEUE_MAX;
- for (j = 0; j < param->num_queue; j++) {
+ for (j = 0; j < param.num_queue; j++) {
queue = odp_queue_create(NULL, &cos->s.queue_param);
if (queue == ODP_QUEUE_INVALID) {
/* unwind the queues */
@@ -296,21 +307,27 @@ odp_cos_t odp_cls_cos_create(const char *name, const odp_cls_cos_param_t *param)
}
} else {
- cos->s.queue = param->queue;
+ cos->s.queue = param.queue;
}
+
+ odp_atomic_init_u64(&cos->s.stats.discards, 0);
+ odp_atomic_init_u64(&cos->s.stats.packets, 0);
+
/* Initialize statistics counters */
for (j = 0; j < cos->s.num_queue; j++) {
- odp_atomic_init_u64(&cos->s.stats[j].discards, 0);
- odp_atomic_init_u64(&cos->s.stats[j].packets, 0);
+ odp_atomic_init_u64(&cos->s.queue_stats[j].discards, 0);
+ odp_atomic_init_u64(&cos->s.queue_stats[j].packets, 0);
}
- cos->s.pool = param->pool;
+ cos->s.action = param.action;
+ cos->s.pool = param.pool;
cos->s.headroom = 0;
cos->s.valid = 1;
cos->s.drop_policy = drop_policy;
odp_atomic_init_u32(&cos->s.num_rule, 0);
cos->s.index = i;
- cos->s.vector = param->vector;
+ cos->s.vector = param.vector;
+ cos->s.stats_enable = param.stats_enable;
UNLOCK(&cos->s.lock);
return _odp_cos_from_ndx(i);
}
@@ -1461,6 +1478,9 @@ static cos_t *match_pmr_cos(cos_t *cos, const uint8_t *pkt_addr, pmr_t *pmr,
/* PMR matched */
pmr_debug_print(pmr, cos);
+ if (cos->s.stats_enable)
+ odp_atomic_inc_u64(&cos->s.stats.packets);
+
hdr->p.input_flags.cls_mark = 0;
if (pmr->s.mark) {
hdr->p.input_flags.cls_mark = 1;
@@ -1537,8 +1557,11 @@ static inline cos_t *cls_select_cos(pktio_entry_t *entry,
default_cos = cls->default_cos;
/* Return error cos for error packet */
- if (pkt_hdr->p.flags.all.error)
- return cls->error_cos;
+ if (pkt_hdr->p.flags.all.error) {
+ cos = cls->error_cos;
+ goto done;
+ }
+
/* Calls all the PMRs attached at the PKTIO level*/
for (i = 0; i < odp_atomic_load_u32(&default_cos->s.num_rule); i++) {
pmr = default_cos->s.pmr[i];
@@ -1551,11 +1574,17 @@ static inline cos_t *cls_select_cos(pktio_entry_t *entry,
cos = match_qos_cos(entry, pkt_addr, pkt_hdr);
if (cos) {
ODP_DBG_RAW(CLS_DBG, " QoS matched -> cos: %s(%u)\n", cos->s.name, cos->s.index);
- return cos;
+ goto done;
}
ODP_DBG_RAW(CLS_DBG, " No match -> default cos\n");
- return cls->default_cos;
+ cos = cls->default_cos;
+
+done:
+ if (cos && cos->s.stats_enable)
+ odp_atomic_inc_u64(&cos->s.stats.packets);
+
+ return cos;
}
static uint32_t packet_rss_hash(odp_packet_hdr_t *pkt_hdr,
@@ -1575,12 +1604,12 @@ static uint32_t packet_rss_hash(odp_packet_hdr_t *pkt_hdr,
* @retval 0 on success
* @retval -EFAULT Bug
* @retval -EINVAL Config error
+ * @retval -ENOENT Drop action
*
* @note *base is not released
*/
int _odp_cls_classify_packet(pktio_entry_t *entry, const uint8_t *base,
- uint16_t pkt_len, uint32_t seg_len, odp_pool_t *pool,
- odp_packet_hdr_t *pkt_hdr, odp_bool_t parse)
+ odp_pool_t *pool, odp_packet_hdr_t *pkt_hdr)
{
cos_t *cos;
uint32_t tbl_index;
@@ -1588,24 +1617,19 @@ int _odp_cls_classify_packet(pktio_entry_t *entry, const uint8_t *base,
ODP_DBG_LVL(CLS_DBG, "Classify packet from %s\n", entry->s.full_name);
- if (parse) {
- packet_parse_reset(pkt_hdr, 1);
- packet_set_len(pkt_hdr, pkt_len);
-
- _odp_packet_parse_common(&pkt_hdr->p, base, pkt_len, seg_len,
- ODP_PROTO_LAYER_ALL,
- entry->s.in_chksums);
- }
cos = cls_select_cos(entry, base, pkt_hdr);
if (cos == NULL)
return -EINVAL;
+ if (cos->s.action == ODP_COS_ACTION_DROP)
+ return -ENOENT;
+
if (cos->s.queue == ODP_QUEUE_INVALID && cos->s.num_queue == 1)
- return -EFAULT;
+ goto error;
if (cos->s.pool == ODP_POOL_INVALID)
- return -EFAULT;
+ goto error;
*pool = cos->s.pool;
pkt_hdr->p.input_flags.dst_queue = 1;
@@ -1623,6 +1647,10 @@ int _odp_cls_classify_packet(pktio_entry_t *entry, const uint8_t *base,
cos->s.num_queue);
pkt_hdr->dst_queue = queue_grp_tbl->s.queue[tbl_index];
return 0;
+
+error:
+ odp_atomic_inc_u64(&cos->s.stats.discards);
+ return -EFAULT;
}
static uint32_t packet_rss_hash(odp_packet_hdr_t *pkt_hdr,
@@ -1780,6 +1808,27 @@ uint64_t odp_pmr_to_u64(odp_pmr_t hdl)
return _odp_pri(hdl);
}
+int odp_cls_cos_stats(odp_cos_t hdl, odp_cls_cos_stats_t *stats)
+{
+ cos_t *cos = get_cos_entry(hdl);
+
+ if (odp_unlikely(cos == NULL)) {
+ ODP_ERR("Invalid odp_cos_t handle\n");
+ return -1;
+ }
+
+ if (odp_unlikely(stats == NULL)) {
+ ODP_ERR("Output structure NULL\n");
+ return -1;
+ }
+
+ memset(stats, 0, sizeof(*stats));
+ stats->discards = odp_atomic_load_u64(&cos->s.stats.discards);
+ stats->packets = odp_atomic_load_u64(&cos->s.stats.packets);
+
+ return 0;
+}
+
int odp_cls_queue_stats(odp_cos_t hdl, odp_queue_t queue,
odp_cls_queue_stats_t *stats)
{
@@ -1803,8 +1852,8 @@ int odp_cls_queue_stats(odp_cos_t hdl, odp_queue_t queue,
}
memset(stats, 0, sizeof(odp_cls_queue_stats_t));
- stats->discards = odp_atomic_load_u64(&cos->s.stats[queue_idx].discards);
- stats->packets = odp_atomic_load_u64(&cos->s.stats[queue_idx].packets);
+ stats->discards = odp_atomic_load_u64(&cos->s.queue_stats[queue_idx].discards);
+ stats->packets = odp_atomic_load_u64(&cos->s.queue_stats[queue_idx].packets);
return 0;
}
diff --git a/platform/linux-generic/odp_crypto_null.c b/platform/linux-generic/odp_crypto_null.c
index bc14bad9a..f276d4659 100644
--- a/platform/linux-generic/odp_crypto_null.c
+++ b/platform/linux-generic/odp_crypto_null.c
@@ -269,9 +269,7 @@ int odp_crypto_session_destroy(odp_crypto_session_t session)
return 0;
}
-/*
- * Shim function around packet operation, can be used by other implementations.
- */
+#if ODP_DEPRECATED_API
int
odp_crypto_operation(odp_crypto_op_param_t *param,
odp_bool_t *posted,
@@ -326,6 +324,7 @@ odp_crypto_operation(odp_crypto_op_param_t *param,
return 0;
}
+#endif
int
_odp_crypto_init_global(void)
@@ -402,6 +401,7 @@ int _odp_crypto_term_local(void)
return 0;
}
+#if ODP_DEPRECATED_API
odp_crypto_compl_t odp_crypto_compl_from_event(odp_event_t ev)
{
/* This check not mandated by the API specification */
@@ -438,6 +438,7 @@ uint64_t odp_crypto_compl_to_u64(odp_crypto_compl_t hdl)
{
return _odp_pri(hdl);
}
+#endif
void odp_crypto_session_param_init(odp_crypto_session_param_t *param)
{
diff --git a/platform/linux-generic/odp_crypto_openssl.c b/platform/linux-generic/odp_crypto_openssl.c
index 13101fbca..6ff0ac041 100644
--- a/platform/linux-generic/odp_crypto_openssl.c
+++ b/platform/linux-generic/odp_crypto_openssl.c
@@ -31,6 +31,7 @@
#include <openssl/hmac.h>
#include <openssl/cmac.h>
#include <openssl/evp.h>
+#include <openssl/opensslv.h>
#if (OPENSSL_VERSION_NUMBER >= 0x10100000L) && !defined(OPENSSL_NO_POLY1305)
#define _ODP_HAVE_CHACHA20_POLY1305 1
@@ -38,6 +39,11 @@
#define _ODP_HAVE_CHACHA20_POLY1305 0
#endif
+/* Ignore warnings about APIs deprecated in OpenSSL 3.0 */
+#if OPENSSL_VERSION_NUMBER >= 0x30000000L
+#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
+#endif
+
#define MAX_SESSIONS 4000
#define AES_BLOCK_SIZE 16
#define AES_KEY_LENGTH 16
@@ -2495,6 +2501,7 @@ int odp_crypto_session_destroy(odp_crypto_session_t session)
/*
* Shim function around packet operation, can be used by other implementations.
*/
+#if ODP_DEPRECATED_API
int
odp_crypto_operation(odp_crypto_op_param_t *param,
odp_bool_t *posted,
@@ -2549,6 +2556,7 @@ odp_crypto_operation(odp_crypto_op_param_t *param,
return 0;
}
+#endif
#if OPENSSL_VERSION_NUMBER < 0x10100000L
static void ODP_UNUSED openssl_thread_id(CRYPTO_THREADID ODP_UNUSED *id)
@@ -2705,6 +2713,7 @@ int _odp_crypto_term_local(void)
return 0;
}
+#if ODP_DEPRECATED_API
odp_crypto_compl_t odp_crypto_compl_from_event(odp_event_t ev)
{
/* This check not mandated by the API specification */
@@ -2741,6 +2750,7 @@ uint64_t odp_crypto_compl_to_u64(odp_crypto_compl_t hdl)
{
return _odp_pri(hdl);
}
+#endif /* ODP_DEPRECATED_API */
void odp_crypto_session_param_init(odp_crypto_session_param_t *param)
{
diff --git a/platform/linux-generic/odp_dma.c b/platform/linux-generic/odp_dma.c
index 87e3b5b73..4c129c616 100644
--- a/platform/linux-generic/odp_dma.c
+++ b/platform/linux-generic/odp_dma.c
@@ -10,6 +10,7 @@
#include <odp/api/align.h>
#include <odp/api/buffer.h>
#include <odp/api/stash.h>
+#include <odp/api/packet.h>
#include <odp/api/pool.h>
#include <odp/api/queue.h>
diff --git a/platform/linux-generic/odp_event.c b/platform/linux-generic/odp_event.c
index 95be9dec2..90c3e3a3c 100644
--- a/platform/linux-generic/odp_event.c
+++ b/platform/linux-generic/odp_event.c
@@ -24,6 +24,7 @@
#include <odp/api/plat/event_inlines.h>
#include <odp/api/plat/packet_inlines.h>
#include <odp/api/plat/packet_vector_inlines.h>
+#include <odp/api/plat/timer_inlines.h>
odp_event_subtype_t odp_event_subtype(odp_event_t event)
{
@@ -70,9 +71,11 @@ void odp_event_free(odp_event_t event)
case ODP_EVENT_TIMEOUT:
odp_timeout_free(odp_timeout_from_event(event));
break;
+#if ODP_DEPRECATED_API
case ODP_EVENT_CRYPTO_COMPL:
odp_crypto_compl_free(odp_crypto_compl_from_event(event));
break;
+#endif
case ODP_EVENT_IPSEC_STATUS:
_odp_ipsec_status_free(_odp_ipsec_status_from_event(event));
break;
@@ -120,8 +123,10 @@ int odp_event_is_valid(odp_event_t event)
/* Fall through */
case ODP_EVENT_TIMEOUT:
/* Fall through */
+#if ODP_DEPRECATED_API
case ODP_EVENT_CRYPTO_COMPL:
/* Fall through */
+#endif
case ODP_EVENT_IPSEC_STATUS:
/* Fall through */
case ODP_EVENT_PACKET_VECTOR:
diff --git a/platform/linux-generic/odp_ipsec.c b/platform/linux-generic/odp_ipsec.c
index 4dec8f440..29f0e4bcd 100644
--- a/platform/linux-generic/odp_ipsec.c
+++ b/platform/linux-generic/odp_ipsec.c
@@ -1803,6 +1803,7 @@ static ipsec_sa_t *ipsec_out_single(odp_packet_t pkt,
state.out_tunnel.ip_tos = 0;
state.out_tunnel.ip_df = 0;
state.out_tunnel.ip_flabel = 0;
+ state.ip_next_hdr = _ODP_IPPROTO_NO_NEXT;
rc = 0;
} else {
rc = -1;
@@ -2269,12 +2270,6 @@ int odp_ipsec_out_inline(const odp_packet_t pkt_in[], int num_in,
ptr) < 0)
status.error.alg = 1;
- packet_subtype_set(pkt, ODP_EVENT_PACKET_IPSEC);
- result = ipsec_pkt_result(pkt);
- memset(result, 0, sizeof(*result));
- result->sa = ipsec_sa->ipsec_sa_hdl;
- result->status = status;
-
if (!status.error.all) {
odp_pktout_queue_t pkqueue;
diff --git a/platform/linux-generic/odp_ipsec_sad.c b/platform/linux-generic/odp_ipsec_sad.c
index c47e4912e..4e4057ede 100644
--- a/platform/linux-generic/odp_ipsec_sad.c
+++ b/platform/linux-generic/odp_ipsec_sad.c
@@ -863,7 +863,10 @@ ipsec_sa_t *_odp_ipsec_sa_lookup(const ipsec_sa_lookup_t *lookup)
lookup->proto == ipsec_sa->proto &&
lookup->spi == ipsec_sa->spi &&
lookup->ver == ipsec_sa->in.lookup_ver &&
- !memcmp(lookup->dst_addr, &ipsec_sa->in.lookup_dst_ipv4,
+ !memcmp(lookup->dst_addr,
+ lookup->ver == ODP_IPSEC_IPV4 ?
+ (void *)&ipsec_sa->in.lookup_dst_ipv4 :
+ (void *)&ipsec_sa->in.lookup_dst_ipv6,
lookup->ver == ODP_IPSEC_IPV4 ?
_ODP_IPV4ADDR_LEN :
_ODP_IPV6ADDR_LEN)) {
diff --git a/platform/linux-generic/odp_packet.c b/platform/linux-generic/odp_packet.c
index b91f8ca70..b0d0fc03c 100644
--- a/platform/linux-generic/odp_packet.c
+++ b/platform/linux-generic/odp_packet.c
@@ -16,6 +16,7 @@
#include <odp/api/proto_stats.h>
#include <odp/api/timer.h>
+#include <odp_parse_internal.h>
#include <odp_chksum_internal.h>
#include <odp_debug_internal.h>
#include <odp_errno_define.h>
@@ -131,11 +132,6 @@ static inline uint32_t packet_seg_len(odp_packet_hdr_t *pkt_hdr,
return pkt_hdr->seg_len;
}
-static inline uint32_t packet_first_seg_len(odp_packet_hdr_t *pkt_hdr)
-{
- return pkt_hdr->seg_len;
-}
-
static inline void *packet_tail(odp_packet_hdr_t *pkt_hdr)
{
odp_packet_hdr_t *last_seg = packet_last_seg(pkt_hdr);
@@ -1771,500 +1767,6 @@ static uint32_t packet_sum_crc32c(odp_packet_hdr_t *pkt_hdr,
return sum;
}
-/*
- * In the worst case we look at the Ethernet header, 8 bytes of LLC/SNAP
- * header and two VLAN tags in the same packet.
- */
-#define PARSE_ETH_BYTES (sizeof(_odp_ethhdr_t) + 8 + 2 * sizeof(_odp_vlanhdr_t))
-/** Parser helper function for Ethernet packets
- *
- * Requires up to PARSE_ETH_BYTES bytes of contiguous packet data.
- */
-static inline uint16_t parse_eth(packet_parser_t *prs, const uint8_t **parseptr,
- uint32_t *offset, uint32_t frame_len)
-{
- uint16_t ethtype;
- const _odp_ethhdr_t *eth;
- uint16_t macaddr0, macaddr2, macaddr4;
- const _odp_vlanhdr_t *vlan;
- _odp_packet_input_flags_t input_flags;
-
- input_flags.all = 0;
- input_flags.l2 = 1;
- input_flags.eth = 1;
-
- eth = (const _odp_ethhdr_t *)*parseptr;
-
- /* Detect jumbo frames */
- if (odp_unlikely(frame_len - *offset > _ODP_ETH_LEN_MAX))
- input_flags.jumbo = 1;
-
- /* Handle Ethernet broadcast/multicast addresses */
- macaddr0 = odp_be_to_cpu_16(*((const uint16_t *)(const void *)eth));
- if (odp_unlikely((macaddr0 & 0x0100) == 0x0100))
- input_flags.eth_mcast = 1;
-
- if (odp_unlikely(macaddr0 == 0xffff)) {
- macaddr2 =
- odp_be_to_cpu_16(*((const uint16_t *)
- (const void *)eth + 1));
- macaddr4 =
- odp_be_to_cpu_16(*((const uint16_t *)
- (const void *)eth + 2));
-
- if ((macaddr2 == 0xffff) && (macaddr4 == 0xffff))
- input_flags.eth_bcast = 1;
- }
-
- /* Get Ethertype */
- ethtype = odp_be_to_cpu_16(eth->type);
- *offset += sizeof(*eth);
- *parseptr += sizeof(*eth);
-
- /* Check for SNAP vs. DIX */
- if (odp_unlikely(ethtype < _ODP_ETH_LEN_MAX)) {
- input_flags.snap = 1;
- if (ethtype > frame_len - *offset) {
- prs->flags.snap_len_err = 1;
- ethtype = 0;
- goto error;
- }
- ethtype = odp_be_to_cpu_16(*((const uint16_t *)(uintptr_t)
- (*parseptr + 6)));
- *offset += 8;
- *parseptr += 8;
- }
-
- /* Parse the VLAN header(s), if present */
- if (odp_unlikely(ethtype == _ODP_ETHTYPE_VLAN_OUTER)) {
- input_flags.vlan_qinq = 1;
- input_flags.vlan = 1;
-
- vlan = (const _odp_vlanhdr_t *)*parseptr;
- ethtype = odp_be_to_cpu_16(vlan->type);
- *offset += sizeof(_odp_vlanhdr_t);
- *parseptr += sizeof(_odp_vlanhdr_t);
- }
-
- if (ethtype == _ODP_ETHTYPE_VLAN) {
- input_flags.vlan = 1;
- vlan = (const _odp_vlanhdr_t *)*parseptr;
- ethtype = odp_be_to_cpu_16(vlan->type);
- *offset += sizeof(_odp_vlanhdr_t);
- *parseptr += sizeof(_odp_vlanhdr_t);
- }
-
- /*
- * The packet was too short for what we parsed. We just give up
- * entirely without trying to parse what fits in the packet.
- */
- if (odp_unlikely(*offset > frame_len)) {
- input_flags.all = 0;
- input_flags.l2 = 1;
- ethtype = 0;
- }
-
-error:
- prs->input_flags.all |= input_flags.all;
-
- return ethtype;
-}
-
-#define PARSE_IPV4_BYTES (0xfU * 4) /* max IPv4 header length with options */
-/**
- * Parser helper function for IPv4
- *
- * Requires up to PARSE_IPV4_BYTES bytes of contiguous packet data.
- */
-static inline uint8_t parse_ipv4(packet_parser_t *prs, const uint8_t **parseptr,
- uint32_t *offset, uint32_t frame_len,
- odp_proto_chksums_t chksums,
- uint64_t *l4_part_sum)
-{
- const _odp_ipv4hdr_t *ipv4 = (const _odp_ipv4hdr_t *)*parseptr;
- uint32_t dstaddr = odp_be_to_cpu_32(ipv4->dst_addr);
- uint32_t l3_len = odp_be_to_cpu_16(ipv4->tot_len);
- uint16_t frag_offset = odp_be_to_cpu_16(ipv4->frag_offset);
- uint8_t ver = _ODP_IPV4HDR_VER(ipv4->ver_ihl);
- uint8_t ihl = _ODP_IPV4HDR_IHL(ipv4->ver_ihl);
-
- if (odp_unlikely(ihl < _ODP_IPV4HDR_IHL_MIN ||
- ver != 4 ||
- sizeof(*ipv4) > frame_len - *offset ||
- (l3_len > frame_len - *offset))) {
- prs->flags.ip_err = 1;
- return 0;
- }
-
- if (chksums.chksum.ipv4) {
- prs->input_flags.l3_chksum_done = 1;
- if (chksum_finalize(chksum_partial(ipv4, ihl * 4, 0)) != 0xffff) {
- prs->flags.ip_err = 1;
- prs->flags.l3_chksum_err = 1;
- return 0;
- }
- }
-
- *offset += ihl * 4;
- *parseptr += ihl * 4;
-
- if (chksums.chksum.udp || chksums.chksum.tcp)
- *l4_part_sum = chksum_partial((const uint8_t *)&ipv4->src_addr,
- 2 * _ODP_IPV4ADDR_LEN, 0);
-
- if (odp_unlikely(ihl > _ODP_IPV4HDR_IHL_MIN))
- prs->input_flags.ipopt = 1;
-
- /* A packet is a fragment if:
- * "more fragments" flag is set (all fragments except the last)
- * OR
- * "fragment offset" field is nonzero (all fragments except the first)
- */
- if (odp_unlikely(_ODP_IPV4HDR_IS_FRAGMENT(frag_offset)))
- prs->input_flags.ipfrag = 1;
-
- /* Handle IPv4 broadcast / multicast */
- if (odp_unlikely(dstaddr == 0xffffffff))
- prs->input_flags.ip_bcast = 1;
-
- if (odp_unlikely((dstaddr >> 28) == 0xe))
- prs->input_flags.ip_mcast = 1;
-
- return ipv4->proto;
-}
-
-/*
- * Peeks 2 bytes beyond IPv6 base header without length check if there
- * are extension headers.
- */
-#define PARSE_IPV6_BYTES (sizeof(_odp_ipv6hdr_t) + 2)
-/**
- * Parser helper function for IPv6
- *
- * Requires at least PARSE_IPV6_BYTES bytes of contiguous packet data.
- */
-static inline uint8_t parse_ipv6(packet_parser_t *prs, const uint8_t **parseptr,
- uint32_t *offset, uint32_t frame_len,
- uint32_t seg_len,
- odp_proto_chksums_t chksums,
- uint64_t *l4_part_sum)
-{
- const _odp_ipv6hdr_t *ipv6 = (const _odp_ipv6hdr_t *)*parseptr;
- const _odp_ipv6hdr_ext_t *ipv6ext;
- uint32_t dstaddr0 = odp_be_to_cpu_32(ipv6->dst_addr.u8[0]);
- uint32_t l3_len = odp_be_to_cpu_16(ipv6->payload_len) +
- _ODP_IPV6HDR_LEN;
-
- /* Basic sanity checks on IPv6 header */
- if (odp_unlikely((odp_be_to_cpu_32(ipv6->ver_tc_flow) >> 28) != 6 ||
- sizeof(*ipv6) > frame_len - *offset ||
- l3_len > frame_len - *offset)) {
- prs->flags.ip_err = 1;
- return 0;
- }
-
- /* IPv6 broadcast / multicast flags */
- prs->input_flags.ip_mcast = (dstaddr0 & 0xff000000) == 0xff000000;
- prs->input_flags.ip_bcast = 0;
-
- /* Skip past IPv6 header */
- *offset += sizeof(_odp_ipv6hdr_t);
- *parseptr += sizeof(_odp_ipv6hdr_t);
-
- if (chksums.chksum.udp || chksums.chksum.tcp)
- *l4_part_sum = chksum_partial((const uint8_t *)&ipv6->src_addr,
- 2 * _ODP_IPV6ADDR_LEN, 0);
-
- /* Skip past any IPv6 extension headers */
- if (ipv6->next_hdr == _ODP_IPPROTO_HOPOPTS ||
- ipv6->next_hdr == _ODP_IPPROTO_ROUTE) {
- prs->input_flags.ipopt = 1;
-
- do {
- ipv6ext = (const _odp_ipv6hdr_ext_t *)*parseptr;
- uint16_t extlen = 8 + ipv6ext->ext_len * 8;
-
- *offset += extlen;
- *parseptr += extlen;
- } while ((ipv6ext->next_hdr == _ODP_IPPROTO_HOPOPTS ||
- ipv6ext->next_hdr == _ODP_IPPROTO_ROUTE) &&
- *offset < seg_len);
-
- if (*offset >= prs->l3_offset +
- odp_be_to_cpu_16(ipv6->payload_len)) {
- prs->flags.ip_err = 1;
- return 0;
- }
-
- if (ipv6ext->next_hdr == _ODP_IPPROTO_FRAG)
- prs->input_flags.ipfrag = 1;
-
- return ipv6ext->next_hdr;
- }
-
- if (odp_unlikely(ipv6->next_hdr == _ODP_IPPROTO_FRAG)) {
- prs->input_flags.ipopt = 1;
- prs->input_flags.ipfrag = 1;
- }
-
- return ipv6->next_hdr;
-}
-
-#define PARSE_TCP_BYTES (sizeof(_odp_tcphdr_t))
-/**
- * Parser helper function for TCP
- *
- * Requires PARSE_TCP_BYTES bytes of contiguous packet data.
- */
-static inline void parse_tcp(packet_parser_t *prs, const uint8_t **parseptr,
- uint16_t tcp_len,
- odp_proto_chksums_t chksums,
- uint64_t *l4_part_sum)
-{
- const _odp_tcphdr_t *tcp = (const _odp_tcphdr_t *)*parseptr;
- uint32_t len = tcp->hl * 4;
-
- if (odp_unlikely(tcp->hl < sizeof(_odp_tcphdr_t) / sizeof(uint32_t)))
- prs->flags.tcp_err = 1;
-
- if (chksums.chksum.tcp &&
- !prs->input_flags.ipfrag) {
- *l4_part_sum += odp_cpu_to_be_16(tcp_len);
-#if ODP_BYTE_ORDER == ODP_BIG_ENDIAN
- *l4_part_sum += _ODP_IPPROTO_TCP;
-#else
- *l4_part_sum += _ODP_IPPROTO_TCP << 8;
-#endif
- }
-
- *parseptr += len;
-}
-
-/*
- * In the worst case we look at the UDP header and 4 bytes of the UDP
- * payload (the non-ESP marker to distinguish IKE packets from ESP packets).
- */
-#define PARSE_UDP_BYTES (sizeof(_odp_udphdr_t) + 4)
-/**
- * Parser helper function for UDP
- *
- * Requires PARSE_UDP_BYTES bytes of contiguous packet data.
- */
-static inline void parse_udp(packet_parser_t *prs, const uint8_t **parseptr,
- odp_proto_chksums_t chksums,
- uint64_t *l4_part_sum)
-{
- const _odp_udphdr_t *udp = (const _odp_udphdr_t *)*parseptr;
- uint32_t udplen = odp_be_to_cpu_16(udp->length);
- uint16_t ipsec_port = odp_cpu_to_be_16(_ODP_UDP_IPSEC_PORT);
-
- if (odp_unlikely(udplen < sizeof(_odp_udphdr_t))) {
- prs->flags.udp_err = 1;
- return;
- }
-
- if (chksums.chksum.udp &&
- !prs->input_flags.ipfrag) {
- if (udp->chksum == 0) {
- prs->input_flags.l4_chksum_done = 1;
- prs->flags.l4_chksum_err =
- (prs->input_flags.ipv4 != 1);
- } else {
- *l4_part_sum += udp->length;
-#if ODP_BYTE_ORDER == ODP_BIG_ENDIAN
- *l4_part_sum += _ODP_IPPROTO_UDP;
-#else
- *l4_part_sum += _ODP_IPPROTO_UDP << 8;
-#endif
- }
- prs->input_flags.udp_chksum_zero = (udp->chksum == 0);
- }
-
- if (odp_unlikely(ipsec_port == udp->dst_port && udplen > 4)) {
- uint32_t val;
-
- memcpy(&val, udp + 1, 4);
- if (val != 0) {
- prs->input_flags.ipsec = 1;
- prs->input_flags.ipsec_udp = 1;
- }
- }
-
- *parseptr += sizeof(_odp_udphdr_t);
-}
-
-#define PARSE_SCTP_BYTES (sizeof(_odp_sctphdr_t))
-/**
- * Parser helper function for SCTP
- *
- * Requires PARSE_SCTP_BYTES bytes of contiguous packet data.
- */
-static inline void parse_sctp(packet_parser_t *prs, const uint8_t **parseptr,
- uint16_t sctp_len,
- odp_proto_chksums_t chksums,
- uint64_t *l4_part_sum)
-{
- if (odp_unlikely(sctp_len < sizeof(_odp_sctphdr_t))) {
- prs->flags.sctp_err = 1;
- return;
- }
-
- if (chksums.chksum.sctp &&
- !prs->input_flags.ipfrag) {
- const _odp_sctphdr_t *sctp =
- (const _odp_sctphdr_t *)*parseptr;
- uint32_t crc = ~0;
- uint32_t zero = 0;
-
- crc = odp_hash_crc32c(sctp, sizeof(*sctp) - 4, crc);
- crc = odp_hash_crc32c(&zero, 4, crc);
- *l4_part_sum = crc;
- }
-
- *parseptr += sizeof(_odp_sctphdr_t);
-}
-
-#define MAX3(a, b, c) (MAX(MAX((a), (b)), (c)))
-#define PARSE_L3_L4_BYTES (MAX(PARSE_IPV4_BYTES, PARSE_IPV6_BYTES) + \
- MAX3(PARSE_TCP_BYTES, PARSE_UDP_BYTES, PARSE_SCTP_BYTES))
-/* Requires up to PARSE_L3_L4_BYTES bytes of contiguous packet data. */
-static inline
-int packet_parse_common_l3_l4(packet_parser_t *prs, const uint8_t *parseptr,
- uint32_t offset,
- uint32_t frame_len, uint32_t seg_len,
- int layer, uint16_t ethtype,
- odp_proto_chksums_t chksums,
- uint64_t *l4_part_sum)
-{
- uint8_t ip_proto;
-
- prs->l3_offset = offset;
-
- if (odp_unlikely(layer <= ODP_PROTO_LAYER_L2))
- return prs->flags.all.error != 0;
-
- /* Set l3 flag only for known ethtypes */
- prs->input_flags.l3 = 1;
-
- /* Parse Layer 3 headers */
- switch (ethtype) {
- case _ODP_ETHTYPE_IPV4:
- prs->input_flags.ipv4 = 1;
- ip_proto = parse_ipv4(prs, &parseptr, &offset, frame_len,
- chksums, l4_part_sum);
- prs->l4_offset = offset;
- break;
-
- case _ODP_ETHTYPE_IPV6:
- prs->input_flags.ipv6 = 1;
- ip_proto = parse_ipv6(prs, &parseptr, &offset, frame_len,
- seg_len, chksums, l4_part_sum);
- prs->l4_offset = offset;
- break;
-
- case _ODP_ETHTYPE_ARP:
- prs->input_flags.arp = 1;
- ip_proto = 255; /* Reserved invalid by IANA */
- break;
-
- default:
- prs->input_flags.l3 = 0;
- ip_proto = 255; /* Reserved invalid by IANA */
- }
-
- if (layer == ODP_PROTO_LAYER_L3)
- return prs->flags.all.error != 0;
-
- /* Set l4 flag only for known ip_proto */
- prs->input_flags.l4 = 1;
-
- /* Parse Layer 4 headers */
- switch (ip_proto) {
- case _ODP_IPPROTO_ICMPV4:
- /* Fall through */
-
- case _ODP_IPPROTO_ICMPV6:
- prs->input_flags.icmp = 1;
- break;
-
- case _ODP_IPPROTO_IPIP:
- /* Do nothing */
- break;
-
- case _ODP_IPPROTO_TCP:
- if (odp_unlikely(offset + _ODP_TCPHDR_LEN > seg_len))
- return -1;
- prs->input_flags.tcp = 1;
- parse_tcp(prs, &parseptr, frame_len - prs->l4_offset, chksums,
- l4_part_sum);
- break;
-
- case _ODP_IPPROTO_UDP:
- if (odp_unlikely(offset + _ODP_UDPHDR_LEN > seg_len))
- return -1;
- prs->input_flags.udp = 1;
- parse_udp(prs, &parseptr, chksums, l4_part_sum);
- break;
-
- case _ODP_IPPROTO_AH:
- prs->input_flags.ipsec = 1;
- prs->input_flags.ipsec_ah = 1;
- break;
-
- case _ODP_IPPROTO_ESP:
- prs->input_flags.ipsec = 1;
- prs->input_flags.ipsec_esp = 1;
- break;
-
- case _ODP_IPPROTO_SCTP:
- prs->input_flags.sctp = 1;
- parse_sctp(prs, &parseptr, frame_len - prs->l4_offset, chksums,
- l4_part_sum);
- break;
-
- case _ODP_IPPROTO_NO_NEXT:
- prs->input_flags.no_next_hdr = 1;
- break;
-
- default:
- prs->input_flags.l4 = 0;
- break;
- }
-
- return prs->flags.all.error != 0;
-}
-
-/**
- * Parse common packet headers up to given layer
- *
- * The function expects at least PACKET_PARSE_SEG_LEN bytes of data to be
- * available from the ptr. Also parse metadata must be already initialized.
- */
-int _odp_packet_parse_common(packet_parser_t *prs, const uint8_t *ptr,
- uint32_t frame_len, uint32_t seg_len,
- int layer, odp_proto_chksums_t chksums)
-{
- uint32_t offset;
- uint16_t ethtype;
- const uint8_t *parseptr;
- uint64_t l4_part_sum;
-
- parseptr = ptr;
- offset = 0;
-
- if (odp_unlikely(layer == ODP_PROTO_LAYER_NONE))
- return 0;
-
- /* Assume valid L2 header, no CRC/FCS check in SW */
- prs->l2_offset = offset;
-
- ethtype = parse_eth(prs, &parseptr, &offset, frame_len);
-
- return packet_parse_common_l3_l4(prs, parseptr, offset, frame_len,
- seg_len, layer, ethtype, chksums,
- &l4_part_sum);
-}
-
static inline int packet_ipv4_chksum(odp_packet_t pkt,
uint32_t offset,
_odp_ipv4hdr_t *ip,
@@ -2442,9 +1944,8 @@ int _odp_packet_sctp_chksum_insert(odp_packet_t pkt)
return odp_packet_copy_from_mem(pkt, pkt_hdr->p.l4_offset + 8, 4, &sum);
}
-static int packet_l4_chksum(odp_packet_hdr_t *pkt_hdr,
- odp_proto_chksums_t chksums,
- uint64_t l4_part_sum)
+int _odp_packet_l4_chksum(odp_packet_hdr_t *pkt_hdr,
+ odp_proto_chksums_t chksums, uint64_t l4_part_sum)
{
/* UDP chksum == 0 case is covered in parse_udp() */
if (chksums.chksum.udp &&
@@ -2518,42 +2019,6 @@ static int packet_l4_chksum(odp_packet_hdr_t *pkt_hdr,
return pkt_hdr->p.flags.all.error != 0;
}
-/**
- * Simple packet parser
- */
-int _odp_packet_parse_layer(odp_packet_hdr_t *pkt_hdr,
- odp_proto_layer_t layer,
- odp_proto_chksums_t chksums)
-{
- uint32_t seg_len = packet_first_seg_len(pkt_hdr);
- const uint8_t *base = packet_data(pkt_hdr);
- uint32_t offset = 0;
- uint16_t ethtype;
- uint64_t l4_part_sum = 0;
- int rc;
-
- if (odp_unlikely(layer == ODP_PROTO_LAYER_NONE))
- return 0;
-
- /* Assume valid L2 header, no CRC/FCS check in SW */
- pkt_hdr->p.l2_offset = offset;
-
- ethtype = parse_eth(&pkt_hdr->p, &base, &offset, pkt_hdr->frame_len);
-
- rc = packet_parse_common_l3_l4(&pkt_hdr->p, base, offset,
- pkt_hdr->frame_len,
- seg_len, layer, ethtype, chksums,
- &l4_part_sum);
-
- if (rc != 0)
- return rc;
-
- if (layer >= ODP_PROTO_LAYER_L4)
- return packet_l4_chksum(pkt_hdr, chksums, l4_part_sum);
- else
- return 0;
-}
-
int odp_packet_parse(odp_packet_t pkt, uint32_t offset,
const odp_packet_parse_param_t *param)
{
@@ -2568,6 +2033,7 @@ int odp_packet_parse(odp_packet_t pkt, uint32_t offset,
uint64_t l4_part_sum = 0;
const uint32_t min_seglen = PARSE_ETH_BYTES + PARSE_L3_L4_BYTES;
uint8_t buf[min_seglen];
+ odp_pktin_config_opt_t opt;
if (proto == ODP_PROTO_NONE || layer == ODP_PROTO_LAYER_NONE)
return -1;
@@ -2598,7 +2064,7 @@ int odp_packet_parse(odp_packet_t pkt, uint32_t offset,
/* Assume valid L2 header, no CRC/FCS check in SW */
pkt_hdr->p.l2_offset = offset;
- ethtype = parse_eth(&pkt_hdr->p, &data, &offset, packet_len);
+ ethtype = _odp_parse_eth(&pkt_hdr->p, &data, &offset, packet_len);
} else if (proto == ODP_PROTO_IPV4) {
ethtype = _ODP_ETHTYPE_IPV4;
} else if (proto == ODP_PROTO_IPV6) {
@@ -2607,17 +2073,18 @@ int odp_packet_parse(odp_packet_t pkt, uint32_t offset,
ethtype = 0; /* Invalid */
}
- ret = packet_parse_common_l3_l4(&pkt_hdr->p, data, offset,
- packet_len, seg_len,
- layer, ethtype,
- param->chksums,
- &l4_part_sum);
+ opt.all_bits = 0;
+
+ ret = _odp_packet_parse_common_l3_l4(&pkt_hdr->p, data, offset,
+ packet_len, seg_len, layer,
+ ethtype, param->chksums,
+ &l4_part_sum, opt);
if (ret)
return -1;
if (layer >= ODP_PROTO_LAYER_L4) {
- ret = packet_l4_chksum(pkt_hdr, param->chksums, l4_part_sum);
+ ret = _odp_packet_l4_chksum(pkt_hdr, param->chksums, l4_part_sum);
if (ret)
return -1;
}
@@ -2879,14 +2346,17 @@ int odp_packet_payload_offset_set(odp_packet_t pkt, uint32_t offset)
void odp_packet_aging_tmo_set(odp_packet_t pkt, uint64_t tmo_ns)
{
- (void)pkt;
- (void)tmo_ns;
+ odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt);
+
+ pkt_hdr->p.flags.tx_aging = tmo_ns ? 1 : 0;
+ pkt_hdr->tx_aging_ns = tmo_ns;
}
uint64_t odp_packet_aging_tmo(odp_packet_t pkt)
{
- (void)pkt;
- return 0;
+ odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt);
+
+ return pkt_hdr->p.flags.tx_aging ? pkt_hdr->tx_aging_ns : 0;
}
int odp_packet_tx_compl_request(odp_packet_t pkt, const odp_packet_tx_compl_opt_t *opt)
diff --git a/platform/linux-generic/odp_packet_flags.c b/platform/linux-generic/odp_packet_flags.c
index bb5c26792..ef542a2cd 100644
--- a/platform/linux-generic/odp_packet_flags.c
+++ b/platform/linux-generic/odp_packet_flags.c
@@ -8,137 +8,11 @@
#include <odp/api/packet_flags.h>
#include <odp_packet_internal.h>
-#define retflag(pkt, x) do { \
- odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); \
- return pkt_hdr->p.x; \
- } while (0)
-
#define setflag(pkt, x, v) do { \
odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); \
pkt_hdr->p.x = (v) & 1; \
} while (0)
-int odp_packet_has_error(odp_packet_t pkt)
-{
- odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt);
-
- return pkt_hdr->p.flags.all.error != 0;
-}
-
-/* Get Input Flags */
-
-int odp_packet_has_l2_error(odp_packet_t pkt)
-{
- odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt);
- /* L2 parsing is always done by default and hence
- no additional check is required */
- return pkt_hdr->p.flags.snap_len_err;
-}
-
-int odp_packet_has_l3(odp_packet_t pkt)
-{
- retflag(pkt, input_flags.l3);
-}
-
-int odp_packet_has_l3_error(odp_packet_t pkt)
-{
- odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt);
-
- return pkt_hdr->p.flags.ip_err;
-}
-
-int odp_packet_has_l4(odp_packet_t pkt)
-{
- retflag(pkt, input_flags.l4);
-}
-
-int odp_packet_has_l4_error(odp_packet_t pkt)
-{
- odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt);
-
- return pkt_hdr->p.flags.tcp_err | pkt_hdr->p.flags.udp_err;
-}
-
-int odp_packet_has_eth_bcast(odp_packet_t pkt)
-{
- retflag(pkt, input_flags.eth_bcast);
-}
-
-int odp_packet_has_eth_mcast(odp_packet_t pkt)
-{
- retflag(pkt, input_flags.eth_mcast);
-}
-
-int odp_packet_has_vlan(odp_packet_t pkt)
-{
- retflag(pkt, input_flags.vlan);
-}
-
-int odp_packet_has_vlan_qinq(odp_packet_t pkt)
-{
- retflag(pkt, input_flags.vlan_qinq);
-}
-
-int odp_packet_has_arp(odp_packet_t pkt)
-{
- retflag(pkt, input_flags.arp);
-}
-
-int odp_packet_has_ipv4(odp_packet_t pkt)
-{
- retflag(pkt, input_flags.ipv4);
-}
-
-int odp_packet_has_ipv6(odp_packet_t pkt)
-{
- retflag(pkt, input_flags.ipv6);
-}
-
-int odp_packet_has_ip_bcast(odp_packet_t pkt)
-{
- retflag(pkt, input_flags.ip_bcast);
-}
-
-int odp_packet_has_ip_mcast(odp_packet_t pkt)
-{
- retflag(pkt, input_flags.ip_mcast);
-}
-
-int odp_packet_has_ipfrag(odp_packet_t pkt)
-{
- retflag(pkt, input_flags.ipfrag);
-}
-
-int odp_packet_has_ipopt(odp_packet_t pkt)
-{
- retflag(pkt, input_flags.ipopt);
-}
-
-int odp_packet_has_udp(odp_packet_t pkt)
-{
- retflag(pkt, input_flags.udp);
-}
-
-int odp_packet_has_tcp(odp_packet_t pkt)
-{
- retflag(pkt, input_flags.tcp);
-}
-
-int odp_packet_has_sctp(odp_packet_t pkt)
-{
- retflag(pkt, input_flags.sctp);
-}
-
-int odp_packet_has_icmp(odp_packet_t pkt)
-{
- retflag(pkt, input_flags.icmp);
-}
-
-odp_packet_color_t odp_packet_color(odp_packet_t pkt)
-{
- retflag(pkt, input_flags.color);
-}
-
void odp_packet_color_set(odp_packet_t pkt, odp_packet_color_t color)
{
odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt);
@@ -146,23 +20,11 @@ void odp_packet_color_set(odp_packet_t pkt, odp_packet_color_t color)
pkt_hdr->p.input_flags.color = color;
}
-odp_bool_t odp_packet_drop_eligible(odp_packet_t pkt)
-{
- odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt);
-
- return !pkt_hdr->p.input_flags.nodrop;
-}
-
void odp_packet_drop_eligible_set(odp_packet_t pkt, odp_bool_t drop)
{
setflag(pkt, input_flags.nodrop, !drop);
}
-int8_t odp_packet_shaper_len_adjust(odp_packet_t pkt)
-{
- retflag(pkt, flags.shaper_len_adj);
-}
-
void odp_packet_shaper_len_adjust_set(odp_packet_t pkt, int8_t adj)
{
odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt);
diff --git a/platform/linux-generic/odp_packet_io.c b/platform/linux-generic/odp_packet_io.c
index 899f10425..7c68b10c5 100644
--- a/platform/linux-generic/odp_packet_io.c
+++ b/platform/linux-generic/odp_packet_io.c
@@ -53,6 +53,10 @@
/* Max wait time supported to avoid potential overflow */
#define MAX_WAIT_TIME (UINT64_MAX / 1024)
+/* One hour maximum aging timeout, no real limitations imposed by the implementation other than
+ * integer width, so just use some value. */
+#define MAX_TX_AGING_TMO_NS 3600000000000ULL
+
typedef struct {
const void *user_ptr;
odp_queue_t queue;
@@ -624,6 +628,8 @@ int odp_pktio_config(odp_pktio_t hdl, const odp_pktio_config_t *config)
return -1;
}
+ entry->s.enabled.tx_aging = config->pktout.bit.aging_ena;
+
if (entry->s.ops->config)
res = entry->s.ops->config(entry, config);
@@ -650,6 +656,9 @@ int odp_pktio_start(odp_pktio_t hdl)
ODP_ERR("Already started\n");
return -1;
}
+ entry->s.parse_layer = pktio_cls_enabled(entry) ?
+ ODP_PROTO_LAYER_ALL :
+ entry->s.config.parser.layer;
if (entry->s.ops->start)
res = entry->s.ops->start(entry);
if (!res)
@@ -660,8 +669,8 @@ int odp_pktio_start(odp_pktio_t hdl)
mode = entry->s.param.in_mode;
if (mode == ODP_PKTIN_MODE_SCHED) {
- unsigned int i;
- unsigned int num = entry->s.num_in_queue;
+ uint32_t i;
+ uint32_t num = entry->s.num_in_queue;
int index[num];
odp_queue_t odpq[num];
@@ -1182,95 +1191,6 @@ static int pktin_deq_multi(odp_queue_t queue, _odp_event_hdr_t *event_hdr[],
return nbr;
}
-int _odp_sched_cb_pktin_poll_one(int pktio_index,
- int rx_queue,
- odp_event_t evt_tbl[])
-{
- int num_rx, num_pkts, i;
- pktio_entry_t *entry = pktio_entry_by_index(pktio_index);
- odp_packet_t pkt;
- odp_packet_hdr_t *pkt_hdr;
- odp_pool_t pool = ODP_POOL_INVALID;
- odp_packet_t packets[QUEUE_MULTI_MAX];
- odp_queue_t queue;
- odp_bool_t vector_enabled = entry->s.in_queue[rx_queue].vector.enable;
- uint32_t num = QUEUE_MULTI_MAX;
- cos_t *cos_hdr = NULL;
-
- if (odp_unlikely(entry->s.state != PKTIO_STATE_STARTED)) {
- if (entry->s.state < PKTIO_STATE_ACTIVE ||
- entry->s.state == PKTIO_STATE_STOP_PENDING)
- return -1;
-
- ODP_DBG("interface not started\n");
- return 0;
- }
-
- if (vector_enabled) {
- /* Make sure all packets will fit into a single packet vector */
- if (entry->s.in_queue[rx_queue].vector.max_size < num)
- num = entry->s.in_queue[rx_queue].vector.max_size;
- pool = entry->s.in_queue[rx_queue].vector.pool;
- }
-
- ODP_ASSERT((unsigned int)rx_queue < entry->s.num_in_queue);
- num_pkts = entry->s.ops->recv(entry, rx_queue, packets, num);
-
- num_rx = 0;
- for (i = 0; i < num_pkts; i++) {
- pkt = packets[i];
- pkt_hdr = packet_hdr(pkt);
- if (odp_unlikely(pkt_hdr->p.input_flags.dst_queue)) {
- odp_event_t event = odp_packet_to_event(pkt);
- uint16_t cos_idx = pkt_hdr->cos;
-
- queue = pkt_hdr->dst_queue;
-
- if (cos_idx != CLS_COS_IDX_NONE) {
- /* Packets from classifier */
- cos_hdr = _odp_cos_entry_from_idx(cos_idx);
-
- if (cos_hdr->s.vector.enable) {
- packet_vector_enq_cos(queue, &event, 1, cos_hdr);
- continue;
- }
- } else if (vector_enabled) {
- /* Packets from inline IPsec */
- packet_vector_enq(queue, &event, 1, pool);
- continue;
- }
-
- if (odp_unlikely(odp_queue_enq(queue, event))) {
- /* Queue full? */
- odp_packet_free(pkt);
- if (cos_idx != CLS_COS_IDX_NONE)
- _odp_cos_queue_stats_add(cos_hdr, queue, 0, 1);
- else
- odp_atomic_inc_u64(&entry->s.stats_extra.in_discards);
- } else {
- if (cos_idx != CLS_COS_IDX_NONE)
- _odp_cos_queue_stats_add(cos_hdr, queue, 1, 0);
- }
- } else {
- evt_tbl[num_rx++] = odp_packet_to_event(pkt);
- }
- }
-
- /* Create packet vector */
- if (vector_enabled && num_rx > 0) {
- odp_packet_vector_t pktv = packet_vector_create((odp_packet_t *)evt_tbl,
- num_rx, pool);
-
- if (odp_unlikely(pktv == ODP_PACKET_VECTOR_INVALID))
- return 0;
-
- evt_tbl[0] = odp_packet_vector_to_event(pktv);
- return 1;
- }
-
- return num_rx;
-}
-
int _odp_sched_cb_pktin_poll(int pktio_index, int pktin_index,
_odp_event_hdr_t *hdr_tbl[], int num)
{
@@ -1886,6 +1806,9 @@ int odp_pktio_capability(odp_pktio_t pktio, odp_pktio_capability_t *capa)
capa->tx_compl.queue_type_sched = 1;
capa->tx_compl.queue_type_plain = 1;
capa->tx_compl.mode_all = 1;
+
+ capa->config.pktout.bit.aging_ena = 1;
+ capa->max_tx_aging_tmo_ns = MAX_TX_AGING_TMO_NS;
}
/* Packet vector generation is common for all pktio types */
@@ -2246,8 +2169,7 @@ int odp_pktin_queue_config(odp_pktio_t pktio,
pktio_entry_t *entry;
odp_pktin_mode_t mode;
odp_pktio_capability_t capa;
- unsigned int num_queues;
- unsigned int i;
+ uint32_t num_queues, i;
int rc;
odp_queue_t queue;
odp_pktin_queue_param_t default_param;
@@ -2410,7 +2332,7 @@ int _odp_pktio_pktout_tm_config(odp_pktio_t pktio_hdl,
bool pktio_started = false;
odp_pktout_mode_t mode;
pktio_entry_t *entry;
- unsigned int i;
+ uint32_t i;
int rc = 0;
odp_pktout_queue_param_init(&param);
@@ -2480,8 +2402,7 @@ int odp_pktout_queue_config(odp_pktio_t pktio,
pktio_entry_t *entry;
odp_pktout_mode_t mode;
odp_pktio_capability_t capa;
- unsigned int num_queues;
- unsigned int i;
+ uint32_t num_queues, i;
int rc;
odp_pktout_queue_param_t default_param;
@@ -2847,18 +2768,17 @@ int odp_pktin_recv_tmo(odp_pktin_queue_t queue, odp_packet_t packets[], int num,
}
}
-int odp_pktin_recv_mq_tmo(const odp_pktin_queue_t queues[], unsigned int num_q,
- unsigned int *from, odp_packet_t packets[], int num,
- uint64_t wait)
+int odp_pktin_recv_mq_tmo(const odp_pktin_queue_t queues[], uint32_t num_q, uint32_t *from,
+ odp_packet_t packets[], int num, uint64_t wait)
{
- unsigned int i;
+ uint32_t i;
int ret;
odp_time_t t1, t2;
struct timespec ts;
int started = 0;
uint64_t sleep_round = 0;
int trial_successful = 0;
- unsigned int lfrom = 0;
+ uint32_t lfrom = 0;
for (i = 0; i < num_q; i++) {
ret = odp_pktin_recv(queues[i], packets, num);
diff --git a/platform/linux-generic/odp_parse.c b/platform/linux-generic/odp_parse.c
new file mode 100644
index 000000000..2342a7e49
--- /dev/null
+++ b/platform/linux-generic/odp_parse.c
@@ -0,0 +1,471 @@
+/* Copyright (c) 2013-2018, Linaro Limited
+ * Copyright (c) 2019-2022, Nokia
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+#include <odp_parse_internal.h>
+#include <odp_chksum_internal.h>
+#include <protocols/eth.h>
+#include <protocols/ip.h>
+#include <protocols/sctp.h>
+#include <protocols/tcp.h>
+#include <protocols/udp.h>
+#include <odp/api/hash.h>
+#include <odp/api/packet_io.h>
+#include <odp/api/packet_types.h>
+#include <stdint.h>
+#include <string.h>
+
+/** Parser helper function for Ethernet packets
+ *
+ * Requires up to PARSE_ETH_BYTES bytes of contiguous packet data.
+ */
+uint16_t _odp_parse_eth(packet_parser_t *prs, const uint8_t **parseptr,
+ uint32_t *offset, uint32_t frame_len)
+{
+ uint16_t ethtype;
+ const _odp_ethhdr_t *eth;
+ uint16_t macaddr0, macaddr2, macaddr4;
+ const _odp_vlanhdr_t *vlan;
+ _odp_packet_input_flags_t input_flags;
+
+ input_flags.all = 0;
+ input_flags.l2 = 1;
+ input_flags.eth = 1;
+
+ eth = (const _odp_ethhdr_t *)*parseptr;
+
+ /* Detect jumbo frames */
+ if (odp_unlikely(frame_len - *offset > _ODP_ETH_LEN_MAX))
+ input_flags.jumbo = 1;
+
+ /* Handle Ethernet broadcast/multicast addresses */
+ macaddr0 = odp_be_to_cpu_16(*((const uint16_t *)(const void *)eth));
+ if (odp_unlikely((macaddr0 & 0x0100) == 0x0100))
+ input_flags.eth_mcast = 1;
+
+ if (odp_unlikely(macaddr0 == 0xffff)) {
+ macaddr2 =
+ odp_be_to_cpu_16(*((const uint16_t *)
+ (const void *)eth + 1));
+ macaddr4 =
+ odp_be_to_cpu_16(*((const uint16_t *)
+ (const void *)eth + 2));
+
+ if ((macaddr2 == 0xffff) && (macaddr4 == 0xffff))
+ input_flags.eth_bcast = 1;
+ }
+
+ /* Get Ethertype */
+ ethtype = odp_be_to_cpu_16(eth->type);
+ *offset += sizeof(*eth);
+ *parseptr += sizeof(*eth);
+
+ /* Check for SNAP vs. DIX */
+ if (odp_unlikely(ethtype < _ODP_ETH_LEN_MAX)) {
+ input_flags.snap = 1;
+ if (ethtype > frame_len - *offset) {
+ prs->flags.snap_len_err = 1;
+ ethtype = 0;
+ goto error;
+ }
+ ethtype = odp_be_to_cpu_16(*((const uint16_t *)(uintptr_t)
+ (*parseptr + 6)));
+ *offset += 8;
+ *parseptr += 8;
+ }
+
+ /* Parse the VLAN header(s), if present */
+ if (odp_unlikely(ethtype == _ODP_ETHTYPE_VLAN_OUTER)) {
+ input_flags.vlan_qinq = 1;
+ input_flags.vlan = 1;
+
+ vlan = (const _odp_vlanhdr_t *)*parseptr;
+ ethtype = odp_be_to_cpu_16(vlan->type);
+ *offset += sizeof(_odp_vlanhdr_t);
+ *parseptr += sizeof(_odp_vlanhdr_t);
+ }
+
+ if (ethtype == _ODP_ETHTYPE_VLAN) {
+ input_flags.vlan = 1;
+ vlan = (const _odp_vlanhdr_t *)*parseptr;
+ ethtype = odp_be_to_cpu_16(vlan->type);
+ *offset += sizeof(_odp_vlanhdr_t);
+ *parseptr += sizeof(_odp_vlanhdr_t);
+ }
+
+ /*
+ * The packet was too short for what we parsed. We just give up
+ * entirely without trying to parse what fits in the packet.
+ */
+ if (odp_unlikely(*offset > frame_len)) {
+ input_flags.all = 0;
+ input_flags.l2 = 1;
+ ethtype = 0;
+ }
+
+error:
+ prs->input_flags.all |= input_flags.all;
+
+ return ethtype;
+}
+
+/**
+ * Parser helper function for IPv4
+ *
+ * Requires up to PARSE_IPV4_BYTES bytes of contiguous packet data.
+ */
+static inline uint8_t parse_ipv4(packet_parser_t *prs, const uint8_t **parseptr,
+ uint32_t *offset, uint32_t frame_len,
+ odp_proto_chksums_t chksums,
+ uint64_t *l4_part_sum)
+{
+ const _odp_ipv4hdr_t *ipv4 = (const _odp_ipv4hdr_t *)*parseptr;
+ uint32_t dstaddr = odp_be_to_cpu_32(ipv4->dst_addr);
+ uint32_t l3_len = odp_be_to_cpu_16(ipv4->tot_len);
+ uint16_t frag_offset = odp_be_to_cpu_16(ipv4->frag_offset);
+ uint8_t ver = _ODP_IPV4HDR_VER(ipv4->ver_ihl);
+ uint8_t ihl = _ODP_IPV4HDR_IHL(ipv4->ver_ihl);
+
+ if (odp_unlikely(ihl < _ODP_IPV4HDR_IHL_MIN ||
+ ver != 4 ||
+ sizeof(*ipv4) > frame_len - *offset ||
+ (l3_len > frame_len - *offset))) {
+ prs->flags.ip_err = 1;
+ return 0;
+ }
+
+ if (chksums.chksum.ipv4) {
+ prs->input_flags.l3_chksum_done = 1;
+ if (chksum_finalize(chksum_partial(ipv4, ihl * 4, 0)) != 0xffff) {
+ prs->flags.ip_err = 1;
+ prs->flags.l3_chksum_err = 1;
+ return 0;
+ }
+ }
+
+ *offset += ihl * 4;
+ *parseptr += ihl * 4;
+
+ if (chksums.chksum.udp || chksums.chksum.tcp)
+ *l4_part_sum = chksum_partial((const uint8_t *)&ipv4->src_addr,
+ 2 * _ODP_IPV4ADDR_LEN, 0);
+
+ if (odp_unlikely(ihl > _ODP_IPV4HDR_IHL_MIN))
+ prs->input_flags.ipopt = 1;
+
+ /* A packet is a fragment if:
+ * "more fragments" flag is set (all fragments except the last)
+ * OR
+ * "fragment offset" field is nonzero (all fragments except the first)
+ */
+ if (odp_unlikely(_ODP_IPV4HDR_IS_FRAGMENT(frag_offset)))
+ prs->input_flags.ipfrag = 1;
+
+ /* Handle IPv4 broadcast / multicast */
+ if (odp_unlikely(dstaddr == 0xffffffff))
+ prs->input_flags.ip_bcast = 1;
+
+ if (odp_unlikely((dstaddr >> 28) == 0xe))
+ prs->input_flags.ip_mcast = 1;
+
+ return ipv4->proto;
+}
+
+/**
+ * Parser helper function for IPv6
+ *
+ * Requires at least PARSE_IPV6_BYTES bytes of contiguous packet data.
+ */
+static inline uint8_t parse_ipv6(packet_parser_t *prs, const uint8_t **parseptr,
+ uint32_t *offset, uint32_t frame_len,
+ uint32_t seg_len,
+ odp_proto_chksums_t chksums,
+ uint64_t *l4_part_sum)
+{
+ const _odp_ipv6hdr_t *ipv6 = (const _odp_ipv6hdr_t *)*parseptr;
+ const _odp_ipv6hdr_ext_t *ipv6ext;
+ uint32_t dstaddr0 = odp_be_to_cpu_32(ipv6->dst_addr.u8[0]);
+ uint32_t l3_len = odp_be_to_cpu_16(ipv6->payload_len) +
+ _ODP_IPV6HDR_LEN;
+
+ /* Basic sanity checks on IPv6 header */
+ if (odp_unlikely((odp_be_to_cpu_32(ipv6->ver_tc_flow) >> 28) != 6 ||
+ sizeof(*ipv6) > frame_len - *offset ||
+ l3_len > frame_len - *offset)) {
+ prs->flags.ip_err = 1;
+ return 0;
+ }
+
+ /* IPv6 broadcast / multicast flags */
+ prs->input_flags.ip_mcast = (dstaddr0 & 0xff000000) == 0xff000000;
+ prs->input_flags.ip_bcast = 0;
+
+ /* Skip past IPv6 header */
+ *offset += sizeof(_odp_ipv6hdr_t);
+ *parseptr += sizeof(_odp_ipv6hdr_t);
+
+ if (chksums.chksum.udp || chksums.chksum.tcp)
+ *l4_part_sum = chksum_partial((const uint8_t *)&ipv6->src_addr,
+ 2 * _ODP_IPV6ADDR_LEN, 0);
+
+ /* Skip past any IPv6 extension headers */
+ if (ipv6->next_hdr == _ODP_IPPROTO_HOPOPTS ||
+ ipv6->next_hdr == _ODP_IPPROTO_ROUTE) {
+ prs->input_flags.ipopt = 1;
+
+ do {
+ ipv6ext = (const _odp_ipv6hdr_ext_t *)*parseptr;
+ uint16_t extlen = 8 + ipv6ext->ext_len * 8;
+
+ *offset += extlen;
+ *parseptr += extlen;
+ } while ((ipv6ext->next_hdr == _ODP_IPPROTO_HOPOPTS ||
+ ipv6ext->next_hdr == _ODP_IPPROTO_ROUTE) &&
+ *offset < seg_len);
+
+ if (*offset >= prs->l3_offset +
+ odp_be_to_cpu_16(ipv6->payload_len)) {
+ prs->flags.ip_err = 1;
+ return 0;
+ }
+
+ if (ipv6ext->next_hdr == _ODP_IPPROTO_FRAG)
+ prs->input_flags.ipfrag = 1;
+
+ return ipv6ext->next_hdr;
+ }
+
+ if (odp_unlikely(ipv6->next_hdr == _ODP_IPPROTO_FRAG)) {
+ prs->input_flags.ipopt = 1;
+ prs->input_flags.ipfrag = 1;
+ }
+
+ return ipv6->next_hdr;
+}
+
+/**
+ * Parser helper function for TCP
+ *
+ * Requires PARSE_TCP_BYTES bytes of contiguous packet data.
+ */
+static inline void parse_tcp(packet_parser_t *prs, const uint8_t **parseptr,
+ uint16_t tcp_len,
+ odp_proto_chksums_t chksums,
+ uint64_t *l4_part_sum)
+{
+ const _odp_tcphdr_t *tcp = (const _odp_tcphdr_t *)*parseptr;
+ uint32_t len = tcp->hl * 4;
+
+ if (odp_unlikely(tcp->hl < sizeof(_odp_tcphdr_t) / sizeof(uint32_t)))
+ prs->flags.tcp_err = 1;
+
+ if (chksums.chksum.tcp &&
+ !prs->input_flags.ipfrag) {
+ *l4_part_sum += odp_cpu_to_be_16(tcp_len);
+#if ODP_BYTE_ORDER == ODP_BIG_ENDIAN
+ *l4_part_sum += _ODP_IPPROTO_TCP;
+#else
+ *l4_part_sum += _ODP_IPPROTO_TCP << 8;
+#endif
+ }
+
+ *parseptr += len;
+}
+
+/**
+ * Parser helper function for UDP
+ *
+ * Requires PARSE_UDP_BYTES bytes of contiguous packet data.
+ */
+static inline void parse_udp(packet_parser_t *prs, const uint8_t **parseptr,
+ odp_proto_chksums_t chksums,
+ uint64_t *l4_part_sum)
+{
+ const _odp_udphdr_t *udp = (const _odp_udphdr_t *)*parseptr;
+ uint32_t udplen = odp_be_to_cpu_16(udp->length);
+ uint16_t ipsec_port = odp_cpu_to_be_16(_ODP_UDP_IPSEC_PORT);
+
+ if (odp_unlikely(udplen < sizeof(_odp_udphdr_t))) {
+ prs->flags.udp_err = 1;
+ return;
+ }
+
+ if (chksums.chksum.udp &&
+ !prs->input_flags.ipfrag) {
+ if (udp->chksum == 0) {
+ prs->input_flags.l4_chksum_done = 1;
+ prs->flags.l4_chksum_err =
+ (prs->input_flags.ipv4 != 1);
+ } else {
+ *l4_part_sum += udp->length;
+#if ODP_BYTE_ORDER == ODP_BIG_ENDIAN
+ *l4_part_sum += _ODP_IPPROTO_UDP;
+#else
+ *l4_part_sum += _ODP_IPPROTO_UDP << 8;
+#endif
+ }
+ prs->input_flags.udp_chksum_zero = (udp->chksum == 0);
+ }
+
+ if (odp_unlikely(ipsec_port == udp->dst_port && udplen > 4)) {
+ uint32_t val;
+
+ memcpy(&val, udp + 1, 4);
+ if (val != 0) {
+ prs->input_flags.ipsec = 1;
+ prs->input_flags.ipsec_udp = 1;
+ }
+ }
+
+ *parseptr += sizeof(_odp_udphdr_t);
+}
+
+/**
+ * Parser helper function for SCTP
+ *
+ * Requires PARSE_SCTP_BYTES bytes of contiguous packet data.
+ */
+static inline void parse_sctp(packet_parser_t *prs, const uint8_t **parseptr,
+ uint16_t sctp_len,
+ odp_proto_chksums_t chksums,
+ uint64_t *l4_part_sum)
+{
+ if (odp_unlikely(sctp_len < sizeof(_odp_sctphdr_t))) {
+ prs->flags.sctp_err = 1;
+ return;
+ }
+
+ if (chksums.chksum.sctp &&
+ !prs->input_flags.ipfrag) {
+ const _odp_sctphdr_t *sctp =
+ (const _odp_sctphdr_t *)*parseptr;
+ uint32_t crc = ~0;
+ uint32_t zero = 0;
+
+ crc = odp_hash_crc32c(sctp, sizeof(*sctp) - 4, crc);
+ crc = odp_hash_crc32c(&zero, 4, crc);
+ *l4_part_sum = crc;
+ }
+
+ *parseptr += sizeof(_odp_sctphdr_t);
+}
+
+/* Requires up to PARSE_L3_L4_BYTES bytes of contiguous packet data. */
+int _odp_packet_parse_common_l3_l4(packet_parser_t *prs,
+ const uint8_t *parseptr, uint32_t offset,
+ uint32_t frame_len, uint32_t seg_len,
+ int layer, uint16_t ethtype,
+ odp_proto_chksums_t chksums,
+ uint64_t *l4_part_sum,
+ odp_pktin_config_opt_t opt)
+{
+ uint8_t ip_proto;
+
+ prs->l3_offset = offset;
+
+ if (odp_unlikely(layer <= ODP_PROTO_LAYER_L2))
+ return prs->flags.all.error != 0;
+
+ /* Set l3 flag only for known ethtypes */
+ prs->input_flags.l3 = 1;
+
+ /* Parse Layer 3 headers */
+ switch (ethtype) {
+ case _ODP_ETHTYPE_IPV4:
+ prs->input_flags.ipv4 = 1;
+ ip_proto = parse_ipv4(prs, &parseptr, &offset, frame_len,
+ chksums, l4_part_sum);
+ prs->l4_offset = offset;
+ if (prs->flags.ip_err && opt.bit.drop_ipv4_err)
+ return -1; /* drop */
+ break;
+
+ case _ODP_ETHTYPE_IPV6:
+ prs->input_flags.ipv6 = 1;
+ ip_proto = parse_ipv6(prs, &parseptr, &offset, frame_len,
+ seg_len, chksums, l4_part_sum);
+ prs->l4_offset = offset;
+ if (prs->flags.ip_err && opt.bit.drop_ipv6_err)
+ return -1; /* drop */
+ break;
+
+ case _ODP_ETHTYPE_ARP:
+ prs->input_flags.arp = 1;
+ ip_proto = 255; /* Reserved invalid by IANA */
+ break;
+
+ default:
+ prs->input_flags.l3 = 0;
+ ip_proto = 255; /* Reserved invalid by IANA */
+ }
+
+ if (layer == ODP_PROTO_LAYER_L3)
+ return prs->flags.all.error != 0;
+
+ /* Set l4 flag only for known ip_proto */
+ prs->input_flags.l4 = 1;
+
+ /* Parse Layer 4 headers */
+ switch (ip_proto) {
+ case _ODP_IPPROTO_ICMPV4:
+ /* Fall through */
+
+ case _ODP_IPPROTO_ICMPV6:
+ prs->input_flags.icmp = 1;
+ break;
+
+ case _ODP_IPPROTO_IPIP:
+ /* Do nothing */
+ break;
+
+ case _ODP_IPPROTO_TCP:
+ if (odp_unlikely(offset + _ODP_TCPHDR_LEN > seg_len))
+ return -1;
+ prs->input_flags.tcp = 1;
+ parse_tcp(prs, &parseptr, frame_len - prs->l4_offset, chksums,
+ l4_part_sum);
+ if (prs->flags.tcp_err && opt.bit.drop_tcp_err)
+ return -1; /* drop */
+ break;
+
+ case _ODP_IPPROTO_UDP:
+ if (odp_unlikely(offset + _ODP_UDPHDR_LEN > seg_len))
+ return -1;
+ prs->input_flags.udp = 1;
+ parse_udp(prs, &parseptr, chksums, l4_part_sum);
+ if (prs->flags.udp_err && opt.bit.drop_udp_err)
+ return -1; /* drop */
+ break;
+
+ case _ODP_IPPROTO_AH:
+ prs->input_flags.ipsec = 1;
+ prs->input_flags.ipsec_ah = 1;
+ break;
+
+ case _ODP_IPPROTO_ESP:
+ prs->input_flags.ipsec = 1;
+ prs->input_flags.ipsec_esp = 1;
+ break;
+
+ case _ODP_IPPROTO_SCTP:
+ prs->input_flags.sctp = 1;
+ parse_sctp(prs, &parseptr, frame_len - prs->l4_offset, chksums,
+ l4_part_sum);
+ if (prs->flags.sctp_err && opt.bit.drop_sctp_err)
+ return -1; /* drop */
+ break;
+
+ case _ODP_IPPROTO_NO_NEXT:
+ prs->input_flags.no_next_hdr = 1;
+ break;
+
+ default:
+ prs->input_flags.l4 = 0;
+ break;
+ }
+
+ return prs->flags.all.error != 0;
+}
diff --git a/platform/linux-generic/odp_pool.c b/platform/linux-generic/odp_pool.c
index cca2d55f0..defdeb4fb 100644
--- a/platform/linux-generic/odp_pool.c
+++ b/platform/linux-generic/odp_pool.c
@@ -494,12 +494,7 @@ static void init_buffers(pool_t *pool)
for (i = 0; i < pool->num + skipped_blocks ; i++) {
int skip = 0;
-
- addr = &pool->base_addr[(i * pool->block_size) +
- pool->block_offset];
- event_hdr = addr;
- buf_hdr = addr;
- pkt_hdr = addr;
+ addr = &pool->base_addr[i * pool->block_size];
/* Skip packet buffers which cross huge page boundaries. Some
* NICs cannot handle buffers which cross page boundaries. */
@@ -517,6 +512,12 @@ static void init_buffers(pool_t *pool)
skip = 1;
}
}
+
+ addr = (uint8_t *)addr + pool->block_offset;
+ event_hdr = addr;
+ buf_hdr = addr;
+ pkt_hdr = addr;
+
if (pool->uarea_size)
uarea = &pool->uarea_base_addr[(i - skipped_blocks) *
pool->uarea_size];
diff --git a/platform/linux-generic/odp_queue_lf.c b/platform/linux-generic/odp_queue_lf.c
index f5c3830e8..58959df31 100644
--- a/platform/linux-generic/odp_queue_lf.c
+++ b/platform/linux-generic/odp_queue_lf.c
@@ -13,6 +13,7 @@
#include <odp_debug_internal.h>
#include <odp_event_internal.h>
#include <odp_queue_basic_internal.h>
+#include <odp_types_internal.h>
#include <string.h>
#include <stdio.h>
@@ -24,9 +25,7 @@
#ifdef __SIZEOF_INT128__
-__extension__ typedef unsigned __int128 u128_t;
-
-static inline void lockfree_zero_u128(u128_t *atomic)
+static inline void lockfree_zero_u128(_odp_u128_t *atomic)
{
__atomic_store_n(atomic, 0, __ATOMIC_RELAXED);
}
@@ -40,21 +39,21 @@ static inline void lockfree_zero_u128(u128_t *atomic)
* So, these are never actually used. */
typedef struct ODP_ALIGNED(16) {
uint64_t u64[2];
-} u128_t;
+} _odp_u128_t;
-static inline u128_t lockfree_load_u128(u128_t *atomic)
+static inline _odp_u128_t lockfree_load_u128(_odp_u128_t *atomic)
{
return *atomic;
}
-static inline void lockfree_zero_u128(u128_t *atomic)
+static inline void lockfree_zero_u128(_odp_u128_t *atomic)
{
atomic->u64[0] = 0;
atomic->u64[1] = 0;
}
-static inline int lockfree_cas_acq_rel_u128(u128_t *atomic, u128_t old_val,
- u128_t new_val)
+static inline int lockfree_cas_acq_rel_u128(_odp_u128_t *atomic, _odp_u128_t old_val,
+ _odp_u128_t new_val)
{
if (atomic->u64[0] == old_val.u64[0] &&
atomic->u64[1] == old_val.u64[1]) {
@@ -75,7 +74,7 @@ static inline int lockfree_check_u128(void)
/* Node in lock-free ring */
typedef union {
- u128_t u128;
+ _odp_u128_t u128;
struct {
/* Data with lowest counter value is the head. Empty node has
diff --git a/platform/linux-generic/odp_schedule_scalable.c b/platform/linux-generic/odp_schedule_scalable.c
index 5343c2834..7482d776d 100644
--- a/platform/linux-generic/odp_schedule_scalable.c
+++ b/platform/linux-generic/odp_schedule_scalable.c
@@ -817,23 +817,19 @@ events_dequeued:
}
/* Ingress queue empty => poll pktio RX queue */
- odp_event_t rx_evts[QUEUE_MULTI_MAX];
- int num_rx = _odp_sched_cb_pktin_poll_one(elem->pktio_idx,
- elem->rx_queue,
- rx_evts);
+ _odp_event_hdr_t *rx_evts[QUEUE_MULTI_MAX];
+ int num_rx = _odp_sched_cb_pktin_poll(elem->pktio_idx, elem->rx_queue,
+ rx_evts, QUEUE_MULTI_MAX);
+
if (odp_likely(num_rx > 0)) {
num = num_rx < num_evts ? num_rx : num_evts;
for (i = 0; i < num; i++) {
/* Return events directly to caller */
- ev[i] = rx_evts[i];
+ ev[i] = _odp_event_from_hdr(rx_evts[i]);
}
if (num_rx > num) {
/* Events remain, enqueue them */
- _odp_event_hdr_t *events[QUEUE_MULTI_MAX];
-
- for (i = num; i < num_rx; i++)
- events[i] = _odp_event_hdr(rx_evts[i]);
- i = _odp_queue_enq_sp(elem, &events[num], num_rx - num);
+ i = _odp_queue_enq_sp(elem, &rx_evts[num], num_rx - num);
/* Enqueue must succeed as the queue was empty */
ODP_ASSERT(i == num_rx - num);
}
diff --git a/platform/linux-generic/odp_schedule_sp.c b/platform/linux-generic/odp_schedule_sp.c
index f5aba1a88..558c3f4fe 100644
--- a/platform/linux-generic/odp_schedule_sp.c
+++ b/platform/linux-generic/odp_schedule_sp.c
@@ -13,6 +13,7 @@
#pragma GCC diagnostic ignored "-Wzero-length-bounds"
#endif
+#include <odp/api/packet.h>
#include <odp/api/ticketlock.h>
#include <odp/api/thread.h>
#include <odp/api/plat/thread_inlines.h>
diff --git a/platform/linux-generic/odp_time.c b/platform/linux-generic/odp_time.c
index d908f02cb..b591d712a 100644
--- a/platform/linux-generic/odp_time.c
+++ b/platform/linux-generic/odp_time.c
@@ -145,15 +145,6 @@ static inline void time_wait_until(odp_time_t time)
} while (odp_time_cmp(time, cur) > 0);
}
-uint64_t odp_time_diff_ns(odp_time_t t2, odp_time_t t1)
-{
- odp_time_t time;
-
- time.u64 = t2.u64 - t1.u64;
-
- return odp_time_to_ns(time);
-}
-
odp_time_t odp_time_local_from_ns(uint64_t ns)
{
return time_from_ns(ns);
diff --git a/platform/linux-generic/odp_timer.c b/platform/linux-generic/odp_timer.c
index d1d4a5bc8..1c54ab740 100644
--- a/platform/linux-generic/odp_timer.c
+++ b/platform/linux-generic/odp_timer.c
@@ -13,49 +13,52 @@
*/
#include <odp_posix_extensions.h>
-#include <errno.h>
-#include <stdlib.h>
-#include <time.h>
-#include <signal.h>
-#include <pthread.h>
-#include <unistd.h>
-#include <sys/syscall.h>
-#include <inttypes.h>
-#include <string.h>
-
-#include <odp/api/std.h>
#include <odp/api/align.h>
-#include <odp_align_internal.h>
#include <odp/api/atomic.h>
-#include <odp/api/plat/atomic_inlines.h>
-#include <odp_atomic_internal.h>
-#include <odp/api/buffer.h>
#include <odp/api/cpu.h>
-#include <odp/api/pool.h>
-#include <odp_pool_internal.h>
#include <odp/api/debug.h>
-#include <odp_debug_internal.h>
#include <odp/api/event.h>
#include <odp/api/hints.h>
-#include <odp_init_internal.h>
-#include <odp_errno_define.h>
+#include <odp/api/pool.h>
#include <odp/api/queue.h>
#include <odp/api/shared_memory.h>
#include <odp/api/spinlock.h>
-#include <odp/api/std_types.h>
+#include <odp/api/std.h>
#include <odp/api/sync.h>
#include <odp/api/time.h>
-#include <odp/api/plat/time_inlines.h>
#include <odp/api/timer.h>
+
+/* Inlined API functions */
+#include <odp/api/plat/atomic_inlines.h>
+#include <odp/api/plat/event_inlines.h>
+#include <odp/api/plat/queue_inlines.h>
+#include <odp/api/plat/time_inlines.h>
+#include <odp/api/plat/timer_inlines.h>
+
+#include <odp/api/plat/timer_inline_types.h>
+
+#include <odp_align_internal.h>
+#include <odp_atomic_internal.h>
+#include <odp_debug_internal.h>
+#include <odp_errno_define.h>
+#include <odp_event_internal.h>
+#include <odp_global_data.h>
+#include <odp_init_internal.h>
#include <odp_libconfig_internal.h>
+#include <odp_pool_internal.h>
#include <odp_queue_if.h>
#include <odp_timer_internal.h>
-#include <odp/api/plat/queue_inlines.h>
-#include <odp_global_data.h>
-#include <odp_event_internal.h>
+#include <odp_types_internal.h>
-/* Inlined API functions */
-#include <odp/api/plat/event_inlines.h>
+#include <errno.h>
+#include <inttypes.h>
+#include <pthread.h>
+#include <signal.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/syscall.h>
+#include <time.h>
+#include <unistd.h>
/* One divided by one nanosecond in Hz */
#define GIGA_HZ 1000000000
@@ -94,6 +97,18 @@
#define IDX2LOCK(idx) (&timer_global->locks[(idx) % NUM_LOCKS])
#endif
+#include <odp/visibility_begin.h>
+
+/* Fill in timeout header field offsets for inline functions */
+const _odp_timeout_inline_offset_t
+_odp_timeout_inline_offset ODP_ALIGNED_CACHE = {
+ .expiration = offsetof(odp_timeout_hdr_t, expiration),
+ .timer = offsetof(odp_timeout_hdr_t, timer),
+ .user_ptr = offsetof(odp_timeout_hdr_t, user_ptr)
+};
+
+#include <odp/visibility_end.h>
+
typedef struct
#ifdef ODP_ATOMIC_U128
ODP_ALIGNED(16) /* 16-byte atomic operations need properly aligned addresses */
@@ -689,7 +704,7 @@ static bool timer_reset(uint32_t idx, uint64_t abs_tck, odp_event_t *tmo_event,
/* Atomic CAS will fail if we experienced torn reads,
* retry update sequence until CAS succeeds */
} while (!_odp_atomic_u128_cmp_xchg_mm((_odp_atomic_u128_t *)tb,
- (_uint128_t *)&old, (_uint128_t *)&new,
+ (_odp_u128_t *)&old, (_odp_u128_t *)&new,
_ODP_MEMMODEL_RLS, _ODP_MEMMODEL_RLX));
#elif __GCC_ATOMIC_LLONG_LOCK_FREE >= 2 && \
defined __GCC_HAVE_SYNC_COMPARE_AND_SWAP_8
@@ -760,8 +775,8 @@ static bool timer_reset(uint32_t idx, uint64_t abs_tck, odp_event_t *tmo_event,
/* We are releasing the new timeout event to some other
* thread */
_odp_atomic_u128_xchg_mm((_odp_atomic_u128_t *)tb,
- (_uint128_t *)&new,
- (_uint128_t *)&old,
+ (_odp_u128_t *)&new,
+ (_odp_u128_t *)&old,
_ODP_MEMMODEL_ACQ_RLS);
old_event = old.tmo_event;
#else
@@ -804,7 +819,7 @@ static odp_event_t timer_set_unused(timer_pool_t *tp, uint32_t idx)
new.tmo_event = ODP_EVENT_INVALID;
_odp_atomic_u128_xchg_mm((_odp_atomic_u128_t *)tb,
- (_uint128_t *)&new, (_uint128_t *)&old,
+ (_odp_u128_t *)&new, (_odp_u128_t *)&old,
_ODP_MEMMODEL_RLX);
old_event = old.tmo_event;
#else
@@ -858,8 +873,8 @@ static odp_event_t timer_cancel(timer_pool_t *tp, uint32_t idx)
/* Atomic CAS will fail if we experienced torn reads,
* retry update sequence until CAS succeeds */
} while (!_odp_atomic_u128_cmp_xchg_mm((_odp_atomic_u128_t *)tb,
- (_uint128_t *)&old,
- (_uint128_t *)&new,
+ (_odp_u128_t *)&old,
+ (_odp_u128_t *)&new,
_ODP_MEMMODEL_RLS,
_ODP_MEMMODEL_RLX));
old_event = old.tmo_event;
@@ -916,7 +931,7 @@ static inline void timer_expire(timer_pool_t *tp, uint32_t idx, uint64_t tick)
new.tmo_event = ODP_EVENT_INVALID;
int succ = _odp_atomic_u128_cmp_xchg_mm((_odp_atomic_u128_t *)tb,
- (_uint128_t *)&old, (_uint128_t *)&new,
+ (_odp_u128_t *)&old, (_odp_u128_t *)&new,
_ODP_MEMMODEL_RLS, _ODP_MEMMODEL_RLX);
if (succ)
tmo_event = old.tmo_event;
@@ -1468,22 +1483,6 @@ void odp_timer_pool_destroy(odp_timer_pool_t tpid)
odp_timer_pool_del(timer_pool_from_hdl(tpid));
}
-uint64_t odp_timer_tick_to_ns(odp_timer_pool_t tpid, uint64_t ticks)
-{
- (void)tpid;
-
- /* Timer ticks in API are nsec */
- return ticks;
-}
-
-uint64_t odp_timer_ns_to_tick(odp_timer_pool_t tpid, uint64_t ns)
-{
- (void)tpid;
-
- /* Timer ticks in API are nsec */
- return ns;
-}
-
uint64_t odp_timer_current_tick(odp_timer_pool_t tpid)
{
timer_pool_t *tp = timer_pool_from_hdl(tpid);
@@ -1812,19 +1811,6 @@ uint64_t odp_timer_to_u64(odp_timer_t hdl)
return _odp_pri(hdl);
}
-odp_timeout_t odp_timeout_from_event(odp_event_t ev)
-{
- /* This check not mandated by the API specification */
- if (odp_event_type(ev) != ODP_EVENT_TIMEOUT)
- ODP_ABORT("Event not a timeout");
- return (odp_timeout_t)ev;
-}
-
-odp_event_t odp_timeout_to_event(odp_timeout_t tmo)
-{
- return (odp_event_t)tmo;
-}
-
uint64_t odp_timeout_to_u64(odp_timeout_t tmo)
{
return _odp_pri(tmo);
@@ -1847,21 +1833,6 @@ int odp_timeout_fresh(odp_timeout_t tmo)
return hdr->expiration == (exp_tck & ~TMO_INACTIVE);
}
-odp_timer_t odp_timeout_timer(odp_timeout_t tmo)
-{
- return timeout_hdr(tmo)->timer;
-}
-
-uint64_t odp_timeout_tick(odp_timeout_t tmo)
-{
- return timeout_hdr(tmo)->expiration;
-}
-
-void *odp_timeout_user_ptr(odp_timeout_t tmo)
-{
- return (void *)(uintptr_t)timeout_hdr(tmo)->user_ptr;
-}
-
odp_timeout_t odp_timeout_alloc(odp_pool_t pool_hdl)
{
odp_event_t event;
diff --git a/platform/linux-generic/odp_timer_api.c b/platform/linux-generic/odp_timer_api.c
new file mode 100644
index 000000000..cd657956b
--- /dev/null
+++ b/platform/linux-generic/odp_timer_api.c
@@ -0,0 +1,11 @@
+/* Copyright (c) 2022, Nokia
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+#include <odp/api/timer.h>
+
+/* Non-inlined functions for ABI compat mode */
+#define _ODP_NO_INLINE
+#include <odp/api/plat/timer_inlines.h>
diff --git a/platform/linux-generic/odp_traffic_mngr.c b/platform/linux-generic/odp_traffic_mngr.c
index fbf33e117..f02297ec9 100644
--- a/platform/linux-generic/odp_traffic_mngr.c
+++ b/platform/linux-generic/odp_traffic_mngr.c
@@ -55,45 +55,60 @@ _odp_int_name_kind_t PROFILE_TO_HANDLE_KIND[ODP_TM_NUM_PROFILES] = {
static const pkt_desc_t EMPTY_PKT_DESC = { .word = 0 };
#define MAX_PRIORITIES ODP_TM_MAX_PRIORITIES
-#define NUM_SHAPER_COLORS ODP_NUM_SHAPER_COLORS
/* Shaper BW limits in bits/sec */
#define TM_MIN_SHAPER_BW 8000ULL
#define TM_MAX_SHAPER_BW (100ULL * 1000ULL * 1000ULL * 1000ULL)
+/* Possible values for running the shaper algorithm. TM_SHAPER_GREEN means that
+ * the traffic is within the commit specification (rate and burst size),
+ * TM_SHAPER_YELLOW means that the traffic is within the peak specification
+ * (rate and burst size) and TM_SHAPER_RED means that the traffic is exceeding
+ * both its commit and peak specifications. Note that packets can also have an
+ * assigned packet color of ODP_PACKET_GREEN, ODP_PACKET_YELLOW or
+ * ODP_PACKET_RED, which has a different meaning and purpose than the shaper
+ * colors.
+ */
+typedef enum {
+ TM_SHAPER_GREEN, TM_SHAPER_YELLOW, TM_SHAPER_RED
+} tm_shaper_color_t;
+
+/* Number of enumeration values defined in tm_shaper_color_t type. */
+#define NUM_SHAPER_COLORS 3
+
static const tm_prop_t basic_prop_tbl[MAX_PRIORITIES][NUM_SHAPER_COLORS] = {
[0] = {
- [ODP_TM_SHAPER_GREEN] = { 0, DECR_BOTH },
- [ODP_TM_SHAPER_YELLOW] = { 0, DECR_BOTH },
- [ODP_TM_SHAPER_RED] = { 0, DELAY_PKT } },
+ [TM_SHAPER_GREEN] = { 0, DECR_BOTH },
+ [TM_SHAPER_YELLOW] = { 0, DECR_BOTH },
+ [TM_SHAPER_RED] = { 0, DELAY_PKT } },
[1] = {
- [ODP_TM_SHAPER_GREEN] = { 1, DECR_BOTH },
- [ODP_TM_SHAPER_YELLOW] = { 1, DECR_BOTH },
- [ODP_TM_SHAPER_RED] = { 1, DELAY_PKT } },
+ [TM_SHAPER_GREEN] = { 1, DECR_BOTH },
+ [TM_SHAPER_YELLOW] = { 1, DECR_BOTH },
+ [TM_SHAPER_RED] = { 1, DELAY_PKT } },
[2] = {
- [ODP_TM_SHAPER_GREEN] = { 2, DECR_BOTH },
- [ODP_TM_SHAPER_YELLOW] = { 2, DECR_BOTH },
- [ODP_TM_SHAPER_RED] = { 2, DELAY_PKT } },
+ [TM_SHAPER_GREEN] = { 2, DECR_BOTH },
+ [TM_SHAPER_YELLOW] = { 2, DECR_BOTH },
+ [TM_SHAPER_RED] = { 2, DELAY_PKT } },
[3] = {
- [ODP_TM_SHAPER_GREEN] = { 3, DECR_BOTH },
- [ODP_TM_SHAPER_YELLOW] = { 3, DECR_BOTH },
- [ODP_TM_SHAPER_RED] = { 3, DELAY_PKT } },
+ [TM_SHAPER_GREEN] = { 3, DECR_BOTH },
+ [TM_SHAPER_YELLOW] = { 3, DECR_BOTH },
+ [TM_SHAPER_RED] = { 3, DELAY_PKT } },
[4] = {
- [ODP_TM_SHAPER_GREEN] = { 4, DECR_BOTH },
- [ODP_TM_SHAPER_YELLOW] = { 4, DECR_BOTH },
- [ODP_TM_SHAPER_RED] = { 4, DELAY_PKT } },
+ [TM_SHAPER_GREEN] = { 4, DECR_BOTH },
+ [TM_SHAPER_YELLOW] = { 4, DECR_BOTH },
+ [TM_SHAPER_RED] = { 4, DELAY_PKT } },
[5] = {
- [ODP_TM_SHAPER_GREEN] = { 5, DECR_BOTH },
- [ODP_TM_SHAPER_YELLOW] = { 5, DECR_BOTH },
- [ODP_TM_SHAPER_RED] = { 5, DELAY_PKT } },
+ [TM_SHAPER_GREEN] = { 5, DECR_BOTH },
+ [TM_SHAPER_YELLOW] = { 5, DECR_BOTH },
+ [TM_SHAPER_RED] = { 5, DELAY_PKT } },
[6] = {
- [ODP_TM_SHAPER_GREEN] = { 6, DECR_BOTH },
- [ODP_TM_SHAPER_YELLOW] = { 6, DECR_BOTH },
- [ODP_TM_SHAPER_RED] = { 6, DELAY_PKT } },
+ [TM_SHAPER_GREEN] = { 6, DECR_BOTH },
+ [TM_SHAPER_YELLOW] = { 6, DECR_BOTH },
+ [TM_SHAPER_RED] = { 6, DELAY_PKT } },
[7] = {
- [ODP_TM_SHAPER_GREEN] = { 7, DECR_BOTH },
- [ODP_TM_SHAPER_YELLOW] = { 7, DECR_BOTH },
- [ODP_TM_SHAPER_RED] = { 7, DELAY_PKT } }
+ [TM_SHAPER_GREEN] = { 7, DECR_BOTH },
+ [TM_SHAPER_YELLOW] = { 7, DECR_BOTH },
+ [TM_SHAPER_RED] = { 7, DELAY_PKT } }
};
#define MAX_SHAPER_PROFILES 128
@@ -1076,30 +1091,30 @@ static odp_bool_t run_shaper(tm_system_t *tm_system,
pkt_desc_t *pkt_desc,
uint8_t priority)
{
- odp_tm_shaper_color_t shaper_color;
+ tm_shaper_color_t shaper_color;
tm_shaper_params_t *shaper_params;
odp_bool_t output_change;
tm_prop_t propagation;
shaper_params = shaper_obj->shaper_params;
- shaper_color = ODP_TM_SHAPER_GREEN;
+ shaper_color = TM_SHAPER_GREEN;
if (shaper_params) {
update_shaper_elapsed_time(tm_system, shaper_params,
shaper_obj);
if (shaper_params->enabled) {
if (0 < shaper_obj->commit_cnt)
- shaper_color = ODP_TM_SHAPER_GREEN;
+ shaper_color = TM_SHAPER_GREEN;
else if (!shaper_params->dual_rate)
- shaper_color = ODP_TM_SHAPER_RED;
+ shaper_color = TM_SHAPER_RED;
else if (shaper_obj->peak_cnt <= 0)
- shaper_color = ODP_TM_SHAPER_RED;
+ shaper_color = TM_SHAPER_RED;
else
- shaper_color = ODP_TM_SHAPER_YELLOW;
+ shaper_color = TM_SHAPER_YELLOW;
- if (shaper_color == ODP_TM_SHAPER_GREEN)
+ if (shaper_color == TM_SHAPER_GREEN)
tm_system->shaper_green_cnt++;
- else if (shaper_color == ODP_TM_SHAPER_YELLOW)
+ else if (shaper_color == TM_SHAPER_YELLOW)
tm_system->shaper_yellow_cnt++;
else
tm_system->shaper_red_cnt++;
@@ -1994,6 +2009,12 @@ static void tm_queue_cnts_decrement(tm_system_t *tm_system,
odp_atomic_sub_u64(&queue_cnts->byte_cnt, frame_len);
}
+static inline void activate_packet_aging(odp_packet_hdr_t *pkt_hdr)
+{
+ if (odp_unlikely(pkt_hdr->p.flags.tx_aging))
+ pkt_hdr->tx_aging_ns = pkt_hdr->tx_aging_ns + odp_time_global_ns();
+}
+
static int tm_enqueue(tm_system_t *tm_system,
tm_queue_obj_t *tm_queue_obj,
odp_packet_t pkt)
@@ -2028,6 +2049,7 @@ static int tm_enqueue(tm_system_t *tm_system,
if (tm_queue_obj->ordered_enqueue)
_odp_sched_fn->order_lock();
+ activate_packet_aging(packet_hdr(pkt));
rc = input_work_queue_append(tm_system, &work_item);
if (tm_queue_obj->ordered_enqueue)
@@ -2224,6 +2246,11 @@ static void tm_egress_marking(tm_system_t *tm_system, odp_packet_t odp_pkt)
}
}
+static inline odp_bool_t is_packet_aged(odp_packet_hdr_t *pkt_hdr)
+{
+ return pkt_hdr->p.flags.tx_aging && pkt_hdr->tx_aging_ns < odp_time_global_ns();
+}
+
static void tm_send_pkt(tm_system_t *tm_system, uint32_t max_sends)
{
tm_queue_obj_t *tm_queue_obj;
@@ -2250,9 +2277,13 @@ static void tm_send_pkt(tm_system_t *tm_system, uint32_t max_sends)
tm_system->egress_pkt_desc = EMPTY_PKT_DESC;
if (tm_system->egress.egress_kind == ODP_TM_EGRESS_PKT_IO) {
- ret = odp_pktout_send(tm_system->pktout, &odp_pkt, 1);
+ pktio_entry = get_pktio_entry(tm_system->pktout.pktio);
+ if (odp_unlikely(_odp_pktio_tx_aging_enabled(pktio_entry) &&
+ is_packet_aged(packet_hdr(odp_pkt))))
+ ret = 0; /* Aged packet handled as a discard */
+ else
+ ret = odp_pktout_send(tm_system->pktout, &odp_pkt, 1);
if (odp_unlikely(ret != 1)) {
- pktio_entry = get_pktio_entry(tm_system->pktout.pktio);
if (odp_unlikely(_odp_pktio_tx_compl_enabled(pktio_entry)))
_odp_pktio_allocate_and_send_tx_compl_events(pktio_entry,
&odp_pkt, 1);
@@ -2581,7 +2612,11 @@ static int tm_capabilities(odp_tm_capabilities_t capabilities[],
cap_ptr->vlan_marking_supported = true;
cap_ptr->ecn_marking_supported = true;
cap_ptr->drop_prec_marking_supported = true;
- cap_ptr->tm_queue_threshold = true;
+
+ cap_ptr->tm_queue_threshold.byte = true;
+ cap_ptr->tm_queue_threshold.packet = true;
+ cap_ptr->tm_queue_threshold.byte_and_packet = true;
+
cap_ptr->tm_queue_query_flags = (ODP_TM_QUERY_PKT_CNT |
ODP_TM_QUERY_BYTE_CNT |
ODP_TM_QUERY_THRESHOLDS);
@@ -2617,7 +2652,10 @@ static int tm_capabilities(odp_tm_capabilities_t capabilities[],
per_level_cap->tm_node_dual_slope_supported = true;
per_level_cap->fair_queuing_supported = true;
per_level_cap->weights_supported = true;
- per_level_cap->tm_node_threshold = true;
+
+ per_level_cap->tm_node_threshold.byte = true;
+ per_level_cap->tm_node_threshold.packet = true;
+ per_level_cap->tm_node_threshold.byte_and_packet = true;
}
cap_ptr->queue_stats.counter.discards = 1;
@@ -2691,7 +2729,11 @@ static void tm_system_capabilities_set(odp_tm_capabilities_t *cap_ptr,
cap_ptr->ecn_marking_supported = req_ptr->ecn_marking_needed;
cap_ptr->drop_prec_marking_supported =
req_ptr->drop_prec_marking_needed;
- cap_ptr->tm_queue_threshold = threshold;
+
+ cap_ptr->tm_queue_threshold.byte = threshold;
+ cap_ptr->tm_queue_threshold.packet = threshold;
+ cap_ptr->tm_queue_threshold.byte_and_packet = threshold;
+
cap_ptr->tm_queue_query_flags = (ODP_TM_QUERY_PKT_CNT |
ODP_TM_QUERY_BYTE_CNT |
ODP_TM_QUERY_THRESHOLDS);
@@ -2743,7 +2785,10 @@ static void tm_system_capabilities_set(odp_tm_capabilities_t *cap_ptr,
per_level_cap->tm_node_dual_slope_supported = dual_slope;
per_level_cap->fair_queuing_supported = true;
per_level_cap->weights_supported = true;
- per_level_cap->tm_node_threshold = threshold;
+
+ per_level_cap->tm_node_threshold.byte = threshold;
+ per_level_cap->tm_node_threshold.packet = threshold;
+ per_level_cap->tm_node_threshold.byte_and_packet = threshold;
}
cap_ptr->queue_stats.counter.discards = 1;
diff --git a/platform/linux-generic/pktio/dpdk.c b/platform/linux-generic/pktio/dpdk.c
index 815ec61b9..030560b0d 100644
--- a/platform/linux-generic/pktio/dpdk.c
+++ b/platform/linux-generic/pktio/dpdk.c
@@ -57,12 +57,14 @@
#include <rte_string_fns.h>
#include <rte_version.h>
-#if RTE_VERSION < RTE_VERSION_NUM(19, 8, 0, 0)
- #define rte_ether_addr ether_addr
- #define rte_ipv4_hdr ipv4_hdr
- #define rte_ipv6_hdr ipv6_hdr
- #define rte_tcp_hdr tcp_hdr
- #define rte_udp_hdr udp_hdr
+#if RTE_VERSION < RTE_VERSION_NUM(21, 11, 0, 0)
+ #define RTE_MBUF_F_RX_RSS_HASH PKT_RX_RSS_HASH
+ #define RTE_MBUF_F_TX_IPV4 PKT_TX_IPV4
+ #define RTE_MBUF_F_TX_IPV6 PKT_TX_IPV6
+ #define RTE_MBUF_F_TX_IP_CKSUM PKT_TX_IP_CKSUM
+ #define RTE_MBUF_F_TX_UDP_CKSUM PKT_TX_UDP_CKSUM
+ #define RTE_MBUF_F_TX_TCP_CKSUM PKT_TX_TCP_CKSUM
+ #define RTE_MEMPOOL_REGISTER_OPS MEMPOOL_REGISTER_OPS
#endif
/* NUMA is not supported on all platforms */
@@ -72,17 +74,7 @@
#define numa_num_configured_nodes() 1
#endif
-#if RTE_VERSION < RTE_VERSION_NUM(17, 5, 0, 0)
-#define rte_log_set_global_level rte_set_log_level
-#endif
-
-/* Release notes v19.11: "Changed the mempool allocation behaviour
- * so that objects no longer cross pages by default" */
-#if RTE_VERSION >= RTE_VERSION_NUM(19, 11, 0, 0)
-#define MEMPOOL_FLAGS MEMPOOL_F_NO_IOVA_CONTIG
-#else
#define MEMPOOL_FLAGS 0
-#endif
#if _ODP_DPDK_ZERO_COPY
ODP_STATIC_ASSERT(CONFIG_PACKET_HEADROOM == RTE_PKTMBUF_HEADROOM,
@@ -107,11 +99,7 @@ ODP_STATIC_ASSERT((DPDK_NB_MBUF % DPDK_MEMPOOL_CACHE_SIZE == 0) &&
#define DPDK_MIN_RX_BURST 4
/* Limits for setting link MTU */
-#if RTE_VERSION >= RTE_VERSION_NUM(19, 11, 0, 0)
#define DPDK_MTU_MIN (RTE_ETHER_MIN_MTU + _ODP_ETHHDR_LEN)
-#else
-#define DPDK_MTU_MIN (68 + _ODP_ETHHDR_LEN)
-#endif
#define DPDK_MTU_MAX (9000 + _ODP_ETHHDR_LEN)
/** DPDK runtime configuration options */
@@ -334,12 +322,16 @@ static void pktmbuf_init(struct rte_mempool *mp, void *opaque_arg ODP_UNUSED,
static struct rte_mempool *mbuf_pool_create(const char *name,
pool_t *pool_entry)
{
+ odp_shm_info_t shm_info;
struct rte_mempool *mp = NULL;
struct rte_pktmbuf_pool_private mbp_priv;
struct rte_mempool_objsz sz;
unsigned int elt_size = pool_entry->dpdk_elt_size;
- unsigned int num = pool_entry->num;
+ unsigned int num = pool_entry->num, populated = 0;
uint32_t total_size;
+ uint64_t page_size, offset = 0, remainder = 0;
+ uint8_t *addr;
+ int ret;
if (!(pool_entry->mem_from_huge_pages)) {
ODP_ERR("DPDK requires memory is allocated from huge pages\n");
@@ -348,11 +340,17 @@ static struct rte_mempool *mbuf_pool_create(const char *name,
if (pool_entry->seg_len < RTE_MBUF_DEFAULT_BUF_SIZE) {
ODP_ERR("Some NICs need at least %dB buffers to not segment "
- "standard ethernet frames. Increase pool seg_len.\n",
- RTE_MBUF_DEFAULT_BUF_SIZE);
+ "standard ethernet frames. Increase pool seg_len.\n",
+ RTE_MBUF_DEFAULT_BUF_SIZE);
goto fail;
}
+ if (odp_shm_info(pool_entry->shm, &shm_info)) {
+ ODP_ERR("Failed to query SHM info.\n");
+ goto fail;
+ }
+
+ page_size = shm_info.page_size;
total_size = rte_mempool_calc_obj_size(elt_size, MEMPOOL_FLAGS, &sz);
if (total_size != pool_entry->block_size) {
ODP_ERR("DPDK pool block size not matching to ODP pool: "
@@ -361,10 +359,7 @@ static struct rte_mempool *mbuf_pool_create(const char *name,
goto fail;
}
- /* Skipped buffers have to be taken into account to populate pool
- * properly. */
- mp = rte_mempool_create_empty(name, num + pool_entry->skipped_blocks,
- elt_size, cache_size(num),
+ mp = rte_mempool_create_empty(name, num, elt_size, cache_size(num),
sizeof(struct rte_pktmbuf_pool_private),
rte_socket_id(), MEMPOOL_FLAGS);
if (mp == NULL) {
@@ -380,16 +375,34 @@ static struct rte_mempool *mbuf_pool_create(const char *name,
}
mbp_priv.mbuf_data_room_size = pool_entry->headroom +
- pool_entry->seg_len;
+ pool_entry->seg_len + pool_entry->tailroom;
mbp_priv.mbuf_priv_size = RTE_ALIGN(sizeof(odp_packet_hdr_t),
RTE_MBUF_PRIV_ALIGN);
rte_pktmbuf_pool_init(mp, &mbp_priv);
- num = rte_mempool_populate_iova(mp, (char *)pool_entry->base_addr,
- RTE_BAD_IOVA, pool_entry->shm_size,
- NULL, NULL);
- if (num <= 0) {
- ODP_ERR("Failed to populate mempool: %d\n", num);
+ /* DPDK expects buffers that would be crossing a hugepage boundary to be aligned to the
+ * boundary. This isn't the case with ODP pools as boundary-crossing buffers are skipped
+ * and unused but still part of the pool. Thus, populate the mempool with several virtually
+ * and physically contiguous chunks as dictated by the skipped buffers. */
+ for (uint64_t i = 0; i < pool_entry->shm_size; i += page_size) {
+ remainder = (page_size - offset) % total_size;
+ addr = pool_entry->base_addr + i + offset;
+ ret = rte_mempool_populate_iova(mp, (char *)addr, rte_mem_virt2iova(addr),
+ page_size - remainder - offset,
+ NULL, NULL);
+
+ if (ret <= 0) {
+ ODP_ERR("Failed to populate mempool: %d\n", ret);
+ goto fail;
+ }
+
+ populated += ret;
+ offset = remainder ? total_size - remainder : 0;
+ }
+
+ if (populated != num) {
+ ODP_ERR("Failed to populate mempool with all requested blocks, populated: %u, "
+ "requested: %u\n", populated, num);
goto fail;
}
@@ -556,7 +569,7 @@ static struct rte_mempool_ops odp_pool_ops = {
.get_count = pool_get_count
};
-MEMPOOL_REGISTER_OPS(odp_pool_ops)
+RTE_MEMPOOL_REGISTER_OPS(odp_pool_ops)
static inline int mbuf_to_pkt(pktio_entry_t *pktio_entry,
odp_packet_t pkt_table[],
@@ -574,9 +587,9 @@ static inline int mbuf_to_pkt(pktio_entry_t *pktio_entry,
pkt_dpdk_t *pkt_dpdk = pkt_priv(pktio_entry);
odp_pool_t pool = pkt_dpdk->pool;
odp_pktin_config_opt_t pktin_cfg = pktio_entry->s.config.pktin;
- odp_proto_layer_t parse_layer = pktio_entry->s.config.parser.layer;
odp_pktio_t input = pktio_entry->s.handle;
uint16_t frame_offset = pktio_entry->s.pktin_frame_offset;
+ const odp_proto_layer_t layer = pktio_entry->s.parse_layer;
/* Allocate maximum sized packets */
max_len = pkt_dpdk->data_room;
@@ -591,8 +604,6 @@ static inline int mbuf_to_pkt(pktio_entry_t *pktio_entry,
}
for (i = 0; i < num; i++) {
- odp_packet_hdr_t parsed_hdr;
-
mbuf = mbuf_table[i];
if (odp_unlikely(mbuf->nb_segs != 1)) {
ODP_ERR("Segmented buffers not supported\n");
@@ -603,31 +614,33 @@ static inline int mbuf_to_pkt(pktio_entry_t *pktio_entry,
odp_prefetch(data);
pkt_len = rte_pktmbuf_pkt_len(mbuf);
+ pkt = pkt_table[i];
+ pkt_hdr = packet_hdr(pkt);
- if (pktio_cls_enabled(pktio_entry)) {
+ if (layer) {
uint32_t supported_ptypes = pkt_dpdk->supported_ptypes;
- packet_parse_reset(&parsed_hdr, 1);
- packet_set_len(&parsed_hdr, pkt_len);
- if (_odp_dpdk_packet_parse_common(&parsed_hdr.p, data,
- pkt_len, pkt_len,
- mbuf,
- ODP_PROTO_LAYER_ALL,
- supported_ptypes,
+ packet_parse_reset(pkt_hdr, 1);
+ if (_odp_dpdk_packet_parse_common(&pkt_hdr->p, data,
+ pkt_len, pkt_len, mbuf,
+ layer, supported_ptypes,
pktin_cfg)) {
odp_packet_free(pkt_table[i]);
rte_pktmbuf_free(mbuf);
continue;
}
- if (_odp_cls_classify_packet(pktio_entry,
- (const uint8_t *)data,
- pkt_len, pkt_len, &pool,
- &parsed_hdr, false))
- goto fail;
+
+ if (pktio_cls_enabled(pktio_entry)) {
+ if (_odp_cls_classify_packet(pktio_entry,
+ (const uint8_t *)data,
+ &pool, pkt_hdr)) {
+ odp_packet_free(pkt_table[i]);
+ rte_pktmbuf_free(mbuf);
+ continue;
+ }
+ }
}
- pkt = pkt_table[i];
- pkt_hdr = packet_hdr(pkt);
pull_tail(pkt_hdr, max_len - pkt_len);
if (frame_offset)
pull_head(pkt_hdr, frame_offset);
@@ -637,22 +650,7 @@ static inline int mbuf_to_pkt(pktio_entry_t *pktio_entry,
pkt_hdr->input = input;
- if (pktio_cls_enabled(pktio_entry)) {
- _odp_packet_copy_cls_md(pkt_hdr, &parsed_hdr);
- } else if (parse_layer != ODP_PROTO_LAYER_NONE) {
- uint32_t supported_ptypes = pkt_dpdk->supported_ptypes;
-
- if (_odp_dpdk_packet_parse_layer(pkt_hdr, mbuf,
- parse_layer,
- supported_ptypes,
- pktin_cfg)) {
- odp_packet_free(pkt);
- rte_pktmbuf_free(mbuf);
- continue;
- }
- }
-
- if (mbuf->ol_flags & PKT_RX_RSS_HASH)
+ if (mbuf->ol_flags & RTE_MBUF_F_RX_RSS_HASH)
packet_set_flow_hash(pkt_hdr, mbuf->hash.rss);
packet_set_ts(pkt_hdr, ts);
@@ -753,12 +751,12 @@ static inline void pkt_set_ol_tx(odp_pktout_config_opt_t *pktout_cfg,
mbuf->l2_len = pkt_p->l3_offset - pkt_p->l2_offset;
if (l3_proto_v4)
- mbuf->ol_flags = PKT_TX_IPV4;
+ mbuf->ol_flags = RTE_MBUF_F_TX_IPV4;
else
- mbuf->ol_flags = PKT_TX_IPV6;
+ mbuf->ol_flags = RTE_MBUF_F_TX_IPV6;
if (ipv4_chksum_pkt) {
- mbuf->ol_flags |= PKT_TX_IP_CKSUM;
+ mbuf->ol_flags |= RTE_MBUF_F_TX_IP_CKSUM;
((struct rte_ipv4_hdr *)l3_hdr)->hdr_checksum = 0;
mbuf->l3_len = _ODP_IPV4HDR_IHL(*(uint8_t *)l3_hdr) * 4;
@@ -772,12 +770,12 @@ static inline void pkt_set_ol_tx(odp_pktout_config_opt_t *pktout_cfg,
l4_hdr = (void *)(mbuf_data + pkt_p->l4_offset);
if (udp_chksum_pkt) {
- mbuf->ol_flags |= PKT_TX_UDP_CKSUM;
+ mbuf->ol_flags |= RTE_MBUF_F_TX_UDP_CKSUM;
((struct rte_udp_hdr *)l4_hdr)->dgram_cksum =
phdr_csum(l3_proto_v4, l3_hdr, mbuf->ol_flags);
} else if (tcp_chksum_pkt) {
- mbuf->ol_flags |= PKT_TX_TCP_CKSUM;
+ mbuf->ol_flags |= RTE_MBUF_F_TX_TCP_CKSUM;
((struct rte_tcp_hdr *)l4_hdr)->cksum =
phdr_csum(l3_proto_v4, l3_hdr, mbuf->ol_flags);
@@ -863,9 +861,9 @@ static inline int mbuf_to_pkt_zero(pktio_entry_t *pktio_entry,
int i, nb_pkts;
odp_pool_t pool;
odp_pktin_config_opt_t pktin_cfg;
- odp_proto_layer_t parse_layer;
odp_pktio_t input;
pkt_dpdk_t *pkt_dpdk;
+ const odp_proto_layer_t layer = pktio_entry->s.parse_layer;
prefetch_pkt(mbuf_table[0]);
@@ -874,15 +872,12 @@ static inline int mbuf_to_pkt_zero(pktio_entry_t *pktio_entry,
pool = pkt_dpdk->pool;
set_flow_hash = pkt_dpdk->opt.set_flow_hash;
pktin_cfg = pktio_entry->s.config.pktin;
- parse_layer = pktio_entry->s.config.parser.layer;
input = pktio_entry->s.handle;
if (odp_likely(mbuf_num > 1))
prefetch_pkt(mbuf_table[1]);
for (i = 0; i < mbuf_num; i++) {
- odp_packet_hdr_t parsed_hdr;
-
if (odp_likely((i + 2) < mbuf_num))
prefetch_pkt(mbuf_table[i + 2]);
@@ -895,30 +890,27 @@ static inline int mbuf_to_pkt_zero(pktio_entry_t *pktio_entry,
data = rte_pktmbuf_mtod(mbuf, char *);
pkt_len = rte_pktmbuf_pkt_len(mbuf);
-
pkt_hdr = pkt_hdr_from_mbuf(mbuf);
+ packet_init(pkt_hdr, pkt_len);
- if (pktio_cls_enabled(pktio_entry)) {
+ if (layer) {
uint32_t supported_ptypes = pkt_dpdk->supported_ptypes;
- packet_parse_reset(&parsed_hdr, 1);
- packet_set_len(&parsed_hdr, pkt_len);
- if (_odp_dpdk_packet_parse_common(&parsed_hdr.p, data,
- pkt_len, pkt_len,
- mbuf,
- ODP_PROTO_LAYER_ALL,
- supported_ptypes,
+ if (_odp_dpdk_packet_parse_common(&pkt_hdr->p, data,
+ pkt_len, pkt_len, mbuf,
+ layer, supported_ptypes,
pktin_cfg)) {
rte_pktmbuf_free(mbuf);
continue;
}
- if (_odp_cls_classify_packet(pktio_entry,
- (const uint8_t *)data,
- pkt_len, pkt_len, &pool,
- &parsed_hdr, false)) {
- ODP_ERR("Unable to classify packet\n");
- rte_pktmbuf_free(mbuf);
- continue;
+
+ if (pktio_cls_enabled(pktio_entry)) {
+ if (_odp_cls_classify_packet(pktio_entry,
+ (const uint8_t *)data,
+ &pool, pkt_hdr)) {
+ rte_pktmbuf_free(mbuf);
+ continue;
+ }
}
}
@@ -926,23 +918,9 @@ static inline int mbuf_to_pkt_zero(pktio_entry_t *pktio_entry,
* are supported. */
pkt_hdr->seg_data = data;
- packet_init(pkt_hdr, pkt_len);
pkt_hdr->input = input;
- if (pktio_cls_enabled(pktio_entry)) {
- _odp_packet_copy_cls_md(pkt_hdr, &parsed_hdr);
- } else if (parse_layer != ODP_PROTO_LAYER_NONE) {
- uint32_t supported_ptypes = pkt_dpdk->supported_ptypes;
-
- if (_odp_dpdk_packet_parse_layer(pkt_hdr, mbuf,
- parse_layer,
- supported_ptypes,
- pktin_cfg)) {
- rte_pktmbuf_free(mbuf);
- continue;
- }
- }
- if (set_flow_hash && (mbuf->ol_flags & PKT_RX_RSS_HASH))
+ if (set_flow_hash && (mbuf->ol_flags & RTE_MBUF_F_RX_RSS_HASH))
packet_set_flow_hash(pkt_hdr, mbuf->hash.rss);
packet_set_ts(pkt_hdr, ts);
@@ -1227,11 +1205,6 @@ static int dpdk_close(pktio_entry_t *pktio_entry)
rte_pktmbuf_free(pkt_dpdk->rx_cache[i].s.pkt[idx++]);
}
-#if RTE_VERSION < RTE_VERSION_NUM(17, 8, 0, 0)
- if (pktio_entry->s.state != PKTIO_STATE_OPENED)
- rte_eth_dev_close(pkt_dpdk->port_id);
-#endif
-
return 0;
}
@@ -1381,18 +1354,6 @@ static void dpdk_mempool_free(struct rte_mempool *mp, void *arg ODP_UNUSED)
rte_mempool_free(mp);
}
-/* RTE_ETH_FOREACH_DEV was introduced in v17.8, but causes a build error in
- * v18.2 (only a warning, but our build system treats warnings as errors). */
-#if (RTE_VERSION >= RTE_VERSION_NUM(18, 2, 0, 0)) && \
- (RTE_VERSION < RTE_VERSION_NUM(18, 5, 0, 0))
- #define ETH_FOREACH_DEV(p) \
- for (p = rte_eth_find_next(0); \
- (unsigned int)p < (unsigned int)RTE_MAX_ETHPORTS; \
- p = rte_eth_find_next(p + 1))
-#elif RTE_VERSION >= RTE_VERSION_NUM(17, 8, 0, 0)
- #define ETH_FOREACH_DEV(p) RTE_ETH_FOREACH_DEV(p)
-#endif
-
static int dpdk_pktio_term(void)
{
uint16_t port_id;
@@ -1400,11 +1361,9 @@ static int dpdk_pktio_term(void)
if (!odp_global_rw->dpdk_initialized)
return 0;
-#if RTE_VERSION >= RTE_VERSION_NUM(17, 8, 0, 0)
- ETH_FOREACH_DEV(port_id) {
+ RTE_ETH_FOREACH_DEV(port_id) {
rte_eth_dev_close(port_id);
}
-#endif
if (!_ODP_DPDK_ZERO_COPY)
rte_mempool_walk(dpdk_mempool_free, NULL);
@@ -1574,16 +1533,18 @@ static int dpdk_init_capability(pktio_entry_t *pktio_entry,
/* Check if setting MTU is supported */
ret = rte_eth_dev_set_mtu(pkt_dpdk->port_id, pkt_dpdk->mtu - _ODP_ETHHDR_LEN);
- if (ret == 0) {
+ /* From DPDK 21.11 onwards, calling rte_eth_dev_set_mtu() before device is configured with
+ * rte_eth_dev_configure() will result in failure. The least hacky (unfortunately still
+ * very hacky) way to continue checking the support is to take into account that the
+ * function will fail earlier with -ENOTSUP if MTU setting is not supported by device than
+ * if the device was not yet configured. */
+ if (ret != -ENOTSUP) {
capa->set_op.op.maxlen = 1;
capa->maxlen.equal = true;
capa->maxlen.min_input = DPDK_MTU_MIN;
capa->maxlen.max_input = pkt_dpdk->mtu_max;
capa->maxlen.min_output = DPDK_MTU_MIN;
capa->maxlen.max_output = pkt_dpdk->mtu_max;
- } else if (ret != -ENOTSUP) {
- ODP_ERR("Failed to set interface MTU: %d\n", ret);
- return -1;
}
ptype_cnt = rte_eth_dev_get_supported_ptypes(pkt_dpdk->port_id,
@@ -1669,15 +1630,6 @@ static int dpdk_init_capability(pktio_entry_t *pktio_entry,
* mode change. Use system call for them. */
static void promisc_mode_check(pkt_dpdk_t *pkt_dpdk)
{
-#if RTE_VERSION < RTE_VERSION_NUM(19, 11, 0, 0)
- /* Enable and disable calls do not have return value */
- rte_eth_promiscuous_enable(pkt_dpdk->port_id);
-
- if (!rte_eth_promiscuous_get(pkt_dpdk->port_id))
- pkt_dpdk->vdev_sysc_promisc = 1;
-
- rte_eth_promiscuous_disable(pkt_dpdk->port_id);
-#else
int ret;
ret = rte_eth_promiscuous_enable(pkt_dpdk->port_id);
@@ -1689,7 +1641,6 @@ static void promisc_mode_check(pkt_dpdk_t *pkt_dpdk)
if (ret)
pkt_dpdk->vdev_sysc_promisc = 1;
-#endif
}
static int dpdk_open(odp_pktio_t id ODP_UNUSED,
@@ -1735,18 +1686,17 @@ static int dpdk_open(odp_pktio_t id ODP_UNUSED,
pkt_dpdk->pool = pool;
- /* rte_eth_dev_count() was removed in v18.05 */
-#if RTE_VERSION < RTE_VERSION_NUM(18, 5, 0, 0)
- if (rte_eth_dev_count() == 0) {
-#else
if (rte_eth_dev_count_avail() == 0) {
-#endif
ODP_ERR("No DPDK ports found\n");
return -1;
}
memset(&dev_info, 0, sizeof(struct rte_eth_dev_info));
- rte_eth_dev_info_get(pkt_dpdk->port_id, &dev_info);
+ ret = rte_eth_dev_info_get(pkt_dpdk->port_id, &dev_info);
+ if (ret) {
+ ODP_ERR("Failed to read device info: %d\n", ret);
+ return -1;
+ }
/* Initialize runtime options */
if (init_options(pktio_entry, &dev_info)) {
@@ -1765,18 +1715,10 @@ static int dpdk_open(odp_pktio_t id ODP_UNUSED,
promisc_mode_check(pkt_dpdk);
-#if RTE_VERSION < RTE_VERSION_NUM(19, 11, 0, 0)
- ret = 0;
- if (pkt_dpdk->opt.multicast_en)
- rte_eth_allmulticast_enable(pkt_dpdk->port_id);
- else
- rte_eth_allmulticast_disable(pkt_dpdk->port_id);
-#else
if (pkt_dpdk->opt.multicast_en)
ret = rte_eth_allmulticast_enable(pkt_dpdk->port_id);
else
ret = rte_eth_allmulticast_disable(pkt_dpdk->port_id);
-#endif
/* Not supported by all PMDs, so ignore the return value */
if (ret)
diff --git a/platform/linux-generic/pktio/dpdk_parse.c b/platform/linux-generic/pktio/dpdk_parse.c
index 0984f06c5..f28f18921 100644
--- a/platform/linux-generic/pktio/dpdk_parse.c
+++ b/platform/linux-generic/pktio/dpdk_parse.c
@@ -1,5 +1,5 @@
/* Copyright (c) 2018, Linaro Limited
- * Copyright (c) 2021, Nokia
+ * Copyright (c) 2021-2022, Nokia
* All rights reserved.
*
* SPDX-License-Identifier: BSD-3-Clause
@@ -26,9 +26,19 @@
#if defined(__PPC64__) && defined(vector)
#undef vector
#endif
+#include <rte_version.h>
+
+#if RTE_VERSION < RTE_VERSION_NUM(21, 11, 0, 0)
+ #define RTE_MBUF_F_RX_IP_CKSUM_MASK PKT_RX_IP_CKSUM_MASK
+ #define RTE_MBUF_F_RX_IP_CKSUM_GOOD PKT_RX_IP_CKSUM_GOOD
+ #define RTE_MBUF_F_RX_IP_CKSUM_UNKNOWN PKT_RX_IP_CKSUM_UNKNOWN
+ #define RTE_MBUF_F_RX_L4_CKSUM_MASK PKT_RX_L4_CKSUM_MASK
+ #define RTE_MBUF_F_RX_L4_CKSUM_GOOD PKT_RX_L4_CKSUM_GOOD
+ #define RTE_MBUF_F_RX_L4_CKSUM_UNKNOWN PKT_RX_L4_CKSUM_UNKNOWN
+#endif
-#define IP4_CSUM_RESULT(ol_flags) (ol_flags & PKT_RX_IP_CKSUM_MASK)
-#define L4_CSUM_RESULT(ol_flags) (ol_flags & PKT_RX_L4_CKSUM_MASK)
+#define IP4_CSUM_RESULT(ol_flags) ((ol_flags) & RTE_MBUF_F_RX_IP_CKSUM_MASK)
+#define L4_CSUM_RESULT(ol_flags) ((ol_flags) & RTE_MBUF_F_RX_L4_CKSUM_MASK)
/** Parser helper function for Ethernet packets */
static inline uint16_t dpdk_parse_eth(packet_parser_t *prs,
@@ -174,9 +184,9 @@ static inline uint8_t dpdk_parse_ipv4(packet_parser_t *prs,
if (do_csum) {
uint64_t packet_csum_result = IP4_CSUM_RESULT(mbuf_ol);
- if (packet_csum_result == PKT_RX_IP_CKSUM_GOOD) {
+ if (packet_csum_result == RTE_MBUF_F_RX_IP_CKSUM_GOOD) {
prs->input_flags.l3_chksum_done = 1;
- } else if (packet_csum_result != PKT_RX_IP_CKSUM_UNKNOWN) {
+ } else if (packet_csum_result != RTE_MBUF_F_RX_IP_CKSUM_UNKNOWN) {
prs->input_flags.l3_chksum_done = 1;
prs->flags.ip_err = 1;
prs->flags.l3_chksum_err = 1;
@@ -302,9 +312,9 @@ static inline void dpdk_parse_tcp(packet_parser_t *prs,
if (do_csum) {
uint64_t packet_csum_result = L4_CSUM_RESULT(mbuf_ol);
- if (packet_csum_result == PKT_RX_L4_CKSUM_GOOD) {
+ if (packet_csum_result == RTE_MBUF_F_RX_L4_CKSUM_GOOD) {
prs->input_flags.l4_chksum_done = 1;
- } else if (packet_csum_result != PKT_RX_L4_CKSUM_UNKNOWN) {
+ } else if (packet_csum_result != RTE_MBUF_F_RX_L4_CKSUM_UNKNOWN) {
prs->input_flags.l4_chksum_done = 1;
prs->flags.tcp_err = 1;
prs->flags.l4_chksum_err = 1;
@@ -332,9 +342,9 @@ static inline void dpdk_parse_udp(packet_parser_t *prs,
if (do_csum) {
uint64_t packet_csum_result = L4_CSUM_RESULT(mbuf_ol);
- if (packet_csum_result == PKT_RX_L4_CKSUM_GOOD) {
+ if (packet_csum_result == RTE_MBUF_F_RX_L4_CKSUM_GOOD) {
prs->input_flags.l4_chksum_done = 1;
- } else if (packet_csum_result != PKT_RX_L4_CKSUM_UNKNOWN) {
+ } else if (packet_csum_result != RTE_MBUF_F_RX_L4_CKSUM_UNKNOWN) {
if (prs->input_flags.ipv4 && !udp->chksum) {
prs->input_flags.l4_chksum_done = 1;
} else {
diff --git a/platform/linux-generic/pktio/loop.c b/platform/linux-generic/pktio/loop.c
index b3232cded..3e21efecd 100644
--- a/platform/linux-generic/pktio/loop.c
+++ b/platform/linux-generic/pktio/loop.c
@@ -18,6 +18,7 @@
#include <odp/api/plat/packet_flag_inlines.h>
#include <odp/api/plat/queue_inlines.h>
+#include <odp_parse_internal.h>
#include <odp_classification_internal.h>
#include <odp_debug_internal.h>
#include <odp_errno_define.h>
@@ -26,6 +27,7 @@
#include <odp_ipsec_internal.h>
#include <odp_packet_internal.h>
#include <odp_packet_io_internal.h>
+#include <odp_macros_internal.h>
#include <odp_queue_if.h>
#include <protocols/eth.h>
@@ -158,7 +160,11 @@ static int loopback_recv(pktio_entry_t *pktio_entry, int index ODP_UNUSED,
odp_time_t ts_val;
odp_time_t *ts = NULL;
int num_rx = 0;
- int failed = 0;
+ int packets = 0, errors = 0;
+ uint32_t octets = 0;
+ const odp_proto_chksums_t chksums = pktio_entry->s.in_chksums;
+ const odp_proto_layer_t layer = pktio_entry->s.parse_layer;
+ const odp_pktin_config_opt_t opt = pktio_entry->s.config.pktin;
if (odp_unlikely(num > QUEUE_MULTI_MAX))
num = QUEUE_MULTI_MAX;
@@ -168,8 +174,7 @@ static int loopback_recv(pktio_entry_t *pktio_entry, int index ODP_UNUSED,
queue = pkt_priv(pktio_entry)->loopq;
nbr = odp_queue_deq_multi(queue, (odp_event_t *)hdr_tbl, num);
- if (pktio_entry->s.config.pktin.bit.ts_all ||
- pktio_entry->s.config.pktin.bit.ts_ptp) {
+ if (opt.bit.ts_all || opt.bit.ts_ptp) {
ts_val = odp_time_global();
ts = &ts_val;
}
@@ -181,61 +186,64 @@ static int loopback_recv(pktio_entry_t *pktio_entry, int index ODP_UNUSED,
pkt_len = odp_packet_len(pkt);
pkt_hdr = packet_hdr(pkt);
- packet_parse_reset(pkt_hdr, 1);
- if (pktio_cls_enabled(pktio_entry)) {
- odp_packet_t new_pkt;
- odp_pool_t new_pool;
+ if (layer) {
uint8_t *pkt_addr;
- uint8_t buf[PACKET_PARSE_SEG_LEN];
+ uint8_t buf[PARSE_BYTES];
int ret;
uint32_t seg_len = odp_packet_seg_len(pkt);
+ uint64_t l4_part_sum = 0;
/* Make sure there is enough data for the packet
* parser in the case of a segmented packet. */
- if (odp_unlikely(seg_len < PACKET_PARSE_SEG_LEN &&
- pkt_len > PACKET_PARSE_SEG_LEN)) {
- odp_packet_copy_to_mem(pkt, 0,
- PACKET_PARSE_SEG_LEN,
- buf);
- seg_len = PACKET_PARSE_SEG_LEN;
+ if (odp_unlikely(seg_len < PARSE_BYTES &&
+ pkt_len > seg_len)) {
+ seg_len = MIN(pkt_len, PARSE_BYTES);
+ odp_packet_copy_to_mem(pkt, 0, seg_len, buf);
pkt_addr = buf;
} else {
pkt_addr = odp_packet_data(pkt);
}
- ret = _odp_cls_classify_packet(pktio_entry, pkt_addr,
- pkt_len, seg_len,
- &new_pool, pkt_hdr, true);
- if (ret) {
- failed++;
+ packet_parse_reset(pkt_hdr, 1);
+ ret = _odp_packet_parse_common(&pkt_hdr->p, pkt_addr, pkt_len,
+ seg_len, layer, chksums,
+ &l4_part_sum, opt);
+ if (ret)
+ errors++;
+
+ if (ret < 0) {
odp_packet_free(pkt);
continue;
}
- if (new_pool != odp_packet_pool(pkt)) {
- new_pkt = odp_packet_copy(pkt, new_pool);
-
- odp_packet_free(pkt);
+ if (pktio_cls_enabled(pktio_entry)) {
+ odp_packet_t new_pkt;
+ odp_pool_t new_pool;
- if (new_pkt == ODP_PACKET_INVALID) {
- failed++;
+ ret = _odp_cls_classify_packet(pktio_entry, pkt_addr,
+ &new_pool, pkt_hdr);
+ if (ret) {
+ odp_packet_free(pkt);
continue;
}
- pkt = new_pkt;
- pkt_hdr = packet_hdr(new_pkt);
+ if (new_pool != odp_packet_pool(pkt)) {
+ new_pkt = odp_packet_copy(pkt, new_pool);
+
+ odp_packet_free(pkt);
+
+ if (new_pkt == ODP_PACKET_INVALID) {
+ pktio_entry->s.stats.in_discards++;
+ continue;
+ }
+
+ pkt = new_pkt;
+ pkt_hdr = packet_hdr(new_pkt);
+ }
}
- } else {
- odp_packet_parse_param_t param;
-
- /*
- * Use odp_packet_parse() which can handle segmented
- * packets.
- */
- param.proto = ODP_PROTO_ETH;
- param.last_layer = pktio_entry->s.config.parser.layer;
- param.chksums = pktio_entry->s.in_chksums;
- odp_packet_parse(packet_handle(pkt_hdr), 0, &param);
+
+ if (layer >= ODP_PROTO_LAYER_L4)
+ _odp_packet_l4_chksum(pkt_hdr, chksums, l4_part_sum);
}
packet_set_ts(pkt_hdr, ts);
@@ -247,12 +255,17 @@ static int loopback_recv(pktio_entry_t *pktio_entry, int index ODP_UNUSED,
odp_packet_has_ipsec(pkt))
_odp_ipsec_try_inline(&pkt);
- pktio_entry->s.stats.in_octets += pkt_len;
+ if (!pkt_hdr->p.flags.all.error) {
+ octets += pkt_len;
+ packets++;
+ }
+
pkts[num_rx++] = pkt;
}
- pktio_entry->s.stats.in_errors += failed;
- pktio_entry->s.stats.in_packets += num_rx - failed;
+ pktio_entry->s.stats.in_octets += octets;
+ pktio_entry->s.stats.in_packets += packets;
+ pktio_entry->s.stats.in_errors += errors;
odp_ticketlock_unlock(&pktio_entry->s.rxl);
@@ -533,6 +546,7 @@ static int loopback_init_capability(pktio_entry_t *pktio_entry)
capa->stats.pktio.counter.in_octets = 1;
capa->stats.pktio.counter.in_packets = 1;
capa->stats.pktio.counter.in_errors = 1;
+ capa->stats.pktio.counter.in_discards = 1;
capa->stats.pktio.counter.out_octets = 1;
capa->stats.pktio.counter.out_packets = 1;
capa->stats.pktin_queue.counter.octets = 1;
diff --git a/platform/linux-generic/pktio/netmap.c b/platform/linux-generic/pktio/netmap.c
index 0b1345e50..94b88e21e 100644
--- a/platform/linux-generic/pktio/netmap.c
+++ b/platform/linux-generic/pktio/netmap.c
@@ -16,6 +16,7 @@
#include <odp/api/time.h>
#include <odp/api/plat/time_inlines.h>
+#include <odp_parse_internal.h>
#include <odp_packet_io_internal.h>
#include <odp_packet_io_stats.h>
#include <odp_ethtool_stats.h>
@@ -822,11 +823,14 @@ static inline int netmap_pkt_to_odp(pktio_entry_t *pktio_entry,
odp_packet_t pkt;
odp_pool_t pool = pkt_priv(pktio_entry)->pool;
odp_packet_hdr_t *pkt_hdr;
- odp_packet_hdr_t parsed_hdr;
int i;
int num;
uint32_t max_len;
uint16_t frame_offset = pktio_entry->s.pktin_frame_offset;
+ int num_rx = 0;
+ const odp_proto_chksums_t chksums = pktio_entry->s.in_chksums;
+ const odp_proto_layer_t layer = pktio_entry->s.parse_layer;
+ const odp_pktin_config_opt_t opt = pktio_entry->s.config.pktin;
/* Allocate maximum sized packets */
max_len = pkt_priv(pktio_entry)->mtu;
@@ -837,45 +841,50 @@ static inline int netmap_pkt_to_odp(pktio_entry_t *pktio_entry,
for (i = 0; i < num; i++) {
netmap_slot_t slot;
uint16_t len;
+ const uint8_t *buf;
+ uint64_t l4_part_sum = 0;
slot = slot_tbl[i];
len = slot.len;
+ buf = (const uint8_t *)slot.buf;
odp_prefetch(slot.buf);
- if (pktio_cls_enabled(pktio_entry)) {
- if (_odp_cls_classify_packet(pktio_entry,
- (const uint8_t *)slot.buf, len,
- len, &pool, &parsed_hdr, true))
- goto fail;
+ pkt = pkt_tbl[num_rx];
+ pkt_hdr = packet_hdr(pkt);
+
+ if (layer) {
+ if (_odp_packet_parse_common(&pkt_hdr->p, buf, len, len,
+ layer, chksums, &l4_part_sum, opt) < 0)
+ continue;
+
+ if (pktio_cls_enabled(pktio_entry)) {
+ if (_odp_cls_classify_packet(pktio_entry, buf, &pool,
+ pkt_hdr))
+ continue;
+ }
}
- pkt = pkt_tbl[i];
- pkt_hdr = packet_hdr(pkt);
pull_tail(pkt_hdr, max_len - len);
if (frame_offset)
pull_head(pkt_hdr, frame_offset);
if (odp_packet_copy_from_mem(pkt, 0, len, slot.buf) != 0)
- goto fail;
+ break;
pkt_hdr->input = pktio_entry->s.handle;
- if (pktio_cls_enabled(pktio_entry))
- _odp_packet_copy_cls_md(pkt_hdr, &parsed_hdr);
- else
- _odp_packet_parse_layer(pkt_hdr,
- pktio_entry->s.config.parser.layer,
- pktio_entry->s.in_chksums);
+ if (layer >= ODP_PROTO_LAYER_L4)
+ _odp_packet_l4_chksum(pkt_hdr, chksums, l4_part_sum);
packet_set_ts(pkt_hdr, ts);
+ num_rx++;
}
- return i;
+ if (num_rx < num)
+ odp_packet_free_multi(&pkt_tbl[num_rx], num - num_rx);
-fail:
- odp_packet_free_multi(&pkt_tbl[i], num - i);
- return i;
+ return num_rx;
}
static inline int netmap_recv_desc(pktio_entry_t *pktio_entry,
@@ -1042,11 +1051,11 @@ static int netmap_recv_tmo(pktio_entry_t *pktio_entry, int index,
}
static int netmap_recv_mq_tmo(pktio_entry_t *pktio_entry[], int index[],
- int num_q, odp_packet_t pkt_table[], int num,
- unsigned *from, uint64_t usecs)
+ uint32_t num_q, odp_packet_t pkt_table[], int num,
+ uint32_t *from, uint64_t usecs)
{
struct timeval timeout;
- int i;
+ uint32_t i;
int ret;
int maxfd = -1, maxfd2;
fd_set readfds;
diff --git a/platform/linux-generic/pktio/null.c b/platform/linux-generic/pktio/null.c
index 90e113ec6..a6498da8c 100644
--- a/platform/linux-generic/pktio/null.c
+++ b/platform/linux-generic/pktio/null.c
@@ -71,9 +71,9 @@ static int null_recv_tmo(pktio_entry_t *pktio_entry ODP_UNUSED,
}
static int null_recv_mq_tmo(pktio_entry_t *pktio_entry[] ODP_UNUSED,
- int index[] ODP_UNUSED, int num_q ODP_UNUSED,
+ int index[] ODP_UNUSED, uint32_t num_q ODP_UNUSED,
odp_packet_t pkt_table[] ODP_UNUSED,
- int num ODP_UNUSED, unsigned *from ODP_UNUSED,
+ int num ODP_UNUSED, uint32_t *from ODP_UNUSED,
uint64_t usecs)
{
struct timeval timeout;
diff --git a/platform/linux-generic/pktio/pcap.c b/platform/linux-generic/pktio/pcap.c
index 7290626c2..af94ffa72 100644
--- a/platform/linux-generic/pktio/pcap.c
+++ b/platform/linux-generic/pktio/pcap.c
@@ -46,6 +46,7 @@
#include <odp/api/plat/packet_inlines.h>
+#include <odp_parse_internal.h>
#include <odp_classification_internal.h>
#include <odp_debug_internal.h>
#include <odp_global_data.h>
@@ -248,7 +249,12 @@ static int pcapif_recv_pkt(pktio_entry_t *pktio_entry, int index ODP_UNUSED,
pkt_pcap_t *pcap = pkt_priv(pktio_entry);
odp_time_t ts_val;
odp_time_t *ts = NULL;
+ int packets = 0, errors = 0;
+ uint32_t octets = 0;
uint16_t frame_offset = pktio_entry->s.pktin_frame_offset;
+ const odp_proto_chksums_t chksums = pktio_entry->s.in_chksums;
+ const odp_proto_layer_t layer = pktio_entry->s.parse_layer;
+ const odp_pktin_config_opt_t opt = pktio_entry->s.config.pktin;
odp_ticketlock_lock(&pktio_entry->s.rxl);
@@ -256,8 +262,7 @@ static int pcapif_recv_pkt(pktio_entry_t *pktio_entry, int index ODP_UNUSED,
odp_ticketlock_unlock(&pktio_entry->s.rxl);
return 0;
}
- if (pktio_entry->s.config.pktin.bit.ts_all ||
- pktio_entry->s.config.pktin.bit.ts_ptp)
+ if (opt.bit.ts_all || opt.bit.ts_ptp)
ts = &ts_val;
for (i = 0; i < num; ) {
@@ -291,42 +296,64 @@ static int pcapif_recv_pkt(pktio_entry_t *pktio_entry, int index ODP_UNUSED,
break;
}
- if (pktio_cls_enabled(pktio_entry)) {
- odp_packet_t new_pkt;
+ if (layer) {
+ uint64_t l4_part_sum = 0;
- ret = _odp_cls_classify_packet(pktio_entry, data,
- pkt_len, pkt_len,
- &new_pool, pkt_hdr, true);
- if (ret) {
+ ret = _odp_packet_parse_common(&pkt_hdr->p, data, pkt_len,
+ pkt_len, layer, chksums,
+ &l4_part_sum, opt);
+ if (ret)
+ errors++;
+
+ if (ret < 0) {
odp_packet_free(pkt);
continue;
}
- if (new_pool != pcap->pool) {
- new_pkt = odp_packet_copy(pkt, new_pool);
- odp_packet_free(pkt);
+ if (pktio_cls_enabled(pktio_entry)) {
+ odp_packet_t new_pkt;
- if (odp_unlikely(new_pkt == ODP_PACKET_INVALID))
+ ret = _odp_cls_classify_packet(pktio_entry, data,
+ &new_pool, pkt_hdr);
+ if (ret) {
+ odp_packet_free(pkt);
continue;
+ }
+ if (new_pool != pcap->pool) {
+ new_pkt = odp_packet_copy(pkt, new_pool);
+
+ odp_packet_free(pkt);
+
+ if (odp_unlikely(new_pkt == ODP_PACKET_INVALID)) {
+ pktio_entry->s.stats.in_discards++;
+ continue;
+ }
- pkt = new_pkt;
- pkt_hdr = packet_hdr(new_pkt);
+ pkt = new_pkt;
+ pkt_hdr = packet_hdr(new_pkt);
+ }
}
- } else {
- _odp_packet_parse_layer(pkt_hdr,
- pktio_entry->s.config.parser.layer,
- pktio_entry->s.in_chksums);
+
+ if (layer >= ODP_PROTO_LAYER_L4)
+ _odp_packet_l4_chksum(pkt_hdr, chksums, l4_part_sum);
}
- pktio_entry->s.stats.in_octets += pkt_hdr->frame_len;
packet_set_ts(pkt_hdr, ts);
pkt_hdr->input = pktio_entry->s.handle;
+ if (!pkt_hdr->p.flags.all.error) {
+ octets += pkt_len;
+ packets++;
+ }
+
pkts[i] = pkt;
i++;
}
- pktio_entry->s.stats.in_packets += i;
+
+ pktio_entry->s.stats.in_octets += octets;
+ pktio_entry->s.stats.in_packets += packets;
+ pktio_entry->s.stats.in_errors += errors;
odp_ticketlock_unlock(&pktio_entry->s.rxl);
@@ -441,6 +468,8 @@ static int pcapif_capability(pktio_entry_t *pktio_entry ODP_UNUSED,
capa->stats.pktio.counter.in_octets = 1;
capa->stats.pktio.counter.in_packets = 1;
+ capa->stats.pktio.counter.in_discards = 1;
+ capa->stats.pktio.counter.in_errors = 1;
capa->stats.pktio.counter.out_octets = 1;
capa->stats.pktio.counter.out_packets = 1;
diff --git a/platform/linux-generic/pktio/pktio_common.c b/platform/linux-generic/pktio/pktio_common.c
index d2f4d7219..1f3cf951e 100644
--- a/platform/linux-generic/pktio/pktio_common.c
+++ b/platform/linux-generic/pktio/pktio_common.c
@@ -11,13 +11,13 @@
static int sock_recv_mq_tmo_select(pktio_entry_t * const *entry,
const int index[],
- unsigned int num_q, unsigned int *from,
+ uint32_t num_q, uint32_t *from,
odp_packet_t packets[], int num,
uint64_t usecs, fd_set *readfds,
int maxfd)
{
struct timeval timeout;
- unsigned int i;
+ uint32_t i;
int ret;
for (i = 0; i < num_q; i++) {
@@ -50,17 +50,17 @@ static int sock_recv_mq_tmo_select(pktio_entry_t * const *entry,
}
int _odp_sock_recv_mq_tmo_try_int_driven(const struct odp_pktin_queue_t queues[],
- unsigned int num_q, unsigned int *from,
+ uint32_t num_q, uint32_t *from,
odp_packet_t packets[], int num,
uint64_t usecs, int *trial_successful)
{
- unsigned int i;
+ uint32_t i;
pktio_entry_t *entry[num_q];
int index[num_q];
fd_set readfds;
int maxfd = -1;
- int (*impl)(pktio_entry_t *entry[], int index[], int num_q,
- odp_packet_t packets[], int num, unsigned int *from,
+ int (*impl)(pktio_entry_t *entry[], int index[], uint32_t num_q,
+ odp_packet_t packets[], int num, uint32_t *from,
uint64_t wait_usecs) = NULL;
int impl_set = 0;
diff --git a/platform/linux-generic/pktio/socket.c b/platform/linux-generic/pktio/socket.c
index 81c178b9d..9d1bbe545 100644
--- a/platform/linux-generic/pktio/socket.c
+++ b/platform/linux-generic/pktio/socket.c
@@ -15,12 +15,14 @@
#include <odp/api/ticketlock.h>
#include <odp_socket_common.h>
+#include <odp_parse_internal.h>
#include <odp_packet_internal.h>
#include <odp_packet_io_internal.h>
#include <odp_packet_io_stats.h>
#include <odp_debug_internal.h>
#include <odp_errno_define.h>
#include <odp_classification_internal.h>
+#include <odp_macros_internal.h>
#include <sys/socket.h>
#include <stdio.h>
@@ -233,6 +235,9 @@ static int sock_mmsg_recv(pktio_entry_t *pktio_entry, int index ODP_UNUSED,
int i;
uint16_t frame_offset = pktio_entry->s.pktin_frame_offset;
uint32_t alloc_len = pkt_sock->mtu + frame_offset;
+ const odp_proto_chksums_t chksums = pktio_entry->s.in_chksums;
+ const odp_proto_layer_t layer = pktio_entry->s.parse_layer;
+ const odp_pktin_config_opt_t opt = pktio_entry->s.config.pktin;
memset(msgvec, 0, sizeof(msgvec));
@@ -249,8 +254,7 @@ static int sock_mmsg_recv(pktio_entry_t *pktio_entry, int index ODP_UNUSED,
recv_msgs = recvmmsg(sockfd, msgvec, nb_pkts, MSG_DONTWAIT, NULL);
odp_ticketlock_unlock(&pkt_sock->rx_lock);
- if (pktio_entry->s.config.pktin.bit.ts_all ||
- pktio_entry->s.config.pktin.bit.ts_ptp) {
+ if (opt.bit.ts_all || opt.bit.ts_ptp) {
ts_val = odp_time_global();
ts = &ts_val;
}
@@ -262,25 +266,48 @@ static int sock_mmsg_recv(pktio_entry_t *pktio_entry, int index ODP_UNUSED,
odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt);
uint16_t pkt_len = msgvec[i].msg_len;
int ret;
+ uint64_t l4_part_sum = 0;
if (odp_unlikely(msgvec[i].msg_hdr.msg_flags & MSG_TRUNC)) {
odp_packet_free(pkt);
ODP_DBG("dropped truncated packet\n");
continue;
}
- if (pktio_cls_enabled(pktio_entry)) {
- uint16_t seg_len = pkt_len;
- if (msgvec[i].msg_hdr.msg_iov->iov_len < pkt_len)
- seg_len = msgvec[i].msg_hdr.msg_iov->iov_len;
+ ret = odp_packet_trunc_tail(&pkt, odp_packet_len(pkt) - pkt_len,
+ NULL, NULL);
+ if (ret < 0) {
+ ODP_ERR("trunc_tail failed");
+ odp_packet_free(pkt);
+ continue;
+ }
+
+ if (layer) {
+ uint8_t buf[PARSE_BYTES];
+ uint16_t seg_len = msgvec[i].msg_hdr.msg_iov->iov_len;
- if (_odp_cls_classify_packet(pktio_entry, base, pkt_len,
- seg_len, &pool, pkt_hdr,
- true)) {
- ODP_ERR("_odp_cls_classify_packet failed");
+ /* Make sure there is enough data for the packet
+ * parser in the case of a segmented packet. */
+ if (odp_unlikely(seg_len < PARSE_BYTES && pkt_len > seg_len)) {
+ seg_len = MIN(pkt_len, PARSE_BYTES);
+ odp_packet_copy_to_mem(pkt, 0, seg_len, buf);
+ base = buf;
+ }
+
+ if (_odp_packet_parse_common(&pkt_hdr->p, base, pkt_len,
+ seg_len, layer, chksums,
+ &l4_part_sum, opt) < 0) {
odp_packet_free(pkt);
continue;
}
+
+ if (pktio_cls_enabled(pktio_entry)) {
+ if (_odp_cls_classify_packet(pktio_entry, base, &pool,
+ pkt_hdr)) {
+ odp_packet_free(pkt);
+ continue;
+ }
+ }
}
/* Don't receive packets sent by ourselves */
@@ -290,20 +317,10 @@ static int sock_mmsg_recv(pktio_entry_t *pktio_entry, int index ODP_UNUSED,
continue;
}
- ret = odp_packet_trunc_tail(&pkt, odp_packet_len(pkt) - pkt_len,
- NULL, NULL);
- if (ret < 0) {
- ODP_ERR("trunk_tail failed");
- odp_packet_free(pkt);
- continue;
- }
-
pkt_hdr->input = pktio_entry->s.handle;
- if (!pktio_cls_enabled(pktio_entry))
- _odp_packet_parse_layer(pkt_hdr,
- pktio_entry->s.config.parser.layer,
- pktio_entry->s.in_chksums);
+ if (layer >= ODP_PROTO_LAYER_L4)
+ _odp_packet_l4_chksum(pkt_hdr, chksums, l4_part_sum);
packet_set_ts(pkt_hdr, ts);
@@ -359,11 +376,11 @@ static int sock_recv_tmo(pktio_entry_t *pktio_entry, int index,
}
static int sock_recv_mq_tmo(pktio_entry_t *pktio_entry[], int index[],
- int num_q, odp_packet_t pkt_table[], int num,
- unsigned *from, uint64_t usecs)
+ uint32_t num_q, odp_packet_t pkt_table[], int num,
+ uint32_t *from, uint64_t usecs)
{
struct timeval timeout;
- int i;
+ uint32_t i;
int ret;
int maxfd = -1, maxfd2;
fd_set readfds;
diff --git a/platform/linux-generic/pktio/socket_mmap.c b/platform/linux-generic/pktio/socket_mmap.c
index f84834610..7824b0e91 100644
--- a/platform/linux-generic/pktio/socket_mmap.c
+++ b/platform/linux-generic/pktio/socket_mmap.c
@@ -16,6 +16,7 @@
#include <odp/api/plat/packet_inlines.h>
#include <odp_socket_common.h>
+#include <odp_parse_internal.h>
#include <odp_packet_internal.h>
#include <odp_packet_io_internal.h>
#include <odp_packet_io_stats.h>
@@ -150,9 +151,11 @@ static inline unsigned pkt_mmap_v2_rx(pktio_entry_t *pktio_entry,
odp_pool_t pool = pkt_sock->pool;
uint16_t frame_offset = pktio_entry->s.pktin_frame_offset;
uint16_t vlan_len = 0;
+ const odp_proto_chksums_t chksums = pktio_entry->s.in_chksums;
+ const odp_proto_layer_t layer = pktio_entry->s.parse_layer;
+ const odp_pktin_config_opt_t opt = pktio_entry->s.config.pktin;
- if (pktio_entry->s.config.pktin.bit.ts_all ||
- pktio_entry->s.config.pktin.bit.ts_ptp)
+ if (opt.bit.ts_all || opt.bit.ts_ptp)
ts = &ts_val;
ring = &pkt_sock->rx_ring;
@@ -163,8 +166,8 @@ static inline unsigned pkt_mmap_v2_rx(pktio_entry_t *pktio_entry,
struct tpacket2_hdr *tp_hdr;
odp_packet_t pkt;
odp_packet_hdr_t *hdr;
- odp_packet_hdr_t parsed_hdr;
int ret;
+ uint64_t l4_part_sum = 0;
tp_hdr = (void *)next_ptr;
@@ -212,18 +215,29 @@ static inline unsigned pkt_mmap_v2_rx(pktio_entry_t *pktio_entry,
continue;
}
- if (pktio_cls_enabled(pktio_entry)) {
- if (_odp_cls_classify_packet(pktio_entry, pkt_buf, pkt_len,
- pkt_len, &pool, &parsed_hdr,
- true)) {
+ hdr = packet_hdr(pkt);
+
+ if (layer) {
+ if (_odp_packet_parse_common(&hdr->p, pkt_buf, pkt_len,
+ pkt_len, layer, chksums,
+ &l4_part_sum, opt) < 0) {
odp_packet_free(pkt);
tp_hdr->tp_status = TP_STATUS_KERNEL;
frame_num = next_frame_num;
continue;
}
+
+ if (pktio_cls_enabled(pktio_entry)) {
+ if (_odp_cls_classify_packet(pktio_entry, pkt_buf,
+ &pool, hdr)) {
+ odp_packet_free(pkt);
+ tp_hdr->tp_status = TP_STATUS_KERNEL;
+ frame_num = next_frame_num;
+ continue;
+ }
+ }
}
- hdr = packet_hdr(pkt);
if (frame_offset)
pull_head(hdr, frame_offset);
@@ -276,12 +290,8 @@ static inline unsigned pkt_mmap_v2_rx(pktio_entry_t *pktio_entry,
hdr->input = pktio_entry->s.handle;
- if (pktio_cls_enabled(pktio_entry))
- _odp_packet_copy_cls_md(hdr, &parsed_hdr);
- else
- _odp_packet_parse_layer(hdr,
- pktio_entry->s.config.parser.layer,
- pktio_entry->s.in_chksums);
+ if (layer >= ODP_PROTO_LAYER_L4)
+ _odp_packet_l4_chksum(hdr, chksums, l4_part_sum);
packet_set_ts(hdr, ts);
@@ -727,11 +737,11 @@ static int sock_mmap_recv_tmo(pktio_entry_t *pktio_entry, int index,
}
static int sock_mmap_recv_mq_tmo(pktio_entry_t *pktio_entry[], int index[],
- int num_q, odp_packet_t pkt_table[], int num,
- unsigned *from, uint64_t usecs)
+ uint32_t num_q, odp_packet_t pkt_table[], int num,
+ uint32_t *from, uint64_t usecs)
{
struct timeval timeout;
- int i;
+ uint32_t i;
int ret;
int maxfd = -1, maxfd2;
fd_set readfds;
diff --git a/platform/linux-generic/pktio/tap.c b/platform/linux-generic/pktio/tap.c
index d298c8982..40c46b43c 100644
--- a/platform/linux-generic/pktio/tap.c
+++ b/platform/linux-generic/pktio/tap.c
@@ -38,6 +38,7 @@
#include <odp/api/plat/packet_inlines.h>
+#include <odp_parse_internal.h>
#include <odp_debug_internal.h>
#include <odp_socket_common.h>
#include <odp_packet_internal.h>
@@ -286,14 +287,27 @@ static odp_packet_t pack_odp_pkt(pktio_entry_t *pktio_entry, const void *data,
odp_packet_hdr_t *pkt_hdr;
odp_packet_hdr_t parsed_hdr;
int num;
+ uint64_t l4_part_sum = 0;
uint16_t frame_offset = pktio_entry->s.pktin_frame_offset;
-
- if (pktio_cls_enabled(pktio_entry)) {
- if (_odp_cls_classify_packet(pktio_entry, data, len, len,
- &pkt_priv(pktio_entry)->pool,
- &parsed_hdr, true)) {
+ const odp_proto_chksums_t chksums = pktio_entry->s.in_chksums;
+ const odp_proto_layer_t layer = pktio_entry->s.parse_layer;
+ const odp_pktin_config_opt_t opt = pktio_entry->s.config.pktin;
+
+ if (layer) {
+ packet_parse_reset(&parsed_hdr, 1);
+ packet_set_len(&parsed_hdr, len);
+ if (_odp_packet_parse_common(&parsed_hdr.p, data, len, len, layer,
+ chksums, &l4_part_sum, opt) < 0) {
return ODP_PACKET_INVALID;
}
+
+ if (pktio_cls_enabled(pktio_entry)) {
+ if (_odp_cls_classify_packet(pktio_entry, data,
+ &pkt_priv(pktio_entry)->pool,
+ &parsed_hdr)) {
+ return ODP_PACKET_INVALID;
+ }
+ }
}
num = _odp_packet_alloc_multi(pkt_priv(pktio_entry)->pool,
@@ -312,12 +326,12 @@ static odp_packet_t pack_odp_pkt(pktio_entry_t *pktio_entry, const void *data,
return ODP_PACKET_INVALID;
}
- if (pktio_cls_enabled(pktio_entry))
+ if (layer) {
_odp_packet_copy_cls_md(pkt_hdr, &parsed_hdr);
- else
- _odp_packet_parse_layer(pkt_hdr,
- pktio_entry->s.config.parser.layer,
- pktio_entry->s.in_chksums);
+
+ if (layer >= ODP_PROTO_LAYER_L4)
+ _odp_packet_l4_chksum(pkt_hdr, chksums, l4_part_sum);
+ }
packet_set_ts(pkt_hdr, ts);
pkt_hdr->input = pktio_entry->s.handle;
@@ -335,6 +349,7 @@ static int tap_pktio_recv(pktio_entry_t *pktio_entry, int index ODP_UNUSED,
uint8_t buf[mtu];
odp_time_t ts_val;
odp_time_t *ts = NULL;
+ int num_rx = 0;
odp_ticketlock_lock(&pktio_entry->s.rxl);
@@ -355,14 +370,15 @@ static int tap_pktio_recv(pktio_entry_t *pktio_entry, int index ODP_UNUSED,
break;
}
- pkts[i] = pack_odp_pkt(pktio_entry, buf, retval, ts);
- if (pkts[i] == ODP_PACKET_INVALID)
+ pkts[num_rx] = pack_odp_pkt(pktio_entry, buf, retval, ts);
+ if (pkts[num_rx] == ODP_PACKET_INVALID)
break;
+ num_rx++;
}
odp_ticketlock_unlock(&pktio_entry->s.rxl);
- return i;
+ return num_rx;
}
static int tap_pktio_send_lockless(pktio_entry_t *pktio_entry,
diff --git a/scripts/ci/build_arm64.sh b/scripts/ci/build_arm64.sh
index 79b55e531..f340a0ca0 100755
--- a/scripts/ci/build_arm64.sh
+++ b/scripts/ci/build_arm64.sh
@@ -6,18 +6,27 @@ if [[ $(uname -m) =~ ^(arm64|aarch64)$ ]]; then
export BUILD_ARCH=aarch64-linux-gnu
fi
-if [ "${CC#clang}" != "${CC}" ] ; then
- export CC="clang --target=${TARGET_ARCH}"
- export CXX="clang++ --target=${TARGET_ARCH}"
+if [ "$TARGET_ARCH" == "$BUILD_ARCH" ]; then
+ # Native build
+ if [ "${CC#clang}" != "${CC}" ] ; then
+ export CXX="clang++"
+ fi
else
- export CC="${TARGET_ARCH}-gcc"
- export CXX="${TARGET_ARCH}-g++"
-fi
-export CPPFLAGS="-I/usr/include/${TARGET_ARCH}/dpdk"
+ # Cross compilation
+ if [ "${CC#clang}" != "${CC}" ] ; then
+ export CC="clang --target=${TARGET_ARCH}"
+ export CXX="clang++ --target=${TARGET_ARCH}"
+ else
+ export CC="${TARGET_ARCH}-gcc"
+ export CXX="${TARGET_ARCH}-g++"
+ fi
+
+ export CPPFLAGS="-I/usr/include/${TARGET_ARCH}/dpdk"
-# Use target libraries
-export PKG_CONFIG_PATH=
-export PKG_CONFIG_LIBDIR=/usr/lib/${TARGET_ARCH}/pkgconfig:/usr/local/lib/${TARGET_ARCH}/pkgconfig
+ # Use target libraries
+ export PKG_CONFIG_PATH=
+ export PKG_CONFIG_LIBDIR=/usr/lib/${TARGET_ARCH}/pkgconfig:/usr/local/lib/${TARGET_ARCH}/pkgconfig
+fi
# ARMv8 crypto
export PKG_CONFIG_PATH=~/aarch64cryptolib/pkgconfig:$PKG_CONFIG_PATH
diff --git a/test/performance/odp_bench_packet.c b/test/performance/odp_bench_packet.c
index 0354ef9b8..23fd17bea 100644
--- a/test/performance/odp_bench_packet.c
+++ b/test/performance/odp_bench_packet.c
@@ -756,7 +756,7 @@ static int bench_packet_headroom(void)
for (i = 0; i < TEST_REPEAT_COUNT; i++)
ret += odp_packet_headroom(gbl_args->pkt_tbl[i]);
- return i;
+ return i + ret;
}
static int bench_packet_tailroom(void)
@@ -767,7 +767,7 @@ static int bench_packet_tailroom(void)
for (i = 0; i < TEST_REPEAT_COUNT; i++)
ret += odp_packet_tailroom(gbl_args->pkt_tbl[i]);
- return i;
+ return i + ret;
}
static int bench_packet_tail(void)
@@ -1412,7 +1412,7 @@ static int bench_packet_has_ref(void)
for (i = 0; i < TEST_REPEAT_COUNT; i++)
ret += odp_packet_has_ref(pkt_tbl[i]);
- return i;
+ return i + ret;
}
static int bench_packet_subtype(void)
diff --git a/test/performance/odp_crypto.c b/test/performance/odp_crypto.c
index a26d072e9..45e770d42 100644
--- a/test/performance/odp_crypto.c
+++ b/test/performance/odp_crypto.c
@@ -610,7 +610,6 @@ create_session_from_config(odp_crypto_session_t *session,
odp_crypto_session_param_init(&params);
memcpy(&params, &config->session, sizeof(odp_crypto_session_param_t));
params.op = ODP_CRYPTO_OP_ENCODE;
- params.pref_mode = ODP_CRYPTO_SYNC;
/* Lookup the packet pool */
pkt_pool = odp_pool_lookup("packet_pool");
diff --git a/test/performance/odp_ipsec.c b/test/performance/odp_ipsec.c
index 5f1f9d4ef..c464ba0d7 100644
--- a/test/performance/odp_ipsec.c
+++ b/test/performance/odp_ipsec.c
@@ -1,4 +1,6 @@
/* Copyright (c) 2018, Linaro Limited
+ * Copyright (c) 2022, Marvell
+ * Copyright (c) 2022, Nokia
* All rights reserved.
*
* SPDX-License-Identifier: BSD-3-Clause
@@ -18,11 +20,14 @@
#include <odp_api.h>
#include <odp/helper/odph_api.h>
+#include <inttypes.h>
/** @def POOL_NUM_PKT
* Number of packets in the pool
*/
-#define POOL_NUM_PKT 64
+#define POOL_NUM_PKT 4096
+
+#define MAX_DEQUEUE_BURST 16
static uint8_t test_salt[16] = "0123456789abcdef";
@@ -97,11 +102,11 @@ typedef struct {
int in_flight;
/**
- * Number of iteration to repeat crypto operation to get good
- * average number. Specified through -i or --terations option.
+ * Number of packets to be IPsec processed to get good average number.
+ * Specified through -c or --count option.
* Default is 10000.
*/
- int iteration_count;
+ int packet_count;
/**
* Payload size to test. If 0 set of predefined payload sizes
@@ -139,6 +144,19 @@ typedef struct {
* Specified through -u argument.
*/
int ah;
+
+ /*
+ * Burst size.
+ * Prepare and submit as many packets for IPsec processing in each
+ * iteration of the loop.
+ */
+ int burst_size;
+
+ /*
+ * Use vector packet completion from IPsec APIs.
+ * Specified through -v or --vector argument.
+ */
+ uint32_t vec_pkt_size;
} ipsec_args_t;
/*
@@ -516,7 +534,7 @@ print_result(ipsec_args_t *cargs,
throughput = (1000000.0 / result->elapsed) * payload_length / 1024;
printf("%30.30s %15d %15d %15.3f %15.3f %15.3f %15d\n",
- config->name, cargs->iteration_count, payload_length,
+ config->name, cargs->packet_count, payload_length,
result->elapsed, result->rusage_self, result->rusage_thread,
throughput);
}
@@ -584,28 +602,49 @@ static uint8_t test_data[] = {
0x08, 0x00, 0xfb, 0x37, 0x12, 0x34, 0x00, 0x00
};
-static odp_packet_t
-make_packet(odp_pool_t pkt_pool, unsigned int payload_length)
+static inline void debug_packets(int debug, odp_packet_t *pkt, int num_pkts)
{
- odp_packet_t pkt;
+ if (odp_likely(!debug))
+ return;
+ for (int i = 0; i < num_pkts; i++)
+ odp_packet_print_data(pkt[i], 0, odp_packet_len(pkt[i]));
+}
- if (payload_length < sizeof(test_data))
- return ODP_PACKET_INVALID;
+static int
+make_packet_multi(odp_pool_t pkt_pool, unsigned int payload_length,
+ odp_packet_t pkt[], int num)
+{
+ int i, ret;
+
+ ret = odp_packet_alloc_multi(pkt_pool, payload_length, pkt, num);
+ if (ret != num) {
+ ODPH_ERR("Could not allocate buffer\n");
+ if (ret > 0)
+ odp_packet_free_sp(pkt, ret);
+ return -1;
+ }
+
+ for (i = 0; i < num; i++) {
+ odp_packet_copy_from_mem(pkt[i], 0, sizeof(test_data), test_data);
+ odp_packet_l3_offset_set(pkt[i], 0);
- pkt = odp_packet_alloc(pkt_pool, payload_length);
- if (pkt == ODP_PACKET_INVALID) {
- ODPH_ERR("failed to allocate buffer\n");
- return pkt;
+ uint8_t *mem = odp_packet_data(pkt[i]);
+ ((odph_ipv4hdr_t *)mem)->tot_len = odp_cpu_to_be_16(payload_length);
+ memset(mem + sizeof(test_data), 1, payload_length - sizeof(test_data));
}
- odp_packet_copy_from_mem(pkt, 0, sizeof(test_data), test_data);
- odp_packet_l3_offset_set(pkt, 0);
+ return 0;
+}
- uint8_t *mem = odp_packet_data(pkt);
- ((odph_ipv4hdr_t *)mem)->tot_len = odp_cpu_to_be_16(payload_length);
- memset(mem + sizeof(test_data), 1, payload_length - sizeof(test_data));
+static inline void check_ipsec_result(odp_packet_t ipsec_pkt)
+{
+ odp_ipsec_packet_result_t result;
- return pkt;
+ if (odp_unlikely(odp_ipsec_result(&result, ipsec_pkt)))
+ ODPH_ERR("odp_ipsec_result() failed\n");
+ else if (odp_unlikely(result.status.error.all))
+ ODPH_ERR("IPsec processing error: %" PRIu32 "\n",
+ result.status.error.all);
}
/**
@@ -619,10 +658,13 @@ run_measure_one(ipsec_args_t *cargs,
time_record_t *start,
time_record_t *end)
{
+ int in_flight, pkts_allowed, num_out, num_pkts, rc = 0;
+ const int max_in_flight = cargs->in_flight;
+ const int burst_size = cargs->burst_size;
+ const int packet_count = cargs->packet_count;
+ const int debug = cargs->debug_packets;
odp_ipsec_out_param_t param;
odp_pool_t pkt_pool;
- odp_packet_t pkt = ODP_PACKET_INVALID;
- int rc = 0;
pkt_pool = odp_pool_lookup("packet_pool");
if (pkt_pool == ODP_POOL_INVALID) {
@@ -630,6 +672,9 @@ run_measure_one(ipsec_args_t *cargs,
return -1;
}
+ if (payload_length < sizeof(test_data))
+ return -1;
+
int packets_sent = 0;
int packets_received = 0;
@@ -641,43 +686,53 @@ run_measure_one(ipsec_args_t *cargs,
fill_time_record(start);
- while ((packets_sent < cargs->iteration_count) ||
- (packets_received < cargs->iteration_count)) {
- if ((packets_sent < cargs->iteration_count) &&
- (packets_sent - packets_received <
- cargs->in_flight)) {
- odp_packet_t out_pkt;
- int num_out = 1;
+ while ((packets_sent < packet_count) ||
+ (packets_received < packet_count)) {
+ num_pkts = packet_count - packets_sent;
+
+ /* Enqueue up to burst size */
+ num_pkts = num_pkts > burst_size ? burst_size : num_pkts;
+
+ /* Enqueue up to (max in flight - current in flight) */
+ in_flight = packets_sent - packets_received;
+ pkts_allowed = max_in_flight - in_flight;
- pkt = make_packet(pkt_pool, payload_length);
- if (ODP_PACKET_INVALID == pkt)
+ /* Enqueue either a burst of packets or skip */
+ num_pkts = num_pkts > pkts_allowed ? 0 : num_pkts;
+
+ if (odp_likely(num_pkts)) {
+ odp_packet_t out_pkt[num_pkts];
+ odp_packet_t pkt[num_pkts];
+ int i;
+
+ if (odp_unlikely(make_packet_multi(pkt_pool,
+ payload_length,
+ pkt,
+ num_pkts)))
return -1;
- if (cargs->debug_packets)
- odp_packet_print_data(pkt, 0,
- odp_packet_len(pkt));
+ debug_packets(debug, pkt, num_pkts);
+ num_out = num_pkts;
- rc = odp_ipsec_out(&pkt, 1,
- &out_pkt, &num_out,
+ rc = odp_ipsec_out(pkt, num_pkts,
+ out_pkt, &num_out,
&param);
- if (rc <= 0) {
- ODPH_ERR("failed odp_ipsec_out: rc = %d\n", rc);
- odp_packet_free(pkt);
+ if (odp_unlikely(rc <= 0)) {
+ ODPH_ERR("Failed odp_ipsec_out: rc = %d\n", rc);
+ odp_packet_free_sp(pkt, num_pkts);
break;
}
- if (odp_packet_has_error(out_pkt)) {
- odp_ipsec_packet_result_t result;
- odp_ipsec_result(&result, out_pkt);
- ODPH_ERR("Received error packet: %d\n",
- result.status.error.all);
- }
+ for (i = 0; i < num_out; i++)
+ check_ipsec_result(out_pkt[i]);
+
packets_sent += rc;
packets_received += num_out;
- if (cargs->debug_packets)
- odp_packet_print_data(out_pkt, 0,
- odp_packet_len(out_pkt));
- odp_packet_free(out_pkt);
+ debug_packets(debug, out_pkt, num_out);
+
+ if (odp_unlikely(rc != num_pkts))
+ odp_packet_free_sp(&pkt[rc], num_pkts - rc);
+ odp_packet_free_sp(out_pkt, num_out);
}
}
@@ -686,6 +741,46 @@ run_measure_one(ipsec_args_t *cargs,
return rc < 0 ? rc : 0;
}
+static uint32_t dequeue_burst(odp_queue_t polled_queue,
+ odp_event_t *events,
+ int max_burst)
+{
+ int num = 0;
+
+ if (polled_queue != ODP_QUEUE_INVALID) {
+ int rc = odp_queue_deq_multi(polled_queue,
+ events,
+ max_burst);
+ num = odp_likely(rc >= 0) ? rc : 0;
+ } else {
+ num = odp_schedule_multi(NULL,
+ ODP_SCHED_NO_WAIT,
+ events,
+ max_burst);
+ }
+ return num;
+}
+
+static inline uint32_t vec_pkt_handle(int debug, odp_event_t ev)
+{
+ odp_packet_vector_t vec = odp_packet_vector_from_event(ev);
+ uint32_t vec_size = odp_packet_vector_size(vec);
+ odp_packet_t *pkt_tbl;
+ uint32_t j;
+
+ odp_packet_vector_tbl(vec, &pkt_tbl);
+
+ for (j = 0; j < vec_size; j++)
+ check_ipsec_result(pkt_tbl[j]);
+
+ debug_packets(debug, pkt_tbl, vec_size);
+
+ odp_packet_free_sp(pkt_tbl, vec_size);
+ odp_packet_vector_free(vec);
+
+ return vec_size;
+}
+
static int
run_measure_one_async(ipsec_args_t *cargs,
odp_ipsec_sa_t sa,
@@ -693,11 +788,14 @@ run_measure_one_async(ipsec_args_t *cargs,
time_record_t *start,
time_record_t *end)
{
+ int in_flight, packets_allowed, num_pkts, rc = 0;
+ const int max_in_flight = cargs->in_flight;
+ const int burst_size = cargs->burst_size;
+ const int packet_count = cargs->packet_count;
+ const int debug = cargs->debug_packets;
odp_ipsec_out_param_t param;
odp_pool_t pkt_pool;
- odp_queue_t out_queue;
- odp_packet_t pkt = ODP_PACKET_INVALID;
- int rc = 0;
+ odp_queue_t polled_queue = ODP_QUEUE_INVALID;
pkt_pool = odp_pool_lookup("packet_pool");
if (pkt_pool == ODP_POOL_INVALID) {
@@ -705,12 +803,17 @@ run_measure_one_async(ipsec_args_t *cargs,
return -1;
}
- out_queue = odp_queue_lookup("ipsec-out");
- if (out_queue == ODP_QUEUE_INVALID) {
- ODPH_ERR("ipsec-out queue not found\n");
- return -1;
+ if (cargs->poll) {
+ polled_queue = odp_queue_lookup("ipsec-out");
+ if (polled_queue == ODP_QUEUE_INVALID) {
+ ODPH_ERR("ipsec-out queue not found\n");
+ return -1;
+ }
}
+ if (payload_length < sizeof(test_data))
+ return -1;
+
int packets_sent = 0;
int packets_received = 0;
@@ -722,55 +825,78 @@ run_measure_one_async(ipsec_args_t *cargs,
fill_time_record(start);
- while ((packets_sent < cargs->iteration_count) ||
- (packets_received < cargs->iteration_count)) {
- odp_event_t ev;
+ while ((packets_sent < packet_count) ||
+ (packets_received < packet_count)) {
+
+ num_pkts = packet_count - packets_sent;
+
+ /* Enqueue up to burst size */
+ num_pkts = num_pkts > burst_size ? burst_size : num_pkts;
+
+ /* Enqueue up to (max in flight - current in flight) */
+ in_flight = packets_sent - packets_received;
+ packets_allowed = max_in_flight - in_flight;
- if ((packets_sent < cargs->iteration_count) &&
- (packets_sent - packets_received <
- cargs->in_flight)) {
- pkt = make_packet(pkt_pool, payload_length);
- if (ODP_PACKET_INVALID == pkt)
+ if (num_pkts > 0 && num_pkts <= packets_allowed) {
+ odp_packet_t pkt[num_pkts];
+
+ if (odp_unlikely(make_packet_multi(pkt_pool,
+ payload_length,
+ pkt,
+ num_pkts)))
return -1;
- if (cargs->debug_packets)
- odp_packet_print_data(pkt, 0,
- odp_packet_len(pkt));
+ debug_packets(debug, pkt, num_pkts);
- rc = odp_ipsec_out_enq(&pkt, 1,
- &param);
- if (rc <= 0) {
- ODPH_ERR("failed odp_crypto_packet_op_enq: rc = %d\n",
+ rc = odp_ipsec_out_enq(pkt, num_pkts, &param);
+ if (odp_unlikely(rc <= 0)) {
+ ODPH_ERR("Failed odp_ipsec_out_enq: rc = %d\n",
rc);
- odp_packet_free(pkt);
+ odp_packet_free_sp(pkt, num_pkts);
break;
}
+
+ if (odp_unlikely(rc != num_pkts))
+ odp_packet_free_sp(&pkt[rc], num_pkts - rc);
+
packets_sent += rc;
- }
+ } else {
+ odp_packet_t pkt_out[max_in_flight];
+ uint32_t i = 0;
+
+ /*
+ * Dequeue packets until we can enqueue the next burst
+ * or until we have received all remaining packets
+ * when there are no more packets to be sent.
+ */
+ while (num_pkts > packets_allowed ||
+ (num_pkts == 0 && packets_received < packet_count)) {
+ odp_event_t events[MAX_DEQUEUE_BURST];
+ uint32_t num;
+
+ num = dequeue_burst(polled_queue, events, MAX_DEQUEUE_BURST);
+
+ for (uint32_t n = 0; n < num; n++) {
+ if (odp_event_type(events[n]) == ODP_EVENT_PACKET_VECTOR) {
+ uint32_t vec_size;
+
+ vec_size = vec_pkt_handle(debug, events[n]);
+ packets_received += vec_size - 1;
+ packets_allowed += vec_size - 1;
+ } else {
+ pkt_out[i] = odp_ipsec_packet_from_event(events[n]);
+ check_ipsec_result(pkt_out[i]);
+ i++;
+ }
+
+ }
+ packets_received += num;
+ packets_allowed += num;
+ }
+ debug_packets(debug, pkt_out, i);
- if (cargs->schedule)
- ev = odp_schedule(NULL,
- ODP_SCHED_NO_WAIT);
- else
- ev = odp_queue_deq(out_queue);
-
- while (ev != ODP_EVENT_INVALID) {
- odp_packet_t out_pkt;
- odp_ipsec_packet_result_t result;
-
- out_pkt = odp_ipsec_packet_from_event(ev);
- odp_ipsec_result(&result, out_pkt);
-
- if (cargs->debug_packets)
- odp_packet_print_data(out_pkt, 0,
- odp_packet_len(out_pkt));
- odp_packet_free(out_pkt);
- packets_received++;
- if (cargs->schedule)
- ev = odp_schedule(NULL,
- ODP_SCHED_NO_WAIT);
- else
- ev = odp_queue_deq(out_queue);
+ if (i)
+ odp_packet_free_sp(pkt_out, i);
}
}
@@ -843,13 +969,13 @@ run_measure_one_config(ipsec_args_t *cargs,
break;
count = get_elapsed_usec(&start, &end);
- result.elapsed = count / cargs->iteration_count;
+ result.elapsed = count / cargs->packet_count;
count = get_rusage_self_diff(&start, &end);
- result.rusage_self = count / cargs->iteration_count;
+ result.rusage_self = count / cargs->packet_count;
count = get_rusage_thread_diff(&start, &end);
- result.rusage_thread = count / cargs->iteration_count;
+ result.rusage_thread = count / cargs->packet_count;
print_result(cargs, payloads[i],
config, &result);
@@ -913,7 +1039,9 @@ static void usage(char *progname)
print_config_names(" ");
printf(" -d, --debug Enable dump of processed packets.\n"
" -f, --flight <number> Max number of packet processed in parallel (default 1)\n"
- " -i, --iterations <number> Number of iterations.\n"
+ " -c, --count <number> Number of packets (default 10000)\n"
+ " -b, --burst <number> Number of packets in one IPsec API submission (default 1)\n"
+ " -v, --vector <number> Enable vector packet completion from IPsec APIs with specified vector size.\n"
" -l, --payload Payload length.\n"
" -s, --schedule Use scheduler for completion events.\n"
" -p, --poll Poll completion queue for completion events.\n"
@@ -932,7 +1060,9 @@ static void parse_args(int argc, char *argv[], ipsec_args_t *cargs)
{"debug", no_argument, NULL, 'd'},
{"flight", optional_argument, NULL, 'f'},
{"help", no_argument, NULL, 'h'},
- {"iterations", optional_argument, NULL, 'i'},
+ {"count", optional_argument, NULL, 'c'},
+ {"burst", optional_argument, NULL, 'b'},
+ {"vector", optional_argument, NULL, 'v'},
{"payload", optional_argument, NULL, 'l'},
{"sessions", optional_argument, NULL, 'm'},
{"poll", no_argument, NULL, 'p'},
@@ -942,11 +1072,13 @@ static void parse_args(int argc, char *argv[], ipsec_args_t *cargs)
{NULL, 0, NULL, 0}
};
- static const char *shortopts = "+a:c:df:hi:m:nl:sptu";
+ static const char *shortopts = "+a:b:c:df:hm:nl:sptuv:";
cargs->in_flight = 1;
cargs->debug_packets = 0;
- cargs->iteration_count = 10000;
+ cargs->packet_count = 10000;
+ cargs->burst_size = 1;
+ cargs->vec_pkt_size = 0;
cargs->payload_length = 0;
cargs->alg_config = NULL;
cargs->schedule = 0;
@@ -971,8 +1103,24 @@ static void parse_args(int argc, char *argv[], ipsec_args_t *cargs)
case 'd':
cargs->debug_packets = 1;
break;
- case 'i':
- cargs->iteration_count = atoi(optarg);
+ case 'c':
+ cargs->packet_count = atoi(optarg);
+ break;
+ case 'b':
+ if (optarg == NULL)
+ cargs->burst_size = 32;
+ else
+ cargs->burst_size = atoi(optarg);
+ if (cargs->burst_size > POOL_NUM_PKT) {
+ printf("Invalid burst size (max allowed: %d)\n", POOL_NUM_PKT);
+ exit(-1);
+ }
+ break;
+ case 'v':
+ if (optarg == NULL)
+ cargs->vec_pkt_size = 32;
+ else
+ cargs->vec_pkt_size = atoi(optarg);
break;
case 'f':
cargs->in_flight = atoi(optarg);
@@ -1001,6 +1149,11 @@ static void parse_args(int argc, char *argv[], ipsec_args_t *cargs)
}
}
+ if (cargs->in_flight < cargs->burst_size) {
+ printf("-f (flight) must be greater than or equal to -b (burst)\n");
+ exit(-1);
+ }
+
optind = 1; /* reset 'extern optind' from the getopt lib */
if (cargs->schedule && cargs->poll) {
@@ -1012,6 +1165,7 @@ static void parse_args(int argc, char *argv[], ipsec_args_t *cargs)
int main(int argc, char *argv[])
{
+ odp_pool_t vec_pool = ODP_POOL_INVALID;
ipsec_args_t cargs;
odp_pool_t pool;
odp_queue_param_t qparam;
@@ -1105,11 +1259,54 @@ int main(int argc, char *argv[])
exit(EXIT_FAILURE);
}
+ if (cargs.vec_pkt_size) {
+ if (capa.vector.max_pools < 1) {
+ ODPH_ERR("Vector packet pool not available");
+ exit(EXIT_FAILURE);
+ }
+
+ if (!ipsec_capa.vector.supported) {
+ ODPH_ERR("Vector packet completion not supported by IPsec.\n");
+ exit(EXIT_FAILURE);
+ }
+
+ if (capa.vector.max_size < cargs.vec_pkt_size) {
+ ODPH_ERR("Vector size larger than max size supported by vector pool.\n");
+ exit(EXIT_FAILURE);
+ }
+
+ if (!cargs.schedule && !cargs.poll) {
+ ODPH_ERR("Vector packet is not supported with sync APIs.\n");
+ exit(EXIT_FAILURE);
+ }
+
+ /* Create vector pool */
+ odp_pool_param_init(&param);
+ param.vector.num = POOL_NUM_PKT;
+ param.vector.max_size = cargs.vec_pkt_size;
+ param.type = ODP_POOL_VECTOR;
+ vec_pool = odp_pool_create("vector_pool", &param);
+
+ if (vec_pool == ODP_POOL_INVALID) {
+ ODPH_ERR("Vector packet pool create failed.\n");
+ exit(EXIT_FAILURE);
+ }
+
+ odp_pool_print(vec_pool);
+ }
+
odp_ipsec_config_init(&config);
config.max_num_sa = 2;
config.inbound.chksums.all_chksum = 0;
config.outbound.all_chksum = 0;
+ if (vec_pool != ODP_POOL_INVALID) {
+ config.vector.enable = true;
+ config.vector.pool = vec_pool;
+ config.vector.max_size = cargs.vec_pkt_size;
+ config.vector.max_tmo_ns = ipsec_capa.vector.max_tmo_ns;
+ }
+
odp_queue_param_init(&qparam);
if (cargs.schedule) {
odp_schedule_config(NULL);
@@ -1196,6 +1393,14 @@ int main(int argc, char *argv[])
if (cargs.schedule || cargs.poll)
odp_queue_destroy(out_queue);
+
+ if (cargs.vec_pkt_size) {
+ if (odp_pool_destroy(vec_pool)) {
+ ODPH_ERR("Error: vector pool destroy\n");
+ exit(EXIT_FAILURE);
+ }
+ }
+
if (odp_pool_destroy(pool)) {
ODPH_ERR("Error: pool destroy\n");
exit(EXIT_FAILURE);
diff --git a/test/performance/odp_ipsec_run.sh b/test/performance/odp_ipsec_run.sh
index 1974a7c55..2ddb48d07 100755
--- a/test/performance/odp_ipsec_run.sh
+++ b/test/performance/odp_ipsec_run.sh
@@ -7,9 +7,9 @@
TEST_DIR="${TEST_DIR:-$(dirname $0)}"
-# Run with a small number of iterations in make check
+# Run with a small number of packets in make check
-$TEST_DIR/odp_ipsec${EXEEXT} -i 100
+$TEST_DIR/odp_ipsec${EXEEXT} -c 100
if [ $? -ne 0 ] ; then
echo Test FAILED
diff --git a/test/performance/odp_pktio_ordered.c b/test/performance/odp_pktio_ordered.c
index 37a0899b6..130eb4688 100644
--- a/test/performance/odp_pktio_ordered.c
+++ b/test/performance/odp_pktio_ordered.c
@@ -1258,7 +1258,7 @@ int main(int argc, char *argv[])
exit(EXIT_FAILURE);
}
- if ((unsigned)gbl_args->appl.num_flows > capa.max_output_queues)
+ if ((uint32_t)gbl_args->appl.num_flows > capa.max_output_queues)
gbl_args->appl.num_flows = capa.max_output_queues;
}
diff --git a/test/performance/odp_random.c b/test/performance/odp_random.c
index 622a58c65..46134ac0c 100644
--- a/test/performance/odp_random.c
+++ b/test/performance/odp_random.c
@@ -14,20 +14,52 @@
#include <odp_api.h>
#include <odp/helper/odph_api.h>
+#define PSEUDO_RANDOM (-1)
+
#define MB (1024ull * 1024ull)
+typedef struct test_global_t test_global_t;
+
+typedef struct thread_arg_t {
+ test_global_t *global;
+ int thread_idx;
+ uint8_t *data;
+
+} thread_arg_t;
+
+struct test_global_t {
+ odp_barrier_t barrier;
+ odp_random_kind_t type;
+ uint8_t *data;
+ uint32_t rounds;
+
+ thread_arg_t thread_arg[ODP_THREAD_COUNT_MAX];
+
+ struct {
+ uint64_t nsec[ODP_THREAD_COUNT_MAX];
+ uint64_t sum[ODP_THREAD_COUNT_MAX];
+ uint64_t min[ODP_THREAD_COUNT_MAX];
+ uint64_t max[ODP_THREAD_COUNT_MAX];
+ } stat;
+};
+
/* Command line options */
typedef struct {
+ int mode;
int num_threads;
uint32_t size;
uint32_t rounds;
+ uint64_t delay;
+
} options_t;
static options_t options;
static const options_t options_def = {
+ .mode = 0,
.num_threads = 1,
.size = 256,
.rounds = 100000,
+ .delay = 0,
};
static void print_usage(void)
@@ -37,13 +69,18 @@ static void print_usage(void)
"\n"
"Usage: odp_random [options]\n"
"\n"
+ " -m, --mode Test mode select (default: 0):\n"
+ " 0: Data throughput\n"
+ " 1: Data generation latency (size: 8B by default)\n"
" -t, --threads Number of worker threads (default %u)\n"
" -s, --size Size of buffer in bytes (default %u)\n"
" -r, --rounds Number of test rounds (default %u)\n"
" Divided by 100 for ODP_RANDOM_TRUE\n"
+ " -d, --delay Delay (nsec) between buffer fills (default %" PRIu64 ").\n"
+ " Affects only latency mode.\n"
" -h, --help This help\n"
"\n",
- options_def.num_threads, options_def.size, options_def.rounds);
+ options_def.num_threads, options_def.size, options_def.rounds, options_def.delay);
}
static int parse_options(int argc, char *argv[])
@@ -53,16 +90,19 @@ static int parse_options(int argc, char *argv[])
int ret = 0;
static const struct option longopts[] = {
+ { "mode", required_argument, NULL, 'm' },
{ "threads", required_argument, NULL, 't' },
{ "size", required_argument, NULL, 's' },
{ "rounds", required_argument, NULL, 'r' },
+ { "delay", required_argument, NULL, 'd' },
{ "help", no_argument, NULL, 'h' },
{ NULL, 0, NULL, 0 }
};
- static const char *shortopts = "+t:s:r:h";
+ static const char *shortopts = "+m:t:s:r:d:h";
options = options_def;
+ options.size = 0;
while (1) {
opt = getopt_long(argc, argv, shortopts, longopts, &long_index);
@@ -71,6 +111,9 @@ static int parse_options(int argc, char *argv[])
break;
switch (opt) {
+ case 'm':
+ options.mode = atoi(optarg);
+ break;
case 't':
options.num_threads = atol(optarg);
break;
@@ -80,6 +123,9 @@ static int parse_options(int argc, char *argv[])
case 'r':
options.rounds = atol(optarg);
break;
+ case 'd':
+ options.delay = atol(optarg);
+ break;
case 'h':
/* fall through */
default:
@@ -89,170 +135,216 @@ static int parse_options(int argc, char *argv[])
}
}
- if (options.size < 1) {
- ODPH_ERR("Invalid size: %" PRIu32 "\n", options.size);
+ if (options.num_threads < 1 || options.num_threads > ODP_THREAD_COUNT_MAX) {
+ ODPH_ERR("Bad number of threads: %i\n", options.num_threads);
return -1;
}
+ if (options.size == 0) {
+ options.size = options_def.size;
+
+ if (options.mode)
+ options.size = 8;
+ }
+
+ printf("\nOptions:\n");
+ printf("------------------------\n");
+ printf(" mode: %i\n", options.mode);
+ printf(" threads: %i\n", options.num_threads);
+ printf(" size: %u\n", options.size);
+ printf(" rounds: %u\n", options.rounds);
+ printf(" delay: %" PRIu64 "\n", options.delay);
+ printf("\n");
+
return ret;
}
-const char *shm_name = "odp_random_test";
+static inline void random_data_loop(odp_random_kind_t type, uint32_t rounds,
+ uint8_t *data, uint32_t size)
+{
+ uint32_t i;
+ int32_t ret;
-typedef struct test_shm_t {
- odp_barrier_t barrier;
- odp_random_kind_t type;
- uint64_t nsec[ODP_THREAD_COUNT_MAX];
-} test_shm_t;
+ if ((int)type == PSEUDO_RANDOM) {
+ uint64_t seed = 0;
-static test_shm_t *shm_lookup(void)
-{
- test_shm_t *shm = NULL;
- odp_shm_t shm_hdl = odp_shm_lookup(shm_name);
+ for (i = 0; i < rounds; i++) {
+ uint32_t pos = 0;
- if (shm_hdl != ODP_SHM_INVALID)
- shm = (test_shm_t *)odp_shm_addr(shm_hdl);
+ while (pos < size) {
+ ret = odp_random_test_data(data + pos, size - pos, &seed);
- return shm;
-}
+ if (ret < 0) {
+ ODPH_ERR("odp_random_test_data() failed\n");
+ exit(EXIT_FAILURE);
+ }
-static uint32_t type_rounds(odp_random_kind_t type)
-{
- switch (type) {
- case ODP_RANDOM_TRUE:
- return options.rounds / 100;
- default:
- return options.rounds;
+ pos += ret;
+ }
+ }
+ } else {
+ for (i = 0; i < rounds; i++) {
+ uint32_t pos = 0;
+
+ while (pos < size) {
+ ret = odp_random_data(data + pos, size - pos, type);
+
+ if (ret < 0) {
+ ODPH_ERR("odp_random_data() failed\n");
+ exit(EXIT_FAILURE);
+ }
+
+ pos += ret;
+ }
+ }
}
}
-static int test_random(void *p)
+static int test_random_perf(void *ptr)
{
- (void)p;
-
- uint8_t *buf, *data;
- const unsigned long page = ODP_PAGE_SIZE;
odp_time_t start;
uint64_t nsec;
- uint32_t rounds;
- test_shm_t *shm = shm_lookup();
+ thread_arg_t *thread_arg = ptr;
+ test_global_t *global = thread_arg->global;
+ odp_random_kind_t type = global->type;
+ int thread_idx = thread_arg->thread_idx;
+ uint8_t *data = thread_arg->data;
+ uint32_t size = options.size;
+ uint32_t rounds = global->rounds;
- if (!shm) {
- ODPH_ERR("Failed to look up shm %s\n", shm_name);
- exit(EXIT_FAILURE);
- }
+ /* One warm up round */
+ random_data_loop(type, 1, data, size);
- rounds = type_rounds(shm->type);
+ odp_barrier_wait(&global->barrier);
- /* One extra page for alignment. */
- buf = (uint8_t *)malloc(options.size + page);
-
- if (!buf) {
- ODPH_ERR("Memory allocation failed.\n");
- exit(EXIT_FAILURE);
- }
-
- /* Align to start of page. */
- data = (uint8_t *)(((uintptr_t)buf + (page - 1)) & ~(page - 1));
-
- odp_barrier_wait(&shm->barrier);
+ /* Test run */
start = odp_time_local();
- for (uint32_t i = 0; i < rounds; i++) {
- uint32_t pos = 0;
-
- while (pos < options.size) {
- int32_t n = odp_random_data(data + pos,
- options.size - pos,
- shm->type);
-
- if (n < 0) {
- ODPH_ERR("odp_random_data() failed\n");
- exit(EXIT_FAILURE);
- }
-
- pos += n;
- }
- }
+ random_data_loop(type, rounds, data, size);
nsec = odp_time_diff_ns(odp_time_local(), start);
- shm->nsec[odp_thread_id()] = nsec;
- free(buf);
+
+ global->stat.nsec[thread_idx] = nsec;
return 0;
}
-static int test_random_test(void *p)
+static inline void random_data_latency(test_global_t *global, int thread_idx,
+ uint32_t rounds, uint8_t *data, uint32_t size)
{
- (void)p;
-
- uint8_t *buf, *data;
- const unsigned long page = ODP_PAGE_SIZE;
- odp_time_t start;
+ uint32_t i;
+ int32_t ret;
+ odp_time_t t1, t2, start;
uint64_t nsec;
+ odp_random_kind_t type = global->type;
+ uint64_t delay = options.delay;
+ uint64_t min = UINT64_MAX;
+ uint64_t max = 0;
+ uint64_t sum = 0;
uint64_t seed = 0;
- uint32_t rounds;
- test_shm_t *shm = shm_lookup();
-
- if (!shm) {
- ODPH_ERR("Failed to look up shm %s\n", shm_name);
- exit(EXIT_FAILURE);
- }
-
- rounds = type_rounds(shm->type);
-
- /* One extra page for alignment. */
- buf = (uint8_t *)malloc(options.size + page);
-
- if (!buf) {
- ODPH_ERR("Memory allocation failed.\n");
- exit(EXIT_FAILURE);
- }
-
- /* Align to start of page. */
- data = (uint8_t *)(((uintptr_t)buf + (page - 1)) & ~(page - 1));
- odp_barrier_wait(&shm->barrier);
start = odp_time_local();
- for (uint32_t i = 0; i < rounds; i++) {
+ for (i = 0; i < rounds; i++) {
uint32_t pos = 0;
- while (pos < options.size) {
- int32_t n = odp_random_test_data(data + pos,
- options.size - pos,
- &seed);
+ if (delay)
+ odp_time_wait_ns(delay);
- if (n < 0) {
- ODPH_ERR("odp_random_data() failed\n");
- exit(EXIT_FAILURE);
- }
+ if ((int)type == PSEUDO_RANDOM) {
+ t1 = odp_time_local_strict();
+ while (pos < size) {
+ ret = odp_random_test_data(data + pos, size - pos, &seed);
+
+ if (ret < 0) {
+ ODPH_ERR("odp_random_test_data() failed\n");
+ exit(EXIT_FAILURE);
+ }
- pos += n;
+ pos += ret;
+ }
+ t2 = odp_time_local_strict();
+ } else {
+ t1 = odp_time_local_strict();
+ while (pos < size) {
+ ret = odp_random_data(data + pos, size - pos, type);
+
+ if (ret < 0) {
+ ODPH_ERR("odp_random_data() failed\n");
+ exit(EXIT_FAILURE);
+ }
+
+ pos += ret;
+ }
+ t2 = odp_time_local_strict();
}
+
+ nsec = odp_time_diff_ns(t2, t1);
+ sum += nsec;
+
+ if (nsec > max)
+ max = nsec;
+ if (nsec < min)
+ min = nsec;
}
nsec = odp_time_diff_ns(odp_time_local(), start);
- shm->nsec[odp_thread_id()] = nsec;
- free(buf);
+
+ global->stat.nsec[thread_idx] = nsec;
+ global->stat.sum[thread_idx] = sum;
+ global->stat.min[thread_idx] = min;
+ global->stat.max[thread_idx] = max;
+}
+
+static int test_random_latency(void *ptr)
+{
+ thread_arg_t *thread_arg = ptr;
+ test_global_t *global = thread_arg->global;
+ odp_random_kind_t type = global->type;
+ int thread_idx = thread_arg->thread_idx;
+ uint8_t *data = thread_arg->data;
+ uint32_t size = options.size;
+ uint32_t rounds = global->rounds;
+
+ /* One warm up round */
+ random_data_loop(type, 1, data, size);
+
+ odp_barrier_wait(&global->barrier);
+
+ /* Test run */
+ random_data_latency(global, thread_idx, rounds, data, size);
return 0;
}
-static void test_type(odp_instance_t instance, test_shm_t *shm,
- odp_random_kind_t type)
+static uint32_t type_rounds(odp_random_kind_t type)
+{
+ switch (type) {
+ case ODP_RANDOM_TRUE:
+ return options.rounds / 100;
+ default:
+ return options.rounds;
+ }
+}
+
+static void test_type(odp_instance_t instance, test_global_t *global, odp_random_kind_t type)
{
- memset(shm, 0, sizeof(test_shm_t));
- shm->type = type;
- odp_barrier_init(&shm->barrier, options.num_threads);
+ int i;
+ int num_threads = options.num_threads;
+ uint32_t rounds = type_rounds(type);
+ uint32_t size = options.size;
+
+ memset(&global->stat, 0, sizeof(global->stat));
+ global->type = type;
+ global->rounds = rounds;
+ odp_barrier_init(&global->barrier, num_threads);
odp_cpumask_t cpumask;
odph_thread_common_param_t thr_common;
- odph_thread_param_t thr_param;
- odph_thread_t thr_worker[options.num_threads];
+ odph_thread_param_t thr_param[num_threads];
+ odph_thread_t thr_worker[num_threads];
- if (odp_cpumask_default_worker(&cpumask, options.num_threads) !=
- options.num_threads) {
+ if (odp_cpumask_default_worker(&cpumask, num_threads) != num_threads) {
ODPH_ERR("Failed to get default CPU mask.\n");
exit(EXIT_FAILURE);
}
@@ -260,35 +352,36 @@ static void test_type(odp_instance_t instance, test_shm_t *shm,
odph_thread_common_param_init(&thr_common);
thr_common.instance = instance;
thr_common.cpumask = &cpumask;
- thr_common.share_param = 1;
- odph_thread_param_init(&thr_param);
- thr_param.thr_type = ODP_THREAD_WORKER;
- thr_param.start = test_random;
+ for (i = 0; i < num_threads; i++) {
+ odph_thread_param_init(&thr_param[i]);
+ thr_param[i].thr_type = ODP_THREAD_WORKER;
+ thr_param[i].arg = &global->thread_arg[i];
- if (type == (odp_random_kind_t)-1)
- thr_param.start = test_random_test;
+ if (options.mode == 0)
+ thr_param[i].start = test_random_perf;
+ else
+ thr_param[i].start = test_random_latency;
+ }
memset(&thr_worker, 0, sizeof(thr_worker));
- if (odph_thread_create(thr_worker, &thr_common, &thr_param,
- options.num_threads) != options.num_threads) {
+ if (odph_thread_create(thr_worker, &thr_common, thr_param, num_threads) != num_threads) {
ODPH_ERR("Failed to create worker threads.\n");
exit(EXIT_FAILURE);
}
- if (odph_thread_join(thr_worker, options.num_threads) !=
- options.num_threads) {
+ if (odph_thread_join(thr_worker, num_threads) != num_threads) {
ODPH_ERR("Failed to join worker threads.\n");
exit(EXIT_FAILURE);
}
double mb, seconds, nsec = 0;
- for (int i = 0; i < ODP_THREAD_COUNT_MAX; i++)
- nsec += shm->nsec[i];
+ for (i = 0; i < num_threads; i++)
+ nsec += global->stat.nsec[i];
- nsec /= options.num_threads;
+ nsec /= num_threads;
switch (type) {
case ODP_RANDOM_BASIC:
@@ -304,25 +397,53 @@ static void test_type(odp_instance_t instance, test_shm_t *shm,
printf("odp_random_test_data\n");
}
- uint32_t rounds = type_rounds(type);
-
printf("--------------------\n");
- printf("threads: %d size: %u B rounds: %u ", options.num_threads,
- options.size, rounds);
- mb = (uint64_t)options.num_threads * (uint64_t)options.size *
- (uint64_t)rounds;
+ printf("threads: %d size: %u B rounds: %u ", num_threads, size, rounds);
+ mb = (uint64_t)num_threads * (uint64_t)size * (uint64_t)rounds;
mb /= MB;
seconds = (double)nsec / (double)ODP_TIME_SEC_IN_NS;
printf("MB: %.3f seconds: %.3f ", mb, seconds);
printf("MB/s: %.3f ", mb / seconds);
- printf("MB/s/thread: %.3f", mb / seconds / (double)options.num_threads);
- printf("\n\n");
+ printf("MB/s/thread: %.3f\n", mb / seconds / (double)num_threads);
+
+ if (options.mode) {
+ double ave;
+ uint64_t min = UINT64_MAX;
+ uint64_t max = 0;
+ uint64_t sum = 0;
+
+ printf(" latency (nsec)\n");
+ printf(" thread min max ave\n");
+ for (i = 0; i < num_threads; i++) {
+ ave = (double)global->stat.sum[i] / rounds;
+ sum += global->stat.sum[i];
+
+ if (global->stat.min[i] < min)
+ min = global->stat.min[i];
+
+ if (global->stat.max[i] > max)
+ max = global->stat.max[i];
+
+ printf("%8i %8" PRIu64 " %8" PRIu64 " %10.1f\n", i, global->stat.min[i],
+ global->stat.max[i], ave);
+ }
+
+ printf(" all %8" PRIu64 " %8" PRIu64 " %10.1f\n",
+ min, max, ((double)sum / rounds) / num_threads);
+ }
+
+ printf("\n");
}
int main(int argc, char **argv)
{
odp_instance_t instance;
odp_init_t init;
+ odp_shm_t shm_glb, shm_data;
+ test_global_t *global;
+ int num_threads, i;
+ uint64_t tot_size, size;
+ uint8_t *addr;
if (parse_options(argc, argv))
exit(EXIT_FAILURE);
@@ -352,31 +473,59 @@ int main(int argc, char **argv)
odp_sys_info_print();
- test_shm_t *shm = NULL;
- odp_shm_t shm_hdl = odp_shm_reserve(shm_name, sizeof(test_shm_t), 64,
- 0);
+ global = NULL;
+ shm_glb = odp_shm_reserve("test_globals", sizeof(test_global_t), ODP_CACHE_LINE_SIZE, 0);
+
+ if (shm_glb != ODP_SHM_INVALID)
+ global = (test_global_t *)odp_shm_addr(shm_glb);
+
+ if (!global) {
+ ODPH_ERR("Failed to reserve shm\n");
+ exit(EXIT_FAILURE);
+ }
+
+ memset(global, 0, sizeof(test_global_t));
+
+ num_threads = options.num_threads;
+ addr = NULL;
+ size = ODP_CACHE_LINE_SIZE + ODP_CACHE_LINE_ROUNDUP(options.size);
+ tot_size = num_threads * size;
+ shm_data = odp_shm_reserve("test_data", tot_size, ODP_CACHE_LINE_SIZE, 0);
- if (shm_hdl != ODP_SHM_INVALID)
- shm = (test_shm_t *)odp_shm_addr(shm_hdl);
+ if (shm_data != ODP_SHM_INVALID)
+ addr = odp_shm_addr(shm_data);
- if (!shm) {
- ODPH_ERR("Failed to reserve shm %s\n", shm_name);
+ if (!addr) {
+ ODPH_ERR("Failed to reserve shm: size %" PRIu64 " bytes\n", tot_size);
exit(EXIT_FAILURE);
}
+ for (i = 0; i < num_threads; i++) {
+ global->thread_arg[i].global = global;
+ global->thread_arg[i].thread_idx = i;
+ global->thread_arg[i].data = addr + i * size;
+ }
+
+ odp_shm_print_all();
+
switch (odp_random_max_kind()) {
case ODP_RANDOM_TRUE:
- test_type(instance, shm, ODP_RANDOM_TRUE);
+ test_type(instance, global, ODP_RANDOM_TRUE);
/* fall through */
case ODP_RANDOM_CRYPTO:
- test_type(instance, shm, ODP_RANDOM_CRYPTO);
+ test_type(instance, global, ODP_RANDOM_CRYPTO);
/* fall through */
default:
- test_type(instance, shm, ODP_RANDOM_BASIC);
- test_type(instance, shm, -1);
+ test_type(instance, global, ODP_RANDOM_BASIC);
+ test_type(instance, global, PSEUDO_RANDOM);
+ }
+
+ if (odp_shm_free(shm_data)) {
+ ODPH_ERR("odp_shm_free() failed\n");
+ exit(EXIT_FAILURE);
}
- if (odp_shm_free(shm_hdl)) {
+ if (odp_shm_free(shm_glb)) {
ODPH_ERR("odp_shm_free() failed\n");
exit(EXIT_FAILURE);
}
diff --git a/test/performance/odp_sched_pktio.c b/test/performance/odp_sched_pktio.c
index 77c1d260f..11be9d367 100644
--- a/test/performance/odp_sched_pktio.c
+++ b/test/performance/odp_sched_pktio.c
@@ -862,10 +862,9 @@ static int open_pktios(test_global_t *test_global)
odp_pktin_queue_param_t pktin_param;
odp_pktout_queue_param_t pktout_param;
odp_schedule_sync_t sched_sync;
- unsigned int num_queue;
+ uint32_t num_queue, j;
char *name;
int i, num_pktio, ret;
- unsigned int j;
num_pktio = test_global->opt.num_pktio;
num_queue = test_global->opt.num_pktio_queue;
diff --git a/test/validation/api/classification/classification.h b/test/validation/api/classification/classification.h
index 4c8613555..70dcc6230 100644
--- a/test/validation/api/classification/classification.h
+++ b/test/validation/api/classification/classification.h
@@ -23,6 +23,11 @@
#define CLS_DEFAULT_SMAC {0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c}
#define CLS_MAGIC_VAL 0xdeadbeef
+/* Config values for Drop CoS */
+#define TEST_DROP 1
+#define CLS_DROP 6
+#define CLS_DROP_PORT 4001
+
/* Config values for Error CoS */
#define TEST_ERROR 1
#define CLS_ERROR 1
@@ -47,7 +52,7 @@
/* Config values for CoS L2 Priority */
#define TEST_L2_QOS 1
-#define CLS_L2_QOS_0 6
+#define CLS_L2_QOS_0 7
#define CLS_L2_QOS_MAX 5
#define CLS_ENTRIES (CLS_L2_QOS_0 + CLS_L2_QOS_MAX)
diff --git a/test/validation/api/classification/odp_classification_basic.c b/test/validation/api/classification/odp_classification_basic.c
index 3aeb89462..fb5ec4ed0 100644
--- a/test/validation/api/classification/odp_classification_basic.c
+++ b/test/validation/api/classification/odp_classification_basic.c
@@ -52,6 +52,49 @@ static void classification_test_create_cos(void)
odp_queue_destroy(queue);
}
+static void classification_test_create_cos_max_common(odp_bool_t stats)
+{
+ uint32_t i;
+ odp_cls_cos_param_t cls_param;
+ odp_cls_capability_t capa;
+
+ CU_ASSERT_FATAL(odp_cls_capability(&capa) == 0);
+
+ uint32_t num = capa.max_cos;
+
+ if (stats && capa.max_cos_stats < num)
+ num = capa.max_cos_stats;
+
+ odp_cos_t cos[num];
+
+ for (i = 0; i < num; i++) {
+ odp_cls_cos_param_init(&cls_param);
+ cls_param.action = ODP_COS_ACTION_DROP;
+ cls_param.stats_enable = stats;
+
+ cos[i] = odp_cls_cos_create(NULL, &cls_param);
+ if (cos[i] == ODP_COS_INVALID) {
+ ODPH_ERR("odp_cls_cos_create() failed at CoS %u out of %u.\n", i + 1, num);
+ break;
+ }
+ }
+
+ CU_ASSERT(i == num);
+
+ for (uint32_t j = 0; j < i; j++)
+ CU_ASSERT(!odp_cos_destroy(cos[j]));
+}
+
+static void classification_test_create_cos_max(void)
+{
+ classification_test_create_cos_max_common(false);
+}
+
+static void classification_test_create_cos_max_stats(void)
+{
+ classification_test_create_cos_max_common(true);
+}
+
static void classification_test_destroy_cos(void)
{
odp_cos_t cos;
@@ -345,6 +388,8 @@ static void classification_test_pmr_composite_create(void)
odp_testinfo_t classification_suite_basic[] = {
ODP_TEST_INFO(classification_test_default_values),
ODP_TEST_INFO(classification_test_create_cos),
+ ODP_TEST_INFO(classification_test_create_cos_max),
+ ODP_TEST_INFO(classification_test_create_cos_max_stats),
ODP_TEST_INFO(classification_test_destroy_cos),
ODP_TEST_INFO(classification_test_create_pmr_match),
ODP_TEST_INFO(classification_test_cos_set_queue),
diff --git a/test/validation/api/classification/odp_classification_tests.c b/test/validation/api/classification/odp_classification_tests.c
index 41b4ab02f..4511fc1d7 100644
--- a/test/validation/api/classification/odp_classification_tests.c
+++ b/test/validation/api/classification/odp_classification_tests.c
@@ -23,6 +23,7 @@ static int global_num_l2_qos;
#define NUM_COS_PMR_CHAIN 2
#define NUM_COS_DEFAULT 1
+#define NUM_COS_DROP 1
#define NUM_COS_ERROR 1
#define NUM_COS_L2_PRIO CLS_L2_QOS_MAX
#define NUM_COS_PMR 1
@@ -453,6 +454,74 @@ void test_pktio_default_cos(odp_bool_t enable_pktv)
odp_packet_free(pkt);
}
+void configure_pktio_drop_cos(odp_bool_t enable_pktv, uint32_t max_cos_stats)
+{
+ uint16_t val;
+ uint16_t mask;
+ odp_pmr_param_t pmr_param;
+ odp_cls_cos_param_t cls_param;
+ char cosname[ODP_COS_NAME_LEN];
+
+ sprintf(cosname, "DropCoS");
+ odp_cls_cos_param_init(&cls_param);
+
+ cls_param.action = ODP_COS_ACTION_DROP;
+ cls_param.stats_enable = max_cos_stats > 0;
+
+ if (enable_pktv) {
+ cls_param.vector.enable = true;
+ cls_param.vector.pool = pktv_config.pool;
+ cls_param.vector.max_size = pktv_config.max_size;
+ cls_param.vector.max_tmo_ns = pktv_config.max_tmo_ns;
+ }
+
+ cos_list[CLS_DROP] = odp_cls_cos_create(cosname, &cls_param);
+ CU_ASSERT_FATAL(cos_list[CLS_DROP] != ODP_COS_INVALID);
+
+ val = odp_cpu_to_be_16(CLS_DROP_PORT);
+ mask = odp_cpu_to_be_16(0xffff);
+ odp_cls_pmr_param_init(&pmr_param);
+ pmr_param.term = find_first_supported_l3_pmr();
+ pmr_param.match.value = &val;
+ pmr_param.match.mask = &mask;
+ pmr_param.val_sz = sizeof(val);
+
+ pmr_list[CLS_DROP] = odp_cls_pmr_create(&pmr_param, 1,
+ cos_list[CLS_DEFAULT],
+ cos_list[CLS_DROP]);
+ CU_ASSERT_FATAL(pmr_list[CLS_DROP] != ODP_PMR_INVALID);
+}
+
+void test_pktio_drop_cos(odp_bool_t enable_pktv)
+{
+ odp_packet_t pkt;
+ odp_queue_t queue;
+ uint32_t seqno = 0;
+ cls_packet_info_t pkt_info;
+ odp_cls_capability_t capa;
+ odp_cls_cos_stats_t start, stop;
+
+ CU_ASSERT_FATAL(odp_cls_capability(&capa) == 0);
+ pkt_info = default_pkt_info;
+ pkt_info.l4_type = CLS_PKT_L4_UDP;
+ pkt = create_packet(pkt_info);
+ CU_ASSERT_FATAL(pkt != ODP_PACKET_INVALID);
+ seqno = cls_pkt_get_seq(pkt);
+ CU_ASSERT(seqno != TEST_SEQ_INVALID);
+ set_first_supported_pmr_port(pkt, CLS_DROP_PORT);
+ CU_ASSERT(odp_cls_cos_stats(cos_list[CLS_DROP], &start) == 0);
+ enqueue_pktio_interface(pkt, pktio_loop);
+ pkt = receive_packet(&queue, ODP_TIME_SEC_IN_NS, enable_pktv);
+ CU_ASSERT(odp_cls_cos_stats(cos_list[CLS_DROP], &stop) == 0);
+ CU_ASSERT_FATAL(pkt == ODP_PACKET_INVALID);
+ if (capa.stats.cos.counter.packets)
+ CU_ASSERT((stop.packets - start.packets) == 1);
+ if (capa.stats.cos.counter.discards)
+ CU_ASSERT((stop.discards - start.discards) == 0);
+ if (capa.stats.cos.counter.errors)
+ CU_ASSERT((stop.errors - start.errors) == 0);
+}
+
static int classification_check_queue_stats(void)
{
odp_cls_capability_t capa;
@@ -926,6 +995,11 @@ static void classification_test_pktio_configure_common(odp_bool_t enable_pktv)
tc.default_cos = 1;
num_cos -= NUM_COS_DEFAULT;
}
+ if (num_cos >= NUM_COS_DEFAULT && TEST_DROP) {
+ configure_pktio_drop_cos(enable_pktv, capa.max_cos_stats);
+ tc.drop_cos = 1;
+ num_cos -= NUM_COS_DROP;
+ }
if (num_cos >= NUM_COS_ERROR && TEST_ERROR) {
configure_pktio_error_cos(enable_pktv);
tc.error_cos = 1;
@@ -969,6 +1043,8 @@ static void classification_test_pktio_test_common(odp_bool_t enable_pktv)
/* Test Different CoS on the pktio interface */
if (tc.default_cos && TEST_DEFAULT)
test_pktio_default_cos(enable_pktv);
+ if (tc.drop_cos && TEST_DROP)
+ test_pktio_drop_cos(enable_pktv);
if (tc.error_cos && TEST_ERROR)
test_pktio_error_cos(enable_pktv);
if (tc.pmr_chain && TEST_PMR_CHAIN)
diff --git a/test/validation/api/classification/odp_classification_testsuites.h b/test/validation/api/classification/odp_classification_testsuites.h
index 2b5da94e2..6b00e138b 100644
--- a/test/validation/api/classification/odp_classification_testsuites.h
+++ b/test/validation/api/classification/odp_classification_testsuites.h
@@ -34,6 +34,7 @@ typedef struct cls_packet_info {
typedef union odp_cls_testcase {
struct {
uint32_t default_cos:1;
+ uint32_t drop_cos:1;
uint32_t error_cos:1;
uint32_t pmr_chain:1;
uint32_t l2_priority:1;
@@ -72,6 +73,8 @@ odp_pool_t pktv_pool_create(const char *poolname);
odp_queue_t queue_create(const char *queuename, bool sched);
void configure_pktio_default_cos(odp_bool_t enable_pktv);
void test_pktio_default_cos(odp_bool_t enable_pktv);
+void configure_pktio_drop_cos(odp_bool_t enable_pktv, uint32_t max_cos_stats);
+void test_pktio_drop_cos(odp_bool_t enable_pktv);
void configure_pktio_error_cos(odp_bool_t enable_pktv);
void test_pktio_error_cos(odp_bool_t enable_pktv);
void configure_cls_pmr_chain(odp_bool_t enable_pktv);
diff --git a/test/validation/api/crypto/odp_crypto_test_inp.c b/test/validation/api/crypto/odp_crypto_test_inp.c
index 721dfe8c9..97f721dd5 100644
--- a/test/validation/api/crypto/odp_crypto_test_inp.c
+++ b/test/validation/api/crypto/odp_crypto_test_inp.c
@@ -37,7 +37,9 @@ static void test_default_values(void)
CU_ASSERT_EQUAL(param.op, ODP_CRYPTO_OP_ENCODE);
CU_ASSERT_EQUAL(param.auth_cipher_text, false);
+#if ODP_DEPRECATED_API
CU_ASSERT_EQUAL(param.pref_mode, ODP_CRYPTO_SYNC);
+#endif
CU_ASSERT_EQUAL(param.op_mode, ODP_CRYPTO_SYNC);
CU_ASSERT_EQUAL(param.cipher_alg, ODP_CIPHER_ALG_NULL);
CU_ASSERT_EQUAL(param.cipher_iv_len, 0);
@@ -194,6 +196,7 @@ static const char *cipher_alg_name(odp_cipher_alg_t cipher)
}
}
+#if ODP_DEPRECATED_API
static int alg_op(odp_packet_t pkt,
odp_bool_t *ok,
odp_crypto_session_t session,
@@ -273,6 +276,7 @@ static int alg_op(odp_packet_t pkt,
return 0;
}
+#endif
static int alg_packet_op(odp_packet_t pkt,
odp_bool_t *ok,
@@ -370,10 +374,14 @@ static int crypto_op(odp_packet_t pkt,
int rc;
if (!suite_context.packet)
+#if ODP_DEPRECATED_API
rc = alg_op(pkt, ok, session,
cipher_iv, auth_iv,
cipher_range, auth_range,
aad, hash_result_offset);
+#else
+ rc = -1;
+#endif
else
rc = alg_packet_op(pkt, ok, session,
cipher_iv, auth_iv,
@@ -657,7 +665,9 @@ static odp_crypto_session_t session_create(odp_crypto_op_t op,
ses_params.op = op;
ses_params.auth_cipher_text = false;
ses_params.op_mode = suite_context.op_mode;
+#if ODP_DEPRECATED_API
ses_params.pref_mode = suite_context.pref_mode;
+#endif
ses_params.cipher_alg = cipher_alg;
ses_params.auth_alg = auth_alg;
ses_params.compl_queue = suite_context.queue;
@@ -2154,6 +2164,7 @@ static odp_event_t plain_compl_queue_deq(void)
return odp_queue_deq(suite_context.queue);
}
+#if ODP_DEPRECATED_API
static int crypto_suite_sync_init(void)
{
suite_context.pool = odp_pool_lookup("packet_pool");
@@ -2206,6 +2217,7 @@ static int crypto_suite_async_sched_init(void)
return 0;
}
+#endif
static int crypto_suite_packet_sync_init(void)
{
@@ -2422,12 +2434,14 @@ odp_testinfo_t crypto_suite[] = {
};
odp_suiteinfo_t crypto_suites[] = {
+#if ODP_DEPRECATED_API
{"odp_crypto_sync_inp", crypto_suite_sync_init,
NULL, crypto_suite},
{"odp_crypto_async_plain_inp", crypto_suite_async_plain_init,
crypto_suite_term, crypto_suite},
{"odp_crypto_async_sched_inp", crypto_suite_async_sched_init,
crypto_suite_term, crypto_suite},
+#endif
{"odp_crypto_packet_sync_inp", crypto_suite_packet_sync_init,
NULL, crypto_suite},
{"odp_crypto_packet_async_plain_inp",
diff --git a/test/validation/api/ipsec/ipsec.c b/test/validation/api/ipsec/ipsec.c
index 88f204ea4..d12234a5a 100644
--- a/test/validation/api/ipsec/ipsec.c
+++ b/test/validation/api/ipsec/ipsec.c
@@ -281,10 +281,6 @@ int ipsec_check_esp_chacha20_poly1305(void)
int ipsec_check_test_sa_update_seq_num(void)
{
- odp_ipsec_capability_t capa;
-
- odp_ipsec_capability(&capa);
-
if (!capa.test.sa_operations.seq_num)
return ODP_TEST_INACTIVE;
@@ -942,9 +938,8 @@ static void verify_in(const ipsec_test_part *part,
result.orig_ip_len == len);
}
}
- ipsec_check_packet(part->out[i].pkt_res,
- pkto[i],
- false);
+ if (part->out[i].l3_type != ODP_PROTO_L3_TYPE_NONE)
+ ipsec_check_packet(part->out[i].pkt_res, pkto[i], false);
if (suite_context.inbound_op_mode == ODP_IPSEC_OP_MODE_INLINE)
expected_user_ptr = NULL;
CU_ASSERT(odp_packet_user_ptr(pkto[i]) == expected_user_ptr);
diff --git a/test/validation/api/pktio/pktio.c b/test/validation/api/pktio/pktio.c
index 9144e1260..f6408c788 100644
--- a/test/validation/api/pktio/pktio.c
+++ b/test/validation/api/pktio/pktio.c
@@ -770,8 +770,8 @@ static int recv_packets_tmo(odp_pktio_t pktio, odp_packet_t pkt_tbl[],
int num_q;
int i;
int n;
- unsigned int from_val = 0;
- unsigned *from = NULL;
+ uint32_t from_val = 0;
+ uint32_t *from = NULL;
if (mode == RECV_MQ_TMO)
from = &from_val;
@@ -787,8 +787,7 @@ static int recv_packets_tmo(odp_pktio_t pktio, odp_packet_t pkt_tbl[],
n = odp_pktin_recv_tmo(pktin[0], pkt_tmp, num - num_rx,
tmo);
else
- n = odp_pktin_recv_mq_tmo(pktin, (unsigned)num_q,
- from, pkt_tmp,
+ n = odp_pktin_recv_mq_tmo(pktin, (uint32_t)num_q, from, pkt_tmp,
num - num_rx, tmo);
ts2 = odp_time_global();
@@ -812,7 +811,7 @@ static int recv_packets_tmo(odp_pktio_t pktio, odp_packet_t pkt_tbl[],
odp_packet_free(pkt_tmp[i]);
}
if (mode == RECV_MQ_TMO)
- CU_ASSERT(from_val < (unsigned)num_q);
+ CU_ASSERT(from_val < (uint32_t)num_q);
} while (num_rx < num);
if (num_rx < num) {
@@ -1265,7 +1264,7 @@ static void test_recv_tmo(recv_tmo_mode_e mode)
odp_packet_t pkt_tbl[test_pkt_count];
uint32_t pkt_seq[test_pkt_count];
uint64_t ns;
- unsigned num_q;
+ uint32_t num_q;
int ret;
int i;
diff --git a/test/validation/api/traffic_mngr/traffic_mngr.c b/test/validation/api/traffic_mngr/traffic_mngr.c
index cab1fd06d..116c9ed6d 100644
--- a/test/validation/api/traffic_mngr/traffic_mngr.c
+++ b/test/validation/api/traffic_mngr/traffic_mngr.c
@@ -120,6 +120,12 @@ typedef enum {
SHAPER_PROFILE, SCHED_PROFILE, THRESHOLD_PROFILE, WRED_PROFILE
} profile_kind_t;
+typedef enum {
+ THRESHOLD_BYTE,
+ THRESHOLD_PACKET,
+ THRESHOLD_BYTE_AND_PACKET
+} threshold_type_t;
+
typedef struct {
uint32_t num_queues;
odp_tm_queue_t tm_queues[];
@@ -516,6 +522,7 @@ static int open_pktios(void)
uint32_t iface;
char pool_name[ODP_POOL_NAME_LEN];
int rc, ret;
+ int pkt_aging = 0;
int lso = 0;
odp_pool_param_init(&pool_param);
@@ -620,6 +627,12 @@ static int open_pktios(void)
odp_pktio_config_init(&pktio_config);
+ /* Enable packet aging if supported */
+ if (xmt_pktio_capa.max_tx_aging_tmo_ns) {
+ pkt_aging = 1;
+ pktio_config.pktout.bit.aging_ena = 1;
+ }
+
/* Enable LSO if supported */
if (xmt_pktio_capa.lso.max_profiles && xmt_pktio_capa.lso.max_profiles_per_pktio) {
lso = 1;
@@ -627,7 +640,7 @@ static int open_pktios(void)
}
/* Enable selected features */
- if (lso) {
+ if (lso || pkt_aging) {
if (odp_pktio_config(xmt_pktio, &pktio_config)) {
ODPH_ERR("pktio configure failed\n");
return -1;
@@ -1818,11 +1831,15 @@ set_reqs_based_on_capas(odp_tm_requirements_t *req)
req->tm_queue_dual_slope_needed = true;
if (tm_capabilities.vlan_marking_supported)
req->vlan_marking_needed = true;
- if (tm_capabilities.tm_queue_threshold)
+ if (tm_capabilities.tm_queue_threshold.byte ||
+ tm_capabilities.tm_queue_threshold.packet ||
+ tm_capabilities.tm_queue_threshold.byte_and_packet)
req->tm_queue_threshold_needed = true;
for (j = 0; j < tm_capabilities.max_levels; j++) {
- if (tm_capabilities.per_level[j].tm_node_threshold)
+ if (tm_capabilities.per_level[j].tm_node_threshold.byte ||
+ tm_capabilities.per_level[j].tm_node_threshold.packet ||
+ tm_capabilities.per_level[j].tm_node_threshold.byte_and_packet)
req->per_level[j].tm_node_threshold_needed = true;
}
@@ -2638,7 +2655,8 @@ static void traffic_mngr_test_sched_profile(void)
}
static void check_threshold_profile(char *threshold_name,
- uint32_t threshold_idx)
+ uint32_t threshold_idx,
+ threshold_type_t threshold)
{
odp_tm_threshold_params_t threshold_params;
odp_tm_threshold_t profile;
@@ -2657,15 +2675,17 @@ static void check_threshold_profile(char *threshold_name,
if (ret)
return;
- CU_ASSERT(threshold_params.max_pkts ==
- threshold_idx * MIN_PKT_THRESHOLD);
- CU_ASSERT(threshold_params.max_bytes ==
- threshold_idx * MIN_BYTE_THRESHOLD);
- CU_ASSERT(threshold_params.enable_max_pkts == 1);
- CU_ASSERT(threshold_params.enable_max_bytes == 1);
+ if (threshold == THRESHOLD_PACKET || threshold == THRESHOLD_BYTE_AND_PACKET) {
+ CU_ASSERT(threshold_params.enable_max_pkts == 1);
+ CU_ASSERT(threshold_params.max_pkts == threshold_idx * MIN_PKT_THRESHOLD);
+ }
+ if (threshold == THRESHOLD_BYTE || threshold == THRESHOLD_BYTE_AND_PACKET) {
+ CU_ASSERT(threshold_params.enable_max_bytes == 1);
+ CU_ASSERT(threshold_params.max_bytes == threshold_idx * MIN_BYTE_THRESHOLD);
+ }
}
-static void traffic_mngr_test_threshold_profile(void)
+static void traffic_mngr_test_threshold_profile(threshold_type_t threshold)
{
odp_tm_threshold_params_t threshold_params;
odp_tm_threshold_t profile;
@@ -2673,14 +2693,19 @@ static void traffic_mngr_test_threshold_profile(void)
char threshold_name[TM_NAME_LEN];
odp_tm_threshold_params_init(&threshold_params);
- threshold_params.enable_max_pkts = 1;
- threshold_params.enable_max_bytes = 1;
+
+ if (threshold == THRESHOLD_PACKET || threshold == THRESHOLD_BYTE_AND_PACKET)
+ threshold_params.enable_max_pkts = 1;
+ if (threshold == THRESHOLD_BYTE || threshold == THRESHOLD_BYTE_AND_PACKET)
+ threshold_params.enable_max_bytes = 1;
for (idx = 1; idx <= NUM_THRESH_TEST_PROFILES; idx++) {
snprintf(threshold_name, sizeof(threshold_name),
"threshold_profile_%" PRIu32, idx);
- threshold_params.max_pkts = idx * MIN_PKT_THRESHOLD;
- threshold_params.max_bytes = idx * MIN_BYTE_THRESHOLD;
+ if (threshold == THRESHOLD_PACKET || threshold == THRESHOLD_BYTE_AND_PACKET)
+ threshold_params.max_pkts = idx * MIN_PKT_THRESHOLD;
+ if (threshold == THRESHOLD_BYTE || threshold == THRESHOLD_BYTE_AND_PACKET)
+ threshold_params.max_bytes = idx * MIN_BYTE_THRESHOLD;
profile = odp_tm_threshold_create(threshold_name,
&threshold_params);
@@ -2702,10 +2727,30 @@ static void traffic_mngr_test_threshold_profile(void)
threshold_idx = ((3 + 7 * idx) % NUM_THRESH_TEST_PROFILES) + 1;
snprintf(threshold_name, sizeof(threshold_name),
"threshold_profile_%" PRIu32, threshold_idx);
- check_threshold_profile(threshold_name, threshold_idx);
+ check_threshold_profile(threshold_name, threshold_idx, threshold);
+ }
+
+ for (i = 0; i < NUM_THRESH_TEST_PROFILES; i++) {
+ CU_ASSERT(odp_tm_threshold_destroy(threshold_profiles[i]) == 0);
+ num_threshold_profiles--;
}
}
+static void traffic_mngr_test_threshold_profile_byte(void)
+{
+ traffic_mngr_test_threshold_profile(THRESHOLD_BYTE);
+}
+
+static void traffic_mngr_test_threshold_profile_packet(void)
+{
+ traffic_mngr_test_threshold_profile(THRESHOLD_PACKET);
+}
+
+static void traffic_mngr_test_threshold_profile_byte_and_packet(void)
+{
+ traffic_mngr_test_threshold_profile(THRESHOLD_BYTE_AND_PACKET);
+}
+
static void check_wred_profile(char *wred_name,
uint32_t wred_idx,
uint32_t color)
@@ -3353,6 +3398,7 @@ static int test_sched_wfq(const char *sched_base_name,
if (FANIN_RATIO <= fanin)
fanin = 0;
}
+ CU_ASSERT(pkts_sent == pkt_cnt + 4);
busy_wait(1000000); /* wait 1 millisecond */
@@ -3433,7 +3479,8 @@ static int test_threshold(const char *threshold_name,
odp_tm_threshold_params_t threshold_params;
odp_tm_queue_t tm_queue;
pkt_info_t pkt_info;
- uint32_t num_pkts, pkt_len, pkts_sent;
+ uint32_t pkt_len, pkts_sent;
+ uint32_t num_pkts = 0;
odp_tm_threshold_params_init(&threshold_params);
if (max_pkts != 0) {
@@ -3442,16 +3489,19 @@ static int test_threshold(const char *threshold_name,
threshold_params.enable_max_pkts = true;
num_pkts = 2 * max_pkts;
pkt_len = 256;
- } else if (max_bytes != 0) {
+ }
+
+ if (max_bytes != 0) {
max_bytes = MIN(max_bytes, MAX_PKTS * MAX_PAYLOAD / 3);
threshold_params.max_bytes = max_bytes;
threshold_params.enable_max_bytes = true;
num_pkts = 2 * max_bytes / MAX_PAYLOAD;
pkt_len = MAX_PAYLOAD;
- } else {
- return -1;
}
+ if (max_pkts == 0 && max_bytes == 0)
+ return -1;
+
/* Pick a tm_queue and set the tm_queue's threshold profile and then
* send in twice the amount of traffic as suggested by the thresholds
* and make sure at least SOME pkts get dropped. */
@@ -4366,6 +4416,40 @@ static int test_fanin_info(const char *node_name)
return walk_tree_backwards(node_desc->node);
}
+static void test_packet_aging(uint64_t tmo_ns, uint32_t pkt_len, odp_bool_t is_dropping)
+{
+ odp_tm_queue_t tm_queue;
+ const char *node_name = "node_1_1_1";
+ const char *shaper_name = "test_shaper";
+ const uint64_t rate = 256 * 1000;
+ pkt_info_t pkt_info;
+ const uint16_t num_pkts = 4;
+ int recv_pkts;
+
+ tm_queue = find_tm_queue(0, node_name, 0);
+ CU_ASSERT_FATAL(tm_queue != ODP_TM_INVALID);
+ init_xmt_pkts(&pkt_info);
+ pkt_info.drop_eligible = false;
+ pkt_info.pkt_class = 1;
+ CU_ASSERT_FATAL(make_pkts(num_pkts, pkt_len, &pkt_info) == 0);
+
+ for (int i = 0; i < num_pkts; i++)
+ odp_packet_aging_tmo_set(xmt_pkts[i], tmo_ns);
+
+ CU_ASSERT_FATAL(set_shaper(node_name, shaper_name, rate, rate) == 0);
+ CU_ASSERT(send_pkts(tm_queue, num_pkts) == num_pkts);
+ recv_pkts = receive_pkts(odp_tm_systems[0], rcv_pktin, num_pkts, MBPS);
+
+ if (is_dropping)
+ CU_ASSERT(recv_pkts < num_pkts)
+ else
+ CU_ASSERT(recv_pkts == num_pkts);
+
+ set_shaper(node_name, NULL, 0, 0);
+ flush_leftover_pkts(odp_tm_systems[0], rcv_pktin);
+ CU_ASSERT(odp_tm_is_idle(odp_tm_systems[0]));
+}
+
static void traffic_mngr_test_default_values(void)
{
odp_tm_requirements_t req;
@@ -4517,25 +4601,51 @@ static void traffic_mngr_test_scheduler(void)
INCREASING_WEIGHTS) == 0);
}
-static int traffic_mngr_check_thresholds(void)
+static int traffic_mngr_check_thresholds_byte(void)
{
- /* Check only for tm queue threshold support as
- * we only test queue threshold.
- */
- if (!tm_capabilities.tm_queue_threshold)
+ /* Check only for TM queue threshold support as we only test queue threshold. */
+ if (!tm_capabilities.tm_queue_threshold.byte)
return ODP_TEST_INACTIVE;
return ODP_TEST_ACTIVE;
}
-static void traffic_mngr_test_thresholds(void)
+static int traffic_mngr_check_thresholds_packet(void)
{
- CU_ASSERT(test_threshold("thresh_A", "shaper_A", "node_1_2_1", 0,
- 16, 0) == 0);
- CU_ASSERT(test_threshold("thresh_B", "shaper_B", "node_1_2_1", 1,
+ /* Check only for TM queue threshold support as we only test queue threshold. */
+ if (!tm_capabilities.tm_queue_threshold.packet)
+ return ODP_TEST_INACTIVE;
+
+ return ODP_TEST_ACTIVE;
+}
+
+static int traffic_mngr_check_thresholds_byte_and_packet(void)
+{
+ /* Check only for TM queue threshold support as we only test queue threshold. */
+ if (!tm_capabilities.tm_queue_threshold.byte_and_packet)
+ return ODP_TEST_INACTIVE;
+
+ return ODP_TEST_ACTIVE;
+}
+
+static void traffic_mngr_test_thresholds_byte(void)
+{
+ CU_ASSERT(test_threshold("thresh_byte", "shaper_B", "node_1_2_1", 1,
0, 6400) == 0);
}
+static void traffic_mngr_test_thresholds_packet(void)
+{
+ CU_ASSERT(test_threshold("thresh_packet", "shaper_A", "node_1_2_1", 0,
+ 16, 0) == 0);
+}
+
+static void traffic_mngr_test_thresholds_byte_and_packet(void)
+{
+ CU_ASSERT(test_threshold("thresh_byte_and_packet", "shaper_A", "node_1_2_1", 0,
+ 16, 6400) == 0);
+}
+
static int traffic_mngr_check_queue_stats(void)
{
if (tm_capabilities.queue_stats.all_counters == 0)
@@ -4617,7 +4727,8 @@ static int traffic_mngr_check_wred(void)
static int traffic_mngr_check_byte_wred(void)
{
/* Check if wred is part of created odp_tm_t capabilities */
- if (!tm_capabilities.tm_queue_wred_supported)
+ if (!tm_capabilities.tm_queue_wred_supported ||
+ !tm_capabilities.tm_queue_threshold.byte)
return ODP_TEST_INACTIVE;
if ((tm_shaper_min_rate > 64 * 1000) ||
@@ -4631,7 +4742,8 @@ static int traffic_mngr_check_byte_wred(void)
static int traffic_mngr_check_pkt_wred(void)
{
/* Check if wred is part of created odp_tm_t capabilities */
- if (!tm_capabilities.tm_queue_wred_supported)
+ if (!tm_capabilities.tm_queue_wred_supported ||
+ !tm_capabilities.tm_queue_threshold.packet)
return ODP_TEST_INACTIVE;
if ((tm_shaper_min_rate > 64 * 1000) ||
@@ -4772,6 +4884,23 @@ static void traffic_mngr_test_ecn_drop_prec_marking(void)
CU_ASSERT(ip_marking_tests("node_1_4_2", true, true) == 0);
}
+static int traffic_mngr_check_tx_aging(void)
+{
+ return xmt_pktio_capa.max_tx_aging_tmo_ns ? ODP_TEST_ACTIVE : ODP_TEST_INACTIVE;
+}
+
+static void traffic_mngr_test_tx_aging_no_drop(void)
+{
+ /* Set very long aging tmo, packets should not be dropped due to aging */
+ test_packet_aging(60000000000, 128, false);
+}
+
+static void traffic_mngr_test_tx_aging_drop(void)
+{
+ /* Set very short aging tmo, there should be drops due to aging */
+ test_packet_aging(10, MAX_PAYLOAD, true);
+}
+
static void traffic_mngr_test_fanin_info(void)
{
CU_ASSERT(test_fanin_info("node_1") == 0);
@@ -4845,16 +4974,24 @@ odp_testinfo_t traffic_mngr_suite[] = {
ODP_TEST_INFO(traffic_mngr_test_tm_create),
ODP_TEST_INFO(traffic_mngr_test_shaper_profile),
ODP_TEST_INFO(traffic_mngr_test_sched_profile),
- ODP_TEST_INFO_CONDITIONAL(traffic_mngr_test_threshold_profile,
- traffic_mngr_check_thresholds),
+ ODP_TEST_INFO_CONDITIONAL(traffic_mngr_test_threshold_profile_byte,
+ traffic_mngr_check_thresholds_byte),
+ ODP_TEST_INFO_CONDITIONAL(traffic_mngr_test_threshold_profile_packet,
+ traffic_mngr_check_thresholds_packet),
+ ODP_TEST_INFO_CONDITIONAL(traffic_mngr_test_threshold_profile_byte_and_packet,
+ traffic_mngr_check_thresholds_byte_and_packet),
ODP_TEST_INFO_CONDITIONAL(traffic_mngr_test_wred_profile,
traffic_mngr_check_wred),
ODP_TEST_INFO_CONDITIONAL(traffic_mngr_test_shaper,
traffic_mngr_check_shaper),
ODP_TEST_INFO_CONDITIONAL(traffic_mngr_test_scheduler,
traffic_mngr_check_scheduler),
- ODP_TEST_INFO_CONDITIONAL(traffic_mngr_test_thresholds,
- traffic_mngr_check_thresholds),
+ ODP_TEST_INFO_CONDITIONAL(traffic_mngr_test_thresholds_byte,
+ traffic_mngr_check_thresholds_byte),
+ ODP_TEST_INFO_CONDITIONAL(traffic_mngr_test_thresholds_packet,
+ traffic_mngr_check_thresholds_packet),
+ ODP_TEST_INFO_CONDITIONAL(traffic_mngr_test_thresholds_byte_and_packet,
+ traffic_mngr_check_thresholds_byte_and_packet),
ODP_TEST_INFO_CONDITIONAL(traffic_mngr_test_byte_wred,
traffic_mngr_check_byte_wred),
ODP_TEST_INFO_CONDITIONAL(traffic_mngr_test_pkt_wred,
@@ -4871,6 +5008,10 @@ odp_testinfo_t traffic_mngr_suite[] = {
traffic_mngr_check_drop_prec_marking),
ODP_TEST_INFO_CONDITIONAL(traffic_mngr_test_ecn_drop_prec_marking,
traffic_mngr_check_ecn_drop_prec_marking),
+ ODP_TEST_INFO_CONDITIONAL(traffic_mngr_test_tx_aging_no_drop,
+ traffic_mngr_check_tx_aging),
+ ODP_TEST_INFO_CONDITIONAL(traffic_mngr_test_tx_aging_drop,
+ traffic_mngr_check_tx_aging),
ODP_TEST_INFO(traffic_mngr_test_fanin_info),
ODP_TEST_INFO_CONDITIONAL(traffic_mngr_test_lso_ipv4, traffic_mngr_check_lso_ipv4),
ODP_TEST_INFO(traffic_mngr_test_destroy),