diff options
author | Matias Elo <matias.elo@nokia.com> | 2022-05-04 15:40:33 +0300 |
---|---|---|
committer | Matias Elo <matias.elo@nokia.com> | 2022-05-04 15:40:33 +0300 |
commit | 2f0df4cb4acdee52f436cad8476ccf662821b7b5 (patch) | |
tree | 085136f3d98d8187b17ea584c992c8117bdedd8f | |
parent | 2f869d2698349bf75ec2404d4ab7d7f3b5b4c88b (diff) | |
parent | e361227ac00851e8720871e927b015c4c0e6a895 (diff) |
Merge tag 'v1.36.0.0' of https://github.com/OpenDataPlane/odp into odp-dpdk
Signed-off-by: Matias Elo <matias.elo@nokia.com>
131 files changed, 3720 insertions, 2375 deletions
diff --git a/.github/workflows/ci-pipeline-arm64.yml b/.github/workflows/ci-pipeline-arm64.yml index 6f32208dc..cacdb7a0d 100644 --- a/.github/workflows/ci-pipeline-arm64.yml +++ b/.github/workflows/ci-pipeline-arm64.yml @@ -31,6 +31,9 @@ jobs: - uses: actions/checkout@v2 - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${{matrix.cc}}" -e CONF="${{matrix.conf}}" $CONTAINER_NAMESPACE/odp-ci-${OS}-${ARCH}-native /odp/scripts/ci/build_${ARCH}.sh + - name: Failure log + if: ${{ failure() }} + run: find . -name config.log -exec cat {} \; Build_OS: if: ${{ github.repository == 'OpenDataPlane/odp-dpdk' }} @@ -39,12 +42,15 @@ jobs: fail-fast: false matrix: cc: [gcc, clang] - os: ['ubuntu_16.04', 'ubuntu_18.04'] + os: ['ubuntu_18.04', 'rocky_linux_8'] steps: - uses: AutoModality/action-clean@v1.1.0 - uses: actions/checkout@v2 - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${{matrix.cc}}" -e CONF="${CONF}" $CONTAINER_NAMESPACE/odp-ci-${{matrix.os}}-${ARCH}-native /odp/scripts/ci/build_${ARCH}.sh + - name: Failure log + if: ${{ failure() }} + run: find . -name config.log -exec cat {} \; Build_gcc-10: if: ${{ github.repository == 'OpenDataPlane/odp-dpdk' }} @@ -60,6 +66,9 @@ jobs: - uses: actions/checkout@v2 - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${CC}" -e CONF="${{matrix.conf}}" $CONTAINER_NAMESPACE/odp-ci-${OS}-${ARCH}-native /odp/scripts/ci/build_${ARCH}.sh + - name: Failure log + if: ${{ failure() }} + run: find . -name config.log -exec cat {} \; Build_out-of-tree: if: ${{ github.repository == 'OpenDataPlane/odp-dpdk' }} @@ -69,6 +78,9 @@ jobs: - uses: actions/checkout@v2 - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${CC}" -e CONF="${CONF}" $CONTAINER_NAMESPACE/odp-ci-${OS}-${ARCH}-native /odp/scripts/ci/out_of_tree.sh + - name: Failure log + if: ${{ failure() }} + run: find . -name config.log -exec cat {} \; Run_distcheck: if: ${{ github.repository == 'OpenDataPlane/odp-dpdk' }} @@ -130,11 +142,11 @@ jobs: strategy: fail-fast: false matrix: - os: ['ubuntu_18.04'] + os: ['ubuntu_18.04', 'ubuntu_22.04'] steps: - uses: AutoModality/action-clean@v1.1.0 - uses: actions/checkout@v2 - - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${{matrix.cc}}" -e ARCH="${ARCH}" + - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${CC}" -e ARCH="${ARCH}" -e CONF="${CONF}" $CONTAINER_NAMESPACE/odp-ci-${{matrix.os}}-${ARCH}-native /odp/scripts/ci/check.sh - name: Failure log if: ${{ failure() }} @@ -177,3 +189,17 @@ jobs: - name: Failure log if: ${{ failure() }} run: find . -name "*.trs" | xargs grep -l '^.test-result. FAIL' | while read trs ; do echo FAILURE detected at $trs; cat ${trs%%.trs}.log ; done + + Run_dpdk-21_11: + if: ${{ github.repository == 'OpenDataPlane/odp' }} + runs-on: [self-hosted, ARM64] + env: + OS: ubuntu_20.04 + steps: + - uses: AutoModality/action-clean@v1.1.0 + - uses: actions/checkout@v2 + - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${CC}" -e ARCH="${ARCH}" + -e CONF="${CONF}" $CONTAINER_NAMESPACE/odp-ci-${OS}-${ARCH}-native-dpdk_21.11 /odp/scripts/ci/check.sh + - name: Failure log + if: ${{ failure() }} + run: find . -name "*.trs" | xargs grep -l '^.test-result. FAIL' | while read trs ; do echo FAILURE detected at $trs; cat ${trs%%.trs}.log ; done diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index 56dd65fcc..102ad0213 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -78,6 +78,9 @@ jobs: - uses: actions/checkout@v2 - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${{matrix.cc}}" -e CONF="${{matrix.conf}}" $CONTAINER_NAMESPACE/odp-ci-${OS}-${ARCH} /odp/scripts/ci/build_${ARCH}.sh + - name: Failure log + if: ${{ failure() }} + run: find . -name config.log -exec cat {} \; Build_arm64: runs-on: ubuntu-18.04 @@ -118,6 +121,9 @@ jobs: OS: ubuntu_20.04 run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${{matrix.compiler}}" -e CONF="${CONF}" $CONTAINER_NAMESPACE/odp-ci-${OS}-${ARCH} /odp/scripts/ci/build_${ARCH}.sh + - name: Failure log + if: ${{ failure() }} + run: find . -name config.log -exec cat {} \; Build_ppc64el: runs-on: ubuntu-18.04 @@ -126,12 +132,14 @@ jobs: strategy: fail-fast: false matrix: - cc: [gcc, clang] conf: ['', '--enable-abi-compat'] steps: - uses: actions/checkout@v2 - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${CC}" -e CONF="${{matrix.conf}}" $CONTAINER_NAMESPACE/odp-ci-${OS}-${ARCH} /odp/scripts/ci/build_${ARCH}.sh + - name: Failure log + if: ${{ failure() }} + run: find . -name config.log -exec cat {} \; Build_i386: runs-on: ubuntu-18.04 @@ -146,6 +154,9 @@ jobs: - uses: actions/checkout@v2 - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${{matrix.cc}}" -e CONF="${{matrix.conf}}" $CONTAINER_NAMESPACE/odp-ci-${OS}-${ARCH} /odp/scripts/ci/build_${ARCH}.sh + - name: Failure log + if: ${{ failure() }} + run: find . -name config.log -exec cat {} \; Build_OS: runs-on: ubuntu-18.04 @@ -153,12 +164,15 @@ jobs: fail-fast: false matrix: cc: [gcc, clang] - os: ['centos_7', 'centos_8', 'ubuntu_16.04'] + os: ['centos_7', 'rocky_linux_8'] conf: ['--enable-abi-compat'] steps: - uses: actions/checkout@v2 - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${{matrix.cc}}" -e CONF="${{matrix.conf}}" $CONTAINER_NAMESPACE/odp-ci-${{matrix.os}}-${ARCH} /odp/scripts/ci/build_${ARCH}.sh + - name: Failure log + if: ${{ failure() }} + run: find . -name config.log -exec cat {} \; Build_gcc-10: runs-on: ubuntu-18.04 @@ -173,6 +187,9 @@ jobs: - uses: actions/checkout@v2 - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${CC}" -e CXX=g++-10 -e CONF="${{matrix.conf}}" $CONTAINER_NAMESPACE/odp-ci-${OS}-${ARCH} /odp/scripts/ci/build_${ARCH}.sh + - name: Failure log + if: ${{ failure() }} + run: find . -name config.log -exec cat {} \; Build_out-of-tree: runs-on: ubuntu-18.04 @@ -180,6 +197,9 @@ jobs: - uses: actions/checkout@v2 - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${CC}" -e CONF="${CONF}" $CONTAINER_NAMESPACE/odp-ci-${OS}-${ARCH} /odp/scripts/ci/out_of_tree.sh + - name: Failure log + if: ${{ failure() }} + run: find . -name config.log -exec cat {} \; Build_linux-generic: runs-on: ubuntu-18.04 @@ -189,6 +209,9 @@ jobs: - uses: actions/checkout@v2 - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${CC}" -e ODP_LIB_NAME="libodp-linux" -e CONF="${CONF}" $CONTAINER_NAMESPACE/odp-ci-${OS}-${ARCH} /odp/scripts/ci/build_${ARCH}.sh + - name: Failure log + if: ${{ failure() }} + run: find . -name config.log -exec cat {} \; Run_coverage: runs-on: ubuntu-18.04 @@ -241,7 +264,7 @@ jobs: fail-fast: false matrix: cc: [gcc, clang] - os: ['ubuntu_20.04'] + os: ['ubuntu_20.04', 'ubuntu_22.04'] steps: - uses: actions/checkout@v2 - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${{matrix.cc}}" -e ARCH="${ARCH}" @@ -303,6 +326,18 @@ jobs: if: ${{ failure() }} run: find . -name "*.trs" | xargs grep -l '^.test-result. FAIL' | while read trs ; do echo FAILURE detected at $trs; cat ${trs%%.trs}.log ; done + Run_dpdk-21_11: + runs-on: ubuntu-18.04 + env: + OS: ubuntu_20.04 + steps: + - uses: actions/checkout@v2 + - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${CC}" -e ARCH="${ARCH}" + -e CONF="${CONF}" $CONTAINER_NAMESPACE/odp-ci-${OS}-${ARCH}-dpdk_21.11 /odp/scripts/ci/check.sh + - name: Failure log + if: ${{ failure() }} + run: find . -name "*.trs" | xargs grep -l '^.test-result. FAIL' | while read trs ; do echo FAILURE detected at $trs; cat ${trs%%.trs}.log ; done + Run_crypto: runs-on: ubuntu-18.04 env: @@ -1,3 +1,36 @@ +== OpenDataPlane (1.36.0.0) + +=== Backward incompatible API changes + +==== Classifier +* Add an action parameter `odp_cos_action_t` to CoS parameters +(`odp_cls_cos_param_t`). Action may be to enqueue or drop the packet classified +to the CoS. The old methods of creating a drop CoS have been replaced by the +new drop action. + +==== Crypto +* Deprecate `odp_crypto_operation()`, the associated data structures, and the +completion event. Use `odp_crypto_op()` or `odp_crypto_op_enq()` instead. + +==== Traffic Manager +* Split `odp_tm_capabilities_t.tm_queue_threshold` capability into byte and +packet modes. +* Split `odp_tm_level_capabilities_t.tm_node_threshold` capability into byte and +packet modes. + +=== Backward compatible API changes +==== Classifier +* Add CoS specific statistics counters (`odp_cls_cos_stats_t`) and +matching capabilities (`odp_cls_stats_capability_t`). Statistics counters can be +read with `odp_cls_cos_stats()`. + +==== Common +* Convert `unsigned int` types to `uint32_t`. + +==== Traffic Manager +* Remove unused TM shaper color enum `odp_tm_shaper_color_t` and +`ODP_NUM_SHAPER_COLORS` define. + == OpenDataPlane (1.35.0.0) === Backward incompatible API changes diff --git a/DEPENDENCIES b/DEPENDENCIES index f319eb36e..85a4cf242 100644 --- a/DEPENDENCIES +++ b/DEPENDENCIES @@ -1,9 +1,8 @@ Prerequisites for building the OpenDataPlane (ODP) API 1. Linux - CentOS 7 (kernel v3.10) and Ubuntu 16.04 (kernel v4.4) are the oldest - Linux distributions tested by the ODP CI. Earlier versions may or may not - work. + CentOS 7 (kernel v3.10) is the oldest Linux distributions tested by the ODP + CI. Earlier versions may or may not work. For CentOS/RedHat distros, configure the system to use Fedora EPEL repos and third-party packages: @@ -206,8 +205,8 @@ Prerequisites for building the OpenDataPlane (ODP) API 3.5 DPDK packet I/O support (optional) - Use DPDK for ODP packet I/O. Currently supported DPDK versions are v19.11 - (recommended) and v20.11. + Use DPDK for ODP packet I/O. Currently supported DPDK versions are v19.11, + v20.11 (recommended), v21.11. Note: only packet I/O is accelerated with DPDK. See https://github.com/OpenDataPlane/odp-dpdk.git @@ -244,8 +243,8 @@ Prerequisites for building the OpenDataPlane (ODP) API # Configure ODP $ ./configure --with-dpdk-path=<dpdk-dir> -3.5.4 Build DPDK v20.11 from source - $ git clone https://dpdk.org/git/dpdk-stable --branch 20.11 --depth 1 ./<dpdk-dir> +3.5.4 Build DPDK v20.11 and onwards from source + $ git clone https://dpdk.org/git/dpdk-stable --branch <version, e.g. 21.11> --depth 1 ./<dpdk-dir> # Prepare the build directory $ cd <dpdk-dir> diff --git a/configure.ac b/configure.ac index 1a3904d53..9a4cce34a 100644 --- a/configure.ac +++ b/configure.ac @@ -3,7 +3,7 @@ AC_PREREQ([2.5]) # ODP API version ########################################################################## m4_define([odpapi_generation_version], [1]) -m4_define([odpapi_major_version], [35]) +m4_define([odpapi_major_version], [36]) m4_define([odpapi_minor_version], [0]) m4_define([odpapi_point_version], [0]) m4_define([odpapi_version], @@ -90,57 +90,6 @@ AM_PROG_LIBTOOL PKG_PROG_PKG_CONFIG -# Checks for library functions. -dnl breaks cross-compilation and malloc(0) behaviour is not that important -dnl AC_FUNC_MALLOC -AC_FUNC_MMAP -AC_CHECK_FUNCS(m4_normalize([ - bzero - clock_gettime - gethostbyname - getpagesize - gettimeofday - malloc - memset - munmap - socket - strchr - strerror - strrchr - strstr - strtoull -])) - -# Checks for header files. -AC_HEADER_RESOLV -AC_CHECK_HEADERS(m4_normalize([ - arpa/inet.h - fcntl.h - inttypes.h - limits.h - netdb.h - netinet/in.h - stddef.h - stdint.h - stdlib.h - string.h - sys/ioctl.h - sys/socket.h - sys/time.h - unistd.h -])) - -# Checks for typedefs, structures, and compiler characteristics. -AC_HEADER_STDBOOL -AC_C_INLINE -AC_TYPE_SIZE_T -AC_TYPE_SSIZE_T -AC_TYPE_UINT8_T -AC_TYPE_UINT16_T -AC_TYPE_INT32_T -AC_TYPE_UINT32_T -AC_TYPE_UINT64_T - ########################################################################## # Default warning setup ########################################################################## diff --git a/doc/users-guide/users-guide-crypto.adoc b/doc/users-guide/users-guide-crypto.adoc index 6393a5e78..0f33d6548 100644 --- a/doc/users-guide/users-guide-crypto.adoc +++ b/doc/users-guide/users-guide-crypto.adoc @@ -13,11 +13,6 @@ ODP provides APIs for following cryptographic services: * Random number generation * Crypto capability inquiries -Ciphering and authentication services are accessible via two complementary -sets of related APIs. The original ODP crypto APIs, and a newer -_packet-oriented_ set of crypto APIs that are designed to be consistent with -the protocol-aware cryptographic services offered by the IPsec API set. - === Crypto Sessions To apply a cryptographic operation to a packet a session must be created. All @@ -49,112 +44,35 @@ with an `odp_crypto_session_t`. === Crypto operations After session creation, a cryptographic operation can be applied to a packet -in one of two ways. - -==== Parameter-based Crypto Operations -This is the original ODP support for cryptographic operations. The -`odp_crypto_operation()` API takes an input `odp_crypto_op_param_t` struct -that describes the cryptographic operation to be performed. This struct -contains the session to use as well as the input packet the operation is to be -performed on. The caller may either specify an output packet to receive the -operation results or may request that the ODP implementation allocate a new -packet to receive these results from the output pool associated with the -`odp_crypto_session_t`. If the input packet is also used as the output packet, -then an "in place" operation is requested. - -When using the `odp_crypto_operation()` API. Applications may indicate a -preference for synchronous or asynchronous processing in the session's -`pref_mode` parameter. However crypto operations may complete synchronously -even if an asynchronous preference is indicated, and applications must examine -the `posted` output parameter from `odp_crypto_operation()` to determine -whether the operation has completed or if an `ODP_EVENT_CRYPTO_COMPL` -notification is expected. In the case of an async operation, the `posted` -output parameter will be set to true. - -The operation arguments specify for each packet the areas that are to be -encrypted or decrypted and authenticated. The parameters include also -pointers to cipher and authentication initialization vectors as needed, -depending on the initialization vector lengths specified at session creation. - -An operation can be executed in in-place, out-of-place or new buffer mode. -In in-place mode output packet is same as the input packet. -In case of out-of-place mode output packet is different from input packet as -specified by the application, while in new buffer mode implementation allocates -a new output buffer from the session’s output pool. - -The application can also specify a context associated with a given operation -that will be retained during async operation and can be retrieved via the -completion event. - -Results of an asynchronous session will be posted as completion events to the -session’s completion queue, which can be accessed directly or via the ODP -scheduler. The completion event contains the status of the operation and the -result. The application has the responsibility to free the completion event. - -Upon receipt of an `ODP_EVENT_CRYPTO_COMPL` event, the -`odp_crypto_compl_result()` API is used to retrieve the -`odp_crypto_op_result_t` associated with the event. This result struct in turn -contains: - -* An indication of the success or failure of the crypto operation -* The user context associated with the event -* The output `odp_packet_t`. -* The `odp_crypto_op_status_t` for the requested cipher operation -* The `odp_crypto_op_status_t` for the requested authentication operation - -==== Packet-based Crypto Operations -To simplify the original cryptographic operation request API, as well as to -be more flexible and consistent with the protocol-aware APIs introduced for -IPsec support, a newer packet-oriented set of cryptographic operation -APIs is also provided. Applications may use either API set, but going forward -it is expected that these newer APIs will be the focus of continued -development. - -Instead of a single `odp_crypto_operation()` API, the packet-based form -provides two APIs: `odp_crypto_op()` is the synchronous form while -`odp_crypto_op_enq()` is the asynchronous form. To check which of these are -supported by the ODP implementation, examine the `sync_mode` and `async_mode` -fields in the `odp_crypto_capability_t` struct returned by the +synchronously or asynchronously. `odp_crypto_op()` is the synchronous API +while `odp_crypto_op_enq()` is the asynchronous API. To check which of these +are supported by the ODP implementation, examine the `sync_mode` and +`async_mode` fields in the `odp_crypto_capability_t` struct returned by the `odp_crypto_capability()` API. Both forms take an input array of packets, an optional output array of packets to receive the results, and an array of `odp_crypto_packet_op_param_t` structs -that describe the operation to be performed on each input packet. As with the -original APIs, the output array may be the same packets to request in-place -operation, or may be specified as `ODP_PACKET_INVALID` to request that ODP -allocate output packets from the pool associated with the -`odp_crypto_session_t` being used. - -The key differences between the `odp_crypto_op_param_t` used by the original -APIs and the `odp_crypto_packet_op_param_t` used by the new APIs are: - -* The original API takes a single `odp_crypto_op_param_t` since it operates on -a single packet whereas the new forms take an array of -`odp_crypto_packet_op_param_t` structs, one for each input packet. - -* The `odp_crypto_packet_op_param_t` does not contain any packet information -since the input and output packets are supplied as API parameters rather than -being encoded in this struct. - -* The `odp_crypto_packet_op_param_t` does not contain a user context field. - -In addition, the `odp_crypto_session_t` field `op_mode` is used instead of -the `pref_mode` field when the packet-oriented APIs are used. If the -`op_mode` is set to `ODP_CRYPTO_SYNC` then the synchronous form of the API -must be used and if `op_mode` is set to `ODP_CRYPTO_ASYNC` then the -asynchronous form of the API must be used. It is an error to attempt to use -a form of the API not properly matched to the mode of the crypto session. - -The output of a packet-based crypto operation is an `odp_packet_t` (one for -each input packet) that is returned either synchronously or -asynchronously. Asynchronous return is in the form of `ODP_EVENT_PACKET` -events that have event subtype `ODP_EVENT_PACKET_CRYPTO`. The packet -associated with such events is obtained via the -`odp_crypto_packet_from_event()` API. The `odp_crypto_result()` API, in turn, -retrieves the `odp_crypto_packet_result_t` from this `odp_packet_t` that -contains: - -* An indication of whether the crypto packet operation was successful or not +that describe the operation to be performed on each input packet. The output +array may be the same packets to request in-place operation, or may be +specified as `ODP_PACKET_INVALID` to request that ODP allocate output packets +from the pool associated with the `odp_crypto_session_t` being used. + +The op_mode field of `odp_crypto_session_t` indicates whether asynchronous +or synchronous operations are used with the session. If `op_mode` is set +to `ODP_CRYPTO_SYNC` then the synchronous API must be used and if `op_mode` +is set to `ODP_CRYPTO_ASYNC` then the asynchronous API must be used. It is +an error to use a form of the API that does not match the mode of the crypto +session. + +The output of a crypto operation is an `odp_packet_t` (one for each input +packet) that is returned either synchronously or asynchronously. Asynchronous +return is in the form of `ODP_EVENT_PACKET` events that have event subtype +`ODP_EVENT_PACKET_CRYPTO`. The packet associated with such events is obtained +via the `odp_crypto_packet_from_event()` API. The `odp_crypto_result()` API, +in turn, retrieves the `odp_crypto_packet_result_t` from this `odp_packet_t` +that contains: + +* An indication of whether the crypto operation was successful or not * The `odp_crypto_op_status_t` for the requested cipher operation * The `odp_crypto_op_status_t` for the requested authentication operation diff --git a/doc/users-guide/users-guide-pktio.adoc b/doc/users-guide/users-guide-pktio.adoc index 43df729fa..1698060e9 100644 --- a/doc/users-guide/users-guide-pktio.adoc +++ b/doc/users-guide/users-guide-pktio.adoc @@ -468,7 +468,7 @@ specified independently at *open* time. .PktIO Input Modes * `ODP_PKTIN_MODE_DIRECT` * `ODP_PKTIN_MODE_QUEUE` -* `ODP_OKTIN_MODE_SCHED` +* `ODP_PKTIN_MODE_SCHED` * `ODP_PKTIN_MODE_DISABLED` .PktIO Output Modes diff --git a/example/generator/odp_generator.c b/example/generator/odp_generator.c index 7a070235b..b980ee180 100644 --- a/example/generator/odp_generator.c +++ b/example/generator/odp_generator.c @@ -55,9 +55,9 @@ typedef struct { odp_pktio_t pktio; odp_pktio_config_t config; odp_pktout_queue_t pktout[MAX_WORKERS]; - unsigned pktout_count; + uint32_t pktout_count; odp_pktin_queue_t pktin[MAX_WORKERS]; - unsigned pktin_count; + uint32_t pktin_count; } interface_t; /** @@ -537,8 +537,8 @@ static int setup_icmp_pkt(odp_packet_t pkt, * @warning This routine aborts if the create is unsuccessful. */ static int create_pktio(const char *dev, odp_pool_t pool, - unsigned num_rx_queues, - unsigned num_tx_queues, + uint32_t num_rx_queues, + uint32_t num_tx_queues, interface_t *itf) { odp_pktio_capability_t capa; @@ -1104,7 +1104,7 @@ int main(int argc, char *argv[]) odph_thread_t thread_tbl[MAX_WORKERS]; odp_pool_t pool; int num_workers; - unsigned num_rx_queues, num_tx_queues; + uint32_t num_rx_queues, num_tx_queues; int i; odp_shm_t shm; odp_cpumask_t cpumask; diff --git a/example/ipfragreass/odp_ipfragreass_reassemble.c b/example/ipfragreass/odp_ipfragreass_reassemble.c index 57e0f375a..2542a3b97 100644 --- a/example/ipfragreass/odp_ipfragreass_reassemble.c +++ b/example/ipfragreass/odp_ipfragreass_reassemble.c @@ -183,11 +183,9 @@ static struct packet *extract_complete_packet(struct packet *tail, struct packet *current = tail; odph_ipv4hdr_t tail_hdr; uint16_t final_frag_offset; - uint16_t expected_frag_offset; tail_hdr = *(odph_ipv4hdr_t *)odp_packet_data(tail->handle); final_frag_offset = ipv4hdr_fragment_offset_oct(tail_hdr); - expected_frag_offset = final_frag_offset; while (current) { odph_ipv4hdr_t curr_hdr; uint16_t curr_offset_oct; @@ -258,7 +256,6 @@ static struct packet *extract_complete_packet(struct packet *tail, break; } - expected_frag_offset -= prev_oct; current = prev; } diff --git a/example/ipsec_api/odp_ipsec_api_run_esp_tun_out.sh b/example/ipsec_api/odp_ipsec_api_run_esp_tun_out.sh index a4c62e226..2bda78665 100755 --- a/example/ipsec_api/odp_ipsec_api_run_esp_tun_out.sh +++ b/example/ipsec_api/odp_ipsec_api_run_esp_tun_out.sh @@ -22,6 +22,7 @@ setup_interfaces -r 192.168.222.2/32,$ROUTE_IF_OUTB,08:00:27:F5:8B:DB \ -p 192.168.111.0/24,192.168.222.0/24,out,esp \ -e 192.168.111.2,192.168.222.2,3des,201,656c8523255ccc23a66c1917aa0cf30991fce83532a4b224 \ + -t 192.168.111.2,192.168.222.2,10.0.111.2,10.0.222.2 \ -s 192.168.111.2,192.168.222.2,$IN_IF,$OUT_IF,10,100 \ -c 2 "$@" diff --git a/example/ipsec_crypto/odp_ipsec_stream.c b/example/ipsec_crypto/odp_ipsec_stream.c index a24d4b4f9..0a0710a13 100644 --- a/example/ipsec_crypto/odp_ipsec_stream.c +++ b/example/ipsec_crypto/odp_ipsec_stream.c @@ -15,6 +15,7 @@ #include <openssl/des.h> #include <openssl/rand.h> #include <openssl/hmac.h> +#include <openssl/opensslv.h> #include <odp_api.h> @@ -26,6 +27,11 @@ #define LOOP_DEQ_COUNT 32 /**< packets to dequeue at once */ +/* Ignore warnings about APIs deprecated in OpenSSL 3.0 */ +#if OPENSSL_VERSION_NUMBER >= 0x30000000L +#pragma GCC diagnostic ignored "-Wdeprecated-declarations" +#endif + /** * Stream packet header */ diff --git a/helper/include/odp/helper/cli.h b/helper/include/odp/helper/cli.h index b2f88d7bc..e9351147f 100644 --- a/helper/include/odp/helper/cli.h +++ b/helper/include/odp/helper/cli.h @@ -21,8 +21,6 @@ extern "C" { #endif -#include <odp_api.h> -#include <odp/helper/ip.h> #include <stdint.h> #include <stdarg.h> diff --git a/helper/include/odp/helper/linux/process.h b/helper/include/odp/helper/linux/process.h index 679c6c907..620db7405 100644 --- a/helper/include/odp/helper/linux/process.h +++ b/helper/include/odp/helper/linux/process.h @@ -18,6 +18,7 @@ #define ODPH_LINUX_PROCESS_H_ #include <odp/helper/threads.h> +#include <odp_api.h> #ifdef __cplusplus extern "C" { diff --git a/helper/include/odp/helper/linux/pthread.h b/helper/include/odp/helper/linux/pthread.h index 33109ec65..298536893 100644 --- a/helper/include/odp/helper/linux/pthread.h +++ b/helper/include/odp/helper/linux/pthread.h @@ -18,6 +18,7 @@ #define ODPH_LINUX_PTHREAD_H_ #include <odp/helper/threads.h> +#include <odp_api.h> #ifdef __cplusplus extern "C" { diff --git a/helper/include/odp/helper/threads.h b/helper/include/odp/helper/threads.h index cfde92830..53ce8cd07 100644 --- a/helper/include/odp/helper/threads.h +++ b/helper/include/odp/helper/threads.h @@ -24,6 +24,7 @@ extern "C" { #endif #include <odp/helper/deprecated.h> +#include <odp_api.h> #include <pthread.h> #include <getopt.h> diff --git a/include/Makefile.am b/include/Makefile.am index 89dfddebc..510154753 100644 --- a/include/Makefile.am +++ b/include/Makefile.am @@ -26,6 +26,7 @@ odpapiinclude_HEADERS = \ odp/api/dma_types.h \ odp/api/errno.h \ odp/api/event.h \ + odp/api/event_types.h \ odp/api/hash.h \ odp/api/hints.h \ odp/api/init.h \ @@ -86,6 +87,7 @@ odpapispecinclude_HEADERS = \ odp/api/spec/dma_types.h \ odp/api/spec/errno.h \ odp/api/spec/event.h \ + odp/api/spec/event_types.h \ odp/api/spec/hash.h \ odp/api/spec/hints.h \ odp/api/spec/init.h \ @@ -148,6 +150,7 @@ odpapiabidefaultinclude_HEADERS = \ odp/api/abi-default/dma_types.h \ odp/api/abi-default/errno.h \ odp/api/abi-default/event.h \ + odp/api/abi-default/event_types.h \ odp/api/abi-default/hash.h \ odp/api/abi-default/init.h \ odp/api/abi-default/ipsec.h \ @@ -178,6 +181,7 @@ odpapiabidefaultinclude_HEADERS = \ odp/api/abi-default/thrmask.h \ odp/api/abi-default/ticketlock.h \ odp/api/abi-default/time.h \ + odp/api/abi-default/timer.h \ odp/api/abi-default/timer_types.h \ odp/api/abi-default/traffic_mngr.h \ odp/api/abi-default/version.h @@ -202,6 +206,7 @@ odpapiabiarchinclude_HEADERS = \ odp/arch/arm32-linux/odp/api/abi/dma_types.h \ odp/arch/arm32-linux/odp/api/abi/errno.h \ odp/arch/arm32-linux/odp/api/abi/event.h \ + odp/arch/arm32-linux/odp/api/abi/event_types.h \ odp/arch/arm32-linux/odp/api/abi/hash.h \ odp/arch/arm32-linux/odp/api/abi/init.h \ odp/arch/arm32-linux/odp/api/abi/ipsec.h \ @@ -232,6 +237,7 @@ odpapiabiarchinclude_HEADERS = \ odp/arch/arm32-linux/odp/api/abi/thrmask.h \ odp/arch/arm32-linux/odp/api/abi/ticketlock.h \ odp/arch/arm32-linux/odp/api/abi/time.h \ + odp/arch/arm32-linux/odp/api/abi/timer.h \ odp/arch/arm32-linux/odp/api/abi/timer_types.h \ odp/arch/arm32-linux/odp/api/abi/traffic_mngr.h \ odp/arch/arm32-linux/odp/api/abi/version.h @@ -252,6 +258,7 @@ odpapiabiarchinclude_HEADERS = \ odp/arch/arm64-linux/odp/api/abi/dma_types.h \ odp/arch/arm64-linux/odp/api/abi/errno.h \ odp/arch/arm64-linux/odp/api/abi/event.h \ + odp/arch/arm64-linux/odp/api/abi/event_types.h \ odp/arch/arm64-linux/odp/api/abi/hash.h \ odp/arch/arm64-linux/odp/api/abi/init.h \ odp/arch/arm64-linux/odp/api/abi/ipsec.h \ @@ -282,6 +289,7 @@ odpapiabiarchinclude_HEADERS = \ odp/arch/arm64-linux/odp/api/abi/thrmask.h \ odp/arch/arm64-linux/odp/api/abi/ticketlock.h \ odp/arch/arm64-linux/odp/api/abi/time.h \ + odp/arch/arm64-linux/odp/api/abi/timer.h \ odp/arch/arm64-linux/odp/api/abi/timer_types.h \ odp/arch/arm64-linux/odp/api/abi/traffic_mngr.h \ odp/arch/arm64-linux/odp/api/abi/version.h @@ -302,6 +310,7 @@ odpapiabiarchinclude_HEADERS = \ odp/arch/default-linux/odp/api/abi/dma_types.h \ odp/arch/default-linux/odp/api/abi/errno.h \ odp/arch/default-linux/odp/api/abi/event.h \ + odp/arch/default-linux/odp/api/abi/event_types.h \ odp/arch/default-linux/odp/api/abi/hash.h \ odp/arch/default-linux/odp/api/abi/init.h \ odp/arch/default-linux/odp/api/abi/ipsec.h \ @@ -332,6 +341,7 @@ odpapiabiarchinclude_HEADERS = \ odp/arch/default-linux/odp/api/abi/thrmask.h \ odp/arch/default-linux/odp/api/abi/ticketlock.h \ odp/arch/default-linux/odp/api/abi/time.h \ + odp/arch/default-linux/odp/api/abi/timer.h \ odp/arch/default-linux/odp/api/abi/timer_types.h \ odp/arch/default-linux/odp/api/abi/traffic_mngr.h \ odp/arch/default-linux/odp/api/abi/version.h @@ -352,6 +362,7 @@ odpapiabiarchinclude_HEADERS = \ odp/arch/power64-linux/odp/api/abi/dma_types.h \ odp/arch/power64-linux/odp/api/abi/errno.h \ odp/arch/power64-linux/odp/api/abi/event.h \ + odp/arch/power64-linux/odp/api/abi/event_types.h \ odp/arch/power64-linux/odp/api/abi/hash.h \ odp/arch/power64-linux/odp/api/abi/init.h \ odp/arch/power64-linux/odp/api/abi/ipsec.h \ @@ -382,6 +393,7 @@ odpapiabiarchinclude_HEADERS = \ odp/arch/power64-linux/odp/api/abi/thrmask.h \ odp/arch/power64-linux/odp/api/abi/ticketlock.h \ odp/arch/power64-linux/odp/api/abi/time.h \ + odp/arch/power64-linux/odp/api/abi/timer.h \ odp/arch/power64-linux/odp/api/abi/timer_types.h \ odp/arch/power64-linux/odp/api/abi/traffic_mngr.h \ odp/arch/power64-linux/odp/api/abi/version.h @@ -402,6 +414,7 @@ odpapiabiarchinclude_HEADERS = \ odp/arch/x86_32-linux/odp/api/abi/dma_types.h \ odp/arch/x86_32-linux/odp/api/abi/errno.h \ odp/arch/x86_32-linux/odp/api/abi/event.h \ + odp/arch/x86_32-linux/odp/api/abi/event_types.h \ odp/arch/x86_32-linux/odp/api/abi/hash.h \ odp/arch/x86_32-linux/odp/api/abi/init.h \ odp/arch/x86_32-linux/odp/api/abi/ipsec.h \ @@ -432,6 +445,7 @@ odpapiabiarchinclude_HEADERS = \ odp/arch/x86_32-linux/odp/api/abi/thrmask.h \ odp/arch/x86_32-linux/odp/api/abi/ticketlock.h \ odp/arch/x86_32-linux/odp/api/abi/time.h \ + odp/arch/x86_32-linux/odp/api/abi/timer.h \ odp/arch/x86_32-linux/odp/api/abi/timer_types.h \ odp/arch/x86_32-linux/odp/api/abi/traffic_mngr.h \ odp/arch/x86_32-linux/odp/api/abi/version.h @@ -452,6 +466,7 @@ odpapiabiarchinclude_HEADERS = \ odp/arch/x86_64-linux/odp/api/abi/dma_types.h \ odp/arch/x86_64-linux/odp/api/abi/errno.h \ odp/arch/x86_64-linux/odp/api/abi/event.h \ + odp/arch/x86_64-linux/odp/api/abi/event_types.h \ odp/arch/x86_64-linux/odp/api/abi/hash.h \ odp/arch/x86_64-linux/odp/api/abi/init.h \ odp/arch/x86_64-linux/odp/api/abi/ipsec.h \ @@ -482,6 +497,7 @@ odpapiabiarchinclude_HEADERS = \ odp/arch/x86_64-linux/odp/api/abi/thrmask.h \ odp/arch/x86_64-linux/odp/api/abi/ticketlock.h \ odp/arch/x86_64-linux/odp/api/abi/time.h \ + odp/arch/x86_64-linux/odp/api/abi/timer.h \ odp/arch/x86_64-linux/odp/api/abi/timer_types.h \ odp/arch/x86_64-linux/odp/api/abi/traffic_mngr.h \ odp/arch/x86_64-linux/odp/api/abi/version.h diff --git a/include/odp/api/abi-default/event.h b/include/odp/api/abi-default/event.h index ecedda3bd..a1563ac9b 100644 --- a/include/odp/api/abi-default/event.h +++ b/include/odp/api/abi-default/event.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2017-2018, Linaro Limited +/* Copyright (c) 2022, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -11,41 +11,7 @@ extern "C" { #endif -#include <stdint.h> - -/** @internal Dummy type for strong typing */ -typedef struct { char dummy; /**< @internal Dummy */ } _odp_abi_event_t; - -/** @ingroup odp_event - * @{ - */ - -typedef _odp_abi_event_t *odp_event_t; - -#define ODP_EVENT_INVALID ((odp_event_t)0) - -typedef enum { - ODP_EVENT_BUFFER = 1, - ODP_EVENT_PACKET = 2, - ODP_EVENT_TIMEOUT = 3, - ODP_EVENT_CRYPTO_COMPL = 4, - ODP_EVENT_IPSEC_STATUS = 5, - ODP_EVENT_PACKET_VECTOR = 6, - ODP_EVENT_PACKET_TX_COMPL = 7, - ODP_EVENT_DMA_COMPL = 8, -} odp_event_type_t; - -typedef enum { - ODP_EVENT_NO_SUBTYPE = 0, - ODP_EVENT_PACKET_BASIC = 1, - ODP_EVENT_PACKET_CRYPTO = 2, - ODP_EVENT_PACKET_IPSEC = 3, - ODP_EVENT_PACKET_COMP = 4 -} odp_event_subtype_t; - -/** - * @} - */ +/* Empty header required due to the inline functions */ #ifdef __cplusplus } diff --git a/include/odp/api/abi-default/event_types.h b/include/odp/api/abi-default/event_types.h new file mode 100644 index 000000000..fc685cfd9 --- /dev/null +++ b/include/odp/api/abi-default/event_types.h @@ -0,0 +1,58 @@ +/* Copyright (c) 2017-2018, Linaro Limited + * Copyright (c) 2022, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_ABI_EVENT_TYPES_H_ +#define ODP_ABI_EVENT_TYPES_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <stdint.h> +#include <odp/api/deprecated.h> + +/** @internal Dummy type for strong typing */ +typedef struct { char dummy; /**< @internal Dummy */ } _odp_abi_event_t; + +/** @ingroup odp_event + * @{ + */ + +typedef _odp_abi_event_t *odp_event_t; + +#define ODP_EVENT_INVALID ((odp_event_t)0) + +typedef enum { + ODP_EVENT_BUFFER = 1, + ODP_EVENT_PACKET = 2, + ODP_EVENT_TIMEOUT = 3, +#if ODP_DEPRECATED_API + ODP_EVENT_CRYPTO_COMPL = 4, +#endif + ODP_EVENT_IPSEC_STATUS = 5, + ODP_EVENT_PACKET_VECTOR = 6, + ODP_EVENT_PACKET_TX_COMPL = 7, + ODP_EVENT_DMA_COMPL = 8, +} odp_event_type_t; + +typedef enum { + ODP_EVENT_NO_SUBTYPE = 0, + ODP_EVENT_PACKET_BASIC = 1, + ODP_EVENT_PACKET_CRYPTO = 2, + ODP_EVENT_PACKET_IPSEC = 3, + ODP_EVENT_PACKET_COMP = 4 +} odp_event_subtype_t; + +/** + * @} + */ + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/include/odp/api/abi-default/timer.h b/include/odp/api/abi-default/timer.h new file mode 100644 index 000000000..faf690ae3 --- /dev/null +++ b/include/odp/api/abi-default/timer.h @@ -0,0 +1,20 @@ +/* Copyright (c) 2022, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_ABI_TIMER_H_ +#define ODP_ABI_TIMER_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +/* Empty header required due to the timer inline functions */ + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/include/odp/api/abi-default/traffic_mngr.h b/include/odp/api/abi-default/traffic_mngr.h index 06dc6c3c7..78985313d 100644 --- a/include/odp/api/abi-default/traffic_mngr.h +++ b/include/odp/api/abi-default/traffic_mngr.h @@ -76,11 +76,6 @@ extern "C" { */ #define ODP_TM_MAX_TM_NODE_FANIN (4 * 1024) -/** The ODP_NUM_SHAPER_COLORS constant just counts the number of enumeration - * values defined in the odp_tm_shaper_color_t type. - */ -#define ODP_NUM_SHAPER_COLORS 3 - /** The INVALID_PRIORITY constant is used when one needs to indicate an * invalid priority value. */ diff --git a/include/odp/api/buffer.h b/include/odp/api/buffer.h index 27622c2a3..a56cd0660 100644 --- a/include/odp/api/buffer.h +++ b/include/odp/api/buffer.h @@ -17,8 +17,6 @@ extern "C" { #endif -#include <odp/api/std_types.h> -#include <odp/api/abi/event.h> #include <odp/api/abi/buffer.h> #include <odp/api/spec/buffer.h> diff --git a/include/odp/api/comp.h b/include/odp/api/comp.h index c3294e79e..300d12a61 100644 --- a/include/odp/api/comp.h +++ b/include/odp/api/comp.h @@ -18,8 +18,6 @@ extern "C" { #endif #include <odp/api/abi/comp.h> -#include <odp/api/abi/event.h> -#include <odp/api/abi/queue_types.h> #include <odp/api/spec/comp.h> diff --git a/include/odp/api/dma_types.h b/include/odp/api/dma_types.h index 5a726a319..eeaa7bee8 100644 --- a/include/odp/api/dma_types.h +++ b/include/odp/api/dma_types.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2021, Nokia +/* Copyright (c) 2021-2022, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -17,7 +17,6 @@ extern "C" { #endif -#include <odp/api/abi/event.h> #include <odp/api/abi/dma_types.h> #include <odp/api/spec/dma_types.h> diff --git a/include/odp/api/event_types.h b/include/odp/api/event_types.h new file mode 100644 index 000000000..a4d20a341 --- /dev/null +++ b/include/odp/api/event_types.h @@ -0,0 +1,28 @@ +/* Copyright (c) 2022, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +/** + * @file + * + * ODP event API type definitions + */ + +#ifndef ODP_API_EVENT_TYPES_H_ +#define ODP_API_EVENT_TYPES_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <odp/api/abi/event_types.h> + +#include <odp/api/spec/event_types.h> + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/include/odp/api/packet.h b/include/odp/api/packet.h index eb75d18fa..1f04c026d 100644 --- a/include/odp/api/packet.h +++ b/include/odp/api/packet.h @@ -1,4 +1,5 @@ /* Copyright (c) 2013-2018, Linaro Limited + * Copyright (c) 2022, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -17,12 +18,7 @@ extern "C" { #endif -#include <odp/api/std_types.h> -#include <odp/api/abi/event.h> -#include <odp/api/abi/packet_io_types.h> -#include <odp/api/abi/packet_types.h> #include <odp/api/abi/packet.h> -#include <odp/api/abi/buffer.h> #include <odp/api/spec/packet.h> diff --git a/include/odp/api/queue.h b/include/odp/api/queue.h index 7897fff86..666d5eaf9 100644 --- a/include/odp/api/queue.h +++ b/include/odp/api/queue.h @@ -1,4 +1,5 @@ /* Copyright (c) 2013-2018, Linaro Limited + * Copyright (c) 2022, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -17,11 +18,7 @@ extern "C" { #endif -#include <odp/api/std_types.h> -#include <odp/api/abi/event.h> -#include <odp/api/abi/queue_types.h> #include <odp/api/abi/queue.h> -#include <odp/api/abi/buffer.h> #include <odp/api/spec/queue.h> diff --git a/include/odp/api/spec/buffer.h b/include/odp/api/spec/buffer.h index 7945b59f9..c11a3c7b4 100644 --- a/include/odp/api/spec/buffer.h +++ b/include/odp/api/spec/buffer.h @@ -18,7 +18,9 @@ extern "C" { #endif +#include <odp/api/event_types.h> #include <odp/api/pool_types.h> +#include <odp/api/std_types.h> /** @defgroup odp_buffer ODP BUFFER * Buffer event metadata and operations. diff --git a/include/odp/api/spec/classification.h b/include/odp/api/spec/classification.h index b9682c7cb..1c76ea192 100644 --- a/include/odp/api/spec/classification.h +++ b/include/odp/api/spec/classification.h @@ -36,9 +36,7 @@ extern "C" { /** * @def ODP_COS_INVALID - * This value is returned from odp_cls_cos_create() on failure, - * May also be used as a sink class of service that - * results in packets being discarded. + * This value is returned from odp_cls_cos_create() on failure. */ /** @@ -179,6 +177,30 @@ typedef struct odp_bp_param_t { } odp_bp_param_t; /** + * Classifier CoS specific statistics counters + * + * Counters are incremented per packet classified to the CoS. In a CoS chain, + * counters are incremented in every CoS for which counters are enabled. + */ +typedef struct odp_cls_cos_stats_t { + /** Number of octets in classified packets. In case of Ethernet, packet + * size includes MAC header. */ + uint64_t octets; + + /** Number of classified packets, including packets dropped due to drop + * action. */ + uint64_t packets; + + /** Number of discarded packets due to other reasons than packet + * errors or drop action. */ + uint64_t discards; + + /** Number of packets with errors. */ + uint64_t errors; + +} odp_cls_cos_stats_t; + +/** * Classifier queue specific statistics counters * * Counters are incremented per packet destined to the queue per originating @@ -206,6 +228,34 @@ typedef struct odp_cls_queue_stats_t { * Classifier statistics capabilities */ typedef struct odp_cls_stats_capability_t { + /** CoS level capabilities */ + struct { + /** Supported counters */ + union { + /** Statistics counters in a bit field structure */ + struct { + /** @see odp_cls_cos_stats_t::octets */ + uint64_t octets : 1; + + /** @see odp_cls_cos_stats_t::packets */ + uint64_t packets : 1; + + /** @see odp_cls_cos_stats_t::discards */ + uint64_t discards : 1; + + /** @see odp_cls_cos_stats_t::errors */ + uint64_t errors : 1; + + } counter; + + /** All bits of the bit field structure + * + * This field can be used to set/clear all flags, or + * for bitwise operations over the entire structure. */ + uint64_t all_counters; + }; + } cos; + /** Queue level capabilities */ struct { /** Supported counters */ @@ -247,17 +297,22 @@ typedef struct odp_cls_capability_t { odp_cls_pmr_terms_t supported_terms; /** Maximum number of PMR terms */ - unsigned int max_pmr_terms; + uint32_t max_pmr_terms; /** Number of PMR terms available for use now */ - unsigned int available_pmr_terms; + uint32_t available_pmr_terms; /** Maximum number of CoS supported */ - unsigned int max_cos; + uint32_t max_cos; + + /** Maximum number of CoSes that can have statistics enabled at the same + * time. If this value is zero, then CoS level statistics are not + * supported. */ + uint32_t max_cos_stats; /** Maximun number of queues supported per CoS * if the value is 1, then hashing is not supported*/ - unsigned int max_hash_queues; + uint32_t max_hash_queues; /** Protocol header combination supported for Hashing */ odp_pktin_hash_proto_t hash_protocols; @@ -315,10 +370,48 @@ typedef enum { } odp_cos_hdr_flow_fields_t; /** + * Enumeration of actions for CoS. + */ +typedef enum { + /** + * Enqueue packet + * + * Packets that arrive in the CoS are enqueued to a destination queue. + */ + ODP_COS_ACTION_ENQUEUE, + + /** + * Drop packet + * + * Packets that arrive in the CoS are dropped. Packets are freed into + * their originating pool. + */ + ODP_COS_ACTION_DROP, +} odp_cos_action_t; + +/** * Class of service parameters * Used to communicate class of service creation options */ typedef struct odp_cls_cos_param { + /** Action to take. When action is ODP_COS_ACTION_DROP, all the other + * parameters are ignored. + * + * The final match in the CoS chain defines the action for a packet. + * I.e. packet is dropped only when the CoS of the last matching rule + * has drop action. Actions in the previous CoSes in the chain are + * ignored. + * + * Default is ODP_COS_ACTION_ENQUEUE. + */ + odp_cos_action_t action; + + /** Enable statistics. If true, counters are incremented when packets + * are classified to the CoS. Default is false. @see + * odp_cls_cos_stats(). + */ + odp_bool_t stats_enable; + /** Number of queues to be linked to this CoS. * If the number is greater than 1 then hashing is enabled. * If number is equal to 1 then hashing is disabled. @@ -391,6 +484,9 @@ int odp_cls_capability(odp_cls_capability_t *capability); /** * Create a class-of-service * + * Depending on the action parameter, packets to the CoS are either enqueued to + * a destination queue, or dropped. + * * The use of class-of-service name is optional. Unique names are not required. * Use odp_cls_cos_param_init() to initialize parameters into their default * values. @@ -401,10 +497,6 @@ int odp_cls_capability(odp_cls_capability_t *capability); * * @retval Class-of-service handle * @retval ODP_COS_INVALID on failure. - * - * @note ODP_QUEUE_INVALID and ODP_POOL_INVALID are valid values for queue - * and pool associated with a class of service. When either of these values - * is configured as INVALID packets assigned to the CoS get dropped. */ odp_cos_t odp_cls_cos_create(const char *name, const odp_cls_cos_param_t *param); @@ -542,6 +634,25 @@ int odp_cos_with_l3_qos(odp_pktio_t pktio_in, odp_bool_t l3_preference); /** + * Get statistics for a CoS + * + * The statistics counters are incremented for packets classified to the + * given CoS. + * + * Counters that are not supported are set to zero. + * + * It's implementation defined if odp_pktio_stats_reset() call affects these + * counters. + * + * @param cos CoS handle + * @param[out] stats Statistics structure for output + * + * @retval 0 on success + * @retval <0 on failure + */ +int odp_cls_cos_stats(odp_cos_t cos, odp_cls_cos_stats_t *stats); + +/** * Get statistics for a queue assigned to a CoS * * The statistics counters are incremented only for packets originating from the diff --git a/include/odp/api/spec/comp.h b/include/odp/api/spec/comp.h index 7f64cca5d..39ecbfde4 100644 --- a/include/odp/api/spec/comp.h +++ b/include/odp/api/spec/comp.h @@ -14,8 +14,11 @@ #define ODP_API_SPEC_COMP_H_ #include <odp/visibility_begin.h> + +#include <odp/api/event_types.h> +#include <odp/api/packet_types.h> +#include <odp/api/queue_types.h> #include <odp/api/std_types.h> -#include <odp/api/packet.h> #ifdef __cplusplus extern "C" { diff --git a/include/odp/api/spec/crypto.h b/include/odp/api/spec/crypto.h index c0ff3710d..7cd69f4d2 100644 --- a/include/odp/api/spec/crypto.h +++ b/include/odp/api/spec/crypto.h @@ -16,6 +16,7 @@ #include <odp/visibility_begin.h> #include <odp/api/deprecated.h> +#include <odp/api/packet_types.h> #include <odp/api/pool_types.h> #include <odp/api/std_types.h> @@ -23,8 +24,6 @@ extern "C" { #endif -#include <odp/api/packet.h> - /** @defgroup odp_crypto ODP CRYPTO * Data ciphering and authentication. * @{ @@ -42,7 +41,7 @@ extern "C" { /** * @typedef odp_crypto_compl_t -* Crypto API completion event (platform dependent). +* @deprecated Crypto API completion event (platform dependent). */ /** @@ -559,8 +558,10 @@ typedef struct odp_crypto_session_param_t { /** Preferred sync vs. async for odp_crypto_operation() * * The default value is ODP_CRYPTO_SYNC. + * + * @deprecated Used only with deprecated odp_crypto_operation() */ - odp_crypto_op_mode_t pref_mode; + odp_crypto_op_mode_t ODP_DEPRECATE(pref_mode); /** Operation mode when using packet interface: sync or async * @@ -672,8 +673,8 @@ typedef struct odp_crypto_session_param_t { /** Async mode completion event queue * * The completion queue is used to return completions from - * odp_crypto_operation() or odp_crypto_op_enq() results to the - * application. + * odp_crypto_op_enq() (and the deprecated odp_crypto_operation()) + * to the application. */ odp_queue_t compl_queue; @@ -689,6 +690,8 @@ typedef struct odp_crypto_session_param_t { /** * Crypto API per packet operation parameters + * + * @deprecated Use odp_crypto_packet_op_param_t instead. */ typedef struct odp_crypto_op_param_t { /** Session handle from creation */ @@ -760,7 +763,7 @@ typedef struct odp_crypto_op_param_t { */ odp_packet_data_range_t auth_range; -} odp_crypto_op_param_t; +} ODP_DEPRECATE(odp_crypto_op_param_t); /** * Crypto packet API per packet operation parameters @@ -895,6 +898,8 @@ typedef struct odp_crypto_op_status { /** * Crypto API operation result + * + * @deprecated Use odp_crypto_packet_result_t instead. */ typedef struct odp_crypto_op_result { /** Request completed successfully */ @@ -912,7 +917,7 @@ typedef struct odp_crypto_op_result { /** Authentication status */ odp_crypto_op_status_t auth_status; -} odp_crypto_op_result_t; +} ODP_DEPRECATE(odp_crypto_op_result_t); /** * Crypto packet API operation result @@ -1141,9 +1146,13 @@ int odp_crypto_session_create(const odp_crypto_session_param_t *param, */ int odp_crypto_session_destroy(odp_crypto_session_t session); +#if ODP_DEPRECATED_API + /** * Return crypto completion handle that is associated with event * + * @deprecated Used only with the deprecated odp_crypto_operation() + * * Note: any invalid parameters will cause undefined behavior and may cause * the application to abort or crash. * @@ -1156,6 +1165,8 @@ odp_crypto_compl_t odp_crypto_compl_from_event(odp_event_t ev); /** * Convert crypto completion handle to event handle * + * @deprecated Used only with the deprecated odp_crypto_operation() + * * @param completion_event Completion event to convert to generic event * * @return Event handle @@ -1165,6 +1176,8 @@ odp_event_t odp_crypto_compl_to_event(odp_crypto_compl_t completion_event); /** * Release crypto completion event * + * @deprecated Used only with the deprecated odp_crypto_operation() + * * @param completion_event Completion event we are done accessing */ void odp_crypto_compl_free(odp_crypto_compl_t completion_event); @@ -1172,6 +1185,8 @@ void odp_crypto_compl_free(odp_crypto_compl_t completion_event); /** * Crypto per packet operation * + * @deprecated Use odp_crypto_op() or odp_crypto_op_enq() instead. + * * Performs the cryptographic operations specified during session creation * on the packet. If the operation is performed synchronously, "posted" * will return FALSE and the result of the operation is immediately available. @@ -1192,12 +1207,16 @@ int odp_crypto_operation(odp_crypto_op_param_t *param, /** * Crypto per packet operation query result from completion event * + * @deprecated Used only with the deprecated odp_crypto_operation() + * * @param completion_event Event containing operation results * @param result Pointer to result structure */ void odp_crypto_compl_result(odp_crypto_compl_t completion_event, odp_crypto_op_result_t *result); +#endif /* ODP_DEPRECATED_API */ + /** * Get printable value for an odp_crypto_session_t * @@ -1211,9 +1230,12 @@ void odp_crypto_compl_result(odp_crypto_compl_t completion_event, */ uint64_t odp_crypto_session_to_u64(odp_crypto_session_t hdl); +#if ODP_DEPRECATED_API /** * Get printable value for an odp_crypto_compl_t * + * @deprecated Used only with the deprecated odp_crypto_operation() + * * @param hdl odp_crypto_compl_t handle to be printed * @return uint64_t value that can be used to print/display this * handle @@ -1223,6 +1245,7 @@ uint64_t odp_crypto_session_to_u64(odp_crypto_session_t hdl); * an odp_crypto_compl_t handle. */ uint64_t odp_crypto_compl_to_u64(odp_crypto_compl_t hdl); +#endif /** * Initialize crypto session parameters diff --git a/include/odp/api/spec/dma_types.h b/include/odp/api/spec/dma_types.h index d7ccc6d56..ae3d05287 100644 --- a/include/odp/api/spec/dma_types.h +++ b/include/odp/api/spec/dma_types.h @@ -18,7 +18,9 @@ extern "C" { #endif +#include <odp/api/event_types.h> #include <odp/api/packet_types.h> +#include <odp/api/queue_types.h> #include <odp/api/std_types.h> /** @defgroup odp_dma ODP DMA diff --git a/include/odp/api/spec/event.h b/include/odp/api/spec/event.h index dc99f35c7..c36d9cef1 100644 --- a/include/odp/api/spec/event.h +++ b/include/odp/api/spec/event.h @@ -1,4 +1,5 @@ /* Copyright (c) 2015-2018, Linaro Limited + * Copyright (c) 2022, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -18,7 +19,8 @@ extern "C" { #endif -#include <odp/api/packet.h> +#include <odp/api/event_types.h> +#include <odp/api/packet_types.h> /** @defgroup odp_event ODP EVENT * Generic event metadata and operations. @@ -26,81 +28,6 @@ extern "C" { */ /** - * @typedef odp_event_t - * ODP event - */ - -/** - * @def ODP_EVENT_INVALID - * Invalid event - */ - -/** - * @typedef odp_event_type_t - * Event type - * - * Event type specifies purpose and general format of an event. It can be - * checked with odp_event_type() or odp_event_types(). Each event type has - * functions (e.g. odp_buffer_from_event()) to convert between the generic event - * handle (odp_event_t) and the type specific handle (e.g. odp_buffer_t). - * Results are undefined, if conversion function of a wrong event type is used. - * Application cannot change event type by chaining conversion functions. - * - * List of event types: - * - ODP_EVENT_BUFFER - * - Buffer event (odp_buffer_t) for simple data storage and message passing - * - ODP_EVENT_PACKET - * - Packet event (odp_packet_t) containing packet data and plenty of - * packet processing related metadata - * - ODP_EVENT_TIMEOUT - * - Timeout event (odp_timeout_t) from a timer - * - ODP_EVENT_CRYPTO_COMPL - * - Crypto completion event (odp_crypto_compl_t) - * - ODP_EVENT_IPSEC_STATUS - * - IPSEC status update event (odp_ipsec_status_t) - * - ODP_EVENT_PACKET_VECTOR - * - Vector of packet events (odp_packet_t) as odp_packet_vector_t - * - ODP_EVENT_PACKET_TX_COMPL - * - Packet Tx completion event (odp_packet_tx_compl_t) generated as a result of a Packet Tx - * completion. - * - ODP_EVENT_DMA_COMPL - * - DMA completion event (odp_dma_compl_t) indicates that a DMA transfer has finished - */ - -/** - * @typedef odp_event_subtype_t - * Event subtype - * - * Event subtype expands event type specification by providing more detailed - * purpose and format of an event. It can be checked with odp_event_subtype() or - * odp_event_types(). Each event subtype may define specific functions - * (e.g. odp_ipsec_packet_from_event()) to convert between the generic event - * handle (odp_event_t) and event type specific handle (e.g. odp_packet_t). When - * subtype is known, these subtype specific functions should be preferred over - * the event type general function (e.g. odp_packet_from_event()). Results are - * undefined, if conversion function of a wrong event subtype is used. - * Application cannot change event subtype by chaining conversion functions. - * - * List of event subtypes: - * - ODP_EVENT_PACKET_BASIC - * - Packet event (odp_packet_t) with basic packet metadata - * - ODP_EVENT_PACKET_COMP - * - Packet event (odp_packet_t) generated as a result of a compression/ - * decompression operation. It contains compression specific metadata in - * addition to the basic packet metadata. - * - ODP_EVENT_PACKET_CRYPTO - * - Packet event (odp_packet_t) generated as a result of a Crypto - * operation. It contains crypto specific metadata in addition to the - * basic packet metadata. - * - ODP_EVENT_PACKET_IPSEC - * - Packet event (odp_packet_t) generated as a result of an IPsec - * operation. It contains IPSEC specific metadata in addition to the basic - * packet metadata. - * - ODP_EVENT_NO_SUBTYPE - * - An event type does not have any subtypes defined - */ - -/** * Event type of an event * * Event type specifies purpose and general format of an event. diff --git a/include/odp/api/spec/event_types.h b/include/odp/api/spec/event_types.h new file mode 100644 index 000000000..529d031e7 --- /dev/null +++ b/include/odp/api/spec/event_types.h @@ -0,0 +1,110 @@ +/* Copyright (c) 2015-2018, Linaro Limited + * Copyright (c) 2022, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +/** + * @file + * + * ODP event API type definitions + */ + +#ifndef ODP_API_SPEC_EVENT_TYPES_H_ +#define ODP_API_SPEC_EVENT_TYPES_H_ +#include <odp/visibility_begin.h> + +#ifdef __cplusplus +extern "C" { +#endif + +/** @addtogroup odp_event + * @{ + */ + +/** + * @typedef odp_event_t + * ODP event + */ + +/** + * @def ODP_EVENT_INVALID + * Invalid event + */ + +/** + * @typedef odp_event_type_t + * Event type + * + * Event type specifies purpose and general format of an event. It can be + * checked with odp_event_type() or odp_event_types(). Each event type has + * functions (e.g. odp_buffer_from_event()) to convert between the generic event + * handle (odp_event_t) and the type specific handle (e.g. odp_buffer_t). + * Results are undefined, if conversion function of a wrong event type is used. + * Application cannot change event type by chaining conversion functions. + * + * List of event types: + * - ODP_EVENT_BUFFER + * - Buffer event (odp_buffer_t) for simple data storage and message passing + * - ODP_EVENT_PACKET + * - Packet event (odp_packet_t) containing packet data and plenty of + * packet processing related metadata + * - ODP_EVENT_TIMEOUT + * - Timeout event (odp_timeout_t) from a timer + * - ODP_EVENT_CRYPTO_COMPL + * - Crypto completion event (odp_crypto_compl_t) + * - ODP_EVENT_IPSEC_STATUS + * - IPSEC status update event (odp_ipsec_status_t) + * - ODP_EVENT_PACKET_VECTOR + * - Vector of packet events (odp_packet_t) as odp_packet_vector_t + * - ODP_EVENT_PACKET_TX_COMPL + * - Packet Tx completion event (odp_packet_tx_compl_t) generated as a result of a Packet Tx + * completion. + * - ODP_EVENT_DMA_COMPL + * - DMA completion event (odp_dma_compl_t) indicates that a DMA transfer has finished + */ + +/** + * @typedef odp_event_subtype_t + * Event subtype + * + * Event subtype expands event type specification by providing more detailed + * purpose and format of an event. It can be checked with odp_event_subtype() or + * odp_event_types(). Each event subtype may define specific functions + * (e.g. odp_ipsec_packet_from_event()) to convert between the generic event + * handle (odp_event_t) and event type specific handle (e.g. odp_packet_t). When + * subtype is known, these subtype specific functions should be preferred over + * the event type general function (e.g. odp_packet_from_event()). Results are + * undefined, if conversion function of a wrong event subtype is used. + * Application cannot change event subtype by chaining conversion functions. + * + * List of event subtypes: + * - ODP_EVENT_PACKET_BASIC + * - Packet event (odp_packet_t) with basic packet metadata + * - ODP_EVENT_PACKET_COMP + * - Packet event (odp_packet_t) generated as a result of a compression/ + * decompression operation. It contains compression specific metadata in + * addition to the basic packet metadata. + * - ODP_EVENT_PACKET_CRYPTO + * - Packet event (odp_packet_t) generated as a result of a Crypto + * operation. It contains crypto specific metadata in addition to the + * basic packet metadata. + * - ODP_EVENT_PACKET_IPSEC + * - Packet event (odp_packet_t) generated as a result of an IPsec + * operation. It contains IPSEC specific metadata in addition to the basic + * packet metadata. + * - ODP_EVENT_NO_SUBTYPE + * - An event type does not have any subtypes defined + */ + +/** + * @} + */ + +#ifdef __cplusplus +} +#endif + +#include <odp/visibility_end.h> +#endif diff --git a/include/odp/api/spec/packet.h b/include/odp/api/spec/packet.h index 8dc0a77bb..c745d29ad 100644 --- a/include/odp/api/spec/packet.h +++ b/include/odp/api/spec/packet.h @@ -19,10 +19,13 @@ extern "C" { #endif -#include <odp/api/proto_stats_types.h> -#include <odp/api/time.h> +#include <odp/api/event_types.h> #include <odp/api/packet_types.h> +#include <odp/api/packet_io_types.h> #include <odp/api/pool_types.h> +#include <odp/api/proto_stats_types.h> +#include <odp/api/std_types.h> +#include <odp/api/time.h> /** @defgroup odp_packet ODP PACKET * Packet event metadata and operations. diff --git a/include/odp/api/spec/packet_flags.h b/include/odp/api/spec/packet_flags.h index 7c1386cb6..98bf2b7bf 100644 --- a/include/odp/api/spec/packet_flags.h +++ b/include/odp/api/spec/packet_flags.h @@ -19,7 +19,7 @@ extern "C" { #endif #include <odp/api/std_types.h> -#include <odp/api/packet.h> +#include <odp/api/packet_types.h> /** @addtogroup odp_packet * Operations on packet metadata flags. diff --git a/include/odp/api/spec/packet_io.h b/include/odp/api/spec/packet_io.h index 3be45446d..4a8635d0a 100644 --- a/include/odp/api/spec/packet_io.h +++ b/include/odp/api/spec/packet_io.h @@ -431,9 +431,8 @@ int odp_pktin_recv_tmo(odp_pktin_queue_t queue, odp_packet_t packets[], * @return Number of packets received * @retval <0 on failure */ -int odp_pktin_recv_mq_tmo(const odp_pktin_queue_t queues[], unsigned int num_q, - unsigned int *from, odp_packet_t packets[], int num, - uint64_t wait); +int odp_pktin_recv_mq_tmo(const odp_pktin_queue_t queues[], uint32_t num_q, uint32_t *from, + odp_packet_t packets[], int num, uint64_t wait); /** * Packet input wait time diff --git a/include/odp/api/spec/packet_io_types.h b/include/odp/api/spec/packet_io_types.h index 2d43822fe..153e188b7 100644 --- a/include/odp/api/spec/packet_io_types.h +++ b/include/odp/api/spec/packet_io_types.h @@ -257,7 +257,7 @@ typedef struct odp_pktin_queue_param_t { * More than one input queues require flow hashing configured. * The maximum value is defined by pktio capability 'max_input_queues'. * Queue type is defined by the input mode. The default value is 1. */ - unsigned int num_queues; + uint32_t num_queues; /** Queue parameters * @@ -304,7 +304,7 @@ typedef struct odp_pktout_queue_param_t { /** Number of output queues to be created. The value must be between * 1 and interface capability. The default value is 1. */ - unsigned int num_queues; + uint32_t num_queues; /** Output queue size array * @@ -820,12 +820,12 @@ typedef struct odp_pktin_vector_capability_t { */ typedef struct odp_pktio_capability_t { /** Maximum number of input queues */ - unsigned int max_input_queues; + uint32_t max_input_queues; /** Maximum number of output queues * * Value does not exceed ODP_PKTOUT_MAX_QUEUES. */ - unsigned int max_output_queues; + uint32_t max_output_queues; /** Minimum output queue size * diff --git a/include/odp/api/spec/pool_types.h b/include/odp/api/spec/pool_types.h index b0d5b37c9..ef272b67d 100644 --- a/include/odp/api/spec/pool_types.h +++ b/include/odp/api/spec/pool_types.h @@ -127,12 +127,12 @@ typedef struct odp_pool_stats_t { */ typedef struct odp_pool_capability_t { /** Maximum number of pools of any type */ - unsigned int max_pools; + uint32_t max_pools; /** Buffer pool capabilities */ struct { /** Maximum number of buffer pools */ - unsigned int max_pools; + uint32_t max_pools; /** Maximum buffer data alignment in bytes */ uint32_t max_align; @@ -162,7 +162,7 @@ typedef struct odp_pool_capability_t { /** Packet pool capabilities */ struct { /** Maximum number of packet pools */ - unsigned int max_pools; + uint32_t max_pools; /** Maximum packet data length in bytes * @@ -250,7 +250,7 @@ typedef struct odp_pool_capability_t { /** Timeout pool capabilities */ struct { /** Maximum number of timeout pools */ - unsigned int max_pools; + uint32_t max_pools; /** Maximum number of timeout events in a pool * @@ -271,7 +271,7 @@ typedef struct odp_pool_capability_t { /** Vector pool capabilities */ struct { /** Maximum number of vector pools */ - unsigned int max_pools; + uint32_t max_pools; /** Maximum number of vector events in a pool * diff --git a/include/odp/api/spec/queue.h b/include/odp/api/spec/queue.h index 9c677c926..65cf3d1ee 100644 --- a/include/odp/api/spec/queue.h +++ b/include/odp/api/spec/queue.h @@ -18,8 +18,9 @@ extern "C" { #endif -#include <odp/api/event.h> +#include <odp/api/event_types.h> #include <odp/api/queue_types.h> +#include <odp/api/std_types.h> /** @defgroup odp_queue ODP QUEUE * Queues for event passing and scheduling. diff --git a/include/odp/api/spec/schedule.h b/include/odp/api/spec/schedule.h index 12da88f7d..524d9e2bf 100644 --- a/include/odp/api/spec/schedule.h +++ b/include/odp/api/spec/schedule.h @@ -19,7 +19,7 @@ extern "C" { #endif #include <odp/api/std_types.h> -#include <odp/api/event.h> +#include <odp/api/event_types.h> #include <odp/api/queue_types.h> #include <odp/api/schedule_types.h> #include <odp/api/thrmask.h> diff --git a/include/odp/api/spec/shared_memory.h b/include/odp/api/spec/shared_memory.h index 6ba6f7fc2..d1955db26 100644 --- a/include/odp/api/spec/shared_memory.h +++ b/include/odp/api/spec/shared_memory.h @@ -133,7 +133,7 @@ typedef struct odp_shm_capability_t { * * This number of separate shared memory blocks can be * reserved concurrently. */ - unsigned int max_blocks; + uint32_t max_blocks; /** Maximum memory block size in bytes * diff --git a/include/odp/api/spec/timer.h b/include/odp/api/spec/timer.h index 9df43d034..b51db7dbc 100644 --- a/include/odp/api/spec/timer.h +++ b/include/odp/api/spec/timer.h @@ -21,7 +21,7 @@ extern "C" { #endif #include <odp/api/timer_types.h> -#include <odp/api/event.h> +#include <odp/api/event_types.h> #include <odp/api/pool_types.h> #include <odp/api/queue_types.h> diff --git a/include/odp/api/spec/timer_types.h b/include/odp/api/spec/timer_types.h index 56c118ce3..754c831b2 100644 --- a/include/odp/api/spec/timer_types.h +++ b/include/odp/api/spec/timer_types.h @@ -21,7 +21,7 @@ extern "C" { #endif #include <odp/api/std_types.h> -#include <odp/api/event.h> +#include <odp/api/event_types.h> /** @defgroup odp_timer ODP TIMER * Timer generating timeout events. diff --git a/include/odp/api/spec/traffic_mngr.h b/include/odp/api/spec/traffic_mngr.h index 64f2eac83..b22887fc6 100644 --- a/include/odp/api/spec/traffic_mngr.h +++ b/include/odp/api/spec/traffic_mngr.h @@ -98,11 +98,6 @@ extern "C" { */ /** - * @def ODP_NUM_SHAPER_COLORS - * The number of enumeration values defined in the odp_tm_shaper_color_t type. - */ - -/** * @def ODP_TM_INVALID_PRIORITY * Used to indicate an invalid priority value. */ @@ -344,9 +339,19 @@ typedef struct { * the legal range of such weights. */ odp_bool_t weights_supported; - /** tm_node_threshold indicates that the tm_nodes at this - * level support threshold profiles. */ - odp_bool_t tm_node_threshold; + /** TM node threshold profile support */ + struct { + /** Threshold given as bytes */ + uint8_t byte : 1; + + /** Threshold given as packets */ + uint8_t packet : 1; + + /** Threshold given as bytes and packets simultaneously */ + uint8_t byte_and_packet : 1; + + } tm_node_threshold; + } odp_tm_level_capabilities_t; /** The tm_pkt_prio_mode_t enumeration type is used to indicate different @@ -515,11 +520,18 @@ typedef struct { /** TM queue statistics counter capabilities */ odp_tm_queue_stats_capability_t queue_stats; - /** tm_queue_threshold indicates support for threshold profile on a - * TM queue. When TRUE, users can set/clear/update threshold profile - * on a TM queue. When false none of it is supported. - */ - odp_bool_t tm_queue_threshold; + /** TM queue threshold profile support */ + struct { + /** Threshold given as bytes */ + uint8_t byte : 1; + + /** Threshold given as packets */ + uint8_t packet : 1; + + /** Threshold given as bytes and packets simultaneously */ + uint8_t byte_and_packet : 1; + + } tm_queue_threshold; /** tm_queue_query_flags indicates supported types of TM queue query. * Types of TM queue query are same as query_flags that are passed to @@ -719,7 +731,6 @@ typedef struct { * * odp_tm_requirements_init() must be called to initialize any * odp_tm_requirements_t record before it is first used or assigned to. - * This is done to allow for vendor specific additions to this record. * * @param requirements A pointer to an odp_tm_requirements_t record which * is to be initialized. @@ -730,7 +741,6 @@ void odp_tm_requirements_init(odp_tm_requirements_t *requirements); * * odp_tm_egress_init() must be called to initialize any odp_tm_egress_t * record before it is first used or assigned to. - * This is done to allow for vendor specific additions to this record. * * @param egress A pointer to an odp_tm_egress_t record which * is to be initialized. @@ -940,9 +950,9 @@ int odp_tm_destroy(odp_tm_t tm); * The odp_tm_vlan_marking() function allows one to configure the TM egress * so as to have it set the one bit VLAN Drop Eligibility Indicator (DEI) * field (but only for pkts that already carry a VLAN tag) of a pkt based upon - * the final pkt (or shaper?) color assigned to the pkt when it reaches the - * egress node. When drop_eligible_enabled is false, then the given color has - * no effect on the VLAN fields. See IEEE 802.1q for more details. + * the final pkt color assigned to the pkt when it reaches the egress node. When + * drop_eligible_enabled is false, then the given color has no effect on the + * VLAN fields. See IEEE 802.1q for more details. * * Note that ALL ODP implementations are required to SUCCESSFULLY handle all * calls to this function with drop_eligible_enabled == FALSE - i.e. must @@ -1037,24 +1047,11 @@ int odp_tm_drop_prec_marking(odp_tm_t tm, /* Shaper profile types and functions * -------------------------------------------------------- */ -/** Possible values of running the shaper algorithm. ODP_TM_SHAPER_GREEN - * means that the traffic is within the commit specification (rate and burst - * size), ODP_TM_SHAPER_YELLOW means that the traffic is within the peak - * specification (rate and burst size) and ODP_TM_SHAPER_RED means that the - * traffic is exceeding both its commit and peak specifications. Note that - * packets can also have an assigned <b> packet color</b> of ODP_PACKET_GREEN, - * ODP_PACKET_YELLOW or ODP_PACKET_RED which has a different meaning and - * purpose than the shaper colors. - */ -typedef enum { - ODP_TM_SHAPER_GREEN, ODP_TM_SHAPER_YELLOW, ODP_TM_SHAPER_RED -} odp_tm_shaper_color_t; - -/** The odp_tm_shaper_params_t record type is used to supply the parameters - * associated with a shaper profile. Since it is expected that - * implementations might augment this record type with platform specific - * additional fields - it is required that odp_tm_shaper_params_init() be - * called on variables of this type before any of the fields are filled in. +/** + * TM shaper parameters + * + * Use odp_tm_shaper_params_init() to initialize parameters into their default + * values. */ typedef struct { /** The committed information rate for this shaper profile. The units @@ -1123,12 +1120,12 @@ typedef struct { odp_bool_t packet_mode; } odp_tm_shaper_params_t; -/** odp_tm_shaper_params_init() must be called to initialize any - * odp_tm_shaper_params_t record before it is first used or assigned to. - * The parameters are initialized to their default values. +/** + * Initialize TM shaper parameters * - * @param params A pointer to an odp_tm_shaper_params_t record which - * is to be initialized. + * Initialize an odp_tm_shaper_params_t to its default values for all fields. + * + * @param params Address of the odp_tm_shaper_params_t to be initialized */ void odp_tm_shaper_params_init(odp_tm_shaper_params_t *params); @@ -1176,8 +1173,8 @@ int odp_tm_shaper_params_read(odp_tm_shaper_t shaper_profile, /** odp_tm_shaper_params_update() "sets" the current set of values associated * with the specified shaper profile object. In addition, this call has the - * effect that all tm_input's and tm_nodes that are associated (attached?) - * with this shaper profile object will be updated with the new values. + * effect that all tm_input's and tm_nodes that are associated with this shaper + * profile object will be updated with the new values. * * @param shaper_profile Specifies the shaper profile object whose * values are to be set. @@ -1215,11 +1212,11 @@ typedef enum { ODP_TM_FRAME_BASED_WEIGHTS /**< Ignore the packet length */ } odp_tm_sched_mode_t; -/** The odp_tm_sched_params_t record type is used to supply the parameters - * associated with a scheduler profile. Since it is expected that - * implementations might augment this record type with platform specific - * additional fields - it is required that odp_tm_sched_params_init() be - * called on variables of this type before any of the fields are filled in. +/** + * TM scheduler parameters + * + * Use odp_tm_sched_params_init() to initialize parameters into their default + * values. */ typedef struct { /** sched_modes indicates whether weighted scheduling should be used @@ -1237,12 +1234,12 @@ typedef struct { uint32_t sched_weights[ODP_TM_MAX_PRIORITIES]; } odp_tm_sched_params_t; -/** odp_tm_sched_params_init() must be called to initialize any - * odp_tm_sched_params_t record before it is first used or assigned to. - * The parameters are initialized to their default values. +/** + * Initialize TM scheduler parameters * - * @param params A pointer to an odp_tm_sched_params_t record which - * is to be initialized. + * Initialize an odp_tm_sched_params_t to its default values for all fields. + * + * @param params Address of the odp_tm_sched_params_t to be initialized */ void odp_tm_sched_params_init(odp_tm_sched_params_t *params); @@ -1289,8 +1286,8 @@ int odp_tm_sched_params_read(odp_tm_sched_t sched_profile, /** odp_tm_sched_params_update() "sets" the current set of values associated * with the specified scheduler profile object. In addition, this call has - * the effect that all tm_nodes that are associated (attached?) with this - * Scheduler profile object will be updated with the new values. + * the effect that all tm_nodes that are associated with this scheduler profile + * object will be updated with the new values. * * @param sched_profile Specifies the Scheduler profile object whose * values are to be set. @@ -1315,11 +1312,11 @@ odp_tm_sched_t odp_tm_sched_lookup(const char *name); /* Queue Threshold Profiles - types and functions * -------------------------------------------------------- */ -/** The odp_tm_threshold_params_t record type is used to supply the parameters - * associated with a queue thresholds profile. Since it is expected that - * implementations might augment this record type with platform specific - * additional fields - it is required that odp_tm_threshold_params_init() be - * called on variables of this type before any of the fields are filled in +/** + * TM threshold parameters + * + * Use odp_tm_threshold_params_init() to initialize parameters into their + * default values. */ typedef struct { uint64_t max_pkts; /**< max pkt cnt for this threshold profile */ @@ -1332,12 +1329,12 @@ typedef struct { odp_bool_t enable_max_bytes; } odp_tm_threshold_params_t; -/** odp_tm_threshold_params_init() must be called to initialize any - * odp_tm_threshold_params_t record before it is first used or assigned to. - * The parameters are initialized to their default values. +/** + * Initialize TM threshold parameters * - * @param params A pointer to an odp_tm_threshold_params_t record which - * is to be initialized. + * Initialize an odp_tm_threshold_params_t to its default values for all fields. + * + * @param params Address of the odp_tm_threshold_params_t to be initialized */ void odp_tm_threshold_params_init(odp_tm_threshold_params_t *params); @@ -1387,8 +1384,8 @@ int odp_tm_thresholds_params_read(odp_tm_threshold_t threshold_profile, /** odp_tm_thresholds_params_update() "sets" the current set of values * associated with the specified queue thresholds profile object. In addition, * this call has the effect that all tm_input's and tm_nodes that are - * associated (attached?) with this queue thresholds profile object will be - * updated with the new values. + * associated with this queue thresholds profile object will be updated with the + * new values. * * @param threshold_profile Specifies the queue thresholds profile * object whose values are to be set. @@ -1413,11 +1410,11 @@ odp_tm_threshold_t odp_tm_thresholds_lookup(const char *name); /* WRED Profiles - types and functions * -------------------------------------------------------- */ -/** The odp_tm_wred_params_t record type is used to supply the parameters - * associated with a Random Early Detection profile. Since it is expected that - * implementations might augment this record type with platform specific - * additional fields - it is required that odp_tm_wred_params_init() be called - * on variables of this type before any of the fields are filled in. +/** + * TM WRED parameters + * + * Use odp_tm_wred_params_init() to initialize parameters into their default + * values. */ typedef struct { /** When min_threshold is set to zero then single-slope WRED is @@ -1471,12 +1468,12 @@ typedef struct { odp_bool_t use_byte_fullness; } odp_tm_wred_params_t; -/** odp_tm_wred_params_init() must be called to initialize any - * odp_tm_wred_params_t record before it is first used or assigned to. - * The parameters are initialized to their default values. +/** + * Initialize TM WRED parameters * - * @param params A pointer to an odp_tm_wred_params_t record which - * is to be initialized. + * Initialize an odp_tm_wred_params_t to its default values for all fields. + * + * @param params Address of the odp_tm_wred_params_t to be initialized */ void odp_tm_wred_params_init(odp_tm_wred_params_t *params); @@ -1524,8 +1521,8 @@ int odp_tm_wred_params_read(odp_tm_wred_t wred_profile, /** odp_tm_wred_params_update() "sets" the current set of values associated * with the specified WRED profile object. In addition, this call has the - * effect that all tm_input's and tm_nodes that are associated (attached?) - * with this WRED profile object will be updated with the new values. + * effect that all tm_input's and tm_nodes that are associated with this WRED + * profile object will be updated with the new values. * * @param wred_profile Specifies the WRED profile object whose * values are to be set. @@ -1547,12 +1544,11 @@ int odp_tm_wred_params_update(odp_tm_wred_t wred_profile, */ odp_tm_wred_t odp_tm_wred_lookup(const char *name); -/** The odp_tm_node_params_t record type is used to hold extra parameters when - * calling the odp_tm_node_create() function. Many of these fields are - * optional EXCEPT for max_fanin and level. Also since it is expected that - * implementations might augment this record type with platform specific - * additional fields - it is required that odp_tm_node_params_init() be called - * on variables of this type before any of the fields are filled in. +/** + * TM node parameters + * + * Many of these fields are optional EXCEPT for max_fanin and level. Use + * odp_tm_node_params_init() to initialize parameters into their default values. */ typedef struct { /** The user_context field is an generic pointer that the user can @@ -1571,9 +1567,9 @@ typedef struct { odp_tm_shaper_t shaper_profile; /** The threshold profile to be used in setting the max queue fullness - * for WRED and/or tail drop? Can be ODP_TM_INVALID and can also be - * set and changed post-creation via odp_tm_node_threshold_config(). - * The default value is ODP_TM_INVALID. */ + * for WRED and/or tail drop. Can be ODP_TM_INVALID and can also be set + * and changed post-creation via odp_tm_node_threshold_config(). The + * default value is ODP_TM_INVALID. */ odp_tm_threshold_t threshold_profile; /** The WRED profile(s) to be associated with this tm_node. Any or @@ -1606,12 +1602,12 @@ typedef struct { uint8_t priority; } odp_tm_node_params_t; -/** odp_tm_node_params_init() must be called to initialize any - * odp_tm_node_params_t record before it is first used or assigned to. - * The parameters are initialized to their default values. +/** + * Initialize TM node parameters * - * @param params A pointer to an odp_tm_node_params_t record which - * is to be initialized. + * Initialize an odp_tm_node_params_t to its default values for all fields. + * + * @param params Address of the odp_tm_node_params_t to be initialized */ void odp_tm_node_params_init(odp_tm_node_params_t *params); @@ -1627,8 +1623,7 @@ void odp_tm_node_params_init(odp_tm_node_params_t *params); * @param name Optional name that can be used later later to find this * same odp_tm_node_t. Can be NULL, otherwise must be * unique across all odp_tm_node objects. - * @param params A pointer to a record holding (an extensible) set of - * properties/attributes of this tm_node. + * @param params TM node parameters. * * @return Returns ODP_TM_INVALID upon failure, otherwise returns * a valid odp_tm_node_t handle if successful. @@ -1739,12 +1734,11 @@ void *odp_tm_node_context(odp_tm_node_t tm_node); */ int odp_tm_node_context_set(odp_tm_node_t tm_node, void *user_context); -/** The odp_tm_queue_params_t record type is used to hold extra parameters - * when calling the odp_tm_queue_create() function. Many of these fields are - * optional EXCEPT for priority. Also since it is expected that - * implementations might augment this record type with platform specific - * additional fields - it is required that odp_tm_queue_params_init() be - * called on variables of this type before any of the fields are filled in. +/** + * TM queue parameters + * + * Use odp_tm_queue_params_init() to initialize parameters into their default + * values. */ typedef struct { /** The user_context field is an generic pointer that the user can @@ -1758,9 +1752,13 @@ typedef struct { odp_tm_shaper_t shaper_profile; /** The threshold profile to be used in setting the max queue fullness - * for WRED and/or tail drop? Can be ODP_TM_INVALID and can also be - * set and changed post-creation via odp_tm_queue_threshold_config(). - * The default value is ODP_TM_INVALID. */ + * for WRED and/or tail drop. Can be ODP_TM_INVALID and can also be set + * and changed post-creation via odp_tm_queue_threshold_config(). The + * default value is ODP_TM_INVALID. + * + * One can specify the maximum queue limits either as a maximum number + * of packets in the queue or as a maximum number of bytes in the queue, + * or if both are specified, then whichever limit is hit first. */ odp_tm_threshold_t threshold_profile; /** The WRED profile(s) to be associated with this tm_queue. Any or @@ -1782,27 +1780,24 @@ typedef struct { odp_bool_t ordered_enqueue; } odp_tm_queue_params_t; -/** odp_tm_queue_params_init() must be called to initialize any - * odp_tm_queue_params_t record before it is first used or assigned to. - * The parameters are initialized to their default values. +/** + * Initialize TM queue parameters * - * @param params A pointer to an odp_tm_queue_params_t record which - * is to be initialized. + * Initialize an odp_tm_queue_params_t to its default values for all fields. + * + * @param params Address of the odp_tm_queue_params_t to be initialized */ void odp_tm_queue_params_init(odp_tm_queue_params_t *params); -/** Create an tm_queue object. One can specify the maximum queue limits - * either as a maximum number of packets in the queue OR as a maximum number - * of bytes in the queue, or if both are specified, then whichever limit is - * hit first. Note that in the case of specifying the maximum queue memory - * size as bytes, the system is free to instead convert this byte value into a - * number of buffers and instead limit the queue memory usage by buffer counts - * versus strictly using byte counts. +/** + * TM queue create + * + * Create a TM queue according to the queue parameters. Use + * odp_tm_queue_params_init() to initialize parameters into their default values. * * @param tm Handle of the TM system into which this odp_tm_queue object is * created. - * @param params A pointer to a record holding (an extensible) set of - * properties/attributes of this tm_queue. + * @param params TM queue parameters. * * @return Returns ODP_TM_INVALID upon failure, otherwise a valid * odp_tm_queue_t handle. @@ -1965,18 +1960,16 @@ int odp_tm_queue_disconnect(odp_tm_queue_t tm_queue); /* Input API * -------------------------------------------------------- */ -/** The odp_tm_enq() function is used to add packets to a given TM system. - * Note that the System Metadata associated with the pkt needed by the TM - * system is (a) a drop_eligible bit, (b) a two bit "pkt_color", (c) a 16-bit - * pkt_len, and MAYBE? (d) a signed 8-bit shaper_len_adjust. +/** Send packet to TM system + * + * Note that the packet metadata utilized by the TM system is (a) + * drop_eligible, (b) pkt_color, (c) pkt_len, and (d) shaper_len_adjust. * * If there is a non-zero shaper_len_adjust, then it is added to the pkt_len * after any non-zero shaper_len_adjust that is part of the shaper profile. * * The pkt_color bits are a result of some earlier Metering/Marking/Policing - * processing (typically ingress based), and should not be confused with the - * shaper_color produced from the TM shaper entities within the tm_inputs and - * tm_nodes. + * processing. * * @param tm_queue Specifies the tm_queue (and indirectly the TM system). * @param pkt Handle to a packet. @@ -2023,8 +2016,8 @@ int odp_tm_enq_multi_lso(odp_tm_queue_t tm_queue, const odp_packet_t packets[], const odp_packet_lso_opt_t *lso_opt); /** The odp_tm_enq_with_cnt() function behaves identically to odp_tm_enq(), - * except that it also returns (an approximation to?) the current tm_queue - * packet queue count. + * except that it also returns the current tm_queue packet queue count (may be + * an approximation). * * @param tm_queue Specifies the tm_queue (and indirectly the TM system). * @param pkt Handle to a packet. @@ -2194,7 +2187,7 @@ int odp_tm_queue_info(odp_tm_queue_t tm_queue, odp_tm_queue_info_t *info); */ #define ODP_TM_QUERY_PKT_CNT 0x01 /**< The total_pkt_cnt value */ #define ODP_TM_QUERY_BYTE_CNT 0x02 /**< The total_byte_cnt value */ -#define ODP_TM_QUERY_THRESHOLDS 0x04 /**< The thresholds??? */ +#define ODP_TM_QUERY_THRESHOLDS 0x04 /**< The threshold values */ /** The odp_tm_query_info_t record type is used to return the various counts * as requested by functions like odp_tm_queue_query() and diff --git a/include/odp/api/timer.h b/include/odp/api/timer.h index 3041594f5..356f7e951 100644 --- a/include/odp/api/timer.h +++ b/include/odp/api/timer.h @@ -17,6 +17,8 @@ extern "C" { #endif +#include <odp/api/abi/timer.h> + #include <odp/api/spec/timer.h> #ifdef __cplusplus diff --git a/include/odp/arch/arm32-linux/odp/api/abi/event_types.h b/include/odp/arch/arm32-linux/odp/api/abi/event_types.h new file mode 100644 index 000000000..a8321920c --- /dev/null +++ b/include/odp/arch/arm32-linux/odp/api/abi/event_types.h @@ -0,0 +1,7 @@ +/* Copyright (c) 2022, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <odp/api/abi-default/event_types.h> diff --git a/include/odp/arch/arm32-linux/odp/api/abi/timer.h b/include/odp/arch/arm32-linux/odp/api/abi/timer.h new file mode 100644 index 000000000..0351468d3 --- /dev/null +++ b/include/odp/arch/arm32-linux/odp/api/abi/timer.h @@ -0,0 +1,7 @@ +/* Copyright (c) 2022, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <odp/api/abi-default/timer.h> diff --git a/include/odp/arch/arm64-linux/odp/api/abi/event_types.h b/include/odp/arch/arm64-linux/odp/api/abi/event_types.h new file mode 100644 index 000000000..a8321920c --- /dev/null +++ b/include/odp/arch/arm64-linux/odp/api/abi/event_types.h @@ -0,0 +1,7 @@ +/* Copyright (c) 2022, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <odp/api/abi-default/event_types.h> diff --git a/include/odp/arch/arm64-linux/odp/api/abi/timer.h b/include/odp/arch/arm64-linux/odp/api/abi/timer.h new file mode 100644 index 000000000..0351468d3 --- /dev/null +++ b/include/odp/arch/arm64-linux/odp/api/abi/timer.h @@ -0,0 +1,7 @@ +/* Copyright (c) 2022, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <odp/api/abi-default/timer.h> diff --git a/include/odp/arch/default-linux/odp/api/abi/event_types.h b/include/odp/arch/default-linux/odp/api/abi/event_types.h new file mode 100644 index 000000000..a8321920c --- /dev/null +++ b/include/odp/arch/default-linux/odp/api/abi/event_types.h @@ -0,0 +1,7 @@ +/* Copyright (c) 2022, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <odp/api/abi-default/event_types.h> diff --git a/include/odp/arch/default-linux/odp/api/abi/timer.h b/include/odp/arch/default-linux/odp/api/abi/timer.h new file mode 100644 index 000000000..0351468d3 --- /dev/null +++ b/include/odp/arch/default-linux/odp/api/abi/timer.h @@ -0,0 +1,7 @@ +/* Copyright (c) 2022, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <odp/api/abi-default/timer.h> diff --git a/include/odp/arch/power64-linux/odp/api/abi/event_types.h b/include/odp/arch/power64-linux/odp/api/abi/event_types.h new file mode 100644 index 000000000..a8321920c --- /dev/null +++ b/include/odp/arch/power64-linux/odp/api/abi/event_types.h @@ -0,0 +1,7 @@ +/* Copyright (c) 2022, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <odp/api/abi-default/event_types.h> diff --git a/include/odp/arch/power64-linux/odp/api/abi/timer.h b/include/odp/arch/power64-linux/odp/api/abi/timer.h new file mode 100644 index 000000000..0351468d3 --- /dev/null +++ b/include/odp/arch/power64-linux/odp/api/abi/timer.h @@ -0,0 +1,7 @@ +/* Copyright (c) 2022, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <odp/api/abi-default/timer.h> diff --git a/include/odp/arch/x86_32-linux/odp/api/abi/event_types.h b/include/odp/arch/x86_32-linux/odp/api/abi/event_types.h new file mode 100644 index 000000000..a8321920c --- /dev/null +++ b/include/odp/arch/x86_32-linux/odp/api/abi/event_types.h @@ -0,0 +1,7 @@ +/* Copyright (c) 2022, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <odp/api/abi-default/event_types.h> diff --git a/include/odp/arch/x86_32-linux/odp/api/abi/timer.h b/include/odp/arch/x86_32-linux/odp/api/abi/timer.h new file mode 100644 index 000000000..0351468d3 --- /dev/null +++ b/include/odp/arch/x86_32-linux/odp/api/abi/timer.h @@ -0,0 +1,7 @@ +/* Copyright (c) 2022, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <odp/api/abi-default/timer.h> diff --git a/include/odp/arch/x86_64-linux/odp/api/abi/event_types.h b/include/odp/arch/x86_64-linux/odp/api/abi/event_types.h new file mode 100644 index 000000000..a8321920c --- /dev/null +++ b/include/odp/arch/x86_64-linux/odp/api/abi/event_types.h @@ -0,0 +1,7 @@ +/* Copyright (c) 2022, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <odp/api/abi-default/event_types.h> diff --git a/include/odp/arch/x86_64-linux/odp/api/abi/timer.h b/include/odp/arch/x86_64-linux/odp/api/abi/timer.h new file mode 100644 index 000000000..0351468d3 --- /dev/null +++ b/include/odp/arch/x86_64-linux/odp/api/abi/timer.h @@ -0,0 +1,7 @@ +/* Copyright (c) 2022, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <odp/api/abi-default/timer.h> diff --git a/platform/linux-generic/Makefile.am b/platform/linux-generic/Makefile.am index 47e5cf759..d4e94f677 100644 --- a/platform/linux-generic/Makefile.am +++ b/platform/linux-generic/Makefile.am @@ -49,7 +49,9 @@ odpapiplatinclude_HEADERS = \ include/odp/api/plat/sync_inlines.h \ include/odp/api/plat/thread_inlines.h \ include/odp/api/plat/ticketlock_inlines.h \ - include/odp/api/plat/time_inlines.h + include/odp/api/plat/time_inlines.h \ + include/odp/api/plat/timer_inlines.h \ + include/odp/api/plat/timer_inline_types.h odpapiabiarchinclude_HEADERS += \ include-abi/odp/api/abi/align.h \ @@ -65,6 +67,7 @@ odpapiabiarchinclude_HEADERS += \ include-abi/odp/api/abi/dma_types.h \ include-abi/odp/api/abi/errno.h \ include-abi/odp/api/abi/event.h \ + include-abi/odp/api/abi/event_types.h \ include-abi/odp/api/abi/hash.h \ include-abi/odp/api/abi/init.h \ include-abi/odp/api/abi/ipsec.h \ @@ -95,6 +98,7 @@ odpapiabiarchinclude_HEADERS += \ include-abi/odp/api/abi/thrmask.h \ include-abi/odp/api/abi/ticketlock.h \ include-abi/odp/api/abi/time.h \ + include-abi/odp/api/abi/timer.h \ include-abi/odp/api/abi/timer_types.h \ include-abi/odp/api/abi/traffic_mngr.h \ include-abi/odp/api/abi/version.h @@ -126,6 +130,7 @@ noinst_HEADERS = \ include/odp_packet_dpdk.h \ include/odp_packet_internal.h \ include/odp_packet_io_internal.h \ + include/odp_parse_internal.h \ include/odp_socket_common.h \ include/odp_packet_io_stats_common.h \ include/odp_packet_io_stats.h \ @@ -160,6 +165,7 @@ noinst_HEADERS = \ include/odp_timer_internal.h \ include/odp_timer_wheel_internal.h \ include/odp_traffic_mngr_internal.h \ + include/odp_types_internal.h \ include/odp_event_vector_internal.h \ include/protocols/eth.h \ include/protocols/ip.h \ @@ -201,6 +207,7 @@ __LIB__libodp_linux_la_SOURCES = \ odp_packet_vector.c \ odp_packet_flags.c \ odp_packet_io.c \ + odp_parse.c \ odp_pkt_queue.c \ odp_pool.c \ odp_queue_basic.c \ @@ -279,7 +286,8 @@ __LIB__libodp_linux_la_SOURCES += \ odp_sync_api.c \ odp_thread_api.c \ odp_ticketlock_api.c \ - odp_time_api.c + odp_time_api.c \ + odp_timer_api.c endif if ARCH_IS_ARM diff --git a/platform/linux-generic/arch/aarch64/odp_atomic.h b/platform/linux-generic/arch/aarch64/odp_atomic.h index 445d91e28..d3b8ea4dc 100644 --- a/platform/linux-generic/arch/aarch64/odp_atomic.h +++ b/platform/linux-generic/arch/aarch64/odp_atomic.h @@ -12,6 +12,7 @@ #error This file should not be included directly, please include odp_cpu.h #endif +#include <odp_types_internal.h> #include <limits.h> #ifdef CONFIG_DMBSTR @@ -36,22 +37,18 @@ do { \ #define LL_MO(mo) (HAS_ACQ((mo)) ? __ATOMIC_ACQUIRE : __ATOMIC_RELAXED) #define SC_MO(mo) (HAS_RLS((mo)) ? __ATOMIC_RELEASE : __ATOMIC_RELAXED) -/* Prevent warnings about ISO C not supporting __int128 */ -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wpedantic" - #ifndef __ARM_FEATURE_QRDMX /* Feature only available in v8.1a and beyond */ static inline bool -__lockfree_compare_exchange_16(register __int128 *var, __int128 *exp, - register __int128 neu, bool weak, int mo_success, +__lockfree_compare_exchange_16(register _odp_u128_t *var, _odp_u128_t *exp, + register _odp_u128_t neu, bool weak, int mo_success, int mo_failure) { (void)weak; /* Always do strong CAS or we can't perform atomic read */ /* Ignore memory ordering for failure, memory order for * success must be stronger or equal. */ (void)mo_failure; - register __int128 old; - register __int128 expected; + register _odp_u128_t old; + register _odp_u128_t expected; int ll_mo = LL_MO(mo_success); int sc_mo = SC_MO(mo_success); @@ -66,10 +63,10 @@ __lockfree_compare_exchange_16(register __int128 *var, __int128 *exp, return old == expected; } -static inline __int128 __lockfree_exchange_16(__int128 *var, __int128 neu, - int mo) +static inline _odp_u128_t __lockfree_exchange_16(_odp_u128_t *var, + _odp_u128_t neu, int mo) { - register __int128 old; + register _odp_u128_t old; int ll_mo = LL_MO(mo); int sc_mo = SC_MO(mo); @@ -81,10 +78,10 @@ static inline __int128 __lockfree_exchange_16(__int128 *var, __int128 neu, return old; } -static inline __int128 __lockfree_fetch_and_16(__int128 *var, __int128 mask, - int mo) +static inline _odp_u128_t __lockfree_fetch_and_16(_odp_u128_t *var, + _odp_u128_t mask, int mo) { - register __int128 old; + register _odp_u128_t old; int ll_mo = LL_MO(mo); int sc_mo = SC_MO(mo); @@ -96,10 +93,10 @@ static inline __int128 __lockfree_fetch_and_16(__int128 *var, __int128 mask, return old; } -static inline __int128 __lockfree_fetch_or_16(__int128 *var, __int128 mask, - int mo) +static inline _odp_u128_t __lockfree_fetch_or_16(_odp_u128_t *var, + _odp_u128_t mask, int mo) { - register __int128 old; + register _odp_u128_t old; int ll_mo = LL_MO(mo); int sc_mo = SC_MO(mo); @@ -113,8 +110,8 @@ static inline __int128 __lockfree_fetch_or_16(__int128 *var, __int128 mask, #else -static inline __int128_t cas_u128(__int128_t *ptr, __int128_t old_val, - __int128_t new_val, int mo) +static inline _odp_u128_t cas_u128(_odp_u128_t *ptr, _odp_u128_t old_val, + _odp_u128_t new_val, int mo) { /* CASP instructions require that the first register number is paired */ register uint64_t old0 __asm__ ("x0"); @@ -151,18 +148,18 @@ static inline __int128_t cas_u128(__int128_t *ptr, __int128_t old_val, abort(); } - return ((__int128)old0) | (((__int128)old1) << 64); + return ((_odp_u128_t)old0) | (((_odp_u128_t)old1) << 64); } static inline bool -__lockfree_compare_exchange_16(register __int128 *var, __int128 *exp, - register __int128 neu, bool weak, int mo_success, +__lockfree_compare_exchange_16(register _odp_u128_t *var, _odp_u128_t *exp, + register _odp_u128_t neu, bool weak, int mo_success, int mo_failure) { (void)weak; (void)mo_failure; - __int128 old; - __int128 expected; + _odp_u128_t old; + _odp_u128_t expected; expected = *exp; old = cas_u128(var, expected, neu, mo_success); @@ -170,11 +167,11 @@ __lockfree_compare_exchange_16(register __int128 *var, __int128 *exp, return old == expected; } -static inline __int128 __lockfree_exchange_16(__int128 *var, __int128 neu, - int mo) +static inline _odp_u128_t __lockfree_exchange_16(_odp_u128_t *var, + _odp_u128_t neu, int mo) { - __int128 old; - __int128 expected; + _odp_u128_t old; + _odp_u128_t expected; do { expected = *var; @@ -183,11 +180,11 @@ static inline __int128 __lockfree_exchange_16(__int128 *var, __int128 neu, return old; } -static inline __int128 __lockfree_fetch_and_16(__int128 *var, __int128 mask, - int mo) +static inline _odp_u128_t __lockfree_fetch_and_16(_odp_u128_t *var, + _odp_u128_t mask, int mo) { - __int128 old; - __int128 expected; + _odp_u128_t old; + _odp_u128_t expected; do { expected = *var; @@ -196,11 +193,11 @@ static inline __int128 __lockfree_fetch_and_16(__int128 *var, __int128 mask, return old; } -static inline __int128 __lockfree_fetch_or_16(__int128 *var, __int128 mask, - int mo) +static inline _odp_u128_t __lockfree_fetch_or_16(_odp_u128_t *var, + _odp_u128_t mask, int mo) { - __int128 old; - __int128 expected; + _odp_u128_t old; + _odp_u128_t expected; do { expected = *var; @@ -211,9 +208,9 @@ static inline __int128 __lockfree_fetch_or_16(__int128 *var, __int128 mask, #endif /* __ARM_FEATURE_QRDMX */ -static inline __int128 __lockfree_load_16(__int128 *var, int mo) +static inline _odp_u128_t __lockfree_load_16(_odp_u128_t *var, int mo) { - __int128 old = *var; /* Possibly torn read */ + _odp_u128_t old = *var; /* Possibly torn read */ /* Do CAS to ensure atomicity * Either CAS succeeds (writing back the same value) @@ -223,19 +220,17 @@ static inline __int128 __lockfree_load_16(__int128 *var, int mo) return old; } -__extension__ typedef unsigned __int128 _u128_t; - -static inline _u128_t lockfree_load_u128(_u128_t *atomic) +static inline _odp_u128_t lockfree_load_u128(_odp_u128_t *atomic) { - return __lockfree_load_16((__int128 *)atomic, __ATOMIC_RELAXED); + return __lockfree_load_16((_odp_u128_t *)atomic, __ATOMIC_RELAXED); } -static inline int lockfree_cas_acq_rel_u128(_u128_t *atomic, - _u128_t old_val, - _u128_t new_val) +static inline int lockfree_cas_acq_rel_u128(_odp_u128_t *atomic, + _odp_u128_t old_val, + _odp_u128_t new_val) { - return __lockfree_compare_exchange_16((__int128 *)atomic, - (__int128 *)&old_val, + return __lockfree_compare_exchange_16((_odp_u128_t *)atomic, + (_odp_u128_t *)&old_val, new_val, 0, __ATOMIC_ACQ_REL, @@ -249,7 +244,7 @@ static inline int lockfree_check_u128(void) /** Atomic bit set operations with memory ordering */ #if defined(__SIZEOF_INT128__) && __SIZEOF_INT128__ == 16 -typedef __int128 bitset_t; +typedef _odp_u128_t bitset_t; #define ATOM_BITSET_SIZE (CHAR_BIT * __SIZEOF_INT128__) #elif __GCC_ATOMIC_LLONG_LOCK_FREE == 2 && \ @@ -292,11 +287,9 @@ static inline bitset_t bitset_mask(uint32_t bit) if (bit < 64) return 1ULL << bit; else - return (unsigned __int128)(1ULL << (bit - 64)) << 64; + return (_odp_u128_t)(1ULL << (bit - 64)) << 64; } -#pragma GCC diagnostic pop - #else #error Unsupported size of bit sets (ATOM_BITSET_SIZE) #endif diff --git a/platform/linux-generic/arch/aarch64/odp_crypto_armv8.c b/platform/linux-generic/arch/aarch64/odp_crypto_armv8.c index d719c6cfa..4d46846ce 100644 --- a/platform/linux-generic/arch/aarch64/odp_crypto_armv8.c +++ b/platform/linux-generic/arch/aarch64/odp_crypto_armv8.c @@ -1,5 +1,6 @@ /* Copyright (c) 2014-2018, Linaro Limited * Copyright (c) 2021, ARM Limited + * Copyright (c) 2022, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -37,9 +38,9 @@ /* * ARM crypto library may read up to 15 bytes past the end of input - * data and AAD. + * data and AAD and write up to 15 bytes past the end of output data. */ -#define OOB_READ_LEN 15 +#define OOB_WRITE_LEN 16 /* rounded up to 16 bytes for efficiency */ /* * Data buffer size must be a multiple of 16, because the ARM crypto @@ -80,9 +81,11 @@ static const odp_crypto_cipher_capability_t cipher_capa_aes_gcm[] = { static const odp_crypto_auth_capability_t auth_capa_null[] = { {.digest_len = 0, .key_len = 0, .aad_len = {.min = 0, .max = 0, .inc = 0} } }; +#define AES_GCM_TAG_LEN 16 + #ifdef __ARM_FEATURE_AES static const odp_crypto_auth_capability_t auth_capa_aes_gcm[] = { -{.digest_len = 16, .key_len = 0, .aad_len = {.min = 8, .max = 12, .inc = 4} } }; +{.digest_len = AES_GCM_TAG_LEN, .key_len = 0, .aad_len = {.min = 8, .max = 12, .inc = 4} } }; #endif /** Forward declaration of session structure */ @@ -92,9 +95,9 @@ typedef struct odp_crypto_generic_session_t odp_crypto_generic_session_t; * Algorithm handler function prototype */ typedef -odp_crypto_alg_err_t (*crypto_func_t)(odp_packet_t pkt, - const odp_crypto_packet_op_param_t *param, - odp_crypto_generic_session_t *session); +odp_bool_t (*crypto_func_t)(odp_packet_t pkt, + const odp_crypto_packet_op_param_t *param, + odp_crypto_generic_session_t *session); /** * Per crypto session data structure @@ -105,15 +108,12 @@ struct odp_crypto_generic_session_t { /* Session creation parameters */ odp_crypto_session_param_t p; - odp_bool_t do_cipher_first; - struct { #if ODP_DEPRECATED_API /* Copy of session IV data */ uint8_t iv_data[ARM_CRYPTO_MAX_IV_LENGTH]; #endif uint8_t key_data[ARM_CRYPTO_MAX_CIPHER_KEY_LENGTH]; - crypto_func_t func; } cipher; struct { @@ -121,11 +121,10 @@ struct odp_crypto_generic_session_t { #if ODP_DEPRECATED_API uint8_t iv_data[ARM_CRYPTO_MAX_IV_LENGTH]; #endif - crypto_func_t func; } auth; + crypto_func_t func; unsigned int idx; - armv8_cipher_constants_t cc; }; @@ -141,7 +140,6 @@ static odp_crypto_global_t *global; typedef struct crypto_local_t { uint8_t buffer[ARM_CRYPTO_MAX_DATA_LENGTH]; - uint8_t digest[ARM_CRYPTO_MAX_DIGEST_LENGTH]; } crypto_local_t; static __thread crypto_local_t local; @@ -176,18 +174,60 @@ void free_session(odp_crypto_generic_session_t *session) odp_spinlock_unlock(&global->lock); } -static odp_crypto_alg_err_t +static +odp_crypto_packet_result_t *get_op_result_from_packet(odp_packet_t pkt) +{ + odp_packet_hdr_t *hdr = packet_hdr(pkt); + + return &hdr->crypto_op_result; +} + +static inline void set_crypto_op_result(odp_packet_t pkt, + odp_crypto_alg_err_t cipher_err, + odp_crypto_alg_err_t auth_err) +{ + odp_crypto_packet_result_t *op_result; + + op_result = get_op_result_from_packet(pkt); + op_result->cipher_status.alg_err = cipher_err; + op_result->cipher_status.hw_err = ODP_CRYPTO_HW_ERR_NONE; + op_result->auth_status.alg_err = auth_err; + op_result->auth_status.hw_err = ODP_CRYPTO_HW_ERR_NONE; + op_result->ok = (cipher_err == ODP_CRYPTO_ALG_ERR_NONE && + auth_err == ODP_CRYPTO_ALG_ERR_NONE); +} + +static inline void set_crypto_op_result_ok(odp_packet_t pkt) +{ + set_crypto_op_result(pkt, + ODP_CRYPTO_ALG_ERR_NONE, + ODP_CRYPTO_ALG_ERR_NONE); +} + +static odp_bool_t null_crypto_routine(odp_packet_t pkt ODP_UNUSED, const odp_crypto_packet_op_param_t *param ODP_UNUSED, odp_crypto_generic_session_t *session ODP_UNUSED) { - return ODP_CRYPTO_ALG_ERR_NONE; + set_crypto_op_result_ok(pkt); + return true; +} + +static inline void copy_aad(uint8_t *dst, uint8_t *src, uint32_t len) +{ + ODP_ASSERT(len == 8 || len == 12); + + /* Use constant length memcpy for better optimization result */ + if (len == 8) + memcpy(dst, src, 8); + else + memcpy(dst, src, 12); } static -odp_crypto_alg_err_t aes_gcm_encrypt(odp_packet_t pkt, - const odp_crypto_packet_op_param_t *param, - odp_crypto_generic_session_t *session) +odp_bool_t aes_gcm_encrypt(odp_packet_t pkt, + const odp_crypto_packet_op_param_t *param, + odp_crypto_generic_session_t *session) { armv8_cipher_state_t cs = { .counter = { @@ -200,12 +240,14 @@ odp_crypto_alg_err_t aes_gcm_encrypt(odp_packet_t pkt, uint64_t aad_bit_length = session->p.auth_aad_len * 8; uint32_t in_pos = param->cipher_range.offset; uint32_t in_len = param->cipher_range.length; - int ret = 0; + odp_bool_t continuous_data; + uint16_t saved_tail[OOB_WRITE_LEN]; + uint8_t tag[AES_GCM_TAG_LEN]; + int rc; /* Fail early if cipher_range is too large */ - if (in_len > ARM_CRYPTO_MAX_DATA_LENGTH) { + if (odp_unlikely(in_len > ARM_CRYPTO_MAX_DATA_LENGTH)) { ODP_DBG("ARM Crypto: Packet size too large for requested operation\n"); - ret = -1; goto err; } @@ -223,9 +265,9 @@ odp_crypto_alg_err_t aes_gcm_encrypt(odp_packet_t pkt, cs.constants = &session->cc; - if (armv8_aes_gcm_set_counter(iv_ptr, iv_bit_length, &cs) != 0) { + rc = armv8_aes_gcm_set_counter(iv_ptr, iv_bit_length, &cs); + if (odp_unlikely(rc)) { ODP_DBG("ARM Crypto: Failure while setting nonce\n"); - ret = -1; goto err; } @@ -233,46 +275,60 @@ odp_crypto_alg_err_t aes_gcm_encrypt(odp_packet_t pkt, * read it in 16 byte chunks. */ uint8_t aad[ARM_CRYPTO_MAX_AAD_LENGTH]; - memcpy(aad, param->aad_ptr, session->p.auth_aad_len); + copy_aad(aad, param->aad_ptr, session->p.auth_aad_len); uint32_t seg_len = 0; uint8_t *data = odp_packet_offset(pkt, in_pos, &seg_len, NULL); if (odp_unlikely(odp_packet_is_segmented(pkt)) || - odp_unlikely(odp_packet_tailroom(pkt) < OOB_READ_LEN)) { - /* Packet is segmented or it may not be safe to read beyond - * the end of packet data. Copy the cipher range to a + odp_unlikely(odp_packet_tailroom(pkt) < OOB_WRITE_LEN)) { + /* Packet is segmented or it may not be safe to read and write + * beyond the end of packet data. Copy the cipher range to a * contiguous buffer. */ odp_packet_copy_to_mem(pkt, in_pos, in_len, local.buffer); data = local.buffer; + continuous_data = false; + } else { + /* Save data that might get overwritten */ + memcpy(saved_tail, data + in_len, OOB_WRITE_LEN); + continuous_data = true; } - if (armv8_enc_aes_gcm_from_state(&cs, - aad, aad_bit_length, - data, plaintext_bit_length, - local.buffer, - local.digest) != 0) { + rc = armv8_enc_aes_gcm_from_state(&cs, + aad, aad_bit_length, + data, plaintext_bit_length, + data, + tag); + if (odp_unlikely(rc)) { ODP_DBG("ARM Crypto: AES GCM Encoding failed\n"); - ret = -1; goto err; } - odp_packet_copy_from_mem(pkt, param->cipher_range.offset, - param->cipher_range.length, local.buffer); + if (odp_likely(continuous_data)) { + memcpy(data + in_len, saved_tail, OOB_WRITE_LEN); + memcpy(data - in_pos + param->hash_result_offset, + tag, AES_GCM_TAG_LEN); + } else { + odp_packet_copy_from_mem(pkt, in_pos, in_len, data); + odp_packet_copy_from_mem(pkt, param->hash_result_offset, + AES_GCM_TAG_LEN, tag); + } - odp_packet_copy_from_mem(pkt, param->hash_result_offset, - session->p.auth_digest_len, local.digest); + set_crypto_op_result_ok(pkt); + return true; err: - return ret < 0 ? ODP_CRYPTO_ALG_ERR_DATA_SIZE : - ODP_CRYPTO_ALG_ERR_NONE; + set_crypto_op_result(pkt, + ODP_CRYPTO_ALG_ERR_DATA_SIZE, + ODP_CRYPTO_ALG_ERR_NONE); + return false; } static -odp_crypto_alg_err_t aes_gcm_decrypt(odp_packet_t pkt, - const odp_crypto_packet_op_param_t *param, - odp_crypto_generic_session_t *session) +odp_bool_t aes_gcm_decrypt(odp_packet_t pkt, + const odp_crypto_packet_op_param_t *param, + odp_crypto_generic_session_t *session) { armv8_cipher_state_t cs = { .counter = { @@ -280,18 +336,19 @@ odp_crypto_alg_err_t aes_gcm_decrypt(odp_packet_t pkt, } }; uint8_t *iv_ptr; - uint8_t tag[16]; + uint8_t tag[AES_GCM_TAG_LEN]; uint64_t iv_bit_length = session->p.cipher_iv_len * 8; uint64_t plaintext_bit_length = param->cipher_range.length * 8; uint64_t aad_bit_length = session->p.auth_aad_len * 8; uint32_t in_pos = param->cipher_range.offset; uint32_t in_len = param->cipher_range.length; - int ret = 0; + odp_bool_t continuous_data; + uint16_t saved_tail[OOB_WRITE_LEN]; + int rc; /* Fail early if cipher_range is too large */ - if (in_len > ARM_CRYPTO_MAX_DATA_LENGTH) { + if (odp_unlikely(in_len > ARM_CRYPTO_MAX_DATA_LENGTH)) { ODP_DBG("ARM Crypto: Packet size too large for requested operation\n"); - ret = -1; goto err; } @@ -309,51 +366,63 @@ odp_crypto_alg_err_t aes_gcm_decrypt(odp_packet_t pkt, cs.constants = &session->cc; - if (armv8_aes_gcm_set_counter(iv_ptr, iv_bit_length, &cs) != 0) { + rc = armv8_aes_gcm_set_counter(iv_ptr, iv_bit_length, &cs); + if (odp_unlikely(rc)) { ODP_DBG("ARM Crypto: Failure while setting nonce\n"); - ret = -1; goto err; } - /* Copy current hash bytes to a buffer and clear it */ - odp_packet_copy_to_mem(pkt, param->hash_result_offset, - session->p.auth_digest_len, tag); - /* Copy AAD in a stack to make sure that the ARM crypto library can * read it in 16 byte chunks. */ uint8_t aad[ARM_CRYPTO_MAX_AAD_LENGTH]; - memcpy(aad, param->aad_ptr, session->p.auth_aad_len); + copy_aad(aad, param->aad_ptr, session->p.auth_aad_len); uint32_t seg_len = 0; uint8_t *data = odp_packet_offset(pkt, in_pos, &seg_len, NULL); if (odp_unlikely(odp_packet_is_segmented(pkt)) || - odp_unlikely(odp_packet_tailroom(pkt) < OOB_READ_LEN)) { - /* Packet is segmented or it may not be safe to read beyond - * the end of packet data. Copy the cipher range to a + odp_unlikely(odp_packet_tailroom(pkt) < OOB_WRITE_LEN)) { + /* Packet is segmented or it may not be safe to read and write + * beyond the end of packet data. Copy the cipher range to a * contiguous buffer. */ odp_packet_copy_to_mem(pkt, in_pos, in_len, local.buffer); - data = local.buffer; + /* Copy tag from the packet to a buffer */ + odp_packet_copy_to_mem(pkt, param->hash_result_offset, + AES_GCM_TAG_LEN, tag); + continuous_data = false; + } else { + /* Save data that might get overwritten */ + memcpy(saved_tail, data + in_len, OOB_WRITE_LEN); + /* Copy tag from the packet to a buffer */ + memcpy(tag, data - in_pos + param->hash_result_offset, AES_GCM_TAG_LEN); + continuous_data = true; } - if (armv8_dec_aes_gcm_from_state(&cs, - aad, aad_bit_length, - data, plaintext_bit_length, - tag, - local.buffer) != 0) { + rc = armv8_dec_aes_gcm_from_state(&cs, + aad, aad_bit_length, + data, plaintext_bit_length, + tag, + data); + if (odp_unlikely(rc)) { ODP_DBG("ARM Crypto: AES GCM Decoding failed\n"); - ret = -1; goto err; } - odp_packet_copy_from_mem(pkt, param->cipher_range.offset, - param->cipher_range.length, local.buffer); + if (odp_likely(continuous_data)) + memcpy(data + in_len, saved_tail, OOB_WRITE_LEN); + else + odp_packet_copy_from_mem(pkt, in_pos, in_len, data); + + set_crypto_op_result_ok(pkt); + return true; err: - return ret < 0 ? ODP_CRYPTO_ALG_ERR_ICV_CHECK : - ODP_CRYPTO_ALG_ERR_NONE; + set_crypto_op_result(pkt, + ODP_CRYPTO_ALG_ERR_NONE, + ODP_CRYPTO_ALG_ERR_ICV_CHECK); + return false; } static int process_aes_gcm_param(odp_crypto_generic_session_t *session) @@ -376,9 +445,9 @@ static int process_aes_gcm_param(odp_crypto_generic_session_t *session) /* Set function */ if (ODP_CRYPTO_OP_ENCODE == session->p.op) - session->cipher.func = aes_gcm_encrypt; + session->func = aes_gcm_encrypt; else - session->cipher.func = aes_gcm_decrypt; + session->func = aes_gcm_decrypt; return 0; } @@ -520,16 +589,10 @@ odp_crypto_session_create(const odp_crypto_session_param_t *param, session->p.auth_iv.length); #endif - /* Derive order */ - if (ODP_CRYPTO_OP_ENCODE == param->op) - session->do_cipher_first = param->auth_cipher_text; - else - session->do_cipher_first = !param->auth_cipher_text; - /* Process based on cipher */ switch (param->cipher_alg) { case ODP_CIPHER_ALG_NULL: - session->cipher.func = null_crypto_routine; + session->func = null_crypto_routine; rc = 0; break; case ODP_CIPHER_ALG_AES_GCM: @@ -584,14 +647,15 @@ odp_crypto_session_create(const odp_crypto_session_param_t *param, /* Process based on auth */ switch (param->auth_alg) { case ODP_AUTH_ALG_NULL: - session->auth.func = null_crypto_routine; - rc = 0; + if (param->cipher_alg == ODP_CIPHER_ALG_NULL) + rc = 0; + else + rc = -1; break; case ODP_AUTH_ALG_AES_GCM: /* AES-GCM requires to do both auth and * cipher at the same time */ if (param->cipher_alg == ODP_CIPHER_ALG_AES_GCM) { - session->auth.func = null_crypto_routine; rc = 0; } else { rc = -1; @@ -630,9 +694,7 @@ int odp_crypto_session_destroy(odp_crypto_session_t session) return 0; } -/* - * Shim function around packet operation, can be used by other implementations. - */ +#if ODP_DEPRECATED_API int odp_crypto_operation(odp_crypto_op_param_t *param, odp_bool_t *posted, @@ -687,6 +749,7 @@ odp_crypto_operation(odp_crypto_op_param_t *param, return 0; } +#endif int _odp_crypto_init_global(void) { @@ -767,6 +830,7 @@ int _odp_crypto_term_local(void) return 0; } +#if ODP_DEPRECATED_API odp_crypto_compl_t odp_crypto_compl_from_event(odp_event_t ev) { /* This check not mandated by the API specification */ @@ -803,6 +867,7 @@ uint64_t odp_crypto_compl_to_u64(odp_crypto_compl_t hdl) { return _odp_pri(hdl); } +#endif /* ODP_DEPRECATED_API */ void odp_crypto_session_param_init(odp_crypto_session_param_t *param) { @@ -828,14 +893,6 @@ odp_event_t odp_crypto_packet_to_event(odp_packet_t pkt) return odp_packet_to_event(pkt); } -static -odp_crypto_packet_result_t *get_op_result_from_packet(odp_packet_t pkt) -{ - odp_packet_hdr_t *hdr = packet_hdr(pkt); - - return &hdr->crypto_op_result; -} - int odp_crypto_result(odp_crypto_packet_result_t *result, odp_packet_t packet) { @@ -856,18 +913,16 @@ int crypto_int(odp_packet_t pkt_in, odp_packet_t *pkt_out, const odp_crypto_packet_op_param_t *param) { - odp_crypto_alg_err_t rc_cipher = ODP_CRYPTO_ALG_ERR_NONE; - odp_crypto_alg_err_t rc_auth = ODP_CRYPTO_ALG_ERR_NONE; odp_crypto_generic_session_t *session; odp_bool_t allocated = false; odp_packet_t out_pkt = *pkt_out; - odp_crypto_packet_result_t *op_result; odp_packet_hdr_t *pkt_hdr; + odp_bool_t ok; session = (odp_crypto_generic_session_t *)(intptr_t)param->session; /* Resolve output buffer */ - if (ODP_PACKET_INVALID == out_pkt && + if (odp_unlikely(ODP_PACKET_INVALID == out_pkt) && ODP_POOL_INVALID != session->p.output_pool) { out_pkt = odp_packet_alloc(session->p.output_pool, odp_packet_len(pkt_in)); @@ -879,7 +934,7 @@ int crypto_int(odp_packet_t pkt_in, return -1; } - if (pkt_in != out_pkt) { + if (odp_unlikely(pkt_in != out_pkt)) { int ret; int md_copy; @@ -903,28 +958,12 @@ int crypto_int(odp_packet_t pkt_in, pkt_in = ODP_PACKET_INVALID; } - /* Invoke the functions */ - if (session->do_cipher_first) { - rc_cipher = session->cipher.func(out_pkt, param, session); - rc_auth = session->auth.func(out_pkt, param, session); - } else { - rc_auth = session->auth.func(out_pkt, param, session); - rc_cipher = session->cipher.func(out_pkt, param, session); - } + /* Invoke the crypto function */ + ok = session->func(out_pkt, param, session); - /* Fill in result */ packet_subtype_set(out_pkt, ODP_EVENT_PACKET_CRYPTO); - op_result = get_op_result_from_packet(out_pkt); - op_result->cipher_status.alg_err = rc_cipher; - op_result->cipher_status.hw_err = ODP_CRYPTO_HW_ERR_NONE; - op_result->auth_status.alg_err = rc_auth; - op_result->auth_status.hw_err = ODP_CRYPTO_HW_ERR_NONE; - op_result->ok = - (rc_cipher == ODP_CRYPTO_ALG_ERR_NONE) && - (rc_auth == ODP_CRYPTO_ALG_ERR_NONE); - pkt_hdr = packet_hdr(out_pkt); - pkt_hdr->p.flags.crypto_err = !op_result->ok; + pkt_hdr->p.flags.crypto_err = !ok; /* Synchronous, simply return results */ *pkt_out = out_pkt; diff --git a/platform/linux-generic/arch/aarch64/odp_llsc.h b/platform/linux-generic/arch/aarch64/odp_llsc.h index 2561451ca..1c2610511 100644 --- a/platform/linux-generic/arch/aarch64/odp_llsc.h +++ b/platform/linux-generic/arch/aarch64/odp_llsc.h @@ -13,6 +13,8 @@ #error This file should not be included directly, please include odp_cpu.h #endif +#include <odp_types_internal.h> + static inline uint16_t ll8(uint8_t *var, int mm) { uint16_t old; @@ -115,11 +117,11 @@ static inline uint32_t sc(uint64_t *var, uint64_t neu, int mm) #define sc64(a, b, c) sc((a), (b), (c)) union i128 { - __extension__ __int128 i128; + _odp_u128_t i128; int64_t i64[2]; }; -__extension__ static inline __int128 lld(__int128 *var, int mm) +static inline _odp_u128_t lld(_odp_u128_t *var, int mm) { union i128 old; @@ -139,7 +141,7 @@ __extension__ static inline __int128 lld(__int128 *var, int mm) } /* Return 0 on success, 1 on failure */ -__extension__ static inline uint32_t scd(__int128 *var, __int128 neu, int mm) +static inline uint32_t scd(_odp_u128_t *var, _odp_u128_t neu, int mm) { uint32_t ret; diff --git a/platform/linux-generic/arch/arm/odp_atomic.h b/platform/linux-generic/arch/arm/odp_atomic.h index bcc89ff73..e400f52d4 100644 --- a/platform/linux-generic/arch/arm/odp_atomic.h +++ b/platform/linux-generic/arch/arm/odp_atomic.h @@ -12,6 +12,7 @@ #error This file should not be included directly, please include odp_cpu.h #endif +#include <odp_types_internal.h> #include <limits.h> #ifdef CONFIG_DMBSTR @@ -70,7 +71,7 @@ static inline bitset_t bitset_mask(uint32_t bit) if (bit < 64) return 1ULL << bit; else - return (unsigned __int128)(1ULL << (bit - 64)) << 64; + return (_odp_u128_t)(1ULL << (bit - 64)) << 64; } #else diff --git a/platform/linux-generic/arch/default/odp_atomic.h b/platform/linux-generic/arch/default/odp_atomic.h index 7650d2b52..4cfc6b4bd 100644 --- a/platform/linux-generic/arch/default/odp_atomic.h +++ b/platform/linux-generic/arch/default/odp_atomic.h @@ -7,18 +7,18 @@ #ifndef ODP_DEFAULT_ATOMIC_H_ #define ODP_DEFAULT_ATOMIC_H_ -#ifdef __SIZEOF_INT128__ +#include <odp_types_internal.h> -__extension__ typedef unsigned __int128 _u128_t; +#ifdef __SIZEOF_INT128__ -static inline _u128_t lockfree_load_u128(_u128_t *atomic) +static inline _odp_u128_t lockfree_load_u128(_odp_u128_t *atomic) { return __atomic_load_n(atomic, __ATOMIC_RELAXED); } -static inline int lockfree_cas_acq_rel_u128(_u128_t *atomic, - _u128_t old_val, - _u128_t new_val) +static inline int lockfree_cas_acq_rel_u128(_odp_u128_t *atomic, + _odp_u128_t old_val, + _odp_u128_t new_val) { return __atomic_compare_exchange_n(atomic, &old_val, new_val, 0 /* strong */, @@ -76,7 +76,7 @@ static inline bitset_t bitset_mask(uint32_t bit) if (bit < 64) return 1ULL << bit; else - return (unsigned __int128)(1ULL << (bit - 64)) << 64; + return (_odp_u128_t)(1ULL << (bit - 64)) << 64; } #else diff --git a/platform/linux-generic/include-abi/odp/api/abi/event.h b/platform/linux-generic/include-abi/odp/api/abi/event.h index 6530ac2e4..e059f318c 100644 --- a/platform/linux-generic/include-abi/odp/api/abi/event.h +++ b/platform/linux-generic/include-abi/odp/api/abi/event.h @@ -1,4 +1,5 @@ /* Copyright (c) 2015-2018, Linaro Limited + * Copyright (c) 2022, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -18,42 +19,9 @@ extern "C" { #endif -#include <odp/api/plat/strong_types.h> - -/** @ingroup odp_event - * @{ - */ - -typedef ODP_HANDLE_T(odp_event_t); - -#define ODP_EVENT_INVALID _odp_cast_scalar(odp_event_t, 0) - -typedef enum odp_event_type_t { - ODP_EVENT_BUFFER = 1, - ODP_EVENT_PACKET = 2, - ODP_EVENT_TIMEOUT = 3, - ODP_EVENT_CRYPTO_COMPL = 4, - ODP_EVENT_IPSEC_STATUS = 5, - ODP_EVENT_PACKET_VECTOR = 6, - ODP_EVENT_PACKET_TX_COMPL = 7, - ODP_EVENT_DMA_COMPL = 8, -} odp_event_type_t; - -typedef enum odp_event_subtype_t { - ODP_EVENT_NO_SUBTYPE = 0, - ODP_EVENT_PACKET_BASIC = 1, - ODP_EVENT_PACKET_CRYPTO = 2, - ODP_EVENT_PACKET_IPSEC = 3, - ODP_EVENT_PACKET_COMP = 4 -} odp_event_subtype_t; - /* Inlined functions for non-ABI compat mode */ #include <odp/api/plat/event_inlines.h> -/** - * @} - */ - #ifdef __cplusplus } #endif diff --git a/platform/linux-generic/include-abi/odp/api/abi/event_types.h b/platform/linux-generic/include-abi/odp/api/abi/event_types.h new file mode 100644 index 000000000..c39322330 --- /dev/null +++ b/platform/linux-generic/include-abi/odp/api/abi/event_types.h @@ -0,0 +1,61 @@ +/* Copyright (c) 2015-2018, Linaro Limited + * Copyright (c) 2022, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +/** + * @file + * + * ODP event type definitions + */ + +#ifndef ODP_API_ABI_EVENT_TYPES_H_ +#define ODP_API_ABI_EVENT_TYPES_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <odp/api/plat/strong_types.h> +#include <odp/api/deprecated.h> + +/** @ingroup odp_event + * @{ + */ + +typedef ODP_HANDLE_T(odp_event_t); + +#define ODP_EVENT_INVALID _odp_cast_scalar(odp_event_t, 0) + +typedef enum odp_event_type_t { + ODP_EVENT_BUFFER = 1, + ODP_EVENT_PACKET = 2, + ODP_EVENT_TIMEOUT = 3, +#if ODP_DEPRECATED_API + ODP_EVENT_CRYPTO_COMPL = 4, +#endif + ODP_EVENT_IPSEC_STATUS = 5, + ODP_EVENT_PACKET_VECTOR = 6, + ODP_EVENT_PACKET_TX_COMPL = 7, + ODP_EVENT_DMA_COMPL = 8, +} odp_event_type_t; + +typedef enum odp_event_subtype_t { + ODP_EVENT_NO_SUBTYPE = 0, + ODP_EVENT_PACKET_BASIC = 1, + ODP_EVENT_PACKET_CRYPTO = 2, + ODP_EVENT_PACKET_IPSEC = 3, + ODP_EVENT_PACKET_COMP = 4 +} odp_event_subtype_t; + +/** + * @} + */ + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/include-abi/odp/api/abi/timer.h b/platform/linux-generic/include-abi/odp/api/abi/timer.h new file mode 100644 index 000000000..0a3b3a9cc --- /dev/null +++ b/platform/linux-generic/include-abi/odp/api/abi/timer.h @@ -0,0 +1,8 @@ +/* Copyright (c) 2022, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +/* Inlined functions for non-ABI compat mode */ +#include <odp/api/plat/timer_inlines.h> diff --git a/platform/linux-generic/include/odp/api/plat/buffer_inlines.h b/platform/linux-generic/include/odp/api/plat/buffer_inlines.h index 632ba0e69..9e0ae3f50 100644 --- a/platform/linux-generic/include/odp/api/plat/buffer_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/buffer_inlines.h @@ -8,7 +8,7 @@ #define ODP_PLAT_BUFFER_INLINES_H_ #include <odp/api/abi/buffer.h> -#include <odp/api/abi/event.h> +#include <odp/api/abi/event_types.h> #include <odp/api/plat/buffer_inline_types.h> diff --git a/platform/linux-generic/include/odp/api/plat/packet_flag_inlines.h b/platform/linux-generic/include/odp/api/plat/packet_flag_inlines.h index 6eb34a39b..04ad9e968 100644 --- a/platform/linux-generic/include/odp/api/plat/packet_flag_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/packet_flag_inlines.h @@ -1,4 +1,5 @@ /* Copyright (c) 2017-2018, Linaro Limited + * Copyright (c) 2022, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -30,11 +31,32 @@ static inline uint64_t _odp_packet_input_flags(odp_packet_t pkt) /* Inline functions by default */ #define _ODP_INLINE static inline #define odp_packet_has_l2 __odp_packet_has_l2 + #define odp_packet_has_l3 __odp_packet_has_l3 + #define odp_packet_has_l4 __odp_packet_has_l4 #define odp_packet_has_eth __odp_packet_has_eth #define odp_packet_has_jumbo __odp_packet_has_jumbo #define odp_packet_has_flow_hash __odp_packet_has_flow_hash #define odp_packet_has_ts __odp_packet_has_ts #define odp_packet_has_ipsec __odp_packet_has_ipsec + #define odp_packet_has_eth_bcast __odp_packet_has_eth_bcast + #define odp_packet_has_eth_mcast __odp_packet_has_eth_mcast + #define odp_packet_has_vlan __odp_packet_has_vlan + #define odp_packet_has_vlan_qinq __odp_packet_has_vlan_qinq + #define odp_packet_has_arp __odp_packet_has_arp + #define odp_packet_has_ipv4 __odp_packet_has_ipv4 + #define odp_packet_has_ipv6 __odp_packet_has_ipv6 + #define odp_packet_has_ip_bcast __odp_packet_has_ip_bcast + #define odp_packet_has_ip_mcast __odp_packet_has_ip_mcast + #define odp_packet_has_ipfrag __odp_packet_has_ipfrag + #define odp_packet_has_ipopt __odp_packet_has_ipopt + #define odp_packet_has_udp __odp_packet_has_udp + #define odp_packet_has_tcp __odp_packet_has_tcp + #define odp_packet_has_sctp __odp_packet_has_sctp + #define odp_packet_has_icmp __odp_packet_has_icmp + #define odp_packet_has_error __odp_packet_has_error + #define odp_packet_has_l2_error __odp_packet_has_l2_error + #define odp_packet_has_l3_error __odp_packet_has_l3_error + #define odp_packet_has_l4_error __odp_packet_has_l4_error #else #undef _ODP_INLINE #define _ODP_INLINE @@ -48,6 +70,22 @@ _ODP_INLINE int odp_packet_has_l2(odp_packet_t pkt) return flags.l2; } +_ODP_INLINE int odp_packet_has_l3(odp_packet_t pkt) +{ + _odp_packet_input_flags_t flags; + + flags.all = _odp_packet_input_flags(pkt); + return flags.l3; +} + +_ODP_INLINE int odp_packet_has_l4(odp_packet_t pkt) +{ + _odp_packet_input_flags_t flags; + + flags.all = _odp_packet_input_flags(pkt); + return flags.l4; +} + _ODP_INLINE int odp_packet_has_eth(odp_packet_t pkt) { _odp_packet_input_flags_t flags; @@ -88,6 +126,163 @@ _ODP_INLINE int odp_packet_has_ipsec(odp_packet_t pkt) return flags.ipsec; } +_ODP_INLINE int odp_packet_has_eth_bcast(odp_packet_t pkt) +{ + _odp_packet_input_flags_t flags; + + flags.all = _odp_packet_input_flags(pkt); + return flags.eth_bcast; +} + +_ODP_INLINE int odp_packet_has_eth_mcast(odp_packet_t pkt) +{ + _odp_packet_input_flags_t flags; + + flags.all = _odp_packet_input_flags(pkt); + return flags.eth_mcast; +} + +_ODP_INLINE int odp_packet_has_vlan(odp_packet_t pkt) +{ + _odp_packet_input_flags_t flags; + + flags.all = _odp_packet_input_flags(pkt); + return flags.vlan; +} + +_ODP_INLINE int odp_packet_has_vlan_qinq(odp_packet_t pkt) +{ + _odp_packet_input_flags_t flags; + + flags.all = _odp_packet_input_flags(pkt); + return flags.vlan_qinq; +} + +_ODP_INLINE int odp_packet_has_arp(odp_packet_t pkt) +{ + _odp_packet_input_flags_t flags; + + flags.all = _odp_packet_input_flags(pkt); + return flags.arp; +} + +_ODP_INLINE int odp_packet_has_ipv4(odp_packet_t pkt) +{ + _odp_packet_input_flags_t flags; + + flags.all = _odp_packet_input_flags(pkt); + return flags.ipv4; +} + +_ODP_INLINE int odp_packet_has_ipv6(odp_packet_t pkt) +{ + _odp_packet_input_flags_t flags; + + flags.all = _odp_packet_input_flags(pkt); + return flags.ipv6; +} + +_ODP_INLINE int odp_packet_has_ip_bcast(odp_packet_t pkt) +{ + _odp_packet_input_flags_t flags; + + flags.all = _odp_packet_input_flags(pkt); + return flags.ip_bcast; +} + +_ODP_INLINE int odp_packet_has_ip_mcast(odp_packet_t pkt) +{ + _odp_packet_input_flags_t flags; + + flags.all = _odp_packet_input_flags(pkt); + return flags.ip_mcast; +} + +_ODP_INLINE int odp_packet_has_ipfrag(odp_packet_t pkt) +{ + _odp_packet_input_flags_t flags; + + flags.all = _odp_packet_input_flags(pkt); + return flags.ipfrag; +} + +_ODP_INLINE int odp_packet_has_ipopt(odp_packet_t pkt) +{ + _odp_packet_input_flags_t flags; + + flags.all = _odp_packet_input_flags(pkt); + return flags.ipopt; +} + +_ODP_INLINE int odp_packet_has_udp(odp_packet_t pkt) +{ + _odp_packet_input_flags_t flags; + + flags.all = _odp_packet_input_flags(pkt); + return flags.udp; +} + +_ODP_INLINE int odp_packet_has_tcp(odp_packet_t pkt) +{ + _odp_packet_input_flags_t flags; + + flags.all = _odp_packet_input_flags(pkt); + return flags.tcp; +} + +_ODP_INLINE int odp_packet_has_sctp(odp_packet_t pkt) +{ + _odp_packet_input_flags_t flags; + + flags.all = _odp_packet_input_flags(pkt); + return flags.sctp; +} + +_ODP_INLINE int odp_packet_has_icmp(odp_packet_t pkt) +{ + _odp_packet_input_flags_t flags; + + flags.all = _odp_packet_input_flags(pkt); + return flags.icmp; +} + +_ODP_INLINE int odp_packet_has_error(odp_packet_t pkt) +{ + _odp_packet_flags_t flags; + + flags.all_flags = _odp_pkt_get(pkt, uint32_t, flags); + return flags.all.error != 0; +} + +_ODP_INLINE int odp_packet_has_l2_error(odp_packet_t pkt) +{ + _odp_packet_flags_t flags; + + flags.all_flags = _odp_pkt_get(pkt, uint32_t, flags); + + /* L2 parsing is always done by default and hence + no additional check is required. */ + return flags.snap_len_err; +} + +_ODP_INLINE int odp_packet_has_l3_error(odp_packet_t pkt) +{ + _odp_packet_flags_t flags; + + flags.all_flags = _odp_pkt_get(pkt, uint32_t, flags); + + return flags.ip_err; +} + +_ODP_INLINE int odp_packet_has_l4_error(odp_packet_t pkt) +{ + _odp_packet_flags_t flags; + + flags.all_flags = _odp_pkt_get(pkt, uint32_t, flags); + + return flags.tcp_err | flags.udp_err; +} + /** @endcond */ #endif diff --git a/platform/linux-generic/include/odp/api/plat/packet_inline_types.h b/platform/linux-generic/include/odp/api/plat/packet_inline_types.h index 66451ce49..126404f2d 100644 --- a/platform/linux-generic/include/odp/api/plat/packet_inline_types.h +++ b/platform/linux-generic/include/odp/api/plat/packet_inline_types.h @@ -115,7 +115,7 @@ typedef union { uint32_t all_flags; struct { - uint32_t reserved1: 7; + uint32_t reserved1: 6; /* * Init flags @@ -133,6 +133,7 @@ typedef union { uint32_t l4_chksum: 1; /* L4 chksum override */ uint32_t ts_set: 1; /* Set Tx timestamp */ uint32_t tx_compl: 1; /* Tx completion event requested */ + uint32_t tx_aging: 1; /* Packet aging at Tx requested */ uint32_t shaper_len_adj: 8; /* Adjustment for traffic mgr */ /* @@ -150,8 +151,8 @@ typedef union { /* Flag groups */ struct { - uint32_t reserved2: 7; - uint32_t other: 17; /* All other flags */ + uint32_t reserved2: 6; + uint32_t other: 18; /* All other flags */ uint32_t error: 8; /* All error flags */ } all; diff --git a/platform/linux-generic/include/odp/api/plat/packet_inlines.h b/platform/linux-generic/include/odp/api/plat/packet_inlines.h index fb63f207b..b72749ea8 100644 --- a/platform/linux-generic/include/odp/api/plat/packet_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/packet_inlines.h @@ -20,7 +20,7 @@ #include <odp/api/time.h> #include <odp/api/abi/buffer.h> -#include <odp/api/abi/event.h> +#include <odp/api/abi/event_types.h> #include <odp/api/abi/packet.h> #include <odp/api/abi/packet_io.h> @@ -73,6 +73,9 @@ #define odp_packet_subtype __odp_packet_subtype #define odp_packet_tx_compl_from_event __odp_packet_tx_compl_from_event #define odp_packet_tx_compl_to_event __odp_packet_tx_compl_to_event + #define odp_packet_color __odp_packet_color + #define odp_packet_drop_eligible __odp_packet_drop_eligible + #define odp_packet_shaper_len_adjust __odp_packet_shaper_len_adjust #else #undef _ODP_INLINE #define _ODP_INLINE @@ -372,6 +375,33 @@ _ODP_INLINE odp_event_t odp_packet_tx_compl_to_event(odp_packet_tx_compl_t tx_co return (odp_event_t)(uintptr_t)tx_compl; } +_ODP_INLINE odp_packet_color_t odp_packet_color(odp_packet_t pkt) +{ + _odp_packet_input_flags_t input_flags; + + input_flags.all = _odp_pkt_get(pkt, uint64_t, input_flags); + + return (odp_packet_color_t)input_flags.color; +} + +_ODP_INLINE odp_bool_t odp_packet_drop_eligible(odp_packet_t pkt) +{ + _odp_packet_input_flags_t input_flags; + + input_flags.all = _odp_pkt_get(pkt, uint64_t, input_flags); + + return !input_flags.nodrop; +} + +_ODP_INLINE int8_t odp_packet_shaper_len_adjust(odp_packet_t pkt) +{ + _odp_packet_flags_t flags; + + flags.all_flags = _odp_pkt_get(pkt, uint32_t, flags); + + return flags.shaper_len_adj; +} + /** @endcond */ #endif diff --git a/platform/linux-generic/include/odp/api/plat/packet_vector_inlines.h b/platform/linux-generic/include/odp/api/plat/packet_vector_inlines.h index 3586aa6d5..54a009a36 100644 --- a/platform/linux-generic/include/odp/api/plat/packet_vector_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/packet_vector_inlines.h @@ -14,7 +14,7 @@ #ifndef _ODP_PLAT_PACKET_VECTOR_INLINES_H_ #define _ODP_PLAT_PACKET_VECTOR_INLINES_H_ -#include <odp/api/abi/event.h> +#include <odp/api/abi/event_types.h> #include <odp/api/abi/packet_types.h> #include <odp/api/abi/pool_types.h> diff --git a/platform/linux-generic/include/odp/api/plat/queue_inline_types.h b/platform/linux-generic/include/odp/api/plat/queue_inline_types.h index 5ce767a16..4eade3dea 100644 --- a/platform/linux-generic/include/odp/api/plat/queue_inline_types.h +++ b/platform/linux-generic/include/odp/api/plat/queue_inline_types.h @@ -12,6 +12,7 @@ extern "C" { #endif #include <stdint.h> +#include <odp/api/event_types.h> #include <odp/api/queue_types.h> /** @cond _ODP_HIDE_FROM_DOXYGEN_ */ diff --git a/platform/linux-generic/include/odp/api/plat/time_inlines.h b/platform/linux-generic/include/odp/api/plat/time_inlines.h index bb2913532..2ffb94c66 100644 --- a/platform/linux-generic/include/odp/api/plat/time_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/time_inlines.h @@ -1,5 +1,5 @@ /* Copyright (c) 2018, Linaro Limited - * Copyright (c) 2020-2021, Nokia + * Copyright (c) 2020-2022, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -99,6 +99,7 @@ static inline uint64_t _odp_time_convert_to_ns(odp_time_t time) #define odp_time_cmp __odp_time_cmp #define odp_time_diff __odp_time_diff + #define odp_time_diff_ns __odp_time_diff_ns #define odp_time_sum __odp_time_sum #else @@ -170,6 +171,15 @@ _ODP_INLINE odp_time_t odp_time_diff(odp_time_t t2, odp_time_t t1) return time; } +_ODP_INLINE uint64_t odp_time_diff_ns(odp_time_t t2, odp_time_t t1) +{ + odp_time_t time; + + time.u64 = t2.u64 - t1.u64; + + return odp_time_to_ns(time); +} + _ODP_INLINE odp_time_t odp_time_sum(odp_time_t t1, odp_time_t t2) { odp_time_t time; diff --git a/platform/linux-generic/include/odp/api/plat/timer_inline_types.h b/platform/linux-generic/include/odp/api/plat/timer_inline_types.h new file mode 100644 index 000000000..e3397c4df --- /dev/null +++ b/platform/linux-generic/include/odp/api/plat/timer_inline_types.h @@ -0,0 +1,37 @@ +/* Copyright (c) 2022, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_PLAT_TIMER_INLINE_TYPES_H_ +#define ODP_PLAT_TIMER_INLINE_TYPES_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <stdint.h> + +/** @cond _ODP_HIDE_FROM_DOXYGEN_ */ + +/* Timeout header field accessor */ +#define _odp_timeout_hdr_field(hdr, cast, field) \ + (*(cast *)(uintptr_t)((uint8_t *)hdr + \ + _odp_timeout_inline_offset.field)) + +/* Timeout header field offsets for inline functions */ +typedef struct _odp_timeout_inline_offset_t { + uint16_t expiration; + uint16_t timer; + uint16_t user_ptr; + +} _odp_timeout_inline_offset_t; + +/** @endcond */ + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/include/odp/api/plat/timer_inlines.h b/platform/linux-generic/include/odp/api/plat/timer_inlines.h new file mode 100644 index 000000000..406aefdf6 --- /dev/null +++ b/platform/linux-generic/include/odp/api/plat/timer_inlines.h @@ -0,0 +1,78 @@ +/* Copyright (c) 2022, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_PLAT_TIMER_INLINES_H_ +#define ODP_PLAT_TIMER_INLINES_H_ + +#include <odp/api/event.h> +#include <odp/api/timer_types.h> + +#include <odp/api/plat/timer_inline_types.h> + +#include <stdint.h> + +/** @cond _ODP_HIDE_FROM_DOXYGEN_ */ + +extern const _odp_timeout_inline_offset_t _odp_timeout_inline_offset; + +#ifndef _ODP_NO_INLINE + /* Inline functions by default */ + #define _ODP_INLINE static inline + #define odp_timeout_timer __odp_timeout_timer + #define odp_timeout_tick __odp_timeout_tick + #define odp_timeout_user_ptr __odp_timeout_user_ptr + #define odp_timer_tick_to_ns __odp_timer_tick_to_ns + #define odp_timer_ns_to_tick __odp_timer_ns_to_tick + #define odp_timeout_from_event __odp_timeout_from_event + #define odp_timeout_to_event __odp_timeout_to_event +#else + #define _ODP_INLINE +#endif + +_ODP_INLINE odp_timer_t odp_timeout_timer(odp_timeout_t tmo) +{ + return _odp_timeout_hdr_field(tmo, odp_timer_t, timer); +} + +_ODP_INLINE uint64_t odp_timeout_tick(odp_timeout_t tmo) +{ + return _odp_timeout_hdr_field(tmo, uint64_t, expiration); +} + +_ODP_INLINE void *odp_timeout_user_ptr(odp_timeout_t tmo) +{ + return _odp_timeout_hdr_field(tmo, void *, user_ptr); +} + +_ODP_INLINE uint64_t odp_timer_tick_to_ns(odp_timer_pool_t tp, uint64_t ticks) +{ + (void)tp; + + /* Timer ticks in API are nsec */ + return ticks; +} + +_ODP_INLINE uint64_t odp_timer_ns_to_tick(odp_timer_pool_t tp, uint64_t ns) +{ + (void)tp; + + /* Timer ticks in API are nsec */ + return ns; +} + +_ODP_INLINE odp_timeout_t odp_timeout_from_event(odp_event_t ev) +{ + return (odp_timeout_t)ev; +} + +_ODP_INLINE odp_event_t odp_timeout_to_event(odp_timeout_t tmo) +{ + return (odp_event_t)tmo; +} + +/** @endcond */ + +#endif diff --git a/platform/linux-generic/include/odp_atomic_internal.h b/platform/linux-generic/include/odp_atomic_internal.h index d98e0e6be..da79b3723 100644 --- a/platform/linux-generic/include/odp_atomic_internal.h +++ b/platform/linux-generic/include/odp_atomic_internal.h @@ -20,6 +20,7 @@ #include <odp/api/align.h> #include <odp/api/hints.h> #include <odp/api/atomic.h> +#include <odp_types_internal.h> #include <stdbool.h> #ifdef __cplusplus @@ -137,12 +138,10 @@ static inline void _odp_atomic_flag_clear(_odp_atomic_flag_t *flag) #endif #ifdef ODP_ATOMIC_U128 -/** An unsigned 128-bit (16-byte) scalar type */ -__extension__ typedef __int128 _uint128_t; /** Atomic 128-bit type */ typedef struct ODP_ALIGNED(16) { - _uint128_t v; /**< Actual storage for the atomic variable */ + _odp_u128_t v; /**< Actual storage for the atomic variable */ } _odp_atomic_u128_t; /** @@ -154,8 +153,8 @@ typedef struct ODP_ALIGNED(16) { * @param mmodel Memory model associated with the exchange operation */ static inline void _odp_atomic_u128_xchg_mm(_odp_atomic_u128_t *ptr, - _uint128_t *val, - _uint128_t *old, + _odp_u128_t *val, + _odp_u128_t *old, _odp_memmodel_t mm) { __atomic_exchange(&ptr->v, val, old, mm); @@ -177,8 +176,8 @@ static inline void _odp_atomic_u128_xchg_mm(_odp_atomic_u128_t *ptr, * @retval 0 exchange failed and '*exp' updated with current value */ static inline int _odp_atomic_u128_cmp_xchg_mm(_odp_atomic_u128_t *ptr, - _uint128_t *exp, - _uint128_t *val, + _odp_u128_t *exp, + _odp_u128_t *val, _odp_memmodel_t succ, _odp_memmodel_t fail) { diff --git a/platform/linux-generic/include/odp_classification_datamodel.h b/platform/linux-generic/include/odp_classification_datamodel.h index cc0e7f081..6b50fef68 100644 --- a/platform/linux-generic/include/odp_classification_datamodel.h +++ b/platform/linux-generic/include/odp_classification_datamodel.h @@ -129,6 +129,7 @@ typedef struct pmr_term_value { Class Of Service */ struct cos_s { + odp_cos_action_t action; /* Action */ odp_queue_t queue; /* Associated Queue */ odp_pool_t pool; /* Associated Buffer pool */ odp_pktin_vector_config_t vector; /* Packet vector config */ @@ -145,10 +146,11 @@ struct cos_s { odp_queue_param_t queue_param; char name[ODP_COS_NAME_LEN]; /* name */ uint8_t index; + odp_bool_t stats_enable; struct { odp_atomic_u64_t discards; odp_atomic_u64_t packets; - } stats[CLS_COS_QUEUE_MAX]; + } stats, queue_stats[CLS_COS_QUEUE_MAX]; }; typedef union cos_u { diff --git a/platform/linux-generic/include/odp_classification_internal.h b/platform/linux-generic/include/odp_classification_internal.h index bc0a12f8f..c20d24c58 100644 --- a/platform/linux-generic/include/odp_classification_internal.h +++ b/platform/linux-generic/include/odp_classification_internal.h @@ -67,9 +67,9 @@ static inline void _odp_cos_queue_stats_add(cos_t *cos, odp_queue_t queue, } if (packets) - odp_atomic_add_u64(&cos->s.stats[queue_idx].packets, packets); + odp_atomic_add_u64(&cos->s.queue_stats[queue_idx].packets, packets); if (discards) - odp_atomic_add_u64(&cos->s.stats[queue_idx].discards, discards); + odp_atomic_add_u64(&cos->s.queue_stats[queue_idx].discards, discards); } /** Classification Internal function **/ @@ -84,8 +84,7 @@ This function calls Classifier module internal functions for a given packet and selects destination queue and packet pool based on selected PMR and CoS. **/ int _odp_cls_classify_packet(pktio_entry_t *entry, const uint8_t *base, - uint16_t pkt_len, uint32_t seg_len, odp_pool_t *pool, - odp_packet_hdr_t *pkt_hdr, odp_bool_t parse); + odp_pool_t *pool, odp_packet_hdr_t *pkt_hdr); /** Packet IO classifier init diff --git a/platform/linux-generic/include/odp_llqueue.h b/platform/linux-generic/include/odp_llqueue.h index e9cf9945e..68325624a 100644 --- a/platform/linux-generic/include/odp_llqueue.h +++ b/platform/linux-generic/include/odp_llqueue.h @@ -15,6 +15,7 @@ #include <odp_config_internal.h> #include <odp_debug_internal.h> +#include <odp_types_internal.h> #include <odp_cpu.h> #include <stdint.h> @@ -49,7 +50,7 @@ static odp_bool_t llq_on_queue(struct llnode *node); typedef uint64_t dintptr_t; #endif #if __SIZEOF_PTRDIFF_T__ == 8 -__extension__ typedef __int128 dintptr_t; +typedef _odp_u128_t dintptr_t; #endif struct llnode { diff --git a/platform/linux-generic/include/odp_macros_internal.h b/platform/linux-generic/include/odp_macros_internal.h index 997e0fd5b..b8be7f938 100644 --- a/platform/linux-generic/include/odp_macros_internal.h +++ b/platform/linux-generic/include/odp_macros_internal.h @@ -35,6 +35,8 @@ extern "C" { tmp_a > tmp_b ? tmp_a : tmp_b; \ }) +#define MAX3(a, b, c) (MAX(MAX((a), (b)), (c))) + #define odp_container_of(pointer, type, member) \ ((type *)(void *)(((char *)pointer) - offsetof(type, member))) diff --git a/platform/linux-generic/include/odp_packet_dpdk.h b/platform/linux-generic/include/odp_packet_dpdk.h index 3add83f19..b326000e6 100644 --- a/platform/linux-generic/include/odp_packet_dpdk.h +++ b/platform/linux-generic/include/odp_packet_dpdk.h @@ -46,18 +46,4 @@ int _odp_dpdk_packet_parse_common(packet_parser_t *pkt_hdr, uint32_t supported_ptypes, odp_pktin_config_opt_t pktin_cfg); -static inline int _odp_dpdk_packet_parse_layer(odp_packet_hdr_t *pkt_hdr, - struct rte_mbuf *mbuf, - odp_proto_layer_t layer, - uint32_t supported_ptypes, - odp_pktin_config_opt_t pktin_cfg) -{ - uint32_t seg_len = pkt_hdr->seg_len; - void *base = pkt_hdr->seg_data; - - return _odp_dpdk_packet_parse_common(&pkt_hdr->p, base, - pkt_hdr->frame_len, seg_len, mbuf, - layer, supported_ptypes, - pktin_cfg); -} #endif diff --git a/platform/linux-generic/include/odp_packet_internal.h b/platform/linux-generic/include/odp_packet_internal.h index 5a29210dd..a8f58146a 100644 --- a/platform/linux-generic/include/odp_packet_internal.h +++ b/platform/linux-generic/include/odp_packet_internal.h @@ -38,15 +38,23 @@ extern "C" { #include <stdint.h> #include <string.h> -/** Minimum segment length expected by _odp_packet_parse_common() */ -#define PACKET_PARSE_SEG_LEN 96 - ODP_STATIC_ASSERT(sizeof(_odp_packet_input_flags_t) == sizeof(uint64_t), "INPUT_FLAGS_SIZE_ERROR"); ODP_STATIC_ASSERT(sizeof(_odp_packet_flags_t) == sizeof(uint32_t), "PACKET_FLAGS_SIZE_ERROR"); +/* Packet extra data length */ +#define PKT_EXTRA_LEN 128 + +/* Packet extra data types */ +#define PKT_EXTRA_TYPE_DPDK 1 + +/* Maximum number of segments per packet */ +#define PKT_MAX_SEGS 255 + +ODP_STATIC_ASSERT(PKT_MAX_SEGS < UINT16_MAX, "PACKET_MAX_SEGS_ERROR"); + /** * Packet parser metadata */ @@ -67,17 +75,6 @@ typedef struct { uint16_t l4_offset; } packet_parser_t; -/* Packet extra data length */ -#define PKT_EXTRA_LEN 128 - -/* Packet extra data types */ -#define PKT_EXTRA_TYPE_DPDK 1 - -/* Maximum number of segments per packet */ -#define PKT_MAX_SEGS 255 - -ODP_STATIC_ASSERT(PKT_MAX_SEGS < UINT16_MAX, "PACKET_MAX_SEGS_ERROR"); - /** * Internal Packet header * @@ -146,6 +143,10 @@ typedef struct ODP_ALIGNED_CACHE odp_packet_hdr_t { /* Max payload size in a LSO segment */ uint16_t lso_max_payload; + /* Packet aging drop timeout before enqueue. Once enqueued holds the maximum age (time of + * request + requested drop timeout). */ + uint64_t tx_aging_ns; + /* LSO profile index */ uint8_t lso_profile_idx; @@ -194,6 +195,11 @@ static inline odp_packet_t packet_from_event_hdr(_odp_event_hdr_t *event_hdr) return (odp_packet_t)(uintptr_t)event_hdr; } +static inline uint32_t packet_first_seg_len(odp_packet_hdr_t *pkt_hdr) +{ + return pkt_hdr->seg_len; +} + static inline odp_packet_hdr_t *packet_last_seg(odp_packet_hdr_t *hdr) { while (hdr->seg_next != NULL) @@ -400,11 +406,6 @@ static inline void packet_set_len(odp_packet_hdr_t *pkt_hdr, uint32_t len) int _odp_packet_alloc_multi(odp_pool_t pool_hdl, uint32_t len, odp_packet_t pkt[], int max_num); -/* Perform packet parse up to a given protocol layer */ -int _odp_packet_parse_layer(odp_packet_hdr_t *pkt_hdr, - odp_proto_layer_t layer, - odp_proto_chksums_t chksums); - /* Reset parser metadata for a new parse */ static inline void packet_parse_reset(odp_packet_hdr_t *pkt_hdr, int all) { @@ -454,12 +455,6 @@ static inline void packet_set_ts(odp_packet_hdr_t *pkt_hdr, odp_time_t *ts) } } -int _odp_packet_parse_common(packet_parser_t *pkt_hdr, const uint8_t *ptr, - uint32_t pkt_len, uint32_t seg_len, int layer, - odp_proto_chksums_t chksums); - -int _odp_cls_parse(odp_packet_hdr_t *pkt_hdr, const uint8_t *parseptr); - int _odp_packet_set_data(odp_packet_t pkt, uint32_t offset, uint8_t c, uint32_t len); @@ -471,6 +466,9 @@ int _odp_packet_tcp_chksum_insert(odp_packet_t pkt); int _odp_packet_udp_chksum_insert(odp_packet_t pkt); int _odp_packet_sctp_chksum_insert(odp_packet_t pkt); +int _odp_packet_l4_chksum(odp_packet_hdr_t *pkt_hdr, + odp_proto_chksums_t chksums, uint64_t l4_part_sum); + #ifdef __cplusplus } #endif diff --git a/platform/linux-generic/include/odp_packet_io_internal.h b/platform/linux-generic/include/odp_packet_io_internal.h index 3e9f94100..aed6de412 100644 --- a/platform/linux-generic/include/odp_packet_io_internal.h +++ b/platform/linux-generic/include/odp_packet_io_internal.h @@ -78,6 +78,7 @@ struct pktio_entry { /* These two locks together lock the whole pktio device */ odp_ticketlock_t rxl; /**< RX ticketlock */ odp_ticketlock_t txl; /**< TX ticketlock */ + odp_proto_layer_t parse_layer; uint16_t pktin_frame_offset; struct { @@ -93,6 +94,8 @@ struct pktio_entry { uint8_t tx_ts : 1; /* Tx completion events */ uint8_t tx_compl : 1; + /* Packet aging */ + uint8_t tx_aging : 1; }; }; } enabled; @@ -144,8 +147,8 @@ struct pktio_entry { /* Storage for queue handles * Multi-queue support is pktio driver specific */ - unsigned num_in_queue; - unsigned num_out_queue; + uint32_t num_in_queue; + uint32_t num_out_queue; struct { odp_queue_t queue; @@ -227,8 +230,8 @@ typedef struct pktio_if_ops { int num); int (*recv_tmo)(pktio_entry_t *entry, int index, odp_packet_t packets[], int num, uint64_t wait_usecs); - int (*recv_mq_tmo)(pktio_entry_t *entry[], int index[], int num_q, - odp_packet_t packets[], int num, unsigned *from, + int (*recv_mq_tmo)(pktio_entry_t *entry[], int index[], uint32_t num_q, + odp_packet_t packets[], int num, uint32_t *from, uint64_t wait_usecs); int (*fd_set)(pktio_entry_t *entry, int index, fd_set *readfds); int (*send)(pktio_entry_t *entry, int index, @@ -291,6 +294,11 @@ static inline int _odp_pktio_tx_compl_enabled(const pktio_entry_t *entry) return entry->s.enabled.tx_compl; } +static inline int _odp_pktio_tx_aging_enabled(pktio_entry_t *entry) +{ + return entry->s.enabled.tx_aging; +} + static inline void _odp_pktio_tx_ts_set(pktio_entry_t *entry) { odp_time_t ts_val = odp_time_global(); @@ -326,7 +334,7 @@ extern const pktio_if_ops_t * const _odp_pktio_if_ops[]; * @return <0 on failure */ int _odp_sock_recv_mq_tmo_try_int_driven(const struct odp_pktin_queue_t queues[], - unsigned int num_q, unsigned int *from, + uint32_t num_q, uint32_t *from, odp_packet_t packets[], int num, uint64_t usecs, int *trial_successful); diff --git a/platform/linux-generic/include/odp_parse_internal.h b/platform/linux-generic/include/odp_parse_internal.h new file mode 100644 index 000000000..8aa5e118b --- /dev/null +++ b/platform/linux-generic/include/odp_parse_internal.h @@ -0,0 +1,112 @@ +/* Copyright (c) 2013-2018, Linaro Limited + * Copyright (c) 2019-2022, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_PARSE_INTERNAL_H_ +#define ODP_PARSE_INTERNAL_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <odp_packet_internal.h> +#include <odp_macros_internal.h> +#include <protocols/eth.h> +#include <protocols/ip.h> +#include <protocols/sctp.h> +#include <protocols/tcp.h> +#include <protocols/udp.h> +#include <odp/api/plat/packet_inline_types.h> +#include <odp/api/hints.h> +#include <odp/api/packet_io.h> +#include <odp/api/packet_types.h> +#include <stdint.h> + +/* + * In the worst case we look at the Ethernet header, 8 bytes of LLC/SNAP + * header and two VLAN tags in the same packet. + */ +#define PARSE_ETH_BYTES (sizeof(_odp_ethhdr_t) + 8 + 2 * sizeof(_odp_vlanhdr_t)) +#define PARSE_IPV4_BYTES (0xfU * 4) /* max IPv4 header length with options */ +/* + * Peek 2 bytes beyond IPv6 base header without length check if there are + * extension headers. + */ +#define PARSE_IPV6_BYTES (sizeof(_odp_ipv6hdr_t) + 2) +#define PARSE_TCP_BYTES (sizeof(_odp_tcphdr_t)) +/* + * In the worst case we look at the UDP header and 4 bytes of the UDP + * payload (the non-ESP marker to distinguish IKE packets from ESP packets). + */ +#define PARSE_UDP_BYTES (sizeof(_odp_udphdr_t) + 4) +#define PARSE_SCTP_BYTES (sizeof(_odp_sctphdr_t)) + +/* _odp_packet_parse_common_l3_l4() requires up to this many bytes. */ +#define PARSE_L3_L4_BYTES (MAX(PARSE_IPV4_BYTES, PARSE_IPV6_BYTES) + \ + MAX3(PARSE_TCP_BYTES, PARSE_UDP_BYTES, PARSE_SCTP_BYTES)) + +/* _odp_packet_parse_common() requires up to this many bytes. */ +#define PARSE_BYTES (PARSE_ETH_BYTES + PARSE_L3_L4_BYTES) + +uint16_t _odp_parse_eth(packet_parser_t *prs, const uint8_t **parseptr, + uint32_t *offset, uint32_t frame_len); + +/* + * Parse common L3 and L4 packet headers up to given layer + * + * See _odp_packet_parse_common(). Requires up to PARSE_L3_L4_BYTES bytes of + * contiguous packet data. + */ +int _odp_packet_parse_common_l3_l4(packet_parser_t *prs, + const uint8_t *parseptr, uint32_t offset, + uint32_t frame_len, uint32_t seg_len, + int layer, uint16_t ethtype, + odp_proto_chksums_t chksums, + uint64_t *l4_part_sum, + odp_pktin_config_opt_t opt); + +/** + * Parse common packet headers up to given layer + * + * Requires up to PARSE_BYTES bytes of contiguous packet data. Also parse + * metadata must be already initialized. + * + * Returns 0 on success, 1 on packet errors, and -1 if the packet should be + * dropped. + */ +static inline int _odp_packet_parse_common(packet_parser_t *prs, + const uint8_t *ptr, + uint32_t frame_len, uint32_t seg_len, + int layer, + odp_proto_chksums_t chksums, + uint64_t *l4_part_sum, + odp_pktin_config_opt_t opt) +{ + uint32_t offset; + uint16_t ethtype; + const uint8_t *parseptr; + + parseptr = ptr; + offset = 0; + + if (odp_unlikely(layer == ODP_PROTO_LAYER_NONE)) + return 0; + + /* Assume valid L2 header, no CRC/FCS check in SW */ + prs->l2_offset = offset; + + ethtype = _odp_parse_eth(prs, &parseptr, &offset, frame_len); + + return _odp_packet_parse_common_l3_l4(prs, parseptr, offset, frame_len, + seg_len, layer, ethtype, chksums, + l4_part_sum, opt); +} + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/include/odp_schedule_if.h b/platform/linux-generic/include/odp_schedule_if.h index ba4ad53e8..cec9c8bb8 100644 --- a/platform/linux-generic/include/odp_schedule_if.h +++ b/platform/linux-generic/include/odp_schedule_if.h @@ -89,8 +89,6 @@ extern const schedule_fn_t *_odp_sched_fn; /* Interface for the scheduler */ int _odp_sched_cb_pktin_poll(int pktio_index, int pktin_index, _odp_event_hdr_t *hdr_tbl[], int num); -/* evts must have at least QUEUE_MULTI_MAX elements */ -int _odp_sched_cb_pktin_poll_one(int pktio_index, int rx_queue, odp_event_t evts[]); void _odp_sched_cb_pktio_stop_finalize(int pktio_index); /* API functions */ diff --git a/platform/linux-generic/include/odp_types_internal.h b/platform/linux-generic/include/odp_types_internal.h new file mode 100644 index 000000000..a97ac9cd4 --- /dev/null +++ b/platform/linux-generic/include/odp_types_internal.h @@ -0,0 +1,24 @@ +/* Copyright (c) 2022, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_TYPES_INTERNAL_H_ +#define ODP_TYPES_INTERNAL_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#ifdef __SIZEOF_INT128__ + +__extension__ typedef unsigned __int128 _odp_u128_t; + +#endif + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/odp_classification.c b/platform/linux-generic/odp_classification.c index 3b50232d6..2fdecbc51 100644 --- a/platform/linux-generic/odp_classification.c +++ b/platform/linux-generic/odp_classification.c @@ -150,7 +150,7 @@ void odp_cls_pmr_param_init(odp_pmr_param_t *param) int odp_cls_capability(odp_cls_capability_t *capability) { - unsigned int count = 0; + uint32_t count = 0; memset(capability, 0, sizeof(odp_cls_capability_t)); @@ -161,6 +161,7 @@ int odp_cls_capability(odp_cls_capability_t *capability) capability->max_pmr_terms = CLS_PMR_MAX_ENTRY; capability->available_pmr_terms = count; capability->max_cos = CLS_COS_MAX_ENTRY; + capability->max_cos_stats = capability->max_cos; capability->pmr_range_supported = false; capability->supported_terms.all_bits = 0; capability->supported_terms.bit.len = 1; @@ -186,6 +187,8 @@ int odp_cls_capability(odp_cls_capability_t *capability) capability->threshold_bp.all_bits = 0; capability->max_hash_queues = CLS_COS_QUEUE_MAX; capability->max_mark = MAX_MARK; + capability->stats.cos.counter.discards = 1; + capability->stats.cos.counter.packets = 1; capability->stats.queue.counter.discards = 1; capability->stats.queue.counter.packets = 1; @@ -220,21 +223,29 @@ static inline void _cls_queue_unwind(uint32_t tbl_index, uint32_t j) odp_queue_destroy(queue_grp_tbl->s.queue[tbl_index + --j]); } -odp_cos_t odp_cls_cos_create(const char *name, const odp_cls_cos_param_t *param) +odp_cos_t odp_cls_cos_create(const char *name, const odp_cls_cos_param_t *param_in) { uint32_t i, j; odp_queue_t queue; odp_cls_drop_t drop_policy; cos_t *cos; uint32_t tbl_index; + odp_cls_cos_param_t param = *param_in; + + if (param.action == ODP_COS_ACTION_DROP) { + param.num_queue = 1; + param.queue = ODP_QUEUE_INVALID; + param.pool = ODP_POOL_INVALID; + param.vector.enable = false; + } /* num_queue should not be zero */ - if (param->num_queue > CLS_COS_QUEUE_MAX || param->num_queue < 1) + if (param.num_queue > CLS_COS_QUEUE_MAX || param.num_queue < 1) return ODP_COS_INVALID; /* Validate packet vector parameters */ - if (param->vector.enable) { - odp_pool_t pool = param->vector.pool; + if (param.vector.enable) { + odp_pool_t pool = param.vector.pool; odp_pool_info_t pool_info; if (pool == ODP_POOL_INVALID || odp_pool_info(pool, &pool_info)) { @@ -245,17 +256,17 @@ odp_cos_t odp_cls_cos_create(const char *name, const odp_cls_cos_param_t *param) ODP_ERR("wrong pool type\n"); return ODP_COS_INVALID; } - if (param->vector.max_size == 0) { + if (param.vector.max_size == 0) { ODP_ERR("vector.max_size is zero\n"); return ODP_COS_INVALID; } - if (param->vector.max_size > pool_info.params.vector.max_size) { + if (param.vector.max_size > pool_info.params.vector.max_size) { ODP_ERR("vector.max_size larger than pool max vector size\n"); return ODP_COS_INVALID; } } - drop_policy = param->drop_policy; + drop_policy = param.drop_policy; for (i = 0; i < CLS_COS_MAX_ENTRY; i++) { cos = &cos_tbl->cos_entry[i]; @@ -274,16 +285,16 @@ odp_cos_t odp_cls_cos_create(const char *name, const odp_cls_cos_param_t *param) cos->s.linked_cos[j] = NULL; } - cos->s.num_queue = param->num_queue; + cos->s.num_queue = param.num_queue; - if (param->num_queue > 1) { + if (param.num_queue > 1) { odp_queue_param_init(&cos->s.queue_param); cos->s.queue_group = true; cos->s.queue = ODP_QUEUE_INVALID; _odp_cls_update_hash_proto(cos, - param->hash_proto); + param.hash_proto); tbl_index = i * CLS_COS_QUEUE_MAX; - for (j = 0; j < param->num_queue; j++) { + for (j = 0; j < param.num_queue; j++) { queue = odp_queue_create(NULL, &cos->s.queue_param); if (queue == ODP_QUEUE_INVALID) { /* unwind the queues */ @@ -296,21 +307,27 @@ odp_cos_t odp_cls_cos_create(const char *name, const odp_cls_cos_param_t *param) } } else { - cos->s.queue = param->queue; + cos->s.queue = param.queue; } + + odp_atomic_init_u64(&cos->s.stats.discards, 0); + odp_atomic_init_u64(&cos->s.stats.packets, 0); + /* Initialize statistics counters */ for (j = 0; j < cos->s.num_queue; j++) { - odp_atomic_init_u64(&cos->s.stats[j].discards, 0); - odp_atomic_init_u64(&cos->s.stats[j].packets, 0); + odp_atomic_init_u64(&cos->s.queue_stats[j].discards, 0); + odp_atomic_init_u64(&cos->s.queue_stats[j].packets, 0); } - cos->s.pool = param->pool; + cos->s.action = param.action; + cos->s.pool = param.pool; cos->s.headroom = 0; cos->s.valid = 1; cos->s.drop_policy = drop_policy; odp_atomic_init_u32(&cos->s.num_rule, 0); cos->s.index = i; - cos->s.vector = param->vector; + cos->s.vector = param.vector; + cos->s.stats_enable = param.stats_enable; UNLOCK(&cos->s.lock); return _odp_cos_from_ndx(i); } @@ -1461,6 +1478,9 @@ static cos_t *match_pmr_cos(cos_t *cos, const uint8_t *pkt_addr, pmr_t *pmr, /* PMR matched */ pmr_debug_print(pmr, cos); + if (cos->s.stats_enable) + odp_atomic_inc_u64(&cos->s.stats.packets); + hdr->p.input_flags.cls_mark = 0; if (pmr->s.mark) { hdr->p.input_flags.cls_mark = 1; @@ -1537,8 +1557,11 @@ static inline cos_t *cls_select_cos(pktio_entry_t *entry, default_cos = cls->default_cos; /* Return error cos for error packet */ - if (pkt_hdr->p.flags.all.error) - return cls->error_cos; + if (pkt_hdr->p.flags.all.error) { + cos = cls->error_cos; + goto done; + } + /* Calls all the PMRs attached at the PKTIO level*/ for (i = 0; i < odp_atomic_load_u32(&default_cos->s.num_rule); i++) { pmr = default_cos->s.pmr[i]; @@ -1551,11 +1574,17 @@ static inline cos_t *cls_select_cos(pktio_entry_t *entry, cos = match_qos_cos(entry, pkt_addr, pkt_hdr); if (cos) { ODP_DBG_RAW(CLS_DBG, " QoS matched -> cos: %s(%u)\n", cos->s.name, cos->s.index); - return cos; + goto done; } ODP_DBG_RAW(CLS_DBG, " No match -> default cos\n"); - return cls->default_cos; + cos = cls->default_cos; + +done: + if (cos && cos->s.stats_enable) + odp_atomic_inc_u64(&cos->s.stats.packets); + + return cos; } static uint32_t packet_rss_hash(odp_packet_hdr_t *pkt_hdr, @@ -1575,12 +1604,12 @@ static uint32_t packet_rss_hash(odp_packet_hdr_t *pkt_hdr, * @retval 0 on success * @retval -EFAULT Bug * @retval -EINVAL Config error + * @retval -ENOENT Drop action * * @note *base is not released */ int _odp_cls_classify_packet(pktio_entry_t *entry, const uint8_t *base, - uint16_t pkt_len, uint32_t seg_len, odp_pool_t *pool, - odp_packet_hdr_t *pkt_hdr, odp_bool_t parse) + odp_pool_t *pool, odp_packet_hdr_t *pkt_hdr) { cos_t *cos; uint32_t tbl_index; @@ -1588,24 +1617,19 @@ int _odp_cls_classify_packet(pktio_entry_t *entry, const uint8_t *base, ODP_DBG_LVL(CLS_DBG, "Classify packet from %s\n", entry->s.full_name); - if (parse) { - packet_parse_reset(pkt_hdr, 1); - packet_set_len(pkt_hdr, pkt_len); - - _odp_packet_parse_common(&pkt_hdr->p, base, pkt_len, seg_len, - ODP_PROTO_LAYER_ALL, - entry->s.in_chksums); - } cos = cls_select_cos(entry, base, pkt_hdr); if (cos == NULL) return -EINVAL; + if (cos->s.action == ODP_COS_ACTION_DROP) + return -ENOENT; + if (cos->s.queue == ODP_QUEUE_INVALID && cos->s.num_queue == 1) - return -EFAULT; + goto error; if (cos->s.pool == ODP_POOL_INVALID) - return -EFAULT; + goto error; *pool = cos->s.pool; pkt_hdr->p.input_flags.dst_queue = 1; @@ -1623,6 +1647,10 @@ int _odp_cls_classify_packet(pktio_entry_t *entry, const uint8_t *base, cos->s.num_queue); pkt_hdr->dst_queue = queue_grp_tbl->s.queue[tbl_index]; return 0; + +error: + odp_atomic_inc_u64(&cos->s.stats.discards); + return -EFAULT; } static uint32_t packet_rss_hash(odp_packet_hdr_t *pkt_hdr, @@ -1780,6 +1808,27 @@ uint64_t odp_pmr_to_u64(odp_pmr_t hdl) return _odp_pri(hdl); } +int odp_cls_cos_stats(odp_cos_t hdl, odp_cls_cos_stats_t *stats) +{ + cos_t *cos = get_cos_entry(hdl); + + if (odp_unlikely(cos == NULL)) { + ODP_ERR("Invalid odp_cos_t handle\n"); + return -1; + } + + if (odp_unlikely(stats == NULL)) { + ODP_ERR("Output structure NULL\n"); + return -1; + } + + memset(stats, 0, sizeof(*stats)); + stats->discards = odp_atomic_load_u64(&cos->s.stats.discards); + stats->packets = odp_atomic_load_u64(&cos->s.stats.packets); + + return 0; +} + int odp_cls_queue_stats(odp_cos_t hdl, odp_queue_t queue, odp_cls_queue_stats_t *stats) { @@ -1803,8 +1852,8 @@ int odp_cls_queue_stats(odp_cos_t hdl, odp_queue_t queue, } memset(stats, 0, sizeof(odp_cls_queue_stats_t)); - stats->discards = odp_atomic_load_u64(&cos->s.stats[queue_idx].discards); - stats->packets = odp_atomic_load_u64(&cos->s.stats[queue_idx].packets); + stats->discards = odp_atomic_load_u64(&cos->s.queue_stats[queue_idx].discards); + stats->packets = odp_atomic_load_u64(&cos->s.queue_stats[queue_idx].packets); return 0; } diff --git a/platform/linux-generic/odp_crypto_null.c b/platform/linux-generic/odp_crypto_null.c index bc14bad9a..f276d4659 100644 --- a/platform/linux-generic/odp_crypto_null.c +++ b/platform/linux-generic/odp_crypto_null.c @@ -269,9 +269,7 @@ int odp_crypto_session_destroy(odp_crypto_session_t session) return 0; } -/* - * Shim function around packet operation, can be used by other implementations. - */ +#if ODP_DEPRECATED_API int odp_crypto_operation(odp_crypto_op_param_t *param, odp_bool_t *posted, @@ -326,6 +324,7 @@ odp_crypto_operation(odp_crypto_op_param_t *param, return 0; } +#endif int _odp_crypto_init_global(void) @@ -402,6 +401,7 @@ int _odp_crypto_term_local(void) return 0; } +#if ODP_DEPRECATED_API odp_crypto_compl_t odp_crypto_compl_from_event(odp_event_t ev) { /* This check not mandated by the API specification */ @@ -438,6 +438,7 @@ uint64_t odp_crypto_compl_to_u64(odp_crypto_compl_t hdl) { return _odp_pri(hdl); } +#endif void odp_crypto_session_param_init(odp_crypto_session_param_t *param) { diff --git a/platform/linux-generic/odp_crypto_openssl.c b/platform/linux-generic/odp_crypto_openssl.c index 13101fbca..6ff0ac041 100644 --- a/platform/linux-generic/odp_crypto_openssl.c +++ b/platform/linux-generic/odp_crypto_openssl.c @@ -31,6 +31,7 @@ #include <openssl/hmac.h> #include <openssl/cmac.h> #include <openssl/evp.h> +#include <openssl/opensslv.h> #if (OPENSSL_VERSION_NUMBER >= 0x10100000L) && !defined(OPENSSL_NO_POLY1305) #define _ODP_HAVE_CHACHA20_POLY1305 1 @@ -38,6 +39,11 @@ #define _ODP_HAVE_CHACHA20_POLY1305 0 #endif +/* Ignore warnings about APIs deprecated in OpenSSL 3.0 */ +#if OPENSSL_VERSION_NUMBER >= 0x30000000L +#pragma GCC diagnostic ignored "-Wdeprecated-declarations" +#endif + #define MAX_SESSIONS 4000 #define AES_BLOCK_SIZE 16 #define AES_KEY_LENGTH 16 @@ -2495,6 +2501,7 @@ int odp_crypto_session_destroy(odp_crypto_session_t session) /* * Shim function around packet operation, can be used by other implementations. */ +#if ODP_DEPRECATED_API int odp_crypto_operation(odp_crypto_op_param_t *param, odp_bool_t *posted, @@ -2549,6 +2556,7 @@ odp_crypto_operation(odp_crypto_op_param_t *param, return 0; } +#endif #if OPENSSL_VERSION_NUMBER < 0x10100000L static void ODP_UNUSED openssl_thread_id(CRYPTO_THREADID ODP_UNUSED *id) @@ -2705,6 +2713,7 @@ int _odp_crypto_term_local(void) return 0; } +#if ODP_DEPRECATED_API odp_crypto_compl_t odp_crypto_compl_from_event(odp_event_t ev) { /* This check not mandated by the API specification */ @@ -2741,6 +2750,7 @@ uint64_t odp_crypto_compl_to_u64(odp_crypto_compl_t hdl) { return _odp_pri(hdl); } +#endif /* ODP_DEPRECATED_API */ void odp_crypto_session_param_init(odp_crypto_session_param_t *param) { diff --git a/platform/linux-generic/odp_dma.c b/platform/linux-generic/odp_dma.c index 87e3b5b73..4c129c616 100644 --- a/platform/linux-generic/odp_dma.c +++ b/platform/linux-generic/odp_dma.c @@ -10,6 +10,7 @@ #include <odp/api/align.h> #include <odp/api/buffer.h> #include <odp/api/stash.h> +#include <odp/api/packet.h> #include <odp/api/pool.h> #include <odp/api/queue.h> diff --git a/platform/linux-generic/odp_event.c b/platform/linux-generic/odp_event.c index 95be9dec2..90c3e3a3c 100644 --- a/platform/linux-generic/odp_event.c +++ b/platform/linux-generic/odp_event.c @@ -24,6 +24,7 @@ #include <odp/api/plat/event_inlines.h> #include <odp/api/plat/packet_inlines.h> #include <odp/api/plat/packet_vector_inlines.h> +#include <odp/api/plat/timer_inlines.h> odp_event_subtype_t odp_event_subtype(odp_event_t event) { @@ -70,9 +71,11 @@ void odp_event_free(odp_event_t event) case ODP_EVENT_TIMEOUT: odp_timeout_free(odp_timeout_from_event(event)); break; +#if ODP_DEPRECATED_API case ODP_EVENT_CRYPTO_COMPL: odp_crypto_compl_free(odp_crypto_compl_from_event(event)); break; +#endif case ODP_EVENT_IPSEC_STATUS: _odp_ipsec_status_free(_odp_ipsec_status_from_event(event)); break; @@ -120,8 +123,10 @@ int odp_event_is_valid(odp_event_t event) /* Fall through */ case ODP_EVENT_TIMEOUT: /* Fall through */ +#if ODP_DEPRECATED_API case ODP_EVENT_CRYPTO_COMPL: /* Fall through */ +#endif case ODP_EVENT_IPSEC_STATUS: /* Fall through */ case ODP_EVENT_PACKET_VECTOR: diff --git a/platform/linux-generic/odp_ipsec.c b/platform/linux-generic/odp_ipsec.c index 4dec8f440..29f0e4bcd 100644 --- a/platform/linux-generic/odp_ipsec.c +++ b/platform/linux-generic/odp_ipsec.c @@ -1803,6 +1803,7 @@ static ipsec_sa_t *ipsec_out_single(odp_packet_t pkt, state.out_tunnel.ip_tos = 0; state.out_tunnel.ip_df = 0; state.out_tunnel.ip_flabel = 0; + state.ip_next_hdr = _ODP_IPPROTO_NO_NEXT; rc = 0; } else { rc = -1; @@ -2269,12 +2270,6 @@ int odp_ipsec_out_inline(const odp_packet_t pkt_in[], int num_in, ptr) < 0) status.error.alg = 1; - packet_subtype_set(pkt, ODP_EVENT_PACKET_IPSEC); - result = ipsec_pkt_result(pkt); - memset(result, 0, sizeof(*result)); - result->sa = ipsec_sa->ipsec_sa_hdl; - result->status = status; - if (!status.error.all) { odp_pktout_queue_t pkqueue; diff --git a/platform/linux-generic/odp_ipsec_sad.c b/platform/linux-generic/odp_ipsec_sad.c index c47e4912e..4e4057ede 100644 --- a/platform/linux-generic/odp_ipsec_sad.c +++ b/platform/linux-generic/odp_ipsec_sad.c @@ -863,7 +863,10 @@ ipsec_sa_t *_odp_ipsec_sa_lookup(const ipsec_sa_lookup_t *lookup) lookup->proto == ipsec_sa->proto && lookup->spi == ipsec_sa->spi && lookup->ver == ipsec_sa->in.lookup_ver && - !memcmp(lookup->dst_addr, &ipsec_sa->in.lookup_dst_ipv4, + !memcmp(lookup->dst_addr, + lookup->ver == ODP_IPSEC_IPV4 ? + (void *)&ipsec_sa->in.lookup_dst_ipv4 : + (void *)&ipsec_sa->in.lookup_dst_ipv6, lookup->ver == ODP_IPSEC_IPV4 ? _ODP_IPV4ADDR_LEN : _ODP_IPV6ADDR_LEN)) { diff --git a/platform/linux-generic/odp_packet.c b/platform/linux-generic/odp_packet.c index b91f8ca70..b0d0fc03c 100644 --- a/platform/linux-generic/odp_packet.c +++ b/platform/linux-generic/odp_packet.c @@ -16,6 +16,7 @@ #include <odp/api/proto_stats.h> #include <odp/api/timer.h> +#include <odp_parse_internal.h> #include <odp_chksum_internal.h> #include <odp_debug_internal.h> #include <odp_errno_define.h> @@ -131,11 +132,6 @@ static inline uint32_t packet_seg_len(odp_packet_hdr_t *pkt_hdr, return pkt_hdr->seg_len; } -static inline uint32_t packet_first_seg_len(odp_packet_hdr_t *pkt_hdr) -{ - return pkt_hdr->seg_len; -} - static inline void *packet_tail(odp_packet_hdr_t *pkt_hdr) { odp_packet_hdr_t *last_seg = packet_last_seg(pkt_hdr); @@ -1771,500 +1767,6 @@ static uint32_t packet_sum_crc32c(odp_packet_hdr_t *pkt_hdr, return sum; } -/* - * In the worst case we look at the Ethernet header, 8 bytes of LLC/SNAP - * header and two VLAN tags in the same packet. - */ -#define PARSE_ETH_BYTES (sizeof(_odp_ethhdr_t) + 8 + 2 * sizeof(_odp_vlanhdr_t)) -/** Parser helper function for Ethernet packets - * - * Requires up to PARSE_ETH_BYTES bytes of contiguous packet data. - */ -static inline uint16_t parse_eth(packet_parser_t *prs, const uint8_t **parseptr, - uint32_t *offset, uint32_t frame_len) -{ - uint16_t ethtype; - const _odp_ethhdr_t *eth; - uint16_t macaddr0, macaddr2, macaddr4; - const _odp_vlanhdr_t *vlan; - _odp_packet_input_flags_t input_flags; - - input_flags.all = 0; - input_flags.l2 = 1; - input_flags.eth = 1; - - eth = (const _odp_ethhdr_t *)*parseptr; - - /* Detect jumbo frames */ - if (odp_unlikely(frame_len - *offset > _ODP_ETH_LEN_MAX)) - input_flags.jumbo = 1; - - /* Handle Ethernet broadcast/multicast addresses */ - macaddr0 = odp_be_to_cpu_16(*((const uint16_t *)(const void *)eth)); - if (odp_unlikely((macaddr0 & 0x0100) == 0x0100)) - input_flags.eth_mcast = 1; - - if (odp_unlikely(macaddr0 == 0xffff)) { - macaddr2 = - odp_be_to_cpu_16(*((const uint16_t *) - (const void *)eth + 1)); - macaddr4 = - odp_be_to_cpu_16(*((const uint16_t *) - (const void *)eth + 2)); - - if ((macaddr2 == 0xffff) && (macaddr4 == 0xffff)) - input_flags.eth_bcast = 1; - } - - /* Get Ethertype */ - ethtype = odp_be_to_cpu_16(eth->type); - *offset += sizeof(*eth); - *parseptr += sizeof(*eth); - - /* Check for SNAP vs. DIX */ - if (odp_unlikely(ethtype < _ODP_ETH_LEN_MAX)) { - input_flags.snap = 1; - if (ethtype > frame_len - *offset) { - prs->flags.snap_len_err = 1; - ethtype = 0; - goto error; - } - ethtype = odp_be_to_cpu_16(*((const uint16_t *)(uintptr_t) - (*parseptr + 6))); - *offset += 8; - *parseptr += 8; - } - - /* Parse the VLAN header(s), if present */ - if (odp_unlikely(ethtype == _ODP_ETHTYPE_VLAN_OUTER)) { - input_flags.vlan_qinq = 1; - input_flags.vlan = 1; - - vlan = (const _odp_vlanhdr_t *)*parseptr; - ethtype = odp_be_to_cpu_16(vlan->type); - *offset += sizeof(_odp_vlanhdr_t); - *parseptr += sizeof(_odp_vlanhdr_t); - } - - if (ethtype == _ODP_ETHTYPE_VLAN) { - input_flags.vlan = 1; - vlan = (const _odp_vlanhdr_t *)*parseptr; - ethtype = odp_be_to_cpu_16(vlan->type); - *offset += sizeof(_odp_vlanhdr_t); - *parseptr += sizeof(_odp_vlanhdr_t); - } - - /* - * The packet was too short for what we parsed. We just give up - * entirely without trying to parse what fits in the packet. - */ - if (odp_unlikely(*offset > frame_len)) { - input_flags.all = 0; - input_flags.l2 = 1; - ethtype = 0; - } - -error: - prs->input_flags.all |= input_flags.all; - - return ethtype; -} - -#define PARSE_IPV4_BYTES (0xfU * 4) /* max IPv4 header length with options */ -/** - * Parser helper function for IPv4 - * - * Requires up to PARSE_IPV4_BYTES bytes of contiguous packet data. - */ -static inline uint8_t parse_ipv4(packet_parser_t *prs, const uint8_t **parseptr, - uint32_t *offset, uint32_t frame_len, - odp_proto_chksums_t chksums, - uint64_t *l4_part_sum) -{ - const _odp_ipv4hdr_t *ipv4 = (const _odp_ipv4hdr_t *)*parseptr; - uint32_t dstaddr = odp_be_to_cpu_32(ipv4->dst_addr); - uint32_t l3_len = odp_be_to_cpu_16(ipv4->tot_len); - uint16_t frag_offset = odp_be_to_cpu_16(ipv4->frag_offset); - uint8_t ver = _ODP_IPV4HDR_VER(ipv4->ver_ihl); - uint8_t ihl = _ODP_IPV4HDR_IHL(ipv4->ver_ihl); - - if (odp_unlikely(ihl < _ODP_IPV4HDR_IHL_MIN || - ver != 4 || - sizeof(*ipv4) > frame_len - *offset || - (l3_len > frame_len - *offset))) { - prs->flags.ip_err = 1; - return 0; - } - - if (chksums.chksum.ipv4) { - prs->input_flags.l3_chksum_done = 1; - if (chksum_finalize(chksum_partial(ipv4, ihl * 4, 0)) != 0xffff) { - prs->flags.ip_err = 1; - prs->flags.l3_chksum_err = 1; - return 0; - } - } - - *offset += ihl * 4; - *parseptr += ihl * 4; - - if (chksums.chksum.udp || chksums.chksum.tcp) - *l4_part_sum = chksum_partial((const uint8_t *)&ipv4->src_addr, - 2 * _ODP_IPV4ADDR_LEN, 0); - - if (odp_unlikely(ihl > _ODP_IPV4HDR_IHL_MIN)) - prs->input_flags.ipopt = 1; - - /* A packet is a fragment if: - * "more fragments" flag is set (all fragments except the last) - * OR - * "fragment offset" field is nonzero (all fragments except the first) - */ - if (odp_unlikely(_ODP_IPV4HDR_IS_FRAGMENT(frag_offset))) - prs->input_flags.ipfrag = 1; - - /* Handle IPv4 broadcast / multicast */ - if (odp_unlikely(dstaddr == 0xffffffff)) - prs->input_flags.ip_bcast = 1; - - if (odp_unlikely((dstaddr >> 28) == 0xe)) - prs->input_flags.ip_mcast = 1; - - return ipv4->proto; -} - -/* - * Peeks 2 bytes beyond IPv6 base header without length check if there - * are extension headers. - */ -#define PARSE_IPV6_BYTES (sizeof(_odp_ipv6hdr_t) + 2) -/** - * Parser helper function for IPv6 - * - * Requires at least PARSE_IPV6_BYTES bytes of contiguous packet data. - */ -static inline uint8_t parse_ipv6(packet_parser_t *prs, const uint8_t **parseptr, - uint32_t *offset, uint32_t frame_len, - uint32_t seg_len, - odp_proto_chksums_t chksums, - uint64_t *l4_part_sum) -{ - const _odp_ipv6hdr_t *ipv6 = (const _odp_ipv6hdr_t *)*parseptr; - const _odp_ipv6hdr_ext_t *ipv6ext; - uint32_t dstaddr0 = odp_be_to_cpu_32(ipv6->dst_addr.u8[0]); - uint32_t l3_len = odp_be_to_cpu_16(ipv6->payload_len) + - _ODP_IPV6HDR_LEN; - - /* Basic sanity checks on IPv6 header */ - if (odp_unlikely((odp_be_to_cpu_32(ipv6->ver_tc_flow) >> 28) != 6 || - sizeof(*ipv6) > frame_len - *offset || - l3_len > frame_len - *offset)) { - prs->flags.ip_err = 1; - return 0; - } - - /* IPv6 broadcast / multicast flags */ - prs->input_flags.ip_mcast = (dstaddr0 & 0xff000000) == 0xff000000; - prs->input_flags.ip_bcast = 0; - - /* Skip past IPv6 header */ - *offset += sizeof(_odp_ipv6hdr_t); - *parseptr += sizeof(_odp_ipv6hdr_t); - - if (chksums.chksum.udp || chksums.chksum.tcp) - *l4_part_sum = chksum_partial((const uint8_t *)&ipv6->src_addr, - 2 * _ODP_IPV6ADDR_LEN, 0); - - /* Skip past any IPv6 extension headers */ - if (ipv6->next_hdr == _ODP_IPPROTO_HOPOPTS || - ipv6->next_hdr == _ODP_IPPROTO_ROUTE) { - prs->input_flags.ipopt = 1; - - do { - ipv6ext = (const _odp_ipv6hdr_ext_t *)*parseptr; - uint16_t extlen = 8 + ipv6ext->ext_len * 8; - - *offset += extlen; - *parseptr += extlen; - } while ((ipv6ext->next_hdr == _ODP_IPPROTO_HOPOPTS || - ipv6ext->next_hdr == _ODP_IPPROTO_ROUTE) && - *offset < seg_len); - - if (*offset >= prs->l3_offset + - odp_be_to_cpu_16(ipv6->payload_len)) { - prs->flags.ip_err = 1; - return 0; - } - - if (ipv6ext->next_hdr == _ODP_IPPROTO_FRAG) - prs->input_flags.ipfrag = 1; - - return ipv6ext->next_hdr; - } - - if (odp_unlikely(ipv6->next_hdr == _ODP_IPPROTO_FRAG)) { - prs->input_flags.ipopt = 1; - prs->input_flags.ipfrag = 1; - } - - return ipv6->next_hdr; -} - -#define PARSE_TCP_BYTES (sizeof(_odp_tcphdr_t)) -/** - * Parser helper function for TCP - * - * Requires PARSE_TCP_BYTES bytes of contiguous packet data. - */ -static inline void parse_tcp(packet_parser_t *prs, const uint8_t **parseptr, - uint16_t tcp_len, - odp_proto_chksums_t chksums, - uint64_t *l4_part_sum) -{ - const _odp_tcphdr_t *tcp = (const _odp_tcphdr_t *)*parseptr; - uint32_t len = tcp->hl * 4; - - if (odp_unlikely(tcp->hl < sizeof(_odp_tcphdr_t) / sizeof(uint32_t))) - prs->flags.tcp_err = 1; - - if (chksums.chksum.tcp && - !prs->input_flags.ipfrag) { - *l4_part_sum += odp_cpu_to_be_16(tcp_len); -#if ODP_BYTE_ORDER == ODP_BIG_ENDIAN - *l4_part_sum += _ODP_IPPROTO_TCP; -#else - *l4_part_sum += _ODP_IPPROTO_TCP << 8; -#endif - } - - *parseptr += len; -} - -/* - * In the worst case we look at the UDP header and 4 bytes of the UDP - * payload (the non-ESP marker to distinguish IKE packets from ESP packets). - */ -#define PARSE_UDP_BYTES (sizeof(_odp_udphdr_t) + 4) -/** - * Parser helper function for UDP - * - * Requires PARSE_UDP_BYTES bytes of contiguous packet data. - */ -static inline void parse_udp(packet_parser_t *prs, const uint8_t **parseptr, - odp_proto_chksums_t chksums, - uint64_t *l4_part_sum) -{ - const _odp_udphdr_t *udp = (const _odp_udphdr_t *)*parseptr; - uint32_t udplen = odp_be_to_cpu_16(udp->length); - uint16_t ipsec_port = odp_cpu_to_be_16(_ODP_UDP_IPSEC_PORT); - - if (odp_unlikely(udplen < sizeof(_odp_udphdr_t))) { - prs->flags.udp_err = 1; - return; - } - - if (chksums.chksum.udp && - !prs->input_flags.ipfrag) { - if (udp->chksum == 0) { - prs->input_flags.l4_chksum_done = 1; - prs->flags.l4_chksum_err = - (prs->input_flags.ipv4 != 1); - } else { - *l4_part_sum += udp->length; -#if ODP_BYTE_ORDER == ODP_BIG_ENDIAN - *l4_part_sum += _ODP_IPPROTO_UDP; -#else - *l4_part_sum += _ODP_IPPROTO_UDP << 8; -#endif - } - prs->input_flags.udp_chksum_zero = (udp->chksum == 0); - } - - if (odp_unlikely(ipsec_port == udp->dst_port && udplen > 4)) { - uint32_t val; - - memcpy(&val, udp + 1, 4); - if (val != 0) { - prs->input_flags.ipsec = 1; - prs->input_flags.ipsec_udp = 1; - } - } - - *parseptr += sizeof(_odp_udphdr_t); -} - -#define PARSE_SCTP_BYTES (sizeof(_odp_sctphdr_t)) -/** - * Parser helper function for SCTP - * - * Requires PARSE_SCTP_BYTES bytes of contiguous packet data. - */ -static inline void parse_sctp(packet_parser_t *prs, const uint8_t **parseptr, - uint16_t sctp_len, - odp_proto_chksums_t chksums, - uint64_t *l4_part_sum) -{ - if (odp_unlikely(sctp_len < sizeof(_odp_sctphdr_t))) { - prs->flags.sctp_err = 1; - return; - } - - if (chksums.chksum.sctp && - !prs->input_flags.ipfrag) { - const _odp_sctphdr_t *sctp = - (const _odp_sctphdr_t *)*parseptr; - uint32_t crc = ~0; - uint32_t zero = 0; - - crc = odp_hash_crc32c(sctp, sizeof(*sctp) - 4, crc); - crc = odp_hash_crc32c(&zero, 4, crc); - *l4_part_sum = crc; - } - - *parseptr += sizeof(_odp_sctphdr_t); -} - -#define MAX3(a, b, c) (MAX(MAX((a), (b)), (c))) -#define PARSE_L3_L4_BYTES (MAX(PARSE_IPV4_BYTES, PARSE_IPV6_BYTES) + \ - MAX3(PARSE_TCP_BYTES, PARSE_UDP_BYTES, PARSE_SCTP_BYTES)) -/* Requires up to PARSE_L3_L4_BYTES bytes of contiguous packet data. */ -static inline -int packet_parse_common_l3_l4(packet_parser_t *prs, const uint8_t *parseptr, - uint32_t offset, - uint32_t frame_len, uint32_t seg_len, - int layer, uint16_t ethtype, - odp_proto_chksums_t chksums, - uint64_t *l4_part_sum) -{ - uint8_t ip_proto; - - prs->l3_offset = offset; - - if (odp_unlikely(layer <= ODP_PROTO_LAYER_L2)) - return prs->flags.all.error != 0; - - /* Set l3 flag only for known ethtypes */ - prs->input_flags.l3 = 1; - - /* Parse Layer 3 headers */ - switch (ethtype) { - case _ODP_ETHTYPE_IPV4: - prs->input_flags.ipv4 = 1; - ip_proto = parse_ipv4(prs, &parseptr, &offset, frame_len, - chksums, l4_part_sum); - prs->l4_offset = offset; - break; - - case _ODP_ETHTYPE_IPV6: - prs->input_flags.ipv6 = 1; - ip_proto = parse_ipv6(prs, &parseptr, &offset, frame_len, - seg_len, chksums, l4_part_sum); - prs->l4_offset = offset; - break; - - case _ODP_ETHTYPE_ARP: - prs->input_flags.arp = 1; - ip_proto = 255; /* Reserved invalid by IANA */ - break; - - default: - prs->input_flags.l3 = 0; - ip_proto = 255; /* Reserved invalid by IANA */ - } - - if (layer == ODP_PROTO_LAYER_L3) - return prs->flags.all.error != 0; - - /* Set l4 flag only for known ip_proto */ - prs->input_flags.l4 = 1; - - /* Parse Layer 4 headers */ - switch (ip_proto) { - case _ODP_IPPROTO_ICMPV4: - /* Fall through */ - - case _ODP_IPPROTO_ICMPV6: - prs->input_flags.icmp = 1; - break; - - case _ODP_IPPROTO_IPIP: - /* Do nothing */ - break; - - case _ODP_IPPROTO_TCP: - if (odp_unlikely(offset + _ODP_TCPHDR_LEN > seg_len)) - return -1; - prs->input_flags.tcp = 1; - parse_tcp(prs, &parseptr, frame_len - prs->l4_offset, chksums, - l4_part_sum); - break; - - case _ODP_IPPROTO_UDP: - if (odp_unlikely(offset + _ODP_UDPHDR_LEN > seg_len)) - return -1; - prs->input_flags.udp = 1; - parse_udp(prs, &parseptr, chksums, l4_part_sum); - break; - - case _ODP_IPPROTO_AH: - prs->input_flags.ipsec = 1; - prs->input_flags.ipsec_ah = 1; - break; - - case _ODP_IPPROTO_ESP: - prs->input_flags.ipsec = 1; - prs->input_flags.ipsec_esp = 1; - break; - - case _ODP_IPPROTO_SCTP: - prs->input_flags.sctp = 1; - parse_sctp(prs, &parseptr, frame_len - prs->l4_offset, chksums, - l4_part_sum); - break; - - case _ODP_IPPROTO_NO_NEXT: - prs->input_flags.no_next_hdr = 1; - break; - - default: - prs->input_flags.l4 = 0; - break; - } - - return prs->flags.all.error != 0; -} - -/** - * Parse common packet headers up to given layer - * - * The function expects at least PACKET_PARSE_SEG_LEN bytes of data to be - * available from the ptr. Also parse metadata must be already initialized. - */ -int _odp_packet_parse_common(packet_parser_t *prs, const uint8_t *ptr, - uint32_t frame_len, uint32_t seg_len, - int layer, odp_proto_chksums_t chksums) -{ - uint32_t offset; - uint16_t ethtype; - const uint8_t *parseptr; - uint64_t l4_part_sum; - - parseptr = ptr; - offset = 0; - - if (odp_unlikely(layer == ODP_PROTO_LAYER_NONE)) - return 0; - - /* Assume valid L2 header, no CRC/FCS check in SW */ - prs->l2_offset = offset; - - ethtype = parse_eth(prs, &parseptr, &offset, frame_len); - - return packet_parse_common_l3_l4(prs, parseptr, offset, frame_len, - seg_len, layer, ethtype, chksums, - &l4_part_sum); -} - static inline int packet_ipv4_chksum(odp_packet_t pkt, uint32_t offset, _odp_ipv4hdr_t *ip, @@ -2442,9 +1944,8 @@ int _odp_packet_sctp_chksum_insert(odp_packet_t pkt) return odp_packet_copy_from_mem(pkt, pkt_hdr->p.l4_offset + 8, 4, &sum); } -static int packet_l4_chksum(odp_packet_hdr_t *pkt_hdr, - odp_proto_chksums_t chksums, - uint64_t l4_part_sum) +int _odp_packet_l4_chksum(odp_packet_hdr_t *pkt_hdr, + odp_proto_chksums_t chksums, uint64_t l4_part_sum) { /* UDP chksum == 0 case is covered in parse_udp() */ if (chksums.chksum.udp && @@ -2518,42 +2019,6 @@ static int packet_l4_chksum(odp_packet_hdr_t *pkt_hdr, return pkt_hdr->p.flags.all.error != 0; } -/** - * Simple packet parser - */ -int _odp_packet_parse_layer(odp_packet_hdr_t *pkt_hdr, - odp_proto_layer_t layer, - odp_proto_chksums_t chksums) -{ - uint32_t seg_len = packet_first_seg_len(pkt_hdr); - const uint8_t *base = packet_data(pkt_hdr); - uint32_t offset = 0; - uint16_t ethtype; - uint64_t l4_part_sum = 0; - int rc; - - if (odp_unlikely(layer == ODP_PROTO_LAYER_NONE)) - return 0; - - /* Assume valid L2 header, no CRC/FCS check in SW */ - pkt_hdr->p.l2_offset = offset; - - ethtype = parse_eth(&pkt_hdr->p, &base, &offset, pkt_hdr->frame_len); - - rc = packet_parse_common_l3_l4(&pkt_hdr->p, base, offset, - pkt_hdr->frame_len, - seg_len, layer, ethtype, chksums, - &l4_part_sum); - - if (rc != 0) - return rc; - - if (layer >= ODP_PROTO_LAYER_L4) - return packet_l4_chksum(pkt_hdr, chksums, l4_part_sum); - else - return 0; -} - int odp_packet_parse(odp_packet_t pkt, uint32_t offset, const odp_packet_parse_param_t *param) { @@ -2568,6 +2033,7 @@ int odp_packet_parse(odp_packet_t pkt, uint32_t offset, uint64_t l4_part_sum = 0; const uint32_t min_seglen = PARSE_ETH_BYTES + PARSE_L3_L4_BYTES; uint8_t buf[min_seglen]; + odp_pktin_config_opt_t opt; if (proto == ODP_PROTO_NONE || layer == ODP_PROTO_LAYER_NONE) return -1; @@ -2598,7 +2064,7 @@ int odp_packet_parse(odp_packet_t pkt, uint32_t offset, /* Assume valid L2 header, no CRC/FCS check in SW */ pkt_hdr->p.l2_offset = offset; - ethtype = parse_eth(&pkt_hdr->p, &data, &offset, packet_len); + ethtype = _odp_parse_eth(&pkt_hdr->p, &data, &offset, packet_len); } else if (proto == ODP_PROTO_IPV4) { ethtype = _ODP_ETHTYPE_IPV4; } else if (proto == ODP_PROTO_IPV6) { @@ -2607,17 +2073,18 @@ int odp_packet_parse(odp_packet_t pkt, uint32_t offset, ethtype = 0; /* Invalid */ } - ret = packet_parse_common_l3_l4(&pkt_hdr->p, data, offset, - packet_len, seg_len, - layer, ethtype, - param->chksums, - &l4_part_sum); + opt.all_bits = 0; + + ret = _odp_packet_parse_common_l3_l4(&pkt_hdr->p, data, offset, + packet_len, seg_len, layer, + ethtype, param->chksums, + &l4_part_sum, opt); if (ret) return -1; if (layer >= ODP_PROTO_LAYER_L4) { - ret = packet_l4_chksum(pkt_hdr, param->chksums, l4_part_sum); + ret = _odp_packet_l4_chksum(pkt_hdr, param->chksums, l4_part_sum); if (ret) return -1; } @@ -2879,14 +2346,17 @@ int odp_packet_payload_offset_set(odp_packet_t pkt, uint32_t offset) void odp_packet_aging_tmo_set(odp_packet_t pkt, uint64_t tmo_ns) { - (void)pkt; - (void)tmo_ns; + odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); + + pkt_hdr->p.flags.tx_aging = tmo_ns ? 1 : 0; + pkt_hdr->tx_aging_ns = tmo_ns; } uint64_t odp_packet_aging_tmo(odp_packet_t pkt) { - (void)pkt; - return 0; + odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); + + return pkt_hdr->p.flags.tx_aging ? pkt_hdr->tx_aging_ns : 0; } int odp_packet_tx_compl_request(odp_packet_t pkt, const odp_packet_tx_compl_opt_t *opt) diff --git a/platform/linux-generic/odp_packet_flags.c b/platform/linux-generic/odp_packet_flags.c index bb5c26792..ef542a2cd 100644 --- a/platform/linux-generic/odp_packet_flags.c +++ b/platform/linux-generic/odp_packet_flags.c @@ -8,137 +8,11 @@ #include <odp/api/packet_flags.h> #include <odp_packet_internal.h> -#define retflag(pkt, x) do { \ - odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); \ - return pkt_hdr->p.x; \ - } while (0) - #define setflag(pkt, x, v) do { \ odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); \ pkt_hdr->p.x = (v) & 1; \ } while (0) -int odp_packet_has_error(odp_packet_t pkt) -{ - odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - - return pkt_hdr->p.flags.all.error != 0; -} - -/* Get Input Flags */ - -int odp_packet_has_l2_error(odp_packet_t pkt) -{ - odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - /* L2 parsing is always done by default and hence - no additional check is required */ - return pkt_hdr->p.flags.snap_len_err; -} - -int odp_packet_has_l3(odp_packet_t pkt) -{ - retflag(pkt, input_flags.l3); -} - -int odp_packet_has_l3_error(odp_packet_t pkt) -{ - odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - - return pkt_hdr->p.flags.ip_err; -} - -int odp_packet_has_l4(odp_packet_t pkt) -{ - retflag(pkt, input_flags.l4); -} - -int odp_packet_has_l4_error(odp_packet_t pkt) -{ - odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - - return pkt_hdr->p.flags.tcp_err | pkt_hdr->p.flags.udp_err; -} - -int odp_packet_has_eth_bcast(odp_packet_t pkt) -{ - retflag(pkt, input_flags.eth_bcast); -} - -int odp_packet_has_eth_mcast(odp_packet_t pkt) -{ - retflag(pkt, input_flags.eth_mcast); -} - -int odp_packet_has_vlan(odp_packet_t pkt) -{ - retflag(pkt, input_flags.vlan); -} - -int odp_packet_has_vlan_qinq(odp_packet_t pkt) -{ - retflag(pkt, input_flags.vlan_qinq); -} - -int odp_packet_has_arp(odp_packet_t pkt) -{ - retflag(pkt, input_flags.arp); -} - -int odp_packet_has_ipv4(odp_packet_t pkt) -{ - retflag(pkt, input_flags.ipv4); -} - -int odp_packet_has_ipv6(odp_packet_t pkt) -{ - retflag(pkt, input_flags.ipv6); -} - -int odp_packet_has_ip_bcast(odp_packet_t pkt) -{ - retflag(pkt, input_flags.ip_bcast); -} - -int odp_packet_has_ip_mcast(odp_packet_t pkt) -{ - retflag(pkt, input_flags.ip_mcast); -} - -int odp_packet_has_ipfrag(odp_packet_t pkt) -{ - retflag(pkt, input_flags.ipfrag); -} - -int odp_packet_has_ipopt(odp_packet_t pkt) -{ - retflag(pkt, input_flags.ipopt); -} - -int odp_packet_has_udp(odp_packet_t pkt) -{ - retflag(pkt, input_flags.udp); -} - -int odp_packet_has_tcp(odp_packet_t pkt) -{ - retflag(pkt, input_flags.tcp); -} - -int odp_packet_has_sctp(odp_packet_t pkt) -{ - retflag(pkt, input_flags.sctp); -} - -int odp_packet_has_icmp(odp_packet_t pkt) -{ - retflag(pkt, input_flags.icmp); -} - -odp_packet_color_t odp_packet_color(odp_packet_t pkt) -{ - retflag(pkt, input_flags.color); -} - void odp_packet_color_set(odp_packet_t pkt, odp_packet_color_t color) { odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); @@ -146,23 +20,11 @@ void odp_packet_color_set(odp_packet_t pkt, odp_packet_color_t color) pkt_hdr->p.input_flags.color = color; } -odp_bool_t odp_packet_drop_eligible(odp_packet_t pkt) -{ - odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - - return !pkt_hdr->p.input_flags.nodrop; -} - void odp_packet_drop_eligible_set(odp_packet_t pkt, odp_bool_t drop) { setflag(pkt, input_flags.nodrop, !drop); } -int8_t odp_packet_shaper_len_adjust(odp_packet_t pkt) -{ - retflag(pkt, flags.shaper_len_adj); -} - void odp_packet_shaper_len_adjust_set(odp_packet_t pkt, int8_t adj) { odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); diff --git a/platform/linux-generic/odp_packet_io.c b/platform/linux-generic/odp_packet_io.c index 899f10425..7c68b10c5 100644 --- a/platform/linux-generic/odp_packet_io.c +++ b/platform/linux-generic/odp_packet_io.c @@ -53,6 +53,10 @@ /* Max wait time supported to avoid potential overflow */ #define MAX_WAIT_TIME (UINT64_MAX / 1024) +/* One hour maximum aging timeout, no real limitations imposed by the implementation other than + * integer width, so just use some value. */ +#define MAX_TX_AGING_TMO_NS 3600000000000ULL + typedef struct { const void *user_ptr; odp_queue_t queue; @@ -624,6 +628,8 @@ int odp_pktio_config(odp_pktio_t hdl, const odp_pktio_config_t *config) return -1; } + entry->s.enabled.tx_aging = config->pktout.bit.aging_ena; + if (entry->s.ops->config) res = entry->s.ops->config(entry, config); @@ -650,6 +656,9 @@ int odp_pktio_start(odp_pktio_t hdl) ODP_ERR("Already started\n"); return -1; } + entry->s.parse_layer = pktio_cls_enabled(entry) ? + ODP_PROTO_LAYER_ALL : + entry->s.config.parser.layer; if (entry->s.ops->start) res = entry->s.ops->start(entry); if (!res) @@ -660,8 +669,8 @@ int odp_pktio_start(odp_pktio_t hdl) mode = entry->s.param.in_mode; if (mode == ODP_PKTIN_MODE_SCHED) { - unsigned int i; - unsigned int num = entry->s.num_in_queue; + uint32_t i; + uint32_t num = entry->s.num_in_queue; int index[num]; odp_queue_t odpq[num]; @@ -1182,95 +1191,6 @@ static int pktin_deq_multi(odp_queue_t queue, _odp_event_hdr_t *event_hdr[], return nbr; } -int _odp_sched_cb_pktin_poll_one(int pktio_index, - int rx_queue, - odp_event_t evt_tbl[]) -{ - int num_rx, num_pkts, i; - pktio_entry_t *entry = pktio_entry_by_index(pktio_index); - odp_packet_t pkt; - odp_packet_hdr_t *pkt_hdr; - odp_pool_t pool = ODP_POOL_INVALID; - odp_packet_t packets[QUEUE_MULTI_MAX]; - odp_queue_t queue; - odp_bool_t vector_enabled = entry->s.in_queue[rx_queue].vector.enable; - uint32_t num = QUEUE_MULTI_MAX; - cos_t *cos_hdr = NULL; - - if (odp_unlikely(entry->s.state != PKTIO_STATE_STARTED)) { - if (entry->s.state < PKTIO_STATE_ACTIVE || - entry->s.state == PKTIO_STATE_STOP_PENDING) - return -1; - - ODP_DBG("interface not started\n"); - return 0; - } - - if (vector_enabled) { - /* Make sure all packets will fit into a single packet vector */ - if (entry->s.in_queue[rx_queue].vector.max_size < num) - num = entry->s.in_queue[rx_queue].vector.max_size; - pool = entry->s.in_queue[rx_queue].vector.pool; - } - - ODP_ASSERT((unsigned int)rx_queue < entry->s.num_in_queue); - num_pkts = entry->s.ops->recv(entry, rx_queue, packets, num); - - num_rx = 0; - for (i = 0; i < num_pkts; i++) { - pkt = packets[i]; - pkt_hdr = packet_hdr(pkt); - if (odp_unlikely(pkt_hdr->p.input_flags.dst_queue)) { - odp_event_t event = odp_packet_to_event(pkt); - uint16_t cos_idx = pkt_hdr->cos; - - queue = pkt_hdr->dst_queue; - - if (cos_idx != CLS_COS_IDX_NONE) { - /* Packets from classifier */ - cos_hdr = _odp_cos_entry_from_idx(cos_idx); - - if (cos_hdr->s.vector.enable) { - packet_vector_enq_cos(queue, &event, 1, cos_hdr); - continue; - } - } else if (vector_enabled) { - /* Packets from inline IPsec */ - packet_vector_enq(queue, &event, 1, pool); - continue; - } - - if (odp_unlikely(odp_queue_enq(queue, event))) { - /* Queue full? */ - odp_packet_free(pkt); - if (cos_idx != CLS_COS_IDX_NONE) - _odp_cos_queue_stats_add(cos_hdr, queue, 0, 1); - else - odp_atomic_inc_u64(&entry->s.stats_extra.in_discards); - } else { - if (cos_idx != CLS_COS_IDX_NONE) - _odp_cos_queue_stats_add(cos_hdr, queue, 1, 0); - } - } else { - evt_tbl[num_rx++] = odp_packet_to_event(pkt); - } - } - - /* Create packet vector */ - if (vector_enabled && num_rx > 0) { - odp_packet_vector_t pktv = packet_vector_create((odp_packet_t *)evt_tbl, - num_rx, pool); - - if (odp_unlikely(pktv == ODP_PACKET_VECTOR_INVALID)) - return 0; - - evt_tbl[0] = odp_packet_vector_to_event(pktv); - return 1; - } - - return num_rx; -} - int _odp_sched_cb_pktin_poll(int pktio_index, int pktin_index, _odp_event_hdr_t *hdr_tbl[], int num) { @@ -1886,6 +1806,9 @@ int odp_pktio_capability(odp_pktio_t pktio, odp_pktio_capability_t *capa) capa->tx_compl.queue_type_sched = 1; capa->tx_compl.queue_type_plain = 1; capa->tx_compl.mode_all = 1; + + capa->config.pktout.bit.aging_ena = 1; + capa->max_tx_aging_tmo_ns = MAX_TX_AGING_TMO_NS; } /* Packet vector generation is common for all pktio types */ @@ -2246,8 +2169,7 @@ int odp_pktin_queue_config(odp_pktio_t pktio, pktio_entry_t *entry; odp_pktin_mode_t mode; odp_pktio_capability_t capa; - unsigned int num_queues; - unsigned int i; + uint32_t num_queues, i; int rc; odp_queue_t queue; odp_pktin_queue_param_t default_param; @@ -2410,7 +2332,7 @@ int _odp_pktio_pktout_tm_config(odp_pktio_t pktio_hdl, bool pktio_started = false; odp_pktout_mode_t mode; pktio_entry_t *entry; - unsigned int i; + uint32_t i; int rc = 0; odp_pktout_queue_param_init(¶m); @@ -2480,8 +2402,7 @@ int odp_pktout_queue_config(odp_pktio_t pktio, pktio_entry_t *entry; odp_pktout_mode_t mode; odp_pktio_capability_t capa; - unsigned int num_queues; - unsigned int i; + uint32_t num_queues, i; int rc; odp_pktout_queue_param_t default_param; @@ -2847,18 +2768,17 @@ int odp_pktin_recv_tmo(odp_pktin_queue_t queue, odp_packet_t packets[], int num, } } -int odp_pktin_recv_mq_tmo(const odp_pktin_queue_t queues[], unsigned int num_q, - unsigned int *from, odp_packet_t packets[], int num, - uint64_t wait) +int odp_pktin_recv_mq_tmo(const odp_pktin_queue_t queues[], uint32_t num_q, uint32_t *from, + odp_packet_t packets[], int num, uint64_t wait) { - unsigned int i; + uint32_t i; int ret; odp_time_t t1, t2; struct timespec ts; int started = 0; uint64_t sleep_round = 0; int trial_successful = 0; - unsigned int lfrom = 0; + uint32_t lfrom = 0; for (i = 0; i < num_q; i++) { ret = odp_pktin_recv(queues[i], packets, num); diff --git a/platform/linux-generic/odp_parse.c b/platform/linux-generic/odp_parse.c new file mode 100644 index 000000000..2342a7e49 --- /dev/null +++ b/platform/linux-generic/odp_parse.c @@ -0,0 +1,471 @@ +/* Copyright (c) 2013-2018, Linaro Limited + * Copyright (c) 2019-2022, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <odp_parse_internal.h> +#include <odp_chksum_internal.h> +#include <protocols/eth.h> +#include <protocols/ip.h> +#include <protocols/sctp.h> +#include <protocols/tcp.h> +#include <protocols/udp.h> +#include <odp/api/hash.h> +#include <odp/api/packet_io.h> +#include <odp/api/packet_types.h> +#include <stdint.h> +#include <string.h> + +/** Parser helper function for Ethernet packets + * + * Requires up to PARSE_ETH_BYTES bytes of contiguous packet data. + */ +uint16_t _odp_parse_eth(packet_parser_t *prs, const uint8_t **parseptr, + uint32_t *offset, uint32_t frame_len) +{ + uint16_t ethtype; + const _odp_ethhdr_t *eth; + uint16_t macaddr0, macaddr2, macaddr4; + const _odp_vlanhdr_t *vlan; + _odp_packet_input_flags_t input_flags; + + input_flags.all = 0; + input_flags.l2 = 1; + input_flags.eth = 1; + + eth = (const _odp_ethhdr_t *)*parseptr; + + /* Detect jumbo frames */ + if (odp_unlikely(frame_len - *offset > _ODP_ETH_LEN_MAX)) + input_flags.jumbo = 1; + + /* Handle Ethernet broadcast/multicast addresses */ + macaddr0 = odp_be_to_cpu_16(*((const uint16_t *)(const void *)eth)); + if (odp_unlikely((macaddr0 & 0x0100) == 0x0100)) + input_flags.eth_mcast = 1; + + if (odp_unlikely(macaddr0 == 0xffff)) { + macaddr2 = + odp_be_to_cpu_16(*((const uint16_t *) + (const void *)eth + 1)); + macaddr4 = + odp_be_to_cpu_16(*((const uint16_t *) + (const void *)eth + 2)); + + if ((macaddr2 == 0xffff) && (macaddr4 == 0xffff)) + input_flags.eth_bcast = 1; + } + + /* Get Ethertype */ + ethtype = odp_be_to_cpu_16(eth->type); + *offset += sizeof(*eth); + *parseptr += sizeof(*eth); + + /* Check for SNAP vs. DIX */ + if (odp_unlikely(ethtype < _ODP_ETH_LEN_MAX)) { + input_flags.snap = 1; + if (ethtype > frame_len - *offset) { + prs->flags.snap_len_err = 1; + ethtype = 0; + goto error; + } + ethtype = odp_be_to_cpu_16(*((const uint16_t *)(uintptr_t) + (*parseptr + 6))); + *offset += 8; + *parseptr += 8; + } + + /* Parse the VLAN header(s), if present */ + if (odp_unlikely(ethtype == _ODP_ETHTYPE_VLAN_OUTER)) { + input_flags.vlan_qinq = 1; + input_flags.vlan = 1; + + vlan = (const _odp_vlanhdr_t *)*parseptr; + ethtype = odp_be_to_cpu_16(vlan->type); + *offset += sizeof(_odp_vlanhdr_t); + *parseptr += sizeof(_odp_vlanhdr_t); + } + + if (ethtype == _ODP_ETHTYPE_VLAN) { + input_flags.vlan = 1; + vlan = (const _odp_vlanhdr_t *)*parseptr; + ethtype = odp_be_to_cpu_16(vlan->type); + *offset += sizeof(_odp_vlanhdr_t); + *parseptr += sizeof(_odp_vlanhdr_t); + } + + /* + * The packet was too short for what we parsed. We just give up + * entirely without trying to parse what fits in the packet. + */ + if (odp_unlikely(*offset > frame_len)) { + input_flags.all = 0; + input_flags.l2 = 1; + ethtype = 0; + } + +error: + prs->input_flags.all |= input_flags.all; + + return ethtype; +} + +/** + * Parser helper function for IPv4 + * + * Requires up to PARSE_IPV4_BYTES bytes of contiguous packet data. + */ +static inline uint8_t parse_ipv4(packet_parser_t *prs, const uint8_t **parseptr, + uint32_t *offset, uint32_t frame_len, + odp_proto_chksums_t chksums, + uint64_t *l4_part_sum) +{ + const _odp_ipv4hdr_t *ipv4 = (const _odp_ipv4hdr_t *)*parseptr; + uint32_t dstaddr = odp_be_to_cpu_32(ipv4->dst_addr); + uint32_t l3_len = odp_be_to_cpu_16(ipv4->tot_len); + uint16_t frag_offset = odp_be_to_cpu_16(ipv4->frag_offset); + uint8_t ver = _ODP_IPV4HDR_VER(ipv4->ver_ihl); + uint8_t ihl = _ODP_IPV4HDR_IHL(ipv4->ver_ihl); + + if (odp_unlikely(ihl < _ODP_IPV4HDR_IHL_MIN || + ver != 4 || + sizeof(*ipv4) > frame_len - *offset || + (l3_len > frame_len - *offset))) { + prs->flags.ip_err = 1; + return 0; + } + + if (chksums.chksum.ipv4) { + prs->input_flags.l3_chksum_done = 1; + if (chksum_finalize(chksum_partial(ipv4, ihl * 4, 0)) != 0xffff) { + prs->flags.ip_err = 1; + prs->flags.l3_chksum_err = 1; + return 0; + } + } + + *offset += ihl * 4; + *parseptr += ihl * 4; + + if (chksums.chksum.udp || chksums.chksum.tcp) + *l4_part_sum = chksum_partial((const uint8_t *)&ipv4->src_addr, + 2 * _ODP_IPV4ADDR_LEN, 0); + + if (odp_unlikely(ihl > _ODP_IPV4HDR_IHL_MIN)) + prs->input_flags.ipopt = 1; + + /* A packet is a fragment if: + * "more fragments" flag is set (all fragments except the last) + * OR + * "fragment offset" field is nonzero (all fragments except the first) + */ + if (odp_unlikely(_ODP_IPV4HDR_IS_FRAGMENT(frag_offset))) + prs->input_flags.ipfrag = 1; + + /* Handle IPv4 broadcast / multicast */ + if (odp_unlikely(dstaddr == 0xffffffff)) + prs->input_flags.ip_bcast = 1; + + if (odp_unlikely((dstaddr >> 28) == 0xe)) + prs->input_flags.ip_mcast = 1; + + return ipv4->proto; +} + +/** + * Parser helper function for IPv6 + * + * Requires at least PARSE_IPV6_BYTES bytes of contiguous packet data. + */ +static inline uint8_t parse_ipv6(packet_parser_t *prs, const uint8_t **parseptr, + uint32_t *offset, uint32_t frame_len, + uint32_t seg_len, + odp_proto_chksums_t chksums, + uint64_t *l4_part_sum) +{ + const _odp_ipv6hdr_t *ipv6 = (const _odp_ipv6hdr_t *)*parseptr; + const _odp_ipv6hdr_ext_t *ipv6ext; + uint32_t dstaddr0 = odp_be_to_cpu_32(ipv6->dst_addr.u8[0]); + uint32_t l3_len = odp_be_to_cpu_16(ipv6->payload_len) + + _ODP_IPV6HDR_LEN; + + /* Basic sanity checks on IPv6 header */ + if (odp_unlikely((odp_be_to_cpu_32(ipv6->ver_tc_flow) >> 28) != 6 || + sizeof(*ipv6) > frame_len - *offset || + l3_len > frame_len - *offset)) { + prs->flags.ip_err = 1; + return 0; + } + + /* IPv6 broadcast / multicast flags */ + prs->input_flags.ip_mcast = (dstaddr0 & 0xff000000) == 0xff000000; + prs->input_flags.ip_bcast = 0; + + /* Skip past IPv6 header */ + *offset += sizeof(_odp_ipv6hdr_t); + *parseptr += sizeof(_odp_ipv6hdr_t); + + if (chksums.chksum.udp || chksums.chksum.tcp) + *l4_part_sum = chksum_partial((const uint8_t *)&ipv6->src_addr, + 2 * _ODP_IPV6ADDR_LEN, 0); + + /* Skip past any IPv6 extension headers */ + if (ipv6->next_hdr == _ODP_IPPROTO_HOPOPTS || + ipv6->next_hdr == _ODP_IPPROTO_ROUTE) { + prs->input_flags.ipopt = 1; + + do { + ipv6ext = (const _odp_ipv6hdr_ext_t *)*parseptr; + uint16_t extlen = 8 + ipv6ext->ext_len * 8; + + *offset += extlen; + *parseptr += extlen; + } while ((ipv6ext->next_hdr == _ODP_IPPROTO_HOPOPTS || + ipv6ext->next_hdr == _ODP_IPPROTO_ROUTE) && + *offset < seg_len); + + if (*offset >= prs->l3_offset + + odp_be_to_cpu_16(ipv6->payload_len)) { + prs->flags.ip_err = 1; + return 0; + } + + if (ipv6ext->next_hdr == _ODP_IPPROTO_FRAG) + prs->input_flags.ipfrag = 1; + + return ipv6ext->next_hdr; + } + + if (odp_unlikely(ipv6->next_hdr == _ODP_IPPROTO_FRAG)) { + prs->input_flags.ipopt = 1; + prs->input_flags.ipfrag = 1; + } + + return ipv6->next_hdr; +} + +/** + * Parser helper function for TCP + * + * Requires PARSE_TCP_BYTES bytes of contiguous packet data. + */ +static inline void parse_tcp(packet_parser_t *prs, const uint8_t **parseptr, + uint16_t tcp_len, + odp_proto_chksums_t chksums, + uint64_t *l4_part_sum) +{ + const _odp_tcphdr_t *tcp = (const _odp_tcphdr_t *)*parseptr; + uint32_t len = tcp->hl * 4; + + if (odp_unlikely(tcp->hl < sizeof(_odp_tcphdr_t) / sizeof(uint32_t))) + prs->flags.tcp_err = 1; + + if (chksums.chksum.tcp && + !prs->input_flags.ipfrag) { + *l4_part_sum += odp_cpu_to_be_16(tcp_len); +#if ODP_BYTE_ORDER == ODP_BIG_ENDIAN + *l4_part_sum += _ODP_IPPROTO_TCP; +#else + *l4_part_sum += _ODP_IPPROTO_TCP << 8; +#endif + } + + *parseptr += len; +} + +/** + * Parser helper function for UDP + * + * Requires PARSE_UDP_BYTES bytes of contiguous packet data. + */ +static inline void parse_udp(packet_parser_t *prs, const uint8_t **parseptr, + odp_proto_chksums_t chksums, + uint64_t *l4_part_sum) +{ + const _odp_udphdr_t *udp = (const _odp_udphdr_t *)*parseptr; + uint32_t udplen = odp_be_to_cpu_16(udp->length); + uint16_t ipsec_port = odp_cpu_to_be_16(_ODP_UDP_IPSEC_PORT); + + if (odp_unlikely(udplen < sizeof(_odp_udphdr_t))) { + prs->flags.udp_err = 1; + return; + } + + if (chksums.chksum.udp && + !prs->input_flags.ipfrag) { + if (udp->chksum == 0) { + prs->input_flags.l4_chksum_done = 1; + prs->flags.l4_chksum_err = + (prs->input_flags.ipv4 != 1); + } else { + *l4_part_sum += udp->length; +#if ODP_BYTE_ORDER == ODP_BIG_ENDIAN + *l4_part_sum += _ODP_IPPROTO_UDP; +#else + *l4_part_sum += _ODP_IPPROTO_UDP << 8; +#endif + } + prs->input_flags.udp_chksum_zero = (udp->chksum == 0); + } + + if (odp_unlikely(ipsec_port == udp->dst_port && udplen > 4)) { + uint32_t val; + + memcpy(&val, udp + 1, 4); + if (val != 0) { + prs->input_flags.ipsec = 1; + prs->input_flags.ipsec_udp = 1; + } + } + + *parseptr += sizeof(_odp_udphdr_t); +} + +/** + * Parser helper function for SCTP + * + * Requires PARSE_SCTP_BYTES bytes of contiguous packet data. + */ +static inline void parse_sctp(packet_parser_t *prs, const uint8_t **parseptr, + uint16_t sctp_len, + odp_proto_chksums_t chksums, + uint64_t *l4_part_sum) +{ + if (odp_unlikely(sctp_len < sizeof(_odp_sctphdr_t))) { + prs->flags.sctp_err = 1; + return; + } + + if (chksums.chksum.sctp && + !prs->input_flags.ipfrag) { + const _odp_sctphdr_t *sctp = + (const _odp_sctphdr_t *)*parseptr; + uint32_t crc = ~0; + uint32_t zero = 0; + + crc = odp_hash_crc32c(sctp, sizeof(*sctp) - 4, crc); + crc = odp_hash_crc32c(&zero, 4, crc); + *l4_part_sum = crc; + } + + *parseptr += sizeof(_odp_sctphdr_t); +} + +/* Requires up to PARSE_L3_L4_BYTES bytes of contiguous packet data. */ +int _odp_packet_parse_common_l3_l4(packet_parser_t *prs, + const uint8_t *parseptr, uint32_t offset, + uint32_t frame_len, uint32_t seg_len, + int layer, uint16_t ethtype, + odp_proto_chksums_t chksums, + uint64_t *l4_part_sum, + odp_pktin_config_opt_t opt) +{ + uint8_t ip_proto; + + prs->l3_offset = offset; + + if (odp_unlikely(layer <= ODP_PROTO_LAYER_L2)) + return prs->flags.all.error != 0; + + /* Set l3 flag only for known ethtypes */ + prs->input_flags.l3 = 1; + + /* Parse Layer 3 headers */ + switch (ethtype) { + case _ODP_ETHTYPE_IPV4: + prs->input_flags.ipv4 = 1; + ip_proto = parse_ipv4(prs, &parseptr, &offset, frame_len, + chksums, l4_part_sum); + prs->l4_offset = offset; + if (prs->flags.ip_err && opt.bit.drop_ipv4_err) + return -1; /* drop */ + break; + + case _ODP_ETHTYPE_IPV6: + prs->input_flags.ipv6 = 1; + ip_proto = parse_ipv6(prs, &parseptr, &offset, frame_len, + seg_len, chksums, l4_part_sum); + prs->l4_offset = offset; + if (prs->flags.ip_err && opt.bit.drop_ipv6_err) + return -1; /* drop */ + break; + + case _ODP_ETHTYPE_ARP: + prs->input_flags.arp = 1; + ip_proto = 255; /* Reserved invalid by IANA */ + break; + + default: + prs->input_flags.l3 = 0; + ip_proto = 255; /* Reserved invalid by IANA */ + } + + if (layer == ODP_PROTO_LAYER_L3) + return prs->flags.all.error != 0; + + /* Set l4 flag only for known ip_proto */ + prs->input_flags.l4 = 1; + + /* Parse Layer 4 headers */ + switch (ip_proto) { + case _ODP_IPPROTO_ICMPV4: + /* Fall through */ + + case _ODP_IPPROTO_ICMPV6: + prs->input_flags.icmp = 1; + break; + + case _ODP_IPPROTO_IPIP: + /* Do nothing */ + break; + + case _ODP_IPPROTO_TCP: + if (odp_unlikely(offset + _ODP_TCPHDR_LEN > seg_len)) + return -1; + prs->input_flags.tcp = 1; + parse_tcp(prs, &parseptr, frame_len - prs->l4_offset, chksums, + l4_part_sum); + if (prs->flags.tcp_err && opt.bit.drop_tcp_err) + return -1; /* drop */ + break; + + case _ODP_IPPROTO_UDP: + if (odp_unlikely(offset + _ODP_UDPHDR_LEN > seg_len)) + return -1; + prs->input_flags.udp = 1; + parse_udp(prs, &parseptr, chksums, l4_part_sum); + if (prs->flags.udp_err && opt.bit.drop_udp_err) + return -1; /* drop */ + break; + + case _ODP_IPPROTO_AH: + prs->input_flags.ipsec = 1; + prs->input_flags.ipsec_ah = 1; + break; + + case _ODP_IPPROTO_ESP: + prs->input_flags.ipsec = 1; + prs->input_flags.ipsec_esp = 1; + break; + + case _ODP_IPPROTO_SCTP: + prs->input_flags.sctp = 1; + parse_sctp(prs, &parseptr, frame_len - prs->l4_offset, chksums, + l4_part_sum); + if (prs->flags.sctp_err && opt.bit.drop_sctp_err) + return -1; /* drop */ + break; + + case _ODP_IPPROTO_NO_NEXT: + prs->input_flags.no_next_hdr = 1; + break; + + default: + prs->input_flags.l4 = 0; + break; + } + + return prs->flags.all.error != 0; +} diff --git a/platform/linux-generic/odp_pool.c b/platform/linux-generic/odp_pool.c index cca2d55f0..defdeb4fb 100644 --- a/platform/linux-generic/odp_pool.c +++ b/platform/linux-generic/odp_pool.c @@ -494,12 +494,7 @@ static void init_buffers(pool_t *pool) for (i = 0; i < pool->num + skipped_blocks ; i++) { int skip = 0; - - addr = &pool->base_addr[(i * pool->block_size) + - pool->block_offset]; - event_hdr = addr; - buf_hdr = addr; - pkt_hdr = addr; + addr = &pool->base_addr[i * pool->block_size]; /* Skip packet buffers which cross huge page boundaries. Some * NICs cannot handle buffers which cross page boundaries. */ @@ -517,6 +512,12 @@ static void init_buffers(pool_t *pool) skip = 1; } } + + addr = (uint8_t *)addr + pool->block_offset; + event_hdr = addr; + buf_hdr = addr; + pkt_hdr = addr; + if (pool->uarea_size) uarea = &pool->uarea_base_addr[(i - skipped_blocks) * pool->uarea_size]; diff --git a/platform/linux-generic/odp_queue_lf.c b/platform/linux-generic/odp_queue_lf.c index f5c3830e8..58959df31 100644 --- a/platform/linux-generic/odp_queue_lf.c +++ b/platform/linux-generic/odp_queue_lf.c @@ -13,6 +13,7 @@ #include <odp_debug_internal.h> #include <odp_event_internal.h> #include <odp_queue_basic_internal.h> +#include <odp_types_internal.h> #include <string.h> #include <stdio.h> @@ -24,9 +25,7 @@ #ifdef __SIZEOF_INT128__ -__extension__ typedef unsigned __int128 u128_t; - -static inline void lockfree_zero_u128(u128_t *atomic) +static inline void lockfree_zero_u128(_odp_u128_t *atomic) { __atomic_store_n(atomic, 0, __ATOMIC_RELAXED); } @@ -40,21 +39,21 @@ static inline void lockfree_zero_u128(u128_t *atomic) * So, these are never actually used. */ typedef struct ODP_ALIGNED(16) { uint64_t u64[2]; -} u128_t; +} _odp_u128_t; -static inline u128_t lockfree_load_u128(u128_t *atomic) +static inline _odp_u128_t lockfree_load_u128(_odp_u128_t *atomic) { return *atomic; } -static inline void lockfree_zero_u128(u128_t *atomic) +static inline void lockfree_zero_u128(_odp_u128_t *atomic) { atomic->u64[0] = 0; atomic->u64[1] = 0; } -static inline int lockfree_cas_acq_rel_u128(u128_t *atomic, u128_t old_val, - u128_t new_val) +static inline int lockfree_cas_acq_rel_u128(_odp_u128_t *atomic, _odp_u128_t old_val, + _odp_u128_t new_val) { if (atomic->u64[0] == old_val.u64[0] && atomic->u64[1] == old_val.u64[1]) { @@ -75,7 +74,7 @@ static inline int lockfree_check_u128(void) /* Node in lock-free ring */ typedef union { - u128_t u128; + _odp_u128_t u128; struct { /* Data with lowest counter value is the head. Empty node has diff --git a/platform/linux-generic/odp_schedule_scalable.c b/platform/linux-generic/odp_schedule_scalable.c index 5343c2834..7482d776d 100644 --- a/platform/linux-generic/odp_schedule_scalable.c +++ b/platform/linux-generic/odp_schedule_scalable.c @@ -817,23 +817,19 @@ events_dequeued: } /* Ingress queue empty => poll pktio RX queue */ - odp_event_t rx_evts[QUEUE_MULTI_MAX]; - int num_rx = _odp_sched_cb_pktin_poll_one(elem->pktio_idx, - elem->rx_queue, - rx_evts); + _odp_event_hdr_t *rx_evts[QUEUE_MULTI_MAX]; + int num_rx = _odp_sched_cb_pktin_poll(elem->pktio_idx, elem->rx_queue, + rx_evts, QUEUE_MULTI_MAX); + if (odp_likely(num_rx > 0)) { num = num_rx < num_evts ? num_rx : num_evts; for (i = 0; i < num; i++) { /* Return events directly to caller */ - ev[i] = rx_evts[i]; + ev[i] = _odp_event_from_hdr(rx_evts[i]); } if (num_rx > num) { /* Events remain, enqueue them */ - _odp_event_hdr_t *events[QUEUE_MULTI_MAX]; - - for (i = num; i < num_rx; i++) - events[i] = _odp_event_hdr(rx_evts[i]); - i = _odp_queue_enq_sp(elem, &events[num], num_rx - num); + i = _odp_queue_enq_sp(elem, &rx_evts[num], num_rx - num); /* Enqueue must succeed as the queue was empty */ ODP_ASSERT(i == num_rx - num); } diff --git a/platform/linux-generic/odp_schedule_sp.c b/platform/linux-generic/odp_schedule_sp.c index f5aba1a88..558c3f4fe 100644 --- a/platform/linux-generic/odp_schedule_sp.c +++ b/platform/linux-generic/odp_schedule_sp.c @@ -13,6 +13,7 @@ #pragma GCC diagnostic ignored "-Wzero-length-bounds" #endif +#include <odp/api/packet.h> #include <odp/api/ticketlock.h> #include <odp/api/thread.h> #include <odp/api/plat/thread_inlines.h> diff --git a/platform/linux-generic/odp_time.c b/platform/linux-generic/odp_time.c index d908f02cb..b591d712a 100644 --- a/platform/linux-generic/odp_time.c +++ b/platform/linux-generic/odp_time.c @@ -145,15 +145,6 @@ static inline void time_wait_until(odp_time_t time) } while (odp_time_cmp(time, cur) > 0); } -uint64_t odp_time_diff_ns(odp_time_t t2, odp_time_t t1) -{ - odp_time_t time; - - time.u64 = t2.u64 - t1.u64; - - return odp_time_to_ns(time); -} - odp_time_t odp_time_local_from_ns(uint64_t ns) { return time_from_ns(ns); diff --git a/platform/linux-generic/odp_timer.c b/platform/linux-generic/odp_timer.c index d1d4a5bc8..1c54ab740 100644 --- a/platform/linux-generic/odp_timer.c +++ b/platform/linux-generic/odp_timer.c @@ -13,49 +13,52 @@ */ #include <odp_posix_extensions.h> -#include <errno.h> -#include <stdlib.h> -#include <time.h> -#include <signal.h> -#include <pthread.h> -#include <unistd.h> -#include <sys/syscall.h> -#include <inttypes.h> -#include <string.h> - -#include <odp/api/std.h> #include <odp/api/align.h> -#include <odp_align_internal.h> #include <odp/api/atomic.h> -#include <odp/api/plat/atomic_inlines.h> -#include <odp_atomic_internal.h> -#include <odp/api/buffer.h> #include <odp/api/cpu.h> -#include <odp/api/pool.h> -#include <odp_pool_internal.h> #include <odp/api/debug.h> -#include <odp_debug_internal.h> #include <odp/api/event.h> #include <odp/api/hints.h> -#include <odp_init_internal.h> -#include <odp_errno_define.h> +#include <odp/api/pool.h> #include <odp/api/queue.h> #include <odp/api/shared_memory.h> #include <odp/api/spinlock.h> -#include <odp/api/std_types.h> +#include <odp/api/std.h> #include <odp/api/sync.h> #include <odp/api/time.h> -#include <odp/api/plat/time_inlines.h> #include <odp/api/timer.h> + +/* Inlined API functions */ +#include <odp/api/plat/atomic_inlines.h> +#include <odp/api/plat/event_inlines.h> +#include <odp/api/plat/queue_inlines.h> +#include <odp/api/plat/time_inlines.h> +#include <odp/api/plat/timer_inlines.h> + +#include <odp/api/plat/timer_inline_types.h> + +#include <odp_align_internal.h> +#include <odp_atomic_internal.h> +#include <odp_debug_internal.h> +#include <odp_errno_define.h> +#include <odp_event_internal.h> +#include <odp_global_data.h> +#include <odp_init_internal.h> #include <odp_libconfig_internal.h> +#include <odp_pool_internal.h> #include <odp_queue_if.h> #include <odp_timer_internal.h> -#include <odp/api/plat/queue_inlines.h> -#include <odp_global_data.h> -#include <odp_event_internal.h> +#include <odp_types_internal.h> -/* Inlined API functions */ -#include <odp/api/plat/event_inlines.h> +#include <errno.h> +#include <inttypes.h> +#include <pthread.h> +#include <signal.h> +#include <stdlib.h> +#include <string.h> +#include <sys/syscall.h> +#include <time.h> +#include <unistd.h> /* One divided by one nanosecond in Hz */ #define GIGA_HZ 1000000000 @@ -94,6 +97,18 @@ #define IDX2LOCK(idx) (&timer_global->locks[(idx) % NUM_LOCKS]) #endif +#include <odp/visibility_begin.h> + +/* Fill in timeout header field offsets for inline functions */ +const _odp_timeout_inline_offset_t +_odp_timeout_inline_offset ODP_ALIGNED_CACHE = { + .expiration = offsetof(odp_timeout_hdr_t, expiration), + .timer = offsetof(odp_timeout_hdr_t, timer), + .user_ptr = offsetof(odp_timeout_hdr_t, user_ptr) +}; + +#include <odp/visibility_end.h> + typedef struct #ifdef ODP_ATOMIC_U128 ODP_ALIGNED(16) /* 16-byte atomic operations need properly aligned addresses */ @@ -689,7 +704,7 @@ static bool timer_reset(uint32_t idx, uint64_t abs_tck, odp_event_t *tmo_event, /* Atomic CAS will fail if we experienced torn reads, * retry update sequence until CAS succeeds */ } while (!_odp_atomic_u128_cmp_xchg_mm((_odp_atomic_u128_t *)tb, - (_uint128_t *)&old, (_uint128_t *)&new, + (_odp_u128_t *)&old, (_odp_u128_t *)&new, _ODP_MEMMODEL_RLS, _ODP_MEMMODEL_RLX)); #elif __GCC_ATOMIC_LLONG_LOCK_FREE >= 2 && \ defined __GCC_HAVE_SYNC_COMPARE_AND_SWAP_8 @@ -760,8 +775,8 @@ static bool timer_reset(uint32_t idx, uint64_t abs_tck, odp_event_t *tmo_event, /* We are releasing the new timeout event to some other * thread */ _odp_atomic_u128_xchg_mm((_odp_atomic_u128_t *)tb, - (_uint128_t *)&new, - (_uint128_t *)&old, + (_odp_u128_t *)&new, + (_odp_u128_t *)&old, _ODP_MEMMODEL_ACQ_RLS); old_event = old.tmo_event; #else @@ -804,7 +819,7 @@ static odp_event_t timer_set_unused(timer_pool_t *tp, uint32_t idx) new.tmo_event = ODP_EVENT_INVALID; _odp_atomic_u128_xchg_mm((_odp_atomic_u128_t *)tb, - (_uint128_t *)&new, (_uint128_t *)&old, + (_odp_u128_t *)&new, (_odp_u128_t *)&old, _ODP_MEMMODEL_RLX); old_event = old.tmo_event; #else @@ -858,8 +873,8 @@ static odp_event_t timer_cancel(timer_pool_t *tp, uint32_t idx) /* Atomic CAS will fail if we experienced torn reads, * retry update sequence until CAS succeeds */ } while (!_odp_atomic_u128_cmp_xchg_mm((_odp_atomic_u128_t *)tb, - (_uint128_t *)&old, - (_uint128_t *)&new, + (_odp_u128_t *)&old, + (_odp_u128_t *)&new, _ODP_MEMMODEL_RLS, _ODP_MEMMODEL_RLX)); old_event = old.tmo_event; @@ -916,7 +931,7 @@ static inline void timer_expire(timer_pool_t *tp, uint32_t idx, uint64_t tick) new.tmo_event = ODP_EVENT_INVALID; int succ = _odp_atomic_u128_cmp_xchg_mm((_odp_atomic_u128_t *)tb, - (_uint128_t *)&old, (_uint128_t *)&new, + (_odp_u128_t *)&old, (_odp_u128_t *)&new, _ODP_MEMMODEL_RLS, _ODP_MEMMODEL_RLX); if (succ) tmo_event = old.tmo_event; @@ -1468,22 +1483,6 @@ void odp_timer_pool_destroy(odp_timer_pool_t tpid) odp_timer_pool_del(timer_pool_from_hdl(tpid)); } -uint64_t odp_timer_tick_to_ns(odp_timer_pool_t tpid, uint64_t ticks) -{ - (void)tpid; - - /* Timer ticks in API are nsec */ - return ticks; -} - -uint64_t odp_timer_ns_to_tick(odp_timer_pool_t tpid, uint64_t ns) -{ - (void)tpid; - - /* Timer ticks in API are nsec */ - return ns; -} - uint64_t odp_timer_current_tick(odp_timer_pool_t tpid) { timer_pool_t *tp = timer_pool_from_hdl(tpid); @@ -1812,19 +1811,6 @@ uint64_t odp_timer_to_u64(odp_timer_t hdl) return _odp_pri(hdl); } -odp_timeout_t odp_timeout_from_event(odp_event_t ev) -{ - /* This check not mandated by the API specification */ - if (odp_event_type(ev) != ODP_EVENT_TIMEOUT) - ODP_ABORT("Event not a timeout"); - return (odp_timeout_t)ev; -} - -odp_event_t odp_timeout_to_event(odp_timeout_t tmo) -{ - return (odp_event_t)tmo; -} - uint64_t odp_timeout_to_u64(odp_timeout_t tmo) { return _odp_pri(tmo); @@ -1847,21 +1833,6 @@ int odp_timeout_fresh(odp_timeout_t tmo) return hdr->expiration == (exp_tck & ~TMO_INACTIVE); } -odp_timer_t odp_timeout_timer(odp_timeout_t tmo) -{ - return timeout_hdr(tmo)->timer; -} - -uint64_t odp_timeout_tick(odp_timeout_t tmo) -{ - return timeout_hdr(tmo)->expiration; -} - -void *odp_timeout_user_ptr(odp_timeout_t tmo) -{ - return (void *)(uintptr_t)timeout_hdr(tmo)->user_ptr; -} - odp_timeout_t odp_timeout_alloc(odp_pool_t pool_hdl) { odp_event_t event; diff --git a/platform/linux-generic/odp_timer_api.c b/platform/linux-generic/odp_timer_api.c new file mode 100644 index 000000000..cd657956b --- /dev/null +++ b/platform/linux-generic/odp_timer_api.c @@ -0,0 +1,11 @@ +/* Copyright (c) 2022, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <odp/api/timer.h> + +/* Non-inlined functions for ABI compat mode */ +#define _ODP_NO_INLINE +#include <odp/api/plat/timer_inlines.h> diff --git a/platform/linux-generic/odp_traffic_mngr.c b/platform/linux-generic/odp_traffic_mngr.c index fbf33e117..f02297ec9 100644 --- a/platform/linux-generic/odp_traffic_mngr.c +++ b/platform/linux-generic/odp_traffic_mngr.c @@ -55,45 +55,60 @@ _odp_int_name_kind_t PROFILE_TO_HANDLE_KIND[ODP_TM_NUM_PROFILES] = { static const pkt_desc_t EMPTY_PKT_DESC = { .word = 0 }; #define MAX_PRIORITIES ODP_TM_MAX_PRIORITIES -#define NUM_SHAPER_COLORS ODP_NUM_SHAPER_COLORS /* Shaper BW limits in bits/sec */ #define TM_MIN_SHAPER_BW 8000ULL #define TM_MAX_SHAPER_BW (100ULL * 1000ULL * 1000ULL * 1000ULL) +/* Possible values for running the shaper algorithm. TM_SHAPER_GREEN means that + * the traffic is within the commit specification (rate and burst size), + * TM_SHAPER_YELLOW means that the traffic is within the peak specification + * (rate and burst size) and TM_SHAPER_RED means that the traffic is exceeding + * both its commit and peak specifications. Note that packets can also have an + * assigned packet color of ODP_PACKET_GREEN, ODP_PACKET_YELLOW or + * ODP_PACKET_RED, which has a different meaning and purpose than the shaper + * colors. + */ +typedef enum { + TM_SHAPER_GREEN, TM_SHAPER_YELLOW, TM_SHAPER_RED +} tm_shaper_color_t; + +/* Number of enumeration values defined in tm_shaper_color_t type. */ +#define NUM_SHAPER_COLORS 3 + static const tm_prop_t basic_prop_tbl[MAX_PRIORITIES][NUM_SHAPER_COLORS] = { [0] = { - [ODP_TM_SHAPER_GREEN] = { 0, DECR_BOTH }, - [ODP_TM_SHAPER_YELLOW] = { 0, DECR_BOTH }, - [ODP_TM_SHAPER_RED] = { 0, DELAY_PKT } }, + [TM_SHAPER_GREEN] = { 0, DECR_BOTH }, + [TM_SHAPER_YELLOW] = { 0, DECR_BOTH }, + [TM_SHAPER_RED] = { 0, DELAY_PKT } }, [1] = { - [ODP_TM_SHAPER_GREEN] = { 1, DECR_BOTH }, - [ODP_TM_SHAPER_YELLOW] = { 1, DECR_BOTH }, - [ODP_TM_SHAPER_RED] = { 1, DELAY_PKT } }, + [TM_SHAPER_GREEN] = { 1, DECR_BOTH }, + [TM_SHAPER_YELLOW] = { 1, DECR_BOTH }, + [TM_SHAPER_RED] = { 1, DELAY_PKT } }, [2] = { - [ODP_TM_SHAPER_GREEN] = { 2, DECR_BOTH }, - [ODP_TM_SHAPER_YELLOW] = { 2, DECR_BOTH }, - [ODP_TM_SHAPER_RED] = { 2, DELAY_PKT } }, + [TM_SHAPER_GREEN] = { 2, DECR_BOTH }, + [TM_SHAPER_YELLOW] = { 2, DECR_BOTH }, + [TM_SHAPER_RED] = { 2, DELAY_PKT } }, [3] = { - [ODP_TM_SHAPER_GREEN] = { 3, DECR_BOTH }, - [ODP_TM_SHAPER_YELLOW] = { 3, DECR_BOTH }, - [ODP_TM_SHAPER_RED] = { 3, DELAY_PKT } }, + [TM_SHAPER_GREEN] = { 3, DECR_BOTH }, + [TM_SHAPER_YELLOW] = { 3, DECR_BOTH }, + [TM_SHAPER_RED] = { 3, DELAY_PKT } }, [4] = { - [ODP_TM_SHAPER_GREEN] = { 4, DECR_BOTH }, - [ODP_TM_SHAPER_YELLOW] = { 4, DECR_BOTH }, - [ODP_TM_SHAPER_RED] = { 4, DELAY_PKT } }, + [TM_SHAPER_GREEN] = { 4, DECR_BOTH }, + [TM_SHAPER_YELLOW] = { 4, DECR_BOTH }, + [TM_SHAPER_RED] = { 4, DELAY_PKT } }, [5] = { - [ODP_TM_SHAPER_GREEN] = { 5, DECR_BOTH }, - [ODP_TM_SHAPER_YELLOW] = { 5, DECR_BOTH }, - [ODP_TM_SHAPER_RED] = { 5, DELAY_PKT } }, + [TM_SHAPER_GREEN] = { 5, DECR_BOTH }, + [TM_SHAPER_YELLOW] = { 5, DECR_BOTH }, + [TM_SHAPER_RED] = { 5, DELAY_PKT } }, [6] = { - [ODP_TM_SHAPER_GREEN] = { 6, DECR_BOTH }, - [ODP_TM_SHAPER_YELLOW] = { 6, DECR_BOTH }, - [ODP_TM_SHAPER_RED] = { 6, DELAY_PKT } }, + [TM_SHAPER_GREEN] = { 6, DECR_BOTH }, + [TM_SHAPER_YELLOW] = { 6, DECR_BOTH }, + [TM_SHAPER_RED] = { 6, DELAY_PKT } }, [7] = { - [ODP_TM_SHAPER_GREEN] = { 7, DECR_BOTH }, - [ODP_TM_SHAPER_YELLOW] = { 7, DECR_BOTH }, - [ODP_TM_SHAPER_RED] = { 7, DELAY_PKT } } + [TM_SHAPER_GREEN] = { 7, DECR_BOTH }, + [TM_SHAPER_YELLOW] = { 7, DECR_BOTH }, + [TM_SHAPER_RED] = { 7, DELAY_PKT } } }; #define MAX_SHAPER_PROFILES 128 @@ -1076,30 +1091,30 @@ static odp_bool_t run_shaper(tm_system_t *tm_system, pkt_desc_t *pkt_desc, uint8_t priority) { - odp_tm_shaper_color_t shaper_color; + tm_shaper_color_t shaper_color; tm_shaper_params_t *shaper_params; odp_bool_t output_change; tm_prop_t propagation; shaper_params = shaper_obj->shaper_params; - shaper_color = ODP_TM_SHAPER_GREEN; + shaper_color = TM_SHAPER_GREEN; if (shaper_params) { update_shaper_elapsed_time(tm_system, shaper_params, shaper_obj); if (shaper_params->enabled) { if (0 < shaper_obj->commit_cnt) - shaper_color = ODP_TM_SHAPER_GREEN; + shaper_color = TM_SHAPER_GREEN; else if (!shaper_params->dual_rate) - shaper_color = ODP_TM_SHAPER_RED; + shaper_color = TM_SHAPER_RED; else if (shaper_obj->peak_cnt <= 0) - shaper_color = ODP_TM_SHAPER_RED; + shaper_color = TM_SHAPER_RED; else - shaper_color = ODP_TM_SHAPER_YELLOW; + shaper_color = TM_SHAPER_YELLOW; - if (shaper_color == ODP_TM_SHAPER_GREEN) + if (shaper_color == TM_SHAPER_GREEN) tm_system->shaper_green_cnt++; - else if (shaper_color == ODP_TM_SHAPER_YELLOW) + else if (shaper_color == TM_SHAPER_YELLOW) tm_system->shaper_yellow_cnt++; else tm_system->shaper_red_cnt++; @@ -1994,6 +2009,12 @@ static void tm_queue_cnts_decrement(tm_system_t *tm_system, odp_atomic_sub_u64(&queue_cnts->byte_cnt, frame_len); } +static inline void activate_packet_aging(odp_packet_hdr_t *pkt_hdr) +{ + if (odp_unlikely(pkt_hdr->p.flags.tx_aging)) + pkt_hdr->tx_aging_ns = pkt_hdr->tx_aging_ns + odp_time_global_ns(); +} + static int tm_enqueue(tm_system_t *tm_system, tm_queue_obj_t *tm_queue_obj, odp_packet_t pkt) @@ -2028,6 +2049,7 @@ static int tm_enqueue(tm_system_t *tm_system, if (tm_queue_obj->ordered_enqueue) _odp_sched_fn->order_lock(); + activate_packet_aging(packet_hdr(pkt)); rc = input_work_queue_append(tm_system, &work_item); if (tm_queue_obj->ordered_enqueue) @@ -2224,6 +2246,11 @@ static void tm_egress_marking(tm_system_t *tm_system, odp_packet_t odp_pkt) } } +static inline odp_bool_t is_packet_aged(odp_packet_hdr_t *pkt_hdr) +{ + return pkt_hdr->p.flags.tx_aging && pkt_hdr->tx_aging_ns < odp_time_global_ns(); +} + static void tm_send_pkt(tm_system_t *tm_system, uint32_t max_sends) { tm_queue_obj_t *tm_queue_obj; @@ -2250,9 +2277,13 @@ static void tm_send_pkt(tm_system_t *tm_system, uint32_t max_sends) tm_system->egress_pkt_desc = EMPTY_PKT_DESC; if (tm_system->egress.egress_kind == ODP_TM_EGRESS_PKT_IO) { - ret = odp_pktout_send(tm_system->pktout, &odp_pkt, 1); + pktio_entry = get_pktio_entry(tm_system->pktout.pktio); + if (odp_unlikely(_odp_pktio_tx_aging_enabled(pktio_entry) && + is_packet_aged(packet_hdr(odp_pkt)))) + ret = 0; /* Aged packet handled as a discard */ + else + ret = odp_pktout_send(tm_system->pktout, &odp_pkt, 1); if (odp_unlikely(ret != 1)) { - pktio_entry = get_pktio_entry(tm_system->pktout.pktio); if (odp_unlikely(_odp_pktio_tx_compl_enabled(pktio_entry))) _odp_pktio_allocate_and_send_tx_compl_events(pktio_entry, &odp_pkt, 1); @@ -2581,7 +2612,11 @@ static int tm_capabilities(odp_tm_capabilities_t capabilities[], cap_ptr->vlan_marking_supported = true; cap_ptr->ecn_marking_supported = true; cap_ptr->drop_prec_marking_supported = true; - cap_ptr->tm_queue_threshold = true; + + cap_ptr->tm_queue_threshold.byte = true; + cap_ptr->tm_queue_threshold.packet = true; + cap_ptr->tm_queue_threshold.byte_and_packet = true; + cap_ptr->tm_queue_query_flags = (ODP_TM_QUERY_PKT_CNT | ODP_TM_QUERY_BYTE_CNT | ODP_TM_QUERY_THRESHOLDS); @@ -2617,7 +2652,10 @@ static int tm_capabilities(odp_tm_capabilities_t capabilities[], per_level_cap->tm_node_dual_slope_supported = true; per_level_cap->fair_queuing_supported = true; per_level_cap->weights_supported = true; - per_level_cap->tm_node_threshold = true; + + per_level_cap->tm_node_threshold.byte = true; + per_level_cap->tm_node_threshold.packet = true; + per_level_cap->tm_node_threshold.byte_and_packet = true; } cap_ptr->queue_stats.counter.discards = 1; @@ -2691,7 +2729,11 @@ static void tm_system_capabilities_set(odp_tm_capabilities_t *cap_ptr, cap_ptr->ecn_marking_supported = req_ptr->ecn_marking_needed; cap_ptr->drop_prec_marking_supported = req_ptr->drop_prec_marking_needed; - cap_ptr->tm_queue_threshold = threshold; + + cap_ptr->tm_queue_threshold.byte = threshold; + cap_ptr->tm_queue_threshold.packet = threshold; + cap_ptr->tm_queue_threshold.byte_and_packet = threshold; + cap_ptr->tm_queue_query_flags = (ODP_TM_QUERY_PKT_CNT | ODP_TM_QUERY_BYTE_CNT | ODP_TM_QUERY_THRESHOLDS); @@ -2743,7 +2785,10 @@ static void tm_system_capabilities_set(odp_tm_capabilities_t *cap_ptr, per_level_cap->tm_node_dual_slope_supported = dual_slope; per_level_cap->fair_queuing_supported = true; per_level_cap->weights_supported = true; - per_level_cap->tm_node_threshold = threshold; + + per_level_cap->tm_node_threshold.byte = threshold; + per_level_cap->tm_node_threshold.packet = threshold; + per_level_cap->tm_node_threshold.byte_and_packet = threshold; } cap_ptr->queue_stats.counter.discards = 1; diff --git a/platform/linux-generic/pktio/dpdk.c b/platform/linux-generic/pktio/dpdk.c index 815ec61b9..030560b0d 100644 --- a/platform/linux-generic/pktio/dpdk.c +++ b/platform/linux-generic/pktio/dpdk.c @@ -57,12 +57,14 @@ #include <rte_string_fns.h> #include <rte_version.h> -#if RTE_VERSION < RTE_VERSION_NUM(19, 8, 0, 0) - #define rte_ether_addr ether_addr - #define rte_ipv4_hdr ipv4_hdr - #define rte_ipv6_hdr ipv6_hdr - #define rte_tcp_hdr tcp_hdr - #define rte_udp_hdr udp_hdr +#if RTE_VERSION < RTE_VERSION_NUM(21, 11, 0, 0) + #define RTE_MBUF_F_RX_RSS_HASH PKT_RX_RSS_HASH + #define RTE_MBUF_F_TX_IPV4 PKT_TX_IPV4 + #define RTE_MBUF_F_TX_IPV6 PKT_TX_IPV6 + #define RTE_MBUF_F_TX_IP_CKSUM PKT_TX_IP_CKSUM + #define RTE_MBUF_F_TX_UDP_CKSUM PKT_TX_UDP_CKSUM + #define RTE_MBUF_F_TX_TCP_CKSUM PKT_TX_TCP_CKSUM + #define RTE_MEMPOOL_REGISTER_OPS MEMPOOL_REGISTER_OPS #endif /* NUMA is not supported on all platforms */ @@ -72,17 +74,7 @@ #define numa_num_configured_nodes() 1 #endif -#if RTE_VERSION < RTE_VERSION_NUM(17, 5, 0, 0) -#define rte_log_set_global_level rte_set_log_level -#endif - -/* Release notes v19.11: "Changed the mempool allocation behaviour - * so that objects no longer cross pages by default" */ -#if RTE_VERSION >= RTE_VERSION_NUM(19, 11, 0, 0) -#define MEMPOOL_FLAGS MEMPOOL_F_NO_IOVA_CONTIG -#else #define MEMPOOL_FLAGS 0 -#endif #if _ODP_DPDK_ZERO_COPY ODP_STATIC_ASSERT(CONFIG_PACKET_HEADROOM == RTE_PKTMBUF_HEADROOM, @@ -107,11 +99,7 @@ ODP_STATIC_ASSERT((DPDK_NB_MBUF % DPDK_MEMPOOL_CACHE_SIZE == 0) && #define DPDK_MIN_RX_BURST 4 /* Limits for setting link MTU */ -#if RTE_VERSION >= RTE_VERSION_NUM(19, 11, 0, 0) #define DPDK_MTU_MIN (RTE_ETHER_MIN_MTU + _ODP_ETHHDR_LEN) -#else -#define DPDK_MTU_MIN (68 + _ODP_ETHHDR_LEN) -#endif #define DPDK_MTU_MAX (9000 + _ODP_ETHHDR_LEN) /** DPDK runtime configuration options */ @@ -334,12 +322,16 @@ static void pktmbuf_init(struct rte_mempool *mp, void *opaque_arg ODP_UNUSED, static struct rte_mempool *mbuf_pool_create(const char *name, pool_t *pool_entry) { + odp_shm_info_t shm_info; struct rte_mempool *mp = NULL; struct rte_pktmbuf_pool_private mbp_priv; struct rte_mempool_objsz sz; unsigned int elt_size = pool_entry->dpdk_elt_size; - unsigned int num = pool_entry->num; + unsigned int num = pool_entry->num, populated = 0; uint32_t total_size; + uint64_t page_size, offset = 0, remainder = 0; + uint8_t *addr; + int ret; if (!(pool_entry->mem_from_huge_pages)) { ODP_ERR("DPDK requires memory is allocated from huge pages\n"); @@ -348,11 +340,17 @@ static struct rte_mempool *mbuf_pool_create(const char *name, if (pool_entry->seg_len < RTE_MBUF_DEFAULT_BUF_SIZE) { ODP_ERR("Some NICs need at least %dB buffers to not segment " - "standard ethernet frames. Increase pool seg_len.\n", - RTE_MBUF_DEFAULT_BUF_SIZE); + "standard ethernet frames. Increase pool seg_len.\n", + RTE_MBUF_DEFAULT_BUF_SIZE); goto fail; } + if (odp_shm_info(pool_entry->shm, &shm_info)) { + ODP_ERR("Failed to query SHM info.\n"); + goto fail; + } + + page_size = shm_info.page_size; total_size = rte_mempool_calc_obj_size(elt_size, MEMPOOL_FLAGS, &sz); if (total_size != pool_entry->block_size) { ODP_ERR("DPDK pool block size not matching to ODP pool: " @@ -361,10 +359,7 @@ static struct rte_mempool *mbuf_pool_create(const char *name, goto fail; } - /* Skipped buffers have to be taken into account to populate pool - * properly. */ - mp = rte_mempool_create_empty(name, num + pool_entry->skipped_blocks, - elt_size, cache_size(num), + mp = rte_mempool_create_empty(name, num, elt_size, cache_size(num), sizeof(struct rte_pktmbuf_pool_private), rte_socket_id(), MEMPOOL_FLAGS); if (mp == NULL) { @@ -380,16 +375,34 @@ static struct rte_mempool *mbuf_pool_create(const char *name, } mbp_priv.mbuf_data_room_size = pool_entry->headroom + - pool_entry->seg_len; + pool_entry->seg_len + pool_entry->tailroom; mbp_priv.mbuf_priv_size = RTE_ALIGN(sizeof(odp_packet_hdr_t), RTE_MBUF_PRIV_ALIGN); rte_pktmbuf_pool_init(mp, &mbp_priv); - num = rte_mempool_populate_iova(mp, (char *)pool_entry->base_addr, - RTE_BAD_IOVA, pool_entry->shm_size, - NULL, NULL); - if (num <= 0) { - ODP_ERR("Failed to populate mempool: %d\n", num); + /* DPDK expects buffers that would be crossing a hugepage boundary to be aligned to the + * boundary. This isn't the case with ODP pools as boundary-crossing buffers are skipped + * and unused but still part of the pool. Thus, populate the mempool with several virtually + * and physically contiguous chunks as dictated by the skipped buffers. */ + for (uint64_t i = 0; i < pool_entry->shm_size; i += page_size) { + remainder = (page_size - offset) % total_size; + addr = pool_entry->base_addr + i + offset; + ret = rte_mempool_populate_iova(mp, (char *)addr, rte_mem_virt2iova(addr), + page_size - remainder - offset, + NULL, NULL); + + if (ret <= 0) { + ODP_ERR("Failed to populate mempool: %d\n", ret); + goto fail; + } + + populated += ret; + offset = remainder ? total_size - remainder : 0; + } + + if (populated != num) { + ODP_ERR("Failed to populate mempool with all requested blocks, populated: %u, " + "requested: %u\n", populated, num); goto fail; } @@ -556,7 +569,7 @@ static struct rte_mempool_ops odp_pool_ops = { .get_count = pool_get_count }; -MEMPOOL_REGISTER_OPS(odp_pool_ops) +RTE_MEMPOOL_REGISTER_OPS(odp_pool_ops) static inline int mbuf_to_pkt(pktio_entry_t *pktio_entry, odp_packet_t pkt_table[], @@ -574,9 +587,9 @@ static inline int mbuf_to_pkt(pktio_entry_t *pktio_entry, pkt_dpdk_t *pkt_dpdk = pkt_priv(pktio_entry); odp_pool_t pool = pkt_dpdk->pool; odp_pktin_config_opt_t pktin_cfg = pktio_entry->s.config.pktin; - odp_proto_layer_t parse_layer = pktio_entry->s.config.parser.layer; odp_pktio_t input = pktio_entry->s.handle; uint16_t frame_offset = pktio_entry->s.pktin_frame_offset; + const odp_proto_layer_t layer = pktio_entry->s.parse_layer; /* Allocate maximum sized packets */ max_len = pkt_dpdk->data_room; @@ -591,8 +604,6 @@ static inline int mbuf_to_pkt(pktio_entry_t *pktio_entry, } for (i = 0; i < num; i++) { - odp_packet_hdr_t parsed_hdr; - mbuf = mbuf_table[i]; if (odp_unlikely(mbuf->nb_segs != 1)) { ODP_ERR("Segmented buffers not supported\n"); @@ -603,31 +614,33 @@ static inline int mbuf_to_pkt(pktio_entry_t *pktio_entry, odp_prefetch(data); pkt_len = rte_pktmbuf_pkt_len(mbuf); + pkt = pkt_table[i]; + pkt_hdr = packet_hdr(pkt); - if (pktio_cls_enabled(pktio_entry)) { + if (layer) { uint32_t supported_ptypes = pkt_dpdk->supported_ptypes; - packet_parse_reset(&parsed_hdr, 1); - packet_set_len(&parsed_hdr, pkt_len); - if (_odp_dpdk_packet_parse_common(&parsed_hdr.p, data, - pkt_len, pkt_len, - mbuf, - ODP_PROTO_LAYER_ALL, - supported_ptypes, + packet_parse_reset(pkt_hdr, 1); + if (_odp_dpdk_packet_parse_common(&pkt_hdr->p, data, + pkt_len, pkt_len, mbuf, + layer, supported_ptypes, pktin_cfg)) { odp_packet_free(pkt_table[i]); rte_pktmbuf_free(mbuf); continue; } - if (_odp_cls_classify_packet(pktio_entry, - (const uint8_t *)data, - pkt_len, pkt_len, &pool, - &parsed_hdr, false)) - goto fail; + + if (pktio_cls_enabled(pktio_entry)) { + if (_odp_cls_classify_packet(pktio_entry, + (const uint8_t *)data, + &pool, pkt_hdr)) { + odp_packet_free(pkt_table[i]); + rte_pktmbuf_free(mbuf); + continue; + } + } } - pkt = pkt_table[i]; - pkt_hdr = packet_hdr(pkt); pull_tail(pkt_hdr, max_len - pkt_len); if (frame_offset) pull_head(pkt_hdr, frame_offset); @@ -637,22 +650,7 @@ static inline int mbuf_to_pkt(pktio_entry_t *pktio_entry, pkt_hdr->input = input; - if (pktio_cls_enabled(pktio_entry)) { - _odp_packet_copy_cls_md(pkt_hdr, &parsed_hdr); - } else if (parse_layer != ODP_PROTO_LAYER_NONE) { - uint32_t supported_ptypes = pkt_dpdk->supported_ptypes; - - if (_odp_dpdk_packet_parse_layer(pkt_hdr, mbuf, - parse_layer, - supported_ptypes, - pktin_cfg)) { - odp_packet_free(pkt); - rte_pktmbuf_free(mbuf); - continue; - } - } - - if (mbuf->ol_flags & PKT_RX_RSS_HASH) + if (mbuf->ol_flags & RTE_MBUF_F_RX_RSS_HASH) packet_set_flow_hash(pkt_hdr, mbuf->hash.rss); packet_set_ts(pkt_hdr, ts); @@ -753,12 +751,12 @@ static inline void pkt_set_ol_tx(odp_pktout_config_opt_t *pktout_cfg, mbuf->l2_len = pkt_p->l3_offset - pkt_p->l2_offset; if (l3_proto_v4) - mbuf->ol_flags = PKT_TX_IPV4; + mbuf->ol_flags = RTE_MBUF_F_TX_IPV4; else - mbuf->ol_flags = PKT_TX_IPV6; + mbuf->ol_flags = RTE_MBUF_F_TX_IPV6; if (ipv4_chksum_pkt) { - mbuf->ol_flags |= PKT_TX_IP_CKSUM; + mbuf->ol_flags |= RTE_MBUF_F_TX_IP_CKSUM; ((struct rte_ipv4_hdr *)l3_hdr)->hdr_checksum = 0; mbuf->l3_len = _ODP_IPV4HDR_IHL(*(uint8_t *)l3_hdr) * 4; @@ -772,12 +770,12 @@ static inline void pkt_set_ol_tx(odp_pktout_config_opt_t *pktout_cfg, l4_hdr = (void *)(mbuf_data + pkt_p->l4_offset); if (udp_chksum_pkt) { - mbuf->ol_flags |= PKT_TX_UDP_CKSUM; + mbuf->ol_flags |= RTE_MBUF_F_TX_UDP_CKSUM; ((struct rte_udp_hdr *)l4_hdr)->dgram_cksum = phdr_csum(l3_proto_v4, l3_hdr, mbuf->ol_flags); } else if (tcp_chksum_pkt) { - mbuf->ol_flags |= PKT_TX_TCP_CKSUM; + mbuf->ol_flags |= RTE_MBUF_F_TX_TCP_CKSUM; ((struct rte_tcp_hdr *)l4_hdr)->cksum = phdr_csum(l3_proto_v4, l3_hdr, mbuf->ol_flags); @@ -863,9 +861,9 @@ static inline int mbuf_to_pkt_zero(pktio_entry_t *pktio_entry, int i, nb_pkts; odp_pool_t pool; odp_pktin_config_opt_t pktin_cfg; - odp_proto_layer_t parse_layer; odp_pktio_t input; pkt_dpdk_t *pkt_dpdk; + const odp_proto_layer_t layer = pktio_entry->s.parse_layer; prefetch_pkt(mbuf_table[0]); @@ -874,15 +872,12 @@ static inline int mbuf_to_pkt_zero(pktio_entry_t *pktio_entry, pool = pkt_dpdk->pool; set_flow_hash = pkt_dpdk->opt.set_flow_hash; pktin_cfg = pktio_entry->s.config.pktin; - parse_layer = pktio_entry->s.config.parser.layer; input = pktio_entry->s.handle; if (odp_likely(mbuf_num > 1)) prefetch_pkt(mbuf_table[1]); for (i = 0; i < mbuf_num; i++) { - odp_packet_hdr_t parsed_hdr; - if (odp_likely((i + 2) < mbuf_num)) prefetch_pkt(mbuf_table[i + 2]); @@ -895,30 +890,27 @@ static inline int mbuf_to_pkt_zero(pktio_entry_t *pktio_entry, data = rte_pktmbuf_mtod(mbuf, char *); pkt_len = rte_pktmbuf_pkt_len(mbuf); - pkt_hdr = pkt_hdr_from_mbuf(mbuf); + packet_init(pkt_hdr, pkt_len); - if (pktio_cls_enabled(pktio_entry)) { + if (layer) { uint32_t supported_ptypes = pkt_dpdk->supported_ptypes; - packet_parse_reset(&parsed_hdr, 1); - packet_set_len(&parsed_hdr, pkt_len); - if (_odp_dpdk_packet_parse_common(&parsed_hdr.p, data, - pkt_len, pkt_len, - mbuf, - ODP_PROTO_LAYER_ALL, - supported_ptypes, + if (_odp_dpdk_packet_parse_common(&pkt_hdr->p, data, + pkt_len, pkt_len, mbuf, + layer, supported_ptypes, pktin_cfg)) { rte_pktmbuf_free(mbuf); continue; } - if (_odp_cls_classify_packet(pktio_entry, - (const uint8_t *)data, - pkt_len, pkt_len, &pool, - &parsed_hdr, false)) { - ODP_ERR("Unable to classify packet\n"); - rte_pktmbuf_free(mbuf); - continue; + + if (pktio_cls_enabled(pktio_entry)) { + if (_odp_cls_classify_packet(pktio_entry, + (const uint8_t *)data, + &pool, pkt_hdr)) { + rte_pktmbuf_free(mbuf); + continue; + } } } @@ -926,23 +918,9 @@ static inline int mbuf_to_pkt_zero(pktio_entry_t *pktio_entry, * are supported. */ pkt_hdr->seg_data = data; - packet_init(pkt_hdr, pkt_len); pkt_hdr->input = input; - if (pktio_cls_enabled(pktio_entry)) { - _odp_packet_copy_cls_md(pkt_hdr, &parsed_hdr); - } else if (parse_layer != ODP_PROTO_LAYER_NONE) { - uint32_t supported_ptypes = pkt_dpdk->supported_ptypes; - - if (_odp_dpdk_packet_parse_layer(pkt_hdr, mbuf, - parse_layer, - supported_ptypes, - pktin_cfg)) { - rte_pktmbuf_free(mbuf); - continue; - } - } - if (set_flow_hash && (mbuf->ol_flags & PKT_RX_RSS_HASH)) + if (set_flow_hash && (mbuf->ol_flags & RTE_MBUF_F_RX_RSS_HASH)) packet_set_flow_hash(pkt_hdr, mbuf->hash.rss); packet_set_ts(pkt_hdr, ts); @@ -1227,11 +1205,6 @@ static int dpdk_close(pktio_entry_t *pktio_entry) rte_pktmbuf_free(pkt_dpdk->rx_cache[i].s.pkt[idx++]); } -#if RTE_VERSION < RTE_VERSION_NUM(17, 8, 0, 0) - if (pktio_entry->s.state != PKTIO_STATE_OPENED) - rte_eth_dev_close(pkt_dpdk->port_id); -#endif - return 0; } @@ -1381,18 +1354,6 @@ static void dpdk_mempool_free(struct rte_mempool *mp, void *arg ODP_UNUSED) rte_mempool_free(mp); } -/* RTE_ETH_FOREACH_DEV was introduced in v17.8, but causes a build error in - * v18.2 (only a warning, but our build system treats warnings as errors). */ -#if (RTE_VERSION >= RTE_VERSION_NUM(18, 2, 0, 0)) && \ - (RTE_VERSION < RTE_VERSION_NUM(18, 5, 0, 0)) - #define ETH_FOREACH_DEV(p) \ - for (p = rte_eth_find_next(0); \ - (unsigned int)p < (unsigned int)RTE_MAX_ETHPORTS; \ - p = rte_eth_find_next(p + 1)) -#elif RTE_VERSION >= RTE_VERSION_NUM(17, 8, 0, 0) - #define ETH_FOREACH_DEV(p) RTE_ETH_FOREACH_DEV(p) -#endif - static int dpdk_pktio_term(void) { uint16_t port_id; @@ -1400,11 +1361,9 @@ static int dpdk_pktio_term(void) if (!odp_global_rw->dpdk_initialized) return 0; -#if RTE_VERSION >= RTE_VERSION_NUM(17, 8, 0, 0) - ETH_FOREACH_DEV(port_id) { + RTE_ETH_FOREACH_DEV(port_id) { rte_eth_dev_close(port_id); } -#endif if (!_ODP_DPDK_ZERO_COPY) rte_mempool_walk(dpdk_mempool_free, NULL); @@ -1574,16 +1533,18 @@ static int dpdk_init_capability(pktio_entry_t *pktio_entry, /* Check if setting MTU is supported */ ret = rte_eth_dev_set_mtu(pkt_dpdk->port_id, pkt_dpdk->mtu - _ODP_ETHHDR_LEN); - if (ret == 0) { + /* From DPDK 21.11 onwards, calling rte_eth_dev_set_mtu() before device is configured with + * rte_eth_dev_configure() will result in failure. The least hacky (unfortunately still + * very hacky) way to continue checking the support is to take into account that the + * function will fail earlier with -ENOTSUP if MTU setting is not supported by device than + * if the device was not yet configured. */ + if (ret != -ENOTSUP) { capa->set_op.op.maxlen = 1; capa->maxlen.equal = true; capa->maxlen.min_input = DPDK_MTU_MIN; capa->maxlen.max_input = pkt_dpdk->mtu_max; capa->maxlen.min_output = DPDK_MTU_MIN; capa->maxlen.max_output = pkt_dpdk->mtu_max; - } else if (ret != -ENOTSUP) { - ODP_ERR("Failed to set interface MTU: %d\n", ret); - return -1; } ptype_cnt = rte_eth_dev_get_supported_ptypes(pkt_dpdk->port_id, @@ -1669,15 +1630,6 @@ static int dpdk_init_capability(pktio_entry_t *pktio_entry, * mode change. Use system call for them. */ static void promisc_mode_check(pkt_dpdk_t *pkt_dpdk) { -#if RTE_VERSION < RTE_VERSION_NUM(19, 11, 0, 0) - /* Enable and disable calls do not have return value */ - rte_eth_promiscuous_enable(pkt_dpdk->port_id); - - if (!rte_eth_promiscuous_get(pkt_dpdk->port_id)) - pkt_dpdk->vdev_sysc_promisc = 1; - - rte_eth_promiscuous_disable(pkt_dpdk->port_id); -#else int ret; ret = rte_eth_promiscuous_enable(pkt_dpdk->port_id); @@ -1689,7 +1641,6 @@ static void promisc_mode_check(pkt_dpdk_t *pkt_dpdk) if (ret) pkt_dpdk->vdev_sysc_promisc = 1; -#endif } static int dpdk_open(odp_pktio_t id ODP_UNUSED, @@ -1735,18 +1686,17 @@ static int dpdk_open(odp_pktio_t id ODP_UNUSED, pkt_dpdk->pool = pool; - /* rte_eth_dev_count() was removed in v18.05 */ -#if RTE_VERSION < RTE_VERSION_NUM(18, 5, 0, 0) - if (rte_eth_dev_count() == 0) { -#else if (rte_eth_dev_count_avail() == 0) { -#endif ODP_ERR("No DPDK ports found\n"); return -1; } memset(&dev_info, 0, sizeof(struct rte_eth_dev_info)); - rte_eth_dev_info_get(pkt_dpdk->port_id, &dev_info); + ret = rte_eth_dev_info_get(pkt_dpdk->port_id, &dev_info); + if (ret) { + ODP_ERR("Failed to read device info: %d\n", ret); + return -1; + } /* Initialize runtime options */ if (init_options(pktio_entry, &dev_info)) { @@ -1765,18 +1715,10 @@ static int dpdk_open(odp_pktio_t id ODP_UNUSED, promisc_mode_check(pkt_dpdk); -#if RTE_VERSION < RTE_VERSION_NUM(19, 11, 0, 0) - ret = 0; - if (pkt_dpdk->opt.multicast_en) - rte_eth_allmulticast_enable(pkt_dpdk->port_id); - else - rte_eth_allmulticast_disable(pkt_dpdk->port_id); -#else if (pkt_dpdk->opt.multicast_en) ret = rte_eth_allmulticast_enable(pkt_dpdk->port_id); else ret = rte_eth_allmulticast_disable(pkt_dpdk->port_id); -#endif /* Not supported by all PMDs, so ignore the return value */ if (ret) diff --git a/platform/linux-generic/pktio/dpdk_parse.c b/platform/linux-generic/pktio/dpdk_parse.c index 0984f06c5..f28f18921 100644 --- a/platform/linux-generic/pktio/dpdk_parse.c +++ b/platform/linux-generic/pktio/dpdk_parse.c @@ -1,5 +1,5 @@ /* Copyright (c) 2018, Linaro Limited - * Copyright (c) 2021, Nokia + * Copyright (c) 2021-2022, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -26,9 +26,19 @@ #if defined(__PPC64__) && defined(vector) #undef vector #endif +#include <rte_version.h> + +#if RTE_VERSION < RTE_VERSION_NUM(21, 11, 0, 0) + #define RTE_MBUF_F_RX_IP_CKSUM_MASK PKT_RX_IP_CKSUM_MASK + #define RTE_MBUF_F_RX_IP_CKSUM_GOOD PKT_RX_IP_CKSUM_GOOD + #define RTE_MBUF_F_RX_IP_CKSUM_UNKNOWN PKT_RX_IP_CKSUM_UNKNOWN + #define RTE_MBUF_F_RX_L4_CKSUM_MASK PKT_RX_L4_CKSUM_MASK + #define RTE_MBUF_F_RX_L4_CKSUM_GOOD PKT_RX_L4_CKSUM_GOOD + #define RTE_MBUF_F_RX_L4_CKSUM_UNKNOWN PKT_RX_L4_CKSUM_UNKNOWN +#endif -#define IP4_CSUM_RESULT(ol_flags) (ol_flags & PKT_RX_IP_CKSUM_MASK) -#define L4_CSUM_RESULT(ol_flags) (ol_flags & PKT_RX_L4_CKSUM_MASK) +#define IP4_CSUM_RESULT(ol_flags) ((ol_flags) & RTE_MBUF_F_RX_IP_CKSUM_MASK) +#define L4_CSUM_RESULT(ol_flags) ((ol_flags) & RTE_MBUF_F_RX_L4_CKSUM_MASK) /** Parser helper function for Ethernet packets */ static inline uint16_t dpdk_parse_eth(packet_parser_t *prs, @@ -174,9 +184,9 @@ static inline uint8_t dpdk_parse_ipv4(packet_parser_t *prs, if (do_csum) { uint64_t packet_csum_result = IP4_CSUM_RESULT(mbuf_ol); - if (packet_csum_result == PKT_RX_IP_CKSUM_GOOD) { + if (packet_csum_result == RTE_MBUF_F_RX_IP_CKSUM_GOOD) { prs->input_flags.l3_chksum_done = 1; - } else if (packet_csum_result != PKT_RX_IP_CKSUM_UNKNOWN) { + } else if (packet_csum_result != RTE_MBUF_F_RX_IP_CKSUM_UNKNOWN) { prs->input_flags.l3_chksum_done = 1; prs->flags.ip_err = 1; prs->flags.l3_chksum_err = 1; @@ -302,9 +312,9 @@ static inline void dpdk_parse_tcp(packet_parser_t *prs, if (do_csum) { uint64_t packet_csum_result = L4_CSUM_RESULT(mbuf_ol); - if (packet_csum_result == PKT_RX_L4_CKSUM_GOOD) { + if (packet_csum_result == RTE_MBUF_F_RX_L4_CKSUM_GOOD) { prs->input_flags.l4_chksum_done = 1; - } else if (packet_csum_result != PKT_RX_L4_CKSUM_UNKNOWN) { + } else if (packet_csum_result != RTE_MBUF_F_RX_L4_CKSUM_UNKNOWN) { prs->input_flags.l4_chksum_done = 1; prs->flags.tcp_err = 1; prs->flags.l4_chksum_err = 1; @@ -332,9 +342,9 @@ static inline void dpdk_parse_udp(packet_parser_t *prs, if (do_csum) { uint64_t packet_csum_result = L4_CSUM_RESULT(mbuf_ol); - if (packet_csum_result == PKT_RX_L4_CKSUM_GOOD) { + if (packet_csum_result == RTE_MBUF_F_RX_L4_CKSUM_GOOD) { prs->input_flags.l4_chksum_done = 1; - } else if (packet_csum_result != PKT_RX_L4_CKSUM_UNKNOWN) { + } else if (packet_csum_result != RTE_MBUF_F_RX_L4_CKSUM_UNKNOWN) { if (prs->input_flags.ipv4 && !udp->chksum) { prs->input_flags.l4_chksum_done = 1; } else { diff --git a/platform/linux-generic/pktio/loop.c b/platform/linux-generic/pktio/loop.c index b3232cded..3e21efecd 100644 --- a/platform/linux-generic/pktio/loop.c +++ b/platform/linux-generic/pktio/loop.c @@ -18,6 +18,7 @@ #include <odp/api/plat/packet_flag_inlines.h> #include <odp/api/plat/queue_inlines.h> +#include <odp_parse_internal.h> #include <odp_classification_internal.h> #include <odp_debug_internal.h> #include <odp_errno_define.h> @@ -26,6 +27,7 @@ #include <odp_ipsec_internal.h> #include <odp_packet_internal.h> #include <odp_packet_io_internal.h> +#include <odp_macros_internal.h> #include <odp_queue_if.h> #include <protocols/eth.h> @@ -158,7 +160,11 @@ static int loopback_recv(pktio_entry_t *pktio_entry, int index ODP_UNUSED, odp_time_t ts_val; odp_time_t *ts = NULL; int num_rx = 0; - int failed = 0; + int packets = 0, errors = 0; + uint32_t octets = 0; + const odp_proto_chksums_t chksums = pktio_entry->s.in_chksums; + const odp_proto_layer_t layer = pktio_entry->s.parse_layer; + const odp_pktin_config_opt_t opt = pktio_entry->s.config.pktin; if (odp_unlikely(num > QUEUE_MULTI_MAX)) num = QUEUE_MULTI_MAX; @@ -168,8 +174,7 @@ static int loopback_recv(pktio_entry_t *pktio_entry, int index ODP_UNUSED, queue = pkt_priv(pktio_entry)->loopq; nbr = odp_queue_deq_multi(queue, (odp_event_t *)hdr_tbl, num); - if (pktio_entry->s.config.pktin.bit.ts_all || - pktio_entry->s.config.pktin.bit.ts_ptp) { + if (opt.bit.ts_all || opt.bit.ts_ptp) { ts_val = odp_time_global(); ts = &ts_val; } @@ -181,61 +186,64 @@ static int loopback_recv(pktio_entry_t *pktio_entry, int index ODP_UNUSED, pkt_len = odp_packet_len(pkt); pkt_hdr = packet_hdr(pkt); - packet_parse_reset(pkt_hdr, 1); - if (pktio_cls_enabled(pktio_entry)) { - odp_packet_t new_pkt; - odp_pool_t new_pool; + if (layer) { uint8_t *pkt_addr; - uint8_t buf[PACKET_PARSE_SEG_LEN]; + uint8_t buf[PARSE_BYTES]; int ret; uint32_t seg_len = odp_packet_seg_len(pkt); + uint64_t l4_part_sum = 0; /* Make sure there is enough data for the packet * parser in the case of a segmented packet. */ - if (odp_unlikely(seg_len < PACKET_PARSE_SEG_LEN && - pkt_len > PACKET_PARSE_SEG_LEN)) { - odp_packet_copy_to_mem(pkt, 0, - PACKET_PARSE_SEG_LEN, - buf); - seg_len = PACKET_PARSE_SEG_LEN; + if (odp_unlikely(seg_len < PARSE_BYTES && + pkt_len > seg_len)) { + seg_len = MIN(pkt_len, PARSE_BYTES); + odp_packet_copy_to_mem(pkt, 0, seg_len, buf); pkt_addr = buf; } else { pkt_addr = odp_packet_data(pkt); } - ret = _odp_cls_classify_packet(pktio_entry, pkt_addr, - pkt_len, seg_len, - &new_pool, pkt_hdr, true); - if (ret) { - failed++; + packet_parse_reset(pkt_hdr, 1); + ret = _odp_packet_parse_common(&pkt_hdr->p, pkt_addr, pkt_len, + seg_len, layer, chksums, + &l4_part_sum, opt); + if (ret) + errors++; + + if (ret < 0) { odp_packet_free(pkt); continue; } - if (new_pool != odp_packet_pool(pkt)) { - new_pkt = odp_packet_copy(pkt, new_pool); - - odp_packet_free(pkt); + if (pktio_cls_enabled(pktio_entry)) { + odp_packet_t new_pkt; + odp_pool_t new_pool; - if (new_pkt == ODP_PACKET_INVALID) { - failed++; + ret = _odp_cls_classify_packet(pktio_entry, pkt_addr, + &new_pool, pkt_hdr); + if (ret) { + odp_packet_free(pkt); continue; } - pkt = new_pkt; - pkt_hdr = packet_hdr(new_pkt); + if (new_pool != odp_packet_pool(pkt)) { + new_pkt = odp_packet_copy(pkt, new_pool); + + odp_packet_free(pkt); + + if (new_pkt == ODP_PACKET_INVALID) { + pktio_entry->s.stats.in_discards++; + continue; + } + + pkt = new_pkt; + pkt_hdr = packet_hdr(new_pkt); + } } - } else { - odp_packet_parse_param_t param; - - /* - * Use odp_packet_parse() which can handle segmented - * packets. - */ - param.proto = ODP_PROTO_ETH; - param.last_layer = pktio_entry->s.config.parser.layer; - param.chksums = pktio_entry->s.in_chksums; - odp_packet_parse(packet_handle(pkt_hdr), 0, ¶m); + + if (layer >= ODP_PROTO_LAYER_L4) + _odp_packet_l4_chksum(pkt_hdr, chksums, l4_part_sum); } packet_set_ts(pkt_hdr, ts); @@ -247,12 +255,17 @@ static int loopback_recv(pktio_entry_t *pktio_entry, int index ODP_UNUSED, odp_packet_has_ipsec(pkt)) _odp_ipsec_try_inline(&pkt); - pktio_entry->s.stats.in_octets += pkt_len; + if (!pkt_hdr->p.flags.all.error) { + octets += pkt_len; + packets++; + } + pkts[num_rx++] = pkt; } - pktio_entry->s.stats.in_errors += failed; - pktio_entry->s.stats.in_packets += num_rx - failed; + pktio_entry->s.stats.in_octets += octets; + pktio_entry->s.stats.in_packets += packets; + pktio_entry->s.stats.in_errors += errors; odp_ticketlock_unlock(&pktio_entry->s.rxl); @@ -533,6 +546,7 @@ static int loopback_init_capability(pktio_entry_t *pktio_entry) capa->stats.pktio.counter.in_octets = 1; capa->stats.pktio.counter.in_packets = 1; capa->stats.pktio.counter.in_errors = 1; + capa->stats.pktio.counter.in_discards = 1; capa->stats.pktio.counter.out_octets = 1; capa->stats.pktio.counter.out_packets = 1; capa->stats.pktin_queue.counter.octets = 1; diff --git a/platform/linux-generic/pktio/netmap.c b/platform/linux-generic/pktio/netmap.c index 0b1345e50..94b88e21e 100644 --- a/platform/linux-generic/pktio/netmap.c +++ b/platform/linux-generic/pktio/netmap.c @@ -16,6 +16,7 @@ #include <odp/api/time.h> #include <odp/api/plat/time_inlines.h> +#include <odp_parse_internal.h> #include <odp_packet_io_internal.h> #include <odp_packet_io_stats.h> #include <odp_ethtool_stats.h> @@ -822,11 +823,14 @@ static inline int netmap_pkt_to_odp(pktio_entry_t *pktio_entry, odp_packet_t pkt; odp_pool_t pool = pkt_priv(pktio_entry)->pool; odp_packet_hdr_t *pkt_hdr; - odp_packet_hdr_t parsed_hdr; int i; int num; uint32_t max_len; uint16_t frame_offset = pktio_entry->s.pktin_frame_offset; + int num_rx = 0; + const odp_proto_chksums_t chksums = pktio_entry->s.in_chksums; + const odp_proto_layer_t layer = pktio_entry->s.parse_layer; + const odp_pktin_config_opt_t opt = pktio_entry->s.config.pktin; /* Allocate maximum sized packets */ max_len = pkt_priv(pktio_entry)->mtu; @@ -837,45 +841,50 @@ static inline int netmap_pkt_to_odp(pktio_entry_t *pktio_entry, for (i = 0; i < num; i++) { netmap_slot_t slot; uint16_t len; + const uint8_t *buf; + uint64_t l4_part_sum = 0; slot = slot_tbl[i]; len = slot.len; + buf = (const uint8_t *)slot.buf; odp_prefetch(slot.buf); - if (pktio_cls_enabled(pktio_entry)) { - if (_odp_cls_classify_packet(pktio_entry, - (const uint8_t *)slot.buf, len, - len, &pool, &parsed_hdr, true)) - goto fail; + pkt = pkt_tbl[num_rx]; + pkt_hdr = packet_hdr(pkt); + + if (layer) { + if (_odp_packet_parse_common(&pkt_hdr->p, buf, len, len, + layer, chksums, &l4_part_sum, opt) < 0) + continue; + + if (pktio_cls_enabled(pktio_entry)) { + if (_odp_cls_classify_packet(pktio_entry, buf, &pool, + pkt_hdr)) + continue; + } } - pkt = pkt_tbl[i]; - pkt_hdr = packet_hdr(pkt); pull_tail(pkt_hdr, max_len - len); if (frame_offset) pull_head(pkt_hdr, frame_offset); if (odp_packet_copy_from_mem(pkt, 0, len, slot.buf) != 0) - goto fail; + break; pkt_hdr->input = pktio_entry->s.handle; - if (pktio_cls_enabled(pktio_entry)) - _odp_packet_copy_cls_md(pkt_hdr, &parsed_hdr); - else - _odp_packet_parse_layer(pkt_hdr, - pktio_entry->s.config.parser.layer, - pktio_entry->s.in_chksums); + if (layer >= ODP_PROTO_LAYER_L4) + _odp_packet_l4_chksum(pkt_hdr, chksums, l4_part_sum); packet_set_ts(pkt_hdr, ts); + num_rx++; } - return i; + if (num_rx < num) + odp_packet_free_multi(&pkt_tbl[num_rx], num - num_rx); -fail: - odp_packet_free_multi(&pkt_tbl[i], num - i); - return i; + return num_rx; } static inline int netmap_recv_desc(pktio_entry_t *pktio_entry, @@ -1042,11 +1051,11 @@ static int netmap_recv_tmo(pktio_entry_t *pktio_entry, int index, } static int netmap_recv_mq_tmo(pktio_entry_t *pktio_entry[], int index[], - int num_q, odp_packet_t pkt_table[], int num, - unsigned *from, uint64_t usecs) + uint32_t num_q, odp_packet_t pkt_table[], int num, + uint32_t *from, uint64_t usecs) { struct timeval timeout; - int i; + uint32_t i; int ret; int maxfd = -1, maxfd2; fd_set readfds; diff --git a/platform/linux-generic/pktio/null.c b/platform/linux-generic/pktio/null.c index 90e113ec6..a6498da8c 100644 --- a/platform/linux-generic/pktio/null.c +++ b/platform/linux-generic/pktio/null.c @@ -71,9 +71,9 @@ static int null_recv_tmo(pktio_entry_t *pktio_entry ODP_UNUSED, } static int null_recv_mq_tmo(pktio_entry_t *pktio_entry[] ODP_UNUSED, - int index[] ODP_UNUSED, int num_q ODP_UNUSED, + int index[] ODP_UNUSED, uint32_t num_q ODP_UNUSED, odp_packet_t pkt_table[] ODP_UNUSED, - int num ODP_UNUSED, unsigned *from ODP_UNUSED, + int num ODP_UNUSED, uint32_t *from ODP_UNUSED, uint64_t usecs) { struct timeval timeout; diff --git a/platform/linux-generic/pktio/pcap.c b/platform/linux-generic/pktio/pcap.c index 7290626c2..af94ffa72 100644 --- a/platform/linux-generic/pktio/pcap.c +++ b/platform/linux-generic/pktio/pcap.c @@ -46,6 +46,7 @@ #include <odp/api/plat/packet_inlines.h> +#include <odp_parse_internal.h> #include <odp_classification_internal.h> #include <odp_debug_internal.h> #include <odp_global_data.h> @@ -248,7 +249,12 @@ static int pcapif_recv_pkt(pktio_entry_t *pktio_entry, int index ODP_UNUSED, pkt_pcap_t *pcap = pkt_priv(pktio_entry); odp_time_t ts_val; odp_time_t *ts = NULL; + int packets = 0, errors = 0; + uint32_t octets = 0; uint16_t frame_offset = pktio_entry->s.pktin_frame_offset; + const odp_proto_chksums_t chksums = pktio_entry->s.in_chksums; + const odp_proto_layer_t layer = pktio_entry->s.parse_layer; + const odp_pktin_config_opt_t opt = pktio_entry->s.config.pktin; odp_ticketlock_lock(&pktio_entry->s.rxl); @@ -256,8 +262,7 @@ static int pcapif_recv_pkt(pktio_entry_t *pktio_entry, int index ODP_UNUSED, odp_ticketlock_unlock(&pktio_entry->s.rxl); return 0; } - if (pktio_entry->s.config.pktin.bit.ts_all || - pktio_entry->s.config.pktin.bit.ts_ptp) + if (opt.bit.ts_all || opt.bit.ts_ptp) ts = &ts_val; for (i = 0; i < num; ) { @@ -291,42 +296,64 @@ static int pcapif_recv_pkt(pktio_entry_t *pktio_entry, int index ODP_UNUSED, break; } - if (pktio_cls_enabled(pktio_entry)) { - odp_packet_t new_pkt; + if (layer) { + uint64_t l4_part_sum = 0; - ret = _odp_cls_classify_packet(pktio_entry, data, - pkt_len, pkt_len, - &new_pool, pkt_hdr, true); - if (ret) { + ret = _odp_packet_parse_common(&pkt_hdr->p, data, pkt_len, + pkt_len, layer, chksums, + &l4_part_sum, opt); + if (ret) + errors++; + + if (ret < 0) { odp_packet_free(pkt); continue; } - if (new_pool != pcap->pool) { - new_pkt = odp_packet_copy(pkt, new_pool); - odp_packet_free(pkt); + if (pktio_cls_enabled(pktio_entry)) { + odp_packet_t new_pkt; - if (odp_unlikely(new_pkt == ODP_PACKET_INVALID)) + ret = _odp_cls_classify_packet(pktio_entry, data, + &new_pool, pkt_hdr); + if (ret) { + odp_packet_free(pkt); continue; + } + if (new_pool != pcap->pool) { + new_pkt = odp_packet_copy(pkt, new_pool); + + odp_packet_free(pkt); + + if (odp_unlikely(new_pkt == ODP_PACKET_INVALID)) { + pktio_entry->s.stats.in_discards++; + continue; + } - pkt = new_pkt; - pkt_hdr = packet_hdr(new_pkt); + pkt = new_pkt; + pkt_hdr = packet_hdr(new_pkt); + } } - } else { - _odp_packet_parse_layer(pkt_hdr, - pktio_entry->s.config.parser.layer, - pktio_entry->s.in_chksums); + + if (layer >= ODP_PROTO_LAYER_L4) + _odp_packet_l4_chksum(pkt_hdr, chksums, l4_part_sum); } - pktio_entry->s.stats.in_octets += pkt_hdr->frame_len; packet_set_ts(pkt_hdr, ts); pkt_hdr->input = pktio_entry->s.handle; + if (!pkt_hdr->p.flags.all.error) { + octets += pkt_len; + packets++; + } + pkts[i] = pkt; i++; } - pktio_entry->s.stats.in_packets += i; + + pktio_entry->s.stats.in_octets += octets; + pktio_entry->s.stats.in_packets += packets; + pktio_entry->s.stats.in_errors += errors; odp_ticketlock_unlock(&pktio_entry->s.rxl); @@ -441,6 +468,8 @@ static int pcapif_capability(pktio_entry_t *pktio_entry ODP_UNUSED, capa->stats.pktio.counter.in_octets = 1; capa->stats.pktio.counter.in_packets = 1; + capa->stats.pktio.counter.in_discards = 1; + capa->stats.pktio.counter.in_errors = 1; capa->stats.pktio.counter.out_octets = 1; capa->stats.pktio.counter.out_packets = 1; diff --git a/platform/linux-generic/pktio/pktio_common.c b/platform/linux-generic/pktio/pktio_common.c index d2f4d7219..1f3cf951e 100644 --- a/platform/linux-generic/pktio/pktio_common.c +++ b/platform/linux-generic/pktio/pktio_common.c @@ -11,13 +11,13 @@ static int sock_recv_mq_tmo_select(pktio_entry_t * const *entry, const int index[], - unsigned int num_q, unsigned int *from, + uint32_t num_q, uint32_t *from, odp_packet_t packets[], int num, uint64_t usecs, fd_set *readfds, int maxfd) { struct timeval timeout; - unsigned int i; + uint32_t i; int ret; for (i = 0; i < num_q; i++) { @@ -50,17 +50,17 @@ static int sock_recv_mq_tmo_select(pktio_entry_t * const *entry, } int _odp_sock_recv_mq_tmo_try_int_driven(const struct odp_pktin_queue_t queues[], - unsigned int num_q, unsigned int *from, + uint32_t num_q, uint32_t *from, odp_packet_t packets[], int num, uint64_t usecs, int *trial_successful) { - unsigned int i; + uint32_t i; pktio_entry_t *entry[num_q]; int index[num_q]; fd_set readfds; int maxfd = -1; - int (*impl)(pktio_entry_t *entry[], int index[], int num_q, - odp_packet_t packets[], int num, unsigned int *from, + int (*impl)(pktio_entry_t *entry[], int index[], uint32_t num_q, + odp_packet_t packets[], int num, uint32_t *from, uint64_t wait_usecs) = NULL; int impl_set = 0; diff --git a/platform/linux-generic/pktio/socket.c b/platform/linux-generic/pktio/socket.c index 81c178b9d..9d1bbe545 100644 --- a/platform/linux-generic/pktio/socket.c +++ b/platform/linux-generic/pktio/socket.c @@ -15,12 +15,14 @@ #include <odp/api/ticketlock.h> #include <odp_socket_common.h> +#include <odp_parse_internal.h> #include <odp_packet_internal.h> #include <odp_packet_io_internal.h> #include <odp_packet_io_stats.h> #include <odp_debug_internal.h> #include <odp_errno_define.h> #include <odp_classification_internal.h> +#include <odp_macros_internal.h> #include <sys/socket.h> #include <stdio.h> @@ -233,6 +235,9 @@ static int sock_mmsg_recv(pktio_entry_t *pktio_entry, int index ODP_UNUSED, int i; uint16_t frame_offset = pktio_entry->s.pktin_frame_offset; uint32_t alloc_len = pkt_sock->mtu + frame_offset; + const odp_proto_chksums_t chksums = pktio_entry->s.in_chksums; + const odp_proto_layer_t layer = pktio_entry->s.parse_layer; + const odp_pktin_config_opt_t opt = pktio_entry->s.config.pktin; memset(msgvec, 0, sizeof(msgvec)); @@ -249,8 +254,7 @@ static int sock_mmsg_recv(pktio_entry_t *pktio_entry, int index ODP_UNUSED, recv_msgs = recvmmsg(sockfd, msgvec, nb_pkts, MSG_DONTWAIT, NULL); odp_ticketlock_unlock(&pkt_sock->rx_lock); - if (pktio_entry->s.config.pktin.bit.ts_all || - pktio_entry->s.config.pktin.bit.ts_ptp) { + if (opt.bit.ts_all || opt.bit.ts_ptp) { ts_val = odp_time_global(); ts = &ts_val; } @@ -262,25 +266,48 @@ static int sock_mmsg_recv(pktio_entry_t *pktio_entry, int index ODP_UNUSED, odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); uint16_t pkt_len = msgvec[i].msg_len; int ret; + uint64_t l4_part_sum = 0; if (odp_unlikely(msgvec[i].msg_hdr.msg_flags & MSG_TRUNC)) { odp_packet_free(pkt); ODP_DBG("dropped truncated packet\n"); continue; } - if (pktio_cls_enabled(pktio_entry)) { - uint16_t seg_len = pkt_len; - if (msgvec[i].msg_hdr.msg_iov->iov_len < pkt_len) - seg_len = msgvec[i].msg_hdr.msg_iov->iov_len; + ret = odp_packet_trunc_tail(&pkt, odp_packet_len(pkt) - pkt_len, + NULL, NULL); + if (ret < 0) { + ODP_ERR("trunc_tail failed"); + odp_packet_free(pkt); + continue; + } + + if (layer) { + uint8_t buf[PARSE_BYTES]; + uint16_t seg_len = msgvec[i].msg_hdr.msg_iov->iov_len; - if (_odp_cls_classify_packet(pktio_entry, base, pkt_len, - seg_len, &pool, pkt_hdr, - true)) { - ODP_ERR("_odp_cls_classify_packet failed"); + /* Make sure there is enough data for the packet + * parser in the case of a segmented packet. */ + if (odp_unlikely(seg_len < PARSE_BYTES && pkt_len > seg_len)) { + seg_len = MIN(pkt_len, PARSE_BYTES); + odp_packet_copy_to_mem(pkt, 0, seg_len, buf); + base = buf; + } + + if (_odp_packet_parse_common(&pkt_hdr->p, base, pkt_len, + seg_len, layer, chksums, + &l4_part_sum, opt) < 0) { odp_packet_free(pkt); continue; } + + if (pktio_cls_enabled(pktio_entry)) { + if (_odp_cls_classify_packet(pktio_entry, base, &pool, + pkt_hdr)) { + odp_packet_free(pkt); + continue; + } + } } /* Don't receive packets sent by ourselves */ @@ -290,20 +317,10 @@ static int sock_mmsg_recv(pktio_entry_t *pktio_entry, int index ODP_UNUSED, continue; } - ret = odp_packet_trunc_tail(&pkt, odp_packet_len(pkt) - pkt_len, - NULL, NULL); - if (ret < 0) { - ODP_ERR("trunk_tail failed"); - odp_packet_free(pkt); - continue; - } - pkt_hdr->input = pktio_entry->s.handle; - if (!pktio_cls_enabled(pktio_entry)) - _odp_packet_parse_layer(pkt_hdr, - pktio_entry->s.config.parser.layer, - pktio_entry->s.in_chksums); + if (layer >= ODP_PROTO_LAYER_L4) + _odp_packet_l4_chksum(pkt_hdr, chksums, l4_part_sum); packet_set_ts(pkt_hdr, ts); @@ -359,11 +376,11 @@ static int sock_recv_tmo(pktio_entry_t *pktio_entry, int index, } static int sock_recv_mq_tmo(pktio_entry_t *pktio_entry[], int index[], - int num_q, odp_packet_t pkt_table[], int num, - unsigned *from, uint64_t usecs) + uint32_t num_q, odp_packet_t pkt_table[], int num, + uint32_t *from, uint64_t usecs) { struct timeval timeout; - int i; + uint32_t i; int ret; int maxfd = -1, maxfd2; fd_set readfds; diff --git a/platform/linux-generic/pktio/socket_mmap.c b/platform/linux-generic/pktio/socket_mmap.c index f84834610..7824b0e91 100644 --- a/platform/linux-generic/pktio/socket_mmap.c +++ b/platform/linux-generic/pktio/socket_mmap.c @@ -16,6 +16,7 @@ #include <odp/api/plat/packet_inlines.h> #include <odp_socket_common.h> +#include <odp_parse_internal.h> #include <odp_packet_internal.h> #include <odp_packet_io_internal.h> #include <odp_packet_io_stats.h> @@ -150,9 +151,11 @@ static inline unsigned pkt_mmap_v2_rx(pktio_entry_t *pktio_entry, odp_pool_t pool = pkt_sock->pool; uint16_t frame_offset = pktio_entry->s.pktin_frame_offset; uint16_t vlan_len = 0; + const odp_proto_chksums_t chksums = pktio_entry->s.in_chksums; + const odp_proto_layer_t layer = pktio_entry->s.parse_layer; + const odp_pktin_config_opt_t opt = pktio_entry->s.config.pktin; - if (pktio_entry->s.config.pktin.bit.ts_all || - pktio_entry->s.config.pktin.bit.ts_ptp) + if (opt.bit.ts_all || opt.bit.ts_ptp) ts = &ts_val; ring = &pkt_sock->rx_ring; @@ -163,8 +166,8 @@ static inline unsigned pkt_mmap_v2_rx(pktio_entry_t *pktio_entry, struct tpacket2_hdr *tp_hdr; odp_packet_t pkt; odp_packet_hdr_t *hdr; - odp_packet_hdr_t parsed_hdr; int ret; + uint64_t l4_part_sum = 0; tp_hdr = (void *)next_ptr; @@ -212,18 +215,29 @@ static inline unsigned pkt_mmap_v2_rx(pktio_entry_t *pktio_entry, continue; } - if (pktio_cls_enabled(pktio_entry)) { - if (_odp_cls_classify_packet(pktio_entry, pkt_buf, pkt_len, - pkt_len, &pool, &parsed_hdr, - true)) { + hdr = packet_hdr(pkt); + + if (layer) { + if (_odp_packet_parse_common(&hdr->p, pkt_buf, pkt_len, + pkt_len, layer, chksums, + &l4_part_sum, opt) < 0) { odp_packet_free(pkt); tp_hdr->tp_status = TP_STATUS_KERNEL; frame_num = next_frame_num; continue; } + + if (pktio_cls_enabled(pktio_entry)) { + if (_odp_cls_classify_packet(pktio_entry, pkt_buf, + &pool, hdr)) { + odp_packet_free(pkt); + tp_hdr->tp_status = TP_STATUS_KERNEL; + frame_num = next_frame_num; + continue; + } + } } - hdr = packet_hdr(pkt); if (frame_offset) pull_head(hdr, frame_offset); @@ -276,12 +290,8 @@ static inline unsigned pkt_mmap_v2_rx(pktio_entry_t *pktio_entry, hdr->input = pktio_entry->s.handle; - if (pktio_cls_enabled(pktio_entry)) - _odp_packet_copy_cls_md(hdr, &parsed_hdr); - else - _odp_packet_parse_layer(hdr, - pktio_entry->s.config.parser.layer, - pktio_entry->s.in_chksums); + if (layer >= ODP_PROTO_LAYER_L4) + _odp_packet_l4_chksum(hdr, chksums, l4_part_sum); packet_set_ts(hdr, ts); @@ -727,11 +737,11 @@ static int sock_mmap_recv_tmo(pktio_entry_t *pktio_entry, int index, } static int sock_mmap_recv_mq_tmo(pktio_entry_t *pktio_entry[], int index[], - int num_q, odp_packet_t pkt_table[], int num, - unsigned *from, uint64_t usecs) + uint32_t num_q, odp_packet_t pkt_table[], int num, + uint32_t *from, uint64_t usecs) { struct timeval timeout; - int i; + uint32_t i; int ret; int maxfd = -1, maxfd2; fd_set readfds; diff --git a/platform/linux-generic/pktio/tap.c b/platform/linux-generic/pktio/tap.c index d298c8982..40c46b43c 100644 --- a/platform/linux-generic/pktio/tap.c +++ b/platform/linux-generic/pktio/tap.c @@ -38,6 +38,7 @@ #include <odp/api/plat/packet_inlines.h> +#include <odp_parse_internal.h> #include <odp_debug_internal.h> #include <odp_socket_common.h> #include <odp_packet_internal.h> @@ -286,14 +287,27 @@ static odp_packet_t pack_odp_pkt(pktio_entry_t *pktio_entry, const void *data, odp_packet_hdr_t *pkt_hdr; odp_packet_hdr_t parsed_hdr; int num; + uint64_t l4_part_sum = 0; uint16_t frame_offset = pktio_entry->s.pktin_frame_offset; - - if (pktio_cls_enabled(pktio_entry)) { - if (_odp_cls_classify_packet(pktio_entry, data, len, len, - &pkt_priv(pktio_entry)->pool, - &parsed_hdr, true)) { + const odp_proto_chksums_t chksums = pktio_entry->s.in_chksums; + const odp_proto_layer_t layer = pktio_entry->s.parse_layer; + const odp_pktin_config_opt_t opt = pktio_entry->s.config.pktin; + + if (layer) { + packet_parse_reset(&parsed_hdr, 1); + packet_set_len(&parsed_hdr, len); + if (_odp_packet_parse_common(&parsed_hdr.p, data, len, len, layer, + chksums, &l4_part_sum, opt) < 0) { return ODP_PACKET_INVALID; } + + if (pktio_cls_enabled(pktio_entry)) { + if (_odp_cls_classify_packet(pktio_entry, data, + &pkt_priv(pktio_entry)->pool, + &parsed_hdr)) { + return ODP_PACKET_INVALID; + } + } } num = _odp_packet_alloc_multi(pkt_priv(pktio_entry)->pool, @@ -312,12 +326,12 @@ static odp_packet_t pack_odp_pkt(pktio_entry_t *pktio_entry, const void *data, return ODP_PACKET_INVALID; } - if (pktio_cls_enabled(pktio_entry)) + if (layer) { _odp_packet_copy_cls_md(pkt_hdr, &parsed_hdr); - else - _odp_packet_parse_layer(pkt_hdr, - pktio_entry->s.config.parser.layer, - pktio_entry->s.in_chksums); + + if (layer >= ODP_PROTO_LAYER_L4) + _odp_packet_l4_chksum(pkt_hdr, chksums, l4_part_sum); + } packet_set_ts(pkt_hdr, ts); pkt_hdr->input = pktio_entry->s.handle; @@ -335,6 +349,7 @@ static int tap_pktio_recv(pktio_entry_t *pktio_entry, int index ODP_UNUSED, uint8_t buf[mtu]; odp_time_t ts_val; odp_time_t *ts = NULL; + int num_rx = 0; odp_ticketlock_lock(&pktio_entry->s.rxl); @@ -355,14 +370,15 @@ static int tap_pktio_recv(pktio_entry_t *pktio_entry, int index ODP_UNUSED, break; } - pkts[i] = pack_odp_pkt(pktio_entry, buf, retval, ts); - if (pkts[i] == ODP_PACKET_INVALID) + pkts[num_rx] = pack_odp_pkt(pktio_entry, buf, retval, ts); + if (pkts[num_rx] == ODP_PACKET_INVALID) break; + num_rx++; } odp_ticketlock_unlock(&pktio_entry->s.rxl); - return i; + return num_rx; } static int tap_pktio_send_lockless(pktio_entry_t *pktio_entry, diff --git a/scripts/ci/build_arm64.sh b/scripts/ci/build_arm64.sh index 79b55e531..f340a0ca0 100755 --- a/scripts/ci/build_arm64.sh +++ b/scripts/ci/build_arm64.sh @@ -6,18 +6,27 @@ if [[ $(uname -m) =~ ^(arm64|aarch64)$ ]]; then export BUILD_ARCH=aarch64-linux-gnu fi -if [ "${CC#clang}" != "${CC}" ] ; then - export CC="clang --target=${TARGET_ARCH}" - export CXX="clang++ --target=${TARGET_ARCH}" +if [ "$TARGET_ARCH" == "$BUILD_ARCH" ]; then + # Native build + if [ "${CC#clang}" != "${CC}" ] ; then + export CXX="clang++" + fi else - export CC="${TARGET_ARCH}-gcc" - export CXX="${TARGET_ARCH}-g++" -fi -export CPPFLAGS="-I/usr/include/${TARGET_ARCH}/dpdk" + # Cross compilation + if [ "${CC#clang}" != "${CC}" ] ; then + export CC="clang --target=${TARGET_ARCH}" + export CXX="clang++ --target=${TARGET_ARCH}" + else + export CC="${TARGET_ARCH}-gcc" + export CXX="${TARGET_ARCH}-g++" + fi + + export CPPFLAGS="-I/usr/include/${TARGET_ARCH}/dpdk" -# Use target libraries -export PKG_CONFIG_PATH= -export PKG_CONFIG_LIBDIR=/usr/lib/${TARGET_ARCH}/pkgconfig:/usr/local/lib/${TARGET_ARCH}/pkgconfig + # Use target libraries + export PKG_CONFIG_PATH= + export PKG_CONFIG_LIBDIR=/usr/lib/${TARGET_ARCH}/pkgconfig:/usr/local/lib/${TARGET_ARCH}/pkgconfig +fi # ARMv8 crypto export PKG_CONFIG_PATH=~/aarch64cryptolib/pkgconfig:$PKG_CONFIG_PATH diff --git a/test/performance/odp_bench_packet.c b/test/performance/odp_bench_packet.c index 0354ef9b8..23fd17bea 100644 --- a/test/performance/odp_bench_packet.c +++ b/test/performance/odp_bench_packet.c @@ -756,7 +756,7 @@ static int bench_packet_headroom(void) for (i = 0; i < TEST_REPEAT_COUNT; i++) ret += odp_packet_headroom(gbl_args->pkt_tbl[i]); - return i; + return i + ret; } static int bench_packet_tailroom(void) @@ -767,7 +767,7 @@ static int bench_packet_tailroom(void) for (i = 0; i < TEST_REPEAT_COUNT; i++) ret += odp_packet_tailroom(gbl_args->pkt_tbl[i]); - return i; + return i + ret; } static int bench_packet_tail(void) @@ -1412,7 +1412,7 @@ static int bench_packet_has_ref(void) for (i = 0; i < TEST_REPEAT_COUNT; i++) ret += odp_packet_has_ref(pkt_tbl[i]); - return i; + return i + ret; } static int bench_packet_subtype(void) diff --git a/test/performance/odp_crypto.c b/test/performance/odp_crypto.c index a26d072e9..45e770d42 100644 --- a/test/performance/odp_crypto.c +++ b/test/performance/odp_crypto.c @@ -610,7 +610,6 @@ create_session_from_config(odp_crypto_session_t *session, odp_crypto_session_param_init(¶ms); memcpy(¶ms, &config->session, sizeof(odp_crypto_session_param_t)); params.op = ODP_CRYPTO_OP_ENCODE; - params.pref_mode = ODP_CRYPTO_SYNC; /* Lookup the packet pool */ pkt_pool = odp_pool_lookup("packet_pool"); diff --git a/test/performance/odp_ipsec.c b/test/performance/odp_ipsec.c index 5f1f9d4ef..c464ba0d7 100644 --- a/test/performance/odp_ipsec.c +++ b/test/performance/odp_ipsec.c @@ -1,4 +1,6 @@ /* Copyright (c) 2018, Linaro Limited + * Copyright (c) 2022, Marvell + * Copyright (c) 2022, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -18,11 +20,14 @@ #include <odp_api.h> #include <odp/helper/odph_api.h> +#include <inttypes.h> /** @def POOL_NUM_PKT * Number of packets in the pool */ -#define POOL_NUM_PKT 64 +#define POOL_NUM_PKT 4096 + +#define MAX_DEQUEUE_BURST 16 static uint8_t test_salt[16] = "0123456789abcdef"; @@ -97,11 +102,11 @@ typedef struct { int in_flight; /** - * Number of iteration to repeat crypto operation to get good - * average number. Specified through -i or --terations option. + * Number of packets to be IPsec processed to get good average number. + * Specified through -c or --count option. * Default is 10000. */ - int iteration_count; + int packet_count; /** * Payload size to test. If 0 set of predefined payload sizes @@ -139,6 +144,19 @@ typedef struct { * Specified through -u argument. */ int ah; + + /* + * Burst size. + * Prepare and submit as many packets for IPsec processing in each + * iteration of the loop. + */ + int burst_size; + + /* + * Use vector packet completion from IPsec APIs. + * Specified through -v or --vector argument. + */ + uint32_t vec_pkt_size; } ipsec_args_t; /* @@ -516,7 +534,7 @@ print_result(ipsec_args_t *cargs, throughput = (1000000.0 / result->elapsed) * payload_length / 1024; printf("%30.30s %15d %15d %15.3f %15.3f %15.3f %15d\n", - config->name, cargs->iteration_count, payload_length, + config->name, cargs->packet_count, payload_length, result->elapsed, result->rusage_self, result->rusage_thread, throughput); } @@ -584,28 +602,49 @@ static uint8_t test_data[] = { 0x08, 0x00, 0xfb, 0x37, 0x12, 0x34, 0x00, 0x00 }; -static odp_packet_t -make_packet(odp_pool_t pkt_pool, unsigned int payload_length) +static inline void debug_packets(int debug, odp_packet_t *pkt, int num_pkts) { - odp_packet_t pkt; + if (odp_likely(!debug)) + return; + for (int i = 0; i < num_pkts; i++) + odp_packet_print_data(pkt[i], 0, odp_packet_len(pkt[i])); +} - if (payload_length < sizeof(test_data)) - return ODP_PACKET_INVALID; +static int +make_packet_multi(odp_pool_t pkt_pool, unsigned int payload_length, + odp_packet_t pkt[], int num) +{ + int i, ret; + + ret = odp_packet_alloc_multi(pkt_pool, payload_length, pkt, num); + if (ret != num) { + ODPH_ERR("Could not allocate buffer\n"); + if (ret > 0) + odp_packet_free_sp(pkt, ret); + return -1; + } + + for (i = 0; i < num; i++) { + odp_packet_copy_from_mem(pkt[i], 0, sizeof(test_data), test_data); + odp_packet_l3_offset_set(pkt[i], 0); - pkt = odp_packet_alloc(pkt_pool, payload_length); - if (pkt == ODP_PACKET_INVALID) { - ODPH_ERR("failed to allocate buffer\n"); - return pkt; + uint8_t *mem = odp_packet_data(pkt[i]); + ((odph_ipv4hdr_t *)mem)->tot_len = odp_cpu_to_be_16(payload_length); + memset(mem + sizeof(test_data), 1, payload_length - sizeof(test_data)); } - odp_packet_copy_from_mem(pkt, 0, sizeof(test_data), test_data); - odp_packet_l3_offset_set(pkt, 0); + return 0; +} - uint8_t *mem = odp_packet_data(pkt); - ((odph_ipv4hdr_t *)mem)->tot_len = odp_cpu_to_be_16(payload_length); - memset(mem + sizeof(test_data), 1, payload_length - sizeof(test_data)); +static inline void check_ipsec_result(odp_packet_t ipsec_pkt) +{ + odp_ipsec_packet_result_t result; - return pkt; + if (odp_unlikely(odp_ipsec_result(&result, ipsec_pkt))) + ODPH_ERR("odp_ipsec_result() failed\n"); + else if (odp_unlikely(result.status.error.all)) + ODPH_ERR("IPsec processing error: %" PRIu32 "\n", + result.status.error.all); } /** @@ -619,10 +658,13 @@ run_measure_one(ipsec_args_t *cargs, time_record_t *start, time_record_t *end) { + int in_flight, pkts_allowed, num_out, num_pkts, rc = 0; + const int max_in_flight = cargs->in_flight; + const int burst_size = cargs->burst_size; + const int packet_count = cargs->packet_count; + const int debug = cargs->debug_packets; odp_ipsec_out_param_t param; odp_pool_t pkt_pool; - odp_packet_t pkt = ODP_PACKET_INVALID; - int rc = 0; pkt_pool = odp_pool_lookup("packet_pool"); if (pkt_pool == ODP_POOL_INVALID) { @@ -630,6 +672,9 @@ run_measure_one(ipsec_args_t *cargs, return -1; } + if (payload_length < sizeof(test_data)) + return -1; + int packets_sent = 0; int packets_received = 0; @@ -641,43 +686,53 @@ run_measure_one(ipsec_args_t *cargs, fill_time_record(start); - while ((packets_sent < cargs->iteration_count) || - (packets_received < cargs->iteration_count)) { - if ((packets_sent < cargs->iteration_count) && - (packets_sent - packets_received < - cargs->in_flight)) { - odp_packet_t out_pkt; - int num_out = 1; + while ((packets_sent < packet_count) || + (packets_received < packet_count)) { + num_pkts = packet_count - packets_sent; + + /* Enqueue up to burst size */ + num_pkts = num_pkts > burst_size ? burst_size : num_pkts; + + /* Enqueue up to (max in flight - current in flight) */ + in_flight = packets_sent - packets_received; + pkts_allowed = max_in_flight - in_flight; - pkt = make_packet(pkt_pool, payload_length); - if (ODP_PACKET_INVALID == pkt) + /* Enqueue either a burst of packets or skip */ + num_pkts = num_pkts > pkts_allowed ? 0 : num_pkts; + + if (odp_likely(num_pkts)) { + odp_packet_t out_pkt[num_pkts]; + odp_packet_t pkt[num_pkts]; + int i; + + if (odp_unlikely(make_packet_multi(pkt_pool, + payload_length, + pkt, + num_pkts))) return -1; - if (cargs->debug_packets) - odp_packet_print_data(pkt, 0, - odp_packet_len(pkt)); + debug_packets(debug, pkt, num_pkts); + num_out = num_pkts; - rc = odp_ipsec_out(&pkt, 1, - &out_pkt, &num_out, + rc = odp_ipsec_out(pkt, num_pkts, + out_pkt, &num_out, ¶m); - if (rc <= 0) { - ODPH_ERR("failed odp_ipsec_out: rc = %d\n", rc); - odp_packet_free(pkt); + if (odp_unlikely(rc <= 0)) { + ODPH_ERR("Failed odp_ipsec_out: rc = %d\n", rc); + odp_packet_free_sp(pkt, num_pkts); break; } - if (odp_packet_has_error(out_pkt)) { - odp_ipsec_packet_result_t result; - odp_ipsec_result(&result, out_pkt); - ODPH_ERR("Received error packet: %d\n", - result.status.error.all); - } + for (i = 0; i < num_out; i++) + check_ipsec_result(out_pkt[i]); + packets_sent += rc; packets_received += num_out; - if (cargs->debug_packets) - odp_packet_print_data(out_pkt, 0, - odp_packet_len(out_pkt)); - odp_packet_free(out_pkt); + debug_packets(debug, out_pkt, num_out); + + if (odp_unlikely(rc != num_pkts)) + odp_packet_free_sp(&pkt[rc], num_pkts - rc); + odp_packet_free_sp(out_pkt, num_out); } } @@ -686,6 +741,46 @@ run_measure_one(ipsec_args_t *cargs, return rc < 0 ? rc : 0; } +static uint32_t dequeue_burst(odp_queue_t polled_queue, + odp_event_t *events, + int max_burst) +{ + int num = 0; + + if (polled_queue != ODP_QUEUE_INVALID) { + int rc = odp_queue_deq_multi(polled_queue, + events, + max_burst); + num = odp_likely(rc >= 0) ? rc : 0; + } else { + num = odp_schedule_multi(NULL, + ODP_SCHED_NO_WAIT, + events, + max_burst); + } + return num; +} + +static inline uint32_t vec_pkt_handle(int debug, odp_event_t ev) +{ + odp_packet_vector_t vec = odp_packet_vector_from_event(ev); + uint32_t vec_size = odp_packet_vector_size(vec); + odp_packet_t *pkt_tbl; + uint32_t j; + + odp_packet_vector_tbl(vec, &pkt_tbl); + + for (j = 0; j < vec_size; j++) + check_ipsec_result(pkt_tbl[j]); + + debug_packets(debug, pkt_tbl, vec_size); + + odp_packet_free_sp(pkt_tbl, vec_size); + odp_packet_vector_free(vec); + + return vec_size; +} + static int run_measure_one_async(ipsec_args_t *cargs, odp_ipsec_sa_t sa, @@ -693,11 +788,14 @@ run_measure_one_async(ipsec_args_t *cargs, time_record_t *start, time_record_t *end) { + int in_flight, packets_allowed, num_pkts, rc = 0; + const int max_in_flight = cargs->in_flight; + const int burst_size = cargs->burst_size; + const int packet_count = cargs->packet_count; + const int debug = cargs->debug_packets; odp_ipsec_out_param_t param; odp_pool_t pkt_pool; - odp_queue_t out_queue; - odp_packet_t pkt = ODP_PACKET_INVALID; - int rc = 0; + odp_queue_t polled_queue = ODP_QUEUE_INVALID; pkt_pool = odp_pool_lookup("packet_pool"); if (pkt_pool == ODP_POOL_INVALID) { @@ -705,12 +803,17 @@ run_measure_one_async(ipsec_args_t *cargs, return -1; } - out_queue = odp_queue_lookup("ipsec-out"); - if (out_queue == ODP_QUEUE_INVALID) { - ODPH_ERR("ipsec-out queue not found\n"); - return -1; + if (cargs->poll) { + polled_queue = odp_queue_lookup("ipsec-out"); + if (polled_queue == ODP_QUEUE_INVALID) { + ODPH_ERR("ipsec-out queue not found\n"); + return -1; + } } + if (payload_length < sizeof(test_data)) + return -1; + int packets_sent = 0; int packets_received = 0; @@ -722,55 +825,78 @@ run_measure_one_async(ipsec_args_t *cargs, fill_time_record(start); - while ((packets_sent < cargs->iteration_count) || - (packets_received < cargs->iteration_count)) { - odp_event_t ev; + while ((packets_sent < packet_count) || + (packets_received < packet_count)) { + + num_pkts = packet_count - packets_sent; + + /* Enqueue up to burst size */ + num_pkts = num_pkts > burst_size ? burst_size : num_pkts; + + /* Enqueue up to (max in flight - current in flight) */ + in_flight = packets_sent - packets_received; + packets_allowed = max_in_flight - in_flight; - if ((packets_sent < cargs->iteration_count) && - (packets_sent - packets_received < - cargs->in_flight)) { - pkt = make_packet(pkt_pool, payload_length); - if (ODP_PACKET_INVALID == pkt) + if (num_pkts > 0 && num_pkts <= packets_allowed) { + odp_packet_t pkt[num_pkts]; + + if (odp_unlikely(make_packet_multi(pkt_pool, + payload_length, + pkt, + num_pkts))) return -1; - if (cargs->debug_packets) - odp_packet_print_data(pkt, 0, - odp_packet_len(pkt)); + debug_packets(debug, pkt, num_pkts); - rc = odp_ipsec_out_enq(&pkt, 1, - ¶m); - if (rc <= 0) { - ODPH_ERR("failed odp_crypto_packet_op_enq: rc = %d\n", + rc = odp_ipsec_out_enq(pkt, num_pkts, ¶m); + if (odp_unlikely(rc <= 0)) { + ODPH_ERR("Failed odp_ipsec_out_enq: rc = %d\n", rc); - odp_packet_free(pkt); + odp_packet_free_sp(pkt, num_pkts); break; } + + if (odp_unlikely(rc != num_pkts)) + odp_packet_free_sp(&pkt[rc], num_pkts - rc); + packets_sent += rc; - } + } else { + odp_packet_t pkt_out[max_in_flight]; + uint32_t i = 0; + + /* + * Dequeue packets until we can enqueue the next burst + * or until we have received all remaining packets + * when there are no more packets to be sent. + */ + while (num_pkts > packets_allowed || + (num_pkts == 0 && packets_received < packet_count)) { + odp_event_t events[MAX_DEQUEUE_BURST]; + uint32_t num; + + num = dequeue_burst(polled_queue, events, MAX_DEQUEUE_BURST); + + for (uint32_t n = 0; n < num; n++) { + if (odp_event_type(events[n]) == ODP_EVENT_PACKET_VECTOR) { + uint32_t vec_size; + + vec_size = vec_pkt_handle(debug, events[n]); + packets_received += vec_size - 1; + packets_allowed += vec_size - 1; + } else { + pkt_out[i] = odp_ipsec_packet_from_event(events[n]); + check_ipsec_result(pkt_out[i]); + i++; + } + + } + packets_received += num; + packets_allowed += num; + } + debug_packets(debug, pkt_out, i); - if (cargs->schedule) - ev = odp_schedule(NULL, - ODP_SCHED_NO_WAIT); - else - ev = odp_queue_deq(out_queue); - - while (ev != ODP_EVENT_INVALID) { - odp_packet_t out_pkt; - odp_ipsec_packet_result_t result; - - out_pkt = odp_ipsec_packet_from_event(ev); - odp_ipsec_result(&result, out_pkt); - - if (cargs->debug_packets) - odp_packet_print_data(out_pkt, 0, - odp_packet_len(out_pkt)); - odp_packet_free(out_pkt); - packets_received++; - if (cargs->schedule) - ev = odp_schedule(NULL, - ODP_SCHED_NO_WAIT); - else - ev = odp_queue_deq(out_queue); + if (i) + odp_packet_free_sp(pkt_out, i); } } @@ -843,13 +969,13 @@ run_measure_one_config(ipsec_args_t *cargs, break; count = get_elapsed_usec(&start, &end); - result.elapsed = count / cargs->iteration_count; + result.elapsed = count / cargs->packet_count; count = get_rusage_self_diff(&start, &end); - result.rusage_self = count / cargs->iteration_count; + result.rusage_self = count / cargs->packet_count; count = get_rusage_thread_diff(&start, &end); - result.rusage_thread = count / cargs->iteration_count; + result.rusage_thread = count / cargs->packet_count; print_result(cargs, payloads[i], config, &result); @@ -913,7 +1039,9 @@ static void usage(char *progname) print_config_names(" "); printf(" -d, --debug Enable dump of processed packets.\n" " -f, --flight <number> Max number of packet processed in parallel (default 1)\n" - " -i, --iterations <number> Number of iterations.\n" + " -c, --count <number> Number of packets (default 10000)\n" + " -b, --burst <number> Number of packets in one IPsec API submission (default 1)\n" + " -v, --vector <number> Enable vector packet completion from IPsec APIs with specified vector size.\n" " -l, --payload Payload length.\n" " -s, --schedule Use scheduler for completion events.\n" " -p, --poll Poll completion queue for completion events.\n" @@ -932,7 +1060,9 @@ static void parse_args(int argc, char *argv[], ipsec_args_t *cargs) {"debug", no_argument, NULL, 'd'}, {"flight", optional_argument, NULL, 'f'}, {"help", no_argument, NULL, 'h'}, - {"iterations", optional_argument, NULL, 'i'}, + {"count", optional_argument, NULL, 'c'}, + {"burst", optional_argument, NULL, 'b'}, + {"vector", optional_argument, NULL, 'v'}, {"payload", optional_argument, NULL, 'l'}, {"sessions", optional_argument, NULL, 'm'}, {"poll", no_argument, NULL, 'p'}, @@ -942,11 +1072,13 @@ static void parse_args(int argc, char *argv[], ipsec_args_t *cargs) {NULL, 0, NULL, 0} }; - static const char *shortopts = "+a:c:df:hi:m:nl:sptu"; + static const char *shortopts = "+a:b:c:df:hm:nl:sptuv:"; cargs->in_flight = 1; cargs->debug_packets = 0; - cargs->iteration_count = 10000; + cargs->packet_count = 10000; + cargs->burst_size = 1; + cargs->vec_pkt_size = 0; cargs->payload_length = 0; cargs->alg_config = NULL; cargs->schedule = 0; @@ -971,8 +1103,24 @@ static void parse_args(int argc, char *argv[], ipsec_args_t *cargs) case 'd': cargs->debug_packets = 1; break; - case 'i': - cargs->iteration_count = atoi(optarg); + case 'c': + cargs->packet_count = atoi(optarg); + break; + case 'b': + if (optarg == NULL) + cargs->burst_size = 32; + else + cargs->burst_size = atoi(optarg); + if (cargs->burst_size > POOL_NUM_PKT) { + printf("Invalid burst size (max allowed: %d)\n", POOL_NUM_PKT); + exit(-1); + } + break; + case 'v': + if (optarg == NULL) + cargs->vec_pkt_size = 32; + else + cargs->vec_pkt_size = atoi(optarg); break; case 'f': cargs->in_flight = atoi(optarg); @@ -1001,6 +1149,11 @@ static void parse_args(int argc, char *argv[], ipsec_args_t *cargs) } } + if (cargs->in_flight < cargs->burst_size) { + printf("-f (flight) must be greater than or equal to -b (burst)\n"); + exit(-1); + } + optind = 1; /* reset 'extern optind' from the getopt lib */ if (cargs->schedule && cargs->poll) { @@ -1012,6 +1165,7 @@ static void parse_args(int argc, char *argv[], ipsec_args_t *cargs) int main(int argc, char *argv[]) { + odp_pool_t vec_pool = ODP_POOL_INVALID; ipsec_args_t cargs; odp_pool_t pool; odp_queue_param_t qparam; @@ -1105,11 +1259,54 @@ int main(int argc, char *argv[]) exit(EXIT_FAILURE); } + if (cargs.vec_pkt_size) { + if (capa.vector.max_pools < 1) { + ODPH_ERR("Vector packet pool not available"); + exit(EXIT_FAILURE); + } + + if (!ipsec_capa.vector.supported) { + ODPH_ERR("Vector packet completion not supported by IPsec.\n"); + exit(EXIT_FAILURE); + } + + if (capa.vector.max_size < cargs.vec_pkt_size) { + ODPH_ERR("Vector size larger than max size supported by vector pool.\n"); + exit(EXIT_FAILURE); + } + + if (!cargs.schedule && !cargs.poll) { + ODPH_ERR("Vector packet is not supported with sync APIs.\n"); + exit(EXIT_FAILURE); + } + + /* Create vector pool */ + odp_pool_param_init(¶m); + param.vector.num = POOL_NUM_PKT; + param.vector.max_size = cargs.vec_pkt_size; + param.type = ODP_POOL_VECTOR; + vec_pool = odp_pool_create("vector_pool", ¶m); + + if (vec_pool == ODP_POOL_INVALID) { + ODPH_ERR("Vector packet pool create failed.\n"); + exit(EXIT_FAILURE); + } + + odp_pool_print(vec_pool); + } + odp_ipsec_config_init(&config); config.max_num_sa = 2; config.inbound.chksums.all_chksum = 0; config.outbound.all_chksum = 0; + if (vec_pool != ODP_POOL_INVALID) { + config.vector.enable = true; + config.vector.pool = vec_pool; + config.vector.max_size = cargs.vec_pkt_size; + config.vector.max_tmo_ns = ipsec_capa.vector.max_tmo_ns; + } + odp_queue_param_init(&qparam); if (cargs.schedule) { odp_schedule_config(NULL); @@ -1196,6 +1393,14 @@ int main(int argc, char *argv[]) if (cargs.schedule || cargs.poll) odp_queue_destroy(out_queue); + + if (cargs.vec_pkt_size) { + if (odp_pool_destroy(vec_pool)) { + ODPH_ERR("Error: vector pool destroy\n"); + exit(EXIT_FAILURE); + } + } + if (odp_pool_destroy(pool)) { ODPH_ERR("Error: pool destroy\n"); exit(EXIT_FAILURE); diff --git a/test/performance/odp_ipsec_run.sh b/test/performance/odp_ipsec_run.sh index 1974a7c55..2ddb48d07 100755 --- a/test/performance/odp_ipsec_run.sh +++ b/test/performance/odp_ipsec_run.sh @@ -7,9 +7,9 @@ TEST_DIR="${TEST_DIR:-$(dirname $0)}" -# Run with a small number of iterations in make check +# Run with a small number of packets in make check -$TEST_DIR/odp_ipsec${EXEEXT} -i 100 +$TEST_DIR/odp_ipsec${EXEEXT} -c 100 if [ $? -ne 0 ] ; then echo Test FAILED diff --git a/test/performance/odp_pktio_ordered.c b/test/performance/odp_pktio_ordered.c index 37a0899b6..130eb4688 100644 --- a/test/performance/odp_pktio_ordered.c +++ b/test/performance/odp_pktio_ordered.c @@ -1258,7 +1258,7 @@ int main(int argc, char *argv[]) exit(EXIT_FAILURE); } - if ((unsigned)gbl_args->appl.num_flows > capa.max_output_queues) + if ((uint32_t)gbl_args->appl.num_flows > capa.max_output_queues) gbl_args->appl.num_flows = capa.max_output_queues; } diff --git a/test/performance/odp_random.c b/test/performance/odp_random.c index 622a58c65..46134ac0c 100644 --- a/test/performance/odp_random.c +++ b/test/performance/odp_random.c @@ -14,20 +14,52 @@ #include <odp_api.h> #include <odp/helper/odph_api.h> +#define PSEUDO_RANDOM (-1) + #define MB (1024ull * 1024ull) +typedef struct test_global_t test_global_t; + +typedef struct thread_arg_t { + test_global_t *global; + int thread_idx; + uint8_t *data; + +} thread_arg_t; + +struct test_global_t { + odp_barrier_t barrier; + odp_random_kind_t type; + uint8_t *data; + uint32_t rounds; + + thread_arg_t thread_arg[ODP_THREAD_COUNT_MAX]; + + struct { + uint64_t nsec[ODP_THREAD_COUNT_MAX]; + uint64_t sum[ODP_THREAD_COUNT_MAX]; + uint64_t min[ODP_THREAD_COUNT_MAX]; + uint64_t max[ODP_THREAD_COUNT_MAX]; + } stat; +}; + /* Command line options */ typedef struct { + int mode; int num_threads; uint32_t size; uint32_t rounds; + uint64_t delay; + } options_t; static options_t options; static const options_t options_def = { + .mode = 0, .num_threads = 1, .size = 256, .rounds = 100000, + .delay = 0, }; static void print_usage(void) @@ -37,13 +69,18 @@ static void print_usage(void) "\n" "Usage: odp_random [options]\n" "\n" + " -m, --mode Test mode select (default: 0):\n" + " 0: Data throughput\n" + " 1: Data generation latency (size: 8B by default)\n" " -t, --threads Number of worker threads (default %u)\n" " -s, --size Size of buffer in bytes (default %u)\n" " -r, --rounds Number of test rounds (default %u)\n" " Divided by 100 for ODP_RANDOM_TRUE\n" + " -d, --delay Delay (nsec) between buffer fills (default %" PRIu64 ").\n" + " Affects only latency mode.\n" " -h, --help This help\n" "\n", - options_def.num_threads, options_def.size, options_def.rounds); + options_def.num_threads, options_def.size, options_def.rounds, options_def.delay); } static int parse_options(int argc, char *argv[]) @@ -53,16 +90,19 @@ static int parse_options(int argc, char *argv[]) int ret = 0; static const struct option longopts[] = { + { "mode", required_argument, NULL, 'm' }, { "threads", required_argument, NULL, 't' }, { "size", required_argument, NULL, 's' }, { "rounds", required_argument, NULL, 'r' }, + { "delay", required_argument, NULL, 'd' }, { "help", no_argument, NULL, 'h' }, { NULL, 0, NULL, 0 } }; - static const char *shortopts = "+t:s:r:h"; + static const char *shortopts = "+m:t:s:r:d:h"; options = options_def; + options.size = 0; while (1) { opt = getopt_long(argc, argv, shortopts, longopts, &long_index); @@ -71,6 +111,9 @@ static int parse_options(int argc, char *argv[]) break; switch (opt) { + case 'm': + options.mode = atoi(optarg); + break; case 't': options.num_threads = atol(optarg); break; @@ -80,6 +123,9 @@ static int parse_options(int argc, char *argv[]) case 'r': options.rounds = atol(optarg); break; + case 'd': + options.delay = atol(optarg); + break; case 'h': /* fall through */ default: @@ -89,170 +135,216 @@ static int parse_options(int argc, char *argv[]) } } - if (options.size < 1) { - ODPH_ERR("Invalid size: %" PRIu32 "\n", options.size); + if (options.num_threads < 1 || options.num_threads > ODP_THREAD_COUNT_MAX) { + ODPH_ERR("Bad number of threads: %i\n", options.num_threads); return -1; } + if (options.size == 0) { + options.size = options_def.size; + + if (options.mode) + options.size = 8; + } + + printf("\nOptions:\n"); + printf("------------------------\n"); + printf(" mode: %i\n", options.mode); + printf(" threads: %i\n", options.num_threads); + printf(" size: %u\n", options.size); + printf(" rounds: %u\n", options.rounds); + printf(" delay: %" PRIu64 "\n", options.delay); + printf("\n"); + return ret; } -const char *shm_name = "odp_random_test"; +static inline void random_data_loop(odp_random_kind_t type, uint32_t rounds, + uint8_t *data, uint32_t size) +{ + uint32_t i; + int32_t ret; -typedef struct test_shm_t { - odp_barrier_t barrier; - odp_random_kind_t type; - uint64_t nsec[ODP_THREAD_COUNT_MAX]; -} test_shm_t; + if ((int)type == PSEUDO_RANDOM) { + uint64_t seed = 0; -static test_shm_t *shm_lookup(void) -{ - test_shm_t *shm = NULL; - odp_shm_t shm_hdl = odp_shm_lookup(shm_name); + for (i = 0; i < rounds; i++) { + uint32_t pos = 0; - if (shm_hdl != ODP_SHM_INVALID) - shm = (test_shm_t *)odp_shm_addr(shm_hdl); + while (pos < size) { + ret = odp_random_test_data(data + pos, size - pos, &seed); - return shm; -} + if (ret < 0) { + ODPH_ERR("odp_random_test_data() failed\n"); + exit(EXIT_FAILURE); + } -static uint32_t type_rounds(odp_random_kind_t type) -{ - switch (type) { - case ODP_RANDOM_TRUE: - return options.rounds / 100; - default: - return options.rounds; + pos += ret; + } + } + } else { + for (i = 0; i < rounds; i++) { + uint32_t pos = 0; + + while (pos < size) { + ret = odp_random_data(data + pos, size - pos, type); + + if (ret < 0) { + ODPH_ERR("odp_random_data() failed\n"); + exit(EXIT_FAILURE); + } + + pos += ret; + } + } } } -static int test_random(void *p) +static int test_random_perf(void *ptr) { - (void)p; - - uint8_t *buf, *data; - const unsigned long page = ODP_PAGE_SIZE; odp_time_t start; uint64_t nsec; - uint32_t rounds; - test_shm_t *shm = shm_lookup(); + thread_arg_t *thread_arg = ptr; + test_global_t *global = thread_arg->global; + odp_random_kind_t type = global->type; + int thread_idx = thread_arg->thread_idx; + uint8_t *data = thread_arg->data; + uint32_t size = options.size; + uint32_t rounds = global->rounds; - if (!shm) { - ODPH_ERR("Failed to look up shm %s\n", shm_name); - exit(EXIT_FAILURE); - } + /* One warm up round */ + random_data_loop(type, 1, data, size); - rounds = type_rounds(shm->type); + odp_barrier_wait(&global->barrier); - /* One extra page for alignment. */ - buf = (uint8_t *)malloc(options.size + page); - - if (!buf) { - ODPH_ERR("Memory allocation failed.\n"); - exit(EXIT_FAILURE); - } - - /* Align to start of page. */ - data = (uint8_t *)(((uintptr_t)buf + (page - 1)) & ~(page - 1)); - - odp_barrier_wait(&shm->barrier); + /* Test run */ start = odp_time_local(); - for (uint32_t i = 0; i < rounds; i++) { - uint32_t pos = 0; - - while (pos < options.size) { - int32_t n = odp_random_data(data + pos, - options.size - pos, - shm->type); - - if (n < 0) { - ODPH_ERR("odp_random_data() failed\n"); - exit(EXIT_FAILURE); - } - - pos += n; - } - } + random_data_loop(type, rounds, data, size); nsec = odp_time_diff_ns(odp_time_local(), start); - shm->nsec[odp_thread_id()] = nsec; - free(buf); + + global->stat.nsec[thread_idx] = nsec; return 0; } -static int test_random_test(void *p) +static inline void random_data_latency(test_global_t *global, int thread_idx, + uint32_t rounds, uint8_t *data, uint32_t size) { - (void)p; - - uint8_t *buf, *data; - const unsigned long page = ODP_PAGE_SIZE; - odp_time_t start; + uint32_t i; + int32_t ret; + odp_time_t t1, t2, start; uint64_t nsec; + odp_random_kind_t type = global->type; + uint64_t delay = options.delay; + uint64_t min = UINT64_MAX; + uint64_t max = 0; + uint64_t sum = 0; uint64_t seed = 0; - uint32_t rounds; - test_shm_t *shm = shm_lookup(); - - if (!shm) { - ODPH_ERR("Failed to look up shm %s\n", shm_name); - exit(EXIT_FAILURE); - } - - rounds = type_rounds(shm->type); - - /* One extra page for alignment. */ - buf = (uint8_t *)malloc(options.size + page); - - if (!buf) { - ODPH_ERR("Memory allocation failed.\n"); - exit(EXIT_FAILURE); - } - - /* Align to start of page. */ - data = (uint8_t *)(((uintptr_t)buf + (page - 1)) & ~(page - 1)); - odp_barrier_wait(&shm->barrier); start = odp_time_local(); - for (uint32_t i = 0; i < rounds; i++) { + for (i = 0; i < rounds; i++) { uint32_t pos = 0; - while (pos < options.size) { - int32_t n = odp_random_test_data(data + pos, - options.size - pos, - &seed); + if (delay) + odp_time_wait_ns(delay); - if (n < 0) { - ODPH_ERR("odp_random_data() failed\n"); - exit(EXIT_FAILURE); - } + if ((int)type == PSEUDO_RANDOM) { + t1 = odp_time_local_strict(); + while (pos < size) { + ret = odp_random_test_data(data + pos, size - pos, &seed); + + if (ret < 0) { + ODPH_ERR("odp_random_test_data() failed\n"); + exit(EXIT_FAILURE); + } - pos += n; + pos += ret; + } + t2 = odp_time_local_strict(); + } else { + t1 = odp_time_local_strict(); + while (pos < size) { + ret = odp_random_data(data + pos, size - pos, type); + + if (ret < 0) { + ODPH_ERR("odp_random_data() failed\n"); + exit(EXIT_FAILURE); + } + + pos += ret; + } + t2 = odp_time_local_strict(); } + + nsec = odp_time_diff_ns(t2, t1); + sum += nsec; + + if (nsec > max) + max = nsec; + if (nsec < min) + min = nsec; } nsec = odp_time_diff_ns(odp_time_local(), start); - shm->nsec[odp_thread_id()] = nsec; - free(buf); + + global->stat.nsec[thread_idx] = nsec; + global->stat.sum[thread_idx] = sum; + global->stat.min[thread_idx] = min; + global->stat.max[thread_idx] = max; +} + +static int test_random_latency(void *ptr) +{ + thread_arg_t *thread_arg = ptr; + test_global_t *global = thread_arg->global; + odp_random_kind_t type = global->type; + int thread_idx = thread_arg->thread_idx; + uint8_t *data = thread_arg->data; + uint32_t size = options.size; + uint32_t rounds = global->rounds; + + /* One warm up round */ + random_data_loop(type, 1, data, size); + + odp_barrier_wait(&global->barrier); + + /* Test run */ + random_data_latency(global, thread_idx, rounds, data, size); return 0; } -static void test_type(odp_instance_t instance, test_shm_t *shm, - odp_random_kind_t type) +static uint32_t type_rounds(odp_random_kind_t type) +{ + switch (type) { + case ODP_RANDOM_TRUE: + return options.rounds / 100; + default: + return options.rounds; + } +} + +static void test_type(odp_instance_t instance, test_global_t *global, odp_random_kind_t type) { - memset(shm, 0, sizeof(test_shm_t)); - shm->type = type; - odp_barrier_init(&shm->barrier, options.num_threads); + int i; + int num_threads = options.num_threads; + uint32_t rounds = type_rounds(type); + uint32_t size = options.size; + + memset(&global->stat, 0, sizeof(global->stat)); + global->type = type; + global->rounds = rounds; + odp_barrier_init(&global->barrier, num_threads); odp_cpumask_t cpumask; odph_thread_common_param_t thr_common; - odph_thread_param_t thr_param; - odph_thread_t thr_worker[options.num_threads]; + odph_thread_param_t thr_param[num_threads]; + odph_thread_t thr_worker[num_threads]; - if (odp_cpumask_default_worker(&cpumask, options.num_threads) != - options.num_threads) { + if (odp_cpumask_default_worker(&cpumask, num_threads) != num_threads) { ODPH_ERR("Failed to get default CPU mask.\n"); exit(EXIT_FAILURE); } @@ -260,35 +352,36 @@ static void test_type(odp_instance_t instance, test_shm_t *shm, odph_thread_common_param_init(&thr_common); thr_common.instance = instance; thr_common.cpumask = &cpumask; - thr_common.share_param = 1; - odph_thread_param_init(&thr_param); - thr_param.thr_type = ODP_THREAD_WORKER; - thr_param.start = test_random; + for (i = 0; i < num_threads; i++) { + odph_thread_param_init(&thr_param[i]); + thr_param[i].thr_type = ODP_THREAD_WORKER; + thr_param[i].arg = &global->thread_arg[i]; - if (type == (odp_random_kind_t)-1) - thr_param.start = test_random_test; + if (options.mode == 0) + thr_param[i].start = test_random_perf; + else + thr_param[i].start = test_random_latency; + } memset(&thr_worker, 0, sizeof(thr_worker)); - if (odph_thread_create(thr_worker, &thr_common, &thr_param, - options.num_threads) != options.num_threads) { + if (odph_thread_create(thr_worker, &thr_common, thr_param, num_threads) != num_threads) { ODPH_ERR("Failed to create worker threads.\n"); exit(EXIT_FAILURE); } - if (odph_thread_join(thr_worker, options.num_threads) != - options.num_threads) { + if (odph_thread_join(thr_worker, num_threads) != num_threads) { ODPH_ERR("Failed to join worker threads.\n"); exit(EXIT_FAILURE); } double mb, seconds, nsec = 0; - for (int i = 0; i < ODP_THREAD_COUNT_MAX; i++) - nsec += shm->nsec[i]; + for (i = 0; i < num_threads; i++) + nsec += global->stat.nsec[i]; - nsec /= options.num_threads; + nsec /= num_threads; switch (type) { case ODP_RANDOM_BASIC: @@ -304,25 +397,53 @@ static void test_type(odp_instance_t instance, test_shm_t *shm, printf("odp_random_test_data\n"); } - uint32_t rounds = type_rounds(type); - printf("--------------------\n"); - printf("threads: %d size: %u B rounds: %u ", options.num_threads, - options.size, rounds); - mb = (uint64_t)options.num_threads * (uint64_t)options.size * - (uint64_t)rounds; + printf("threads: %d size: %u B rounds: %u ", num_threads, size, rounds); + mb = (uint64_t)num_threads * (uint64_t)size * (uint64_t)rounds; mb /= MB; seconds = (double)nsec / (double)ODP_TIME_SEC_IN_NS; printf("MB: %.3f seconds: %.3f ", mb, seconds); printf("MB/s: %.3f ", mb / seconds); - printf("MB/s/thread: %.3f", mb / seconds / (double)options.num_threads); - printf("\n\n"); + printf("MB/s/thread: %.3f\n", mb / seconds / (double)num_threads); + + if (options.mode) { + double ave; + uint64_t min = UINT64_MAX; + uint64_t max = 0; + uint64_t sum = 0; + + printf(" latency (nsec)\n"); + printf(" thread min max ave\n"); + for (i = 0; i < num_threads; i++) { + ave = (double)global->stat.sum[i] / rounds; + sum += global->stat.sum[i]; + + if (global->stat.min[i] < min) + min = global->stat.min[i]; + + if (global->stat.max[i] > max) + max = global->stat.max[i]; + + printf("%8i %8" PRIu64 " %8" PRIu64 " %10.1f\n", i, global->stat.min[i], + global->stat.max[i], ave); + } + + printf(" all %8" PRIu64 " %8" PRIu64 " %10.1f\n", + min, max, ((double)sum / rounds) / num_threads); + } + + printf("\n"); } int main(int argc, char **argv) { odp_instance_t instance; odp_init_t init; + odp_shm_t shm_glb, shm_data; + test_global_t *global; + int num_threads, i; + uint64_t tot_size, size; + uint8_t *addr; if (parse_options(argc, argv)) exit(EXIT_FAILURE); @@ -352,31 +473,59 @@ int main(int argc, char **argv) odp_sys_info_print(); - test_shm_t *shm = NULL; - odp_shm_t shm_hdl = odp_shm_reserve(shm_name, sizeof(test_shm_t), 64, - 0); + global = NULL; + shm_glb = odp_shm_reserve("test_globals", sizeof(test_global_t), ODP_CACHE_LINE_SIZE, 0); + + if (shm_glb != ODP_SHM_INVALID) + global = (test_global_t *)odp_shm_addr(shm_glb); + + if (!global) { + ODPH_ERR("Failed to reserve shm\n"); + exit(EXIT_FAILURE); + } + + memset(global, 0, sizeof(test_global_t)); + + num_threads = options.num_threads; + addr = NULL; + size = ODP_CACHE_LINE_SIZE + ODP_CACHE_LINE_ROUNDUP(options.size); + tot_size = num_threads * size; + shm_data = odp_shm_reserve("test_data", tot_size, ODP_CACHE_LINE_SIZE, 0); - if (shm_hdl != ODP_SHM_INVALID) - shm = (test_shm_t *)odp_shm_addr(shm_hdl); + if (shm_data != ODP_SHM_INVALID) + addr = odp_shm_addr(shm_data); - if (!shm) { - ODPH_ERR("Failed to reserve shm %s\n", shm_name); + if (!addr) { + ODPH_ERR("Failed to reserve shm: size %" PRIu64 " bytes\n", tot_size); exit(EXIT_FAILURE); } + for (i = 0; i < num_threads; i++) { + global->thread_arg[i].global = global; + global->thread_arg[i].thread_idx = i; + global->thread_arg[i].data = addr + i * size; + } + + odp_shm_print_all(); + switch (odp_random_max_kind()) { case ODP_RANDOM_TRUE: - test_type(instance, shm, ODP_RANDOM_TRUE); + test_type(instance, global, ODP_RANDOM_TRUE); /* fall through */ case ODP_RANDOM_CRYPTO: - test_type(instance, shm, ODP_RANDOM_CRYPTO); + test_type(instance, global, ODP_RANDOM_CRYPTO); /* fall through */ default: - test_type(instance, shm, ODP_RANDOM_BASIC); - test_type(instance, shm, -1); + test_type(instance, global, ODP_RANDOM_BASIC); + test_type(instance, global, PSEUDO_RANDOM); + } + + if (odp_shm_free(shm_data)) { + ODPH_ERR("odp_shm_free() failed\n"); + exit(EXIT_FAILURE); } - if (odp_shm_free(shm_hdl)) { + if (odp_shm_free(shm_glb)) { ODPH_ERR("odp_shm_free() failed\n"); exit(EXIT_FAILURE); } diff --git a/test/performance/odp_sched_pktio.c b/test/performance/odp_sched_pktio.c index 77c1d260f..11be9d367 100644 --- a/test/performance/odp_sched_pktio.c +++ b/test/performance/odp_sched_pktio.c @@ -862,10 +862,9 @@ static int open_pktios(test_global_t *test_global) odp_pktin_queue_param_t pktin_param; odp_pktout_queue_param_t pktout_param; odp_schedule_sync_t sched_sync; - unsigned int num_queue; + uint32_t num_queue, j; char *name; int i, num_pktio, ret; - unsigned int j; num_pktio = test_global->opt.num_pktio; num_queue = test_global->opt.num_pktio_queue; diff --git a/test/validation/api/classification/classification.h b/test/validation/api/classification/classification.h index 4c8613555..70dcc6230 100644 --- a/test/validation/api/classification/classification.h +++ b/test/validation/api/classification/classification.h @@ -23,6 +23,11 @@ #define CLS_DEFAULT_SMAC {0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c} #define CLS_MAGIC_VAL 0xdeadbeef +/* Config values for Drop CoS */ +#define TEST_DROP 1 +#define CLS_DROP 6 +#define CLS_DROP_PORT 4001 + /* Config values for Error CoS */ #define TEST_ERROR 1 #define CLS_ERROR 1 @@ -47,7 +52,7 @@ /* Config values for CoS L2 Priority */ #define TEST_L2_QOS 1 -#define CLS_L2_QOS_0 6 +#define CLS_L2_QOS_0 7 #define CLS_L2_QOS_MAX 5 #define CLS_ENTRIES (CLS_L2_QOS_0 + CLS_L2_QOS_MAX) diff --git a/test/validation/api/classification/odp_classification_basic.c b/test/validation/api/classification/odp_classification_basic.c index 3aeb89462..fb5ec4ed0 100644 --- a/test/validation/api/classification/odp_classification_basic.c +++ b/test/validation/api/classification/odp_classification_basic.c @@ -52,6 +52,49 @@ static void classification_test_create_cos(void) odp_queue_destroy(queue); } +static void classification_test_create_cos_max_common(odp_bool_t stats) +{ + uint32_t i; + odp_cls_cos_param_t cls_param; + odp_cls_capability_t capa; + + CU_ASSERT_FATAL(odp_cls_capability(&capa) == 0); + + uint32_t num = capa.max_cos; + + if (stats && capa.max_cos_stats < num) + num = capa.max_cos_stats; + + odp_cos_t cos[num]; + + for (i = 0; i < num; i++) { + odp_cls_cos_param_init(&cls_param); + cls_param.action = ODP_COS_ACTION_DROP; + cls_param.stats_enable = stats; + + cos[i] = odp_cls_cos_create(NULL, &cls_param); + if (cos[i] == ODP_COS_INVALID) { + ODPH_ERR("odp_cls_cos_create() failed at CoS %u out of %u.\n", i + 1, num); + break; + } + } + + CU_ASSERT(i == num); + + for (uint32_t j = 0; j < i; j++) + CU_ASSERT(!odp_cos_destroy(cos[j])); +} + +static void classification_test_create_cos_max(void) +{ + classification_test_create_cos_max_common(false); +} + +static void classification_test_create_cos_max_stats(void) +{ + classification_test_create_cos_max_common(true); +} + static void classification_test_destroy_cos(void) { odp_cos_t cos; @@ -345,6 +388,8 @@ static void classification_test_pmr_composite_create(void) odp_testinfo_t classification_suite_basic[] = { ODP_TEST_INFO(classification_test_default_values), ODP_TEST_INFO(classification_test_create_cos), + ODP_TEST_INFO(classification_test_create_cos_max), + ODP_TEST_INFO(classification_test_create_cos_max_stats), ODP_TEST_INFO(classification_test_destroy_cos), ODP_TEST_INFO(classification_test_create_pmr_match), ODP_TEST_INFO(classification_test_cos_set_queue), diff --git a/test/validation/api/classification/odp_classification_tests.c b/test/validation/api/classification/odp_classification_tests.c index 41b4ab02f..4511fc1d7 100644 --- a/test/validation/api/classification/odp_classification_tests.c +++ b/test/validation/api/classification/odp_classification_tests.c @@ -23,6 +23,7 @@ static int global_num_l2_qos; #define NUM_COS_PMR_CHAIN 2 #define NUM_COS_DEFAULT 1 +#define NUM_COS_DROP 1 #define NUM_COS_ERROR 1 #define NUM_COS_L2_PRIO CLS_L2_QOS_MAX #define NUM_COS_PMR 1 @@ -453,6 +454,74 @@ void test_pktio_default_cos(odp_bool_t enable_pktv) odp_packet_free(pkt); } +void configure_pktio_drop_cos(odp_bool_t enable_pktv, uint32_t max_cos_stats) +{ + uint16_t val; + uint16_t mask; + odp_pmr_param_t pmr_param; + odp_cls_cos_param_t cls_param; + char cosname[ODP_COS_NAME_LEN]; + + sprintf(cosname, "DropCoS"); + odp_cls_cos_param_init(&cls_param); + + cls_param.action = ODP_COS_ACTION_DROP; + cls_param.stats_enable = max_cos_stats > 0; + + if (enable_pktv) { + cls_param.vector.enable = true; + cls_param.vector.pool = pktv_config.pool; + cls_param.vector.max_size = pktv_config.max_size; + cls_param.vector.max_tmo_ns = pktv_config.max_tmo_ns; + } + + cos_list[CLS_DROP] = odp_cls_cos_create(cosname, &cls_param); + CU_ASSERT_FATAL(cos_list[CLS_DROP] != ODP_COS_INVALID); + + val = odp_cpu_to_be_16(CLS_DROP_PORT); + mask = odp_cpu_to_be_16(0xffff); + odp_cls_pmr_param_init(&pmr_param); + pmr_param.term = find_first_supported_l3_pmr(); + pmr_param.match.value = &val; + pmr_param.match.mask = &mask; + pmr_param.val_sz = sizeof(val); + + pmr_list[CLS_DROP] = odp_cls_pmr_create(&pmr_param, 1, + cos_list[CLS_DEFAULT], + cos_list[CLS_DROP]); + CU_ASSERT_FATAL(pmr_list[CLS_DROP] != ODP_PMR_INVALID); +} + +void test_pktio_drop_cos(odp_bool_t enable_pktv) +{ + odp_packet_t pkt; + odp_queue_t queue; + uint32_t seqno = 0; + cls_packet_info_t pkt_info; + odp_cls_capability_t capa; + odp_cls_cos_stats_t start, stop; + + CU_ASSERT_FATAL(odp_cls_capability(&capa) == 0); + pkt_info = default_pkt_info; + pkt_info.l4_type = CLS_PKT_L4_UDP; + pkt = create_packet(pkt_info); + CU_ASSERT_FATAL(pkt != ODP_PACKET_INVALID); + seqno = cls_pkt_get_seq(pkt); + CU_ASSERT(seqno != TEST_SEQ_INVALID); + set_first_supported_pmr_port(pkt, CLS_DROP_PORT); + CU_ASSERT(odp_cls_cos_stats(cos_list[CLS_DROP], &start) == 0); + enqueue_pktio_interface(pkt, pktio_loop); + pkt = receive_packet(&queue, ODP_TIME_SEC_IN_NS, enable_pktv); + CU_ASSERT(odp_cls_cos_stats(cos_list[CLS_DROP], &stop) == 0); + CU_ASSERT_FATAL(pkt == ODP_PACKET_INVALID); + if (capa.stats.cos.counter.packets) + CU_ASSERT((stop.packets - start.packets) == 1); + if (capa.stats.cos.counter.discards) + CU_ASSERT((stop.discards - start.discards) == 0); + if (capa.stats.cos.counter.errors) + CU_ASSERT((stop.errors - start.errors) == 0); +} + static int classification_check_queue_stats(void) { odp_cls_capability_t capa; @@ -926,6 +995,11 @@ static void classification_test_pktio_configure_common(odp_bool_t enable_pktv) tc.default_cos = 1; num_cos -= NUM_COS_DEFAULT; } + if (num_cos >= NUM_COS_DEFAULT && TEST_DROP) { + configure_pktio_drop_cos(enable_pktv, capa.max_cos_stats); + tc.drop_cos = 1; + num_cos -= NUM_COS_DROP; + } if (num_cos >= NUM_COS_ERROR && TEST_ERROR) { configure_pktio_error_cos(enable_pktv); tc.error_cos = 1; @@ -969,6 +1043,8 @@ static void classification_test_pktio_test_common(odp_bool_t enable_pktv) /* Test Different CoS on the pktio interface */ if (tc.default_cos && TEST_DEFAULT) test_pktio_default_cos(enable_pktv); + if (tc.drop_cos && TEST_DROP) + test_pktio_drop_cos(enable_pktv); if (tc.error_cos && TEST_ERROR) test_pktio_error_cos(enable_pktv); if (tc.pmr_chain && TEST_PMR_CHAIN) diff --git a/test/validation/api/classification/odp_classification_testsuites.h b/test/validation/api/classification/odp_classification_testsuites.h index 2b5da94e2..6b00e138b 100644 --- a/test/validation/api/classification/odp_classification_testsuites.h +++ b/test/validation/api/classification/odp_classification_testsuites.h @@ -34,6 +34,7 @@ typedef struct cls_packet_info { typedef union odp_cls_testcase { struct { uint32_t default_cos:1; + uint32_t drop_cos:1; uint32_t error_cos:1; uint32_t pmr_chain:1; uint32_t l2_priority:1; @@ -72,6 +73,8 @@ odp_pool_t pktv_pool_create(const char *poolname); odp_queue_t queue_create(const char *queuename, bool sched); void configure_pktio_default_cos(odp_bool_t enable_pktv); void test_pktio_default_cos(odp_bool_t enable_pktv); +void configure_pktio_drop_cos(odp_bool_t enable_pktv, uint32_t max_cos_stats); +void test_pktio_drop_cos(odp_bool_t enable_pktv); void configure_pktio_error_cos(odp_bool_t enable_pktv); void test_pktio_error_cos(odp_bool_t enable_pktv); void configure_cls_pmr_chain(odp_bool_t enable_pktv); diff --git a/test/validation/api/crypto/odp_crypto_test_inp.c b/test/validation/api/crypto/odp_crypto_test_inp.c index 721dfe8c9..97f721dd5 100644 --- a/test/validation/api/crypto/odp_crypto_test_inp.c +++ b/test/validation/api/crypto/odp_crypto_test_inp.c @@ -37,7 +37,9 @@ static void test_default_values(void) CU_ASSERT_EQUAL(param.op, ODP_CRYPTO_OP_ENCODE); CU_ASSERT_EQUAL(param.auth_cipher_text, false); +#if ODP_DEPRECATED_API CU_ASSERT_EQUAL(param.pref_mode, ODP_CRYPTO_SYNC); +#endif CU_ASSERT_EQUAL(param.op_mode, ODP_CRYPTO_SYNC); CU_ASSERT_EQUAL(param.cipher_alg, ODP_CIPHER_ALG_NULL); CU_ASSERT_EQUAL(param.cipher_iv_len, 0); @@ -194,6 +196,7 @@ static const char *cipher_alg_name(odp_cipher_alg_t cipher) } } +#if ODP_DEPRECATED_API static int alg_op(odp_packet_t pkt, odp_bool_t *ok, odp_crypto_session_t session, @@ -273,6 +276,7 @@ static int alg_op(odp_packet_t pkt, return 0; } +#endif static int alg_packet_op(odp_packet_t pkt, odp_bool_t *ok, @@ -370,10 +374,14 @@ static int crypto_op(odp_packet_t pkt, int rc; if (!suite_context.packet) +#if ODP_DEPRECATED_API rc = alg_op(pkt, ok, session, cipher_iv, auth_iv, cipher_range, auth_range, aad, hash_result_offset); +#else + rc = -1; +#endif else rc = alg_packet_op(pkt, ok, session, cipher_iv, auth_iv, @@ -657,7 +665,9 @@ static odp_crypto_session_t session_create(odp_crypto_op_t op, ses_params.op = op; ses_params.auth_cipher_text = false; ses_params.op_mode = suite_context.op_mode; +#if ODP_DEPRECATED_API ses_params.pref_mode = suite_context.pref_mode; +#endif ses_params.cipher_alg = cipher_alg; ses_params.auth_alg = auth_alg; ses_params.compl_queue = suite_context.queue; @@ -2154,6 +2164,7 @@ static odp_event_t plain_compl_queue_deq(void) return odp_queue_deq(suite_context.queue); } +#if ODP_DEPRECATED_API static int crypto_suite_sync_init(void) { suite_context.pool = odp_pool_lookup("packet_pool"); @@ -2206,6 +2217,7 @@ static int crypto_suite_async_sched_init(void) return 0; } +#endif static int crypto_suite_packet_sync_init(void) { @@ -2422,12 +2434,14 @@ odp_testinfo_t crypto_suite[] = { }; odp_suiteinfo_t crypto_suites[] = { +#if ODP_DEPRECATED_API {"odp_crypto_sync_inp", crypto_suite_sync_init, NULL, crypto_suite}, {"odp_crypto_async_plain_inp", crypto_suite_async_plain_init, crypto_suite_term, crypto_suite}, {"odp_crypto_async_sched_inp", crypto_suite_async_sched_init, crypto_suite_term, crypto_suite}, +#endif {"odp_crypto_packet_sync_inp", crypto_suite_packet_sync_init, NULL, crypto_suite}, {"odp_crypto_packet_async_plain_inp", diff --git a/test/validation/api/ipsec/ipsec.c b/test/validation/api/ipsec/ipsec.c index 88f204ea4..d12234a5a 100644 --- a/test/validation/api/ipsec/ipsec.c +++ b/test/validation/api/ipsec/ipsec.c @@ -281,10 +281,6 @@ int ipsec_check_esp_chacha20_poly1305(void) int ipsec_check_test_sa_update_seq_num(void) { - odp_ipsec_capability_t capa; - - odp_ipsec_capability(&capa); - if (!capa.test.sa_operations.seq_num) return ODP_TEST_INACTIVE; @@ -942,9 +938,8 @@ static void verify_in(const ipsec_test_part *part, result.orig_ip_len == len); } } - ipsec_check_packet(part->out[i].pkt_res, - pkto[i], - false); + if (part->out[i].l3_type != ODP_PROTO_L3_TYPE_NONE) + ipsec_check_packet(part->out[i].pkt_res, pkto[i], false); if (suite_context.inbound_op_mode == ODP_IPSEC_OP_MODE_INLINE) expected_user_ptr = NULL; CU_ASSERT(odp_packet_user_ptr(pkto[i]) == expected_user_ptr); diff --git a/test/validation/api/pktio/pktio.c b/test/validation/api/pktio/pktio.c index 9144e1260..f6408c788 100644 --- a/test/validation/api/pktio/pktio.c +++ b/test/validation/api/pktio/pktio.c @@ -770,8 +770,8 @@ static int recv_packets_tmo(odp_pktio_t pktio, odp_packet_t pkt_tbl[], int num_q; int i; int n; - unsigned int from_val = 0; - unsigned *from = NULL; + uint32_t from_val = 0; + uint32_t *from = NULL; if (mode == RECV_MQ_TMO) from = &from_val; @@ -787,8 +787,7 @@ static int recv_packets_tmo(odp_pktio_t pktio, odp_packet_t pkt_tbl[], n = odp_pktin_recv_tmo(pktin[0], pkt_tmp, num - num_rx, tmo); else - n = odp_pktin_recv_mq_tmo(pktin, (unsigned)num_q, - from, pkt_tmp, + n = odp_pktin_recv_mq_tmo(pktin, (uint32_t)num_q, from, pkt_tmp, num - num_rx, tmo); ts2 = odp_time_global(); @@ -812,7 +811,7 @@ static int recv_packets_tmo(odp_pktio_t pktio, odp_packet_t pkt_tbl[], odp_packet_free(pkt_tmp[i]); } if (mode == RECV_MQ_TMO) - CU_ASSERT(from_val < (unsigned)num_q); + CU_ASSERT(from_val < (uint32_t)num_q); } while (num_rx < num); if (num_rx < num) { @@ -1265,7 +1264,7 @@ static void test_recv_tmo(recv_tmo_mode_e mode) odp_packet_t pkt_tbl[test_pkt_count]; uint32_t pkt_seq[test_pkt_count]; uint64_t ns; - unsigned num_q; + uint32_t num_q; int ret; int i; diff --git a/test/validation/api/traffic_mngr/traffic_mngr.c b/test/validation/api/traffic_mngr/traffic_mngr.c index cab1fd06d..116c9ed6d 100644 --- a/test/validation/api/traffic_mngr/traffic_mngr.c +++ b/test/validation/api/traffic_mngr/traffic_mngr.c @@ -120,6 +120,12 @@ typedef enum { SHAPER_PROFILE, SCHED_PROFILE, THRESHOLD_PROFILE, WRED_PROFILE } profile_kind_t; +typedef enum { + THRESHOLD_BYTE, + THRESHOLD_PACKET, + THRESHOLD_BYTE_AND_PACKET +} threshold_type_t; + typedef struct { uint32_t num_queues; odp_tm_queue_t tm_queues[]; @@ -516,6 +522,7 @@ static int open_pktios(void) uint32_t iface; char pool_name[ODP_POOL_NAME_LEN]; int rc, ret; + int pkt_aging = 0; int lso = 0; odp_pool_param_init(&pool_param); @@ -620,6 +627,12 @@ static int open_pktios(void) odp_pktio_config_init(&pktio_config); + /* Enable packet aging if supported */ + if (xmt_pktio_capa.max_tx_aging_tmo_ns) { + pkt_aging = 1; + pktio_config.pktout.bit.aging_ena = 1; + } + /* Enable LSO if supported */ if (xmt_pktio_capa.lso.max_profiles && xmt_pktio_capa.lso.max_profiles_per_pktio) { lso = 1; @@ -627,7 +640,7 @@ static int open_pktios(void) } /* Enable selected features */ - if (lso) { + if (lso || pkt_aging) { if (odp_pktio_config(xmt_pktio, &pktio_config)) { ODPH_ERR("pktio configure failed\n"); return -1; @@ -1818,11 +1831,15 @@ set_reqs_based_on_capas(odp_tm_requirements_t *req) req->tm_queue_dual_slope_needed = true; if (tm_capabilities.vlan_marking_supported) req->vlan_marking_needed = true; - if (tm_capabilities.tm_queue_threshold) + if (tm_capabilities.tm_queue_threshold.byte || + tm_capabilities.tm_queue_threshold.packet || + tm_capabilities.tm_queue_threshold.byte_and_packet) req->tm_queue_threshold_needed = true; for (j = 0; j < tm_capabilities.max_levels; j++) { - if (tm_capabilities.per_level[j].tm_node_threshold) + if (tm_capabilities.per_level[j].tm_node_threshold.byte || + tm_capabilities.per_level[j].tm_node_threshold.packet || + tm_capabilities.per_level[j].tm_node_threshold.byte_and_packet) req->per_level[j].tm_node_threshold_needed = true; } @@ -2638,7 +2655,8 @@ static void traffic_mngr_test_sched_profile(void) } static void check_threshold_profile(char *threshold_name, - uint32_t threshold_idx) + uint32_t threshold_idx, + threshold_type_t threshold) { odp_tm_threshold_params_t threshold_params; odp_tm_threshold_t profile; @@ -2657,15 +2675,17 @@ static void check_threshold_profile(char *threshold_name, if (ret) return; - CU_ASSERT(threshold_params.max_pkts == - threshold_idx * MIN_PKT_THRESHOLD); - CU_ASSERT(threshold_params.max_bytes == - threshold_idx * MIN_BYTE_THRESHOLD); - CU_ASSERT(threshold_params.enable_max_pkts == 1); - CU_ASSERT(threshold_params.enable_max_bytes == 1); + if (threshold == THRESHOLD_PACKET || threshold == THRESHOLD_BYTE_AND_PACKET) { + CU_ASSERT(threshold_params.enable_max_pkts == 1); + CU_ASSERT(threshold_params.max_pkts == threshold_idx * MIN_PKT_THRESHOLD); + } + if (threshold == THRESHOLD_BYTE || threshold == THRESHOLD_BYTE_AND_PACKET) { + CU_ASSERT(threshold_params.enable_max_bytes == 1); + CU_ASSERT(threshold_params.max_bytes == threshold_idx * MIN_BYTE_THRESHOLD); + } } -static void traffic_mngr_test_threshold_profile(void) +static void traffic_mngr_test_threshold_profile(threshold_type_t threshold) { odp_tm_threshold_params_t threshold_params; odp_tm_threshold_t profile; @@ -2673,14 +2693,19 @@ static void traffic_mngr_test_threshold_profile(void) char threshold_name[TM_NAME_LEN]; odp_tm_threshold_params_init(&threshold_params); - threshold_params.enable_max_pkts = 1; - threshold_params.enable_max_bytes = 1; + + if (threshold == THRESHOLD_PACKET || threshold == THRESHOLD_BYTE_AND_PACKET) + threshold_params.enable_max_pkts = 1; + if (threshold == THRESHOLD_BYTE || threshold == THRESHOLD_BYTE_AND_PACKET) + threshold_params.enable_max_bytes = 1; for (idx = 1; idx <= NUM_THRESH_TEST_PROFILES; idx++) { snprintf(threshold_name, sizeof(threshold_name), "threshold_profile_%" PRIu32, idx); - threshold_params.max_pkts = idx * MIN_PKT_THRESHOLD; - threshold_params.max_bytes = idx * MIN_BYTE_THRESHOLD; + if (threshold == THRESHOLD_PACKET || threshold == THRESHOLD_BYTE_AND_PACKET) + threshold_params.max_pkts = idx * MIN_PKT_THRESHOLD; + if (threshold == THRESHOLD_BYTE || threshold == THRESHOLD_BYTE_AND_PACKET) + threshold_params.max_bytes = idx * MIN_BYTE_THRESHOLD; profile = odp_tm_threshold_create(threshold_name, &threshold_params); @@ -2702,10 +2727,30 @@ static void traffic_mngr_test_threshold_profile(void) threshold_idx = ((3 + 7 * idx) % NUM_THRESH_TEST_PROFILES) + 1; snprintf(threshold_name, sizeof(threshold_name), "threshold_profile_%" PRIu32, threshold_idx); - check_threshold_profile(threshold_name, threshold_idx); + check_threshold_profile(threshold_name, threshold_idx, threshold); + } + + for (i = 0; i < NUM_THRESH_TEST_PROFILES; i++) { + CU_ASSERT(odp_tm_threshold_destroy(threshold_profiles[i]) == 0); + num_threshold_profiles--; } } +static void traffic_mngr_test_threshold_profile_byte(void) +{ + traffic_mngr_test_threshold_profile(THRESHOLD_BYTE); +} + +static void traffic_mngr_test_threshold_profile_packet(void) +{ + traffic_mngr_test_threshold_profile(THRESHOLD_PACKET); +} + +static void traffic_mngr_test_threshold_profile_byte_and_packet(void) +{ + traffic_mngr_test_threshold_profile(THRESHOLD_BYTE_AND_PACKET); +} + static void check_wred_profile(char *wred_name, uint32_t wred_idx, uint32_t color) @@ -3353,6 +3398,7 @@ static int test_sched_wfq(const char *sched_base_name, if (FANIN_RATIO <= fanin) fanin = 0; } + CU_ASSERT(pkts_sent == pkt_cnt + 4); busy_wait(1000000); /* wait 1 millisecond */ @@ -3433,7 +3479,8 @@ static int test_threshold(const char *threshold_name, odp_tm_threshold_params_t threshold_params; odp_tm_queue_t tm_queue; pkt_info_t pkt_info; - uint32_t num_pkts, pkt_len, pkts_sent; + uint32_t pkt_len, pkts_sent; + uint32_t num_pkts = 0; odp_tm_threshold_params_init(&threshold_params); if (max_pkts != 0) { @@ -3442,16 +3489,19 @@ static int test_threshold(const char *threshold_name, threshold_params.enable_max_pkts = true; num_pkts = 2 * max_pkts; pkt_len = 256; - } else if (max_bytes != 0) { + } + + if (max_bytes != 0) { max_bytes = MIN(max_bytes, MAX_PKTS * MAX_PAYLOAD / 3); threshold_params.max_bytes = max_bytes; threshold_params.enable_max_bytes = true; num_pkts = 2 * max_bytes / MAX_PAYLOAD; pkt_len = MAX_PAYLOAD; - } else { - return -1; } + if (max_pkts == 0 && max_bytes == 0) + return -1; + /* Pick a tm_queue and set the tm_queue's threshold profile and then * send in twice the amount of traffic as suggested by the thresholds * and make sure at least SOME pkts get dropped. */ @@ -4366,6 +4416,40 @@ static int test_fanin_info(const char *node_name) return walk_tree_backwards(node_desc->node); } +static void test_packet_aging(uint64_t tmo_ns, uint32_t pkt_len, odp_bool_t is_dropping) +{ + odp_tm_queue_t tm_queue; + const char *node_name = "node_1_1_1"; + const char *shaper_name = "test_shaper"; + const uint64_t rate = 256 * 1000; + pkt_info_t pkt_info; + const uint16_t num_pkts = 4; + int recv_pkts; + + tm_queue = find_tm_queue(0, node_name, 0); + CU_ASSERT_FATAL(tm_queue != ODP_TM_INVALID); + init_xmt_pkts(&pkt_info); + pkt_info.drop_eligible = false; + pkt_info.pkt_class = 1; + CU_ASSERT_FATAL(make_pkts(num_pkts, pkt_len, &pkt_info) == 0); + + for (int i = 0; i < num_pkts; i++) + odp_packet_aging_tmo_set(xmt_pkts[i], tmo_ns); + + CU_ASSERT_FATAL(set_shaper(node_name, shaper_name, rate, rate) == 0); + CU_ASSERT(send_pkts(tm_queue, num_pkts) == num_pkts); + recv_pkts = receive_pkts(odp_tm_systems[0], rcv_pktin, num_pkts, MBPS); + + if (is_dropping) + CU_ASSERT(recv_pkts < num_pkts) + else + CU_ASSERT(recv_pkts == num_pkts); + + set_shaper(node_name, NULL, 0, 0); + flush_leftover_pkts(odp_tm_systems[0], rcv_pktin); + CU_ASSERT(odp_tm_is_idle(odp_tm_systems[0])); +} + static void traffic_mngr_test_default_values(void) { odp_tm_requirements_t req; @@ -4517,25 +4601,51 @@ static void traffic_mngr_test_scheduler(void) INCREASING_WEIGHTS) == 0); } -static int traffic_mngr_check_thresholds(void) +static int traffic_mngr_check_thresholds_byte(void) { - /* Check only for tm queue threshold support as - * we only test queue threshold. - */ - if (!tm_capabilities.tm_queue_threshold) + /* Check only for TM queue threshold support as we only test queue threshold. */ + if (!tm_capabilities.tm_queue_threshold.byte) return ODP_TEST_INACTIVE; return ODP_TEST_ACTIVE; } -static void traffic_mngr_test_thresholds(void) +static int traffic_mngr_check_thresholds_packet(void) { - CU_ASSERT(test_threshold("thresh_A", "shaper_A", "node_1_2_1", 0, - 16, 0) == 0); - CU_ASSERT(test_threshold("thresh_B", "shaper_B", "node_1_2_1", 1, + /* Check only for TM queue threshold support as we only test queue threshold. */ + if (!tm_capabilities.tm_queue_threshold.packet) + return ODP_TEST_INACTIVE; + + return ODP_TEST_ACTIVE; +} + +static int traffic_mngr_check_thresholds_byte_and_packet(void) +{ + /* Check only for TM queue threshold support as we only test queue threshold. */ + if (!tm_capabilities.tm_queue_threshold.byte_and_packet) + return ODP_TEST_INACTIVE; + + return ODP_TEST_ACTIVE; +} + +static void traffic_mngr_test_thresholds_byte(void) +{ + CU_ASSERT(test_threshold("thresh_byte", "shaper_B", "node_1_2_1", 1, 0, 6400) == 0); } +static void traffic_mngr_test_thresholds_packet(void) +{ + CU_ASSERT(test_threshold("thresh_packet", "shaper_A", "node_1_2_1", 0, + 16, 0) == 0); +} + +static void traffic_mngr_test_thresholds_byte_and_packet(void) +{ + CU_ASSERT(test_threshold("thresh_byte_and_packet", "shaper_A", "node_1_2_1", 0, + 16, 6400) == 0); +} + static int traffic_mngr_check_queue_stats(void) { if (tm_capabilities.queue_stats.all_counters == 0) @@ -4617,7 +4727,8 @@ static int traffic_mngr_check_wred(void) static int traffic_mngr_check_byte_wred(void) { /* Check if wred is part of created odp_tm_t capabilities */ - if (!tm_capabilities.tm_queue_wred_supported) + if (!tm_capabilities.tm_queue_wred_supported || + !tm_capabilities.tm_queue_threshold.byte) return ODP_TEST_INACTIVE; if ((tm_shaper_min_rate > 64 * 1000) || @@ -4631,7 +4742,8 @@ static int traffic_mngr_check_byte_wred(void) static int traffic_mngr_check_pkt_wred(void) { /* Check if wred is part of created odp_tm_t capabilities */ - if (!tm_capabilities.tm_queue_wred_supported) + if (!tm_capabilities.tm_queue_wred_supported || + !tm_capabilities.tm_queue_threshold.packet) return ODP_TEST_INACTIVE; if ((tm_shaper_min_rate > 64 * 1000) || @@ -4772,6 +4884,23 @@ static void traffic_mngr_test_ecn_drop_prec_marking(void) CU_ASSERT(ip_marking_tests("node_1_4_2", true, true) == 0); } +static int traffic_mngr_check_tx_aging(void) +{ + return xmt_pktio_capa.max_tx_aging_tmo_ns ? ODP_TEST_ACTIVE : ODP_TEST_INACTIVE; +} + +static void traffic_mngr_test_tx_aging_no_drop(void) +{ + /* Set very long aging tmo, packets should not be dropped due to aging */ + test_packet_aging(60000000000, 128, false); +} + +static void traffic_mngr_test_tx_aging_drop(void) +{ + /* Set very short aging tmo, there should be drops due to aging */ + test_packet_aging(10, MAX_PAYLOAD, true); +} + static void traffic_mngr_test_fanin_info(void) { CU_ASSERT(test_fanin_info("node_1") == 0); @@ -4845,16 +4974,24 @@ odp_testinfo_t traffic_mngr_suite[] = { ODP_TEST_INFO(traffic_mngr_test_tm_create), ODP_TEST_INFO(traffic_mngr_test_shaper_profile), ODP_TEST_INFO(traffic_mngr_test_sched_profile), - ODP_TEST_INFO_CONDITIONAL(traffic_mngr_test_threshold_profile, - traffic_mngr_check_thresholds), + ODP_TEST_INFO_CONDITIONAL(traffic_mngr_test_threshold_profile_byte, + traffic_mngr_check_thresholds_byte), + ODP_TEST_INFO_CONDITIONAL(traffic_mngr_test_threshold_profile_packet, + traffic_mngr_check_thresholds_packet), + ODP_TEST_INFO_CONDITIONAL(traffic_mngr_test_threshold_profile_byte_and_packet, + traffic_mngr_check_thresholds_byte_and_packet), ODP_TEST_INFO_CONDITIONAL(traffic_mngr_test_wred_profile, traffic_mngr_check_wred), ODP_TEST_INFO_CONDITIONAL(traffic_mngr_test_shaper, traffic_mngr_check_shaper), ODP_TEST_INFO_CONDITIONAL(traffic_mngr_test_scheduler, traffic_mngr_check_scheduler), - ODP_TEST_INFO_CONDITIONAL(traffic_mngr_test_thresholds, - traffic_mngr_check_thresholds), + ODP_TEST_INFO_CONDITIONAL(traffic_mngr_test_thresholds_byte, + traffic_mngr_check_thresholds_byte), + ODP_TEST_INFO_CONDITIONAL(traffic_mngr_test_thresholds_packet, + traffic_mngr_check_thresholds_packet), + ODP_TEST_INFO_CONDITIONAL(traffic_mngr_test_thresholds_byte_and_packet, + traffic_mngr_check_thresholds_byte_and_packet), ODP_TEST_INFO_CONDITIONAL(traffic_mngr_test_byte_wred, traffic_mngr_check_byte_wred), ODP_TEST_INFO_CONDITIONAL(traffic_mngr_test_pkt_wred, @@ -4871,6 +5008,10 @@ odp_testinfo_t traffic_mngr_suite[] = { traffic_mngr_check_drop_prec_marking), ODP_TEST_INFO_CONDITIONAL(traffic_mngr_test_ecn_drop_prec_marking, traffic_mngr_check_ecn_drop_prec_marking), + ODP_TEST_INFO_CONDITIONAL(traffic_mngr_test_tx_aging_no_drop, + traffic_mngr_check_tx_aging), + ODP_TEST_INFO_CONDITIONAL(traffic_mngr_test_tx_aging_drop, + traffic_mngr_check_tx_aging), ODP_TEST_INFO(traffic_mngr_test_fanin_info), ODP_TEST_INFO_CONDITIONAL(traffic_mngr_test_lso_ipv4, traffic_mngr_check_lso_ipv4), ODP_TEST_INFO(traffic_mngr_test_destroy), |