diff options
112 files changed, 3558 insertions, 3702 deletions
diff --git a/.github/workflows/ci-pipeline-arm64.yml b/.github/workflows/ci-pipeline-arm64.yml index 7b27dc248..98f7cf9ac 100644 --- a/.github/workflows/ci-pipeline-arm64.yml +++ b/.github/workflows/ci-pipeline-arm64.yml @@ -19,7 +19,7 @@ jobs: fail-fast: false matrix: cc: [gcc, clang] - conf: ['', 'CFLAGS=-O3', 'CFLAGS=-O1', 'CFLAGS=-O0 --enable-debug=full', 'CFLAGS=-pedantic', + conf: ['', 'CFLAGS=-O3', 'CFLAGS=-O1', 'CFLAGS=-O0 --enable-debug=full', 'CFLAGS=-Os', 'CFLAGS=-pedantic', '--enable-lto', '--enable-lto --enable-abi-compat', '--enable-pcapng-support'] exclude: - cc: clang diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index 93e96f19e..ba41ba565 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -67,7 +67,7 @@ jobs: fail-fast: false matrix: cc: [gcc, clang] - conf: ['', 'CFLAGS=-O3', 'CFLAGS=-O1', 'CFLAGS=-O0 --enable-debug=full', 'CFLAGS=-pedantic', + conf: ['', 'CFLAGS=-O3', 'CFLAGS=-O1', 'CFLAGS=-O0 --enable-debug=full', 'CFLAGS=-Os', 'CFLAGS=-pedantic', '--enable-lto', '--enable-lto --enable-abi-compat', '--enable-pcapng-support'] exclude: - cc: clang diff --git a/.github/workflows/gh-pages.yml b/.github/workflows/gh-pages.yml index 18f269579..325c3b30a 100644 --- a/.github/workflows/gh-pages.yml +++ b/.github/workflows/gh-pages.yml @@ -39,14 +39,14 @@ jobs: cp users-guide/users-guide.html gh-pages/users-guide/index.html mkdir gh-pages/process-guide cp process-guide/*.html gh-pages/process-guide/ - touch gh-pages/.nojekyll popd - name: Deploy env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - uses: crazy-max/ghaction-github-pages@v2 + uses: crazy-max/ghaction-github-pages@v3 with: allow_empty_commit: false build_dir: ./doc/gh-pages + jekyll: false target_branch: gh-pages @@ -1,3 +1,118 @@ +== OpenDataPlane (1.41.0.0) + +=== Backward incompatible API changes +==== Classifier +* Require that PMRs must be destroyed before the CoS they refer to is destroyed. + +==== Crypto +* Deprecate the old session creation error names +(`ODP_CRYPTO_SES_CREATE_ERR_NONE`, `ODP_CRYPTO_SES_CREATE_ERR_ENOMEM`, +`ODP_CRYPTO_SES_CREATE_ERR_INV_CIPHER`, `ODP_CRYPTO_SES_CREATE_ERR_INV_AUTH`) +that have been replaced by shorter error names. +* Change return value of `odp_crypto_result()` to indicate crypto operation +success/failure. +* Deprecate `odp_crypto_packet_result_t.ok` field. Replaced by +`odp_crypto_result()` return value. +* Specify that the status fields of `odp_crypto_packet_result_t` are valid +only when the operation failed (`odp_crypto_result()` returned -1). +* Deprecate `odp_crypto_hw_err_t` type and `odp_crypto_op_status_t.hw_err` +field. All errors are now reported through `alg_err` field. +* Deprecate `ODP_CRYPTO_ALG_ERR_KEY_SIZE` and `ODP_CRYPTO_ALG_ERR_IV_INVALID` +error codes. +* Require that cipher range (`odp_crypto_packet_op_param_t.cipher_range`) and +auth range (`odp_crypto_packet_op_param_t.auth_range`) have zero offset and zero +length when used with null cipher and null auth algorithm, respectively, with +the out-of-place operation type. + +==== Errno +* Remove mention about errno values specified in the API spec. Setting errno has +been removed from all ODP APIs outside errno module. + +==== Packet IO +* Remove legacy reference to errno from `odp_pktio_open()` function. + +==== Stash +* Remove special meaning of `odp_stash_capability_t.max_num_obj` value zero. + +==== Timer +* Remove legacy references to errno from `odp_timer_pool_create()` and +`odp_timer_alloc()` functions. + +=== Backward compatible API changes +==== Classifier +* Clarify that the CoS to be destroyed with `odp_cos_destroy()` must not be in +use. +* Clarify that `odp_cos_queue()` returns `ODP_QUEUE_INVALID` if the queue is not +set. +* Allow CoS pool (`odp_cls_cos_param_t.pool`) to be set to `ODP_POOL_INVALID`, +in which case the originating pktin pool is used. +* Clarify when CoS queue may and may not be invalid. + +==== Crypto +* Change IV (`odp_crypto_packet_op_param_t.cipher_iv_ptr`, +`odp_crypto_packet_op_param_t.auth_iv_ptr`) and AAD ( +`odp_crypto_packet_op_param_t.aad_ptr`) pointers to pointers to constants. +* Clarify that algorithm order (`odp_crypto_session_param_t.auth_cipher_text`) +is ignored with null algorithms. +* Clarify when cipher (`odp_crypto_packet_op_param_t.cipher_range`) and auth +(`odp_crypto_packet_op_param_t.auth_range`) ranges are ignored. +* Allow the result pointer for `odp_crypto_result()` to be null, making filling +the result structure optional. +* Clarify the description of `ODP_CRYPTO_ALG_ERR_DATA_SIZE`. +* Add a new crypto operation error `ODP_CRYPTO_ALG_ERR_OTHER` to cover cases +for which the other errors do not fit. +* Clarify that null cipher and auth algorithms ignore key length, IV length, +digest length, etc. session parameters. +* Clarify that crypto operations do not affect parse flags in packet metadata +and thus `odp_packet_has_error()` cannot be used for checking operation result. + +==== Packet IO +* Allow the default CoS to be removed by passing `ODP_COS_INVALID` to +`odp_pktio_default_cos_set()`. + +==== Pool +* Clarify that the pool to be destroyed with `odp_pool_destroy()` must not be in +use. +* Clarify that `odp_pool_capability_t.max_pools` is used for all pool types +defined in `odp_pool_type_t`. + +==== Stash +* Add new stash create parameter `odp_stash_param_t.strict_size` for performance +optimization. The new option is disabled by default and the total object count +limitation is removed from stash put functions. +* Add new capabilities for maximum number of object handles per stash for +each object size (`odp_stash_capability_t.max_num`). + +==== Timer +* Clarify that zero `odp_timer_periodic_start_t.first_tick` means that the first +expiration time is one period after the current time, not at the current time. + +=== Remove deprecated APIs +==== Crypto +* Remove deprecated `odp_crypto_compl_t` crypto completion event. +* Remove deprecated `odp_crypto_op_param_t` type. +* Remove deprecated `odp_crypto_op_result_t` type. +* Remove deprecated `odp_crypto_session_param_t.pref_mode` field. +* Remove deprecated `odp_crypto_compl_from_event()` function. +* Remove deprecated `odp_crypto_compl_to_event()` function. +* Remove deprecated `odp_crypto_compl_free()` function. +* Remove deprecated `odp_crypto_operation()` function. +* Remove deprecated `odp_crypto_compl_result()` function. +* Remove deprecated `odp_crypto_compl_to_u64()` function. + +==== Event +* Remove deprecated `ODP_EVENT_CRYPTO_COMPL` event type. + +=== Implementation +==== Packet IO +* Remove netmap pktio device. +* Change recommended DPDK version for DPDK pktio device to v22.11. + +==== Stash +* Change implementation to use overflow safe MPMC rings by default. Previous +strict size ring-based implementation can be used by enabling +`odp_stash_param_t.strict_size` parameter. + == OpenDataPlane (1.40.0.0) === Backward incompatible API changes diff --git a/DEPENDENCIES b/DEPENDENCIES index d3a67a6d6..ba17d4992 100644 --- a/DEPENDENCIES +++ b/DEPENDENCIES @@ -152,86 +152,8 @@ Prerequisites for building the OpenDataPlane (ODP) API $ ./configure --with-crypto=ipsecmb $ make -3.5 Netmap packet I/O support (optional) - Netmap accelerated ODP packet I/O. - -3.5.1 Building netmap kernel modules - - ODP works at least with the latest release version of netmap, which is - currently v13.0. However, if possible one should try to use the latest netmap - master branch commit for the best performance and the latest bug fixes. - - # Checkout netmap code - $ git clone https://github.com/luigirizzo/netmap.git - $ cd netmap - $ git checkout v13.0 (optional) - - This is enough to build ODP. If you don't want to build netmap kernel - modules you can jump to section 3.3.2. - - Netmap consists of a core kernel module (netmap.ko), optional modified - device drivers and user space API headers to access the netmap - functionality. It is recommended to build both the core module and modified - device drivers for optimal performance. - - Netmap builds as an out-of-tree kernel module, you need matching kernel - sources to compile it. General build instructions can be found in the packet - README: https://github.com/luigirizzo/netmap/blob/master/LINUX/README. - - If you are running Ubuntu/Debian with the stock kernel and you want to - compile both netmap.ko and modified drivers, these steps will guide you - through it. - - # Download kernel headers - $ sudo apt-get install linux-headers-$(uname -r) - - # Download kernel source matching to the headers - $ sudo apt-get install linux-source - # or - $ apt-get source linux-image-$(uname -r) - - The source archive will be placed in /usr/src/linux-source-<kernel-version> - (or in the current directory if using apt-get source). You will need to - locate it and extract it to a convenient place. - - # Compile netmap - $ cd <netmap_dir>/LINUX - $ ./configure --kernel-sources=<path_to_kernel_src> - $ make - -3.5.2 Building ODP - - $ cd <odp_dir> - $ ./bootstrap - $ ./configure --with-netmap-path=<netmap_dir> - $ make - -3.5.3 Inserting netmap kernel modules - - In order to use netmap I/O you need to insert at least the core netmap - kernel module. - - $ cd <netmap_dir>/LINUX - $ sudo insmod netmap.ko - - To insert the optional modified drivers you first need to remove the - original drivers, if loaded (and if not linked into the kernel). For - example, if using ixgbe: - - $ cd <netmap_path>/LINUX - $ sudo rmmod ixgbe - $ sudo insmod ixgbe/ixgbe.ko - - To restore the original drivers you should be able to use modprobe. - -3.5.4 Running ODP with netmap I/O - - ODP applications will use netmap for packet I/O by default as long as the - netmap kernel module is loaded. If socket I/O is desired instead, it can be - activated by setting the environment variable ODP_PKTIO_DISABLE_NETMAP. - -3.6 DPDK packet I/O support (optional) +3.5 DPDK packet I/O support (optional) Use DPDK for ODP packet I/O. Currently supported DPDK versions are v19.11, v20.11, v21.11, and v22.11 (recommended). @@ -240,7 +162,7 @@ Prerequisites for building the OpenDataPlane (ODP) API https://github.com/OpenDataPlane/odp-dpdk.git for a full DPDK based ODP implementation. -3.6.1 DPDK pktio requirements +3.5.1 DPDK pktio requirements DPDK pktio adds a dependency to NUMA library. # Debian/Ubuntu @@ -249,11 +171,11 @@ Prerequisites for building the OpenDataPlane (ODP) API # CentOS/RedHat/Fedora $ sudo yum install numactl-devel -3.6.2 Native DPDK install +3.5.2 Native DPDK install # Debian/Ubuntu starting from 20.04 $ sudo apt-get install dpdk-dev -3.6.3 Build DPDK v19.11 from source +3.5.3 Build DPDK v19.11 from source $ git clone https://dpdk.org/git/dpdk-stable --branch 19.11 --depth 1 ./<dpdk-dir> # Make and edit DPDK configuration @@ -271,7 +193,7 @@ Prerequisites for building the OpenDataPlane (ODP) API # Configure ODP $ ./configure --with-dpdk-path=<dpdk-dir> -3.6.4 Build DPDK v20.11 and onwards from source +3.5.4 Build DPDK v20.11 and onwards from source $ git clone https://dpdk.org/git/dpdk-stable --branch <version, e.g. 22.11> --depth 1 ./<dpdk-dir> # Prepare the build directory @@ -291,7 +213,7 @@ Prerequisites for building the OpenDataPlane (ODP) API # Or, if DPDK was not installed to the default location, set PKG_CONFIG_PATH: $ PKG_CONFIG_PATH=<dpdk-dir>/install/lib/x86_64-linux-gnu/pkgconfig ./configure --enable-dpdk -3.6.5 Setup system +3.5.5 Setup system # Load DPDK modules $ sudo modprobe uio @@ -303,7 +225,7 @@ Prerequisites for building the OpenDataPlane (ODP) API 512 x 2MB huge pages. All this can be done with the DPDK setup script (<dpdk-dir>/usertools/dpdk-setup.sh). -3.6.6 Running ODP with DPDK pktio +3.5.6 Running ODP with DPDK pktio ODP applications will try use DPDK for packet I/O by default. If some other I/O type is desired instead, DPDK I/O can be disabled by setting the @@ -319,7 +241,7 @@ Prerequisites for building the OpenDataPlane (ODP) API 1024MB of memory: $ sudo ODP_PKTIO_DPDK_PARAMS="-m 1024" ./test/performance/odp_l2fwd -i 0 -c 1 -3.7 AF_XDP socket based packet I/O support (optional) +3.6 AF_XDP socket based packet I/O support (optional) Use AF_XDP socket for packet I/O. At the moment, only zero-copy variant is supported, requiring a kernel version 5.4 or higher. Additionally, if packet @@ -336,7 +258,7 @@ Prerequisites for building the OpenDataPlane (ODP) API Note that, currently, AF_XDP socket packet I/O cannot be instantiated if DPDK zero-copy is enabled. -3.7.1 AF_XDP socket packet I/O requirements +3.6.1 AF_XDP socket packet I/O requirements AF_XDP socket packet I/O implementation requires libxdp and libbpf libraries. They can be fetched from XDP-project in GitHub: @@ -358,14 +280,14 @@ Prerequisites for building the OpenDataPlane (ODP) API $ cd <path to built libbpf> $ make install -3.7.2 Build ODP with AF_XDP socket packet I/O support +3.6.2 Build ODP with AF_XDP socket packet I/O support After building and installing libxdp and libbpf, ODP can be configured to be built with AF_XDP support (pass PKG_CONFIG_PATH if needed). $ ./configure --enable-xdp -3.7.3 Running ODP with AF_XDP socket packet I/O +3.6.3 Running ODP with AF_XDP socket packet I/O AF_XDP socket packet I/Os bind to TRX-combined queues. Based on the packet prosessing needs, NIC(s) of the environment should be configured diff --git a/config/odp-linux-generic.conf b/config/odp-linux-generic.conf index df5328d1f..63ac31fe2 100644 --- a/config/odp-linux-generic.conf +++ b/config/odp-linux-generic.conf @@ -16,7 +16,7 @@ # Mandatory fields odp_implementation = "linux-generic" -config_file_version = "0.1.25" +config_file_version = "0.1.27" # System options system: { @@ -140,15 +140,6 @@ pktio_dpdk: { } } -# netmap pktio options -pktio_netmap: { - # Interface specific options - virt: { - nr_rx_slots = 0 - nr_tx_slots = 0 - } -} - # XDP pktio options pktio_xdp: { # Number of RX and TX descriptors to be reserved for AF_XDP socket @@ -243,6 +234,32 @@ sched_basic: { # > 0, events may be dropped by the implementation if the target queue # is full. To prevent this set 'order_stash_size' to 0. order_stash_size = 512 + + # Power saving options for schedule with wait + # + # When waiting for events during a schedule call, save power by + # sleeping in the poll loop. First, run schedule loop normally for + # poll_time_nsec nanoseconds. If there are no events to schedule in that + # time, continue polling, but sleep for sleep_time_nsec nanoseconds on + # each round. + # + # During sleep, the thread is not polling for packet input or timers. + # Each thread measures time and sleeps independently of other threads. + # + # When using this feature, it may be necessary to decrease + # /proc/<pid>/timerslack_ns, or use a real-time priority. Sleeping may + # have an adverse effect on performance for a short time after sleep. + powersave: { + # Time in nsec to poll before sleeping + # + # <1: Disabled. Never sleep. sleep_time_nsec is ignored. + poll_time_nsec = 0 + + # Time in nsec to sleep + # + # Actual sleep time may vary. + sleep_time_nsec = 0 + } } stash: { @@ -254,12 +271,6 @@ stash: { # The value may be rounded up by the implementation. For optimal memory # usage set value to a power of two - 1. max_num_obj = 4095 - - # Strict size - # - # When set to 0, application can attempt to store more handles into a - # stash than it specified in the creation parameters. - strict_size = 1 } timer: { diff --git a/configure.ac b/configure.ac index 19b23e4b5..6fcca50cf 100644 --- a/configure.ac +++ b/configure.ac @@ -3,7 +3,7 @@ AC_PREREQ([2.5]) # ODP API version ########################################################################## m4_define([odp_version_generation], [1]) -m4_define([odp_version_major], [40]) +m4_define([odp_version_major], [41]) m4_define([odp_version_minor], [0]) m4_define([odp_version_patch], [0]) diff --git a/doc/application-api-guide/Doxyfile b/doc/application-api-guide/Doxyfile index a1777932e..7f8c70c8e 100644 --- a/doc/application-api-guide/Doxyfile +++ b/doc/application-api-guide/Doxyfile @@ -6,5 +6,5 @@ PROJECT_LOGO = $(SRCDIR)/doc/images/ODP-Logo-HQ.svg INPUT = $(SRCDIR)/doc/application-api-guide \ include \ $(SRCDIR)/include -EXAMPLE_PATH = $(SRCDIR)/example $(SRCDIR)/CONTRIBUTING +EXAMPLE_PATH = $(SRCDIR)/example $(SRCDIR)/CONTRIBUTING $(SRCDIR)/include/README WARNINGS = NO diff --git a/doc/application-api-guide/api_guide_lines.dox b/doc/application-api-guide/api_guide_lines.dox index 75f5ab01f..22b9159d4 100644 --- a/doc/application-api-guide/api_guide_lines.dox +++ b/doc/application-api-guide/api_guide_lines.dox @@ -7,7 +7,7 @@ /** -@page api_guide_lines API Guide Lines +@page api_guide_lines API Developer Guidelines @tableofcontents diff --git a/doc/application-api-guide/odp.dox b/doc/application-api-guide/odp.dox index ee328329e..88a85093a 100644 --- a/doc/application-api-guide/odp.dox +++ b/doc/application-api-guide/odp.dox @@ -36,12 +36,16 @@ * for general Linux data plane support. * * @section contact Contact Details - * - The main web site is http://www.opendataplane.org/ + * - The main web site is https://www.opendataplane.org/ * - The git repo is https://github.com/OpenDataPlane/odp.git * - Bug tracking is https://github.com/OpenDataPlane/odp/issues * */ -/**@page contributing Contributing Guide Lines +/**@page api_principles API Principles + * @verbinclude include/README + */ + +/**@page contributing Contributing Guidelines * @verbinclude CONTRIBUTING */ diff --git a/doc/platform-api-guide/Doxyfile b/doc/platform-api-guide/Doxyfile index e581e7b62..8fc251cd3 100644 --- a/doc/platform-api-guide/Doxyfile +++ b/doc/platform-api-guide/Doxyfile @@ -9,4 +9,4 @@ INPUT = $(SRCDIR)/doc/application-api-guide \ $(SRCDIR)/include/odp/api \ $(SRCDIR)/platform/$(WITH_PLATFORM)/doc \ $(SRCDIR)/platform/$(WITH_PLATFORM)/include/odp/api -EXAMPLE_PATH = $(SRCDIR)/example $(SRCDIR)/platform $(SRCDIR)/CONTRIBUTING +EXAMPLE_PATH = $(SRCDIR)/example $(SRCDIR)/platform $(SRCDIR)/CONTRIBUTING $(SRCDIR)/include/README diff --git a/doc/process-guide/faq.adoc b/doc/process-guide/faq.adoc index cbdd05757..8b3de4de7 100644 --- a/doc/process-guide/faq.adoc +++ b/doc/process-guide/faq.adoc @@ -219,9 +219,6 @@ vendors package this capability as a NIC + ODP library. The ODP project also supports its own ODP-DPDK [1] implementation to help migrations from the lower level DPDK API to the ODPs abstraction. -But traditional NICs are supported odp-linux has PKTIO support for Netmap and -DPDK. - == Does ODP support polling mode? ODP does not dictate a model, although the majority of current contributors see diff --git a/example/classifier/odp_classifier.c b/example/classifier/odp_classifier.c index 552da53dc..5b66003fe 100644 --- a/example/classifier/odp_classifier.c +++ b/example/classifier/odp_classifier.c @@ -274,9 +274,6 @@ static odp_pktio_t create_pktio(const char *dev, odp_pool_t pool) /* Open a packet IO instance */ pktio = odp_pktio_open(dev, pool, &pktio_param); if (pktio == ODP_PKTIO_INVALID) { - if (odp_errno() == EPERM) - ODPH_ERR("Root level permission required\n"); - ODPH_ERR("pktio create failed for %s\n", dev); exit(EXIT_FAILURE); } diff --git a/example/ipsec_crypto/odp_ipsec.c b/example/ipsec_crypto/odp_ipsec.c index 565d85a3f..ca7a534d4 100644 --- a/example/ipsec_crypto/odp_ipsec.c +++ b/example/ipsec_crypto/odp_ipsec.c @@ -634,7 +634,6 @@ pkt_disposition_e do_ipsec_in_classify(odp_packet_t *pkt, /* Initialize parameters block */ memset(¶ms, 0, sizeof(params)); params.session = entry->state.session; - out_pkt = entry->in_place ? *pkt : ODP_PACKET_INVALID; /*Save everything to context */ ctx->ipsec.ip_tos = ip->tos; @@ -679,7 +678,7 @@ pkt_disposition_e do_ipsec_in_classify(odp_packet_t *pkt, *skip = FALSE; ctx->state = PKT_STATE_IPSEC_IN_FINISH; if (entry->async) { - if (odp_crypto_op_enq(pkt, &out_pkt, ¶ms, 1) != 1) { + if (odp_crypto_op_enq(pkt, NULL, ¶ms, 1) != 1) { ODPH_ERR("Error: odp_crypto_op_enq() failed\n"); exit(EXIT_FAILURE); } @@ -708,19 +707,12 @@ pkt_disposition_e do_ipsec_in_finish(odp_packet_t pkt, pkt_ctx_t *ctx) { odph_ipv4hdr_t *ip; - odp_crypto_packet_result_t result; int hdr_len = ctx->ipsec.hdr_len; int trl_len = 0; - odp_crypto_result(&result, pkt); + if (odp_crypto_result(NULL, pkt) != 0) + return PKT_DROP; - /* Check crypto result */ - if (!result.ok) { - if (!is_crypto_op_status_ok(&result.cipher_status)) - return PKT_DROP; - if (!is_crypto_op_status_ok(&result.auth_status)) - return PKT_DROP; - } ip = (odph_ipv4hdr_t *)odp_packet_l3_ptr(pkt, NULL); /* @@ -989,11 +981,9 @@ pkt_disposition_e do_ipsec_out_seq(odp_packet_t *pkt, ip->id = odp_cpu_to_be_16((*ctx->ipsec.tun_hdr_id)++); } - out_pkt = entry->in_place ? *pkt : ODP_PACKET_INVALID; - /* Issue crypto request */ if (entry->async) { - if (odp_crypto_op_enq(pkt, &out_pkt, + if (odp_crypto_op_enq(pkt, NULL, &ctx->ipsec.params, 1) != 1) { ODPH_ERR("Error: odp_crypto_op_enq() failed\n"); exit(EXIT_FAILURE); @@ -1023,17 +1013,10 @@ pkt_disposition_e do_ipsec_out_finish(odp_packet_t pkt, pkt_ctx_t *ctx) { odph_ipv4hdr_t *ip; - odp_crypto_packet_result_t result; - odp_crypto_result(&result, pkt); + if (odp_crypto_result(NULL, pkt) != 0) + return PKT_DROP; - /* Check crypto result */ - if (!result.ok) { - if (!is_crypto_op_status_ok(&result.cipher_status)) - return PKT_DROP; - if (!is_crypto_op_status_ok(&result.auth_status)) - return PKT_DROP; - } ip = (odph_ipv4hdr_t *)odp_packet_l3_ptr(pkt, NULL); /* Finalize the IPv4 header */ @@ -1362,9 +1345,7 @@ main(int argc, char *argv[]) /* Populate our IPsec cache */ printf("Using %s mode for crypto API\n\n", - (CRYPTO_API_SYNC == global->appl.mode) ? "SYNC" : - (CRYPTO_API_ASYNC_IN_PLACE == global->appl.mode) ? - "ASYNC_IN_PLACE" : "ASYNC_NEW_BUFFER"); + (CRYPTO_API_SYNC == global->appl.mode) ? "SYNC" : "ASYNC"); ipsec_init_post(global->appl.mode); /* Initialize interfaces (which resolves FWD DB entries */ @@ -1661,8 +1642,7 @@ static void usage(char *progname) "Mandatory OPTIONS:\n" " -i, --interface Eth interfaces (comma-separated, no spaces)\n" " -m, --mode 0: SYNC\n" - " 1: ASYNC_IN_PLACE\n" - " 2: ASYNC_NEW_BUFFER\n" + " 1: ASYNC\n" " Default: 0: SYNC api mode\n" "\n" "Routing / IPSec OPTIONS:\n" diff --git a/example/ipsec_crypto/odp_ipsec_cache.c b/example/ipsec_crypto/odp_ipsec_cache.c index d6ea8ecd4..65a51bd1d 100644 --- a/example/ipsec_crypto/odp_ipsec_cache.c +++ b/example/ipsec_crypto/odp_ipsec_cache.c @@ -71,6 +71,7 @@ int create_ipsec_cache_entry(sa_db_entry_t *cipher_sa, /* Setup parameters and call crypto library to create session */ params.op = (in) ? ODP_CRYPTO_OP_DECODE : ODP_CRYPTO_OP_ENCODE; + params.op_type = ODP_CRYPTO_OP_TYPE_BASIC; params.auth_cipher_text = TRUE; if (CRYPTO_API_SYNC == api_mode) { params.op_mode = ODP_CRYPTO_SYNC; @@ -84,11 +85,6 @@ int create_ipsec_cache_entry(sa_db_entry_t *cipher_sa, entry->async = TRUE; } - if (CRYPTO_API_ASYNC_NEW_BUFFER == api_mode) - entry->in_place = FALSE; - else - entry->in_place = TRUE; - entry->sa_flags = 0; /* Cipher */ diff --git a/example/ipsec_crypto/odp_ipsec_cache.h b/example/ipsec_crypto/odp_ipsec_cache.h index 0634f8a3c..b1e3e7ac1 100644 --- a/example/ipsec_crypto/odp_ipsec_cache.h +++ b/example/ipsec_crypto/odp_ipsec_cache.h @@ -22,8 +22,7 @@ extern "C" { */ typedef enum { CRYPTO_API_SYNC, /**< Synchronous mode */ - CRYPTO_API_ASYNC_IN_PLACE, /**< Asynchronous in place */ - CRYPTO_API_ASYNC_NEW_BUFFER /**< Asynchronous new buffer */ + CRYPTO_API_ASYNC, /**< Asynchronous mode */ } crypto_api_mode_e; /** @@ -31,7 +30,6 @@ typedef enum { */ typedef struct ipsec_cache_entry_s { struct ipsec_cache_entry_s *next; /**< Next entry on list */ - odp_bool_t in_place; /**< Crypto API mode */ odp_bool_t async; /**< ASYNC or SYNC mode */ int sa_flags; uint32_t src_ip; /**< Source v4 address */ diff --git a/example/ipsec_crypto/odp_ipsec_misc.h b/example/ipsec_crypto/odp_ipsec_misc.h index 6186a2369..9f9985c57 100644 --- a/example/ipsec_crypto/odp_ipsec_misc.h +++ b/example/ipsec_crypto/odp_ipsec_misc.h @@ -321,24 +321,6 @@ void ipv4_adjust_len(odph_ipv4hdr_t *ip, int adj) ip->tot_len = odp_cpu_to_be_16(odp_be_to_cpu_16(ip->tot_len) + adj); } -/** - * Verify crypto operation completed successfully - * - * @param status Pointer to cryto completion structure - * - * @return TRUE if all OK else FALSE - */ -static inline -odp_bool_t is_crypto_op_status_ok(odp_crypto_op_status_t *status) -{ - if (status->alg_err != ODP_CRYPTO_ALG_ERR_NONE) - return FALSE; - if (status->hw_err != ODP_CRYPTO_HW_ERR_NONE) - return FALSE; - return TRUE; -} - - #ifdef __cplusplus } #endif diff --git a/example/timer/odp_timer_accuracy.c b/example/timer/odp_timer_accuracy.c index 0952320bb..b8c42ef1e 100644 --- a/example/timer/odp_timer_accuracy.c +++ b/example/timer/odp_timer_accuracy.c @@ -1,5 +1,5 @@ /* Copyright (c) 2018, Linaro Limited - * Copyright (c) 2019-2022, Nokia + * Copyright (c) 2019-2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -18,21 +18,34 @@ #define MAX_FILENAME 128 +enum mode_e { + MODE_ONESHOT = 0, + MODE_RESTART_ABS, + MODE_RESTART_REL, + MODE_PERIODIC, +}; + typedef struct timer_ctx_t { odp_timer_t timer; odp_event_t event; uint64_t nsec; - + uint64_t count; } timer_ctx_t; typedef struct { uint64_t nsec_before_sum; uint64_t nsec_before_min; + uint64_t nsec_before_min_idx; uint64_t nsec_before_max; + uint64_t nsec_before_max_idx; uint64_t nsec_after_sum; uint64_t nsec_after_min; + uint64_t nsec_after_min_idx; uint64_t nsec_after_max; + uint64_t nsec_after_max_idx; + + int64_t nsec_final; uint64_t num_before; uint64_t num_exact; @@ -51,14 +64,18 @@ typedef struct test_log_t { typedef struct test_global_t { struct { unsigned long long period_ns; - unsigned long long res_ns; + long long res_ns; unsigned long long res_hz; unsigned long long offset_ns; unsigned long long max_tmo_ns; unsigned long long num; + unsigned long long num_warmup; unsigned long long burst; unsigned long long burst_gap; - int mode; + odp_fract_u64_t freq; + unsigned long long max_multiplier; + unsigned long long multiplier; + enum mode_e mode; int clk_src; int init; int output; @@ -71,12 +88,15 @@ typedef struct test_global_t { odp_timer_pool_t timer_pool; odp_pool_t timeout_pool; timer_ctx_t *timer_ctx; - uint64_t period_ns; + double res_ns; + uint64_t warmup_timers; uint64_t tot_timers; uint64_t alloc_timers; uint64_t start_tick; uint64_t start_ns; uint64_t period_tick; + double period_dbl; + odp_fract_u64_t base_freq; odp_shm_t log_shm; test_log_t *log; FILE *file; @@ -90,19 +110,28 @@ static void print_usage(void) "Timer accuracy test application.\n" "\n" "OPTIONS:\n" - " -p, --period <nsec> Timeout period in nsec. Default: 200 msec\n" - " -r, --res_ns <nsec> Timeout resolution in nsec. Default: period / 10\n" - " -R, --res_hz <hertz> Timeout resolution in hertz. Note: resolution can be set\n" - " either in nsec or hertz (not both). Default: 0\n" - " -f, --first <nsec> First timer offset in nsec. Default: 300 msec\n" - " -x, --max_tmo <nsec> Maximum timeout in nsec. When 0, max tmo is calculated from other options. Default: 0\n" + " -p, --period <nsec> Timeout period in nsec. Not used in periodic mode. Default: 200 msec\n" + " -r, --res_ns <nsec> Timeout resolution in nsec. Default value is 0. Special values:\n" + " 0: Use period / 10 as the resolution\n" + " -1: In periodic mode, use resolution from capabilities\n" + " -R, --res_hz <hertz> Timeout resolution in hertz. Set resolution either with -r (nsec) or -R (hertz),\n" + " and leave other to 0. Default: 0 (not used)\n" + " -f, --first <nsec> First timer offset in nsec. Default: 0 for periodic mode, otherwise 300 msec\n" + " -x, --max_tmo <nsec> Maximum timeout in nsec. Not used in periodic mode.\n" + " When 0, max tmo is calculated from other options. Default: 0\n" " -n, --num <number> Number of timeout periods. Default: 50\n" + " -w, --warmup <number> Number of warmup periods. Default: 0\n" " -b, --burst <number> Number of timers per a timeout period. Default: 1\n" " -g, --burst_gap <nsec> Gap (in nsec) between timers within a burst. Default: 0\n" + " In periodic mode, first + burst * burst_gap must be less than period length.\n" " -m, --mode <number> Test mode select (default: 0):\n" - " 0: Set all timers at init phase.\n" - " 1: Set first burst of timers at init. Restart timers during test with absolute time.\n" - " 2: Set first burst of timers at init. Restart timers during test with relative time.\n" + " 0: One-shot. Start all timers at init phase.\n" + " 1: One-shot. Each period, restart timers with absolute time.\n" + " 2: One-shot. Each period, restart timers with relative time.\n" + " 3: Periodic.\n" + " -P, --periodic <freq_integer:freq_numer:freq_denom:max_multiplier>\n" + " Periodic timer pool parameters. Default: 5:0:0:1 (5 Hz)\n" + " -M, --multiplier Periodic timer multiplier. Default: 1\n" " -o, --output <file> Output file for measurement logs\n" " -e, --early_retry <num> When timer restart fails due to ODP_TIMER_TOO_NEAR, retry this many times\n" " with expiration time incremented by the period. Default: 0\n" @@ -123,9 +152,12 @@ static int parse_options(int argc, char *argv[], test_global_t *test_global) {"first", required_argument, NULL, 'f'}, {"max_tmo", required_argument, NULL, 'x'}, {"num", required_argument, NULL, 'n'}, + {"warmup", required_argument, NULL, 'w'}, {"burst", required_argument, NULL, 'b'}, {"burst_gap", required_argument, NULL, 'g'}, {"mode", required_argument, NULL, 'm'}, + {"periodic", required_argument, NULL, 'P'}, + {"multiplier", required_argument, NULL, 'M'}, {"output", required_argument, NULL, 'o'}, {"early_retry", required_argument, NULL, 'e'}, {"clk_src", required_argument, NULL, 's'}, @@ -133,18 +165,24 @@ static int parse_options(int argc, char *argv[], test_global_t *test_global) {"help", no_argument, NULL, 'h'}, {NULL, 0, NULL, 0} }; - const char *shortopts = "+p:r:R:f:x:n:b:g:m:o:e:s:ih"; + const char *shortopts = "+p:r:R:f:x:n:w:b:g:m:P:M:o:e:s:ih"; int ret = 0; test_global->opt.period_ns = 200 * ODP_TIME_MSEC_IN_NS; test_global->opt.res_ns = 0; test_global->opt.res_hz = 0; - test_global->opt.offset_ns = 300 * ODP_TIME_MSEC_IN_NS; + test_global->opt.offset_ns = UINT64_MAX; test_global->opt.max_tmo_ns = 0; test_global->opt.num = 50; + test_global->opt.num_warmup = 0; test_global->opt.burst = 1; test_global->opt.burst_gap = 0; - test_global->opt.mode = 0; + test_global->opt.mode = MODE_ONESHOT; + test_global->opt.freq.integer = ODP_TIME_SEC_IN_NS / test_global->opt.period_ns; + test_global->opt.freq.numer = 0; + test_global->opt.freq.denom = 0; + test_global->opt.max_multiplier = 1; + test_global->opt.multiplier = 1; test_global->opt.clk_src = ODP_CLOCK_DEFAULT; test_global->opt.init = 0; test_global->opt.output = 0; @@ -161,7 +199,7 @@ static int parse_options(int argc, char *argv[], test_global_t *test_global) test_global->opt.period_ns = strtoull(optarg, NULL, 0); break; case 'r': - test_global->opt.res_ns = strtoull(optarg, NULL, 0); + test_global->opt.res_ns = strtoll(optarg, NULL, 0); break; case 'R': test_global->opt.res_hz = strtoull(optarg, NULL, 0); @@ -175,6 +213,9 @@ static int parse_options(int argc, char *argv[], test_global_t *test_global) case 'n': test_global->opt.num = strtoull(optarg, NULL, 0); break; + case 'w': + test_global->opt.num_warmup = strtoull(optarg, NULL, 0); + break; case 'b': test_global->opt.burst = strtoull(optarg, NULL, 0); break; @@ -184,6 +225,16 @@ static int parse_options(int argc, char *argv[], test_global_t *test_global) case 'm': test_global->opt.mode = atoi(optarg); break; + case 'P': + sscanf(optarg, "%" SCNu64 ":%" SCNu64 ":%" SCNu64 ":%llu", + &test_global->opt.freq.integer, + &test_global->opt.freq.numer, + &test_global->opt.freq.denom, + &test_global->opt.max_multiplier); + break; + case 'M': + test_global->opt.multiplier = strtoull(optarg, NULL, 0); + break; case 'o': test_global->opt.output = 1; /* filename is NULL terminated in anycase */ @@ -209,20 +260,198 @@ static int parse_options(int argc, char *argv[], test_global_t *test_global) } } - /* Default resolution */ - if (test_global->opt.res_ns == 0 && test_global->opt.res_hz == 0) - test_global->opt.res_ns = test_global->opt.period_ns / 10; + if (test_global->opt.mode == MODE_PERIODIC) { + if ((test_global->opt.freq.integer == 0 && test_global->opt.freq.numer == 0) || + (test_global->opt.freq.numer != 0 && test_global->opt.freq.denom == 0)) { + printf("Bad frequency\n"); + return -1; + } - test_global->tot_timers = test_global->opt.num * test_global->opt.burst; + test_global->opt.period_ns = 0; - if (test_global->opt.mode) - test_global->alloc_timers = test_global->opt.burst; - else + if (test_global->opt.offset_ns == UINT64_MAX) + test_global->opt.offset_ns = 0; + } else { + if (test_global->opt.res_ns < 0) { + printf("Resolution (res_ns) must be >= 0 with single shot timer\n"); + return -1; + } + + if (test_global->opt.offset_ns == UINT64_MAX) + test_global->opt.offset_ns = 300 * ODP_TIME_MSEC_IN_NS; + } + + test_global->warmup_timers = test_global->opt.num_warmup * test_global->opt.burst; + test_global->tot_timers = + test_global->warmup_timers + test_global->opt.num * test_global->opt.burst; + + if (test_global->opt.mode == MODE_ONESHOT) test_global->alloc_timers = test_global->tot_timers; + else + test_global->alloc_timers = test_global->opt.burst; return ret; } +static int single_shot_params(test_global_t *test_global, odp_timer_pool_param_t *timer_param, + odp_timer_capability_t *timer_capa) +{ + uint64_t res_ns, res_hz; + uint64_t max_res_ns, max_res_hz; + uint64_t period_ns = test_global->opt.period_ns; + uint64_t num_tmo = test_global->opt.num; + uint64_t offset_ns = test_global->opt.offset_ns; + enum mode_e mode = test_global->opt.mode; + + max_res_ns = timer_capa->max_res.res_ns; + max_res_hz = timer_capa->max_res.res_hz; + + /* Default resolution */ + if (test_global->opt.res_ns == 0 && test_global->opt.res_hz == 0) { + res_ns = test_global->opt.period_ns / 10; + res_hz = 0; + } else if (test_global->opt.res_ns) { + res_ns = test_global->opt.res_ns; + res_hz = 0; + } else { + res_ns = 0; + res_hz = test_global->opt.res_hz; + } + + if (res_ns && res_ns < max_res_ns) { + printf("Resolution %" PRIu64 " nsec too high. Highest resolution %" PRIu64 " nsec. " + "Default resolution is period / 10.\n\n", + res_ns, max_res_ns); + return -1; + } + + if (res_hz && res_hz > max_res_hz) { + printf("Resolution %" PRIu64 " hz too high. Highest resolution %" PRIu64 " hz. " + "Default resolution is period / 10.\n\n", + res_hz, max_res_hz); + return -1; + } + + if (res_ns) + timer_param->res_ns = res_ns; + else + timer_param->res_hz = res_hz; + + if (mode == MODE_ONESHOT) { + timer_param->min_tmo = offset_ns / 2; + timer_param->max_tmo = offset_ns + ((num_tmo + 1) * period_ns); + } else { + timer_param->min_tmo = period_ns / 10; + timer_param->max_tmo = offset_ns + (2 * period_ns); + } + + if (test_global->opt.max_tmo_ns) { + if (test_global->opt.max_tmo_ns < timer_param->max_tmo) { + printf("Max tmo is too small. Must be at least %" PRIu64 " nsec.\n", + timer_param->max_tmo); + return -1; + } + + timer_param->max_tmo = test_global->opt.max_tmo_ns; + } + + printf(" period: %" PRIu64 " nsec\n", period_ns); + printf(" max res nsec: %" PRIu64 "\n", max_res_ns); + printf(" max res hertz: %" PRIu64 "\n", max_res_hz); + + test_global->period_dbl = period_ns; + + return 0; +} + +static int periodic_params(test_global_t *test_global, odp_timer_pool_param_t *timer_param, + odp_timer_capability_t *timer_capa) +{ + int ret; + uint64_t res_ns; + odp_timer_periodic_capability_t capa; + double freq_dbl, min_freq, max_freq; + double opt_freq = odp_fract_u64_to_dbl(&test_global->opt.freq); + odp_fract_u64_t freq = test_global->opt.freq; + uint64_t res_hz = test_global->opt.res_hz; + uint64_t max_multiplier = test_global->opt.max_multiplier; + uint64_t multiplier = test_global->opt.multiplier; + + if (res_hz) { + res_ns = ODP_TIME_SEC_IN_NS / res_hz; + } else { + res_ns = test_global->opt.res_ns; + + /* Default resolution */ + if (res_ns == 0) + res_ns = ODP_TIME_SEC_IN_NS / (10 * multiplier * opt_freq); + } + + if (res_ns == 0) { + printf("Too high resolution\n"); + return -1; + } + + /* Resolution from capa */ + if (test_global->opt.res_ns < 0) + res_ns = 0; + + min_freq = odp_fract_u64_to_dbl(&timer_capa->periodic.min_base_freq_hz); + max_freq = odp_fract_u64_to_dbl(&timer_capa->periodic.max_base_freq_hz); + + capa.base_freq_hz = freq; + capa.max_multiplier = max_multiplier; + capa.res_ns = res_ns; + + ret = odp_timer_periodic_capability(test_global->opt.clk_src, &capa); + + if (ret < 0) { + printf("Requested periodic timer capabilities are not supported.\n" + "Capabilities: min base freq %g Hz, max base freq %g Hz, " + "max res %" PRIu64 " Hz\n", min_freq, max_freq, timer_capa->max_res.res_hz); + return -1; + } + + if (ret == 0) { + printf("Requested base frequency is not met. Using %.2f Hz instead of %.2f Hz.\n", + odp_fract_u64_to_dbl(&capa.base_freq_hz), opt_freq); + + freq = capa.base_freq_hz; + } + + if (res_ns == 0) + res_ns = capa.res_ns; + + freq_dbl = odp_fract_u64_to_dbl(&freq); + test_global->base_freq = freq; + test_global->period_dbl = ODP_TIME_SEC_IN_NS / (multiplier * freq_dbl); + + /* Min/max tmo are ignored, leave those to default values */ + timer_param->timer_type = ODP_TIMER_TYPE_PERIODIC; + timer_param->periodic.base_freq_hz = freq; + timer_param->periodic.max_multiplier = max_multiplier; + + if (res_hz) + timer_param->res_hz = res_hz; + else + timer_param->res_ns = res_ns; + + printf(" min freq capa: %.2f hz\n", min_freq); + printf(" max freq capa: %.2f hz\n", max_freq); + printf(" freq option: %.2f hz\n", opt_freq); + printf(" freq: %.2f hz\n", freq_dbl); + printf(" freq integer: %" PRIu64 "\n", freq.integer); + printf(" freq numer: %" PRIu64 "\n", freq.numer); + printf(" freq denom: %" PRIu64 "\n", freq.denom); + printf(" max_multiplier: %" PRIu64 "\n", max_multiplier); + printf(" multiplier: %" PRIu64 "\n", multiplier); + printf(" timer freq: %.2f hz\n", multiplier * freq_dbl); + printf(" timer period: %.2f nsec\n", test_global->period_dbl); + printf(" resolution capa: %" PRIu64 " nsec\n", capa.res_ns); + + return 0; +} + static int start_timers(test_global_t *test_global) { odp_pool_t pool; @@ -234,25 +463,26 @@ static int start_timers(test_global_t *test_global) odp_queue_t queue; odp_queue_param_t queue_param; uint64_t start_tick; - uint64_t period_ns, res_ns, res_hz, start_ns, nsec, offset_ns; - uint64_t max_res_ns, max_res_hz; + uint64_t period_ns, start_ns, nsec, offset_ns; + uint32_t max_timers; odp_event_t event; odp_timeout_t timeout; - odp_timer_set_t ret; odp_time_t time; - uint64_t i, j, idx, num_tmo, burst, burst_gap; + uint64_t i, j, idx, num_tmo, num_warmup, burst, burst_gap; uint64_t tot_timers, alloc_timers; - int mode; + enum mode_e mode; odp_timer_clk_src_t clk_src; + int ret; mode = test_global->opt.mode; alloc_timers = test_global->alloc_timers; tot_timers = test_global->tot_timers; num_tmo = test_global->opt.num; + num_warmup = test_global->opt.num_warmup; burst = test_global->opt.burst; burst_gap = test_global->opt.burst_gap; period_ns = test_global->opt.period_ns; - test_global->period_ns = period_ns; + offset_ns = test_global->opt.offset_ns; /* Always init globals for destroy calls */ test_global->queue = ODP_QUEUE_INVALID; @@ -297,94 +527,66 @@ static int start_timers(test_global_t *test_global) return -1; } - if (timer_capa.max_timers && - test_global->alloc_timers > timer_capa.max_timers) { - printf("Error: Too many timers: %" PRIu64 ".\n" - " Max timers: %u\n", test_global->alloc_timers, - timer_capa.max_timers); - return -1; - } - - max_res_ns = timer_capa.max_res.res_ns; - max_res_hz = timer_capa.max_res.res_hz; + max_timers = timer_capa.max_timers; - offset_ns = test_global->opt.offset_ns; - - if (test_global->opt.res_ns) { - res_ns = test_global->opt.res_ns; - res_hz = 0; - } else { - res_ns = 0; - res_hz = test_global->opt.res_hz; + if (mode == MODE_PERIODIC) { + if (timer_capa.periodic.max_pools < 1) { + printf("Error: Periodic timers not supported.\n"); + return -1; + } + max_timers = timer_capa.periodic.max_timers; } - if (res_ns && res_ns < max_res_ns) { - printf("Resolution %" PRIu64 " nsec too high. Highest resolution %" PRIu64 " nsec. " - "Default resolution is period / 10.\n\n", - res_ns, max_res_ns); + if (max_timers && test_global->alloc_timers > max_timers) { + printf("Error: Too many timers: %" PRIu64 ".\n" + " Max timers: %u\n", + test_global->alloc_timers, max_timers); return -1; } - if (res_hz && res_hz > max_res_hz) { - printf("Resolution %" PRIu64 " hz too high. Highest resolution %" PRIu64 " hz. " - "Default resolution is period / 10.\n\n", - res_hz, max_res_hz); - return -1; - } + printf("\nTest parameters:\n"); + printf(" clock source: %i\n", clk_src); + printf(" max timers capa: %" PRIu32 "\n", max_timers); + printf(" mode: %i\n", mode); odp_timer_pool_param_init(&timer_param); - if (res_ns) - timer_param.res_ns = res_ns; + if (mode == MODE_PERIODIC) + ret = periodic_params(test_global, &timer_param, &timer_capa); else - timer_param.res_hz = res_hz; - - if (mode == 0) { - timer_param.min_tmo = offset_ns / 2; - timer_param.max_tmo = offset_ns + ((num_tmo + 1) * period_ns); - } else { - /* periodic mode */ - timer_param.min_tmo = period_ns / 10; - timer_param.max_tmo = offset_ns + (2 * period_ns); - } + ret = single_shot_params(test_global, &timer_param, &timer_capa); - if (test_global->opt.max_tmo_ns) { - if (test_global->opt.max_tmo_ns < timer_param.max_tmo) { - printf("Max tmo is too small. Must be at least %" PRIu64 " nsec.\n", - timer_param.max_tmo); - return -1; - } + if (ret) + return ret; - timer_param.max_tmo = test_global->opt.max_tmo_ns; + if (timer_param.res_hz) { + test_global->res_ns = 1000000000.0 / timer_param.res_hz; + printf(" resolution: %" PRIu64 " Hz\n", timer_param.res_hz); + } else { + test_global->res_ns = timer_param.res_ns; + printf(" resolution: %" PRIu64 " nsec\n", timer_param.res_ns); } timer_param.num_timers = alloc_timers; timer_param.clk_src = clk_src; - printf("\nTest parameters:\n"); - printf(" clock source: %i\n", test_global->opt.clk_src); - printf(" max res nsec: %" PRIu64 "\n", max_res_ns); - printf(" max res hertz: %" PRIu64 "\n", max_res_hz); - printf(" max timers capa: %" PRIu32 "\n", timer_capa.max_timers); - printf(" mode: %i\n", mode); printf(" restart retries: %i\n", test_global->opt.early_retry); if (test_global->opt.output) printf(" log file: %s\n", test_global->filename); printf(" start offset: %" PRIu64 " nsec\n", offset_ns); - printf(" period: %" PRIu64 " nsec\n", period_ns); - if (res_ns) - printf(" resolution: %" PRIu64 " nsec\n", res_ns); - else - printf(" resolution: %" PRIu64 " hz\n", res_hz); printf(" min timeout: %" PRIu64 " nsec\n", timer_param.min_tmo); printf(" max timeout: %" PRIu64 " nsec\n", timer_param.max_tmo); printf(" num timeout: %" PRIu64 "\n", num_tmo); + printf(" num warmup: %" PRIu64 "\n", num_warmup); printf(" burst size: %" PRIu64 "\n", burst); printf(" burst gap: %" PRIu64 "\n", burst_gap); printf(" total timers: %" PRIu64 "\n", tot_timers); + printf(" warmup timers: %" PRIu64 "\n", test_global->warmup_timers); printf(" alloc timers: %" PRIu64 "\n", alloc_timers); + printf(" warmup time: %.2f sec\n", + (offset_ns + (num_warmup * test_global->period_dbl)) / 1000000000.0); printf(" test run time: %.2f sec\n\n", - (offset_ns + (num_tmo * period_ns)) / 1000000000.0); + (offset_ns + (num_tmo * test_global->period_dbl)) / 1000000000.0); timer_pool = odp_timer_pool_create("timer_accuracy", &timer_param); @@ -446,27 +648,48 @@ static int start_timers(test_global_t *test_global) test_global->start_ns = start_ns; test_global->period_tick = odp_timer_ns_to_tick(timer_pool, period_ns); - /* When mode is 1, set only one burst of timers initially */ - if (mode) + /* When mode is not one-shot, set only one burst of timers initially */ + if (mode != MODE_ONESHOT) num_tmo = 1; for (i = 0; i < num_tmo; i++) { + odp_timer_set_t retval; + for (j = 0; j < burst; j++) { timer_ctx_t *ctx = &test_global->timer_ctx[idx]; odp_timer_start_t start_param; - nsec = offset_ns + (i * period_ns) + (j * burst_gap); - ctx->nsec = start_ns + nsec; - - start_param.tick_type = ODP_TIMER_TICK_ABS; - start_param.tick = start_tick + odp_timer_ns_to_tick(timer_pool, nsec); - start_param.tmo_ev = ctx->event; - - ret = odp_timer_start(ctx->timer, &start_param); + if (mode == MODE_PERIODIC) { + odp_timer_periodic_start_t start_param; + + nsec = offset_ns + (j * burst_gap); + + /* By default, timer starts one period after current time. Round + * floating point to closest integer number. */ + ctx->nsec = start_ns + test_global->period_dbl + 0.5; + if (nsec) + ctx->nsec = start_ns + nsec; + + ctx->count = 0; + start_param.freq_multiplier = test_global->opt.multiplier; + start_param.first_tick = 0; + if (nsec) + start_param.first_tick = + start_tick + odp_timer_ns_to_tick(timer_pool, nsec); + start_param.tmo_ev = ctx->event; + retval = odp_timer_periodic_start(ctx->timer, &start_param); + } else { + nsec = offset_ns + (i * period_ns) + (j * burst_gap); + ctx->nsec = start_ns + nsec; + start_param.tick_type = ODP_TIMER_TICK_ABS; + start_param.tick = + start_tick + odp_timer_ns_to_tick(timer_pool, nsec); + start_param.tmo_ev = ctx->event; + retval = odp_timer_start(ctx->timer, &start_param); + } - if (ret != ODP_TIMER_SUCCESS) { - printf("Timer[%" PRIu64 "] set failed: %i\n", - idx, ret); + if (retval != ODP_TIMER_SUCCESS) { + printf("Timer[%" PRIu64 "] set failed: %i\n", idx, retval); return -1; } @@ -518,6 +741,16 @@ static int destroy_timers(test_global_t *test_global) return ret; } +static void print_nsec_error(const char *str, uint64_t nsec, double res_ns, + uint64_t idx) +{ + printf(" %s: %12" PRIu64 " / %.3fx resolution", + str, nsec, (double)nsec / res_ns); + if (idx != UINT64_MAX) + printf(", event %" PRIu64, idx); + printf("\n"); +} + static void print_stat(test_global_t *test_global) { uint64_t i; @@ -526,10 +759,7 @@ static void print_stat(test_global_t *test_global) test_log_t *log = test_global->log; double ave_after = 0.0; double ave_before = 0.0; - double res_ns = test_global->opt.res_ns; - - if (test_global->opt.res_ns == 0) - res_ns = 1000000000.0 / test_global->opt.res_hz; + double res_ns = test_global->res_ns; if (stat->num_after) ave_after = (double)stat->nsec_after_sum / stat->num_after; @@ -546,7 +776,7 @@ static void print_stat(test_global_t *test_global) fprintf(file, " Timer tmo(ns) diff(ns)\n"); - for (i = 0; i < tot_timers; i++) { + for (i = test_global->warmup_timers; i < tot_timers; i++) { fprintf(file, "%8" PRIu64 " %12" PRIu64 " %10" PRIi64 "\n", i, log[i].tmo_ns, log[i].diff_ns); } @@ -554,7 +784,9 @@ static void print_stat(test_global_t *test_global) fprintf(file, "\n"); } - printf("\n Test results:\n"); + tot_timers -= test_global->warmup_timers; + + printf("\nTest results:\n"); printf(" num after: %12" PRIu64 " / %.2f%%\n", stat->num_after, 100.0 * stat->num_after / tot_timers); printf(" num before: %12" PRIu64 " / %.2f%%\n", @@ -564,42 +796,74 @@ static void print_stat(test_global_t *test_global) printf(" num retry: %12" PRIu64 " / %.2f%%\n", stat->num_too_near, 100.0 * stat->num_too_near / tot_timers); printf(" error after (nsec):\n"); - printf(" min: %12" PRIu64 " / %.3fx resolution\n", - stat->nsec_after_min, (double)stat->nsec_after_min / res_ns); - printf(" max: %12" PRIu64 " / %.3fx resolution\n", - stat->nsec_after_max, (double)stat->nsec_after_max / res_ns); - printf(" ave: %12.0f / %.3fx resolution\n", - ave_after, ave_after / res_ns); + print_nsec_error("min", stat->nsec_after_min, res_ns, stat->nsec_after_min_idx); + print_nsec_error("max", stat->nsec_after_max, res_ns, stat->nsec_after_max_idx); + print_nsec_error("ave", ave_after, res_ns, UINT64_MAX); printf(" error before (nsec):\n"); - printf(" min: %12" PRIu64 " / %.3fx resolution\n", - stat->nsec_before_min, (double)stat->nsec_before_min / res_ns); - printf(" max: %12" PRIu64 " / %.3fx resolution\n", - stat->nsec_before_max, (double)stat->nsec_before_max / res_ns); - printf(" ave: %12.0f / %.3fx resolution\n", - ave_before, ave_before / res_ns); + print_nsec_error("min", stat->nsec_before_min, res_ns, stat->nsec_before_min_idx); + print_nsec_error("max", stat->nsec_before_max, res_ns, stat->nsec_before_max_idx); + print_nsec_error("ave", ave_before, res_ns, UINT64_MAX); + printf(" final timeout error (nsec):\n"); + printf(" %12" PRIi64 " / %.3fx resolution\n", + stat->nsec_final, (double)stat->nsec_final / res_ns); printf("\n"); } +static void cancel_periodic_timers(test_global_t *test_global) +{ + uint64_t i, alloc_timers; + odp_timer_t timer; + odp_event_t ev; + + if (test_global->opt.mode != MODE_PERIODIC) + return; + + alloc_timers = test_global->alloc_timers; + + for (i = 0; i < alloc_timers; i++) { + timer = test_global->timer_ctx[i].timer; + + if (timer == ODP_TIMER_INVALID) + break; + + if (odp_timer_periodic_cancel(timer)) + printf("Failed to cancel periodic timer.\n"); + } + + while (alloc_timers) { + odp_timeout_t tmo; + timer_ctx_t *ctx; + + ev = odp_schedule(NULL, ODP_SCHED_WAIT); + tmo = odp_timeout_from_event(ev); + ctx = odp_timeout_user_ptr(tmo); + if (odp_timer_periodic_ack(ctx->timer, ev) != 2) + continue; + odp_event_free(ev); + alloc_timers--; + } +} + static void run_test(test_global_t *test_global) { uint64_t burst, num, num_tmo, next_tmo; uint64_t i, tot_timers; odp_event_t ev; odp_time_t time; - uint64_t time_ns, diff_ns, period_ns; + uint64_t time_ns, diff_ns; odp_timeout_t tmo; uint64_t tmo_ns; timer_ctx_t *ctx; test_stat_t *stat = &test_global->stat; test_log_t *log = test_global->log; - int mode = test_global->opt.mode; + enum mode_e mode = test_global->opt.mode; + double period_dbl = test_global->period_dbl; num = 0; next_tmo = 1; num_tmo = test_global->opt.num; burst = test_global->opt.burst; tot_timers = test_global->tot_timers; - period_ns = test_global->period_ns; for (i = 0; i < tot_timers; i++) { ev = odp_schedule(NULL, ODP_SCHED_WAIT); @@ -609,6 +873,19 @@ static void run_test(test_global_t *test_global) tmo = odp_timeout_from_event(ev); ctx = odp_timeout_user_ptr(tmo); tmo_ns = ctx->nsec; + if (mode == MODE_PERIODIC) { + /* round to closest integer number */ + tmo_ns += ctx->count * period_dbl + 0.5; + ctx->count++; + } + + if (i == test_global->warmup_timers) { + memset(stat, 0, sizeof(*stat)); + stat->nsec_before_min = UINT64_MAX; + stat->nsec_after_min = UINT64_MAX; + } + + stat->nsec_final = (int64_t)time_ns - (int64_t)tmo_ns; if (log) log[i].tmo_ns = tmo_ns; @@ -617,10 +894,14 @@ static void run_test(test_global_t *test_global) diff_ns = time_ns - tmo_ns; stat->num_after++; stat->nsec_after_sum += diff_ns; - if (diff_ns < stat->nsec_after_min) + if (diff_ns < stat->nsec_after_min) { stat->nsec_after_min = diff_ns; - if (diff_ns > stat->nsec_after_max) + stat->nsec_after_min_idx = i; + } + if (diff_ns > stat->nsec_after_max) { stat->nsec_after_max = diff_ns; + stat->nsec_after_max_idx = i; + } if (log) log[i].diff_ns = diff_ns; @@ -628,17 +909,22 @@ static void run_test(test_global_t *test_global) diff_ns = tmo_ns - time_ns; stat->num_before++; stat->nsec_before_sum += diff_ns; - if (diff_ns < stat->nsec_before_min) + if (diff_ns < stat->nsec_before_min) { stat->nsec_before_min = diff_ns; - if (diff_ns > stat->nsec_before_max) + stat->nsec_before_min_idx = i; + } + if (diff_ns > stat->nsec_before_max) { stat->nsec_before_max = diff_ns; + stat->nsec_before_max_idx = i; + } if (log) log[i].diff_ns = -diff_ns; } else { stat->num_exact++; } - if (mode && next_tmo < num_tmo) { + if ((mode == MODE_RESTART_ABS || mode == MODE_RESTART_REL) && + next_tmo < num_tmo) { /* Reset timer for next period */ odp_timer_t tim; uint64_t nsec, tick; @@ -647,6 +933,7 @@ static void run_test(test_global_t *test_global) odp_timer_pool_t tp = test_global->timer_pool; unsigned int retries = test_global->opt.early_retry; uint64_t start_ns = test_global->start_ns; + uint64_t period_ns = test_global->opt.period_ns; odp_timer_start_t start_param; tim = ctx->timer; @@ -654,7 +941,7 @@ static void run_test(test_global_t *test_global) /* Depending on the option, retry when expiration * time is too early */ for (j = 0; j < retries + 1; j++) { - if (mode == 1) { + if (mode == MODE_RESTART_ABS) { /* Absolute time */ ctx->nsec += period_ns; nsec = ctx->nsec - start_ns; @@ -685,6 +972,9 @@ static void run_test(test_global_t *test_global) "%" PRIu64 "\n", ret, ctx->nsec); return; } + } else if (mode == MODE_PERIODIC) { + if (odp_timer_periodic_ack(ctx->timer, ev)) + printf("Failed to ack a periodic timer.\n"); } else { odp_event_free(ev); } @@ -698,6 +988,8 @@ static void run_test(test_global_t *test_global) } + cancel_periodic_timers(test_global); + /* Free current scheduler context. There should be no more events. */ while ((ev = odp_schedule(NULL, ODP_SCHED_NO_WAIT)) != ODP_EVENT_INVALID) { diff --git a/helper/include/odp/helper/odph_debug.h b/helper/include/odp/helper/odph_debug.h index bedef2f66..7cb2ee617 100644 --- a/helper/include/odp/helper/odph_debug.h +++ b/helper/include/odp/helper/odph_debug.h @@ -24,14 +24,16 @@ extern "C" { #endif +#pragma GCC diagnostic push + +#ifdef __clang__ +#pragma GCC diagnostic ignored "-Wgnu-zero-variadic-macro-arguments" +#endif + /** @addtogroup odph_debug ODPH DEBUG * @{ */ -/* Avoid "ISO C99 requires at least one argument for the "..." in a variadic - * macro" errors when building with 'pedantic' option. */ -#pragma GCC system_header - /** * Assert macro for applications and helper code * @@ -61,50 +63,50 @@ typedef enum odph_log_level { */ #define ODPH_LOG(level, fmt, ...) \ do { \ - switch (level) { \ - case ODPH_LOG_ERR: \ - fprintf(stderr, "%s:%d:%s():" fmt, __FILE__, \ - __LINE__, __func__, ##__VA_ARGS__); \ - break; \ - case ODPH_LOG_DBG: \ - if (ODPH_DEBUG_PRINT == 1) \ - fprintf(stderr, "%s:%d:%s():" fmt, __FILE__, \ - __LINE__, __func__, ##__VA_ARGS__); \ - break; \ - case ODPH_LOG_ABORT: \ + if (level != ODPH_LOG_DBG || ODPH_DEBUG_PRINT == 1) \ fprintf(stderr, "%s:%d:%s(): " fmt, __FILE__, \ __LINE__, __func__, ##__VA_ARGS__); \ + if (level == ODPH_LOG_ABORT) \ abort(); \ - break; \ - default: \ - fprintf(stderr, "Unknown LOG level"); \ - break;\ - } \ } while (0) /** * Debug printing macro, which prints output when DEBUG flag is set. */ -#define ODPH_DBG(fmt, ...) \ - ODPH_LOG(ODPH_LOG_DBG, fmt, ##__VA_ARGS__) +#define ODPH_DBG(...) \ + do { \ + __extension__ ({ \ + ODPH_LOG(ODPH_LOG_DBG, ##__VA_ARGS__); \ + }); \ + } while (0) /** * Print output to stderr (file, line and function). */ -#define ODPH_ERR(fmt, ...) \ - ODPH_LOG(ODPH_LOG_ERR, fmt, ##__VA_ARGS__) +#define ODPH_ERR(...) \ + do { \ + __extension__ ({ \ + ODPH_LOG(ODPH_LOG_ERR, ##__VA_ARGS__); \ + }); \ + } while (0) /** * Print output to stderr (file, line and function), * then abort. */ -#define ODPH_ABORT(fmt, ...) \ - ODPH_LOG(ODPH_LOG_ABORT, fmt, ##__VA_ARGS__) +#define ODPH_ABORT(...) \ + do { \ + __extension__ ({ \ + ODPH_LOG(ODPH_LOG_ABORT, ##__VA_ARGS__); \ + }); \ + } while (0) /** * @} */ +#pragma GCC diagnostic pop + #ifdef __cplusplus } #endif diff --git a/helper/test/chksum.c b/helper/test/chksum.c index d14ee20d1..3b1f8ed9f 100644 --- a/helper/test/chksum.c +++ b/helper/test/chksum.c @@ -107,6 +107,9 @@ int main(int argc ODP_UNUSED, char *argv[] ODP_UNUSED) ODPH_IPV4HDR_LEN); ip->proto = ODPH_IPPROTO_UDP; ip->id = odp_cpu_to_be_16(1); + ip->tos = 0; + ip->frag_offset = 0; + ip->ttl = 0; odp_packet_has_ipv4_set(test_packet, 1); if (odph_ipv4_csum_update(test_packet) < 0) status = -1; diff --git a/include/README b/include/README index c8abb6893..180ff62c7 100644 --- a/include/README +++ b/include/README @@ -1,51 +1,94 @@ -ODP specification -================= +Copyright (c) 2017, Linaro Limited +Copyright (c) 2023, Nokia +All rights reserved. + +SPDX-License-Identifier: BSD-3-Clause + +# ODP specification ODP specification consists of several types of files, which together provide full list of types, values and functions that ODP implementation MUST provide. -ODP API specification ---------------------- +## API Principles + +Both applications and implementations must comply with the API specification. If +not otherwise documented, results are undefined if an application acts against +the specification. For example, if an application passes bad parameters to an +ODP API, one implementation may report an error, while another may not check +them (to maximize performance) and would just crash. + +Many ODP component areas provide an odp_xxx_capability() API that returns +platform-specific information regarding supported features in that component. +For best portability applications should always use these capability APIs. + +ODP APIs are described using opaque data types of which definition are left up +to the ODP implementation. For example, ODP packets are referenced by handles of +type odp_packet_t, and packet-related APIs take arguments of this type. What an +odp_packet_t actually is, is not part of the ODP API specification and +applications cannot make assumptions about the underlying type. + +Application gains ODP handle ownership when it receives it from an ODP API call. +For example, application can receive an odp_event_t handle from `odp_schedule()` +call. The ownership ends when the handle is passed back to ODP, for example with +`odp_queue_enq()` call. Application MUST NOT use the handle anymore after it +has lost the ownership. + +## API headers + +ODP API headers are divided into the following directories. +``` +include/ +├── odp_api.h +└── odp/ + ├── api/ + │ ├── abi-default/ + │ └── spec/ + └── arch/ + └── @ARCH_ABI@/ + └── odp/ + └── api/ + └── abi/ +platform/ +└── @with_platform@/ + ├── include/ + └── include-abi/ + └── odp/ + └── api/ + └── abi/ +``` + +### Application header + +This header found at `include/odp_api.h` is an entry point for an application. +Application MUST include only odp_api.h, nothing else. This file includes all +files from ODP specification. + +### API specification These are the files from `include/odp/api/spec` directory. They specify a set -of function prototypes, types, type names, enumerations etc that MUST be +of function prototypes, types, type names, enumerations, etc. that MUST be provided by ODP implementation. Doxygen comments inside these files document -requirements for ABI interface provided by an implementation. Content of some -types and value of some enumerations are left undefined in API spec. These are -defined either in ABI spec or implementation specific (non-ABI compatible) -headers .An implementation MUST use these headers AS IS, without any -modifications to be compatible with ODP specification. +the API specification. Content of some types and value of some enumerations are +left undefined in API spec. These are defined either in ABI spec or +implementation specific (non-ABI compatible) headers. An implementation MUST use +these headers AS IS, without any modifications to be compatible with ODP +specification. -ODP ABI compatibility specification ------------------------------------ +### ABI compatibility specification These are the files from `include/odp/arch/@ARCH_ABI@/odp/api/abi/` directory. They specify a set of types and values that MUST be used AS IS without any modifications by an implementation if it supports and is compiled for ABI-compatibility mode. -ODP default ABI headers ------------------------ +### Default ABI headers These are the files from `include/odp/api/abi-default` directory. They provide default specification for ODP types and values for ABI compatibility. CPU architecture specific ABI compatibility files heavily depend on these headers. These files MUST NOT be changed by an implementation. -odp_api.h header ----------------- - -This header found at `include/odp_api.h` is an entry point for an application. -Application MUST include only odp_api.h, nothing else. This file includes all -files from ODP specification. - -Additional ODP headers -====================== - -These are the headers provided by an ODP to supplement ODP specification. - -ODP API headers ---------------- +### Additional API headers These are the files from `include/odp/api` directory. They glue together API and ABI specification headers. Although they are not part of ODP specification @@ -53,11 +96,9 @@ itself, they provide an easy way for an implementation to use ODP API/ABI header files. An implementation SHOULD use these headers AS IS unless it has strong reason not to do so. -Platform-specific headers -========================= +## Platform-specific headers -Platform ABI headers --------------------- +### Platform ABI headers These are the headers found at `platform/@with_platform@/include-abi/odp/api/abi` directory. They are used by @@ -66,14 +107,13 @@ disabled. They should implement at least a set of types and values documented in ODP API specification headers. They are permitted to provide any platform specific optimizations (i.e. they might provide types and/or values that map directly onto the hardware details, they might provide inline functions to -speed up execution of the application, etc). These headers MAY use ODP default +speed up execution of the application, etc.). These headers MAY use ODP default ABI headers if they do fit. -Rest of platform-specific headers ---------------------------------- +### Additional platform-specific headers Platform MAY provide additional headers at `platform/@with_platform/include`. -However these headers SHOULD NOT be used directly by an application, because +However, these headers SHOULD NOT be used directly by an application, because this will tie it to the exact implementation details. Application MUST include only <odp_api.h> header. Platform ABI headers MAY use these headers to implement platform-specific optimizations. diff --git a/include/odp/api/abi-default/crypto_types.h b/include/odp/api/abi-default/crypto_types.h index cc9155bfa..4fa9a63a4 100644 --- a/include/odp/api/abi-default/crypto_types.h +++ b/include/odp/api/abi-default/crypto_types.h @@ -14,9 +14,6 @@ extern "C" { #include <stdint.h> -/** @internal Dummy type for strong typing */ -typedef struct { char dummy; /**< @internal Dummy */ } _odp_abi_crypto_compl_t; - /** @ingroup odp_crypto * @{ */ @@ -24,7 +21,6 @@ typedef struct { char dummy; /**< @internal Dummy */ } _odp_abi_crypto_compl_t; #define ODP_CRYPTO_SESSION_INVALID (0xffffffffffffffffULL) typedef uint64_t odp_crypto_session_t; -typedef _odp_abi_crypto_compl_t *odp_crypto_compl_t; /** * @} diff --git a/include/odp/api/abi-default/event_types.h b/include/odp/api/abi-default/event_types.h index fc685cfd9..0fa20e692 100644 --- a/include/odp/api/abi-default/event_types.h +++ b/include/odp/api/abi-default/event_types.h @@ -30,9 +30,6 @@ typedef enum { ODP_EVENT_BUFFER = 1, ODP_EVENT_PACKET = 2, ODP_EVENT_TIMEOUT = 3, -#if ODP_DEPRECATED_API - ODP_EVENT_CRYPTO_COMPL = 4, -#endif ODP_EVENT_IPSEC_STATUS = 5, ODP_EVENT_PACKET_VECTOR = 6, ODP_EVENT_PACKET_TX_COMPL = 7, diff --git a/include/odp/api/spec/classification.h b/include/odp/api/spec/classification.h index 0c4294a6a..592ad1050 100644 --- a/include/odp/api/spec/classification.h +++ b/include/odp/api/spec/classification.h @@ -622,7 +622,8 @@ typedef enum { */ typedef struct odp_cls_cos_param { /** Action to take. When action is ODP_COS_ACTION_DROP, all the other - * parameters are ignored. + * parameters are ignored. If action is ODP_COS_ACTION_ENQUEUE, then + * queue must be set, or num_queue must be greater than one. * * The final match in the CoS chain defines the action for a packet. * I.e. packet is dropped only when the CoS of the last matching rule @@ -671,7 +672,15 @@ typedef struct odp_cls_cos_param { odp_pktin_hash_proto_t hash_proto; }; }; - /** Pool associated with CoS */ + /** Pool associated with CoS + * + * May be set to ODP_POOL_INVALID, in which case the default pool of + * the originating packet input is used (@see odp_pktio_open()). If + * there is no originating packet input (e.g. with lookaside IPsec), + * then this parameter must be set to a valid pool. + * + * Default is ODP_POOL_INVALID. + */ odp_pool_t pool; #if ODP_DEPRECATED_API @@ -750,6 +759,11 @@ odp_queue_t odp_cls_hash_result(odp_cos_t cos, odp_packet_t packet); /** * Discard a class-of-service along with all its associated resources * + * Before destroying a CoS, all the PMRs referring to the CoS (as a source or + * destination CoS) must be destroyed first. Also, the CoS must not be in use + * as the default CoS in any pktio (@see odp_pktio_default_cos_set()) or as the + * destination CoS of any IPsec SA. + * * @param cos CoS handle * * @retval 0 on success @@ -760,9 +774,12 @@ int odp_cos_destroy(odp_cos_t cos); /** * Assign a queue for a class-of-service * + * Action of the given CoS may not be ODP_COS_ACTION_DROP. + * * @param cos CoS handle * @param queue Handle of the queue where all packets of this specific - * class of service will be enqueued. + * class of service will be enqueued. Must not be + * ODP_QUEUE_INVALID. * * @retval 0 on success * @retval <0 on failure @@ -775,7 +792,8 @@ int odp_cos_queue_set(odp_cos_t cos, odp_queue_t queue); * @param cos CoS handle * * @retval Queue handle associated with the given class-of-service -* @retval ODP_QUEUE_INVALID on failure +* @retval ODP_QUEUE_INVALID on failure, or if there are multiple queues, or if +* the CoS action is ODP_COS_ACTION_DROP. */ odp_queue_t odp_cos_queue(odp_cos_t cos); @@ -1012,7 +1030,7 @@ int odp_cls_cos_pool_set(odp_cos_t cos, odp_pool_t pool_id); * @param cos CoS handle * * @retval pool handle of the associated pool -* @retval ODP_POOL_INVALID if no associated pool found or in case of an error +* @retval ODP_POOL_INVALID on failure, or if the pool has not been set */ odp_pool_t odp_cls_cos_pool(odp_cos_t cos); diff --git a/include/odp/api/spec/crypto.h b/include/odp/api/spec/crypto.h index 4292b78c1..1c477336e 100644 --- a/include/odp/api/spec/crypto.h +++ b/include/odp/api/spec/crypto.h @@ -115,80 +115,6 @@ int odp_crypto_session_create(const odp_crypto_session_param_t *param, */ int odp_crypto_session_destroy(odp_crypto_session_t session); -#if ODP_DEPRECATED_API - -/** - * Return crypto completion handle that is associated with event - * - * @deprecated Used only with the deprecated odp_crypto_operation() - * - * Note: any invalid parameters will cause undefined behavior and may cause - * the application to abort or crash. - * - * @param ev An event of type ODP_EVENT_CRYPTO_COMPL - * - * @return crypto completion handle - */ -odp_crypto_compl_t odp_crypto_compl_from_event(odp_event_t ev); - -/** - * Convert crypto completion handle to event handle - * - * @deprecated Used only with the deprecated odp_crypto_operation() - * - * @param completion_event Completion event to convert to generic event - * - * @return Event handle - */ -odp_event_t odp_crypto_compl_to_event(odp_crypto_compl_t completion_event); - -/** - * Release crypto completion event - * - * @deprecated Used only with the deprecated odp_crypto_operation() - * - * @param completion_event Completion event we are done accessing - */ -void odp_crypto_compl_free(odp_crypto_compl_t completion_event); - -/** - * Crypto per packet operation - * - * @deprecated Use odp_crypto_op() or odp_crypto_op_enq() instead. - * - * This function may be called only for sessions configured with - * the ODP_CRYPTO_OP_TYPE_LEGACY operation type. - * - * Performs the cryptographic operations specified during session creation - * on the packet. If the operation is performed synchronously, "posted" - * will return FALSE and the result of the operation is immediately available. - * If "posted" returns TRUE the result will be delivered via the completion - * queue specified when the session was created. - * - * @param param Operation parameters - * @param posted Pointer to return posted, TRUE for async operation - * @param result Results of operation (when posted returns FALSE) - * - * @retval 0 on success - * @retval <0 on failure - */ -int odp_crypto_operation(odp_crypto_op_param_t *param, - odp_bool_t *posted, - odp_crypto_op_result_t *result); - -/** - * Crypto per packet operation query result from completion event - * - * @deprecated Used only with the deprecated odp_crypto_operation() - * - * @param completion_event Event containing operation results - * @param result Pointer to result structure - */ -void odp_crypto_compl_result(odp_crypto_compl_t completion_event, - odp_crypto_op_result_t *result); - -#endif /* ODP_DEPRECATED_API */ - /** * Get printable value for an odp_crypto_session_t * @@ -202,23 +128,6 @@ void odp_crypto_compl_result(odp_crypto_compl_t completion_event, */ uint64_t odp_crypto_session_to_u64(odp_crypto_session_t hdl); -#if ODP_DEPRECATED_API -/** - * Get printable value for an odp_crypto_compl_t - * - * @deprecated Used only with the deprecated odp_crypto_operation() - * - * @param hdl odp_crypto_compl_t handle to be printed - * @return uint64_t value that can be used to print/display this - * handle - * - * @note This routine is intended to be used for diagnostic purposes - * to enable applications to generate a printable value that represents - * an odp_crypto_compl_t handle. - */ -uint64_t odp_crypto_compl_to_u64(odp_crypto_compl_t hdl); -#endif - /** * Initialize crypto session parameters * @@ -259,17 +168,28 @@ odp_event_t odp_crypto_packet_to_event(odp_packet_t pkt); /** * Get crypto operation results from a crypto processed packet * - * Successful crypto operations of all types (SYNC and ASYNC) produce packets - * which contain crypto result metadata. This function copies the operation - * results from a crypto processed packet. Event subtype of this kind of - * packet is ODP_EVENT_PACKET_CRYPTO. Results are undefined if a non-crypto - * processed packet is passed as input. + * Crypto operations of all types (SYNC and ASYNC) produce packets which + * contain crypto result metadata. This function returns success status + * of the crypto operation that was applied to a packet and optionally + * writes additional information in a result structure. + * + * If the crypto operation succeeded, zero is returned and the values + * written in the cipher_status and auth_status fields of the result + * structure have undefined values. + * + * If the crypto operation failed, -1 is returned and the cipher_status + * and auth_status fields of the result structure indicate the reason for + * the failure. + * + * The subtype of the passed packet must be ODP_EVENT_PACKET_CRYPTO, + * otherwise the result of the call is undefined. * * @param packet A crypto processed packet (ODP_EVENT_PACKET_CRYPTO) - * @param[out] result Pointer to operation result for output + * @param[out] result Pointer to operation result for output or NULL * - * @retval 0 On success - * @retval <0 On failure + * @retval 0 Crypto operation succeeded + * @retval -1 Crypto operation failed + * @retval <-1 Failed to get crypto operation status of the packet */ int odp_crypto_result(odp_crypto_packet_result_t *result, odp_packet_t packet); @@ -280,6 +200,11 @@ int odp_crypto_result(odp_crypto_packet_result_t *result, * Performs the SYNC cryptographic operations specified during session creation * on the packets. All arrays should be of num_pkt size. * + * Result of the crypto operation can be checked using odp_crypto_result(). + * Parse flags in packet metadata are not affected by the crypto operation. + * In particular, odp_packet_has_error() can not be used for checking if the + * crypto operation succeeded. + * * Use of the pkt_out parameter depends on the configured crypto operation * type as described below. * diff --git a/include/odp/api/spec/crypto_types.h b/include/odp/api/spec/crypto_types.h index 574f56445..46d9ae3f7 100644 --- a/include/odp/api/spec/crypto_types.h +++ b/include/odp/api/spec/crypto_types.h @@ -38,11 +38,6 @@ extern "C" { */ /** - * @typedef odp_crypto_compl_t -* @deprecated Crypto API completion event (platform dependent). -*/ - -/** * Crypto API operation mode */ typedef enum { @@ -516,8 +511,7 @@ typedef enum odp_crypto_op_type_t { /** * Input packet data and metadata are copied in the output packet * and then processed. Output packet is allocated by the caller - * or by ODP. odp_crypto_op(), odp_crypto_op_enq() and - * odp_crypto_operation() can be used. + * or by ODP. * * This is the default value but will be deprecated in the future. */ @@ -526,7 +520,6 @@ typedef enum odp_crypto_op_type_t { /** * Input packet data and metadata are copied in the output packet * and then processed. Output packet is allocated by ODP. - * odp_crypto_op() and odp_crypto_op_enq() can be used. */ ODP_CRYPTO_OP_TYPE_BASIC, @@ -541,8 +534,6 @@ typedef enum odp_crypto_op_type_t { * * Crypto output is the processed crypto_range, auth_range and * MAC/digest (in encode sessions) of the input packet. - * - * odp_crypto_op() and odp_crypto_op_enq() can be used. */ ODP_CRYPTO_OP_TYPE_OOP, } odp_crypto_op_type_t; @@ -578,7 +569,8 @@ typedef struct odp_crypto_session_param_t { * indicates the reverse order of operation. * * The value is ignored with authenticated encryption algorithms - * such as AES-GCM. + * such as AES-GCM. The value is also ignored when one of the + * algorithms is null. * * true: Authenticate cipher text * false: Authenticate plain text @@ -597,14 +589,6 @@ typedef struct odp_crypto_session_param_t { */ odp_bool_t hash_result_in_auth_range; - /** Preferred sync vs. async for odp_crypto_operation() - * - * The default value is ODP_CRYPTO_SYNC. - * - * @deprecated Used only with deprecated odp_crypto_operation() - */ - odp_crypto_op_mode_t ODP_DEPRECATE(pref_mode); - /** Operation mode when using packet interface: sync or async * * The default value is ODP_CRYPTO_SYNC. @@ -619,6 +603,9 @@ typedef struct odp_crypto_session_param_t { * the pairing authentication algorithm. When ciphering is enabled * cipher key and IV need to be set. The default value is * ODP_CIPHER_ALG_NULL. + * + * When ciphering is disabled, i.e. cipher_alg is ODP_CIPHER_ALG_NULL, + * cipher_key and cipher_iv_len parameters are ignored. */ odp_cipher_alg_t cipher_alg; @@ -645,6 +632,10 @@ typedef struct odp_crypto_session_param_t { * ODP_AUTH_ALG_CHACHA20_POLY1305. Otherwise, all authentication side * parameters must be set when authentication is enabled. The default * value is ODP_AUTH_ALG_NULL. + * + * When authentication is disabled, i.e. auth_alg is + * ODP_AUTH_ALG_NULL, auth_key, auth_iv_len, auth_digest_len, + * auth_aad_len and hash_result_in_auth_range parameters are ignored. */ odp_auth_alg_t auth_alg; @@ -674,8 +665,7 @@ typedef struct odp_crypto_session_param_t { /** Async mode completion event queue * * The completion queue is used to return completions from - * odp_crypto_op_enq() (and the deprecated odp_crypto_operation()) - * to the application. + * odp_crypto_op_enq() to the application. */ odp_queue_t compl_queue; @@ -693,83 +683,6 @@ typedef struct odp_crypto_session_param_t { } odp_crypto_session_param_t; /** - * Crypto API per packet operation parameters - * - * @deprecated Use odp_crypto_packet_op_param_t instead. - */ -typedef struct odp_crypto_op_param_t { - /** Session handle from creation */ - odp_crypto_session_t session; - - /** User context */ - void *ctx; - - /** Input packet - * - * Specifies the input packet for the crypto operation. When the - * 'out_pkt' variable is set to ODP_PACKET_INVALID (indicating a new - * packet should be allocated for the resulting packet). - */ - odp_packet_t pkt; - - /** Output packet - * - * Both "in place" (the original packet 'pkt' is modified) and - * "copy" (the packet is replicated to a new packet which contains - * the modified data) modes are supported. The "in place" mode of - * operation is indicated by setting 'out_pkt' equal to 'pkt'. - * For the copy mode of operation, setting 'out_pkt' to a valid packet - * value indicates the caller wishes to specify the destination packet. - * Setting 'out_pkt' to ODP_PACKET_INVALID indicates the caller wishes - * the destination packet be allocated from the output pool specified - * during session creation. - */ - odp_packet_t out_pkt; - - /** IV pointer for cipher */ - uint8_t *cipher_iv_ptr; - - /** Authentication IV pointer */ - uint8_t *auth_iv_ptr; - - /** Offset from start of packet for hash result - * - * In case of decode sessions, the expected hash will be read from - * this offset and compared with the calculated hash. After the - * operation the hash bytes will have undefined values. - * - * In case of encode sessions the calculated hash will be stored in - * this offset. - * - * If the hash_result_in_auth_range session parameter is true, - * the hash result location may overlap auth_range. In that case - * the result location will be zeroed in decode sessions before - * hash calculation. Zeroing is not done in encode sessions. - */ - uint32_t hash_result_offset; - - /** Pointer to AAD. AAD length is defined by 'auth_aad_len' - * session parameter. - */ - uint8_t *aad_ptr; - - /** Data range to be ciphered */ - odp_packet_data_range_t cipher_range; - - /** Data range to be authenticated - * - * The value is ignored with authenticated encryption algorithms, - * such as AES-GCM, which authenticate data in the cipher range - * and the AAD. - * - * As a special case AES-GMAC uses this field instead of aad_ptr - * for the data bytes to be authenticated. - */ - odp_packet_data_range_t auth_range; - -} ODP_DEPRECATE(odp_crypto_op_param_t); - -/** * Crypto packet API per packet operation parameters */ typedef struct odp_crypto_packet_op_param_t { @@ -777,10 +690,10 @@ typedef struct odp_crypto_packet_op_param_t { odp_crypto_session_t session; /** IV pointer for cipher */ - uint8_t *cipher_iv_ptr; + const uint8_t *cipher_iv_ptr; /** IV pointer for authentication */ - uint8_t *auth_iv_ptr; + const uint8_t *auth_iv_ptr; /** Offset from start of packet for hash result * @@ -808,12 +721,31 @@ typedef struct odp_crypto_packet_op_param_t { /** Pointer to AAD. AAD length is defined by 'auth_aad_len' * session parameter. */ - uint8_t *aad_ptr; + const uint8_t *aad_ptr; - /** Data range to apply cipher */ + /** Data range to be ciphered. + * + * Ignored by the null cipher with operation types other than + * ODP_CRYPTO_OP_TYPE_OOP. Must be set to zero range (zero offset + * and zero length) with the null cipher used with the out-of-place + * operation type. + **/ odp_packet_data_range_t cipher_range; - /** Data range to authenticate */ + /** Data range to be authenticated + * + * The value is ignored with authenticated encryption algorithms, + * such as AES-GCM, which authenticate data in the cipher range + * and the AAD. + * + * Ignored by the null auth algorithm with operation types other than + * ODP_CRYPTO_OP_TYPE_OOP. Must be set to zero range (zero offset + * and zero length) with the null cipher used with the out-of-place + * operation type. + * + * As a special case AES-GMAC uses this field instead of aad_ptr + * for the data bytes to be authenticated. + */ odp_packet_data_range_t auth_range; /** Shift of the output offsets with ODP_CRYPTO_OP_TYPE_OOP @@ -871,14 +803,16 @@ typedef enum { ODP_CRYPTO_SES_ERR_PARAMS, } odp_crypto_ses_create_err_t; -/** This synonym for backward compatibility will be deprecated later */ +#if ODP_DEPRECATED_API +/** This synonym for backward compatibility has been deprecated */ #define ODP_CRYPTO_SES_CREATE_ERR_NONE ODP_CRYPTO_SES_ERR_NONE -/** This synonym for backward compatibility will be deprecated later */ +/** This synonym for backward compatibility has been deprecated */ #define ODP_CRYPTO_SES_CREATE_ERR_ENOMEM ODP_CRYPTO_SES_ERR_ENOMEM -/** This synonym for backward compatibility will be deprecated later */ +/** This synonym for backward compatibility has been deprecated */ #define ODP_CRYPTO_SES_CREATE_ERR_INV_CIPHER ODP_CRYPTO_SES_ERR_CIPHER -/** This synonym for backward compatibility will be deprecated later */ +/** This synonym for backward compatibility has been deprecated */ #define ODP_CRYPTO_SES_CREATE_ERR_INV_AUTH ODP_CRYPTO_SES_ERR_AUTH +#endif /** * Crypto API algorithm return code @@ -886,14 +820,16 @@ typedef enum { typedef enum { /** Algorithm successful */ ODP_CRYPTO_ALG_ERR_NONE, - /** Invalid data block size */ + /** Invalid range or packet size */ ODP_CRYPTO_ALG_ERR_DATA_SIZE, /** Key size invalid for algorithm */ - ODP_CRYPTO_ALG_ERR_KEY_SIZE, + ODP_DEPRECATE(ODP_CRYPTO_ALG_ERR_KEY_SIZE), /** Computed ICV value mismatch */ ODP_CRYPTO_ALG_ERR_ICV_CHECK, /** IV value not specified */ - ODP_CRYPTO_ALG_ERR_IV_INVALID, + ODP_DEPRECATE(ODP_CRYPTO_ALG_ERR_IV_INVALID), + /** Other error */ + ODP_CRYPTO_ALG_ERR_OTHER, } odp_crypto_alg_err_t; /** @@ -901,12 +837,12 @@ typedef enum { */ typedef enum { /** Operation completed successfully */ - ODP_CRYPTO_HW_ERR_NONE, + ODP_DEPRECATE(ODP_CRYPTO_HW_ERR_NONE), /** Error detected during DMA of data */ - ODP_CRYPTO_HW_ERR_DMA, + ODP_DEPRECATE(ODP_CRYPTO_HW_ERR_DMA), /** Operation failed due to pool depletion */ - ODP_CRYPTO_HW_ERR_BP_DEPLETED, -} odp_crypto_hw_err_t; + ODP_DEPRECATE(ODP_CRYPTO_HW_ERR_BP_DEPLETED), +} ODP_DEPRECATE(odp_crypto_hw_err_t); /** * Cryto API per packet operation completion status @@ -916,39 +852,18 @@ typedef struct odp_crypto_op_status { odp_crypto_alg_err_t alg_err; /** Hardware specific return code */ - odp_crypto_hw_err_t hw_err; - + ODP_DEPRECATE(odp_crypto_hw_err_t) ODP_DEPRECATE(hw_err); } odp_crypto_op_status_t; /** - * Crypto API operation result - * - * @deprecated Use odp_crypto_packet_result_t instead. - */ -typedef struct odp_crypto_op_result { - /** Request completed successfully */ - odp_bool_t ok; - - /** User context from request */ - void *ctx; - - /** Output packet */ - odp_packet_t pkt; - - /** Cipher status */ - odp_crypto_op_status_t cipher_status; - - /** Authentication status */ - odp_crypto_op_status_t auth_status; - -} ODP_DEPRECATE(odp_crypto_op_result_t); - -/** * Crypto packet API operation result */ typedef struct odp_crypto_packet_result_t { - /** Request completed successfully */ - odp_bool_t ok; + /** Request completed successfully. + * + * @deprecated Check the return value of odp_crypto_result() instead. + */ + odp_bool_t ODP_DEPRECATE(ok); /** Input packet passed to odp_crypo_op_enq() when the operation * type of the session is ODP_CRYPTO_OP_TYPE_OOP. In other cases diff --git a/include/odp/api/spec/dma_types.h b/include/odp/api/spec/dma_types.h index ae3d05287..caa51700b 100644 --- a/include/odp/api/spec/dma_types.h +++ b/include/odp/api/spec/dma_types.h @@ -69,7 +69,9 @@ extern "C" { * Pool statistics are not supported with DMA completion event pools. */ typedef struct odp_dma_pool_capability_t { - /** Maximum number of DMA completion event pools */ + /** Maximum number of DMA completion event pools + * + * See odp_pool_capability_t::max_pools for total capability. */ uint32_t max_pools; /** Maximum number of DMA completion events in a pool */ diff --git a/include/odp/api/spec/errno.h b/include/odp/api/spec/errno.h index bbd820192..98884da01 100644 --- a/include/odp/api/spec/errno.h +++ b/include/odp/api/spec/errno.h @@ -31,8 +31,7 @@ extern "C" { * may initialize errno to zero at anytime by calling odp_errno_zero(). Other * ODP functions never set errno to zero. Valid errno values are non-zero * and implementation specific. It's also implementation specific which - * functions set errno in addition to those explicitly specified by - * the API spec. ODP errno is initially zero. + * functions set errno. ODP errno is initially zero. * * @{ */ diff --git a/include/odp/api/spec/event_types.h b/include/odp/api/spec/event_types.h index 529d031e7..489119247 100644 --- a/include/odp/api/spec/event_types.h +++ b/include/odp/api/spec/event_types.h @@ -52,8 +52,6 @@ extern "C" { * packet processing related metadata * - ODP_EVENT_TIMEOUT * - Timeout event (odp_timeout_t) from a timer - * - ODP_EVENT_CRYPTO_COMPL - * - Crypto completion event (odp_crypto_compl_t) * - ODP_EVENT_IPSEC_STATUS * - IPSEC status update event (odp_ipsec_status_t) * - ODP_EVENT_PACKET_VECTOR diff --git a/include/odp/api/spec/packet_io.h b/include/odp/api/spec/packet_io.h index 9a6117cee..602093764 100644 --- a/include/odp/api/spec/packet_io.h +++ b/include/odp/api/spec/packet_io.h @@ -47,9 +47,9 @@ extern "C" { * Open a packet IO interface * * An ODP program can open a single packet IO interface per device, attempts - * to open an already open device will fail, returning ODP_PKTIO_INVALID with - * errno set. Use odp_pktio_lookup() to obtain a handle to an already open - * device. Packet IO parameters provide interface level configuration options. + * to open an already open device will fail, returning ODP_PKTIO_INVALID. Use + * odp_pktio_lookup() to obtain a handle to an already open device. Packet IO + * parameters provide interface level configuration options. * * Use odp_pktio_param_init() to initialize packet IO parameters into their * default values. Default values are also used when 'param' pointer is NULL. @@ -103,7 +103,7 @@ extern "C" { * the odp_pktio_default_cos_set() routine, or because a matching PMR * assigned the packet to a specific CoS. The default pool specified * here is applicable only for those packets that are not assigned to a - * more specific CoS. + * more specific CoS that specifies another pool. * * @see odp_pktio_start(), odp_pktio_stop(), odp_pktio_close() */ @@ -500,7 +500,7 @@ odp_lso_profile_t odp_lso_profile_create(odp_pktio_t pktio, const odp_lso_profil /** * Destroy LSO profile * - * LSO profiles can be destoyed only when the packet IO interface is not active (i.e. after it + * LSO profiles can be destroyed only when the packet IO interface is not active (i.e. after it * has been stopped). * * @param lso_profile LSO profile to be destroyed @@ -663,8 +663,8 @@ int odp_pktio_mac_addr_set(odp_pktio_t pktio, const void *mac_addr, * * @param pktio Ingress port pktio handle. * @param default_cos Class-of-service set to all packets arriving at this - * ingress port, unless overridden by subsequent - * header-based filters. + * ingress port. Use ODP_COS_INVALID to remove the default + * CoS. * * @retval 0 on success * @retval <0 on failure diff --git a/include/odp/api/spec/pool.h b/include/odp/api/spec/pool.h index b0a57a36e..a72dcd3af 100644 --- a/include/odp/api/spec/pool.h +++ b/include/odp/api/spec/pool.h @@ -59,15 +59,16 @@ odp_pool_t odp_pool_create(const char *name, const odp_pool_param_t *param); /** * Destroy a pool previously created by odp_pool_create() * + * This routine destroys a previously created pool, and will destroy any + * internal shared memory objects associated with the pool. The pool must not + * be in use (in pktio, classifier, timer, etc.) when calling this function. + * Results are undefined if an attempt is made to destroy a pool that contains + * allocated or otherwise active buffers. + * * @param pool Handle of the pool to be destroyed * * @retval 0 Success * @retval -1 Failure - * - * @note This routine destroys a previously created pool, and will destroy any - * internal shared memory objects associated with the pool. Results are - * undefined if an attempt is made to destroy a pool that contains allocated - * or otherwise active buffers. */ int odp_pool_destroy(odp_pool_t pool); diff --git a/include/odp/api/spec/pool_types.h b/include/odp/api/spec/pool_types.h index 9f433fba5..e9b1f9d52 100644 --- a/include/odp/api/spec/pool_types.h +++ b/include/odp/api/spec/pool_types.h @@ -151,7 +151,7 @@ typedef struct odp_pool_stats_t { * Pool capabilities */ typedef struct odp_pool_capability_t { - /** Maximum number of pools of any type */ + /** Maximum number of pools of any type (odp_pool_type_t) */ uint32_t max_pools; /** Buffer pool capabilities */ diff --git a/include/odp/api/spec/queue.h b/include/odp/api/spec/queue.h index 65cf3d1ee..0315d1312 100644 --- a/include/odp/api/spec/queue.h +++ b/include/odp/api/spec/queue.h @@ -50,9 +50,9 @@ odp_queue_t odp_queue_create(const char *name, const odp_queue_param_t *param); * Destroy ODP queue * * Destroys ODP queue. The queue must be empty and detached from other - * ODP API (crypto, pktio, etc). Application must ensure that no other - * operations on this queue are invoked in parallel. Otherwise behavior - * is undefined. + * ODP API (crypto, pktio, classifier, timer, etc). Application must ensure + * that no other operations on this queue are invoked in parallel. Otherwise + * behavior is undefined. * * @param queue Queue handle * diff --git a/include/odp/api/spec/stash.h b/include/odp/api/spec/stash.h index f5929c45a..2453ed6bb 100644 --- a/include/odp/api/spec/stash.h +++ b/include/odp/api/spec/stash.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2020-2022, Nokia +/* Copyright (c) 2020-2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -109,8 +109,7 @@ uint64_t odp_stash_to_u64(odp_stash_t stash); * Store object handles into the stash. Handle values are opaque data to * ODP implementation and may be e.g. pointers or indexes to arbitrary objects. * Application specifies object handle size and maximum number of handles to be - * stored in stash creation parameters. Application must not attempt to store - * more handles into the stash than it specifies in the creation parameters. + * stored in stash creation parameters. * * A successful operation returns the actual number of object handles stored. * If the return value is less than 'num', the remaining handles at the end of diff --git a/include/odp/api/spec/stash_types.h b/include/odp/api/spec/stash_types.h index 5f3e608bb..30fff26c4 100644 --- a/include/odp/api/spec/stash_types.h +++ b/include/odp/api/spec/stash_types.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2020-2022, Nokia +/* Copyright (c) 2020-2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -156,13 +156,38 @@ typedef struct odp_stash_capability_t { */ uint32_t max_stashes; - /** Maximum number of object handles per stash + /** Maximum common number of object handles per stash for any object size * - * The value of zero means that limited only by the available - * memory size. + * An application is able to store at least this many objects when using any of the + * supported object sizes. Some of the per object size values listed in 'max_num' may be + * larger than this common value. */ uint64_t max_num_obj; + /** Maximum number of object handles per stash for each object size + * + * Values for unsupported object handle sizes are set to zero. + */ + struct { + /** Maximum number of 1 byte object handles */ + uint64_t u8; + + /** Maximum number of 2 byte object handles */ + uint64_t u16; + + /** Maximum number of 4 byte object handles */ + uint64_t u32; + + /** Maximum number of 8 byte object handles */ + uint64_t u64; + + /** Maximum number of 16 byte object handles */ + uint64_t u128; + + /** Maximum number of 'max_obj_size' object handles */ + uint64_t max_obj_size; + } max_num; + /** Maximum object handle size in bytes * * At least 4 byte object handle size is always supported. @@ -219,10 +244,11 @@ typedef struct odp_stash_param_t { */ odp_stash_op_mode_t get_mode; - /** Maximum number of object handles + /** Number of object handles * - * This is the maximum number of object handles application will store - * in the stash. The value must not exceed 'max_num_obj' capability. + * Application must be able to store at least this many object handles + * into the stash. An implementation may round up the value. The given + * value must not exceed 'max_num' capability. */ uint64_t num_obj; @@ -265,6 +291,15 @@ typedef struct odp_stash_param_t { */ odp_stash_stats_opt_t stats; + /** + * Strict size + * + * If true, application never attempts to store more handles into the stash than specified + * in the 'num_obj' parameter. Implementation may use this value as a hint for performance + * optimizations. The default value is false. + */ + odp_bool_t strict_size; + } odp_stash_param_t; /** diff --git a/include/odp/api/spec/timer.h b/include/odp/api/spec/timer.h index 0e1f07f09..7982e391b 100644 --- a/include/odp/api/spec/timer.h +++ b/include/odp/api/spec/timer.h @@ -123,7 +123,7 @@ void odp_timer_pool_param_init(odp_timer_pool_param_t *param); * @param params Timer pool parameters. The content will be copied. * * @return Timer pool handle on success - * @retval ODP_TIMER_POOL_INVALID on failure and errno set + * @retval ODP_TIMER_POOL_INVALID on failure */ odp_timer_pool_t odp_timer_pool_create(const char *name, const odp_timer_pool_param_t *params); @@ -209,7 +209,7 @@ int odp_timer_pool_info(odp_timer_pool_t timer_pool, * @param user_ptr User defined pointer or NULL to be copied to timeouts * * @return Timer handle on success - * @retval ODP_TIMER_INVALID on failure and errno set. + * @retval ODP_TIMER_INVALID on failure */ odp_timer_t odp_timer_alloc(odp_timer_pool_t timer_pool, odp_queue_t queue, const void *user_ptr); diff --git a/include/odp/api/spec/timer_types.h b/include/odp/api/spec/timer_types.h index d61992fba..fd23bdeca 100644 --- a/include/odp/api/spec/timer_types.h +++ b/include/odp/api/spec/timer_types.h @@ -476,9 +476,10 @@ typedef struct odp_timer_start_t { typedef struct odp_timer_periodic_start_t { /** First expiration time * - * The first expiration time in absolute timer ticks. When zero, timer expiration starts - * from the current time. After the first expiration, timer expiration continues with the - * defined frequency. The tick value must be within one timer period from the current time. + * The first expiration time in absolute timer ticks. When zero, the first expiration time + * is one period after the current time. After the first expiration, timer expiration + * continues with the defined frequency. The tick value must be within one timer period + * from the current time. */ uint64_t first_tick; diff --git a/include/odp/autoheader_internal.h.in b/include/odp/autoheader_internal.h.in index a42b34ad0..87cefa131 100644 --- a/include/odp/autoheader_internal.h.in +++ b/include/odp/autoheader_internal.h.in @@ -17,9 +17,6 @@ /* Define to 1 to enable DPDK zero copy support */ #undef _ODP_DPDK_ZERO_COPY -/* Define to 1 to enable netmap packet I/O support */ -#undef _ODP_PKTIO_NETMAP - /* Define to 1 to enable pcap packet I/O support */ #undef _ODP_PKTIO_PCAP diff --git a/m4/odp_libconfig.m4 b/m4/odp_libconfig.m4 index 29cc8c85b..31619934c 100644 --- a/m4/odp_libconfig.m4 +++ b/m4/odp_libconfig.m4 @@ -5,7 +5,8 @@ AC_DEFUN([ODP_LIBCONFIG], ########################################################################## # Check for libconfig availability ########################################################################## -PKG_CHECK_MODULES([LIBCONFIG], [libconfig]) +PKG_CHECK_MODULES([LIBCONFIG], [libconfig], [have_libconfig=yes]) +AM_CONDITIONAL([LIBCONFIG], [test x$have_libconfig = xyes]) ########################################################################## # Check for od availability diff --git a/platform/linux-generic/Makefile.am b/platform/linux-generic/Makefile.am index d49a2138b..324507ee5 100644 --- a/platform/linux-generic/Makefile.am +++ b/platform/linux-generic/Makefile.am @@ -12,7 +12,6 @@ AM_CPPFLAGS += -I$(top_srcdir)/platform/$(with_platform)/arch/@ARCH_DIR@ AM_CPPFLAGS += -I$(top_srcdir)/platform/$(with_platform)/arch/default AM_CPPFLAGS += $(OPENSSL_CPPFLAGS) -AM_CPPFLAGS += $(NETMAP_CPPFLAGS) AM_CFLAGS += $(AARCH64CRYPTO_CFLAGS) AM_CFLAGS += $(DPDK_CFLAGS) @@ -270,7 +269,6 @@ __LIB__libodp_linux_la_SOURCES = \ pktio/io_ops.c \ pktio/ipc.c \ pktio/loop.c \ - pktio/netmap.c \ pktio/null.c \ pktio/pktio_common.c \ pktio/socket.c \ diff --git a/platform/linux-generic/README b/platform/linux-generic/README index 138e6040c..8fa50c127 100644 --- a/platform/linux-generic/README +++ b/platform/linux-generic/README @@ -29,17 +29,17 @@ SPDX-License-Identifier: BSD-3-Clause E.g. socket:eth1 - netmap:eth2 + socket_xdp:eth2 The supported pktio types are: dpdk ipc loop - netmap null pcap socket socket_mmap + socket_xdp tap 5. Random data diff --git a/platform/linux-generic/arch/aarch64/odp/api/abi/atomic_inlines.h b/platform/linux-generic/arch/aarch64/odp/api/abi/atomic_inlines.h index e8f33f09e..3b0f94efe 100644 --- a/platform/linux-generic/arch/aarch64/odp/api/abi/atomic_inlines.h +++ b/platform/linux-generic/arch/aarch64/odp/api/abi/atomic_inlines.h @@ -205,6 +205,34 @@ static inline void _odp_atomic_dec_u64(odp_atomic_u64_t *atom) _odp_atomic_sub_u64(atom, 1); } +static inline void _odp_atomic_max_u32(odp_atomic_u32_t *atom, uint32_t val) +{ + __asm__ volatile("stumax %w[val], %[atom]" + : [atom] "+Q" (atom->v) + : [val] "r" (val)); +} + +static inline void _odp_atomic_min_u32(odp_atomic_u32_t *atom, uint32_t val) +{ + __asm__ volatile("stumin %w[val], %[atom]" + : [atom] "+Q" (atom->v) + : [val] "r" (val)); +} + +static inline void _odp_atomic_max_u64(odp_atomic_u64_t *atom, uint64_t val) +{ + __asm__ volatile("stumax %[val], %[atom]" + : [atom] "+Q" (atom->v) + : [val] "r" (val)); +} + +static inline void _odp_atomic_min_u64(odp_atomic_u64_t *atom, uint64_t val) +{ + __asm__ volatile("stumin %[val], %[atom]" + : [atom] "+Q" (atom->v) + : [val] "r" (val)); +} + static inline void _odp_atomic_add_rel_u32(odp_atomic_u32_t *atom, uint32_t val) { __asm__ volatile("staddl %w[val], %[atom]" diff --git a/platform/linux-generic/arch/aarch64/odp_crypto_armv8.c b/platform/linux-generic/arch/aarch64/odp_crypto_armv8.c index 67ae6a389..2eefc6302 100644 --- a/platform/linux-generic/arch/aarch64/odp_crypto_armv8.c +++ b/platform/linux-generic/arch/aarch64/odp_crypto_armv8.c @@ -181,11 +181,7 @@ static inline void set_crypto_op_result(odp_packet_t pkt, op_result = &packet_hdr(pkt)->crypto_op_result; op_result->cipher_status.alg_err = cipher_err; - op_result->cipher_status.hw_err = ODP_CRYPTO_HW_ERR_NONE; op_result->auth_status.alg_err = auth_err; - op_result->auth_status.hw_err = ODP_CRYPTO_HW_ERR_NONE; - op_result->ok = (cipher_err == ODP_CRYPTO_ALG_ERR_NONE && - auth_err == ODP_CRYPTO_ALG_ERR_NONE); } static inline void set_crypto_op_result_ok(odp_packet_t pkt) @@ -203,7 +199,7 @@ null_crypto_routine(odp_packet_t pkt ODP_UNUSED, set_crypto_op_result_ok(pkt); } -static inline void copy_aad(uint8_t *dst, uint8_t *src, uint32_t len) +static inline void copy_aad(uint8_t *dst, const uint8_t *src, uint32_t len) { _ODP_ASSERT(len == 8 || len == 12); @@ -661,66 +657,6 @@ int odp_crypto_session_destroy(odp_crypto_session_t session) return 0; } -#if ODP_DEPRECATED_API -int -odp_crypto_operation(odp_crypto_op_param_t *param, - odp_bool_t *posted, - odp_crypto_op_result_t *result) -{ - odp_crypto_packet_op_param_t packet_param; - odp_packet_t out_pkt = param->out_pkt; - odp_crypto_packet_result_t packet_result; - odp_crypto_op_result_t local_result; - int rc; - - if (((odp_crypto_generic_session_t *)(intptr_t)param->session)->p.op_type != - ODP_CRYPTO_OP_TYPE_LEGACY) - return -1; - - packet_param.session = param->session; - packet_param.cipher_iv_ptr = param->cipher_iv_ptr; - packet_param.auth_iv_ptr = param->auth_iv_ptr; - packet_param.hash_result_offset = param->hash_result_offset; - packet_param.aad_ptr = param->aad_ptr; - packet_param.cipher_range = param->cipher_range; - packet_param.auth_range = param->auth_range; - - rc = odp_crypto_op(¶m->pkt, &out_pkt, &packet_param, 1); - if (rc <= 0) - return -1; - - rc = odp_crypto_result(&packet_result, out_pkt); - if (rc < 0) { - /* - * We cannot fail since odp_crypto_op() has already processed - * the packet. Let's indicate error in the result instead. - */ - packet_result.ok = false; - } - - /* Indicate to caller operation was sync */ - *posted = 0; - - packet_subtype_set(out_pkt, ODP_EVENT_PACKET_BASIC); - - /* Fill in result */ - local_result.ctx = param->ctx; - local_result.pkt = out_pkt; - local_result.cipher_status = packet_result.cipher_status; - local_result.auth_status = packet_result.auth_status; - local_result.ok = packet_result.ok; - - /* - * Be bug-to-bug compatible. Return output packet also through params. - */ - param->out_pkt = out_pkt; - - *result = local_result; - - return 0; -} -#endif - int _odp_crypto_init_global(void) { size_t mem_size; @@ -800,45 +736,6 @@ int _odp_crypto_term_local(void) return 0; } -#if ODP_DEPRECATED_API -odp_crypto_compl_t odp_crypto_compl_from_event(odp_event_t ev) -{ - /* This check not mandated by the API specification */ - if (odp_event_type(ev) != ODP_EVENT_CRYPTO_COMPL) - _ODP_ABORT("Event not a crypto completion"); - return (odp_crypto_compl_t)ev; -} - -odp_event_t odp_crypto_compl_to_event(odp_crypto_compl_t completion_event) -{ - return (odp_event_t)completion_event; -} - -void -odp_crypto_compl_result(odp_crypto_compl_t completion_event, - odp_crypto_op_result_t *result) -{ - (void)completion_event; - (void)result; - - /* We won't get such events anyway, so there can be no result */ - _ODP_ASSERT(0); -} - -void -odp_crypto_compl_free(odp_crypto_compl_t completion_event) -{ - odp_event_t ev = odp_crypto_compl_to_event(completion_event); - - odp_buffer_free(odp_buffer_from_event(ev)); -} - -uint64_t odp_crypto_compl_to_u64(odp_crypto_compl_t hdl) -{ - return _odp_pri(hdl); -} -#endif /* ODP_DEPRECATED_API */ - void odp_crypto_session_param_init(odp_crypto_session_param_t *param) { memset(param, 0, sizeof(odp_crypto_session_param_t)); @@ -877,9 +774,6 @@ static odp_packet_t get_output_packet(const odp_crypto_generic_session_t *sessio { int rc; - if (odp_likely(session->p.op_type == ODP_CRYPTO_OP_TYPE_BASIC)) - return pkt_in; - if (odp_likely(pkt_in == pkt_out)) return pkt_out; @@ -914,9 +808,13 @@ int crypto_int(odp_packet_t pkt_in, session = (odp_crypto_generic_session_t *)(intptr_t)param->session; - out_pkt = get_output_packet(session, pkt_in, *pkt_out); - if (odp_unlikely(out_pkt == ODP_PACKET_INVALID)) - return -1; + if (odp_likely(session->p.op_type == ODP_CRYPTO_OP_TYPE_BASIC)) { + out_pkt = pkt_in; + } else { + out_pkt = get_output_packet(session, pkt_in, *pkt_out); + if (odp_unlikely(out_pkt == ODP_PACKET_INVALID)) + return -1; + } /* Invoke the crypto function */ session->func(out_pkt, param, session); diff --git a/platform/linux-generic/arch/default/odp/api/abi/atomic_generic.h b/platform/linux-generic/arch/default/odp/api/abi/atomic_generic.h index 2a1bccbc4..af435e495 100644 --- a/platform/linux-generic/arch/default/odp/api/abi/atomic_generic.h +++ b/platform/linux-generic/arch/default/odp/api/abi/atomic_generic.h @@ -30,6 +30,32 @@ static inline void _odp_atomic_dec_u32(odp_atomic_u32_t *atom) (void)__atomic_fetch_sub(&atom->v, 1, __ATOMIC_RELAXED); } +static inline void _odp_atomic_max_u32(odp_atomic_u32_t *atom, uint32_t new_val) +{ + uint32_t old_val; + + old_val = __atomic_load_n(&atom->v, __ATOMIC_RELAXED); + + while (new_val > old_val) { + if (__atomic_compare_exchange_n(&atom->v, &old_val, new_val, 0 /* strong */, + __ATOMIC_RELAXED, __ATOMIC_RELAXED)) + break; + } +} + +static inline void _odp_atomic_min_u32(odp_atomic_u32_t *atom, uint32_t new_val) +{ + uint32_t old_val; + + old_val = __atomic_load_n(&atom->v, __ATOMIC_RELAXED); + + while (new_val < old_val) { + if (__atomic_compare_exchange_n(&atom->v, &old_val, new_val, 0 /* strong */, + __ATOMIC_RELAXED, __ATOMIC_RELAXED)) + break; + } +} + static inline void _odp_atomic_add_rel_u32(odp_atomic_u32_t *atom, uint32_t val) { (void)__atomic_fetch_add(&atom->v, val, __ATOMIC_RELEASE); @@ -60,6 +86,32 @@ static inline void _odp_atomic_dec_u64(odp_atomic_u64_t *atom) (void)__atomic_fetch_sub(&atom->v, 1, __ATOMIC_RELAXED); } +static inline void _odp_atomic_max_u64(odp_atomic_u64_t *atom, uint64_t new_val) +{ + uint64_t old_val; + + old_val = __atomic_load_n(&atom->v, __ATOMIC_RELAXED); + + while (new_val > old_val) { + if (__atomic_compare_exchange_n(&atom->v, &old_val, new_val, 0 /* strong */, + __ATOMIC_RELAXED, __ATOMIC_RELAXED)) + break; + } +} + +static inline void _odp_atomic_min_u64(odp_atomic_u64_t *atom, uint64_t new_val) +{ + uint64_t old_val; + + old_val = __atomic_load_n(&atom->v, __ATOMIC_RELAXED); + + while (new_val < old_val) { + if (__atomic_compare_exchange_n(&atom->v, &old_val, new_val, 0 /* strong */, + __ATOMIC_RELAXED, __ATOMIC_RELAXED)) + break; + } +} + #ifndef ODP_ATOMIC_U64_LOCK static inline void _odp_atomic_add_rel_u64(odp_atomic_u64_t *atom, uint64_t val) { diff --git a/platform/linux-generic/include-abi/odp/api/abi/crypto_types.h b/platform/linux-generic/include-abi/odp/api/abi/crypto_types.h index a5cb43c5d..d49caf89a 100644 --- a/platform/linux-generic/include-abi/odp/api/abi/crypto_types.h +++ b/platform/linux-generic/include-abi/odp/api/abi/crypto_types.h @@ -29,7 +29,6 @@ extern "C" { #define ODP_CRYPTO_SESSION_INVALID (0xffffffffffffffffULL) typedef uint64_t odp_crypto_session_t; -typedef ODP_HANDLE_T(odp_crypto_compl_t); /** * @} diff --git a/platform/linux-generic/include-abi/odp/api/abi/event_types.h b/platform/linux-generic/include-abi/odp/api/abi/event_types.h index c39322330..7eb539827 100644 --- a/platform/linux-generic/include-abi/odp/api/abi/event_types.h +++ b/platform/linux-generic/include-abi/odp/api/abi/event_types.h @@ -33,9 +33,6 @@ typedef enum odp_event_type_t { ODP_EVENT_BUFFER = 1, ODP_EVENT_PACKET = 2, ODP_EVENT_TIMEOUT = 3, -#if ODP_DEPRECATED_API - ODP_EVENT_CRYPTO_COMPL = 4, -#endif ODP_EVENT_IPSEC_STATUS = 5, ODP_EVENT_PACKET_VECTOR = 6, ODP_EVENT_PACKET_TX_COMPL = 7, diff --git a/platform/linux-generic/include/odp/api/plat/atomic_inlines.h b/platform/linux-generic/include/odp/api/plat/atomic_inlines.h index 5ce4bfc28..e47559102 100644 --- a/platform/linux-generic/include/odp/api/plat/atomic_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/atomic_inlines.h @@ -149,28 +149,14 @@ _ODP_INLINE uint32_t odp_atomic_xchg_u32(odp_atomic_u32_t *atom, return __atomic_exchange_n(&atom->v, new_val, __ATOMIC_RELAXED); } -_ODP_INLINE void odp_atomic_max_u32(odp_atomic_u32_t *atom, uint32_t new_max) +_ODP_INLINE void odp_atomic_max_u32(odp_atomic_u32_t *atom, uint32_t val) { - uint32_t old_val; - - old_val = odp_atomic_load_u32(atom); - - while (new_max > old_val) { - if (odp_atomic_cas_u32(atom, &old_val, new_max)) - break; - } + _odp_atomic_max_u32(atom, val); } -_ODP_INLINE void odp_atomic_min_u32(odp_atomic_u32_t *atom, uint32_t new_min) +_ODP_INLINE void odp_atomic_min_u32(odp_atomic_u32_t *atom, uint32_t val) { - uint32_t old_val; - - old_val = odp_atomic_load_u32(atom); - - while (new_min < old_val) { - if (odp_atomic_cas_u32(atom, &old_val, new_min)) - break; - } + _odp_atomic_min_u32(atom, val); } #ifdef ODP_ATOMIC_U64_LOCK @@ -325,6 +311,30 @@ _ODP_INLINE int odp_atomic_cas_acq_rel_u64(odp_atomic_u64_t *atom, return ret; } +_ODP_INLINE void odp_atomic_max_u64(odp_atomic_u64_t *atom, uint64_t new_val) +{ + uint64_t old_val; + + old_val = odp_atomic_load_u64(atom); + + while (new_val > old_val) { + if (odp_atomic_cas_u64(atom, &old_val, new_val)) + break; + } +} + +_ODP_INLINE void odp_atomic_min_u64(odp_atomic_u64_t *atom, uint64_t new_val) +{ + uint64_t old_val; + + old_val = odp_atomic_load_u64(atom); + + while (new_val < old_val) { + if (odp_atomic_cas_u64(atom, &old_val, new_val)) + break; + } +} + #else /* !ODP_ATOMIC_U64_LOCK */ _ODP_INLINE void odp_atomic_init_u64(odp_atomic_u64_t *atom, uint64_t val) @@ -447,32 +457,18 @@ _ODP_INLINE int odp_atomic_cas_acq_rel_u64(odp_atomic_u64_t *atom, __ATOMIC_RELAXED); } -#endif /* !ODP_ATOMIC_U64_LOCK */ - -_ODP_INLINE void odp_atomic_max_u64(odp_atomic_u64_t *atom, uint64_t new_max) +_ODP_INLINE void odp_atomic_max_u64(odp_atomic_u64_t *atom, uint64_t val) { - uint64_t old_val; - - old_val = odp_atomic_load_u64(atom); - - while (new_max > old_val) { - if (odp_atomic_cas_u64(atom, &old_val, new_max)) - break; - } + _odp_atomic_max_u64(atom, val); } -_ODP_INLINE void odp_atomic_min_u64(odp_atomic_u64_t *atom, uint64_t new_min) +_ODP_INLINE void odp_atomic_min_u64(odp_atomic_u64_t *atom, uint64_t val) { - uint64_t old_val; - - old_val = odp_atomic_load_u64(atom); - - while (new_min < old_val) { - if (odp_atomic_cas_u64(atom, &old_val, new_min)) - break; - } + _odp_atomic_min_u64(atom, val); } +#endif /* !ODP_ATOMIC_U64_LOCK */ + _ODP_INLINE uint32_t odp_atomic_load_acq_u32(odp_atomic_u32_t *atom) { return __atomic_load_n(&atom->v, __ATOMIC_ACQUIRE); diff --git a/platform/linux-generic/include/odp/api/plat/crypto_inlines.h b/platform/linux-generic/include/odp/api/plat/crypto_inlines.h index 8e98d8580..ddf7debf4 100644 --- a/platform/linux-generic/include/odp/api/plat/crypto_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/crypto_inlines.h @@ -46,14 +46,25 @@ _ODP_INLINE odp_event_t odp_crypto_packet_to_event(odp_packet_t pkt) _ODP_INLINE int odp_crypto_result(odp_crypto_packet_result_t *result, odp_packet_t pkt) { odp_crypto_packet_result_t *op_result; + odp_bool_t ok; _ODP_ASSERT(odp_packet_subtype(pkt) == ODP_EVENT_PACKET_CRYPTO); op_result = _odp_pkt_get_ptr(pkt, odp_crypto_packet_result_t, crypto_op); - *result = *op_result; + ok = op_result->cipher_status.alg_err == ODP_CRYPTO_ALG_ERR_NONE && + op_result->auth_status.alg_err == ODP_CRYPTO_ALG_ERR_NONE; - return 0; + if (result) { + *result = *op_result; +#if ODP_DEPRECATED_API + result->ok = ok; + result->cipher_status.hw_err = ODP_CRYPTO_HW_ERR_NONE; + result->auth_status.hw_err = ODP_CRYPTO_HW_ERR_NONE; +#endif + } + + return ok ? 0 : -1; } /** @endcond */ diff --git a/platform/linux-generic/include/odp/api/plat/debug_inlines.h b/platform/linux-generic/include/odp/api/plat/debug_inlines.h index bada48f9b..41af3dca4 100644 --- a/platform/linux-generic/include/odp/api/plat/debug_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/debug_inlines.h @@ -30,26 +30,28 @@ extern "C" { #endif -/* Avoid "ISO C99 requires at least one argument for the "..." in a variadic - * macro" errors when building with 'pedantic' option. */ -#pragma GCC system_header +#pragma GCC diagnostic push + +#ifdef __clang__ +#pragma GCC diagnostic ignored "-Wgnu-zero-variadic-macro-arguments" +#endif extern odp_log_func_t ODP_PRINTF_FORMAT(2, 3) _odp_log_fn; extern odp_abort_func_t _odp_abort_fn; -#define _ODP_LOG_FN(level, fmt, ...) \ +#define _ODP_LOG_FN(level, ...) \ do { \ if (_odp_this_thread && _odp_this_thread->log_fn) \ - _odp_this_thread->log_fn(level, fmt, ##__VA_ARGS__); \ + _odp_this_thread->log_fn(level, ##__VA_ARGS__); \ else \ - _odp_log_fn(level, fmt, ##__VA_ARGS__); \ + _odp_log_fn(level, ##__VA_ARGS__); \ } while (0) /** * ODP LOG macro. */ #define _ODP_LOG(level, fmt, ...) \ - _ODP_LOG_FN(level, "%s:%d:%s():" fmt, __FILE__, \ + _ODP_LOG_FN(level, "%s:%d:%s(): " fmt, __FILE__, \ __LINE__, __func__, ##__VA_ARGS__) /** @@ -64,25 +66,33 @@ extern odp_abort_func_t _odp_abort_fn; /* * Print debug message to log, if ODP_DEBUG_PRINT flag is set (ignores CONFIG_DEBUG_LEVEL). */ -#define _ODP_DBG(fmt, ...) \ +#define _ODP_DBG(...) \ do { \ if (ODP_DEBUG_PRINT == 1) \ - _ODP_LOG(ODP_LOG_DBG, fmt, ##__VA_ARGS__);\ + __extension__ ({ \ + _ODP_LOG(ODP_LOG_DBG, ##__VA_ARGS__); \ + }); \ } while (0) /** * Log error message. */ -#define _ODP_ERR(fmt, ...) \ - _ODP_LOG(ODP_LOG_ERR, fmt, ##__VA_ARGS__) +#define _ODP_ERR(...) \ + do { \ + __extension__ ({ \ + _ODP_LOG(ODP_LOG_ERR, ##__VA_ARGS__); \ + }); \ + } while (0) /** * Log abort message and then stop execution (by default call abort()). * This function should not return. */ -#define _ODP_ABORT(fmt, ...) \ +#define _ODP_ABORT(...) \ do { \ - _ODP_LOG(ODP_LOG_ABORT, fmt, ##__VA_ARGS__); \ + __extension__ ({ \ + _ODP_LOG(ODP_LOG_ABORT, ##__VA_ARGS__); \ + }); \ _odp_abort_fn(); \ } while (0) @@ -90,8 +100,10 @@ extern odp_abort_func_t _odp_abort_fn; * Log print message when the application calls one of the ODP APIs * specifically for dumping internal data. */ -#define _ODP_PRINT(fmt, ...) \ - _ODP_LOG_FN(ODP_LOG_PRINT, fmt, ##__VA_ARGS__) +#define _ODP_PRINT(...) \ + _ODP_LOG_FN(ODP_LOG_PRINT, ##__VA_ARGS__) + +#pragma GCC diagnostic pop #ifdef __cplusplus } diff --git a/platform/linux-generic/include/odp/api/plat/packet_inlines.h b/platform/linux-generic/include/odp/api/plat/packet_inlines.h index 93e95e21c..f03c88e10 100644 --- a/platform/linux-generic/include/odp/api/plat/packet_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/packet_inlines.h @@ -650,8 +650,9 @@ _ODP_INLINE void *odp_packet_buf_head(odp_packet_buf_t pkt_buf) _ODP_INLINE uint32_t odp_packet_buf_data_offset(odp_packet_buf_t pkt_buf) { - return (uint32_t)((uintptr_t)_odp_pkt_get(pkt_buf, void *, seg_data) - - (uintptr_t)odp_packet_buf_head(pkt_buf)); + void *buf_head = odp_packet_buf_head(pkt_buf); + + return (uint32_t)((uintptr_t)_odp_pkt_get(pkt_buf, void *, seg_data) - (uintptr_t)buf_head); } _ODP_INLINE void odp_packet_buf_data_set(odp_packet_buf_t pkt_buf, uint32_t data_offset, diff --git a/platform/linux-generic/include/odp_debug_internal.h b/platform/linux-generic/include/odp_debug_internal.h index 4184d5170..4c1fa8bbb 100644 --- a/platform/linux-generic/include/odp_debug_internal.h +++ b/platform/linux-generic/include/odp_debug_internal.h @@ -30,9 +30,11 @@ extern "C" { #endif -/* Avoid "ISO C99 requires at least one argument for the "..." in a variadic - * macro" errors when building with 'pedantic' option. */ -#pragma GCC system_header +#pragma GCC diagnostic push + +#ifdef __clang__ +#pragma GCC diagnostic ignored "-Wgnu-zero-variadic-macro-arguments" +#endif /* Debug level configure option. Zero is the highest level. Value of N prints debug messages from * level 0 to N. */ @@ -49,21 +51,25 @@ extern "C" { /* * Print debug message to log, if ODP_DEBUG_PRINT flag is set and CONFIG_DEBUG_LEVEL is high enough. */ -#define ODP_DBG_LVL(level, fmt, ...) \ +#define ODP_DBG_LVL(level, ...) \ do { \ if (ODP_DEBUG_PRINT == 1 && CONFIG_DEBUG_LEVEL >= (level)) \ - _ODP_LOG(ODP_LOG_DBG, fmt, ##__VA_ARGS__);\ + __extension__ ({ \ + _ODP_LOG(ODP_LOG_DBG, ##__VA_ARGS__); \ + }); \ } while (0) /* * Same as ODP_DBG_LVL() but does not add file/line/function name prefix */ -#define ODP_DBG_RAW(level, fmt, ...) \ +#define ODP_DBG_RAW(level, ...) \ do { \ if (ODP_DEBUG_PRINT == 1 && CONFIG_DEBUG_LEVEL >= (level)) \ - _ODP_LOG_FN(ODP_LOG_DBG, fmt, ##__VA_ARGS__);\ + _ODP_LOG_FN(ODP_LOG_DBG, ##__VA_ARGS__); \ } while (0) +#pragma GCC diagnostic pop + #ifdef __cplusplus } #endif diff --git a/platform/linux-generic/include/odp_packet_io_internal.h b/platform/linux-generic/include/odp_packet_io_internal.h index 187a3a76f..9caf3c2e9 100644 --- a/platform/linux-generic/include/odp_packet_io_internal.h +++ b/platform/linux-generic/include/odp_packet_io_internal.h @@ -62,9 +62,7 @@ ODP_STATIC_ASSERT(PKTIO_LSO_PROFILES < UINT8_MAX, "PKTIO_LSO_PROFILES_ERROR"); /* Forward declaration */ struct pktio_if_ops; -#if defined(_ODP_PKTIO_NETMAP) -#define PKTIO_PRIVATE_SIZE 74752 -#elif defined(_ODP_PKTIO_XDP) && ODP_CACHE_LINE_SIZE == 128 +#if defined(_ODP_PKTIO_XDP) && ODP_CACHE_LINE_SIZE == 128 #define PKTIO_PRIVATE_SIZE 33792 #elif defined(_ODP_PKTIO_XDP) #define PKTIO_PRIVATE_SIZE 29696 @@ -292,7 +290,6 @@ static inline void _odp_pktio_tx_ts_set(pktio_entry_t *entry) odp_atomic_store_u64(&entry->tx_ts, ts_val.u64); } -extern const pktio_if_ops_t _odp_netmap_pktio_ops; extern const pktio_if_ops_t _odp_dpdk_pktio_ops; extern const pktio_if_ops_t _odp_sock_xdp_pktio_ops; extern const pktio_if_ops_t _odp_sock_mmsg_pktio_ops; diff --git a/platform/linux-generic/include/odp_schedule_if.h b/platform/linux-generic/include/odp_schedule_if.h index c70fdcf7c..07f935bc9 100644 --- a/platform/linux-generic/include/odp_schedule_if.h +++ b/platform/linux-generic/include/odp_schedule_if.h @@ -14,6 +14,7 @@ extern "C" { #include <odp/api/queue.h> #include <odp/api/schedule.h> +#include <odp/api/plat/schedule_inline_types.h> #include <odp_event_internal.h> #include <odp_queue_if.h> @@ -59,6 +60,7 @@ typedef void (*schedule_order_lock_start_fn_t)(void); typedef void (*schedule_order_lock_wait_fn_t)(void); typedef uint32_t (*schedule_max_ordered_locks_fn_t)(void); typedef void (*schedule_get_config_fn_t)(schedule_config_t *config); +typedef const _odp_schedule_api_fn_t *(*schedule_sched_api_fn_t)(void); typedef struct schedule_fn_t { schedule_pktio_start_fn_t pktio_start; @@ -80,6 +82,7 @@ typedef struct schedule_fn_t { schedule_order_unlock_lock_fn_t order_unlock_lock; schedule_max_ordered_locks_fn_t max_ordered_locks; schedule_get_config_fn_t get_config; + schedule_sched_api_fn_t sched_api; } schedule_fn_t; diff --git a/platform/linux-generic/m4/configure.m4 b/platform/linux-generic/m4/configure.m4 index 515c85239..a48d3e4dd 100644 --- a/platform/linux-generic/m4/configure.m4 +++ b/platform/linux-generic/m4/configure.m4 @@ -26,7 +26,6 @@ m4_include([platform/linux-generic/m4/odp_openssl.m4]) m4_include([platform/linux-generic/m4/odp_crypto.m4]) m4_include([platform/linux-generic/m4/odp_ipsec_mb.m4]) m4_include([platform/linux-generic/m4/odp_pcapng.m4]) -m4_include([platform/linux-generic/m4/odp_netmap.m4]) m4_include([platform/linux-generic/m4/odp_dpdk.m4]) m4_include([platform/linux-generic/m4/odp_xdp.m4]) ODP_EVENT_VALIDATION diff --git a/platform/linux-generic/m4/odp_libconfig.m4 b/platform/linux-generic/m4/odp_libconfig.m4 index 0d268c935..a6d19f661 100644 --- a/platform/linux-generic/m4/odp_libconfig.m4 +++ b/platform/linux-generic/m4/odp_libconfig.m4 @@ -3,7 +3,7 @@ ########################################################################## m4_define([_odp_config_version_generation], [0]) m4_define([_odp_config_version_major], [1]) -m4_define([_odp_config_version_minor], [25]) +m4_define([_odp_config_version_minor], [27]) m4_define([_odp_config_version], [_odp_config_version_generation._odp_config_version_major._odp_config_version_minor]) diff --git a/platform/linux-generic/m4/odp_netmap.m4 b/platform/linux-generic/m4/odp_netmap.m4 deleted file mode 100644 index 7688c080e..000000000 --- a/platform/linux-generic/m4/odp_netmap.m4 +++ /dev/null @@ -1,49 +0,0 @@ -########################################################################## -# Enable netmap support -########################################################################## -netmap_support=no -AC_ARG_ENABLE([netmap_support], - [AS_HELP_STRING([--enable-netmap-support], [include netmap IO support] - [[default=disabled] (linux-generic)])], - [if test x$enableval = xyes; then - netmap_support=yes - fi]) - -########################################################################## -# Set optional netmap path -########################################################################## -AC_ARG_WITH([netmap-path], -AS_HELP_STRING([--with-netmap-path=DIR], [path to netmap root directory] - [[default=system] (linux-generic)]), - [NETMAP_PATH=$withval - NETMAP_CPPFLAGS="-isystem $NETMAP_PATH/sys" - netmap_support=yes],[]) - -########################################################################## -# Save and set temporary compilation flags -########################################################################## -OLD_CPPFLAGS=$CPPFLAGS -CPPFLAGS="$NETMAP_CPPFLAGS $CPPFLAGS" - -########################################################################## -# Check for netmap availability -########################################################################## -if test x$netmap_support = xyes -then - AC_CHECK_HEADERS([net/netmap_user.h], [], - [AC_MSG_FAILURE(["can't find netmap header"])]) - AC_DEFINE([_ODP_PKTIO_NETMAP], [1], - [Define to 1 to enable netmap packet I/O support]) - AC_SUBST([NETMAP_CPPFLAGS]) -else - netmap_support=no -fi - -########################################################################## -# Restore old saved variables -########################################################################## -CPPFLAGS=$OLD_CPPFLAGS - -AC_CONFIG_COMMANDS_PRE([dnl -AM_CONDITIONAL([netmap_support], [test x$netmap_support = xyes ]) -]) diff --git a/platform/linux-generic/odp_classification.c b/platform/linux-generic/odp_classification.c index 96ac4c640..7c90ee8ac 100644 --- a/platform/linux-generic/odp_classification.c +++ b/platform/linux-generic/odp_classification.c @@ -250,6 +250,9 @@ odp_cos_t odp_cls_cos_create(const char *name, const odp_cls_cos_param_t *param_ param.queue = ODP_QUEUE_INVALID; param.pool = ODP_POOL_INVALID; param.vector.enable = false; + } else { + if (param.num_queue == 1 && param.queue == ODP_QUEUE_INVALID) + return ODP_COS_INVALID; } /* num_queue should not be zero */ @@ -432,6 +435,11 @@ int odp_cos_queue_set(odp_cos_t cos_id, odp_queue_t queue_id) return -1; } + if (queue_id == ODP_QUEUE_INVALID) { + _ODP_ERR("Invalid queue\n"); + return -1; + } + if (cos->num_queue != 1) { _ODP_ERR("Hashing enabled, cannot set queue\n"); return -1; @@ -534,17 +542,20 @@ odp_cls_drop_t odp_cos_drop(odp_cos_t cos_id) int odp_pktio_default_cos_set(odp_pktio_t pktio_in, odp_cos_t default_cos) { pktio_entry_t *entry; - cos_t *cos; + cos_t *cos = NULL; entry = get_pktio_entry(pktio_in); if (entry == NULL) { _ODP_ERR("Invalid odp_pktio_t handle\n"); return -1; } - cos = get_cos_entry(default_cos); - if (cos == NULL) { - _ODP_ERR("Invalid odp_cos_t handle\n"); - return -1; + + if (default_cos != ODP_COS_INVALID) { + cos = get_cos_entry(default_cos); + if (cos == NULL) { + _ODP_ERR("Invalid odp_cos_t handle\n"); + return -1; + } } entry->cls.default_cos = cos; @@ -1718,13 +1729,10 @@ int _odp_cls_classify_packet(pktio_entry_t *entry, const uint8_t *base, if (cos->action == ODP_COS_ACTION_DROP) return 1; - if (cos->queue == ODP_QUEUE_INVALID && cos->num_queue == 1) - goto error; - - if (cos->pool == ODP_POOL_INVALID) - goto error; - *pool = cos->pool; + if (*pool == ODP_POOL_INVALID) + *pool = entry->pool; + pkt_hdr->p.input_flags.dst_queue = 1; pkt_hdr->cos = cos->index; @@ -1740,10 +1748,6 @@ int _odp_cls_classify_packet(pktio_entry_t *entry, const uint8_t *base, cos->num_queue); pkt_hdr->dst_queue = queue_grp_tbl->queue[tbl_index]; return 0; - -error: - odp_atomic_inc_u64(&cos->stats.discards); - return 1; } static uint32_t packet_rss_hash(odp_packet_hdr_t *pkt_hdr, diff --git a/platform/linux-generic/odp_crypto_ipsecmb.c b/platform/linux-generic/odp_crypto_ipsecmb.c index a7ce6077a..0b83adecc 100644 --- a/platform/linux-generic/odp_crypto_ipsecmb.c +++ b/platform/linux-generic/odp_crypto_ipsecmb.c @@ -34,6 +34,7 @@ #define IPSEC_MB_CRYPTO_MAX_AUTH_KEY_LENGTH 32 #define IPSEC_MB_CRYPTO_MAX_DATA_LENGTH 65536 #define ZUC_DIGEST_LENGTH 4 +#define SNOW3G_DIGEST_LENGTH 4 #define ODP_CRYPTO_IPSEC_MB_SHM_NAME "_odp_crypto_ipsecmb" /* @@ -48,6 +49,9 @@ static const odp_crypto_cipher_capability_t cipher_capa_zuc_eea3[] = { {.key_len = 16, .iv_len = 16}, {.key_len = 32, .iv_len = 25} }; +static const odp_crypto_cipher_capability_t cipher_capa_snow3g_uea2[] = { +{.key_len = 16, .iv_len = 16} }; + /* * Authentication algorithm capabilities * @@ -62,6 +66,10 @@ static const odp_crypto_auth_capability_t auth_capa_zuc_eia3[] = { {.digest_len = 4, .key_len = 32, .aad_len = {.min = 0, .max = 0, .inc = 0}, .iv_len = 25} }; +static const odp_crypto_auth_capability_t auth_capa_snow3g_uia2[] = { +{.digest_len = SNOW3G_DIGEST_LENGTH, .key_len = 16, .aad_len = {.min = 0, .max = 0, .inc = 0}, + .iv_len = 16} }; + /** Forward declaration of session structure */ typedef struct odp_crypto_generic_session_t odp_crypto_generic_session_t; @@ -84,12 +92,18 @@ struct odp_crypto_generic_session_t { odp_bool_t do_cipher_first; struct { - uint8_t key_data[IPSEC_MB_CRYPTO_MAX_CIPHER_KEY_LENGTH]; + union { + uint8_t key_data[IPSEC_MB_CRYPTO_MAX_CIPHER_KEY_LENGTH]; + snow3g_key_schedule_t key_sched; + }; crypto_func_t func; } cipher; struct { - uint8_t key[IPSEC_MB_CRYPTO_MAX_AUTH_KEY_LENGTH]; + union { + uint8_t key[IPSEC_MB_CRYPTO_MAX_AUTH_KEY_LENGTH]; + snow3g_key_schedule_t key_sched; + }; crypto_func_t func; } auth; @@ -152,12 +166,12 @@ null_crypto_routine(odp_packet_t pkt ODP_UNUSED, } static -odp_crypto_alg_err_t zuc_eea3_cipher_op(odp_packet_t pkt, +odp_crypto_alg_err_t ipsec_mb_cipher_op(odp_packet_t pkt, const odp_crypto_packet_op_param_t *param, odp_crypto_generic_session_t *session) { IMB_MGR *mb_mgr = local.mb_mgr; - uint8_t *iv_ptr = param->cipher_iv_ptr; + const uint8_t *iv_ptr = param->cipher_iv_ptr; uint32_t in_pos = param->cipher_range.offset; uint32_t in_len = param->cipher_range.length; @@ -177,22 +191,32 @@ odp_crypto_alg_err_t zuc_eea3_cipher_op(odp_packet_t pkt, data = local.buffer; } - if (session->p.cipher_key.length == 16) { - /* ZUC128 EEA3 */ - IMB_ZUC_EEA3_1_BUFFER(mb_mgr, session->cipher.key_data, - iv_ptr, - data, - data, - in_len); + if (session->p.cipher_alg == ODP_CIPHER_ALG_ZUC_EEA3) { + if (session->p.cipher_key.length == 16) { + /* ZUC128 EEA3 */ + IMB_ZUC_EEA3_1_BUFFER(mb_mgr, session->cipher.key_data, + iv_ptr, + data, + data, + in_len); + } else { + /* Only 16 and 32 byte keys are supported + * ZUC256 EEA3 */ + IMB_ZUC256_EEA3_1_BUFFER(mb_mgr, session->cipher.key_data, + iv_ptr, + data, + data, + in_len); + } } else { - /* Only 16 and 32 byte keys are supported - * ZUC256 EEA3 */ - IMB_ZUC256_EEA3_1_BUFFER(mb_mgr, session->cipher.key_data, - iv_ptr, - data, - data, - in_len); + /* Only ODP_CIPHER_ALG_SNOW3G_UEA2 */ + IMB_SNOW3G_F8_1_BUFFER(mb_mgr, &session->cipher.key_sched, + iv_ptr, + data, + data, + in_len); } + if (odp_unlikely(imb_get_errno(mb_mgr) != 0)) return ODP_CRYPTO_ALG_ERR_DATA_SIZE; @@ -213,18 +237,18 @@ static int process_zuc_eea3_param(odp_crypto_generic_session_t *session) memcpy(session->cipher.key_data, session->p.cipher_key.data, session->p.cipher_key.length); - session->cipher.func = zuc_eea3_cipher_op; + session->cipher.func = ipsec_mb_cipher_op; return 0; } static -odp_crypto_alg_err_t auth_zuc_eia3_gen(odp_packet_t pkt, +odp_crypto_alg_err_t auth_ipsec_mb_gen(odp_packet_t pkt, const odp_crypto_packet_op_param_t *param, odp_crypto_generic_session_t *session) { IMB_MGR *mb_mgr = local.mb_mgr; - uint8_t *iv_ptr = param->auth_iv_ptr; + const uint8_t *iv_ptr = param->auth_iv_ptr; uint32_t in_pos = param->auth_range.offset; uint32_t in_len = param->auth_range.length; uint32_t auth_tag; @@ -245,22 +269,32 @@ odp_crypto_alg_err_t auth_zuc_eia3_gen(odp_packet_t pkt, data = local.buffer; } - if (session->p.auth_key.length == 16) { - /* ZUC128 EIA3 */ - IMB_ZUC_EIA3_1_BUFFER(mb_mgr, session->auth.key, - iv_ptr, - data, - param->auth_range.length * 8, - &auth_tag); + if (session->p.auth_alg == ODP_AUTH_ALG_ZUC_EIA3) { + if (session->p.auth_key.length == 16) { + /* ZUC128 EIA3 */ + IMB_ZUC_EIA3_1_BUFFER(mb_mgr, session->auth.key, + iv_ptr, + data, + param->auth_range.length * 8, + &auth_tag); + } else { + /* Only 16 and 32 byte keys are supported + * ZUC256 EIA3 */ + IMB_ZUC256_EIA3_1_BUFFER(mb_mgr, session->auth.key, + iv_ptr, + data, + param->auth_range.length * 8, + &auth_tag); + } } else { - /* Only 16 and 32 byte keys are supported - * ZUC256 EIA3 */ - IMB_ZUC256_EIA3_1_BUFFER(mb_mgr, session->auth.key, - iv_ptr, - data, - param->auth_range.length * 8, - &auth_tag); + /* Only ODP_AUTH_ALG_SNOW3G_UIA2 */ + IMB_SNOW3G_F9_1_BUFFER(mb_mgr, &session->auth.key_sched, + iv_ptr, + data, + param->auth_range.length * 8, + &auth_tag); } + if (odp_unlikely(imb_get_errno(mb_mgr) != 0)) return ODP_CRYPTO_ALG_ERR_DATA_SIZE; @@ -273,12 +307,12 @@ odp_crypto_alg_err_t auth_zuc_eia3_gen(odp_packet_t pkt, } static -odp_crypto_alg_err_t auth_zuc_eia3_check(odp_packet_t pkt, +odp_crypto_alg_err_t auth_ipsec_mb_check(odp_packet_t pkt, const odp_crypto_packet_op_param_t *param, odp_crypto_generic_session_t *session) { IMB_MGR *mb_mgr = local.mb_mgr; - uint8_t *iv_ptr = param->auth_iv_ptr; + const uint8_t *iv_ptr = param->auth_iv_ptr; uint32_t in_pos = param->auth_range.offset; uint32_t in_len = param->auth_range.length; uint32_t bytes = ZUC_DIGEST_LENGTH; @@ -308,22 +342,32 @@ odp_crypto_alg_err_t auth_zuc_eia3_check(odp_packet_t pkt, data = local.buffer; } - if (session->p.auth_key.length == 16) { - /* ZUC128 EIA3 */ - IMB_ZUC_EIA3_1_BUFFER(mb_mgr, session->auth.key, - iv_ptr, - data, - param->auth_range.length * 8, - &hash_out); + if (session->p.auth_alg == ODP_AUTH_ALG_ZUC_EIA3) { + if (session->p.auth_key.length == 16) { + /* ZUC128 EIA3 */ + IMB_ZUC_EIA3_1_BUFFER(mb_mgr, session->auth.key, + iv_ptr, + data, + param->auth_range.length * 8, + &hash_out); + } else { + /* Only 16 and 32 byte keys are supported + * ZUC256 EIA3 */ + IMB_ZUC256_EIA3_1_BUFFER(mb_mgr, session->auth.key, + iv_ptr, + data, + param->auth_range.length * 8, + &hash_out); + } } else { - /* Only 16 and 32 byte keys are supported - * ZUC256 EIA3 */ - IMB_ZUC256_EIA3_1_BUFFER(mb_mgr, session->auth.key, - iv_ptr, - data, - param->auth_range.length * 8, - &hash_out); + /* Only ODP_AUTH_ALG_SNOW3G_UIA2 */ + IMB_SNOW3G_F9_1_BUFFER(mb_mgr, &session->auth.key_sched, + iv_ptr, + data, + param->auth_range.length * 8, + &hash_out); } + if (odp_unlikely(imb_get_errno(mb_mgr) != 0)) return ODP_CRYPTO_ALG_ERR_DATA_SIZE; @@ -343,9 +387,9 @@ static int process_auth_zuc_eia3_param(odp_crypto_generic_session_t *session) return -1; if (ODP_CRYPTO_OP_ENCODE == session->p.op) - session->auth.func = auth_zuc_eia3_gen; + session->auth.func = auth_ipsec_mb_gen; else - session->auth.func = auth_zuc_eia3_check; + session->auth.func = auth_ipsec_mb_check; if (session->p.auth_digest_len != ZUC_DIGEST_LENGTH) return -1; @@ -356,6 +400,42 @@ static int process_auth_zuc_eia3_param(odp_crypto_generic_session_t *session) return 0; } +static int process_snow3g_uea2_param(odp_crypto_generic_session_t *session) +{ + if (!(16 == session->p.cipher_key.length && + 16 == session->p.cipher_iv_len)) + return -1; + + memcpy(session->cipher.key_data, session->p.cipher_key.data, + session->p.cipher_key.length); + + session->cipher.func = ipsec_mb_cipher_op; + + return IMB_SNOW3G_INIT_KEY_SCHED(local.mb_mgr, session->p.cipher_key.data, + &session->cipher.key_sched); +} + +static int process_auth_snow3g_uia2_param(odp_crypto_generic_session_t *session) +{ + if (!(16 == session->p.auth_key.length && + 16 == session->p.auth_iv_len)) + return -1; + + if (ODP_CRYPTO_OP_ENCODE == session->p.op) + session->auth.func = auth_ipsec_mb_gen; + else + session->auth.func = auth_ipsec_mb_check; + + if (session->p.auth_digest_len != SNOW3G_DIGEST_LENGTH) + return -1; + + memcpy(session->auth.key, session->p.auth_key.data, + session->p.auth_key.length); + + return IMB_SNOW3G_INIT_KEY_SCHED(local.mb_mgr, session->p.auth_key.data, + &session->auth.key_sched); +} + int odp_crypto_capability(odp_crypto_capability_t *capa) { if (NULL == capa) @@ -374,6 +454,9 @@ int odp_crypto_capability(odp_crypto_capability_t *capa) capa->ciphers.bit.zuc_eea3 = 1; capa->auths.bit.zuc_eia3 = 1; + capa->ciphers.bit.snow3g_uea2 = 1; + capa->auths.bit.snow3g_uia2 = 1; + capa->max_sessions = MAX_SESSIONS; return 0; @@ -396,6 +479,10 @@ int odp_crypto_cipher_capability(odp_cipher_alg_t cipher, src = cipher_capa_zuc_eea3; num = sizeof(cipher_capa_zuc_eea3) / size; break; + case ODP_CIPHER_ALG_SNOW3G_UEA2: + src = cipher_capa_snow3g_uea2; + num = sizeof(cipher_capa_snow3g_uea2) / size; + break; default: return -1; } @@ -424,6 +511,10 @@ int odp_crypto_auth_capability(odp_auth_alg_t auth, src = auth_capa_zuc_eia3; num = sizeof(auth_capa_zuc_eia3) / size; break; + case ODP_AUTH_ALG_SNOW3G_UIA2: + src = auth_capa_snow3g_uia2; + num = sizeof(auth_capa_snow3g_uia2) / size; + break; default: return -1; } @@ -482,6 +573,9 @@ odp_crypto_session_create(const odp_crypto_session_param_t *param, case ODP_CIPHER_ALG_ZUC_EEA3: rc = process_zuc_eea3_param(session); break; + case ODP_CIPHER_ALG_SNOW3G_UEA2: + rc = process_snow3g_uea2_param(session); + break; default: rc = -1; } @@ -500,6 +594,9 @@ odp_crypto_session_create(const odp_crypto_session_param_t *param, case ODP_AUTH_ALG_ZUC_EIA3: rc = process_auth_zuc_eia3_param(session); break; + case ODP_AUTH_ALG_SNOW3G_UIA2: + rc = process_auth_snow3g_uia2_param(session); + break; default: rc = -1; } @@ -531,66 +628,6 @@ int odp_crypto_session_destroy(odp_crypto_session_t session) return 0; } -#if ODP_DEPRECATED_API -int -odp_crypto_operation(odp_crypto_op_param_t *param, - odp_bool_t *posted, - odp_crypto_op_result_t *result) -{ - odp_crypto_packet_op_param_t packet_param; - odp_packet_t out_pkt = param->out_pkt; - odp_crypto_packet_result_t packet_result; - odp_crypto_op_result_t local_result; - int rc; - - if (((odp_crypto_generic_session_t *)(intptr_t)param->session)->p.op_type != - ODP_CRYPTO_OP_TYPE_LEGACY) - return -1; - - packet_param.session = param->session; - packet_param.cipher_iv_ptr = param->cipher_iv_ptr; - packet_param.auth_iv_ptr = param->auth_iv_ptr; - packet_param.hash_result_offset = param->hash_result_offset; - packet_param.aad_ptr = param->aad_ptr; - packet_param.cipher_range = param->cipher_range; - packet_param.auth_range = param->auth_range; - - rc = odp_crypto_op(¶m->pkt, &out_pkt, &packet_param, 1); - if (rc <= 0) - return -1; - - rc = odp_crypto_result(&packet_result, out_pkt); - if (rc < 0) { - /* - * We cannot fail since odp_crypto_op() has already processed - * the packet. Let's indicate error in the result instead. - */ - packet_result.ok = false; - } - - /* Indicate to caller operation was sync */ - *posted = 0; - - packet_subtype_set(out_pkt, ODP_EVENT_PACKET_BASIC); - - /* Fill in result */ - local_result.ctx = param->ctx; - local_result.pkt = out_pkt; - local_result.cipher_status = packet_result.cipher_status; - local_result.auth_status = packet_result.auth_status; - local_result.ok = packet_result.ok; - - /* - * Be bug-to-bug compatible. Return output packet also through params. - */ - param->out_pkt = out_pkt; - - *result = local_result; - - return 0; -} -#endif - int _odp_crypto_init_global(void) { size_t mem_size; @@ -682,45 +719,6 @@ int _odp_crypto_term_local(void) return 0; } -#if ODP_DEPRECATED_API -odp_crypto_compl_t odp_crypto_compl_from_event(odp_event_t ev) -{ - /* This check not mandated by the API specification */ - if (odp_event_type(ev) != ODP_EVENT_CRYPTO_COMPL) - _ODP_ABORT("Event not a crypto completion"); - return (odp_crypto_compl_t)ev; -} - -odp_event_t odp_crypto_compl_to_event(odp_crypto_compl_t completion_event) -{ - return (odp_event_t)completion_event; -} - -void -odp_crypto_compl_result(odp_crypto_compl_t completion_event, - odp_crypto_op_result_t *result) -{ - (void)completion_event; - (void)result; - - /* We won't get such events anyway, so there can be no result */ - _ODP_ASSERT(0); -} - -void -odp_crypto_compl_free(odp_crypto_compl_t completion_event) -{ - odp_event_t ev = odp_crypto_compl_to_event(completion_event); - - odp_buffer_free(odp_buffer_from_event(ev)); -} - -uint64_t odp_crypto_compl_to_u64(odp_crypto_compl_t hdl) -{ - return _odp_pri(hdl); -} -#endif /* ODP_DEPRECATED_API */ - void odp_crypto_session_param_init(odp_crypto_session_param_t *param) { memset(param, 0, sizeof(odp_crypto_session_param_t)); @@ -759,9 +757,6 @@ static odp_packet_t get_output_packet(const odp_crypto_generic_session_t *sessio { int rc; - if (odp_likely(session->p.op_type == ODP_CRYPTO_OP_TYPE_BASIC)) - return pkt_in; - if (odp_likely(pkt_in == pkt_out)) return pkt_out; @@ -799,9 +794,13 @@ int crypto_int(odp_packet_t pkt_in, session = (odp_crypto_generic_session_t *)(intptr_t)param->session; - out_pkt = get_output_packet(session, pkt_in, *pkt_out); - if (odp_unlikely(out_pkt == ODP_PACKET_INVALID)) - return -1; + if (odp_likely(session->p.op_type == ODP_CRYPTO_OP_TYPE_BASIC)) { + out_pkt = pkt_in; + } else { + out_pkt = get_output_packet(session, pkt_in, *pkt_out); + if (odp_unlikely(out_pkt == ODP_PACKET_INVALID)) + return -1; + } /* Invoke the crypto function */ if (session->do_cipher_first) { @@ -815,12 +814,7 @@ int crypto_int(odp_packet_t pkt_in, packet_subtype_set(out_pkt, ODP_EVENT_PACKET_CRYPTO); op_result = &packet_hdr(out_pkt)->crypto_op_result; op_result->cipher_status.alg_err = rc_cipher; - op_result->cipher_status.hw_err = ODP_CRYPTO_HW_ERR_NONE; op_result->auth_status.alg_err = rc_auth; - op_result->auth_status.hw_err = ODP_CRYPTO_HW_ERR_NONE; - op_result->ok = - (rc_cipher == ODP_CRYPTO_ALG_ERR_NONE) && - (rc_auth == ODP_CRYPTO_ALG_ERR_NONE); /* Synchronous, simply return results */ *pkt_out = out_pkt; diff --git a/platform/linux-generic/odp_crypto_null.c b/platform/linux-generic/odp_crypto_null.c index 3ca27ca00..c93691e92 100644 --- a/platform/linux-generic/odp_crypto_null.c +++ b/platform/linux-generic/odp_crypto_null.c @@ -275,66 +275,6 @@ int odp_crypto_session_destroy(odp_crypto_session_t session) return 0; } -#if ODP_DEPRECATED_API -int -odp_crypto_operation(odp_crypto_op_param_t *param, - odp_bool_t *posted, - odp_crypto_op_result_t *result) -{ - odp_crypto_packet_op_param_t packet_param; - odp_packet_t out_pkt = param->out_pkt; - odp_crypto_packet_result_t packet_result; - odp_crypto_op_result_t local_result; - int rc; - - if (((odp_crypto_generic_session_t *)(intptr_t)param->session)->p.op_type != - ODP_CRYPTO_OP_TYPE_LEGACY) - return -1; - - packet_param.session = param->session; - packet_param.cipher_iv_ptr = param->cipher_iv_ptr; - packet_param.auth_iv_ptr = param->auth_iv_ptr; - packet_param.hash_result_offset = param->hash_result_offset; - packet_param.aad_ptr = param->aad_ptr; - packet_param.cipher_range = param->cipher_range; - packet_param.auth_range = param->auth_range; - - rc = odp_crypto_op(¶m->pkt, &out_pkt, &packet_param, 1); - if (rc <= 0) - return -1; - - rc = odp_crypto_result(&packet_result, out_pkt); - if (rc < 0) { - /* - * We cannot fail since odp_crypto_op() has already processed - * the packet. Let's indicate error in the result instead. - */ - packet_result.ok = false; - } - - /* Indicate to caller operation was sync */ - *posted = 0; - - packet_subtype_set(out_pkt, ODP_EVENT_PACKET_BASIC); - - /* Fill in result */ - local_result.ctx = param->ctx; - local_result.pkt = out_pkt; - local_result.cipher_status = packet_result.cipher_status; - local_result.auth_status = packet_result.auth_status; - local_result.ok = packet_result.ok; - - /* - * Be bug-to-bug compatible. Return output packet also through params. - */ - param->out_pkt = out_pkt; - - *result = local_result; - - return 0; -} -#endif - int _odp_crypto_init_global(void) { @@ -410,45 +350,6 @@ int _odp_crypto_term_local(void) return 0; } -#if ODP_DEPRECATED_API -odp_crypto_compl_t odp_crypto_compl_from_event(odp_event_t ev) -{ - /* This check not mandated by the API specification */ - if (odp_event_type(ev) != ODP_EVENT_CRYPTO_COMPL) - _ODP_ABORT("Event not a crypto completion"); - return (odp_crypto_compl_t)ev; -} - -odp_event_t odp_crypto_compl_to_event(odp_crypto_compl_t completion_event) -{ - return (odp_event_t)completion_event; -} - -void -odp_crypto_compl_result(odp_crypto_compl_t completion_event, - odp_crypto_op_result_t *result) -{ - (void)completion_event; - (void)result; - - /* We won't get such events anyway, so there can be no result */ - _ODP_ASSERT(0); -} - -void -odp_crypto_compl_free(odp_crypto_compl_t completion_event) -{ - odp_event_t ev = odp_crypto_compl_to_event(completion_event); - - odp_buffer_free(odp_buffer_from_event(ev)); -} - -uint64_t odp_crypto_compl_to_u64(odp_crypto_compl_t hdl) -{ - return _odp_pri(hdl); -} -#endif - void odp_crypto_session_param_init(odp_crypto_session_param_t *param) { memset(param, 0, sizeof(odp_crypto_session_param_t)); @@ -487,9 +388,6 @@ static odp_packet_t get_output_packet(const odp_crypto_generic_session_t *sessio { int rc; - if (odp_likely(session->p.op_type == ODP_CRYPTO_OP_TYPE_BASIC)) - return pkt_in; - if (odp_likely(pkt_in == pkt_out)) return pkt_out; @@ -525,18 +423,19 @@ int crypto_int(odp_packet_t pkt_in, session = (odp_crypto_generic_session_t *)(intptr_t)param->session; - out_pkt = get_output_packet(session, pkt_in, *pkt_out); - if (odp_unlikely(out_pkt == ODP_PACKET_INVALID)) - return -1; + if (odp_likely(session->p.op_type == ODP_CRYPTO_OP_TYPE_BASIC)) { + out_pkt = pkt_in; + } else { + out_pkt = get_output_packet(session, pkt_in, *pkt_out); + if (odp_unlikely(out_pkt == ODP_PACKET_INVALID)) + return -1; + } /* Fill in result */ packet_subtype_set(out_pkt, ODP_EVENT_PACKET_CRYPTO); op_result = &packet_hdr(out_pkt)->crypto_op_result; op_result->cipher_status.alg_err = ODP_CRYPTO_ALG_ERR_NONE; - op_result->cipher_status.hw_err = ODP_CRYPTO_HW_ERR_NONE; op_result->auth_status.alg_err = ODP_CRYPTO_ALG_ERR_NONE; - op_result->auth_status.hw_err = ODP_CRYPTO_HW_ERR_NONE; - op_result->ok = true; /* Synchronous, simply return results */ *pkt_out = out_pkt; @@ -569,7 +468,7 @@ int odp_crypto_op_enq(const odp_packet_t pkt_in[], const odp_crypto_packet_op_param_t param[], int num_pkt) { - odp_packet_t pkt; + odp_packet_t pkt = ODP_PACKET_INVALID; odp_event_t event; odp_crypto_generic_session_t *session; int i, rc; diff --git a/platform/linux-generic/odp_crypto_openssl.c b/platform/linux-generic/odp_crypto_openssl.c index 19925fc09..d4c7a3f1a 100644 --- a/platform/linux-generic/odp_crypto_openssl.c +++ b/platform/linux-generic/odp_crypto_openssl.c @@ -705,7 +705,7 @@ int packet_cmac_eia2(odp_packet_t pkt, uint8_t *hash) { CMAC_CTX *ctx = local.cmac_ctx[session->idx]; - void *iv_ptr = param->auth_iv_ptr; + const void *iv_ptr = param->auth_iv_ptr; uint32_t offset = param->auth_range.offset; uint32_t len = param->auth_range.length; size_t outlen; @@ -2343,69 +2343,6 @@ int odp_crypto_session_destroy(odp_crypto_session_t session) return 0; } -/* - * Shim function around packet operation, can be used by other implementations. - */ -#if ODP_DEPRECATED_API -int -odp_crypto_operation(odp_crypto_op_param_t *param, - odp_bool_t *posted, - odp_crypto_op_result_t *result) -{ - odp_crypto_packet_op_param_t packet_param; - odp_packet_t out_pkt = param->out_pkt; - odp_crypto_packet_result_t packet_result; - odp_crypto_op_result_t local_result; - int rc; - - if (((odp_crypto_generic_session_t *)(intptr_t)param->session)->p.op_type != - ODP_CRYPTO_OP_TYPE_LEGACY) - return -1; - - packet_param.session = param->session; - packet_param.cipher_iv_ptr = param->cipher_iv_ptr; - packet_param.auth_iv_ptr = param->auth_iv_ptr; - packet_param.hash_result_offset = param->hash_result_offset; - packet_param.aad_ptr = param->aad_ptr; - packet_param.cipher_range = param->cipher_range; - packet_param.auth_range = param->auth_range; - - rc = odp_crypto_op(¶m->pkt, &out_pkt, &packet_param, 1); - if (rc <= 0) - return -1; - - rc = odp_crypto_result(&packet_result, out_pkt); - if (rc < 0) { - /* - * We cannot fail since odp_crypto_op() has already processed - * the packet. Let's indicate error in the result instead. - */ - packet_result.ok = false; - } - - /* Indicate to caller operation was sync */ - *posted = 0; - - packet_subtype_set(out_pkt, ODP_EVENT_PACKET_BASIC); - - /* Fill in result */ - local_result.ctx = param->ctx; - local_result.pkt = out_pkt; - local_result.cipher_status = packet_result.cipher_status; - local_result.auth_status = packet_result.auth_status; - local_result.ok = packet_result.ok; - - /* - * Be bug-to-bug compatible. Return output packet also through params. - */ - param->out_pkt = out_pkt; - - *result = local_result; - - return 0; -} -#endif - #if OPENSSL_VERSION_NUMBER < 0x10100000L static void ODP_UNUSED openssl_thread_id(CRYPTO_THREADID ODP_UNUSED *id) { @@ -2561,45 +2498,6 @@ int _odp_crypto_term_local(void) return 0; } -#if ODP_DEPRECATED_API -odp_crypto_compl_t odp_crypto_compl_from_event(odp_event_t ev) -{ - /* This check not mandated by the API specification */ - if (odp_event_type(ev) != ODP_EVENT_CRYPTO_COMPL) - _ODP_ABORT("Event not a crypto completion"); - return (odp_crypto_compl_t)ev; -} - -odp_event_t odp_crypto_compl_to_event(odp_crypto_compl_t completion_event) -{ - return (odp_event_t)completion_event; -} - -void -odp_crypto_compl_result(odp_crypto_compl_t completion_event, - odp_crypto_op_result_t *result) -{ - (void)completion_event; - (void)result; - - /* We won't get such events anyway, so there can be no result */ - _ODP_ASSERT(0); -} - -void -odp_crypto_compl_free(odp_crypto_compl_t completion_event) -{ - odp_event_t ev = odp_crypto_compl_to_event(completion_event); - - odp_buffer_free(odp_buffer_from_event(ev)); -} - -uint64_t odp_crypto_compl_to_u64(odp_crypto_compl_t hdl) -{ - return _odp_pri(hdl); -} -#endif /* ODP_DEPRECATED_API */ - void odp_crypto_session_param_init(odp_crypto_session_param_t *param) { memset(param, 0, sizeof(odp_crypto_session_param_t)); @@ -2638,9 +2536,6 @@ static odp_packet_t get_output_packet(const odp_crypto_generic_session_t *sessio { int rc; - if (odp_likely(session->p.op_type == ODP_CRYPTO_OP_TYPE_BASIC)) - return pkt_in; - if (odp_likely(pkt_in == pkt_out)) return pkt_out; @@ -2678,9 +2573,13 @@ int crypto_int(odp_packet_t pkt_in, session = (odp_crypto_generic_session_t *)(intptr_t)param->session; - out_pkt = get_output_packet(session, pkt_in, *pkt_out); - if (odp_unlikely(out_pkt == ODP_PACKET_INVALID)) - return -1; + if (odp_likely(session->p.op_type == ODP_CRYPTO_OP_TYPE_BASIC)) { + out_pkt = pkt_in; + } else { + out_pkt = get_output_packet(session, pkt_in, *pkt_out); + if (odp_unlikely(out_pkt == ODP_PACKET_INVALID)) + return -1; + } if (ODP_DEBUG) { if (session->p.auth_alg != ODP_AUTH_ALG_NULL && @@ -2711,12 +2610,7 @@ out: packet_subtype_set(out_pkt, ODP_EVENT_PACKET_CRYPTO); op_result = &packet_hdr(out_pkt)->crypto_op_result; op_result->cipher_status.alg_err = rc_cipher; - op_result->cipher_status.hw_err = ODP_CRYPTO_HW_ERR_NONE; op_result->auth_status.alg_err = rc_auth; - op_result->auth_status.hw_err = ODP_CRYPTO_HW_ERR_NONE; - op_result->ok = - (rc_cipher == ODP_CRYPTO_ALG_ERR_NONE) && - (rc_auth == ODP_CRYPTO_ALG_ERR_NONE); /* Synchronous, simply return results */ *pkt_out = out_pkt; diff --git a/platform/linux-generic/odp_event.c b/platform/linux-generic/odp_event.c index edf77e2dc..e15cb1c50 100644 --- a/platform/linux-generic/odp_event.c +++ b/platform/linux-generic/odp_event.c @@ -59,11 +59,6 @@ static inline void event_free(odp_event_t event, _odp_ev_id_t id) case ODP_EVENT_TIMEOUT: odp_timeout_free(odp_timeout_from_event(event)); break; -#if ODP_DEPRECATED_API - case ODP_EVENT_CRYPTO_COMPL: - odp_crypto_compl_free(odp_crypto_compl_from_event(event)); - break; -#endif case ODP_EVENT_IPSEC_STATUS: _odp_ipsec_status_free(_odp_ipsec_status_from_event(event)); break; @@ -115,10 +110,6 @@ int odp_event_is_valid(odp_event_t event) return !_odp_packet_validate(odp_packet_from_event(event), _ODP_EV_EVENT_IS_VALID); case ODP_EVENT_TIMEOUT: /* Fall through */ -#if ODP_DEPRECATED_API - case ODP_EVENT_CRYPTO_COMPL: - /* Fall through */ -#endif case ODP_EVENT_IPSEC_STATUS: /* Fall through */ case ODP_EVENT_PACKET_VECTOR: diff --git a/platform/linux-generic/odp_ipsec.c b/platform/linux-generic/odp_ipsec.c index 28d46c794..861dc1337 100644 --- a/platform/linux-generic/odp_ipsec.c +++ b/platform/linux-generic/odp_ipsec.c @@ -955,27 +955,27 @@ static int ipsec_in_prepare_packet(odp_packet_t *pkt, ipsec_state_t *state, ipse static int ipsec_in_do_crypto(odp_packet_t *pkt, odp_crypto_packet_op_param_t *param, odp_ipsec_op_status_t *status) { - odp_crypto_packet_result_t crypto; + odp_crypto_packet_result_t result; + int rc; if (odp_unlikely(odp_crypto_op(pkt, pkt, param, 1) < 0)) { _ODP_DBG("Crypto failed\n"); goto alg_err; } - if (odp_unlikely(odp_crypto_result(&crypto, *pkt) < 0)) { + rc = odp_crypto_result(&result, *pkt); + + if (odp_likely(rc == 0)) + return 0; + + if (odp_unlikely(rc < -1)) { _ODP_DBG("Crypto failed\n"); goto alg_err; } - if (odp_unlikely(!crypto.ok)) { - if (crypto.cipher_status.alg_err == ODP_CRYPTO_ALG_ERR_ICV_CHECK || - crypto.auth_status.alg_err == ODP_CRYPTO_ALG_ERR_ICV_CHECK) - goto auth_err; - else - goto alg_err; - } - - return 0; + if (result.cipher_status.alg_err == ODP_CRYPTO_ALG_ERR_ICV_CHECK || + result.auth_status.alg_err == ODP_CRYPTO_ALG_ERR_ICV_CHECK) + goto auth_err; alg_err: status->error.alg = 1; @@ -2085,25 +2085,25 @@ static void ipsec_do_crypto_burst(odp_packet_t pkts[], odp_crypto_packet_op_para static int ipsec_in_check_crypto_result(odp_packet_t pkt, odp_ipsec_op_status_t *status) { - odp_crypto_packet_result_t crypto; + odp_crypto_packet_result_t result; + int rc = odp_crypto_result(&result, pkt); + + if (odp_likely(rc == 0)) + return 0; - if (odp_unlikely(odp_crypto_result(&crypto, pkt) < 0)) { + if (odp_unlikely(rc < -1)) { _ODP_DBG("Crypto failed\n"); status->error.alg = 1; return -1; } - if (odp_unlikely(!crypto.ok)) { - if (crypto.cipher_status.alg_err == ODP_CRYPTO_ALG_ERR_ICV_CHECK || - crypto.auth_status.alg_err == ODP_CRYPTO_ALG_ERR_ICV_CHECK) - status->error.auth = 1; - else - status->error.alg = 1; - - return -1; - } + if (result.cipher_status.alg_err == ODP_CRYPTO_ALG_ERR_ICV_CHECK || + result.auth_status.alg_err == ODP_CRYPTO_ALG_ERR_ICV_CHECK) + status->error.auth = 1; + else + status->error.alg = 1; - return 0; + return -1; } static inline void update_post_lifetime_stats(ipsec_sa_t *sa, ipsec_state_t *state) @@ -2250,20 +2250,12 @@ static void ipsec_out_prepare(const odp_packet_t pkt_in[], odp_packet_t pkt_out[ static int ipsec_out_check_crypto_result(odp_packet_t pkt, odp_ipsec_op_status_t *status) { - odp_crypto_packet_result_t crypto; - - if (odp_unlikely(odp_crypto_result(&crypto, pkt) < 0)) { + if (odp_unlikely(odp_crypto_result(NULL, pkt) != 0)) { _ODP_DBG("Crypto failed\n"); status->error.alg = 1; return -1; } - if (odp_unlikely(!crypto.ok)) { - status->error.alg = 1; - - return -1; - } - return 0; } diff --git a/platform/linux-generic/odp_ipsec_sad.c b/platform/linux-generic/odp_ipsec_sad.c index 4cf265694..daa80c7e6 100644 --- a/platform/linux-generic/odp_ipsec_sad.c +++ b/platform/linux-generic/odp_ipsec_sad.c @@ -631,6 +631,7 @@ odp_ipsec_sa_t odp_ipsec_sa_create(const odp_ipsec_sa_param_t *param) crypto_param.op = (ODP_IPSEC_DIR_INBOUND == param->dir) ? ODP_CRYPTO_OP_DECODE : ODP_CRYPTO_OP_ENCODE; + crypto_param.op_type = ODP_CRYPTO_OP_TYPE_BASIC; crypto_param.auth_cipher_text = 1; if (param->proto == ODP_IPSEC_AH) crypto_param.hash_result_in_auth_range = 1; diff --git a/platform/linux-generic/odp_ishm.c b/platform/linux-generic/odp_ishm.c index 8bb558be2..926e20a8a 100644 --- a/platform/linux-generic/odp_ishm.c +++ b/platform/linux-generic/odp_ishm.c @@ -42,7 +42,6 @@ #include <odp/api/system_info.h> #include <odp/api/debug.h> #include <odp_init_internal.h> -#include <odp_errno_define.h> #include <odp_shm_internal.h> #include <odp_debug_internal.h> #include <odp_fdserver_internal.h> diff --git a/platform/linux-generic/odp_packet.c b/platform/linux-generic/odp_packet.c index 639a74e0c..869713098 100644 --- a/platform/linux-generic/odp_packet.c +++ b/platform/linux-generic/odp_packet.c @@ -19,7 +19,6 @@ #include <odp_parse_internal.h> #include <odp_chksum_internal.h> #include <odp_debug_internal.h> -#include <odp_errno_define.h> #include <odp_event_internal.h> #include <odp_event_validation_internal.h> #include <odp_macros_internal.h> @@ -655,10 +654,7 @@ odp_packet_t odp_packet_alloc(odp_pool_t pool_hdl, uint32_t len) odp_packet_t pkt; int num, num_seg; - if (odp_unlikely(pool->type != ODP_POOL_PACKET)) { - _odp_errno = EINVAL; - return ODP_PACKET_INVALID; - } + _ODP_ASSERT(pool->type == ODP_POOL_PACKET); if (odp_unlikely(len > pool->max_len || len == 0)) return ODP_PACKET_INVALID; @@ -678,10 +674,7 @@ int odp_packet_alloc_multi(odp_pool_t pool_hdl, uint32_t len, pool_t *pool = _odp_pool_entry(pool_hdl); int num, num_seg; - if (odp_unlikely(pool->type != ODP_POOL_PACKET)) { - _odp_errno = EINVAL; - return -1; - } + _ODP_ASSERT(pool->type == ODP_POOL_PACKET); if (odp_unlikely(len > pool->max_len || len == 0)) return -1; diff --git a/platform/linux-generic/odp_packet_io.c b/platform/linux-generic/odp_packet_io.c index 5a6c0f460..78cd93885 100644 --- a/platform/linux-generic/odp_packet_io.c +++ b/platform/linux-generic/odp_packet_io.c @@ -25,7 +25,6 @@ #include <odp_classification_internal.h> #include <odp_config_internal.h> #include <odp_debug_internal.h> -#include <odp_errno_define.h> #include <odp_event_vector_internal.h> #include <odp_init_internal.h> #include <odp_libconfig_internal.h> @@ -35,7 +34,6 @@ #include <odp_queue_if.h> #include <odp_schedule_if.h> -#include <errno.h> #include <ifaddrs.h> #include <inttypes.h> #include <string.h> @@ -405,8 +403,7 @@ odp_pktio_t odp_pktio_open(const char *name, odp_pool_t pool, hdl = odp_pktio_lookup(name); if (hdl != ODP_PKTIO_INVALID) { - /* interface is already open */ - _odp_errno = EEXIST; + _ODP_ERR("pktio device %s already opened\n", name); return ODP_PKTIO_INVALID; } @@ -2357,7 +2354,7 @@ int _odp_pktio_pktout_tm_config(odp_pktio_t pktio_hdl, odp_pktout_mode_t mode; pktio_entry_t *entry; uint32_t i; - int rc = 0; + int rc; odp_pktout_queue_param_init(¶m); param.num_queues = 1; @@ -2368,11 +2365,10 @@ int _odp_pktio_pktout_tm_config(odp_pktio_t pktio_hdl, return -1; } - rc = -ENOTSUP; mode = entry->param.out_mode; /* Don't proceed further if mode is not TM */ if (mode != ODP_PKTOUT_MODE_TM) - return rc; + return -1; /* Don't reconfigure unless requested */ if (entry->num_out_queue && !reconf) { @@ -3507,7 +3503,7 @@ odp_proto_stats_capability(odp_pktio_t pktio, odp_proto_stats_capability_t *capa (void)pktio; if (capa == NULL) - return -EINVAL; + return -1; memset(capa, 0, sizeof(*capa)); diff --git a/platform/linux-generic/odp_schedule_basic.c b/platform/linux-generic/odp_schedule_basic.c index 3fc9c1c17..d289f1e21 100644 --- a/platform/linux-generic/odp_schedule_basic.c +++ b/platform/linux-generic/odp_schedule_basic.c @@ -13,6 +13,8 @@ #pragma GCC diagnostic ignored "-Wzero-length-bounds" #endif +#include <odp_posix_extensions.h> + #include <odp/api/schedule.h> #include <odp_schedule_if.h> #include <odp/api/align.h> @@ -41,6 +43,7 @@ #include <odp_print_internal.h> #include <string.h> +#include <time.h> /* No synchronization context */ #define NO_SYNC_CONTEXT ODP_SCHED_SYNC_PARALLEL @@ -292,6 +295,11 @@ typedef struct { order_context_t order[CONFIG_MAX_SCHED_QUEUES]; + struct { + uint32_t poll_time; + struct timespec sleep_time; + } powersave; + /* Scheduler interface config options (not used in fast path) */ schedule_config_t config_if; uint32_t max_queues; @@ -323,7 +331,7 @@ static void prio_grp_mask_init(void) static inline void prio_grp_mask_set(int prio, int grp) { - uint64_t grp_mask = 0x1u << grp; + uint64_t grp_mask = (uint64_t)1 << grp; uint64_t mask = odp_atomic_load_u64(&sched->prio_grp_mask[prio]); odp_atomic_store_u64(&sched->prio_grp_mask[prio], mask | grp_mask); @@ -333,7 +341,7 @@ static inline void prio_grp_mask_set(int prio, int grp) static inline void prio_grp_mask_clear(int prio, int grp) { - uint64_t grp_mask = 0x1u << grp; + uint64_t grp_mask = (uint64_t)1 << grp; uint64_t mask = odp_atomic_load_u64(&sched->prio_grp_mask[prio]); sched->prio_grp_count[prio][grp]--; @@ -521,6 +529,26 @@ static int read_config_file(sched_global_t *sched) sched->config_if.group_enable.control = val; _ODP_PRINT(" %s: %i\n", str, val); + str = "sched_basic.powersave.poll_time_nsec"; + if (!_odp_libconfig_lookup_int(str, &val)) { + _ODP_ERR("Config option '%s' not found.\n", str); + return -1; + } + + sched->powersave.poll_time = _ODP_MAX(0, val); + _ODP_PRINT(" %s: %i\n", str, val); + + str = "sched_basic.powersave.sleep_time_nsec"; + if (!_odp_libconfig_lookup_int(str, &val)) { + _ODP_ERR("Config option '%s' not found.\n", str); + return -1; + } + + val = _ODP_MAX(0, val); + sched->powersave.sleep_time.tv_sec = val / 1000000000; + sched->powersave.sleep_time.tv_nsec = val % 1000000000; + _ODP_PRINT(" %s: %i\n", str, val); + _ODP_PRINT(" dynamic load balance: %s\n", sched->load_balance ? "ON" : "OFF"); _ODP_PRINT("\n"); @@ -725,7 +753,7 @@ static inline int grp_update_tbl(void) if (odp_thrmask_isset(&sched->sched_grp[i].mask, thr)) { sched_local.grp[num] = i; num++; - mask |= 0x1u << i; + mask |= (uint64_t)1 << i; } } @@ -1632,6 +1660,51 @@ static inline int schedule_loop(odp_queue_t *out_queue, uint64_t wait, return ret; } +static inline int schedule_loop_sleep(odp_queue_t *out_queue, uint64_t wait, + odp_event_t out_ev[], uint32_t max_num) +{ + int ret; + odp_time_t start, end, current, start_sleep; + int first = 1, sleep = 0; + + while (1) { + ret = do_schedule(out_queue, out_ev, max_num); + if (ret) { + timer_run(2); + break; + } + timer_run(1); + + if (first) { + start = odp_time_local(); + start_sleep = + odp_time_sum(start, + odp_time_local_from_ns(sched->powersave.poll_time)); + if (wait != ODP_SCHED_WAIT) + end = odp_time_sum(start, odp_time_local_from_ns(wait)); + first = 0; + continue; + } + + if (sleep) + nanosleep(&sched->powersave.sleep_time, NULL); + + if (wait != ODP_SCHED_WAIT || !sleep) { + current = odp_time_local(); + if (odp_time_cmp(start_sleep, current) < 0) + sleep = 1; + } + + if (wait == ODP_SCHED_WAIT) + continue; + + if (odp_time_cmp(end, current) < 0) + break; + } + + return ret; +} + static odp_event_t schedule(odp_queue_t *out_queue, uint64_t wait) { odp_event_t ev; @@ -1643,12 +1716,35 @@ static odp_event_t schedule(odp_queue_t *out_queue, uint64_t wait) return ev; } +static odp_event_t schedule_sleep(odp_queue_t *out_queue, uint64_t wait) +{ + odp_event_t ev; + + ev = ODP_EVENT_INVALID; + + if (wait == ODP_SCHED_NO_WAIT) + schedule_loop(out_queue, wait, &ev, 1); + else + schedule_loop_sleep(out_queue, wait, &ev, 1); + + return ev; +} + static int schedule_multi(odp_queue_t *out_queue, uint64_t wait, odp_event_t events[], int num) { return schedule_loop(out_queue, wait, events, num); } +static int schedule_multi_sleep(odp_queue_t *out_queue, uint64_t wait, + odp_event_t events[], int num) +{ + if (wait == ODP_SCHED_NO_WAIT) + return schedule_loop(out_queue, wait, events, num); + + return schedule_loop_sleep(out_queue, wait, events, num); +} + static int schedule_multi_no_wait(odp_queue_t *out_queue, odp_event_t events[], int num) { @@ -1667,6 +1763,12 @@ static int schedule_multi_wait(odp_queue_t *out_queue, odp_event_t events[], return ret; } +static int schedule_multi_wait_sleep(odp_queue_t *out_queue, odp_event_t events[], + int num) +{ + return schedule_loop_sleep(out_queue, ODP_SCHED_WAIT, events, num); +} + static inline void order_lock(void) { if (sched_local.sync_ctx != ODP_SCHED_SYNC_ORDERED) @@ -2199,6 +2301,17 @@ int _odp_sched_basic_get_spread(uint32_t queue_index) return sched->queue[queue_index].spread; } +const _odp_schedule_api_fn_t _odp_schedule_basic_api; +const _odp_schedule_api_fn_t _odp_schedule_basic_sleep_api; + +static const _odp_schedule_api_fn_t *sched_api(void) +{ + if (sched->powersave.poll_time > 0) + return &_odp_schedule_basic_sleep_api; + + return &_odp_schedule_basic_api; +} + /* Fill in scheduler interface */ const schedule_fn_t _odp_schedule_basic_fn = { .pktio_start = schedule_pktio_start, @@ -2216,7 +2329,8 @@ const schedule_fn_t _odp_schedule_basic_fn = { .order_lock = order_lock, .order_unlock = order_unlock, .max_ordered_locks = schedule_max_ordered_locks, - .get_config = schedule_get_config + .get_config = schedule_get_config, + .sched_api = sched_api, }; /* Fill in scheduler API calls */ @@ -2253,3 +2367,40 @@ const _odp_schedule_api_fn_t _odp_schedule_basic_api = { .schedule_order_wait = order_lock, .schedule_print = schedule_print }; + +/* API functions used when powersave is enabled in the config file. */ +const _odp_schedule_api_fn_t _odp_schedule_basic_sleep_api = { + .schedule_wait_time = schedule_wait_time, + .schedule_capability = schedule_capability, + .schedule_config_init = schedule_config_init, + .schedule_config = schedule_config, + /* Only the following *_sleep functions differ from _odp_schedule_basic_api */ + .schedule = schedule_sleep, + .schedule_multi = schedule_multi_sleep, + .schedule_multi_wait = schedule_multi_wait_sleep, + /* End of powersave specific functions */ + .schedule_multi_no_wait = schedule_multi_no_wait, + .schedule_pause = schedule_pause, + .schedule_resume = schedule_resume, + .schedule_release_atomic = schedule_release_atomic, + .schedule_release_ordered = schedule_release_ordered, + .schedule_prefetch = schedule_prefetch, + .schedule_min_prio = schedule_min_prio, + .schedule_max_prio = schedule_max_prio, + .schedule_default_prio = schedule_default_prio, + .schedule_num_prio = schedule_num_prio, + .schedule_group_create = schedule_group_create, + .schedule_group_destroy = schedule_group_destroy, + .schedule_group_lookup = schedule_group_lookup, + .schedule_group_join = schedule_group_join, + .schedule_group_leave = schedule_group_leave, + .schedule_group_thrmask = schedule_group_thrmask, + .schedule_group_info = schedule_group_info, + .schedule_order_lock = schedule_order_lock, + .schedule_order_unlock = schedule_order_unlock, + .schedule_order_unlock_lock = schedule_order_unlock_lock, + .schedule_order_lock_start = schedule_order_lock_start, + .schedule_order_lock_wait = schedule_order_lock_wait, + .schedule_order_wait = order_lock, + .schedule_print = schedule_print +}; diff --git a/platform/linux-generic/odp_schedule_if.c b/platform/linux-generic/odp_schedule_if.c index 5f2d6f901..f4d50f84e 100644 --- a/platform/linux-generic/odp_schedule_if.c +++ b/platform/linux-generic/odp_schedule_if.c @@ -30,14 +30,8 @@ int _odp_schedule_configured(void) #include <odp/visibility_end.h> extern const schedule_fn_t _odp_schedule_sp_fn; -extern const _odp_schedule_api_fn_t _odp_schedule_sp_api; - extern const schedule_fn_t _odp_schedule_basic_fn; -extern const _odp_schedule_api_fn_t _odp_schedule_basic_api; - -extern const schedule_fn_t _odp_schedule_scalable_fn; -extern const _odp_schedule_api_fn_t _odp_schedule_scalable_api; - +extern const schedule_fn_t _odp_schedule_scalable_fn; const schedule_fn_t *_odp_sched_fn; int _odp_sched_id; @@ -153,21 +147,23 @@ int _odp_schedule_init_global(void) if (!strcmp(sched, "basic")) { _odp_sched_id = _ODP_SCHED_ID_BASIC; _odp_sched_fn = &_odp_schedule_basic_fn; - _odp_sched_api = &_odp_schedule_basic_api; } else if (!strcmp(sched, "sp")) { _odp_sched_id = _ODP_SCHED_ID_SP; _odp_sched_fn = &_odp_schedule_sp_fn; - _odp_sched_api = &_odp_schedule_sp_api; } else if (!strcmp(sched, "scalable")) { _odp_sched_id = _ODP_SCHED_ID_SCALABLE; _odp_sched_fn = &_odp_schedule_scalable_fn; - _odp_sched_api = &_odp_schedule_scalable_api; } else { _ODP_ABORT("Unknown scheduler specified via ODP_SCHEDULER\n"); return -1; } - return _odp_sched_fn->init_global(); + if (_odp_sched_fn->init_global()) + return -1; + + _odp_sched_api = _odp_sched_fn->sched_api(); + + return 0; } int _odp_schedule_term_global(void) diff --git a/platform/linux-generic/odp_schedule_scalable.c b/platform/linux-generic/odp_schedule_scalable.c index 172a4d336..bca88e4b5 100644 --- a/platform/linux-generic/odp_schedule_scalable.c +++ b/platform/linux-generic/odp_schedule_scalable.c @@ -1846,19 +1846,19 @@ static int schedule_init_global(void) * GROUP_CONTROL groups. */ odp_thrmask_zero(&mask); - tmp_all = odp_schedule_group_create("__group_all", &mask); + tmp_all = schedule_group_create("__group_all", &mask); if (tmp_all != ODP_SCHED_GROUP_ALL) { _ODP_ERR("Could not create ODP_SCHED_GROUP_ALL()\n"); goto failed_create_group_all; } - tmp_wrkr = odp_schedule_group_create("__group_worker", &mask); + tmp_wrkr = schedule_group_create("__group_worker", &mask); if (tmp_wrkr != ODP_SCHED_GROUP_WORKER) { _ODP_ERR("Could not create ODP_SCHED_GROUP_WORKER()\n"); goto failed_create_group_worker; } - tmp_ctrl = odp_schedule_group_create("__group_control", &mask); + tmp_ctrl = schedule_group_create("__group_control", &mask); if (tmp_ctrl != ODP_SCHED_GROUP_CONTROL) { _ODP_ERR("Could not create ODP_SCHED_GROUP_CONTROL()\n"); goto failed_create_group_control; @@ -1872,15 +1872,15 @@ static int schedule_init_global(void) failed_create_group_control: if (tmp_ctrl != ODP_SCHED_GROUP_INVALID) - odp_schedule_group_destroy(ODP_SCHED_GROUP_CONTROL); + schedule_group_destroy(ODP_SCHED_GROUP_CONTROL); failed_create_group_worker: if (tmp_wrkr != ODP_SCHED_GROUP_INVALID) - odp_schedule_group_destroy(ODP_SCHED_GROUP_WORKER); + schedule_group_destroy(ODP_SCHED_GROUP_WORKER); failed_create_group_all: if (tmp_all != ODP_SCHED_GROUP_INVALID) - odp_schedule_group_destroy(ODP_SCHED_GROUP_ALL); + schedule_group_destroy(ODP_SCHED_GROUP_ALL); failed_sched_shm_pool_create: @@ -1892,15 +1892,15 @@ static int schedule_term_global(void) /* Destroy enabled sched groups for default GROUP_ALL, GROUP_WORKER and * GROUP_CONTROL groups. */ if (global->config_if.group_enable.all) { - if (odp_schedule_group_destroy(ODP_SCHED_GROUP_ALL) != 0) + if (schedule_group_destroy(ODP_SCHED_GROUP_ALL) != 0) _ODP_ERR("Failed to destroy ODP_SCHED_GROUP_ALL\n"); } if (global->config_if.group_enable.worker) { - if (odp_schedule_group_destroy(ODP_SCHED_GROUP_WORKER) != 0) + if (schedule_group_destroy(ODP_SCHED_GROUP_WORKER) != 0) _ODP_ERR("Failed to destroy ODP_SCHED_GROUP_WORKER\n"); } if (global->config_if.group_enable.control) { - if (odp_schedule_group_destroy(ODP_SCHED_GROUP_CONTROL) != 0) + if (schedule_group_destroy(ODP_SCHED_GROUP_CONTROL) != 0) _ODP_ERR("Failed to destroy ODP_SCHED_GROUP_CONTROL\n"); } @@ -1932,21 +1932,19 @@ static int schedule_init_local(void) odp_spinlock_lock(&global->init_lock); if (global->config_if.group_enable.all) { - if (odp_schedule_group_join(ODP_SCHED_GROUP_ALL, &mask) != 0) { + if (schedule_group_join(ODP_SCHED_GROUP_ALL, &mask) != 0) { _ODP_ERR("Failed to join ODP_SCHED_GROUP_ALL\n"); goto failed_to_join_grp_all; } } if (global->config_if.group_enable.control && thr_type == ODP_THREAD_CONTROL) { - if (odp_schedule_group_join(ODP_SCHED_GROUP_CONTROL, - &mask) != 0) { + if (schedule_group_join(ODP_SCHED_GROUP_CONTROL, &mask) != 0) { _ODP_ERR("Failed to join ODP_SCHED_GROUP_CONTROL\n"); goto failed_to_join_grp_ctrl; } } if (global->config_if.group_enable.worker && thr_type == ODP_THREAD_WORKER) { - if (odp_schedule_group_join(ODP_SCHED_GROUP_WORKER, - &mask) != 0) { + if (schedule_group_join(ODP_SCHED_GROUP_WORKER, &mask) != 0) { _ODP_ERR("Failed to join ODP_SCHED_GROUP_WORKER\n"); goto failed_to_join_grp_wrkr; } @@ -1959,7 +1957,7 @@ static int schedule_init_local(void) failed_to_join_grp_wrkr: failed_to_join_grp_ctrl: if (global->config_if.group_enable.all) - odp_schedule_group_leave(ODP_SCHED_GROUP_ALL, &mask); + schedule_group_leave(ODP_SCHED_GROUP_ALL, &mask); failed_to_join_grp_all: odp_spinlock_unlock(&global->init_lock); @@ -1982,17 +1980,15 @@ static int schedule_term_local(void) odp_thrmask_set(&mask, thr_id); if (global->config_if.group_enable.all) { - if (odp_schedule_group_leave(ODP_SCHED_GROUP_ALL, &mask) != 0) + if (schedule_group_leave(ODP_SCHED_GROUP_ALL, &mask) != 0) _ODP_ERR("Failed to leave ODP_SCHED_GROUP_ALL\n"); } if (global->config_if.group_enable.control && thr_type == ODP_THREAD_CONTROL) { - if (odp_schedule_group_leave(ODP_SCHED_GROUP_CONTROL, - &mask) != 0) + if (schedule_group_leave(ODP_SCHED_GROUP_CONTROL, &mask) != 0) _ODP_ERR("Failed to leave ODP_SCHED_GROUP_CONTROL\n"); } if (global->config_if.group_enable.worker && thr_type == ODP_THREAD_WORKER) { - if (odp_schedule_group_leave(ODP_SCHED_GROUP_WORKER, - &mask) != 0) + if (schedule_group_leave(ODP_SCHED_GROUP_WORKER, &mask) != 0) _ODP_ERR("Failed to leave ODP_SCHED_GROUP_WORKER\n"); } @@ -2026,16 +2022,16 @@ static int schedule_config(const odp_schedule_config_t *config) /* Destroy disabled predefined scheduling groups. */ if (!config->sched_group.all) { - if (odp_schedule_group_destroy(ODP_SCHED_GROUP_ALL) != 0) + if (schedule_group_destroy(ODP_SCHED_GROUP_ALL) != 0) _ODP_ERR("Failed to destroy ODP_SCHED_GROUP_ALL\n"); } if (!config->sched_group.worker) { - if (odp_schedule_group_destroy(ODP_SCHED_GROUP_WORKER) != 0) + if (schedule_group_destroy(ODP_SCHED_GROUP_WORKER) != 0) _ODP_ERR("Failed to destroy ODP_SCHED_GROUP_WORKER\n"); } if (!config->sched_group.control) { - if (odp_schedule_group_destroy(ODP_SCHED_GROUP_CONTROL) != 0) + if (schedule_group_destroy(ODP_SCHED_GROUP_CONTROL) != 0) _ODP_ERR("Failed to destroy ODP_SCHED_GROUP_CONTROL\n"); } @@ -2187,6 +2183,13 @@ static void schedule_print(void) _ODP_PRINT("\n"); } +const _odp_schedule_api_fn_t _odp_schedule_scalable_api; + +static const _odp_schedule_api_fn_t *sched_api(void) +{ + return &_odp_schedule_scalable_api; +} + const schedule_fn_t _odp_schedule_scalable_fn = { .pktio_start = pktio_start, .thr_add = thr_add, @@ -2203,6 +2206,7 @@ const schedule_fn_t _odp_schedule_scalable_fn = { .order_lock = order_lock, .order_unlock = order_unlock, .max_ordered_locks = schedule_max_ordered_locks, + .sched_api = sched_api, }; const _odp_schedule_api_fn_t _odp_schedule_scalable_api = { diff --git a/platform/linux-generic/odp_schedule_sp.c b/platform/linux-generic/odp_schedule_sp.c index c829f9044..c4d18e66e 100644 --- a/platform/linux-generic/odp_schedule_sp.c +++ b/platform/linux-generic/odp_schedule_sp.c @@ -1059,6 +1059,13 @@ static void get_config(schedule_config_t *config) *config = sched_global->config_if; } +const _odp_schedule_api_fn_t _odp_schedule_sp_api; + +static const _odp_schedule_api_fn_t *sched_api(void) +{ + return &_odp_schedule_sp_api; +} + /* Fill in scheduler interface */ const schedule_fn_t _odp_schedule_sp_fn = { .pktio_start = pktio_start, @@ -1076,7 +1083,8 @@ const schedule_fn_t _odp_schedule_sp_fn = { .order_lock = order_lock, .order_unlock = order_unlock, .max_ordered_locks = max_ordered_locks, - .get_config = get_config + .get_config = get_config, + .sched_api = sched_api, }; /* Fill in scheduler API calls */ diff --git a/platform/linux-generic/odp_stash.c b/platform/linux-generic/odp_stash.c index 5ff499843..cb87a5fa5 100644 --- a/platform/linux-generic/odp_stash.c +++ b/platform/linux-generic/odp_stash.c @@ -84,6 +84,7 @@ typedef struct ODP_ALIGNED_CACHE stash_t { char name[ODP_STASH_NAME_LEN]; int index; + uint8_t strict_size; /* Ring header followed by variable sized data (object handles) */ union { @@ -117,7 +118,6 @@ typedef struct stash_global_t { uint32_t max_num; uint32_t max_num_obj; uint32_t num_internal; - uint8_t strict_size; uint8_t stash_state[CONFIG_MAX_STASHES]; stash_t *stash[CONFIG_MAX_STASHES]; uint8_t data[] ODP_ALIGNED_CACHE; @@ -144,7 +144,6 @@ int _odp_stash_init_global(void) uint64_t ring_max_size, stash_max_size, stash_data_size, offset; const uint32_t internal_stashes = odp_global_ro.disable.dma ? 0 : CONFIG_INTERNAL_STASHES; uint8_t *stash_data; - uint8_t strict_size; int val = 0; if (odp_global_ro.disable.stash && odp_global_ro.disable.dma) { @@ -170,14 +169,6 @@ int _odp_stash_init_global(void) _ODP_PRINT(" %s: %i\n", str, val); max_num_obj = val; - str = "stash.strict_size"; - if (!_odp_libconfig_lookup_int(str, &val)) { - _ODP_ERR("Config option '%s' not found.\n", str); - return -1; - } - _ODP_PRINT(" %s: %i\n", str, val); - strict_size = !!val; - _ODP_PRINT("\n"); /* Reserve resources for implementation internal stashes */ @@ -213,7 +204,6 @@ int _odp_stash_init_global(void) stash_global->shm = shm; stash_global->max_num = max_num; stash_global->max_num_obj = max_num_obj; - stash_global->strict_size = strict_size; stash_global->num_internal = internal_stashes; odp_ticketlock_init(&stash_global->lock); @@ -261,7 +251,14 @@ int odp_stash_capability(odp_stash_capability_t *capa, odp_stash_type_t type) capa->max_stashes_any_type = max_stashes; capa->max_stashes = max_stashes; + capa->max_num_obj = stash_global->max_num_obj; + capa->max_num.u8 = stash_global->max_num_obj; + capa->max_num.u16 = stash_global->max_num_obj; + capa->max_num.u32 = stash_global->max_num_obj; + capa->max_num.u64 = stash_global->max_num_obj; + capa->max_num.max_obj_size = stash_global->max_num_obj; + capa->max_obj_size = sizeof(uint64_t); capa->max_get_batch = MIN_RING_SIZE; capa->max_put_batch = MIN_RING_SIZE; @@ -491,7 +488,8 @@ odp_stash_t odp_stash_create(const char *name, const odp_stash_param_t *param) memset(stash, 0, sizeof(stash_t)); /* Set ring function pointers */ - if (stash_global->strict_size) { + stash->strict_size = !!param->strict_size; + if (stash->strict_size) { if (ring_u64) { stash->ring_fn.u64.init = strict_ring_u64_init; stash->ring_fn.u64.enq_multi = strict_ring_u64_enq_multi; @@ -914,6 +912,7 @@ void odp_stash_print(odp_stash_t st) _ODP_PRINT(" obj size %u\n", stash->obj_size); _ODP_PRINT(" obj count %u\n", stash_obj_count(stash)); _ODP_PRINT(" ring size %u\n", stash->ring_size); + _ODP_PRINT(" strict size %u\n", stash->strict_size); _ODP_PRINT("\n"); } diff --git a/platform/linux-generic/odp_timer.c b/platform/linux-generic/odp_timer.c index 8597c3f4e..18e1a7e23 100644 --- a/platform/linux-generic/odp_timer.c +++ b/platform/linux-generic/odp_timer.c @@ -41,7 +41,6 @@ #include <odp_atomic_internal.h> #include <odp_config_internal.h> #include <odp_debug_internal.h> -#include <odp_errno_define.h> #include <odp_event_internal.h> #include <odp_global_data.h> #include <odp_init_internal.h> @@ -409,7 +408,7 @@ static inline odp_timer_t timer_alloc(timer_pool_t *tp, odp_queue_t queue, const /* Add timer to queue */ _odp_queue_fn->timer_add(queue); } else { - _odp_errno = ENFILE; /* Reusing file table overflow */ + /* Reusing file table overflow */ hdl = ODP_TIMER_INVALID; } odp_spinlock_unlock(&tp->lock); @@ -1401,23 +1400,14 @@ odp_timer_pool_t odp_timer_pool_create(const char *name, return ODP_TIMER_POOL_INVALID; } - if ((param->res_ns && param->res_hz) || - (param->res_ns == 0 && param->res_hz == 0)) { - _odp_errno = EINVAL; + if ((param->res_ns && param->res_hz) || (param->res_ns == 0 && param->res_hz == 0)) return ODP_TIMER_POOL_INVALID; - } - if (param->res_hz == 0 && - param->res_ns < timer_global->highest_res_ns) { - _odp_errno = EINVAL; + if (param->res_hz == 0 && param->res_ns < timer_global->highest_res_ns) return ODP_TIMER_POOL_INVALID; - } - if (param->res_ns == 0 && - param->res_hz > timer_global->highest_res_hz) { - _odp_errno = EINVAL; + if (param->res_ns == 0 && param->res_hz > timer_global->highest_res_hz) return ODP_TIMER_POOL_INVALID; - } return timer_pool_new(name, param); } @@ -1643,7 +1633,7 @@ int odp_timer_periodic_start(odp_timer_t timer, const odp_timer_periodic_start_t if (odp_unlikely(abs_tick > cur_tick + tim->periodic_ticks)) return ODP_TIMER_TOO_FAR; } else { - abs_tick = cur_tick; + abs_tick = cur_tick + tim->periodic_ticks; } if (!timer_reset(idx, abs_tick, &tmo_ev, tp)) diff --git a/platform/linux-generic/odp_traffic_mngr.c b/platform/linux-generic/odp_traffic_mngr.c index c8d8bc5c0..82339361d 100644 --- a/platform/linux-generic/odp_traffic_mngr.c +++ b/platform/linux-generic/odp_traffic_mngr.c @@ -23,7 +23,6 @@ #include <odp_traffic_mngr_internal.h> #include <odp_macros_internal.h> #include <odp_init_internal.h> -#include <odp_errno_define.h> #include <odp_global_data.h> #include <odp_schedule_if.h> #include <odp_event_internal.h> diff --git a/platform/linux-generic/pktio/dpdk.c b/platform/linux-generic/pktio/dpdk.c index fa40d1bde..67779e073 100644 --- a/platform/linux-generic/pktio/dpdk.c +++ b/platform/linux-generic/pktio/dpdk.c @@ -1,5 +1,5 @@ /* Copyright (c) 2016-2018, Linaro Limited - * Copyright (c) 2019-2022, Nokia + * Copyright (c) 2019-2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -24,7 +24,6 @@ #include <odp_classification_internal.h> #include <odp_debug_internal.h> -#include <odp_errno_define.h> #include <odp_global_data.h> #include <odp_libconfig_internal.h> #include <odp_macros_internal.h> @@ -59,6 +58,7 @@ #endif #include <ctype.h> +#include <errno.h> #include <sched.h> #include <stdint.h> #include <unistd.h> @@ -71,6 +71,41 @@ #define RTE_MBUF_F_TX_UDP_CKSUM PKT_TX_UDP_CKSUM #define RTE_MBUF_F_TX_TCP_CKSUM PKT_TX_TCP_CKSUM #define RTE_MEMPOOL_REGISTER_OPS MEMPOOL_REGISTER_OPS + + #define RTE_ETH_RSS_IPV4 ETH_RSS_IPV4 + #define RTE_ETH_RSS_FRAG_IPV4 ETH_RSS_FRAG_IPV4 + #define RTE_ETH_RSS_NONFRAG_IPV4_TCP ETH_RSS_NONFRAG_IPV4_TCP + #define RTE_ETH_RSS_NONFRAG_IPV4_UDP ETH_RSS_NONFRAG_IPV4_UDP + #define RTE_ETH_RSS_NONFRAG_IPV4_OTHER ETH_RSS_NONFRAG_IPV4_OTHER + + #define RTE_ETH_RSS_IPV6 ETH_RSS_IPV6 + #define RTE_ETH_RSS_IPV6_EX ETH_RSS_IPV6_EX + #define RTE_ETH_RSS_IPV6_UDP_EX ETH_RSS_IPV6_UDP_EX + #define RTE_ETH_RSS_IPV6_TCP_EX ETH_RSS_IPV6_TCP_EX + #define RTE_ETH_RSS_FRAG_IPV6 ETH_RSS_FRAG_IPV6 + #define RTE_ETH_RSS_NONFRAG_IPV6_TCP ETH_RSS_NONFRAG_IPV6_TCP + #define RTE_ETH_RSS_NONFRAG_IPV6_UDP ETH_RSS_NONFRAG_IPV6_UDP + #define RTE_ETH_RSS_NONFRAG_IPV6_OTHER ETH_RSS_NONFRAG_IPV6_OTHER + + #define RTE_ETH_MQ_RX_RSS ETH_MQ_RX_RSS + #define RTE_ETH_MQ_TX_NONE ETH_MQ_TX_NONE + + #define RTE_ETH_RX_OFFLOAD_IPV4_CKSUM DEV_RX_OFFLOAD_IPV4_CKSUM + #define RTE_ETH_RX_OFFLOAD_TCP_CKSUM DEV_RX_OFFLOAD_TCP_CKSUM + #define RTE_ETH_RX_OFFLOAD_UDP_CKSUM DEV_RX_OFFLOAD_UDP_CKSUM + + #define RTE_ETH_TX_OFFLOAD_IPV4_CKSUM DEV_TX_OFFLOAD_IPV4_CKSUM + #define RTE_ETH_TX_OFFLOAD_SCTP_CKSUM DEV_TX_OFFLOAD_SCTP_CKSUM + #define RTE_ETH_TX_OFFLOAD_TCP_CKSUM DEV_TX_OFFLOAD_TCP_CKSUM + #define RTE_ETH_TX_OFFLOAD_UDP_CKSUM DEV_TX_OFFLOAD_UDP_CKSUM + + #define RTE_ETH_FC_FULL RTE_FC_FULL + #define RTE_ETH_FC_RX_PAUSE RTE_FC_RX_PAUSE + #define RTE_ETH_FC_TX_PAUSE RTE_FC_TX_PAUSE + #define RTE_ETH_LINK_AUTONEG ETH_LINK_AUTONEG + #define RTE_ETH_LINK_FULL_DUPLEX ETH_LINK_FULL_DUPLEX + #define RTE_ETH_LINK_UP ETH_LINK_UP + #define RTE_ETH_SPEED_NUM_NONE ETH_SPEED_NUM_NONE #endif #define MEMPOOL_FLAGS 0 @@ -848,12 +883,11 @@ static inline void pkt_set_ol_tx(odp_pktout_config_opt_t *pktout_cfg, static inline int pkt_to_mbuf(pktio_entry_t *pktio_entry, struct rte_mbuf *mbuf_table[], const odp_packet_t pkt_table[], uint16_t num, - int *tx_ts_idx) + uint16_t *tx_ts_idx) { pkt_dpdk_t *pkt_dpdk = pkt_priv(pktio_entry); - int i, j; char *data; - uint16_t pkt_len; + uint16_t i, j, pkt_len; uint8_t chksum_enabled = pktio_entry->enabled.chksum_insert; uint8_t tx_ts_enabled = _odp_pktio_tx_ts_enabled(pktio_entry); odp_pktout_config_opt_t *pktout_cfg = &pktio_entry->config.pktout; @@ -868,11 +902,8 @@ static inline int pkt_to_mbuf(pktio_entry_t *pktio_entry, pkt_len = packet_len(pkt_hdr); - if (odp_unlikely(pkt_len > pkt_dpdk->mtu)) { - if (i == 0) - _odp_errno = EMSGSIZE; + if (odp_unlikely(pkt_len > pkt_dpdk->mtu)) goto fail; - } /* Packet always fits in mbuf */ data = rte_pktmbuf_append(mbuf_table[i], pkt_len); @@ -894,7 +925,7 @@ fail: for (j = i; j < num; j++) rte_pktmbuf_free(mbuf_table[j]); - return i; + return i > 0 ? i : -1; } static inline void prefetch_pkt(struct rte_mbuf *mbuf) @@ -1007,15 +1038,15 @@ static inline int mbuf_to_pkt_zero(pktio_entry_t *pktio_entry, static inline int pkt_to_mbuf_zero(pktio_entry_t *pktio_entry, struct rte_mbuf *mbuf_table[], const odp_packet_t pkt_table[], uint16_t num, - uint16_t *copy_count, uint16_t cpy_idx[], int *tx_ts_idx) + uint16_t *copy_count, uint16_t cpy_idx[], uint16_t *tx_ts_idx) { pkt_dpdk_t *pkt_dpdk = pkt_priv(pktio_entry); odp_pktout_config_opt_t *pktout_cfg = &pktio_entry->config.pktout; odp_pktout_config_opt_t *pktout_capa = &pkt_dpdk->pktout_capa; uint16_t mtu = pkt_dpdk->mtu; + uint16_t i; uint8_t chksum_enabled = pktio_entry->enabled.chksum_insert; uint8_t tx_ts_enabled = _odp_pktio_tx_ts_enabled(pktio_entry); - int i; *copy_count = 0; for (i = 0; i < num; i++) { @@ -1034,7 +1065,7 @@ static inline int pkt_to_mbuf_zero(pktio_entry_t *pktio_entry, pkt_set_ol_tx(pktout_cfg, pktout_capa, pkt_hdr, mbuf, odp_packet_data(pkt)); } else { - int dummy_idx = 0; + uint16_t dummy_idx = 0; /* Fall back to packet copy */ if (odp_unlikely(pkt_to_mbuf(pktio_entry, &mbuf, @@ -1052,9 +1083,7 @@ static inline int pkt_to_mbuf_zero(pktio_entry_t *pktio_entry, return i; fail: - if (i == 0) - _odp_errno = EMSGSIZE; - return i; + return i > 0 ? i : -1; } /* Test if s has only digits or not. Dpdk pktio uses only digits.*/ @@ -1189,22 +1218,22 @@ static void hash_proto_to_rss_conf(struct rte_eth_rss_conf *rss_conf, const odp_pktin_hash_proto_t *hash_proto) { if (hash_proto->proto.ipv4_udp) - rss_conf->rss_hf |= ETH_RSS_NONFRAG_IPV4_UDP; + rss_conf->rss_hf |= RTE_ETH_RSS_NONFRAG_IPV4_UDP; if (hash_proto->proto.ipv4_tcp) - rss_conf->rss_hf |= ETH_RSS_NONFRAG_IPV4_TCP; + rss_conf->rss_hf |= RTE_ETH_RSS_NONFRAG_IPV4_TCP; if (hash_proto->proto.ipv4) - rss_conf->rss_hf |= ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 | - ETH_RSS_NONFRAG_IPV4_OTHER; + rss_conf->rss_hf |= RTE_ETH_RSS_IPV4 | RTE_ETH_RSS_FRAG_IPV4 | + RTE_ETH_RSS_NONFRAG_IPV4_OTHER; if (hash_proto->proto.ipv6_udp) - rss_conf->rss_hf |= ETH_RSS_NONFRAG_IPV6_UDP | - ETH_RSS_IPV6_UDP_EX; + rss_conf->rss_hf |= RTE_ETH_RSS_NONFRAG_IPV6_UDP | + RTE_ETH_RSS_IPV6_UDP_EX; if (hash_proto->proto.ipv6_tcp) - rss_conf->rss_hf |= ETH_RSS_NONFRAG_IPV6_TCP | - ETH_RSS_IPV6_TCP_EX; + rss_conf->rss_hf |= RTE_ETH_RSS_NONFRAG_IPV6_TCP | + RTE_ETH_RSS_IPV6_TCP_EX; if (hash_proto->proto.ipv6) - rss_conf->rss_hf |= ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6 | - ETH_RSS_NONFRAG_IPV6_OTHER | - ETH_RSS_IPV6_EX; + rss_conf->rss_hf |= RTE_ETH_RSS_IPV6 | RTE_ETH_RSS_FRAG_IPV6 | + RTE_ETH_RSS_NONFRAG_IPV6_OTHER | + RTE_ETH_RSS_IPV6_EX; rss_conf->rss_key = NULL; } @@ -1218,34 +1247,34 @@ static int dpdk_setup_eth_dev(pktio_entry_t *pktio_entry) memset(ð_conf, 0, sizeof(eth_conf)); - eth_conf.rxmode.mq_mode = ETH_MQ_RX_RSS; - eth_conf.txmode.mq_mode = ETH_MQ_TX_NONE; + eth_conf.rxmode.mq_mode = RTE_ETH_MQ_RX_RSS; + eth_conf.txmode.mq_mode = RTE_ETH_MQ_TX_NONE; eth_conf.rx_adv_conf.rss_conf = pkt_dpdk->rss_conf; /* Setup RX checksum offloads */ if (pktio_entry->config.pktin.bit.ipv4_chksum) - rx_offloads |= DEV_RX_OFFLOAD_IPV4_CKSUM; + rx_offloads |= RTE_ETH_RX_OFFLOAD_IPV4_CKSUM; if (pktio_entry->config.pktin.bit.udp_chksum) - rx_offloads |= DEV_RX_OFFLOAD_UDP_CKSUM; + rx_offloads |= RTE_ETH_RX_OFFLOAD_UDP_CKSUM; if (pktio_entry->config.pktin.bit.tcp_chksum) - rx_offloads |= DEV_RX_OFFLOAD_TCP_CKSUM; + rx_offloads |= RTE_ETH_RX_OFFLOAD_TCP_CKSUM; eth_conf.rxmode.offloads = rx_offloads; /* Setup TX checksum offloads */ if (pktio_entry->config.pktout.bit.ipv4_chksum_ena) - tx_offloads |= DEV_TX_OFFLOAD_IPV4_CKSUM; + tx_offloads |= RTE_ETH_TX_OFFLOAD_IPV4_CKSUM; if (pktio_entry->config.pktout.bit.udp_chksum_ena) - tx_offloads |= DEV_TX_OFFLOAD_UDP_CKSUM; + tx_offloads |= RTE_ETH_TX_OFFLOAD_UDP_CKSUM; if (pktio_entry->config.pktout.bit.tcp_chksum_ena) - tx_offloads |= DEV_TX_OFFLOAD_TCP_CKSUM; + tx_offloads |= RTE_ETH_TX_OFFLOAD_TCP_CKSUM; if (pktio_entry->config.pktout.bit.sctp_chksum_ena) - tx_offloads |= DEV_TX_OFFLOAD_SCTP_CKSUM; + tx_offloads |= RTE_ETH_TX_OFFLOAD_SCTP_CKSUM; eth_conf.txmode.offloads = tx_offloads; @@ -1461,32 +1490,32 @@ static void prepare_rss_conf(pktio_entry_t *pktio_entry, /* Print debug info about unsupported hash protocols */ if (p->hash_proto.proto.ipv4 && - ((rss_hf_capa & ETH_RSS_IPV4) == 0)) + ((rss_hf_capa & RTE_ETH_RSS_IPV4) == 0)) _ODP_PRINT("DPDK: hash_proto.ipv4 not supported (rss_hf_capa 0x%" PRIx64 ")\n", rss_hf_capa); if (p->hash_proto.proto.ipv4_udp && - ((rss_hf_capa & ETH_RSS_NONFRAG_IPV4_UDP) == 0)) + ((rss_hf_capa & RTE_ETH_RSS_NONFRAG_IPV4_UDP) == 0)) _ODP_PRINT("DPDK: hash_proto.ipv4_udp not supported (rss_hf_capa 0x%" PRIx64 ")\n", rss_hf_capa); if (p->hash_proto.proto.ipv4_tcp && - ((rss_hf_capa & ETH_RSS_NONFRAG_IPV4_TCP) == 0)) + ((rss_hf_capa & RTE_ETH_RSS_NONFRAG_IPV4_TCP) == 0)) _ODP_PRINT("DPDK: hash_proto.ipv4_tcp not supported (rss_hf_capa 0x%" PRIx64 ")\n", rss_hf_capa); if (p->hash_proto.proto.ipv6 && - ((rss_hf_capa & ETH_RSS_IPV6) == 0)) + ((rss_hf_capa & RTE_ETH_RSS_IPV6) == 0)) _ODP_PRINT("DPDK: hash_proto.ipv6 not supported (rss_hf_capa 0x%" PRIx64 ")\n", rss_hf_capa); if (p->hash_proto.proto.ipv6_udp && - ((rss_hf_capa & ETH_RSS_NONFRAG_IPV6_UDP) == 0)) + ((rss_hf_capa & RTE_ETH_RSS_NONFRAG_IPV6_UDP) == 0)) _ODP_PRINT("DPDK: hash_proto.ipv6_udp not supported (rss_hf_capa 0x%" PRIx64 ")\n", rss_hf_capa); if (p->hash_proto.proto.ipv6_tcp && - ((rss_hf_capa & ETH_RSS_NONFRAG_IPV6_TCP) == 0)) + ((rss_hf_capa & RTE_ETH_RSS_NONFRAG_IPV6_TCP) == 0)) _ODP_PRINT("DPDK: hash_proto.ipv6_tcp not supported (rss_hf_capa 0x%" PRIx64 ")\n", rss_hf_capa); @@ -1684,26 +1713,26 @@ static int dpdk_init_capability(pktio_entry_t *pktio_entry, capa->config.pktin.bit.ts_ptp = 1; capa->config.pktin.bit.ipv4_chksum = ptype_l3_ipv4 && - (dev_info->rx_offload_capa & DEV_RX_OFFLOAD_IPV4_CKSUM) ? 1 : 0; + (dev_info->rx_offload_capa & RTE_ETH_RX_OFFLOAD_IPV4_CKSUM) ? 1 : 0; if (capa->config.pktin.bit.ipv4_chksum) capa->config.pktin.bit.drop_ipv4_err = 1; capa->config.pktin.bit.udp_chksum = ptype_l4_udp && - (dev_info->rx_offload_capa & DEV_RX_OFFLOAD_UDP_CKSUM) ? 1 : 0; + (dev_info->rx_offload_capa & RTE_ETH_RX_OFFLOAD_UDP_CKSUM) ? 1 : 0; if (capa->config.pktin.bit.udp_chksum) capa->config.pktin.bit.drop_udp_err = 1; capa->config.pktin.bit.tcp_chksum = ptype_l4_tcp && - (dev_info->rx_offload_capa & DEV_RX_OFFLOAD_TCP_CKSUM) ? 1 : 0; + (dev_info->rx_offload_capa & RTE_ETH_RX_OFFLOAD_TCP_CKSUM) ? 1 : 0; if (capa->config.pktin.bit.tcp_chksum) capa->config.pktin.bit.drop_tcp_err = 1; capa->config.pktout.bit.ipv4_chksum = - (dev_info->tx_offload_capa & DEV_TX_OFFLOAD_IPV4_CKSUM) ? 1 : 0; + (dev_info->tx_offload_capa & RTE_ETH_TX_OFFLOAD_IPV4_CKSUM) ? 1 : 0; capa->config.pktout.bit.udp_chksum = - (dev_info->tx_offload_capa & DEV_TX_OFFLOAD_UDP_CKSUM) ? 1 : 0; + (dev_info->tx_offload_capa & RTE_ETH_TX_OFFLOAD_UDP_CKSUM) ? 1 : 0; capa->config.pktout.bit.tcp_chksum = - (dev_info->tx_offload_capa & DEV_TX_OFFLOAD_TCP_CKSUM) ? 1 : 0; + (dev_info->tx_offload_capa & RTE_ETH_TX_OFFLOAD_TCP_CKSUM) ? 1 : 0; capa->config.pktout.bit.ipv4_chksum_ena = capa->config.pktout.bit.ipv4_chksum; @@ -2099,10 +2128,9 @@ static int dpdk_send(pktio_entry_t *pktio_entry, int index, pkt_dpdk_t *pkt_dpdk = pkt_priv(pktio_entry); uint16_t copy_count = 0; uint16_t cpy_idx[num]; - int tx_pkts; - int i; + uint16_t tx_pkts; int mbufs; - int tx_ts_idx = 0; + uint16_t tx_ts_idx = 0; if (_ODP_DPDK_ZERO_COPY) mbufs = pkt_to_mbuf_zero(pktio_entry, tx_mbufs, pkt_table, num, @@ -2111,6 +2139,9 @@ static int dpdk_send(pktio_entry_t *pktio_entry, int index, mbufs = pkt_to_mbuf(pktio_entry, tx_mbufs, pkt_table, num, &tx_ts_idx); + if (odp_unlikely(mbufs < 1)) + return mbufs; + if (!pkt_dpdk->flags.lockless_tx) odp_ticketlock_lock(&pkt_dpdk->tx_lock[index]); @@ -2128,7 +2159,7 @@ static int dpdk_send(pktio_entry_t *pktio_entry, int index, if (odp_unlikely(copy_count)) { uint16_t idx; - for (i = 0; i < copy_count; i++) { + for (uint16_t i = 0; i < copy_count; i++) { idx = cpy_idx[i]; if (odp_likely(idx < tx_pkts)) @@ -2137,20 +2168,14 @@ static int dpdk_send(pktio_entry_t *pktio_entry, int index, rte_pktmbuf_free(tx_mbufs[idx]); } } - if (odp_unlikely(tx_pkts == 0 && _odp_errno != 0)) - return -1; } else { if (odp_unlikely(tx_pkts < mbufs)) { - for (i = tx_pkts; i < mbufs; i++) + for (uint16_t i = tx_pkts; i < mbufs; i++) rte_pktmbuf_free(tx_mbufs[i]); } - if (odp_unlikely(tx_pkts == 0)) { - if (_odp_errno != 0) - return -1; - } else { + if (odp_likely(tx_pkts)) odp_packet_free_multi(pkt_table, tx_pkts); - } } return tx_pkts; @@ -2234,32 +2259,32 @@ static int dpdk_link_info(pktio_entry_t *pktio_entry, odp_pktio_link_info_t *inf memset(info, 0, sizeof(odp_pktio_link_info_t)); info->pause_rx = ODP_PKTIO_LINK_PAUSE_OFF; info->pause_tx = ODP_PKTIO_LINK_PAUSE_OFF; - if (fc_conf.mode == RTE_FC_RX_PAUSE) { + if (fc_conf.mode == RTE_ETH_FC_RX_PAUSE) { info->pause_rx = ODP_PKTIO_LINK_PAUSE_ON; - } else if (fc_conf.mode == RTE_FC_TX_PAUSE) { + } else if (fc_conf.mode == RTE_ETH_FC_TX_PAUSE) { info->pause_tx = ODP_PKTIO_LINK_PAUSE_ON; - } else if (fc_conf.mode == RTE_FC_FULL) { + } else if (fc_conf.mode == RTE_ETH_FC_FULL) { info->pause_rx = ODP_PKTIO_LINK_PAUSE_ON; info->pause_tx = ODP_PKTIO_LINK_PAUSE_ON; } rte_eth_link_get_nowait(port_id, &link); - if (link.link_autoneg == ETH_LINK_AUTONEG) + if (link.link_autoneg == RTE_ETH_LINK_AUTONEG) info->autoneg = ODP_PKTIO_LINK_AUTONEG_ON; else info->autoneg = ODP_PKTIO_LINK_AUTONEG_OFF; - if (link.link_duplex == ETH_LINK_FULL_DUPLEX) + if (link.link_duplex == RTE_ETH_LINK_FULL_DUPLEX) info->duplex = ODP_PKTIO_LINK_DUPLEX_FULL; else info->duplex = ODP_PKTIO_LINK_DUPLEX_HALF; - if (link.link_speed == ETH_SPEED_NUM_NONE) + if (link.link_speed == RTE_ETH_SPEED_NUM_NONE) info->speed = ODP_PKTIO_LINK_SPEED_UNKNOWN; else info->speed = link.link_speed; - if (link.link_status == ETH_LINK_UP) + if (link.link_status == RTE_ETH_LINK_UP) info->status = ODP_PKTIO_LINK_STATUS_UP; else info->status = ODP_PKTIO_LINK_STATUS_DOWN; diff --git a/platform/linux-generic/pktio/io_ops.c b/platform/linux-generic/pktio/io_ops.c index f9ea89f71..141b881e3 100644 --- a/platform/linux-generic/pktio/io_ops.c +++ b/platform/linux-generic/pktio/io_ops.c @@ -19,9 +19,6 @@ const pktio_if_ops_t * const _odp_pktio_if_ops[] = { #ifdef _ODP_PKTIO_XDP &_odp_sock_xdp_pktio_ops, #endif -#ifdef _ODP_PKTIO_NETMAP - &_odp_netmap_pktio_ops, -#endif #ifdef _ODP_PKTIO_PCAP &_odp_pcap_pktio_ops, #endif diff --git a/platform/linux-generic/pktio/ipc.c b/platform/linux-generic/pktio/ipc.c index 586bc0aa7..58b949402 100644 --- a/platform/linux-generic/pktio/ipc.c +++ b/platform/linux-generic/pktio/ipc.c @@ -10,13 +10,11 @@ #include <odp_debug_internal.h> #include <odp_packet_io_internal.h> #include <odp_pool_internal.h> -#include <odp_errno_define.h> #include <odp_macros_internal.h> #include <odp_shm_internal.h> #include <odp_ring_ptr_internal.h> #include <odp_global_data.h> -#include <errno.h> #include <fcntl.h> #include <stdint.h> #include <sys/mman.h> @@ -129,7 +127,6 @@ static ring_ptr_t *_ring_create(const char *name, uint32_t count, /* count must be a power of 2 */ if (!_ODP_CHECK_IS_POWER2(count)) { _ODP_ERR("Requested size is invalid, must be a power of 2\n"); - _odp_errno = EINVAL; return NULL; } @@ -144,7 +141,6 @@ static ring_ptr_t *_ring_create(const char *name, uint32_t count, ring_ptr_init(r); } else { - _odp_errno = ENOMEM; _ODP_ERR("Cannot reserve memory\n"); } @@ -322,7 +318,7 @@ static int _ipc_init_master(pktio_entry_t *pktio_entry, goto free_s_prod; } - memcpy(pinfo->master.pool_name, pool_name, strlen(pool_name)); + strcpy(pinfo->master.pool_name, pool_name); /* Export ring info for the slave process to use */ pinfo->master.ring_size = ring_size; diff --git a/platform/linux-generic/pktio/loop.c b/platform/linux-generic/pktio/loop.c index a69daf56b..f9c72db25 100644 --- a/platform/linux-generic/pktio/loop.c +++ b/platform/linux-generic/pktio/loop.c @@ -21,7 +21,6 @@ #include <odp_parse_internal.h> #include <odp_classification_internal.h> #include <odp_debug_internal.h> -#include <odp_errno_define.h> #include <odp_event_internal.h> #include <odp_global_data.h> #include <odp_ipsec_internal.h> @@ -33,7 +32,6 @@ #include <protocols/eth.h> #include <protocols/ip.h> -#include <errno.h> #include <inttypes.h> #include <limits.h> #include <stdint.h> @@ -336,8 +334,10 @@ static int loopback_recv(pktio_entry_t *pktio_entry, int index, odp_packet_t pkt /* Try IPsec inline processing */ if (pktio_entry->config.inbound_ipsec && !pkt_hdr->p.flags.ip_err && - odp_packet_has_ipsec(pkt)) + odp_packet_has_ipsec(pkt)) { _odp_ipsec_try_inline(&pkt); + pkt_hdr = packet_hdr(pkt); + } if (!pkt_hdr->p.flags.all.error) { octets += pkt_len; @@ -525,10 +525,8 @@ static int loopback_send(pktio_entry_t *pktio_entry, int index, const odp_packet uint32_t pkt_len = odp_packet_len(pkt_tbl[i]); if (odp_unlikely(pkt_len > pkt_loop->mtu)) { - if (nb_tx == 0) { - _odp_errno = EMSGSIZE; + if (nb_tx == 0) return -1; - } break; } diff --git a/platform/linux-generic/pktio/netmap.c b/platform/linux-generic/pktio/netmap.c deleted file mode 100644 index 0352e33f7..000000000 --- a/platform/linux-generic/pktio/netmap.c +++ /dev/null @@ -1,1347 +0,0 @@ -/* Copyright (c) 2015-2018, Linaro Limited - * Copyright (c) 2019-2022, Nokia - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -#include <odp/autoheader_internal.h> - -#ifdef _ODP_PKTIO_NETMAP - -#include <odp_posix_extensions.h> - -#include <odp/api/packet.h> -#include <odp/api/plat/packet_inlines.h> -#include <odp/api/time.h> -#include <odp/api/plat/time_inlines.h> - -#include <odp_parse_internal.h> -#include <odp_packet_io_internal.h> -#include <odp_packet_io_stats.h> -#include <odp_ethtool_stats.h> -#include <odp_ethtool_rss.h> -#include <odp_socket_common.h> -#include <odp_debug_internal.h> -#include <odp_errno_define.h> -#include <odp_classification_datamodel.h> -#include <odp_classification_internal.h> -#include <odp_libconfig_internal.h> -#include <odp_macros_internal.h> - -#include <protocols/eth.h> -#include <sys/ioctl.h> -#include <sys/syscall.h> -#include <poll.h> -#include <linux/ethtool.h> -#include <linux/sockios.h> -#include <inttypes.h> - -/* Disable netmap debug prints */ -#ifndef ND -#define ND(_fmt, ...) do {} while (0) -#define D(_fmt, ...) do {} while (0) -#define RD(lps, format, ...) do {} while (0) -#endif - -#define NETMAP_WITH_LIBS -#include <net/netmap_user.h> - -#define NM_WAIT_TIMEOUT 10 /* netmap_wait_for_link() timeout in seconds */ -#define NM_INJECT_RETRIES 10 - -#define NM_MAX_DESC 64 - -#define NM_BUF_SIZE "/sys/module/netmap/parameters/buf_size" - -/** netmap runtime configuration options */ -typedef struct { - int nr_rx_slots; - int nr_tx_slots; -} netmap_opt_t; - -/** Ring for mapping pktin/pktout queues to netmap descriptors */ -typedef struct ODP_ALIGNED_CACHE { - unsigned int first; /**< Index of first netmap descriptor */ - unsigned int last; /**< Index of last netmap descriptor */ - unsigned int num; /**< Number of netmap descriptors */ - /** Netmap metadata for the device */ - struct nm_desc *desc[NM_MAX_DESC]; - unsigned int cur; /**< Index of current netmap descriptor */ - odp_ticketlock_t lock; /**< Queue lock */ -} netmap_ring_t; - -/** Netmap ring slot */ -typedef struct { - char *buf; /**< Slot buffer pointer */ - uint16_t len; /**< Slot length */ -} netmap_slot_t; - -/** Packet socket using netmap mmaped rings for both Rx and Tx */ -typedef struct { - odp_pool_t pool; /**< pool to alloc packets from */ - uint32_t if_flags; /**< interface flags */ - uint32_t mtu; /**< maximum transmission unit */ - uint32_t mtu_max; /**< maximum supported MTU value */ - int sockfd; /**< control socket */ - unsigned char if_mac[ETH_ALEN]; /**< eth mac address */ - char nm_name[IF_NAMESIZE + 7]; /**< netmap:<ifname> */ - char if_name[IF_NAMESIZE]; /**< interface name used in ioctl */ - odp_bool_t is_virtual; /**< nm virtual port (VALE/pipe) */ - uint32_t num_rx_rings; /**< number of nm rx rings */ - uint32_t num_tx_rings; /**< number of nm tx rings */ - unsigned int num_rx_desc_rings; /**< number of rx descriptor rings */ - unsigned int num_tx_desc_rings; /**< number of tx descriptor rings */ - odp_bool_t lockless_rx; /**< no locking for rx */ - odp_bool_t lockless_tx; /**< no locking for tx */ - /** mapping of pktin queues to netmap rx descriptors */ - netmap_ring_t rx_desc_ring[ODP_PKTIN_MAX_QUEUES]; - /** mapping of pktout queues to netmap tx descriptors */ - netmap_ring_t tx_desc_ring[ODP_PKTOUT_MAX_QUEUES]; - netmap_opt_t opt; /**< options */ -} pkt_netmap_t; - -ODP_STATIC_ASSERT(PKTIO_PRIVATE_SIZE >= sizeof(pkt_netmap_t), - "PKTIO_PRIVATE_SIZE too small"); - -static inline pkt_netmap_t *pkt_priv(pktio_entry_t *pktio_entry) -{ - return (pkt_netmap_t *)(uintptr_t)(pktio_entry->pkt_priv); -} - -static int disable_pktio; /** !0 this pktio disabled, 0 enabled */ -static int netmap_stats_reset(pktio_entry_t *pktio_entry); - -static int read_netmap_buf_size(void) -{ - FILE *file; - char str[128]; - int size = 0; - - file = fopen(NM_BUF_SIZE, "rt"); - if (file == NULL) { - /* File not found */ - return 0; - } - - if (fgets(str, sizeof(str), file) != NULL) { - /* Read netmap buffer size */ - if (sscanf(str, "%i", &size) != 1) - size = 0; - } - - fclose(file); - - return size; -} - -static int lookup_opt(const char *opt_name, const char *drv_name, int *val) -{ - const char *base = "pktio_netmap"; - int ret; - - ret = _odp_libconfig_lookup_ext_int(base, drv_name, opt_name, val); - if (ret == 0) - _ODP_ERR("Unable to find netmap configuration option: %s\n", opt_name); - - return ret; -} - -static int init_options(pktio_entry_t *pktio_entry) -{ - netmap_opt_t *opt = &pkt_priv(pktio_entry)->opt; - - if (!lookup_opt("nr_rx_slots", "virt", - &opt->nr_rx_slots)) - return -1; - if (opt->nr_rx_slots < 0 || - opt->nr_rx_slots > 4096) { - _ODP_ERR("Invalid number of RX slots\n"); - return -1; - } - - if (!lookup_opt("nr_tx_slots", "virt", - &opt->nr_tx_slots)) - return -1; - if (opt->nr_tx_slots < 0 || - opt->nr_tx_slots > 4096) { - _ODP_ERR("Invalid number of TX slots\n"); - return -1; - } - - _ODP_DBG("netmap interface: %s\n", pkt_priv(pktio_entry)->if_name); - _ODP_DBG(" num_rx_desc: %d\n", opt->nr_rx_slots); - _ODP_DBG(" num_tx_desc: %d\n", opt->nr_tx_slots); - - return 0; -} - -static int netmap_do_ioctl(pktio_entry_t *pktio_entry, unsigned long cmd, - int subcmd) -{ - pkt_netmap_t *pkt_nm = pkt_priv(pktio_entry); - struct ethtool_value eval; - struct ifreq ifr; - int err; - int fd = pkt_nm->sockfd; - - memset(&ifr, 0, sizeof(ifr)); - snprintf(ifr.ifr_name, sizeof(ifr.ifr_name), "%s", - pkt_priv(pktio_entry)->if_name); - - switch (cmd) { - case SIOCSIFFLAGS: - ifr.ifr_flags = pkt_nm->if_flags & 0xffff; - break; - case SIOCETHTOOL: - eval.cmd = subcmd; - eval.data = 0; - ifr.ifr_data = (caddr_t)&eval; - break; - default: - break; - } - err = ioctl(fd, cmd, &ifr); - if (err) - goto done; - - switch (cmd) { - case SIOCGIFFLAGS: - pkt_nm->if_flags = (ifr.ifr_flags << 16) | - (0xffff & ifr.ifr_flags); - break; - case SIOCETHTOOL: - if (subcmd == ETHTOOL_GLINK) - return eval.data; - break; - default: - break; - } -done: - if (err) - _ODP_ERR("ioctl err %d %lu: %s\n", err, cmd, strerror(errno)); - - return err; -} - -/** - * Map netmap rings to pktin/pktout queues - * - * @param rings Array of netmap descriptor rings - * @param num_queues Number of pktin/pktout queues - * @param num_rings Number of matching netmap rings - */ -static inline void map_netmap_rings(netmap_ring_t *rings, - unsigned int num_queues, unsigned int num_rings) -{ - netmap_ring_t *desc_ring; - unsigned int rings_per_queue; - unsigned int remainder; - unsigned int mapped_rings; - unsigned int i; - unsigned int desc_id = 0; - - rings_per_queue = num_rings / num_queues; - remainder = num_rings % num_queues; - - if (remainder) - _ODP_DBG("WARNING: Netmap rings mapped unevenly to queues\n"); - - for (i = 0; i < num_queues; i++) { - desc_ring = &rings[i]; - if (i < remainder) - mapped_rings = rings_per_queue + 1; - else - mapped_rings = rings_per_queue; - - desc_ring->first = desc_id; - desc_ring->cur = desc_id; - desc_ring->last = desc_ring->first + mapped_rings - 1; - desc_ring->num = mapped_rings; - - desc_id = desc_ring->last + 1; - } -} - -static int netmap_input_queues_config(pktio_entry_t *pktio_entry, - const odp_pktin_queue_param_t *p) -{ - pkt_netmap_t *pkt_nm = pkt_priv(pktio_entry); - odp_pktin_mode_t mode = pktio_entry->param.in_mode; - unsigned int num_queues = p->num_queues; - odp_bool_t lockless; - - /* Scheduler synchronizes input queue polls. Only single thread - * at a time polls a queue */ - if (mode == ODP_PKTIN_MODE_SCHED) - lockless = 1; - else - lockless = (p->op_mode == ODP_PKTIO_OP_MT_UNSAFE); - - if (p->hash_enable && num_queues > 1) { - if (_odp_rss_conf_set_fd(pkt_priv(pktio_entry)->sockfd, - pkt_priv(pktio_entry)->if_name, - &p->hash_proto)) { - _ODP_ERR("Failed to configure input hash\n"); - return -1; - } - } - - pkt_nm->lockless_rx = lockless; - - return 0; -} - -static int netmap_output_queues_config(pktio_entry_t *pktio_entry, - const odp_pktout_queue_param_t *p) -{ - pkt_netmap_t *pkt_nm = pkt_priv(pktio_entry); - - pkt_nm->lockless_tx = (p->op_mode == ODP_PKTIO_OP_MT_UNSAFE); - - return 0; -} - -/** - * Close netmap descriptors - * - * Can be reopened using netmap_start() function. - * - * @param pktio_entry Packet IO entry - */ -static inline void netmap_close_descriptors(pktio_entry_t *pktio_entry) -{ - int i, j; - pkt_netmap_t *pkt_nm = pkt_priv(pktio_entry); - - for (i = 0; i < ODP_PKTIN_MAX_QUEUES; i++) { - for (j = 0; j < NM_MAX_DESC; j++) { - if (pkt_nm->rx_desc_ring[i].desc[j] != NULL) { - nm_close(pkt_nm->rx_desc_ring[i].desc[j]); - pkt_nm->rx_desc_ring[i].desc[j] = NULL; - } - } - } - for (i = 0; i < ODP_PKTOUT_MAX_QUEUES; i++) { - for (j = 0; j < NM_MAX_DESC; j++) { - if (pkt_nm->tx_desc_ring[i].desc[j] != NULL) { - nm_close(pkt_nm->tx_desc_ring[i].desc[j]); - pkt_nm->tx_desc_ring[i].desc[j] = NULL; - } - } - } - - pkt_nm->num_rx_desc_rings = 0; - pkt_nm->num_tx_desc_rings = 0; -} - -static int netmap_close(pktio_entry_t *pktio_entry) -{ - pkt_netmap_t *pkt_nm = pkt_priv(pktio_entry); - - netmap_close_descriptors(pktio_entry); - - if (pkt_nm->sockfd != -1 && close(pkt_nm->sockfd) != 0) { - _odp_errno = errno; - _ODP_ERR("close(sockfd): %s\n", strerror(errno)); - return -1; - } - return 0; -} - -static int netmap_link_status(pktio_entry_t *pktio_entry) -{ - if (pkt_priv(pktio_entry)->is_virtual) - return ODP_PKTIO_LINK_STATUS_UP; - - return _odp_link_status_fd(pkt_priv(pktio_entry)->sockfd, - pkt_priv(pktio_entry)->if_name); -} - -static int netmap_link_info(pktio_entry_t *pktio_entry, odp_pktio_link_info_t *info) -{ - pkt_netmap_t *pkt_nm = pkt_priv(pktio_entry); - - if (pkt_nm->is_virtual) { - memset(info, 0, sizeof(odp_pktio_link_info_t)); - - info->autoneg = ODP_PKTIO_LINK_AUTONEG_OFF; - info->duplex = ODP_PKTIO_LINK_DUPLEX_FULL; - info->media = "virtual"; - info->pause_rx = ODP_PKTIO_LINK_PAUSE_OFF; - info->pause_tx = ODP_PKTIO_LINK_PAUSE_OFF; - info->speed = ODP_PKTIO_LINK_SPEED_UNKNOWN; - info->status = ODP_PKTIO_LINK_STATUS_UP; - - return 0; - } - - return _odp_link_info_fd(pkt_nm->sockfd, pkt_nm->if_name, info); -} - -/** - * Wait for netmap link to come up - * - * @param pktio_entry Packet IO entry - * - * @retval 1 link is up - * @retval 0 link is down - * @retval <0 on failure - */ -static inline int netmap_wait_for_link(pktio_entry_t *pktio_entry) -{ - int i; - int ret; - - /* Wait for the link to come up */ - for (i = 0; i <= NM_WAIT_TIMEOUT; i++) { - ret = netmap_link_status(pktio_entry); - if (ret == -1) - return -1; - /* nm_open() causes the physical link to reset. When using a - * direct attached loopback cable there may be a small delay - * until the opposing end's interface comes back up again. In - * this case without the additional sleep pktio validation - * tests fail. */ - if (!pkt_priv(pktio_entry)->is_virtual) - sleep(1); - if (ret == 1) - return 1; - } - _ODP_DBG("%s link is down\n", pkt_priv(pktio_entry)->if_name); - return 0; -} - -/** - * Initialize netmap capability values - * - * @param pktio_entry Packet IO entry - */ -static void netmap_init_capability(pktio_entry_t *pktio_entry) -{ - pkt_netmap_t *pkt_nm = pkt_priv(pktio_entry); - odp_pktio_capability_t *capa = &pktio_entry->capa; - - memset(capa, 0, sizeof(odp_pktio_capability_t)); - - capa->max_input_queues = ODP_PKTIN_MAX_QUEUES; - if (pkt_nm->num_rx_rings < ODP_PKTIN_MAX_QUEUES) - capa->max_input_queues = pkt_nm->num_rx_rings; - if (capa->max_input_queues > NM_MAX_DESC) { - /* Have to use a single descriptor to fetch packets from all - * netmap rings */ - capa->max_input_queues = 1; - _ODP_DBG("Unable to store all %" PRIu32 " rx rings (max %d)\n" - " max input queues: %u\n", pkt_nm->num_rx_rings, - NM_MAX_DESC, capa->max_input_queues); - } - - capa->max_output_queues = ODP_PKTOUT_MAX_QUEUES; - if (pkt_nm->num_tx_rings < ODP_PKTOUT_MAX_QUEUES) - capa->max_output_queues = pkt_nm->num_tx_rings; - if (capa->max_output_queues > NM_MAX_DESC) { - capa->max_output_queues = NM_MAX_DESC; - _ODP_DBG("Unable to store all %" PRIu32 " tx rings (max %d)\n" - " max output queues: %u\n", pkt_nm->num_tx_rings, - NM_MAX_DESC, capa->max_output_queues); - } - - capa->set_op.op.promisc_mode = 1; - capa->set_op.op.maxlen = 1; - - capa->maxlen.equal = true; - capa->maxlen.min_input = _ODP_SOCKET_MTU_MIN; - capa->maxlen.max_input = pkt_nm->mtu_max; - capa->maxlen.min_output = _ODP_SOCKET_MTU_MIN; - capa->maxlen.max_output = pkt_nm->mtu_max; - - odp_pktio_config_init(&capa->config); - capa->config.pktin.bit.ts_all = 1; - capa->config.pktin.bit.ts_ptp = 1; - - capa->config.pktout.bit.ts_ena = 1; -} - -/** - * Open a netmap interface - * - * In addition to standard interfaces (with or without modified netmap drivers) - * virtual VALE and pipe interfaces are also supported. These can be used for - * example for testing packet IO functionality without any physical interfaces. - * - * To use virtual interfaces the 'netdev' device name has to begin with 'vale' - * prefix. A valid VALE device name would be e.g. 'vale0'. Pipe device names - * have to include also '{NN' (master) or '}NN' (slave) suffix. A valid pipe - * master would be e.g. 'vale0{0' and a slave to the same pipe 'vale0}0'. - * - * Netmap requires standard interface names to begin with 'netmap:' prefix. - * netmap_open() adds the prefix if it is missing. Virtual interfaces don't - * require the 'netmap:' prefix. - * - * @param id Packet IO handle - * @param pktio_entry Packet IO entry - * @param netdev Packet IO device name - * @param pool Default pool from which to allocate storage for packets - * - * @retval 0 on success - * @retval <0 on failure - */ -static int netmap_open(odp_pktio_t id ODP_UNUSED, pktio_entry_t *pktio_entry, - const char *netdev, odp_pool_t pool) -{ - int i; - int err; - int sockfd; - const char *prefix; - uint32_t mtu; - uint32_t nm_buf_size; - pkt_netmap_t *pkt_nm = pkt_priv(pktio_entry); - struct nm_desc *desc; - odp_pktin_hash_proto_t hash_proto; - odp_pktio_stats_t cur_stats; - - if (disable_pktio) - return -1; - - if (pool == ODP_POOL_INVALID) - return -1; - - /* Init pktio entry */ - memset(pkt_nm, 0, sizeof(*pkt_nm)); - pkt_nm->sockfd = -1; - pkt_nm->pool = pool; - - /* allow interface to be opened with or without the 'netmap:' prefix */ - prefix = "netmap:"; - if (strncmp(netdev, "netmap:", 7) == 0) - netdev += 7; - if (strncmp(netdev, "vale", 4) == 0) { - pkt_nm->is_virtual = 1; - prefix = ""; - } - - snprintf(pkt_nm->nm_name, sizeof(pkt_nm->nm_name), "%s%s", prefix, - netdev); - snprintf(pkt_nm->if_name, sizeof(pkt_nm->if_name), "%s", netdev); - - /* Initialize runtime options */ - if (init_options(pktio_entry)) { - _ODP_ERR("Initializing runtime options failed\n"); - return -1; - } - - /* Read netmap buffer size */ - nm_buf_size = read_netmap_buf_size(); - if (!nm_buf_size) { - _ODP_ERR("Unable to read netmap buf size\n"); - return -1; - } - pkt_nm->mtu_max = _ODP_SOCKET_MTU_MAX; - if (pkt_nm->mtu_max > nm_buf_size) - pkt_nm->mtu_max = nm_buf_size; - - if (!pkt_nm->is_virtual) { - sockfd = socket(AF_INET, SOCK_DGRAM, 0); - if (sockfd == -1) { - _ODP_ERR("Cannot get device control socket\n"); - return -1; - } - pkt_nm->sockfd = sockfd; - - /* Use either interface MTU or netmap buffer size as MTU, - * whichever is smaller. */ - mtu = _odp_mtu_get_fd(pkt_nm->sockfd, pkt_nm->if_name); - if (mtu == 0) { - _ODP_ERR("Unable to read interface MTU\n"); - goto error; - } - pkt_nm->mtu = (mtu < nm_buf_size) ? mtu : nm_buf_size; - - /* Netmap requires that interface MTU size <= nm buf size */ - if (mtu > nm_buf_size) { - if (_odp_mtu_set_fd(pkt_nm->sockfd, pkt_nm->if_name, - nm_buf_size)) { - _ODP_ERR("Unable to set interface MTU\n"); - goto error; - } - } - } - /* Dummy open here to check if netmap module is available and to read - * capability info. */ - desc = nm_open(pkt_nm->nm_name, NULL, 0, NULL); - if (desc == NULL) { - _ODP_ERR("nm_open(%s) failed\n", pkt_nm->nm_name); - goto error; - } - pkt_nm->num_rx_rings = desc->nifp->ni_rx_rings; - pkt_nm->num_tx_rings = desc->nifp->ni_tx_rings; - - netmap_init_capability(pktio_entry); - - nm_close(desc); - - for (i = 0; i < ODP_PKTIN_MAX_QUEUES; i++) - odp_ticketlock_init(&pkt_nm->rx_desc_ring[i].lock); - - for (i = 0; i < ODP_PKTOUT_MAX_QUEUES; i++) - odp_ticketlock_init(&pkt_nm->tx_desc_ring[i].lock); - - if (pkt_nm->is_virtual) { - static unsigned int mac; - uint32_t tid = syscall(SYS_gettid); - - if ((int)tid == -1) - _ODP_DBG("Unable to fetch thread ID. VALE port MAC " - "addresses may not be unique.\n"); - - pktio_entry->capa.max_input_queues = 1; - pktio_entry->capa.set_op.op.maxlen = 0; - pktio_entry->capa.set_op.op.promisc_mode = 0; - pkt_nm->mtu = nm_buf_size; - pktio_entry->stats_type = STATS_UNSUPPORTED; - /* Set MAC address for virtual interface */ - pkt_nm->if_mac[0] = 0x2; - pkt_nm->if_mac[1] = (tid >> 24) & 0xff; - pkt_nm->if_mac[2] = (tid >> 16) & 0xff; - pkt_nm->if_mac[3] = (tid >> 8) & 0xff; - pkt_nm->if_mac[4] = tid & 0xff; - pkt_nm->if_mac[5] = ++mac; - - return 0; - } - - /* Check if RSS is supported. If not, set 'max_input_queues' to 1. */ - if (_odp_rss_conf_get_supported_fd(pkt_nm->sockfd, netdev, - &hash_proto) == 0) { - _ODP_DBG("RSS not supported\n"); - pktio_entry->capa.max_input_queues = 1; - } - - err = netmap_do_ioctl(pktio_entry, SIOCGIFFLAGS, 0); - if (err) - goto error; - if ((pkt_nm->if_flags & IFF_UP) == 0) - _ODP_DBG("%s is down\n", pkt_nm->if_name); - - err = _odp_mac_addr_get_fd(pkt_nm->sockfd, netdev, pkt_nm->if_mac); - if (err) - goto error; - - /* netmap uses only ethtool to get statistics counters */ - err = _odp_ethtool_stats_get_fd(pkt_nm->sockfd, pkt_nm->if_name, &cur_stats); - if (err) { - _ODP_DBG("netmap pktio %s does not support statistics counters\n", - pkt_nm->if_name); - pktio_entry->stats_type = STATS_UNSUPPORTED; - } else { - pktio_entry->stats_type = STATS_ETHTOOL; - _odp_sock_stats_capa(pktio_entry, &pktio_entry->capa); - } - - (void)netmap_stats_reset(pktio_entry); - - return 0; - -error: - netmap_close(pktio_entry); - return -1; -} - -static int netmap_start(pktio_entry_t *pktio_entry) -{ - pkt_netmap_t *pkt_nm = pkt_priv(pktio_entry); - netmap_ring_t *desc_ring; - struct nm_desc *desc_ptr; - unsigned int i; - unsigned int j; - unsigned int num_rx_desc = 0; - uint64_t flags; - odp_pktin_mode_t in_mode = pktio_entry->param.in_mode; - odp_pktout_mode_t out_mode = pktio_entry->param.out_mode; - - /* If no pktin/pktout queues have been configured. Configure one - * for each direction. */ - if (!pktio_entry->num_in_queue && - in_mode != ODP_PKTIN_MODE_DISABLED) { - odp_pktin_queue_param_t param; - - odp_pktin_queue_param_init(¶m); - param.num_queues = 1; - if (odp_pktin_queue_config(pktio_entry->handle, ¶m)) - return -1; - } - if (!pktio_entry->num_out_queue && - out_mode == ODP_PKTOUT_MODE_DIRECT) { - odp_pktout_queue_param_t param; - - odp_pktout_queue_param_init(¶m); - param.num_queues = 1; - if (odp_pktout_queue_config(pktio_entry->handle, ¶m)) - return -1; - } - - if (pkt_nm->num_rx_desc_rings == pktio_entry->num_in_queue && - pkt_nm->num_tx_desc_rings == pktio_entry->num_out_queue) - return (netmap_wait_for_link(pktio_entry) == 1) ? 0 : -1; - - netmap_close_descriptors(pktio_entry); - - /* Map pktin/pktout queues to netmap rings */ - if (pktio_entry->num_in_queue) { - /* In single queue case only one netmap descriptor is - * required. */ - num_rx_desc = (pktio_entry->num_in_queue == 1) ? 1 : - pkt_nm->num_rx_rings; - - map_netmap_rings(pkt_nm->rx_desc_ring, - pktio_entry->num_in_queue, num_rx_desc); - } - if (pktio_entry->num_out_queue) - /* Enough to map only one netmap tx ring per pktout queue */ - map_netmap_rings(pkt_nm->tx_desc_ring, - pktio_entry->num_out_queue, - pktio_entry->num_out_queue); - - /* Use nm_open() to parse netmap flags from interface name */ - desc_ptr = nm_open(pkt_nm->nm_name, NULL, 0, NULL); - if (desc_ptr == NULL) { - _ODP_ERR("nm_start(%s) failed\n", pkt_nm->nm_name); - goto error; - } - struct nm_desc base_desc = *desc_ptr; - - nm_close(desc_ptr); - - base_desc.self = &base_desc; - base_desc.mem = NULL; - if (pkt_priv(pktio_entry)->is_virtual) { - base_desc.req.nr_rx_slots = - pkt_priv(pktio_entry)->opt.nr_rx_slots; - base_desc.req.nr_tx_slots = - pkt_priv(pktio_entry)->opt.nr_tx_slots; - } - base_desc.req.nr_ringid = 0; - if ((base_desc.req.nr_flags & NR_REG_MASK) == NR_REG_ALL_NIC || - (base_desc.req.nr_flags & NR_REG_MASK) == NR_REG_ONE_NIC) { - base_desc.req.nr_flags &= ~NR_REG_MASK; - if (num_rx_desc == 1) - base_desc.req.nr_flags |= NR_REG_ALL_NIC; - else - base_desc.req.nr_flags |= NR_REG_ONE_NIC; - } - - /* Only the first rx descriptor does mmap */ - desc_ring = pkt_nm->rx_desc_ring; - flags = NM_OPEN_IFNAME | NETMAP_NO_TX_POLL; - if (pkt_priv(pktio_entry)->is_virtual) - flags |= NM_OPEN_RING_CFG; - desc_ring[0].desc[0] = nm_open(pkt_nm->nm_name, NULL, flags, - &base_desc); - if (desc_ring[0].desc[0] == NULL) { - _ODP_ERR("nm_start(%s) failed\n", pkt_nm->nm_name); - goto error; - } - /* Open rest of the rx descriptors (one per netmap ring) */ - flags = NM_OPEN_IFNAME | NETMAP_NO_TX_POLL | NM_OPEN_NO_MMAP; - if (pkt_priv(pktio_entry)->is_virtual) - flags |= NM_OPEN_RING_CFG; - for (i = 0; i < pktio_entry->num_in_queue; i++) { - for (j = desc_ring[i].first; j <= desc_ring[i].last; j++) { - if (i == 0 && j == 0) { /* First already opened */ - if (num_rx_desc > 1) - continue; - else - break; - } - base_desc.req.nr_ringid = j; - desc_ring[i].desc[j] = nm_open(pkt_nm->nm_name, NULL, - flags, &base_desc); - if (desc_ring[i].desc[j] == NULL) { - _ODP_ERR("nm_start(%s) failed\n", pkt_nm->nm_name); - goto error; - } - } - } - /* Open tx descriptors */ - desc_ring = pkt_nm->tx_desc_ring; - flags = NM_OPEN_IFNAME | NM_OPEN_NO_MMAP; - if (pkt_priv(pktio_entry)->is_virtual) - flags |= NM_OPEN_RING_CFG; - - if ((base_desc.req.nr_flags & NR_REG_MASK) == NR_REG_ALL_NIC) { - base_desc.req.nr_flags &= ~NR_REG_ALL_NIC; - base_desc.req.nr_flags |= NR_REG_ONE_NIC; - } - - for (i = 0; i < pktio_entry->num_out_queue; i++) { - for (j = desc_ring[i].first; j <= desc_ring[i].last; j++) { - base_desc.req.nr_ringid = j; - desc_ring[i].desc[j] = nm_open(pkt_nm->nm_name, NULL, - flags, &base_desc); - if (desc_ring[i].desc[j] == NULL) { - _ODP_ERR("nm_start(%s) failed\n", pkt_nm->nm_name); - goto error; - } - } - } - pkt_nm->num_rx_desc_rings = pktio_entry->num_in_queue; - pkt_nm->num_tx_desc_rings = pktio_entry->num_out_queue; - /* Wait for the link to come up */ - return (netmap_wait_for_link(pktio_entry) == 1) ? 0 : -1; - -error: - netmap_close_descriptors(pktio_entry); - return -1; -} - -static int netmap_stop(pktio_entry_t *pktio_entry ODP_UNUSED) -{ - return 0; -} - -/** - * Create ODP packets from netmap packets - * - * @param pktio_entry Packet IO entry - * @param pkt_tbl Array for new ODP packet handles - * @param slot_tbl Array of netmap ring slots - * @param slot_num Number of netmap ring slots - * @param ts Pointer to pktin timestamp - * - * @retval Number of created packets - */ -static inline int netmap_pkt_to_odp(pktio_entry_t *pktio_entry, - odp_packet_t pkt_tbl[], - netmap_slot_t slot_tbl[], int16_t slot_num, - odp_time_t *ts) -{ - odp_packet_t pkt; - odp_pool_t pool = pkt_priv(pktio_entry)->pool; - odp_packet_hdr_t *pkt_hdr; - int i; - int num; - uint32_t max_len; - uint16_t frame_offset = pktio_entry->pktin_frame_offset; - int num_rx = 0; - const odp_proto_layer_t layer = pktio_entry->parse_layer; - const odp_pktin_config_opt_t opt = pktio_entry->config.pktin; - - /* Allocate maximum sized packets */ - max_len = pkt_priv(pktio_entry)->mtu; - - num = _odp_packet_alloc_multi(pool, max_len + frame_offset, - pkt_tbl, slot_num); - - for (i = 0; i < num; i++) { - netmap_slot_t slot; - uint16_t len; - const uint8_t *buf; - - slot = slot_tbl[i]; - len = slot.len; - buf = (const uint8_t *)slot.buf; - - odp_prefetch(slot.buf); - - pkt = pkt_tbl[i]; - pkt_hdr = packet_hdr(pkt); - - pull_tail(pkt_hdr, max_len - len); - if (frame_offset) - pull_head(pkt_hdr, frame_offset); - - if (odp_packet_copy_from_mem(pkt, 0, len, slot.buf) != 0) { - odp_packet_free(pkt); - continue; - } - - if (layer) { - int ret; - - ret = _odp_packet_parse_common(pkt_hdr, buf, len, len, - layer, opt); - if (ret) - odp_atomic_inc_u64(&pktio_entry->stats_extra.in_errors); - - if (ret < 0) { - odp_packet_free(pkt); - continue; - } - - if (pktio_cls_enabled(pktio_entry)) { - odp_pool_t new_pool; - - ret = _odp_cls_classify_packet(pktio_entry, buf, - &new_pool, pkt_hdr); - if (ret < 0) - odp_atomic_inc_u64(&pktio_entry->stats_extra.in_discards); - - if (ret) { - odp_packet_free(pkt); - continue; - } - - if (odp_unlikely(_odp_pktio_packet_to_pool( - &pkt, &pkt_hdr, new_pool))) { - odp_packet_free(pkt); - odp_atomic_inc_u64(&pktio_entry->stats_extra.in_discards); - continue; - } - } - } - - pkt_hdr->input = pktio_entry->handle; - packet_set_ts(pkt_hdr, ts); - - pkt_tbl[num_rx++] = pkt; - } - - return num_rx; -} - -static inline int netmap_recv_desc(pktio_entry_t *pktio_entry, - struct nm_desc *desc, - odp_packet_t pkt_table[], int num) -{ - struct netmap_ring *ring; - odp_time_t ts_val; - odp_time_t *ts = NULL; - netmap_slot_t slot_tbl[num]; - char *buf; - uint32_t slot_id; - uint32_t mtu = pkt_priv(pktio_entry)->mtu; - int i; - int ring_id = desc->cur_rx_ring; - int num_rx = 0; - int num_rings = desc->last_rx_ring - desc->first_rx_ring + 1; - - if (pktio_entry->config.pktin.bit.ts_all || - pktio_entry->config.pktin.bit.ts_ptp) - ts = &ts_val; - - for (i = 0; i < num_rings && num_rx != num; i++) { - if (ring_id > desc->last_rx_ring) - ring_id = desc->first_rx_ring; - - ring = NETMAP_RXRING(desc->nifp, ring_id); - - while (!nm_ring_empty(ring) && num_rx != num) { - slot_id = ring->cur; - buf = NETMAP_BUF(ring, ring->slot[slot_id].buf_idx); - - if (odp_likely(ring->slot[slot_id].len <= mtu)) { - slot_tbl[num_rx].buf = buf; - slot_tbl[num_rx].len = ring->slot[slot_id].len; - num_rx++; - } else { - _ODP_DBG("Dropped oversized packet: %" PRIu16 " " - "B\n", ring->slot[slot_id].len); - } - ring->cur = nm_ring_next(ring, slot_id); - ring->head = ring->cur; - } - ring_id++; - } - desc->cur_rx_ring = ring_id; - - if (num_rx) { - if (ts != NULL) - ts_val = odp_time_global(); - return netmap_pkt_to_odp(pktio_entry, pkt_table, slot_tbl, - num_rx, ts); - } - return 0; -} - -static int netmap_fd_set(pktio_entry_t *pktio_entry, int index, fd_set *readfds) -{ - struct nm_desc *desc; - pkt_netmap_t *pkt_nm = pkt_priv(pktio_entry); - unsigned int first_desc_id = pkt_nm->rx_desc_ring[index].first; - unsigned int last_desc_id = pkt_nm->rx_desc_ring[index].last; - unsigned int desc_id; - int num_desc = pkt_nm->rx_desc_ring[index].num; - int i; - int max_fd = 0; - - if (!pkt_nm->lockless_rx) - odp_ticketlock_lock(&pkt_nm->rx_desc_ring[index].lock); - - desc_id = pkt_nm->rx_desc_ring[index].cur; - - for (i = 0; i < num_desc; i++) { - if (desc_id > last_desc_id) - desc_id = first_desc_id; - - desc = pkt_nm->rx_desc_ring[index].desc[desc_id]; - - FD_SET(desc->fd, readfds); - if (desc->fd > max_fd) - max_fd = desc->fd; - desc_id++; - } - pkt_nm->rx_desc_ring[index].cur = desc_id; - - if (!pkt_nm->lockless_rx) - odp_ticketlock_unlock(&pkt_nm->rx_desc_ring[index].lock); - - return max_fd; -} - -static int netmap_recv(pktio_entry_t *pktio_entry, int index, - odp_packet_t pkt_table[], int num) -{ - struct nm_desc *desc; - pkt_netmap_t *pkt_nm = pkt_priv(pktio_entry); - unsigned int first_desc_id = pkt_nm->rx_desc_ring[index].first; - unsigned int last_desc_id = pkt_nm->rx_desc_ring[index].last; - unsigned int desc_id; - int num_desc = pkt_nm->rx_desc_ring[index].num; - int i; - int num_rx = 0; - int max_fd = 0; - fd_set empty_rings; - - FD_ZERO(&empty_rings); - - if (!pkt_nm->lockless_rx) - odp_ticketlock_lock(&pkt_nm->rx_desc_ring[index].lock); - - desc_id = pkt_nm->rx_desc_ring[index].cur; - - for (i = 0; i < num_desc && num_rx != num; i++) { - if (desc_id > last_desc_id) - desc_id = first_desc_id; - - desc = pkt_nm->rx_desc_ring[index].desc[desc_id]; - - num_rx += netmap_recv_desc(pktio_entry, desc, - &pkt_table[num_rx], num - num_rx); - - if (num_rx != num) { - FD_SET(desc->fd, &empty_rings); - if (desc->fd > max_fd) - max_fd = desc->fd; - } - desc_id++; - } - pkt_nm->rx_desc_ring[index].cur = desc_id; - - if (num_rx != num) { - struct timeval tout = {.tv_sec = 0, .tv_usec = 0}; - - if (select(max_fd + 1, &empty_rings, NULL, NULL, &tout) == -1) - _ODP_ERR("RX: select error\n"); - } - if (!pkt_nm->lockless_rx) - odp_ticketlock_unlock(&pkt_nm->rx_desc_ring[index].lock); - - return num_rx; -} - -static int netmap_recv_tmo(pktio_entry_t *pktio_entry, int index, - odp_packet_t pkt_table[], int num, uint64_t usecs) -{ - struct timeval timeout; - int ret; - int maxfd; - fd_set readfds; - - ret = netmap_recv(pktio_entry, index, pkt_table, num); - if (ret != 0) - return ret; - - timeout.tv_sec = usecs / (1000 * 1000); - timeout.tv_usec = usecs - timeout.tv_sec * (1000ULL * 1000ULL); - FD_ZERO(&readfds); - maxfd = netmap_fd_set(pktio_entry, index, &readfds); - - if (select(maxfd + 1, &readfds, NULL, NULL, &timeout) == 0) - return 0; - - return netmap_recv(pktio_entry, index, pkt_table, num); -} - -static int netmap_recv_mq_tmo(pktio_entry_t *pktio_entry[], int index[], - uint32_t num_q, odp_packet_t pkt_table[], int num, - uint32_t *from, uint64_t usecs) -{ - struct timeval timeout; - uint32_t i; - int ret; - int maxfd = -1, maxfd2; - fd_set readfds; - - for (i = 0; i < num_q; i++) { - ret = netmap_recv(pktio_entry[i], index[i], pkt_table, num); - - if (ret > 0 && from) - *from = i; - - if (ret != 0) - return ret; - } - - FD_ZERO(&readfds); - - for (i = 0; i < num_q; i++) { - maxfd2 = netmap_fd_set(pktio_entry[i], index[i], &readfds); - if (maxfd2 > maxfd) - maxfd = maxfd2; - } - - timeout.tv_sec = usecs / (1000 * 1000); - timeout.tv_usec = usecs - timeout.tv_sec * (1000ULL * 1000ULL); - - if (select(maxfd + 1, &readfds, NULL, NULL, &timeout) == 0) - return 0; - - for (i = 0; i < num_q; i++) { - ret = netmap_recv(pktio_entry[i], index[i], pkt_table, num); - - if (ret > 0 && from) - *from = i; - - if (ret != 0) - return ret; - } - - return 0; -} - -static int netmap_send(pktio_entry_t *pktio_entry, int index, - const odp_packet_t pkt_table[], int num) -{ - pkt_netmap_t *pkt_nm = pkt_priv(pktio_entry); - struct pollfd polld; - struct nm_desc *desc; - struct netmap_ring *ring; - int i; - int nb_tx; - int desc_id; - int tx_ts_idx = 0; - uint8_t tx_ts_enabled = _odp_pktio_tx_ts_enabled(pktio_entry); - odp_packet_t pkt; - uint32_t pkt_len; - unsigned int slot_id; - char *buf; - - /* Only one netmap tx ring per pktout queue */ - desc_id = pkt_nm->tx_desc_ring[index].cur; - desc = pkt_nm->tx_desc_ring[index].desc[desc_id]; - ring = NETMAP_TXRING(desc->nifp, desc->cur_tx_ring); - - if (!pkt_nm->lockless_tx) - odp_ticketlock_lock(&pkt_nm->tx_desc_ring[index].lock); - - polld.fd = desc->fd; - polld.events = POLLOUT; - - for (nb_tx = 0; nb_tx < num; nb_tx++) { - pkt = pkt_table[nb_tx]; - pkt_len = odp_packet_len(pkt); - - if (pkt_len > pkt_nm->mtu) { - if (nb_tx == 0) - _odp_errno = EMSGSIZE; - break; - } - for (i = 0; i < NM_INJECT_RETRIES; i++) { - if (nm_ring_empty(ring)) { - poll(&polld, 1, 0); - continue; - } - slot_id = ring->cur; - ring->slot[slot_id].flags = 0; - ring->slot[slot_id].len = pkt_len; - - buf = NETMAP_BUF(ring, ring->slot[slot_id].buf_idx); - - if (odp_packet_copy_to_mem(pkt, 0, pkt_len, buf)) { - i = NM_INJECT_RETRIES; - break; - } - ring->cur = nm_ring_next(ring, slot_id); - ring->head = ring->cur; - break; - } - if (i == NM_INJECT_RETRIES) - break; - - if (tx_ts_enabled && tx_ts_idx == 0) { - if (odp_unlikely(packet_hdr(pkt)->p.flags.ts_set)) - tx_ts_idx = i + 1; - } - } - /* Send pending packets */ - poll(&polld, 1, 0); - - if (!pkt_nm->lockless_tx) - odp_ticketlock_unlock(&pkt_nm->tx_desc_ring[index].lock); - - if (odp_unlikely(nb_tx == 0)) { - if (_odp_errno != 0) - return -1; - } else { - if (odp_unlikely(tx_ts_idx && nb_tx >= tx_ts_idx)) - _odp_pktio_tx_ts_set(pktio_entry); - - odp_packet_free_multi(pkt_table, nb_tx); - } - - return nb_tx; -} - -static int netmap_mac_addr_get(pktio_entry_t *pktio_entry, void *mac_addr) -{ - memcpy(mac_addr, pkt_priv(pktio_entry)->if_mac, ETH_ALEN); - return ETH_ALEN; -} - -static uint32_t netmap_mtu_get(pktio_entry_t *pktio_entry) -{ - return pkt_priv(pktio_entry)->mtu; -} - -static int netmap_mtu_set(pktio_entry_t *pktio_entry, uint32_t maxlen_input, - uint32_t maxlen_output ODP_UNUSED) -{ - pkt_netmap_t *pkt_nm = pkt_priv(pktio_entry); - int ret; - - ret = _odp_mtu_set_fd(pkt_nm->sockfd, pktio_entry->name, maxlen_input); - if (ret) - return ret; - - pkt_nm->mtu = maxlen_input; - - return 0; -} - -static int netmap_promisc_mode_set(pktio_entry_t *pktio_entry, - odp_bool_t enable) -{ - if (pkt_priv(pktio_entry)->is_virtual) { - _odp_errno = ENOTSUP; - return -1; - } - - return _odp_promisc_mode_set_fd(pkt_priv(pktio_entry)->sockfd, - pkt_priv(pktio_entry)->if_name, enable); -} - -static int netmap_promisc_mode_get(pktio_entry_t *pktio_entry) -{ - if (pkt_priv(pktio_entry)->is_virtual) - return 0; - - return _odp_promisc_mode_get_fd(pkt_priv(pktio_entry)->sockfd, - pkt_priv(pktio_entry)->if_name); -} - -static int netmap_capability(pktio_entry_t *pktio_entry, - odp_pktio_capability_t *capa) -{ - *capa = pktio_entry->capa; - return 0; -} - -static int netmap_stats(pktio_entry_t *pktio_entry, - odp_pktio_stats_t *stats) -{ - return _odp_sock_stats_fd(pktio_entry, - stats, - pkt_priv(pktio_entry)->sockfd); -} - -static int netmap_stats_reset(pktio_entry_t *pktio_entry) -{ - return _odp_sock_stats_reset_fd(pktio_entry, - pkt_priv(pktio_entry)->sockfd); -} - -static int netmap_extra_stat_info(pktio_entry_t *pktio_entry, - odp_pktio_extra_stat_info_t info[], int num) -{ - return _odp_sock_extra_stat_info(pktio_entry, info, num, - pkt_priv(pktio_entry)->sockfd); -} - -static int netmap_extra_stats(pktio_entry_t *pktio_entry, uint64_t stats[], - int num) -{ - return _odp_sock_extra_stats(pktio_entry, stats, num, - pkt_priv(pktio_entry)->sockfd); -} - -static int netmap_extra_stat_counter(pktio_entry_t *pktio_entry, uint32_t id, - uint64_t *stat) -{ - return _odp_sock_extra_stat_counter(pktio_entry, id, stat, - pkt_priv(pktio_entry)->sockfd); -} - -static void netmap_print(pktio_entry_t *pktio_entry) -{ - odp_pktin_hash_proto_t hash_proto; - - if (_odp_rss_conf_get_fd(pkt_priv(pktio_entry)->sockfd, - pkt_priv(pktio_entry)->if_name, &hash_proto)) - _odp_rss_conf_print(&hash_proto); -} - -static int netmap_init_global(void) -{ - if (getenv("ODP_PKTIO_DISABLE_NETMAP")) { - _ODP_PRINT("PKTIO: netmap pktio skipped," - " enabled export ODP_PKTIO_DISABLE_NETMAP=1.\n"); - disable_pktio = 1; - } else { - _ODP_PRINT("PKTIO: initialized netmap pktio," - " use export ODP_PKTIO_DISABLE_NETMAP=1 to disable.\n" - " Netmap prefixes are netmap:eth0 or vale:eth0. Refer to" - " Netmap documentation for usage information.\n"); - } - return 0; -} - -const pktio_if_ops_t _odp_netmap_pktio_ops = { - .name = "netmap", - .print = netmap_print, - .init_global = netmap_init_global, - .init_local = NULL, - .term = NULL, - .open = netmap_open, - .close = netmap_close, - .start = netmap_start, - .stop = netmap_stop, - .link_status = netmap_link_status, - .link_info = netmap_link_info, - .stats = netmap_stats, - .stats_reset = netmap_stats_reset, - .extra_stat_info = netmap_extra_stat_info, - .extra_stats = netmap_extra_stats, - .extra_stat_counter = netmap_extra_stat_counter, - .maxlen_get = netmap_mtu_get, - .maxlen_set = netmap_mtu_set, - .promisc_mode_set = netmap_promisc_mode_set, - .promisc_mode_get = netmap_promisc_mode_get, - .mac_get = netmap_mac_addr_get, - .mac_set = NULL, - .capability = netmap_capability, - .pktio_ts_res = NULL, - .pktio_ts_from_ns = NULL, - .pktio_time = NULL, - .config = NULL, - .input_queues_config = netmap_input_queues_config, - .output_queues_config = netmap_output_queues_config, - .recv = netmap_recv, - .recv_tmo = netmap_recv_tmo, - .recv_mq_tmo = netmap_recv_mq_tmo, - .send = netmap_send, - .fd_set = netmap_fd_set -}; -#else /* _ODP_PKTIO_NETMAP */ -/* Avoid warning about empty translation unit */ -typedef int _odp_dummy; -#endif diff --git a/platform/linux-generic/pktio/socket.c b/platform/linux-generic/pktio/socket.c index 97574186c..371831961 100644 --- a/platform/linux-generic/pktio/socket.c +++ b/platform/linux-generic/pktio/socket.c @@ -20,7 +20,6 @@ #include <odp_packet_io_internal.h> #include <odp_packet_io_stats.h> #include <odp_debug_internal.h> -#include <odp_errno_define.h> #include <odp_classification_internal.h> #include <odp_macros_internal.h> @@ -96,7 +95,6 @@ static int sock_close(pktio_entry_t *pktio_entry) pkt_sock_t *pkt_sock = pkt_priv(pktio_entry); if (pkt_sock->sockfd != -1 && close(pkt_sock->sockfd) != 0) { - _odp_errno = errno; _ODP_ERR("close(sockfd): %s\n", strerror(errno)); return -1; } @@ -128,7 +126,6 @@ static int sock_setup_pkt(pktio_entry_t *pktio_entry, const char *netdev, sockfd = socket(AF_PACKET, SOCK_RAW, htons(ETH_P_ALL)); if (sockfd == -1) { - _odp_errno = errno; _ODP_ERR("socket(): %s\n", strerror(errno)); goto error; } @@ -139,7 +136,6 @@ static int sock_setup_pkt(pktio_entry_t *pktio_entry, const char *netdev, snprintf(ethreq.ifr_name, IF_NAMESIZE, "%s", netdev); err = ioctl(sockfd, SIOCGIFINDEX, ðreq); if (err != 0) { - _odp_errno = errno; _ODP_ERR("ioctl(SIOCGIFINDEX): %s: \"%s\".\n", strerror(errno), ethreq.ifr_name); goto error; } @@ -162,7 +158,6 @@ static int sock_setup_pkt(pktio_entry_t *pktio_entry, const char *netdev, sa_ll.sll_ifindex = if_idx; sa_ll.sll_protocol = htons(ETH_P_ALL); if (bind(sockfd, (struct sockaddr *)&sa_ll, sizeof(sa_ll)) < 0) { - _odp_errno = errno; _ODP_ERR("bind(to IF): %s\n", strerror(errno)); goto error; } @@ -486,7 +481,6 @@ static int sock_mmsg_send(pktio_entry_t *pktio_entry, int index ODP_UNUSED, ret = sendmmsg(sockfd, &msgvec[i], num - i, MSG_DONTWAIT); if (odp_unlikely(ret <= -1)) { if (i == 0 && SOCK_ERR_REPORT(errno)) { - _odp_errno = errno; _ODP_ERR("sendmmsg(): %s\n", strerror(errno)); odp_ticketlock_unlock(&pkt_sock->tx_lock); return -1; diff --git a/platform/linux-generic/pktio/socket_common.c b/platform/linux-generic/pktio/socket_common.c index b6ae7b9ae..dabe86aa2 100644 --- a/platform/linux-generic/pktio/socket_common.c +++ b/platform/linux-generic/pktio/socket_common.c @@ -19,7 +19,6 @@ #include <linux/sockios.h> #include <errno.h> #include <odp_debug_internal.h> -#include <odp_errno_define.h> #include <odp_socket_common.h> #include <protocols/eth.h> @@ -46,7 +45,6 @@ struct ethtool_link_settings { /** * ODP_PACKET_SOCKET_MMSG: * ODP_PACKET_SOCKET_MMAP: - * ODP_PACKET_NETMAP: */ int _odp_mac_addr_get_fd(int fd, const char *name, unsigned char mac_dst[]) { @@ -57,7 +55,6 @@ int _odp_mac_addr_get_fd(int fd, const char *name, unsigned char mac_dst[]) snprintf(ethreq.ifr_name, IF_NAMESIZE, "%s", name); ret = ioctl(fd, SIOCGIFHWADDR, ðreq); if (ret != 0) { - _odp_errno = errno; _ODP_ERR("ioctl(SIOCGIFHWADDR): %s: \"%s\".\n", strerror(errno), ethreq.ifr_name); return -1; } @@ -70,7 +67,6 @@ int _odp_mac_addr_get_fd(int fd, const char *name, unsigned char mac_dst[]) /* * ODP_PACKET_SOCKET_MMSG: * ODP_PACKET_SOCKET_MMAP: - * ODP_PACKET_NETMAP: * ODP_PACKET_TAP: */ uint32_t _odp_mtu_get_fd(int fd, const char *name) @@ -81,7 +77,6 @@ uint32_t _odp_mtu_get_fd(int fd, const char *name) snprintf(ifr.ifr_name, IF_NAMESIZE, "%s", name); ret = ioctl(fd, SIOCGIFMTU, &ifr); if (ret < 0) { - _odp_errno = errno; _ODP_ERR("ioctl(SIOCGIFMTU): %s: \"%s\".\n", strerror(errno), ifr.ifr_name); return 0; } @@ -91,7 +86,6 @@ uint32_t _odp_mtu_get_fd(int fd, const char *name) /* * ODP_PACKET_SOCKET_MMAP: * ODP_PACKET_SOCKET_MMSG: - * ODP_PACKET_NETMAP: * ODP_PACKET_TAP: */ int _odp_mtu_set_fd(int fd, const char *name, int mtu) @@ -104,7 +98,6 @@ int _odp_mtu_set_fd(int fd, const char *name, int mtu) ret = ioctl(fd, SIOCSIFMTU, &ifr); if (ret < 0) { - _odp_errno = errno; _ODP_ERR("ioctl(SIOCSIFMTU): %s: \"%s\".\n", strerror(errno), ifr.ifr_name); return -1; } @@ -114,7 +107,6 @@ int _odp_mtu_set_fd(int fd, const char *name, int mtu) /* * ODP_PACKET_SOCKET_MMSG: * ODP_PACKET_SOCKET_MMAP: - * ODP_PACKET_NETMAP: */ int _odp_promisc_mode_set_fd(int fd, const char *name, int enable) { @@ -124,7 +116,6 @@ int _odp_promisc_mode_set_fd(int fd, const char *name, int enable) snprintf(ifr.ifr_name, IF_NAMESIZE, "%s", name); ret = ioctl(fd, SIOCGIFFLAGS, &ifr); if (ret < 0) { - _odp_errno = errno; _ODP_DBG("ioctl(SIOCGIFFLAGS): %s: \"%s\".\n", strerror(errno), ifr.ifr_name); return -1; } @@ -136,7 +127,6 @@ int _odp_promisc_mode_set_fd(int fd, const char *name, int enable) ret = ioctl(fd, SIOCSIFFLAGS, &ifr); if (ret < 0) { - _odp_errno = errno; _ODP_DBG("ioctl(SIOCSIFFLAGS): %s: \"%s\".\n", strerror(errno), ifr.ifr_name); return -1; } @@ -146,7 +136,6 @@ int _odp_promisc_mode_set_fd(int fd, const char *name, int enable) /* * ODP_PACKET_SOCKET_MMSG: * ODP_PACKET_SOCKET_MMAP: - * ODP_PACKET_NETMAP: */ int _odp_promisc_mode_get_fd(int fd, const char *name) { @@ -156,7 +145,6 @@ int _odp_promisc_mode_get_fd(int fd, const char *name) snprintf(ifr.ifr_name, IF_NAMESIZE, "%s", name); ret = ioctl(fd, SIOCGIFFLAGS, &ifr); if (ret < 0) { - _odp_errno = errno; _ODP_DBG("ioctl(SIOCGIFFLAGS): %s: \"%s\".\n", strerror(errno), ifr.ifr_name); return -1; } @@ -172,7 +160,6 @@ int _odp_link_status_fd(int fd, const char *name) snprintf(ifr.ifr_name, IF_NAMESIZE, "%s", name); ret = ioctl(fd, SIOCGIFFLAGS, &ifr); if (ret < 0) { - _odp_errno = errno; _ODP_DBG("ioctl(SIOCGIFFLAGS): %s: \"%s\".\n", strerror(errno), ifr.ifr_name); return ODP_PKTIO_LINK_STATUS_UNKNOWN; } @@ -199,7 +186,6 @@ int _odp_link_info_fd(int fd, const char *name, odp_pktio_link_info_t *info) /* Link pause status */ ifr.ifr_data = (void *)&pcmd; if (ioctl(fd, SIOCETHTOOL, &ifr) && errno != EOPNOTSUPP) { - _odp_errno = errno; _ODP_ERR("ioctl(SIOCETHTOOL): %s: \"%s\".\n", strerror(errno), ifr.ifr_name); return -1; } @@ -211,7 +197,6 @@ int _odp_link_info_fd(int fd, const char *name, odp_pktio_link_info_t *info) ifr.ifr_data = (void *)&ecmd_old; if (ioctl(fd, SIOCETHTOOL, &ifr) < 0) { - _odp_errno = errno; _ODP_ERR("ioctl(SIOCETHTOOL): %s: \"%s\".\n", strerror(errno), ifr.ifr_name); return -1; @@ -270,7 +255,6 @@ int _odp_link_info_fd(int fd, const char *name, odp_pktio_link_info_t *info) *ecmd = hcmd; ifr.ifr_data = (void *)ecmd; if (ioctl(fd, SIOCETHTOOL, &ifr) < 0) { - _odp_errno = errno; _ODP_ERR("ioctl(SIOCETHTOOL): %s: \"%s\".\n", strerror(errno), ifr.ifr_name); return -1; } diff --git a/platform/linux-generic/pktio/socket_mmap.c b/platform/linux-generic/pktio/socket_mmap.c index ea6c0b9b7..0c04acd03 100644 --- a/platform/linux-generic/pktio/socket_mmap.c +++ b/platform/linux-generic/pktio/socket_mmap.c @@ -21,7 +21,6 @@ #include <odp_packet_io_internal.h> #include <odp_packet_io_stats.h> #include <odp_debug_internal.h> -#include <odp_errno_define.h> #include <odp_classification_datamodel.h> #include <odp_classification_internal.h> #include <odp_global_data.h> @@ -113,14 +112,12 @@ static int mmap_pkt_socket(void) int ret, sock = socket(PF_PACKET, SOCK_RAW, htons(ETH_P_ALL)); if (sock == -1) { - _odp_errno = errno; _ODP_ERR("socket(SOCK_RAW): %s\n", strerror(errno)); return -1; } ret = setsockopt(sock, SOL_PACKET, PACKET_VERSION, &ver, sizeof(ver)); if (ret == -1) { - _odp_errno = errno; _ODP_ERR("setsockopt(PACKET_VERSION): %s\n", strerror(errno)); close(sock); return -1; @@ -483,7 +480,6 @@ static int mmap_setup_ring(pkt_sock_mmap_t *pkt_sock, struct ring *ring, ret = setsockopt(sock, SOL_PACKET, type, &ring->req, sizeof(ring->req)); if (ret == -1) { - _odp_errno = errno; _ODP_ERR("setsockopt(pkt mmap): %s\n", strerror(errno)); return -1; } @@ -514,7 +510,6 @@ static int mmap_sock(pkt_sock_mmap_t *pkt_sock) MAP_SHARED | MAP_LOCKED | MAP_POPULATE, sock, 0); if (pkt_sock->mmap_base == MAP_FAILED) { - _odp_errno = errno; _ODP_ERR("mmap rx&tx buffer failed: %s\n", strerror(errno)); return -1; } @@ -570,7 +565,6 @@ static int mmap_bind_sock(pkt_sock_mmap_t *pkt_sock, const char *netdev) ret = bind(pkt_sock->sockfd, (struct sockaddr *)&pkt_sock->ll, sizeof(pkt_sock->ll)); if (ret == -1) { - _odp_errno = errno; _ODP_ERR("bind(to IF): %s\n", strerror(errno)); return -1; } @@ -590,7 +584,6 @@ static int sock_mmap_close(pktio_entry_t *entry) } if (pkt_sock->sockfd != -1 && close(pkt_sock->sockfd) != 0) { - _odp_errno = errno; _ODP_ERR("close(sockfd): %s\n", strerror(errno)); return -1; } @@ -664,7 +657,6 @@ static int sock_mmap_open(odp_pktio_t id ODP_UNUSED, if_idx = if_nametoindex(netdev); if (if_idx == 0) { - _odp_errno = errno; _ODP_ERR("if_nametoindex(): %s\n", strerror(errno)); goto error; } diff --git a/platform/linux-generic/pktio/stats/ethtool_stats.c b/platform/linux-generic/pktio/stats/ethtool_stats.c index 678ec45ee..deb00bf59 100644 --- a/platform/linux-generic/pktio/stats/ethtool_stats.c +++ b/platform/linux-generic/pktio/stats/ethtool_stats.c @@ -10,7 +10,6 @@ #include <odp/api/packet_io_stats.h> #include <odp_debug_internal.h> -#include <odp_errno_define.h> #include <odp_ethtool_stats.h> #include <sys/ioctl.h> @@ -50,13 +49,11 @@ static struct ethtool_gstrings *get_stringset(int fd, struct ifreq *ifr) drvinfo.cmd = ETHTOOL_GDRVINFO; ifr->ifr_data = (void *)&drvinfo; if (ioctl(fd, SIOCETHTOOL, ifr)) { - _odp_errno = errno; - _ODP_ERR("Cannot get stats information\n"); + _ODP_ERR("Cannot get stats information: %s\n", strerror(errno)); return NULL; } len = *(uint32_t *)(void *)((char *)&drvinfo + drvinfo_offset); } else { - _odp_errno = errno; return NULL; } @@ -76,8 +73,7 @@ static struct ethtool_gstrings *get_stringset(int fd, struct ifreq *ifr) strings->len = len; ifr->ifr_data = (void *)strings; if (ioctl(fd, SIOCETHTOOL, ifr)) { - _odp_errno = errno; - _ODP_ERR("Cannot get stats information\n"); + _ODP_ERR("Cannot get stats information: %s\n", strerror(errno)); free(strings); return NULL; } @@ -122,7 +118,6 @@ static int ethtool_stats_get(int fd, const char *name, ifr.ifr_data = (void *)estats; err = ioctl(fd, SIOCETHTOOL, &ifr); if (err < 0) { - _odp_errno = errno; free(strings); free(estats); return -1; diff --git a/platform/linux-generic/pktio/stats/sysfs_stats.c b/platform/linux-generic/pktio/stats/sysfs_stats.c index d9fb0754c..2b47d4b83 100644 --- a/platform/linux-generic/pktio/stats/sysfs_stats.c +++ b/platform/linux-generic/pktio/stats/sysfs_stats.c @@ -8,7 +8,6 @@ #include <odp/api/packet_io_stats.h> #include <odp_debug_internal.h> -#include <odp_errno_define.h> #include <odp_sysfs_stats.h> #include <dirent.h> @@ -27,7 +26,6 @@ static int sysfs_get_val(const char *fname, uint64_t *val) file = fopen(fname, "rt"); if (file == NULL) { - _odp_errno = errno; /* do not print debug err if sysfs is not supported by * kernel driver. */ diff --git a/platform/linux-generic/pktio/tap.c b/platform/linux-generic/pktio/tap.c index 604e53170..5f5b081c5 100644 --- a/platform/linux-generic/pktio/tap.c +++ b/platform/linux-generic/pktio/tap.c @@ -44,7 +44,6 @@ #include <odp_packet_internal.h> #include <odp_packet_io_internal.h> #include <odp_classification_internal.h> -#include <odp_errno_define.h> #include <errno.h> #include <fcntl.h> @@ -98,7 +97,6 @@ static int mac_addr_set_fd(int fd, const char *name, ret = ioctl(fd, SIOCSIFHWADDR, ðreq); if (ret != 0) { - _odp_errno = errno; _ODP_ERR("ioctl(SIOCSIFHWADDR): %s: \"%s\".\n", strerror(errno), ethreq.ifr_name); return -1; } @@ -128,7 +126,6 @@ static int tap_pktio_open(odp_pktio_t id ODP_UNUSED, fd = open("/dev/net/tun", O_RDWR); if (fd < 0) { - _odp_errno = errno; _ODP_ERR("failed to open /dev/net/tun: %s\n", strerror(errno)); return -1; } @@ -143,7 +140,6 @@ static int tap_pktio_open(odp_pktio_t id ODP_UNUSED, snprintf(ifr.ifr_name, IF_NAMESIZE, "%s", devname + 4); if (ioctl(fd, TUNSETIFF, (void *)&ifr) < 0) { - _odp_errno = errno; _ODP_ERR("%s: creating tap device failed: %s\n", ifr.ifr_name, strerror(errno)); goto tap_err; } @@ -151,13 +147,11 @@ static int tap_pktio_open(odp_pktio_t id ODP_UNUSED, /* Set nonblocking mode on interface. */ flags = fcntl(fd, F_GETFL, 0); if (flags < 0) { - _odp_errno = errno; _ODP_ERR("fcntl(F_GETFL) failed: %s\n", strerror(errno)); goto tap_err; } if (fcntl(fd, F_SETFL, flags | O_NONBLOCK) < 0) { - _odp_errno = errno; _ODP_ERR("fcntl(F_SETFL) failed: %s\n", strerror(errno)); goto tap_err; } @@ -168,14 +162,12 @@ static int tap_pktio_open(odp_pktio_t id ODP_UNUSED, /* Create AF_INET socket for network interface related operations. */ skfd = socket(AF_INET, SOCK_DGRAM, 0); if (skfd < 0) { - _odp_errno = errno; _ODP_ERR("socket creation failed: %s\n", strerror(errno)); goto tap_err; } mtu = _odp_mtu_get_fd(skfd, devname + 4); if (mtu == 0) { - _odp_errno = errno; _ODP_ERR("_odp_mtu_get_fd failed: %s\n", strerror(errno)); goto sock_err; } @@ -207,7 +199,6 @@ static int tap_pktio_start(pktio_entry_t *pktio_entry) /* Up interface by default. */ if (ioctl(tap->skfd, SIOCGIFFLAGS, &ifr) < 0) { - _odp_errno = errno; _ODP_ERR("ioctl(SIOCGIFFLAGS) failed: %s\n", strerror(errno)); goto sock_err; } @@ -216,7 +207,6 @@ static int tap_pktio_start(pktio_entry_t *pktio_entry) ifr.ifr_flags |= IFF_RUNNING; if (ioctl(tap->skfd, SIOCSIFFLAGS, &ifr) < 0) { - _odp_errno = errno; _ODP_ERR("failed to come up: %s\n", strerror(errno)); goto sock_err; } @@ -238,7 +228,6 @@ static int tap_pktio_stop(pktio_entry_t *pktio_entry) /* Up interface by default. */ if (ioctl(tap->skfd, SIOCGIFFLAGS, &ifr) < 0) { - _odp_errno = errno; _ODP_ERR("ioctl(SIOCGIFFLAGS) failed: %s\n", strerror(errno)); goto sock_err; } @@ -247,7 +236,6 @@ static int tap_pktio_stop(pktio_entry_t *pktio_entry) ifr.ifr_flags &= ~IFF_RUNNING; if (ioctl(tap->skfd, SIOCSIFFLAGS, &ifr) < 0) { - _odp_errno = errno; _ODP_ERR("failed to come up: %s\n", strerror(errno)); goto sock_err; } @@ -264,13 +252,11 @@ static int tap_pktio_close(pktio_entry_t *pktio_entry) pkt_tap_t *tap = pkt_priv(pktio_entry); if (tap->fd != -1 && close(tap->fd) != 0) { - _odp_errno = errno; _ODP_ERR("close(tap->fd): %s\n", strerror(errno)); ret = -1; } if (tap->skfd != -1 && close(tap->skfd) != 0) { - _odp_errno = errno; _ODP_ERR("close(tap->skfd): %s\n", strerror(errno)); ret = -1; } @@ -361,7 +347,6 @@ static int tap_pktio_recv(pktio_entry_t *pktio_entry, int index ODP_UNUSED, ts_val = odp_time_global(); if (retval < 0) { - _odp_errno = errno; break; } @@ -392,7 +377,6 @@ static int tap_pktio_send_lockless(pktio_entry_t *pktio_entry, if (odp_unlikely(pkt_len > mtu)) { if (i == 0) { - _odp_errno = EMSGSIZE; return -1; } break; @@ -409,7 +393,6 @@ static int tap_pktio_send_lockless(pktio_entry_t *pktio_entry, if (retval < 0) { if (i == 0 && SOCK_ERR_REPORT(errno)) { - _odp_errno = errno; _ODP_ERR("write(): %s\n", strerror(errno)); return -1; } @@ -417,7 +400,6 @@ static int tap_pktio_send_lockless(pktio_entry_t *pktio_entry, } else if ((uint32_t)retval != pkt_len) { _ODP_ERR("sent partial ethernet packet\n"); if (i == 0) { - _odp_errno = EMSGSIZE; return -1; } break; diff --git a/platform/linux-generic/test/Makefile.am b/platform/linux-generic/test/Makefile.am index 06e5a3ba2..99d7199ad 100644 --- a/platform/linux-generic/test/Makefile.am +++ b/platform/linux-generic/test/Makefile.am @@ -23,9 +23,6 @@ SUBDIRS += validation/api/pktio\ if ODP_PKTIO_PCAP TESTS += validation/api/pktio/pktio_run_pcap.sh endif -if netmap_support -TESTS += validation/api/pktio/pktio_run_netmap.sh -endif if PKTIO_DPDK TESTS += validation/api/pktio/pktio_run_dpdk.sh endif diff --git a/platform/linux-generic/test/example/ipsec_api/pktio_env b/platform/linux-generic/test/example/ipsec_api/pktio_env index f8cadb5cb..b3a073631 100644 --- a/platform/linux-generic/test/example/ipsec_api/pktio_env +++ b/platform/linux-generic/test/example/ipsec_api/pktio_env @@ -35,7 +35,7 @@ if [ -n "$WITH_OPENSSL_CRYPTO" ] && [ ${WITH_OPENSSL_CRYPTO} -eq 0 ]; then exit 77 fi -if [ ${ODPH_PROC_MODE} -eq 1 ]; then +if [ -n "$ODPH_PROC_MODE" ] && [ ${ODPH_PROC_MODE} -ne 0 ]; then echo "Process mode not supported. Skipping." exit 77 fi diff --git a/platform/linux-generic/test/example/ipsec_crypto/pktio_env b/platform/linux-generic/test/example/ipsec_crypto/pktio_env index d2550063e..fd770ac41 100644 --- a/platform/linux-generic/test/example/ipsec_crypto/pktio_env +++ b/platform/linux-generic/test/example/ipsec_crypto/pktio_env @@ -35,7 +35,7 @@ if [ -n "$WITH_OPENSSL_CRYPTO" ] && [ ${WITH_OPENSSL_CRYPTO} -eq 0 ]; then exit 77 fi -if [ ${ODPH_PROC_MODE} -eq 1 ]; then +if [ -n "$ODPH_PROC_MODE" ] && [ ${ODPH_PROC_MODE} -ne 0 ]; then echo "Process mode not supported. Skipping." exit 77 fi diff --git a/platform/linux-generic/test/inline-timer.conf b/platform/linux-generic/test/inline-timer.conf index 44db4e337..d645bef3c 100644 --- a/platform/linux-generic/test/inline-timer.conf +++ b/platform/linux-generic/test/inline-timer.conf @@ -1,6 +1,6 @@ # Mandatory fields odp_implementation = "linux-generic" -config_file_version = "0.1.25" +config_file_version = "0.1.27" timer: { # Enable inline timer implementation diff --git a/platform/linux-generic/test/packet_align.conf b/platform/linux-generic/test/packet_align.conf index 26491bd53..427674bb2 100644 --- a/platform/linux-generic/test/packet_align.conf +++ b/platform/linux-generic/test/packet_align.conf @@ -1,6 +1,6 @@ # Mandatory fields odp_implementation = "linux-generic" -config_file_version = "0.1.25" +config_file_version = "0.1.27" pool: { pkt: { diff --git a/platform/linux-generic/test/process-mode.conf b/platform/linux-generic/test/process-mode.conf index 2277aabdf..5bfcb9f2f 100644 --- a/platform/linux-generic/test/process-mode.conf +++ b/platform/linux-generic/test/process-mode.conf @@ -1,6 +1,6 @@ # Mandatory fields odp_implementation = "linux-generic" -config_file_version = "0.1.25" +config_file_version = "0.1.27" # Shared memory options shm: { diff --git a/platform/linux-generic/test/sched-basic.conf b/platform/linux-generic/test/sched-basic.conf index c9f7c79fd..1a401298e 100644 --- a/platform/linux-generic/test/sched-basic.conf +++ b/platform/linux-generic/test/sched-basic.conf @@ -1,10 +1,13 @@ # Mandatory fields odp_implementation = "linux-generic" -config_file_version = "0.1.25" +config_file_version = "0.1.27" # Test scheduler with an odd spread value and without dynamic load balance sched_basic: { prio_spread = 3 load_balance = 0 - order_stash_size = 0 + powersave: { + poll_time_nsec = 5000 + sleep_time_nsec = 50000 + } } diff --git a/platform/linux-generic/test/stash-custom.conf b/platform/linux-generic/test/stash-custom.conf index 95af7a259..b96c1cf45 100644 --- a/platform/linux-generic/test/stash-custom.conf +++ b/platform/linux-generic/test/stash-custom.conf @@ -1,6 +1,6 @@ # Mandatory fields odp_implementation = "linux-generic" -config_file_version = "0.1.25" +config_file_version = "0.1.27" # Test overflow safe stash variant stash: { diff --git a/platform/linux-generic/test/validation/api/pktio/Makefile.am b/platform/linux-generic/test/validation/api/pktio/Makefile.am index 98b8b233d..1646743fe 100644 --- a/platform/linux-generic/test/validation/api/pktio/Makefile.am +++ b/platform/linux-generic/test/validation/api/pktio/Makefile.am @@ -5,9 +5,6 @@ dist_check_SCRIPTS = pktio_env \ if ODP_PKTIO_PCAP dist_check_SCRIPTS += pktio_run_pcap.sh endif -if netmap_support -dist_check_SCRIPTS += pktio_run_netmap.sh -endif if PKTIO_DPDK dist_check_SCRIPTS += pktio_run_dpdk.sh endif diff --git a/platform/linux-generic/test/validation/api/pktio/pktio_run.sh b/platform/linux-generic/test/validation/api/pktio/pktio_run.sh index 9820ca339..cac297768 100755 --- a/platform/linux-generic/test/validation/api/pktio/pktio_run.sh +++ b/platform/linux-generic/test/validation/api/pktio/pktio_run.sh @@ -62,9 +62,6 @@ run_test() unset ODP_PKTIO_DISABLE_SOCKET_${distype} done - # this script doesn't support testing with netmap - export ODP_PKTIO_DISABLE_NETMAP=y - for distype in SKIP MMAP; do if [ "$disabletype" != "SKIP" ]; then export ODP_PKTIO_DISABLE_SOCKET_${distype}=y diff --git a/platform/linux-generic/test/validation/api/pktio/pktio_run_netmap.sh b/platform/linux-generic/test/validation/api/pktio/pktio_run_netmap.sh deleted file mode 100755 index f4851483b..000000000 --- a/platform/linux-generic/test/validation/api/pktio/pktio_run_netmap.sh +++ /dev/null @@ -1,123 +0,0 @@ -#!/bin/sh -# -# Copyright (c) 2016-2018, Linaro Limited -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause -# - -# any parameter passed as arguments to this script is passed unchanged to -# the test itself (pktio_main) - -# directories where pktio_main binary can be found: -# -in the validation dir when running make check (intree or out of tree) -# -in the script directory, when running after 'make install', or -# -in the validation when running standalone (./pktio_run) intree. -# -in the current directory. -# running stand alone out of tree requires setting PATH -PATH=${TEST_DIR}/api/pktio:$PATH -PATH=$(dirname $0):$PATH -PATH=$(dirname $0)/../../../../../../test/validation/api/pktio:$PATH -PATH=.:$PATH - -pktio_main_path=$(which pktio_main${EXEEXT}) -if [ -x "$pktio_main_path" ] ; then - echo "running with pktio_main: $pktio_main_path" -else - echo "cannot find pktio_main: please set you PATH for it." -fi - -# directory where platform test sources are, including scripts -TEST_SRC_DIR=$(dirname $0) - -# exit codes expected by automake for skipped tests -TEST_SKIPPED=77 - -# Use installed pktio env or for make check take it from the test directory -if [ -f "./pktio_env" ]; then - . ./pktio_env -elif [ -f ${TEST_SRC_DIR}/pktio_env ]; then - . ${TEST_SRC_DIR}/pktio_env -else - echo "ERROR: unable to find pktio_env!" - echo "pktio_env has to be in current directory or in ${TEST_SRC_DIR}" - exit 1 -fi - -run_test() -{ - local ret=0 - - pktio_main${EXEEXT} $* - ret=$? - - if [ $ret -ne 0 ]; then - echo "!!! FAILED !!!" - fi - - return $ret -} - -run_test_vale() -{ - # use two vale ports on the same switch - export ODP_PKTIO_IF0=valetest:0 - export ODP_PKTIO_IF1=valetest:1 - run_test - return $? -} - -run_test_pipe() -{ - # use a netmap pipe - export ODP_PKTIO_IF0=valetest:0{0 - export ODP_PKTIO_IF1=valetest:0}0 - run_test - return $? -} - -run_test_veth() -{ - if [ "$(lsmod | grep veth)" = "" ]; then - echo "netmap enabled veth module not loaded, skipping test." - return 0 - fi - - setup_pktio_env clean - export ODP_PKTIO_IF0=netmap:$IF0 - export ODP_PKTIO_IF1=netmap:$IF1 - run_test - return $? -} - -run() -{ - local ret=0 - - # need to be root to run these tests - if [ "$(id -u)" != "0" ]; then - echo "netmap tests must be run as root, skipping test." - exit $TEST_SKIPPED - fi - - if [ "$(lsmod | grep netmap)" = "" ]; then - echo "netmap kernel module not loaded, skipping test." - exit $TEST_SKIPPED - fi - - if [ "$ODP_PKTIO_IF0" != "" ]; then - run_test - ret=$? - else - run_test_vale - r=$?; [ $ret = 0 ] && ret=$r - run_test_pipe - r=$?; [ $ret = 0 ] && ret=$r - run_test_veth - r=$?; [ $ret = 0 ] && ret=$r - fi - - exit $ret -} - -run diff --git a/scripts/ci/build_x86_64.sh b/scripts/ci/build_x86_64.sh index 309e58b50..57d077227 100755 --- a/scripts/ci/build_x86_64.sh +++ b/scripts/ci/build_x86_64.sh @@ -7,6 +7,6 @@ if [ "${CC#clang}" != "${CC}" ] ; then fi # Required by CentOS and Rocky Linux to find DPDK install -export PKG_CONFIG_PATH=$PKG_CONFIG_PATH:/usr/local/lib64/pkgconfig +export PKG_CONFIG_PATH=$PKG_CONFIG_PATH:/usr/local/lib64/pkgconfig:/usr/lib/pkgconfig/ exec "$(dirname "$0")"/build.sh diff --git a/test/performance/Makefile.am b/test/performance/Makefile.am index beeb652cc..9dc83fd22 100644 --- a/test/performance/Makefile.am +++ b/test/performance/Makefile.am @@ -18,7 +18,6 @@ COMPILE_ONLY = odp_cpu_bench \ odp_crypto \ odp_dma_perf \ odp_ipsec \ - odp_ipsecfwd \ odp_l2fwd \ odp_packet_gen \ odp_pktio_ordered \ @@ -29,6 +28,10 @@ COMPILE_ONLY = odp_cpu_bench \ odp_stress \ odp_timer_perf +if LIBCONFIG +COMPILE_ONLY += odp_ipsecfwd +endif + TESTSCRIPTS = odp_cpu_bench_run.sh \ odp_crypto_run.sh \ odp_dma_perf_run.sh \ @@ -62,7 +65,6 @@ odp_crc_SOURCES = odp_crc.c odp_crypto_SOURCES = odp_crypto.c odp_dma_perf_SOURCES = odp_dma_perf.c odp_ipsec_SOURCES = odp_ipsec.c -odp_ipsecfwd_SOURCES = odp_ipsecfwd.c odp_lock_perf_SOURCES = odp_lock_perf.c odp_mem_perf_SOURCES = odp_mem_perf.c odp_packet_gen_SOURCES = odp_packet_gen.c @@ -78,6 +80,10 @@ odp_sched_perf_SOURCES = odp_sched_perf.c odp_stress_SOURCES = odp_stress.c odp_timer_perf_SOURCES = odp_timer_perf.c +if LIBCONFIG +odp_ipsecfwd_SOURCES = odp_ipsecfwd.c +endif + # l2fwd test depends on generator example EXTRA_odp_l2fwd_DEPENDENCIES = $(top_builddir)/example/generator/odp_generator$(EXEEXT) $(top_builddir)/example/generator/odp_generator$(EXEEXT): diff --git a/test/performance/odp_atomic_perf.c b/test/performance/odp_atomic_perf.c index ee760babf..5eeba8442 100644 --- a/test/performance/odp_atomic_perf.c +++ b/test/performance/odp_atomic_perf.c @@ -18,8 +18,9 @@ /* Default number of test rounds */ #define NUM_ROUNDS 1000000u -/* Initial value for atomic variables */ -#define INIT_VAL 1234567 +/* Initial value for atomic variables. Supports up to 2 billion + * rounds of 32-bit min and max tests. */ +#define INIT_VAL 0x80000000 /* Max number of workers if num_cpu=0 */ #define DEFAULT_MAX_WORKERS 10 diff --git a/test/performance/odp_crypto.c b/test/performance/odp_crypto.c index 2f79ae644..52af6d2fc 100644 --- a/test/performance/odp_crypto.c +++ b/test/performance/odp_crypto.c @@ -90,20 +90,20 @@ typedef struct { */ typedef struct { /** - * If non zero prints content of packets. Enabled by -d or + * If non-zero, prints content of packets. Enabled by -d or * --debug option. */ int debug_packets; /** - * If non zero Try to run crypto operation in place. Note some + * If non-zero, try to run crypto operation in place. Note some * implementation may not support such mode. Enabled by -n or * --inplace option. */ int in_place; /** - * If non zeor output of previous operation taken as input for + * If non-zero, output of previous operation taken as input for * next encrypt operations. Enabled by -r or --reuse option. */ int reuse_packet; @@ -118,7 +118,7 @@ typedef struct { /** * Number of iteration to repeat crypto operation to get good - * average number. Specified through -i or --terations option. + * average number. Specified through -i or --iterations option. * Default is 10000. */ int iteration_count; @@ -136,7 +136,7 @@ typedef struct { /** * Pointer to selected algorithm to test. If NULL all available - * alogorthims are tested. Name of algorithm is passed through + * algorithms are tested. Name of algorithm is passed through * -a or --algorithm option. */ crypto_alg_config_t *alg_config; @@ -451,6 +451,49 @@ static crypto_alg_config_t algs_config[] = { .auth_digest_len = 4, }, }, + { + .name = "snow3g-uea2", + .session = { + .cipher_alg = ODP_CIPHER_ALG_SNOW3G_UEA2, + .cipher_key = { + .data = test_key16, + .length = sizeof(test_key16) + }, + .cipher_iv_len = 16, + .auth_alg = ODP_AUTH_ALG_NULL, + }, + }, + { + .name = "snow3g-uia2", + .session = { + .cipher_alg = ODP_CIPHER_ALG_NULL, + .auth_alg = ODP_AUTH_ALG_SNOW3G_UIA2, + .auth_key = { + .data = test_key16, + .length = sizeof(test_key16) + }, + .auth_iv_len = 16, + .auth_digest_len = 4, + }, + }, + { + .name = "snow3g-uea2-snow3g-uia2", + .session = { + .cipher_alg = ODP_CIPHER_ALG_SNOW3G_UEA2, + .cipher_key = { + .data = test_key16, + .length = sizeof(test_key16) + }, + .cipher_iv_len = 16, + .auth_alg = ODP_AUTH_ALG_SNOW3G_UIA2, + .auth_key = { + .data = test_key16, + .length = sizeof(test_key16) + }, + .auth_iv_len = 16, + .auth_digest_len = 4, + }, + }, }; /** @@ -810,8 +853,6 @@ run_measure_one(crypto_args_t *cargs, } packets_sent += rc; } else { - odp_crypto_packet_result_t result; - rc = odp_crypto_op(&pkt, &out_pkt, ¶ms, 1); if (rc <= 0) { @@ -822,8 +863,7 @@ run_measure_one(crypto_args_t *cargs, } packets_sent += rc; packets_received++; - if (odp_unlikely(odp_crypto_result(&result, out_pkt) != 0) || - odp_unlikely(!result.ok)) { + if (odp_unlikely(odp_crypto_result(NULL, out_pkt) != 0)) { ODPH_ERR("Crypto operation failed\n"); odp_packet_free(out_pkt); return -1; @@ -846,7 +886,6 @@ run_measure_one(crypto_args_t *cargs, if (cargs->schedule || cargs->poll) { odp_event_t ev; - odp_crypto_packet_result_t result; odp_packet_t out_pkt; if (cargs->schedule) @@ -857,8 +896,7 @@ run_measure_one(crypto_args_t *cargs, while (ev != ODP_EVENT_INVALID) { out_pkt = odp_crypto_packet_from_event(ev); - if (odp_unlikely(odp_crypto_result(&result, out_pkt) != 0) || - odp_unlikely(!result.ok)) { + if (odp_unlikely(odp_crypto_result(NULL, out_pkt) != 0)) { ODPH_ERR("Crypto operation failed\n"); odp_packet_free(out_pkt); return -1; @@ -948,6 +986,11 @@ static int check_cipher_alg(const odp_crypto_capability_t *capa, case ODP_CIPHER_ALG_ZUC_EEA3: if (capa->ciphers.bit.zuc_eea3) return 0; + break; + case ODP_CIPHER_ALG_SNOW3G_UEA2: + if (capa->ciphers.bit.snow3g_uea2) + return 0; + break; default: break; } @@ -1002,6 +1045,11 @@ static int check_auth_alg(const odp_crypto_capability_t *capa, case ODP_AUTH_ALG_ZUC_EIA3: if (capa->auths.bit.zuc_eia3) return 0; + break; + case ODP_AUTH_ALG_SNOW3G_UIA2: + if (capa->auths.bit.snow3g_uia2) + return 0; + break; default: break; } @@ -1096,7 +1144,7 @@ static int check_auth_params(const odp_crypto_capability_t *crypto_capa, } /** - * Process one algorithm. Note if paload size is specicified it is + * Process one algorithm. Note if payload size is specified it is * only one run. Or iterate over set of predefined payloads. */ static int run_measure_one_config(test_run_arg_t *arg) @@ -1442,7 +1490,7 @@ static void parse_args(int argc, char *argv[], crypto_args_t *cargs) } /** - * Prinf usage information + * Print usage information */ static void usage(char *progname) { diff --git a/test/performance/odp_ipsecfwd.c b/test/performance/odp_ipsecfwd.c index 6098fd964..a6df747f3 100644 --- a/test/performance/odp_ipsecfwd.c +++ b/test/performance/odp_ipsecfwd.c @@ -19,6 +19,7 @@ #include <odp_api.h> #include <odp/helper/odph_api.h> +#include <libconfig.h> #define PROG_NAME "odp_ipsecfwd" #define SHORT_PROG_NAME "ipsfwd" @@ -105,6 +106,17 @@ typedef struct ODP_ALIGNED_CACHE { uint8_t pktio; } thread_config_t; +typedef struct { + odp_ipsec_sa_param_t sa_param; + char cipher_key[65U]; + char cipher_key_extra[5U]; + char auth_key[65U]; + char auth_key_extra[5U]; + odp_u32be_t lkp_dst_ip; + odp_u32be_t src_ip; + odp_u32be_t dst_ip; +} sa_config_t; + typedef uint32_t (*rx_fn_t)(thread_config_t *config, odp_event_t evs[], int num); typedef void (*ipsec_fn_t)(odp_packet_t pkts[], int num, odph_table_t fwd_tbl, stats_t *stats); typedef void (*drain_fn_t)(prog_config_t *config); @@ -123,9 +135,9 @@ typedef struct prog_config_s { fwd_entry_t fwd_entries[MAX_FWDS]; odp_queue_t sa_qs[MAX_SA_QUEUES]; pktio_t pktios[MAX_IFS]; + sa_config_t default_cfg; ops_t ops; - char *sa_conf_file; - char *fwd_conf_file; + char *conf_file; odp_instance_t odp_instance; odp_queue_t compl_q; odp_pool_t pktio_pool; @@ -203,6 +215,7 @@ static __thread pkt_ifs_t ifs; static void init_config(prog_config_t *config) { memset(config, 0, sizeof(*config)); + odp_ipsec_sa_param_init(&config->default_cfg.sa_param); config->compl_q = ODP_QUEUE_INVALID; config->pktio_pool = ODP_POOL_INVALID; config->num_input_qs = 1; @@ -315,14 +328,50 @@ static void print_usage(void) "Simple IPsec performance tester. Forward and process plain and ipsec packets.\n" "\n" "Examples:\n" - " %s -i ens9f1 -s /etc/odp/sa.conf -f /etc/odp/fwd.conf\n" + " %s -i ens9f1 -C /etc/odp/ipsecfwd.conf\n" "\n" - " With sa.conf containing, for example:\n" - " 0 222 192.168.1.10 192.168.1.16 4 jWnZr4t7w!zwC*F- 0 2" - " n2r5u7x!A%%D*G-KaPdSg 0 12\n" + " With ipsecfwd.conf containing, for example:\n" + " default: {\n" + " dir = 1\n" + " proto = 0\n" + " mode = 0\n" + " crypto: {\n" + " cipher_alg = 4\n" + " cipher_key = \"jWnZr4t7w!zwC*F-\"\n" + " auth_alg = 2\n" + " auth_key = \"n2r5u7x!A%%D*\"\n" + " icv_len = 12\n" + " };\n" + " };\n" "\n" - " With fwd.conf containing, for example:\n" - " 192.168.1.0/24 ens9f1 aa:bb:cc:dd:11:22\n" + " sa: (\n" + " {\n" + " spi = 1337\n" + " outbound: {\n" + " tunnel: {\n" + " src_addr = \"192.168.1.10\"\n" + " dst_addr = \"192.168.1.16\"\n" + " };\n" + " };\n" + " },\n" + " {\n" + " spi = 1338\n" + " outbound: {\n" + " tunnel: {\n" + " src_addr = \"192.168.3.110\"\n" + " dst_addr = \"192.168.3.116\"\n" + " };\n" + " };\n" + " }\n" + " );\n" + "\n" + " fwd: (\n" + " {\n" + " prefix: \"192.168.1.0/24\"\n" + " if: \"ens9f1\"\n" + " dst_mac: \"00:00:05:00:07:00\"\n" + " }\n" + " );\n" "\n" "Usage: %s [options]\n" "\n" @@ -336,39 +385,35 @@ static void print_usage(void) " -m, --mode Queueing mode.\n" " 0: ordered (default)\n" " 1: parallel\n" - " -s, --sa SA configuration file. Individual SA configuration is\n" - " expected to be within a single line, values whitespace\n" - " separated:\n" - "\n" - " <line in file> Dir SPI TunSrcIPv4 TunDstIPv4" - " CipherAlgoIdx CipherKey CipherKeyExtra AuthAlgIdx AuthKey AuthKeyExtra ICVLen\n" + " -C, --conf Configuration file. 'libconfig' syntax is expected.\n" + " SA configuration supports default fallback, i.e.\n" + " individual SA configuration blocks may omit some\n" + " parameters and instead set these once in default block\n" + " which then are used to fill missing parameters. The only\n" + " required argument for an SA is the 'spi' parameter.\n" + " Individual SA parameter blocks are expected to be in\n" + " 'sa'-named list. Parameter naming follows API\n" + " specification, see 'odp_ipsec_sa_param_t' for parameter\n" + " names and hierarchy. Traffic is mapped to SAs based on UDP\n" + " port: the port is used as the SPI. For forwarding entries,\n" + " individual parameter blocks are similarly expected to be\n" + " in 'fwd'-named list. With forwarding entries, every\n" + " parameter is always required and interfaces present in\n" + " forwarding entries should be one of the interfaces passed\n" + " with '--interfaces' option. See example above for\n" + " potential SA and forwarding configuration.\n" "\n" - " With combined algorithms, authentication data is ignored.\n" - " Traffic is mapped to SAs based on UDP port: the port is\n" - " used as the SPI. Non-zero Dir value declares an outbound\n" - " SA whereas zero Dir value declares an inbound SA.\n" - "\n" - " Supported cipher and authentication algorithms:\n", + " Supported cipher and authentication algorithms for SAs:\n", PROG_NAME, PROG_NAME, MIN(pool_capa.pkt.max_num, PKT_CNT), MIN(pool_capa.pkt.max_len, PKT_SIZE)); print_supported_algos(&ipsec_capa); - printf(" -f, --fwd_table Forwarding configuration file. Individual forwarding\n" - " configuration is expected to be within a single line,\n" - " values whitespace separated:\n" - "\n" - " <line in file> IPv4Prefix/MaskLen NetIf DstMac\n" - "\n" - " IPv4Prefix and MaskLen define a matchable prefix and NetIf\n" - " and DstMac define the outgoing interface and destination\n" - " MAC address for a match. NetIf should be one of the\n" - " interfaces passed with \"--interfaces\" option\n" - " -I, --num_input_qs Input queue count. 1 by default.\n" + printf(" -I, --num_input_qs Input queue count. 1 by default.\n" " -S, --num_sa_qs SA queue count. 1 by default.\n" " -O, --num_output_qs Output queue count. 1 by default.\n" " -d, --direct_rx Use direct RX. Interfaces will be polled by workers\n" - " directly. \"--mode\", \"--num_input_qs\" and\n" - " \"--num_output_qs\" options are ignored, input and output\n" - " queue counts will match worker count.\n" + " directly. '--mode', '--num_input_qs' and '--num_output_qs'\n" + " options are ignored, input and output queue counts will\n" + " match worker count.\n" " -h, --help This help.\n" "\n"); } @@ -949,25 +994,206 @@ static odp_bool_t create_sa_dest_queues(odp_ipsec_capability_t *ipsec_capa, return true; } -static void create_sa_entry(uint32_t dir, uint32_t spi, const char *src_ip_str, - const char *dst_ip_str, int cipher_idx, uint8_t *cipher_key, - uint8_t *cipher_key_extra, int auth_idx, uint8_t *auth_key, - uint8_t *auth_key_extra, uint32_t icv_len, uint32_t ar_ws, - uint32_t max_num_sa, prog_config_t *config) +static void parse_crypto(config_setting_t *cfg, sa_config_t *config) { - uint32_t src_ip, dst_ip; - odp_ipsec_sa_param_t sa_param; - odp_ipsec_crypto_param_t crypto_param; - odp_ipsec_sa_t sa; + int val; + const char *val_str; + config_setting_t *cs = config_setting_lookup(cfg, "crypto"); - if (config->num_sas == max_num_sa) { - ODPH_ERR("Maximum number of SAs parsed (%u), ignoring rest\n", max_num_sa); + if (cs == NULL) + return; + + if (config_setting_lookup_int(cs, "cipher_alg", &val) == CONFIG_TRUE) + config->sa_param.crypto.cipher_alg = val; + + if (config_setting_lookup_string(cs, "cipher_key", &val_str) == CONFIG_TRUE) { + strcpy(config->cipher_key, val_str); + config->sa_param.crypto.cipher_key.data = (uint8_t *)config->cipher_key; + config->sa_param.crypto.cipher_key.length = + strlen((const char *)config->cipher_key); + } + + if (config_setting_lookup_string(cs, "cipher_key_extra", &val_str) == CONFIG_TRUE) { + strcpy(config->cipher_key_extra, val_str); + config->sa_param.crypto.cipher_key_extra.data = + (uint8_t *)config->cipher_key_extra; + config->sa_param.crypto.cipher_key_extra.length = + strlen((const char *)config->cipher_key_extra); + } + + if (config_setting_lookup_int(cs, "auth_alg", &val) == CONFIG_TRUE) + config->sa_param.crypto.auth_alg = val; + + if (config_setting_lookup_string(cs, "auth_key", &val_str) == CONFIG_TRUE) { + strcpy(config->auth_key, val_str); + config->sa_param.crypto.auth_key.data = (uint8_t *)config->auth_key; + config->sa_param.crypto.auth_key.length = strlen((const char *)config->auth_key); + } + + if (config_setting_lookup_string(cs, "auth_key_extra", &val_str) == CONFIG_TRUE) { + strcpy(config->auth_key_extra, val_str); + config->sa_param.crypto.auth_key_extra.data = (uint8_t *)config->auth_key_extra; + config->sa_param.crypto.auth_key_extra.length = + strlen((const char *)config->auth_key_extra); + } + + if (config_setting_lookup_int(cs, "icv_len", &val) == CONFIG_TRUE) + config->sa_param.crypto.icv_len = val; +} + +static void parse_opt(config_setting_t *cfg, sa_config_t *config) +{ + int val; + config_setting_t *cs = config_setting_lookup(cfg, "opt"); + + if (cs == NULL) + return; + + if (config_setting_lookup_int(cs, "esn", &val) == CONFIG_TRUE) + config->sa_param.opt.esn = val; + + if (config_setting_lookup_int(cs, "udp_encap", &val) == CONFIG_TRUE) + config->sa_param.opt.udp_encap = val; + + if (config_setting_lookup_int(cs, "copy_dscp", &val) == CONFIG_TRUE) + config->sa_param.opt.copy_dscp = val; + + if (config_setting_lookup_int(cs, "copy_flabel", &val) == CONFIG_TRUE) + config->sa_param.opt.copy_flabel = val; + + if (config_setting_lookup_int(cs, "copy_df", &val) == CONFIG_TRUE) + config->sa_param.opt.copy_df = val; + + if (config_setting_lookup_int(cs, "dec_ttl", &val) == CONFIG_TRUE) + config->sa_param.opt.dec_ttl = val; +} + +static void parse_limits(config_setting_t *cfg, sa_config_t *config) +{ + config_setting_t *cs = config_setting_lookup(cfg, "lifetime"), *soft, *hard; + long long val; + + if (cs == NULL) return; + + soft = config_setting_lookup(cs, "soft_limit"); + hard = config_setting_lookup(cs, "hard_limit"); + + if (soft != NULL) { + if (config_setting_lookup_int64(soft, "bytes", &val) == CONFIG_TRUE) + config->sa_param.lifetime.soft_limit.bytes = val; + + if (config_setting_lookup_int64(soft, "packets", &val) == CONFIG_TRUE) + config->sa_param.lifetime.soft_limit.packets = val; + } + + if (hard != NULL) { + if (config_setting_lookup_int64(hard, "bytes", &val) == CONFIG_TRUE) + config->sa_param.lifetime.hard_limit.bytes = val; + + if (config_setting_lookup_int64(hard, "packets", &val) == CONFIG_TRUE) + config->sa_param.lifetime.hard_limit.packets = val; } +} - if (odph_ipv4_addr_parse(&src_ip, src_ip_str) < 0 || - odph_ipv4_addr_parse(&dst_ip, dst_ip_str) < 0) { - ODPH_ERR("Error parsing IP addresses for SA %u\n", spi); +static void parse_inbound(config_setting_t *cfg, sa_config_t *config) +{ + config_setting_t *cs = config_setting_lookup(cfg, "inbound"); + int val; + const char *val_str; + + if (cs == NULL) + return; + + if (config_setting_lookup_int(cs, "lookup_mode", &val) == CONFIG_TRUE) + config->sa_param.inbound.lookup_mode = val; + + if (config_setting_lookup_string(cs, "lookup_dst_addr", &val_str) == CONFIG_TRUE) { + odph_ipv4_addr_parse(&config->lkp_dst_ip, val_str); + config->lkp_dst_ip = odp_cpu_to_be_32(config->lkp_dst_ip); + config->sa_param.inbound.lookup_param.dst_addr = &config->lkp_dst_ip; + } + + if (config_setting_lookup_int(cs, "antireplay_ws", &val) == CONFIG_TRUE) + config->sa_param.inbound.antireplay_ws = val; + + if (config_setting_lookup_int(cs, "reassembly_en", &val) == CONFIG_TRUE) + config->sa_param.inbound.reassembly_en = val; +} + +static void parse_outbound(config_setting_t *cfg, sa_config_t *config) +{ + config_setting_t *cs = config_setting_lookup(cfg, "outbound"), *tunnel; + const char *val_str; + int val; + + if (cs == NULL) + return; + + tunnel = config_setting_lookup(cs, "tunnel"); + + if (tunnel != NULL) { + if (config_setting_lookup_string(tunnel, "src_addr", &val_str) == CONFIG_TRUE) { + odph_ipv4_addr_parse(&config->src_ip, val_str); + config->src_ip = odp_cpu_to_be_32(config->src_ip); + config->sa_param.outbound.tunnel.ipv4.src_addr = &config->src_ip; + } + + if (config_setting_lookup_string(tunnel, "dst_addr", &val_str) == CONFIG_TRUE) { + odph_ipv4_addr_parse(&config->dst_ip, val_str); + config->dst_ip = odp_cpu_to_be_32(config->dst_ip); + config->sa_param.outbound.tunnel.ipv4.dst_addr = &config->dst_ip; + } + + if (config_setting_lookup_int(tunnel, "dscp", &val) == CONFIG_TRUE) + config->sa_param.outbound.tunnel.ipv4.dscp = val; + + if (config_setting_lookup_int(tunnel, "df", &val) == CONFIG_TRUE) + config->sa_param.outbound.tunnel.ipv4.df = val; + + if (config_setting_lookup_int(tunnel, "ttl", &val) == CONFIG_TRUE) + config->sa_param.outbound.tunnel.ipv4.ttl = val; + } + + if (config_setting_lookup_int(cs, "frag_mode", &val) == CONFIG_TRUE) + config->sa_param.outbound.frag_mode = val; + + if (config_setting_lookup_int(cs, "mtu", &val) == CONFIG_TRUE) + config->sa_param.outbound.mtu = val; +} + +static void parse_sa_entry(config_setting_t *cfg, sa_config_t *config) +{ + int val; + + if (config_setting_lookup_int(cfg, "dir", &val) == CONFIG_TRUE) + config->sa_param.dir = val; + + if (config_setting_lookup_int(cfg, "proto", &val) == CONFIG_TRUE) + config->sa_param.proto = val; + + if (config_setting_lookup_int(cfg, "mode", &val) == CONFIG_TRUE) + config->sa_param.mode = val; + + if (config_setting_lookup_int(cfg, "spi", &val) == CONFIG_TRUE) + config->sa_param.spi = val; + + parse_crypto(cfg, config); + parse_opt(cfg, config); + parse_limits(cfg, config); + parse_inbound(cfg, config); + parse_outbound(cfg, config); +} + +static void create_sa_entry(odp_ipsec_sa_param_t *sa_param, prog_config_t *config, + uint32_t max_num_sa) +{ + uint32_t dir = sa_param->dir; + uint32_t spi = sa_param->spi; + odp_ipsec_sa_t sa; + + if (config->num_sas == max_num_sa) { + ODPH_ERR("Maximum number of SAs parsed (%u), ignoring rest\n", max_num_sa); return; } @@ -981,37 +1207,8 @@ static void create_sa_entry(uint32_t dir, uint32_t spi, const char *src_ip_str, return; } - src_ip = odp_cpu_to_be_32(src_ip); - dst_ip = odp_cpu_to_be_32(dst_ip); - odp_ipsec_sa_param_init(&sa_param); - sa_param.proto = ODP_IPSEC_ESP; - sa_param.mode = ODP_IPSEC_MODE_TUNNEL; - sa_param.spi = spi; - sa_param.dest_queue = config->sa_qs[config->num_sas % config->num_sa_qs]; - - if (dir > 0U) { - sa_param.dir = ODP_IPSEC_DIR_OUTBOUND; - sa_param.outbound.tunnel.ipv4.src_addr = &src_ip; - sa_param.outbound.tunnel.ipv4.dst_addr = &dst_ip; - } else { - sa_param.dir = ODP_IPSEC_DIR_INBOUND; - sa_param.inbound.lookup_mode = ODP_IPSEC_LOOKUP_DISABLED; - sa_param.inbound.antireplay_ws = ar_ws; - } - - crypto_param.cipher_alg = cipher_idx; - crypto_param.cipher_key.data = cipher_key; - crypto_param.cipher_key.length = strlen((const char *)cipher_key); - crypto_param.cipher_key_extra.data = cipher_key_extra; - crypto_param.cipher_key_extra.length = strlen((const char *)cipher_key_extra); - crypto_param.auth_alg = auth_idx; - crypto_param.auth_key.data = auth_key; - crypto_param.auth_key.length = strlen((const char *)auth_key); - crypto_param.auth_key_extra.data = auth_key_extra; - crypto_param.auth_key_extra.length = strlen((const char *)auth_key_extra); - crypto_param.icv_len = icv_len; - sa_param.crypto = crypto_param; - sa = odp_ipsec_sa_create(&sa_param); + sa_param->dest_queue = config->sa_qs[config->num_sas % config->num_sa_qs]; + sa = odp_ipsec_sa_create(sa_param); if (sa == ODP_IPSEC_SA_INVALID) { ODPH_ERR("Error creating SA handle for SA %u\n", spi); @@ -1023,19 +1220,46 @@ static void create_sa_entry(uint32_t dir, uint32_t spi, const char *src_ip_str, ++config->num_sas; } -static void parse_sas(prog_config_t *config) +static void parse_and_create_sa_entries(config_t *cfg, prog_config_t *config, uint32_t max_num_sa) { - odp_ipsec_capability_t ipsec_capa; - FILE *file; - int cipher_idx, auth_idx; - uint32_t ar_ws, max_num_sa, dir, spi, icv_len; - char src_ip[16U] = { 0 }, dst_ip[16U] = { 0 }; - uint8_t cipher_key[65U] = { 0U }, cipher_key_extra[5U] = { 0U }, auth_key[65U] = { 0U }, - auth_key_extra[5U] = { 0U }; - - if (config->sa_conf_file == NULL) + config_setting_t *cs; + int count; + + cs = config_lookup(cfg, "default"); + + if (cs != NULL) + parse_sa_entry(cs, &config->default_cfg); + + cs = config_lookup(cfg, "sa"); + + if (cs == NULL) return; + count = config_setting_length(cs); + + for (int i = 0; i < count; i++) { + sa_config_t sa_cfg; + config_setting_t *sa; + int val; + + sa_cfg = config->default_cfg; + sa = config_setting_get_elem(cs, i); + + if (sa == NULL) + continue; + + if (config_setting_lookup_int(sa, "spi", &val) == CONFIG_TRUE) { + parse_sa_entry(sa, &sa_cfg); + create_sa_entry(&sa_cfg.sa_param, config, max_num_sa); + } + } +} + +static void parse_sas(config_t *cfg, prog_config_t *config) +{ + odp_ipsec_capability_t ipsec_capa; + uint32_t max_num_sa; + if (odp_ipsec_capability(&ipsec_capa) < 0) { ODPH_ERR("Error querying IPsec capabilities\n"); return; @@ -1047,24 +1271,8 @@ static void parse_sas(prog_config_t *config) if (!config->is_dir_rx && !create_sa_dest_queues(&ipsec_capa, config)) return; - file = fopen(config->sa_conf_file, "r"); - - if (file == NULL) { - ODPH_ERR("Error opening SA configuration file: %s\n", strerror(errno)); - return; - } - - ar_ws = MIN(32U, ipsec_capa.max_antireplay_ws); max_num_sa = MIN(MAX_SAS, ipsec_capa.max_num_sa); - - while (fscanf(file, "%u%u%s%s%d%s%s%d%s%s%u", &dir, &spi, src_ip, dst_ip, - &cipher_idx, cipher_key, cipher_key_extra, &auth_idx, auth_key, - auth_key_extra, &icv_len) == 11) - create_sa_entry(!!dir, spi, src_ip, dst_ip, cipher_idx, cipher_key, - cipher_key_extra, auth_idx, auth_key, auth_key_extra, icv_len, - ar_ws, max_num_sa, config); - - (void)fclose(file); + parse_and_create_sa_entries(cfg, config, max_num_sa); } static const pktio_t *get_pktio(const char *iface, const prog_config_t *config) @@ -1077,12 +1285,14 @@ static const pktio_t *get_pktio(const char *iface, const prog_config_t *config) return NULL; } -static void create_fwd_table_entry(const char *dst_ip_str, const char *iface, - const char *dst_mac_str, uint8_t mask, prog_config_t *config) +static void create_fwd_table_entry(config_setting_t *cfg, prog_config_t *config) { - fwd_entry_t *entry; + const char *val_str; + char dst_ip_str[16U] = { 0 }; + uint32_t mask, dst_ip; odph_ethaddr_t dst_mac; - uint32_t dst_ip; + const pktio_t *pktio = NULL; + fwd_entry_t *entry; odph_iplookup_prefix_t prefix; if (config->num_fwds == MAX_FWDS) { @@ -1091,50 +1301,69 @@ static void create_fwd_table_entry(const char *dst_ip_str, const char *iface, return; } - entry = &config->fwd_entries[config->num_fwds]; + if (config_setting_lookup_string(cfg, "prefix", &val_str) == CONFIG_TRUE) { + if (sscanf(val_str, "%[^/]/%u", dst_ip_str, &mask) != 2) { + ODPH_ERR("Error parsing IP and subnet mask for forwarding entry\n"); + return; + } - if (odph_eth_addr_parse(&dst_mac, dst_mac_str) < 0 || - odph_ipv4_addr_parse(&dst_ip, dst_ip_str) < 0) { - ODPH_ERR("Error parsing MAC and IP addresses for forwarding entry\n"); + if (odph_ipv4_addr_parse(&dst_ip, dst_ip_str) < 0) { + ODPH_ERR("Syntax error in IP address for forwarding entry\n"); + return; + } + } else { return; } - entry->pktio = get_pktio(iface, config); + if (config_setting_lookup_string(cfg, "if", &val_str) == CONFIG_TRUE) { + pktio = get_pktio(val_str, config); + + if (pktio == NULL) { + ODPH_ERR("Error parsing next interface for forwarding entry\n"); + return; + } + } else { + return; + } - if (entry->pktio == NULL) { - ODPH_ERR("Invalid interface in forwarding entry: %s\n", iface); + if (config_setting_lookup_string(cfg, "dst_mac", &val_str) == CONFIG_TRUE) { + if (odph_eth_addr_parse(&dst_mac, val_str) < 0) { + ODPH_ERR("Syntax error in destination MAC for forwarding entry\n"); + return; + } + } else { return; } + entry = &config->fwd_entries[config->num_fwds]; entry->dst_mac = dst_mac; + entry->pktio = pktio; prefix.ip = dst_ip; prefix.cidr = mask; entry->prefix = prefix; ++config->num_fwds; } -static void parse_fwd_table(prog_config_t *config) +static void parse_fwd_table(config_t *cfg, prog_config_t *config) { - FILE *file; - char dst_ip[16U] = { 0 }, iface[64U] = { 0 }, dst_mac[18U] = { 0 }; - uint32_t mask; + config_setting_t *cs; + int count; - if (config->fwd_conf_file == NULL) { - ODPH_ERR("Invalid forwarding configuration file\n"); + cs = config_lookup(cfg, "fwd"); + + if (cs == NULL) return; - } - file = fopen(config->fwd_conf_file, "r"); + count = config_setting_length(cs); - if (file == NULL) { - ODPH_ERR("Error opening forwarding configuration file: %s\n", strerror(errno)); - return; - } + for (int i = 0; i < count; i++) { + config_setting_t *fwd = config_setting_get_elem(cs, i); - while (fscanf(file, " %[^/]/%u%s%s", dst_ip, &mask, iface, dst_mac) == 4) - create_fwd_table_entry(dst_ip, iface, dst_mac, mask, config); + if (fwd == NULL) + continue; - (void)fclose(file); + create_fwd_table_entry(fwd, config); + } } static parse_result_t check_options(prog_config_t *config) @@ -1152,11 +1381,6 @@ static parse_result_t check_options(prog_config_t *config) return PRS_NOK; } - if (config->sa_conf_file != NULL && config->num_sas == 0U) { - ODPH_ERR("Invalid SA configuration\n"); - return PRS_NOK; - } - if (config->num_fwds == 0U) { ODPH_ERR("Invalid number of forwarding entries: %u (min: 1, max: %u)\n", config->num_fwds, MAX_FWDS); @@ -1198,24 +1422,24 @@ static parse_result_t check_options(prog_config_t *config) static parse_result_t parse_options(int argc, char **argv, prog_config_t *config) { int opt, long_index; + config_t cfg; static const struct option longopts[] = { - { "interfaces", required_argument, NULL, 'i'}, - { "num_pkts", required_argument, NULL, 'n'}, - { "pkt_len", required_argument, NULL, 'l'}, + { "interfaces", required_argument, NULL, 'i' }, + { "num_pkts", required_argument, NULL, 'n' }, + { "pkt_len", required_argument, NULL, 'l' }, { "count", required_argument, NULL, 'c' }, { "mode", required_argument, NULL, 'm' }, - { "sa", required_argument, NULL, 's'}, - { "fwd_table", required_argument, NULL, 'f' }, + { "conf", required_argument, NULL, 'C' }, { "num_input_qs", required_argument, NULL, 'I' }, { "num_sa_qs", required_argument, NULL, 'S' }, { "num_output_qs", required_argument, NULL, 'O' }, - { "direct_rx", no_argument, NULL, 'd'}, + { "direct_rx", no_argument, NULL, 'd' }, { "help", no_argument, NULL, 'h' }, { NULL, 0, NULL, 0 } }; - static const char *shortopts = "i:n:l:c:m:s:f:I:S:O:dh"; + static const char *shortopts = "i:n:l:c:m:C:I:S:O:dh"; while (true) { opt = getopt_long(argc, argv, shortopts, longopts, &long_index); @@ -1239,11 +1463,8 @@ static parse_result_t parse_options(int argc, char **argv, prog_config_t *config case 'm': config->mode = !!atoi(optarg); break; - case 's': - config->sa_conf_file = strdup(optarg); - break; - case 'f': - config->fwd_conf_file = strdup(optarg); + case 'C': + config->conf_file = strdup(optarg); break; case 'I': config->num_input_qs = atoi(optarg); @@ -1267,8 +1488,17 @@ static parse_result_t parse_options(int argc, char **argv, prog_config_t *config } } - parse_sas(config); - parse_fwd_table(config); + config_init(&cfg); + + if (config_read_file(&cfg, config->conf_file) == CONFIG_FALSE) { + ODPH_ERR("Error opening SA configuration file: %s\n", config_error_text(&cfg)); + config_destroy(&cfg); + return PRS_NOK; + } + + parse_sas(&cfg, config); + parse_fwd_table(&cfg, config); + config_destroy(&cfg); return check_options(config); } @@ -1686,8 +1916,7 @@ static void teardown_test(const prog_config_t *config) if (config->compl_q != ODP_QUEUE_INVALID) (void)odp_queue_destroy(config->compl_q); - free(config->sa_conf_file); - free(config->fwd_conf_file); + free(config->conf_file); } static void print_stats(const prog_config_t *config) diff --git a/test/performance/odp_ipsecfwd.conf b/test/performance/odp_ipsecfwd.conf new file mode 100644 index 000000000..81ddaef7e --- /dev/null +++ b/test/performance/odp_ipsecfwd.conf @@ -0,0 +1,41 @@ +default: { + dir = 1 + proto = 0 + mode = 0 + crypto: { + cipher_alg = 4 + cipher_key = "jWnZr4t7w!zwC*F-" + auth_alg = 2 + auth_key = "n2r5u7x!A%D*" + icv_len = 12 + }; +}; + +sa: ( + { + spi = 1337 + outbound: { + tunnel: { + src_addr = "192.168.1.10" + dst_addr = "192.168.1.16" + }; + }; + }, + { + spi = 1338 + outbound: { + tunnel: { + src_addr = "192.168.3.110" + dst_addr = "192.168.3.116" + }; + }; + } +); + +fwd: ( + { + prefix: "192.168.1.0/24" + if: "ens9f1" + dst_mac: "00:00:05:00:07:00" + } +); diff --git a/test/performance/odp_packet_gen.c b/test/performance/odp_packet_gen.c index e02ffe95e..85b0c740b 100644 --- a/test/performance/odp_packet_gen.c +++ b/test/performance/odp_packet_gen.c @@ -1312,7 +1312,6 @@ static int init_packets(test_global_t *global, int pktio, void *data; uint8_t *u8; odph_ethhdr_t *eth; - odph_vlanhdr_t *vlan; odph_ipv4hdr_t *ip; odph_udphdr_t *udp; uint16_t tpid; @@ -1323,6 +1322,7 @@ static int init_packets(test_global_t *global, int pktio, uint16_t udp_dst = test_options->udp_dst; uint32_t udp_src_cnt = 0; uint32_t udp_dst_cnt = 0; + odph_vlanhdr_t *vlan = NULL; /* Fixes bogus compiler warning */ if (num_vlan > MAX_VLANS) num_vlan = MAX_VLANS; diff --git a/test/performance/odp_timer_perf.c b/test/performance/odp_timer_perf.c index ccea30a14..3df9a875f 100644 --- a/test/performance/odp_timer_perf.c +++ b/test/performance/odp_timer_perf.c @@ -292,6 +292,8 @@ static int create_timer_pools(test_global_t *global) printf(" num timer %u\n", num_timer); printf(" resolution %" PRIu64 " nsec\n", res_ns); printf(" period %" PRIu64 " nsec\n", period_ns); + printf(" max timeout %" PRIu64 " nsec\n", max_tmo_ns); + printf(" min timeout %" PRIu64 " nsec\n", min_tmo_ns); printf(" first timer at %.2f sec\n", (double)START_NS / ODP_TIME_SEC_IN_NS); if (mode == MODE_SCHED_OVERH) printf(" test duration %.2f sec\n", (double)max_tmo_ns / ODP_TIME_SEC_IN_NS); diff --git a/test/validation/api/classification/odp_classification_basic.c b/test/validation/api/classification/odp_classification_basic.c index f7aa96e52..2969dedab 100644 --- a/test/validation/api/classification/odp_classification_basic.c +++ b/test/validation/api/classification/odp_classification_basic.c @@ -203,13 +203,14 @@ static void classification_test_create_pmr_match(void) retval = odp_cls_pmr_destroy(ODP_PMR_INVALID); CU_ASSERT(retval < 0); + odp_cos_destroy(cos); odp_queue_destroy(queue); odp_pool_destroy(pool); odp_pool_destroy(pkt_pool); - odp_cos_destroy(cos); + odp_pktio_default_cos_set(pktio, ODP_COS_INVALID); + odp_cos_destroy(default_cos); odp_queue_destroy(default_queue); odp_pool_destroy(default_pool); - odp_cos_destroy(default_cos); odp_pktio_close(pktio); } @@ -401,13 +402,14 @@ static void classification_test_pmr_composite_create(void) retval = odp_cls_pmr_destroy(pmr_composite); CU_ASSERT(retval == 0); + odp_cos_destroy(cos); odp_queue_destroy(queue); odp_pool_destroy(pool); odp_pool_destroy(pkt_pool); - odp_cos_destroy(cos); + odp_pktio_default_cos_set(pktio, ODP_COS_INVALID); + odp_cos_destroy(default_cos); odp_queue_destroy(default_queue); odp_pool_destroy(default_pool); - odp_cos_destroy(default_cos); odp_pktio_close(pktio); } diff --git a/test/validation/api/classification/odp_classification_test_pmr.c b/test/validation/api/classification/odp_classification_test_pmr.c index 0b29783c0..bf067719b 100644 --- a/test/validation/api/classification/odp_classification_test_pmr.c +++ b/test/validation/api/classification/odp_classification_test_pmr.c @@ -194,9 +194,10 @@ static void classification_test_pktin_classifier_flag(void) CU_ASSERT(seqno == cls_pkt_get_seq(pkt)); odp_packet_free(pkt); + odp_cls_pmr_destroy(pmr); odp_cos_destroy(cos); + odp_pktio_default_cos_set(pktio, ODP_COS_INVALID); odp_cos_destroy(default_cos); - odp_cls_pmr_destroy(pmr); stop_pktio(pktio); odp_queue_destroy(queue); odp_queue_destroy(default_queue); @@ -366,9 +367,10 @@ static void _classification_test_pmr_term_tcp_dport(int num_pkt) CU_ASSERT(sent_queue == recv_queue); CU_ASSERT(sent_default == recv_default); + odp_cls_pmr_destroy(pmr); odp_cos_destroy(cos); + odp_pktio_default_cos_set(pktio, ODP_COS_INVALID); odp_cos_destroy(default_cos); - odp_cls_pmr_destroy(pmr); stop_pktio(pktio); odp_queue_destroy(queue); odp_queue_destroy(default_queue); @@ -457,9 +459,10 @@ static void test_pmr(const odp_pmr_param_t *pmr_param, odp_packet_t pkt, } odp_packet_free(pkt); + odp_cls_pmr_destroy(pmr); odp_cos_destroy(cos); + odp_pktio_default_cos_set(pktio, ODP_COS_INVALID); odp_cos_destroy(default_cos); - odp_cls_pmr_destroy(pmr); /* XXX ordering */ stop_pktio(pktio); odp_pool_destroy(default_pool); odp_pool_destroy(pool); @@ -747,9 +750,10 @@ static void classification_test_pmr_term_dmac(void) CU_ASSERT(recvpool == default_pool); CU_ASSERT(retqueue == default_queue); + odp_cls_pmr_destroy(pmr); odp_cos_destroy(cos); + odp_pktio_default_cos_set(pktio, ODP_COS_INVALID); odp_cos_destroy(default_cos); - odp_cls_pmr_destroy(pmr); odp_packet_free(pkt); stop_pktio(pktio); odp_pool_destroy(default_pool); @@ -1080,9 +1084,10 @@ static void classification_test_pmr_pool_set(void) CU_ASSERT(retqueue == queue); odp_packet_free(pkt); + odp_cls_pmr_destroy(pmr); odp_cos_destroy(cos); + odp_pktio_default_cos_set(pktio, ODP_COS_INVALID); odp_cos_destroy(default_cos); - odp_cls_pmr_destroy(pmr); stop_pktio(pktio); odp_pool_destroy(default_pool); odp_pool_destroy(pool); @@ -1180,9 +1185,10 @@ static void classification_test_pmr_queue_set(void) CU_ASSERT(retqueue == queue_new); odp_packet_free(pkt); + odp_cls_pmr_destroy(pmr); odp_cos_destroy(cos); + odp_pktio_default_cos_set(pktio, ODP_COS_INVALID); odp_cos_destroy(default_cos); - odp_cls_pmr_destroy(pmr); stop_pktio(pktio); odp_pool_destroy(default_pool); odp_pool_destroy(pool); @@ -1603,15 +1609,15 @@ static void test_pmr_series(const int num_udp, int marking) CU_ASSERT(seqno == cls_pkt_get_seq(pkt)); CU_ASSERT(retqueue == default_queue); odp_packet_free(pkt); + odp_cls_pmr_destroy(pmr_ip); for (i = 0; i < num_udp; i++) { - odp_cos_destroy(cos_udp[i]); odp_cls_pmr_destroy(pmr_udp[i]); + odp_cos_destroy(cos_udp[i]); } odp_cos_destroy(cos_ip); - odp_cls_pmr_destroy(pmr_ip); - + odp_pktio_default_cos_set(pktio, ODP_COS_INVALID); odp_cos_destroy(default_cos); stop_pktio(pktio); odp_pool_destroy(default_pool); diff --git a/test/validation/api/classification/odp_classification_tests.c b/test/validation/api/classification/odp_classification_tests.c index 962885c06..002beb2aa 100644 --- a/test/validation/api/classification/odp_classification_tests.c +++ b/test/validation/api/classification/odp_classification_tests.c @@ -162,6 +162,16 @@ static int classification_suite_common_term(odp_bool_t enable_pktv) retcode = -1; } + for (i = 0; i < CLS_ENTRIES; i++) { + if (pmr_list[i] != ODP_PMR_INVALID) + odp_cls_pmr_destroy(pmr_list[i]); + } + + for (i = 0; i < CLS_ENTRIES; i++) { + if (cos_list[i] != ODP_COS_INVALID) + odp_cos_destroy(cos_list[i]); + } + if (0 != odp_pool_destroy(pool_default)) { ODPH_ERR("Pool_default destroy failed\n"); retcode = -1; @@ -175,16 +185,6 @@ static int classification_suite_common_term(odp_bool_t enable_pktv) } for (i = 0; i < CLS_ENTRIES; i++) { - if (cos_list[i] != ODP_COS_INVALID) - odp_cos_destroy(cos_list[i]); - } - - for (i = 0; i < CLS_ENTRIES; i++) { - if (pmr_list[i] != ODP_PMR_INVALID) - odp_cls_pmr_destroy(pmr_list[i]); - } - - for (i = 0; i < CLS_ENTRIES; i++) { if (queue_list[i] != ODP_QUEUE_INVALID) odp_queue_destroy(queue_list[i]); } diff --git a/test/validation/api/crypto/odp_crypto_test_inp.c b/test/validation/api/crypto/odp_crypto_test_inp.c index 7d5e50c71..c6a7767cc 100644 --- a/test/validation/api/crypto/odp_crypto_test_inp.c +++ b/test/validation/api/crypto/odp_crypto_test_inp.c @@ -6,14 +6,23 @@ */ #include <string.h> +#include <stdlib.h> #include <odp_api.h> #include <odp/helper/odph_api.h> #include <odp_cunit_common.h> #include <packet_common.h> #include "test_vectors.h" +/* + * If nonzero, run time consuming tests too. + * Set through FULL_TEST environment variable. + */ +static int full_test; + +#define MAX_FAILURE_PRINTS 20 + #define PKT_POOL_NUM 64 -#define PKT_POOL_LEN (1 * 1024) +#define PKT_POOL_LEN 1200 /* enough for a test packet and some headroom */ #define UAREA_SIZE 8 #define ARRAY_SIZE(a) (sizeof(a) / sizeof(a[0])) @@ -40,9 +49,6 @@ static void test_defaults(uint8_t fill) CU_ASSERT_EQUAL(param.op, ODP_CRYPTO_OP_ENCODE); CU_ASSERT_EQUAL(param.op_type, ODP_CRYPTO_OP_TYPE_LEGACY); CU_ASSERT_EQUAL(param.auth_cipher_text, false); -#if ODP_DEPRECATED_API - CU_ASSERT_EQUAL(param.pref_mode, ODP_CRYPTO_SYNC); -#endif CU_ASSERT_EQUAL(param.op_mode, ODP_CRYPTO_SYNC); CU_ASSERT_EQUAL(param.cipher_alg, ODP_CIPHER_ALG_NULL); CU_ASSERT_EQUAL(param.cipher_iv_len, 0); @@ -151,88 +157,6 @@ static const char *cipher_alg_name(odp_cipher_alg_t cipher) } } -#if ODP_DEPRECATED_API -static int alg_op(odp_packet_t pkt, - odp_bool_t *ok, - odp_crypto_session_t session, - uint8_t *cipher_iv_ptr, - uint8_t *auth_iv_ptr, - odp_packet_data_range_t *cipher_range, - odp_packet_data_range_t *auth_range, - uint8_t *aad, - unsigned int hash_result_offset) -{ - int rc; - odp_crypto_op_result_t result; - odp_crypto_op_param_t op_params; - odp_bool_t posted; - odp_event_subtype_t subtype; - - /* Prepare input/output params */ - memset(&op_params, 0, sizeof(op_params)); - op_params.session = session; - op_params.pkt = pkt; - op_params.out_pkt = pkt; - op_params.ctx = (void *)0xdeadbeef; - - op_params.cipher_range = *cipher_range; - op_params.auth_range = *auth_range; - if (cipher_iv_ptr) - op_params.cipher_iv_ptr = cipher_iv_ptr; - if (auth_iv_ptr) - op_params.auth_iv_ptr = auth_iv_ptr; - - op_params.aad_ptr = aad; - - op_params.hash_result_offset = hash_result_offset; - - rc = odp_crypto_operation(&op_params, &posted, &result); - if (rc < 0) { - CU_FAIL("Failed odp_crypto_operation()"); - return rc; - } - - if (posted) { - odp_event_t event; - odp_crypto_compl_t compl_event; - - /* Get crypto completion event from compl_queue. */ - CU_ASSERT_FATAL(NULL != suite_context.compl_queue_deq); - do { - event = suite_context.compl_queue_deq(); - } while (event == ODP_EVENT_INVALID); - - CU_ASSERT(odp_event_is_valid(event) == 1); - CU_ASSERT(ODP_EVENT_CRYPTO_COMPL == odp_event_type(event)); - CU_ASSERT(ODP_EVENT_NO_SUBTYPE == odp_event_subtype(event)); - CU_ASSERT(ODP_EVENT_CRYPTO_COMPL == - odp_event_types(event, &subtype)); - CU_ASSERT(ODP_EVENT_NO_SUBTYPE == subtype); - - compl_event = odp_crypto_compl_from_event(event); - CU_ASSERT(odp_crypto_compl_to_u64(compl_event) == - odp_crypto_compl_to_u64( - odp_crypto_compl_from_event(event))); - odp_crypto_compl_result(compl_event, &result); - odp_crypto_compl_free(compl_event); - } - - CU_ASSERT(result.pkt == pkt); - CU_ASSERT(result.ctx == (void *)0xdeadbeef); - CU_ASSERT(ODP_EVENT_PACKET == - odp_event_type(odp_packet_to_event(result.pkt))); - CU_ASSERT(ODP_EVENT_PACKET_BASIC == - odp_event_subtype(odp_packet_to_event(result.pkt))); - CU_ASSERT(ODP_EVENT_PACKET == - odp_event_types(odp_packet_to_event(result.pkt), &subtype)); - CU_ASSERT(ODP_EVENT_PACKET_BASIC == subtype); - - *ok = result.ok; - - return 0; -} -#endif - static int alg_packet_op(odp_packet_t pkt_in, odp_packet_t *pkt_out, odp_bool_t *ok, @@ -241,8 +165,8 @@ static int alg_packet_op(odp_packet_t pkt_in, int32_t oop_shift, uint8_t *cipher_iv_ptr, uint8_t *auth_iv_ptr, - odp_packet_data_range_t *cipher_range, - odp_packet_data_range_t *auth_range, + const odp_packet_data_range_t *cipher_range, + const odp_packet_data_range_t *auth_range, uint8_t *aad, unsigned int hash_result_offset) { @@ -319,17 +243,21 @@ static int alg_packet_op(odp_packet_t pkt_in, CU_ASSERT(odp_packet_subtype(*pkt_out) == ODP_EVENT_PACKET_CRYPTO); rc = odp_crypto_result(&result, *pkt_out); - if (rc < 0) { + if (rc < -1) { CU_FAIL("Failed odp_crypto_packet_result()"); return rc; } - CU_ASSERT(rc == 0); + CU_ASSERT(rc == 0 || rc == -1); if (op_type == ODP_CRYPTO_OP_TYPE_OOP && suite_context.op_mode == ODP_CRYPTO_ASYNC) CU_ASSERT(result.pkt_in == pkt_in); - *ok = result.ok; + *ok = (rc == 0); + +#if ODP_DEPRECATED_API + CU_ASSERT(*ok == result.ok); +#endif return 0; } @@ -342,30 +270,18 @@ static int crypto_op(odp_packet_t pkt_in, int32_t oop_shift, uint8_t *cipher_iv, uint8_t *auth_iv, - odp_packet_data_range_t *cipher_range, - odp_packet_data_range_t *auth_range, + const odp_packet_data_range_t *cipher_range, + const odp_packet_data_range_t *auth_range, uint8_t *aad, unsigned int hash_result_offset) { int rc; - if (!suite_context.packet) { -#if ODP_DEPRECATED_API - rc = alg_op(pkt_in, ok, session, - cipher_iv, auth_iv, - cipher_range, auth_range, - aad, hash_result_offset); - *pkt_out = pkt_in; -#else - rc = -1; -#endif - } else { - rc = alg_packet_op(pkt_in, pkt_out, ok, session, - op_type, oop_shift, - cipher_iv, auth_iv, - cipher_range, auth_range, - aad, hash_result_offset); - } + rc = alg_packet_op(pkt_in, pkt_out, ok, session, + op_type, oop_shift, + cipher_iv, auth_iv, + cipher_range, auth_range, + aad, hash_result_offset); if (rc < 0) odp_packet_free(pkt_in); @@ -407,8 +323,10 @@ static void adjust_segments(odp_packet_t *pkt, uint32_t first_seg_len) static void fill_with_pattern(uint8_t *buf, uint32_t len) { + static uint8_t value; + for (uint32_t n = 0; n < len; n++) - buf[n] = n; + buf[n] = value++; } static void write_header_and_trailer(odp_packet_t pkt, @@ -435,6 +353,8 @@ typedef struct alg_test_param_t { odp_cipher_alg_t cipher_alg; odp_auth_alg_t auth_alg; crypto_test_reference_t *ref; + odp_packet_data_range_t cipher_range; + odp_packet_data_range_t auth_range; uint32_t digest_offset; odp_bool_t is_bit_mode_cipher; odp_bool_t is_bit_mode_auth; @@ -450,23 +370,14 @@ static void prepare_crypto_ranges(const alg_test_param_t *param, odp_packet_data_range_t *auth_range) { odp_packet_data_range_t zero_range = {.offset = 0, .length = 0}; + uint32_t c_scale = param->is_bit_mode_cipher ? 8 : 1; + uint32_t a_scale = param->is_bit_mode_auth ? 8 : 1; + + *cipher_range = param->cipher_range; + *auth_range = param->auth_range; + cipher_range->offset += c_scale * param->header_len; + auth_range->offset += a_scale * param->header_len; - cipher_range->offset = param->header_len; - cipher_range->length = ref_length_in_bytes(param->ref); - auth_range->offset = param->header_len; - auth_range->length = ref_length_in_bytes(param->ref); - if (param->is_bit_mode_cipher) { - cipher_range->offset *= 8; - cipher_range->length = ref_length_in_bits(param->ref); - } - if (param->is_bit_mode_auth) { - auth_range->offset *= 8; - auth_range->length = ref_length_in_bits(param->ref); - } - /* - * We did not check the bit mode of the null algorithms, so let's - * not pass potentially invalid ranges to them. - */ if (param->cipher_alg == ODP_CIPHER_ALG_NULL) *cipher_range = zero_range; if (param->auth_alg == ODP_AUTH_ALG_NULL) @@ -571,7 +482,7 @@ static int is_in_range(uint32_t offs, uint32_t range_offs, uint32_t range_len) */ typedef struct ignore_t { uint32_t byte_offset; /* offset to a byte which has bits to be ignored */ - uint32_t byte_mask; /* mask of ignored bits in the byte */ + uint8_t byte_mask; /* mask of ignored bits in the byte */ struct { uint32_t offset; uint32_t length; @@ -619,10 +530,13 @@ static void prepare_ignore_info(const alg_test_param_t *param, */ if (param->is_bit_mode_cipher && param->cipher_alg != ODP_CIPHER_ALG_NULL) { - uint8_t leftover_bits = param->ref->length % 8; + uint8_t leftover_bits = ref_length_in_bits(param->ref) % 8; ignore->byte_offset = cipher_offset + cipher_len - 1 + shift; - ignore->byte_mask = ~(0xff << (8 - leftover_bits)); + if (leftover_bits > 0) + ignore->byte_mask = ~(0xff << (8 - leftover_bits)); + else + ignore->byte_mask = 0; } /* @@ -679,11 +593,13 @@ static void prepare_expected_data(const alg_test_param_t *param, const int32_t shift = param->op_type == ODP_CRYPTO_OP_TYPE_OOP ? param->oop_shift : 0; const odp_packet_t base_pkt = param->op_type == ODP_CRYPTO_OP_TYPE_OOP ? pkt_out : pkt_in; int rc; + uint32_t cipher_offset_in_ref = param->cipher_range.offset; if (param->op == ODP_CRYPTO_OP_ENCODE) digest_offset += shift; if (param->is_bit_mode_cipher) { + cipher_offset_in_ref /= 8; cipher_offset /= 8; cipher_len = (cipher_len + 7) / 8; } @@ -725,11 +641,11 @@ static void prepare_expected_data(const alg_test_param_t *param, * text) does not work in any real use case anyway. */ memcpy(ex->data + cipher_offset + shift, - param->ref->ciphertext, + param->ref->ciphertext + cipher_offset_in_ref, cipher_len); } else { memcpy(ex->data + cipher_offset + shift, - param->ref->plaintext, + param->ref->plaintext + cipher_offset_in_ref, cipher_len); } @@ -743,7 +659,7 @@ static void print_data(const char *title, uint8_t *data, uint32_t len) { static uint64_t limit; - if (limit++ > 20) + if (limit++ > MAX_FAILURE_PRINTS) return; printf("%s\n", title); @@ -830,14 +746,7 @@ static void alg_test_execute(const alg_test_param_t *param) param->ref->aad, digest_offset)) return; - /* - * API is not explicit about whether a failed crypto op - * sets the has_error packet flag or leaves it unchanged. - * Let's allow both behaviours. - */ test_packet_get_md(pkt_out, &md_out); - if (param->wrong_digest) - md_out.has_error = 0; if (param->op_type == ODP_CRYPTO_OP_TYPE_OOP) { test_packet_md_t md; @@ -855,15 +764,89 @@ static void alg_test_execute(const alg_test_param_t *param) CU_ASSERT(test_packet_is_md_equal(&md_out, &md_in)); } + if (param->cipher_alg != ODP_CIPHER_ALG_NULL && + param->auth_alg != ODP_AUTH_ALG_NULL && + param->digest_offset >= cipher_range.offset && + param->digest_offset < cipher_range.offset + cipher_range.length) { + /* + * Not all implementations support digest offset in cipher + * range, so allow crypto op failure without further checks + * in this case. + */ + if (!ok) + goto out; + } + if (param->wrong_digest) { CU_ASSERT(!ok); } else { CU_ASSERT(ok); } check_output_packet_data(pkt_out, &expected); +out: odp_packet_free(pkt_out); } +static void print_alg_test_param(const alg_test_param_t *p) +{ + const char *cipher_mode = p->is_bit_mode_cipher ? "bit" : "byte"; + const char *auth_mode = p->is_bit_mode_auth ? "bit" : "byte"; + + switch (p->op_type) { + case ODP_CRYPTO_OP_TYPE_LEGACY: + printf("legacy "); + break; + case ODP_CRYPTO_OP_TYPE_BASIC: + printf("basic "); + break; + case ODP_CRYPTO_OP_TYPE_OOP: + printf("out-of-place "); + break; + } + printf("%s\n", p->op == ODP_CRYPTO_OP_ENCODE ? "encode" : "decode"); + + printf("cipher: %s, %s mode\n", cipher_alg_name(p->cipher_alg), cipher_mode); + printf(" key length: %d, iv length: %d\n", + p->ref->cipher_key_length, p->ref->cipher_iv_length); + printf(" range: offset %d, length %d\n", + p->cipher_range.offset, p->cipher_range.length); + + printf("auth: %s, %s mode\n", auth_alg_name(p->auth_alg), auth_mode); + printf(" key length: %d, iv length: %d\n", + p->ref->auth_key_length, p->ref->auth_iv_length); + printf(" range: offset %d, length %d; aad length: %d\n", + p->auth_range.offset, p->auth_range.length, p->ref->aad_length); + printf(" digest offset: %d, digest length %d\n", + p->digest_offset, p->ref->digest_length); + + if (p->wrong_digest) + printf("wrong digest test\n"); + printf("header length: %d, trailer length: %d\n", p->header_len, p->trailer_len); + if (p->adjust_segmentation) + printf("segmentation adjusted, first_seg_len: %d\n", p->first_seg_len); + if (p->op_type == ODP_CRYPTO_OP_TYPE_OOP) + printf("oop_shift: %d\n", p->oop_shift); +} + +static void alg_test_execute_and_print(alg_test_param_t *param) +{ + static int print_limit = MAX_FAILURE_PRINTS; + unsigned int num = CU_get_number_of_failures(); + + alg_test_execute(param); + + if (CU_get_number_of_failures() > num) { + if (print_limit > 0) { + printf("\nTest failed:\n"); + print_alg_test_param(param); + printf("\n"); + print_limit--; + if (print_limit == 0) + printf("Suppressing further failure output\n"); + } + } +} + static void alg_test_op(alg_test_param_t *param) { int32_t oop_shifts[] = {0, 3, 130, -10}; @@ -877,13 +860,15 @@ static void alg_test_op(alg_test_param_t *param) param->oop_shift = oop_shifts[n]; param->wrong_digest = false; - alg_test_execute(param); - alg_test_execute(param); /* rerun with the same parameters */ + alg_test_execute_and_print(param); + if (full_test) + alg_test_execute_and_print(param); /* rerun with the same parameters */ param->wrong_digest = true; - alg_test_execute(param); + alg_test_execute_and_print(param); } } +static int combo_warning_shown; static int oop_warning_shown; typedef enum { @@ -891,10 +876,16 @@ typedef enum { HASH_OVERLAP, } hash_test_mode_t; +typedef enum { + AUTH_CIPHERTEXT, + AUTH_PLAINTEXT +} alg_order_t; + static odp_crypto_session_t session_create(odp_crypto_op_t op, odp_crypto_op_type_t op_type, odp_cipher_alg_t cipher_alg, odp_auth_alg_t auth_alg, + alg_order_t order, crypto_test_reference_t *ref, hash_test_mode_t hash_mode) { @@ -920,11 +911,8 @@ static odp_crypto_session_t session_create(odp_crypto_op_t op, odp_crypto_session_param_init(&ses_params); ses_params.op = op; ses_params.op_type = op_type; - ses_params.auth_cipher_text = false; + ses_params.auth_cipher_text = (order == AUTH_CIPHERTEXT); ses_params.op_mode = suite_context.op_mode; -#if ODP_DEPRECATED_API - ses_params.pref_mode = suite_context.pref_mode; -#endif ses_params.cipher_alg = cipher_alg; ses_params.auth_alg = auth_alg; ses_params.compl_queue = suite_context.queue; @@ -936,17 +924,29 @@ static odp_crypto_session_t session_create(odp_crypto_op_t op, ses_params.auth_digest_len = ref->digest_length; ses_params.auth_aad_len = ref->aad_length; ses_params.hash_result_in_auth_range = (hash_mode == HASH_OVERLAP); - rc = odp_crypto_session_create(&ses_params, &session, &status); + + if (rc < 0 && status == ODP_CRYPTO_SES_ERR_ALG_COMBO) { + if (!combo_warning_shown) { + combo_warning_shown = 1; + printf("\n Unsupported algorithm combination: %s, %s\n", + cipher_alg_name(cipher_alg), + auth_alg_name(auth_alg)); + } + return ODP_CRYPTO_SESSION_INVALID; + } + /* - * In some cases an individual algorithm cannot be used alone, - * i.e. with the null cipher/auth algorithm. + * Allow ODP_CRYPTO_SES_ERR_ALG_ORDER only in async op mode. + * In sync mode an implementation should be able to support both + * orders without much difficulty. */ - if (rc < 0 && - status == ODP_CRYPTO_SES_ERR_ALG_COMBO) { - printf("\n Unsupported algorithm combination: %s, %s\n", + if (rc < 0 && status == ODP_CRYPTO_SES_ERR_ALG_ORDER && + ses_params.op_mode == ODP_CRYPTO_ASYNC) { + printf("\n Unsupported algorithm order: %s, %s, auth_cipher_text: %d\n", cipher_alg_name(cipher_alg), - auth_alg_name(auth_alg)); + auth_alg_name(auth_alg), + ses_params.auth_cipher_text); return ODP_CRYPTO_SESSION_INVALID; } @@ -959,12 +959,6 @@ static odp_crypto_session_t session_create(odp_crypto_op_t op, return ODP_CRYPTO_SESSION_INVALID; } - /* - * We do not allow ODP_CRYPTO_SES_ERR_ALG_ORDER since we do - * not combine individual non-null crypto and auth algorithms - * with each other in the tests. Both orders should work when - * only one algorithm is used (i.e. the other one is null). - */ CU_ASSERT_FATAL(!rc); CU_ASSERT(status == ODP_CRYPTO_SES_ERR_NONE); CU_ASSERT(odp_crypto_session_to_u64(session) != @@ -985,22 +979,28 @@ static void alg_test_ses(odp_crypto_op_t op, odp_crypto_op_type_t op_type, odp_cipher_alg_t cipher_alg, odp_auth_alg_t auth_alg, + alg_order_t order, crypto_test_reference_t *ref, + odp_packet_data_range_t cipher_range, + odp_packet_data_range_t auth_range, uint32_t digest_offset, odp_bool_t is_bit_mode_cipher, odp_bool_t is_bit_mode_auth) { unsigned int initial_num_failures = CU_get_number_of_failures(); const uint32_t reflength = ref_length_in_bytes(ref); - const hash_test_mode_t hash_mode = digest_offset < reflength ? HASH_OVERLAP - : HASH_NO_OVERLAP; + hash_test_mode_t hash_mode = HASH_NO_OVERLAP; odp_crypto_session_t session; int rc; uint32_t seg_len; uint32_t max_shift; alg_test_param_t test_param; - session = session_create(op, op_type, cipher_alg, auth_alg, ref, hash_mode); + if (digest_offset >= auth_range.offset && + digest_offset < auth_range.offset + auth_range.length) + hash_mode = HASH_OVERLAP; + + session = session_create(op, op_type, cipher_alg, auth_alg, order, ref, hash_mode); if (session == ODP_CRYPTO_SESSION_INVALID) return; @@ -1011,6 +1011,8 @@ static void alg_test_ses(odp_crypto_op_t op, test_param.cipher_alg = cipher_alg; test_param.auth_alg = auth_alg; test_param.ref = ref; + test_param.cipher_range = cipher_range; + test_param.auth_range = auth_range; test_param.is_bit_mode_cipher = is_bit_mode_cipher; test_param.is_bit_mode_auth = is_bit_mode_auth; test_param.digest_offset = digest_offset; @@ -1018,12 +1020,21 @@ static void alg_test_ses(odp_crypto_op_t op, alg_test_op(&test_param); max_shift = reflength + ref->digest_length; + seg_len = 0; + + if (!full_test && + cipher_alg != ODP_CIPHER_ALG_NULL && + auth_alg != ODP_AUTH_ALG_NULL) { + /* run the loop body just once */ + seg_len = max_shift / 2; + max_shift = seg_len; + } /* * Test with segmented packets with all possible segment boundaries * within the packet data */ - for (seg_len = 0; seg_len <= max_shift; seg_len++) { + for (; seg_len <= max_shift; seg_len++) { /* * CUnit chokes on too many assertion failures, so bail * out if this test has already failed. @@ -1052,7 +1063,10 @@ static void alg_test_ses(odp_crypto_op_t op, static void alg_test(odp_crypto_op_t op, odp_cipher_alg_t cipher_alg, odp_auth_alg_t auth_alg, + alg_order_t order, crypto_test_reference_t *ref, + odp_packet_data_range_t cipher_range, + odp_packet_data_range_t auth_range, uint32_t digest_offset, odp_bool_t is_bit_mode_cipher, odp_bool_t is_bit_mode_auth) @@ -1071,7 +1085,10 @@ static void alg_test(odp_crypto_op_t op, op_types[n], cipher_alg, auth_alg, + order, ref, + cipher_range, + auth_range, digest_offset, is_bit_mode_cipher, is_bit_mode_auth); @@ -1131,6 +1148,8 @@ static void check_alg(odp_crypto_op_t op, odp_bool_t is_bit_mode_cipher = false; odp_bool_t is_bit_mode_auth = false; uint32_t digest_offs = ref_length_in_bytes(&ref[idx]); + odp_packet_data_range_t cipher_range = {.offset = 0}; + odp_packet_data_range_t auth_range = {.offset = 0}; if (ref_length_in_bits(&ref[idx]) % 8 != 0) bit_mode_needed = true; @@ -1190,7 +1209,18 @@ static void check_alg(odp_crypto_op_t op, continue; } - alg_test(op, cipher_alg, auth_alg, &ref[idx], digest_offs, + cipher_range.length = is_bit_mode_cipher ? + ref_length_in_bits(&ref[idx]) : + ref_length_in_bytes(&ref[idx]); + auth_range.length = is_bit_mode_auth ? + ref_length_in_bits(&ref[idx]) : + ref_length_in_bytes(&ref[idx]); + + alg_test(op, cipher_alg, auth_alg, AUTH_PLAINTEXT, &ref[idx], + cipher_range, auth_range, digest_offs, + is_bit_mode_cipher, is_bit_mode_auth); + alg_test(op, cipher_alg, auth_alg, AUTH_CIPHERTEXT, &ref[idx], + cipher_range, auth_range, digest_offs, is_bit_mode_cipher, is_bit_mode_auth); cipher_tested[cipher_idx] = true; @@ -1252,14 +1282,12 @@ static int check_alg_support(odp_cipher_alg_t cipher, odp_auth_alg_t auth) return ODP_TEST_INACTIVE; } - if (suite_context.packet) { - if (suite_context.op_mode == ODP_CRYPTO_SYNC && - capability.sync_mode == ODP_SUPPORT_NO) - return ODP_TEST_INACTIVE; - if (suite_context.op_mode == ODP_CRYPTO_ASYNC && - capability.async_mode == ODP_SUPPORT_NO) - return ODP_TEST_INACTIVE; - } + if (suite_context.op_mode == ODP_CRYPTO_SYNC && + capability.sync_mode == ODP_SUPPORT_NO) + return ODP_TEST_INACTIVE; + if (suite_context.op_mode == ODP_CRYPTO_ASYNC && + capability.async_mode == ODP_SUPPORT_NO) + return ODP_TEST_INACTIVE; /* Cipher algorithms */ switch (cipher) { @@ -1498,7 +1526,7 @@ static int create_hash_test_reference(odp_auth_alg_t auth, session = session_create(ODP_CRYPTO_OP_ENCODE, ODP_CRYPTO_OP_TYPE_LEGACY, ODP_CIPHER_ALG_NULL, - auth, ref, HASH_NO_OVERLAP); + auth, AUTH_PLAINTEXT, ref, HASH_NO_OVERLAP); if (session == ODP_CRYPTO_SESSION_INVALID) return -1; @@ -1531,10 +1559,16 @@ static int create_hash_test_reference(odp_auth_alg_t auth, } static void test_auth_hash_in_auth_range(odp_auth_alg_t auth, - const odp_crypto_auth_capability_t *capa) + const odp_crypto_auth_capability_t *capa, + alg_order_t order) { static crypto_test_reference_t ref = {.length = 0}; uint32_t digest_offset = 13; + const odp_packet_data_range_t cipher_range = {.offset = 0, .length = 0}; + odp_packet_data_range_t auth_range; + + if (!full_test && capa->digest_len % 4 != 0) + return; /* * Create test packets with auth hash in the authenticated range and @@ -1543,6 +1577,11 @@ static void test_auth_hash_in_auth_range(odp_auth_alg_t auth, if (create_hash_test_reference(auth, capa, &ref, digest_offset, 0)) return; + auth_range.offset = 0; + auth_range.length = capa->bit_mode ? + ref_length_in_bits(&ref) : + ref_length_in_bytes(&ref); + /* * Decode the ciphertext packet. * @@ -1553,7 +1592,9 @@ static void test_auth_hash_in_auth_range(odp_auth_alg_t auth, alg_test(ODP_CRYPTO_OP_DECODE, ODP_CIPHER_ALG_NULL, auth, + order, &ref, + cipher_range, auth_range, digest_offset, false, capa->bit_mode); @@ -1565,6 +1606,11 @@ static void test_auth_hash_in_auth_range(odp_auth_alg_t auth, if (create_hash_test_reference(auth, capa, &ref, digest_offset, 1)) return; + auth_range.offset = 0; + auth_range.length = capa->bit_mode ? + ref_length_in_bits(&ref) : + ref_length_in_bytes(&ref); + /* * Encode the plaintext packet. * @@ -1575,41 +1621,60 @@ static void test_auth_hash_in_auth_range(odp_auth_alg_t auth, alg_test(ODP_CRYPTO_OP_ENCODE, ODP_CIPHER_ALG_NULL, auth, + order, &ref, + cipher_range, auth_range, digest_offset, false, capa->bit_mode); } +/* + * Cipher algorithms that are not AEAD algorithms + */ +static odp_cipher_alg_t cipher_algs[] = { + ODP_CIPHER_ALG_DES, + ODP_CIPHER_ALG_3DES_CBC, + ODP_CIPHER_ALG_3DES_ECB, + ODP_CIPHER_ALG_AES_CBC, + ODP_CIPHER_ALG_AES_CTR, + ODP_CIPHER_ALG_AES_ECB, + ODP_CIPHER_ALG_AES_CFB128, + ODP_CIPHER_ALG_AES_XTS, + ODP_CIPHER_ALG_KASUMI_F8, + ODP_CIPHER_ALG_SNOW3G_UEA2, + ODP_CIPHER_ALG_AES_EEA2, + ODP_CIPHER_ALG_ZUC_EEA3, +}; + +/* + * Authentication algorithms and hashes that use auth_range + * parameter. AEAD algorithms are excluded. + */ +static odp_auth_alg_t auth_algs[] = { + ODP_AUTH_ALG_MD5_HMAC, + ODP_AUTH_ALG_SHA1_HMAC, + ODP_AUTH_ALG_SHA224_HMAC, + ODP_AUTH_ALG_SHA256_HMAC, + ODP_AUTH_ALG_SHA384_HMAC, + ODP_AUTH_ALG_SHA512_HMAC, + ODP_AUTH_ALG_AES_GMAC, + ODP_AUTH_ALG_AES_CMAC, + ODP_AUTH_ALG_AES_XCBC_MAC, + ODP_AUTH_ALG_KASUMI_F9, + ODP_AUTH_ALG_SNOW3G_UIA2, + ODP_AUTH_ALG_AES_EIA2, + ODP_AUTH_ALG_ZUC_EIA3, + ODP_AUTH_ALG_MD5, + ODP_AUTH_ALG_SHA1, + ODP_AUTH_ALG_SHA224, + ODP_AUTH_ALG_SHA256, + ODP_AUTH_ALG_SHA384, + ODP_AUTH_ALG_SHA512, +}; + static void test_auth_hashes_in_auth_range(void) { - /* - * Authentication algorithms and hashes that use auth_range - * parameter. AEAD algorithms are excluded. - */ - static odp_auth_alg_t auth_algs[] = { - ODP_AUTH_ALG_NULL, - ODP_AUTH_ALG_MD5_HMAC, - ODP_AUTH_ALG_SHA1_HMAC, - ODP_AUTH_ALG_SHA224_HMAC, - ODP_AUTH_ALG_SHA256_HMAC, - ODP_AUTH_ALG_SHA384_HMAC, - ODP_AUTH_ALG_SHA512_HMAC, - ODP_AUTH_ALG_AES_GMAC, - ODP_AUTH_ALG_AES_CMAC, - ODP_AUTH_ALG_AES_XCBC_MAC, - ODP_AUTH_ALG_KASUMI_F9, - ODP_AUTH_ALG_SNOW3G_UIA2, - ODP_AUTH_ALG_AES_EIA2, - ODP_AUTH_ALG_ZUC_EIA3, - ODP_AUTH_ALG_MD5, - ODP_AUTH_ALG_SHA1, - ODP_AUTH_ALG_SHA224, - ODP_AUTH_ALG_SHA256, - ODP_AUTH_ALG_SHA384, - ODP_AUTH_ALG_SHA512, - }; - for (size_t n = 0; n < ARRAY_SIZE(auth_algs); n++) { odp_auth_alg_t auth = auth_algs[n]; int num; @@ -1624,11 +1689,547 @@ static void test_auth_hashes_in_auth_range(void) num = odp_crypto_auth_capability(auth, capa, num); - for (int i = 0; i < num; i++) - test_auth_hash_in_auth_range(auth, &capa[i]); + for (int i = 0; i < num; i++) { + test_auth_hash_in_auth_range(auth, &capa[i], AUTH_PLAINTEXT); + test_auth_hash_in_auth_range(auth, &capa[i], AUTH_CIPHERTEXT); + } } } +/* + * Encode ref->plaintext and save result in ref->ciphertext. + */ +static int crypto_encode_ref(crypto_test_reference_t *ref, + odp_cipher_alg_t cipher, + odp_auth_alg_t auth, + odp_packet_data_range_t cipher_range, + odp_packet_data_range_t auth_range, + uint32_t hash_result_offset) +{ + odp_packet_data_range_t zero_range = {.offset = 0, .length = 0}; + odp_packet_t pkt; + int rc; + odp_crypto_session_t session; + odp_bool_t ok; + + pkt = odp_packet_alloc(suite_context.pool, ref->length); + CU_ASSERT_FATAL(pkt != ODP_PACKET_INVALID); + + rc = odp_packet_copy_from_mem(pkt, 0, ref->length, ref->plaintext); + CU_ASSERT(rc == 0); + + session = session_create(ODP_CRYPTO_OP_ENCODE, + ODP_CRYPTO_OP_TYPE_LEGACY, + cipher, + auth, + AUTH_PLAINTEXT, + ref, + HASH_OVERLAP); + + if (session == ODP_CRYPTO_SESSION_INVALID) { + odp_packet_free(pkt); + return 1; + } + + if (cipher == ODP_CIPHER_ALG_NULL) + cipher_range = zero_range; + if (auth == ODP_AUTH_ALG_NULL) { + auth_range = zero_range; + hash_result_offset = 0; + } + + CU_ASSERT_FATAL(hash_result_offset + ref->digest_length <= ref->length); + + rc = crypto_op(pkt, &pkt, &ok, session, ODP_CRYPTO_OP_TYPE_LEGACY, 0, + ref->cipher_iv, ref->auth_iv, + &cipher_range, &auth_range, + ref->aad, hash_result_offset); + CU_ASSERT(rc == 0); + CU_ASSERT(ok); + + rc = odp_crypto_session_destroy(session); + CU_ASSERT(rc == 0); + + rc = odp_packet_copy_to_mem(pkt, 0, ref->length, ref->ciphertext); + CU_ASSERT(rc == 0); + + odp_packet_free(pkt); + return 0; +} + +typedef struct crypto_suite_t { + odp_cipher_alg_t cipher; + odp_auth_alg_t auth; + alg_order_t order; + const odp_crypto_cipher_capability_t *cipher_capa; + const odp_crypto_auth_capability_t *auth_capa; +} crypto_suite_t; + +/* + * Create test reference for combined auth & cipher by doing authentication + * and ciphering through separate ODP crypto operations. + */ +static int create_combined_ref(const crypto_suite_t *suite, + crypto_test_reference_t *ref, + odp_packet_data_range_t *cipher_range, + odp_packet_data_range_t *auth_range, + uint32_t digest_offset) +{ + uint32_t total_len; + int rc; + crypto_test_reference_t ref_cipher_only; + crypto_test_reference_t ref_auth_only; + crypto_test_reference_t *first_ref, *second_ref; + odp_auth_alg_t first_auth, second_auth; + odp_cipher_alg_t first_cipher, second_cipher; + + total_len = cipher_range->offset + cipher_range->length; + if (auth_range->offset + auth_range->length > total_len) + total_len = auth_range->offset + auth_range->length; + if (digest_offset + suite->auth_capa->digest_len > total_len) + total_len = digest_offset + suite->auth_capa->digest_len; + + ref->cipher_key_length = suite->cipher_capa->key_len; + ref->cipher_iv_length = suite->cipher_capa->iv_len; + ref->auth_key_length = suite->auth_capa->key_len; + ref->auth_iv_length = suite->auth_capa->iv_len; + ref->digest_length = suite->auth_capa->digest_len; + ref->aad_length = 0; + ref->is_length_in_bits = false; + ref->length = total_len; + + if (suite->cipher_capa->bit_mode) { + cipher_range->offset *= 8; + cipher_range->length *= 8; + } + if (suite->auth_capa->bit_mode) { + auth_range->offset *= 8; + auth_range->length *= 8; + } + + if (ref->auth_key_length > MAX_KEY_LEN || + ref->auth_iv_length > MAX_IV_LEN || + total_len > MAX_DATA_LEN || + digest_offset + ref->digest_length > MAX_DATA_LEN) + CU_FAIL_FATAL("Internal error\n"); + + fill_with_pattern(ref->cipher_key, ref->cipher_key_length); + fill_with_pattern(ref->cipher_iv, ref->cipher_iv_length); + fill_with_pattern(ref->auth_key, ref->auth_key_length); + fill_with_pattern(ref->auth_iv, ref->auth_iv_length); + fill_with_pattern(ref->plaintext, ref->length); + memset(ref->plaintext + digest_offset, 0, ref->digest_length); + + ref_cipher_only = *ref; + ref_cipher_only.auth_key_length = 0; + ref_cipher_only.auth_iv_length = 0; + ref_cipher_only.aad_length = 0; + ref_cipher_only.digest_length = 0; + + ref_auth_only = *ref; + ref_auth_only.cipher_key_length = 0; + ref_auth_only.cipher_iv_length = 0; + + if (suite->order == AUTH_CIPHERTEXT) { + first_ref = &ref_cipher_only; + first_cipher = suite->cipher; + first_auth = ODP_AUTH_ALG_NULL; + second_ref = &ref_auth_only; + second_cipher = ODP_CIPHER_ALG_NULL; + second_auth = suite->auth; + } else { + first_ref = &ref_auth_only; + first_cipher = ODP_CIPHER_ALG_NULL; + first_auth = suite->auth; + second_ref = &ref_cipher_only; + second_cipher = suite->cipher; + second_auth = ODP_AUTH_ALG_NULL; + } + rc = crypto_encode_ref(first_ref, + first_cipher, first_auth, + *cipher_range, *auth_range, + digest_offset); + if (rc) + return 1; + memcpy(second_ref->plaintext, first_ref->ciphertext, ref->length); + rc = crypto_encode_ref(second_ref, + second_cipher, second_auth, + *cipher_range, *auth_range, + digest_offset); + if (rc) + return 1; + memcpy(ref->ciphertext, second_ref->ciphertext, ref->length); + /* + * These may be encrypted bytes, but that is what alg_test wants if + * the digest is encrypted in the input packet. + */ + memcpy(ref->digest, second_ref->ciphertext + digest_offset, ref->digest_length); + + return 0; +} + +/* + * Return cipher range that is at least min_len bytes long, multiple of the + * block size and at least 3 blocks. + */ +static uint32_t get_cipher_range_len(uint32_t min_len) +{ +#define MAX_BLOCK_SIZE 16 + uint32_t bs = MAX_BLOCK_SIZE; + uint32_t len = 3 * bs; + + if (min_len > len) + len = ((min_len + bs - 1) / bs) * bs; + return len; +} + +typedef enum range_overlap_t { + SEPARATE_AUTH_AND_CIPHER_RANGES, + SAME_AUTH_AND_CIPHER_RANGE, + RANGES_PARTIALLY_OVERLAP, + AUTH_RANGE_IN_CIPHER_RANGE, + CIPHER_RANGE_IN_AUTH_RANGE, +} range_overlap_t; +#define NUM_RANGE_OVERLAPS 5 + +typedef enum hash_location_t { + HASH_SEPARATE, + HASH_IN_AUTH_RANGE_ONLY, + HASH_IN_CIPHER_RANGE_ONLY, + HASH_IN_AUTH_AND_CIPHER_RANGE, +} hash_location_t; +#define NUM_HASH_LOCATIONS 4 + +static int make_byte_ranges(range_overlap_t overlap, + hash_location_t hash_location, + uint32_t hash_len, + odp_packet_data_range_t *cipher_range, + odp_packet_data_range_t *auth_range, + uint32_t *digest_offset) +{ + const uint32_t padding = 5; /* padding between parts, could also be zero */ + const uint32_t nonzero_len = 3; + uint32_t c_offs = 0, c_len = 0, a_offs = 0, a_len = 0, digest_offs = 0; + + switch (overlap) { + case SEPARATE_AUTH_AND_CIPHER_RANGES: + switch (hash_location) { + case HASH_SEPARATE: + /* |cccc_aaaa_dd| */ + c_offs = 0; + c_len = get_cipher_range_len(nonzero_len); + a_offs = c_offs + c_len + padding; + a_len = nonzero_len; + digest_offs = a_offs + a_len + padding; + break; + case HASH_IN_AUTH_RANGE_ONLY: + /* + * |cccc_aaaa| + * | _dd_| + */ + c_offs = 0; + c_len = get_cipher_range_len(nonzero_len); + a_offs = c_offs + c_len + padding; + a_len = hash_len + 2 * padding; + digest_offs = a_offs + padding; + break; + case HASH_IN_CIPHER_RANGE_ONLY: + /* + * |cccc_aaaa| + * |_dd_ | + */ + c_offs = 0; + c_len = get_cipher_range_len(hash_len + 2 * padding); + a_offs = c_offs + c_len + padding; + a_len = nonzero_len; + digest_offs = c_offs + padding; + break; + case HASH_IN_AUTH_AND_CIPHER_RANGE: + /* not possible when ranges are separate */ + return 1; + } + break; + case SAME_AUTH_AND_CIPHER_RANGE: + c_offs = 0; + a_offs = 0; + switch (hash_location) { + case HASH_SEPARATE: + /* + * |cccc_dd| + * |aaaa | + */ + c_len = get_cipher_range_len(nonzero_len); + a_len = c_len; + digest_offs = c_len + padding; + break; + case HASH_IN_AUTH_RANGE_ONLY: + case HASH_IN_CIPHER_RANGE_ONLY: + /* not possible when ranges are the same */ + return 1; + case HASH_IN_AUTH_AND_CIPHER_RANGE: + /* + * |cccc| + * |aaaa| + * |_dd_| + */ + c_len = get_cipher_range_len(hash_len + 2 * padding); + a_len = c_len; + digest_offs = padding; + break; + } + break; + case RANGES_PARTIALLY_OVERLAP: + a_offs = 0; + switch (hash_location) { + case HASH_SEPARATE: + /* + * |aaaa | + * | cccc_dd| + */ + a_len = 2 * nonzero_len; + c_offs = nonzero_len; + c_len = get_cipher_range_len(a_len); + digest_offs = c_offs + c_len + padding; + break; + case HASH_IN_AUTH_RANGE_ONLY: + /* + * |aaaaa | + * |_dd_ccc| + */ + digest_offs = padding; + a_len = hash_len + 2 * padding + nonzero_len; + c_offs = hash_len + 2 * padding; + c_len = get_cipher_range_len(2 * nonzero_len); + break; + case HASH_IN_CIPHER_RANGE_ONLY: + /* PDCP case when AUTH_PLAINTEXT */ + /* + * |aaaadd| + * | ccccc| + */ + c_offs = nonzero_len; + c_len = get_cipher_range_len(nonzero_len + hash_len); + a_len = nonzero_len + c_len - hash_len; + digest_offs = c_offs + c_len - hash_len; + break; + case HASH_IN_AUTH_AND_CIPHER_RANGE: + /* + * |aaaaaa | + * | cccccc| + * |¨_dd_ | + */ + c_offs = nonzero_len; + c_len = get_cipher_range_len(hash_len + 2 * padding + nonzero_len); + a_len = c_offs + hash_len + 2 * padding; + digest_offs = c_offs + padding; + break; + } + break; + case AUTH_RANGE_IN_CIPHER_RANGE: + c_offs = 0; + a_offs = nonzero_len; + switch (hash_location) { + case HASH_SEPARATE: + /* + * |cccc_dd| + * | aa_ | + */ + a_len = nonzero_len; + c_len = get_cipher_range_len(a_offs + a_len + padding); + digest_offs = c_len + padding; + break; + case HASH_IN_AUTH_RANGE_ONLY: + /* not possible since auth range is in cipher range */ + return 1; + case HASH_IN_CIPHER_RANGE_ONLY: + /* + * |ccccccc| + * | aa_dd_| + */ + a_len = nonzero_len; + digest_offs = a_offs + a_len + padding; + c_len = get_cipher_range_len(digest_offs + hash_len + padding); + break; + case HASH_IN_AUTH_AND_CIPHER_RANGE: + /* + * |cccccc| + * | aaaa_| + * | _dd_ | + */ + a_len = /**/ hash_len + 2 * padding; + c_len = get_cipher_range_len(a_offs + a_len + padding); + digest_offs = a_offs + /**/ padding; + break; + } + break; + case CIPHER_RANGE_IN_AUTH_RANGE: + a_offs = 0; + c_offs = nonzero_len; + switch (hash_location) { + case HASH_SEPARATE: + /* + * |aaaa_dd| + * | cc_ | + */ + c_len = get_cipher_range_len(nonzero_len); + a_len = c_offs + c_len + padding; + digest_offs = a_len + padding; + break; + case HASH_IN_AUTH_RANGE_ONLY: + /* + * |aaaaaaa| + * | cc_dd_| + */ + c_len = get_cipher_range_len(nonzero_len); + digest_offs = c_offs + c_len + padding; + a_len = digest_offs + hash_len + padding; + break; + case HASH_IN_CIPHER_RANGE_ONLY: + /* not possible since cipher range is in auth range */ + return 1; + case HASH_IN_AUTH_AND_CIPHER_RANGE: + /* + * |aaaaaa| + * | cccc_| + * | _dd_ | + */ + c_len = get_cipher_range_len(hash_len + 2 * padding); + a_len = c_offs + c_len + padding; + digest_offs = c_offs + padding; + break; + } + break; + } + cipher_range->offset = c_offs; + cipher_range->length = c_len; + auth_range->offset = a_offs; + auth_range->length = a_len; + *digest_offset = digest_offs; + return 0; +} + +static void test_combo(const crypto_suite_t *suite, + range_overlap_t overlap, + hash_location_t location) +{ + int rc; + + odp_packet_data_range_t cipher_range = {0, 0}; + odp_packet_data_range_t auth_range = {0, 0}; + uint32_t digest_offset = 0; + crypto_test_reference_t ref; + + rc = make_byte_ranges(overlap, + location, + suite->auth_capa->digest_len, + &cipher_range, + &auth_range, + &digest_offset); + if (rc) + return; + + rc = create_combined_ref(suite, &ref, + &cipher_range, &auth_range, + digest_offset); + if (rc) + return; + + alg_test(ODP_CRYPTO_OP_ENCODE, + suite->cipher, + suite->auth, + suite->order, + &ref, + cipher_range, auth_range, + digest_offset, + suite->cipher_capa->bit_mode, + suite->auth_capa->bit_mode); + + alg_test(ODP_CRYPTO_OP_DECODE, + suite->cipher, + suite->auth, + suite->order, + &ref, + cipher_range, auth_range, + digest_offset, + suite->cipher_capa->bit_mode, + suite->auth_capa->bit_mode); +} + +/* Iterate and test different cipher/auth range and hash locations */ +static void test_combo_ranges(const crypto_suite_t *suite) +{ + if (!full_test && suite->auth_capa->digest_len % 4 != 0) + return; + + for (int overlap = 0; overlap < NUM_RANGE_OVERLAPS; overlap++) + for (int location = 0; location < NUM_HASH_LOCATIONS; location++) { + if (suite->order == AUTH_CIPHERTEXT && + (location == HASH_IN_CIPHER_RANGE_ONLY || + location == HASH_IN_AUTH_AND_CIPHER_RANGE)) { + /* + * This combination Ãs not valid since + * the generated hash would overwrite some + * ciphertext, preventing decryption. + */ + continue; + } + test_combo(suite, overlap, location); + } +} + +/* Iterate and test all variants (key sizes etc) of an alg combo */ +static void test_combo_variants(odp_cipher_alg_t cipher, odp_auth_alg_t auth) +{ + int num, num_ciphers, num_auths; + + /* ODP API says AES-GMAC can be combined with the null cipher only */ + if (auth == ODP_AUTH_ALG_AES_GMAC && + cipher != ODP_CIPHER_ALG_NULL) + return; + + if (check_alg_support(cipher, auth) == ODP_TEST_INACTIVE) + return; + + printf(" %s, %s\n", + cipher_alg_name(cipher), + auth_alg_name(auth)); + + num_ciphers = odp_crypto_cipher_capability(cipher, NULL, 0); + num_auths = odp_crypto_auth_capability(auth, NULL, 0); + CU_ASSERT_FATAL(num_ciphers > 0); + CU_ASSERT_FATAL(num_auths > 0); + + odp_crypto_cipher_capability_t cipher_capa[num_ciphers]; + odp_crypto_auth_capability_t auth_capa[num_auths]; + + num = odp_crypto_cipher_capability(cipher, cipher_capa, num_ciphers); + CU_ASSERT(num == num_ciphers); + num = odp_crypto_auth_capability(auth, auth_capa, num_auths); + CU_ASSERT(num == num_auths); + + combo_warning_shown = 0; + + for (int n = 0; n < num_ciphers; n++) + for (int i = 0; i < num_auths; i++) { + crypto_suite_t suite = {.cipher = cipher, + .auth = auth, + .cipher_capa = &cipher_capa[n], + .auth_capa = &auth_capa[i]}; + suite.order = AUTH_PLAINTEXT; + test_combo_ranges(&suite); + suite.order = AUTH_CIPHERTEXT; + test_combo_ranges(&suite); + } +} + +static void test_all_combinations(void) +{ + printf("\n"); + for (size_t n = 0; n < ARRAY_SIZE(cipher_algs); n++) + for (size_t i = 0; i < ARRAY_SIZE(auth_algs); i++) + test_combo_variants(cipher_algs[n], auth_algs[i]); +} + static int check_alg_null(void) { return check_alg_support(ODP_CIPHER_ALG_NULL, ODP_AUTH_ALG_NULL); @@ -2443,64 +3044,8 @@ static odp_event_t plain_compl_queue_deq(void) return odp_queue_deq(suite_context.queue); } -#if ODP_DEPRECATED_API -static int crypto_suite_sync_init(void) -{ - suite_context.pool = odp_pool_lookup("packet_pool"); - if (suite_context.pool == ODP_POOL_INVALID) - return -1; - - suite_context.queue = ODP_QUEUE_INVALID; - suite_context.pref_mode = ODP_CRYPTO_SYNC; - return 0; -} - -static int crypto_suite_async_plain_init(void) -{ - odp_queue_t out_queue; - - suite_context.pool = odp_pool_lookup("packet_pool"); - if (suite_context.pool == ODP_POOL_INVALID) - return -1; - - out_queue = plain_compl_queue_create(); - if (ODP_QUEUE_INVALID == out_queue) { - fprintf(stderr, "Crypto outq creation failed.\n"); - return -1; - } - suite_context.queue = out_queue; - suite_context.q_type = ODP_QUEUE_TYPE_PLAIN; - suite_context.compl_queue_deq = plain_compl_queue_deq; - suite_context.pref_mode = ODP_CRYPTO_ASYNC; - - return 0; -} - -static int crypto_suite_async_sched_init(void) -{ - odp_queue_t out_queue; - - suite_context.pool = odp_pool_lookup("packet_pool"); - if (suite_context.pool == ODP_POOL_INVALID) - return -1; - - out_queue = sched_compl_queue_create(); - if (ODP_QUEUE_INVALID == out_queue) { - fprintf(stderr, "Crypto outq creation failed.\n"); - return -1; - } - suite_context.queue = out_queue; - suite_context.q_type = ODP_QUEUE_TYPE_SCHED; - suite_context.compl_queue_deq = sched_compl_queue_deq; - suite_context.pref_mode = ODP_CRYPTO_ASYNC; - - return 0; -} -#endif - static int crypto_suite_packet_sync_init(void) { - suite_context.packet = true; suite_context.op_mode = ODP_CRYPTO_SYNC; suite_context.pool = odp_pool_lookup("packet_pool"); @@ -2515,7 +3060,6 @@ static int crypto_suite_packet_async_plain_init(void) { odp_queue_t out_queue; - suite_context.packet = true; suite_context.op_mode = ODP_CRYPTO_ASYNC; suite_context.pool = odp_pool_lookup("packet_pool"); @@ -2538,7 +3082,6 @@ static int crypto_suite_packet_async_sched_init(void) { odp_queue_t out_queue; - suite_context.packet = true; suite_context.op_mode = ODP_CRYPTO_ASYNC; suite_context.pool = odp_pool_lookup("packet_pool"); @@ -2709,18 +3252,11 @@ odp_testinfo_t crypto_suite[] = { ODP_TEST_INFO_CONDITIONAL(crypto_test_check_alg_sha512, check_alg_sha512), ODP_TEST_INFO(test_auth_hashes_in_auth_range), + ODP_TEST_INFO(test_all_combinations), ODP_TEST_INFO_NULL, }; odp_suiteinfo_t crypto_suites[] = { -#if ODP_DEPRECATED_API - {"odp_crypto_sync_inp", crypto_suite_sync_init, - NULL, crypto_suite}, - {"odp_crypto_async_plain_inp", crypto_suite_async_plain_init, - crypto_suite_term, crypto_suite}, - {"odp_crypto_async_sched_inp", crypto_suite_async_sched_init, - crypto_suite_term, crypto_suite}, -#endif {"odp_crypto_packet_sync_inp", crypto_suite_packet_sync_init, NULL, crypto_suite}, {"odp_crypto_packet_async_plain_inp", @@ -2834,6 +3370,11 @@ static int crypto_term(odp_instance_t inst) int main(int argc, char *argv[]) { int ret; + char *env = getenv("FULL_TEST"); + + if (env && strcmp(env, "0")) + full_test = 1; + printf("Test mode: %s\n", full_test ? "full" : "partial"); /* parse common options: */ if (odp_cunit_parse_options(argc, argv)) diff --git a/test/validation/api/dma/dma.c b/test/validation/api/dma/dma.c index f3e967193..ad42c9bb3 100644 --- a/test/validation/api/dma/dma.c +++ b/test/validation/api/dma/dma.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2021-2022, Nokia +/* Copyright (c) 2021-2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -10,9 +10,8 @@ #define COMPL_POOL_NAME "DMA compl pool" -#define SHM_SIZE (1024 * 1024) +#define MIN_SEG_LEN 1024 #define SHM_ALIGN ODP_CACHE_LINE_SIZE -#define NUM_COMPL 10 #define RETRIES 5 #define TIMEOUT 5 #define OFFSET 10 @@ -21,6 +20,8 @@ #define RESULT 1 #define USER_DATA 0xdeadbeef +#define MIN(a, b) (a < b ? a : b) + typedef struct global_t { odp_dma_capability_t dma_capa; odp_shm_t shm; @@ -47,7 +48,7 @@ static int dma_suite_init(void) odp_dma_pool_param_t dma_pool_param; odp_pool_capability_t pool_capa; odp_queue_param_t queue_param; - uint32_t pkt_len; + uint32_t shm_len, pkt_len; void *addr; memset(&global, 0, sizeof(global_t)); @@ -67,7 +68,8 @@ static int dma_suite_init(void) return 0; } - shm = odp_shm_reserve("DMA test", SHM_SIZE, SHM_ALIGN, 0); + shm_len = MIN_SEG_LEN * global.dma_capa.max_segs * global.dma_capa.max_transfers; + shm = odp_shm_reserve("DMA test", shm_len, SHM_ALIGN, 0); if (shm == ODP_SHM_INVALID) { ODPH_ERR("SHM reserve failed\n"); @@ -82,7 +84,7 @@ static int dma_suite_init(void) } global.shm = shm; - global.data_size = SHM_SIZE / 2; + global.data_size = shm_len / 2; global.src_addr = addr; global.dst_addr = (uint8_t *)global.src_addr + global.data_size; global.len = global.data_size - OFFSET - TRAILER; @@ -96,6 +98,7 @@ static int dma_suite_init(void) if (pkt_len == 0) pkt_len = 4000; + pkt_len = MIN(pkt_len, global.dma_capa.max_seg_len); odp_pool_param_init(&pool_param); pool_param.type = ODP_POOL_PACKET; pool_param.pkt.num = 4; @@ -124,13 +127,13 @@ static int dma_suite_init(void) } if (global.dma_capa.compl_mode_mask & ODP_DMA_COMPL_EVENT) { - if (global.dma_capa.pool.max_num < NUM_COMPL) { + if (global.dma_capa.pool.max_num < global.dma_capa.max_transfers) { ODPH_ERR("Too small DMA compl pool %u\n", global.dma_capa.pool.max_num); return -1; } odp_dma_pool_param_init(&dma_pool_param); - dma_pool_param.num = NUM_COMPL; + dma_pool_param.num = global.dma_capa.max_transfers; global.cache_size = dma_pool_param.cache_size; global.compl_pool = odp_dma_pool_create(COMPL_POOL_NAME, &dma_pool_param); @@ -178,7 +181,7 @@ static void test_dma_capability(void) odp_dma_capability_t capa; memset(&capa, 0, sizeof(odp_dma_capability_t)); - CU_ASSERT(odp_dma_capability(&capa) == 0); + CU_ASSERT_FATAL(odp_dma_capability(&capa) == 0); if (capa.max_sessions == 0) return; @@ -193,8 +196,12 @@ static void test_dma_capability(void) CU_ASSERT(capa.compl_mode_mask & ODP_DMA_COMPL_SYNC); if (capa.compl_mode_mask & ODP_DMA_COMPL_EVENT) { + odp_pool_capability_t pool_capa; + + CU_ASSERT_FATAL(odp_pool_capability(&pool_capa) == 0); + CU_ASSERT(capa.queue_type_sched || capa.queue_type_plain); - CU_ASSERT(capa.pool.max_pools > 0); + CU_ASSERT(capa.pool.max_pools > 0 && capa.pool.max_pools <= pool_capa.max_pools); CU_ASSERT(capa.pool.max_num > 0); CU_ASSERT(capa.pool.min_cache_size <= capa.pool.max_cache_size); } @@ -266,9 +273,10 @@ static void test_dma_compl_pool(void) { odp_pool_t pool; odp_pool_info_t pool_info; - odp_dma_compl_t compl; + odp_dma_compl_t compl[global.dma_capa.max_transfers]; uint64_t u64; int ret; + uint32_t i, j; const char *name = COMPL_POOL_NAME; CU_ASSERT_FATAL(global.compl_pool != ODP_POOL_INVALID); @@ -282,17 +290,24 @@ static void test_dma_compl_pool(void) CU_ASSERT(strcmp(pool_info.name, name) == 0); CU_ASSERT(pool_info.pool_ext == 0); CU_ASSERT(pool_info.type == ODP_POOL_DMA_COMPL); - CU_ASSERT(pool_info.dma_pool_param.num == NUM_COMPL); + CU_ASSERT(pool_info.dma_pool_param.num == global.dma_capa.max_transfers); CU_ASSERT(pool_info.dma_pool_param.cache_size == global.cache_size); - compl = odp_dma_compl_alloc(global.compl_pool); + for (i = 0; i < global.dma_capa.max_transfers; i++) { + compl[i] = odp_dma_compl_alloc(global.compl_pool); + + u64 = odp_dma_compl_to_u64(compl[i]); + CU_ASSERT(u64 != odp_dma_compl_to_u64(ODP_DMA_COMPL_INVALID)); + + if (compl[i] == ODP_DMA_COMPL_INVALID) + break; - u64 = odp_dma_compl_to_u64(compl); - CU_ASSERT(u64 != odp_dma_compl_to_u64(ODP_DMA_COMPL_INVALID)); - printf("\n DMA compl handle: 0x%" PRIx64 "\n", u64); - odp_dma_compl_print(compl); + printf("\n DMA compl handle: 0x%" PRIx64 "\n", u64); + odp_dma_compl_print(compl[i]); + } - odp_dma_compl_free(compl); + for (j = 0; j < i; j++) + odp_dma_compl_free(compl[j]); } static void test_dma_compl_pool_same_name(void) @@ -307,7 +322,7 @@ static void test_dma_compl_pool_same_name(void) CU_ASSERT(pool == pool_a); odp_dma_pool_param_init(&dma_pool_param); - dma_pool_param.num = NUM_COMPL; + dma_pool_param.num = global.dma_capa.max_transfers; /* Second pool with the same name */ pool_b = odp_dma_pool_create(name, &dma_pool_param); @@ -319,6 +334,36 @@ static void test_dma_compl_pool_same_name(void) CU_ASSERT_FATAL(odp_pool_destroy(pool_b) == 0); } +static void test_dma_compl_pool_max_pools(void) +{ + odp_dma_pool_param_t dma_pool_param; + /* Max pools minus the ones already created in global init */ + uint32_t num = global.dma_capa.pool.max_pools - 2, i, j; + odp_pool_t pools[num]; + int ret; + + odp_dma_pool_param_init(&dma_pool_param); + dma_pool_param.num = global.dma_capa.max_transfers; + + for (i = 0; i < num; i++) { + pools[i] = odp_dma_pool_create(NULL, &dma_pool_param); + CU_ASSERT(pools[i] != ODP_POOL_INVALID); + + if (pools[i] == ODP_POOL_INVALID) { + ODPH_ERR("DMA completion pool create failed: %u / %u\n", i, num); + break; + } + } + + for (j = 0; j < i; j++) { + ret = odp_pool_destroy(pools[j]); + CU_ASSERT(ret == 0); + + if (ret == -1) + ODPH_ERR("DMA completion pool destroy failed: %u / %u\n", j, i); + } +} + static void init_source(uint8_t *src, uint32_t len) { uint32_t i; @@ -380,126 +425,132 @@ static int do_transfer(odp_dma_t dma, const odp_dma_transfer_param_t *trs_param, return ret; } -static int do_transfer_async(odp_dma_t dma, const odp_dma_transfer_param_t *trs_param, +static int do_transfer_async(odp_dma_t dma, odp_dma_transfer_param_t *trs_param, odp_dma_compl_mode_t compl_mode, int multi) { - odp_dma_compl_param_t compl_param; + int num_trs = multi ? multi : 1; + odp_dma_compl_param_t compl_param[num_trs]; + const odp_dma_compl_param_t *compl_ptr[num_trs]; + const odp_dma_transfer_param_t *trs_ptr[num_trs]; odp_event_t ev; odp_dma_compl_t compl; - int i, ret, done; + int i, j, ret, done; uint32_t user_data = USER_DATA; odp_dma_result_t result; - odp_dma_transfer_id_t transfer_id = ODP_DMA_TRANSFER_ID_INVALID; uint64_t wait_ns = 500 * ODP_TIME_MSEC_IN_NS; uint64_t sched_wait = odp_schedule_wait_time(wait_ns); void *user_ptr = &user_data; - odp_dma_compl_param_init(&compl_param); - compl_param.compl_mode = compl_mode; - - if (compl_mode == ODP_DMA_COMPL_EVENT) { - compl = odp_dma_compl_alloc(global.compl_pool); - - CU_ASSERT(compl != ODP_DMA_COMPL_INVALID); - if (compl == ODP_DMA_COMPL_INVALID) + for (i = 0; i < num_trs; i++) { + odp_dma_compl_param_init(&compl_param[i]); + compl_param[i].compl_mode = compl_mode; + + if (compl_mode == ODP_DMA_COMPL_EVENT) { + compl = odp_dma_compl_alloc(global.compl_pool); + + CU_ASSERT(compl != ODP_DMA_COMPL_INVALID); + if (compl == ODP_DMA_COMPL_INVALID) + return -1; + + compl_param[i].event = odp_dma_compl_to_event(compl); + compl_param[i].queue = global.queue; + } else if (compl_mode == ODP_DMA_COMPL_POLL) { + compl_param[i].transfer_id = odp_dma_transfer_id_alloc(dma); + + CU_ASSERT(compl_param[i].transfer_id != ODP_DMA_TRANSFER_ID_INVALID); + if (compl_param[i].transfer_id == ODP_DMA_TRANSFER_ID_INVALID) + return -1; + } else if (compl_mode != ODP_DMA_COMPL_NONE) { + ODPH_ERR("Wrong compl mode: %u\n", compl_mode); return -1; + } - compl_param.event = odp_dma_compl_to_event(compl); - compl_param.queue = global.queue; - } else if (compl_mode == ODP_DMA_COMPL_POLL) { - transfer_id = odp_dma_transfer_id_alloc(dma); - - CU_ASSERT(transfer_id != ODP_DMA_TRANSFER_ID_INVALID); - if (transfer_id == ODP_DMA_TRANSFER_ID_INVALID) - return -1; + compl_param[i].user_ptr = user_ptr; - compl_param.transfer_id = transfer_id; - } else if (compl_mode != ODP_DMA_COMPL_NONE) { - ODPH_ERR("Wrong compl mode: %u\n", compl_mode); - return -1; + if (multi) { + trs_ptr[i] = &trs_param[i]; + compl_ptr[i] = &compl_param[i]; + } } for (i = 0; i < RETRIES; i++) { - compl_param.user_ptr = user_ptr; - - if (multi) { - const odp_dma_compl_param_t *compl_ptr[1] = {&compl_param}; - const odp_dma_transfer_param_t *trs_ptr[1] = {trs_param}; - - ret = odp_dma_transfer_start_multi(dma, trs_ptr, compl_ptr, 1); - } else { - ret = odp_dma_transfer_start(dma, trs_param, &compl_param); - } + if (multi) + ret = odp_dma_transfer_start_multi(dma, trs_ptr, compl_ptr, num_trs); + else + ret = odp_dma_transfer_start(dma, trs_param, compl_param); if (ret) break; } - CU_ASSERT(ret == 1); + CU_ASSERT(ret == num_trs); if (ret < 1) return ret; - memset(&result, 0, sizeof(odp_dma_result_t)); + for (i = 0; i < ret; i++) { + memset(&result, 0, sizeof(odp_dma_result_t)); - if (compl_mode == ODP_DMA_COMPL_POLL) { - for (i = 0; i < TIMEOUT; i++) { - done = odp_dma_transfer_done(dma, transfer_id, &result); - if (done) - break; + if (compl_mode == ODP_DMA_COMPL_POLL) { + for (j = 0; j < TIMEOUT; j++) { + done = odp_dma_transfer_done(dma, compl_param[i].transfer_id, + &result); + if (done) + break; - odp_time_wait_ns(wait_ns); - } + odp_time_wait_ns(wait_ns); + } - CU_ASSERT(done == 1); - CU_ASSERT(result.success); - CU_ASSERT(result.user_ptr == user_ptr); - CU_ASSERT(user_data == USER_DATA); + CU_ASSERT(done == 1); + CU_ASSERT(result.success); + CU_ASSERT(result.user_ptr == user_ptr); + CU_ASSERT(user_data == USER_DATA); - odp_dma_transfer_id_free(dma, transfer_id); + odp_dma_transfer_id_free(dma, compl_param[i].transfer_id); - return done; + return done; - } else if (compl_mode == ODP_DMA_COMPL_EVENT) { - odp_queue_t from = ODP_QUEUE_INVALID; + } else if (compl_mode == ODP_DMA_COMPL_EVENT) { + odp_queue_t from = ODP_QUEUE_INVALID; - for (i = 0; i < TIMEOUT; i++) { - ev = odp_schedule(&from, sched_wait); - if (ev != ODP_EVENT_INVALID) - break; - } + for (j = 0; j < TIMEOUT; j++) { + ev = odp_schedule(&from, sched_wait); + if (ev != ODP_EVENT_INVALID) + break; + } - CU_ASSERT(ev != ODP_EVENT_INVALID); - if (ev == ODP_EVENT_INVALID) - return -1; + CU_ASSERT(ev != ODP_EVENT_INVALID); + if (ev == ODP_EVENT_INVALID) + return -1; - CU_ASSERT(from == global.queue); - CU_ASSERT(odp_event_type(ev) == ODP_EVENT_DMA_COMPL); + CU_ASSERT(from == global.queue); + CU_ASSERT(odp_event_type(ev) == ODP_EVENT_DMA_COMPL); - compl = odp_dma_compl_from_event(ev); - CU_ASSERT(compl != ODP_DMA_COMPL_INVALID); + compl = odp_dma_compl_from_event(ev); + CU_ASSERT(compl != ODP_DMA_COMPL_INVALID); - CU_ASSERT(odp_dma_compl_result(compl, &result) == 0); - CU_ASSERT(result.success); - CU_ASSERT(result.user_ptr == user_ptr); - CU_ASSERT(user_data == USER_DATA); + CU_ASSERT(odp_dma_compl_result(compl, &result) == 0); + CU_ASSERT(result.success); + CU_ASSERT(result.user_ptr == user_ptr); + CU_ASSERT(user_data == USER_DATA); - /* Test also without result struct output */ - CU_ASSERT(odp_dma_compl_result(compl, NULL) == 0); + /* Test also without result struct output */ + CU_ASSERT(odp_dma_compl_result(compl, NULL) == 0); - /* Test compl event print on the first event */ - if (global.event_count == 0) { - printf("\n\n"); - odp_dma_compl_print(compl); - } + /* Test compl event print on the first event */ + if (global.event_count == 0) { + printf("\n\n"); + odp_dma_compl_print(compl); + } - /* Test both ways to free the event */ - if (global.event_count % 2) - odp_event_free(ev); - else - odp_dma_compl_free(compl); + /* Test both ways to free the event */ + if (global.event_count % 2) + odp_event_free(ev); + else + odp_dma_compl_free(compl); - global.event_count++; + global.event_count++; + } } return 1; @@ -517,8 +568,8 @@ static void test_dma_addr_to_addr(odp_dma_compl_mode_t compl_mode_mask, uint32_t uint32_t i, cur_len; uint8_t *src = global.src_addr + OFFSET; uint8_t *dst = global.dst_addr + OFFSET; - uint32_t len = global.len; - uint32_t seg_len = len / num; + uint32_t seg_len = MIN(global.len / num, global.dma_capa.max_seg_len); + uint32_t len = seg_len * num; uint32_t offset = 0; init_source(global.src_addr, global.data_size); @@ -558,7 +609,7 @@ static void test_dma_addr_to_addr(odp_dma_compl_mode_t compl_mode_mask, uint32_t if (ret > 0) { CU_ASSERT(check_equal(src, dst, len) == 0); CU_ASSERT(check_zero(global.dst_addr, OFFSET) == 0); - CU_ASSERT(check_zero(dst + len, TRAILER) == 0); + CU_ASSERT(check_zero(dst + len, global.len - len + TRAILER) == 0); } CU_ASSERT(odp_dma_destroy(dma) == 0); @@ -577,8 +628,8 @@ static void test_dma_addr_to_addr_trs(odp_dma_compl_mode_t compl_mode_mask, uint odp_dma_compl_mode_t compl_mode; uint8_t *src = global.src_addr + OFFSET; uint8_t *dst = global.dst_addr + OFFSET; - uint32_t len = global.len; - uint32_t trs_len = len / num_trs; + uint32_t trs_len = MIN(global.len / num_trs, global.dma_capa.max_seg_len); + uint32_t len = trs_len * num_trs; uint32_t offset = 0; int ret = -1; @@ -630,7 +681,65 @@ static void test_dma_addr_to_addr_trs(odp_dma_compl_mode_t compl_mode_mask, uint if (ret > 0) { CU_ASSERT(check_equal(src, dst, len) == 0); CU_ASSERT(check_zero(global.dst_addr, OFFSET) == 0); - CU_ASSERT(check_zero(dst + len, TRAILER) == 0); + CU_ASSERT(check_zero(dst + len, global.len - len + TRAILER) == 0); + } + + CU_ASSERT(odp_dma_destroy(dma) == 0); +} + +static void test_dma_addr_to_addr_max_trs(odp_dma_compl_mode_t compl_mode_mask) +{ + odp_dma_param_t dma_param; + uint32_t num_trs = global.dma_capa.max_transfers; + odp_dma_transfer_param_t trs_param[num_trs]; + odp_dma_t dma; + odp_dma_seg_t src_seg[num_trs]; + odp_dma_seg_t dst_seg[num_trs]; + int ret; + uint32_t i, cur_len; + uint8_t *src = global.src_addr + OFFSET; + uint8_t *dst = global.dst_addr + OFFSET; + uint32_t seg_len = MIN(global.len / num_trs, global.dma_capa.max_seg_len); + uint32_t len = seg_len * num_trs; + uint32_t offset = 0; + + init_source(global.src_addr, global.data_size); + memset(global.dst_addr, 0, global.data_size); + + odp_dma_param_init(&dma_param); + dma_param.compl_mode_mask = compl_mode_mask; + dma = odp_dma_create("addr_to_addr", &dma_param); + CU_ASSERT_FATAL(dma != ODP_DMA_INVALID); + + memset(src_seg, 0, sizeof(src_seg)); + memset(dst_seg, 0, sizeof(dst_seg)); + + for (i = 0; i < num_trs; i++) { + cur_len = seg_len; + if (i == num_trs - 1) + cur_len = len - seg_len * i; + + src_seg[i].addr = src + offset; + src_seg[i].len = cur_len; + dst_seg[i].addr = dst + offset; + dst_seg[i].len = cur_len; + offset += cur_len; + } + + for (i = 0; i < num_trs; i++) { + odp_dma_transfer_param_init(&trs_param[i]); + trs_param[i].num_src = 1; + trs_param[i].num_dst = 1; + trs_param[i].src_seg = &src_seg[i]; + trs_param[i].dst_seg = &dst_seg[i]; + } + + ret = do_transfer_async(dma, trs_param, compl_mode_mask, num_trs); + + if (ret > 0) { + CU_ASSERT(check_equal(src, dst, len) == 0); + CU_ASSERT(check_zero(global.dst_addr, OFFSET) == 0); + CU_ASSERT(check_zero(dst + len, global.len - len + TRAILER) == 0); } CU_ASSERT(odp_dma_destroy(dma) == 0); @@ -1000,7 +1109,7 @@ static void test_dma_addr_to_addr_sync(void) static void test_dma_addr_to_addr_sync_mtrs(void) { - test_dma_addr_to_addr_trs(ODP_DMA_COMPL_SYNC, 2, 0, 0); + test_dma_addr_to_addr_trs(ODP_DMA_COMPL_SYNC, global.dma_capa.max_transfers * 2, 0, 0); } static void test_dma_addr_to_addr_sync_mseg(void) @@ -1140,6 +1249,11 @@ static void test_dma_multi_addr_to_addr_poll(void) test_dma_addr_to_addr(ODP_DMA_COMPL_POLL, 1, MULTI, 0); } +static void test_dma_multi_addr_to_addr_poll_max_trs(void) +{ + test_dma_addr_to_addr_max_trs(ODP_DMA_COMPL_POLL); +} + static void test_dma_multi_addr_to_pkt_poll(void) { test_dma_addr_to_pkt(ODP_DMA_COMPL_POLL, MULTI); @@ -1160,6 +1274,11 @@ static void test_dma_multi_addr_to_addr_event(void) test_dma_addr_to_addr(ODP_DMA_COMPL_EVENT, 1, MULTI, 0); } +static void test_dma_multi_addr_to_addr_event_max_trs(void) +{ + test_dma_addr_to_addr_max_trs(ODP_DMA_COMPL_EVENT); +} + static void test_dma_multi_addr_to_pkt_event(void) { test_dma_addr_to_pkt(ODP_DMA_COMPL_EVENT, MULTI); @@ -1181,6 +1300,7 @@ odp_testinfo_t dma_suite[] = { ODP_TEST_INFO_CONDITIONAL(test_dma_debug, check_sync), ODP_TEST_INFO_CONDITIONAL(test_dma_compl_pool, check_event), ODP_TEST_INFO_CONDITIONAL(test_dma_compl_pool_same_name, check_event), + ODP_TEST_INFO_CONDITIONAL(test_dma_compl_pool_max_pools, check_event), ODP_TEST_INFO_CONDITIONAL(test_dma_addr_to_addr_sync, check_sync), ODP_TEST_INFO_CONDITIONAL(test_dma_addr_to_addr_sync_mtrs, check_sync), ODP_TEST_INFO_CONDITIONAL(test_dma_addr_to_addr_sync_mseg, check_sync), @@ -1208,10 +1328,12 @@ odp_testinfo_t dma_suite[] = { ODP_TEST_INFO_CONDITIONAL(test_dma_multi_pkt_to_addr_sync, check_sync), ODP_TEST_INFO_CONDITIONAL(test_dma_multi_pkt_to_pkt_sync, check_sync), ODP_TEST_INFO_CONDITIONAL(test_dma_multi_addr_to_addr_poll, check_poll), + ODP_TEST_INFO_CONDITIONAL(test_dma_multi_addr_to_addr_poll_max_trs, check_poll), ODP_TEST_INFO_CONDITIONAL(test_dma_multi_addr_to_pkt_poll, check_poll), ODP_TEST_INFO_CONDITIONAL(test_dma_multi_pkt_to_addr_poll, check_poll), ODP_TEST_INFO_CONDITIONAL(test_dma_multi_pkt_to_pkt_poll, check_poll), ODP_TEST_INFO_CONDITIONAL(test_dma_multi_addr_to_addr_event, check_scheduled), + ODP_TEST_INFO_CONDITIONAL(test_dma_multi_addr_to_addr_event_max_trs, check_scheduled), ODP_TEST_INFO_CONDITIONAL(test_dma_multi_addr_to_pkt_event, check_scheduled), ODP_TEST_INFO_CONDITIONAL(test_dma_multi_pkt_to_addr_event, check_scheduled), ODP_TEST_INFO_CONDITIONAL(test_dma_multi_pkt_to_pkt_event, check_scheduled), diff --git a/test/validation/api/ipsec/ipsec_test_out.c b/test/validation/api/ipsec/ipsec_test_out.c index bb318edad..1a013289b 100644 --- a/test/validation/api/ipsec/ipsec_test_out.c +++ b/test/validation/api/ipsec/ipsec_test_out.c @@ -475,6 +475,30 @@ static void ipsec_check_out_in_one(const ipsec_test_part *part_outbound, } } +static int sa_creation_failure_ok(const odp_ipsec_sa_param_t *param) +{ + odp_cipher_alg_t cipher = param->crypto.cipher_alg; + odp_auth_alg_t auth = param->crypto.auth_alg; + + /* Single algorithm must not fail */ + if (cipher == ODP_CIPHER_ALG_NULL || auth == ODP_AUTH_ALG_NULL) + return 0; + + /* Combined mode algorithms must not fail */ + if (cipher == ODP_CIPHER_ALG_AES_GCM || + cipher == ODP_CIPHER_ALG_AES_CCM || + cipher == ODP_CIPHER_ALG_CHACHA20_POLY1305) + return 0; + + /* Combination of mandatory algorithms must not fail */ + if (cipher == ODP_CIPHER_ALG_AES_CBC && auth == ODP_AUTH_ALG_SHA1_HMAC) + return 0; + + printf("\n Algorithm combination (%d, %d) maybe not supported.\n", cipher, auth); + printf(" SA creation failed, skipping test.\n"); + return 1; +} + static void test_out_in_common(const ipsec_test_flags *flags, odp_cipher_alg_t cipher, const odp_crypto_key_t *cipher_key, @@ -539,6 +563,9 @@ static void test_out_in_common(const ipsec_test_flags *flags, sa_out = odp_ipsec_sa_create(¶m); + if (sa_out == ODP_IPSEC_SA_INVALID && sa_creation_failure_ok(¶m)) + return; + CU_ASSERT_NOT_EQUAL_FATAL(ODP_IPSEC_SA_INVALID, sa_out); ipsec_sa_param_fill(¶m, diff --git a/test/validation/api/pktio/pktio.c b/test/validation/api/pktio/pktio.c index 446411b22..2f4dda4a4 100644 --- a/test/validation/api/pktio/pktio.c +++ b/test/validation/api/pktio/pktio.c @@ -300,6 +300,8 @@ static void pktio_init_packet_eth_ipv4(odp_packet_t pkt, uint8_t proto) seq = odp_atomic_fetch_inc_u32(&ip_seq); ip->id = odp_cpu_to_be_16(seq); ip->chksum = 0; + ip->frag_offset = 0; + ip->tos = 0; odph_ipv4_csum_update(pkt); } @@ -1681,7 +1683,6 @@ static void pktio_test_lookup(void) pktio_inval = odp_pktio_open(iface_name[0], default_pkt_pool, &pktio_param); - CU_ASSERT(odp_errno() != 0); CU_ASSERT(pktio_inval == ODP_PKTIO_INVALID); CU_ASSERT(odp_pktio_close(pktio) == 0); @@ -2039,8 +2040,10 @@ static void pktio_config_flow_control(int pfc, int rx, int tx) if (cos != ODP_COS_INVALID) odp_cos_destroy(cos); - if (default_cos != ODP_COS_INVALID) + if (default_cos != ODP_COS_INVALID) { + odp_pktio_default_cos_set(pktio, ODP_COS_INVALID); odp_cos_destroy(default_cos); + } if (queue != ODP_QUEUE_INVALID) odp_queue_destroy(queue); @@ -4433,7 +4436,7 @@ static int create_pool(const char *iface, int num) pool[num] = odp_pool_create(pool_name, ¶ms); if (ODP_POOL_INVALID == pool[num]) { - ODPH_ERR("failed to create pool: %d", odp_errno()); + ODPH_ERR("failed to create pool: %s\n", pool_name); return -1; } @@ -4463,7 +4466,7 @@ static int create_pktv_pool(const char *iface, int num) pktv_pool[num] = odp_pool_create(pool_name, ¶ms); if (ODP_POOL_INVALID == pktv_pool[num]) { - ODPH_ERR("failed to create pool: %d", odp_errno()); + ODPH_ERR("failed to create pool: %s\n", pool_name); return -1; } diff --git a/test/validation/api/stash/stash.c b/test/validation/api/stash/stash.c index f4ddfa2e1..fd4c04577 100644 --- a/test/validation/api/stash/stash.c +++ b/test/validation/api/stash/stash.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2020-2022, Nokia +/* Copyright (c) 2020-2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -7,6 +7,8 @@ #include <odp_api.h> #include "odp_cunit_common.h" +#include <string.h> + #define MAGIC_U64 0x8b7438fa56c82e96 #define MAGIC_U32 0x74a13b94 #define MAGIC_U16 0x25bf @@ -111,10 +113,9 @@ static int stash_suite_init(void) static void stash_capability(void) { odp_stash_capability_t capa; - int ret; memset(&capa, 0, sizeof(odp_stash_capability_t)); - CU_ASSERT(odp_stash_capability(&capa, ODP_STASH_TYPE_DEFAULT) == 0); + CU_ASSERT_FATAL(odp_stash_capability(&capa, ODP_STASH_TYPE_DEFAULT) == 0); CU_ASSERT(capa.max_stashes_any_type > 0); CU_ASSERT(capa.max_stashes > 0); CU_ASSERT(capa.max_num_obj > 0); @@ -122,16 +123,43 @@ static void stash_capability(void) CU_ASSERT(capa.max_get_batch >= 1); CU_ASSERT(capa.max_put_batch >= 1); + CU_ASSERT(capa.max_num.u8 >= capa.max_num_obj); + CU_ASSERT(capa.max_num.u16 >= capa.max_num_obj); + CU_ASSERT(capa.max_num.u32 >= capa.max_num_obj); + CU_ASSERT(capa.max_num.max_obj_size >= capa.max_num_obj); + if (capa.max_obj_size >= 8) + CU_ASSERT(capa.max_num.u64 >= capa.max_num_obj); + if (capa.max_obj_size < 8) + CU_ASSERT(capa.max_num.u64 == 0); + if (capa.max_obj_size >= 16) + CU_ASSERT(capa.max_num.u128 >= capa.max_num_obj); + if (capa.max_obj_size < 16) + CU_ASSERT(capa.max_num.u128 == 0); + memset(&capa, 0, sizeof(odp_stash_capability_t)); - ret = odp_stash_capability(&capa, ODP_STASH_TYPE_FIFO); - CU_ASSERT(ret == 0); + CU_ASSERT_FATAL(odp_stash_capability(&capa, ODP_STASH_TYPE_FIFO) == 0); CU_ASSERT(capa.max_stashes_any_type > 0); - if (ret == 0 && capa.max_stashes) { - CU_ASSERT(capa.max_num_obj > 0); - CU_ASSERT(capa.max_obj_size >= sizeof(uint32_t)); - CU_ASSERT(capa.max_get_batch >= 1); - CU_ASSERT(capa.max_put_batch >= 1); - } + + if (capa.max_stashes == 0) + return; + + CU_ASSERT(capa.max_num_obj > 0); + CU_ASSERT(capa.max_obj_size >= sizeof(uint32_t)); + CU_ASSERT(capa.max_get_batch >= 1); + CU_ASSERT(capa.max_put_batch >= 1); + + CU_ASSERT(capa.max_num.u8 >= capa.max_num_obj); + CU_ASSERT(capa.max_num.u16 >= capa.max_num_obj); + CU_ASSERT(capa.max_num.u32 >= capa.max_num_obj); + CU_ASSERT(capa.max_num.max_obj_size >= capa.max_num_obj); + if (capa.max_obj_size >= 8) + CU_ASSERT(capa.max_num.u64 >= capa.max_num_obj); + if (capa.max_obj_size < 8) + CU_ASSERT(capa.max_num.u64 == 0); + if (capa.max_obj_size >= 16) + CU_ASSERT(capa.max_num.u128 >= capa.max_num_obj); + if (capa.max_obj_size < 16) + CU_ASSERT(capa.max_num.u128 == 0); } static void param_defaults(uint8_t fill) @@ -147,6 +175,7 @@ static void param_defaults(uint8_t fill) CU_ASSERT(param.stats.all == 0); CU_ASSERT(param.stats.bit.count == 0); CU_ASSERT(param.stats.bit.cache_count == 0); + CU_ASSERT(param.strict_size == 0); } static void stash_param_defaults(void) @@ -449,12 +478,13 @@ static void stash_stats_u32(void) CU_ASSERT_FATAL(odp_stash_destroy(stash) == 0); } -static void stash_default_put(uint32_t size, int32_t burst, stash_op_t op, int batch) +static void stash_default_put(uint32_t size, int32_t burst, stash_op_t op, int batch, + odp_bool_t strict_size) { odp_stash_t stash; odp_stash_param_t param; int32_t i, ret, retry, num_left; - int32_t num, max_burst; + int32_t num, max_burst, num_stashed; void *input, *output; if (batch) { @@ -505,6 +535,7 @@ static void stash_default_put(uint32_t size, int32_t burst, stash_op_t op, int b param.num_obj = num; param.obj_size = size; param.cache_size = global.cache_size_default; + param.strict_size = strict_size; stash = odp_stash_create("test_stash_default", ¶m); @@ -516,6 +547,12 @@ static void stash_default_put(uint32_t size, int32_t burst, stash_op_t op, int b retry = MAX_RETRY; num_left = num; max_burst = burst; + num_stashed = 0; + + /* Try to store extra objects if strict mode is not enabled */ + if (!strict_size) + num_left += burst; + while (num_left > 0) { if (op == STASH_GEN) { if (batch) @@ -542,6 +579,9 @@ static void stash_default_put(uint32_t size, int32_t burst, stash_op_t op, int b } CU_ASSERT_FATAL(ret >= 0); CU_ASSERT_FATAL(ret <= burst); + + num_stashed += ret; + if (batch) { CU_ASSERT(ret == 0 || ret == burst); if (num_left - ret < burst) @@ -552,6 +592,9 @@ static void stash_default_put(uint32_t size, int32_t burst, stash_op_t op, int b num_left -= ret; retry = MAX_RETRY; } else { + /* Stash full */ + if (num_stashed >= num) + break; retry--; CU_ASSERT_FATAL(retry > 0); } @@ -559,7 +602,7 @@ static void stash_default_put(uint32_t size, int32_t burst, stash_op_t op, int b burst = max_burst; retry = MAX_RETRY; - num_left = num; + num_left = num_stashed; while (num_left > 0) { memset(output, 0, burst * size); @@ -652,12 +695,13 @@ static void stash_default_put(uint32_t size, int32_t burst, stash_op_t op, int b CU_ASSERT_FATAL(odp_stash_destroy(stash) == 0); } -static void stash_fifo_put(uint32_t size, int32_t burst, stash_op_t op, int batch) +static void stash_fifo_put(uint32_t size, int32_t burst, stash_op_t op, int batch, + odp_bool_t strict_size) { odp_stash_t stash; odp_stash_param_t param; int32_t i, ret, retry, num_left; - int32_t num, max_burst; + int32_t num, max_burst, num_stashed; void *input, *output; if (batch) { @@ -701,6 +745,7 @@ static void stash_fifo_put(uint32_t size, int32_t burst, stash_op_t op, int batc param.type = ODP_STASH_TYPE_FIFO; param.num_obj = num; param.obj_size = size; + param.strict_size = strict_size; stash = odp_stash_create("test_stash_fifo", ¶m); @@ -712,6 +757,12 @@ static void stash_fifo_put(uint32_t size, int32_t burst, stash_op_t op, int batc retry = MAX_RETRY; num_left = num; max_burst = burst; + num_stashed = 0; + + /* Try to store extra objects if strict mode is not enabled */ + if (!strict_size) + num_left += burst; + while (num_left > 0) { for (i = 0; i < burst; i++) { if (size == sizeof(uint64_t)) @@ -750,6 +801,8 @@ static void stash_fifo_put(uint32_t size, int32_t burst, stash_op_t op, int batc CU_ASSERT_FATAL(ret >= 0); CU_ASSERT_FATAL(ret <= burst); + num_stashed += ret; + if (batch) { CU_ASSERT(ret == 0 || ret == burst); if (num_left - ret < burst) @@ -760,6 +813,9 @@ static void stash_fifo_put(uint32_t size, int32_t burst, stash_op_t op, int batc num_left -= ret; retry = MAX_RETRY; } else { + /* Stash full */ + if (num_stashed >= num) + break; retry--; CU_ASSERT_FATAL(retry > 0); } @@ -767,7 +823,7 @@ static void stash_fifo_put(uint32_t size, int32_t burst, stash_op_t op, int batc burst = max_burst; retry = MAX_RETRY; - num_left = num; + num_left = num_stashed; while (num_left > 0) { memset(output, 0, burst * size); @@ -912,282 +968,338 @@ static int check_support_fifo(void) static void stash_default_put_u64_1(void) { - stash_default_put(sizeof(uint64_t), 1, STASH_GEN, 0); + stash_default_put(sizeof(uint64_t), 1, STASH_GEN, 0, false); + stash_default_put(sizeof(uint64_t), 1, STASH_GEN, 0, true); } static void stash_default_put_u64_n(void) { - stash_default_put(sizeof(uint64_t), BURST, STASH_GEN, 0); + stash_default_put(sizeof(uint64_t), BURST, STASH_GEN, 0, false); + stash_default_put(sizeof(uint64_t), BURST, STASH_GEN, 0, true); } static void stash_default_u64_put_u64_1(void) { - stash_default_put(sizeof(uint64_t), 1, STASH_U64, 0); + stash_default_put(sizeof(uint64_t), 1, STASH_U64, 0, false); + stash_default_put(sizeof(uint64_t), 1, STASH_U64, 0, true); } static void stash_default_u64_put_u64_n(void) { - stash_default_put(sizeof(uint64_t), BURST, STASH_U64, 0); + stash_default_put(sizeof(uint64_t), BURST, STASH_U64, 0, false); + stash_default_put(sizeof(uint64_t), BURST, STASH_U64, 0, true); } static void stash_default_put_ptr_1(void) { - stash_default_put(sizeof(uintptr_t), 1, STASH_PTR, 0); + stash_default_put(sizeof(uintptr_t), 1, STASH_PTR, 0, false); + stash_default_put(sizeof(uintptr_t), 1, STASH_PTR, 0, true); } static void stash_default_put_ptr_n(void) { - stash_default_put(sizeof(uintptr_t), BURST, STASH_PTR, 0); + stash_default_put(sizeof(uintptr_t), BURST, STASH_PTR, 0, false); + stash_default_put(sizeof(uintptr_t), BURST, STASH_PTR, 0, true); } static void stash_default_put_u64_1_batch(void) { - stash_default_put(sizeof(uint64_t), 1, STASH_GEN, 1); + stash_default_put(sizeof(uint64_t), 1, STASH_GEN, 1, false); + stash_default_put(sizeof(uint64_t), 1, STASH_GEN, 1, true); } static void stash_default_put_u64_n_batch(void) { - stash_default_put(sizeof(uint64_t), BATCH, STASH_GEN, 1); + stash_default_put(sizeof(uint64_t), BATCH, STASH_GEN, 1, false); + stash_default_put(sizeof(uint64_t), BATCH, STASH_GEN, 1, true); } static void stash_default_u64_put_u64_1_batch(void) { - stash_default_put(sizeof(uint64_t), 1, STASH_U64, 1); + stash_default_put(sizeof(uint64_t), 1, STASH_U64, 1, false); + stash_default_put(sizeof(uint64_t), 1, STASH_U64, 1, true); } static void stash_default_u64_put_u64_n_batch(void) { - stash_default_put(sizeof(uint64_t), BATCH, STASH_U64, 1); + stash_default_put(sizeof(uint64_t), BATCH, STASH_U64, 1, false); + stash_default_put(sizeof(uint64_t), BATCH, STASH_U64, 1, true); } static void stash_default_put_ptr_1_batch(void) { - stash_default_put(sizeof(uintptr_t), 1, STASH_PTR, 1); + stash_default_put(sizeof(uintptr_t), 1, STASH_PTR, 1, false); + stash_default_put(sizeof(uintptr_t), 1, STASH_PTR, 1, true); } static void stash_default_put_ptr_n_batch(void) { - stash_default_put(sizeof(uintptr_t), BATCH, STASH_PTR, 1); + stash_default_put(sizeof(uintptr_t), BATCH, STASH_PTR, 1, false); + stash_default_put(sizeof(uintptr_t), BATCH, STASH_PTR, 1, true); } static void stash_default_put_u32_1(void) { - stash_default_put(sizeof(uint32_t), 1, STASH_GEN, 0); + stash_default_put(sizeof(uint32_t), 1, STASH_GEN, 0, false); + stash_default_put(sizeof(uint32_t), 1, STASH_GEN, 0, true); } static void stash_default_put_u32_n(void) { - stash_default_put(sizeof(uint32_t), BURST, STASH_GEN, 0); + stash_default_put(sizeof(uint32_t), BURST, STASH_GEN, 0, false); + stash_default_put(sizeof(uint32_t), BURST, STASH_GEN, 0, true); } static void stash_default_u32_put_u32_1(void) { - stash_default_put(sizeof(uint32_t), 1, STASH_U32, 0); + stash_default_put(sizeof(uint32_t), 1, STASH_U32, 0, false); + stash_default_put(sizeof(uint32_t), 1, STASH_U32, 0, true); } static void stash_default_u32_put_u32_n(void) { - stash_default_put(sizeof(uint32_t), BURST, STASH_U32, 0); + stash_default_put(sizeof(uint32_t), BURST, STASH_U32, 0, false); + stash_default_put(sizeof(uint32_t), BURST, STASH_U32, 0, true); } static void stash_default_put_u16_1(void) { - stash_default_put(sizeof(uint16_t), 1, STASH_GEN, 0); + stash_default_put(sizeof(uint16_t), 1, STASH_GEN, 0, false); + stash_default_put(sizeof(uint16_t), 1, STASH_GEN, 0, true); } static void stash_default_put_u16_n(void) { - stash_default_put(sizeof(uint16_t), BURST, STASH_GEN, 0); + stash_default_put(sizeof(uint16_t), BURST, STASH_GEN, 0, false); + stash_default_put(sizeof(uint16_t), BURST, STASH_GEN, 0, true); } static void stash_default_put_u8_1(void) { - stash_default_put(sizeof(uint8_t), 1, STASH_GEN, 0); + stash_default_put(sizeof(uint8_t), 1, STASH_GEN, 0, false); + stash_default_put(sizeof(uint8_t), 1, STASH_GEN, 0, true); } static void stash_default_put_u8_n(void) { - stash_default_put(sizeof(uint8_t), BURST, STASH_GEN, 0); + stash_default_put(sizeof(uint8_t), BURST, STASH_GEN, 0, false); + stash_default_put(sizeof(uint8_t), BURST, STASH_GEN, 0, true); } static void stash_default_put_u32_1_batch(void) { - stash_default_put(sizeof(uint32_t), 1, STASH_GEN, 1); + stash_default_put(sizeof(uint32_t), 1, STASH_GEN, 1, false); + stash_default_put(sizeof(uint32_t), 1, STASH_GEN, 1, true); } static void stash_default_put_u32_n_batch(void) { - stash_default_put(sizeof(uint32_t), BATCH, STASH_GEN, 1); + stash_default_put(sizeof(uint32_t), BATCH, STASH_GEN, 1, false); + stash_default_put(sizeof(uint32_t), BATCH, STASH_GEN, 1, true); } static void stash_default_u32_put_u32_1_batch(void) { - stash_default_put(sizeof(uint32_t), 1, STASH_U32, 1); + stash_default_put(sizeof(uint32_t), 1, STASH_U32, 1, false); + stash_default_put(sizeof(uint32_t), 1, STASH_U32, 1, true); } static void stash_default_u32_put_u32_n_batch(void) { - stash_default_put(sizeof(uint32_t), BATCH, STASH_U32, 1); + stash_default_put(sizeof(uint32_t), BATCH, STASH_U32, 1, false); + stash_default_put(sizeof(uint32_t), BATCH, STASH_U32, 1, true); } static void stash_default_put_u16_1_batch(void) { - stash_default_put(sizeof(uint16_t), 1, STASH_GEN, 1); + stash_default_put(sizeof(uint16_t), 1, STASH_GEN, 1, false); + stash_default_put(sizeof(uint16_t), 1, STASH_GEN, 1, true); } static void stash_default_put_u16_n_batch(void) { - stash_default_put(sizeof(uint16_t), BATCH, STASH_GEN, 1); + stash_default_put(sizeof(uint16_t), BATCH, STASH_GEN, 1, false); + stash_default_put(sizeof(uint16_t), BATCH, STASH_GEN, 1, true); } static void stash_default_put_u8_1_batch(void) { - stash_default_put(sizeof(uint8_t), 1, STASH_GEN, 1); + stash_default_put(sizeof(uint8_t), 1, STASH_GEN, 1, false); + stash_default_put(sizeof(uint8_t), 1, STASH_GEN, 1, true); } static void stash_default_put_u8_n_batch(void) { - stash_default_put(sizeof(uint8_t), BATCH, STASH_GEN, 1); + stash_default_put(sizeof(uint8_t), BATCH, STASH_GEN, 1, false); + stash_default_put(sizeof(uint8_t), BATCH, STASH_GEN, 1, true); } static void stash_fifo_put_u64_1(void) { - stash_fifo_put(sizeof(uint64_t), 1, STASH_GEN, 0); + stash_fifo_put(sizeof(uint64_t), 1, STASH_GEN, 0, false); + stash_fifo_put(sizeof(uint64_t), 1, STASH_GEN, 0, true); } static void stash_fifo_put_u64_n(void) { - stash_fifo_put(sizeof(uint64_t), BURST, STASH_GEN, 0); + stash_fifo_put(sizeof(uint64_t), BURST, STASH_GEN, 0, false); + stash_fifo_put(sizeof(uint64_t), BURST, STASH_GEN, 0, true); } static void stash_fifo_u64_put_u64_1(void) { - stash_fifo_put(sizeof(uint64_t), 1, STASH_U64, 0); + stash_fifo_put(sizeof(uint64_t), 1, STASH_U64, 0, false); + stash_fifo_put(sizeof(uint64_t), 1, STASH_U64, 0, true); } static void stash_fifo_u64_put_u64_n(void) { - stash_fifo_put(sizeof(uint64_t), BURST, STASH_U64, 0); + stash_fifo_put(sizeof(uint64_t), BURST, STASH_U64, 0, false); + stash_fifo_put(sizeof(uint64_t), BURST, STASH_U64, 0, true); } static void stash_fifo_put_ptr_1(void) { - stash_fifo_put(sizeof(uintptr_t), 1, STASH_PTR, 0); + stash_fifo_put(sizeof(uintptr_t), 1, STASH_PTR, 0, false); + stash_fifo_put(sizeof(uintptr_t), 1, STASH_PTR, 0, true); } static void stash_fifo_put_ptr_n(void) { - stash_fifo_put(sizeof(uintptr_t), BURST, STASH_PTR, 0); + stash_fifo_put(sizeof(uintptr_t), BURST, STASH_PTR, 0, false); + stash_fifo_put(sizeof(uintptr_t), BURST, STASH_PTR, 0, true); } static void stash_fifo_put_u32_1(void) { - stash_fifo_put(sizeof(uint32_t), 1, STASH_GEN, 0); + stash_fifo_put(sizeof(uint32_t), 1, STASH_GEN, 0, false); + stash_fifo_put(sizeof(uint32_t), 1, STASH_GEN, 0, true); } static void stash_fifo_put_u32_n(void) { - stash_fifo_put(sizeof(uint32_t), BURST, STASH_GEN, 0); + stash_fifo_put(sizeof(uint32_t), BURST, STASH_GEN, 0, false); + stash_fifo_put(sizeof(uint32_t), BURST, STASH_GEN, 0, true); } static void stash_fifo_u32_put_u32_1(void) { - stash_fifo_put(sizeof(uint32_t), 1, STASH_U32, 0); + stash_fifo_put(sizeof(uint32_t), 1, STASH_U32, 0, false); + stash_fifo_put(sizeof(uint32_t), 1, STASH_U32, 0, true); } static void stash_fifo_u32_put_u32_n(void) { - stash_fifo_put(sizeof(uint32_t), BURST, STASH_U32, 0); + stash_fifo_put(sizeof(uint32_t), BURST, STASH_U32, 0, false); + stash_fifo_put(sizeof(uint32_t), BURST, STASH_U32, 0, true); } static void stash_fifo_put_u16_1(void) { - stash_fifo_put(sizeof(uint16_t), 1, STASH_GEN, 0); + stash_fifo_put(sizeof(uint16_t), 1, STASH_GEN, 0, false); + stash_fifo_put(sizeof(uint16_t), 1, STASH_GEN, 0, true); } static void stash_fifo_put_u16_n(void) { - stash_fifo_put(sizeof(uint16_t), BURST, STASH_GEN, 0); + stash_fifo_put(sizeof(uint16_t), BURST, STASH_GEN, 0, false); + stash_fifo_put(sizeof(uint16_t), BURST, STASH_GEN, 0, true); } static void stash_fifo_put_u8_1(void) { - stash_fifo_put(sizeof(uint8_t), 1, STASH_GEN, 0); + stash_fifo_put(sizeof(uint8_t), 1, STASH_GEN, 0, false); + stash_fifo_put(sizeof(uint8_t), 1, STASH_GEN, 0, true); } static void stash_fifo_put_u8_n(void) { - stash_fifo_put(sizeof(uint8_t), BURST, STASH_GEN, 0); + stash_fifo_put(sizeof(uint8_t), BURST, STASH_GEN, 0, false); + stash_fifo_put(sizeof(uint8_t), BURST, STASH_GEN, 0, true); } static void stash_fifo_put_u64_1_batch(void) { - stash_fifo_put(sizeof(uint64_t), 1, STASH_GEN, 1); + stash_fifo_put(sizeof(uint64_t), 1, STASH_GEN, 1, false); + stash_fifo_put(sizeof(uint64_t), 1, STASH_GEN, 1, true); } static void stash_fifo_put_u64_n_batch(void) { - stash_fifo_put(sizeof(uint64_t), BATCH, STASH_GEN, 1); + stash_fifo_put(sizeof(uint64_t), BATCH, STASH_GEN, 1, false); + stash_fifo_put(sizeof(uint64_t), BATCH, STASH_GEN, 1, true); } static void stash_fifo_u64_put_u64_1_batch(void) { - stash_fifo_put(sizeof(uint64_t), 1, STASH_U64, 1); + stash_fifo_put(sizeof(uint64_t), 1, STASH_U64, 1, false); + stash_fifo_put(sizeof(uint64_t), 1, STASH_U64, 1, true); } static void stash_fifo_u64_put_u64_n_batch(void) { - stash_fifo_put(sizeof(uint64_t), BATCH, STASH_U64, 1); + stash_fifo_put(sizeof(uint64_t), BATCH, STASH_U64, 1, false); + stash_fifo_put(sizeof(uint64_t), BATCH, STASH_U64, 1, true); } static void stash_fifo_put_ptr_1_batch(void) { - stash_fifo_put(sizeof(uintptr_t), 1, STASH_PTR, 1); + stash_fifo_put(sizeof(uintptr_t), 1, STASH_PTR, 1, false); + stash_fifo_put(sizeof(uintptr_t), 1, STASH_PTR, 1, true); } static void stash_fifo_put_ptr_n_batch(void) { - stash_fifo_put(sizeof(uintptr_t), BATCH, STASH_PTR, 1); + stash_fifo_put(sizeof(uintptr_t), BATCH, STASH_PTR, 1, false); + stash_fifo_put(sizeof(uintptr_t), BATCH, STASH_PTR, 1, true); } static void stash_fifo_put_u32_1_batch(void) { - stash_fifo_put(sizeof(uint32_t), 1, STASH_GEN, 1); + stash_fifo_put(sizeof(uint32_t), 1, STASH_GEN, 1, false); + stash_fifo_put(sizeof(uint32_t), 1, STASH_GEN, 1, true); } static void stash_fifo_put_u32_n_batch(void) { - stash_fifo_put(sizeof(uint32_t), BATCH, STASH_GEN, 1); + stash_fifo_put(sizeof(uint32_t), BATCH, STASH_GEN, 1, false); + stash_fifo_put(sizeof(uint32_t), BATCH, STASH_GEN, 1, true); } static void stash_fifo_u32_put_u32_1_batch(void) { - stash_fifo_put(sizeof(uint32_t), 1, STASH_U32, 1); + stash_fifo_put(sizeof(uint32_t), 1, STASH_U32, 1, false); + stash_fifo_put(sizeof(uint32_t), 1, STASH_U32, 1, true); } static void stash_fifo_u32_put_u32_n_batch(void) { - stash_fifo_put(sizeof(uint32_t), BATCH, STASH_U32, 1); + stash_fifo_put(sizeof(uint32_t), BATCH, STASH_U32, 1, false); + stash_fifo_put(sizeof(uint32_t), BATCH, STASH_U32, 1, true); } static void stash_fifo_put_u16_1_batch(void) { - stash_fifo_put(sizeof(uint16_t), 1, STASH_GEN, 1); + stash_fifo_put(sizeof(uint16_t), 1, STASH_GEN, 1, false); + stash_fifo_put(sizeof(uint16_t), 1, STASH_GEN, 1, true); } static void stash_fifo_put_u16_n_batch(void) { - stash_fifo_put(sizeof(uint16_t), BATCH, STASH_GEN, 1); + stash_fifo_put(sizeof(uint16_t), BATCH, STASH_GEN, 1, false); + stash_fifo_put(sizeof(uint16_t), BATCH, STASH_GEN, 1, true); } static void stash_fifo_put_u8_1_batch(void) { - stash_fifo_put(sizeof(uint8_t), 1, STASH_GEN, 1); + stash_fifo_put(sizeof(uint8_t), 1, STASH_GEN, 1, false); + stash_fifo_put(sizeof(uint8_t), 1, STASH_GEN, 1, true); } static void stash_fifo_put_u8_n_batch(void) { - stash_fifo_put(sizeof(uint8_t), BATCH, STASH_GEN, 1); + stash_fifo_put(sizeof(uint8_t), BATCH, STASH_GEN, 1, false); + stash_fifo_put(sizeof(uint8_t), BATCH, STASH_GEN, 1, true); } odp_testinfo_t stash_suite[] = { diff --git a/test/validation/api/timer/timer.c b/test/validation/api/timer/timer.c index 0e3919b73..eb7f0772a 100644 --- a/test/validation/api/timer/timer.c +++ b/test/validation/api/timer/timer.c @@ -1,5 +1,5 @@ /* Copyright (c) 2015-2018, Linaro Limited - * Copyright (c) 2019-2022, Nokia + * Copyright (c) 2019-2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -244,6 +244,14 @@ check_plain_queue_support(void) return ODP_TEST_INACTIVE; } +static int check_periodic_support(void) +{ + if (global_mem->periodic) + return ODP_TEST_ACTIVE; + + return ODP_TEST_INACTIVE; +} + static int check_periodic_sched_support(void) { if (global_mem->periodic && global_mem->param.queue_type_sched) @@ -2377,6 +2385,139 @@ static void timer_test_sched_all(void) timer_test_all(ODP_QUEUE_TYPE_SCHED); } +static void timer_test_periodic_capa(void) +{ + odp_timer_capability_t timer_capa; + odp_timer_periodic_capability_t capa; + odp_fract_u64_t min_fract, max_fract, base_freq; + uint64_t freq_range, freq_step, first_hz, res_ns, max_multiplier; + double freq, min_freq, max_freq; + int ret; + uint32_t i, j; + uint32_t num = 100; + + memset(&timer_capa, 0, sizeof(odp_timer_capability_t)); + CU_ASSERT_FATAL(odp_timer_capability(ODP_CLOCK_DEFAULT, &timer_capa) == 0); + CU_ASSERT(timer_capa.periodic.max_pools); + CU_ASSERT(timer_capa.periodic.max_timers); + + min_fract = timer_capa.periodic.min_base_freq_hz; + max_fract = timer_capa.periodic.max_base_freq_hz; + + CU_ASSERT_FATAL(min_fract.integer || min_fract.numer); + CU_ASSERT_FATAL(max_fract.integer || max_fract.numer); + + if (min_fract.numer) + CU_ASSERT_FATAL(min_fract.denom); + + if (max_fract.numer) + CU_ASSERT_FATAL(max_fract.denom); + + min_freq = odp_fract_u64_to_dbl(&min_fract); + max_freq = odp_fract_u64_to_dbl(&max_fract); + CU_ASSERT(min_freq <= max_freq); + + memset(&capa, 0, sizeof(odp_timer_periodic_capability_t)); + + /* Min freq, capa fills in resolution */ + capa.base_freq_hz = min_fract; + capa.max_multiplier = 1; + capa.res_ns = 0; + + CU_ASSERT(odp_timer_periodic_capability(ODP_CLOCK_DEFAULT, &capa) == 1); + CU_ASSERT(capa.base_freq_hz.integer == min_fract.integer); + CU_ASSERT(capa.base_freq_hz.numer == min_fract.numer); + CU_ASSERT(capa.base_freq_hz.denom == min_fract.denom); + CU_ASSERT(capa.max_multiplier >= 1); + CU_ASSERT(capa.res_ns > 0); + + /* Max freq, capa fills in resolution */ + capa.base_freq_hz = max_fract; + capa.max_multiplier = 1; + capa.res_ns = 0; + + CU_ASSERT(odp_timer_periodic_capability(ODP_CLOCK_DEFAULT, &capa) == 1); + CU_ASSERT(capa.base_freq_hz.integer == max_fract.integer); + CU_ASSERT(capa.base_freq_hz.numer == max_fract.numer); + CU_ASSERT(capa.base_freq_hz.denom == max_fract.denom); + CU_ASSERT(capa.max_multiplier >= 1); + CU_ASSERT(capa.res_ns > 0); + + freq_range = max_fract.integer - min_fract.integer; + + if (freq_range < 10 * num) + num = freq_range / 10; + + /* Too short frequency range */ + if (num == 0) + return; + + freq_step = freq_range / num; + first_hz = min_fract.integer + 1; + + ODPH_DBG("min %" PRIu64 ", max %" PRIu64 ", range %" PRIu64 ", step %" PRIu64 "\n", + min_fract.integer, max_fract.integer, freq_range, freq_step); + + for (i = 0; i < num; i++) { + base_freq.integer = first_hz + i * freq_step; + base_freq.numer = 0; + base_freq.denom = 0; + + freq = odp_fract_u64_to_dbl(&base_freq); + + if (freq > max_freq) + base_freq = max_fract; + + for (j = 0; j < 4; j++) { + capa.base_freq_hz = base_freq; + + max_multiplier = 1; + res_ns = 0; + + if (j & 0x1) + max_multiplier = 2; + + if (j & 0x2) + res_ns = 1 + (ODP_TIME_SEC_IN_NS / (10 * base_freq.integer)); + + capa.max_multiplier = max_multiplier; + capa.res_ns = res_ns; + + ODPH_DBG("freq %" PRIu64 ", multip %" PRIu64 ", res %" PRIu64 ",\n", + base_freq.integer, max_multiplier, res_ns); + + ret = odp_timer_periodic_capability(ODP_CLOCK_DEFAULT, &capa); + + if (ret == 1) { + CU_ASSERT(capa.base_freq_hz.integer == base_freq.integer); + CU_ASSERT(capa.base_freq_hz.numer == base_freq.numer); + CU_ASSERT(capa.base_freq_hz.denom == base_freq.denom); + } else if (ret == 0) { + CU_ASSERT(capa.base_freq_hz.integer != base_freq.integer || + capa.base_freq_hz.numer != base_freq.numer || + capa.base_freq_hz.denom != base_freq.denom) + + if (capa.base_freq_hz.numer) + CU_ASSERT_FATAL(capa.base_freq_hz.denom); + + CU_ASSERT(odp_fract_u64_to_dbl(&capa.base_freq_hz) >= min_freq); + CU_ASSERT(odp_fract_u64_to_dbl(&capa.base_freq_hz) <= max_freq); + } + + if (ret >= 0) { + CU_ASSERT(capa.max_multiplier >= max_multiplier); + + if (res_ns) { + /* Same or better resolution */ + CU_ASSERT(capa.res_ns <= res_ns); + } else { + CU_ASSERT(capa.res_ns > 0); + } + } + } + } +} + static void timer_test_periodic(odp_queue_type_t queue_type, int use_first) { odp_timer_capability_t timer_capa; @@ -2392,7 +2533,7 @@ static void timer_test_periodic(odp_queue_type_t queue_type, int use_first) odp_event_t ev; odp_timer_t timer; odp_time_t t1, t2; - uint64_t tick, cur_tick, period_ns, duration_ns, diff_ns; + uint64_t tick, cur_tick, period_ns, duration_ns, diff_ns, offset_ns; double freq, freq_out, min_freq, max_freq; int ret; const char *user_ctx = "User context"; @@ -2509,10 +2650,14 @@ static void timer_test_periodic(odp_queue_type_t queue_type, int use_first) memset(&start_param, 0, sizeof(odp_timer_periodic_start_t)); cur_tick = odp_timer_current_tick(timer_pool); - tick = cur_tick + odp_timer_ns_to_tick(timer_pool, period_ns / 2); + offset_ns = period_ns / 2; + tick = cur_tick + odp_timer_ns_to_tick(timer_pool, offset_ns); - if (use_first) + if (use_first) { + /* First tick moves timer to start before the first period */ + duration_ns -= (period_ns - offset_ns); start_param.first_tick = tick; + } start_param.freq_multiplier = multiplier; start_param.tmo_ev = ev; @@ -2521,7 +2666,8 @@ static void timer_test_periodic(odp_queue_type_t queue_type, int use_first) ODPH_DBG(" Current tick: %" PRIu64 "\n", cur_tick); ODPH_DBG(" First tick: %" PRIu64 "\n", start_param.first_tick); ODPH_DBG(" Multiplier: %" PRIu64 "\n", start_param.freq_multiplier); - ODPH_DBG("Test duration ns: %" PRIu64 "\n", duration_ns); + ODPH_DBG(" Period: %" PRIu64 " nsec\n", period_ns); + ODPH_DBG("Expected duration: %" PRIu64 " nsec\n", duration_ns); ret = odp_timer_periodic_start(timer, &start_param); @@ -2576,6 +2722,8 @@ static void timer_test_periodic(odp_queue_type_t queue_type, int use_first) ret = odp_timer_periodic_cancel(timer); CU_ASSERT_FATAL(ret == 0); + ODPH_DBG("Measured duration: %" PRIu64 " nsec\n", diff_ns); + t1 = odp_time_local(); while (1) { if (queue_type == ODP_QUEUE_TYPE_SCHED) @@ -2714,6 +2862,8 @@ odp_testinfo_t timer_suite[] = { check_plain_queue_support), ODP_TEST_INFO_CONDITIONAL(timer_test_sched_all, check_sched_queue_support), + ODP_TEST_INFO_CONDITIONAL(timer_test_periodic_capa, + check_periodic_support), ODP_TEST_INFO_CONDITIONAL(timer_test_periodic_sched, check_periodic_sched_support), ODP_TEST_INFO_CONDITIONAL(timer_test_periodic_sched_first, |