diff options
| author | Matias Elo <matias.elo@nokia.com> | 2021-11-24 09:23:14 +0200 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2021-11-24 09:23:14 +0200 |
| commit | 450eed7b050496cc6dd981fe20f38915063f8a7c (patch) | |
| tree | d46c0c4439dab28ad6f353c2ad3bc3d166f28ce1 | |
| parent | f5e9dc2da1202c6afc09559e74a63eb4126e7eef (diff) | |
| parent | a323b89bd7296b7774d51774a7426d432735d061 (diff) | |
Merge ODP v1.32.1.0v1.32.1.0_DPDK_19.11
Merge ODP linux-generic v1.32.1.0 into linux-dpdk
136 files changed, 3314 insertions, 1028 deletions
diff --git a/.github/workflows/ci-pipeline-arm64.yml b/.github/workflows/ci-pipeline-arm64.yml index ec7268463..396ab70b8 100644 --- a/.github/workflows/ci-pipeline-arm64.yml +++ b/.github/workflows/ci-pipeline-arm64.yml @@ -19,8 +19,8 @@ jobs: fail-fast: false matrix: cc: [gcc, clang] - conf: ['', 'CFLAGS=-O3', 'CFLAGS=-O1', 'CFLAGS=-O0 --enable-debug=full', '--enable-lto', - '--enable-lto --enable-abi-compat', '--enable-pcapng-support'] + conf: ['', 'CFLAGS=-O3', 'CFLAGS=-O1', 'CFLAGS=-O0 --enable-debug=full', 'CFLAGS=-pedantic', + '--enable-lto', '--enable-lto --enable-abi-compat', '--enable-pcapng-support'] exclude: - cc: clang conf: '--enable-lto' @@ -95,7 +95,7 @@ jobs: fail-fast: false matrix: cc: [gcc, clang] - conf: ['', '--enable-abi-compat', '--enable-deprecated --enable-debug=full', + conf: ['', '--enable-abi-compat', '--enable-deprecated --enable-helper-deprecated --enable-debug=full', '--enable-dpdk-zero-copy --disable-static-applications', '--disable-host-optimization', '--disable-host-optimization --enable-abi-compat', '--without-openssl --without-pcap'] @@ -108,6 +108,23 @@ jobs: if: ${{ failure() }} run: find . -name "*.trs" | xargs grep -l '^.test-result. FAIL' | while read trs ; do echo FAILURE detected at $trs; cat ${trs%%.trs}.log ; done + Run_CFLAGS: + if: ${{ github.repository == 'OpenDataPlane/odp' }} + runs-on: [self-hosted, ARM64] + strategy: + fail-fast: false + matrix: + cc: [gcc, clang] + cflags: ['-march=armv8.2-a -O2', '-march=armv8-a+lse -O2'] + steps: + - uses: AutoModality/action-clean@v1.1.0 + - uses: actions/checkout@v2 + - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${{matrix.cc}}" -e ARCH="${ARCH}" + -e CXX=g++-10 -e CFLAGS="${{matrix.cflags}}" $CONTAINER_NAMESPACE/odp-ci-${OS}-${ARCH}-native /odp/scripts/ci/check.sh + - name: Failure log + if: ${{ failure() }} + run: find . -name "*.trs" | xargs grep -l '^.test-result. FAIL' | while read trs ; do echo FAILURE detected at $trs; cat ${trs%%.trs}.log ; done + Run_OS: if: ${{ github.repository == 'OpenDataPlane/odp' }} diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index 47e8deb1f..99081d12c 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -67,8 +67,8 @@ jobs: fail-fast: false matrix: cc: [gcc, clang] - conf: ['', 'CFLAGS=-O3', 'CFLAGS=-O1', 'CFLAGS=-O0 --enable-debug=full', '--enable-lto', - '--enable-lto --enable-abi-compat', '--enable-pcapng-support'] + conf: ['', 'CFLAGS=-O3', 'CFLAGS=-O1', 'CFLAGS=-O0 --enable-debug=full', 'CFLAGS=-pedantic', + '--enable-lto', '--enable-lto --enable-abi-compat', '--enable-pcapng-support'] exclude: - cc: clang conf: '--enable-lto' @@ -100,6 +100,18 @@ jobs: run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${{matrix.cc}}" -e CONF="${CONF}" $CONTAINER_NAMESPACE/odp-ci-${OS}-${ARCH} /odp/scripts/ci/build_${ARCH}.sh + - name: CFLAGS="-march=armv8.2-a" + env: + CONF: "CFLAGS=-march=armv8.2-a" + run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${{matrix.cc}}" + -e CONF="${CONF}" $CONTAINER_NAMESPACE/odp-ci-${OS}-${ARCH} /odp/scripts/ci/build_${ARCH}.sh + + - name: CFLAGS="-march=armv8-a+lse" + env: + CONF: "CFLAGS=-march=armv8-a+lse" + run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${{matrix.cc}}" + -e CONF="${CONF}" $CONTAINER_NAMESPACE/odp-ci-${OS}-${ARCH} /odp/scripts/ci/build_${ARCH}.sh + - name: Ubuntu 20.04 env: CONF: "--enable-dpdk-shared" @@ -211,7 +223,7 @@ jobs: fail-fast: false matrix: cc: [gcc, clang] - conf: ['', '--enable-abi-compat', '--enable-deprecated --enable-debug=full', + conf: ['', '--enable-abi-compat', '--enable-deprecated --enable-helper-deprecated --enable-debug=full', '--disable-static-applications', '--disable-host-optimization', '--disable-host-optimization --enable-abi-compat', '--without-openssl --without-pcap'] @@ -1,3 +1,49 @@ +== OpenDataPlane (1.32.1.0) +=== Backward compatible API changes +==== Init +* Added `odp_instance()` function for reading the current ODP instance handle + +==== IPsec +* Clarified bit field unions' `all` member intended usage + +==== Packet +* Clarified `odp_proto_chksums_t.all_chksum` intended usage + +==== Packet IO +* Clarified `odp_pktin_hash_proto_t.all_bits` intended usage + +==== Pool +* Added `odp_pool_print_all()` function for printing implementation defined +information about all created pools to the ODP log +* Clarified `odp_pool_param_t.pkt.seg_len` documentation + +==== Stash +* Added 32-bit only `odp_stash_put_u32()` and `odp_stash_get_u32()` functions +* Added 64-bit only `odp_stash_put_u64()` and `odp_stash_get_u64()` functions +* Added pointer only `odp_stash_put_ptr()` and `odp_stash_get_ptr()` functions + +=== Helper (1.3.0) +==== Backward incompatible changes +===== CLI +* Removed unused `instance` parameter from `odph_cli_init()` + +===== Deprecation Framework +Added a deprecation framework to enable controlled API deprecation. + +When a helper API is deprecated, validation tests will be updated to use the +replacement API. By default, attempts to compile code with deprecated helper +APIs will fail. To make moving to a new API version easier, helper library +supports `--enable-helper-deprecated` `configure` option, which makes the +deprecated APIs visible again. + +===== Linux +* Deprecated `odph_odpthreads_create()` function. Use `odph_thread_create()` +instead. +* Deprecated `odph_odpthreads_join()` function. Use `odph_thread_join()` +instead. +* Deprecated unused `odph_thread_param_t.instance` struct member +* Deprecated `odph_odpthread_t` and `odph_odpthread_params_t` types + == OpenDataPlane (1.32.0.0) === Backward incompatible API changes ==== IPsec diff --git a/configure.ac b/configure.ac index 912bc61ef..127e7c591 100644 --- a/configure.ac +++ b/configure.ac @@ -4,8 +4,8 @@ AC_PREREQ([2.5]) ########################################################################## m4_define([odpapi_generation_version], [1]) m4_define([odpapi_major_version], [32]) -m4_define([odpapi_minor_version], [0]) -m4_define([odpapi_point_version], [1]) +m4_define([odpapi_minor_version], [1]) +m4_define([odpapi_point_version], [0]) m4_define([odpapi_version], [odpapi_generation_version.odpapi_major_version.odpapi_minor_version.odpapi_point_version]) AC_INIT([OpenDataPlane],[odpapi_version],[odp@lists.opendataplane.org]) @@ -21,7 +21,7 @@ AC_SUBST(ODP_VERSION_API_MINOR) # Helper library version ########################################################################## m4_define([odph_version_generation], [1]) -m4_define([odph_version_major], [2]) +m4_define([odph_version_major], [3]) m4_define([odph_version_minor], [0]) m4_define([odph_version], @@ -43,7 +43,12 @@ AS_IF([test "$ac_cv_env_CFLAGS_set" = ""], [user_cflags=0], [user_cflags=1]) # Initialize automake AM_INIT_AUTOMAKE([1.9 tar-pax subdir-objects foreign nostdinc -Wall -Werror]) AC_CONFIG_SRCDIR([include/odp/api/spec/init.h]) -AC_CONFIG_HEADERS([include/odp/autoheader_build.h include/odp/autoheader_external.h include/odp/autoheader_internal.h helper/include/odp/helper/autoheader_external.h]) +AC_CONFIG_HEADERS(m4_normalize([ + include/odp/autoheader_build.h + include/odp/autoheader_external.h + include/odp/autoheader_internal.h + helper/include/odp/helper/autoheader_external.h +])) AC_USE_SYSTEM_EXTENSIONS AC_SYS_LARGEFILE @@ -89,11 +94,41 @@ PKG_PROG_PKG_CONFIG dnl breaks cross-compilation and malloc(0) behaviour is not that important dnl AC_FUNC_MALLOC AC_FUNC_MMAP -AC_CHECK_FUNCS([bzero clock_gettime gethostbyname getpagesize gettimeofday malloc memset munmap socket strchr strerror strrchr strstr strtoull]) +AC_CHECK_FUNCS(m4_normalize([ + bzero + clock_gettime + gethostbyname + getpagesize + gettimeofday + malloc + memset + munmap + socket + strchr + strerror + strrchr + strstr + strtoull +])) # Checks for header files. AC_HEADER_RESOLV -AC_CHECK_HEADERS([arpa/inet.h fcntl.h inttypes.h limits.h netdb.h netinet/in.h stddef.h stdint.h stdlib.h string.h sys/ioctl.h sys/socket.h sys/time.h unistd.h]) +AC_CHECK_HEADERS(m4_normalize([ + arpa/inet.h + fcntl.h + inttypes.h + limits.h + netdb.h + netinet/in.h + stddef.h + stdint.h + stdlib.h + string.h + sys/ioctl.h + sys/socket.h + sys/time.h + unistd.h +])) # Checks for typedefs, structures, and compiler characteristics. AC_HEADER_STDBOOL @@ -137,7 +172,7 @@ AS_IF([test "$GCC" == yes], ) ) -ODP_CFLAGS="$ODP_CFLAGS -std=c99 -D_GNU_SOURCE" +ODP_CFLAGS="$ODP_CFLAGS -std=c11 -D_GNU_SOURCE" ODP_CXXFLAGS="$ODP_CXXFLAGS -std=c++11" # Extra flags for example to suppress certain warning types @@ -506,6 +541,7 @@ AC_MSG_RESULT([ ABI compatible: ${abi_compat} link time optimization: ${lto_enabled} deprecated APIs: ${deprecated} + deprecated helper APIs: ${enable_helper_deprecated} debug: ${debug_settings} cunit: ${cunit_support} static tests linkage: ${enable_static_applications} diff --git a/doc/helper-guide/Doxyfile b/doc/helper-guide/Doxyfile index 52d542e73..74eb6b94f 100644 --- a/doc/helper-guide/Doxyfile +++ b/doc/helper-guide/Doxyfile @@ -35,3 +35,4 @@ MACRO_EXPANSION = YES EXPAND_ONLY_PREDEF = YES INTERNAL_DOCS = YES DOT_IMAGE_FORMAT = svg +PREDEFINED = ODPH_DEPRECATE(x)=x diff --git a/doc/images/pktio_fsm.gv b/doc/images/pktio_fsm.gv index 09199c607..b28fbb0d4 100644 --- a/doc/images/pktio_fsm.gv +++ b/doc/images/pktio_fsm.gv @@ -1,6 +1,5 @@ digraph pktio_state_machine { rankdir=LR; - size="9,12"; node [fontsize=28]; edge [fontsize=28]; node [shape=doublecircle]; Unallocated Ready; diff --git a/doc/images/timeout_fsm.gv b/doc/images/timeout_fsm.gv index 21ecb59d9..dab24d0d6 100644 --- a/doc/images/timeout_fsm.gv +++ b/doc/images/timeout_fsm.gv @@ -1,6 +1,5 @@ digraph timer_state_machine { rankdir=LR; - size="12,20"; node [fontsize=28]; edge [fontsize=28]; node [shape=doublecircle]; TO_Unalloc; diff --git a/doc/images/timer_fsm.gv b/doc/images/timer_fsm.gv index 1798d31f2..819d0f979 100644 --- a/doc/images/timer_fsm.gv +++ b/doc/images/timer_fsm.gv @@ -1,6 +1,5 @@ digraph timer_state_machine { rankdir=LR; - size="12,20"; node [fontsize=28]; edge [fontsize=28]; node [shape=doublecircle]; Timer_Unalloc; diff --git a/doc/implementers-guide/implementers-guide.adoc b/doc/implementers-guide/implementers-guide.adoc index 398934482..0b039630c 100644 --- a/doc/implementers-guide/implementers-guide.adoc +++ b/doc/implementers-guide/implementers-guide.adoc @@ -182,9 +182,9 @@ divided in two distinct areas: This grouping defines tests that are expected to be executable and succeed on any platform, though possibly with very different performance, depending on the underlying platform. They are written in plain C code, and may only use -functions defined in the standard libC (C99) library (besides the ODP -functions being tested, of course). A free C99 specification can be found at -the http://www.open-std.org/JTC1/sc22/wg14/www/docs/n1256.pdf[open-std.org] +functions defined in the standard libC (C11) library (besides the ODP +functions being tested, of course). A free C11 draft specification can be found +at the http://www.open-std.org/jtc1/sc22/wg14/www/docs/n1570.pdf[open-std.org] web site. No other languages (like scripting) are allowed as their usage would make assumptions on the platform capability. diff --git a/example/cli/odp_cli.c b/example/cli/odp_cli.c index 48badb5cd..5f87b2a78 100644 --- a/example/cli/odp_cli.c +++ b/example/cli/odp_cli.c @@ -160,7 +160,7 @@ int main(int argc, char *argv[]) cli_param.port = opt.port; /* Initialize CLI helper. */ - if (odph_cli_init(inst, &cli_param)) { + if (odph_cli_init(&cli_param)) { ODPH_ERR("CLI helper initialization failed.\n"); exit(EXIT_FAILURE); } diff --git a/example/generator/odp_generator.c b/example/generator/odp_generator.c index 8ad73601f..7a070235b 100644 --- a/example/generator/odp_generator.c +++ b/example/generator/odp_generator.c @@ -980,7 +980,7 @@ static int gen_recv_direct_thread(void *arg) } #define COUNTER_SUM(_c, _nw) \ -({ \ +__extension__ ({ \ int _itr; \ uint64_t _result = 0; \ \ diff --git a/example/ipfragreass/odp_ipfragreass_atomics.h b/example/ipfragreass/odp_ipfragreass_atomics.h index 7db64911f..743a004e2 100644 --- a/example/ipfragreass/odp_ipfragreass_atomics.h +++ b/example/ipfragreass/odp_ipfragreass_atomics.h @@ -42,9 +42,11 @@ static inline bool atomic_strong_cas_dblptr(uint64_t *var, uint64_t *exp, * * @return Whether the operation succeeded */ -static inline bool atomic_strong_cas_dblptr(__int128 *var, __int128 *exp, - __int128 neu, int mo_success, - int mo_failure) +__extension__ static inline bool atomic_strong_cas_dblptr(__int128 *var, + __int128 *exp, + __int128 neu, + int mo_success, + int mo_failure) { return __atomic_compare_exchange_n(var, exp, neu, 0, mo_success, mo_failure); diff --git a/example/ipfragreass/odp_ipfragreass_atomics_arm.h b/example/ipfragreass/odp_ipfragreass_atomics_arm.h index 5d61909ab..4bea45f9e 100644 --- a/example/ipfragreass/odp_ipfragreass_atomics_arm.h +++ b/example/ipfragreass/odp_ipfragreass_atomics_arm.h @@ -10,9 +10,12 @@ #include <odp_api.h> #if __SIZEOF_POINTER__ == 8 && defined(__aarch64__) -static inline __int128 lld(__int128 *var, int mo) + +__extension__ typedef __int128 _int128_t; + +static inline _int128_t lld(_int128_t *var, int mo) { - __int128 old; + _int128_t old; uint64_t lo, hi; if (mo == __ATOMIC_ACQUIRE) @@ -29,7 +32,7 @@ static inline __int128 lld(__int128 *var, int mo) } -static inline uint32_t scd(__int128 *var, __int128 neu, int mo) +static inline uint32_t scd(_int128_t *var, _int128_t neu, int mo) { uint32_t ret; uint64_t lo = neu, hi = neu >> 64; @@ -43,12 +46,12 @@ static inline uint32_t scd(__int128 *var, __int128 neu, int mo) return ret; } -static inline bool atomic_strong_cas_dblptr(__int128 *var, __int128 *exp, - __int128 neu, int mo_success, +static inline bool atomic_strong_cas_dblptr(_int128_t *var, _int128_t *exp, + _int128_t neu, int mo_success, int mo_failure ODP_UNUSED) { - register __int128 old; - register __int128 expected = *exp; + register _int128_t old; + register _int128_t expected = *exp; int ll_mo, sc_mo; ll_mo = (mo_success != __ATOMIC_RELAXED && diff --git a/example/ipfragreass/odp_ipfragreass_reassemble.h b/example/ipfragreass/odp_ipfragreass_reassemble.h index 26826edf1..e0e132b6c 100644 --- a/example/ipfragreass/odp_ipfragreass_reassemble.h +++ b/example/ipfragreass/odp_ipfragreass_reassemble.h @@ -125,7 +125,7 @@ union fraglist { struct { uint64_t half[2]; }; - __int128 raw; + __extension__ __int128 raw; #endif }; diff --git a/example/ipsec_crypto/odp_ipsec_misc.h b/example/ipsec_crypto/odp_ipsec_misc.h index 23b89ae84..4f50f27ec 100644 --- a/example/ipsec_crypto/odp_ipsec_misc.h +++ b/example/ipsec_crypto/odp_ipsec_misc.h @@ -255,7 +255,7 @@ char *mac_addr_str(char *b, uint8_t *mac) static inline int parse_mac_string(char *macaddress, uint8_t *mac) { - int macwords[ODPH_ETHADDR_LEN]; + unsigned int macwords[ODPH_ETHADDR_LEN]; int converted; converted = sscanf(macaddress, diff --git a/example/ipsec_crypto/odp_ipsec_stream.c b/example/ipsec_crypto/odp_ipsec_stream.c index 110f7d5df..a24d4b4f9 100644 --- a/example/ipsec_crypto/odp_ipsec_stream.c +++ b/example/ipsec_crypto/odp_ipsec_stream.c @@ -31,7 +31,7 @@ */ typedef struct ODP_PACKED stream_pkt_hdr_s { odp_u64be_t magic; /**< Stream magic value for verification */ - uint8_t data[0]; /**< Incrementing data stream */ + uint8_t data[]; /**< Incrementing data stream */ } stream_pkt_hdr_t; stream_db_t *stream_db; diff --git a/example/timer/odp_timer_test.c b/example/timer/odp_timer_test.c index b10559e6e..2efe55e5a 100644 --- a/example/timer/odp_timer_test.c +++ b/example/timer/odp_timer_test.c @@ -65,7 +65,7 @@ static const char *timerset2str(odp_timer_set_t val) default: return "?"; } -}; +} /** @private test timeout */ static void remove_prescheduled_events(void) diff --git a/example/traffic_mgmt/odp_traffic_mgmt.c b/example/traffic_mgmt/odp_traffic_mgmt.c index 521338785..c82c1d5b2 100644 --- a/example/traffic_mgmt/odp_traffic_mgmt.c +++ b/example/traffic_mgmt/odp_traffic_mgmt.c @@ -73,7 +73,16 @@ static profile_params_set_t COMPANY_PROFILE_PARAMS = { }, .wred_params = { - [ODP_PACKET_GREEN ... ODP_PACKET_YELLOW] = { + [ODP_PACKET_GREEN] = { + .min_threshold = PERCENT(70), + .med_threshold = PERCENT(90), + .med_drop_prob = PERCENT(80), + .max_drop_prob = PERCENT(100), + .enable_wred = TRUE, + .use_byte_fullness = FALSE, + }, + + [ODP_PACKET_YELLOW] = { .min_threshold = PERCENT(70), .med_threshold = PERCENT(90), .med_drop_prob = PERCENT(80), @@ -106,7 +115,16 @@ static profile_params_set_t COS0_PROFILE_PARAMS = { }, .wred_params = { - [ODP_PACKET_GREEN ... ODP_PACKET_YELLOW] = { + [ODP_PACKET_GREEN] = { + .min_threshold = PERCENT(80), + .med_threshold = PERCENT(90), + .med_drop_prob = PERCENT(50), + .max_drop_prob = PERCENT(100), + .enable_wred = TRUE, + .use_byte_fullness = FALSE, + }, + + [ODP_PACKET_YELLOW] = { .min_threshold = PERCENT(80), .med_threshold = PERCENT(90), .med_drop_prob = PERCENT(50), @@ -139,7 +157,16 @@ static profile_params_set_t COS1_PROFILE_PARAMS = { }, .wred_params = { - [ODP_PACKET_GREEN ... ODP_PACKET_YELLOW] = { + [ODP_PACKET_GREEN] = { + .min_threshold = PERCENT(40), + .med_threshold = PERCENT(90), + .med_drop_prob = PERCENT(70), + .max_drop_prob = PERCENT(100), + .enable_wred = TRUE, + .use_byte_fullness = FALSE, + }, + + [ODP_PACKET_YELLOW] = { .min_threshold = PERCENT(40), .med_threshold = PERCENT(90), .med_drop_prob = PERCENT(70), @@ -172,7 +199,16 @@ static profile_params_set_t COS2_PROFILE_PARAMS = { }, .wred_params = { - [ODP_PACKET_GREEN ... ODP_PACKET_YELLOW] = { + [ODP_PACKET_GREEN] = { + .min_threshold = PERCENT(50), + .med_threshold = PERCENT(80), + .med_drop_prob = PERCENT(70), + .max_drop_prob = PERCENT(100), + .enable_wred = TRUE, + .use_byte_fullness = FALSE, + }, + + [ODP_PACKET_YELLOW] = { .min_threshold = PERCENT(50), .med_threshold = PERCENT(80), .med_drop_prob = PERCENT(70), @@ -205,7 +241,16 @@ static profile_params_set_t COS3_PROFILE_PARAMS = { }, .wred_params = { - [ODP_PACKET_GREEN ... ODP_PACKET_YELLOW] = { + [ODP_PACKET_GREEN] = { + .min_threshold = PERCENT(40), + .med_threshold = PERCENT(70), + .med_drop_prob = PERCENT(80), + .max_drop_prob = PERCENT(100), + .enable_wred = TRUE, + .use_byte_fullness = FALSE, + }, + + [ODP_PACKET_YELLOW] = { .min_threshold = PERCENT(40), .med_threshold = PERCENT(70), .med_drop_prob = PERCENT(80), @@ -578,13 +623,17 @@ static uint32_t pkt_service_class(void) * of delayed traffic so as to stimulate more interesting behaviors. */ rand8 = random_8(); - switch (rand8) { - case 0 ... 24: return 0; - case 25 ... 49: return 1; - case 50 ... 150: return 2; - case 151 ... 255: return 3; - default: return 3; - } + + if (rand8 <= 24) + return 0; + else if (rand8 >= 25 && rand8 <= 49) + return 1; + else if (rand8 >= 50 && rand8 <= 150) + return 2; + else if (rand8 >= 151 && rand8 <= 255) + return 3; + else + return 3; } static odp_packet_t make_odp_packet(uint16_t pkt_len) diff --git a/helper/Makefile.am b/helper/Makefile.am index 56fd5bd6b..8e410703c 100644 --- a/helper/Makefile.am +++ b/helper/Makefile.am @@ -18,6 +18,7 @@ AM_LDFLAGS = -version-number '$(ODPHELPER_LIBSO_VERSION)' helperincludedir = $(includedir)/odp/helper/ helperinclude_HEADERS = \ include/odp/helper/autoheader_external.h\ + include/odp/helper/deprecated.h\ include/odp/helper/chksum.h\ include/odp/helper/odph_debug.h \ include/odp/helper/eth.h\ @@ -84,7 +85,7 @@ __LIB__libodphelper_la_LIBADD += $(LIBCLI_LIBS) lib_LTLIBRARIES = $(LIB)/libodphelper.la -CHECK_GLOBALS_REGEX = " odph_" +CHECK_GLOBALS_REGEX = " (odph_|_deprecated_odph_)" TESTS_ENVIRONMENT = \ LIBTOOL="$(LIBTOOL)" \ diff --git a/helper/cli.c b/helper/cli.c index 8dc6b66fa..97fa232a3 100644 --- a/helper/cli.c +++ b/helper/cli.c @@ -43,11 +43,10 @@ typedef struct { /* Guards cli_fd and run, which must be accessed atomically. */ odp_spinlock_t lock; odp_spinlock_t api_lock; - odp_instance_t instance; odph_cli_param_t cli_param; struct sockaddr_in addr; uint32_t num_user_commands; - user_cmd_t user_cmd[0]; + user_cmd_t user_cmd[]; } cli_shm_t; static const char *shm_name = "_odp_cli"; @@ -75,7 +74,7 @@ static cli_shm_t *shm_lookup(void) return shm; } -int odph_cli_init(odp_instance_t instance, const odph_cli_param_t *param) +int odph_cli_init(const odph_cli_param_t *param) { if (odp_shm_lookup(shm_name) != ODP_SHM_INVALID) { ODPH_ERR("Error: shm %s already exists\n", shm_name); @@ -101,7 +100,6 @@ int odph_cli_init(odp_instance_t instance, const odph_cli_param_t *param) odp_spinlock_init(&shm->api_lock); shm->listen_fd = -1; shm->cli_fd = -1; - shm->instance = instance; shm->addr.sin_family = AF_INET; shm->addr.sin_port = htons(param->port); diff --git a/helper/hashtable.c b/helper/hashtable.c index ac11acb15..39da586a8 100644 --- a/helper/hashtable.c +++ b/helper/hashtable.c @@ -40,7 +40,7 @@ typedef struct odph_hash_node { * its structure is like: * k_byte1 k_byte2...k_byten v_byte1...v_bytem */ - char content[0]; + char content[]; } odph_hash_node; typedef struct { diff --git a/helper/include/odp/helper/autoheader_external.h.in b/helper/include/odp/helper/autoheader_external.h.in index 6f5187a5b..81f10a737 100644 --- a/helper/include/odp/helper/autoheader_external.h.in +++ b/helper/include/odp/helper/autoheader_external.h.in @@ -11,4 +11,7 @@ /* Define to 1 to display helper debug information */ #undef ODPH_DEBUG_PRINT +/* Define to 1 to enable deprecated helper API definitions */ +#undef ODPH_DEPRECATED_API + #endif diff --git a/helper/include/odp/helper/cli.h b/helper/include/odp/helper/cli.h index 982509f53..b2f88d7bc 100644 --- a/helper/include/odp/helper/cli.h +++ b/helper/include/odp/helper/cli.h @@ -76,12 +76,11 @@ void odph_cli_param_init(odph_cli_param_t *param); * In process mode (ODPH_PROC_MODE), this function must be called before * creating the thread which calls odph_cli_run(). * - * @param instance ODP instance * @param param CLI server parameters to use * @retval 0 Success * @retval <0 Failure */ -int odph_cli_init(odp_instance_t instance, const odph_cli_param_t *param); +int odph_cli_init(const odph_cli_param_t *param); /** * Register a user defined command diff --git a/helper/include/odp/helper/deprecated.h b/helper/include/odp/helper/deprecated.h new file mode 100644 index 000000000..3415b2127 --- /dev/null +++ b/helper/include/odp/helper/deprecated.h @@ -0,0 +1,39 @@ +/* Copyright (c) 2017-2018, Linaro Limited + * Copyright (c) 2021, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +/** + * @file + * + * Macro for deprecated API definitions + */ + +#ifndef ODPH_DEPRECATED_H_ +#define ODPH_DEPRECATED_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <odp/helper/autoheader_external.h> + +/** + * @def ODPH_DEPRECATE + * + * Macro to deprecate helper API definitions + */ + +#if ODPH_DEPRECATED_API +#define ODPH_DEPRECATE(x) x +#else +#define ODPH_DEPRECATE(x) _deprecated_ ## x +#endif + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/helper/include/odp/helper/ipsec.h b/helper/include/odp/helper/ipsec.h index 1b2dbb77b..b149ce9ff 100644 --- a/helper/include/odp/helper/ipsec.h +++ b/helper/include/odp/helper/ipsec.h @@ -35,7 +35,7 @@ extern "C" { typedef struct ODP_PACKED { odp_u32be_t spi; /**< Security Parameter Index */ odp_u32be_t seq_no; /**< Sequence Number */ - uint8_t iv[0]; /**< Initialization vector */ + uint8_t iv[]; /**< Initialization vector */ } odph_esphdr_t; /** @internal Compile time assert */ @@ -48,7 +48,7 @@ ODP_STATIC_ASSERT(sizeof(odph_esphdr_t) == ODPH_ESPHDR_LEN, typedef struct ODP_PACKED { uint8_t pad_len; /**< Padding length (0-255) */ uint8_t next_header; /**< Next header protocol */ - uint8_t icv[0]; /**< Integrity Check Value (optional) */ + uint8_t icv[]; /**< Integrity Check Value (optional) */ } odph_esptrl_t; /** @internal Compile time assert */ @@ -64,7 +64,7 @@ typedef struct ODP_PACKED { odp_u16be_t pad; /**< Padding (must be 0) */ odp_u32be_t spi; /**< Security Parameter Index */ odp_u32be_t seq_no; /**< Sequence Number */ - uint8_t icv[0]; /**< Integrity Check Value */ + uint8_t icv[]; /**< Integrity Check Value */ } odph_ahhdr_t; /** @internal Compile time assert */ diff --git a/helper/include/odp/helper/odph_api.h b/helper/include/odp/helper/odph_api.h index 7ab875c6e..3c321b9d4 100644 --- a/helper/include/odp/helper/odph_api.h +++ b/helper/include/odp/helper/odph_api.h @@ -18,6 +18,8 @@ extern "C" { #endif +#include <odp/helper/autoheader_external.h> + #include <odp/helper/odph_debug.h> #include <odp/helper/chksum.h> #include <odp/helper/odph_cuckootable.h> @@ -38,8 +40,6 @@ extern "C" { #include <odp/helper/udp.h> #include <odp/helper/version.h> -#include <odp/helper/autoheader_external.h> - #ifdef ODPH_CLI #include <odp/helper/cli.h> #endif diff --git a/helper/include/odp/helper/odph_debug.h b/helper/include/odp/helper/odph_debug.h index 39ce3f0d9..bedef2f66 100644 --- a/helper/include/odp/helper/odph_debug.h +++ b/helper/include/odp/helper/odph_debug.h @@ -28,6 +28,10 @@ extern "C" { * @{ */ +/* Avoid "ISO C99 requires at least one argument for the "..." in a variadic + * macro" errors when building with 'pedantic' option. */ +#pragma GCC system_header + /** * Assert macro for applications and helper code * diff --git a/helper/include/odp/helper/threads.h b/helper/include/odp/helper/threads.h index d7cb0ac2a..f16626abf 100644 --- a/helper/include/odp/helper/threads.h +++ b/helper/include/odp/helper/threads.h @@ -23,6 +23,8 @@ extern "C" { #endif +#include <odp/helper/deprecated.h> + #include <pthread.h> #include <getopt.h> #include <sys/types.h> @@ -67,7 +69,7 @@ typedef struct { odp_thread_type_t thr_type; /** @deprecated ODP instance handle for odph_odpthreads_create(). */ - odp_instance_t instance; + odp_instance_t ODPH_DEPRECATE(instance); /** * Minimum stack size in bytes. 0 = use default. Ignored by @@ -128,10 +130,10 @@ typedef struct { } odph_helper_options_t; /** @deprecated Legacy thread table entry */ -typedef odph_thread_t odph_odpthread_t; +typedef odph_thread_t ODPH_DEPRECATE(odph_odpthread_t); /** @deprecated Legacy thread parameters */ -typedef odph_thread_param_t odph_odpthread_params_t; +typedef odph_thread_param_t ODPH_DEPRECATE(odph_odpthread_params_t); /** Common parameters for odph_thread_create() call */ typedef struct { @@ -292,9 +294,10 @@ int odph_thread_join(odph_thread_t thread[], int num); * * @deprecated Use odph_thread_create() instead. */ -int odph_odpthreads_create(odph_odpthread_t *thread_tbl, - const odp_cpumask_t *mask, - const odph_odpthread_params_t *thr_params); +int ODPH_DEPRECATE(odph_odpthreads_create)( + ODPH_DEPRECATE(odph_odpthread_t) *thread_tbl, + const odp_cpumask_t *mask, + const ODPH_DEPRECATE(odph_odpthread_params_t) *thr_params); /** * Waits odpthreads (as linux threads or processes) to exit. @@ -308,7 +311,8 @@ int odph_odpthreads_create(odph_odpthread_t *thread_tbl, * * @deprecated Use odph_thread_join() instead. */ -int odph_odpthreads_join(odph_odpthread_t *thread_tbl); +int ODPH_DEPRECATE(odph_odpthreads_join)( + ODPH_DEPRECATE(odph_odpthread_t) *thread_tbl); /** * Set CPU affinity of the current odp thread diff --git a/helper/m4/configure.m4 b/helper/m4/configure.m4 index 36629c81e..2e1076172 100644 --- a/helper/m4/configure.m4 +++ b/helper/m4/configure.m4 @@ -47,5 +47,17 @@ AS_IF([test "x$enable_helper_debug_print" != "xno"], [ODPH_DEBUG_PRINT=1], AC_DEFINE_UNQUOTED([ODPH_DEBUG_PRINT], [$ODPH_DEBUG_PRINT], [Define to 1 to display helper debug information]) +########################################################################## +# Enable/disable deprecated helper API definitions +########################################################################## +AC_ARG_ENABLE([helper-deprecated], + [AS_HELP_STRING([--enable-helper-deprecated], + [enable deprecated helper API definitions [default=disabled]])], + [], [enable_helper_deprecated=no]) +AS_IF([test "x$enable_helper_deprecated" != "xno"], [ODPH_DEPRECATED_API=1], + [ODPH_DEPRECATED_API=0]) +AC_DEFINE_UNQUOTED([ODPH_DEPRECATED_API], [$ODPH_DEPRECATED_API], + [Define to 1 to enable deprecated helper API definitions]) + AC_CONFIG_FILES([helper/libodphelper.pc helper/test/Makefile]) diff --git a/helper/test/cli.c b/helper/test/cli.c index 475a99b90..4fa403f5a 100644 --- a/helper/test/cli.c +++ b/helper/test/cli.c @@ -48,7 +48,7 @@ int main(int argc, char *argv[]) odph_cli_param_init(&cli_param); - if (odph_cli_init(instance, &cli_param)) { + if (odph_cli_init(&cli_param)) { ODPH_ERR("Error: odph_cli_init() failed.\n"); exit(EXIT_FAILURE); } diff --git a/helper/test/odpthreads.c b/helper/test/odpthreads.c index aa06df188..b7b4c2104 100644 --- a/helper/test/odpthreads.c +++ b/helper/test/odpthreads.c @@ -28,7 +28,7 @@ static void main_exit(void); /* ODP application instance */ -static odp_instance_t odp_instance; +static odp_instance_t instance; static int worker_fn(void *arg ODP_UNUSED) { @@ -91,12 +91,12 @@ int main(int argc, char *argv[]) odp_init_param_init(&init_param); init_param.mem_model = helper_options.mem_model; - if (odp_init_global(&odp_instance, &init_param, NULL)) { + if (odp_init_global(&instance, &init_param, NULL)) { ODPH_ERR("Error: ODP global init failed.\n"); exit(EXIT_FAILURE); } - if (odp_init_local(odp_instance, ODP_THREAD_CONTROL)) { + if (odp_init_local(instance, ODP_THREAD_CONTROL)) { ODPH_ERR("Error: ODP local init failed.\n"); exit(EXIT_FAILURE); } @@ -154,7 +154,7 @@ int main(int argc, char *argv[]) printf("new num worker threads: %i\n\n", num_workers); odph_thread_common_param_init(&thr_common); - thr_common.instance = odp_instance; + thr_common.instance = instance; thr_common.cpumask = &cpu_mask; thr_common.share_param = 1; @@ -213,7 +213,7 @@ static void main_exit(void) _exit(EXIT_FAILURE); } - if (odp_term_global(odp_instance)) { + if (odp_term_global(instance)) { ODPH_ERR("Error: ODP global term failed.\n"); _exit(EXIT_FAILURE); } diff --git a/helper/threads.c b/helper/threads.c index 9326e12e6..221709a04 100644 --- a/helper/threads.c +++ b/helper/threads.c @@ -43,7 +43,7 @@ static void *run_thread(void *arg) int status; int ret; odp_instance_t instance; - odph_odpthread_params_t *thr_params; + ODPH_DEPRECATE(odph_odpthread_params_t) *thr_params; odph_thread_start_args_t *start_args = arg; @@ -395,9 +395,10 @@ int odph_thread_join(odph_thread_t thread[], int num) /* * create an odpthread set (as linux processes or linux threads or both) */ -int odph_odpthreads_create(odph_odpthread_t *thread_tbl, - const odp_cpumask_t *mask, - const odph_odpthread_params_t *thr_params) +int ODPH_DEPRECATE(odph_odpthreads_create)( + ODPH_DEPRECATE(odph_odpthread_t) *thread_tbl, + const odp_cpumask_t *mask, + const ODPH_DEPRECATE(odph_odpthread_params_t) *thr_params) { int i; int num; @@ -406,7 +407,7 @@ int odph_odpthreads_create(odph_odpthread_t *thread_tbl, num = odp_cpumask_count(mask); - memset(thread_tbl, 0, num * sizeof(odph_odpthread_t)); + memset(thread_tbl, 0, num * sizeof(*thread_tbl)); cpu_count = odp_cpu_count(); @@ -425,7 +426,7 @@ int odph_odpthreads_create(odph_odpthread_t *thread_tbl, /* Copy thread parameters */ start_args->thr_params = *thr_params; - start_args->instance = thr_params->instance; + start_args->instance = thr_params->ODPH_DEPRECATE(instance); if (helper_options.mem_model == ODP_MEM_MODEL_THREAD) { if (create_pthread(&thread_tbl[i], cpu, 0)) @@ -445,7 +446,8 @@ int odph_odpthreads_create(odph_odpthread_t *thread_tbl, /* * wait for the odpthreads termination (linux processes and threads) */ -int odph_odpthreads_join(odph_odpthread_t *thread_tbl) +int ODPH_DEPRECATE(odph_odpthreads_join)( + ODPH_DEPRECATE(odph_odpthread_t) *thread_tbl) { pid_t pid; int i = 0; diff --git a/include/odp/api/spec/init.h b/include/odp/api/spec/init.h index 67e616fce..69ca55767 100644 --- a/include/odp/api/spec/init.h +++ b/include/odp/api/spec/init.h @@ -1,5 +1,5 @@ /* Copyright (c) 2013-2018, Linaro Limited - * Copyright (c) 2019, Nokia + * Copyright (c) 2019-2021, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -36,11 +36,21 @@ extern "C" { * ODP log level. */ typedef enum { + /** Debug */ ODP_LOG_DBG, + + /** Error */ ODP_LOG_ERR, + + /** Unimplemented */ ODP_LOG_UNIMPLEMENTED, + + /** Abort */ ODP_LOG_ABORT, + + /** Print */ ODP_LOG_PRINT + } odp_log_level_t; /** @@ -346,6 +356,20 @@ int odp_term_global(odp_instance_t instance); void odp_log_thread_fn_set(odp_log_func_t func); /** + * Get instance handle + * + * A successful call outputs the calling thread's ODP instance handle. + * + * @param[out] instance Instance handle pointer for output + * + * @retval 0 on success + * @retval <0 on failure + * + * @see odp_init_global(), odp_init_local() + */ +int odp_instance(odp_instance_t *instance); + +/** * @} */ diff --git a/include/odp/api/spec/ipsec.h b/include/odp/api/spec/ipsec.h index 6e28e7fb4..f66758341 100644 --- a/include/odp/api/spec/ipsec.h +++ b/include/odp/api/spec/ipsec.h @@ -1296,7 +1296,10 @@ typedef struct odp_ipsec_warn_t { uint32_t soft_exp_packets : 1; }; - /** All warnings bits */ + /** All warning bits + * + * This field can be used to set/clear all bits, or to perform + * bitwise operations over those. */ uint32_t all; }; @@ -1335,7 +1338,10 @@ typedef struct odp_ipsec_op_flag_t { }; - /** All flag bits */ + /** All flag bits + * + * This field can be used to set/clear all flags, or to perform + * bitwise operations over those. */ uint32_t all; }; @@ -1373,7 +1379,10 @@ typedef struct odp_ipsec_out_opt_t { } flag; - /** All flag bits */ + /** All flag bits + * + * This field can be used to set/clear all flags, or to perform + * bitwise operations over those. */ uint32_t all_flags; }; diff --git a/include/odp/api/spec/packet_io.h b/include/odp/api/spec/packet_io.h index 2ccd7b4c8..0d5938e9e 100644 --- a/include/odp/api/spec/packet_io.h +++ b/include/odp/api/spec/packet_io.h @@ -134,7 +134,10 @@ typedef union odp_pktin_hash_proto_t { uint32_t ipv6 : 1; } proto; - /** All bits of the bit field structure */ + /** All bits of the bit field structure + * + * This field can be used to set/clear all bits, or to perform bitwise + * operations over those. */ uint32_t all_bits; } odp_pktin_hash_proto_t; diff --git a/include/odp/api/spec/packet_types.h b/include/odp/api/spec/packet_types.h index 52b0e22b6..113f24d94 100644 --- a/include/odp/api/spec/packet_types.h +++ b/include/odp/api/spec/packet_types.h @@ -324,7 +324,10 @@ typedef union odp_proto_chksums_t { } chksum; - /** All checksum bits. This can be used to set/clear all flags. */ + /** All checksum bits + * + * This field can be used to set/clear all flags, or to perform bitwise + * operations over those. */ uint32_t all_chksum; } odp_proto_chksums_t; diff --git a/include/odp/api/spec/pool.h b/include/odp/api/spec/pool.h index 6314b827c..6ebfafe05 100644 --- a/include/odp/api/spec/pool.h +++ b/include/odp/api/spec/pool.h @@ -105,6 +105,14 @@ int odp_pool_info(odp_pool_t pool, odp_pool_info_t *info); void odp_pool_print(odp_pool_t pool); /** + * Print debug info about all pools + * + * Print implementation defined information about all created pools to the ODP + * log. The information is intended to be used for debugging. + */ +void odp_pool_print_all(void); + +/** * Get printable value for an odp_pool_t * * @param hdl odp_pool_t handle to be printed diff --git a/include/odp/api/spec/pool_types.h b/include/odp/api/spec/pool_types.h index 44d9297c1..583a81b67 100644 --- a/include/odp/api/spec/pool_types.h +++ b/include/odp/api/spec/pool_types.h @@ -418,8 +418,9 @@ typedef struct odp_pool_param_t { */ uint32_t align; - /** Minimum number of packet data bytes that are stored in the - * first segment of a packet. The maximum value is defined by + /** Minimum number of packet data bytes that can be stored in + * the first segment of a newly allocated packet (starting from + * odp_packet_data()). The maximum value is defined by * pool capability pkt.max_seg_len. Use 0 for default. */ uint32_t seg_len; diff --git a/include/odp/api/spec/protocols.h b/include/odp/api/spec/protocols.h index 71fc59909..95c9bc0ed 100644 --- a/include/odp/api/spec/protocols.h +++ b/include/odp/api/spec/protocols.h @@ -19,11 +19,9 @@ extern "C" { #endif /** - * @addtogroup odp_protocols - * @details - * <b> Protocols </b> - * - * @{ + * @defgroup odp_protocols ODP PROTOCOLS + * Network protocols + * @{ */ /** IPv4 address size */ diff --git a/include/odp/api/spec/stash.h b/include/odp/api/spec/stash.h index 42b3bd9d7..12214e200 100644 --- a/include/odp/api/spec/stash.h +++ b/include/odp/api/spec/stash.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2020, Nokia +/* Copyright (c) 2020-2021, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -307,6 +307,55 @@ uint64_t odp_stash_to_u64(odp_stash_t stash); int32_t odp_stash_put(odp_stash_t stash, const void *obj, int32_t num); /** + * Put 32-bit integers into a stash + * + * Otherwise like odp_stash_put(), except that this function operates on 32-bit + * integers. The stash must have been created with 'obj_size' of 4. + * + * @param stash Stash handle + * @param u32 Points to an array of 32-bit integers to be stored. The array + * must be 32-bit aligned in memory. + * @param num Number of integers to store + * + * @return Number of integers actually stored (0 ... num) + * @retval <0 on failure + */ +int32_t odp_stash_put_u32(odp_stash_t stash, const uint32_t u32[], int32_t num); + +/** + * Put 64-bit integers into a stash + * + * Otherwise like odp_stash_put(), except that this function operates on 64-bit + * integers. The stash must have been created with 'obj_size' of 8. + * + * @param stash Stash handle + * @param u64 Points to an array of 64-bit integers to be stored. The array + * must be 64-bit aligned in memory. + * @param num Number of integers to store + * + * @return Number of integers actually stored (0 ... num) + * @retval <0 on failure + */ +int32_t odp_stash_put_u64(odp_stash_t stash, const uint64_t u64[], int32_t num); + +/** + * Put pointers into a stash + * + * Otherwise like odp_stash_put(), except that this function operates on + * pointers. The stash must have been created with 'obj_size' matching to the + * size of uintptr_t. + * + * @param stash Stash handle + * @param ptr Points to an array of pointers to be stored. The array must be + * pointer size aligned in memory. + * @param num Number of pointers to store + * + * @return Number of pointers actually stored (0 ... num) + * @retval <0 on failure + */ +int32_t odp_stash_put_ptr(odp_stash_t stash, const uintptr_t ptr[], int32_t num); + +/** * Get object handles from a stash * * Get previously stored object handles from the stash. Application specifies @@ -325,6 +374,55 @@ int32_t odp_stash_put(odp_stash_t stash, const void *obj, int32_t num); int32_t odp_stash_get(odp_stash_t stash, void *obj, int32_t num); /** + * Get 32-bit integers from a stash + * + * Otherwise like odp_stash_get(), except that this function operates on 32-bit + * integers. The stash must have been created with 'obj_size' of 4. + * + * @param stash Stash handle + * @param[out] u32 Points to an array of 32-bit integers for output. The + * array must be 32-bit aligned in memory. + * @param num Maximum number of integers to get from the stash + * + * @return Number of integers actually output (0 ... num) to 'u32' array + * @retval <0 on failure + */ +int32_t odp_stash_get_u32(odp_stash_t stash, uint32_t u32[], int32_t num); + +/** + * Get 64-bit integers from a stash + * + * Otherwise like odp_stash_get(), except that this function operates on 64-bit + * integers. The stash must have been created with 'obj_size' of 8. + * + * @param stash Stash handle + * @param[out] u64 Points to an array of 64-bit integers for output. The + * array must be 64-bit aligned in memory. + * @param num Maximum number of integers to get from the stash + * + * @return Number of integers actually output (0 ... num) to 'u64' array + * @retval <0 on failure + */ +int32_t odp_stash_get_u64(odp_stash_t stash, uint64_t u64[], int32_t num); + +/** + * Get pointers from a stash + * + * Otherwise like odp_stash_get(), except that this function operates on + * pointers. The stash must have been created with 'obj_size' matching to the + * size of uintptr_t. + * + * @param stash Stash handle + * @param[out] ptr Points to an array of pointers for output. The array must + * be pointer size aligned in memory. + * @param num Maximum number of pointers to get from the stash + * + * @return Number of pointers actually output (0 ... num) to 'ptr' array + * @retval <0 on failure + */ +int32_t odp_stash_get_ptr(odp_stash_t stash, uintptr_t ptr[], int32_t num); + +/** * Flush object handles from the thread local cache * * Flushes all object handles from the thread local cache into the stash, so diff --git a/include/odp/api/spec/version.h.in b/include/odp/api/spec/version.h.in index 21a5f9a77..f07a9125f 100644 --- a/include/odp/api/spec/version.h.in +++ b/include/odp/api/spec/version.h.in @@ -1,5 +1,5 @@ /* Copyright (c) 2013-2018, Linaro Limited - * Copyright (c) 2020, Nokia + * Copyright (c) 2020-2021, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -21,8 +21,7 @@ extern "C" { /** * @defgroup odp_version ODP VERSION - * @details - * <b> ODP API and implementation versions </b> + * API and implementation versions * * ODP API version is identified by ODP_VERSION_API_XXX preprocessor macros. * In addition to these macros, API calls can be used to identify implementation diff --git a/platform/linux-dpdk/Makefile.am b/platform/linux-dpdk/Makefile.am index 26fa5d734..bd2ce4e8d 100644 --- a/platform/linux-dpdk/Makefile.am +++ b/platform/linux-dpdk/Makefile.am @@ -134,6 +134,7 @@ noinst_HEADERS = \ include/odp_ptr_ring_spsc_internal.h \ include/odp_ptr_ring_st_internal.h \ ${top_srcdir}/platform/linux-generic/include/odp_ring_u32_internal.h \ + ${top_srcdir}/platform/linux-generic/include/odp_ring_u64_internal.h \ ${top_srcdir}/platform/linux-generic/include/odp_schedule_if.h \ ${top_srcdir}/platform/linux-generic/include/odp_sorted_list_internal.h \ ${top_srcdir}/platform/linux-generic/include/odp_sysinfo_internal.h \ @@ -263,6 +264,7 @@ endif if ARCH_IS_AARCH64 __LIB__libodp_dpdk_la_SOURCES += arch/aarch64/odp_atomic.c \ arch/default/odp_cpu_cycles.c \ + arch/aarch64/cpu_flags.c \ arch/aarch64/odp_global_time.c \ arch/default/odp_hash_crc32.c \ arch/aarch64/odp_sysinfo_parse.c @@ -278,6 +280,7 @@ odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ endif noinst_HEADERS += arch/aarch64/odp_atomic.h \ arch/aarch64/odp_cpu.h \ + arch/aarch64/cpu_flags.h \ arch/aarch64/odp_cpu_idling.h \ arch/aarch64/odp_llsc.h endif diff --git a/platform/linux-dpdk/arch/aarch64/cpu_flags.c b/platform/linux-dpdk/arch/aarch64/cpu_flags.c new file mode 120000 index 000000000..a5f786955 --- /dev/null +++ b/platform/linux-dpdk/arch/aarch64/cpu_flags.c @@ -0,0 +1 @@ +../../../linux-generic/arch/aarch64/cpu_flags.c
\ No newline at end of file diff --git a/platform/linux-dpdk/arch/aarch64/cpu_flags.h b/platform/linux-dpdk/arch/aarch64/cpu_flags.h new file mode 120000 index 000000000..27ca5d5ea --- /dev/null +++ b/platform/linux-dpdk/arch/aarch64/cpu_flags.h @@ -0,0 +1 @@ +../../../linux-generic/arch/aarch64/cpu_flags.h
\ No newline at end of file diff --git a/platform/linux-dpdk/include/odp_config_internal.h b/platform/linux-dpdk/include/odp_config_internal.h index 0d2e6f478..0f86f4da5 100644 --- a/platform/linux-dpdk/include/odp_config_internal.h +++ b/platform/linux-dpdk/include/odp_config_internal.h @@ -54,6 +54,11 @@ extern "C" { #define CONFIG_QUEUE_MAX_ORD_LOCKS 2 /* + * Maximum number of stashes + */ +#define CONFIG_MAX_STASHES 128 + +/* * Maximum number of packet IO resources */ #define ODP_CONFIG_PKTIO_ENTRIES 64 @@ -126,8 +131,11 @@ extern "C" { /* * Number of shared memory blocks reserved for implementation internal use. + * + * Each stash and packet pool requires one SHM block, and 20 blocks are + * reserved for per ODP module global data. */ -#define CONFIG_INTERNAL_SHM_BLOCKS (ODP_CONFIG_POOLS + 20) +#define CONFIG_INTERNAL_SHM_BLOCKS (CONFIG_MAX_STASHES + ODP_CONFIG_POOLS + 20) /* * Maximum number of shared memory blocks. diff --git a/platform/linux-dpdk/include/odp_pool_internal.h b/platform/linux-dpdk/include/odp_pool_internal.h index 0b5df61af..39ca0f002 100644 --- a/platform/linux-dpdk/include/odp_pool_internal.h +++ b/platform/linux-dpdk/include/odp_pool_internal.h @@ -106,6 +106,25 @@ static inline pool_t *pool_entry_from_hdl(odp_pool_t pool_hdl) return &_odp_pool_glb->pool[_odp_typeval(pool_hdl) - 1]; } +static inline int _odp_buffer_alloc_multi(pool_t *pool, + odp_buffer_hdr_t *buf_hdr[], int num) +{ + int i; + struct rte_mempool *mp = pool->rte_mempool; + + for (i = 0; i < num; i++) { + struct rte_mbuf *mbuf; + + mbuf = rte_mbuf_raw_alloc(mp); + if (odp_unlikely(mbuf == NULL)) + return i; + + buf_hdr[i] = mbuf_to_buf_hdr(mbuf); + } + + return i; +} + static inline void _odp_buffer_free_multi(odp_buffer_hdr_t *buf_hdr[], int num) { int i; diff --git a/platform/linux-dpdk/m4/configure.m4 b/platform/linux-dpdk/m4/configure.m4 index 1e6e8306c..aa369415c 100644 --- a/platform/linux-dpdk/m4/configure.m4 +++ b/platform/linux-dpdk/m4/configure.m4 @@ -72,6 +72,13 @@ AS_VAR_APPEND([PLAT_CFG_TEXT], [" default_config_path: ${default_config_path}"]) ODP_CHECK_CFLAG([-Wno-error=cast-align]) + +# Ignore Clang specific errors about fields with variable sized type not at the +# end of a struct or usage of these structs in arrays. This style is used by +# e.g. timer_pool_t. +ODP_CHECK_CFLAG([-Wno-error=gnu-variable-sized-type-not-at-end]) +ODP_CHECK_CFLAG([-Wno-error=flexible-array-extensions]) + AC_DEFINE([_ODP_PKTIO_DPDK], [1]) AC_CONFIG_COMMANDS_PRE([dnl AM_CONDITIONAL([PLATFORM_IS_LINUX_DPDK], diff --git a/platform/linux-dpdk/odp_init.c b/platform/linux-dpdk/odp_init.c index a79e2fd5e..546203a5d 100644 --- a/platform/linux-dpdk/odp_init.c +++ b/platform/linux-dpdk/odp_init.c @@ -812,3 +812,10 @@ void odp_log_thread_fn_set(odp_log_func_t func) { _odp_this_thread->log_fn = func; } + +int odp_instance(odp_instance_t *instance) +{ + *instance = (odp_instance_t)odp_global_ro.main_pid; + + return 0; +} diff --git a/platform/linux-dpdk/odp_packet.c b/platform/linux-dpdk/odp_packet.c index 653eda84c..5b16428db 100644 --- a/platform/linux-dpdk/odp_packet.c +++ b/platform/linux-dpdk/odp_packet.c @@ -77,13 +77,22 @@ ODP_STATIC_ASSERT(sizeof(_odp_dummy_mbuf.hash.rss) == sizeof(uint32_t), ODP_STATIC_ASSERT(sizeof(_odp_dummy_mbuf.ol_flags) == sizeof(uint64_t), "ol_flags should be uint64_t"); -/* Check that invalid values are the same. Some versions of Clang have trouble - * with the strong type casting, and complain that these invalid values are not - * integral constants. */ +/* Check that invalid values are the same. Some versions of Clang and pedantic + * build have trouble with the strong type casting, and complain that these + * invalid values are not integral constants. + * + * Invalid values are required to be equal for _odp_buffer_is_valid() to work + * properly. */ #ifndef __clang__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wpedantic" ODP_STATIC_ASSERT(ODP_PACKET_INVALID == 0, "Packet invalid not 0"); ODP_STATIC_ASSERT(ODP_BUFFER_INVALID == 0, "Buffer invalid not 0"); ODP_STATIC_ASSERT(ODP_EVENT_INVALID == 0, "Event invalid not 0"); +ODP_STATIC_ASSERT(ODP_PACKET_VECTOR_INVALID == 0, "Packet vector invalid not 0"); +ODP_STATIC_ASSERT(ODP_PACKET_TX_COMPL_INVALID == 0, "Packet TX completion invalid not 0"); +ODP_STATIC_ASSERT(ODP_TIMEOUT_INVALID == 0, "Timeout invalid not 0"); +#pragma GCC diagnostic pop #endif static inline odp_buffer_t packet_to_buffer(odp_packet_t pkt) @@ -2067,7 +2076,7 @@ static int packet_l4_chksum(odp_packet_hdr_t *pkt_hdr, } } - return pkt_hdr->p.flags.all_flags != 0; + return pkt_hdr->p.flags.all.error != 0; } /** diff --git a/platform/linux-dpdk/odp_pool.c b/platform/linux-dpdk/odp_pool.c index 89770292e..9a37de2d5 100644 --- a/platform/linux-dpdk/odp_pool.c +++ b/platform/linux-dpdk/odp_pool.c @@ -858,29 +858,6 @@ odp_pool_t odp_pool_lookup(const char *name) return ODP_POOL_INVALID; } -static inline int buffer_alloc_multi(pool_t *pool, odp_buffer_hdr_t *buf_hdr[], - int num) -{ - int i; - struct rte_mempool *mp = pool->rte_mempool; - - ODP_ASSERT(pool->type == ODP_POOL_BUFFER || - pool->type == ODP_POOL_TIMEOUT || - pool->type == ODP_POOL_VECTOR); - - for (i = 0; i < num; i++) { - struct rte_mbuf *mbuf; - - mbuf = rte_mbuf_raw_alloc(mp); - if (odp_unlikely(mbuf == NULL)) - return i; - - buf_hdr[i] = mbuf_to_buf_hdr(mbuf); - } - - return i; -} - odp_buffer_t odp_buffer_alloc(odp_pool_t pool_hdl) { odp_buffer_t buf; @@ -890,7 +867,10 @@ odp_buffer_t odp_buffer_alloc(odp_pool_t pool_hdl) ODP_ASSERT(ODP_POOL_INVALID != pool_hdl); pool = pool_entry_from_hdl(pool_hdl); - ret = buffer_alloc_multi(pool, (odp_buffer_hdr_t **)&buf, 1); + + ODP_ASSERT(pool->type == ODP_POOL_BUFFER); + + ret = _odp_buffer_alloc_multi(pool, (odp_buffer_hdr_t **)&buf, 1); if (odp_likely(ret == 1)) return buf; @@ -906,7 +886,9 @@ int odp_buffer_alloc_multi(odp_pool_t pool_hdl, odp_buffer_t buf[], int num) pool = pool_entry_from_hdl(pool_hdl); - return buffer_alloc_multi(pool, (odp_buffer_hdr_t **)buf, num); + ODP_ASSERT(pool->type == ODP_POOL_BUFFER); + + return _odp_buffer_alloc_multi(pool, (odp_buffer_hdr_t **)buf, num); } void odp_buffer_free(odp_buffer_t buf) @@ -926,6 +908,56 @@ void odp_pool_print(odp_pool_t pool_hdl) rte_mempool_dump(stdout, pool->rte_mempool); } +void odp_pool_print_all(void) +{ + uint64_t available; + uint32_t i, index, tot, cache_size; + uint32_t elt_size, elt_len = 0; + uint8_t type, ext; + const int col_width = 24; + const char *name; + char type_c; + + ODP_PRINT("\nList of all pools\n"); + ODP_PRINT("-----------------\n"); + ODP_PRINT(" idx %-*s type free tot cache elt_len ext\n", col_width, "name"); + + for (i = 0; i < ODP_CONFIG_POOLS; i++) { + pool_t *pool = pool_entry(i); + + LOCK(&pool->lock); + + if (pool->rte_mempool == NULL) { + UNLOCK(&pool->lock); + continue; + } + + available = rte_mempool_avail_count(pool->rte_mempool); + cache_size = pool->rte_mempool->cache_size; + ext = pool->pool_ext; + index = pool->pool_idx; + name = pool->name; + tot = pool->rte_mempool->size; + type = pool->type; + elt_size = pool->rte_mempool->elt_size; + + UNLOCK(&pool->lock); + + if (type == ODP_POOL_BUFFER || type == ODP_POOL_PACKET) + elt_len = elt_size; + + type_c = (type == ODP_POOL_BUFFER) ? 'B' : + (type == ODP_POOL_PACKET) ? 'P' : + (type == ODP_POOL_TIMEOUT) ? 'T' : + (type == ODP_POOL_VECTOR) ? 'V' : '-'; + + ODP_PRINT("%4u %-*s %c %6" PRIu64 " %6" PRIu32 " %6" PRIu32 " %8" PRIu32 " " + "%" PRIu8 "\n", index, col_width, name, type_c, available, tot, + cache_size, elt_len, ext); + } + ODP_PRINT("\n"); +} + static void mempool_addr_range(struct rte_mempool *mp ODP_UNUSED, void *opaque, struct rte_mempool_memhdr *memhdr, unsigned int mem_idx ODP_UNUSED) diff --git a/platform/linux-dpdk/odp_queue_basic.c b/platform/linux-dpdk/odp_queue_basic.c index 33e0ba6bc..e3c3241c4 100644 --- a/platform/linux-dpdk/odp_queue_basic.c +++ b/platform/linux-dpdk/odp_queue_basic.c @@ -658,7 +658,7 @@ static void queue_print(odp_queue_t handle) } ODP_PRINT("\nQueue info\n"); ODP_PRINT("----------\n"); - ODP_PRINT(" handle %p\n", queue->s.handle); + ODP_PRINT(" handle %p\n", (void *)queue->s.handle); ODP_PRINT(" index %" PRIu32 "\n", queue_id); ODP_PRINT(" name %s\n", queue->s.name); ODP_PRINT(" enq mode %s\n", diff --git a/platform/linux-dpdk/odp_queue_eventdev.c b/platform/linux-dpdk/odp_queue_eventdev.c index 0960c456e..b650874ec 100644 --- a/platform/linux-dpdk/odp_queue_eventdev.c +++ b/platform/linux-dpdk/odp_queue_eventdev.c @@ -1008,7 +1008,7 @@ static void queue_print(odp_queue_t handle) } ODP_PRINT("\nQueue info\n"); ODP_PRINT("----------\n"); - ODP_PRINT(" handle %p\n", handle); + ODP_PRINT(" handle %p\n", (void *)handle); ODP_PRINT(" index %" PRIu32 "\n", queue->s.index); ODP_PRINT(" name %s\n", queue->s.name); ODP_PRINT(" enq mode %s\n", diff --git a/platform/linux-dpdk/odp_schedule_if.c b/platform/linux-dpdk/odp_schedule_if.c index 21f5fc85b..73475ae88 100644 --- a/platform/linux-dpdk/odp_schedule_if.c +++ b/platform/linux-dpdk/odp_schedule_if.c @@ -98,27 +98,27 @@ int odp_schedule_multi_no_wait(odp_queue_t *from, odp_event_t events[], int num) void odp_schedule_pause(void) { - return _odp_sched_api->schedule_pause(); + _odp_sched_api->schedule_pause(); } void odp_schedule_resume(void) { - return _odp_sched_api->schedule_resume(); + _odp_sched_api->schedule_resume(); } void odp_schedule_release_atomic(void) { - return _odp_sched_api->schedule_release_atomic(); + _odp_sched_api->schedule_release_atomic(); } void odp_schedule_release_ordered(void) { - return _odp_sched_api->schedule_release_ordered(); + _odp_sched_api->schedule_release_ordered(); } void odp_schedule_prefetch(int num) { - return _odp_sched_api->schedule_prefetch(num); + _odp_sched_api->schedule_prefetch(num); } int odp_schedule_min_prio(void) @@ -183,12 +183,12 @@ int odp_schedule_group_info(odp_schedule_group_t group, void odp_schedule_order_lock(uint32_t lock_index) { - return _odp_sched_api->schedule_order_lock(lock_index); + _odp_sched_api->schedule_order_lock(lock_index); } void odp_schedule_order_unlock(uint32_t lock_index) { - return _odp_sched_api->schedule_order_unlock(lock_index); + _odp_sched_api->schedule_order_unlock(lock_index); } void odp_schedule_order_unlock_lock(uint32_t unlock_index, uint32_t lock_index) diff --git a/platform/linux-dpdk/odp_time.c b/platform/linux-dpdk/odp_time.c index 1f3fb2719..a53240564 100644 --- a/platform/linux-dpdk/odp_time.c +++ b/platform/linux-dpdk/odp_time.c @@ -214,7 +214,7 @@ void odp_time_wait_ns(uint64_t ns) void odp_time_wait_until(odp_time_t time) { - return time_wait_until(time); + time_wait_until(time); } static odp_bool_t is_invariant_tsc_supported(void) diff --git a/platform/linux-dpdk/odp_timer.c b/platform/linux-dpdk/odp_timer.c index 68ce550f5..96df1bf79 100644 --- a/platform/linux-dpdk/odp_timer.c +++ b/platform/linux-dpdk/odp_timer.c @@ -16,6 +16,7 @@ #include <odp_debug_internal.h> #include <odp_init_internal.h> #include <odp_libconfig_internal.h> +#include <odp_pool_internal.h> #include <odp_queue_if.h> #include <odp_ring_u32_internal.h> #include <odp_thread_internal.h> @@ -81,6 +82,8 @@ typedef struct { } timer_entry_t; +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wpedantic" typedef struct timer_pool_s { timer_entry_t timer[MAX_TIMER_RING_SIZE]; @@ -100,6 +103,7 @@ typedef struct timer_pool_s { uint32_t hwm_timers; } timer_pool_t; +#pragma GCC diagnostic pop /* Wrappers for alternative DPDK timer implementation */ typedef int (*timer_stop_fn)(struct rte_timer *tim); @@ -953,20 +957,29 @@ void *odp_timeout_user_ptr(odp_timeout_t tmo) return (void *)(uintptr_t)timeout_hdr->user_ptr; } -odp_timeout_t odp_timeout_alloc(odp_pool_t pool) +odp_timeout_t odp_timeout_alloc(odp_pool_t pool_hdl) { - odp_buffer_t buf = odp_buffer_alloc(pool); + odp_timeout_t tmo; + pool_t *pool; + int ret; + + ODP_ASSERT(pool_hdl != ODP_POOL_INVALID); + + pool = pool_entry_from_hdl(pool_hdl); + + ODP_ASSERT(pool->type == ODP_POOL_TIMEOUT); - if (odp_unlikely(buf == ODP_BUFFER_INVALID)) - return ODP_TIMEOUT_INVALID; - return odp_timeout_from_event(odp_buffer_to_event(buf)); + ret = _odp_buffer_alloc_multi(pool, (odp_buffer_hdr_t **)&tmo, 1); + + if (odp_likely(ret == 1)) + return tmo; + + return ODP_TIMEOUT_INVALID; } void odp_timeout_free(odp_timeout_t tmo) { - odp_event_t ev = odp_timeout_to_event(tmo); - - odp_buffer_free(odp_buffer_from_event(ev)); + _odp_buffer_free_multi((odp_buffer_hdr_t **)&tmo, 1); } void odp_timer_pool_print(odp_timer_pool_t timer_pool) @@ -982,7 +995,7 @@ void odp_timer_pool_print(odp_timer_pool_t timer_pool) ODP_PRINT("\nTimer pool info\n"); ODP_PRINT("---------------\n"); - ODP_PRINT(" timer pool %p\n", tp); + ODP_PRINT(" timer pool %p\n", (void *)tp); ODP_PRINT(" name %s\n", tp->name); ODP_PRINT(" num timers %u\n", tp->cur_timers); ODP_PRINT(" hwm timers %u\n", tp->hwm_timers); @@ -1001,7 +1014,7 @@ void odp_timer_print(odp_timer_t timer_hdl) ODP_PRINT("\nTimer info\n"); ODP_PRINT("----------\n"); - ODP_PRINT(" timer pool %p\n", timer->timer_pool); + ODP_PRINT(" timer pool %p\n", (void *)timer->timer_pool); ODP_PRINT(" timer index %" PRIu32 "\n", timer->timer_idx); ODP_PRINT(" dest queue 0x%" PRIx64 "\n", odp_queue_to_u64(timer->queue)); ODP_PRINT(" user ptr %p\n", timer->user_ptr); @@ -1035,7 +1048,7 @@ void odp_timeout_print(odp_timeout_t tmo) ODP_PRINT("\nTimeout info\n"); ODP_PRINT("------------\n"); ODP_PRINT(" tmo handle 0x%" PRIx64 "\n", odp_timeout_to_u64(tmo)); - ODP_PRINT(" timer pool %p\n", tp); + ODP_PRINT(" timer pool %p\n", (void *)tp); ODP_PRINT(" timer index %u\n", idx); ODP_PRINT(" expiration %" PRIu64 "\n", timeout_hdr->expiration); ODP_PRINT(" user ptr %p\n", timeout_hdr->user_ptr); diff --git a/platform/linux-generic/Makefile.am b/platform/linux-generic/Makefile.am index e763c0abc..8763606ad 100644 --- a/platform/linux-generic/Makefile.am +++ b/platform/linux-generic/Makefile.am @@ -143,6 +143,7 @@ noinst_HEADERS = \ include/odp_ring_spsc_internal.h \ include/odp_ring_st_internal.h \ include/odp_ring_u32_internal.h \ + include/odp_ring_u64_internal.h \ include/odp_schedule_if.h \ include/odp_schedule_scalable_config.h \ include/odp_schedule_scalable.h \ @@ -295,6 +296,7 @@ endif if ARCH_IS_AARCH64 __LIB__libodp_linux_la_SOURCES += arch/aarch64/odp_atomic.c \ arch/default/odp_cpu_cycles.c \ + arch/aarch64/cpu_flags.c \ arch/aarch64/odp_global_time.c \ arch/default/odp_hash_crc32.c \ arch/aarch64/odp_sysinfo_parse.c @@ -310,6 +312,7 @@ odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ endif noinst_HEADERS += arch/aarch64/odp_atomic.h \ arch/aarch64/odp_cpu.h \ + arch/aarch64/cpu_flags.h \ arch/aarch64/odp_cpu_idling.h \ arch/aarch64/odp_llsc.h endif diff --git a/platform/linux-generic/arch/aarch64/cpu_flags.c b/platform/linux-generic/arch/aarch64/cpu_flags.c new file mode 100644 index 000000000..d70e26271 --- /dev/null +++ b/platform/linux-generic/arch/aarch64/cpu_flags.c @@ -0,0 +1,976 @@ +/* Copyright (c) 2018, Linaro Limited + * Copyright (c) 2020-2021, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <string.h> +#include <stdlib.h> +#include <sys/auxv.h> +#include <asm/hwcap.h> + +#include <odp/api/hints.h> +#include <odp_debug_internal.h> +#include "cpu_flags.h" + +typedef struct { + const char *feat_flag; + const unsigned int hwcap_field; + bool valid; +} hwcap_feat_flag_t; + +static hwcap_feat_flag_t hwcap_flags[] = { + { + /* Floating-point Extensions */ + .feat_flag = "FEAT_FP", +#ifdef HWCAP_FP + .hwcap_field = HWCAP_FP, + .valid = 1, +#endif + }, + + { + /* Floating-point Extensions */ + .feat_flag = "FEAT_FP", +#ifdef HWCAP_ASIMD + .hwcap_field = HWCAP_ASIMD, + .valid = 1, +#endif + }, + + { + /* Generic Timer is configured to generate events at approx. 10KHz */ + .feat_flag = "HWCAP_EVTSTRM", +#ifdef HWCAP_EVTSTRM + .hwcap_field = HWCAP_EVTSTRM, + .valid = 1, +#endif + }, + + { + /* Advanced SIMD AES Instructions */ + .feat_flag = "FEAT_AES", +#ifdef HWCAP_AES + .hwcap_field = HWCAP_AES, + .valid = 1, +#endif + }, + + { + /* Advanced SIMD PMULL Instructions */ + .feat_flag = "FEAT_PMULL", +#ifdef HWCAP_PMULL + .hwcap_field = HWCAP_PMULL, + .valid = 1, +#endif + }, + + { + /* Advanced SIMD SHA1 Instructions */ + .feat_flag = "FEAT_SHA1", +#ifdef HWCAP_SHA1 + .hwcap_field = HWCAP_SHA1, + .valid = 1, +#endif + }, + + { + /* Advanced SIMD SHA256 Instructions */ + .feat_flag = "FEAT_SHA256", +#ifdef HWCAP_SHA2 + .hwcap_field = HWCAP_SHA2, + .valid = 1, +#endif + }, + + { + /* CRC32 Instructions */ + .feat_flag = "FEAT_CRC32", +#ifdef HWCAP_CRC32 + .hwcap_field = HWCAP_CRC32, + .valid = 1, +#endif + }, + + { + /* Large System Extensions */ + .feat_flag = "FEAT_LSE", +#ifdef HWCAP_ATOMICS + .hwcap_field = HWCAP_ATOMICS, + .valid = 1, +#endif + }, + + { + /* Half-precision Floating-point Data Processing Instructions */ + .feat_flag = "FEAT_FP16", +#ifdef HWCAP_FPHP + .hwcap_field = HWCAP_FPHP, + .valid = 1, +#endif + }, + + { + /* Half-precision Floating-point Data Processing Instructions */ + .feat_flag = "FEAT_FP16", +#ifdef HWCAP_ASIMDHP + .hwcap_field = HWCAP_ASIMDHP, + .valid = 1, +#endif + }, + + { + /* Availability of EL0 Access to certain ID Registers */ + .feat_flag = "HWCAP_CPUID", +#ifdef HWCAP_CPUID + .hwcap_field = HWCAP_CPUID, + .valid = 1, +#endif + }, + + { + /* Rounding Double Multiply Accumulate Extensions */ + .feat_flag = "FEAT_RDM", +#ifdef HWCAP_ASIMDRDM + .hwcap_field = HWCAP_ASIMDRDM, + .valid = 1, +#endif + }, + + { + /* JavaScript FJCVTS Conversion Instructions */ + .feat_flag = "FEAT_JSCVT", +#ifdef HWCAP_JSCVT + .hwcap_field = HWCAP_JSCVT, + .valid = 1, +#endif + }, + + { + /* Floating-point FCMLA and FCADD Instructions */ + .feat_flag = "FEAT_FCMA", +#ifdef HWCAP_FCMA + .hwcap_field = HWCAP_FCMA, + .valid = 1, +#endif + }, + + { + /* Load-acquire RCpc Instructions */ + .feat_flag = "FEAT_LRCPC", +#ifdef HWCAP_LRCPC + .hwcap_field = HWCAP_LRCPC, + .valid = 1, +#endif + }, + + { + /* DC CVAP Instructions */ + .feat_flag = "FEAT_DPB", +#ifdef HWCAP_DCPOP + .hwcap_field = HWCAP_DCPOP, + .valid = 1, +#endif + }, + + { + /* Advanced SIMD EOR3, RAX1, XAR, and BCAX Instructions */ + .feat_flag = "FEAT_SHA3", +#ifdef HWCAP_SHA3 + .hwcap_field = HWCAP_SHA3, + .valid = 1, +#endif + }, + + { + /* Advanced SIMD SM3 Instructions */ + .feat_flag = "FEAT_SM3", +#ifdef HWCAP_SM3 + .hwcap_field = HWCAP_SM3, + .valid = 1, +#endif + }, + + { + /* Advanced SIMD SM4 Instructions */ + .feat_flag = "FEAT_SM4", +#ifdef HWCAP_SM4 + .hwcap_field = HWCAP_SM4, + .valid = 1, +#endif + }, + + { + /* Advanced SIMD Int8 Dot Product Instructions */ + .feat_flag = "FEAT_DotProd", +#ifdef HWCAP_ASIMDDP + .hwcap_field = HWCAP_ASIMDDP, + .valid = 1, +#endif + }, + + { + /* Advanced SIMD SHA512 Instructions */ + .feat_flag = "FEAT_SHA512", +#ifdef HWCAP_SHA512 + .hwcap_field = HWCAP_SHA512, + .valid = 1, +#endif + }, + + { + /* Scalable Vector Extensions */ + .feat_flag = "FEAT_SVE", +#ifdef HWCAP_SVE + .hwcap_field = HWCAP_SVE, + .valid = 1, +#endif + }, + + { + /* Half-precision Floating-point FMLAL Instructions */ + .feat_flag = "FEAT_FHM", +#ifdef HWCAP_ASIMDFHM + .hwcap_field = HWCAP_ASIMDFHM, + .valid = 1, +#endif + }, + + { + /* Data Independent Timing Instructions */ + .feat_flag = "FEAT_DIT", +#ifdef HWCAP_DIT + .hwcap_field = HWCAP_DIT, + .valid = 1, +#endif + }, + + { + /* Large System Extensions Version 2 */ + .feat_flag = "FEAT_LSE2", +#ifdef HWCAP_USCAT + .hwcap_field = HWCAP_USCAT, + .valid = 1, +#endif + }, + + { + /* Load-acquire RCpc Instructions Version 2 */ + .feat_flag = "FEAT_LRCPC2", +#ifdef HWCAP_ILRCPC + .hwcap_field = HWCAP_ILRCPC, + .valid = 1, +#endif + }, + + { + /* Condition Flag Manipulation Extensions */ + .feat_flag = "FEAT_FlagM", +#ifdef HWCAP_FLAGM + .hwcap_field = HWCAP_FLAGM, + .valid = 1, +#endif + }, + + { + /* Speculative Store Bypass Safe Instructions */ + .feat_flag = "FEAT_SSBS2", +#ifdef HWCAP_SSBS + .hwcap_field = HWCAP_SSBS, + .valid = 1, +#endif + }, + + { + /* Speculation Barrier Instructions */ + .feat_flag = "FEAT_SB", +#ifdef HWCAP_SB + .hwcap_field = HWCAP_SB, + .valid = 1, +#endif + }, + + { + /* Pointer Authentication Extensions */ + .feat_flag = "FEAT_PAuth", +#ifdef HWCAP_PACA + .hwcap_field = HWCAP_PACA, + .valid = 1, +#endif + }, + + { + /* Generic Authentication Extensions */ + .feat_flag = "HWCAP_PACG", +#ifdef HWCAP_PACG + .hwcap_field = HWCAP_PACG, + .valid = 1, +#endif + } +}; + +static hwcap_feat_flag_t hwcap2_flags[] = { + { + /* DC CVADP instructions */ + .feat_flag = "FEAT_DPB2", +#ifdef HWCAP2_DCPODP + .hwcap_field = HWCAP2_DCPODP, + .valid = 1, +#endif + }, + + { + /* Scalable Vector Extensions Version 2 */ + .feat_flag = "FEAT_SVE2", +#ifdef HWCAP2_SVE2 + .hwcap_field = HWCAP2_SVE2, + .valid = 1, +#endif + }, + + { + /* SVE AES Instructions */ + .feat_flag = "FEAT_SVE_AES", +#ifdef HWCAP2_SVEAES + .hwcap_field = HWCAP2_SVEAES, + .valid = 1, +#endif + }, + + { + /* SVE PMULL Instructions */ + .feat_flag = "FEAT_SVE_PMULL128", +#ifdef HWCAP2_SVEPMULL + .hwcap_field = HWCAP2_SVEPMULL, + .valid = 1, +#endif + }, + + { + /* SVE Bit Permute Instructions */ + .feat_flag = "FEAT_SVE_BitPerm", +#ifdef HWCAP2_SVEBITPERM + .hwcap_field = HWCAP2_SVEBITPERM, + .valid = 1, +#endif + }, + + { + /* SVE SHA-3 Instructions */ + .feat_flag = "FEAT_SVE_SHA3", +#ifdef HWCAP2_SVESHA3 + .hwcap_field = HWCAP2_SVESHA3, + .valid = 1, +#endif + }, + + { + /* SVE SM4 Instructions */ + .feat_flag = "FEAT_SVE_SM4", +#ifdef HWCAP2_SVESM4 + .hwcap_field = HWCAP2_SVESM4, + .valid = 1, +#endif + }, + + { + /* Condition Flag Manipulation Extensions Version 2 */ + .feat_flag = "FEAT_FlagM2", +#ifdef HWCAP2_FLAGM2 + .hwcap_field = HWCAP2_FLAGM2, + .valid = 1, +#endif + }, + + { + /* FRINT32Z, FRINT32X, FRINT64Z, and FRINT64X instructions */ + .feat_flag = "FEAT_FRINTTS", +#ifdef HWCAP2_FRINT + .hwcap_field = HWCAP2_FRINT, + .valid = 1, +#endif + }, + + { + /* SVE Int8 Matrix Multiplication Instructions */ + .feat_flag = "FEAT_I8MM", +#ifdef HWCAP2_SVEI8MM + .hwcap_field = HWCAP2_SVEI8MM, + .valid = 1, +#endif + }, + + { + /* SVE Single-precision Floating-point Matrix Multiply Instructions */ + .feat_flag = "FEAT_F32MM", +#ifdef HWCAP2_SVEF32MM + .hwcap_field = HWCAP2_SVEF32MM, + .valid = 1, +#endif + }, + + { + /* SVE Double-precision Floating-point Matrix Multiply Instructions */ + .feat_flag = "FEAT_F64MM", +#ifdef HWCAP2_SVEF64MM + .hwcap_field = HWCAP2_SVEF64MM, + .valid = 1, +#endif + }, + + { + /* SVE BFloat16 Instructions */ + .feat_flag = "FEAT_BF16", +#ifdef HWCAP2_SVEBF16 + .hwcap_field = HWCAP2_SVEBF16, + .valid = 1, +#endif + }, + + { + /* Advanced SIMD and Floating-point Int8 Matrix Multiplication Instructions */ + .feat_flag = "FEAT_I8MM", +#ifdef HWCAP2_I8MM + .hwcap_field = HWCAP2_I8MM, + .valid = 1, +#endif + }, + + { + /* Advanced SIMD and Floating-point BFloat16 Instructions */ + .feat_flag = "FEAT_BF16", +#ifdef HWCAP2_BF16 + .hwcap_field = HWCAP2_BF16, + .valid = 1, +#endif + }, + + { + /* Data Gathering Hint Extensions */ + .feat_flag = "FEAT_DGH", +#ifdef HWCAP2_DGH + .hwcap_field = HWCAP2_DGH, + .valid = 1, +#endif + }, + + { + /* Random Number Generation Extensions */ + .feat_flag = "FEAT_RNG", +#ifdef HWCAP2_RNG + .hwcap_field = HWCAP2_RNG, + .valid = 1, +#endif + }, + + { + /* Branch Target Identification Extensions */ + .feat_flag = "FEAT_BTI", +#ifdef HWCAP2_BTI + .hwcap_field = HWCAP2_BTI, + .valid = 1, +#endif + }, + + { + /* Full Memory Tagging Extensions */ + .feat_flag = "FEAT_MTE2", +#ifdef HWCAP2_MTE + .hwcap_field = HWCAP2_MTE, + .valid = 1, +#endif + } +}; + +static void _odp_sys_info_print_acle_flags(void) +{ + const char *ndef = "n/a"; + + /* Avoid compiler warning about unused variable */ + (void)ndef; + + /* See ARM C Language Extensions documentation for details */ + ODP_PRINT("ARM FEATURES:\n"); + + ODP_PRINT(" __ARM_ALIGN_MAX_PWR "); +#ifdef __ARM_ALIGN_MAX_PWR + ODP_PRINT("%i\n", __ARM_ALIGN_MAX_PWR); +#else + ODP_PRINT("%s\n", ndef); +#endif + + ODP_PRINT(" __ARM_ALIGN_MAX_STACK_PWR "); +#ifdef __ARM_ALIGN_MAX_STACK_PWR + ODP_PRINT("%i\n", __ARM_ALIGN_MAX_STACK_PWR); +#else + ODP_PRINT("%s\n", ndef); +#endif + + ODP_PRINT(" __ARM_ARCH "); +#ifdef __ARM_ARCH + ODP_PRINT("%i\n", __ARM_ARCH); +#else + ODP_PRINT("%s\n", ndef); +#endif + + ODP_PRINT(" __ARM_ARCH_ISA_A64 "); +#ifdef __ARM_ARCH_ISA_A64 + ODP_PRINT("%i\n", __ARM_ARCH_ISA_A64); +#else + ODP_PRINT("%s\n", ndef); +#endif + + ODP_PRINT(" __ARM_BIG_ENDIAN "); +#ifdef __ARM_BIG_ENDIAN + ODP_PRINT("%i\n", __ARM_BIG_ENDIAN); +#else + ODP_PRINT("%s\n", ndef); +#endif + + ODP_PRINT(" __ARM_BF16_FORMAT_ALTERNATIVE "); +#ifdef __ARM_BF16_FORMAT_ALTERNATIVE + ODP_PRINT("%i\n", __ARM_BF16_FORMAT_ALTERNATIVE); +#else + ODP_PRINT("%s\n", ndef); +#endif + + ODP_PRINT(" __ARM_FEATURE_ATOMICS "); +#ifdef __ARM_FEATURE_ATOMICS + ODP_PRINT("%i\n", __ARM_FEATURE_ATOMICS); +#else + ODP_PRINT("%s\n", ndef); +#endif + + ODP_PRINT(" __ARM_FEATURE_BF16 "); +#ifdef __ARM_FEATURE_BF16 + ODP_PRINT("%i\n", __ARM_FEATURE_BF16); +#else + ODP_PRINT("%s\n", ndef); +#endif + + ODP_PRINT(" __ARM_FEATURE_BTI_DEFAULT "); +#ifdef __ARM_FEATURE_BTI_DEFAULT + ODP_PRINT("%i\n", __ARM_FEATURE_BTI_DEFAULT); +#else + ODP_PRINT("%s\n", ndef); +#endif + + ODP_PRINT(" __ARM_FEATURE_CDE "); +#ifdef __ARM_FEATURE_CDE + ODP_PRINT("%i\n", __ARM_FEATURE_CDE); +#else + ODP_PRINT("%s\n", ndef); +#endif + + ODP_PRINT(" __ARM_FEATURE_CDE_COPROC "); +#ifdef __ARM_FEATURE_CDE_COPROC + ODP_PRINT("0x%X\n", __ARM_FEATURE_CDE_COPROC); +#else + ODP_PRINT("%s\n", ndef); +#endif + + ODP_PRINT(" __ARM_FEATURE_CLZ "); +#ifdef __ARM_FEATURE_CLZ + ODP_PRINT("%i\n", __ARM_FEATURE_CLZ); +#else + ODP_PRINT("%s\n", ndef); +#endif + + ODP_PRINT(" __ARM_FEATURE_COMPLEX "); +#ifdef __ARM_FEATURE_COMPLEX + ODP_PRINT("%i\n", __ARM_FEATURE_COMPLEX); +#else + ODP_PRINT("%s\n", ndef); +#endif + + ODP_PRINT(" __ARM_FEATURE_COPROC "); +#ifdef __ARM_FEATURE_COPROC + ODP_PRINT("0x%X\n", __ARM_FEATURE_COPROC); +#else + ODP_PRINT("%s\n", ndef); +#endif + + ODP_PRINT(" __ARM_FEATURE_CRC32 "); +#ifdef __ARM_FEATURE_CRC32 + ODP_PRINT("%i\n", __ARM_FEATURE_CRC32); +#else + ODP_PRINT("%s\n", ndef); +#endif + + ODP_PRINT(" __ARM_FEATURE_CRYPTO "); +#ifdef __ARM_FEATURE_CRYPTO + ODP_PRINT("%i\n", __ARM_FEATURE_CRYPTO); +#else + ODP_PRINT("%s\n", ndef); +#endif + + ODP_PRINT(" __ARM_FEATURE_DIRECTED_ROUNDING "); +#ifdef __ARM_FEATURE_DIRECTED_ROUNDING + ODP_PRINT("%i\n", __ARM_FEATURE_DIRECTED_ROUNDING); +#else + ODP_PRINT("%s\n", ndef); +#endif + + ODP_PRINT(" __ARM_FEATURE_DOTPROD "); +#ifdef __ARM_FEATURE_DOTPROD + ODP_PRINT("%i\n", __ARM_FEATURE_DOTPROD); +#else + ODP_PRINT("%s\n", ndef); +#endif + + ODP_PRINT(" __ARM_FEATURE_FMA "); +#ifdef __ARM_FEATURE_FMA + ODP_PRINT("%i\n", __ARM_FEATURE_FMA); +#else + ODP_PRINT("%s\n", ndef); +#endif + + ODP_PRINT(" __ARM_FEATURE_FP16_FML "); +#ifdef __ARM_FEATURE_FP16_FML + ODP_PRINT("%i\n", __ARM_FEATURE_FP16_FML); +#else + ODP_PRINT("%s\n", ndef); +#endif + + ODP_PRINT(" __ARM_FEATURE_FRINT "); +#ifdef __ARM_FEATURE_FRINT + ODP_PRINT("%i\n", __ARM_FEATURE_FRINT); +#else + ODP_PRINT("%s\n", ndef); +#endif + + ODP_PRINT(" __ARM_FEATURE_IDIV "); +#ifdef __ARM_FEATURE_IDIV + ODP_PRINT("%i\n", __ARM_FEATURE_IDIV); +#else + ODP_PRINT("%s\n", ndef); +#endif + + ODP_PRINT(" __ARM_FEATURE_JCVT "); +#ifdef __ARM_FEATURE_JCVT + ODP_PRINT("%i\n", __ARM_FEATURE_JCVT); +#else + ODP_PRINT("%s\n", ndef); +#endif + + ODP_PRINT(" __ARM_FEATURE_MATMUL_INT8 "); +#ifdef __ARM_FEATURE_MATMUL_INT8 + ODP_PRINT("%i\n", __ARM_FEATURE_MATMUL_INT8); +#else + ODP_PRINT("%s\n", ndef); +#endif + + ODP_PRINT(" __ARM_FEATURE_MEMORY_TAGGING "); +#ifdef __ARM_FEATURE_MEMORY_TAGGING + ODP_PRINT("%i\n", __ARM_FEATURE_MEMORY_TAGGING); +#else + ODP_PRINT("%s\n", ndef); +#endif + + ODP_PRINT(" __ARM_FEATURE_MVE "); +#ifdef __ARM_FEATURE_MVE + ODP_PRINT("0x%X\n", __ARM_FEATURE_MVE); +#else + ODP_PRINT("%s\n", ndef); +#endif + + ODP_PRINT(" __ARM_FEATURE_NUMERIC_MAXMIN "); +#ifdef __ARM_FEATURE_NUMERIC_MAXMIN + ODP_PRINT("%i\n", __ARM_FEATURE_NUMERIC_MAXMIN); +#else + ODP_PRINT("%s\n", ndef); +#endif + + ODP_PRINT(" __ARM_FEATURE_PAC_DEFAULT "); +#ifdef __ARM_FEATURE_PAC_DEFAULT + ODP_PRINT("0x%X\n", __ARM_FEATURE_PAC_DEFAULT); +#else + ODP_PRINT("%s\n", ndef); +#endif + + ODP_PRINT(" __ARM_FEATURE_QRDMX "); +#ifdef __ARM_FEATURE_QRDMX + ODP_PRINT("%i\n", __ARM_FEATURE_QRDMX); +#else + ODP_PRINT("%s\n", ndef); +#endif + + ODP_PRINT(" __ARM_FEATURE_RNG "); +#ifdef __ARM_FEATURE_RNG + ODP_PRINT("%i\n", __ARM_FEATURE_RNG); +#else + ODP_PRINT("%s\n", ndef); +#endif + + ODP_PRINT(" __ARM_FEATURE_SHA3 "); +#ifdef __ARM_FEATURE_SHA3 + ODP_PRINT("%i\n", __ARM_FEATURE_SHA3); +#else + ODP_PRINT("%s\n", ndef); +#endif + + ODP_PRINT(" __ARM_FEATURE_SHA512 "); +#ifdef __ARM_FEATURE_SHA512 + ODP_PRINT("%i\n", __ARM_FEATURE_SHA512); +#else + ODP_PRINT("%s\n", ndef); +#endif + + ODP_PRINT(" __ARM_FEATURE_SM3 "); +#ifdef __ARM_FEATURE_SM3 + ODP_PRINT("%i\n", __ARM_FEATURE_SM3); +#else + ODP_PRINT("%s\n", ndef); +#endif + + ODP_PRINT(" __ARM_FEATURE_SM4 "); +#ifdef __ARM_FEATURE_SM4 + ODP_PRINT("%i\n", __ARM_FEATURE_SM4); +#else + ODP_PRINT("%s\n", ndef); +#endif + + ODP_PRINT(" __ARM_FEATURE_TME "); +#ifdef __ARM_FEATURE_TME + ODP_PRINT("%i\n", __ARM_FEATURE_TME); +#else + ODP_PRINT("%s\n", ndef); +#endif + + ODP_PRINT(" __ARM_FEATURE_UNALIGNED "); +#ifdef __ARM_FEATURE_UNALIGNED + ODP_PRINT("%i\n", __ARM_FEATURE_UNALIGNED); +#else + ODP_PRINT("%s\n", ndef); +#endif + + ODP_PRINT(" __ARM_FP "); +#ifdef __ARM_FP + ODP_PRINT("0x%X\n", __ARM_FP); +#else + ODP_PRINT("%s\n", ndef); +#endif + + ODP_PRINT(" __ARM_FP_FAST "); +#ifdef __ARM_FP_FAST + ODP_PRINT("%i\n", __ARM_FP_FAST); +#else + ODP_PRINT("%s\n", ndef); +#endif + + ODP_PRINT(" __ARM_FP_FENV_ROUNDING "); +#ifdef __ARM_FP_FENV_ROUNDING + ODP_PRINT("%i\n", __ARM_FP_FENV_ROUNDING); +#else + ODP_PRINT("%s\n", ndef); +#endif + + ODP_PRINT(" __ARM_FP16_ARGS "); +#ifdef __ARM_FP16_ARGS + ODP_PRINT("%i\n", __ARM_FP16_ARGS); +#else + ODP_PRINT("%s\n", ndef); +#endif + + ODP_PRINT(" __ARM_FP16_FORMAT_ALTERNATIVE "); +#ifdef __ARM_FP16_FORMAT_ALTERNATIVE + ODP_PRINT("%i\n", __ARM_FP16_FORMAT_ALTERNATIVE); +#else + ODP_PRINT("%s\n", ndef); +#endif + + ODP_PRINT(" __ARM_FP16_FORMAT_IEEE "); +#ifdef __ARM_FP16_FORMAT_IEEE + ODP_PRINT("%i\n", __ARM_FP16_FORMAT_IEEE); +#else + ODP_PRINT("%s\n", ndef); +#endif + + ODP_PRINT(" __ARM_NEON "); +#ifdef __ARM_NEON + ODP_PRINT("%i\n", __ARM_NEON); +#else + ODP_PRINT("%s\n", ndef); +#endif + + ODP_PRINT(" __ARM_NEON_FP "); +#ifdef __ARM_NEON_FP + ODP_PRINT("0x%X\n", __ARM_NEON_FP); +#else + ODP_PRINT("%s\n", ndef); +#endif + + ODP_PRINT(" __ARM_PCS_AAPCS64 "); +#ifdef __ARM_PCS_AAPCS64 + ODP_PRINT("%i\n", __ARM_PCS_AAPCS64); +#else + ODP_PRINT("%s\n", ndef); +#endif + + ODP_PRINT(" __ARM_ROPI "); +#ifdef __ARM_ROPI + ODP_PRINT("%i\n", __ARM_ROPI); +#else + ODP_PRINT("%s\n", ndef); +#endif + + ODP_PRINT(" __ARM_RWPI "); +#ifdef __ARM_RWPI + ODP_PRINT("%i\n", __ARM_RWPI); +#else + ODP_PRINT("%s\n", ndef); +#endif + + ODP_PRINT(" __ARM_SIZEOF_MINIMAL_ENUM "); +#ifdef __ARM_SIZEOF_MINIMAL_ENUM + ODP_PRINT("%i\n", __ARM_SIZEOF_MINIMAL_ENUM); +#else + ODP_PRINT("%s\n", ndef); +#endif + + ODP_PRINT(" __ARM_SIZEOF_WCHAR_T "); +#ifdef __ARM_SIZEOF_WCHAR_T + ODP_PRINT("%i\n", __ARM_SIZEOF_WCHAR_T); +#else + ODP_PRINT("%s\n", ndef); +#endif + + ODP_PRINT(" ARM ISA version: "); +#if defined(__ARM_ARCH) + if (__ARM_ARCH < 8) { + ODP_PRINT("v%i\n", __ARM_ARCH); + } else if (__ARM_ARCH == 8) { + /* Actually, this checks for new NEON instructions in + * v8.1, but is currently the only way to distinguish + * v8.0 and >=v8.1. */ + #ifdef __ARM_FEATURE_QRDMX + ODP_PRINT("v8.1 or higher\n"); + #else + ODP_PRINT("v8.0\n"); + #endif + } else { + /* ACLE 2018 defines that from v8.1 onwards the value includes + * the minor version number: __ARM_ARCH = X * 100 + Y + * E.g. for Armv8.1 __ARM_ARCH = 801 */ + int major = __ARM_ARCH / 100; + int minor = __ARM_ARCH - (major * 100); + + ODP_PRINT("v%i.%i\n", major, minor); + } +#else + ODP_PRINT("%s\n", ndef); +#endif + + ODP_PRINT("\n"); +} + +static int check_hwcap_duplicates(unsigned int hwcap_field) +{ + int ret = 0; + + /* FP and AdvSIMD fields of the AArch64 Processor + * Feature Register 0 must have the same value and are + * defined by the same feature flag. Print the flag + * only once. */ +#ifdef HWCAP_ASIMD + if (hwcap_field == HWCAP_ASIMD) + ret = 1; +#endif +#ifdef HWCAP_ASIMDHP + if (hwcap_field == HWCAP_ASIMDHP) + ret = 1; +#endif + + return ret; +} + +static void _odp_sys_info_print_hwcap_flags(void) +{ + unsigned long hwcaps, hwcaps2; + unsigned int size, size2; + + ODP_PRINT("ARM FEATURES SUPPORTED BY HARDWARE:\n"); + + /* Print supported hardware flags via AT_HWCAP entry of the hwcaps + * auxiliary vector. */ + hwcaps = getauxval(AT_HWCAP); + size = sizeof(hwcap_flags) / sizeof(hwcap_feat_flag_t); + for (unsigned int i = 0; i < size; i++) { + if (hwcap_flags[i].valid) { + if (check_hwcap_duplicates(hwcap_flags[i].hwcap_field)) { + hwcaps = hwcaps >> 1; + continue; + } + + if (hwcaps & 0x01) + ODP_PRINT("%s ", hwcap_flags[i].feat_flag); + hwcaps = hwcaps >> 1; + } + } + + /* Print supported hardware flags via AT_HWCAP2 entry of the hwcaps + * auxiliary vector. */ + hwcaps2 = getauxval(AT_HWCAP2); + size2 = sizeof(hwcap2_flags) / sizeof(hwcap_feat_flag_t); + for (unsigned long i = 0; i < size2; i++) { + if (hwcap2_flags[i].valid) { + if (hwcaps2 & 0x01) + ODP_PRINT("%s ", hwcap2_flags[i].feat_flag); + hwcaps2 = hwcaps2 >> 1; + } + } + + ODP_PRINT("\n"); + + /* Re-initialize hwcaps and hwcaps2 */ + hwcaps = 0; + hwcaps2 = 0; + + ODP_PRINT("\nARM FEATURES NOT SUPPORTED BY HARDWARE:\n"); + + hwcaps = getauxval(AT_HWCAP); + for (unsigned long i = 0; i < size; i++) { + if (hwcap_flags[i].valid) { + if (check_hwcap_duplicates(hwcap_flags[i].hwcap_field)) { + hwcaps = hwcaps >> 1; + continue; + } + + if (!(hwcaps & 0x01)) + ODP_PRINT("%s ", hwcap_flags[i].feat_flag); + hwcaps = hwcaps >> 1; + } + } + + hwcaps2 = getauxval(AT_HWCAP2); + for (unsigned long i = 0; i < size2; i++) { + if (hwcap2_flags[i].valid) { + if (!(hwcaps2 & 0x01)) + ODP_PRINT("%s ", hwcap2_flags[i].feat_flag); + hwcaps2 = hwcaps2 >> 1; + } + } + + ODP_PRINT("\n"); + + ODP_PRINT("\nARM FEATURES NOT SUPPORTED BY KERNEL:\n"); + + for (unsigned long i = 0; i < size; i++) { + if (!hwcap_flags[i].valid) + ODP_PRINT("%s ", hwcap_flags[i].feat_flag); + } + + for (unsigned long i = 0; i < size2; i++) { + if (!hwcap2_flags[i].valid) + ODP_PRINT("%s ", hwcap2_flags[i].feat_flag); + } + + ODP_PRINT("\n\n"); +} + +void _odp_cpu_flags_print_all(void) +{ + _odp_sys_info_print_acle_flags(); + _odp_sys_info_print_hwcap_flags(); +} diff --git a/platform/linux-generic/arch/aarch64/cpu_flags.h b/platform/linux-generic/arch/aarch64/cpu_flags.h new file mode 100644 index 000000000..177b1c44f --- /dev/null +++ b/platform/linux-generic/arch/aarch64/cpu_flags.h @@ -0,0 +1,20 @@ +/* Copyright (c) 2021, ARM Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_PLAT_CPU_FLAGS_H_ +#define ODP_PLAT_CPU_FLAGS_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +void _odp_cpu_flags_print_all(void); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/arch/aarch64/odp/api/abi/atomic_inlines.h b/platform/linux-generic/arch/aarch64/odp/api/abi/atomic_inlines.h index 446f9cc50..f530afd4f 100644 --- a/platform/linux-generic/arch/aarch64/odp/api/abi/atomic_inlines.h +++ b/platform/linux-generic/arch/aarch64/odp/api/abi/atomic_inlines.h @@ -22,7 +22,7 @@ * @return 1 for success and 0 for fail */ #define ATOMIC_CAS_OP_128(atom, oper, old_val, new_val, val) \ -({ \ +__extension__ ({ \ odp_u128_t _val; \ odp_atomic_u128_t *_atom = atom; \ odp_u128_t *_old_val = old_val; \ @@ -152,7 +152,96 @@ static inline int _odp_atomic_cas_acq_rel_u128(odp_atomic_u128_t *atom, odp_u128 return 0; } -#else /* _ODP_LOCK_FREE_128BIT_ATOMICS */ + +static inline void _odp_atomic_add_u32(odp_atomic_u32_t *atom, uint32_t val) +{ + __asm__ volatile("stadd %w[val], %[atom]" + : [atom] "+Q" (atom->v) + : [val] "r" (val)); +} + +static inline void _odp_atomic_sub_u32(odp_atomic_u32_t *atom, uint32_t val) +{ + int32_t neg_val = -val; + + __asm__ volatile("stadd %w[neg_val], %[atom]" + : [atom] "+Q" (atom->v) + : [neg_val] "r" (neg_val)); +} + +static inline void _odp_atomic_inc_u32(odp_atomic_u32_t *atom) +{ + _odp_atomic_add_u32(atom, 1); +} + +static inline void _odp_atomic_dec_u32(odp_atomic_u32_t *atom) +{ + _odp_atomic_sub_u32(atom, 1); +} + +static inline void _odp_atomic_add_u64(odp_atomic_u64_t *atom, uint64_t val) +{ + __asm__ volatile("stadd %[val], %[atom]" + : [atom] "+Q" (atom->v) + : [val] "r" (val)); +} + +static inline void _odp_atomic_sub_u64(odp_atomic_u64_t *atom, uint64_t val) +{ + int64_t neg_val = -val; + + __asm__ volatile("stadd %[neg_val], %[atom]" + : [atom] "+Q" (atom->v) + : [neg_val] "r" (neg_val)); +} + +static inline void _odp_atomic_inc_u64(odp_atomic_u64_t *atom) +{ + _odp_atomic_add_u64(atom, 1); +} + +static inline void _odp_atomic_dec_u64(odp_atomic_u64_t *atom) +{ + _odp_atomic_sub_u64(atom, 1); +} + +static inline void _odp_atomic_add_rel_u32(odp_atomic_u32_t *atom, uint32_t val) +{ + __asm__ volatile("staddl %w[val], %[atom]" + : [atom] "+Q" (atom->v) + : [val] "r" (val) + : "memory"); +} + +static inline void _odp_atomic_sub_rel_u32(odp_atomic_u32_t *atom, uint32_t val) +{ + int32_t neg_val = -val; + + __asm__ volatile("staddl %w[neg_val], %[atom]" + : [atom] "+Q" (atom->v) + : [neg_val] "r" (neg_val) + : "memory"); +} + +static inline void _odp_atomic_add_rel_u64(odp_atomic_u64_t *atom, uint64_t val) +{ + __asm__ volatile("staddl %[val], %[atom]" + : [atom] "+Q" (atom->v) + : [val] "r" (val) + : "memory"); +} + +static inline void _odp_atomic_sub_rel_u64(odp_atomic_u64_t *atom, uint64_t val) +{ + int64_t neg_val = -val; + + __asm__ volatile("staddl %[neg_val], %[atom]" + : [atom] "+Q" (atom->v) + : [neg_val] "r" (neg_val) + : "memory"); +} + +#else /* !_ODP_LOCK_FREE_128BIT_ATOMICS */ /* Use generic implementation */ #include <odp/api/abi/atomic_generic.h> diff --git a/platform/linux-generic/arch/aarch64/odp_atomic.h b/platform/linux-generic/arch/aarch64/odp_atomic.h index c7a28fc2c..445d91e28 100644 --- a/platform/linux-generic/arch/aarch64/odp_atomic.h +++ b/platform/linux-generic/arch/aarch64/odp_atomic.h @@ -36,6 +36,10 @@ do { \ #define LL_MO(mo) (HAS_ACQ((mo)) ? __ATOMIC_ACQUIRE : __ATOMIC_RELAXED) #define SC_MO(mo) (HAS_RLS((mo)) ? __ATOMIC_RELEASE : __ATOMIC_RELAXED) +/* Prevent warnings about ISO C not supporting __int128 */ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wpedantic" + #ifndef __ARM_FEATURE_QRDMX /* Feature only available in v8.1a and beyond */ static inline bool __lockfree_compare_exchange_16(register __int128 *var, __int128 *exp, @@ -219,7 +223,7 @@ static inline __int128 __lockfree_load_16(__int128 *var, int mo) return old; } -typedef unsigned __int128 _u128_t; +__extension__ typedef unsigned __int128 _u128_t; static inline _u128_t lockfree_load_u128(_u128_t *atomic) { @@ -291,6 +295,8 @@ static inline bitset_t bitset_mask(uint32_t bit) return (unsigned __int128)(1ULL << (bit - 64)) << 64; } +#pragma GCC diagnostic pop + #else #error Unsupported size of bit sets (ATOM_BITSET_SIZE) #endif diff --git a/platform/linux-generic/arch/aarch64/odp_llsc.h b/platform/linux-generic/arch/aarch64/odp_llsc.h index cf55be132..2561451ca 100644 --- a/platform/linux-generic/arch/aarch64/odp_llsc.h +++ b/platform/linux-generic/arch/aarch64/odp_llsc.h @@ -115,11 +115,11 @@ static inline uint32_t sc(uint64_t *var, uint64_t neu, int mm) #define sc64(a, b, c) sc((a), (b), (c)) union i128 { - __int128 i128; + __extension__ __int128 i128; int64_t i64[2]; }; -static inline __int128 lld(__int128 *var, int mm) +__extension__ static inline __int128 lld(__int128 *var, int mm) { union i128 old; @@ -139,10 +139,12 @@ static inline __int128 lld(__int128 *var, int mm) } /* Return 0 on success, 1 on failure */ -static inline uint32_t scd(__int128 *var, __int128 neu, int mm) +__extension__ static inline uint32_t scd(__int128 *var, __int128 neu, int mm) { uint32_t ret; +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wpedantic" if (mm == __ATOMIC_RELEASE) __asm__ volatile("stlxp %w0, %1, %2, [%3]" : "=&r" (ret) @@ -159,6 +161,7 @@ static inline uint32_t scd(__int128 *var, __int128 neu, int mm) : ); else ODP_ABORT(); +#pragma GCC diagnostic pop return ret; } diff --git a/platform/linux-generic/arch/aarch64/odp_sysinfo_parse.c b/platform/linux-generic/arch/aarch64/odp_sysinfo_parse.c index ab64d501e..dbb6d43af 100644 --- a/platform/linux-generic/arch/aarch64/odp_sysinfo_parse.c +++ b/platform/linux-generic/arch/aarch64/odp_sysinfo_parse.c @@ -13,6 +13,7 @@ #include <odp_global_data.h> #include <odp_sysinfo_internal.h> #include <odp_debug_internal.h> +#include "cpu_flags.h" #define TMP_STR_LEN 64 @@ -334,139 +335,7 @@ int _odp_cpuinfo_parser(FILE *file, system_info_t *sysinfo) void _odp_sys_info_print_arch(void) { - const char *ndef = "n/a"; - - /* Avoid compiler warning about unused variable */ - (void)ndef; - - /* See ARM C Language Extensions documentation for details */ - ODP_PRINT("ARM FEATURES:\n"); - - ODP_PRINT(" __ARM_ARCH "); -#ifdef __ARM_ARCH - ODP_PRINT("%i\n", __ARM_ARCH); -#else - ODP_PRINT("%s\n", ndef); -#endif - - ODP_PRINT(" __ARM_ARCH_ISA_A64 "); -#ifdef __ARM_ARCH_ISA_A64 - ODP_PRINT("%i\n", __ARM_ARCH_ISA_A64); -#else - ODP_PRINT("%s\n", ndef); -#endif - - ODP_PRINT(" __ARM_NEON "); -#ifdef __ARM_NEON - ODP_PRINT("%i\n", __ARM_NEON); -#else - ODP_PRINT("%s\n", ndef); -#endif - - ODP_PRINT(" __ARM_FEATURE_ATOMICS "); -#ifdef __ARM_FEATURE_ATOMICS - ODP_PRINT("%i\n", __ARM_FEATURE_ATOMICS); -#else - ODP_PRINT("%s\n", ndef); -#endif - - ODP_PRINT(" __ARM_FEATURE_UNALIGNED "); -#ifdef __ARM_FEATURE_UNALIGNED - ODP_PRINT("%i\n", __ARM_FEATURE_UNALIGNED); -#else - ODP_PRINT("%s\n", ndef); -#endif - - ODP_PRINT(" __ARM_FEATURE_IDIV "); -#ifdef __ARM_FEATURE_IDIV - ODP_PRINT("%i\n", __ARM_FEATURE_IDIV); -#else - ODP_PRINT("%s\n", ndef); -#endif - - ODP_PRINT(" __ARM_FEATURE_QRDMX "); -#ifdef __ARM_FEATURE_QRDMX - ODP_PRINT("%i\n", __ARM_FEATURE_QRDMX); -#else - ODP_PRINT("%s\n", ndef); -#endif - - ODP_PRINT(" __ARM_FEATURE_DOTPROD "); -#ifdef __ARM_FEATURE_DOTPROD - ODP_PRINT("%i\n", __ARM_FEATURE_DOTPROD); -#else - ODP_PRINT("%s\n", ndef); -#endif - - ODP_PRINT(" __ARM_FEATURE_CRYPTO "); -#ifdef __ARM_FEATURE_CRYPTO - ODP_PRINT("%i\n", __ARM_FEATURE_CRYPTO); -#else - ODP_PRINT("%s\n", ndef); -#endif - - ODP_PRINT(" __ARM_FEATURE_SHA512 "); -#ifdef __ARM_FEATURE_SHA512 - ODP_PRINT("%i\n", __ARM_FEATURE_SHA512); -#else - ODP_PRINT("%s\n", ndef); -#endif - - ODP_PRINT(" __ARM_FEATURE_SHA3 "); -#ifdef __ARM_FEATURE_SHA3 - ODP_PRINT("%i\n", __ARM_FEATURE_SHA3); -#else - ODP_PRINT("%s\n", ndef); -#endif - - ODP_PRINT(" __ARM_FEATURE_SM3 "); -#ifdef __ARM_FEATURE_SM3 - ODP_PRINT("%i\n", __ARM_FEATURE_SM3); -#else - ODP_PRINT("%s\n", ndef); -#endif - - ODP_PRINT(" __ARM_FEATURE_SM4 "); -#ifdef __ARM_FEATURE_SM4 - ODP_PRINT("%i\n", __ARM_FEATURE_SM4); -#else - ODP_PRINT("%s\n", ndef); -#endif - - ODP_PRINT(" __ARM_FEATURE_CRC32 "); -#ifdef __ARM_FEATURE_CRC32 - ODP_PRINT("%i\n", __ARM_FEATURE_CRC32); -#else - ODP_PRINT("%s\n", ndef); -#endif - - ODP_PRINT(" ARM ISA version: "); -#if defined(__ARM_ARCH) - if (__ARM_ARCH < 8) { - ODP_PRINT("v%i\n", __ARM_ARCH); - } else if (__ARM_ARCH == 8) { - /* Actually, this checks for new NEON instructions in - * v8.1, but is currently the only way to distinguish - * v8.0 and >=v8.1. */ - #ifdef __ARM_FEATURE_QRDMX - ODP_PRINT("v8.1 or higher\n"); - #else - ODP_PRINT("v8.0\n"); - #endif - } else { - /* ACLE 2018 defines that from v8.1 onwards the value includes - * the minor version number: __ARM_ARCH = X * 100 + Y - * E.g. for Armv8.1 __ARM_ARCH = 801 */ - int major = __ARM_ARCH / 100; - int minor = __ARM_ARCH - (major * 100); - - ODP_PRINT("v%i.%i\n", major, minor); - } -#else - ODP_PRINT("%s\n", ndef); -#endif - - ODP_PRINT("\n"); + _odp_cpu_flags_print_all(); } uint64_t odp_cpu_arch_hz_current(int id ODP_UNUSED) diff --git a/platform/linux-generic/arch/default/odp/api/abi/atomic_generic.h b/platform/linux-generic/arch/default/odp/api/abi/atomic_generic.h index e274a4d64..aec8b9f00 100644 --- a/platform/linux-generic/arch/default/odp/api/abi/atomic_generic.h +++ b/platform/linux-generic/arch/default/odp/api/abi/atomic_generic.h @@ -10,6 +10,68 @@ #include <odp/api/atomic.h> +static inline void _odp_atomic_add_u32(odp_atomic_u32_t *atom, uint32_t val) +{ + (void)__atomic_fetch_add(&atom->v, val, __ATOMIC_RELAXED); +} + +static inline void _odp_atomic_sub_u32(odp_atomic_u32_t *atom, uint32_t val) +{ + (void)__atomic_fetch_sub(&atom->v, val, __ATOMIC_RELAXED); +} + +static inline void _odp_atomic_inc_u32(odp_atomic_u32_t *atom) +{ + (void)__atomic_fetch_add(&atom->v, 1, __ATOMIC_RELAXED); +} + +static inline void _odp_atomic_dec_u32(odp_atomic_u32_t *atom) +{ + (void)__atomic_fetch_sub(&atom->v, 1, __ATOMIC_RELAXED); +} + +static inline void _odp_atomic_add_rel_u32(odp_atomic_u32_t *atom, uint32_t val) +{ + (void)__atomic_fetch_add(&atom->v, val, __ATOMIC_RELEASE); +} + +static inline void _odp_atomic_sub_rel_u32(odp_atomic_u32_t *atom, uint32_t val) +{ + (void)__atomic_fetch_sub(&atom->v, val, __ATOMIC_RELEASE); +} + +static inline void _odp_atomic_add_u64(odp_atomic_u64_t *atom, uint64_t val) +{ + (void)__atomic_fetch_add(&atom->v, val, __ATOMIC_RELAXED); +} + +static inline void _odp_atomic_sub_u64(odp_atomic_u64_t *atom, uint64_t val) +{ + (void)__atomic_fetch_sub(&atom->v, val, __ATOMIC_RELAXED); +} + +static inline void _odp_atomic_inc_u64(odp_atomic_u64_t *atom) +{ + (void)__atomic_fetch_add(&atom->v, 1, __ATOMIC_RELAXED); +} + +static inline void _odp_atomic_dec_u64(odp_atomic_u64_t *atom) +{ + (void)__atomic_fetch_sub(&atom->v, 1, __ATOMIC_RELAXED); +} + +#ifndef ODP_ATOMIC_U64_LOCK +static inline void _odp_atomic_add_rel_u64(odp_atomic_u64_t *atom, uint64_t val) +{ + (void)__atomic_fetch_add(&atom->v, val, __ATOMIC_RELEASE); +} + +static inline void _odp_atomic_sub_rel_u64(odp_atomic_u64_t *atom, uint64_t val) +{ + (void)__atomic_fetch_sub(&atom->v, val, __ATOMIC_RELEASE); +} +#endif + #ifdef __SIZEOF_INT128__ static inline void _odp_atomic_init_u128(odp_atomic_u128_t *atom, odp_u128_t val) @@ -78,7 +140,7 @@ static inline int _odp_atomic_cas_acq_rel_u128(odp_atomic_u128_t *atom, odp_u128 * 128 bit CAS operation expression for the ATOMIC_OP macro */ #define ATOMIC_CAS_OP_128(ret_ptr, old_val, new_val) \ -({ \ +__extension__ ({ \ int *_ret_ptr = ret_ptr; \ odp_u128_t *_old_val = old_val; \ odp_u128_t _new_val = new_val; \ @@ -99,7 +161,7 @@ static inline int _odp_atomic_cas_acq_rel_u128(odp_atomic_u128_t *atom, odp_u128 * @return The old value of the variable. */ #define ATOMIC_OP_128(atom, expr) \ -({ \ +__extension__ ({ \ odp_u128_t _old_val; \ odp_atomic_u128_t *_atom = atom; \ /* Loop while lock is already taken, stop when lock becomes clear */ \ diff --git a/platform/linux-generic/arch/default/odp_atomic.h b/platform/linux-generic/arch/default/odp_atomic.h index fc8260194..7650d2b52 100644 --- a/platform/linux-generic/arch/default/odp_atomic.h +++ b/platform/linux-generic/arch/default/odp_atomic.h @@ -9,7 +9,7 @@ #ifdef __SIZEOF_INT128__ -typedef unsigned __int128 _u128_t; +__extension__ typedef unsigned __int128 _u128_t; static inline _u128_t lockfree_load_u128(_u128_t *atomic) { diff --git a/platform/linux-generic/include/odp/api/plat/atomic_inlines.h b/platform/linux-generic/include/odp/api/plat/atomic_inlines.h index a9da70890..5ce4bfc28 100644 --- a/platform/linux-generic/include/odp/api/plat/atomic_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/atomic_inlines.h @@ -100,7 +100,7 @@ _ODP_INLINE uint32_t odp_atomic_fetch_add_u32(odp_atomic_u32_t *atom, _ODP_INLINE void odp_atomic_add_u32(odp_atomic_u32_t *atom, uint32_t val) { - (void)__atomic_fetch_add(&atom->v, val, __ATOMIC_RELAXED); + _odp_atomic_add_u32(atom, val); } _ODP_INLINE uint32_t odp_atomic_fetch_sub_u32(odp_atomic_u32_t *atom, @@ -111,7 +111,7 @@ _ODP_INLINE uint32_t odp_atomic_fetch_sub_u32(odp_atomic_u32_t *atom, _ODP_INLINE void odp_atomic_sub_u32(odp_atomic_u32_t *atom, uint32_t val) { - (void)__atomic_fetch_sub(&atom->v, val, __ATOMIC_RELAXED); + _odp_atomic_sub_u32(atom, val); } _ODP_INLINE uint32_t odp_atomic_fetch_inc_u32(odp_atomic_u32_t *atom) @@ -121,7 +121,7 @@ _ODP_INLINE uint32_t odp_atomic_fetch_inc_u32(odp_atomic_u32_t *atom) _ODP_INLINE void odp_atomic_inc_u32(odp_atomic_u32_t *atom) { - (void)__atomic_fetch_add(&atom->v, 1, __ATOMIC_RELAXED); + _odp_atomic_inc_u32(atom); } _ODP_INLINE uint32_t odp_atomic_fetch_dec_u32(odp_atomic_u32_t *atom) @@ -131,7 +131,7 @@ _ODP_INLINE uint32_t odp_atomic_fetch_dec_u32(odp_atomic_u32_t *atom) _ODP_INLINE void odp_atomic_dec_u32(odp_atomic_u32_t *atom) { - (void)__atomic_fetch_sub(&atom->v, 1, __ATOMIC_RELAXED); + _odp_atomic_dec_u32(atom); } _ODP_INLINE int odp_atomic_cas_u32(odp_atomic_u32_t *atom, uint32_t *old_val, @@ -180,7 +180,7 @@ _ODP_INLINE void odp_atomic_min_u32(odp_atomic_u32_t *atom, uint32_t new_min) * CAS operation expression for the ATOMIC_OP macro */ #define ATOMIC_CAS_OP(ret_ptr, old_val, new_val) \ -({ \ +__extension__ ({ \ if (atom->v == (old_val)) { \ atom->v = (new_val); \ *(ret_ptr) = 1; \ @@ -197,7 +197,7 @@ _ODP_INLINE void odp_atomic_min_u32(odp_atomic_u32_t *atom, uint32_t new_min) * @return The old value of the variable. */ #define ATOMIC_OP(atom, expr) \ -({ \ +__extension__ ({ \ uint64_t _old_val; \ /* Loop while lock is already taken, stop when lock becomes clear */ \ while (__atomic_test_and_set(&(atom)->lock, __ATOMIC_ACQUIRE)) \ @@ -350,7 +350,7 @@ _ODP_INLINE uint64_t odp_atomic_fetch_add_u64(odp_atomic_u64_t *atom, _ODP_INLINE void odp_atomic_add_u64(odp_atomic_u64_t *atom, uint64_t val) { - (void)__atomic_fetch_add(&atom->v, val, __ATOMIC_RELAXED); + _odp_atomic_add_u64(atom, val); } _ODP_INLINE uint64_t odp_atomic_fetch_sub_u64(odp_atomic_u64_t *atom, @@ -361,7 +361,7 @@ _ODP_INLINE uint64_t odp_atomic_fetch_sub_u64(odp_atomic_u64_t *atom, _ODP_INLINE void odp_atomic_sub_u64(odp_atomic_u64_t *atom, uint64_t val) { - (void)__atomic_fetch_sub(&atom->v, val, __ATOMIC_RELAXED); + _odp_atomic_sub_u64(atom, val); } _ODP_INLINE uint64_t odp_atomic_fetch_inc_u64(odp_atomic_u64_t *atom) @@ -371,7 +371,7 @@ _ODP_INLINE uint64_t odp_atomic_fetch_inc_u64(odp_atomic_u64_t *atom) _ODP_INLINE void odp_atomic_inc_u64(odp_atomic_u64_t *atom) { - (void)__atomic_fetch_add(&atom->v, 1, __ATOMIC_RELAXED); + _odp_atomic_inc_u64(atom); } _ODP_INLINE uint64_t odp_atomic_fetch_dec_u64(odp_atomic_u64_t *atom) @@ -381,7 +381,7 @@ _ODP_INLINE uint64_t odp_atomic_fetch_dec_u64(odp_atomic_u64_t *atom) _ODP_INLINE void odp_atomic_dec_u64(odp_atomic_u64_t *atom) { - (void)__atomic_fetch_sub(&atom->v, 1, __ATOMIC_RELAXED); + _odp_atomic_dec_u64(atom); } _ODP_INLINE int odp_atomic_cas_u64(odp_atomic_u64_t *atom, uint64_t *old_val, @@ -411,12 +411,12 @@ _ODP_INLINE void odp_atomic_store_rel_u64(odp_atomic_u64_t *atom, uint64_t val) _ODP_INLINE void odp_atomic_add_rel_u64(odp_atomic_u64_t *atom, uint64_t val) { - (void)__atomic_fetch_add(&atom->v, val, __ATOMIC_RELEASE); + _odp_atomic_add_rel_u64(atom, val); } _ODP_INLINE void odp_atomic_sub_rel_u64(odp_atomic_u64_t *atom, uint64_t val) { - (void)__atomic_fetch_sub(&atom->v, val, __ATOMIC_RELEASE); + _odp_atomic_sub_rel_u64(atom, val); } _ODP_INLINE int odp_atomic_cas_acq_u64(odp_atomic_u64_t *atom, @@ -485,12 +485,12 @@ _ODP_INLINE void odp_atomic_store_rel_u32(odp_atomic_u32_t *atom, uint32_t val) _ODP_INLINE void odp_atomic_add_rel_u32(odp_atomic_u32_t *atom, uint32_t val) { - (void)__atomic_fetch_add(&atom->v, val, __ATOMIC_RELEASE); + _odp_atomic_add_rel_u32(atom, val); } _ODP_INLINE void odp_atomic_sub_rel_u32(odp_atomic_u32_t *atom, uint32_t val) { - (void)__atomic_fetch_sub(&atom->v, val, __ATOMIC_RELEASE); + _odp_atomic_sub_rel_u32(atom, val); } _ODP_INLINE int odp_atomic_cas_acq_u32(odp_atomic_u32_t *atom, diff --git a/platform/linux-generic/include/odp_atomic_internal.h b/platform/linux-generic/include/odp_atomic_internal.h index 81280b1fa..d98e0e6be 100644 --- a/platform/linux-generic/include/odp_atomic_internal.h +++ b/platform/linux-generic/include/odp_atomic_internal.h @@ -138,7 +138,7 @@ static inline void _odp_atomic_flag_clear(_odp_atomic_flag_t *flag) #ifdef ODP_ATOMIC_U128 /** An unsigned 128-bit (16-byte) scalar type */ -typedef __int128 _uint128_t; +__extension__ typedef __int128 _uint128_t; /** Atomic 128-bit type */ typedef struct ODP_ALIGNED(16) { diff --git a/platform/linux-generic/include/odp_buffer_internal.h b/platform/linux-generic/include/odp_buffer_internal.h index e7bc78d6e..dec85f9d3 100644 --- a/platform/linux-generic/include/odp_buffer_internal.h +++ b/platform/linux-generic/include/odp_buffer_internal.h @@ -83,7 +83,7 @@ struct ODP_ALIGNED_CACHE odp_buffer_hdr_t { uint8_t flow_id; /* Data or next header */ - uint8_t data[0]; + uint8_t data[]; }; /* Buffer header size is critical for performance. Ensure that it does not accidentally diff --git a/platform/linux-generic/include/odp_config_internal.h b/platform/linux-generic/include/odp_config_internal.h index 40d2639f1..899b261bd 100644 --- a/platform/linux-generic/include/odp_config_internal.h +++ b/platform/linux-generic/include/odp_config_internal.h @@ -54,6 +54,11 @@ extern "C" { #define CONFIG_QUEUE_MAX_ORD_LOCKS 2 /* + * Maximum number of stashes + */ +#define CONFIG_MAX_STASHES 128 + +/* * Maximum number of packet IO resources */ #define ODP_CONFIG_PKTIO_ENTRIES 64 @@ -119,10 +124,10 @@ extern "C" { /* * Number of shared memory blocks reserved for implementation internal use. * - * Each pool requires three SHM blocks (buffers, ring, user area). 20 blocks are - * reserved for per ODP module global data. + * Each stash requires one SHM block, each pool requires three blocks (buffers, + * ring, user area), and 20 blocks are reserved for per ODP module global data. */ -#define CONFIG_INTERNAL_SHM_BLOCKS ((ODP_CONFIG_POOLS * 3) + 20) +#define CONFIG_INTERNAL_SHM_BLOCKS (CONFIG_MAX_STASHES + (ODP_CONFIG_POOLS * 3) + 20) /* * Maximum number of shared memory blocks. diff --git a/platform/linux-generic/include/odp_debug_internal.h b/platform/linux-generic/include/odp_debug_internal.h index 5027cfe4a..c5c1890c3 100644 --- a/platform/linux-generic/include/odp_debug_internal.h +++ b/platform/linux-generic/include/odp_debug_internal.h @@ -29,6 +29,10 @@ extern "C" { #endif +/* Avoid "ISO C99 requires at least one argument for the "..." in a variadic + * macro" errors when building with 'pedantic' option. */ +#pragma GCC system_header + /* Debug level configure option. Zero is the highest level. Value of N prints debug messages from * level 0 to N. */ #define CONFIG_DEBUG_LEVEL 0 diff --git a/platform/linux-generic/include/odp_event_vector_internal.h b/platform/linux-generic/include/odp_event_vector_internal.h index d3322eff5..2d51801df 100644 --- a/platform/linux-generic/include/odp_event_vector_internal.h +++ b/platform/linux-generic/include/odp_event_vector_internal.h @@ -17,6 +17,8 @@ #include <odp/api/packet.h> #include <odp_buffer_internal.h> +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wpedantic" /** * Internal event vector header */ @@ -28,9 +30,10 @@ typedef struct { uint32_t size; /* Vector of packet handles */ - odp_packet_t packet[0]; + odp_packet_t packet[]; } odp_event_vector_hdr_t; +#pragma GCC diagnostic pop /** * Return the vector header diff --git a/platform/linux-generic/include/odp_ipsec_internal.h b/platform/linux-generic/include/odp_ipsec_internal.h index 66d85d119..cc224e4cc 100644 --- a/platform/linux-generic/include/odp_ipsec_internal.h +++ b/platform/linux-generic/include/odp_ipsec_internal.h @@ -84,6 +84,8 @@ int _odp_ipsec_status_send(odp_queue_t queue, #define IPSEC_MAX_SALT_LEN 4 /**< Maximum salt length in bytes */ +#define IPSEC_SEQ_HI_LEN 4 /**< ESN Higher bits length in bytes */ + /* The minimum supported AR window size */ #define IPSEC_AR_WIN_SIZE_MIN 32 @@ -175,6 +177,8 @@ struct ipsec_sa_s { unsigned copy_flabel : 1; unsigned aes_ctr_iv : 1; unsigned udp_encap : 1; + unsigned esn : 1; + unsigned insert_seq_hi : 1; /* Only for outbound */ unsigned use_counter_iv : 1; @@ -264,8 +268,14 @@ typedef struct odp_ipsec_sa_lookup_s { /** IPSEC AAD */ typedef struct ODP_PACKED { - odp_u32be_t spi; /**< Security Parameter Index */ - odp_u32be_t seq_no; /**< Sequence Number */ + /**< Security Parameter Index */ + odp_u32be_t spi; + + /**< Sequence Number */ + union { + odp_u32be_t seq_no; + odp_u64be_t seq_no64; + }; } ipsec_aad_t; /* Return IV length required for the cipher for IPsec use */ @@ -317,14 +327,14 @@ int _odp_ipsec_sa_lifetime_update(ipsec_sa_t *ipsec_sa, uint32_t len, * * @retval <0 if the packet falls out of window */ -int _odp_ipsec_sa_replay_precheck(ipsec_sa_t *ipsec_sa, uint32_t seq, +int _odp_ipsec_sa_replay_precheck(ipsec_sa_t *ipsec_sa, uint64_t seq, odp_ipsec_op_status_t *status); /* Run check on sequence number of the packet and update window if necessary. * * @retval <0 if the packet falls out of window */ -int _odp_ipsec_sa_replay_update(ipsec_sa_t *ipsec_sa, uint32_t seq, +int _odp_ipsec_sa_replay_update(ipsec_sa_t *ipsec_sa, uint64_t seq, odp_ipsec_op_status_t *status); /** diff --git a/platform/linux-generic/include/odp_llqueue.h b/platform/linux-generic/include/odp_llqueue.h index cc4b84ecd..e9cf9945e 100644 --- a/platform/linux-generic/include/odp_llqueue.h +++ b/platform/linux-generic/include/odp_llqueue.h @@ -49,7 +49,7 @@ static odp_bool_t llq_on_queue(struct llnode *node); typedef uint64_t dintptr_t; #endif #if __SIZEOF_PTRDIFF_T__ == 8 -typedef __int128 dintptr_t; +__extension__ typedef __int128 dintptr_t; #endif struct llnode { diff --git a/platform/linux-generic/include/odp_macros_internal.h b/platform/linux-generic/include/odp_macros_internal.h index 229772373..997e0fd5b 100644 --- a/platform/linux-generic/include/odp_macros_internal.h +++ b/platform/linux-generic/include/odp_macros_internal.h @@ -22,14 +22,14 @@ extern "C" { #define ARRAY_SIZE(x) (sizeof(x) / sizeof((x)[0])) #define MIN(a, b) \ - ({ \ + __extension__ ({ \ __typeof__(a) tmp_a = (a); \ __typeof__(b) tmp_b = (b); \ tmp_a < tmp_b ? tmp_a : tmp_b; \ }) #define MAX(a, b) \ - ({ \ + __extension__ ({ \ __typeof__(a) tmp_a = (a); \ __typeof__(b) tmp_b = (b); \ tmp_a > tmp_b ? tmp_a : tmp_b; \ @@ -39,7 +39,7 @@ extern "C" { ((type *)(void *)(((char *)pointer) - offsetof(type, member))) #define DIV_ROUND_UP(a, b) \ - ({ \ + __extension__ ({ \ __typeof__(a) tmp_a = (a); \ __typeof__(b) tmp_b = (b); \ ODP_STATIC_ASSERT(__builtin_constant_p(b), ""); \ diff --git a/platform/linux-generic/include/odp_packet_internal.h b/platform/linux-generic/include/odp_packet_internal.h index 497ea4aee..62f8aea25 100644 --- a/platform/linux-generic/include/odp_packet_internal.h +++ b/platform/linux-generic/include/odp_packet_internal.h @@ -73,6 +73,8 @@ typedef struct { ODP_STATIC_ASSERT(PKT_MAX_SEGS < UINT16_MAX, "PACKET_MAX_SEGS_ERROR"); +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wpedantic" /** * Internal Packet header * @@ -148,9 +150,10 @@ typedef struct odp_packet_hdr_t { }; /* Packet data storage */ - uint8_t data[0]; + uint8_t data[]; } odp_packet_hdr_t; +#pragma GCC diagnostic pop /* Packet header size is critical for performance. Ensure that it does not accidentally * grow over 256 bytes when cache line size is 64 bytes (or less). With larger cache line sizes, diff --git a/platform/linux-generic/include/odp_pool_internal.h b/platform/linux-generic/include/odp_pool_internal.h index dc9f0d207..001bdfc37 100644 --- a/platform/linux-generic/include/odp_pool_internal.h +++ b/platform/linux-generic/include/odp_pool_internal.h @@ -35,6 +35,8 @@ typedef struct ODP_ALIGNED_CACHE pool_cache_t { } pool_cache_t; +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wpedantic" /* Buffer header ring */ typedef struct ODP_ALIGNED_CACHE { /* Ring header */ @@ -44,9 +46,10 @@ typedef struct ODP_ALIGNED_CACHE { odp_buffer_hdr_t *buf_hdr[CONFIG_POOL_MAX_NUM + 1]; /* Index to pointer look-up table for external memory pool */ - odp_buffer_hdr_t *buf_hdr_by_index[0]; + odp_buffer_hdr_t *buf_hdr_by_index[]; } pool_ring_t; +#pragma GCC diagnostic pop /* Callback function for pool destroy */ typedef void (*pool_destroy_cb_fn)(void *pool); diff --git a/platform/linux-generic/include/odp_ring_common.h b/platform/linux-generic/include/odp_ring_common.h index 88e6bf880..a2d9e4be5 100644 --- a/platform/linux-generic/include/odp_ring_common.h +++ b/platform/linux-generic/include/odp_ring_common.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2019, Nokia +/* Copyright (c) 2019-2021, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -12,7 +12,8 @@ extern "C" { #endif #define _ODP_RING_TYPE_U32 1 -#define _ODP_RING_TYPE_PTR 2 +#define _ODP_RING_TYPE_U64 2 +#define _ODP_RING_TYPE_PTR 3 #ifdef __cplusplus } diff --git a/platform/linux-generic/include/odp_ring_internal.h b/platform/linux-generic/include/odp_ring_internal.h index 6ac6d0ee3..d11e81bf2 100644 --- a/platform/linux-generic/include/odp_ring_internal.h +++ b/platform/linux-generic/include/odp_ring_internal.h @@ -1,5 +1,5 @@ /* Copyright (c) 2016-2018, Linaro Limited - * Copyright (c) 2019, Nokia + * Copyright (c) 2019-2021, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -45,12 +45,17 @@ struct ring_common { typedef struct ODP_ALIGNED_CACHE { struct ring_common r; - uint32_t data[0]; + uint32_t data[]; } ring_u32_t; typedef struct ODP_ALIGNED_CACHE { struct ring_common r; - void *data[0]; + uint64_t data[]; +} ring_u64_t; + +typedef struct ODP_ALIGNED_CACHE { + struct ring_common r; + void *data[]; } ring_ptr_t; /* 32-bit CAS with memory order selection */ @@ -87,6 +92,16 @@ static inline int cas_mo_u32(odp_atomic_u32_t *atom, uint32_t *old_val, #define _RING_ENQ ring_u32_enq #define _RING_ENQ_MULTI ring_u32_enq_multi #define _RING_LEN ring_u32_len +#elif _ODP_RING_TYPE == _ODP_RING_TYPE_U64 + #define _ring_gen_t ring_u64_t + #define _ring_data_t uint64_t + + #define _RING_INIT ring_u64_init + #define _RING_DEQ ring_u64_deq + #define _RING_DEQ_MULTI ring_u64_deq_multi + #define _RING_ENQ ring_u64_enq + #define _RING_ENQ_MULTI ring_u64_enq_multi + #define _RING_LEN ring_u64_len #elif _ODP_RING_TYPE == _ODP_RING_TYPE_PTR #define _ring_gen_t ring_ptr_t #define _ring_data_t void * diff --git a/platform/linux-generic/include/odp_ring_u64_internal.h b/platform/linux-generic/include/odp_ring_u64_internal.h new file mode 100644 index 000000000..4b4c7b1b5 --- /dev/null +++ b/platform/linux-generic/include/odp_ring_u64_internal.h @@ -0,0 +1,25 @@ +/* Copyright (c) 2021, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_RING_U64_INTERNAL_H_ +#define ODP_RING_U64_INTERNAL_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <odp_ring_common.h> + +#undef _ODP_RING_TYPE +#define _ODP_RING_TYPE _ODP_RING_TYPE_U64 + +#include <odp_ring_internal.h> + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/include/odp_timer_internal.h b/platform/linux-generic/include/odp_timer_internal.h index 23fd54bf9..435fa8b70 100644 --- a/platform/linux-generic/include/odp_timer_internal.h +++ b/platform/linux-generic/include/odp_timer_internal.h @@ -20,6 +20,8 @@ #include <odp/api/timer.h> #include <odp_global_data.h> +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wpedantic" /** * Internal Timeout header */ @@ -37,6 +39,7 @@ typedef struct { odp_timer_t timer; } odp_timeout_hdr_t; +#pragma GCC diagnostic pop /* A larger decrement value should be used after receiving events compared to * an 'empty' call. */ diff --git a/platform/linux-generic/include/protocols/ipsec.h b/platform/linux-generic/include/protocols/ipsec.h index 7838076d4..eabd2d1d9 100644 --- a/platform/linux-generic/include/protocols/ipsec.h +++ b/platform/linux-generic/include/protocols/ipsec.h @@ -34,7 +34,7 @@ extern "C" { typedef struct ODP_PACKED { odp_u32be_t spi; /**< Security Parameter Index */ odp_u32be_t seq_no; /**< Sequence Number */ - uint8_t iv[0]; /**< Initialization vector */ + uint8_t iv[]; /**< Initialization vector */ } _odp_esphdr_t; /** @internal Compile time assert */ @@ -47,7 +47,7 @@ ODP_STATIC_ASSERT(sizeof(_odp_esphdr_t) == _ODP_ESPHDR_LEN, typedef struct ODP_PACKED { uint8_t pad_len; /**< Padding length (0-255) */ uint8_t next_header; /**< Next header protocol */ - uint8_t icv[0]; /**< Integrity Check Value (optional) */ + uint8_t icv[]; /**< Integrity Check Value (optional) */ } _odp_esptrl_t; /** @internal Compile time assert */ @@ -63,7 +63,7 @@ typedef struct ODP_PACKED { odp_u16be_t pad; /**< Padding (must be 0) */ odp_u32be_t spi; /**< Security Parameter Index */ odp_u32be_t seq_no; /**< Sequence Number */ - uint8_t icv[0]; /**< Integrity Check Value */ + uint8_t icv[]; /**< Integrity Check Value */ } _odp_ahhdr_t; /** @internal Compile time assert */ diff --git a/platform/linux-generic/m4/configure.m4 b/platform/linux-generic/m4/configure.m4 index 01593baea..ffac70414 100644 --- a/platform/linux-generic/m4/configure.m4 +++ b/platform/linux-generic/m4/configure.m4 @@ -35,6 +35,11 @@ AS_VAR_APPEND([PLAT_CFG_TEXT], [" pcapng: ${have_pcapng} default_config_path: ${default_config_path}"]) +# Ignore Clang specific errors about fields with variable sized type not at the +# end of a struct. This style is used by e.g. odp_packet_hdr_t and +# odp_timeout_hdr_t. +ODP_CHECK_CFLAG([-Wno-error=gnu-variable-sized-type-not-at-end]) + AC_CONFIG_COMMANDS_PRE([dnl AM_CONDITIONAL([PLATFORM_IS_LINUX_GENERIC], [test "${with_platform}" = "linux-generic"]) diff --git a/platform/linux-generic/odp_buffer.c b/platform/linux-generic/odp_buffer.c index 4f06ce2f0..fed113923 100644 --- a/platform/linux-generic/odp_buffer.c +++ b/platform/linux-generic/odp_buffer.c @@ -52,7 +52,7 @@ void odp_buffer_print(odp_buffer_t buf) len += snprintf(&str[len], n - len, "Buffer\n------\n"); len += snprintf(&str[len], n - len, " pool index %u\n", hdr->index.pool); len += snprintf(&str[len], n - len, " buffer index %u\n", hdr->index.buffer); - len += snprintf(&str[len], n - len, " addr %p\n", hdr->base_data); + len += snprintf(&str[len], n - len, " addr %p\n", (void *)hdr->base_data); len += snprintf(&str[len], n - len, " size %u\n", odp_buffer_size(buf)); str[len] = 0; diff --git a/platform/linux-generic/odp_comp.c b/platform/linux-generic/odp_comp.c index 579bdc03b..5c286caed 100644 --- a/platform/linux-generic/odp_comp.c +++ b/platform/linux-generic/odp_comp.c @@ -134,8 +134,8 @@ static void process_input(odp_packet_t pkt_out, out_data = odp_packet_offset(pkt_out, start, &out_len, &cur_seg); ODP_DBG("out_data %p seg_data_ptr %p out_len %d seg %p\n", - out_data, odp_packet_seg_data(pkt_out, cur_seg), - out_len, cur_seg); + (void *)out_data, odp_packet_seg_data(pkt_out, cur_seg), + out_len, (void *)cur_seg); if (0 == out_len) { /* there are no more segments */ @@ -156,8 +156,8 @@ static void process_input(odp_packet_t pkt_out, ODP_DBG("next_in %p, avail_in %d next_out %p" " avail_out %d, sync %d\n", - streamp->next_in, streamp->avail_in, - streamp->next_out, + (const void *)streamp->next_in, streamp->avail_in, + (void *)streamp->next_out, streamp->avail_out, sync); @@ -259,7 +259,7 @@ static int deflate_comp(odp_packet_t pkt_in, &in_len, &in_seg); ODP_DBG("data %p in_len %d seg %p len %d\n", - data, in_len, in_seg, len); + (void *)data, in_len, (void *)in_seg, len); if (in_len > len) in_len = len; diff --git a/platform/linux-generic/odp_crypto_null.c b/platform/linux-generic/odp_crypto_null.c index aa8e8a2b3..8bd3433f0 100644 --- a/platform/linux-generic/odp_crypto_null.c +++ b/platform/linux-generic/odp_crypto_null.c @@ -77,7 +77,7 @@ struct odp_crypto_global_s { /* These flags are cleared at alloc_session() */ uint8_t ctx_valid[ODP_THREAD_COUNT_MAX][MAX_SESSIONS]; - odp_ticketlock_t openssl_lock[0]; + odp_ticketlock_t openssl_lock[]; }; static odp_crypto_global_t *global; diff --git a/platform/linux-generic/odp_crypto_openssl.c b/platform/linux-generic/odp_crypto_openssl.c index d8276b4b4..6e702acab 100644 --- a/platform/linux-generic/odp_crypto_openssl.c +++ b/platform/linux-generic/odp_crypto_openssl.c @@ -245,7 +245,7 @@ struct odp_crypto_global_s { /* These flags are cleared at alloc_session() */ uint8_t ctx_valid[ODP_THREAD_COUNT_MAX][MAX_SESSIONS]; - odp_ticketlock_t openssl_lock[0]; + odp_ticketlock_t openssl_lock[]; }; static odp_crypto_global_t *global; diff --git a/platform/linux-generic/odp_fdserver.c b/platform/linux-generic/odp_fdserver.c index 3103ddd5c..8bb12b64a 100644 --- a/platform/linux-generic/odp_fdserver.c +++ b/platform/linux-generic/odp_fdserver.c @@ -68,13 +68,8 @@ #define MAP_ANONYMOUS MAP_ANON #endif -#define FD_ODP_DEBUG_PRINT 0 - -#define FD_ODP_DBG(fmt, ...) \ - do { \ - if (FD_ODP_DEBUG_PRINT == 1) \ - ODP_DBG(fmt, ##__VA_ARGS__);\ - } while (0) +/* Debug level for the FD server */ +#define FD_DBG 3 /* define the tables of file descriptors handled by this server: */ #define FDSERVER_MAX_ENTRIES 256 @@ -287,8 +282,8 @@ int _odp_fdserver_register_fd(fd_server_context_e context, uint64_t key, int command; int fd; - FD_ODP_DBG("FD client register: pid=%d key=%" PRIu64 ", fd=%d\n", - getpid(), key, fd_to_send); + ODP_DBG_LVL(FD_DBG, "FD client register: pid=%d key=%" PRIu64 ", fd=%d\n", + getpid(), key, fd_to_send); s_sock = get_socket(); if (s_sock < 0) @@ -326,8 +321,8 @@ int _odp_fdserver_deregister_fd(fd_server_context_e context, uint64_t key) int command; int fd; - FD_ODP_DBG("FD client deregister: pid=%d key=%" PRIu64 "\n", - getpid(), key); + ODP_DBG_LVL(FD_DBG, "FD client deregister: pid=%d key=%" PRIu64 "\n", + getpid(), key); s_sock = get_socket(); if (s_sock < 0) @@ -399,7 +394,7 @@ static int stop_server(void) int s_sock; /* server socket */ int res; - FD_ODP_DBG("FD sending server stop request\n"); + ODP_DBG_LVL(FD_DBG, "FD sending server stop request\n"); s_sock = get_socket(); if (s_sock < 0) @@ -446,8 +441,8 @@ static int handle_request(int client_sock) fd_table[fd_table_nb_entries].context = context; fd_table[fd_table_nb_entries].key = key; fd_table[fd_table_nb_entries++].fd = fd; - FD_ODP_DBG("storing {ctx=%d, key=%" PRIu64 "}->fd=%d\n", - context, key, fd); + ODP_DBG_LVL(FD_DBG, "storing {ctx=%d, key=%" PRIu64 "}->fd=%d\n", + context, key, fd); } else { ODP_ERR("FD table full\n"); send_fdserver_msg(client_sock, FD_REGISTER_NACK, @@ -499,8 +494,8 @@ static int handle_request(int client_sock) for (i = 0; i < fd_table_nb_entries; i++) { if ((fd_table[i].context == context) && (fd_table[i].key == key)) { - FD_ODP_DBG("drop {ctx=%d," - " key=%" PRIu64 "}->fd=%d\n", + ODP_DBG_LVL(FD_DBG, "drop {ctx=%d," + " key=%" PRIu64 "}->fd=%d\n", context, key, fd_table[i].fd); close(fd_table[i].fd); fd_table[i] = fd_table[--fd_table_nb_entries]; @@ -517,7 +512,7 @@ static int handle_request(int client_sock) break; case FD_SERVERSTOP_REQ: - FD_ODP_DBG("Stopping FD server\n"); + ODP_DBG_LVL(FD_DBG, "Stopping FD server\n"); return 1; default: diff --git a/platform/linux-generic/odp_init.c b/platform/linux-generic/odp_init.c index d5530c1f1..18646dc08 100644 --- a/platform/linux-generic/odp_init.c +++ b/platform/linux-generic/odp_init.c @@ -630,3 +630,10 @@ void odp_log_thread_fn_set(odp_log_func_t func) { _odp_this_thread->log_fn = func; } + +int odp_instance(odp_instance_t *instance) +{ + *instance = (odp_instance_t)odp_global_ro.main_pid; + + return 0; +} diff --git a/platform/linux-generic/odp_ipsec.c b/platform/linux-generic/odp_ipsec.c index 09596b502..3bd524c3c 100644 --- a/platform/linux-generic/odp_ipsec.c +++ b/platform/linux-generic/odp_ipsec.c @@ -357,7 +357,7 @@ typedef struct { struct { uint16_t hdr_len; uint16_t trl_len; - odp_u32be_t seq_no; + uint64_t seq_no; } in; odp_u32be_t ipv4_addr; uint8_t ipv6_addr[_ODP_IPV6ADDR_LEN]; @@ -379,6 +379,40 @@ typedef struct { uint8_t iv[IPSEC_MAX_IV_LEN]; } ipsec_state_t; +/* + * Computes 64-bit seq number according to RFC4303 A2 + */ +static inline uint64_t ipsec_compute_esn(ipsec_sa_t *ipsec_sa, uint32_t seq) +{ + uint32_t wintop_h, wintop_l, winbot_l, ws; + uint64_t seq64 = 0, wintop = 0; + + wintop = odp_atomic_load_u64(&ipsec_sa->hot.in.wintop_seq); + wintop_l = wintop & 0xffffffff; + wintop_h = wintop >> 32; + + ws = ipsec_sa->in.ar.win_size; + winbot_l = wintop_l - ws + 1; + + /* case A: window is within one sequence number subspace */ + if (wintop_l >= (ws - 1)) { + if (seq < winbot_l) + wintop_h++; + /* case B: window spans two sequence number subspaces */ + } else { + if (seq >= winbot_l) + wintop_h--; + } + + seq64 = ((uint64_t)wintop_h << 32) | seq; + return seq64; +} + +static inline uint32_t ipsec_get_seqh_len(ipsec_sa_t *ipsec_sa) +{ + return ipsec_sa->insert_seq_hi * IPSEC_SEQ_HI_LEN; +} + static int ipsec_parse_ipv4(ipsec_state_t *state, odp_packet_t pkt) { _odp_ipv4hdr_t ipv4hdr; @@ -569,17 +603,41 @@ static int ipsec_in_esp(odp_packet_t *pkt, param->auth_iv_ptr = state->iv; state->esp.aad.spi = esp.spi; - state->esp.aad.seq_no = esp.seq_no; state->in.seq_no = odp_be_to_cpu_32(esp.seq_no); + if (ipsec_sa->esn) { + state->in.seq_no = ipsec_compute_esn(ipsec_sa, state->in.seq_no); + state->esp.aad.seq_no64 = odp_cpu_to_be_64(state->in.seq_no); + } else { + state->esp.aad.seq_no = esp.seq_no; + } param->aad_ptr = (uint8_t *)&state->esp.aad; + /* Insert high-order bits of ESN before the ICV for ICV check + * with non-combined mode algorithms. + */ + if (ipsec_sa->insert_seq_hi) { + uint32_t inb_seqh = odp_cpu_to_be_32(state->in.seq_no >> 32); + uint32_t icv_offset = odp_packet_len(*pkt) - ipsec_sa->icv_len; + + if (odp_packet_extend_tail(pkt, IPSEC_SEQ_HI_LEN, NULL, NULL) < 0) { + status->error.alg = 1; + ODP_ERR("odp_packet_extend_tail failed\n"); + return -1; + } + odp_packet_move_data(*pkt, icv_offset + IPSEC_SEQ_HI_LEN, icv_offset, + ipsec_sa->icv_len); + odp_packet_copy_from_mem(*pkt, icv_offset, IPSEC_SEQ_HI_LEN, &inb_seqh); + } + param->auth_range.offset = ipsec_offset; param->auth_range.length = state->ip_tot_len - - state->ip_hdr_len - + state->ip_hdr_len + + ipsec_get_seqh_len(ipsec_sa) - ipsec_sa->icv_len; param->hash_result_offset = state->ip_offset + - state->ip_tot_len - + state->ip_tot_len + + ipsec_get_seqh_len(ipsec_sa) - ipsec_sa->icv_len; state->stats_length = param->cipher_range.length; @@ -681,6 +739,23 @@ static int ipsec_in_ah(odp_packet_t *pkt, } state->in.seq_no = odp_be_to_cpu_32(ah.seq_no); + if (ipsec_sa->esn) + state->in.seq_no = ipsec_compute_esn(ipsec_sa, state->in.seq_no); + + /* ESN higher 32 bits are included at the end of the packet data + * for inbound ICV computation. + */ + if (ipsec_sa->insert_seq_hi) { + uint32_t inb_seqh = odp_cpu_to_be_32(state->in.seq_no >> 32); + uint32_t seqh_offset = odp_packet_len(*pkt); + + if (odp_packet_extend_tail(pkt, IPSEC_SEQ_HI_LEN, NULL, NULL) < 0) { + status->error.alg = 1; + ODP_ERR("odp_packet_extend_tail failed\n"); + return -1; + } + odp_packet_copy_from_mem(*pkt, seqh_offset, IPSEC_SEQ_HI_LEN, &inb_seqh); + } param->auth_range.offset = state->ip_offset; param->auth_range.length = state->ip_tot_len; @@ -688,6 +763,7 @@ static int ipsec_in_ah(odp_packet_t *pkt, ipsec_sa->esp_iv_len; state->stats_length = param->auth_range.length; + param->auth_range.length += ipsec_get_seqh_len(ipsec_sa); return 0; } @@ -728,31 +804,35 @@ static int ipsec_in_ah_post(odp_packet_t pkt, static void ipsec_sa_err_stats_update(ipsec_sa_t *sa, odp_ipsec_op_status_t *status) { + odp_ipsec_op_status_t err_status; + if (odp_likely(ODP_IPSEC_OK == status->error.all)) return; if (NULL == sa) return; - if (status->error.proto) + err_status = *status; + + if (err_status.error.proto) odp_atomic_inc_u64(&sa->stats.proto_err); - if (status->error.auth) + if (err_status.error.auth) odp_atomic_inc_u64(&sa->stats.auth_err); - if (status->error.antireplay) + if (err_status.error.antireplay) odp_atomic_inc_u64(&sa->stats.antireplay_err); - if (status->error.alg) + if (err_status.error.alg) odp_atomic_inc_u64(&sa->stats.alg_err); - if (status->error.mtu) + if (err_status.error.mtu) odp_atomic_inc_u64(&sa->stats.mtu_err); - if (status->error.hard_exp_bytes) + if (err_status.error.hard_exp_bytes) odp_atomic_inc_u64(&sa->stats.hard_exp_bytes_err); - if (status->error.hard_exp_packets) + if (err_status.error.hard_exp_packets) odp_atomic_inc_u64(&sa->stats.hard_exp_pkts_err); } @@ -876,7 +956,8 @@ static ipsec_sa_t *ipsec_in_single(odp_packet_t pkt, goto post_lifetime_err_cnt_update; } - if (odp_packet_trunc_tail(&pkt, state.in.trl_len, NULL, NULL) < 0) { + if (odp_packet_trunc_tail(&pkt, state.in.trl_len + ipsec_get_seqh_len(ipsec_sa), + NULL, NULL) < 0) { status->error.alg = 1; goto post_lifetime_err_cnt_update; } @@ -1280,10 +1361,13 @@ static int ipsec_out_esp(odp_packet_t *pkt, memset(&esp, 0, sizeof(esp)); esp.spi = odp_cpu_to_be_32(ipsec_sa->spi); + state->esp.aad.spi = esp.spi; esp.seq_no = odp_cpu_to_be_32(seq_no & 0xffffffff); - state->esp.aad.spi = esp.spi; - state->esp.aad.seq_no = esp.seq_no; + if (ipsec_sa->esn) + state->esp.aad.seq_no64 = odp_cpu_to_be_64(seq_no); + else + state->esp.aad.seq_no = esp.seq_no; param->aad_ptr = (uint8_t *)&state->esp.aad; @@ -1359,6 +1443,22 @@ static int ipsec_out_esp(odp_packet_t *pkt, esptrl_offset, _ODP_ESPTRL_LEN, &esptrl); + /* Outbound ICV computation includes ESN higher 32 bits as part of ESP + * implicit trailer for individual algo's. + */ + if (ipsec_sa->insert_seq_hi) { + uint32_t outb_seqh = odp_cpu_to_be_32(seq_no >> 32); + + if (odp_packet_extend_tail(pkt, IPSEC_SEQ_HI_LEN, NULL, NULL) < 0) { + status->error.alg = 1; + ODP_ERR("odp_packet_extend_tail failed\n"); + return -1; + } + odp_packet_copy_from_mem(*pkt, + esptrl_offset + _ODP_ESPTRL_LEN, + IPSEC_SEQ_HI_LEN, &outb_seqh); + } + if (odp_unlikely(state->ip_tot_len < state->ip_hdr_len + hdr_len + ipsec_sa->icv_len)) { status->error.proto = 1; @@ -1373,10 +1473,12 @@ static int ipsec_out_esp(odp_packet_t *pkt, param->auth_range.offset = ipsec_offset; param->auth_range.length = state->ip_tot_len - - state->ip_hdr_len - + state->ip_hdr_len + + ipsec_get_seqh_len(ipsec_sa) - ipsec_sa->icv_len; param->hash_result_offset = state->ip_offset + - state->ip_tot_len - + state->ip_tot_len + + ipsec_get_seqh_len(ipsec_sa) - ipsec_sa->icv_len; state->stats_length = param->cipher_range.length; @@ -1384,10 +1486,27 @@ static int ipsec_out_esp(odp_packet_t *pkt, return 0; } -static void ipsec_out_esp_post(ipsec_state_t *state, odp_packet_t pkt) +static int ipsec_out_esp_post(ipsec_state_t *state, odp_packet_t *pkt, + ipsec_sa_t *ipsec_sa) { if (state->is_ipv4) - _odp_packet_ipv4_chksum_insert(pkt); + _odp_packet_ipv4_chksum_insert(*pkt); + + /* Remove the high order ESN bits that were added in the packet for ICV + * computation. + */ + if (ipsec_sa->insert_seq_hi) { + uint32_t icv_offset = odp_packet_len(*pkt) - ipsec_sa->icv_len; + + odp_packet_move_data(*pkt, icv_offset - IPSEC_SEQ_HI_LEN, icv_offset, + ipsec_sa->icv_len); + if (odp_packet_trunc_tail(pkt, IPSEC_SEQ_HI_LEN, NULL, NULL) < 0) { + ODP_ERR("odp_packet_trunc_tail failed\n"); + return -1; + } + } + + return 0; } static int ipsec_out_ah(odp_packet_t *pkt, @@ -1479,32 +1598,62 @@ static int ipsec_out_ah(odp_packet_t *pkt, 0, hdr_len - _ODP_AHHDR_LEN - ipsec_sa->esp_iv_len); + /* ESN higher 32 bits are included at the end of the packet data + * for outbound ICV computation. + */ + if (ipsec_sa->insert_seq_hi) { + uint32_t outb_seqh = odp_cpu_to_be_32(seq_no >> 32); + uint32_t seqh_offset = odp_packet_len(*pkt); + + if (odp_packet_extend_tail(pkt, IPSEC_SEQ_HI_LEN, NULL, NULL) < 0) { + status->error.alg = 1; + ODP_ERR("odp_packet_extend_tail failed\n"); + return -1; + } + odp_packet_copy_from_mem(*pkt, + seqh_offset, IPSEC_SEQ_HI_LEN, &outb_seqh); + } + param->auth_range.offset = state->ip_offset; param->auth_range.length = state->ip_tot_len; param->hash_result_offset = ipsec_offset + _ODP_AHHDR_LEN + ipsec_sa->esp_iv_len; state->stats_length = param->auth_range.length; + param->auth_range.length += ipsec_get_seqh_len(ipsec_sa); return 0; } -static void ipsec_out_ah_post(ipsec_state_t *state, odp_packet_t pkt) +static int ipsec_out_ah_post(ipsec_state_t *state, odp_packet_t *pkt, + ipsec_sa_t *ipsec_sa) { if (state->is_ipv4) { - _odp_ipv4hdr_t *ipv4hdr = odp_packet_l3_ptr(pkt, NULL); + _odp_ipv4hdr_t *ipv4hdr = odp_packet_l3_ptr(*pkt, NULL); ipv4hdr->ttl = state->ah_ipv4.ttl; ipv4hdr->tos = state->ah_ipv4.tos; ipv4hdr->frag_offset = state->ah_ipv4.frag_offset; - _odp_packet_ipv4_chksum_insert(pkt); + _odp_packet_ipv4_chksum_insert(*pkt); } else { - _odp_ipv6hdr_t *ipv6hdr = odp_packet_l3_ptr(pkt, NULL); + _odp_ipv6hdr_t *ipv6hdr = odp_packet_l3_ptr(*pkt, NULL); ipv6hdr->ver_tc_flow = state->ah_ipv6.ver_tc_flow; ipv6hdr->hop_limit = state->ah_ipv6.hop_limit; } + + /* Remove the high order ESN bits that were added in the packet for ICV + * computation. + */ + if (ipsec_sa->insert_seq_hi) { + if (odp_packet_trunc_tail(pkt, IPSEC_SEQ_HI_LEN, NULL, NULL) < 0) { + ODP_ERR("odp_packet_trunc_tail failed\n"); + return -1; + } + } + + return 0; } #define OL_TX_CHKSUM_PKT(_cfg, _proto, _ovr_set, _ovr) \ @@ -1714,9 +1863,14 @@ static ipsec_sa_t *ipsec_out_single(odp_packet_t pkt, /* Finalize the IP header */ if (ODP_IPSEC_ESP == ipsec_sa->proto) - ipsec_out_esp_post(&state, pkt); + rc = ipsec_out_esp_post(&state, &pkt, ipsec_sa); else if (ODP_IPSEC_AH == ipsec_sa->proto) - ipsec_out_ah_post(&state, pkt); + rc = ipsec_out_ah_post(&state, &pkt, ipsec_sa); + + if (rc < 0) { + status->error.alg = 1; + goto post_lifetime_err_cnt_update; + } goto exit; diff --git a/platform/linux-generic/odp_ipsec_events.c b/platform/linux-generic/odp_ipsec_events.c index bb786614a..a199ffdf3 100644 --- a/platform/linux-generic/odp_ipsec_events.c +++ b/platform/linux-generic/odp_ipsec_events.c @@ -18,12 +18,15 @@ #include <odp/api/plat/event_inlines.h> #include <odp/api/plat/queue_inlines.h> +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wpedantic" typedef struct { /* common buffer header */ odp_buffer_hdr_t buf_hdr; odp_ipsec_status_t status; } ipsec_status_hdr_t; +#pragma GCC diagnostic pop static odp_pool_t ipsec_status_pool = ODP_POOL_INVALID; diff --git a/platform/linux-generic/odp_ipsec_sad.c b/platform/linux-generic/odp_ipsec_sad.c index 0eea57a10..756370516 100644 --- a/platform/linux-generic/odp_ipsec_sad.c +++ b/platform/linux-generic/odp_ipsec_sad.c @@ -503,10 +503,8 @@ odp_ipsec_sa_t odp_ipsec_sa_create(const odp_ipsec_sa_param_t *param) } ipsec_sa->mode = param->mode; ipsec_sa->flags = 0; - if (param->opt.esn) { - ODP_ERR("ESN is not supported!\n"); - return ODP_IPSEC_SA_INVALID; - } + ipsec_sa->esn = param->opt.esn; + if (ODP_IPSEC_DIR_INBOUND == param->dir) { ipsec_sa->lookup_mode = param->inbound.lookup_mode; if (ODP_IPSEC_LOOKUP_DSTADDR_SPI == ipsec_sa->lookup_mode) { @@ -640,6 +638,11 @@ odp_ipsec_sa_t odp_ipsec_sa_create(const odp_ipsec_sa_param_t *param) goto error; ipsec_sa->salt_length = 0; + /* ESN higher 32 bits flag. + * This flag is set for individual algo's. + * This flag is reset for combined mode algo's and ODP_AUTH_ALG_NULL. + */ + ipsec_sa->insert_seq_hi = (ipsec_sa->esn) ? 1 : 0; switch (crypto_param.cipher_alg) { case ODP_CIPHER_ALG_NULL: @@ -700,10 +703,17 @@ odp_ipsec_sa_t odp_ipsec_sa_create(const odp_ipsec_sa_param_t *param) case ODP_AUTH_ALG_AES128_GCM: #endif case ODP_AUTH_ALG_AES_GCM: - crypto_param.auth_aad_len = sizeof(ipsec_aad_t); + case ODP_AUTH_ALG_AES_CCM: + if (ipsec_sa->esn) { + crypto_param.auth_aad_len = 12; + ipsec_sa->insert_seq_hi = 0; + } else { + crypto_param.auth_aad_len = 8; + } break; case ODP_AUTH_ALG_AES_GMAC: - if (ODP_CIPHER_ALG_NULL != crypto_param.cipher_alg) + if ((ODP_CIPHER_ALG_NULL != crypto_param.cipher_alg) || + ipsec_sa->esn) goto error; ipsec_sa->use_counter_iv = 1; ipsec_sa->esp_iv_len = 8; @@ -713,7 +723,12 @@ odp_ipsec_sa_t odp_ipsec_sa_create(const odp_ipsec_sa_param_t *param) salt_param = ¶m->crypto.auth_key_extra; break; case ODP_AUTH_ALG_CHACHA20_POLY1305: - crypto_param.auth_aad_len = sizeof(ipsec_aad_t); + if (ipsec_sa->esn) { + crypto_param.auth_aad_len = 12; + ipsec_sa->insert_seq_hi = 0; + } else { + crypto_param.auth_aad_len = 8; + } break; default: break; @@ -721,6 +736,10 @@ odp_ipsec_sa_t odp_ipsec_sa_create(const odp_ipsec_sa_param_t *param) ipsec_sa->icv_len = crypto_param.auth_digest_len; + /* For ODP_AUTH_ALG_NULL */ + if (!ipsec_sa->icv_len) + ipsec_sa->insert_seq_hi = 0; + if (ipsec_sa->salt_length) { if (ipsec_sa->salt_length > IPSEC_MAX_SALT_LEN) { ODP_ERR("IPSEC_MAX_SALT_LEN too small\n"); @@ -947,30 +966,37 @@ static uint64_t ipsec_sa_antireplay_max_seq(ipsec_sa_t *ipsec_sa) { uint64_t max_seq = 0; - max_seq = odp_atomic_load_u64(&ipsec_sa->hot.in.wintop_seq) & 0xffffffff; + max_seq = odp_atomic_load_u64(&ipsec_sa->hot.in.wintop_seq); + if (!ipsec_sa->esn) + max_seq &= 0xffffffff; return max_seq; } -int _odp_ipsec_sa_replay_precheck(ipsec_sa_t *ipsec_sa, uint32_t seq, +int _odp_ipsec_sa_replay_precheck(ipsec_sa_t *ipsec_sa, uint64_t seq, odp_ipsec_op_status_t *status) { /* Try to be as quick as possible, we will discard packets later */ - if (ipsec_sa->antireplay && ((seq + ipsec_sa->in.ar.win_size) <= - (odp_atomic_load_u64(&ipsec_sa->hot.in.wintop_seq) & 0xffffffff))) { - status->error.antireplay = 1; - return -1; - } + if (ipsec_sa->antireplay) { + uint64_t wintop = odp_atomic_load_u64(&ipsec_sa->hot.in.wintop_seq); + + if (!ipsec_sa->esn) + wintop &= 0xffffffff; + if ((seq + ipsec_sa->in.ar.win_size) <= wintop) { + status->error.antireplay = 1; + return -1; + } + } return 0; } -static inline int ipsec_wslarge_replay_update(ipsec_sa_t *ipsec_sa, uint32_t seq, +static inline int ipsec_wslarge_replay_update(ipsec_sa_t *ipsec_sa, uint64_t seq, odp_ipsec_op_status_t *status) { - uint32_t bucket, wintop_bucket, new_bucket; - uint32_t bkt_diff, bkt_cnt, top_seq; - uint64_t bit = 0; + uint64_t bucket, wintop_bucket, new_bucket; + uint64_t bkt_diff, bkt_cnt; + uint64_t bit = 0, top_seq; odp_spinlock_lock(&ipsec_sa->hot.in.lock); @@ -1052,14 +1078,14 @@ static inline int ipsec_ws32_replay_update(ipsec_sa_t *ipsec_sa, uint32_t seq, return 0; } -int _odp_ipsec_sa_replay_update(ipsec_sa_t *ipsec_sa, uint32_t seq, +int _odp_ipsec_sa_replay_update(ipsec_sa_t *ipsec_sa, uint64_t seq, odp_ipsec_op_status_t *status) { int ret; /* Window update for ws equal to 32 */ - if (ipsec_sa->in.ar.win_size == IPSEC_AR_WIN_SIZE_MIN) - ret = ipsec_ws32_replay_update(ipsec_sa, seq, status); + if ((!ipsec_sa->esn) && (ipsec_sa->in.ar.win_size == IPSEC_AR_WIN_SIZE_MIN)) + ret = ipsec_ws32_replay_update(ipsec_sa, (seq & 0xffffffff), status); else ret = ipsec_wslarge_replay_update(ipsec_sa, seq, status); diff --git a/platform/linux-generic/odp_ishm.c b/platform/linux-generic/odp_ishm.c index ebc931658..a10b9d5df 100644 --- a/platform/linux-generic/odp_ishm.c +++ b/platform/linux-generic/odp_ishm.c @@ -1621,17 +1621,18 @@ int _odp_ishm_cleanup_files(const char *dirpath) char userdir[PATH_MAX]; char prefix[PATH_MAX]; char *fullpath; - int d_len = strlen(dirpath); + int d_len; int p_len; int f_len; snprintf(userdir, PATH_MAX, "%s/%s", dirpath, odp_global_ro.uid); + d_len = strlen(userdir); dir = opendir(userdir); if (!dir) { /* ok if the dir does not exist. no much to delete then! */ ODP_DBG("opendir failed for %s: %s\n", - dirpath, strerror(errno)); + userdir, strerror(errno)); return 0; } snprintf(prefix, PATH_MAX, _ODP_FILES_FMT, odp_global_ro.main_pid); @@ -1645,7 +1646,7 @@ int _odp_ishm_cleanup_files(const char *dirpath) return -1; } snprintf(fullpath, PATH_MAX, "%s/%s", - dirpath, e->d_name); + userdir, e->d_name); ODP_DBG("deleting obsolete file: %s\n", fullpath); if (unlink(fullpath)) ODP_ERR("unlink failed for %s: %s\n", diff --git a/platform/linux-generic/odp_name_table.c b/platform/linux-generic/odp_name_table.c index 9c1814cae..b9ba6e3fe 100644 --- a/platform/linux-generic/odp_name_table.c +++ b/platform/linux-generic/odp_name_table.c @@ -70,7 +70,7 @@ typedef struct ODP_ALIGNED_CACHE { uint32_t num_avail_to_add; uint32_t base_id; name_tbl_entry_t *free_list_head; - name_tbl_entry_t entries[0]; + name_tbl_entry_t entries[]; } name_tbl_t; typedef struct { diff --git a/platform/linux-generic/odp_packet.c b/platform/linux-generic/odp_packet.c index c6a50bf84..ed5d81952 100644 --- a/platform/linux-generic/odp_packet.c +++ b/platform/linux-generic/odp_packet.c @@ -63,13 +63,22 @@ const _odp_packet_inline_offset_t _odp_packet_inline ODP_ALIGNED_CACHE = { #include <odp/visibility_end.h> -/* Check that invalid values are the same. Some versions of Clang have trouble - * with the strong type casting, and complain that these invalid values are not - * integral constants. */ +/* Check that invalid values are the same. Some versions of Clang and pedantic + * build have trouble with the strong type casting, and complain that these + * invalid values are not integral constants. + * + * Invalid values are required to be equal for _odp_buffer_is_valid() to work + * properly. */ #ifndef __clang__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wpedantic" ODP_STATIC_ASSERT(ODP_PACKET_INVALID == 0, "Packet invalid not 0"); ODP_STATIC_ASSERT(ODP_BUFFER_INVALID == 0, "Buffer invalid not 0"); ODP_STATIC_ASSERT(ODP_EVENT_INVALID == 0, "Event invalid not 0"); +ODP_STATIC_ASSERT(ODP_PACKET_VECTOR_INVALID == 0, "Packet vector invalid not 0"); +ODP_STATIC_ASSERT(ODP_PACKET_TX_COMPL_INVALID == 0, "Packet TX completion invalid not 0"); +ODP_STATIC_ASSERT(ODP_TIMEOUT_INVALID == 0, "Timeout invalid not 0"); +#pragma GCC diagnostic pop #endif static inline odp_buffer_t packet_to_buffer(odp_packet_t pkt) @@ -2592,7 +2601,7 @@ static int packet_l4_chksum(odp_packet_hdr_t *pkt_hdr, } } - return pkt_hdr->p.flags.all_flags != 0; + return pkt_hdr->p.flags.all.error != 0; } /** diff --git a/platform/linux-generic/odp_packet_vector.c b/platform/linux-generic/odp_packet_vector.c index d97bb96a1..6f0ee201a 100644 --- a/platform/linux-generic/odp_packet_vector.c +++ b/platform/linux-generic/odp_packet_vector.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2020, Nokia +/* Copyright (c) 2020-2021, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -30,34 +30,33 @@ const _odp_event_vector_inline_offset_t _odp_event_vector_inline ODP_ALIGNED_CAC #include <odp/visibility_end.h> -static inline odp_event_vector_hdr_t *event_vector_hdr_from_buffer(odp_buffer_t buf) +odp_packet_vector_t odp_packet_vector_alloc(odp_pool_t pool_hdl) { - return (odp_event_vector_hdr_t *)(uintptr_t)buf; -} + odp_packet_vector_t pktv; + pool_t *pool; + int ret; -odp_packet_vector_t odp_packet_vector_alloc(odp_pool_t pool) -{ - odp_buffer_t buf; + ODP_ASSERT(pool_hdl != ODP_POOL_INVALID); + + pool = pool_entry_from_hdl(pool_hdl); - ODP_ASSERT(pool_entry_from_hdl(pool)->type == ODP_POOL_VECTOR); + ODP_ASSERT(pool->type == ODP_POOL_VECTOR); - buf = odp_buffer_alloc(pool); - if (odp_unlikely(buf == ODP_BUFFER_INVALID)) - return ODP_PACKET_VECTOR_INVALID; + ret = _odp_buffer_alloc_multi(pool, (odp_buffer_hdr_t **)&pktv, 1); - ODP_ASSERT(event_vector_hdr_from_buffer(buf)->size == 0); + if (odp_likely(ret == 1)) + return pktv; - return odp_packet_vector_from_event(odp_buffer_to_event(buf)); + return ODP_PACKET_VECTOR_INVALID; } void odp_packet_vector_free(odp_packet_vector_t pktv) { odp_event_vector_hdr_t *pktv_hdr = _odp_packet_vector_hdr(pktv); - odp_event_t ev = odp_packet_vector_to_event(pktv); pktv_hdr->size = 0; - odp_buffer_free(odp_buffer_from_event(ev)); + _odp_buffer_free_multi((odp_buffer_hdr_t **)&pktv_hdr, 1); } int odp_packet_vector_valid(odp_packet_vector_t pktv) @@ -103,7 +102,7 @@ void odp_packet_vector_print(odp_packet_vector_t pktv) len += snprintf(&str[len], n - len, "Packet Vector\n"); len += snprintf(&str[len], n - len, - " handle %p\n", pktv); + " handle %p\n", (void *)pktv); len += snprintf(&str[len], n - len, " size %" PRIu32 "\n", pktv_hdr->size); @@ -114,7 +113,7 @@ void odp_packet_vector_print(odp_packet_vector_t pktv) str_len = snprintf(seg_str, max_len, " packet %p len %" PRIu32 "\n", - pkt, odp_packet_len(pkt)); + (void *)pkt, odp_packet_len(pkt)); /* Prevent print buffer overflow */ if (n - len - str_len < 10) { diff --git a/platform/linux-generic/odp_pcapng.c b/platform/linux-generic/odp_pcapng.c index ce950760e..6bdd1fd69 100644 --- a/platform/linux-generic/odp_pcapng.c +++ b/platform/linux-generic/odp_pcapng.c @@ -577,4 +577,7 @@ int _odp_pcapng_write_pkts(pktio_entry_t *entry, int qidx, return len; } -#endif /* _ODP_PCAPNG */ +#else /* _ODP_PCAPNG */ +/* Avoid warning about empty translation unit */ +typedef int _odp_dummy; +#endif diff --git a/platform/linux-generic/odp_pkt_queue.c b/platform/linux-generic/odp_pkt_queue.c index 556e034f8..ad70f0e0a 100644 --- a/platform/linux-generic/odp_pkt_queue.c +++ b/platform/linux-generic/odp_pkt_queue.c @@ -26,19 +26,12 @@ typedef struct ODP_ALIGNED_CACHE { odp_packet_t pkts[NUM_PKTS]; } queue_blk_t; +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wpedantic" typedef struct ODP_ALIGNED_CACHE { queue_blk_t blks[0]; } queue_blks_t; - -/* The queue_num_tbl is used to map from a queue_num to a queue_num_desc. - * The reason is based on the assumption that usually only a small fraction - * of the max_num_queues will have more than 1 pkt associated with it. This - * way the active queue_desc's can be dynamically allocated and freed according - * to the actual usage pattern. - */ -typedef struct { - uint32_t queue_num_to_blk_idx[0]; -} queue_num_tbl_t; +#pragma GCC diagnostic pop typedef struct { uint32_t num_blks; diff --git a/platform/linux-generic/odp_pool.c b/platform/linux-generic/odp_pool.c index 9c60a9458..49d2e74f5 100644 --- a/platform/linux-generic/odp_pool.c +++ b/platform/linux-generic/odp_pool.c @@ -669,6 +669,7 @@ static odp_pool_t pool_create(const char *name, const odp_pool_param_t *params, return ODP_POOL_INVALID; } + num = params->pkt.num; seg_len = CONFIG_PACKET_MAX_SEG_LEN; max_len = _odp_pool_glb->config.pkt_max_len; @@ -691,9 +692,19 @@ static odp_pool_t pool_create(const char *name, const odp_pool_param_t *params, return ODP_POOL_INVALID; } + /* Multiple segments required per 'params->pkt.len' packet */ + if (params->pkt.len > seg_len) + num *= (params->pkt.len + seg_len - 1) / seg_len; + + /* Make sure 'params->pkt.max_num' limitation holds */ + if (params->pkt.max_num && num > params->pkt.max_num) { + ODP_ERR("Pool 'max_num' parameter too small (%u/%u)\n", + params->pkt.max_num, num); + return ODP_POOL_INVALID; + } + headroom = CONFIG_PACKET_HEADROOM; tailroom = CONFIG_PACKET_TAILROOM; - num = params->pkt.num; uarea_size = params->pkt.uarea_size; cache_size = params->pkt.cache_size; break; @@ -853,12 +864,15 @@ static int check_params(const odp_pool_param_t *params) uint32_t cache_size, num; int num_threads = odp_global_ro.init_param.num_control + odp_global_ro.init_param.num_worker; + int cur_threads = odp_thread_count(); if (!params || odp_pool_capability(&capa) < 0) return -1; num = 0; cache_size = 0; + if (num_threads < cur_threads) + num_threads = cur_threads; switch (params->type) { case ODP_POOL_BUFFER: @@ -1270,6 +1284,9 @@ odp_buffer_t odp_buffer_alloc(odp_pool_t pool_hdl) ODP_ASSERT(ODP_POOL_INVALID != pool_hdl); pool = pool_entry_from_hdl(pool_hdl); + + ODP_ASSERT(pool->type == ODP_POOL_BUFFER); + ret = _odp_buffer_alloc_multi(pool, (odp_buffer_hdr_t **)&buf, 1); if (odp_likely(ret == 1)) @@ -1286,6 +1303,8 @@ int odp_buffer_alloc_multi(odp_pool_t pool_hdl, odp_buffer_t buf[], int num) pool = pool_entry_from_hdl(pool_hdl); + ODP_ASSERT(pool->type == ODP_POOL_BUFFER); + return _odp_buffer_alloc_multi(pool, (odp_buffer_hdr_t **)buf, num); } @@ -1392,15 +1411,65 @@ void odp_pool_print(odp_pool_t pool_hdl) ODP_PRINT(" block size %u\n", pool->block_size); ODP_PRINT(" uarea size %u\n", pool->uarea_size); ODP_PRINT(" shm size %" PRIu64 "\n", pool->shm_size); - ODP_PRINT(" base addr %p\n", pool->base_addr); - ODP_PRINT(" max addr %p\n", pool->max_addr); + ODP_PRINT(" base addr %p\n", (void *)pool->base_addr); + ODP_PRINT(" max addr %p\n", (void *)pool->max_addr); ODP_PRINT(" uarea shm size %" PRIu64 "\n", pool->uarea_shm_size); - ODP_PRINT(" uarea base addr %p\n", pool->uarea_base_addr); + ODP_PRINT(" uarea base addr %p\n", (void *)pool->uarea_base_addr); ODP_PRINT(" cache size %u\n", pool->cache_size); ODP_PRINT(" burst size %u\n", pool->burst_size); ODP_PRINT("\n"); } +void odp_pool_print_all(void) +{ + uint64_t available; + uint32_t i, index, tot, cache_size, seg_len; + uint32_t buf_len = 0; + uint8_t type, ext; + const int col_width = 24; + const char *name; + char type_c; + + ODP_PRINT("\nList of all pools\n"); + ODP_PRINT("-----------------\n"); + ODP_PRINT(" idx %-*s type free tot cache buf_len ext\n", col_width, "name"); + + for (i = 0; i < ODP_CONFIG_POOLS; i++) { + pool_t *pool = pool_entry(i); + + LOCK(&pool->lock); + + if (!pool->reserved) { + UNLOCK(&pool->lock); + continue; + } + + available = ring_ptr_len(&pool->ring->hdr); + cache_size = pool->cache_size; + ext = pool->pool_ext; + index = pool->pool_idx; + name = pool->name; + tot = pool->num; + type = pool->type; + seg_len = pool->seg_len; + + UNLOCK(&pool->lock); + + if (type == ODP_POOL_BUFFER || type == ODP_POOL_PACKET) + buf_len = seg_len; + + type_c = (type == ODP_POOL_BUFFER) ? 'B' : + (type == ODP_POOL_PACKET) ? 'P' : + (type == ODP_POOL_TIMEOUT) ? 'T' : + (type == ODP_POOL_VECTOR) ? 'V' : '-'; + + ODP_PRINT("%4u %-*s %c %6" PRIu64 " %6" PRIu32 " %6" PRIu32 " %8" PRIu32 " " + "%" PRIu8 "\n", index, col_width, name, type_c, available, tot, + cache_size, buf_len, ext); + } + ODP_PRINT("\n"); +} + odp_pool_t odp_buffer_pool(odp_buffer_t buf) { pool_t *pool = pool_from_buf(buf); diff --git a/platform/linux-generic/odp_queue_basic.c b/platform/linux-generic/odp_queue_basic.c index e35f01f7d..fe4d90930 100644 --- a/platform/linux-generic/odp_queue_basic.c +++ b/platform/linux-generic/odp_queue_basic.c @@ -701,7 +701,7 @@ static void queue_print(odp_queue_t handle) } ODP_PRINT("\nQueue info\n"); ODP_PRINT("----------\n"); - ODP_PRINT(" handle %p\n", queue->s.handle); + ODP_PRINT(" handle %p\n", (void *)queue->s.handle); ODP_PRINT(" index %" PRIu32 "\n", queue_id); ODP_PRINT(" name %s\n", queue->s.name); ODP_PRINT(" enq mode %s\n", diff --git a/platform/linux-generic/odp_queue_lf.c b/platform/linux-generic/odp_queue_lf.c index 206172bb8..f1b265970 100644 --- a/platform/linux-generic/odp_queue_lf.c +++ b/platform/linux-generic/odp_queue_lf.c @@ -21,7 +21,7 @@ #ifdef __SIZEOF_INT128__ -typedef unsigned __int128 u128_t; +__extension__ typedef unsigned __int128 u128_t; static inline void lockfree_zero_u128(u128_t *atomic) { diff --git a/platform/linux-generic/odp_queue_scalable.c b/platform/linux-generic/odp_queue_scalable.c index ad7807bf9..916cb6739 100644 --- a/platform/linux-generic/odp_queue_scalable.c +++ b/platform/linux-generic/odp_queue_scalable.c @@ -29,17 +29,11 @@ #include <odp_ishmpool_internal.h> #include <odp/api/plat/queue_inline_types.h> #include <odp_global_data.h> +#include <odp_macros_internal.h> #include <string.h> #include <inttypes.h> -#define MIN(a, b) \ - ({ \ - __typeof__(a) tmp_a = (a); \ - __typeof__(b) tmp_b = (b); \ - tmp_a < tmp_b ? tmp_a : tmp_b; \ - }) - #define LOCK(a) odp_ticketlock_lock(a) #define UNLOCK(a) odp_ticketlock_unlock(a) #define LOCK_INIT(a) odp_ticketlock_init(a) @@ -745,7 +739,7 @@ int _odp_queue_deq_sc(sched_elem_t *q, odp_event_t *evp, int num) return actual; } -inline int _odp_queue_deq(sched_elem_t *q, odp_buffer_hdr_t *buf_hdr[], int num) +int _odp_queue_deq(sched_elem_t *q, odp_buffer_hdr_t *buf_hdr[], int num) { int actual; ringidx_t old_read; @@ -812,7 +806,7 @@ inline int _odp_queue_deq(sched_elem_t *q, odp_buffer_hdr_t *buf_hdr[], int num) return actual; } -inline int _odp_queue_deq_mc(sched_elem_t *q, odp_event_t *evp, int num) +int _odp_queue_deq_mc(sched_elem_t *q, odp_event_t *evp, int num) { int ret, evt_idx; odp_buffer_hdr_t *hdr_tbl[QUEUE_MULTI_MAX]; @@ -965,7 +959,7 @@ static void queue_print(odp_queue_t handle) } ODP_PRINT("\nQueue info\n"); ODP_PRINT("----------\n"); - ODP_PRINT(" handle %p\n", queue->s.handle); + ODP_PRINT(" handle %p\n", (void *)queue->s.handle); ODP_PRINT(" index %" PRIu32 "\n", queue->s.index); ODP_PRINT(" name %s\n", queue->s.name); ODP_PRINT(" enq mode %s\n", diff --git a/platform/linux-generic/odp_schedule_basic.c b/platform/linux-generic/odp_schedule_basic.c index eb17e95eb..f86914722 100644 --- a/platform/linux-generic/odp_schedule_basic.c +++ b/platform/linux-generic/odp_schedule_basic.c @@ -195,6 +195,8 @@ typedef struct ODP_ALIGNED_CACHE { } sched_local_t; +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wpedantic" /* Priority queue */ typedef struct ODP_ALIGNED_CACHE { /* Ring header */ @@ -204,6 +206,7 @@ typedef struct ODP_ALIGNED_CACHE { uint32_t queue_index[MAX_RING_SIZE]; /* overlaps with ring.data[] */ } prio_queue_t; +#pragma GCC diagnostic pop /* Order context of a queue */ typedef struct ODP_ALIGNED_CACHE { @@ -247,8 +250,10 @@ typedef struct { } queue[CONFIG_MAX_SCHED_QUEUES]; /* Scheduler priority queues */ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wpedantic" prio_queue_t prio_q[NUM_SCHED_GRPS][NUM_PRIO][MAX_SPREAD]; - +#pragma GCC diagnostic pop uint32_t prio_q_count[NUM_SCHED_GRPS][NUM_PRIO][MAX_SPREAD]; odp_thrmask_t mask_all; @@ -1960,7 +1965,7 @@ static int schedule_num_grps(void) static void schedule_get_config(schedule_config_t *config) { *config = sched->config_if; -}; +} static int schedule_capability(odp_schedule_capability_t *capa) { diff --git a/platform/linux-generic/odp_schedule_if.c b/platform/linux-generic/odp_schedule_if.c index 528c4065e..3b908ea87 100644 --- a/platform/linux-generic/odp_schedule_if.c +++ b/platform/linux-generic/odp_schedule_if.c @@ -95,27 +95,27 @@ int odp_schedule_multi_no_wait(odp_queue_t *from, odp_event_t events[], int num) void odp_schedule_pause(void) { - return _odp_sched_api->schedule_pause(); + _odp_sched_api->schedule_pause(); } void odp_schedule_resume(void) { - return _odp_sched_api->schedule_resume(); + _odp_sched_api->schedule_resume(); } void odp_schedule_release_atomic(void) { - return _odp_sched_api->schedule_release_atomic(); + _odp_sched_api->schedule_release_atomic(); } void odp_schedule_release_ordered(void) { - return _odp_sched_api->schedule_release_ordered(); + _odp_sched_api->schedule_release_ordered(); } void odp_schedule_prefetch(int num) { - return _odp_sched_api->schedule_prefetch(num); + _odp_sched_api->schedule_prefetch(num); } int odp_schedule_min_prio(void) @@ -180,12 +180,12 @@ int odp_schedule_group_info(odp_schedule_group_t group, void odp_schedule_order_lock(uint32_t lock_index) { - return _odp_sched_api->schedule_order_lock(lock_index); + _odp_sched_api->schedule_order_lock(lock_index); } void odp_schedule_order_unlock(uint32_t lock_index) { - return _odp_sched_api->schedule_order_unlock(lock_index); + _odp_sched_api->schedule_order_unlock(lock_index); } void odp_schedule_order_unlock_lock(uint32_t unlock_index, uint32_t lock_index) diff --git a/platform/linux-generic/odp_schedule_sp.c b/platform/linux-generic/odp_schedule_sp.c index 4f6ce736d..470075cea 100644 --- a/platform/linux-generic/odp_schedule_sp.c +++ b/platform/linux-generic/odp_schedule_sp.c @@ -81,6 +81,8 @@ typedef struct ODP_ALIGNED_CACHE sched_cmd_t { sizeof(struct sched_cmd_s)]; } sched_cmd_t; +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wpedantic" typedef struct ODP_ALIGNED_CACHE { /* Ring header */ ring_u32_t ring; @@ -89,6 +91,7 @@ typedef struct ODP_ALIGNED_CACHE { uint32_t ring_idx[RING_SIZE]; /* overlaps with ring.data[] */ } prio_queue_t; +#pragma GCC diagnostic pop typedef struct thr_group_t { /* A generation counter for fast comparison if groups have changed */ @@ -123,7 +126,11 @@ typedef struct ODP_ALIGNED_CACHE sched_group_t { typedef struct { sched_cmd_t queue_cmd[NUM_QUEUE]; sched_cmd_t pktio_cmd[NUM_PKTIO]; + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wpedantic" prio_queue_t prio_queue[NUM_GROUP][NUM_PRIO]; +#pragma GCC diagnostic pop sched_group_t sched_group; odp_shm_t shm; /* Scheduler interface config options (not used in fast path) */ @@ -1056,7 +1063,7 @@ static void schedule_print(void) static void get_config(schedule_config_t *config) { *config = sched_global->config_if; -}; +} /* Fill in scheduler interface */ const schedule_fn_t _odp_schedule_sp_fn = { diff --git a/platform/linux-generic/odp_sorted_list.c b/platform/linux-generic/odp_sorted_list.c index 042818c83..8f998238d 100644 --- a/platform/linux-generic/odp_sorted_list.c +++ b/platform/linux-generic/odp_sorted_list.c @@ -28,9 +28,12 @@ typedef struct { uint32_t pad; } sorted_list_desc_t; +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wpedantic" typedef struct { sorted_list_desc_t descs[0]; } sorted_list_descs_t; +#pragma GCC diagnostic pop typedef struct { uint64_t total_inserts; diff --git a/platform/linux-generic/odp_stash.c b/platform/linux-generic/odp_stash.c index a1bf5b63d..57cf747f1 100644 --- a/platform/linux-generic/odp_stash.c +++ b/platform/linux-generic/odp_stash.c @@ -4,6 +4,7 @@ * SPDX-License-Identifier: BSD-3-Clause */ +#include <stdint.h> #include <stdio.h> #include <string.h> @@ -12,16 +13,18 @@ #include <odp/api/stash.h> #include <odp/api/plat/strong_types.h> +#include <odp_config_internal.h> #include <odp_debug_internal.h> #include <odp_global_data.h> #include <odp_init_internal.h> -#include <odp_ring_ptr_internal.h> #include <odp_ring_u32_internal.h> +#include <odp_ring_u64_internal.h> -#define MAX_STASHES 32 #define MAX_RING_SIZE (1024 * 1024) #define MIN_RING_SIZE 64 +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wpedantic" typedef struct stash_t { char name[ODP_STASH_NAME_LEN]; odp_shm_t shm; @@ -32,23 +35,24 @@ typedef struct stash_t { /* Ring header followed by variable sized data (object handles) */ union { struct ODP_ALIGNED_CACHE { - ring_ptr_t hdr; - uintptr_t data[0]; - } ring_ptr; - - struct ODP_ALIGNED_CACHE { ring_u32_t hdr; - uint32_t data[0]; + uint32_t data[]; } ring_u32; + + struct ODP_ALIGNED_CACHE { + ring_u64_t hdr; + uint64_t data[]; + } ring_u64; }; } stash_t; +#pragma GCC diagnostic pop typedef struct stash_global_t { odp_ticketlock_t lock; odp_shm_t shm; - uint8_t stash_reserved[MAX_STASHES]; - stash_t *stash[MAX_STASHES]; + uint8_t stash_reserved[CONFIG_MAX_STASHES]; + stash_t *stash[CONFIG_MAX_STASHES]; } stash_global_t; @@ -106,10 +110,10 @@ int odp_stash_capability(odp_stash_capability_t *capa, odp_stash_type_t type) (void)type; memset(capa, 0, sizeof(odp_stash_capability_t)); - capa->max_stashes_any_type = MAX_STASHES; - capa->max_stashes = MAX_STASHES; + capa->max_stashes_any_type = CONFIG_MAX_STASHES; + capa->max_stashes = CONFIG_MAX_STASHES; capa->max_num_obj = MAX_RING_SIZE; - capa->max_obj_size = sizeof(uintptr_t); + capa->max_obj_size = sizeof(uint64_t); return 0; } @@ -129,7 +133,7 @@ static int reserve_index(void) odp_ticketlock_lock(&stash_global->lock); - for (i = 0; i < MAX_STASHES; i++) { + for (i = 0; i < CONFIG_MAX_STASHES; i++) { if (stash_global->stash_reserved[i] == 0) { index = i; stash_global->stash_reserved[i] = 1; @@ -157,7 +161,7 @@ odp_stash_t odp_stash_create(const char *name, const odp_stash_param_t *param) odp_shm_t shm; stash_t *stash; uint64_t i, ring_size, shm_size; - int ring_ptr, index; + int ring_u64, index; char shm_name[ODP_STASH_NAME_LEN + 8]; if (odp_global_ro.disable.stash) { @@ -165,7 +169,7 @@ odp_stash_t odp_stash_create(const char *name, const odp_stash_param_t *param) return ODP_STASH_INVALID; } - if (param->obj_size > sizeof(uintptr_t)) { + if (param->obj_size > sizeof(uint64_t)) { ODP_ERR("Too large object handle.\n"); return ODP_STASH_INVALID; } @@ -187,9 +191,9 @@ odp_stash_t odp_stash_create(const char *name, const odp_stash_param_t *param) return ODP_STASH_INVALID; } - ring_ptr = 0; + ring_u64 = 0; if (param->obj_size > sizeof(uint32_t)) - ring_ptr = 1; + ring_u64 = 1; ring_size = param->num_obj; @@ -202,8 +206,8 @@ odp_stash_t odp_stash_create(const char *name, const odp_stash_param_t *param) memset(shm_name, 0, sizeof(shm_name)); snprintf(shm_name, sizeof(shm_name) - 1, "_stash_%s", name); - if (ring_ptr) - shm_size = sizeof(stash_t) + (ring_size * sizeof(uintptr_t)); + if (ring_u64) + shm_size = sizeof(stash_t) + (ring_size * sizeof(uint64_t)); else shm_size = sizeof(stash_t) + (ring_size * sizeof(uint32_t)); @@ -218,11 +222,11 @@ odp_stash_t odp_stash_create(const char *name, const odp_stash_param_t *param) stash = odp_shm_addr(shm); memset(stash, 0, sizeof(stash_t)); - if (ring_ptr) { - ring_ptr_init(&stash->ring_ptr.hdr); + if (ring_u64) { + ring_u64_init(&stash->ring_u64.hdr); for (i = 0; i < ring_size; i++) - stash->ring_ptr.data[i] = 0; + stash->ring_u64.data[i] = 0; } else { ring_u32_init(&stash->ring_u32.hdr); @@ -284,7 +288,7 @@ odp_stash_t odp_stash_lookup(const char *name) odp_ticketlock_lock(&stash_global->lock); - for (i = 0; i < MAX_STASHES; i++) { + for (i = 0; i < CONFIG_MAX_STASHES; i++) { stash = stash_global->stash[i]; if (stash && strcmp(stash->name, name) == 0) { @@ -311,11 +315,11 @@ int32_t odp_stash_put(odp_stash_t st, const void *obj, int32_t num) obj_size = stash->obj_size; - if (obj_size == sizeof(uintptr_t)) { - ring_ptr_t *ring_ptr = &stash->ring_ptr.hdr; + if (obj_size == sizeof(uint64_t)) { + ring_u64_t *ring_u64 = &stash->ring_u64.hdr; - ring_ptr_enq_multi(ring_ptr, stash->ring_mask, - (void *)(uintptr_t)obj, num); + ring_u64_enq_multi(ring_u64, stash->ring_mask, + (uint64_t *)(uintptr_t)obj, num); return num; } @@ -354,6 +358,55 @@ int32_t odp_stash_put(odp_stash_t st, const void *obj, int32_t num) return -1; } +int32_t odp_stash_put_u32(odp_stash_t st, const uint32_t u32[], int32_t num) +{ + stash_t *stash = (stash_t *)(uintptr_t)st; + + if (odp_unlikely(st == ODP_STASH_INVALID)) + return -1; + + ODP_ASSERT(stash->obj_size == sizeof(uint32_t)); + + ring_u32_enq_multi(&stash->ring_u32.hdr, stash->ring_mask, + (uint32_t *)(uintptr_t)u32, num); + return num; +} + +int32_t odp_stash_put_u64(odp_stash_t st, const uint64_t u64[], int32_t num) +{ + stash_t *stash = (stash_t *)(uintptr_t)st; + + if (odp_unlikely(st == ODP_STASH_INVALID)) + return -1; + + ODP_ASSERT(stash->obj_size == sizeof(uint64_t)); + + ring_u64_enq_multi(&stash->ring_u64.hdr, stash->ring_mask, + (uint64_t *)(uintptr_t)u64, num); + return num; +} + +int32_t odp_stash_put_ptr(odp_stash_t st, const uintptr_t ptr[], int32_t num) +{ + stash_t *stash = (stash_t *)(uintptr_t)st; + + if (odp_unlikely(st == ODP_STASH_INVALID)) + return -1; + + ODP_ASSERT(stash->obj_size == sizeof(uintptr_t)); + + if (sizeof(uintptr_t) == sizeof(uint32_t)) + ring_u32_enq_multi(&stash->ring_u32.hdr, stash->ring_mask, + (uint32_t *)(uintptr_t)ptr, num); + else if (sizeof(uintptr_t) == sizeof(uint64_t)) + ring_u64_enq_multi(&stash->ring_u64.hdr, stash->ring_mask, + (uint64_t *)(uintptr_t)ptr, num); + else + return -1; + + return num; +} + int32_t odp_stash_get(odp_stash_t st, void *obj, int32_t num) { stash_t *stash; @@ -367,10 +420,10 @@ int32_t odp_stash_get(odp_stash_t st, void *obj, int32_t num) obj_size = stash->obj_size; - if (obj_size == sizeof(uintptr_t)) { - ring_ptr_t *ring_ptr = &stash->ring_ptr.hdr; + if (obj_size == sizeof(uint64_t)) { + ring_u64_t *ring_u64 = &stash->ring_u64.hdr; - return ring_ptr_deq_multi(ring_ptr, stash->ring_mask, obj, num); + return ring_u64_deq_multi(ring_u64, stash->ring_mask, obj, num); } if (obj_size == sizeof(uint32_t)) { @@ -410,6 +463,52 @@ int32_t odp_stash_get(odp_stash_t st, void *obj, int32_t num) return -1; } +int32_t odp_stash_get_u32(odp_stash_t st, uint32_t u32[], int32_t num) +{ + stash_t *stash = (stash_t *)(uintptr_t)st; + + if (odp_unlikely(st == ODP_STASH_INVALID)) + return -1; + + ODP_ASSERT(stash->obj_size == sizeof(uint32_t)); + + return ring_u32_deq_multi(&stash->ring_u32.hdr, stash->ring_mask, u32, + num); +} + +int32_t odp_stash_get_u64(odp_stash_t st, uint64_t u64[], int32_t num) +{ + stash_t *stash = (stash_t *)(uintptr_t)st; + + if (odp_unlikely(st == ODP_STASH_INVALID)) + return -1; + + ODP_ASSERT(stash->obj_size == sizeof(uint64_t)); + + return ring_u64_deq_multi(&stash->ring_u64.hdr, stash->ring_mask, u64, + num); +} + +int32_t odp_stash_get_ptr(odp_stash_t st, uintptr_t ptr[], int32_t num) +{ + stash_t *stash = (stash_t *)(uintptr_t)st; + + if (odp_unlikely(st == ODP_STASH_INVALID)) + return -1; + + ODP_ASSERT(stash->obj_size == sizeof(uintptr_t)); + + if (sizeof(uintptr_t) == sizeof(uint32_t)) + return ring_u32_deq_multi(&stash->ring_u32.hdr, + stash->ring_mask, + (uint32_t *)(uintptr_t)ptr, num); + else if (sizeof(uintptr_t) == sizeof(uint64_t)) + return ring_u64_deq_multi(&stash->ring_u64.hdr, + stash->ring_mask, + (uint64_t *)(uintptr_t)ptr, num); + return -1; +} + int odp_stash_flush_cache(odp_stash_t st) { if (odp_unlikely(st == ODP_STASH_INVALID)) diff --git a/platform/linux-generic/odp_time.c b/platform/linux-generic/odp_time.c index 8d528727f..d908f02cb 100644 --- a/platform/linux-generic/odp_time.c +++ b/platform/linux-generic/odp_time.c @@ -185,7 +185,7 @@ void odp_time_wait_ns(uint64_t ns) void odp_time_wait_until(odp_time_t time) { - return time_wait_until(time); + time_wait_until(time); } int _odp_time_init_global(void) diff --git a/platform/linux-generic/odp_timer.c b/platform/linux-generic/odp_timer.c index f1ca28346..a28d31245 100644 --- a/platform/linux-generic/odp_timer.c +++ b/platform/linux-generic/odp_timer.c @@ -244,7 +244,7 @@ static inline timer_pool_t *handle_to_tp(odp_timer_t hdl) if (odp_likely(tp != NULL)) return timer_global->timer_pool[tp_idx]; } - ODP_ABORT("Invalid timer handle %p\n", hdl); + ODP_ABORT("Invalid timer handle %p\n", (void *)hdl); } static inline uint32_t handle_to_idx(odp_timer_t hdl, @@ -255,7 +255,7 @@ static inline uint32_t handle_to_idx(odp_timer_t hdl, __builtin_prefetch(&tp->tick_buf[idx], 0, 0); if (odp_likely(idx < odp_atomic_load_u32(&tp->high_wm))) return idx; - ODP_ABORT("Invalid timer handle %p\n", hdl); + ODP_ABORT("Invalid timer handle %p\n", (void *)hdl); } static inline odp_timer_t tp_idx_to_handle(timer_pool_t *tp, @@ -1496,20 +1496,29 @@ void *odp_timeout_user_ptr(odp_timeout_t tmo) return (void *)(uintptr_t)timeout_hdr(tmo)->user_ptr; } -odp_timeout_t odp_timeout_alloc(odp_pool_t pool) +odp_timeout_t odp_timeout_alloc(odp_pool_t pool_hdl) { - odp_buffer_t buf = odp_buffer_alloc(pool); + odp_timeout_t tmo; + pool_t *pool; + int ret; + + ODP_ASSERT(pool_hdl != ODP_POOL_INVALID); + + pool = pool_entry_from_hdl(pool_hdl); + + ODP_ASSERT(pool->type == ODP_POOL_TIMEOUT); - if (odp_unlikely(buf == ODP_BUFFER_INVALID)) - return ODP_TIMEOUT_INVALID; - return odp_timeout_from_event(odp_buffer_to_event(buf)); + ret = _odp_buffer_alloc_multi(pool, (odp_buffer_hdr_t **)&tmo, 1); + + if (odp_likely(ret == 1)) + return tmo; + + return ODP_TIMEOUT_INVALID; } void odp_timeout_free(odp_timeout_t tmo) { - odp_event_t ev = odp_timeout_to_event(tmo); - - odp_buffer_free(odp_buffer_from_event(ev)); + _odp_buffer_free_multi((odp_buffer_hdr_t **)&tmo, 1); } void odp_timer_pool_print(odp_timer_pool_t timer_pool) @@ -1525,7 +1534,7 @@ void odp_timer_pool_print(odp_timer_pool_t timer_pool) ODP_PRINT("\nTimer pool info\n"); ODP_PRINT("---------------\n"); - ODP_PRINT(" timer pool %p\n", tp); + ODP_PRINT(" timer pool %p\n", (void *)tp); ODP_PRINT(" tp index %u\n", tp->tp_idx); ODP_PRINT(" num timers %u\n", tp->num_alloc); ODP_PRINT(" num tp %i\n", timer_global->num_timer_pools); @@ -1550,7 +1559,7 @@ void odp_timer_print(odp_timer_t timer) ODP_PRINT("\nTimer info\n"); ODP_PRINT("----------\n"); - ODP_PRINT(" timer pool %p\n", tp); + ODP_PRINT(" timer pool %p\n", (void *)tp); ODP_PRINT(" timer index %u\n", idx); ODP_PRINT(" dest queue 0x%" PRIx64 "\n", odp_queue_to_u64(tim->queue)); ODP_PRINT(" user ptr %p\n", tim->user_ptr); @@ -1580,7 +1589,7 @@ void odp_timeout_print(odp_timeout_t tmo) ODP_PRINT("\nTimeout info\n"); ODP_PRINT("------------\n"); ODP_PRINT(" tmo handle 0x%" PRIx64 "\n", odp_timeout_to_u64(tmo)); - ODP_PRINT(" timer pool %p\n", tp); + ODP_PRINT(" timer pool %p\n", (void *)tp); ODP_PRINT(" timer index %u\n", idx); ODP_PRINT(" expiration %" PRIu64 "\n", tmo_hdr->expiration); ODP_PRINT(" user ptr %p\n", tmo_hdr->user_ptr); diff --git a/platform/linux-generic/odp_timer_wheel.c b/platform/linux-generic/odp_timer_wheel.c index 71b3370b5..c50d3a13d 100644 --- a/platform/linux-generic/odp_timer_wheel.c +++ b/platform/linux-generic/odp_timer_wheel.c @@ -94,9 +94,12 @@ typedef union { timer_blk_t *timer_blk_list; } current_timer_slot_t; +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wpedantic" typedef struct { current_timer_slot_t slots[0]; } current_wheel_t; +#pragma GCC diagnostic pop typedef struct { uint32_t count; @@ -105,7 +108,7 @@ typedef struct { uint32_t head_idx; uint32_t tail_idx; uint32_t max_idx; - current_timer_slot_t entries[0]; + current_timer_slot_t entries[]; } expired_ring_t; typedef struct { @@ -130,9 +133,12 @@ typedef union { /* Each general_timer_slot is 16 bytes long. */ list_entry_t list_entry; } general_timer_slot_t; +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wpedantic" typedef struct { general_timer_slot_t slots[0]; } general_wheel_t; +#pragma GCC diagnostic pop typedef struct { /* Note that rev stands for revolution - one complete sweep through diff --git a/platform/linux-generic/pktio/dpdk.c b/platform/linux-generic/pktio/dpdk.c index 9abba6292..36c1ec06e 100644 --- a/platform/linux-generic/pktio/dpdk.c +++ b/platform/linux-generic/pktio/dpdk.c @@ -559,7 +559,7 @@ static struct rte_mempool_ops odp_pool_ops = { .get_count = pool_get_count }; -MEMPOOL_REGISTER_OPS(odp_pool_ops); +MEMPOOL_REGISTER_OPS(odp_pool_ops) static inline int mbuf_to_pkt(pktio_entry_t *pktio_entry, odp_packet_t pkt_table[], diff --git a/platform/linux-generic/pktio/dpdk_parse.c b/platform/linux-generic/pktio/dpdk_parse.c index c1b75f126..0984f06c5 100644 --- a/platform/linux-generic/pktio/dpdk_parse.c +++ b/platform/linux-generic/pktio/dpdk_parse.c @@ -510,4 +510,7 @@ int _odp_dpdk_packet_parse_common(packet_parser_t *prs, const uint8_t *ptr, pktin_cfg); } -#endif /* _ODP_PKTIO_DPDK */ +#else /* _ODP_PKTIO_DPDK */ +/* Avoid warning about empty translation unit */ +typedef int _odp_dummy; +#endif diff --git a/platform/linux-generic/pktio/ipc.c b/platform/linux-generic/pktio/ipc.c index ad633e231..b89252303 100644 --- a/platform/linux-generic/pktio/ipc.c +++ b/platform/linux-generic/pktio/ipc.c @@ -1,5 +1,5 @@ /* Copyright (c) 2015-2018, Linaro Limited - * Copyright (c) 2019, Nokia + * Copyright (c) 2019-2021, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -15,17 +15,13 @@ #include <errno.h> #include <fcntl.h> +#include <stdint.h> #include <sys/mman.h> #include <sys/stat.h> #include <unistd.h> -#define IPC_ODP_DEBUG_PRINT 0 - -#define IPC_ODP_DBG(fmt, ...) \ - do { \ - if (IPC_ODP_DEBUG_PRINT == 1) \ - ODP_DBG(fmt, ##__VA_ARGS__);\ - } while (0) +/* Debug level for IPC */ +#define IPC_DBG 3 /* Burst size for IPC free operations */ #define IPC_BURST_SIZE 32 @@ -112,7 +108,7 @@ static inline pkt_ipc_t *pkt_priv(pktio_entry_t *pktio_entry) } /* MAC address for the "ipc" interface */ -static const char pktio_ipc_mac[] = {0x12, 0x12, 0x12, 0x12, 0x12, 0x12}; +static const uint8_t pktio_ipc_mac[] = {0x12, 0x12, 0x12, 0x12, 0x12, 0x12}; static odp_shm_t _ipc_map_remote_pool(const char *name, int pid); @@ -221,7 +217,7 @@ static int _ipc_master_start(pktio_entry_t *pktio_entry) odp_atomic_store_u32(&pktio_ipc->ready, 1); - IPC_ODP_DBG("%s started.\n", pktio_entry->s.name); + ODP_DBG_LVL(IPC_DBG, "%s started.\n", pktio_entry->s.name); return 0; } @@ -382,7 +378,7 @@ static odp_shm_t _ipc_map_remote_pool(const char *name, int pid) return ODP_SHM_INVALID; } - IPC_ODP_DBG("Mapped remote pool %s to local %s\n", name, rname); + ODP_DBG_LVL(IPC_DBG, "Mapped remote pool %s to local %s\n", name, rname); return shm; } @@ -547,7 +543,7 @@ static int ipc_pktio_open(odp_pktio_t id ODP_UNUSED, pktio_ipc->type = PKTIO_TYPE_IPC_SLAVE; snprintf(name, sizeof(name), "ipc:%s_info", tail); - IPC_ODP_DBG("lookup for name %s for pid %d\n", name, pid); + ODP_DBG_LVL(IPC_DBG, "lookup for name %s for pid %d\n", name, pid); shm = odp_shm_import(name, pid, name); if (ODP_SHM_INVALID == shm) return -1; @@ -639,7 +635,7 @@ static int ipc_pktio_recv_lockless(pktio_entry_t *pktio_entry, ready = odp_atomic_load_u32(&pktio_ipc->ready); if (odp_unlikely(!ready)) { - IPC_ODP_DBG("start pktio is missing before usage?\n"); + ODP_DBG_LVL(IPC_DBG, "start pktio is missing before usage?\n"); return 0; } @@ -692,7 +688,7 @@ static int ipc_pktio_recv_lockless(pktio_entry_t *pktio_entry, * packet ordering store such packets in local * cache. */ - IPC_ODP_DBG("unable to allocate packet %d/%d\n", + ODP_DBG_LVL(IPC_DBG, "unable to allocate packet %d/%d\n", i, pkts); break; } @@ -742,7 +738,7 @@ static int ipc_pktio_recv_lockless(pktio_entry_t *pktio_entry, ring_ptr_enq_multi(r_p, ring_mask, ipcbufs_p, pkts); for (i = 0; i < pkts; i++) { - IPC_ODP_DBG("%d/%d send to be free packet offset %" PRIuPTR "\n", + ODP_DBG_LVL(IPC_DBG, "%d/%d send to be free packet offset %" PRIuPTR "\n", i, pkts, offsets[i]); } @@ -819,11 +815,11 @@ static int ipc_pktio_send_lockless(pktio_entry_t *pktio_entry, (uint8_t *)odp_shm_addr(pool->shm); /* compile all function code even if ipc disabled with config */ - IPC_ODP_DBG("%d/%d send packet %" PRIu64 ", pool %" PRIu64 "," + ODP_DBG_LVL(IPC_DBG, "%d/%d send packet %" PRIu64 ", pool %" PRIu64 "," "phdr = %p, offset %td, sendoff %" PRIxPTR ", addr %p iaddr " "%p\n", i, num, odp_packet_to_u64(pkt), odp_pool_to_u64(pool_hdl), - pkt_hdr, (uint8_t *)pkt_hdr->seg_data - + (void *)pkt_hdr, (uint8_t *)pkt_hdr->seg_data - (uint8_t *)odp_shm_addr(pool->shm), offsets[i], odp_shm_addr(pool->shm), odp_shm_addr(ipc_pool->shm)); diff --git a/platform/linux-generic/pktio/loop.c b/platform/linux-generic/pktio/loop.c index 889a270ea..666e7ce90 100644 --- a/platform/linux-generic/pktio/loop.c +++ b/platform/linux-generic/pktio/loop.c @@ -49,7 +49,7 @@ static inline pkt_loop_t *pkt_priv(pktio_entry_t *pktio_entry) } /* MAC address for the "loop" interface */ -static const char pktio_loop_mac[] = {0x02, 0xe9, 0x34, 0x80, 0x73, 0x01}; +static const uint8_t pktio_loop_mac[] = {0x02, 0xe9, 0x34, 0x80, 0x73, 0x01}; static int loopback_stats_reset(pktio_entry_t *pktio_entry); static int loopback_init_capability(pktio_entry_t *pktio_entry); diff --git a/platform/linux-generic/pktio/netmap.c b/platform/linux-generic/pktio/netmap.c index 18d9b3a33..b45cdbe47 100644 --- a/platform/linux-generic/pktio/netmap.c +++ b/platform/linux-generic/pktio/netmap.c @@ -1331,5 +1331,7 @@ const pktio_if_ops_t _odp_netmap_pktio_ops = { .send = netmap_send, .fd_set = netmap_fd_set }; - -#endif /* _ODP_PKTIO_NETMAP */ +#else /* _ODP_PKTIO_NETMAP */ +/* Avoid warning about empty translation unit */ +typedef int _odp_dummy; +#endif diff --git a/platform/linux-generic/pktio/null.c b/platform/linux-generic/pktio/null.c index cbcde323c..8a421986b 100644 --- a/platform/linux-generic/pktio/null.c +++ b/platform/linux-generic/pktio/null.c @@ -7,6 +7,8 @@ #include <odp_api.h> #include <odp_packet_io_internal.h> +#include <stdint.h> + typedef struct { int promisc; /**< whether promiscuous mode is on */ } pkt_null_t; @@ -116,7 +118,7 @@ static uint32_t null_mtu_get(pktio_entry_t *pktio_entry ODP_UNUSED) return PKTIO_NULL_MTU; } -static const char null_mac[] = {0x02, 0xe9, 0x34, 0x80, 0x73, 0x05}; +static const uint8_t null_mac[] = {0x02, 0xe9, 0x34, 0x80, 0x73, 0x05}; static int null_mac_addr_get(pktio_entry_t *pktio_entry ODP_UNUSED, void *mac_addr) diff --git a/platform/linux-generic/pktio/pcap.c b/platform/linux-generic/pktio/pcap.c index bf4c87c02..d42840da9 100644 --- a/platform/linux-generic/pktio/pcap.c +++ b/platform/linux-generic/pktio/pcap.c @@ -50,6 +50,7 @@ #include <errno.h> #include <pcap/pcap.h> #include <pcap/bpf.h> +#include <stdint.h> typedef struct { char *fname_rx; /**< name of pcap file for rx */ @@ -75,7 +76,7 @@ static inline pkt_pcap_t *pkt_priv(pktio_entry_t *pktio_entry) #define PKTIO_PCAP_MTU_MIN (68 + _ODP_ETHHDR_LEN) #define PKTIO_PCAP_MTU_MAX (64 * 1024) -static const char pcap_mac[] = {0x02, 0xe9, 0x34, 0x80, 0x73, 0x04}; +static const uint8_t pcap_mac[] = {0x02, 0xe9, 0x34, 0x80, 0x73, 0x04}; static int pcapif_stats_reset(pktio_entry_t *pktio_entry); diff --git a/test/common/odp_cunit_common.h b/test/common/odp_cunit_common.h index 3e06ba10c..2ccab6076 100644 --- a/test/common/odp_cunit_common.h +++ b/test/common/odp_cunit_common.h @@ -46,7 +46,7 @@ static inline void odp_cunit_test_missing(void) { } /* A test case that is unconditionally inactive. Its name will be registered * with CUnit but it won't be executed and will be reported as inactive in * the result summary. */ -#define ODP_TEST_INFO_INACTIVE(test_func, args...) \ +#define ODP_TEST_INFO_INACTIVE(test_func, ...) \ {#test_func, odp_cunit_test_missing, odp_cunit_test_inactive} #define ODP_TEST_INACTIVE 0 diff --git a/test/performance/odp_crypto.c b/test/performance/odp_crypto.c index 4f81dab17..341782630 100644 --- a/test/performance/odp_crypto.c +++ b/test/performance/odp_crypto.c @@ -19,10 +19,6 @@ #include <odp_api.h> #include <odp/helper/odph_api.h> -#define app_err(fmt, ...) \ - fprintf(stderr, "%s:%d:%s(): Error: " fmt, __FILE__, \ - __LINE__, __func__, ##__VA_ARGS__) - /** @def POOL_NUM_PKT * Number of packets in the pool */ @@ -640,7 +636,7 @@ create_session_from_config(odp_crypto_session_t *session, /* Lookup the packet pool */ pkt_pool = odp_pool_lookup("packet_pool"); if (pkt_pool == ODP_POOL_INVALID) { - app_err("packet_pool pool not found\n"); + ODPH_ERR("packet_pool pool not found\n"); return -1; } params.output_pool = pkt_pool; @@ -648,7 +644,7 @@ create_session_from_config(odp_crypto_session_t *session, if (cargs->schedule || cargs->poll) { out_queue = odp_queue_lookup("crypto-out"); if (out_queue == ODP_QUEUE_INVALID) { - app_err("crypto-out queue not found\n"); + ODPH_ERR("crypto-out queue not found\n"); return -1; } params.compl_queue = out_queue; @@ -659,7 +655,7 @@ create_session_from_config(odp_crypto_session_t *session, } if (odp_crypto_session_create(¶ms, session, &ses_create_rc)) { - app_err("crypto session create failed.\n"); + ODPH_ERR("crypto session create failed.\n"); return -1; } @@ -673,7 +669,7 @@ make_packet(odp_pool_t pkt_pool, unsigned int payload_length) pkt = odp_packet_alloc(pkt_pool, payload_length); if (pkt == ODP_PACKET_INVALID) { - app_err("failed to allocate buffer\n"); + ODPH_ERR("failed to allocate buffer\n"); return pkt; } @@ -704,14 +700,14 @@ run_measure_one(crypto_args_t *cargs, pkt_pool = odp_pool_lookup("packet_pool"); if (pkt_pool == ODP_POOL_INVALID) { - app_err("pkt_pool not found\n"); + ODPH_ERR("pkt_pool not found\n"); return -1; } out_queue = odp_queue_lookup("crypto-out"); if (cargs->schedule || cargs->poll) { if (out_queue == ODP_QUEUE_INVALID) { - app_err("crypto-out queue not found\n"); + ODPH_ERR("crypto-out queue not found\n"); return -1; } } @@ -766,8 +762,7 @@ run_measure_one(crypto_args_t *cargs, rc = odp_crypto_op_enq(&pkt, &out_pkt, ¶ms, 1); if (rc <= 0) { - app_err("failed odp_crypto_packet_op_enq: rc = %d\n", - rc); + ODPH_ERR("failed odp_crypto_packet_op_enq: rc = %d\n", rc); if (!cargs->reuse_packet) odp_packet_free(pkt); break; @@ -777,8 +772,7 @@ run_measure_one(crypto_args_t *cargs, rc = odp_crypto_op(&pkt, &out_pkt, ¶ms, 1); if (rc <= 0) { - app_err("failed odp_crypto_packet_op: rc = %d\n", - rc); + ODPH_ERR("failed odp_crypto_packet_op: rc = %d\n", rc); if (!cargs->reuse_packet) odp_packet_free(pkt); break; @@ -1045,7 +1039,7 @@ int main(int argc, char *argv[]) /* Let helper collect its own arguments (e.g. --odph_proc) */ argc = odph_parse_options(argc, argv); if (odph_options(&helper_options)) { - app_err("Reading ODP helper options failed.\n"); + ODPH_ERR("Reading ODP helper options failed.\n"); exit(EXIT_FAILURE); } @@ -1059,7 +1053,7 @@ int main(int argc, char *argv[]) /* Init ODP before calling anything else */ if (odp_init_global(&instance, &init_param, NULL)) { - app_err("ODP global init failed.\n"); + ODPH_ERR("ODP global init failed.\n"); exit(EXIT_FAILURE); } @@ -1069,22 +1063,22 @@ int main(int argc, char *argv[]) odp_sys_info_print(); if (odp_crypto_capability(&crypto_capa)) { - app_err("Crypto capability request failed.\n"); + ODPH_ERR("Crypto capability request failed.\n"); exit(EXIT_FAILURE); } if (cargs.schedule && crypto_capa.queue_type_sched == 0) { - app_err("scheduled type completion queue not supported.\n"); + ODPH_ERR("scheduled type completion queue not supported.\n"); exit(EXIT_FAILURE); } if (cargs.poll && crypto_capa.queue_type_plain == 0) { - app_err("plain type completion queue not supported.\n"); + ODPH_ERR("plain type completion queue not supported.\n"); exit(EXIT_FAILURE); } if (odp_pool_capability(&pool_capa)) { - app_err("Pool capability request failed.\n"); + ODPH_ERR("Pool capability request failed.\n"); exit(EXIT_FAILURE); } @@ -1106,7 +1100,7 @@ int main(int argc, char *argv[]) pool = odp_pool_create("packet_pool", ¶ms); if (pool == ODP_POOL_INVALID) { - app_err("packet pool create failed.\n"); + ODPH_ERR("packet pool create failed.\n"); exit(EXIT_FAILURE); } odp_pool_print(pool); @@ -1125,7 +1119,7 @@ int main(int argc, char *argv[]) } if (cargs.schedule || cargs.poll) { if (out_queue == ODP_QUEUE_INVALID) { - app_err("crypto-out queue create failed.\n"); + ODPH_ERR("crypto-out queue create failed.\n"); exit(EXIT_FAILURE); } } @@ -1185,17 +1179,17 @@ int main(int argc, char *argv[]) if (cargs.schedule || cargs.poll) odp_queue_destroy(out_queue); if (odp_pool_destroy(pool)) { - app_err("Error: pool destroy\n"); + ODPH_ERR("Error: pool destroy\n"); exit(EXIT_FAILURE); } if (odp_term_local()) { - app_err("Error: term local\n"); + ODPH_ERR("Error: term local\n"); exit(EXIT_FAILURE); } if (odp_term_global(instance)) { - app_err("Error: term global\n"); + ODPH_ERR("Error: term global\n"); exit(EXIT_FAILURE); } diff --git a/test/performance/odp_ipsec.c b/test/performance/odp_ipsec.c index 04788995e..ee3e062b7 100644 --- a/test/performance/odp_ipsec.c +++ b/test/performance/odp_ipsec.c @@ -19,10 +19,6 @@ #include <odp_api.h> #include <odp/helper/odph_api.h> -#define app_err(fmt, ...) \ - fprintf(stderr, "%s:%d:%s(): Error: " fmt, __FILE__, \ - __LINE__, __func__, ##__VA_ARGS__) - /** @def POOL_NUM_PKT * Number of packets in the pool */ @@ -567,7 +563,7 @@ create_sa_from_config(ipsec_alg_config_t *config, if (cargs->schedule || cargs->poll) { out_queue = odp_queue_lookup("ipsec-out"); if (out_queue == ODP_QUEUE_INVALID) { - app_err("ipsec-out queue not found\n"); + ODPH_ERR("ipsec-out queue not found\n"); return ODP_IPSEC_SA_INVALID; } param.dest_queue = out_queue; @@ -598,7 +594,7 @@ make_packet(odp_pool_t pkt_pool, unsigned int payload_length) pkt = odp_packet_alloc(pkt_pool, payload_length); if (pkt == ODP_PACKET_INVALID) { - app_err("failed to allocate buffer\n"); + ODPH_ERR("failed to allocate buffer\n"); return pkt; } @@ -630,7 +626,7 @@ run_measure_one(ipsec_args_t *cargs, pkt_pool = odp_pool_lookup("packet_pool"); if (pkt_pool == ODP_POOL_INVALID) { - app_err("pkt_pool not found\n"); + ODPH_ERR("pkt_pool not found\n"); return -1; } @@ -665,8 +661,7 @@ run_measure_one(ipsec_args_t *cargs, &out_pkt, &num_out, ¶m); if (rc <= 0) { - app_err("failed odp_ipsec_out: rc = %d\n", - rc); + ODPH_ERR("failed odp_ipsec_out: rc = %d\n", rc); odp_packet_free(pkt); break; } @@ -674,8 +669,8 @@ run_measure_one(ipsec_args_t *cargs, odp_ipsec_packet_result_t result; odp_ipsec_result(&result, out_pkt); - app_err("Received error packet: %d\n", - result.status.error.all); + ODPH_ERR("Received error packet: %d\n", + result.status.error.all); } packets_sent += rc; packets_received += num_out; @@ -706,13 +701,13 @@ run_measure_one_async(ipsec_args_t *cargs, pkt_pool = odp_pool_lookup("packet_pool"); if (pkt_pool == ODP_POOL_INVALID) { - app_err("pkt_pool not found\n"); + ODPH_ERR("pkt_pool not found\n"); return -1; } out_queue = odp_queue_lookup("ipsec-out"); if (out_queue == ODP_QUEUE_INVALID) { - app_err("ipsec-out queue not found\n"); + ODPH_ERR("ipsec-out queue not found\n"); return -1; } @@ -745,8 +740,8 @@ run_measure_one_async(ipsec_args_t *cargs, rc = odp_ipsec_out_enq(&pkt, 1, ¶m); if (rc <= 0) { - app_err("failed odp_crypto_packet_op_enq: rc = %d\n", - rc); + ODPH_ERR("failed odp_crypto_packet_op_enq: rc = %d\n", + rc); odp_packet_free(pkt); break; } @@ -800,12 +795,12 @@ run_measure_one_config(ipsec_args_t *cargs, int rc = 0; if (odp_ipsec_capability(&capa) < 0) { - app_err("IPSEC capability call failed.\n"); + ODPH_ERR("IPSEC capability call failed.\n"); return -1; } if (cargs->ah && (ODP_SUPPORT_NO == capa.proto_ah)) { - app_err("IPSEC AH protocol not supported.\n"); + ODPH_ERR("IPSEC AH protocol not supported.\n"); return -1; } @@ -821,7 +816,7 @@ run_measure_one_config(ipsec_args_t *cargs, sa = create_sa_from_config(config, cargs); if (sa == ODP_IPSEC_SA_INVALID) { - app_err("IPsec SA create failed.\n"); + ODPH_ERR("IPsec SA create failed.\n"); return -1; } @@ -1041,7 +1036,7 @@ int main(int argc, char *argv[]) /* Let helper collect its own arguments (e.g. --odph_proc) */ argc = odph_parse_options(argc, argv); if (odph_options(&helper_options)) { - app_err("Reading ODP helper options failed.\n"); + ODPH_ERR("Reading ODP helper options failed.\n"); exit(EXIT_FAILURE); } @@ -1055,20 +1050,20 @@ int main(int argc, char *argv[]) /* Init ODP before calling anything else */ if (odp_init_global(&instance, &init_param, NULL)) { - app_err("ODP global init failed.\n"); + ODPH_ERR("ODP global init failed.\n"); exit(EXIT_FAILURE); } /* Init this thread */ if (odp_init_local(instance, ODP_THREAD_WORKER)) { - app_err("ODP local init failed.\n"); + ODPH_ERR("ODP local init failed.\n"); exit(EXIT_FAILURE); } odp_sys_info_print(); if (odp_pool_capability(&capa)) { - app_err("Pool capability request failed.\n"); + ODPH_ERR("Pool capability request failed.\n"); exit(EXIT_FAILURE); } @@ -1090,23 +1085,23 @@ int main(int argc, char *argv[]) pool = odp_pool_create("packet_pool", ¶m); if (pool == ODP_POOL_INVALID) { - app_err("packet pool create failed.\n"); + ODPH_ERR("packet pool create failed.\n"); exit(EXIT_FAILURE); } odp_pool_print(pool); if (odp_ipsec_capability(&ipsec_capa) < 0) { - app_err("IPSEC capability call failed.\n"); + ODPH_ERR("IPSEC capability call failed.\n"); exit(EXIT_FAILURE); } if (cargs.schedule && !ipsec_capa.queue_type_sched) { - app_err("Scheduled type destination queue not supported.\n"); + ODPH_ERR("Scheduled type destination queue not supported.\n"); exit(EXIT_FAILURE); } if (cargs.poll && !ipsec_capa.queue_type_plain) { - app_err("Plain type destination queue not supported.\n"); + ODPH_ERR("Plain type destination queue not supported.\n"); exit(EXIT_FAILURE); } @@ -1129,7 +1124,7 @@ int main(int argc, char *argv[]) } if (cargs.schedule || cargs.poll) { if (out_queue == ODP_QUEUE_INVALID) { - app_err("ipsec-out queue create failed.\n"); + ODPH_ERR("ipsec-out queue create failed.\n"); exit(EXIT_FAILURE); } config.inbound_mode = ODP_IPSEC_OP_MODE_ASYNC; @@ -1141,7 +1136,7 @@ int main(int argc, char *argv[]) config.inbound.default_queue = ODP_QUEUE_INVALID; } if (odp_ipsec_config(&config)) { - app_err("odp_ipsec_config() failed\n"); + ODPH_ERR("odp_ipsec_config() failed\n"); exit(EXIT_FAILURE); } @@ -1202,17 +1197,17 @@ int main(int argc, char *argv[]) if (cargs.schedule || cargs.poll) odp_queue_destroy(out_queue); if (odp_pool_destroy(pool)) { - app_err("Error: pool destroy\n"); + ODPH_ERR("Error: pool destroy\n"); exit(EXIT_FAILURE); } if (odp_term_local()) { - app_err("Error: term local\n"); + ODPH_ERR("Error: term local\n"); exit(EXIT_FAILURE); } if (odp_term_global(instance)) { - app_err("Error: term global\n"); + ODPH_ERR("Error: term global\n"); exit(EXIT_FAILURE); } diff --git a/test/performance/odp_pktio_perf.c b/test/performance/odp_pktio_perf.c index 593465f4f..bf635abd9 100644 --- a/test/performance/odp_pktio_perf.c +++ b/test/performance/odp_pktio_perf.c @@ -52,12 +52,6 @@ * received by this time will be assumed to have been lost. */ #define SHUTDOWN_DELAY_NS (ODP_TIME_MSEC_IN_NS * 100) -#define VPRINT(fmt, ...) \ - do { \ - if (gbl_args->args.verbose) \ - printf(fmt, ##__VA_ARGS__); \ - } while (0) - #define CACHE_ALIGN_ROUNDUP(x)\ ((ODP_CACHE_LINE_SIZE) * \ (((x) + ODP_CACHE_LINE_SIZE - 1) / (ODP_CACHE_LINE_SIZE))) @@ -369,12 +363,13 @@ static int run_thread_tx(void *arg) cur_time = odp_time_local(); } - VPRINT(" %02d: TxPkts %-8" PRIu64 " EnqFail %-6" PRIu64 - " AllocFail %-6" PRIu64 " Idle %" PRIu64 "ms\n", - thr_id, stats->s.tx_cnt, - stats->s.enq_failures, stats->s.alloc_failures, - odp_time_to_ns(stats->s.idle_ticks) / - (uint64_t)ODP_TIME_MSEC_IN_NS); + if (gbl_args->args.verbose) + printf(" %02d: TxPkts %-8" PRIu64 " EnqFail %-6" PRIu64 + " AllocFail %-6" PRIu64 " Idle %" PRIu64 "ms\n", + thr_id, stats->s.tx_cnt, stats->s.enq_failures, + stats->s.alloc_failures, + odp_time_to_ns(stats->s.idle_ticks) / + (uint64_t)ODP_TIME_MSEC_IN_NS); return 0; } diff --git a/test/performance/odp_sched_perf.c b/test/performance/odp_sched_perf.c index f9b4ce548..af3a11c97 100644 --- a/test/performance/odp_sched_perf.c +++ b/test/performance/odp_sched_perf.c @@ -40,6 +40,7 @@ typedef struct test_options_t { uint32_t max_burst; int queue_type; int forward; + int fairness; uint32_t queue_size; uint32_t tot_queue; uint32_t tot_event; @@ -88,6 +89,11 @@ typedef struct test_global_t { } test_global_t; +typedef struct { + odp_queue_t next; + odp_atomic_u64_t count; +} queue_context_t; + static void print_usage(void) { printf("\n" @@ -113,6 +119,7 @@ static void print_usage(void) " -b, --burst Maximum number of events per operation. Default: 100.\n" " -t, --type Queue type. 0: parallel, 1: atomic, 2: ordered. Default: 0.\n" " -f, --forward 0: Keep event in the original queue, 1: Forward event to the next queue. Default: 0.\n" + " -a, --fairness 0: Don't count events per queue, 1: Count and report events relative to average. Default: 0.\n" " -w, --wait_ns Number of nsec to wait before enqueueing events. Default: 0.\n" " -k, --ctx_rd_words Number of queue context words (uint64_t) to read on every event. Default: 0.\n" " -l, --ctx_rw_words Number of queue context words (uint64_t) to modify on every event. Default: 0.\n" @@ -142,6 +149,7 @@ static int parse_options(int argc, char *argv[], test_options_t *test_options) {"burst", required_argument, NULL, 'b'}, {"type", required_argument, NULL, 't'}, {"forward", required_argument, NULL, 'f'}, + {"fairness", required_argument, NULL, 'a'}, {"wait_ns", required_argument, NULL, 'w'}, {"ctx_rd_words", required_argument, NULL, 'k'}, {"ctx_rw_words", required_argument, NULL, 'l'}, @@ -152,7 +160,7 @@ static int parse_options(int argc, char *argv[], test_options_t *test_options) {NULL, 0, NULL, 0} }; - static const char *shortopts = "+c:q:L:H:d:e:s:g:j:b:t:f:w:k:l:n:m:vh"; + static const char *shortopts = "+c:q:L:H:d:e:s:g:j:b:t:f:a:w:k:l:n:m:vh"; test_options->num_cpu = 1; test_options->num_queue = 1; @@ -166,6 +174,7 @@ static int parse_options(int argc, char *argv[], test_options_t *test_options) test_options->max_burst = 100; test_options->queue_type = 0; test_options->forward = 0; + test_options->fairness = 0; test_options->ctx_rd_words = 0; test_options->ctx_rw_words = 0; test_options->rd_words = 0; @@ -216,6 +225,9 @@ static int parse_options(int argc, char *argv[], test_options_t *test_options) case 'f': test_options->forward = atoi(optarg); break; + case 'a': + test_options->fairness = atoi(optarg); + break; case 'k': test_options->ctx_rd_words = atoi(optarg); break; @@ -290,9 +302,11 @@ static int parse_options(int argc, char *argv[], test_options_t *test_options) /* When forwarding, all events may end up into * a single queue */ test_options->queue_size = test_options->tot_event; - ctx_size = sizeof(odp_queue_t); } + if (test_options->forward || test_options->fairness) + ctx_size = sizeof(queue_context_t); + if (test_options->ctx_rd_words || test_options->ctx_rw_words) { /* Round up queue handle size to a multiple of 8 for correct * context data alignment */ @@ -303,7 +317,7 @@ static int parse_options(int argc, char *argv[], test_options_t *test_options) /* When context data is modified, round up to cache line size to avoid * false sharing */ - if (test_options->ctx_rw_words) + if (test_options->fairness || test_options->ctx_rw_words) ctx_size = ROUNDUP(ctx_size, ODP_CACHE_LINE_SIZE); test_options->ctx_size = ctx_size; @@ -624,17 +638,24 @@ static int create_queues(test_global_t *global) queue = global->queue[i]; if (ctx_size) { + /* + * Cast increases alignment, but it's ok, since ctx and + * ctx_size are both cache line aligned. + */ + queue_context_t *qc = (queue_context_t *)(uintptr_t)ctx; + if (test_options->forward) { - odp_queue_t *next_queue; uint32_t next = i + 1; if (next == tot_queue) next = first; - next_queue = (odp_queue_t *)(uintptr_t)ctx; - *next_queue = global->queue[next]; + qc->next = global->queue[next]; } + if (test_options->fairness) + odp_atomic_init_u64(&qc->count, 0); + if (odp_queue_context_set(queue, ctx, ctx_size)) { printf("Error: Context set failed %u\n", i); return -1; @@ -699,6 +720,45 @@ static int join_all_groups(test_global_t *global, int thr) return 0; } +static void print_queue_fairness(test_global_t *global) +{ + uint32_t i; + queue_context_t *ctx; + test_options_t *test_options = &global->test_options; + uint32_t first = test_options->num_dummy; + uint32_t num_queue = test_options->num_queue; + uint32_t tot_queue = test_options->tot_queue; + uint64_t total = 0; + double average; + + if (!test_options->fairness) + return; + + for (i = first; i < tot_queue; i++) { + ctx = odp_queue_context(global->queue[i]); + total += odp_atomic_load_u64(&ctx->count); + } + + average = (double)total / (double)num_queue; + + printf("\n"); + printf("RESULTS - events per queue (percent of average):\n"); + printf("------------------------------------------------\n"); + printf(" 1 2 3 4 5 6 7 8 9 10"); + + for (i = first; i < tot_queue; i++) { + ctx = odp_queue_context(global->queue[i]); + + if ((i % 10) == 0) + printf("\n "); + + printf("%6.1f ", (double)odp_atomic_load_u64(&ctx->count) / + average * 100.0); + } + + printf("\n"); +} + static int destroy_queues(test_global_t *global) { uint32_t i; @@ -802,7 +862,6 @@ static int test_sched(void *arg) uint64_t events, enqueues, waits, events_prev; odp_time_t t1, t2, last_retry_ts; odp_queue_t queue; - odp_queue_t *next; thread_arg_t *thread_arg = arg; test_global_t *global = thread_arg->global; test_options_t *test_options = &global->test_options; @@ -810,6 +869,7 @@ static int test_sched(void *arg) uint32_t max_burst = test_options->max_burst; int num_group = test_options->num_group; int forward = test_options->forward; + int fairness = test_options->fairness; int touch_data = test_options->touch_data; uint32_t rd_words = test_options->rd_words; uint32_t rw_words = test_options->rw_words; @@ -826,8 +886,8 @@ static int test_sched(void *arg) thr = odp_thread_id(); - if (forward) - ctx_offset = ROUNDUP(sizeof(odp_queue_t), 8); + if (forward || fairness) + ctx_offset = ROUNDUP(sizeof(queue_context_t), 8); if (num_group > 0) { uint32_t num_join = test_options->num_join; @@ -885,12 +945,13 @@ static int test_sched(void *arg) i = 0; if (odp_unlikely(ctx_size)) { - void *ctx = odp_queue_context(queue); + queue_context_t *ctx = odp_queue_context(queue); - if (forward) { - next = ctx; - queue = *next; - } + if (forward) + queue = ctx->next; + + if (fairness) + odp_atomic_add_u64(&ctx->count, num); if (odp_unlikely(touch_ctx)) ctx_sum += rw_ctx_data(ctx, ctx_offset, @@ -999,10 +1060,8 @@ static int test_sched(void *arg) if (ev[0] == ODP_EVENT_INVALID) break; - if (odp_unlikely(forward)) { - next = odp_queue_context(queue); - queue = *next; - } + if (odp_unlikely(forward)) + queue = ((queue_context_t *)odp_queue_context(queue))->next; if (odp_queue_enq(queue, ev[0])) { printf("Error: Queue enqueue failed\n"); @@ -1276,6 +1335,8 @@ int main(int argc, char **argv) /* Wait workers to exit */ odph_thread_join(global->thread_tbl, global->test_options.num_cpu); + print_queue_fairness(global); + if (destroy_queues(global)) return -1; diff --git a/test/validation/api/crypto/odp_crypto_test_inp.c b/test/validation/api/crypto/odp_crypto_test_inp.c index 06ece9f50..a5987d2e7 100644 --- a/test/validation/api/crypto/odp_crypto_test_inp.c +++ b/test/validation/api/crypto/odp_crypto_test_inp.c @@ -710,6 +710,8 @@ static void check_alg(odp_crypto_op_t op, CU_ASSERT_FATAL(cipher_num > 0); CU_ASSERT_FATAL(auth_num > 0); + init_reference(ref, count); + odp_crypto_cipher_capability_t cipher_capa[cipher_num]; odp_crypto_auth_capability_t auth_capa[auth_num]; odp_bool_t cipher_tested[cipher_num]; diff --git a/test/validation/api/crypto/test_vectors.h b/test/validation/api/crypto/test_vectors.h index c377a5e0a..98fbcd343 100644 --- a/test/validation/api/crypto/test_vectors.h +++ b/test/validation/api/crypto/test_vectors.h @@ -1,4 +1,5 @@ /* Copyright (c) 2014-2018, Linaro Limited + * Copyright (c) 2021, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -10,6 +11,7 @@ #include "test_vectors_len.h" typedef struct crypto_test_reference_s { + uint8_t copy_previous_vector; /* does not copy digest_length */ uint32_t cipher_key_length; uint8_t cipher_key[MAX_KEY_LEN]; uint32_t auth_key_length; @@ -1461,6 +1463,10 @@ static crypto_test_reference_t aes_cmac_reference[] = { 0xf7, 0x9b, 0xdd, 0x9d, 0xd0, 0x4a, 0x28, 0x7c }, }, { + .copy_previous_vector = 1, + .digest_length = 12, + }, + { .auth_key_length = AES192_KEY_LEN, .auth_key = { 0x8e, 0x73, 0xb0, 0xf7, 0xda, 0x0e, 0x64, 0x52, 0xc8, 0x10, 0xf3, 0x2b, 0x80, 0x90, 0x79, 0xe5, @@ -1476,6 +1482,10 @@ static crypto_test_reference_t aes_cmac_reference[] = { 0x06, 0x62, 0xf6, 0x5e, 0x61, 0x7c, 0x51, 0x84 }, }, { + .copy_previous_vector = 1, + .digest_length = 12, + }, + { .auth_key_length = AES256_KEY_LEN, .auth_key = { 0x60, 0x3d, 0xeb, 0x10, 0x15, 0xca, 0x71, 0xbe, 0x2b, 0x73, 0xae, 0xf0, 0x85, 0x7d, 0x77, 0x81, @@ -1492,50 +1502,9 @@ static crypto_test_reference_t aes_cmac_reference[] = { 0xbd, 0x4b, 0xf2, 0x8d, 0x8c, 0x37, 0xc3, 0x5c }, }, { - .auth_key_length = AES128_KEY_LEN, - .auth_key = { 0x2b, 0x7e, 0x15, 0x16, 0x28, 0xae, 0xd2, 0xa6, - 0xab, 0xf7, 0x15, 0x88, 0x09, 0xcf, 0x4f, 0x3c }, - .length = 16, - .plaintext = { 0x6b, 0xc1, 0xbe, 0xe2, 0x2e, 0x40, 0x9f, 0x96, - 0xe9, 0x3d, 0x7e, 0x11, 0x73, 0x93, 0x17, 0x2a}, - .ciphertext = { 0x6b, 0xc1, 0xbe, 0xe2, 0x2e, 0x40, 0x9f, 0x96, - 0xe9, 0x3d, 0x7e, 0x11, 0x73, 0x93, 0x17, 0x2a - }, + .copy_previous_vector = 1, .digest_length = 12, - .digest = { 0x07, 0x0a, 0x16, 0xb4, 0x6b, 0x4d, 0x41, 0x44, - 0xf7, 0x9b, 0xdd, 0x9d }, }, - { - .auth_key_length = AES192_KEY_LEN, - .auth_key = { 0x8e, 0x73, 0xb0, 0xf7, 0xda, 0x0e, 0x64, 0x52, - 0xc8, 0x10, 0xf3, 0x2b, 0x80, 0x90, 0x79, 0xe5, - 0x62, 0xf8, 0xea, 0xd2, 0x52, 0x2c, 0x6b, 0x7b }, - .length = 16, - .plaintext = { 0x6b, 0xc1, 0xbe, 0xe2, 0x2e, 0x40, 0x9f, 0x96, - 0xe9, 0x3d, 0x7e, 0x11, 0x73, 0x93, 0x17, 0x2a}, - .ciphertext = { 0x6b, 0xc1, 0xbe, 0xe2, 0x2e, 0x40, 0x9f, 0x96, - 0xe9, 0x3d, 0x7e, 0x11, 0x73, 0x93, 0x17, 0x2a - }, - .digest_length = 12, - .digest = { 0x9e, 0x99, 0xa7, 0xbf, 0x31, 0xe7, 0x10, 0x90, - 0x06, 0x62, 0xf6, 0x5e }, - }, - { - .auth_key_length = AES256_KEY_LEN, - .auth_key = { 0x60, 0x3d, 0xeb, 0x10, 0x15, 0xca, 0x71, 0xbe, - 0x2b, 0x73, 0xae, 0xf0, 0x85, 0x7d, 0x77, 0x81, - 0x1f, 0x35, 0x2c, 0x07, 0x3b, 0x61, 0x08, 0xd7, - 0x2d, 0x98, 0x10, 0xa3, 0x09, 0x14, 0xdf, 0xf4 }, - .length = 16, - .plaintext = { 0x6b, 0xc1, 0xbe, 0xe2, 0x2e, 0x40, 0x9f, 0x96, - 0xe9, 0x3d, 0x7e, 0x11, 0x73, 0x93, 0x17, 0x2a}, - .ciphertext = { 0x6b, 0xc1, 0xbe, 0xe2, 0x2e, 0x40, 0x9f, 0x96, - 0xe9, 0x3d, 0x7e, 0x11, 0x73, 0x93, 0x17, 0x2a - }, - .digest_length = 12, - .digest = { 0x28, 0xa7, 0x02, 0x3f, 0x45, 0x2e, 0x8f, 0x82, - 0xbd, 0x4b, 0xf2, 0x8d }, - } }; /* @@ -1685,65 +1654,14 @@ static crypto_test_reference_t hmac_md5_reference[] = { /* "Hi There" */ .plaintext = { 0x48, 0x69, 0x20, 0x54, 0x68, 0x65, 0x72, 0x65}, .ciphertext = { 0x48, 0x69, 0x20, 0x54, 0x68, 0x65, 0x72, 0x65}, - .digest_length = HMAC_MD5_96_CHECK_LEN, + .digest_length = HMAC_MD5_CHECK_LEN, .digest = { 0x92, 0x94, 0x72, 0x7a, 0x36, 0x38, 0xbb, 0x1c, - 0x13, 0xf4, 0x8e, 0xf8 }, - - }, - { - .auth_key_length = HMAC_MD5_KEY_LEN, - /* "Jefe" */ - .auth_key = { 0x4a, 0x65, 0x66, 0x65 }, - .length = 28, - /* what do ya want for nothing?*/ - .plaintext = { 0x77, 0x68, 0x61, 0x74, 0x20, 0x64, 0x6f, 0x20, - 0x79, 0x61, 0x20, 0x77, 0x61, 0x6e, 0x74, 0x20, - 0x66, 0x6f, 0x72, 0x20, 0x6e, 0x6f, 0x74, 0x68, - 0x69, 0x6e, 0x67, 0x3f }, - .ciphertext = { 0x77, 0x68, 0x61, 0x74, 0x20, 0x64, 0x6f, 0x20, - 0x79, 0x61, 0x20, 0x77, 0x61, 0x6e, 0x74, 0x20, - 0x66, 0x6f, 0x72, 0x20, 0x6e, 0x6f, 0x74, 0x68, - 0x69, 0x6e, 0x67, 0x3f }, - .digest_length = HMAC_MD5_96_CHECK_LEN, - .digest = { 0x75, 0x0c, 0x78, 0x3e, 0x6a, 0xb0, 0xb5, 0x03, - 0xea, 0xa8, 0x6e, 0x31 }, + 0x13, 0xf4, 0x8e, 0xf8, 0x15, 0x8b, 0xfc, 0x9d }, }, { - .auth_key_length = HMAC_MD5_KEY_LEN, - .auth_key = { 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, - 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa }, - .length = 50, - .plaintext = { 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, - 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, - 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, - 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, - 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, - 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, - 0xdd, 0xdd }, - .ciphertext = { 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, - 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, - 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, - 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, - 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, - 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, - 0xdd, 0xdd }, + .copy_previous_vector = 1, .digest_length = HMAC_MD5_96_CHECK_LEN, - .digest = { 0x56, 0xbe, 0x34, 0x52, 0x1d, 0x14, 0x4c, 0x88, - 0xdb, 0xb8, 0xc7, 0x33 } - }, - { - .auth_key_length = HMAC_MD5_KEY_LEN, - .auth_key = { 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, - 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b }, - .length = 8, - /* "Hi There" */ - .plaintext = { 0x48, 0x69, 0x20, 0x54, 0x68, 0x65, 0x72, 0x65}, - .ciphertext = { 0x48, 0x69, 0x20, 0x54, 0x68, 0x65, 0x72, 0x65}, - .digest_length = HMAC_MD5_CHECK_LEN, - .digest = { 0x92, 0x94, 0x72, 0x7a, 0x36, 0x38, 0xbb, 0x1c, - 0x13, 0xf4, 0x8e, 0xf8, 0x15, 0x8b, 0xfc, 0x9d }, - }, { .auth_key_length = HMAC_MD5_KEY_LEN, @@ -1765,6 +1683,10 @@ static crypto_test_reference_t hmac_md5_reference[] = { }, { + .copy_previous_vector = 1, + .digest_length = HMAC_MD5_96_CHECK_LEN, + }, + { .auth_key_length = HMAC_MD5_KEY_LEN, .auth_key = { 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa }, @@ -1786,65 +1708,14 @@ static crypto_test_reference_t hmac_md5_reference[] = { .digest_length = HMAC_MD5_CHECK_LEN, .digest = { 0x56, 0xbe, 0x34, 0x52, 0x1d, 0x14, 0x4c, 0x88, 0xdb, 0xb8, 0xc7, 0x33, 0xf0, 0xe8, 0xb3, 0xf6 } - } -}; - -static crypto_test_reference_t hmac_sha1_reference[] = { - { - .auth_key_length = HMAC_SHA1_KEY_LEN, - .auth_key = { 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, - 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, - 0x0b, 0x0b, 0x0b, 0x0b }, - .length = 8, - /* "Hi There" */ - .plaintext = { 0x48, 0x69, 0x20, 0x54, 0x68, 0x65, 0x72, 0x65}, - .ciphertext = { 0x48, 0x69, 0x20, 0x54, 0x68, 0x65, 0x72, 0x65}, - .digest_length = HMAC_SHA1_96_CHECK_LEN, - .digest = { 0xb6, 0x17, 0x31, 0x86, 0x55, 0x05, - 0x72, 0x64, 0xe2, 0x8b, 0xc0, 0xb6 } - }, - { - .auth_key_length = HMAC_SHA1_KEY_LEN, - /* "Jefe" */ - .auth_key = { 0x4a, 0x65, 0x66, 0x65 }, - .length = 28, - /* what do ya want for nothing?*/ - .plaintext = { 0x77, 0x68, 0x61, 0x74, 0x20, 0x64, 0x6f, 0x20, - 0x79, 0x61, 0x20, 0x77, 0x61, 0x6e, 0x74, 0x20, - 0x66, 0x6f, 0x72, 0x20, 0x6e, 0x6f, 0x74, 0x68, - 0x69, 0x6e, 0x67, 0x3f }, - .ciphertext = { 0x77, 0x68, 0x61, 0x74, 0x20, 0x64, 0x6f, 0x20, - 0x79, 0x61, 0x20, 0x77, 0x61, 0x6e, 0x74, 0x20, - 0x66, 0x6f, 0x72, 0x20, 0x6e, 0x6f, 0x74, 0x68, - 0x69, 0x6e, 0x67, 0x3f }, - .digest_length = HMAC_SHA1_96_CHECK_LEN, - .digest = { 0xef, 0xfc, 0xdf, 0x6a, 0xe5, 0xeb, - 0x2f, 0xa2, 0xd2, 0x74, 0x16, 0xd5 } }, { - .auth_key_length = HMAC_SHA1_KEY_LEN, - .auth_key = { 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, - 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, - 0xaa, 0xaa, 0xaa, 0xaa }, - .length = 50, - .plaintext = { 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, - 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, - 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, - 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, - 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, - 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, - 0xdd, 0xdd }, - .ciphertext = { 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, - 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, - 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, - 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, - 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, - 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, - 0xdd, 0xdd }, - .digest_length = HMAC_SHA1_96_CHECK_LEN, - .digest = { 0x12, 0x5d, 0x73, 0x42, 0xb9, 0xac, - 0x11, 0xcd, 0x91, 0xa3, 0x9a, 0xf4 } + .copy_previous_vector = 1, + .digest_length = HMAC_MD5_96_CHECK_LEN, }, +}; + +static crypto_test_reference_t hmac_sha1_reference[] = { { .auth_key_length = HMAC_SHA1_KEY_LEN, .auth_key = { 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, @@ -1860,6 +1731,10 @@ static crypto_test_reference_t hmac_sha1_reference[] = { 0xf1, 0x46, 0xbe, 0x00 } }, { + .copy_previous_vector = 1, + .digest_length = HMAC_SHA1_96_CHECK_LEN, + }, + { .auth_key_length = HMAC_SHA1_KEY_LEN, /* "Jefe" */ .auth_key = { 0x4a, 0x65, 0x66, 0x65 }, @@ -1879,6 +1754,10 @@ static crypto_test_reference_t hmac_sha1_reference[] = { 0x25, 0x9a, 0x7c, 0x79 } }, { + .copy_previous_vector = 1, + .digest_length = HMAC_SHA1_96_CHECK_LEN, + }, + { .auth_key_length = HMAC_SHA1_KEY_LEN, .auth_key = { 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, @@ -1902,7 +1781,11 @@ static crypto_test_reference_t hmac_sha1_reference[] = { .digest = { 0x12, 0x5d, 0x73, 0x42, 0xb9, 0xac, 0x11, 0xcd, 0x91, 0xa3, 0x9a, 0xf4, 0x8a, 0xa1, 0x7b, 0x4f, 0x63, 0xf1, 0x75, 0xd3 } - } + }, + { + .copy_previous_vector = 1, + .digest_length = HMAC_SHA1_96_CHECK_LEN, + }, }; static crypto_test_reference_t hmac_sha224_reference[] = { @@ -1979,61 +1862,6 @@ static crypto_test_reference_t hmac_sha256_reference[] = { /* "Hi There" */ .plaintext = { 0x48, 0x69, 0x20, 0x54, 0x68, 0x65, 0x72, 0x65}, .ciphertext = { 0x48, 0x69, 0x20, 0x54, 0x68, 0x65, 0x72, 0x65}, - .digest_length = HMAC_SHA256_128_CHECK_LEN, - .digest = { 0xb0, 0x34, 0x4c, 0x61, 0xd8, 0xdb, 0x38, 0x53, - 0x5c, 0xa8, 0xaf, 0xce, 0xaf, 0x0b, 0xf1, 0x2b } - }, - { - .auth_key_length = HMAC_SHA256_KEY_LEN, - /* "Jefe" */ - .auth_key = { 0x4a, 0x65, 0x66, 0x65 }, - .length = 28, - /* what do ya want for nothing?*/ - .plaintext = { 0x77, 0x68, 0x61, 0x74, 0x20, 0x64, 0x6f, 0x20, - 0x79, 0x61, 0x20, 0x77, 0x61, 0x6e, 0x74, 0x20, - 0x66, 0x6f, 0x72, 0x20, 0x6e, 0x6f, 0x74, 0x68, - 0x69, 0x6e, 0x67, 0x3f }, - .ciphertext = { 0x77, 0x68, 0x61, 0x74, 0x20, 0x64, 0x6f, 0x20, - 0x79, 0x61, 0x20, 0x77, 0x61, 0x6e, 0x74, 0x20, - 0x66, 0x6f, 0x72, 0x20, 0x6e, 0x6f, 0x74, 0x68, - 0x69, 0x6e, 0x67, 0x3f }, - .digest_length = HMAC_SHA256_128_CHECK_LEN, - .digest = { 0x5b, 0xdc, 0xc1, 0x46, 0xbf, 0x60, 0x75, 0x4e, - 0x6a, 0x04, 0x24, 0x26, 0x08, 0x95, 0x75, 0xc7 } - }, - { - .auth_key_length = HMAC_SHA256_KEY_LEN, - .auth_key = { 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, - 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, - 0xaa, 0xaa, 0xaa, 0xaa }, - .length = 50, - .plaintext = { 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, - 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, - 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, - 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, - 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, - 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, - 0xdd, 0xdd }, - .ciphertext = { 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, - 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, - 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, - 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, - 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, - 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, - 0xdd, 0xdd }, - .digest_length = HMAC_SHA256_128_CHECK_LEN, - .digest = { 0x77, 0x3e, 0xa9, 0x1e, 0x36, 0x80, 0x0e, 0x46, - 0x85, 0x4d, 0xb8, 0xeb, 0xd0, 0x91, 0x81, 0xa7 } - }, - { - .auth_key_length = HMAC_SHA256_KEY_LEN, - .auth_key = { 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, - 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, - 0x0b, 0x0b, 0x0b, 0x0b }, - .length = 8, - /* "Hi There" */ - .plaintext = { 0x48, 0x69, 0x20, 0x54, 0x68, 0x65, 0x72, 0x65}, - .ciphertext = { 0x48, 0x69, 0x20, 0x54, 0x68, 0x65, 0x72, 0x65}, .digest_length = HMAC_SHA256_CHECK_LEN, .digest = { 0xb0, 0x34, 0x4c, 0x61, 0xd8, 0xdb, 0x38, 0x53, 0x5c, 0xa8, 0xaf, 0xce, 0xaf, 0x0b, 0xf1, 0x2b, @@ -2041,6 +1869,10 @@ static crypto_test_reference_t hmac_sha256_reference[] = { 0x26, 0xe9, 0x37, 0x6c, 0x2e, 0x32, 0xcf, 0xf7 } }, { + .copy_previous_vector = 1, + .digest_length = HMAC_SHA256_128_CHECK_LEN, + }, + { .auth_key_length = HMAC_SHA256_KEY_LEN, /* "Jefe" */ .auth_key = { 0x4a, 0x65, 0x66, 0x65 }, @@ -2061,6 +1893,10 @@ static crypto_test_reference_t hmac_sha256_reference[] = { 0x9d, 0xec, 0x58, 0xb9, 0x64, 0xec, 0x38, 0x43 } }, { + .copy_previous_vector = 1, + .digest_length = HMAC_SHA256_128_CHECK_LEN, + }, + { .auth_key_length = HMAC_SHA256_KEY_LEN, .auth_key = { 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, @@ -2085,68 +1921,14 @@ static crypto_test_reference_t hmac_sha256_reference[] = { 0x85, 0x4d, 0xb8, 0xeb, 0xd0, 0x91, 0x81, 0xa7, 0x29, 0x59, 0x09, 0x8b, 0x3e, 0xf8, 0xc1, 0x22, 0xd9, 0x63, 0x55, 0x14, 0xce, 0xd5, 0x65, 0xfe } - } -}; - -static crypto_test_reference_t hmac_sha384_reference[] = { - { - .auth_key_length = HMAC_SHA384_KEY_LEN, - .auth_key = { 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, - 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, - 0x0b, 0x0b, 0x0b, 0x0b }, - .length = 8, - /* "Hi There" */ - .plaintext = { 0x48, 0x69, 0x20, 0x54, 0x68, 0x65, 0x72, 0x65}, - .ciphertext = { 0x48, 0x69, 0x20, 0x54, 0x68, 0x65, 0x72, 0x65}, - .digest_length = HMAC_SHA384_192_CHECK_LEN, - .digest = { 0xaf, 0xd0, 0x39, 0x44, 0xd8, 0x48, 0x95, 0x62, - 0x6b, 0x08, 0x25, 0xf4, 0xab, 0x46, 0x90, 0x7f, - 0x15, 0xf9, 0xda, 0xdb, 0xe4, 0x10, 0x1e, 0xc6 } }, { - .auth_key_length = HMAC_SHA384_KEY_LEN, - /* "Jefe" */ - .auth_key = { 0x4a, 0x65, 0x66, 0x65 }, - .length = 28, - /* what do ya want for nothing?*/ - .plaintext = { 0x77, 0x68, 0x61, 0x74, 0x20, 0x64, 0x6f, 0x20, - 0x79, 0x61, 0x20, 0x77, 0x61, 0x6e, 0x74, 0x20, - 0x66, 0x6f, 0x72, 0x20, 0x6e, 0x6f, 0x74, 0x68, - 0x69, 0x6e, 0x67, 0x3f }, - .ciphertext = { 0x77, 0x68, 0x61, 0x74, 0x20, 0x64, 0x6f, 0x20, - 0x79, 0x61, 0x20, 0x77, 0x61, 0x6e, 0x74, 0x20, - 0x66, 0x6f, 0x72, 0x20, 0x6e, 0x6f, 0x74, 0x68, - 0x69, 0x6e, 0x67, 0x3f }, - .digest_length = HMAC_SHA384_192_CHECK_LEN, - .digest = { 0xaf, 0x45, 0xd2, 0xe3, 0x76, 0x48, 0x40, 0x31, - 0x61, 0x7f, 0x78, 0xd2, 0xb5, 0x8a, 0x6b, 0x1b, - 0x9c, 0x7e, 0xf4, 0x64, 0xf5, 0xa0, 0x1b, 0x47 } - }, - { - .auth_key_length = HMAC_SHA384_KEY_LEN, - .auth_key = { 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, - 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, - 0xaa, 0xaa, 0xaa, 0xaa }, - .length = 50, - .plaintext = { 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, - 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, - 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, - 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, - 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, - 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, - 0xdd, 0xdd }, - .ciphertext = { 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, - 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, - 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, - 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, - 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, - 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, - 0xdd, 0xdd }, - .digest_length = HMAC_SHA384_192_CHECK_LEN, - .digest = { 0x88, 0x06, 0x26, 0x08, 0xd3, 0xe6, 0xad, 0x8a, - 0x0a, 0xa2, 0xac, 0xe0, 0x14, 0xc8, 0xa8, 0x6f, - 0x0a, 0xa6, 0x35, 0xd9, 0x47, 0xac, 0x9f, 0xeb } + .copy_previous_vector = 1, + .digest_length = HMAC_SHA256_128_CHECK_LEN, }, +}; + +static crypto_test_reference_t hmac_sha384_reference[] = { { .auth_key_length = HMAC_SHA384_KEY_LEN, .auth_key = { 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, @@ -2165,6 +1947,10 @@ static crypto_test_reference_t hmac_sha384_reference[] = { 0x4a, 0xf1, 0x52, 0xe8, 0xb2, 0xfa, 0x9c, 0xb6 } }, { + .copy_previous_vector = 1, + .digest_length = HMAC_SHA384_192_CHECK_LEN, + }, + { .auth_key_length = HMAC_SHA384_KEY_LEN, /* "Jefe" */ .auth_key = { 0x4a, 0x65, 0x66, 0x65 }, @@ -2187,6 +1973,10 @@ static crypto_test_reference_t hmac_sha384_reference[] = { 0x8b, 0x32, 0x39, 0xec, 0xfa, 0xb2, 0x16, 0x49 } }, { + .copy_previous_vector = 1, + .digest_length = HMAC_SHA384_192_CHECK_LEN, + }, + { .auth_key_length = HMAC_SHA384_KEY_LEN, .auth_key = { 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, @@ -2213,71 +2003,14 @@ static crypto_test_reference_t hmac_sha384_reference[] = { 0xe8, 0x3e, 0xf4, 0xe5, 0x59, 0x66, 0x14, 0x4b, 0x2a, 0x5a, 0xb3, 0x9d, 0xc1, 0x38, 0x14, 0xb9, 0x4e, 0x3a, 0xb6, 0xe1, 0x01, 0xa3, 0x4f, 0x27 } - } -}; - -static crypto_test_reference_t hmac_sha512_reference[] = { - { - .auth_key_length = HMAC_SHA512_KEY_LEN, - .auth_key = { 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, - 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, - 0x0b, 0x0b, 0x0b, 0x0b }, - .length = 8, - /* "Hi There" */ - .plaintext = { 0x48, 0x69, 0x20, 0x54, 0x68, 0x65, 0x72, 0x65}, - .ciphertext = { 0x48, 0x69, 0x20, 0x54, 0x68, 0x65, 0x72, 0x65}, - .digest_length = HMAC_SHA512_256_CHECK_LEN, - .digest = { 0x87, 0xaa, 0x7c, 0xde, 0xa5, 0xef, 0x61, 0x9d, - 0x4f, 0xf0, 0xb4, 0x24, 0x1a, 0x1d, 0x6c, 0xb0, - 0x23, 0x79, 0xf4, 0xe2, 0xce, 0x4e, 0xc2, 0x78, - 0x7a, 0xd0, 0xb3, 0x05, 0x45, 0xe1, 0x7c, 0xde } }, { - .auth_key_length = HMAC_SHA512_KEY_LEN, - /* "Jefe" */ - .auth_key = { 0x4a, 0x65, 0x66, 0x65 }, - .length = 28, - /* what do ya want for nothing?*/ - .plaintext = { 0x77, 0x68, 0x61, 0x74, 0x20, 0x64, 0x6f, 0x20, - 0x79, 0x61, 0x20, 0x77, 0x61, 0x6e, 0x74, 0x20, - 0x66, 0x6f, 0x72, 0x20, 0x6e, 0x6f, 0x74, 0x68, - 0x69, 0x6e, 0x67, 0x3f }, - .ciphertext = { 0x77, 0x68, 0x61, 0x74, 0x20, 0x64, 0x6f, 0x20, - 0x79, 0x61, 0x20, 0x77, 0x61, 0x6e, 0x74, 0x20, - 0x66, 0x6f, 0x72, 0x20, 0x6e, 0x6f, 0x74, 0x68, - 0x69, 0x6e, 0x67, 0x3f }, - .digest_length = HMAC_SHA512_256_CHECK_LEN, - .digest = { 0x16, 0x4b, 0x7a, 0x7b, 0xfc, 0xf8, 0x19, 0xe2, - 0xe3, 0x95, 0xfb, 0xe7, 0x3b, 0x56, 0xe0, 0xa3, - 0x87, 0xbd, 0x64, 0x22, 0x2e, 0x83, 0x1f, 0xd6, - 0x10, 0x27, 0x0c, 0xd7, 0xea, 0x25, 0x05, 0x54 } - }, - { - .auth_key_length = HMAC_SHA512_KEY_LEN, - .auth_key = { 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, - 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, - 0xaa, 0xaa, 0xaa, 0xaa }, - .length = 50, - .plaintext = { 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, - 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, - 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, - 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, - 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, - 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, - 0xdd, 0xdd }, - .ciphertext = { 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, - 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, - 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, - 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, - 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, - 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, - 0xdd, 0xdd }, - .digest_length = HMAC_SHA512_256_CHECK_LEN, - .digest = { 0xfa, 0x73, 0xb0, 0x08, 0x9d, 0x56, 0xa2, 0x84, - 0xef, 0xb0, 0xf0, 0x75, 0x6c, 0x89, 0x0b, 0xe9, - 0xb1, 0xb5, 0xdb, 0xdd, 0x8e, 0xe8, 0x1a, 0x36, - 0x55, 0xf8, 0x3e, 0x33, 0xb2, 0x27, 0x9d, 0x39 } + .copy_previous_vector = 1, + .digest_length = HMAC_SHA384_192_CHECK_LEN, }, +}; + +static crypto_test_reference_t hmac_sha512_reference[] = { { .auth_key_length = HMAC_SHA512_KEY_LEN, .auth_key = { 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, @@ -2298,6 +2031,10 @@ static crypto_test_reference_t hmac_sha512_reference[] = { 0x2e, 0x69, 0x6c, 0x20, 0x3a, 0x12, 0x68, 0x54 } }, { + .copy_previous_vector = 1, + .digest_length = HMAC_SHA512_256_CHECK_LEN, + }, + { .auth_key_length = HMAC_SHA512_KEY_LEN, /* "Jefe" */ .auth_key = { 0x4a, 0x65, 0x66, 0x65 }, @@ -2322,6 +2059,10 @@ static crypto_test_reference_t hmac_sha512_reference[] = { 0x63, 0x6e, 0x07, 0x0a, 0x38, 0xbc, 0xe7, 0x37 } }, { + .copy_previous_vector = 1, + .digest_length = HMAC_SHA512_256_CHECK_LEN, + }, + { .auth_key_length = HMAC_SHA512_KEY_LEN, .auth_key = { 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, @@ -2350,49 +2091,32 @@ static crypto_test_reference_t hmac_sha512_reference[] = { 0x06, 0xb4, 0x85, 0xa4, 0x7e, 0x67, 0xc8, 0x07, 0xb9, 0x46, 0xa3, 0x37, 0xbe, 0xe8, 0x94, 0x26, 0x74, 0x27, 0x88, 0x59, 0xe1, 0x32, 0x92, 0xfb } - } + }, + { + .copy_previous_vector = 1, + .digest_length = HMAC_SHA512_256_CHECK_LEN, + }, }; +/* + * RFC 3566 + */ static crypto_test_reference_t aes_xcbc_reference[] = { + /* Test Case #1 */ { .auth_key_length = AES_XCBC_MAC_KEY_LEN, - .auth_key = { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, - 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f }, - .length = 3, - .plaintext = { 0x00, 0x01, 0x02 }, - .ciphertext = { 0x00, 0x01, 0x02 }, - .digest_length = AES_XCBC_MAC_96_CHECK_LEN, - .digest = { 0x5b, 0x37, 0x65, 0x80, 0xae, 0x2f, 0x19, 0xaf, - 0xe7, 0x21, 0x9c, 0xee } + .auth_key = {0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f}, + .length = 0, + .digest_length = AES_XCBC_MAC_CHECK_LEN, + .digest = {0x75, 0xf0, 0x25, 0x1d, 0x52, 0x8a, 0xc0, 0x1c, + 0x45, 0x73, 0xdf, 0xd5, 0x84, 0xd7, 0x9f, 0x29} }, { - .auth_key_length = AES_XCBC_MAC_KEY_LEN, - .auth_key = { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, - 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f }, - .length = 16, - .plaintext = { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, - 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f }, - .ciphertext = { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, - 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f }, + .copy_previous_vector = 1, .digest_length = AES_XCBC_MAC_96_CHECK_LEN, - .digest = { 0xd2, 0xa2, 0x46, 0xfa, 0x34, 0x9b, 0x68, 0xa7, - 0x99, 0x98, 0xa4, 0x39 } - }, - { - .auth_key_length = AES_XCBC_MAC_KEY_LEN, - .auth_key = { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, - 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f }, - .length = 20, - .plaintext = { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, - 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, - 0x10, 0x11, 0x12, 0x13 }, - .ciphertext = { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, - 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, - 0x10, 0x11, 0x12, 0x13 }, - .digest_length = AES_XCBC_MAC_96_CHECK_LEN, - .digest = { 0x47, 0xf5, 0x1b, 0x45, 0x64, 0x96, 0x62, 0x15, - 0xb8, 0x98, 0x5c, 0x63 } }, + /* Test Case #2 */ { .auth_key_length = AES_XCBC_MAC_KEY_LEN, .auth_key = { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, @@ -2405,6 +2129,11 @@ static crypto_test_reference_t aes_xcbc_reference[] = { 0xe7, 0x21, 0x9c, 0xee, 0xf1, 0x72, 0x75, 0x6f } }, { + .copy_previous_vector = 1, + .digest_length = AES_XCBC_MAC_96_CHECK_LEN, + }, + /* Test Case #3 */ + { .auth_key_length = AES_XCBC_MAC_KEY_LEN, .auth_key = { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f }, @@ -2418,6 +2147,11 @@ static crypto_test_reference_t aes_xcbc_reference[] = { 0x99, 0x98, 0xa4, 0x39, 0x4f, 0xf7, 0xa2, 0x63 } }, { + .copy_previous_vector = 1, + .digest_length = AES_XCBC_MAC_96_CHECK_LEN, + }, + /* Test Case #4 */ + { .auth_key_length = AES_XCBC_MAC_KEY_LEN, .auth_key = { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f }, @@ -2431,7 +2165,72 @@ static crypto_test_reference_t aes_xcbc_reference[] = { .digest_length = AES_XCBC_MAC_CHECK_LEN, .digest = { 0x47, 0xf5, 0x1b, 0x45, 0x64, 0x96, 0x62, 0x15, 0xb8, 0x98, 0x5c, 0x63, 0x05, 0x5e, 0xd3, 0x08 } - } + }, + { + .copy_previous_vector = 1, + .digest_length = AES_XCBC_MAC_96_CHECK_LEN, + }, + /* Test Case #5 */ + { + .auth_key_length = AES_XCBC_MAC_KEY_LEN, + .auth_key = {0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f}, + .length = 32, + .plaintext = {0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f}, + .ciphertext = {0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f}, + .digest_length = AES_XCBC_MAC_CHECK_LEN, + .digest = {0xf5, 0x4f, 0x0e, 0xc8, 0xd2, 0xb9, 0xf3, 0xd3, + 0x68, 0x07, 0x73, 0x4b, 0xd5, 0x28, 0x3f, 0xd4} + }, + { + .copy_previous_vector = 1, + .digest_length = AES_XCBC_MAC_96_CHECK_LEN, + }, + /* Test Case #6 */ + { + .auth_key_length = AES_XCBC_MAC_KEY_LEN, + .auth_key = {0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f}, + .length = 34, + .plaintext = {0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, + 0x20, 0x21}, + .ciphertext = {0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, + 0x20, 0x21}, + .digest_length = AES_XCBC_MAC_CHECK_LEN, + .digest = {0xbe, 0xcb, 0xb3, 0xbc, 0xcd, 0xb5, 0x18, 0xa3, + 0x06, 0x77, 0xd5, 0x48, 0x1f, 0xb6, 0xb4, 0xd8} + }, + { + .copy_previous_vector = 1, + .digest_length = AES_XCBC_MAC_96_CHECK_LEN, + }, + /* Test Case #7 */ + { + .auth_key_length = AES_XCBC_MAC_KEY_LEN, + .auth_key = {0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f}, + .length = 1000, + /* Plaintext is 1000 zero bytes. No explicit init needed. */ + .digest_length = AES_XCBC_MAC_CHECK_LEN, + .digest = {0xf0, 0xda, 0xfe, 0xe8, 0x95, 0xdb, 0x30, 0x25, + 0x37, 0x61, 0x10, 0x3b, 0x5d, 0x84, 0x52, 0x8f} + }, + { + .copy_previous_vector = 1, + .digest_length = AES_XCBC_MAC_96_CHECK_LEN, + }, }; /* @@ -2941,4 +2740,21 @@ static crypto_test_reference_t sha512_reference[] = { } }; +static void init_reference(crypto_test_reference_t *ref, int size) +{ + int n; + crypto_test_reference_t *prev = NULL; + + for (n = 0; n < size; n++) { + if (prev && ref[n].copy_previous_vector) { + uint32_t len; + + len = ref[n].digest_length; + ref[n] = *prev; + ref[n].digest_length = len; + } + prev = &ref[n]; + } +} + #endif diff --git a/test/validation/api/crypto/test_vectors_len.h b/test/validation/api/crypto/test_vectors_len.h index 17121e12e..9edf2999e 100644 --- a/test/validation/api/crypto/test_vectors_len.h +++ b/test/validation/api/crypto/test_vectors_len.h @@ -9,7 +9,7 @@ /* Maximum */ #define MAX_KEY_LEN 64 #define MAX_IV_LEN 16 -#define MAX_DATA_LEN 715 +#define MAX_DATA_LEN 1000 #define MAX_AAD_LEN 12 #define MAX_DIGEST_LEN 64 diff --git a/test/validation/api/init/init_main.c b/test/validation/api/init/init_main.c index c335e10a3..f1716db25 100644 --- a/test/validation/api/init/init_main.c +++ b/test/validation/api/init/init_main.c @@ -1,5 +1,5 @@ /* Copyright (c) 2015-2018, Linaro Limited - * Copyright (c) 2019, Nokia + * Copyright (c) 2019-2021, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -11,6 +11,7 @@ #include <stdarg.h> #include <stdlib.h> #include <stdio.h> +#include <string.h> /* Replacement abort function */ static void ODP_NORETURN my_abort_func(void) @@ -51,6 +52,7 @@ static void init_test_defaults(void) { int ret; odp_instance_t instance; + odp_instance_t current_instance; odp_init_t param; odp_init_param_init(¶m); @@ -61,6 +63,9 @@ static void init_test_defaults(void) ret = odp_init_local(instance, ODP_THREAD_WORKER); CU_ASSERT_FATAL(ret == 0); + CU_ASSERT_FATAL(odp_instance(¤t_instance) == 0); + CU_ASSERT(memcmp(¤t_instance, &instance, sizeof(odp_instance_t)) == 0); + ret = odp_term_local(); CU_ASSERT_FATAL(ret == 0); diff --git a/test/validation/api/ipsec/ipsec.c b/test/validation/api/ipsec/ipsec.c index c6fbe6f91..cd1824fb1 100644 --- a/test/validation/api/ipsec/ipsec.c +++ b/test/validation/api/ipsec/ipsec.c @@ -329,6 +329,12 @@ int ipsec_check_esp_aes_gcm_128_reass_ipv6(void) return ODP_TEST_INACTIVE; } +int ipsec_check_esp_null_aes_xcbc(void) +{ + return ipsec_check_esp(ODP_CIPHER_ALG_NULL, 0, + ODP_AUTH_ALG_AES_XCBC_MAC, 128); +} + void ipsec_sa_param_fill(odp_ipsec_sa_param_t *param, odp_bool_t in, odp_bool_t ah, @@ -949,9 +955,15 @@ static void verify_in(const ipsec_test_part *part, CU_ASSERT_EQUAL(IPSEC_SA_CTX, odp_ipsec_sa_context(sa)); if (suite_context.inbound_op_mode != ODP_IPSEC_OP_MODE_SYNC) { - uint32_t len = part->pkt_in->len - part->pkt_in->l3_offset; + uint32_t len; - CU_ASSERT(result.orig_ip_len == len); + if (part->out[i].orig_ip_len) + len = part->out[i].orig_ip_len; + else + len = part->pkt_in->len - part->pkt_in->l3_offset; + + CU_ASSERT(result.orig_ip_len == 0 || + result.orig_ip_len == len); } } ipsec_check_packet(part->out[i].pkt_res, @@ -991,11 +1003,7 @@ static void parse_ip(odp_packet_t pkt) .proto = proto, .last_layer = ODP_PROTO_LAYER_L4, }; - /* - * odp_packet_parse() is buggy in linux generic ODP. Intentionally - * ignore the return value until the bug has been fixed. - */ - (void)odp_packet_parse(pkt, l3, ¶m); + CU_ASSERT(odp_packet_parse(pkt, l3, ¶m) == 0); } int ipsec_check_out(const ipsec_test_part *part, odp_ipsec_sa_t sa, diff --git a/test/validation/api/ipsec/ipsec.h b/test/validation/api/ipsec/ipsec.h index 6edce03d0..cbb533913 100644 --- a/test/validation/api/ipsec/ipsec.h +++ b/test/validation/api/ipsec/ipsec.h @@ -96,6 +96,11 @@ typedef struct { odp_proto_l3_type_t l3_type; odp_proto_l4_type_t l4_type; uint32_t seq_num; + /* + * Expected original IP length. Non zero only when expected len + * differs from that of input test packet (pkt_in). + */ + uint32_t orig_ip_len; } out[MAX_FRAGS]; } ipsec_test_part; @@ -149,5 +154,6 @@ int ipsec_check_esp_chacha20_poly1305(void); int ipsec_check_test_sa_update_seq_num(void); int ipsec_check_esp_aes_gcm_128_reass_ipv4(void); int ipsec_check_esp_aes_gcm_128_reass_ipv6(void); +int ipsec_check_esp_null_aes_xcbc(void); #endif diff --git a/test/validation/api/ipsec/ipsec_test_in.c b/test/validation/api/ipsec/ipsec_test_in.c index bcdcd4e83..508d7aaa5 100644 --- a/test/validation/api/ipsec/ipsec_test_in.c +++ b/test/validation/api/ipsec/ipsec_test_in.c @@ -1777,6 +1777,7 @@ static void test_multi_out_in(odp_ipsec_sa_t out_sa, { uint8_t ver_ihl = result_packet->data[result_packet->l3_offset]; odp_bool_t is_result_ipv6 = (ODPH_IPV4HDR_VER(ver_ihl) == ODPH_IPV6); + uint32_t orig_ip_len = 0; int i; for (i = 0; i < num_input_packets; i++) { @@ -1784,6 +1785,7 @@ static void test_multi_out_in(odp_ipsec_sa_t out_sa, ipsec_test_part test_in; ipsec_test_packet test_pkt; odp_packet_t pkt = ODP_PACKET_INVALID; + uint32_t l3_off, pkt_len; /* * Convert plain text packet to IPsec packet through @@ -1798,11 +1800,26 @@ static void test_multi_out_in(odp_ipsec_sa_t out_sa, * Expect result packet only for the last packet. */ memset(&test_in, 0, sizeof(test_in)); + + /* + * In case of complete reassembly, the original IP length is the + * sum of IP lengths of the ESP packets that contained the + * individual fragments. + */ + if (reass_status == ODP_PACKET_REASS_COMPLETE) { + pkt_len = odp_packet_len(pkt); + l3_off = odp_packet_l3_offset(pkt); + CU_ASSERT(ODP_PACKET_OFFSET_INVALID != l3_off) + + orig_ip_len += pkt_len - l3_off; + } + if (i == num_input_packets - 1) { part_prep_plain(&test_in, 1, is_result_ipv6, true); test_in.out[0].pkt_res = result_packet; test_in.out[0].reass_status = reass_status; test_in.out[0].num_frags = num_input_packets; + test_in.out[0].orig_ip_len = orig_ip_len; } ipsec_test_packet_from_pkt(&test_pkt, &pkt); test_in.pkt_in = &test_pkt; @@ -2192,6 +2209,42 @@ static void test_in_ipv6_esp_reass_incomp(void) ipsec_sa_destroy(out_sa); } +static void test_in_ipv4_null_aes_xcbc_esp(void) +{ + odp_ipsec_tunnel_param_t tunnel; + odp_ipsec_sa_param_t param; + odp_ipsec_sa_t sa; + + memset(&tunnel, 0, sizeof(odp_ipsec_tunnel_param_t)); + + ipsec_sa_param_fill(¶m, + true, false, 0x100, &tunnel, + ODP_CIPHER_ALG_NULL, NULL, + ODP_AUTH_ALG_AES_XCBC_MAC, &key_auth_aes_xcbc_128, + NULL, NULL); + + sa = odp_ipsec_sa_create(¶m); + + CU_ASSERT_NOT_EQUAL_FATAL(ODP_IPSEC_SA_INVALID, sa); + + ipsec_test_part test = { + .pkt_in = &pkt_ipv4_null_aes_xcbc_esp, + .num_pkt = 1, + .out = { + { .status.warn.all = 0, + .status.error.all = 0, + .l3_type = ODP_PROTO_L3_TYPE_IPV4, + .l4_type = ODP_PROTO_L4_TYPE_UDP, + .pkt_res = &pkt_ipv4_null_aes_xcbc_plain, + }, + }, + }; + + ipsec_check_in_one(&test, sa); + + ipsec_sa_destroy(sa); +} + static void ipsec_test_capability(void) { odp_ipsec_capability_t capa; @@ -2312,5 +2365,7 @@ odp_testinfo_t ipsec_in_suite[] = { ipsec_check_esp_aes_gcm_128_reass_ipv6), ODP_TEST_INFO_CONDITIONAL(test_in_ipv6_esp_reass_incomp, ipsec_check_esp_aes_gcm_128_reass_ipv6), + ODP_TEST_INFO_CONDITIONAL(test_in_ipv4_null_aes_xcbc_esp, + ipsec_check_esp_null_aes_xcbc), ODP_TEST_INFO_NULL, }; diff --git a/test/validation/api/ipsec/ipsec_test_out.c b/test/validation/api/ipsec/ipsec_test_out.c index 733db10b9..aab480bdd 100644 --- a/test/validation/api/ipsec/ipsec_test_out.c +++ b/test/validation/api/ipsec/ipsec_test_out.c @@ -1377,6 +1377,50 @@ static void test_out_ipv4_udp_esp_null_sha256(void) ipsec_sa_destroy(sa); } +static void test_out_ipv4_null_aes_xcbc(void) +{ + odp_ipsec_tunnel_param_t tunnel; + odp_ipsec_sa_param_t param; + odp_ipsec_sa_t sa; + + memset(&tunnel, 0, sizeof(odp_ipsec_tunnel_param_t)); + uint32_t src = IPV4ADDR(10, 0, 111, 2); + uint32_t dst = IPV4ADDR(10, 0, 222, 2); + + memset(&tunnel, 0, sizeof(odp_ipsec_tunnel_param_t)); + tunnel.type = ODP_IPSEC_TUNNEL_IPV4; + tunnel.ipv4.src_addr = &src; + tunnel.ipv4.dst_addr = &dst; + tunnel.ipv4.ttl = 64; + + ipsec_sa_param_fill(¶m, + false, false, 0x100, &tunnel, + ODP_CIPHER_ALG_NULL, NULL, + ODP_AUTH_ALG_AES_XCBC_MAC, &key_auth_aes_xcbc_128, + NULL, NULL); + + sa = odp_ipsec_sa_create(¶m); + + CU_ASSERT_NOT_EQUAL_FATAL(ODP_IPSEC_SA_INVALID, sa); + + ipsec_test_part test = { + .pkt_in = &pkt_ipv4_null_aes_xcbc_plain, + .num_pkt = 1, + .out = { + { .status.warn.all = 0, + .status.error.all = 0, + .l3_type = ODP_PROTO_L3_TYPE_IPV4, + .l4_type = _ODP_PROTO_L4_TYPE_UNDEF, + .pkt_res = &pkt_ipv4_null_aes_xcbc_esp, + }, + }, + }; + + ipsec_check_out_one(&test, sa); + + ipsec_sa_destroy(sa); +} + static void test_sa_info(void) { uint32_t src = IPV4ADDR(10, 0, 111, 2); @@ -1849,6 +1893,8 @@ odp_testinfo_t ipsec_out_suite[] = { ipsec_check_esp_null_sha256), ODP_TEST_INFO_CONDITIONAL(test_out_ipv4_udp_esp_null_sha256, ipsec_check_esp_null_sha256), + ODP_TEST_INFO_CONDITIONAL(test_out_ipv4_null_aes_xcbc, + ipsec_check_esp_null_aes_xcbc), ODP_TEST_INFO_CONDITIONAL(test_sa_info, ipsec_check_esp_aes_cbc_128_sha1), ODP_TEST_INFO_CONDITIONAL(test_test_sa_update_seq_num, diff --git a/test/validation/api/ipsec/test_vectors.h b/test/validation/api/ipsec/test_vectors.h index 136794c75..b032ef973 100644 --- a/test/validation/api/ipsec/test_vectors.h +++ b/test/validation/api/ipsec/test_vectors.h @@ -75,6 +75,8 @@ KEY(key_rfc7634, 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f); KEY(key_rfc7634_salt, 0xa0, 0xa1, 0xa2, 0xa3); KEY(key_3byte_salt, 0x01, 0x02, 0x03); +KEY(key_auth_aes_xcbc_128, 0x61, 0x31, 0x62, 0x32, 0x63, 0x33, 0x64, 0x34, + 0x65, 0x35, 0x66, 0x36, 0x67, 0x37, 0x68, 0x38); /* DES keys have parity bits so that each byte has odd parity */ KEY(key_des_64, 0xa4, 0xa4, 0xa4, 0xa4, 0xa4, 0xa4, 0xa4, 0xa4); @@ -2108,4 +2110,59 @@ static const ipsec_test_packet pkt_ipv4_udp_esp_null_sha256 = { }, }; +static const ipsec_test_packet pkt_ipv4_null_aes_xcbc_esp = { + .len = 106, + .l2_offset = 0, + .l3_offset = 14, + .l4_offset = 34, + .data = { + /* ETH */ + 0xf1, 0xf1, 0xf1, 0xf1, 0xf1, 0xf1, + 0xf2, 0xf2, 0xf2, 0xf2, 0xf2, 0xf2, 0x08, 0x00, + + /* IP */ + 0x45, 0x00, 0x00, 0x5c, 0x06, 0x00, 0x00, 0x00, + 0x40, 0x32, 0x13, 0x6c, 0x0a, 0x00, 0x6f, 0x02, + 0x0a, 0x00, 0xde, 0x02, + + /* ESP */ + 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x01, + + /* IP */ + 0x45, 0x00, 0x00, 0x2f, 0x49, 0x37, 0x00, 0x00, + 0x40, 0x11, 0x22, 0x84, 0x0d, 0x00, 0x00, 0x02, + 0x02, 0x00, 0x00, 0x02, 0x08, 0x00, 0x08, 0x00, + 0x00, 0x1b, 0x6d, 0x99, 0x58, 0x58, 0x58, 0x58, + 0x58, 0x58, 0x58, 0x58, 0x58, 0x58, 0x58, 0x58, + 0x58, 0x58, 0x58, 0x58, 0x58, 0x58, 0x58, + + /* ESP trailer */ + 0x01, 0x02, 0x03, 0x03, 0x04, + + /* ICV */ + 0xf1, 0x52, 0x64, 0xd1, 0x9b, 0x62, 0x24, 0xdd, + 0xcc, 0x14, 0xf5, 0xc1, + }, +}; + +static const ipsec_test_packet pkt_ipv4_null_aes_xcbc_plain = { + .len = 61, + .l2_offset = 0, + .l3_offset = 14, + .l4_offset = 34, + .data = { + /* ETH */ + 0xf1, 0xf1, 0xf1, 0xf1, 0xf1, 0xf1, + 0xf2, 0xf2, 0xf2, 0xf2, 0xf2, 0xf2, 0x08, 0x00, + + /* IP */ + 0x45, 0x00, 0x00, 0x2f, 0x49, 0x37, 0x00, 0x00, + 0x40, 0x11, 0x22, 0x84, 0x0d, 0x00, 0x00, 0x02, + 0x02, 0x00, 0x00, 0x02, 0x08, 0x00, 0x08, 0x00, + 0x00, 0x1b, 0x6d, 0x99, 0x58, 0x58, 0x58, 0x58, + 0x58, 0x58, 0x58, 0x58, 0x58, 0x58, 0x58, 0x58, + 0x58, 0x58, 0x58, 0x58, 0x58, 0x58, 0x58, + }, +}; + #endif diff --git a/test/validation/api/packet/packet.c b/test/validation/api/packet/packet.c index e25260b1d..4a9b7cdc5 100644 --- a/test/validation/api/packet/packet.c +++ b/test/validation/api/packet/packet.c @@ -36,6 +36,9 @@ ODP_STATIC_ASSERT(PACKET_POOL_NUM_SEG > 1 && /* Number of preallocated packet vector test packets */ #define PKT_VEC_PACKET_NUM PKT_VEC_NUM +/* Maximum packet length when 'pool_capa.pkt.max_len == 0' */ +#define DEFAULT_MAX_LEN (32 * 1024) + static odp_pool_capability_t pool_capa; static odp_pool_param_t default_param; static odp_pool_t default_pool; @@ -130,6 +133,25 @@ static void _packet_compare_data(odp_packet_t pkt1, odp_packet_t pkt2, } } +static int packet_sanity_check(odp_packet_t pkt) +{ + odp_packet_seg_t seg; + uint32_t len = 0; + + for (seg = odp_packet_first_seg(pkt); + seg != ODP_PACKET_SEG_INVALID; + seg = odp_packet_next_seg(pkt, seg)) { + uint32_t seglen = odp_packet_seg_data_len(pkt, seg); + + CU_ASSERT(seglen != 0); + if (seglen == 0) + return 1; + len += seglen; + } + CU_ASSERT(len == odp_packet_len(pkt)); + return len != odp_packet_len(pkt); +} + static int fill_data_forward(odp_packet_t pkt, uint32_t offset, uint32_t len, uint32_t *cur_data) { @@ -566,6 +588,89 @@ static void packet_test_alloc_segmented(void) CU_ASSERT(odp_pool_destroy(pool) == 0); } +static void packet_test_alloc_max_len(void) +{ + const int num = 5; + odp_packet_t pkts[num]; + odp_packet_t pkt; + uint32_t max_len; + odp_pool_t pool; + odp_pool_param_t params; + int ret, i, num_alloc; + + max_len = pool_capa.pkt.max_len; + if (!max_len) + max_len = DEFAULT_MAX_LEN; + + odp_pool_param_init(¶ms); + params.type = ODP_POOL_PACKET; + params.pkt.len = max_len; + params.pkt.num = num; + + pool = odp_pool_create("pool_alloc_max_len", ¶ms); + CU_ASSERT_FATAL(pool != ODP_POOL_INVALID); + + pkt = odp_packet_alloc(pool, max_len); + CU_ASSERT_FATAL(pkt != ODP_PACKET_INVALID); + CU_ASSERT(odp_packet_len(pkt) == max_len); + + odp_packet_free(pkt); + + num_alloc = 0; + for (i = 0; i < num; i++) { + ret = odp_packet_alloc_multi(pool, max_len, + &pkts[num_alloc], num - num_alloc); + CU_ASSERT_FATAL(ret >= 0); + num_alloc += ret; + if (num_alloc >= num) + break; + } + + CU_ASSERT(num_alloc == num); + + for (i = 0; i < num_alloc; i++) + CU_ASSERT(odp_packet_len(pkts[i]) == max_len); + + odp_packet_free_multi(pkts, num_alloc); + + CU_ASSERT(odp_pool_destroy(pool) == 0); +} + +static void packet_test_alloc_max_segment(void) +{ + const int num = 5; + uint32_t max_len, max_seg_len; + odp_packet_t pkt; + odp_pool_t pool; + odp_pool_param_t params; + + max_len = pool_capa.pkt.max_len; + if (max_len == 0) + max_len = DEFAULT_MAX_LEN; + + max_seg_len = pool_capa.pkt.max_seg_len; + if (max_seg_len == 0 || max_seg_len > max_len) + max_seg_len = max_len; + + odp_pool_param_init(¶ms); + params.type = ODP_POOL_PACKET; + params.pkt.seg_len = max_seg_len; + params.pkt.len = max_len; + params.pkt.num = num; + + pool = odp_pool_create("pool_alloc_max_segment", ¶ms); + CU_ASSERT_FATAL(pool != ODP_POOL_INVALID); + + pkt = odp_packet_alloc(pool, max_len); + CU_ASSERT_FATAL(pkt != ODP_PACKET_INVALID); + CU_ASSERT(odp_packet_len(pkt) == max_len); + CU_ASSERT(odp_packet_seg_len(pkt) >= max_seg_len); + + odp_packet_free(pkt); + + CU_ASSERT(odp_pool_destroy(pool) == 0); +} + static void packet_test_alloc_align(void) { odp_pool_t pool; @@ -925,6 +1030,7 @@ static void _verify_headroom_shift(odp_packet_t *pkt, extended = 1; } } + packet_sanity_check(*pkt); CU_ASSERT_PTR_NOT_NULL(data); if (extended) { @@ -1022,6 +1128,7 @@ static void _verify_tailroom_shift(odp_packet_t *pkt, extended = 1; } } + packet_sanity_check(*pkt); CU_ASSERT_PTR_NOT_NULL(tail); if (extended) { @@ -1289,6 +1396,7 @@ static void packet_test_add_rem_data(void) CU_ASSERT(ret >= 0); if (ret < 0) goto free_packet; + packet_sanity_check(new_pkt); CU_ASSERT(odp_packet_len(new_pkt) == pkt_len + add_len); /* Verify that user metadata is preserved */ CU_ASSERT(odp_packet_user_ptr(new_pkt) == usr_ptr); @@ -1310,6 +1418,7 @@ static void packet_test_add_rem_data(void) CU_ASSERT(ret >= 0); if (ret < 0) goto free_packet; + packet_sanity_check(new_pkt); CU_ASSERT(odp_packet_len(new_pkt) == pkt_len - add_len); CU_ASSERT(odp_packet_user_ptr(new_pkt) == usr_ptr); @@ -1715,12 +1824,15 @@ static void packet_test_concatsplit(void) CU_ASSERT(odp_packet_concat(&pkt, pkt2) >= 0); CU_ASSERT(odp_packet_len(pkt) == pkt_len * 2); + packet_sanity_check(pkt); packet_compare_offset(pkt, 0, pkt, pkt_len, pkt_len); CU_ASSERT(odp_packet_split(&pkt, pkt_len, &pkt2) == 0); CU_ASSERT(pkt != pkt2); CU_ASSERT(odp_packet_data(pkt) != odp_packet_data(pkt2)); CU_ASSERT(odp_packet_len(pkt) == odp_packet_len(pkt2)); + packet_sanity_check(pkt); + packet_sanity_check(pkt2); packet_compare_data(pkt, pkt2); packet_compare_data(pkt, test_packet); @@ -1731,19 +1843,22 @@ static void packet_test_concatsplit(void) odp_packet_pool(segmented_test_packet)); CU_ASSERT_FATAL(pkt != ODP_PACKET_INVALID); pkt_len = odp_packet_len(pkt); - + packet_sanity_check(pkt); packet_compare_data(pkt, segmented_test_packet); + CU_ASSERT(odp_packet_split(&pkt, pkt_len / 2, &splits[0]) == 0); CU_ASSERT(pkt != splits[0]); CU_ASSERT(odp_packet_data(pkt) != odp_packet_data(splits[0])); CU_ASSERT(odp_packet_len(pkt) == pkt_len / 2); CU_ASSERT(odp_packet_len(pkt) + odp_packet_len(splits[0]) == pkt_len); - + packet_sanity_check(pkt); + packet_sanity_check(splits[0]); packet_compare_offset(pkt, 0, segmented_test_packet, 0, pkt_len / 2); packet_compare_offset(splits[0], 0, segmented_test_packet, pkt_len / 2, odp_packet_len(splits[0])); CU_ASSERT(odp_packet_concat(&pkt, splits[0]) >= 0); + packet_sanity_check(pkt); packet_compare_offset(pkt, 0, segmented_test_packet, 0, pkt_len / 2); packet_compare_offset(pkt, pkt_len / 2, segmented_test_packet, pkt_len / 2, pkt_len / 2); @@ -1754,15 +1869,24 @@ static void packet_test_concatsplit(void) packet_compare_data(pkt, segmented_test_packet); CU_ASSERT(odp_packet_split(&pkt, pkt_len / 2, &splits[0]) == 0); + packet_sanity_check(pkt); CU_ASSERT(odp_packet_split(&pkt, pkt_len / 4, &splits[1]) == 0); + packet_sanity_check(pkt); CU_ASSERT(odp_packet_split(&pkt, pkt_len / 8, &splits[2]) == 0); + packet_sanity_check(pkt); + packet_sanity_check(splits[0]); + packet_sanity_check(splits[1]); + packet_sanity_check(splits[2]); CU_ASSERT(odp_packet_len(splits[0]) + odp_packet_len(splits[1]) + odp_packet_len(splits[2]) + odp_packet_len(pkt) == pkt_len); CU_ASSERT(odp_packet_concat(&pkt, splits[2]) >= 0); + packet_sanity_check(pkt); CU_ASSERT(odp_packet_concat(&pkt, splits[1]) >= 0); + packet_sanity_check(pkt); CU_ASSERT(odp_packet_concat(&pkt, splits[0]) >= 0); + packet_sanity_check(pkt); CU_ASSERT(odp_packet_len(pkt) == odp_packet_len(segmented_test_packet)); packet_compare_data(pkt, segmented_test_packet); @@ -1813,6 +1937,9 @@ static void packet_test_concat_small(void) odp_packet_free(pkt2); break; } + + if (packet_sanity_check(pkt)) + break; } CU_ASSERT(odp_packet_len(pkt) == len); @@ -1866,6 +1993,7 @@ static void packet_test_concat_extend_trunc(void) ret = odp_packet_concat(&pkt, pkt2); CU_ASSERT(ret >= 0); + packet_sanity_check(pkt); if (ret < 0) odp_packet_free(pkt2); @@ -1876,13 +2004,13 @@ static void packet_test_concat_extend_trunc(void) ret = odp_packet_extend_tail(&pkt, ext_len, NULL, NULL); CU_ASSERT(ret >= 0); - + packet_sanity_check(pkt); CU_ASSERT(odp_packet_len(pkt) == (cur_len + ext_len)); cur_len = odp_packet_len(pkt); ret = odp_packet_extend_head(&pkt, ext_len, NULL, NULL); CU_ASSERT(ret >= 0); - + packet_sanity_check(pkt); CU_ASSERT(odp_packet_len(pkt) == (cur_len + ext_len)); cur_len = odp_packet_len(pkt); @@ -1895,18 +2023,19 @@ static void packet_test_concat_extend_trunc(void) if (ret < 0) odp_packet_free(pkt2); + packet_sanity_check(pkt); CU_ASSERT(odp_packet_len(pkt) == (cur_len + alloc_len)); cur_len = odp_packet_len(pkt); ret = odp_packet_trunc_head(&pkt, trunc_len, NULL, NULL); CU_ASSERT(ret >= 0); - + packet_sanity_check(pkt); CU_ASSERT(odp_packet_len(pkt) == (cur_len - trunc_len)); cur_len = odp_packet_len(pkt); ret = odp_packet_trunc_tail(&pkt, trunc_len, NULL, NULL); CU_ASSERT(ret >= 0); - + packet_sanity_check(pkt); CU_ASSERT(odp_packet_len(pkt) == (cur_len - trunc_len)); cur_len = odp_packet_len(pkt); @@ -1962,6 +2091,9 @@ static void packet_test_extend_small(void) if (ret < 0) break; + if (packet_sanity_check(pkt)) + break; + if (tail) { /* assert needs brackets */ CU_ASSERT(seg_len == 1); @@ -2066,6 +2198,9 @@ static void packet_test_extend_large(void) if (ret < 0) break; + if (packet_sanity_check(pkt)) + break; + if (tail) { /* assert needs brackets */ CU_ASSERT((seg_len > 0) && @@ -2197,6 +2332,9 @@ static void packet_test_extend_mix(void) CU_ASSERT(ret == 0); } + if (packet_sanity_check(pkt)) + break; + cur_len += ext_len; } @@ -2259,9 +2397,11 @@ static void packet_test_extend_ref(void) NULL, NULL) >= 0); CU_ASSERT(odp_packet_extend_head(&max_pkt, 1, NULL, NULL) >= 0); CU_ASSERT(odp_packet_len(max_pkt) == max_len); + packet_sanity_check(max_pkt); /* Now try with a reference in place */ CU_ASSERT(odp_packet_trunc_tail(&max_pkt, 100, NULL, NULL) >= 0); + packet_sanity_check(max_pkt); ref = odp_packet_ref(max_pkt, 100); /* Verify ref lengths */ @@ -2275,6 +2415,7 @@ static void packet_test_extend_ref(void) /* Now extend max_pkt and verify effect */ CU_ASSERT(odp_packet_extend_head(&max_pkt, 10, NULL, NULL) >= 0); CU_ASSERT(odp_packet_len(max_pkt) == max_len - 90); + packet_sanity_check(max_pkt); /* Extend on max_pkt should not affect ref */ CU_ASSERT(odp_packet_len(ref) == max_len - 200); @@ -2282,12 +2423,14 @@ static void packet_test_extend_ref(void) /* Now extend ref and verify effect*/ CU_ASSERT(odp_packet_extend_head(&ref, 20, NULL, NULL) >= 0); CU_ASSERT(odp_packet_len(ref) == max_len - 180); + packet_sanity_check(max_pkt); /* Extend on ref should not affect max_pkt */ CU_ASSERT(odp_packet_len(max_pkt) == max_len - 90); /* Trunc max_pkt of all unshared len */ CU_ASSERT(odp_packet_trunc_head(&max_pkt, 110, NULL, NULL) >= 0); + packet_sanity_check(max_pkt); /* Verify effect on max_pkt */ CU_ASSERT(odp_packet_len(max_pkt) == max_len - 200); @@ -2299,6 +2442,7 @@ static void packet_test_extend_ref(void) odp_packet_free(ref); CU_ASSERT(odp_packet_has_ref(max_pkt) == 0); CU_ASSERT(odp_packet_len(max_pkt) == max_len - 200); + packet_sanity_check(max_pkt); odp_packet_free(max_pkt); } @@ -2339,6 +2483,8 @@ static void packet_test_align(void) /* Verify requested contiguous addressabilty */ CU_ASSERT(aligned_seglen >= seg_len + 2); + + packet_sanity_check(pkt); } /* Get a misaligned address */ @@ -2358,6 +2504,7 @@ static void packet_test_align(void) packet_compare_offset(pkt, offset, segmented_test_packet, offset, aligned_seglen); CU_ASSERT((uintptr_t)aligned_data % max_align == 0); + packet_sanity_check(pkt); odp_packet_free(pkt); } @@ -4080,6 +4227,8 @@ odp_testinfo_t packet_suite[] = { ODP_TEST_INFO(packet_test_alloc_free_multi), ODP_TEST_INFO(packet_test_free_sp), ODP_TEST_INFO(packet_test_alloc_segmented), + ODP_TEST_INFO(packet_test_alloc_max_len), + ODP_TEST_INFO(packet_test_alloc_max_segment), ODP_TEST_INFO(packet_test_alloc_align), ODP_TEST_INFO(packet_test_basic_metadata), ODP_TEST_INFO(packet_test_debug), diff --git a/test/validation/api/pool/pool.c b/test/validation/api/pool/pool.c index 861ac18ae..866f93020 100644 --- a/test/validation/api/pool/pool.c +++ b/test/validation/api/pool/pool.c @@ -140,6 +140,7 @@ static void pool_test_lookup_info_print(void) CU_ASSERT(param.type == info.params.type); odp_pool_print(pool); + odp_pool_print_all(); CU_ASSERT(odp_pool_destroy(pool) == 0); } diff --git a/test/validation/api/queue/queue.c b/test/validation/api/queue/queue.c index 6f59cbf71..7b39da433 100644 --- a/test/validation/api/queue/queue.c +++ b/test/validation/api/queue/queue.c @@ -669,7 +669,9 @@ static void queue_test_param(void) CU_ASSERT_FATAL(buf != ODP_BUFFER_INVALID); ev = odp_buffer_to_event(buf); - if (!(CU_ASSERT(odp_queue_enq(queue, ev) == 0))) { + ret = odp_queue_enq(queue, ev); + CU_ASSERT(ret == 0); + if (ret) { odp_buffer_free(buf); } else { CU_ASSERT(ev == odp_queue_deq(queue)); diff --git a/test/validation/api/scheduler/scheduler.c b/test/validation/api/scheduler/scheduler.c index 9c84eacd3..da0eae267 100644 --- a/test/validation/api/scheduler/scheduler.c +++ b/test/validation/api/scheduler/scheduler.c @@ -274,7 +274,7 @@ static void scheduler_test_queue_destroy(void) odp_buffer_t buf; odp_event_t ev; uint32_t *u32; - int i; + int i, ret; odp_schedule_sync_t sync[] = {ODP_SCHED_SYNC_PARALLEL, ODP_SCHED_SYNC_ATOMIC, ODP_SCHED_SYNC_ORDERED}; @@ -305,7 +305,10 @@ static void scheduler_test_queue_destroy(void) u32[0] = MAGIC; ev = odp_buffer_to_event(buf); - if (!(CU_ASSERT(odp_queue_enq(queue, ev) == 0))) + + ret = odp_queue_enq(queue, ev); + CU_ASSERT(ret == 0); + if (ret) odp_buffer_free(buf); ev = odp_schedule(&from, ODP_SCHED_WAIT); @@ -371,7 +374,9 @@ static void scheduler_test_wait(void) u32[0] = MAGIC; ev = odp_buffer_to_event(buf); - if (!(CU_ASSERT(odp_queue_enq(queue, ev) == 0))) { + ret = odp_queue_enq(queue, ev); + CU_ASSERT(ret == 0); + if (ret) { odp_buffer_free(buf); continue; } diff --git a/test/validation/api/stash/stash.c b/test/validation/api/stash/stash.c index c99d5a471..49da49dfd 100644 --- a/test/validation/api/stash/stash.c +++ b/test/validation/api/stash/stash.c @@ -27,6 +27,13 @@ #define MAX_RETRY 1024 #define RETRY_MSEC 100 +typedef enum stash_op_t { + STASH_GEN, + STASH_U32, + STASH_U64, + STASH_PTR +} stash_op_t; + typedef struct num_obj_t { uint32_t u64; uint32_t u32; @@ -358,7 +365,7 @@ static void stash_create_fifo_u32_all(void) CU_ASSERT_FATAL(odp_stash_destroy(stash[i]) == 0); } -static void stash_default_put(uint32_t size, int32_t burst) +static void stash_default_put(uint32_t size, int32_t burst, stash_op_t op) { odp_stash_t stash; odp_stash_param_t param; @@ -414,7 +421,16 @@ static void stash_default_put(uint32_t size, int32_t burst) retry = MAX_RETRY; num_left = num; while (num_left) { - ret = odp_stash_put(stash, input, burst); + if (op == STASH_GEN) + ret = odp_stash_put(stash, input, burst); + else if (op == STASH_U32) + ret = odp_stash_put_u32(stash, input_u32, burst); + else if (op == STASH_U64) + ret = odp_stash_put_u64(stash, input_u64, burst); + else if (op == STASH_PTR) + ret = odp_stash_put_ptr(stash, input, burst); + else + ret = -1; CU_ASSERT_FATAL(ret >= 0); CU_ASSERT_FATAL(ret <= burst); @@ -446,8 +462,16 @@ static void stash_default_put(uint32_t size, int32_t burst) output_u8[0] = MAGIC_U8; output_u8[burst + 1] = MAGIC_U8; } - - ret = odp_stash_get(stash, output, burst); + if (op == STASH_GEN) + ret = odp_stash_get(stash, output, burst); + else if (op == STASH_U32) + ret = odp_stash_get_u32(stash, &output_u32[1], burst); + else if (op == STASH_U64) + ret = odp_stash_get_u64(stash, &output_u64[1], burst); + else if (op == STASH_PTR) + ret = odp_stash_get_ptr(stash, output, burst); + else + ret = -1; CU_ASSERT_FATAL(ret >= 0); CU_ASSERT_FATAL(ret <= burst); @@ -494,7 +518,7 @@ static void stash_default_put(uint32_t size, int32_t burst) CU_ASSERT_FATAL(odp_stash_destroy(stash) == 0); } -static void stash_fifo_put(uint32_t size, int32_t burst) +static void stash_fifo_put(uint32_t size, int32_t burst, stash_op_t op) { odp_stash_t stash; odp_stash_param_t param; @@ -553,8 +577,16 @@ static void stash_fifo_put(uint32_t size, int32_t burst) else input_u8[i] = VAL_U8 + num_left - i; } - - ret = odp_stash_put(stash, input, burst); + if (op == STASH_GEN) + ret = odp_stash_put(stash, input, burst); + else if (op == STASH_U32) + ret = odp_stash_put_u32(stash, input_u32, burst); + else if (op == STASH_U64) + ret = odp_stash_put_u64(stash, input_u64, burst); + else if (op == STASH_PTR) + ret = odp_stash_put_ptr(stash, input, burst); + else + ret = -1; CU_ASSERT_FATAL(ret >= 0); if (ret) { @@ -587,7 +619,16 @@ static void stash_fifo_put(uint32_t size, int32_t burst) output_u8[burst + 1] = MAGIC_U8; } - ret = odp_stash_get(stash, output, burst); + if (op == STASH_GEN) + ret = odp_stash_get(stash, output, burst); + else if (op == STASH_U32) + ret = odp_stash_get_u32(stash, &output_u32[1], burst); + else if (op == STASH_U64) + ret = odp_stash_get_u64(stash, &output_u64[1], burst); + else if (op == STASH_PTR) + ret = odp_stash_get_ptr(stash, output, burst); + else + ret = -1; CU_ASSERT_FATAL(ret >= 0); if (size == sizeof(uint64_t)) { @@ -649,6 +690,14 @@ static int check_support_64(void) return ODP_TEST_INACTIVE; } +static int check_support_ptr(void) +{ + if (global.capa_default.max_obj_size >= sizeof(uintptr_t)) + return ODP_TEST_ACTIVE; + + return ODP_TEST_INACTIVE; +} + static int check_support_fifo_64(void) { if (global.fifo_supported && @@ -658,6 +707,15 @@ static int check_support_fifo_64(void) return ODP_TEST_INACTIVE; } +static int check_support_fifo_ptr(void) +{ + if (global.fifo_supported && + global.capa_fifo.max_obj_size >= sizeof(uintptr_t)) + return ODP_TEST_ACTIVE; + + return ODP_TEST_INACTIVE; +} + static int check_support_fifo(void) { if (global.fifo_supported) @@ -668,82 +726,142 @@ static int check_support_fifo(void) static void stash_default_put_u64_1(void) { - stash_default_put(sizeof(uint64_t), 1); + stash_default_put(sizeof(uint64_t), 1, STASH_GEN); } static void stash_default_put_u64_n(void) { - stash_default_put(sizeof(uint64_t), BURST); + stash_default_put(sizeof(uint64_t), BURST, STASH_GEN); +} + +static void stash_default_u64_put_u64_1(void) +{ + stash_default_put(sizeof(uint64_t), 1, STASH_U64); +} + +static void stash_default_u64_put_u64_n(void) +{ + stash_default_put(sizeof(uint64_t), BURST, STASH_U64); +} + +static void stash_default_put_ptr_1(void) +{ + stash_default_put(sizeof(uintptr_t), 1, STASH_PTR); +} + +static void stash_default_put_ptr_n(void) +{ + stash_default_put(sizeof(uintptr_t), BURST, STASH_PTR); } static void stash_default_put_u32_1(void) { - stash_default_put(sizeof(uint32_t), 1); + stash_default_put(sizeof(uint32_t), 1, STASH_GEN); } static void stash_default_put_u32_n(void) { - stash_default_put(sizeof(uint32_t), BURST); + stash_default_put(sizeof(uint32_t), BURST, STASH_GEN); +} + +static void stash_default_u32_put_u32_1(void) +{ + stash_default_put(sizeof(uint32_t), 1, STASH_U32); +} + +static void stash_default_u32_put_u32_n(void) +{ + stash_default_put(sizeof(uint32_t), BURST, STASH_U32); } static void stash_default_put_u16_1(void) { - stash_default_put(sizeof(uint16_t), 1); + stash_default_put(sizeof(uint16_t), 1, STASH_GEN); } static void stash_default_put_u16_n(void) { - stash_default_put(sizeof(uint16_t), BURST); + stash_default_put(sizeof(uint16_t), BURST, STASH_GEN); } static void stash_default_put_u8_1(void) { - stash_default_put(sizeof(uint8_t), 1); + stash_default_put(sizeof(uint8_t), 1, STASH_GEN); } static void stash_default_put_u8_n(void) { - stash_default_put(sizeof(uint8_t), BURST); + stash_default_put(sizeof(uint8_t), BURST, STASH_GEN); } static void stash_fifo_put_u64_1(void) { - stash_fifo_put(sizeof(uint64_t), 1); + stash_fifo_put(sizeof(uint64_t), 1, STASH_GEN); } static void stash_fifo_put_u64_n(void) { - stash_fifo_put(sizeof(uint64_t), BURST); + stash_fifo_put(sizeof(uint64_t), BURST, STASH_GEN); +} + +static void stash_fifo_u64_put_u64_1(void) +{ + stash_fifo_put(sizeof(uint64_t), 1, STASH_U64); +} + +static void stash_fifo_u64_put_u64_n(void) +{ + stash_fifo_put(sizeof(uint64_t), BURST, STASH_U64); +} + +static void stash_fifo_put_ptr_1(void) +{ + stash_fifo_put(sizeof(uintptr_t), 1, STASH_PTR); +} + +static void stash_fifo_put_ptr_n(void) +{ + stash_fifo_put(sizeof(uintptr_t), BURST, STASH_PTR); } static void stash_fifo_put_u32_1(void) { - stash_fifo_put(sizeof(uint32_t), 1); + stash_fifo_put(sizeof(uint32_t), 1, STASH_GEN); } static void stash_fifo_put_u32_n(void) { - stash_fifo_put(sizeof(uint32_t), BURST); + stash_fifo_put(sizeof(uint32_t), BURST, STASH_GEN); +} + +static void stash_fifo_u32_put_u32_1(void) +{ + stash_fifo_put(sizeof(uint32_t), 1, STASH_U32); +} + +static void stash_fifo_u32_put_u32_n(void) +{ + stash_fifo_put(sizeof(uint32_t), BURST, STASH_U32); } static void stash_fifo_put_u16_1(void) { - stash_fifo_put(sizeof(uint16_t), 1); + stash_fifo_put(sizeof(uint16_t), 1, STASH_GEN); } static void stash_fifo_put_u16_n(void) { - stash_fifo_put(sizeof(uint16_t), BURST); + stash_fifo_put(sizeof(uint16_t), BURST, STASH_GEN); } static void stash_fifo_put_u8_1(void) { - stash_fifo_put(sizeof(uint8_t), 1); + stash_fifo_put(sizeof(uint8_t), 1, STASH_GEN); } static void stash_fifo_put_u8_n(void) { - stash_fifo_put(sizeof(uint8_t), BURST); + stash_fifo_put(sizeof(uint8_t), BURST, STASH_GEN); } odp_testinfo_t stash_suite[] = { @@ -753,8 +871,14 @@ odp_testinfo_t stash_suite[] = { ODP_TEST_INFO(stash_create_u32), ODP_TEST_INFO_CONDITIONAL(stash_default_put_u64_1, check_support_64), ODP_TEST_INFO_CONDITIONAL(stash_default_put_u64_n, check_support_64), + ODP_TEST_INFO_CONDITIONAL(stash_default_u64_put_u64_1, check_support_64), + ODP_TEST_INFO_CONDITIONAL(stash_default_u64_put_u64_n, check_support_64), + ODP_TEST_INFO_CONDITIONAL(stash_default_put_ptr_1, check_support_ptr), + ODP_TEST_INFO_CONDITIONAL(stash_default_put_ptr_n, check_support_ptr), ODP_TEST_INFO(stash_default_put_u32_1), ODP_TEST_INFO(stash_default_put_u32_n), + ODP_TEST_INFO(stash_default_u32_put_u32_1), + ODP_TEST_INFO(stash_default_u32_put_u32_n), ODP_TEST_INFO(stash_default_put_u16_1), ODP_TEST_INFO(stash_default_put_u16_n), ODP_TEST_INFO(stash_default_put_u8_1), @@ -763,8 +887,14 @@ odp_testinfo_t stash_suite[] = { ODP_TEST_INFO(stash_create_u32_all), ODP_TEST_INFO_CONDITIONAL(stash_fifo_put_u64_1, check_support_fifo_64), ODP_TEST_INFO_CONDITIONAL(stash_fifo_put_u64_n, check_support_fifo_64), + ODP_TEST_INFO_CONDITIONAL(stash_fifo_u64_put_u64_1, check_support_fifo_64), + ODP_TEST_INFO_CONDITIONAL(stash_fifo_u64_put_u64_n, check_support_fifo_64), + ODP_TEST_INFO_CONDITIONAL(stash_fifo_put_ptr_1, check_support_fifo_ptr), + ODP_TEST_INFO_CONDITIONAL(stash_fifo_put_ptr_n, check_support_fifo_ptr), ODP_TEST_INFO_CONDITIONAL(stash_fifo_put_u32_1, check_support_fifo), ODP_TEST_INFO_CONDITIONAL(stash_fifo_put_u32_n, check_support_fifo), + ODP_TEST_INFO_CONDITIONAL(stash_fifo_u32_put_u32_1, check_support_fifo), + ODP_TEST_INFO_CONDITIONAL(stash_fifo_u32_put_u32_n, check_support_fifo), ODP_TEST_INFO_CONDITIONAL(stash_fifo_put_u16_1, check_support_fifo), ODP_TEST_INFO_CONDITIONAL(stash_fifo_put_u16_n, check_support_fifo), ODP_TEST_INFO_CONDITIONAL(stash_fifo_put_u8_1, check_support_fifo), diff --git a/test/validation/api/timer/timer.c b/test/validation/api/timer/timer.c index 0716b7999..4cdbf05c2 100644 --- a/test/validation/api/timer/timer.c +++ b/test/validation/api/timer/timer.c @@ -717,7 +717,7 @@ static void timer_test_event_type(odp_queue_type_t queue_type, ODPH_DBG(" period_ns %" PRIu64 "\n", period_ns); ODPH_DBG(" period_tick %" PRIu64 "\n", period_tick); ODPH_DBG(" duration_ns %" PRIu64 "\n", duration_ns); - ODPH_DBG(" user_ptr %p\n\n", user_ctx); + ODPH_DBG(" user_ptr %p\n\n", (const void *)user_ctx); for (i = 0; i < num; i++) { if (event_type == ODP_EVENT_BUFFER) { diff --git a/test/validation/api/traffic_mngr/traffic_mngr.c b/test/validation/api/traffic_mngr/traffic_mngr.c index 2ba0d1c9d..0899272ba 100644 --- a/test/validation/api/traffic_mngr/traffic_mngr.c +++ b/test/validation/api/traffic_mngr/traffic_mngr.c @@ -120,7 +120,7 @@ typedef enum { typedef struct { uint32_t num_queues; - odp_tm_queue_t tm_queues[0]; + odp_tm_queue_t tm_queues[]; } tm_queue_desc_t; typedef struct tm_node_desc_s tm_node_desc_t; @@ -133,7 +133,7 @@ struct tm_node_desc_s { odp_tm_node_t node; odp_tm_node_t parent_node; tm_queue_desc_t *queue_desc; - tm_node_desc_t *children[0]; + tm_node_desc_t *children[]; }; typedef struct { @@ -1813,7 +1813,7 @@ static void dump_tm_subtree(tm_node_desc_t *node_desc) "level=%" PRIu32" parent=0x%" PRIX64 " children=%" PRIu32 " " "queues=%" PRIu32 " queue_fanin=%" PRIu32 " " "node_fanin=%" PRIu32 "\n", - node_desc, node_desc->node_name, + (void *)node_desc, node_desc->node_name, odp_tm_node_to_u64(node_desc->node), node_desc->node_idx, node_desc->level, odp_tm_node_to_u64(node_desc->parent_node), node_desc->num_children, num_queues, node_info.tm_queue_fanin, |