diff options
author | Matias Elo <matias.elo@nokia.com> | 2021-05-26 17:03:10 +0300 |
---|---|---|
committer | GitHub <noreply@github.com> | 2021-05-26 17:03:10 +0300 |
commit | 3cc4177b551f32aa4b3b0d76c248f5342362a028 (patch) | |
tree | ec8f42fce582fb0400074c34cdf1aa2a4f070cf7 | |
parent | cb9db0fa8ca6dd0d58ffe6180affe43abe53956c (diff) | |
parent | ce0f58291aa87b856c7df2501c8e7de9a16a9b5d (diff) |
Merge ODP v1.28.0.0v1.28.0.0_DPDK_19.11
Merge ODP linux-generic v1.28.0.0 into ODP-DPDK.
141 files changed, 4370 insertions, 1550 deletions
diff --git a/.checkpatch.conf b/.checkpatch.conf index fa31c1edf..b276680bf 100644 --- a/.checkpatch.conf +++ b/.checkpatch.conf @@ -20,5 +20,6 @@ --ignore=PREFER_FALLTHROUGH --ignore=LONG_LINE_STRING --ignore=EMAIL_SUBJECT +--ignore=C99_COMMENT_TOLERANCE --codespell --codespellfile=/usr/share/codespell/dictionary.txt diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index b0fddfc9f..f21f85a03 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -66,7 +66,7 @@ jobs: fail-fast: false matrix: cc: [gcc, clang] - conf: ['', 'CFLAGS=-O3', 'CFLAGS=-O0 --enable-debug --enable-debug-print', '--enable-lto', '--enable-lto --disable-abi-compat', '--enable-pcapng-support'] + conf: ['', 'CFLAGS=-O3', 'CFLAGS=-O1', 'CFLAGS=-O0 --enable-debug=full', '--enable-lto', '--enable-lto --disable-abi-compat', '--enable-pcapng-support'] exclude: - cc: clang conf: '--enable-lto' @@ -140,7 +140,7 @@ jobs: fail-fast: false matrix: cc: [gcc, clang] - os: ['centos_7', 'centos_8'] + os: ['centos_7', 'centos_8', 'ubuntu_16.04'] steps: - uses: actions/checkout@v2 - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${{matrix.cc}}" @@ -177,6 +177,14 @@ jobs: - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${CC}" -e ODP_LIB_NAME="libodp-linux" -e CONF="${CONF}" $CONTAINER_NAMESPACE/odp-ci-${OS}-${ARCH} /odp/scripts/ci/build_${ARCH}.sh + Build_sched_config: + runs-on: ubuntu-18.04 + steps: + - uses: actions/checkout@v2 + - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${CC}" + -e CONF="--enable-debug=full" -e ODP_CONFIG_FILE=/odp/platform/linux-dpdk/test/sched-basic.conf + $CONTAINER_NAMESPACE/odp-ci-${OS}-${ARCH} /odp/scripts/ci/build_${ARCH}.sh + Run_coverage: runs-on: ubuntu-18.04 steps: @@ -227,7 +235,7 @@ jobs: fail-fast: false matrix: cc: [gcc, clang] - os: ['ubuntu_16.04', 'ubuntu_20.04'] + os: ['ubuntu_20.04'] steps: - uses: actions/checkout@v2 - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${{matrix.cc}}" diff --git a/.github/workflows/coverity.yml b/.github/workflows/coverity.yml new file mode 100644 index 000000000..601744e1b --- /dev/null +++ b/.github/workflows/coverity.yml @@ -0,0 +1,25 @@ +name: Coverity Scan + +on: + schedule: + - cron: '0 0 * * *' # Once every day at 00:00 UTC +env: + CC: gcc + ARCH: x86_64 + CONTAINER_NAMESPACE: ghcr.io/opendataplane/odp-docker-images + OS: ubuntu_20.04 + COVERITY_EMAIL: odp@lists.opendataplane.org + COVERITY_PROJECT: ODP-DPDK + +jobs: + Coverity-analysis: + runs-on: ubuntu-18.04 + steps: + - uses: actions/checkout@v2 + - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g + -e CC="${CC}" -e GITHUB_SHA="${GITHUB_SHA}" + -e COVERITY_TOKEN="${{ secrets.COVERITY_TOKEN }}" + -e COVERITY_EMAIL="${COVERITY_EMAIL}" + -e COVERITY_PROJECT="${COVERITY_PROJECT}" + ${CONTAINER_NAMESPACE}/odp-ci-${OS}-${ARCH}-coverity-linux-dpdk + /odp/scripts/ci/coverity.sh @@ -1,3 +1,24 @@ +== OpenDataPlane (1.28.0.0) +=== API +==== Crypto +* New crypto capabilities for defining if a queue type (scheduled/plain) can be +used as a crypto completion event destination + +=== Event +* New packet TX completion event type and related functions + +=== Packet +* New packet APIs for requesting packet TX completion events +* New packet APIS for requesting packet TX drop based on packet age + +=== Classifier +* New `odp_cls_print_all()` function for printing implementation specific +debug information about all classifier rules + +=== System +* New enumerations for ARMv8.7-A, ARMv9.0-A, ARMv9.1-A, and ARMv9.2-A ISA +versions + == OpenDataPlane (1.24.0.0) === Summary of Changes diff --git a/configure.ac b/configure.ac index 1987c4c2c..f92195772 100644 --- a/configure.ac +++ b/configure.ac @@ -3,7 +3,7 @@ AC_PREREQ([2.5]) # ODP API version ########################################################################## m4_define([odpapi_generation_version], [1]) -m4_define([odpapi_major_version], [27]) +m4_define([odpapi_major_version], [28]) m4_define([odpapi_minor_version], [0]) m4_define([odpapi_point_version], [0]) m4_define([odpapi_version], @@ -43,7 +43,7 @@ AS_IF([test "$ac_cv_env_CFLAGS_set" = ""], [user_cflags=0], [user_cflags=1]) # Initialize automake AM_INIT_AUTOMAKE([1.9 tar-pax subdir-objects foreign nostdinc -Wall -Werror]) AC_CONFIG_SRCDIR([include/odp/api/spec/init.h]) -AM_CONFIG_HEADER([include/odp/autoheader_build.h include/odp/autoheader_external.h include/odp/autoheader_internal.h helper/include/odp/helper/autoheader_external.h]) +AC_CONFIG_HEADERS([include/odp/autoheader_build.h include/odp/autoheader_external.h include/odp/autoheader_internal.h helper/include/odp/helper/autoheader_external.h]) AC_USE_SYSTEM_EXTENSIONS AC_SYS_LARGEFILE @@ -138,7 +138,7 @@ AS_IF([test "$GCC" == yes], ) ODP_CFLAGS="$ODP_CFLAGS -std=c99 -D_GNU_SOURCE" -ODP_CXXFLAGS="$ODP_CXXFLAGS -std=c++11 -Wno-deprecated-register" +ODP_CXXFLAGS="$ODP_CXXFLAGS -std=c++11" # Extra flags for example to suppress certain warning types ODP_CFLAGS="$ODP_CFLAGS $ODP_CFLAGS_EXTRA" @@ -240,11 +240,10 @@ AC_ARG_ENABLE([abi-compat], ODP_LIBSO_VERSION=0:0:0 # do not use -march=native with clang (due to possible failures on - # clang optimizations) or when user have defined CFLAGS (may contain - # another -march option). + # clang optimizations). $CC --version | grep -q clang - if test $? -ne 0 -a $user_cflags -eq 0; then + if test $? -ne 0; then ODP_CHECK_CFLAG([-march=native]) fi fi]) @@ -445,9 +444,9 @@ DX_INIT_DOXYGEN($PACKAGE_NAME, ########################################################################## # Default include setup ########################################################################## -CFLAGS="$CFLAGS $ODP_CFLAGS $EXTRA_CFLAGS" -CXXFLAGS="$CXXFLAGS $ODP_CXXFLAGS" -LDFLAGS="$LDFLAGS $ODP_LTO_FLAGS" +CFLAGS="$ODP_CFLAGS $CFLAGS" +CXXFLAGS="$ODP_CXXFLAGS $CXXFLAGS" +LDFLAGS="$ODP_LTO_FLAGS $LDFLAGS" AC_CONFIG_FILES([Makefile]) AC_CONFIG_FILES([include/Makefile diff --git a/example/classifier/odp_classifier.c b/example/classifier/odp_classifier.c index 859802358..ebac378a6 100644 --- a/example/classifier/odp_classifier.c +++ b/example/classifier/odp_classifier.c @@ -673,6 +673,9 @@ int main(int argc, char *argv[]) configure_cos(default_cos, args); + printf("\n"); + odp_cls_print_all(); + if (odp_pktio_start(pktio)) { ODPH_ERR("Error: unable to start pktio\n"); exit(EXIT_FAILURE); diff --git a/example/ipsec_crypto/odp_ipsec.c b/example/ipsec_crypto/odp_ipsec.c index bc70f643c..d6102b866 100644 --- a/example/ipsec_crypto/odp_ipsec.c +++ b/example/ipsec_crypto/odp_ipsec.c @@ -1210,6 +1210,7 @@ main(int argc, char *argv[]) odp_shm_t shm; odp_cpumask_t cpumask; char cpumaskstr[ODP_CPUMASK_STR_SIZE]; + odp_crypto_capability_t crypto_capa; odp_pool_param_t params; odp_instance_t instance; odp_init_t init_param; @@ -1251,6 +1252,21 @@ main(int argc, char *argv[]) exit(EXIT_FAILURE); } + if (odp_crypto_capability(&crypto_capa)) { + ODPH_ERR("Error: Crypto capability request failed.\n"); + exit(EXIT_FAILURE); + } + + if ((NULL == getenv("ODP_IPSEC_USE_POLL_QUEUES")) && + (crypto_capa.queue_type_sched == 0)) { + ODPH_ERR("Error: scheduled type compl queue not supported.\n"); + exit(EXIT_FAILURE); + } else if ((NULL != getenv("ODP_IPSEC_USE_POLL_QUEUES")) && + crypto_capa.queue_type_plain == 0) { + ODPH_ERR("Error: Plain type compl queue not supported.\n"); + exit(EXIT_FAILURE); + } + /* Reserve memory for args from shared mem */ shm = odp_shm_reserve("shm_args", sizeof(global_data_t), ODP_CACHE_LINE_SIZE, 0); diff --git a/example/ipsec_crypto/odp_ipsec_stream.c b/example/ipsec_crypto/odp_ipsec_stream.c index db4130fe7..e53dbce0d 100644 --- a/example/ipsec_crypto/odp_ipsec_stream.c +++ b/example/ipsec_crypto/odp_ipsec_stream.c @@ -425,6 +425,13 @@ odp_bool_t verify_ipv4_packet(stream_db_entry_t *stream, /* Find IPsec headers if any and compare against entry */ hdr_len = locate_ipsec_headers(ip, &ah, &esp); + /* Verify if the packet is IPsec encapsulated or is cleartext as + * expected + */ + if (((stream->output.entry && (!ah && !esp))) || + (stream->input.entry && (ah || esp))) + return FALSE; + /* Cleartext packet */ if (!ah && !esp) goto clear_packet; diff --git a/example/packet/Makefile.am b/example/packet/Makefile.am index 5e4d9f5ea..cf33b2ef7 100644 --- a/example/packet/Makefile.am +++ b/example/packet/Makefile.am @@ -35,3 +35,5 @@ clean-local: rm -f $(builddir)/$$f; \ done \ fi + +.NOTPARALLEL: diff --git a/example/sysinfo/odp_sysinfo.c b/example/sysinfo/odp_sysinfo.c index c28fd065c..cad87b121 100644 --- a/example/sysinfo/odp_sysinfo.c +++ b/example/sysinfo/odp_sysinfo.c @@ -66,6 +66,14 @@ static const char *arm_isa(odp_cpu_arch_arm_t isa) return "ARMv8.5-A"; case ODP_CPU_ARCH_ARMV8_6: return "ARMv8.6-A"; + case ODP_CPU_ARCH_ARMV8_7: + return "ARMv8.7-A"; + case ODP_CPU_ARCH_ARMV9_0: + return "ARMv9.0-A"; + case ODP_CPU_ARCH_ARMV9_1: + return "ARMv9.1-A"; + case ODP_CPU_ARCH_ARMV9_2: + return "ARMv9.2-A"; default: return "Unknown"; } diff --git a/example/timer/Makefile.am b/example/timer/Makefile.am index c7d58922d..e4f50feaf 100644 --- a/example/timer/Makefile.am +++ b/example/timer/Makefile.am @@ -14,3 +14,5 @@ if test_example TESTS = odp_timer_accuracy \ odp_timer_simple endif + +.NOTPARALLEL: diff --git a/helper/Makefile.am b/helper/Makefile.am index 15e968310..56fd5bd6b 100644 --- a/helper/Makefile.am +++ b/helper/Makefile.am @@ -83,3 +83,16 @@ __LIB__libodphelper_la_LIBADD = $(PTHREAD_LIBS) __LIB__libodphelper_la_LIBADD += $(LIBCLI_LIBS) lib_LTLIBRARIES = $(LIB)/libodphelper.la + +CHECK_GLOBALS_REGEX = " odph_" + +TESTS_ENVIRONMENT = \ + LIBTOOL="$(LIBTOOL)" \ + NM="$(NM)" \ + LIB="$(LIB)" \ + lib_LTLIBRARIES="$(lib_LTLIBRARIES)" \ + CHECK_GLOBALS_REGEX=$(CHECK_GLOBALS_REGEX) + +dist_check_SCRIPTS = check-globals.sh + +TESTS = $(dist_check_SCRIPTS) diff --git a/helper/check-globals.sh b/helper/check-globals.sh new file mode 120000 index 000000000..821608c6e --- /dev/null +++ b/helper/check-globals.sh @@ -0,0 +1 @@ +../scripts/check-globals.sh
\ No newline at end of file diff --git a/helper/test/Makefile.am b/helper/test/Makefile.am index 57e54cfb8..84dbd3e03 100644 --- a/helper/test/Makefile.am +++ b/helper/test/Makefile.am @@ -65,3 +65,5 @@ clean-local: rm -f $(builddir)/$$f; \ done \ fi + +.NOTPARALLEL: diff --git a/include/odp/api/abi-default/atomic.h b/include/odp/api/abi-default/atomic.h index a58f42b8f..ce454499c 100644 --- a/include/odp/api/abi-default/atomic.h +++ b/include/odp/api/abi-default/atomic.h @@ -60,7 +60,7 @@ typedef struct ODP_ALIGNED(sizeof(uint64_t)) odp_atomic_u64_s { #endif -#ifdef _ODP_LOCK_FREE_128BIT_ATOMICS +#if defined(__SIZEOF_INT128__) || defined(_ODP_LOCK_FREE_128BIT_ATOMICS) /** * @internal diff --git a/include/odp/api/abi-default/event.h b/include/odp/api/abi-default/event.h index a63571ca0..8976252f3 100644 --- a/include/odp/api/abi-default/event.h +++ b/include/odp/api/abi-default/event.h @@ -30,7 +30,8 @@ typedef enum { ODP_EVENT_TIMEOUT = 3, ODP_EVENT_CRYPTO_COMPL = 4, ODP_EVENT_IPSEC_STATUS = 5, - ODP_EVENT_PACKET_VECTOR = 6 + ODP_EVENT_PACKET_VECTOR = 6, + ODP_EVENT_PACKET_TX_COMPL = 7, } odp_event_type_t; typedef enum { diff --git a/include/odp/api/abi-default/packet.h b/include/odp/api/abi-default/packet.h index 3660cfa29..712f83ef6 100644 --- a/include/odp/api/abi-default/packet.h +++ b/include/odp/api/abi-default/packet.h @@ -22,6 +22,9 @@ typedef struct { char dummy; /**< *internal Dummy */ } _odp_abi_packet_seg_t; /** @internal Dummy type for strong typing */ typedef struct { char dummy; /**< *internal Dummy */ } _odp_abi_packet_vector_t; +/** @internal Dummy type for strong typing */ +typedef struct { char dummy; /**< *internal Dummy */ } _odp_abi_packet_tx_compl_t; + /** @ingroup odp_packet * @{ */ @@ -29,11 +32,13 @@ typedef struct { char dummy; /**< *internal Dummy */ } _odp_abi_packet_vector_t; typedef _odp_abi_packet_t *odp_packet_t; typedef _odp_abi_packet_seg_t *odp_packet_seg_t; typedef _odp_abi_packet_vector_t *odp_packet_vector_t; +typedef _odp_abi_packet_tx_compl_t *odp_packet_tx_compl_t; #define ODP_PACKET_INVALID ((odp_packet_t)0) #define ODP_PACKET_SEG_INVALID ((odp_packet_seg_t)0) #define ODP_PACKET_OFFSET_INVALID 0xffff #define ODP_PACKET_VECTOR_INVALID ((odp_packet_vector_t)0) +#define ODP_PACKET_TX_COMPL_INVALID ((odp_packet_tx_compl_t)0) typedef uint8_t odp_proto_l2_type_t; diff --git a/include/odp/api/packet.h b/include/odp/api/packet.h index 46b5e7752..e4b2427a0 100644 --- a/include/odp/api/packet.h +++ b/include/odp/api/packet.h @@ -21,6 +21,7 @@ extern "C" { #include <odp/api/abi/event.h> #include <odp/api/abi/packet_io.h> #include <odp/api/abi/packet.h> +#include <odp/api/abi/queue.h> #include <odp/api/abi/buffer.h> #include <odp/api/abi/pool.h> diff --git a/include/odp/api/spec/classification.h b/include/odp/api/spec/classification.h index c33aca13f..31ce3e794 100644 --- a/include/odp/api/spec/classification.h +++ b/include/odp/api/spec/classification.h @@ -818,6 +818,14 @@ uint64_t odp_cos_to_u64(odp_cos_t hdl); uint64_t odp_pmr_to_u64(odp_pmr_t hdl); /** + * Print classifier info + * + * Print implementation defined information about classifier. The information is + * intended to be used for debugging. + */ +void odp_cls_print_all(void); + +/** * @} */ diff --git a/include/odp/api/spec/crypto.h b/include/odp/api/spec/crypto.h index 446d93a51..c5c3f0936 100644 --- a/include/odp/api/spec/crypto.h +++ b/include/odp/api/spec/crypto.h @@ -893,6 +893,27 @@ typedef struct odp_crypto_capability_t { /** Authentication algorithms implemented with HW offload */ odp_crypto_auth_algos_t hw_auths; + /** + * Scheduled crypto completion queue support + * + * This defines whether scheduled queues are supported as crypto + * compl_queue. + * 0: Scheduled queues are not supported as crypto completion queues + * 1: Scheduled queues are supported as crypto completion queues + * @see odp_crypto_session_param_t + */ + odp_bool_t queue_type_sched; + + /** + * Plain crypto completion queue support + * + * This defines whether plain queues are supported as crypto + * compl_queue. + * 0: Plain queues are not supported as crypto completion queues + * 1: Plain queues are supported as crypto completion queues + * @see odp_crypto_session_param_t + */ + odp_bool_t queue_type_plain; } odp_crypto_capability_t; /** diff --git a/include/odp/api/spec/event.h b/include/odp/api/spec/event.h index dc61b6e31..32fd37c29 100644 --- a/include/odp/api/spec/event.h +++ b/include/odp/api/spec/event.h @@ -60,6 +60,9 @@ extern "C" { * - IPSEC status update event (odp_ipsec_status_t) * - ODP_EVENT_PACKET_VECTOR * - Vector of packet events (odp_packet_t) as odp_packet_vector_t + * - ODP_EVENT_PACKET_TX_COMPL + * - Packet Tx completion event (odp_packet_tx_compl_t) generated as a result of a Packet Tx + * completion. */ /** diff --git a/include/odp/api/spec/packet.h b/include/odp/api/spec/packet.h index 3810c279a..d973b26a5 100644 --- a/include/odp/api/spec/packet.h +++ b/include/odp/api/spec/packet.h @@ -188,6 +188,16 @@ extern "C" { */ /** + * @typedef odp_packet_tx_compl_t + * ODP Packet Tx completion + */ + +/** + * @def ODP_PACKET_TX_COMPL_INVALID + * Invalid packet Tx completion + */ + +/** * Protocol */ typedef enum odp_proto_t { @@ -2140,6 +2150,92 @@ uint32_t odp_packet_payload_offset(odp_packet_t pkt); */ int odp_packet_payload_offset_set(odp_packet_t pkt, uint32_t offset); +/** + * Enable or disable Tx packet aging + * + * Enable or disable Tx packet drop based on packet age. When enabled, packet will be dropped + * if it is in Tx pktout queue or traffic shapers/schedulers for longer than timeout set. + * + * When tmo_ns is + * !0: Aging is enabled + * 0: Aging is disabled + * + * Aging is disabled by default. Maximum tmo value is defined by max_tx_aging_tmo_ns capa. + * + * @param pkt Packet handle + * @param tmo_ns Packet aging drop timeout in nsec. When 0, aging drop is disabled (default). + * + * @see odp_pktio_capability_t::max_tx_aging_tmo_ns + */ +void odp_packet_aging_tmo_set(odp_packet_t pkt, uint64_t tmo_ns); + +/** + * Check if packet has Tx aging drop enabled + * + * @param pkt Packet handle + * + * @return Aging drop timeout if enabled. + * @retval >0 Aging drop timeout in nano seconds and implies aging drop is enabled. + * @retval 0 If Aging drop is disabled. + */ +uint64_t odp_packet_aging_tmo(odp_packet_t pkt); + +/** Packet Tx completion mode */ +typedef enum odp_packet_tx_compl_mode_t { + /** Packet Tx completion event is disabled + * + * When mode is disabled, all other fields of odp_packet_tx_compl_opt_t are ignored. + */ + ODP_PACKET_TX_COMPL_DISABLED, + /** Packet Tx completion event is sent for all packets (both transmitted and dropped) */ + ODP_PACKET_TX_COMPL_ALL, +} odp_packet_tx_compl_mode_t; + +/** + * Tx completion request options + */ +typedef struct odp_packet_tx_compl_opt_t { + /** Queue handle + * + * Tx completion event will be posted to ODP queue identified by this handle. + */ + odp_queue_t queue; + + /** Packet Tx completion event mode */ + odp_packet_tx_compl_mode_t mode; + +} odp_packet_tx_compl_opt_t; + +/** + * Request Tx completion event. + * + * Enables or disables TX completion event request for the packet. When + * enabled, an event of type ODP_EVENT_PACKET_TX_COMPL will be sent to the + * destination queue based on the TX completion mode. The event is sent only + * after pktio interface has finished processing the packet. A previously + * enabled request can be disabled by setting the mode to + * ODP_PACKET_TX_COMPL_DISABLED. + * + * TX completion event request is disabled by default. + * + * @param pkt Packet handle + * @param opt Points to TX completion event generation options + * + * @retval 0 On success + * @retval <0 On failure + */ +int odp_packet_tx_compl_request(odp_packet_t pkt, const odp_packet_tx_compl_opt_t *opt); + +/** + * Check if TX completion event is requested for the packet + * + * @param pkt Packet handle + * + * @retval non-zero TX completion event is requested + * @retval 0 TX completion event is not requested + */ +int odp_packet_has_tx_compl_request(odp_packet_t pkt); + /* * * Packet vector handling routines @@ -2317,6 +2413,61 @@ uint64_t odp_packet_vector_to_u64(odp_packet_vector_t hdl); /* * + * Packet Tx completion event handling routines + * ******************************************************** + */ + +/** + * Get packet Tx completion handle from event + * + * Converts an ODP_EVENT_PACKET_TX_COMPL type event to packet Tx completion + * handle. + * + * @param ev Event handle + * + * @return Packet Tx completion handle + * + * @see odp_event_type() + */ +odp_packet_tx_compl_t odp_packet_tx_compl_from_event(odp_event_t ev); + +/** Convert packet Tx completion to event + * + * @param tx_compl Packet Tx completion + * + * @return Event handle + */ +odp_event_t odp_packet_tx_compl_to_event(odp_packet_tx_compl_t tx_compl); + +/** + * Free packet Tx completion + * + * Frees the packet Tx completion back to platform. It frees packet Tx + * completion that gets allocated as part of packet Tx completion request + * for a given packet. + * + * @param tx_compl Packet Tx completion handle + * + * @see odp_packet_tx_completion_request() + */ +void odp_packet_tx_compl_free(odp_packet_tx_compl_t tx_compl); + +/** + * User context pointer + * + * Return user context pointer from packet Tx completion event. + * This is the same value set to packet using odp_packet_user_ptr_set(). + * + * @param tx_compl Packet Tx completion handle + * + * @return User context pointer + * + * @see odp_packet_user_ptr_set() + */ +void *odp_packet_tx_compl_user_ptr(odp_packet_tx_compl_t tx_compl); + +/* + * * Debugging * ******************************************************** * diff --git a/include/odp/api/spec/packet_io.h b/include/odp/api/spec/packet_io.h index af3c72dde..03b7d094a 100644 --- a/include/odp/api/spec/packet_io.h +++ b/include/odp/api/spec/packet_io.h @@ -489,6 +489,22 @@ typedef union odp_pktout_config_opt_t { */ uint64_t no_packet_refs : 1; + /** Enable packet aging and drop + * + * 0: application will not request packet aging (default) + * 1: application may request packet aging + */ + uint64_t aging_ena : 1; + + /** Enable packet transmit completion event requests + * + * Use pktio capability tx_compl to check if TX completion events are supported. + * + * 0: Application will not request packet TX completion events (default) + * 1: Application may request packet TX completion events + */ + uint64_t tx_compl_ena : 1; + } bit; /** All bits of the bit field structure @@ -845,6 +861,46 @@ typedef struct odp_pktio_capability_t { uint32_t max_output; } maxlen; + /** + * Max Tx aging timeout in nano seconds supported when packet aging + * feature is supported. + * + * 0: aging is not supported + * >0: maximum aging timeout supported in nanoseconds + */ + uint64_t max_tx_aging_tmo_ns; + + /** Supported packet Tx completion options */ + struct { + /** + * Scheduled queue support + * + * This defines whether schedule queues are supported for receiving Tx + * completion events. + * + * 0: Scheduled queues are not supported for receiving Tx completion events. + * 1: Scheduled queues are supported for receiving Tx completion events. + * @see odp_packet_tx_compl_request() + */ + odp_bool_t queue_type_sched; + + /** + * Plain queue support + * + * This defines whether plain queues are supported for receiving Tx + * completion events. + * + * 0: Plain queues are not supported for receiving Tx completion events. + * 1: Plain queues are supported for receiving Tx completion events. + * @see odp_packet_tx_compl_request() + */ + odp_bool_t queue_type_plain; + + /** ODP_PACKET_TX_COMPL_ALL supported */ + uint32_t mode_all:1; + + } tx_compl; + } odp_pktio_capability_t; /** diff --git a/include/odp/api/spec/system_info.h b/include/odp/api/spec/system_info.h index 5d4e3cb87..a73263984 100644 --- a/include/odp/api/spec/system_info.h +++ b/include/odp/api/spec/system_info.h @@ -1,5 +1,5 @@ /* Copyright (c) 2013-2018, Linaro Limited - * Copyright (c) 2020, Nokia + * Copyright (c) 2020-2021, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -84,6 +84,18 @@ typedef enum odp_cpu_arch_arm_t { /** ARMv8.6-A ISA */ ODP_CPU_ARCH_ARMV8_6, + /** ARMv8.7-A ISA */ + ODP_CPU_ARCH_ARMV8_7, + + /** ARMv9.0-A ISA */ + ODP_CPU_ARCH_ARMV9_0, + + /** ARMv9.1-A ISA */ + ODP_CPU_ARCH_ARMV9_1, + + /** ARMv9.2-A ISA */ + ODP_CPU_ARCH_ARMV9_2, + } odp_cpu_arch_arm_t; /** diff --git a/m4/odp_atomic.m4 b/m4/odp_atomic.m4 index cde313261..2554502fd 100644 --- a/m4/odp_atomic.m4 +++ b/m4/odp_atomic.m4 @@ -15,6 +15,15 @@ if test "x$use_libatomic" = "xyes"; then fi AC_SUBST([ATOMIC_LIBS]) +# In non-abi-compat mode libatomic is exposed to the application +if test $ODP_ABI_COMPAT -eq 1; then + ATOMIC_LIBS_ABI_COMPAT=$ATOMIC_LIBS + AC_SUBST([ATOMIC_LIBS_ABI_COMPAT]) +else + ATOMIC_LIBS_NON_ABI_COMPAT=$ATOMIC_LIBS + AC_SUBST([ATOMIC_LIBS_NON_ABI_COMPAT]) +fi + # Double wide __atomic_compare_exchange_n is required by ipfragreass example use_libatomic_opt=no; have_atomic_cmp_exc=yes; diff --git a/platform/Makefile.inc b/platform/Makefile.inc index 475d38c37..c820727c2 100644 --- a/platform/Makefile.inc +++ b/platform/Makefile.inc @@ -17,9 +17,9 @@ lib_LTLIBRARIES = AM_LDFLAGS = -version-number '$(ODP_LIBSO_VERSION)' if ODP_ABI_COMPAT -AM_LDFLAGS += -export-symbols-regex '^(_deprecated_)?odp_' +AM_LDFLAGS += -export-symbols-regex '^(odp_|_deprecated_odp_)' else -AM_LDFLAGS += -export-symbols-regex '^(_deprecated_)?_?odp_' +AM_LDFLAGS += -export-symbols-regex '^(odp_|_odp_|_deprecated_odp_)' endif AM_CFLAGS = "-DODP_VERSION_BUILD=$(VERSION)" diff --git a/platform/linux-dpdk/Makefile.am b/platform/linux-dpdk/Makefile.am index 38a0258a0..e8c2531f0 100644 --- a/platform/linux-dpdk/Makefile.am +++ b/platform/linux-dpdk/Makefile.am @@ -151,7 +151,6 @@ BUILT_SOURCES = \ include/odp_libconfig_config.h __LIB__libodp_dpdk_la_SOURCES = \ - ../linux-generic/odp_atomic.c \ ../linux-generic/odp_barrier.c \ odp_buffer.c \ ../linux-generic/odp_chksum.c \ @@ -233,23 +232,29 @@ __LIB__libodp_dpdk_la_SOURCES += \ endif if ARCH_IS_ARM -__LIB__libodp_dpdk_la_SOURCES += arch/default/odp_cpu_cycles.c \ - arch/default/odp_global_time.c \ +__LIB__libodp_dpdk_la_SOURCES += arch/default/odp_atomic.c \ + arch/default/odp_cpu_cycles.c \ arch/default/odp_hash_crc32.c \ arch/arm/odp_sysinfo_parse.c odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_inlines.h \ arch/default/odp/api/abi/cpu_time.h \ arch/default/odp/api/abi/hash_crc32.h if !ODP_ABI_COMPAT -odpapiabiarchinclude_HEADERS += arch/arm/odp/api/abi/cpu.h +odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ + arch/default/odp/api/abi/atomic_inlines.h \ + arch/arm/odp/api/abi/cpu.h endif noinst_HEADERS += arch/arm/odp_atomic.h \ arch/arm/odp_cpu.h \ arch/arm/odp_cpu_idling.h \ - arch/arm/odp_llsc.h + arch/arm/odp_llsc.h \ + arch/default/odp_atomic.h \ + arch/default/odp_cpu.h \ + arch/default/odp_cpu_idling.h endif if ARCH_IS_AARCH64 -__LIB__libodp_dpdk_la_SOURCES += arch/default/odp_cpu_cycles.c \ +__LIB__libodp_dpdk_la_SOURCES += arch/aarch64/odp_atomic.c \ + arch/default/odp_cpu_cycles.c \ arch/aarch64/odp_global_time.c \ arch/default/odp_hash_crc32.c \ arch/aarch64/odp_sysinfo_parse.c @@ -257,7 +262,9 @@ odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_inlines.h \ arch/default/odp/api/abi/cpu_time.h \ arch/aarch64/odp/api/abi/hash_crc32.h if !ODP_ABI_COMPAT -odpapiabiarchinclude_HEADERS += arch/aarch64/odp/api/abi/atomic.h \ +odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ + arch/aarch64/odp/api/abi/atomic_inlines.h \ + arch/aarch64/odp/api/abi/atomic.h \ arch/aarch64/odp/api/abi/cpu.h endif noinst_HEADERS += arch/aarch64/odp_atomic.h \ @@ -266,7 +273,8 @@ noinst_HEADERS += arch/aarch64/odp_atomic.h \ arch/aarch64/odp_llsc.h endif if ARCH_IS_DEFAULT -__LIB__libodp_dpdk_la_SOURCES += arch/default/odp_cpu_cycles.c \ +__LIB__libodp_dpdk_la_SOURCES += arch/default/odp_atomic.c \ + arch/default/odp_cpu_cycles.c \ arch/default/odp_global_time.c \ arch/default/odp_hash_crc32.c \ arch/default/odp_sysinfo_parse.c @@ -274,13 +282,17 @@ odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_inlines.h \ arch/default/odp/api/abi/cpu_time.h \ arch/default/odp/api/abi/hash_crc32.h if !ODP_ABI_COMPAT -odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu.h +odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ + arch/default/odp/api/abi/atomic_inlines.h \ + arch/default/odp/api/abi/cpu.h endif -noinst_HEADERS += arch/default/odp_cpu.h \ +noinst_HEADERS += arch/default/odp_atomic.h \ + arch/default/odp_cpu.h \ arch/default/odp_cpu_idling.h endif if ARCH_IS_MIPS64 -__LIB__libodp_dpdk_la_SOURCES += arch/mips64/odp_cpu_cycles.c \ +__LIB__libodp_dpdk_la_SOURCES += arch/default/odp_atomic.c \ + arch/mips64/odp_cpu_cycles.c \ arch/default/odp_global_time.c \ arch/default/odp_hash_crc32.c \ arch/mips64/odp_sysinfo_parse.c @@ -288,13 +300,17 @@ odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_inlines.h \ arch/default/odp/api/abi/cpu_time.h \ arch/default/odp/api/abi/hash_crc32.h if !ODP_ABI_COMPAT -odpapiabiarchinclude_HEADERS += arch/mips64/odp/api/abi/cpu.h +odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ + arch/default/odp/api/abi/atomic_inlines.h \ + arch/mips64/odp/api/abi/cpu.h endif -noinst_HEADERS += arch/default/odp_cpu.h \ +noinst_HEADERS += arch/default/odp_atomic.h \ + arch/default/odp_cpu.h \ arch/default/odp_cpu_idling.h endif if ARCH_IS_POWERPC -__LIB__libodp_dpdk_la_SOURCES += arch/default/odp_cpu_cycles.c \ +__LIB__libodp_dpdk_la_SOURCES += arch/default/odp_atomic.c \ + arch/default/odp_cpu_cycles.c \ arch/default/odp_global_time.c \ arch/default/odp_hash_crc32.c \ arch/powerpc/odp_sysinfo_parse.c @@ -302,13 +318,17 @@ odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_inlines.h \ arch/default/odp/api/abi/cpu_time.h \ arch/default/odp/api/abi/hash_crc32.h if !ODP_ABI_COMPAT -odpapiabiarchinclude_HEADERS += arch/powerpc/odp/api/abi/cpu.h +odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ + arch/default/odp/api/abi/atomic_inlines.h \ + arch/powerpc/odp/api/abi/cpu.h endif -noinst_HEADERS += arch/default/odp_cpu.h \ +noinst_HEADERS += arch/default/odp_atomic.h \ + arch/default/odp_cpu.h \ arch/default/odp_cpu_idling.h endif if ARCH_IS_X86 -__LIB__libodp_dpdk_la_SOURCES += arch/x86/cpu_flags.c \ +__LIB__libodp_dpdk_la_SOURCES += arch/default/odp_atomic.c \ + arch/x86/cpu_flags.c \ arch/x86/odp_cpu_cycles.c \ arch/x86/odp_global_time.c \ arch/default/odp_hash_crc32.c \ @@ -318,10 +338,13 @@ odpapiabiarchinclude_HEADERS += arch/x86/odp/api/abi/cpu_inlines.h \ arch/x86/odp/api/abi/cpu_time.h \ arch/default/odp/api/abi/hash_crc32.h if !ODP_ABI_COMPAT -odpapiabiarchinclude_HEADERS += arch/x86/odp/api/abi/cpu.h +odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ + arch/default/odp/api/abi/atomic_inlines.h \ + arch/x86/odp/api/abi/cpu.h endif noinst_HEADERS += arch/x86/cpu_flags.h \ arch/x86/odp_cpu.h \ + arch/default/odp_atomic.h \ arch/default/odp_cpu.h \ arch/default/odp_cpu_idling.h endif @@ -332,3 +355,16 @@ __LIB__libodp_dpdk_la_LIBADD += $(LIBCONFIG_LIBS) __LIB__libodp_dpdk_la_LIBADD += $(DPDK_LIBS_LIBODP) __LIB__libodp_dpdk_la_LIBADD += $(PTHREAD_LIBS) __LIB__libodp_dpdk_la_LIBADD += $(TIMER_LIBS) + +CHECK_GLOBALS_REGEX = " (odp_|_odp_|_deprecated_odp_|miniz_|mz_|tdefl_|tinfl_|mp_hdlr_init_odp_pool_ops)" + +TESTS_ENVIRONMENT = \ + LIBTOOL="$(LIBTOOL)" \ + NM="$(NM)" \ + LIB="$(LIB)" \ + lib_LTLIBRARIES="$(lib_LTLIBRARIES)" \ + CHECK_GLOBALS_REGEX=$(CHECK_GLOBALS_REGEX) + +dist_check_SCRIPTS = check-globals.sh + +TESTS = $(dist_check_SCRIPTS) diff --git a/platform/linux-dpdk/Makefile.inc b/platform/linux-dpdk/Makefile.inc index 241eb5eac..6e153015b 100644 --- a/platform/linux-dpdk/Makefile.inc +++ b/platform/linux-dpdk/Makefile.inc @@ -3,4 +3,4 @@ AM_CXXFLAGS += $(DPDK_CFLAGS) LOG_COMPILER = $(top_builddir)/platform/linux-dpdk/test/wrapper-script.sh SH_LOG_COMPILER = $(LOG_COMPILER) -dist_check_SCRIPTS = $(top_builddir)/platform/linux-dpdk/test/wrapper-script.sh +EXTRA_DIST += $(top_builddir)/platform/linux-dpdk/test/wrapper-script.sh diff --git a/platform/linux-dpdk/check-globals.sh b/platform/linux-dpdk/check-globals.sh new file mode 120000 index 000000000..c999a29ef --- /dev/null +++ b/platform/linux-dpdk/check-globals.sh @@ -0,0 +1 @@ +../../scripts/check-globals.sh
\ No newline at end of file diff --git a/platform/linux-dpdk/include-abi/odp/api/abi/packet.h b/platform/linux-dpdk/include-abi/odp/api/abi/packet.h index e99f0d49f..3b55e2693 100644 --- a/platform/linux-dpdk/include-abi/odp/api/abi/packet.h +++ b/platform/linux-dpdk/include-abi/odp/api/abi/packet.h @@ -38,6 +38,10 @@ typedef ODP_HANDLE_T(odp_packet_vector_t); #define ODP_PACKET_VECTOR_INVALID _odp_cast_scalar(odp_packet_vector_t, 0) +typedef ODP_HANDLE_T(odp_packet_tx_compl_t); + +#define ODP_PACKET_TX_COMPL_INVALID _odp_cast_scalar(odp_packet_tx_compl_t, 0) + typedef uint8_t odp_proto_l2_type_t; #define ODP_PROTO_L2_TYPE_NONE 0 diff --git a/platform/linux-dpdk/include/odp_config_internal.h b/platform/linux-dpdk/include/odp_config_internal.h index fd3d075ec..73d2304c9 100644 --- a/platform/linux-dpdk/include/odp_config_internal.h +++ b/platform/linux-dpdk/include/odp_config_internal.h @@ -1,5 +1,5 @@ /* Copyright (c) 2016-2018, Linaro Limited - * Copyright (c) 2020, Nokia + * Copyright (c) 2020-2021, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -147,6 +147,13 @@ extern "C" { /* Maximum packet vector size */ #define CONFIG_PACKET_VECTOR_MAX_SIZE 256 +/* + * Maximum number of IPsec SAs. The actual maximum number can be further + * limited by the number of sessions supported by the crypto subsystem and + * is reported by odp_ipsec_capability(). + */ +#define CONFIG_IPSEC_MAX_NUM_SA 4000 + #ifdef __cplusplus } #endif diff --git a/platform/linux-dpdk/include/odp_errno_define.h b/platform/linux-dpdk/include/odp_errno_define.h index 4af5e1b98..9f9486267 100644 --- a/platform/linux-dpdk/include/odp_errno_define.h +++ b/platform/linux-dpdk/include/odp_errno_define.h @@ -19,7 +19,7 @@ extern "C" { #include <rte_errno.h> -#define __odp_errno (rte_errno) +#define _odp_errno (rte_errno) #ifdef __cplusplus } diff --git a/platform/linux-dpdk/include/odp_eventdev_internal.h b/platform/linux-dpdk/include/odp_eventdev_internal.h index 29e9d7a58..496a2238f 100644 --- a/platform/linux-dpdk/include/odp_eventdev_internal.h +++ b/platform/linux-dpdk/include/odp_eventdev_internal.h @@ -154,15 +154,15 @@ typedef struct { extern eventdev_global_t *_odp_eventdev_gbl; extern __thread eventdev_local_t _odp_eventdev_local; -int service_setup(uint32_t service_id); +int _odp_service_setup(uint32_t service_id); -int dummy_link_queues(uint8_t dev_id, uint8_t dummy_linked_queues[], int num); +int _odp_dummy_link_queues(uint8_t dev_id, uint8_t dummy_linked_queues[], int num); -int dummy_unlink_queues(uint8_t dev_id, uint8_t dummy_linked_queues[], int num); +int _odp_dummy_unlink_queues(uint8_t dev_id, uint8_t dummy_linked_queues[], int num); -void rx_adapter_port_stop(uint16_t port_id); +void _odp_rx_adapter_port_stop(uint16_t port_id); -int rx_adapter_close(void); +int _odp_rx_adapter_close(void); static inline uint8_t event_schedule_type(odp_schedule_sync_t sync) { diff --git a/platform/linux-dpdk/include/odp_packet_io_internal.h b/platform/linux-dpdk/include/odp_packet_io_internal.h index 631e36fb2..fe8d4770c 100644 --- a/platform/linux-dpdk/include/odp_packet_io_internal.h +++ b/platform/linux-dpdk/include/odp_packet_io_internal.h @@ -234,9 +234,9 @@ static inline void pktio_cls_enabled_set(pktio_entry_t *entry, int ena) entry->s.enabled.cls = !!ena; } -uint16_t dpdk_pktio_port_id(pktio_entry_t *entry); +uint16_t _odp_dpdk_pktio_port_id(pktio_entry_t *entry); -int input_pkts(pktio_entry_t *pktio_entry, odp_packet_t pkt_table[], int num); +int _odp_input_pkts(pktio_entry_t *pktio_entry, odp_packet_t pkt_table[], int num); static inline int _odp_pktio_tx_ts_enabled(pktio_entry_t *entry) { @@ -253,7 +253,7 @@ static inline void _odp_pktio_tx_ts_set(pktio_entry_t *entry) extern const pktio_if_ops_t _odp_loopback_pktio_ops; extern const pktio_if_ops_t _odp_null_pktio_ops; extern const pktio_if_ops_t _odp_dpdk_pktio_ops; -extern const pktio_if_ops_t * const pktio_if_ops[]; +extern const pktio_if_ops_t * const _odp_pktio_if_ops[]; /* Dummy function required by odp_pktin_recv_mq_tmo() */ static inline int diff --git a/platform/linux-dpdk/include/odp_pool_internal.h b/platform/linux-dpdk/include/odp_pool_internal.h index d0682ec43..0db8cf056 100644 --- a/platform/linux-dpdk/include/odp_pool_internal.h +++ b/platform/linux-dpdk/include/odp_pool_internal.h @@ -92,7 +92,7 @@ static inline pool_t *pool_entry_from_hdl(odp_pool_t pool_hdl) return &_odp_pool_glb->pool[_odp_typeval(pool_hdl) - 1]; } -static inline void buffer_free_multi(odp_buffer_hdr_t *buf_hdr[], int num) +static inline void _odp_buffer_free_multi(odp_buffer_hdr_t *buf_hdr[], int num) { int i; diff --git a/platform/linux-dpdk/libodp-dpdk.pc.in b/platform/linux-dpdk/libodp-dpdk.pc.in index 729191d5f..c3ee4f7b9 100644 --- a/platform/linux-dpdk/libodp-dpdk.pc.in +++ b/platform/linux-dpdk/libodp-dpdk.pc.in @@ -7,6 +7,6 @@ Name: lib@ODP_LIB_NAME@ Description: The ODP packet processing engine Version: @PKGCONFIG_VERSION@ Requires.private: libconfig -Libs: -L${libdir} -l@ODP_LIB_NAME@ @DPDK_LIBS_NON_ABI_COMPAT@ -Libs.private: @DPDK_LIBS_ABI_COMPAT@ @OPENSSL_STATIC_LIBS@ @PTHREAD_LIBS@ @TIMER_LIBS@ -lpthread @ATOMIC_LIBS@ +Libs: -L${libdir} -l@ODP_LIB_NAME@ @DPDK_LIBS_NON_ABI_COMPAT@ @ATOMIC_LIBS_NON_ABI_COMPAT@ +Libs.private: @DPDK_LIBS_ABI_COMPAT@ @OPENSSL_STATIC_LIBS@ @PTHREAD_LIBS@ @TIMER_LIBS@ -lpthread @ATOMIC_LIBS_ABI_COMPAT@ Cflags: -I${includedir} @DPDK_CFLAGS@ diff --git a/platform/linux-dpdk/m4/configure.m4 b/platform/linux-dpdk/m4/configure.m4 index b86bbbbb5..2006c0834 100644 --- a/platform/linux-dpdk/m4/configure.m4 +++ b/platform/linux-dpdk/m4/configure.m4 @@ -61,7 +61,7 @@ esac # Required for experimental rte_event_port_unlinks_in_progress() API DPDK_CFLAGS="${DPDK_CFLAGS} -DALLOW_EXPERIMENTAL_API" -AS_VAR_APPEND([PLAT_DEP_LIBS], ["${LIBCONFIG_LIBS} ${OPENSSL_LIBS} ${DPDK_LIBS_LT} ${LIBCLI_LIBS}"]) +AS_VAR_APPEND([PLAT_DEP_LIBS], ["${ATOMIC_LIBS} ${LIBCONFIG_LIBS} ${OPENSSL_LIBS} ${DPDK_LIBS_LT} ${LIBCLI_LIBS}"]) # Add text to the end of configure with platform specific settings. # Make sure it's aligned same as other lines in configure.ac. diff --git a/platform/linux-dpdk/odp_crypto.c b/platform/linux-dpdk/odp_crypto.c index 1ae54bb4a..96e6b00d3 100644 --- a/platform/linux-dpdk/odp_crypto.c +++ b/platform/linux-dpdk/odp_crypto.c @@ -623,6 +623,8 @@ int odp_crypto_capability(odp_crypto_capability_t *capability) capability->sync_mode = ODP_SUPPORT_YES; capability->async_mode = ODP_SUPPORT_PREFERRED; capability->max_sessions = MAX_SESSIONS; + capability->queue_type_plain = 1; + capability->queue_type_sched = 1; for (cdev_id = 0; cdev_id < cdev_count; cdev_id++) { struct rte_cryptodev_info dev_info; diff --git a/platform/linux-dpdk/odp_packet.c b/platform/linux-dpdk/odp_packet.c index e65a08d0e..d1b7558e5 100644 --- a/platform/linux-dpdk/odp_packet.c +++ b/platform/linux-dpdk/odp_packet.c @@ -61,18 +61,18 @@ const _odp_packet_inline_offset_t _odp_packet_inline ODP_ALIGNED_CACHE = { #include <odp/visibility_end.h> /* Catch if DPDK mbuf members sizes have changed */ -struct rte_mbuf dummy; -ODP_STATIC_ASSERT(sizeof(dummy.data_off) == sizeof(uint16_t), +struct rte_mbuf _odp_dummy_mbuf; +ODP_STATIC_ASSERT(sizeof(_odp_dummy_mbuf.data_off) == sizeof(uint16_t), "data_off should be uint16_t"); -ODP_STATIC_ASSERT(sizeof(dummy.pkt_len) == sizeof(uint32_t), +ODP_STATIC_ASSERT(sizeof(_odp_dummy_mbuf.pkt_len) == sizeof(uint32_t), "pkt_len should be uint32_t"); -ODP_STATIC_ASSERT(sizeof(dummy.data_len) == sizeof(uint16_t), +ODP_STATIC_ASSERT(sizeof(_odp_dummy_mbuf.data_len) == sizeof(uint16_t), "data_len should be uint16_t"); -ODP_STATIC_ASSERT(sizeof(dummy.nb_segs) == sizeof(uint16_t), +ODP_STATIC_ASSERT(sizeof(_odp_dummy_mbuf.nb_segs) == sizeof(uint16_t), "nb_segs should be uint16_t"); -ODP_STATIC_ASSERT(sizeof(dummy.hash.rss) == sizeof(uint32_t), +ODP_STATIC_ASSERT(sizeof(_odp_dummy_mbuf.hash.rss) == sizeof(uint32_t), "hash.rss should be uint32_t"); -ODP_STATIC_ASSERT(sizeof(dummy.ol_flags) == sizeof(uint64_t), +ODP_STATIC_ASSERT(sizeof(_odp_dummy_mbuf.ol_flags) == sizeof(uint64_t), "ol_flags should be uint64_t"); /* Check that invalid values are the same. Some versions of Clang have trouble @@ -118,7 +118,7 @@ static odp_packet_t packet_alloc(pool_t *pool, uint32_t len) struct rte_mbuf *mbuf = rte_pktmbuf_alloc(pool->rte_mempool); if (odp_unlikely(mbuf == NULL)) { - __odp_errno = ENOMEM; + _odp_errno = ENOMEM; return ODP_PACKET_INVALID; } pkt_hdr = (odp_packet_hdr_t *)mbuf; @@ -133,13 +133,13 @@ static odp_packet_t packet_alloc(pool_t *pool, uint32_t len) /* Check num_seg here so rte_pktmbuf_chain() always succeeds */ if (odp_unlikely(num_seg > RTE_MBUF_MAX_NB_SEGS)) { - __odp_errno = EOVERFLOW; + _odp_errno = EOVERFLOW; return ODP_PACKET_INVALID; } ret = rte_pktmbuf_alloc_bulk(pool->rte_mempool, mbufs, num_seg); if (odp_unlikely(ret)) { - __odp_errno = ENOMEM; + _odp_errno = ENOMEM; return ODP_PACKET_INVALID; } @@ -171,7 +171,7 @@ odp_packet_t odp_packet_alloc(odp_pool_t pool_hdl, uint32_t len) pool_t *pool = pool_entry_from_hdl(pool_hdl); if (odp_unlikely(pool->params.type != ODP_POOL_PACKET)) { - __odp_errno = EINVAL; + _odp_errno = EINVAL; return ODP_PACKET_INVALID; } @@ -188,7 +188,7 @@ int odp_packet_alloc_multi(odp_pool_t pool_hdl, uint32_t len, pool_t *pool = pool_entry_from_hdl(pool_hdl); if (odp_unlikely(pool->params.type != ODP_POOL_PACKET)) { - __odp_errno = EINVAL; + _odp_errno = EINVAL; return -1; } @@ -198,7 +198,7 @@ int odp_packet_alloc_multi(odp_pool_t pool_hdl, uint32_t len, for (i = 0; i < num; i++) { pkt[i] = packet_alloc(pool, len); if (odp_unlikely(pkt[i] == ODP_PACKET_INVALID)) - return (i == 0 && __odp_errno != ENOMEM) ? -1 : i; + return (i == 0 && _odp_errno != ENOMEM) ? -1 : i; } return i; } @@ -2321,3 +2321,56 @@ int odp_packet_payload_offset_set(odp_packet_t pkt, uint32_t offset) return 0; } + +void odp_packet_aging_tmo_set(odp_packet_t pkt, uint64_t tmo_ns) +{ + (void)pkt; + (void)tmo_ns; +} + +uint64_t odp_packet_aging_tmo(odp_packet_t pkt) +{ + (void)pkt; + return 0; +} + +int odp_packet_tx_compl_request(odp_packet_t pkt, const odp_packet_tx_compl_opt_t *opt) +{ + (void)pkt; + (void)opt; + + return -1; +} + +int odp_packet_has_tx_compl_request(odp_packet_t pkt) +{ + (void)pkt; + + return 0; +} + +odp_packet_tx_compl_t odp_packet_tx_compl_from_event(odp_event_t ev) +{ + (void)ev; + + return ODP_PACKET_TX_COMPL_INVALID; +} + +odp_event_t odp_packet_tx_compl_to_event(odp_packet_tx_compl_t tx_compl) +{ + (void)tx_compl; + + return ODP_EVENT_INVALID; +} + +void odp_packet_tx_compl_free(odp_packet_tx_compl_t tx_compl) +{ + (void)tx_compl; +} + +void *odp_packet_tx_compl_user_ptr(odp_packet_tx_compl_t tx_compl) +{ + (void)tx_compl; + + return NULL; +} diff --git a/platform/linux-dpdk/odp_packet_dpdk.c b/platform/linux-dpdk/odp_packet_dpdk.c index 4ef7bdc0c..7fd4de06d 100644 --- a/platform/linux-dpdk/odp_packet_dpdk.c +++ b/platform/linux-dpdk/odp_packet_dpdk.c @@ -129,7 +129,7 @@ static inline pkt_dpdk_t *pkt_priv(pktio_entry_t *pktio_entry) * Order matters. The first implementation to setup successfully * will be picked. * Array must be NULL terminated */ -const pktio_if_ops_t * const pktio_if_ops[] = { +const pktio_if_ops_t * const _odp_pktio_if_ops[] = { &_odp_loopback_pktio_ops, &_odp_dpdk_pktio_ops, &_odp_null_pktio_ops, @@ -140,7 +140,7 @@ extern void *pktio_entry_ptr[ODP_CONFIG_PKTIO_ENTRIES]; static uint32_t mtu_get_pkt_dpdk(pktio_entry_t *pktio_entry); -uint16_t dpdk_pktio_port_id(pktio_entry_t *pktio_entry) +uint16_t _odp_dpdk_pktio_port_id(pktio_entry_t *pktio_entry) { const pkt_dpdk_t *pkt_dpdk = pkt_priv(pktio_entry); @@ -693,7 +693,7 @@ static int close_pkt_dpdk(pktio_entry_t *pktio_entry) if (_odp_eventdev_gbl && _odp_eventdev_gbl->rx_adapter.status != RX_ADAPTER_INIT) - rx_adapter_port_stop(pkt_dpdk->port_id); + _odp_rx_adapter_port_stop(pkt_dpdk->port_id); else rte_eth_dev_stop(pkt_dpdk->port_id); @@ -873,7 +873,7 @@ static inline void prefetch_pkt(odp_packet_t pkt) odp_prefetch(&pkt_hdr->p); } -int input_pkts(pktio_entry_t *pktio_entry, odp_packet_t pkt_table[], int num) +int _odp_input_pkts(pktio_entry_t *pktio_entry, odp_packet_t pkt_table[], int num) { pkt_dpdk_t * const pkt_dpdk = pkt_priv(pktio_entry); uint16_t i; @@ -1012,10 +1012,10 @@ static int recv_pkt_dpdk(pktio_entry_t *pktio_entry, int index, odp_ticketlock_unlock(&pkt_dpdk->rx_lock[index]); /* Packets may also me received through eventdev, so don't add any - * processing here. Instead, perform all processing in input_pkts() + * processing here. Instead, perform all processing in _odp_input_pkts() * which is also called by eventdev. */ if (nb_rx) - return input_pkts(pktio_entry, pkt_table, nb_rx); + return _odp_input_pkts(pktio_entry, pkt_table, nb_rx); return 0; } @@ -1177,7 +1177,7 @@ static int send_pkt_dpdk(pktio_entry_t *pktio_entry, int index, return -1; if (odp_unlikely(mbuf->pkt_len > pkt_dpdk->mtu)) { - __odp_errno = EMSGSIZE; + _odp_errno = EMSGSIZE; return -1; } } else if (odp_unlikely(tx_ts_idx && pkts >= tx_ts_idx)) { diff --git a/platform/linux-dpdk/odp_pool.c b/platform/linux-dpdk/odp_pool.c index e36fff903..24c9c2aec 100644 --- a/platform/linux-dpdk/odp_pool.c +++ b/platform/linux-dpdk/odp_pool.c @@ -855,7 +855,7 @@ void odp_buffer_free(odp_buffer_t buf) void odp_buffer_free_multi(const odp_buffer_t buf[], int num) { - buffer_free_multi((odp_buffer_hdr_t **)(uintptr_t)buf, num); + _odp_buffer_free_multi((odp_buffer_hdr_t **)(uintptr_t)buf, num); } void odp_pool_print(odp_pool_t pool_hdl) diff --git a/platform/linux-dpdk/odp_queue_basic.c b/platform/linux-dpdk/odp_queue_basic.c index 4cf619965..4a4388b2b 100644 --- a/platform/linux-dpdk/odp_queue_basic.c +++ b/platform/linux-dpdk/odp_queue_basic.c @@ -144,6 +144,7 @@ static int queue_init_global(void) for (i = 0; i < CONFIG_MAX_QUEUES; i++) { /* init locks */ queue_entry_t *queue = qentry_from_index(i); + LOCK_INIT(queue); queue->s.index = i; queue->s.handle = (odp_queue_t)queue; @@ -370,6 +371,7 @@ static int queue_destroy(odp_queue_t handle) { int empty; queue_entry_t *queue; + queue = qentry_from_handle(handle); if (handle == ODP_QUEUE_INVALID) @@ -1168,7 +1170,7 @@ static odp_event_t queue_api_deq(odp_queue_t handle) } /* API functions */ -_odp_queue_api_fn_t queue_basic_api = { +_odp_queue_api_fn_t _odp_queue_basic_api = { .queue_create = queue_create, .queue_destroy = queue_destroy, .queue_lookup = queue_lookup, @@ -1192,7 +1194,7 @@ _odp_queue_api_fn_t queue_basic_api = { }; /* Functions towards internal components */ -queue_fn_t queue_basic_fn = { +queue_fn_t _odp_queue_basic_fn = { .init_global = queue_init_global, .term_global = queue_term_global, .init_local = queue_init_local, diff --git a/platform/linux-dpdk/odp_queue_eventdev.c b/platform/linux-dpdk/odp_queue_eventdev.c index 195a0fc68..5233031f6 100644 --- a/platform/linux-dpdk/odp_queue_eventdev.c +++ b/platform/linux-dpdk/odp_queue_eventdev.c @@ -185,7 +185,7 @@ static void print_dev_info(const struct rte_event_dev_info *info) info->event_dev_cap); } -int service_setup(uint32_t service_id) +int _odp_service_setup(uint32_t service_id) { uint32_t cores[RTE_MAX_LCORE]; uint32_t lcore = 0; @@ -315,7 +315,7 @@ static int queue_is_linked(uint8_t dev_id, uint8_t queue_id) } /* Dummy link all unlinked queues to port zero to pass evendev start */ -int dummy_link_queues(uint8_t dev_id, uint8_t dummy_linked_queues[], int num) +int _odp_dummy_link_queues(uint8_t dev_id, uint8_t dummy_linked_queues[], int num) { uint8_t priority = RTE_EVENT_DEV_PRIORITY_NORMAL; uint8_t queue_id; @@ -337,7 +337,7 @@ int dummy_link_queues(uint8_t dev_id, uint8_t dummy_linked_queues[], int num) } /* Remove dummy links to port zero */ -int dummy_unlink_queues(uint8_t dev_id, uint8_t dummy_linked_queues[], int num) +int _odp_dummy_unlink_queues(uint8_t dev_id, uint8_t dummy_linked_queues[], int num) { int i; @@ -465,8 +465,8 @@ static int init_event_dev(void) /* Eventdev requires that each queue is linked to at least one * port at startup. */ - num_dummy_links = dummy_link_queues(dev_id, dummy_links, - config.nb_event_queues); + num_dummy_links = _odp_dummy_link_queues(dev_id, dummy_links, + config.nb_event_queues); if (!(info.event_dev_cap & RTE_EVENT_DEV_CAP_DISTRIBUTED_SCHED)) { uint32_t service_id; @@ -476,7 +476,7 @@ static int init_event_dev(void) ODP_ERR("Unable to retrieve service ID\n"); return -1; } - if (service_setup(service_id)) { + if (_odp_service_setup(service_id)) { ODP_ERR("Failed to setup service core\n"); return -1; } @@ -489,7 +489,7 @@ static int init_event_dev(void) /* Unlink all ports from queues. Thread specific ports will be linked * when the application calls schedule/enqueue for the first time. */ - if (dummy_unlink_queues(dev_id, dummy_links, num_dummy_links)) { + if (_odp_dummy_unlink_queues(dev_id, dummy_links, num_dummy_links)) { rte_event_dev_stop(dev_id); rte_event_dev_close(dev_id); return -1; @@ -602,7 +602,7 @@ static int queue_term_global(void) UNLOCK(queue); } - if (rx_adapter_close()) + if (_odp_rx_adapter_close()) ret = -1; rte_event_dev_stop(_odp_eventdev_gbl->dev_id); @@ -1345,7 +1345,7 @@ static odp_event_t queue_api_deq(odp_queue_t handle) } /* API functions */ -_odp_queue_api_fn_t queue_eventdev_api = { +_odp_queue_api_fn_t _odp_queue_eventdev_api = { .queue_create = queue_create, .queue_destroy = queue_destroy, .queue_lookup = queue_lookup, @@ -1367,7 +1367,7 @@ _odp_queue_api_fn_t queue_eventdev_api = { }; /* Functions towards internal components */ -queue_fn_t queue_eventdev_fn = { +queue_fn_t _odp_queue_eventdev_fn = { .init_global = queue_init_global, .term_global = queue_term_global, .init_local = queue_init_local, diff --git a/platform/linux-dpdk/odp_queue_if.c b/platform/linux-dpdk/odp_queue_if.c index 310664fb8..3768dca54 100644 --- a/platform/linux-dpdk/odp_queue_if.c +++ b/platform/linux-dpdk/odp_queue_if.c @@ -23,11 +23,11 @@ const _odp_queue_api_fn_t *_odp_queue_api; #include <odp/visibility_end.h> -extern const _odp_queue_api_fn_t queue_basic_api; -extern const queue_fn_t queue_basic_fn; +extern const _odp_queue_api_fn_t _odp_queue_basic_api; +extern const queue_fn_t _odp_queue_basic_fn; -extern const _odp_queue_api_fn_t queue_eventdev_api; -extern const queue_fn_t queue_eventdev_fn; +extern const _odp_queue_api_fn_t _odp_queue_eventdev_api; +extern const queue_fn_t _odp_queue_eventdev_fn; const queue_fn_t *_odp_queue_fn; @@ -114,11 +114,11 @@ int _odp_queue_init_global(void) sched = _ODP_SCHEDULE_DEFAULT; if (!strcmp(sched, "basic") || !strcmp(sched, "sp")) { - _odp_queue_fn = &queue_basic_fn; - _odp_queue_api = &queue_basic_api; + _odp_queue_fn = &_odp_queue_basic_fn; + _odp_queue_api = &_odp_queue_basic_api; } else if (!strcmp(sched, "eventdev")) { - _odp_queue_fn = &queue_eventdev_fn; - _odp_queue_api = &queue_eventdev_api; + _odp_queue_fn = &_odp_queue_eventdev_fn; + _odp_queue_api = &_odp_queue_eventdev_api; } else { ODP_ABORT("Unknown scheduler specified via ODP_SCHEDULER\n"); return -1; diff --git a/platform/linux-dpdk/odp_schedule_eventdev.c b/platform/linux-dpdk/odp_schedule_eventdev.c index 4ab235d70..f7b6bbb8f 100644 --- a/platform/linux-dpdk/odp_schedule_eventdev.c +++ b/platform/linux-dpdk/odp_schedule_eventdev.c @@ -242,8 +242,8 @@ static int rx_adapter_add_queues(uint8_t rx_adapter_id, uint8_t port_id, int i; /* SW eventdev requires that all queues have ports linked */ - num_dummy_links = dummy_link_queues(_odp_eventdev_gbl->dev_id, dummy_links, - num_dummy_links); + num_dummy_links = _odp_dummy_link_queues(_odp_eventdev_gbl->dev_id, dummy_links, + num_dummy_links); for (i = 0; i < num_pktin; i++) { queue_entry_t *queue = qentry_from_handle(queues[i]); @@ -277,14 +277,14 @@ static int rx_adapter_add_queues(uint8_t rx_adapter_id, uint8_t port_id, break; } - if (dummy_unlink_queues(_odp_eventdev_gbl->dev_id, dummy_links, - num_dummy_links)) + if (_odp_dummy_unlink_queues(_odp_eventdev_gbl->dev_id, dummy_links, + num_dummy_links)) return -1; return ret; } -int rx_adapter_close(void) +int _odp_rx_adapter_close(void) { uint16_t port_id; uint8_t rx_adapter_id = _odp_eventdev_gbl->rx_adapter.id; @@ -308,7 +308,7 @@ int rx_adapter_close(void) return ret; } -void rx_adapter_port_stop(uint16_t port_id) +void _odp_rx_adapter_port_stop(uint16_t port_id) { uint8_t rx_adapter_id = _odp_eventdev_gbl->rx_adapter.id; @@ -415,7 +415,7 @@ static void schedule_pktio_start(int pktio_index, int num_pktin, int pktin_idx[], odp_queue_t queue[]) { pktio_entry_t *entry = get_pktio_entry(index_to_pktio(pktio_index)); - uint16_t port_id = dpdk_pktio_port_id(entry); + uint16_t port_id = _odp_dpdk_pktio_port_id(entry); uint8_t rx_adapter_id = _odp_eventdev_gbl->rx_adapter.id; /* All eventdev pktio devices should to be started before calling @@ -449,7 +449,7 @@ static void schedule_pktio_start(int pktio_index, int num_pktin, if (ret && ret != -ESRCH) { ODP_ABORT("Unable to retrieve service ID\n"); } else if (!ret) { - if (service_setup(service_id)) + if (_odp_service_setup(service_id)) ODP_ABORT("Unable to start RX service\n"); } @@ -539,7 +539,12 @@ static inline uint16_t event_input(struct rte_event ev[], odp_event_t out_ev[], uint16_t num_pkts = 0; uint16_t num_events = 0; uint16_t i; - uint8_t first_queue = ev[0].queue_id; + uint8_t first_queue; + + if (odp_unlikely(nb_events == 0)) + return 0; + + first_queue = ev[0].queue_id; for (i = 0; i < nb_events; i++) { struct rte_event *event = &ev[i]; @@ -568,8 +573,7 @@ static inline uint16_t event_input(struct rte_event ev[], odp_event_t out_ev[], if (num_pkts) { pktio_entry_t *entry = _odp_eventdev_gbl->pktio[pkt_table[0]->port]; - num_pkts = input_pkts(entry, (odp_packet_t *)pkt_table, - num_pkts); + num_pkts = _odp_input_pkts(entry, (odp_packet_t *)pkt_table, num_pkts); if (!odp_global_ro.init_param.not_used.feat.cls) num_pkts = classify_pkts((odp_packet_t *)pkt_table, @@ -722,10 +726,12 @@ static void schedule_resume(void) static void schedule_release_atomic(void) { + /* Nothing to do */ } static void schedule_release_ordered(void) { + /* Nothing to do */ } static uint64_t schedule_wait_time(uint64_t ns) @@ -784,9 +790,9 @@ static int schedule_thr_rem(odp_schedule_group_t group, int thr) return 0; } -/* This function is a no-op */ -static void schedule_prefetch(int num ODP_UNUSED) +static void schedule_prefetch(int num) { + (void)num; } static int schedule_num_prio(void) @@ -968,33 +974,41 @@ static int schedule_group_info(odp_schedule_group_t group, return ret; } -static void schedule_order_lock(uint32_t lock_index ODP_UNUSED) +static void schedule_order_lock(uint32_t lock_index) { + (void)lock_index; } -static void schedule_order_unlock(uint32_t lock_index ODP_UNUSED) +static void schedule_order_unlock(uint32_t lock_index) { + (void)lock_index; } -static void schedule_order_unlock_lock(uint32_t unlock_index ODP_UNUSED, - uint32_t lock_index ODP_UNUSED) +static void schedule_order_unlock_lock(uint32_t unlock_index, + uint32_t lock_index) { + (void)unlock_index; + (void)lock_index; } -static void schedule_order_lock_start(uint32_t lock_index ODP_UNUSED) +static void schedule_order_lock_start(uint32_t lock_index) { + (void)lock_index; } -static void schedule_order_lock_wait(uint32_t lock_index ODP_UNUSED) +static void schedule_order_lock_wait(uint32_t lock_index) { + (void)lock_index; } static void order_lock(void) { + /* Nothing to do */ } static void order_unlock(void) { + /* Nothing to do */ } static int schedule_capability(odp_schedule_capability_t *capa) diff --git a/platform/linux-dpdk/odp_system_info.c b/platform/linux-dpdk/odp_system_info.c index b6a723fd8..42c502fda 100644 --- a/platform/linux-dpdk/odp_system_info.c +++ b/platform/linux-dpdk/odp_system_info.c @@ -172,7 +172,8 @@ static char *get_hugepage_dir(uint64_t hugepage_sz) retval = strdup(tokens[MOUNTPT]); break; } - } else { /* there is an explicit page size, so check it */ + } else { + /* there is an explicit page size, so check it */ pagesz = rte_str_to_size(&pagesz_str[pagesize_opt_len]); if (pagesz == hugepage_sz) { retval = strdup(tokens[MOUNTPT]); diff --git a/platform/linux-dpdk/odp_timer.c b/platform/linux-dpdk/odp_timer.c index 0910d09ac..28b4b17a9 100644 --- a/platform/linux-dpdk/odp_timer.c +++ b/platform/linux-dpdk/odp_timer.c @@ -118,7 +118,7 @@ typedef struct timer_local_t { } timer_local_t; /* Points to timer global data */ -timer_global_t *timer_global; +static timer_global_t *timer_global; /* Timer thread local data */ static __thread timer_local_t timer_local; @@ -214,7 +214,7 @@ int _odp_timer_term_local(void) return 0; } -void _timer_run_inline(int dec) +void _odp_timer_run_inline(int dec) { int poll_interval = timer_global->poll_interval; odp_time_t now; @@ -428,6 +428,7 @@ odp_timer_pool_t odp_timer_pool_create(const char *name, void odp_timer_pool_start(void) { + /* Nothing to do */ } void odp_timer_pool_destroy(odp_timer_pool_t tp) diff --git a/platform/linux-dpdk/test/Makefile.am b/platform/linux-dpdk/test/Makefile.am index 6e4edf952..b21fd5314 100644 --- a/platform/linux-dpdk/test/Makefile.am +++ b/platform/linux-dpdk/test/Makefile.am @@ -56,3 +56,5 @@ clean-local: rm -f $(builddir)/$$f; \ done \ fi + +.NOTPARALLEL: diff --git a/platform/linux-dpdk/test/sched-basic.conf b/platform/linux-dpdk/test/sched-basic.conf new file mode 100644 index 000000000..c54089a0d --- /dev/null +++ b/platform/linux-dpdk/test/sched-basic.conf @@ -0,0 +1,8 @@ +# Mandatory fields +odp_implementation = "linux-dpdk" +config_file_version = "0.1.11" + +sched_basic: { + # Test scheduler with an odd spread value + prio_spread = 3 +} diff --git a/platform/linux-generic/Makefile.am b/platform/linux-generic/Makefile.am index 1d9a3bbdb..495ff1c88 100644 --- a/platform/linux-generic/Makefile.am +++ b/platform/linux-generic/Makefile.am @@ -161,7 +161,6 @@ BUILT_SOURCES = \ include/odp_libconfig_config.h __LIB__libodp_linux_la_SOURCES = \ - odp_atomic.c \ odp_barrier.c \ odp_buffer.c \ odp_chksum.c \ @@ -266,7 +265,8 @@ __LIB__libodp_linux_la_SOURCES += \ endif if ARCH_IS_ARM -__LIB__libodp_linux_la_SOURCES += arch/default/odp_cpu_cycles.c \ +__LIB__libodp_linux_la_SOURCES += arch/default/odp_atomic.c \ + arch/default/odp_cpu_cycles.c \ arch/default/odp_global_time.c \ arch/default/odp_hash_crc32.c \ arch/arm/odp_sysinfo_parse.c @@ -274,15 +274,21 @@ odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_inlines.h \ arch/default/odp/api/abi/cpu_time.h \ arch/default/odp/api/abi/hash_crc32.h if !ODP_ABI_COMPAT -odpapiabiarchinclude_HEADERS += arch/arm/odp/api/abi/cpu.h +odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ + arch/default/odp/api/abi/atomic_inlines.h \ + arch/arm/odp/api/abi/cpu.h endif noinst_HEADERS += arch/arm/odp_atomic.h \ arch/arm/odp_cpu.h \ arch/arm/odp_cpu_idling.h \ - arch/arm/odp_llsc.h + arch/arm/odp_llsc.h \ + arch/default/odp_atomic.h \ + arch/default/odp_cpu.h \ + arch/default/odp_cpu_idling.h endif if ARCH_IS_AARCH64 -__LIB__libodp_linux_la_SOURCES += arch/default/odp_cpu_cycles.c \ +__LIB__libodp_linux_la_SOURCES += arch/aarch64/odp_atomic.c \ + arch/default/odp_cpu_cycles.c \ arch/aarch64/odp_global_time.c \ arch/default/odp_hash_crc32.c \ arch/aarch64/odp_sysinfo_parse.c @@ -290,7 +296,9 @@ odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_inlines.h \ arch/default/odp/api/abi/cpu_time.h \ arch/aarch64/odp/api/abi/hash_crc32.h if !ODP_ABI_COMPAT -odpapiabiarchinclude_HEADERS += arch/aarch64/odp/api/abi/atomic.h \ +odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ + arch/aarch64/odp/api/abi/atomic_inlines.h \ + arch/aarch64/odp/api/abi/atomic.h \ arch/aarch64/odp/api/abi/cpu.h endif noinst_HEADERS += arch/aarch64/odp_atomic.h \ @@ -299,7 +307,8 @@ noinst_HEADERS += arch/aarch64/odp_atomic.h \ arch/aarch64/odp_llsc.h endif if ARCH_IS_DEFAULT -__LIB__libodp_linux_la_SOURCES += arch/default/odp_cpu_cycles.c \ +__LIB__libodp_linux_la_SOURCES += arch/default/odp_atomic.c \ + arch/default/odp_cpu_cycles.c \ arch/default/odp_global_time.c \ arch/default/odp_hash_crc32.c \ arch/default/odp_sysinfo_parse.c @@ -307,13 +316,17 @@ odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_inlines.h \ arch/default/odp/api/abi/cpu_time.h \ arch/default/odp/api/abi/hash_crc32.h if !ODP_ABI_COMPAT -odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu.h +odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ + arch/default/odp/api/abi/atomic_inlines.h \ + arch/default/odp/api/abi/cpu.h endif -noinst_HEADERS += arch/default/odp_cpu.h \ +noinst_HEADERS += arch/default/odp_atomic.h \ + arch/default/odp_cpu.h \ arch/default/odp_cpu_idling.h endif if ARCH_IS_MIPS64 -__LIB__libodp_linux_la_SOURCES += arch/mips64/odp_cpu_cycles.c \ +__LIB__libodp_linux_la_SOURCES += arch/default/odp_atomic.c \ + arch/mips64/odp_cpu_cycles.c \ arch/default/odp_global_time.c \ arch/default/odp_hash_crc32.c \ arch/mips64/odp_sysinfo_parse.c @@ -321,13 +334,17 @@ odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_inlines.h \ arch/default/odp/api/abi/cpu_time.h \ arch/default/odp/api/abi/hash_crc32.h if !ODP_ABI_COMPAT -odpapiabiarchinclude_HEADERS += arch/mips64/odp/api/abi/cpu.h +odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ + arch/default/odp/api/abi/atomic_inlines.h \ + arch/mips64/odp/api/abi/cpu.h endif -noinst_HEADERS += arch/default/odp_cpu.h \ +noinst_HEADERS += arch/default/odp_atomic.h \ + arch/default/odp_cpu.h \ arch/default/odp_cpu_idling.h endif if ARCH_IS_POWERPC -__LIB__libodp_linux_la_SOURCES += arch/default/odp_cpu_cycles.c \ +__LIB__libodp_linux_la_SOURCES += arch/default/odp_atomic.c \ + arch/default/odp_cpu_cycles.c \ arch/default/odp_global_time.c \ arch/default/odp_hash_crc32.c \ arch/powerpc/odp_sysinfo_parse.c @@ -335,13 +352,17 @@ odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_inlines.h \ arch/default/odp/api/abi/cpu_time.h \ arch/default/odp/api/abi/hash_crc32.h if !ODP_ABI_COMPAT -odpapiabiarchinclude_HEADERS += arch/powerpc/odp/api/abi/cpu.h +odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ + arch/default/odp/api/abi/atomic_inlines.h \ + arch/powerpc/odp/api/abi/cpu.h endif -noinst_HEADERS += arch/default/odp_cpu.h \ +noinst_HEADERS += arch/default/odp_atomic.h \ + arch/default/odp_cpu.h \ arch/default/odp_cpu_idling.h endif if ARCH_IS_X86 -__LIB__libodp_linux_la_SOURCES += arch/x86/cpu_flags.c \ +__LIB__libodp_linux_la_SOURCES += arch/default/odp_atomic.c \ + arch/x86/cpu_flags.c \ arch/x86/odp_cpu_cycles.c \ arch/x86/odp_global_time.c \ arch/default/odp_hash_crc32.c \ @@ -351,10 +372,13 @@ odpapiabiarchinclude_HEADERS += arch/x86/odp/api/abi/cpu_inlines.h \ arch/x86/odp/api/abi/cpu_time.h \ arch/default/odp/api/abi/hash_crc32.h if !ODP_ABI_COMPAT -odpapiabiarchinclude_HEADERS += arch/x86/odp/api/abi/cpu.h +odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ + arch/default/odp/api/abi/atomic_inlines.h \ + arch/x86/odp/api/abi/cpu.h endif noinst_HEADERS += arch/x86/cpu_flags.h \ arch/x86/odp_cpu.h \ + arch/default/odp_atomic.h \ arch/default/odp_cpu.h \ arch/default/odp_cpu_idling.h endif @@ -373,3 +397,16 @@ __LIB__libodp_linux_la_LIBADD += $(TIMER_LIBS) if ODP_PKTIO_PCAP __LIB__libodp_linux_la_LIBADD += $(PCAP_LIBS) endif + +CHECK_GLOBALS_REGEX = " (odp_|_odp_|_deprecated_odp_|miniz_|mz_|tdefl_|tinfl_|mp_hdlr_init_odp_pool_ops)" + +TESTS_ENVIRONMENT = \ + LIBTOOL="$(LIBTOOL)" \ + NM="$(NM)" \ + LIB="$(LIB)" \ + lib_LTLIBRARIES="$(lib_LTLIBRARIES)" \ + CHECK_GLOBALS_REGEX=$(CHECK_GLOBALS_REGEX) + +dist_check_SCRIPTS = check-globals.sh + +TESTS = $(dist_check_SCRIPTS) diff --git a/platform/linux-generic/arch/aarch64/odp/api/abi/atomic_inlines.h b/platform/linux-generic/arch/aarch64/odp/api/abi/atomic_inlines.h new file mode 100644 index 000000000..446f9cc50 --- /dev/null +++ b/platform/linux-generic/arch/aarch64/odp/api/abi/atomic_inlines.h @@ -0,0 +1,161 @@ +/* Copyright (c) 2021, ARM Limited + * Copyright (c) 2021, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_API_ABI_ATOMIC_INLINES_H_ +#define ODP_API_ABI_ATOMIC_INLINES_H_ + +#include <odp/api/atomic.h> + +#ifdef _ODP_LOCK_FREE_128BIT_ATOMICS + +/** + * @internal + * Helper macro for lockless atomic CAS operations on 128-bit integers + * @param[in,out] atom Pointer to the 128-bit atomic variable + * @param oper CAS operation + * @param old_val Old value + * @param new_val New value to be swapped + * @return 1 for success and 0 for fail + */ +#define ATOMIC_CAS_OP_128(atom, oper, old_val, new_val, val) \ +({ \ + odp_u128_t _val; \ + odp_atomic_u128_t *_atom = atom; \ + odp_u128_t *_old_val = old_val; \ + odp_u128_t _new_val = new_val; \ + odp_u128_t *ptr = (odp_u128_t *)(_atom); \ + register uint64_t old0 __asm__ ("x0"); \ + register uint64_t old1 __asm__ ("x1"); \ + register uint64_t new0 __asm__ ("x2"); \ + register uint64_t new1 __asm__ ("x3"); \ + old0 = (uint64_t)(_old_val)->u64[0]; \ + old1 = (uint64_t)(_old_val)->u64[1]; \ + new0 = (uint64_t)(_new_val).u64[0]; \ + new1 = (uint64_t)(_new_val).u64[1]; \ + __asm__ volatile(oper " %[old0], %[old1], %[new0], %[new1], [%[ptr]]" \ + : [old0] "+r" (old0), [old1] "+r" (old1) \ + : [new0] "r" (new0), [new1] "r" (new1), \ + [ptr] "r" (ptr) \ + : "memory"); \ + _val.u64[0] = old0; \ + _val.u64[1] = old1; \ + val = _val; \ +}) + +#define ATOMIC_CAS_OP_128_NO_ORDER(atom, old_value, new_value, val) \ + ATOMIC_CAS_OP_128(atom, "casp", old_value, new_value, val) + +#define ATOMIC_CAS_OP_128_ACQ(atom, old_value, new_value, val) \ + ATOMIC_CAS_OP_128(atom, "caspa", old_value, new_value, val) + +#define ATOMIC_CAS_OP_128_REL(atom, old_value, new_value, val) \ + ATOMIC_CAS_OP_128(atom, "caspl", old_value, new_value, val) + +#define ATOMIC_CAS_OP_128_ACQ_REL(atom, old_value, new_value, val) \ + ATOMIC_CAS_OP_128(atom, "caspal", old_value, new_value, val) + +static inline void _odp_atomic_init_u128(odp_atomic_u128_t *atom, odp_u128_t new_val) +{ + atom->v = new_val; +} + +static inline odp_u128_t _odp_atomic_load_u128(odp_atomic_u128_t *atom) +{ + odp_u128_t val, exp; + + exp.u64[0] = 0; + exp.u64[1] = 0; + ATOMIC_CAS_OP_128_NO_ORDER(atom, &exp, exp, val); + return val; +} + +static inline void _odp_atomic_store_u128(odp_atomic_u128_t *atom, odp_u128_t new_val) +{ + odp_u128_t old, val; + + old = atom->v; + + while (1) { + ATOMIC_CAS_OP_128_NO_ORDER(atom, &old, new_val, val); + + if ((val.u64[0] == old.u64[0]) && (val.u64[1] == old.u64[1])) + return; + + old = val; + } +} + +static inline int _odp_atomic_cas_u128(odp_atomic_u128_t *atom, odp_u128_t *old_val, + odp_u128_t new_val) +{ + odp_u128_t val; + + ATOMIC_CAS_OP_128_NO_ORDER(atom, old_val, new_val, val); + + if ((val.u64[0] == old_val->u64[0]) && (val.u64[1] == old_val->u64[1])) + return 1; + + old_val->u64[0] = val.u64[0]; + old_val->u64[1] = val.u64[1]; + + return 0; +} + +static inline int _odp_atomic_cas_acq_u128(odp_atomic_u128_t *atom, odp_u128_t *old_val, + odp_u128_t new_val) +{ + odp_u128_t val; + + ATOMIC_CAS_OP_128_ACQ(atom, old_val, new_val, val); + + if ((val.u64[0] == old_val->u64[0]) && (val.u64[1] == old_val->u64[1])) + return 1; + + old_val->u64[0] = val.u64[0]; + old_val->u64[1] = val.u64[1]; + + return 0; +} + +static inline int _odp_atomic_cas_rel_u128(odp_atomic_u128_t *atom, odp_u128_t *old_val, + odp_u128_t new_val) +{ + odp_u128_t val; + + ATOMIC_CAS_OP_128_REL(atom, old_val, new_val, val); + + if ((val.u64[0] == old_val->u64[0]) && (val.u64[1] == old_val->u64[1])) + return 1; + + old_val->u64[0] = val.u64[0]; + old_val->u64[1] = val.u64[1]; + + return 0; +} + +static inline int _odp_atomic_cas_acq_rel_u128(odp_atomic_u128_t *atom, odp_u128_t *old_val, + odp_u128_t new_val) +{ + odp_u128_t val; + + ATOMIC_CAS_OP_128_ACQ_REL(atom, old_val, new_val, val); + + if ((val.u64[0] == old_val->u64[0]) && (val.u64[1] == old_val->u64[1])) + return 1; + + old_val->u64[0] = val.u64[0]; + old_val->u64[1] = val.u64[1]; + + return 0; +} +#else /* _ODP_LOCK_FREE_128BIT_ATOMICS */ + +/* Use generic implementation */ +#include <odp/api/abi/atomic_generic.h> + +#endif +#endif diff --git a/platform/linux-generic/arch/aarch64/odp_atomic.c b/platform/linux-generic/arch/aarch64/odp_atomic.c new file mode 100644 index 000000000..c6b809768 --- /dev/null +++ b/platform/linux-generic/arch/aarch64/odp_atomic.c @@ -0,0 +1,56 @@ +/* Copyright (c) 2015-2018, Linaro Limited + * Copyright (c) 2021, ARM Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <odp/api/atomic.h> + +int odp_atomic_lock_free_u64(odp_atomic_op_t *atomic_op) +{ +#if __GCC_ATOMIC_LLONG_LOCK_FREE < 2 + /* All operations have locks */ + if (atomic_op) + atomic_op->all_bits = 0; + + return 0; +#else + /* All operations are lock-free */ + if (atomic_op) { + atomic_op->all_bits = ~((uint32_t)0); + atomic_op->op.init = 0; + } + + return 2; +#endif +} + +int odp_atomic_lock_free_u128(odp_atomic_op_t *atomic_op) +{ +#ifdef _ODP_LOCK_FREE_128BIT_ATOMICS + if (atomic_op) { + atomic_op->all_bits = 0; + atomic_op->op.load = 1; + atomic_op->op.store = 1; + atomic_op->op.cas = 1; + } + + return 2; +#elif defined(__SIZEOF_INT128__) + if (__atomic_is_lock_free(16, NULL)) { + if (atomic_op) { + atomic_op->all_bits = 0; + atomic_op->op.load = 1; + atomic_op->op.store = 1; + atomic_op->op.cas = 1; + } + return 2; + } +#endif + /* All operations have locks */ + if (atomic_op) + atomic_op->all_bits = 0; + + return 0; +} diff --git a/platform/linux-generic/arch/aarch64/odp_atomic.h b/platform/linux-generic/arch/aarch64/odp_atomic.h index dbeccebde..c7a28fc2c 100644 --- a/platform/linux-generic/arch/aarch64/odp_atomic.h +++ b/platform/linux-generic/arch/aarch64/odp_atomic.h @@ -12,6 +12,8 @@ #error This file should not be included directly, please include odp_cpu.h #endif +#include <limits.h> + #ifdef CONFIG_DMBSTR #define atomic_store_release(loc, val, ro) \ @@ -217,54 +219,108 @@ static inline __int128 __lockfree_load_16(__int128 *var, int mo) return old; } -#ifdef _ODP_LOCK_FREE_128BIT_ATOMICS +typedef unsigned __int128 _u128_t; -/** - * @internal - * Helper macro for lockless atomic CAS operations on 128-bit integers - * @param[in,out] atom Pointer to the 128-bit atomic variable - * @param oper CAS operation - * @param old_val Old value - * @param new_val New value to be swapped - * @return 1 for success and 0 for fail - */ -#define ATOMIC_CAS_OP_128(atom, oper, old_val, new_val, val) \ -({ \ - odp_u128_t _val; \ - odp_atomic_u128_t *_atom = atom; \ - odp_u128_t *_old_val = old_val; \ - odp_u128_t _new_val = new_val; \ - odp_u128_t *ptr = (odp_u128_t *)(_atom); \ - register uint64_t old0 __asm__ ("x0"); \ - register uint64_t old1 __asm__ ("x1"); \ - register uint64_t new0 __asm__ ("x2"); \ - register uint64_t new1 __asm__ ("x3"); \ - old0 = (uint64_t)(_old_val)->u64[0]; \ - old1 = (uint64_t)(_old_val)->u64[1]; \ - new0 = (uint64_t)(_new_val).u64[0]; \ - new1 = (uint64_t)(_new_val).u64[1]; \ - __asm__ volatile(oper " %[old0], %[old1], %[new0], %[new1], [%[ptr]]" \ - : [old0] "+r" (old0), [old1] "+r" (old1) \ - : [new0] "r" (new0), [new1] "r" (new1), \ - [ptr] "r" (ptr) \ - : "memory"); \ - _val.u64[0] = old0; \ - _val.u64[1] = old1; \ - val = _val; \ -}) - -#define ATOMIC_CAS_OP_128_NO_ORDER(atom, old_value, new_value, val) \ - ATOMIC_CAS_OP_128(atom, "casp", old_value, new_value, val) - -#define ATOMIC_CAS_OP_128_ACQ(atom, old_value, new_value, val) \ - ATOMIC_CAS_OP_128(atom, "caspa", old_value, new_value, val) - -#define ATOMIC_CAS_OP_128_REL(atom, old_value, new_value, val) \ - ATOMIC_CAS_OP_128(atom, "caspl", old_value, new_value, val) - -#define ATOMIC_CAS_OP_128_ACQ_REL(atom, old_value, new_value, val) \ - ATOMIC_CAS_OP_128(atom, "caspal", old_value, new_value, val) +static inline _u128_t lockfree_load_u128(_u128_t *atomic) +{ + return __lockfree_load_16((__int128 *)atomic, __ATOMIC_RELAXED); +} + +static inline int lockfree_cas_acq_rel_u128(_u128_t *atomic, + _u128_t old_val, + _u128_t new_val) +{ + return __lockfree_compare_exchange_16((__int128 *)atomic, + (__int128 *)&old_val, + new_val, + 0, + __ATOMIC_ACQ_REL, + __ATOMIC_RELAXED); +} + +static inline int lockfree_check_u128(void) +{ + return 1; +} + +/** Atomic bit set operations with memory ordering */ +#if defined(__SIZEOF_INT128__) && __SIZEOF_INT128__ == 16 +typedef __int128 bitset_t; +#define ATOM_BITSET_SIZE (CHAR_BIT * __SIZEOF_INT128__) + +#elif __GCC_ATOMIC_LLONG_LOCK_FREE == 2 && \ + __SIZEOF_LONG_LONG__ != __SIZEOF_LONG__ +typedef unsigned long long bitset_t; +#define ATOM_BITSET_SIZE (CHAR_BIT * __SIZEOF_LONG_LONG__) +#elif __GCC_ATOMIC_LONG_LOCK_FREE == 2 && __SIZEOF_LONG__ != __SIZEOF_INT__ +typedef unsigned long bitset_t; +#define ATOM_BITSET_SIZE (CHAR_BIT * __SIZEOF_LONG__) + +#elif __GCC_ATOMIC_INT_LOCK_FREE == 2 +typedef unsigned int bitset_t; +#define ATOM_BITSET_SIZE (CHAR_BIT * __SIZEOF_INT__) + +#else +/* Target does not support lock-free atomic operations */ +typedef unsigned int bitset_t; +#define ATOM_BITSET_SIZE (CHAR_BIT * __SIZEOF_INT__) #endif +#if ATOM_BITSET_SIZE <= 32 + +static inline bitset_t bitset_mask(uint32_t bit) +{ + return 1UL << bit; +} + +#elif ATOM_BITSET_SIZE <= 64 + +static inline bitset_t bitset_mask(uint32_t bit) +{ + return 1ULL << bit; +} + +#elif ATOM_BITSET_SIZE <= 128 + +static inline bitset_t bitset_mask(uint32_t bit) +{ + if (bit < 64) + return 1ULL << bit; + else + return (unsigned __int128)(1ULL << (bit - 64)) << 64; +} + +#else +#error Unsupported size of bit sets (ATOM_BITSET_SIZE) +#endif + +static inline bitset_t atom_bitset_load(bitset_t *bs, int mo) +{ + return __lockfree_load_16(bs, mo); +} + +static inline void atom_bitset_set(bitset_t *bs, uint32_t bit, int mo) +{ + (void)__lockfree_fetch_or_16(bs, bitset_mask(bit), mo); +} + +static inline void atom_bitset_clr(bitset_t *bs, uint32_t bit, int mo) +{ + (void)__lockfree_fetch_and_16(bs, ~bitset_mask(bit), mo); +} + +static inline bitset_t atom_bitset_xchg(bitset_t *bs, bitset_t neu, int mo) +{ + return __lockfree_exchange_16(bs, neu, mo); +} + +static inline bitset_t atom_bitset_cmpxchg(bitset_t *bs, bitset_t *old, + bitset_t neu, bool weak, + int mo_success, int mo_failure) +{ + return __lockfree_compare_exchange_16(bs, old, neu, weak, mo_success, + mo_failure); +} + #endif /* PLATFORM_LINUXGENERIC_ARCH_ARM_ODP_ATOMIC_H */ diff --git a/platform/linux-generic/arch/aarch64/odp_sysinfo_parse.c b/platform/linux-generic/arch/aarch64/odp_sysinfo_parse.c index 77e115756..ab64d501e 100644 --- a/platform/linux-generic/arch/aarch64/odp_sysinfo_parse.c +++ b/platform/linux-generic/arch/aarch64/odp_sysinfo_parse.c @@ -1,4 +1,5 @@ /* Copyright (c) 2018, Linaro Limited + * Copyright (c) 2020-2021, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -64,6 +65,7 @@ static void aarch64_part_info(char *str, int maxlen, odp_cpu_arch_arm_t *cpu_isa *cpu_isa = ODP_CPU_ARCH_ARM_UNKNOWN; if (implementer == 0x41) { + /* Part numbers are specified in Main ID Register (MIDR_EL1) documentation */ switch (part) { case 0xd02: snprintf(str, maxlen, "Cortex-A34"); @@ -105,6 +107,10 @@ static void aarch64_part_info(char *str, int maxlen, odp_cpu_arch_arm_t *cpu_isa snprintf(str, maxlen, "Cortex-A76"); *cpu_isa = ODP_CPU_ARCH_ARMV8_2; return; + case 0xd0c: + snprintf(str, maxlen, "Neoverse N1"); + *cpu_isa = ODP_CPU_ARCH_ARMV8_2; + return; case 0xd0e: snprintf(str, maxlen, "Cortex-A76AE"); *cpu_isa = ODP_CPU_ARCH_ARMV8_2; @@ -117,6 +123,18 @@ static void aarch64_part_info(char *str, int maxlen, odp_cpu_arch_arm_t *cpu_isa snprintf(str, maxlen, "Cortex-A78"); *cpu_isa = ODP_CPU_ARCH_ARMV8_2; return; + case 0xd42: + snprintf(str, maxlen, "Cortex-A78AE"); + *cpu_isa = ODP_CPU_ARCH_ARMV8_2; + return; + case 0xd4a: + snprintf(str, maxlen, "Neoverse E1"); + *cpu_isa = ODP_CPU_ARCH_ARMV8_2; + return; + case 0xd4b: + snprintf(str, maxlen, "Cortex-A78C"); + *cpu_isa = ODP_CPU_ARCH_ARMV8_2; + return; default: break; } @@ -195,6 +213,19 @@ static odp_cpu_arch_arm_t arm_isa_version(void) return ODP_CPU_ARCH_ARMV8_5; case 6: return ODP_CPU_ARCH_ARMV8_6; + case 7: + return ODP_CPU_ARCH_ARMV8_7; + default: + return ODP_CPU_ARCH_ARM_UNKNOWN; + } + } else if (major == 9) { + switch (minor) { + case 0: + return ODP_CPU_ARCH_ARMV9_0; + case 1: + return ODP_CPU_ARCH_ARMV9_1; + case 2: + return ODP_CPU_ARCH_ARMV9_2; default: return ODP_CPU_ARCH_ARM_UNKNOWN; } diff --git a/platform/linux-generic/arch/arm/odp_atomic.h b/platform/linux-generic/arch/arm/odp_atomic.h index d85a01841..bcc89ff73 100644 --- a/platform/linux-generic/arch/arm/odp_atomic.h +++ b/platform/linux-generic/arch/arm/odp_atomic.h @@ -1,5 +1,4 @@ -/* Copyright (c) 2017, ARM Limited. All rights reserved. - * +/* Copyright (c) 2017-2021, ARM Limited. All rights reserved. * Copyright (c) 2017-2018, Linaro Limited * All rights reserved. * @@ -13,6 +12,8 @@ #error This file should not be included directly, please include odp_cpu.h #endif +#include <limits.h> + #ifdef CONFIG_DMBSTR #define atomic_store_release(loc, val, ro) \ @@ -28,4 +29,80 @@ do { \ #endif /* CONFIG_DMBSTR */ +/** Atomic bit set operations with memory ordering */ +#if __GCC_ATOMIC_LLONG_LOCK_FREE == 2 && \ + __SIZEOF_LONG_LONG__ != __SIZEOF_LONG__ +typedef unsigned long long bitset_t; +#define ATOM_BITSET_SIZE (CHAR_BIT * __SIZEOF_LONG_LONG__) + +#elif __GCC_ATOMIC_LONG_LOCK_FREE == 2 && __SIZEOF_LONG__ != __SIZEOF_INT__ +typedef unsigned long bitset_t; +#define ATOM_BITSET_SIZE (CHAR_BIT * __SIZEOF_LONG__) + +#elif __GCC_ATOMIC_INT_LOCK_FREE == 2 +typedef unsigned int bitset_t; +#define ATOM_BITSET_SIZE (CHAR_BIT * __SIZEOF_INT__) + +#else +/* Target does not support lock-free atomic operations */ +typedef unsigned int bitset_t; +#define ATOM_BITSET_SIZE (CHAR_BIT * __SIZEOF_INT__) +#endif + +#if ATOM_BITSET_SIZE <= 32 + +static inline bitset_t bitset_mask(uint32_t bit) +{ + return 1UL << bit; +} + +#elif ATOM_BITSET_SIZE <= 64 + +static inline bitset_t bitset_mask(uint32_t bit) +{ + return 1ULL << bit; +} + +#elif ATOM_BITSET_SIZE <= 128 + +static inline bitset_t bitset_mask(uint32_t bit) +{ + if (bit < 64) + return 1ULL << bit; + else + return (unsigned __int128)(1ULL << (bit - 64)) << 64; +} + +#else +#error Unsupported size of bit sets (ATOM_BITSET_SIZE) +#endif + +static inline bitset_t atom_bitset_load(bitset_t *bs, int mo) +{ + return __atomic_load_n(bs, mo); +} + +static inline void atom_bitset_set(bitset_t *bs, uint32_t bit, int mo) +{ + (void)__atomic_fetch_or(bs, bitset_mask(bit), mo); +} + +static inline void atom_bitset_clr(bitset_t *bs, uint32_t bit, int mo) +{ + (void)__atomic_fetch_and(bs, ~bitset_mask(bit), mo); +} + +static inline bitset_t atom_bitset_xchg(bitset_t *bs, bitset_t neu, int mo) +{ + return __atomic_exchange_n(bs, neu, mo); +} + +static inline bitset_t atom_bitset_cmpxchg(bitset_t *bs, bitset_t *old, + bitset_t neu, bool weak, + int mo_success, int mo_failure) +{ + return __atomic_compare_exchange_n(bs, old, neu, weak, mo_success, + mo_failure); +} + #endif /* PLATFORM_LINUXGENERIC_ARCH_ARM_ODP_ATOMIC_H */ diff --git a/platform/linux-generic/arch/default/odp/api/abi/atomic_generic.h b/platform/linux-generic/arch/default/odp/api/abi/atomic_generic.h new file mode 100644 index 000000000..e274a4d64 --- /dev/null +++ b/platform/linux-generic/arch/default/odp/api/abi/atomic_generic.h @@ -0,0 +1,159 @@ +/* Copyright (c) 2021, ARM Limited + * Copyright (c) 2021, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_API_ABI_ATOMIC_GENERIC_H_ +#define ODP_API_ABI_ATOMIC_GENERIC_H_ + +#include <odp/api/atomic.h> + +#ifdef __SIZEOF_INT128__ + +static inline void _odp_atomic_init_u128(odp_atomic_u128_t *atom, odp_u128_t val) +{ + atom->v = val; +} + +static inline odp_u128_t _odp_atomic_load_u128(odp_atomic_u128_t *atom) +{ + odp_u128_t val; + + *(__int128_t *)&val = __atomic_load_n((__int128_t *)&atom->v, __ATOMIC_RELAXED); + return val; +} + +static inline void _odp_atomic_store_u128(odp_atomic_u128_t *atom, odp_u128_t val) +{ + __atomic_store_n((__int128_t *)&atom->v, *(__int128_t *)&val, __ATOMIC_RELAXED); +} + +static inline int _odp_atomic_cas_u128(odp_atomic_u128_t *atom, odp_u128_t *old_val, + odp_u128_t new_val) +{ + return __atomic_compare_exchange_n((__int128_t *)&atom->v, (__int128_t *)old_val, + *(__int128_t *)&new_val, 0 /* strong */, + __ATOMIC_RELAXED, __ATOMIC_RELAXED); +} + +static inline int _odp_atomic_cas_acq_u128(odp_atomic_u128_t *atom, odp_u128_t *old_val, + odp_u128_t new_val) +{ + return __atomic_compare_exchange_n((__int128_t *)&atom->v, (__int128_t *)old_val, + *(__int128_t *)&new_val, 0 /* strong */, + __ATOMIC_ACQUIRE, __ATOMIC_RELAXED); +} + +static inline int _odp_atomic_cas_rel_u128(odp_atomic_u128_t *atom, odp_u128_t *old_val, + odp_u128_t new_val) +{ + return __atomic_compare_exchange_n((__int128_t *)&atom->v, (__int128_t *)old_val, + *(__int128_t *)&new_val, 0 /* strong */, + __ATOMIC_RELEASE, __ATOMIC_RELAXED); +} + +static inline int _odp_atomic_cas_acq_rel_u128(odp_atomic_u128_t *atom, odp_u128_t *old_val, + odp_u128_t new_val) +{ + return __atomic_compare_exchange_n((__int128_t *)&atom->v, (__int128_t *)old_val, + *(__int128_t *)&new_val, 0 /* strong */, + __ATOMIC_ACQ_REL, __ATOMIC_RELAXED); +} + +#else /* Lock-based implementation */ + +/** + * @internal + * 128 bit store operation expression for the ATOMIC_OP macro + */ +#define ATOMIC_STORE_OP_128(new_val) \ +({ \ + (_atom)->v = (new_val); \ +}) + +/** + * @internal + * 128 bit CAS operation expression for the ATOMIC_OP macro + */ +#define ATOMIC_CAS_OP_128(ret_ptr, old_val, new_val) \ +({ \ + int *_ret_ptr = ret_ptr; \ + odp_u128_t *_old_val = old_val; \ + odp_u128_t _new_val = new_val; \ + if (((_atom)->v.u64[0] == (_old_val)->u64[0]) && \ + ((_atom)->v.u64[1] == (_old_val)->u64[1])) { \ + (_atom)->v = (_new_val); \ + *(_ret_ptr) = 1; \ + } else { \ + *(_ret_ptr) = 0; \ + } \ +}) + +/** + * @internal + * Helper macro for lock-based atomic operations on 128-bit integers + * @param[in,out] atom Pointer to the 128-bit atomic variable + * @param expr Expression used update the variable. + * @return The old value of the variable. + */ +#define ATOMIC_OP_128(atom, expr) \ +({ \ + odp_u128_t _old_val; \ + odp_atomic_u128_t *_atom = atom; \ + /* Loop while lock is already taken, stop when lock becomes clear */ \ + while (__atomic_test_and_set(&(_atom)->lock, __ATOMIC_ACQUIRE)) \ + (void)0; \ + _old_val = (_atom)->v; \ + (expr); /* Perform whatever update is desired */ \ + __atomic_clear(&(_atom)->lock, __ATOMIC_RELEASE); \ + _old_val; /* Return old value */ \ +}) + +static inline void _odp_atomic_init_u128(odp_atomic_u128_t *atom, odp_u128_t val) +{ + atom->v.u64[0] = val.u64[0]; + atom->v.u64[1] = val.u64[1]; + atom->lock = 0; +} + +static inline odp_u128_t _odp_atomic_load_u128(odp_atomic_u128_t *atom) +{ + return ATOMIC_OP_128(atom, (void)0); +} + +static inline void _odp_atomic_store_u128(odp_atomic_u128_t *atom, odp_u128_t val) +{ + ATOMIC_OP_128(atom, ATOMIC_STORE_OP_128(val)); +} + +static inline int _odp_atomic_cas_u128(odp_atomic_u128_t *atom, odp_u128_t *old_val, + odp_u128_t new_val) +{ + int ret; + + *old_val = ATOMIC_OP_128(atom, ATOMIC_CAS_OP_128(&ret, old_val, new_val)); + return ret; +} + +static inline int _odp_atomic_cas_acq_u128(odp_atomic_u128_t *atom, odp_u128_t *old_val, + odp_u128_t new_val) +{ + return _odp_atomic_cas_u128(atom, old_val, new_val); +} + +static inline int _odp_atomic_cas_rel_u128(odp_atomic_u128_t *atom, odp_u128_t *old_val, + odp_u128_t new_val) +{ + return _odp_atomic_cas_u128(atom, old_val, new_val); +} + +static inline int _odp_atomic_cas_acq_rel_u128(odp_atomic_u128_t *atom, odp_u128_t *old_val, + odp_u128_t new_val) +{ + return _odp_atomic_cas_u128(atom, old_val, new_val); +} +#endif + +#endif diff --git a/platform/linux-generic/arch/default/odp/api/abi/atomic_inlines.h b/platform/linux-generic/arch/default/odp/api/abi/atomic_inlines.h new file mode 100644 index 000000000..f1072d11f --- /dev/null +++ b/platform/linux-generic/arch/default/odp/api/abi/atomic_inlines.h @@ -0,0 +1,7 @@ +/* Copyright (c) 2021, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <odp/api/abi/atomic_generic.h> diff --git a/platform/linux-generic/arch/default/odp_atomic.c b/platform/linux-generic/arch/default/odp_atomic.c new file mode 100644 index 000000000..36fc5e8ea --- /dev/null +++ b/platform/linux-generic/arch/default/odp_atomic.c @@ -0,0 +1,47 @@ +/* Copyright (c) 2015-2018, Linaro Limited + * Copyright (c) 2021, ARM Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <odp/api/atomic.h> + +int odp_atomic_lock_free_u64(odp_atomic_op_t *atomic_op) +{ +#if __GCC_ATOMIC_LLONG_LOCK_FREE < 2 + /* All operations have locks */ + if (atomic_op) + atomic_op->all_bits = 0; + + return 0; +#else + /* All operations are lock-free */ + if (atomic_op) { + atomic_op->all_bits = ~((uint32_t)0); + atomic_op->op.init = 0; + } + + return 2; +#endif +} + +int odp_atomic_lock_free_u128(odp_atomic_op_t *atomic_op) +{ +#ifdef __SIZEOF_INT128__ + if (__atomic_is_lock_free(16, NULL)) { + if (atomic_op) { + atomic_op->all_bits = 0; + atomic_op->op.load = 1; + atomic_op->op.store = 1; + atomic_op->op.cas = 1; + } + return 2; + } +#endif + /* All operations have locks */ + if (atomic_op) + atomic_op->all_bits = 0; + + return 0; +} diff --git a/platform/linux-generic/arch/default/odp_atomic.h b/platform/linux-generic/arch/default/odp_atomic.h new file mode 100644 index 000000000..fc8260194 --- /dev/null +++ b/platform/linux-generic/arch/default/odp_atomic.h @@ -0,0 +1,114 @@ +/* Copyright (c) 2021, ARM Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_DEFAULT_ATOMIC_H_ +#define ODP_DEFAULT_ATOMIC_H_ + +#ifdef __SIZEOF_INT128__ + +typedef unsigned __int128 _u128_t; + +static inline _u128_t lockfree_load_u128(_u128_t *atomic) +{ + return __atomic_load_n(atomic, __ATOMIC_RELAXED); +} + +static inline int lockfree_cas_acq_rel_u128(_u128_t *atomic, + _u128_t old_val, + _u128_t new_val) +{ + return __atomic_compare_exchange_n(atomic, &old_val, new_val, + 0 /* strong */, + __ATOMIC_ACQ_REL, + __ATOMIC_RELAXED); +} + +static inline int lockfree_check_u128(void) +{ + return __atomic_is_lock_free(16, NULL); +} + +#endif + +#include <limits.h> + +/** Atomic bit set operations with memory ordering */ +#if __GCC_ATOMIC_LLONG_LOCK_FREE == 2 && \ + __SIZEOF_LONG_LONG__ != __SIZEOF_LONG__ +typedef unsigned long long bitset_t; +#define ATOM_BITSET_SIZE (CHAR_BIT * __SIZEOF_LONG_LONG__) + +#elif __GCC_ATOMIC_LONG_LOCK_FREE == 2 && __SIZEOF_LONG__ != __SIZEOF_INT__ +typedef unsigned long bitset_t; +#define ATOM_BITSET_SIZE (CHAR_BIT * __SIZEOF_LONG__) + +#elif __GCC_ATOMIC_INT_LOCK_FREE == 2 +typedef unsigned int bitset_t; +#define ATOM_BITSET_SIZE (CHAR_BIT * __SIZEOF_INT__) + +#else +/* Target does not support lock-free atomic operations */ +typedef unsigned int bitset_t; +#define ATOM_BITSET_SIZE (CHAR_BIT * __SIZEOF_INT__) +#endif + +#if ATOM_BITSET_SIZE <= 32 + +static inline bitset_t bitset_mask(uint32_t bit) +{ + return 1UL << bit; +} + +#elif ATOM_BITSET_SIZE <= 64 + +static inline bitset_t bitset_mask(uint32_t bit) +{ + return 1ULL << bit; +} + +#elif ATOM_BITSET_SIZE <= 128 + +static inline bitset_t bitset_mask(uint32_t bit) +{ + if (bit < 64) + return 1ULL << bit; + else + return (unsigned __int128)(1ULL << (bit - 64)) << 64; +} + +#else +#error Unsupported size of bit sets (ATOM_BITSET_SIZE) +#endif + +static inline bitset_t atom_bitset_load(bitset_t *bs, int mo) +{ + return __atomic_load_n(bs, mo); +} + +static inline void atom_bitset_set(bitset_t *bs, uint32_t bit, int mo) +{ + (void)__atomic_fetch_or(bs, bitset_mask(bit), mo); +} + +static inline void atom_bitset_clr(bitset_t *bs, uint32_t bit, int mo) +{ + (void)__atomic_fetch_and(bs, ~bitset_mask(bit), mo); +} + +static inline bitset_t atom_bitset_xchg(bitset_t *bs, bitset_t neu, int mo) +{ + return __atomic_exchange_n(bs, neu, mo); +} + +static inline bitset_t atom_bitset_cmpxchg(bitset_t *bs, bitset_t *old, + bitset_t neu, bool weak, + int mo_success, int mo_failure) +{ + return __atomic_compare_exchange_n(bs, old, neu, weak, mo_success, + mo_failure); +} + +#endif diff --git a/platform/linux-generic/arch/default/odp_cpu.h b/platform/linux-generic/arch/default/odp_cpu.h index d8bc125c8..821956819 100644 --- a/platform/linux-generic/arch/default/odp_cpu.h +++ b/platform/linux-generic/arch/default/odp_cpu.h @@ -20,6 +20,7 @@ #define atomic_store_release(loc, val, ro) \ __atomic_store_n(loc, val, __ATOMIC_RELEASE) +#include "odp_atomic.h" #include "odp_cpu_idling.h" #endif diff --git a/platform/linux-generic/check-globals.sh b/platform/linux-generic/check-globals.sh new file mode 120000 index 000000000..c999a29ef --- /dev/null +++ b/platform/linux-generic/check-globals.sh @@ -0,0 +1 @@ +../../scripts/check-globals.sh
\ No newline at end of file diff --git a/platform/linux-generic/include-abi/odp/api/abi/atomic.h b/platform/linux-generic/include-abi/odp/api/abi/atomic.h index 13c12a79f..7c11b0ab2 100644 --- a/platform/linux-generic/include-abi/odp/api/abi/atomic.h +++ b/platform/linux-generic/include-abi/odp/api/abi/atomic.h @@ -55,7 +55,7 @@ typedef struct ODP_ALIGNED(sizeof(uint64_t)) odp_atomic_u64_s { #endif -#ifdef _ODP_LOCK_FREE_128BIT_ATOMICS +#if defined(__SIZEOF_INT128__) || defined(_ODP_LOCK_FREE_128BIT_ATOMICS) /** * @internal diff --git a/platform/linux-generic/include-abi/odp/api/abi/event.h b/platform/linux-generic/include-abi/odp/api/abi/event.h index 27d750d16..1cbb81afe 100644 --- a/platform/linux-generic/include-abi/odp/api/abi/event.h +++ b/platform/linux-generic/include-abi/odp/api/abi/event.h @@ -34,7 +34,8 @@ typedef enum odp_event_type_t { ODP_EVENT_TIMEOUT = 3, ODP_EVENT_CRYPTO_COMPL = 4, ODP_EVENT_IPSEC_STATUS = 5, - ODP_EVENT_PACKET_VECTOR = 6 + ODP_EVENT_PACKET_VECTOR = 6, + ODP_EVENT_PACKET_TX_COMPL = 7 } odp_event_type_t; typedef enum odp_event_subtype_t { diff --git a/platform/linux-generic/include-abi/odp/api/abi/packet.h b/platform/linux-generic/include-abi/odp/api/abi/packet.h index 76ec97dc7..28e97637c 100644 --- a/platform/linux-generic/include-abi/odp/api/abi/packet.h +++ b/platform/linux-generic/include-abi/odp/api/abi/packet.h @@ -39,6 +39,10 @@ typedef ODP_HANDLE_T(odp_packet_vector_t); #define ODP_PACKET_VECTOR_INVALID _odp_cast_scalar(odp_packet_vector_t, 0) +typedef ODP_HANDLE_T(odp_packet_tx_compl_t); + +#define ODP_PACKET_TX_COMPL_INVALID _odp_cast_scalar(odp_packet_tx_compl_t, 0) + #define ODP_PACKET_OFFSET_INVALID 0xffff typedef uint8_t odp_proto_l2_type_t; diff --git a/platform/linux-generic/include/odp/api/plat/atomic_inlines.h b/platform/linux-generic/include/odp/api/plat/atomic_inlines.h index 5f0cba05e..4ab8bb411 100644 --- a/platform/linux-generic/include/odp/api/plat/atomic_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/atomic_inlines.h @@ -1,4 +1,5 @@ /* Copyright (c) 2016-2018, Linaro Limited + * Copyright (c) 2021, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -13,6 +14,8 @@ #ifndef _ODP_PLAT_ATOMIC_INLINES_H_ #define _ODP_PLAT_ATOMIC_INLINES_H_ +#include <odp/api/abi/atomic_inlines.h> + /** @cond _ODP_HIDE_FROM_DOXYGEN_ */ #ifndef _ODP_NO_INLINE @@ -82,6 +85,14 @@ #define odp_atomic_cas_acq_u32 __odp_atomic_cas_acq_u32 #define odp_atomic_cas_rel_u32 __odp_atomic_cas_rel_u32 #define odp_atomic_cas_acq_rel_u32 __odp_atomic_cas_acq_rel_u32 + #define odp_atomic_init_u128 __odp_atomic_init_u128 + #define odp_atomic_load_u128 __odp_atomic_load_u128 + #define odp_atomic_store_u128 __odp_atomic_store_u128 + #define odp_atomic_cas_u128 __odp_atomic_cas_u128 + #define odp_atomic_cas_acq_u128 __odp_atomic_cas_acq_u128 + #define odp_atomic_cas_rel_u128 __odp_atomic_cas_rel_u128 + #define odp_atomic_cas_acq_rel_u128 __odp_atomic_cas_acq_rel_u128 + #else #define _ODP_INLINE #endif @@ -530,6 +541,45 @@ _ODP_INLINE int odp_atomic_cas_acq_rel_u32(odp_atomic_u32_t *atom, __ATOMIC_RELAXED); } +_ODP_INLINE void odp_atomic_init_u128(odp_atomic_u128_t *atom, odp_u128_t val) +{ + _odp_atomic_init_u128(atom, val); +} + +_ODP_INLINE odp_u128_t odp_atomic_load_u128(odp_atomic_u128_t *atom) +{ + return _odp_atomic_load_u128(atom); +} + +_ODP_INLINE void odp_atomic_store_u128(odp_atomic_u128_t *atom, odp_u128_t val) +{ + _odp_atomic_store_u128(atom, val); +} + +_ODP_INLINE int odp_atomic_cas_u128(odp_atomic_u128_t *atom, + odp_u128_t *old_val, odp_u128_t new_val) +{ + return _odp_atomic_cas_u128(atom, old_val, new_val); +} + +_ODP_INLINE int odp_atomic_cas_acq_u128(odp_atomic_u128_t *atom, + odp_u128_t *old_val, odp_u128_t new_val) +{ + return _odp_atomic_cas_acq_u128(atom, old_val, new_val); +} + +_ODP_INLINE int odp_atomic_cas_rel_u128(odp_atomic_u128_t *atom, + odp_u128_t *old_val, odp_u128_t new_val) +{ + return _odp_atomic_cas_rel_u128(atom, old_val, new_val); +} + +_ODP_INLINE int odp_atomic_cas_acq_rel_u128(odp_atomic_u128_t *atom, + odp_u128_t *old_val, odp_u128_t new_val) +{ + return _odp_atomic_cas_acq_rel_u128(atom, old_val, new_val); +} + /** @endcond */ #endif diff --git a/platform/linux-generic/include/odp_bitset.h b/platform/linux-generic/include/odp_bitset.h index 5c10ef9e5..0931fb337 100644 --- a/platform/linux-generic/include/odp_bitset.h +++ b/platform/linux-generic/include/odp_bitset.h @@ -24,39 +24,8 @@ * (lock-free) max is 128 */ -/* Find a suitable data type that supports lock-free atomic operations */ -#if defined(__aarch64__) && defined(__SIZEOF_INT128__) && \ - __SIZEOF_INT128__ == 16 -#define LOCKFREE16 -typedef __int128 bitset_t; -#define ATOM_BITSET_SIZE (CHAR_BIT * __SIZEOF_INT128__) - -#elif __GCC_ATOMIC_LLONG_LOCK_FREE == 2 && \ - __SIZEOF_LONG_LONG__ != __SIZEOF_LONG__ -typedef unsigned long long bitset_t; -#define ATOM_BITSET_SIZE (CHAR_BIT * __SIZEOF_LONG_LONG__) - -#elif __GCC_ATOMIC_LONG_LOCK_FREE == 2 && __SIZEOF_LONG__ != __SIZEOF_INT__ -typedef unsigned long bitset_t; -#define ATOM_BITSET_SIZE (CHAR_BIT * __SIZEOF_LONG__) - -#elif __GCC_ATOMIC_INT_LOCK_FREE == 2 -typedef unsigned int bitset_t; -#define ATOM_BITSET_SIZE (CHAR_BIT * __SIZEOF_INT__) - -#else -/* Target does not support lock-free atomic operations */ -typedef unsigned int bitset_t; -#define ATOM_BITSET_SIZE (CHAR_BIT * __SIZEOF_INT__) -#endif - #if ATOM_BITSET_SIZE <= 32 -static inline bitset_t bitset_mask(uint32_t bit) -{ - return 1UL << bit; -} - /* Return first-bit-set with StdC ffs() semantics */ static inline uint32_t bitset_ffs(bitset_t b) { @@ -71,11 +40,6 @@ static inline bitset_t bitset_monitor(bitset_t *bs, int mo) #elif ATOM_BITSET_SIZE <= 64 -static inline bitset_t bitset_mask(uint32_t bit) -{ - return 1ULL << bit; -} - /* Return first-bit-set with StdC ffs() semantics */ static inline uint32_t bitset_ffs(bitset_t b) { @@ -90,14 +54,6 @@ static inline bitset_t bitset_monitor(bitset_t *bs, int mo) #elif ATOM_BITSET_SIZE <= 128 -static inline bitset_t bitset_mask(uint32_t bit) -{ - if (bit < 64) - return 1ULL << bit; - else - return (unsigned __int128)(1ULL << (bit - 64)) << 64; -} - /* Return first-bit-set with StdC ffs() semantics */ static inline uint32_t bitset_ffs(bitset_t b) { @@ -119,60 +75,6 @@ static inline bitset_t bitset_monitor(bitset_t *bs, int mo) #error Unsupported size of bit sets (ATOM_BITSET_SIZE) #endif -/* Atomic load with memory ordering */ -static inline bitset_t atom_bitset_load(bitset_t *bs, int mo) -{ -#ifdef LOCKFREE16 - return __lockfree_load_16(bs, mo); -#else - return __atomic_load_n(bs, mo); -#endif -} - -/* Atomic bit set with memory ordering */ -static inline void atom_bitset_set(bitset_t *bs, uint32_t bit, int mo) -{ -#ifdef LOCKFREE16 - (void)__lockfree_fetch_or_16(bs, bitset_mask(bit), mo); -#else - (void)__atomic_fetch_or(bs, bitset_mask(bit), mo); -#endif -} - -/* Atomic bit clear with memory ordering */ -static inline void atom_bitset_clr(bitset_t *bs, uint32_t bit, int mo) -{ -#ifdef LOCKFREE16 - (void)__lockfree_fetch_and_16(bs, ~bitset_mask(bit), mo); -#else - (void)__atomic_fetch_and(bs, ~bitset_mask(bit), mo); -#endif -} - -/* Atomic exchange with memory ordering */ -static inline bitset_t atom_bitset_xchg(bitset_t *bs, bitset_t neu, int mo) -{ -#ifdef LOCKFREE16 - return __lockfree_exchange_16(bs, neu, mo); -#else - return __atomic_exchange_n(bs, neu, mo); -#endif -} - -/* Atomic compare&exchange with memory ordering */ -static inline bitset_t atom_bitset_cmpxchg(bitset_t *bs, bitset_t *old, - bitset_t neu, bool weak, - int mo_success, int mo_failure) -{ -#ifdef LOCKFREE16 - return __lockfree_compare_exchange_16(bs, old, neu, weak, mo_success, - mo_failure); -#else - return __atomic_compare_exchange_n(bs, old, neu, weak, mo_success, - mo_failure); -#endif -} - /* Return a & ~b */ static inline bitset_t bitset_andn(bitset_t a, bitset_t b) { diff --git a/platform/linux-generic/include/odp_config_internal.h b/platform/linux-generic/include/odp_config_internal.h index 35f1c9142..1e8b390dd 100644 --- a/platform/linux-generic/include/odp_config_internal.h +++ b/platform/linux-generic/include/odp_config_internal.h @@ -1,5 +1,5 @@ /* Copyright (c) 2016-2018, Linaro Limited - * Copyright (c) 2019-2020, Nokia + * Copyright (c) 2019-2021, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -147,6 +147,13 @@ extern "C" { /* Enable pool statistics collection */ #define CONFIG_POOL_STATISTICS 1 +/* + * Maximum number of IPsec SAs. The actual maximum number can be further + * limited by the number of sessions supported by the crypto subsystem and + * is reported by odp_ipsec_capability(). + */ +#define CONFIG_IPSEC_MAX_NUM_SA 4000 + #ifdef __cplusplus } #endif diff --git a/platform/linux-generic/include/odp_errno_define.h b/platform/linux-generic/include/odp_errno_define.h index 570d37387..3f97618b0 100644 --- a/platform/linux-generic/include/odp_errno_define.h +++ b/platform/linux-generic/include/odp_errno_define.h @@ -17,7 +17,7 @@ extern "C" { #endif -extern __thread int __odp_errno; +extern __thread int _odp_errno; #ifdef __cplusplus } diff --git a/platform/linux-generic/include/odp_ipsec_internal.h b/platform/linux-generic/include/odp_ipsec_internal.h index 311c0e50c..2509d22ab 100644 --- a/platform/linux-generic/include/odp_ipsec_internal.h +++ b/platform/linux-generic/include/odp_ipsec_internal.h @@ -86,11 +86,6 @@ int _odp_ipsec_status_send(odp_queue_t queue, /* 32 is minimum required by the standard. We do not support more */ #define IPSEC_ANTIREPLAY_WS 32 -/** - * Maximum number of available SAs - */ -#define ODP_CONFIG_IPSEC_SAS 8 - struct ipsec_sa_s { odp_atomic_u32_t state ODP_ALIGNED_CACHE; @@ -240,6 +235,9 @@ uint32_t _odp_ipsec_cipher_iv_len(odp_cipher_alg_t cipher); /* Return digest length required for the cipher for IPsec use */ uint32_t _odp_ipsec_auth_digest_len(odp_auth_alg_t auth); +/* Return the maximum number of SAs supported by the implementation */ +uint32_t _odp_ipsec_max_num_sa(void); + /* * Get SA entry from handle without obtaining a reference */ diff --git a/platform/linux-generic/include/odp_packet_internal.h b/platform/linux-generic/include/odp_packet_internal.h index 8349df43a..4af4bf062 100644 --- a/platform/linux-generic/include/odp_packet_internal.h +++ b/platform/linux-generic/include/odp_packet_internal.h @@ -300,8 +300,8 @@ static inline void packet_set_len(odp_packet_hdr_t *pkt_hdr, uint32_t len) int _odp_packet_copy_md_to_packet(odp_packet_t srcpkt, odp_packet_t dstpkt); /* Packet alloc of pktios */ -int packet_alloc_multi(odp_pool_t pool_hdl, uint32_t len, - odp_packet_t pkt[], int max_num); +int _odp_packet_alloc_multi(odp_pool_t pool_hdl, uint32_t len, + odp_packet_t pkt[], int max_num); /* Perform packet parse up to a given protocol layer */ int _odp_packet_parse_layer(odp_packet_hdr_t *pkt_hdr, diff --git a/platform/linux-generic/include/odp_packet_io_internal.h b/platform/linux-generic/include/odp_packet_io_internal.h index c6b916240..07b0b4b69 100644 --- a/platform/linux-generic/include/odp_packet_io_internal.h +++ b/platform/linux-generic/include/odp_packet_io_internal.h @@ -265,8 +265,8 @@ static inline void _odp_pktio_tx_ts_set(pktio_entry_t *entry) odp_atomic_store_u64(&entry->s.tx_ts, ts_val.u64); } -extern const pktio_if_ops_t netmap_pktio_ops; -extern const pktio_if_ops_t dpdk_pktio_ops; +extern const pktio_if_ops_t _odp_netmap_pktio_ops; +extern const pktio_if_ops_t _odp_dpdk_pktio_ops; extern const pktio_if_ops_t _odp_sock_mmsg_pktio_ops; extern const pktio_if_ops_t _odp_sock_mmap_pktio_ops; extern const pktio_if_ops_t _odp_loopback_pktio_ops; @@ -276,7 +276,7 @@ extern const pktio_if_ops_t _odp_pcap_pktio_ops; extern const pktio_if_ops_t _odp_tap_pktio_ops; extern const pktio_if_ops_t _odp_null_pktio_ops; extern const pktio_if_ops_t _odp_ipc_pktio_ops; -extern const pktio_if_ops_t * const pktio_if_ops[]; +extern const pktio_if_ops_t * const _odp_pktio_if_ops[]; /** * Try interrupt-driven receive diff --git a/platform/linux-generic/include/odp_pool_internal.h b/platform/linux-generic/include/odp_pool_internal.h index 9deb63cd0..a0e4c5c65 100644 --- a/platform/linux-generic/include/odp_pool_internal.h +++ b/platform/linux-generic/include/odp_pool_internal.h @@ -160,8 +160,8 @@ static inline odp_buffer_hdr_t *buf_hdr_from_index_u32(uint32_t u32) return buf_hdr_from_index(pool, buffer_idx); } -int buffer_alloc_multi(pool_t *pool, odp_buffer_hdr_t *buf_hdr[], int num); -void buffer_free_multi(odp_buffer_hdr_t *buf_hdr[], int num_free); +int _odp_buffer_alloc_multi(pool_t *pool, odp_buffer_hdr_t *buf_hdr[], int num); +void _odp_buffer_free_multi(odp_buffer_hdr_t *buf_hdr[], int num_free); int _odp_buffer_is_valid(odp_buffer_t buf); #ifdef __cplusplus diff --git a/platform/linux-generic/include/odp_random_openssl_internal.h b/platform/linux-generic/include/odp_random_openssl_internal.h index 7784d560e..3205a2c32 100644 --- a/platform/linux-generic/include/odp_random_openssl_internal.h +++ b/platform/linux-generic/include/odp_random_openssl_internal.h @@ -16,7 +16,6 @@ extern "C" { #include <odp/api/random.h> odp_random_kind_t _odp_random_openssl_max_kind(void); -int32_t _odp_random_openssl_test_data(uint8_t *buf, uint32_t len, uint64_t *seed); int32_t _odp_random_openssl_data(uint8_t *buf, uint32_t len, odp_random_kind_t kind); int _odp_random_openssl_init_local(void); int _odp_random_openssl_term_local(void); diff --git a/platform/linux-generic/include/odp_timer_internal.h b/platform/linux-generic/include/odp_timer_internal.h index 0c2a0fe4b..23fd54bf9 100644 --- a/platform/linux-generic/include/odp_timer_internal.h +++ b/platform/linux-generic/include/odp_timer_internal.h @@ -40,13 +40,13 @@ typedef struct { /* A larger decrement value should be used after receiving events compared to * an 'empty' call. */ -void _timer_run_inline(int dec); +void _odp_timer_run_inline(int dec); /* Static inline wrapper to minimize modification of schedulers. */ static inline void timer_run(int dec) { if (odp_global_rw->inline_timers) - _timer_run_inline(dec); + _odp_timer_run_inline(dec); } #endif diff --git a/platform/linux-generic/libodp-linux.pc.in b/platform/linux-generic/libodp-linux.pc.in index a15ff2dbc..2b28414e9 100644 --- a/platform/linux-generic/libodp-linux.pc.in +++ b/platform/linux-generic/libodp-linux.pc.in @@ -7,6 +7,6 @@ Name: lib@ODP_LIB_NAME@ Description: The ODP packet processing engine Version: @PKGCONFIG_VERSION@ Requires.private: libconfig -Libs: -L${libdir} -l@ODP_LIB_NAME@ -Libs.private: @OPENSSL_STATIC_LIBS@ @DPDK_LIBS@ @PCAP_LIBS@ @PTHREAD_LIBS@ @TIMER_LIBS@ -lpthread @ATOMIC_LIBS@ +Libs: -L${libdir} -l@ODP_LIB_NAME@ @ATOMIC_LIBS_NON_ABI_COMPAT@ +Libs.private: @OPENSSL_STATIC_LIBS@ @DPDK_LIBS@ @PCAP_LIBS@ @PTHREAD_LIBS@ @TIMER_LIBS@ -lpthread @ATOMIC_LIBS_ABI_COMPAT@ Cflags: -I${includedir} diff --git a/platform/linux-generic/m4/configure.m4 b/platform/linux-generic/m4/configure.m4 index 924346c1f..2ebc23174 100644 --- a/platform/linux-generic/m4/configure.m4 +++ b/platform/linux-generic/m4/configure.m4 @@ -25,7 +25,7 @@ m4_include([platform/linux-generic/m4/odp_netmap.m4]) m4_include([platform/linux-generic/m4/odp_dpdk.m4]) ODP_SCHEDULER -AS_VAR_APPEND([PLAT_DEP_LIBS], ["${LIBCONFIG_LIBS} ${OPENSSL_LIBS} ${DPDK_LIBS_LT} ${LIBCLI_LIBS}"]) +AS_VAR_APPEND([PLAT_DEP_LIBS], ["${ATOMIC_LIBS} ${LIBCONFIG_LIBS} ${OPENSSL_LIBS} ${DPDK_LIBS_LT} ${LIBCLI_LIBS}"]) # Add text to the end of configure with platform specific settings. # Make sure it's aligned same as other lines in configure.ac. diff --git a/platform/linux-generic/odp_atomic.c b/platform/linux-generic/odp_atomic.c deleted file mode 100644 index 59253c645..000000000 --- a/platform/linux-generic/odp_atomic.c +++ /dev/null @@ -1,306 +0,0 @@ -/* Copyright (c) 2015-2018, Linaro Limited - * Copyright (c) 2021, ARM Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -#include <odp/api/atomic.h> -#include <odp_cpu.h> - -int odp_atomic_lock_free_u64(odp_atomic_op_t *atomic_op) -{ -#if __GCC_ATOMIC_LLONG_LOCK_FREE < 2 - /* All operations have locks */ - if (atomic_op) - atomic_op->all_bits = 0; - - return 0; -#else - /* All operations are lock-free */ - if (atomic_op) { - atomic_op->all_bits = ~((uint32_t)0); - atomic_op->op.init = 0; - } - - return 2; -#endif -} - -int odp_atomic_lock_free_u128(odp_atomic_op_t *atomic_op) -{ -#ifdef _ODP_LOCK_FREE_128BIT_ATOMICS - if (atomic_op) { - atomic_op->all_bits = 0; - atomic_op->op.load = 1; - atomic_op->op.store = 1; - atomic_op->op.cas = 1; - } - - return 2; -#else - /* All operations have locks */ - if (atomic_op) - atomic_op->all_bits = 0; - - return 0; -#endif -} - -#ifdef _ODP_LOCK_FREE_128BIT_ATOMICS - -static void __atomic_init_u128(odp_atomic_u128_t *atom, odp_u128_t new_val) -{ - odp_u128_t old, val; - - old = atom->v; - - while (1) { - ATOMIC_CAS_OP_128_NO_ORDER(atom, &old, new_val, val); - - if ((val.u64[0] == old.u64[0]) && (val.u64[1] == old.u64[1])) - return; - - old = val; - } -} - -static odp_u128_t __atomic_load_u128(odp_atomic_u128_t *atom) -{ - odp_u128_t val, exp; - - exp.u64[0] = 0; - exp.u64[1] = 0; - ATOMIC_CAS_OP_128_NO_ORDER(atom, &exp, exp, val); - return val; -} - -static void __atomic_store_u128(odp_atomic_u128_t *atom, odp_u128_t new_val) -{ - odp_u128_t old, val; - - old = atom->v; - - while (1) { - ATOMIC_CAS_OP_128_NO_ORDER(atom, &old, new_val, val); - - if ((val.u64[0] == old.u64[0]) && (val.u64[1] == old.u64[1])) - return; - - old = val; - } -} - -static int __atomic_cas_u128(odp_atomic_u128_t *atom, - odp_u128_t *old_val, odp_u128_t new_val) -{ - int ret = 0; - odp_u128_t val; - - ATOMIC_CAS_OP_128_NO_ORDER(atom, old_val, new_val, val); - - if ((val.u64[0] == old_val->u64[0]) && (val.u64[1] == old_val->u64[1])) - ret = 1; - - old_val->u64[0] = val.u64[0]; - old_val->u64[1] = val.u64[1]; - - return ret; -} - -static int __atomic_cas_acq_u128(odp_atomic_u128_t *atom, - odp_u128_t *old_val, odp_u128_t new_val) -{ - int ret = 0; - odp_u128_t val; - - ATOMIC_CAS_OP_128_ACQ(atom, old_val, new_val, val); - - if ((val.u64[0] == old_val->u64[0]) && (val.u64[1] == old_val->u64[1])) - ret = 1; - - old_val->u64[0] = val.u64[0]; - old_val->u64[1] = val.u64[1]; - - return ret; -} - -static int __atomic_cas_rel_u128(odp_atomic_u128_t *atom, - odp_u128_t *old_val, odp_u128_t new_val) -{ - int ret = 0; - odp_u128_t val; - - ATOMIC_CAS_OP_128_REL(atom, old_val, new_val, val); - - if ((val.u64[0] == old_val->u64[0]) && (val.u64[1] == old_val->u64[1])) - ret = 1; - - old_val->u64[0] = val.u64[0]; - old_val->u64[1] = val.u64[1]; - - return ret; -} - -static int __atomic_cas_acq_rel_u128(odp_atomic_u128_t *atom, - odp_u128_t *old_val, - odp_u128_t new_val) -{ - int ret = 0; - odp_u128_t val; - - ATOMIC_CAS_OP_128_ACQ_REL(atom, old_val, new_val, val); - - if ((val.u64[0] == old_val->u64[0]) && (val.u64[1] == old_val->u64[1])) - ret = 1; - - old_val->u64[0] = val.u64[0]; - old_val->u64[1] = val.u64[1]; - - return ret; -} - -#else /* Locked version */ - -/** - * @internal - * 128 bit store operation expression for the ATOMIC_OP macro - */ -#define ATOMIC_STORE_OP_128(new_val) \ -({ \ - (_atom)->v = (new_val); \ -}) - -/** - * @internal - * 128 bit CAS operation expression for the ATOMIC_OP macro - */ -#define ATOMIC_CAS_OP_128(ret_ptr, old_val, new_val) \ -({ \ - int *_ret_ptr = ret_ptr; \ - odp_u128_t *_old_val = old_val; \ - odp_u128_t _new_val = new_val; \ - if (((_atom)->v.u64[0] == (_old_val)->u64[0]) && \ - ((_atom)->v.u64[1] == (_old_val)->u64[1])) { \ - (_atom)->v = (_new_val); \ - *(_ret_ptr) = 1; \ - } else { \ - *(_ret_ptr) = 0; \ - } \ -}) - -/** - * @internal - * Helper macro for lock-based atomic operations on 128-bit integers - * @param[in,out] atom Pointer to the 128-bit atomic variable - * @param expr Expression used update the variable. - * @return The old value of the variable. - */ -#define ATOMIC_OP_128(atom, expr) \ -({ \ - odp_u128_t _old_val; \ - odp_atomic_u128_t *_atom = atom; \ - /* Loop while lock is already taken, stop when lock becomes clear */ \ - while (__atomic_test_and_set(&(_atom)->lock, __ATOMIC_ACQUIRE)) \ - (void)0; \ - _old_val = (_atom)->v; \ - (expr); /* Perform whatever update is desired */ \ - __atomic_clear(&(_atom)->lock, __ATOMIC_RELEASE); \ - _old_val; /* Return old value */ \ -}) - -static void __atomic_init_u128(odp_atomic_u128_t *atom, odp_u128_t val) -{ - atom->lock = 0; - ATOMIC_OP_128(atom, ATOMIC_STORE_OP_128(val)); -} - -static odp_u128_t __atomic_load_u128(odp_atomic_u128_t *atom) -{ - return ATOMIC_OP_128(atom, (void)0); -} - -static void __atomic_store_u128(odp_atomic_u128_t *atom, odp_u128_t val) -{ - ATOMIC_OP_128(atom, ATOMIC_STORE_OP_128(val)); -} - -static int __atomic_cas_u128(odp_atomic_u128_t *atom, - odp_u128_t *old_val, odp_u128_t new_val) -{ - int ret; - *old_val = ATOMIC_OP_128(atom, ATOMIC_CAS_OP_128(&ret, old_val, - new_val)); - return ret; -} - -static int __atomic_cas_acq_u128(odp_atomic_u128_t *atom, - odp_u128_t *old_val, - odp_u128_t new_val) -{ - int ret; - *old_val = ATOMIC_OP_128(atom, ATOMIC_CAS_OP_128(&ret, old_val, - new_val)); - return ret; -} - -static int __atomic_cas_rel_u128(odp_atomic_u128_t *atom, - odp_u128_t *old_val, - odp_u128_t new_val) -{ - int ret; - *old_val = ATOMIC_OP_128(atom, ATOMIC_CAS_OP_128(&ret, old_val, - new_val)); - return ret; -} - -static int __atomic_cas_acq_rel_u128(odp_atomic_u128_t *atom, - odp_u128_t *old_val, - odp_u128_t new_val) -{ - int ret; - *old_val = ATOMIC_OP_128(atom, ATOMIC_CAS_OP_128(&ret, old_val, - new_val)); - return ret; -} - -#endif - -void odp_atomic_init_u128(odp_atomic_u128_t *atom, odp_u128_t val) -{ - __atomic_init_u128(atom, val); -} - -odp_u128_t odp_atomic_load_u128(odp_atomic_u128_t *atom) -{ - return __atomic_load_u128(atom); -} - -void odp_atomic_store_u128(odp_atomic_u128_t *atom, odp_u128_t val) -{ - __atomic_store_u128(atom, val); -} - -int odp_atomic_cas_u128(odp_atomic_u128_t *atom, - odp_u128_t *old_val, odp_u128_t new_val) -{ - return __atomic_cas_u128(atom, old_val, new_val); -} - -int odp_atomic_cas_acq_u128(odp_atomic_u128_t *atom, - odp_u128_t *old_val, odp_u128_t new_val) -{ - return __atomic_cas_acq_u128(atom, old_val, new_val); -} - -int odp_atomic_cas_rel_u128(odp_atomic_u128_t *atom, - odp_u128_t *old_val, odp_u128_t new_val) -{ - return __atomic_cas_rel_u128(atom, old_val, new_val); -} - -int odp_atomic_cas_acq_rel_u128(odp_atomic_u128_t *atom, - odp_u128_t *old_val, odp_u128_t new_val) -{ - return __atomic_cas_acq_rel_u128(atom, old_val, new_val); -} diff --git a/platform/linux-generic/odp_classification.c b/platform/linux-generic/odp_classification.c index bfa410b44..9b31f99a8 100644 --- a/platform/linux-generic/odp_classification.c +++ b/platform/linux-generic/odp_classification.c @@ -120,12 +120,14 @@ int _odp_classification_init_global(void) for (i = 0; i < CLS_COS_MAX_ENTRY; i++) { /* init locks */ cos_t *cos = get_cos_entry_internal(_odp_cos_from_ndx(i)); + LOCK_INIT(&cos->s.lock); } for (i = 0; i < CLS_PMR_MAX_ENTRY; i++) { /* init locks */ pmr_t *pmr = get_pmr_entry_internal(_odp_pmr_from_ndx(i)); + LOCK_INIT(&pmr->s.lock); } @@ -290,8 +292,7 @@ odp_cos_t odp_cls_cos_create(const char *name, const odp_cls_cos_param_t *param) param->hash_proto); tbl_index = i * CLS_COS_QUEUE_MAX; for (j = 0; j < param->num_queue; j++) { - queue = odp_queue_create(NULL, &cos->s. - queue_param); + queue = odp_queue_create(NULL, &cos->s.queue_param); if (queue == ODP_QUEUE_INVALID) { /* unwind the queues */ _cls_queue_unwind(tbl_index, j); @@ -1781,3 +1782,101 @@ uint64_t odp_pmr_to_u64(odp_pmr_t hdl) { return _odp_pri(hdl); } + +static +void print_cos_ident(struct cos_s *cos) +{ + if (strlen(cos->name)) + ODP_PRINT("%s", cos->name); + + ODP_PRINT("(%" PRIu64 ")\n", + odp_cos_to_u64(_odp_cos_from_ndx(cos->index))); +} + +static +void print_queue_ident(odp_queue_t q) +{ + odp_queue_info_t info; + + if (!odp_queue_info(q, &info) && strlen(info.name)) + ODP_PRINT("%s", info.name); + else + ODP_PRINT("%" PRIx64, odp_queue_to_u64(q)); + + ODP_PRINT("\n"); +} + +static +void print_hex(const void *vp, int len) +{ + const uint8_t *p = vp; + + for (int i = 0; i < len; i++) + ODP_PRINT("%02x", *p++); +} + +static +void cls_print_cos(struct cos_s *cos) +{ + uint32_t num_rule = odp_atomic_load_u32(&cos->num_rule); + bool first = true; + + ODP_PRINT("cos: "); + print_cos_ident(cos); + ODP_PRINT(" queue: "); + print_queue_ident(cos->queue); + + for (uint32_t j = 0; j < num_rule; j++) { + struct pmr_s *pmr = &cos->pmr[j]->s; + + LOCK(&pmr->lock); + for (uint32_t k = 0; k < pmr->num_pmr; k++) { + pmr_term_value_t *v = &pmr->pmr_term_value[k]; + + if (first) + ODP_PRINT(" rules: "); + else + ODP_PRINT(" "); + + first = false; + + ODP_PRINT("%s: ", format_pmr_name(v->term)); + + if (v->term == ODP_PMR_CUSTOM_FRAME || + v->term == ODP_PMR_CUSTOM_L3) + ODP_PRINT("offset:%" PRIu32 " ", v->offset); + + if (v->range_term) { + ODP_PRINT("<range>"); + } else { + print_hex(v->match.value_u8, v->val_sz); + ODP_PRINT(" "); + print_hex(v->match.mask_u8, v->val_sz); + } + + ODP_PRINT(" -> "); + + if (pmr->mark) + ODP_PRINT("mark:%" PRIu16 " ", pmr->mark); + + print_cos_ident(&cos->linked_cos[j]->s); + } + UNLOCK(&pmr->lock); + } +} + +void odp_cls_print_all(void) +{ + ODP_PRINT("\n" + "Classifier info\n" + "---------------\n\n"); + + for (uint32_t i = 0; i < CLS_COS_MAX_ENTRY; i++) { + struct cos_s *cos = &cos_tbl->cos_entry[i].s; + + LOCK(&cos->lock); + if (cos->valid) + cls_print_cos(cos); + UNLOCK(&cos->lock); + } +} diff --git a/platform/linux-generic/odp_crypto_null.c b/platform/linux-generic/odp_crypto_null.c index 6dd98a2b7..cf9abc99f 100644 --- a/platform/linux-generic/odp_crypto_null.c +++ b/platform/linux-generic/odp_crypto_null.c @@ -121,6 +121,8 @@ int odp_crypto_capability(odp_crypto_capability_t *capa) capa->sync_mode = ODP_SUPPORT_PREFERRED; capa->async_mode = ODP_SUPPORT_YES; + capa->queue_type_plain = 1; + capa->queue_type_sched = 1; capa->ciphers.bit.null = 1; diff --git a/platform/linux-generic/odp_crypto_openssl.c b/platform/linux-generic/odp_crypto_openssl.c index 64984e201..07a91cc46 100644 --- a/platform/linux-generic/odp_crypto_openssl.c +++ b/platform/linux-generic/odp_crypto_openssl.c @@ -37,7 +37,7 @@ #define _ODP_HAVE_CHACHA20_POLY1305 0 #endif -#define MAX_SESSIONS 32 +#define MAX_SESSIONS 4000 #define AES_BLOCK_SIZE 16 #define AES_KEY_LENGTH 16 @@ -314,9 +314,9 @@ null_crypto_routine(odp_packet_t pkt ODP_UNUSED, } static void -null_crypto_init_routine(odp_crypto_generic_session_t *session ODP_UNUSED) +null_crypto_init_routine(odp_crypto_generic_session_t *session) { - return; + (void)session; } /* Mimic new OpenSSL 1.1.y API */ @@ -1849,6 +1849,8 @@ int odp_crypto_capability(odp_crypto_capability_t *capa) capa->sync_mode = ODP_SUPPORT_PREFERRED; capa->async_mode = ODP_SUPPORT_YES; + capa->queue_type_plain = 1; + capa->queue_type_sched = 1; capa->ciphers.bit.null = 1; capa->ciphers.bit.trides_cbc = 1; diff --git a/platform/linux-generic/odp_errno.c b/platform/linux-generic/odp_errno.c index 162300991..71fc2da77 100644 --- a/platform/linux-generic/odp_errno.c +++ b/platform/linux-generic/odp_errno.c @@ -9,24 +9,24 @@ #include <stdio.h> #include <odp_debug_internal.h> -__thread int __odp_errno; +__thread int _odp_errno; int odp_errno(void) { - return __odp_errno; + return _odp_errno; } void odp_errno_zero(void) { - __odp_errno = 0; + _odp_errno = 0; } void odp_errno_print(const char *str) { if (str != NULL) - ODP_PRINT("%s %s\n", str, strerror(__odp_errno)); + ODP_PRINT("%s %s\n", str, strerror(_odp_errno)); else - ODP_PRINT("%s\n", strerror(__odp_errno)); + ODP_PRINT("%s\n", strerror(_odp_errno)); } const char *odp_errno_str(int errnum) diff --git a/platform/linux-generic/odp_fdserver.c b/platform/linux-generic/odp_fdserver.c index 11e7602af..9e0d75de3 100644 --- a/platform/linux-generic/odp_fdserver.c +++ b/platform/linux-generic/odp_fdserver.c @@ -513,7 +513,7 @@ static int handle_request(int client_sock) break; case FD_SERVERSTOP_REQ: - FD_ODP_DBG("Stoping FD server\n"); + FD_ODP_DBG("Stopping FD server\n"); return 1; default: diff --git a/platform/linux-generic/odp_init.c b/platform/linux-generic/odp_init.c index 27390c13b..9e734f1a6 100644 --- a/platform/linux-generic/odp_init.c +++ b/platform/linux-generic/odp_init.c @@ -294,10 +294,10 @@ int odp_init_global(odp_instance_t *instance, const odp_init_t *params, const odp_platform_init_t *platform_params ODP_UNUSED) { + enum init_stage stage = NO_INIT; + memset(&odp_global_ro, 0, sizeof(odp_global_data_ro_t)); odp_global_ro.main_pid = getpid(); - - enum init_stage stage = NO_INIT; odp_global_ro.log_fn = odp_override_log; odp_global_ro.abort_fn = odp_override_abort; diff --git a/platform/linux-generic/odp_ipsec.c b/platform/linux-generic/odp_ipsec.c index f8746f812..766fd10b7 100644 --- a/platform/linux-generic/odp_ipsec.c +++ b/platform/linux-generic/odp_ipsec.c @@ -1,4 +1,5 @@ /* Copyright (c) 2017-2018, Linaro Limited + * Copyright (c) 2018-2021, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -121,7 +122,7 @@ int odp_ipsec_capability(odp_ipsec_capability_t *capa) capa->proto_ah = ODP_SUPPORT_YES; - capa->max_num_sa = ODP_CONFIG_IPSEC_SAS; + capa->max_num_sa = _odp_ipsec_max_num_sa(); capa->max_antireplay_ws = IPSEC_ANTIREPLAY_WS; @@ -215,7 +216,7 @@ void odp_ipsec_config_init(odp_ipsec_config_t *config) memset(config, 0, sizeof(odp_ipsec_config_t)); config->inbound_mode = ODP_IPSEC_OP_MODE_SYNC; config->outbound_mode = ODP_IPSEC_OP_MODE_SYNC; - config->max_num_sa = ODP_CONFIG_IPSEC_SAS; + config->max_num_sa = _odp_ipsec_max_num_sa(); config->inbound.default_queue = ODP_QUEUE_INVALID; config->inbound.lookup.min_spi = 0; config->inbound.lookup.max_spi = UINT32_MAX; @@ -224,7 +225,7 @@ void odp_ipsec_config_init(odp_ipsec_config_t *config) int odp_ipsec_config(const odp_ipsec_config_t *config) { - if (ODP_CONFIG_IPSEC_SAS < config->max_num_sa) + if (config->max_num_sa > _odp_ipsec_max_num_sa()) return -1; *ipsec_config = *config; @@ -1552,17 +1553,19 @@ static ipsec_sa_t *ipsec_out_single(odp_packet_t pkt, ipsec_out_checksums(pkt, &state); } } else { - if (state.is_ipv4) + if (state.is_ipv4) { rc = ipsec_out_tunnel_parse_ipv4(&state, ipsec_sa); - else if (state.is_ipv6) + } else if (state.is_ipv6) { rc = ipsec_out_tunnel_parse_ipv6(&state, ipsec_sa); - else if (opt->flag.tfc_dummy) { + } else if (opt->flag.tfc_dummy) { state.out_tunnel.ip_tos = 0; state.out_tunnel.ip_df = 0; state.out_tunnel.ip_flabel = 0; rc = 0; - } else + } else { rc = -1; + } + if (rc < 0) { status->error.alg = 1; goto exit; diff --git a/platform/linux-generic/odp_ipsec_sad.c b/platform/linux-generic/odp_ipsec_sad.c index 7cae32703..da8232b01 100644 --- a/platform/linux-generic/odp_ipsec_sad.c +++ b/platform/linux-generic/odp_ipsec_sad.c @@ -1,4 +1,5 @@ /* Copyright (c) 2017-2018, Linaro Limited + * Copyright (c) 2018-2021, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -9,6 +10,7 @@ #include <odp/api/random.h> #include <odp/api/shared_memory.h> +#include <odp_config_internal.h> #include <odp_init_internal.h> #include <odp_debug_internal.h> #include <odp_ipsec_internal.h> @@ -64,7 +66,7 @@ typedef struct sa_thread_local_s { odp_atomic_u32_t packet_quota; /* * Bytes that can be processed in this thread before looking at - * at the SA-global byte counter and checking hard and soft limits. + * the SA-global byte counter and checking hard and soft limits. */ uint32_t byte_quota; /* @@ -75,19 +77,20 @@ typedef struct sa_thread_local_s { } sa_thread_local_t; typedef struct ODP_ALIGNED_CACHE ipsec_thread_local_s { - sa_thread_local_t sa[ODP_CONFIG_IPSEC_SAS]; + sa_thread_local_t sa[CONFIG_IPSEC_MAX_NUM_SA]; uint16_t first_ipv4_id; /* first ID of current block of IDs */ uint16_t next_ipv4_id; /* next ID to be used */ } ipsec_thread_local_t; typedef struct ipsec_sa_table_t { - ipsec_sa_t ipsec_sa[ODP_CONFIG_IPSEC_SAS]; - ipsec_thread_local_t per_thread[ODP_THREAD_COUNT_MAX]; + ipsec_sa_t ipsec_sa[CONFIG_IPSEC_MAX_NUM_SA]; struct ODP_ALIGNED_CACHE { ring_mpmc_t ipv4_id_ring; uint32_t ipv4_id_data[IPV4_ID_RING_SIZE] ODP_ALIGNED_CACHE; } hot; + uint32_t max_num_sa; odp_shm_t shm; + ipsec_thread_local_t per_thread[]; } ipsec_sa_table_t; static ipsec_sa_table_t *ipsec_sa_tbl; @@ -122,8 +125,9 @@ static void init_sa_thread_local(ipsec_sa_t *sa) { sa_thread_local_t *sa_tl; int n; + int thread_count_max = odp_thread_count_max(); - for (n = 0; n < ODP_THREAD_COUNT_MAX; n++) { + for (n = 0; n < thread_count_max; n++) { sa_tl = &ipsec_sa_tbl->per_thread[n].sa[sa->ipsec_sa_idx]; odp_atomic_init_u32(&sa_tl->packet_quota, 0); sa_tl->byte_quota = 0; @@ -133,14 +137,28 @@ static void init_sa_thread_local(ipsec_sa_t *sa) int _odp_ipsec_sad_init_global(void) { + odp_crypto_capability_t crypto_capa; + uint32_t max_num_sa = CONFIG_IPSEC_MAX_NUM_SA; + uint64_t shm_size; + unsigned int thread_count_max = odp_thread_count_max(); odp_shm_t shm; unsigned i; if (odp_global_ro.disable.ipsec) return 0; + if (odp_crypto_capability(&crypto_capa)) { + ODP_ERR("odp_crypto_capability() failed\n"); + return -1; + } + if (max_num_sa > crypto_capa.max_sessions) + max_num_sa = crypto_capa.max_sessions; + + shm_size = sizeof(ipsec_sa_table_t) + + sizeof(ipsec_thread_local_t) * thread_count_max; + shm = odp_shm_reserve("_odp_ipsec_sa_table", - sizeof(ipsec_sa_table_t), + shm_size, ODP_CACHE_LINE_SIZE, 0); if (shm == ODP_SHM_INVALID) @@ -149,9 +167,10 @@ int _odp_ipsec_sad_init_global(void) ipsec_sa_tbl = odp_shm_addr(shm); memset(ipsec_sa_tbl, 0, sizeof(ipsec_sa_table_t)); ipsec_sa_tbl->shm = shm; + ipsec_sa_tbl->max_num_sa = max_num_sa; ring_mpmc_init(&ipsec_sa_tbl->hot.ipv4_id_ring); - for (i = 0; i < ODP_THREAD_COUNT_MAX; i++) { + for (i = 0; i < thread_count_max; i++) { /* * Make the current ID block fully used, forcing allocation * of a fresh block at first use. @@ -175,7 +194,7 @@ int _odp_ipsec_sad_init_global(void) 1); } - for (i = 0; i < ODP_CONFIG_IPSEC_SAS; i++) { + for (i = 0; i < ipsec_sa_tbl->max_num_sa; i++) { ipsec_sa_t *ipsec_sa = ipsec_sa_entry(i); ipsec_sa->ipsec_sa_hdl = ipsec_sa_index_to_handle(i); @@ -190,7 +209,7 @@ int _odp_ipsec_sad_init_global(void) int _odp_ipsec_sad_term_global(void) { - int i; + uint32_t i; ipsec_sa_t *ipsec_sa; int ret = 0; int rc = 0; @@ -198,7 +217,7 @@ int _odp_ipsec_sad_term_global(void) if (odp_global_ro.disable.ipsec) return 0; - for (i = 0; i < ODP_CONFIG_IPSEC_SAS; i++) { + for (i = 0; i < ipsec_sa_tbl->max_num_sa; i++) { ipsec_sa = ipsec_sa_entry(i); if (odp_atomic_load_u32(&ipsec_sa->state) != @@ -219,12 +238,17 @@ int _odp_ipsec_sad_term_global(void) return rc; } +uint32_t _odp_ipsec_max_num_sa(void) +{ + return ipsec_sa_tbl->max_num_sa; +} + static ipsec_sa_t *ipsec_sa_reserve(void) { - int i; + uint32_t i; ipsec_sa_t *ipsec_sa; - for (i = 0; i < ODP_CONFIG_IPSEC_SAS; i++) { + for (i = 0; i < ipsec_sa_tbl->max_num_sa; i++) { uint32_t state = IPSEC_SA_STATE_FREE; ipsec_sa = ipsec_sa_entry(i); @@ -443,7 +467,7 @@ odp_ipsec_sa_t odp_ipsec_sa_create(const odp_ipsec_sa_param_t *param) odp_atomic_init_u64(&ipsec_sa->hot.in.antireplay, 0); } else { ipsec_sa->lookup_mode = ODP_IPSEC_LOOKUP_DISABLED; - odp_atomic_store_u64(&ipsec_sa->hot.out.seq, 1); + odp_atomic_init_u64(&ipsec_sa->hot.out.seq, 1); ipsec_sa->out.frag_mode = param->outbound.frag_mode; ipsec_sa->out.mtu = param->outbound.mtu; } @@ -453,8 +477,8 @@ odp_ipsec_sa_t odp_ipsec_sa_create(const odp_ipsec_sa_param_t *param) ipsec_sa->copy_flabel = param->opt.copy_flabel; ipsec_sa->udp_encap = param->opt.udp_encap; - odp_atomic_store_u64(&ipsec_sa->hot.bytes, 0); - odp_atomic_store_u64(&ipsec_sa->hot.packets, 0); + odp_atomic_init_u64(&ipsec_sa->hot.bytes, 0); + odp_atomic_init_u64(&ipsec_sa->hot.packets, 0); ipsec_sa->soft_limit_bytes = param->lifetime.soft_limit.bytes; ipsec_sa->soft_limit_packets = param->lifetime.soft_limit.packets; ipsec_sa->hard_limit_bytes = param->lifetime.hard_limit.bytes; @@ -758,10 +782,10 @@ int odp_ipsec_sa_mtu_update(odp_ipsec_sa_t sa, uint32_t mtu) ipsec_sa_t *_odp_ipsec_sa_lookup(const ipsec_sa_lookup_t *lookup) { - int i; + uint32_t i; ipsec_sa_t *best = NULL; - for (i = 0; i < ODP_CONFIG_IPSEC_SAS; i++) { + for (i = 0; i < ipsec_sa_tbl->max_num_sa; i++) { ipsec_sa_t *ipsec_sa = ipsec_sa_entry(i); if (ipsec_sa_lock(ipsec_sa) < 0) @@ -922,7 +946,7 @@ int _odp_ipsec_sa_replay_update(ipsec_sa_t *ipsec_sa, uint32_t seq, uint16_t _odp_ipsec_sa_alloc_ipv4_id(ipsec_sa_t *ipsec_sa) { - (void) ipsec_sa; + (void)ipsec_sa; ipsec_thread_local_t *tl = &ipsec_sa_tbl->per_thread[odp_thread_id()]; uint32_t data; @@ -951,6 +975,7 @@ uint16_t _odp_ipsec_sa_alloc_ipv4_id(ipsec_sa_t *ipsec_sa) uint64_t _odp_ipsec_sa_stats_pkts(ipsec_sa_t *sa) { + int thread_count_max = odp_thread_count_max(); uint64_t tl_pkt_quota = 0; sa_thread_local_t *sa_tl; int n; @@ -966,7 +991,7 @@ uint64_t _odp_ipsec_sa_stats_pkts(ipsec_sa_t *sa) * need to be accounted for. */ - for (n = 0; n < ODP_THREAD_COUNT_MAX; n++) { + for (n = 0; n < thread_count_max; n++) { sa_tl = &ipsec_sa_tbl->per_thread[n].sa[sa->ipsec_sa_idx]; tl_pkt_quota += odp_atomic_load_u32(&sa_tl->packet_quota); } @@ -1012,7 +1037,6 @@ static void ipsec_in_sa_info(ipsec_sa_t *ipsec_sa, odp_ipsec_sa_info_t *sa_info) uint8_t *dst = sa_info->inbound.lookup_param.dst_addr; if (ipsec_sa->lookup_mode == ODP_IPSEC_LOOKUP_DSTADDR_SPI) { - if (ipsec_sa->param.inbound.lookup_param.ip_version == ODP_IPSEC_IPV4) memcpy(dst, &ipsec_sa->in.lookup_dst_ipv4, @@ -1020,8 +1044,8 @@ static void ipsec_in_sa_info(ipsec_sa_t *ipsec_sa, odp_ipsec_sa_info_t *sa_info) else memcpy(dst, &ipsec_sa->in.lookup_dst_ipv6, ODP_IPV6_ADDR_SIZE); - } + sa_info->param.inbound.lookup_param.dst_addr = dst; if (ipsec_sa->antireplay) { diff --git a/platform/linux-generic/odp_ishm.c b/platform/linux-generic/odp_ishm.c index 6b2d03d40..0014f3c34 100644 --- a/platform/linux-generic/odp_ishm.c +++ b/platform/linux-generic/odp_ishm.c @@ -393,7 +393,10 @@ static int hp_get_cached(uint64_t len) { int fd; - if (NULL == hpc || hpc->idx < 0 || len != hpc->len) + if (hpc == NULL) + return -1; + + if (hpc->idx < 0 || len != hpc->len) return -1; fd = hpc->fd[hpc->idx]; @@ -404,12 +407,17 @@ static int hp_get_cached(uint64_t len) static int hp_put_cached(int fd) { - if (NULL == hpc || odp_unlikely(++hpc->idx >= hpc->total)) { - hpc->idx--; + if (hpc == NULL) { + ODP_ERR("Bad hpc state\n"); + return -1; + } + + if (odp_unlikely((hpc->idx + 1) >= hpc->total)) { ODP_ERR("Trying to put more FD than allowed: %d\n", fd); return -1; } + hpc->idx++; hpc->fd[hpc->idx] = fd; return 0; @@ -428,7 +436,7 @@ static void *alloc_fragment(uintptr_t size, int block_index, intptr_t align, ishm_fragment_t *fragmnt; *best_fragmnt = NULL; ishm_fragment_t *rem_fragmnt; - uintptr_t border;/* possible start of new fragment (next alignement) */ + uintptr_t border;/* possible start of new fragment (next alignment) */ intptr_t left; /* room remaining after, if the segment is allocated */ uintptr_t remainder = odp_global_ro.shm_max_memory; @@ -1120,7 +1128,7 @@ int _odp_ishm_reserve(const char *name, uint64_t size, int fd, * can request more: If the user requirement exceeds the page * size then we have to make sure the block will be mapped at * the same address every where, otherwise alignment may be - * be wrong for some process */ + * wrong for some process */ hp_align = align; if (hp_align <= page_hp_size) hp_align = page_hp_size; @@ -1177,7 +1185,7 @@ int _odp_ishm_reserve(const char *name, uint64_t size, int fd, * can request more: If the user requirement exceeds the page * size then we have to make sure the block will be mapped at * the same address every where, otherwise alignment may be - * be wrong for some process */ + * wrong for some process */ if (align <= odp_sys_page_size()) align = odp_sys_page_size(); else @@ -2058,7 +2066,6 @@ int _odp_ishm_status(const char *title) "", len_total / 1024 / 1024, "", lost_total / 1024 / 1024); - /* display the virtual space allocations... : */ ODP_PRINT("\nishm virtual space:\n"); for (fragmnt = ishm_ftbl->used_fragmnts; @@ -2080,7 +2087,7 @@ int _odp_ishm_status(const char *title) /* some other sanity checks: */ if (fragmnt->prev != previous) - ODP_ERR("chaining error\n"); + ODP_ERR("chaining error\n"); if (fragmnt != ishm_ftbl->used_fragmnts) { if ((uintptr_t)fragmnt->start != last_address + 1) diff --git a/platform/linux-generic/odp_packet.c b/platform/linux-generic/odp_packet.c index 752ff8416..99b0b3ae2 100644 --- a/platform/linux-generic/odp_packet.c +++ b/platform/linux-generic/odp_packet.c @@ -405,11 +405,11 @@ static inline odp_packet_hdr_t *alloc_segments(pool_t *pool, int num) odp_packet_hdr_t *pkt_hdr[num]; int ret; - ret = buffer_alloc_multi(pool, (odp_buffer_hdr_t **)pkt_hdr, num); + ret = _odp_buffer_alloc_multi(pool, (odp_buffer_hdr_t **)pkt_hdr, num); if (odp_unlikely(ret != num)) { if (ret > 0) - buffer_free_multi((odp_buffer_hdr_t **)pkt_hdr, ret); + _odp_buffer_free_multi((odp_buffer_hdr_t **)pkt_hdr, ret); return NULL; } @@ -523,13 +523,12 @@ static inline void packet_free_multi(odp_buffer_hdr_t *hdr[], int num) /* Skip references and pack to be freed headers to array head */ if (odp_unlikely(num_ref)) hdr[i - num_ref] = hdr[i]; - } num -= num_ref; if (odp_likely(num)) - buffer_free_multi(hdr, num); + _odp_buffer_free_multi(hdr, num); } static inline void free_all_segments(odp_packet_hdr_t *pkt_hdr, int num) @@ -626,8 +625,8 @@ static inline int packet_alloc(pool_t *pool, uint32_t len, int max_pkt, odp_packet_hdr_t *hdr_next; odp_packet_hdr_t *hdr; - num_buf = buffer_alloc_multi(pool, (odp_buffer_hdr_t **)pkt_hdr, - max_buf); + num_buf = _odp_buffer_alloc_multi(pool, (odp_buffer_hdr_t **)pkt_hdr, + max_buf); /* Failed to allocate all segments */ if (odp_unlikely(num_buf != max_buf)) { @@ -640,7 +639,7 @@ static inline int packet_alloc(pool_t *pool, uint32_t len, int max_pkt, odp_buffer_hdr_t **p; p = (odp_buffer_hdr_t **)&pkt_hdr[num_buf - num_free]; - buffer_free_multi(p, num_free); + _odp_buffer_free_multi(p, num_free); } if (num == 0) @@ -673,8 +672,8 @@ static inline int packet_alloc(pool_t *pool, uint32_t len, int max_pkt, return num; } -int packet_alloc_multi(odp_pool_t pool_hdl, uint32_t len, - odp_packet_t pkt[], int max_num) +int _odp_packet_alloc_multi(odp_pool_t pool_hdl, uint32_t len, + odp_packet_t pkt[], int max_num) { pool_t *pool = pool_entry_from_hdl(pool_hdl); int num, num_seg; @@ -692,7 +691,7 @@ odp_packet_t odp_packet_alloc(odp_pool_t pool_hdl, uint32_t len) int num, num_seg; if (odp_unlikely(pool->params.type != ODP_POOL_PACKET)) { - __odp_errno = EINVAL; + _odp_errno = EINVAL; return ODP_PACKET_INVALID; } @@ -715,7 +714,7 @@ int odp_packet_alloc_multi(odp_pool_t pool_hdl, uint32_t len, int num, num_seg; if (odp_unlikely(pool->params.type != ODP_POOL_PACKET)) { - __odp_errno = EINVAL; + _odp_errno = EINVAL; return -1; } @@ -2880,3 +2879,56 @@ int odp_packet_payload_offset_set(odp_packet_t pkt, uint32_t offset) return 0; } + +void odp_packet_aging_tmo_set(odp_packet_t pkt, uint64_t tmo_ns) +{ + (void)pkt; + (void)tmo_ns; +} + +uint64_t odp_packet_aging_tmo(odp_packet_t pkt) +{ + (void)pkt; + return 0; +} + +int odp_packet_tx_compl_request(odp_packet_t pkt, const odp_packet_tx_compl_opt_t *opt) +{ + (void)pkt; + (void)opt; + + return -1; +} + +int odp_packet_has_tx_compl_request(odp_packet_t pkt) +{ + (void)pkt; + + return 0; +} + +odp_packet_tx_compl_t odp_packet_tx_compl_from_event(odp_event_t ev) +{ + (void)ev; + + return ODP_PACKET_TX_COMPL_INVALID; +} + +odp_event_t odp_packet_tx_compl_to_event(odp_packet_tx_compl_t tx_compl) +{ + (void)tx_compl; + + return ODP_EVENT_INVALID; +} + +void odp_packet_tx_compl_free(odp_packet_tx_compl_t tx_compl) +{ + (void)tx_compl; +} + +void *odp_packet_tx_compl_user_ptr(odp_packet_tx_compl_t tx_compl) +{ + (void)tx_compl; + + return NULL; +} diff --git a/platform/linux-generic/odp_packet_io.c b/platform/linux-generic/odp_packet_io.c index 442aa0d94..369c319c2 100644 --- a/platform/linux-generic/odp_packet_io.c +++ b/platform/linux-generic/odp_packet_io.c @@ -126,9 +126,9 @@ int _odp_pktio_init_global(void) _odp_pktio_entry_ptr[i] = pktio_entry; } - for (pktio_if = 0; pktio_if_ops[pktio_if]; ++pktio_if) { - if (pktio_if_ops[pktio_if]->init_global) - if (pktio_if_ops[pktio_if]->init_global()) { + for (pktio_if = 0; _odp_pktio_if_ops[pktio_if]; ++pktio_if) { + if (_odp_pktio_if_ops[pktio_if]->init_global) + if (_odp_pktio_if_ops[pktio_if]->init_global()) { ODP_ERR("failed to initialized pktio type %d", pktio_if); return -1; @@ -149,9 +149,9 @@ int _odp_pktio_init_local(void) { int pktio_if; - for (pktio_if = 0; pktio_if_ops[pktio_if]; ++pktio_if) { - if (pktio_if_ops[pktio_if]->init_local) - if (pktio_if_ops[pktio_if]->init_local()) { + for (pktio_if = 0; _odp_pktio_if_ops[pktio_if]; ++pktio_if) { + if (_odp_pktio_if_ops[pktio_if]->init_local) + if (_odp_pktio_if_ops[pktio_if]->init_local()) { ODP_ERR("failed to initialized pktio type %d", pktio_if); return -1; @@ -263,8 +263,8 @@ static const char *strip_pktio_type(const char *name, char *type_out) if_name++; /* Match if_type to enabled pktio devices */ - for (pktio_if = 0; pktio_if_ops[pktio_if]; pktio_if++) { - if (!strcmp(pktio_type, pktio_if_ops[pktio_if]->name)) { + for (pktio_if = 0; _odp_pktio_if_ops[pktio_if]; pktio_if++) { + if (!strcmp(pktio_type, _odp_pktio_if_ops[pktio_if]->name)) { if (type_out) strcpy(type_out, pktio_type); /* Some pktio devices expect device names to @@ -328,14 +328,14 @@ static odp_pktio_t setup_pktio_entry(const char *name, odp_pool_t pool, odp_pktio_config_init(&pktio_entry->s.config); - for (pktio_if = 0; pktio_if_ops[pktio_if]; ++pktio_if) { + for (pktio_if = 0; _odp_pktio_if_ops[pktio_if]; ++pktio_if) { /* Only use explicitly defined pktio type */ if (strlen(pktio_type) && - strcmp(pktio_if_ops[pktio_if]->name, pktio_type)) + strcmp(_odp_pktio_if_ops[pktio_if]->name, pktio_type)) continue; - ret = pktio_if_ops[pktio_if]->open(hdl, pktio_entry, if_name, - pool); + ret = _odp_pktio_if_ops[pktio_if]->open(hdl, pktio_entry, if_name, + pool); if (!ret) break; } @@ -352,7 +352,7 @@ static odp_pktio_t setup_pktio_entry(const char *name, odp_pool_t pool, snprintf(pktio_entry->s.full_name, sizeof(pktio_entry->s.full_name), "%s", name); pktio_entry->s.state = PKTIO_STATE_OPENED; - pktio_entry->s.ops = pktio_if_ops[pktio_if]; + pktio_entry->s.ops = _odp_pktio_if_ops[pktio_if]; unlock_entry(pktio_entry); return hdl; @@ -400,7 +400,7 @@ odp_pktio_t odp_pktio_open(const char *name, odp_pool_t pool, hdl = odp_pktio_lookup(name); if (hdl != ODP_PKTIO_INVALID) { /* interface is already open */ - __odp_errno = EEXIST; + _odp_errno = EEXIST; return ODP_PKTIO_INVALID; } @@ -1062,7 +1062,7 @@ static odp_buffer_hdr_t *pktin_dequeue(odp_queue_t queue) ODP_DBG("Interface %s dropped %i packets\n", entry->s.name, num - num_enq); - buffer_free_multi(&hdr_tbl[num_enq + 1], num - num_enq); + _odp_buffer_free_multi(&hdr_tbl[num_enq + 1], num - num_enq); } } @@ -1087,7 +1087,7 @@ static int pktin_deq_multi(odp_queue_t queue, odp_buffer_hdr_t *buf_hdr[], if (odp_unlikely(nbr > num)) ODP_ABORT("queue_deq_multi req: %d, returned %d\n", num, nbr); - /** queue already has number of requsted buffers, + /** queue already has number of requested buffers, * do not do receive in that case. */ if (nbr == num) @@ -1116,7 +1116,7 @@ static int pktin_deq_multi(odp_queue_t queue, odp_buffer_hdr_t *buf_hdr[], ODP_DBG("Interface %s dropped %i packets\n", entry->s.name, j - num_enq); - buffer_free_multi(&buf_hdr[num_enq], j - num_enq); + _odp_buffer_free_multi(&buf_hdr[num_enq], j - num_enq); } } @@ -1742,9 +1742,9 @@ int _odp_pktio_term_global(void) unlock_entry(pktio_entry); } - for (pktio_if = 0; pktio_if_ops[pktio_if]; ++pktio_if) { - if (pktio_if_ops[pktio_if]->term) - if (pktio_if_ops[pktio_if]->term()) + for (pktio_if = 0; _odp_pktio_if_ops[pktio_if]; ++pktio_if) { + if (_odp_pktio_if_ops[pktio_if]->term) + if (_odp_pktio_if_ops[pktio_if]->term()) ODP_ABORT("failed to terminate pktio type %d", pktio_if); } diff --git a/platform/linux-generic/odp_pool.c b/platform/linux-generic/odp_pool.c index 0b1d0a7f4..07da3d9cc 100644 --- a/platform/linux-generic/odp_pool.c +++ b/platform/linux-generic/odp_pool.c @@ -1056,7 +1056,7 @@ int odp_pool_info(odp_pool_t pool_hdl, odp_pool_info_t *info) return 0; } -int buffer_alloc_multi(pool_t *pool, odp_buffer_hdr_t *buf_hdr[], int max_num) +int _odp_buffer_alloc_multi(pool_t *pool, odp_buffer_hdr_t *buf_hdr[], int max_num) { uint32_t pool_idx = pool->pool_idx; pool_cache_t *cache = local.cache[pool_idx]; @@ -1171,7 +1171,7 @@ static inline void buffer_free_to_pool(pool_t *pool, odp_atomic_inc_u64(&pool->stats.cache_free_ops); } -void buffer_free_multi(odp_buffer_hdr_t *buf_hdr[], int num_total) +void _odp_buffer_free_multi(odp_buffer_hdr_t *buf_hdr[], int num_total) { pool_t *pool; int num; @@ -1210,7 +1210,7 @@ odp_buffer_t odp_buffer_alloc(odp_pool_t pool_hdl) ODP_ASSERT(ODP_POOL_INVALID != pool_hdl); pool = pool_entry_from_hdl(pool_hdl); - ret = buffer_alloc_multi(pool, (odp_buffer_hdr_t **)&buf, 1); + ret = _odp_buffer_alloc_multi(pool, (odp_buffer_hdr_t **)&buf, 1); if (odp_likely(ret == 1)) return buf; @@ -1226,17 +1226,17 @@ int odp_buffer_alloc_multi(odp_pool_t pool_hdl, odp_buffer_t buf[], int num) pool = pool_entry_from_hdl(pool_hdl); - return buffer_alloc_multi(pool, (odp_buffer_hdr_t **)buf, num); + return _odp_buffer_alloc_multi(pool, (odp_buffer_hdr_t **)buf, num); } void odp_buffer_free(odp_buffer_t buf) { - buffer_free_multi((odp_buffer_hdr_t **)&buf, 1); + _odp_buffer_free_multi((odp_buffer_hdr_t **)&buf, 1); } void odp_buffer_free_multi(const odp_buffer_t buf[], int num) { - buffer_free_multi((odp_buffer_hdr_t **)(uintptr_t)buf, num); + _odp_buffer_free_multi((odp_buffer_hdr_t **)(uintptr_t)buf, num); } int odp_pool_capability(odp_pool_capability_t *capa) diff --git a/platform/linux-generic/odp_queue_basic.c b/platform/linux-generic/odp_queue_basic.c index 2d2ed3c7f..8dc10467a 100644 --- a/platform/linux-generic/odp_queue_basic.c +++ b/platform/linux-generic/odp_queue_basic.c @@ -146,6 +146,7 @@ static int queue_init_global(void) for (i = 0; i < CONFIG_MAX_QUEUES; i++) { /* init locks */ queue_entry_t *queue = qentry_from_index(i); + LOCK_INIT(queue); queue->s.index = i; queue->s.handle = (odp_queue_t)queue; @@ -389,6 +390,7 @@ static int queue_destroy(odp_queue_t handle) { int empty; queue_entry_t *queue; + queue = qentry_from_handle(handle); if (handle == ODP_QUEUE_INVALID) diff --git a/platform/linux-generic/odp_queue_lf.c b/platform/linux-generic/odp_queue_lf.c index 82b95c34d..70d555ab5 100644 --- a/platform/linux-generic/odp_queue_lf.c +++ b/platform/linux-generic/odp_queue_lf.c @@ -23,62 +23,13 @@ typedef unsigned __int128 u128_t; -static inline void atomic_zero_u128(u128_t *atomic) +static inline void lockfree_zero_u128(u128_t *atomic) { __atomic_store_n(atomic, 0, __ATOMIC_RELAXED); } -#if defined(__aarch64__) -/* ARMv8 has atomic load-acq/store-rel instructions for a pair of - * 64bit of data. GCC atomic built-in for 128bits does not utilize these - * instructions but uses locks instead. Override GCC built-in for ARMv8. - */ #include <odp_cpu.h> -static inline int atomic_cas_acq_rel_u128(u128_t *atomic, u128_t old_val, - u128_t new_val) -{ - return __lockfree_compare_exchange_16((__int128 *)atomic, - (__int128 *)&old_val, - new_val, - 0, - __ATOMIC_ACQ_REL, - __ATOMIC_RELAXED); -} - -static inline u128_t atomic_load_u128(u128_t *atomic) -{ - return __lockfree_load_16((__int128 *)atomic, __ATOMIC_RELAXED); -} - -static inline int atomic_is_lockfree_u128(void) -{ - return 1; -} - -#else - -static inline u128_t atomic_load_u128(u128_t *atomic) -{ - return __atomic_load_n(atomic, __ATOMIC_RELAXED); -} - -static inline int atomic_cas_acq_rel_u128(u128_t *atomic, u128_t old_val, - u128_t new_val) -{ - return __atomic_compare_exchange_n(atomic, &old_val, new_val, - 0 /* strong */, - __ATOMIC_ACQ_REL, - __ATOMIC_RELAXED); -} - -static inline int atomic_is_lockfree_u128(void) -{ - return __atomic_is_lock_free(16, NULL); -} - -#endif - #else /* These definitions enable build in non 128 bit compatible systems. @@ -88,19 +39,19 @@ typedef struct ODP_ALIGNED(16) { uint64_t u64[2]; } u128_t; -static inline u128_t atomic_load_u128(u128_t *atomic) +static inline u128_t lockfree_load_u128(u128_t *atomic) { return *atomic; } -static inline void atomic_zero_u128(u128_t *atomic) +static inline void lockfree_zero_u128(u128_t *atomic) { atomic->u64[0] = 0; atomic->u64[1] = 0; } -static inline int atomic_cas_acq_rel_u128(u128_t *atomic, u128_t old_val, - u128_t new_val) +static inline int lockfree_cas_acq_rel_u128(u128_t *atomic, u128_t old_val, + u128_t new_val) { if (atomic->u64[0] == old_val.u64[0] && atomic->u64[1] == old_val.u64[1]) { @@ -112,7 +63,7 @@ static inline int atomic_cas_acq_rel_u128(u128_t *atomic, u128_t old_val, return 0; } -static inline int atomic_is_lockfree_u128(void) +static inline int lockfree_check_u128(void) { return 0; } @@ -187,7 +138,7 @@ static int queue_lf_enq(odp_queue_t handle, odp_buffer_hdr_t *buf_hdr) node = &queue_lf->node[idx]; idx = next_idx(idx); - node_val.u128 = atomic_load_u128(&node->u128); + node_val.u128 = lockfree_load_u128(&node->u128); if (node_val.s.counter == 0) { found = 1; @@ -200,7 +151,7 @@ static int queue_lf_enq(odp_queue_t handle, odp_buffer_hdr_t *buf_hdr) return -1; /* Try to insert data */ - if (atomic_cas_acq_rel_u128(&node->u128, node_val.u128, + if (lockfree_cas_acq_rel_u128(&node->u128, node_val.u128, new_val.u128)) return 0; } @@ -244,7 +195,7 @@ static odp_buffer_hdr_t *queue_lf_deq(odp_queue_t handle) * the lowest counter. */ for (i = 0; i < RING_LF_SIZE; i++) { node = &queue_lf->node[i]; - node_val.u128 = atomic_load_u128(&node->u128); + node_val.u128 = lockfree_load_u128(&node->u128); counter = node_val.s.counter; if (counter && counter < lowest) { @@ -265,7 +216,7 @@ static odp_buffer_hdr_t *queue_lf_deq(odp_queue_t handle) * values. */ for (i = 0; i < i_lowest; i++) { node = &queue_lf->node[i]; - node_val.u128 = atomic_load_u128(&node->u128); + node_val.u128 = lockfree_load_u128(&node->u128); counter = node_val.s.counter; if (counter && counter < lowest) { @@ -278,7 +229,7 @@ static odp_buffer_hdr_t *queue_lf_deq(odp_queue_t handle) buf_hdr = (void *)(uintptr_t)old_val.s.ptr; /* Try to remove data */ - if (atomic_cas_acq_rel_u128(&old->u128, old_val.u128, + if (lockfree_cas_acq_rel_u128(&old->u128, old_val.u128, new_val.u128)) return buf_hdr; } @@ -309,7 +260,7 @@ uint32_t _odp_queue_lf_init_global(uint32_t *queue_lf_size, int lockfree; /* 16 byte lockfree CAS operation is needed. */ - lockfree = atomic_is_lockfree_u128(); + lockfree = lockfree_check_u128(); ODP_DBG("\nLock-free queue init\n"); ODP_DBG(" u128 lock-free: %i\n\n", lockfree); @@ -359,7 +310,7 @@ static void init_queue(queue_lf_t *queue_lf) odp_atomic_init_u64(&queue_lf->enq_counter, 1); for (i = 0; i < RING_LF_SIZE; i++) - atomic_zero_u128(&queue_lf->node[i].u128); + lockfree_zero_u128(&queue_lf->node[i].u128); } void *_odp_queue_lf_create(queue_entry_t *queue) @@ -403,7 +354,7 @@ uint32_t _odp_queue_lf_length(void *queue_lf_ptr) uint32_t num = 0; for (i = 0; i < RING_LF_SIZE; i++) { - node_val.u128 = atomic_load_u128(&queue_lf->node[i].u128); + node_val.u128 = lockfree_load_u128(&queue_lf->node[i].u128); if (node_val.s.counter) num++; } diff --git a/platform/linux-generic/odp_queue_scalable.c b/platform/linux-generic/odp_queue_scalable.c index d70e174e0..248855be4 100644 --- a/platform/linux-generic/odp_queue_scalable.c +++ b/platform/linux-generic/odp_queue_scalable.c @@ -181,7 +181,6 @@ static int queue_init(queue_entry_t *queue, const char *name, sched_elem->schedq = _odp_sched_queue_add(param->sched.group, prio); ODP_ASSERT(sched_elem->schedq != NULL); - } return 0; @@ -218,9 +217,7 @@ static int queue_init_global(void) /* Add the reorder window size */ pool_size += sizeof(reorder_window_t) * CONFIG_MAX_QUEUES; - /* Choose min_alloc and max_alloc such that buddy allocator is - * is selected. - */ + /* Choose min_alloc and max_alloc such that buddy allocator is selected. */ min_alloc = 0; max_alloc = CONFIG_SCAL_QUEUE_SIZE * sizeof(odp_buffer_hdr_t *); queue_shm_pool = _odp_ishm_pool_create("queue_shm_pool", @@ -674,8 +671,7 @@ static int _queue_enq_multi(odp_queue_t handle, odp_buffer_hdr_t *buf_hdr[], static int _queue_enq(odp_queue_t handle, odp_buffer_hdr_t *buf_hdr) { - return odp_likely( - _queue_enq_multi(handle, &buf_hdr, 1) == 1) ? 0 : -1; + return odp_likely(_queue_enq_multi(handle, &buf_hdr, 1) == 1) ? 0 : -1; } static int queue_enq_multi(odp_queue_t handle, const odp_event_t ev[], int num) @@ -734,8 +730,7 @@ int _odp_queue_deq_sc(sched_elem_t *q, odp_event_t *evp, int num) mask = q->cons_mask; ring = q->cons_ring; do { - *evp++ = odp_buffer_to_event( - buf_from_buf_hdr(ring[old_read & mask])); + *evp++ = odp_buffer_to_event(buf_from_buf_hdr(ring[old_read & mask])); } while (++old_read != new_read); /* Signal producers that empty slots are available @@ -828,8 +823,7 @@ inline int _odp_queue_deq_mc(sched_elem_t *q, odp_event_t *evp, int num) ret = _odp_queue_deq(q, hdr_tbl, num); if (odp_likely(ret != 0)) { for (evt_idx = 0; evt_idx < num; evt_idx++) - evp[evt_idx] = odp_buffer_to_event( - buf_from_buf_hdr(hdr_tbl[evt_idx])); + evp[evt_idx] = odp_buffer_to_event(buf_from_buf_hdr(hdr_tbl[evt_idx])); } return ret; @@ -1024,11 +1018,11 @@ static void queue_print_all(void) odp_queue_op_mode_t enq_mode; odp_queue_op_mode_t deq_mode; odp_queue_order_t order; + odp_schedule_sync_t sync; + int prio; const char *bl_str; char type_c, enq_c, deq_c, order_c, sync_c; const int col_width = 24; - int prio = 0; - odp_schedule_sync_t sync = ODP_SCHED_SYNC_PARALLEL; ODP_PRINT("\nList of all queues\n"); ODP_PRINT("------------------\n"); @@ -1050,6 +1044,8 @@ static void queue_print_all(void) enq_mode = queue->s.param.enq_mode; deq_mode = queue->s.param.deq_mode; order = queue->s.param.order; + prio = queue->s.param.sched.prio; + sync = queue->s.param.sched.sync; UNLOCK(&queue->s.lock); diff --git a/platform/linux-generic/odp_random.c b/platform/linux-generic/odp_random.c index aae12a5d3..acae9663d 100644 --- a/platform/linux-generic/odp_random.c +++ b/platform/linux-generic/odp_random.c @@ -29,8 +29,6 @@ int32_t odp_random_data(uint8_t *buf, uint32_t len, odp_random_kind_t kind) int32_t odp_random_test_data(uint8_t *buf, uint32_t len, uint64_t *seed) { - if (_ODP_OPENSSL) - return _odp_random_openssl_test_data(buf, len, seed); return _odp_random_std_test_data(buf, len, seed); } diff --git a/platform/linux-generic/odp_random_openssl.c b/platform/linux-generic/odp_random_openssl.c index e1d45c4fe..a74f99bd4 100644 --- a/platform/linux-generic/odp_random_openssl.c +++ b/platform/linux-generic/odp_random_openssl.c @@ -36,27 +36,6 @@ int32_t _odp_random_openssl_data(uint8_t *buf, uint32_t len, return -1; } } - -int32_t _odp_random_openssl_test_data(uint8_t *buf, uint32_t len, - uint64_t *seed) -{ - union { - uint32_t rand_word; - uint8_t rand_byte[4]; - } u; - uint32_t i = 0, j; - uint32_t seed32 = (*seed) & 0xffffffff; - - while (i < len) { - u.rand_word = rand_r(&seed32); - - for (j = 0; j < 4 && i < len; j++, i++) - *buf++ = u.rand_byte[j]; - } - - *seed = seed32; - return len; -} #else /* Dummy functions for building without OpenSSL support */ odp_random_kind_t _odp_random_openssl_max_kind(void) @@ -70,13 +49,6 @@ int32_t _odp_random_openssl_data(uint8_t *buf ODP_UNUSED, { return -1; } - -int32_t _odp_random_openssl_test_data(uint8_t *buf ODP_UNUSED, - uint32_t len ODP_UNUSED, - uint64_t *seed ODP_UNUSED) -{ - return -1; -} #endif /* _ODP_OPENSSL */ int _odp_random_openssl_init_local(void) diff --git a/platform/linux-generic/odp_rwlock.c b/platform/linux-generic/odp_rwlock.c index 74f5307aa..03af7d26c 100644 --- a/platform/linux-generic/odp_rwlock.c +++ b/platform/linux-generic/odp_rwlock.c @@ -58,6 +58,7 @@ void odp_rwlock_write_lock(odp_rwlock_t *rwlock) while (is_locked == 0) { uint32_t zero = 0; + cnt = odp_atomic_load_u32(&rwlock->cnt); /* lock acquired, wait */ if (cnt != 0) { diff --git a/platform/linux-generic/odp_schedule_basic.c b/platform/linux-generic/odp_schedule_basic.c index 32a21442d..21f707be7 100644 --- a/platform/linux-generic/odp_schedule_basic.c +++ b/platform/linux-generic/odp_schedule_basic.c @@ -229,6 +229,7 @@ typedef struct { /* Scheduler interface config options (not used in fast path) */ schedule_config_t config_if; + uint32_t max_queues; } sched_global_t; @@ -399,6 +400,7 @@ static int schedule_init_global(void) odp_shm_t shm; int i, j, grp; int prefer_ratio; + uint32_t ring_size; ODP_DBG("Schedule init ... "); @@ -419,11 +421,23 @@ static int schedule_init_global(void) return -1; } + sched->shm = shm; prefer_ratio = sched->config.prefer_ratio; /* When num_spread == 1, only spread_tbl[0] is used. */ sched->max_spread = (sched->config.num_spread - 1) * prefer_ratio; - sched->shm = shm; + + ring_size = MAX_RING_SIZE / sched->config.num_spread; + ring_size = ROUNDUP_POWER2_U32(ring_size); + ODP_ASSERT(ring_size <= MAX_RING_SIZE); + sched->ring_mask = ring_size - 1; + + /* Each ring can hold in maximum ring_size-1 queues. Due to ring size round up, + * total capacity of rings may be larger than CONFIG_MAX_SCHED_QUEUES. */ + sched->max_queues = sched->ring_mask * sched->config.num_spread; + if (sched->max_queues > CONFIG_MAX_SCHED_QUEUES) + sched->max_queues = CONFIG_MAX_SCHED_QUEUES; + odp_spinlock_init(&sched->mask_lock); for (grp = 0; grp < NUM_SCHED_GRPS; grp++) { @@ -573,7 +587,6 @@ static inline int prio_level_from_api(int api_prio) static int schedule_create_queue(uint32_t queue_index, const odp_schedule_param_t *sched_param) { - uint32_t ring_size; int i; int prio = prio_level_from_api(sched_param->prio); uint8_t spread = spread_index(queue_index); @@ -620,11 +633,6 @@ static int schedule_create_queue(uint32_t queue_index, sched->queue[queue_index].pktio_index = 0; sched->queue[queue_index].pktin_index = 0; - ring_size = MAX_RING_SIZE / sched->config.num_spread; - ring_size = ROUNDUP_POWER2_U32(ring_size); - ODP_ASSERT(ring_size <= MAX_RING_SIZE); - sched->ring_mask = ring_size - 1; - odp_atomic_init_u64(&sched->order[queue_index].ctx, 0); odp_atomic_init_u64(&sched->order[queue_index].next_ctx, 0); @@ -762,7 +770,7 @@ static inline void ordered_stash_release(void) num_enq = 0; ODP_DBG("Dropped %i packets\n", num - num_enq); - buffer_free_multi(&buf_hdr[num_enq], num - num_enq); + _odp_buffer_free_multi(&buf_hdr[num_enq], num - num_enq); } } sched_local.ordered.stash_num = 0; @@ -822,7 +830,7 @@ static int schedule_term_local(void) static void schedule_config_init(odp_schedule_config_t *config) { - config->num_queues = CONFIG_MAX_SCHED_QUEUES; + config->num_queues = sched->max_queues; config->queue_size = _odp_queue_glb->config.max_queue_size; config->sched_group.all = sched->config_if.group_enable.all; config->sched_group.control = sched->config_if.group_enable.control; @@ -840,7 +848,6 @@ static void schedule_group_clear(odp_schedule_group_t group) grp_update_mask(group, &zero); sched->sched_grp[group].allocated = 0; - } static int schedule_config(const odp_schedule_config_t *config) @@ -999,7 +1006,7 @@ static inline int poll_pktin(uint32_t qi, int direct_recv, num_enq = 0; ODP_DBG("Dropped %i packets\n", num - num_enq); - buffer_free_multi(&b_hdr[num_enq], num - num_enq); + _odp_buffer_free_multi(&b_hdr[num_enq], num - num_enq); } return ret; @@ -1018,7 +1025,6 @@ static inline int do_schedule_grp(odp_queue_t *out_queue, odp_event_t out_ev[], /* Schedule events */ for (prio = 0; prio < NUM_PRIO; prio++) { - if (sched->prio_q_mask[prio] == 0) continue; @@ -1253,7 +1259,6 @@ static inline int schedule_loop(odp_queue_t *out_queue, uint64_t wait, int ret; while (1) { - ret = do_schedule(out_queue, out_ev, max_num); if (ret) { timer_run(2); @@ -1326,6 +1331,7 @@ static inline void order_lock(void) static void order_unlock(void) { + /* Nothing to do */ } static void schedule_order_lock(uint32_t lock_index) @@ -1621,9 +1627,9 @@ static int schedule_thr_rem(odp_schedule_group_t group, int thr) return 0; } -/* This function is a no-op */ -static void schedule_prefetch(int num ODP_UNUSED) +static void schedule_prefetch(int num) { + (void)num; } static int schedule_num_grps(void) @@ -1643,7 +1649,7 @@ static int schedule_capability(odp_schedule_capability_t *capa) capa->max_ordered_locks = schedule_max_ordered_locks(); capa->max_groups = schedule_num_grps(); capa->max_prios = schedule_num_prio(); - capa->max_queues = CONFIG_MAX_SCHED_QUEUES; + capa->max_queues = sched->max_queues; capa->max_queue_size = _odp_queue_glb->config.max_queue_size; capa->max_flow_id = BUF_HDR_MAX_FLOW_ID; diff --git a/platform/linux-generic/odp_schedule_scalable.c b/platform/linux-generic/odp_schedule_scalable.c index 2ba40256b..c9991c3f3 100644 --- a/platform/linux-generic/odp_schedule_scalable.c +++ b/platform/linux-generic/odp_schedule_scalable.c @@ -214,15 +214,13 @@ void _odp_sched_update_enq(sched_elem_t *q, uint32_t actual) ticket = nss.nxt_ticket++; /* Else queue already was non-empty. */ /* Attempt to update numevts counter and optionally take ticket. */ - } while (!__atomic_compare_exchange( - &q->qschst, &oss, &nss, - true, __ATOMIC_RELAXED, __ATOMIC_RELAXED)); + } while (!__atomic_compare_exchange(&q->qschst, &oss, &nss, + true, __ATOMIC_RELAXED, __ATOMIC_RELAXED)); if (odp_unlikely(ticket != TICKET_INVALID)) { /* Wait for our turn to update schedq. */ - if (odp_unlikely( - __atomic_load_n(&q->qschst.cur_ticket, - __ATOMIC_ACQUIRE) != ticket)) { + if (odp_unlikely(__atomic_load_n(&q->qschst.cur_ticket, + __ATOMIC_ACQUIRE) != ticket)) { sevl(); while (wfe() && monitor8(&q->qschst.cur_ticket, @@ -333,12 +331,9 @@ sched_update_deq(sched_elem_t *q, * the CAS operation */ nss.cur_ticket = _odp_sched_ts->ticket + 1; - } while (odp_unlikely(!__atomic_compare_exchange( - &q->qschst, - &oss, &nss, - true, - __ATOMIC_RELEASE, - __ATOMIC_RELAXED))); + } while (odp_unlikely(!__atomic_compare_exchange(&q->qschst, &oss, &nss, true, + __ATOMIC_RELEASE, + __ATOMIC_RELAXED))); return; } @@ -361,16 +356,14 @@ sched_update_deq(sched_elem_t *q, nss.wrr_budget = CONFIG_WRR_WEIGHT; } /* Attempt to update numevts and optionally take ticket. */ - } while (!__atomic_compare_exchange( - &q->qschst, &oss, &nss, - true, __ATOMIC_RELAXED, __ATOMIC_RELAXED)); + } while (!__atomic_compare_exchange(&q->qschst, &oss, &nss, + true, __ATOMIC_RELAXED, __ATOMIC_RELAXED)); if (odp_unlikely(ticket != TICKET_INVALID)) { ODP_ASSERT(q->qschst_type != ODP_SCHED_SYNC_ATOMIC); /* Wait for our turn to update schedq. */ - if (odp_unlikely( - __atomic_load_n(&q->qschst.cur_ticket, - __ATOMIC_ACQUIRE) != ticket)) { + if (odp_unlikely(__atomic_load_n(&q->qschst.cur_ticket, + __ATOMIC_ACQUIRE) != ticket)) { sevl(); while (wfe() && monitor8(&q->qschst.cur_ticket, @@ -728,8 +721,7 @@ static void pktio_start(int pktio_idx, elem->cons_type |= FLAG_PKTIN; /* Set pktin queue flag */ elem->pktio_idx = pktio_idx; elem->rx_queue = rxq; - elem->xoffset = sched_pktin_add(elem->sched_grp, - elem->sched_prio); + elem->xoffset = sched_pktin_add(elem->sched_grp, elem->sched_prio); ODP_ASSERT(elem->schedq != NULL); schedq_push(elem->schedq, elem); } @@ -899,8 +891,7 @@ static int _schedule(odp_queue_t *from, odp_event_t ev[], int num_evts) num = poll_pktin(atomq, ev, num_evts); if (odp_likely(num != 0)) { if (from) - *from = queue_get_handle( - (queue_entry_t *)atomq); + *from = queue_get_handle((queue_entry_t *)atomq); return num; } } else { @@ -932,8 +923,7 @@ dequeue_atomic: * scheduler. */ if (from) - *from = queue_get_handle( - (queue_entry_t *)atomq); + *from = queue_get_handle((queue_entry_t *)atomq); return num; } } @@ -1006,11 +996,10 @@ restart_same: * responsibility. */ /* The ticket taken below will signal producers */ - ts->ticket = __atomic_fetch_add( - &atomq->qschst.nxt_ticket, 1, __ATOMIC_RELAXED); - while (__atomic_load_n( - &atomq->qschst.cur_ticket, - __ATOMIC_ACQUIRE) != ts->ticket) { + ts->ticket = __atomic_fetch_add(&atomq->qschst.nxt_ticket, 1, + __ATOMIC_RELAXED); + while (__atomic_load_n(&atomq->qschst.cur_ticket, + __ATOMIC_ACQUIRE) != ts->ticket) { /* No need to use WFE, spinning here seems * very infrequent. */ @@ -1111,8 +1100,7 @@ restart_same: ts->rctx = rctx; if (from) - *from = queue_get_handle( - (queue_entry_t *)elem); + *from = queue_get_handle((queue_entry_t *)elem); return num; } #ifdef CONFIG_QSCHST_LOCK @@ -1571,8 +1559,7 @@ static int schedule_group_destroy(odp_schedule_group_t group) */ for (p = 0; p < ODP_SCHED_PRIO_NUM; p++) { if (sg->xcount[p] != 0) { - bitset_t wanted = atom_bitset_load( - &sg->thr_wanted, __ATOMIC_RELAXED); + bitset_t wanted = atom_bitset_load(&sg->thr_wanted, __ATOMIC_RELAXED); sevl(); while (wfe() && @@ -2167,6 +2154,7 @@ static void order_lock(void) */ static void order_unlock(void) { + /* Nothing to do */ } static uint32_t schedule_max_ordered_locks(void) diff --git a/platform/linux-generic/odp_schedule_scalable_ordered.c b/platform/linux-generic/odp_schedule_scalable_ordered.c index 239d18239..991be658e 100644 --- a/platform/linux-generic/odp_schedule_scalable_ordered.c +++ b/platform/linux-generic/odp_schedule_scalable_ordered.c @@ -332,15 +332,11 @@ int _odp_rctx_save(queue_entry_t *queue, odp_buffer_hdr_t *buf_hdr[], int num) /* No more space in current reorder context * Try to allocate another. */ - if (odp_unlikely( - bitset_is_null(ts->priv_rvec_free))) { - ts->priv_rvec_free = - atom_bitset_xchg( - &ts->rvec_free, - 0, - __ATOMIC_RELAXED); - if (odp_unlikely(bitset_is_null( - ts->priv_rvec_free))) + if (odp_unlikely(bitset_is_null(ts->priv_rvec_free))) { + ts->priv_rvec_free = atom_bitset_xchg(&ts->rvec_free, 0, + __ATOMIC_RELAXED); + + if (odp_unlikely(bitset_is_null(ts->priv_rvec_free))) /* Out of reorder contexts. * Return the number of events * stored so far. diff --git a/platform/linux-generic/odp_schedule_sp.c b/platform/linux-generic/odp_schedule_sp.c index cad391ed4..a91b9fbca 100644 --- a/platform/linux-generic/odp_schedule_sp.c +++ b/platform/linux-generic/odp_schedule_sp.c @@ -255,7 +255,6 @@ static int term_global(void) } odp_event_free(event); } - } } @@ -751,10 +750,12 @@ static void schedule_resume(void) static void schedule_release_atomic(void) { + /* Nothing to do */ } static void schedule_release_ordered(void) { + /* Nothing to do */ } static void schedule_prefetch(int num) @@ -1016,10 +1017,12 @@ static void schedule_order_lock_wait(uint32_t lock_index) static void order_lock(void) { + /* Nothing to do */ } static void order_unlock(void) { + /* Nothing to do */ } static int schedule_capability(odp_schedule_capability_t *capa) diff --git a/platform/linux-generic/odp_system_info.c b/platform/linux-generic/odp_system_info.c index cbbadf336..079210652 100644 --- a/platform/linux-generic/odp_system_info.c +++ b/platform/linux-generic/odp_system_info.c @@ -89,7 +89,6 @@ static int systemcpu_cache_line_size(void) } #endif - static uint64_t default_huge_page_size(void) { char str[1024]; @@ -248,11 +247,9 @@ static char *get_hugepage_dir(uint64_t hugepage_sz) retval = strdup(tokens[MOUNTPT]); break; } - } - /* there is an explicit page size, so check it */ - else { - pagesz = - str_to_size(&pagesz_str[pagesize_opt_len]); + } else { + /* there is an explicit page size, so check it */ + pagesz = str_to_size(&pagesz_str[pagesize_opt_len]); if (pagesz == hugepage_sz) { retval = strdup(tokens[MOUNTPT]); break; @@ -304,7 +301,6 @@ static int systemcpu(system_info_t *sysinfo) sysinfo->cpu_count = ret; - ret = systemcpu_cache_line_size(); if (ret == 0) { ODP_ERR("systemcpu_cache_line_size failed.\n"); diff --git a/platform/linux-generic/odp_timer.c b/platform/linux-generic/odp_timer.c index 8b2ba16c2..2db5bc7b1 100644 --- a/platform/linux-generic/odp_timer.c +++ b/platform/linux-generic/odp_timer.c @@ -510,7 +510,7 @@ static inline odp_timer_t timer_alloc(timer_pool_t *tp, odp_queue_t queue, const /* Add timer to queue */ _odp_queue_fn->timer_add(queue); } else { - __odp_errno = ENFILE; /* Reusing file table overflow */ + _odp_errno = ENFILE; /* Reusing file table overflow */ hdl = ODP_TIMER_INVALID; } odp_spinlock_unlock(&tp->lock); @@ -583,12 +583,9 @@ static bool timer_reset(uint32_t idx, uint64_t abs_tck, odp_buffer_t *tmo_buf, /* Atomic CAS will fail if we experienced torn reads, * retry update sequence until CAS succeeds */ - } while (!_odp_atomic_u128_cmp_xchg_mm( - (_odp_atomic_u128_t *)tb, - (_uint128_t *)&old, - (_uint128_t *)&new, - _ODP_MEMMODEL_RLS, - _ODP_MEMMODEL_RLX)); + } while (!_odp_atomic_u128_cmp_xchg_mm((_odp_atomic_u128_t *)tb, + (_uint128_t *)&old, (_uint128_t *)&new, + _ODP_MEMMODEL_RLS, _ODP_MEMMODEL_RLX)); #elif __GCC_ATOMIC_LLONG_LOCK_FREE >= 2 && \ defined __GCC_HAVE_SYNC_COMPARE_AND_SWAP_8 /* Target supports lock-free 64-bit CAS (and probably exchange) */ @@ -816,10 +813,9 @@ static inline void timer_expire(timer_pool_t *tp, uint32_t idx, uint64_t tick) new.exp_tck.v = exp_tck | TMO_INACTIVE; new.tmo_buf = ODP_BUFFER_INVALID; - int succ = _odp_atomic_u128_cmp_xchg_mm( - (_odp_atomic_u128_t *)tb, - (_uint128_t *)&old, (_uint128_t *)&new, - _ODP_MEMMODEL_RLS, _ODP_MEMMODEL_RLX); + int succ = _odp_atomic_u128_cmp_xchg_mm((_odp_atomic_u128_t *)tb, + (_uint128_t *)&old, (_uint128_t *)&new, + _ODP_MEMMODEL_RLS, _ODP_MEMMODEL_RLX); if (succ) tmo_buf = old.tmo_buf; /* Else CAS failed, something changed => skip timer @@ -966,7 +962,7 @@ static inline void timer_pool_scan_inline(int num, odp_time_t now) } } -void _timer_run_inline(int dec) +void _odp_timer_run_inline(int dec) { odp_time_t now; int num = timer_global->highest_tp_idx + 1; @@ -1284,19 +1280,19 @@ odp_timer_pool_t odp_timer_pool_create(const char *name, if ((param->res_ns && param->res_hz) || (param->res_ns == 0 && param->res_hz == 0)) { - __odp_errno = EINVAL; + _odp_errno = EINVAL; return ODP_TIMER_POOL_INVALID; } if (param->res_hz == 0 && param->res_ns < timer_global->highest_res_ns) { - __odp_errno = EINVAL; + _odp_errno = EINVAL; return ODP_TIMER_POOL_INVALID; } if (param->res_ns == 0 && param->res_hz > timer_global->highest_res_hz) { - __odp_errno = EINVAL; + _odp_errno = EINVAL; return ODP_TIMER_POOL_INVALID; } diff --git a/platform/linux-generic/odp_timer_wheel.c b/platform/linux-generic/odp_timer_wheel.c index 84640d215..71b3370b5 100644 --- a/platform/linux-generic/odp_timer_wheel.c +++ b/platform/linux-generic/odp_timer_wheel.c @@ -637,9 +637,7 @@ static int timer_current_wheel_update(timer_wheels_t *timer_wheels, if (timer_slot->user_data != 0) { rc = expired_timers_append(timer_wheels, timer_slot); if (rc < 0) - timer_wheels-> - expired_timers_ring-> - expired_ring_full_cnt++; + timer_wheels->expired_timers_ring->expired_ring_full_cnt++; timer_slot->user_data = 0; } @@ -954,9 +952,7 @@ void _odp_timer_wheel_stats_print(_odp_timer_wheel_t timer_wheel) ODP_PRINT("_odp_int_timer_wheel_stats current_ticks=%" PRIu64 "\n", timer_wheels->current_ticks); for (wheel_idx = 0; wheel_idx < 4; wheel_idx++) - _odp_int_timer_wheel_desc_print( - &timer_wheels->wheel_descs[wheel_idx], - wheel_idx); + _odp_int_timer_wheel_desc_print(&timer_wheels->wheel_descs[wheel_idx], wheel_idx); ODP_PRINT(" total timer_inserts=%" PRIu64 " timer_removes=%" PRIu64 " insert_fails=%" PRIu64 "\n", diff --git a/platform/linux-generic/odp_traffic_mngr.c b/platform/linux-generic/odp_traffic_mngr.c index e1aff6e8c..0ae107509 100644 --- a/platform/linux-generic/odp_traffic_mngr.c +++ b/platform/linux-generic/odp_traffic_mngr.c @@ -356,7 +356,6 @@ static void *alloc_entry_in_tbl(profile_tbl_t *profile_tbl, default: ODP_ERR("Invalid TM profile\n"); return NULL; - } } @@ -392,7 +391,6 @@ static void free_tbl_entry(profile_tbl_t *profile_tbl, default: ODP_ERR("Invalid TM profile\n"); return; - } } @@ -1238,10 +1236,9 @@ static odp_bool_t run_sched(tm_system_t *tm_system, * virtual finish time, just insert it into this * sched_state's list sorted by virtual finish times. */ - rc = _odp_sorted_list_insert( - tm_system->_odp_int_sorted_pool, - new_sched_state->sorted_list, - new_finish_time, new_pkt_desc->word); + rc = _odp_sorted_list_insert(tm_system->_odp_int_sorted_pool, + new_sched_state->sorted_list, + new_finish_time, new_pkt_desc->word); if (0 <= rc) { new_sched_state->sorted_list_cnt++; @@ -2067,8 +2064,7 @@ static void egress_vlan_marking(tm_vlan_marking_t *vlan_marking, * correctness rather then performance. */ split_hdr = hdr_len < (_ODP_ETHHDR_LEN + _ODP_VLANHDR_LEN); if (split_hdr) { - odp_packet_copy_to_mem(odp_pkt, _ODP_ETHHDR_LEN, - _ODP_VLANHDR_LEN, &vlan_hdr); + odp_packet_copy_to_mem(odp_pkt, _ODP_ETHHDR_LEN, _ODP_VLANHDR_LEN, &vlan_hdr); vlan_hdr_ptr = &vlan_hdr; } @@ -2082,8 +2078,7 @@ static void egress_vlan_marking(tm_vlan_marking_t *vlan_marking, vlan_hdr_ptr->tci = odp_cpu_to_be_16(new_tci); if (split_hdr) - odp_packet_copy_from_mem(odp_pkt, _ODP_ETHHDR_LEN, - _ODP_VLANHDR_LEN, &vlan_hdr); + odp_packet_copy_from_mem(odp_pkt, _ODP_ETHHDR_LEN, _ODP_VLANHDR_LEN, &vlan_hdr); } static void egress_ipv4_tos_marking(tm_tos_marking_t *tos_marking, @@ -2107,8 +2102,7 @@ static void egress_ipv4_tos_marking(tm_tos_marking_t *tos_marking, * correctness rather then performance. */ split_hdr = hdr_len < 12; if (split_hdr) { - odp_packet_copy_to_mem(odp_pkt, l3_offset, - _ODP_IPV4HDR_LEN, &ipv4_hdr); + odp_packet_copy_to_mem(odp_pkt, l3_offset, _ODP_IPV4HDR_LEN, &ipv4_hdr); ipv4_hdr_ptr = &ipv4_hdr; } @@ -2149,8 +2143,7 @@ static void egress_ipv4_tos_marking(tm_tos_marking_t *tos_marking, ipv4_hdr_ptr->tos = new_tos; ipv4_hdr_ptr->chksum = odp_cpu_to_be_16((~ones_compl_sum) & 0xFFFF); if (split_hdr) - odp_packet_copy_from_mem(odp_pkt, l3_offset, - _ODP_IPV4HDR_LEN, &ipv4_hdr); + odp_packet_copy_from_mem(odp_pkt, l3_offset, _ODP_IPV4HDR_LEN, &ipv4_hdr); } static void egress_ipv6_tc_marking(tm_tos_marking_t *tos_marking, @@ -2174,8 +2167,7 @@ static void egress_ipv6_tc_marking(tm_tos_marking_t *tos_marking, * correctness rather then performance. */ split_hdr = hdr_len < 4; if (split_hdr) { - odp_packet_copy_to_mem(odp_pkt, l3_offset, - _ODP_IPV6HDR_LEN, &ipv6_hdr); + odp_packet_copy_to_mem(odp_pkt, l3_offset, _ODP_IPV6HDR_LEN, &ipv6_hdr); ipv6_hdr_ptr = &ipv6_hdr; } @@ -2203,8 +2195,7 @@ static void egress_ipv6_tc_marking(tm_tos_marking_t *tos_marking, ipv6_hdr_ptr->ver_tc_flow = odp_cpu_to_be_32(new_ver_tc_flow); if (split_hdr) - odp_packet_copy_from_mem(odp_pkt, l3_offset, - _ODP_IPV6HDR_LEN, &ipv6_hdr); + odp_packet_copy_from_mem(odp_pkt, l3_offset, _ODP_IPV6HDR_LEN, &ipv6_hdr); } static void tm_egress_marking(tm_system_t *tm_system, odp_packet_t odp_pkt) @@ -2307,9 +2298,8 @@ static int tm_process_input_work_queue(tm_system_t *tm_system, /* If the tm_queue_obj already has a pkt to work with, * then just add this new pkt to the associated * _odp_int_pkt_queue. */ - (void)_odp_pkt_queue_append( - tm_system->_odp_int_queue_pool, - tm_queue_obj->_odp_int_pkt_queue, pkt); + (void)_odp_pkt_queue_append(tm_system->_odp_int_queue_pool, + tm_queue_obj->_odp_int_pkt_queue, pkt); tm_queue_obj->pkts_enqueued_cnt++; } else { /* If the tm_queue_obj doesn't have a pkt to work @@ -2749,6 +2739,7 @@ static int tm_thread_create(tm_system_group_t *tm_group) rc); return rc; } + static void _odp_tm_group_destroy(_odp_tm_group_t odp_tm_group) { tm_system_group_t *tm_group; @@ -2976,21 +2967,19 @@ odp_tm_t odp_tm_create(const char *name, odp_ticketlock_init(&tm_system->tm_system_lock); odp_atomic_init_u64(&tm_system->destroying, 0); - tm_system->_odp_int_sorted_pool = _odp_sorted_pool_create( - max_sorted_lists); + tm_system->_odp_int_sorted_pool = _odp_sorted_pool_create(max_sorted_lists); create_fail |= tm_system->_odp_int_sorted_pool == _ODP_INT_SORTED_POOL_INVALID; if (create_fail == 0) { - tm_system->_odp_int_queue_pool = _odp_queue_pool_create( - max_num_queues, max_queued_pkts); + tm_system->_odp_int_queue_pool = _odp_queue_pool_create(max_num_queues, + max_queued_pkts); create_fail |= tm_system->_odp_int_queue_pool == _ODP_INT_QUEUE_POOL_INVALID; } if (create_fail == 0) { - tm_system->_odp_int_timer_wheel = _odp_timer_wheel_create( - max_timers, tm_system); + tm_system->_odp_int_timer_wheel = _odp_timer_wheel_create(max_timers, tm_system); create_fail |= tm_system->_odp_int_timer_wheel == _ODP_INT_TIMER_WHEEL_INVALID; } @@ -3007,20 +2996,14 @@ odp_tm_t odp_tm_create(const char *name, if (create_fail) { _odp_int_name_tbl_delete(name_tbl_id); - if (tm_system->_odp_int_sorted_pool - != _ODP_INT_SORTED_POOL_INVALID) - _odp_sorted_pool_destroy( - tm_system->_odp_int_sorted_pool); + if (tm_system->_odp_int_sorted_pool != _ODP_INT_SORTED_POOL_INVALID) + _odp_sorted_pool_destroy(tm_system->_odp_int_sorted_pool); - if (tm_system->_odp_int_queue_pool != - _ODP_INT_QUEUE_POOL_INVALID) - _odp_queue_pool_destroy( - tm_system->_odp_int_queue_pool); + if (tm_system->_odp_int_queue_pool != _ODP_INT_QUEUE_POOL_INVALID) + _odp_queue_pool_destroy(tm_system->_odp_int_queue_pool); - if (tm_system->_odp_int_timer_wheel - != _ODP_INT_TIMER_WHEEL_INVALID) - _odp_timer_wheel_destroy( - tm_system->_odp_int_timer_wheel); + if (tm_system->_odp_int_timer_wheel != _ODP_INT_TIMER_WHEEL_INVALID) + _odp_timer_wheel_destroy(tm_system->_odp_int_timer_wheel); tm_system_free(tm_system); odp_ticketlock_unlock(&tm_glb->create_lock); @@ -3708,9 +3691,8 @@ odp_tm_node_t odp_tm_node_create(odp_tm_t odp_tm, const char *name, schedulers_obj = &tm_node_obj->schedulers_obj; schedulers_obj->num_priorities = num_priorities; for (priority = 0; priority < num_priorities; priority++) { - sorted_list = _odp_sorted_list_create( - tm_system->_odp_int_sorted_pool, - params->max_fanin); + sorted_list = _odp_sorted_list_create(tm_system->_odp_int_sorted_pool, + params->max_fanin); schedulers_obj->sched_states[priority].sorted_list = sorted_list; } @@ -4673,8 +4655,8 @@ int odp_tm_total_threshold_config(odp_tm_t odp_tm, return -1; odp_ticketlock_lock(&tm_glb->profile_lock); - tm_system->total_info.threshold_params = tm_get_profile_params( - thresholds_profile, TM_THRESHOLD_PROFILE); + tm_system->total_info.threshold_params = tm_get_profile_params(thresholds_profile, + TM_THRESHOLD_PROFILE); odp_ticketlock_unlock(&tm_glb->profile_lock); return 0; } diff --git a/platform/linux-generic/pktio/dpdk.c b/platform/linux-generic/pktio/dpdk.c index 97fb6ef1d..fb99d8ed5 100644 --- a/platform/linux-generic/pktio/dpdk.c +++ b/platform/linux-generic/pktio/dpdk.c @@ -432,8 +432,8 @@ static int pool_dequeue_bulk(struct rte_mempool *mp, void **obj_table, int pkts; int i; - pkts = packet_alloc_multi(pool, pool_entry->seg_len, packet_tbl, - num); + pkts = _odp_packet_alloc_multi(pool, pool_entry->seg_len, packet_tbl, + num); if (odp_unlikely(pkts != (int)num)) { if (pkts > 0) @@ -577,10 +577,10 @@ static inline int mbuf_to_pkt(pktio_entry_t *pktio_entry, /* Allocate maximum sized packets */ max_len = pkt_dpdk->data_room; - num = packet_alloc_multi(pool, max_len + frame_offset, - pkt_table, mbuf_num); + num = _odp_packet_alloc_multi(pool, max_len + frame_offset, + pkt_table, mbuf_num); if (num != mbuf_num) { - ODP_DBG("packet_alloc_multi() unable to allocate all packets: " + ODP_DBG("_odp_packet_alloc_multi() unable to allocate all packets: " "%d/%" PRIu16 " allocated\n", num, mbuf_num); for (i = num; i < mbuf_num; i++) rte_pktmbuf_free(mbuf_table[i]); @@ -805,7 +805,7 @@ static inline int pkt_to_mbuf(pktio_entry_t *pktio_entry, if (odp_unlikely(pkt_len > pkt_dpdk->mtu)) { if (i == 0) - __odp_errno = EMSGSIZE; + _odp_errno = EMSGSIZE; goto fail; } @@ -999,7 +999,7 @@ static inline int pkt_to_mbuf_zero(pktio_entry_t *pktio_entry, fail: if (i == 0) - __odp_errno = EMSGSIZE; + _odp_errno = EMSGSIZE; return i; } @@ -2063,8 +2063,8 @@ static int dpdk_send(pktio_entry_t *pktio_entry, int index, } } } - if (odp_unlikely(tx_pkts == 0 && __odp_errno != 0)) - return -1; + if (odp_unlikely(tx_pkts == 0 && _odp_errno != 0)) + return -1; } else { if (odp_unlikely(tx_pkts < mbufs)) { for (i = tx_pkts; i < mbufs; i++) @@ -2072,7 +2072,7 @@ static int dpdk_send(pktio_entry_t *pktio_entry, int index, } if (odp_unlikely(tx_pkts == 0)) { - if (__odp_errno != 0) + if (_odp_errno != 0) return -1; } else { odp_packet_free_multi(pkt_table, tx_pkts); @@ -2227,7 +2227,7 @@ static int dpdk_stats_reset(pktio_entry_t *pktio_entry) return 0; } -const pktio_if_ops_t dpdk_pktio_ops = { +const pktio_if_ops_t _odp_dpdk_pktio_ops = { .name = "dpdk", .init_global = dpdk_pktio_init_global, .init_local = dpdk_pktio_init_local, diff --git a/platform/linux-generic/pktio/io_ops.c b/platform/linux-generic/pktio/io_ops.c index cd85164e6..b5a08b58a 100644 --- a/platform/linux-generic/pktio/io_ops.c +++ b/platform/linux-generic/pktio/io_ops.c @@ -11,13 +11,13 @@ * Order matters. The first implementation to setup successfully * will be picked. * Array must be NULL terminated */ -const pktio_if_ops_t * const pktio_if_ops[] = { +const pktio_if_ops_t * const _odp_pktio_if_ops[] = { &_odp_loopback_pktio_ops, #ifdef _ODP_PKTIO_DPDK - &dpdk_pktio_ops, + &_odp_dpdk_pktio_ops, #endif #ifdef _ODP_PKTIO_NETMAP - &netmap_pktio_ops, + &_odp_netmap_pktio_ops, #endif #ifdef _ODP_PKTIO_PCAP &_odp_pcap_pktio_ops, diff --git a/platform/linux-generic/pktio/ipc.c b/platform/linux-generic/pktio/ipc.c index 845fd821d..e0c8cba3b 100644 --- a/platform/linux-generic/pktio/ipc.c +++ b/platform/linux-generic/pktio/ipc.c @@ -130,7 +130,7 @@ static ring_ptr_t *_ring_create(const char *name, uint32_t count, /* count must be a power of 2 */ if (!CHECK_IS_POWER2(count)) { ODP_ERR("Requested size is invalid, must be a power of 2\n"); - __odp_errno = EINVAL; + _odp_errno = EINVAL; return NULL; } @@ -145,7 +145,7 @@ static ring_ptr_t *_ring_create(const char *name, uint32_t count, ring_ptr_init(r); } else { - __odp_errno = ENOMEM; + _odp_errno = ENOMEM; ODP_ERR("Cannot reserve memory\n"); } diff --git a/platform/linux-generic/pktio/loop.c b/platform/linux-generic/pktio/loop.c index 1d5751eb8..d58c9a55c 100644 --- a/platform/linux-generic/pktio/loop.c +++ b/platform/linux-generic/pktio/loop.c @@ -311,7 +311,7 @@ static int loopback_send(pktio_entry_t *pktio_entry, int index ODP_UNUSED, if (odp_unlikely(pkt_len > pkt_loop->mtu)) { if (nb_tx == 0) { - __odp_errno = EMSGSIZE; + _odp_errno = EMSGSIZE; return -1; } break; @@ -431,6 +431,7 @@ static int loopback_init_capability(pktio_entry_t *pktio_entry) capa->maxlen.max_output = LOOP_MTU_MAX; odp_pktio_config_init(&capa->config); + capa->config.enable_loop = 1; capa->config.pktin.bit.ts_all = 1; capa->config.pktin.bit.ts_ptp = 1; capa->config.pktin.bit.ipv4_chksum = 1; diff --git a/platform/linux-generic/pktio/netmap.c b/platform/linux-generic/pktio/netmap.c index a6bebaa11..544503aa3 100644 --- a/platform/linux-generic/pktio/netmap.c +++ b/platform/linux-generic/pktio/netmap.c @@ -345,7 +345,7 @@ static int netmap_close(pktio_entry_t *pktio_entry) netmap_close_descriptors(pktio_entry); if (pkt_nm->sockfd != -1 && close(pkt_nm->sockfd) != 0) { - __odp_errno = errno; + _odp_errno = errno; ODP_ERR("close(sockfd): %s\n", strerror(errno)); return -1; } @@ -830,8 +830,8 @@ static inline int netmap_pkt_to_odp(pktio_entry_t *pktio_entry, /* Allocate maximum sized packets */ max_len = pkt_priv(pktio_entry)->mtu; - num = packet_alloc_multi(pool, max_len + frame_offset, - pkt_tbl, slot_num); + num = _odp_packet_alloc_multi(pool, max_len + frame_offset, + pkt_tbl, slot_num); for (i = 0; i < num; i++) { netmap_slot_t slot; @@ -1121,7 +1121,7 @@ static int netmap_send(pktio_entry_t *pktio_entry, int index, if (pkt_len > pkt_nm->mtu) { if (nb_tx == 0) - __odp_errno = EMSGSIZE; + _odp_errno = EMSGSIZE; break; } for (i = 0; i < NM_INJECT_RETRIES; i++) { @@ -1158,7 +1158,7 @@ static int netmap_send(pktio_entry_t *pktio_entry, int index, odp_ticketlock_unlock(&pkt_nm->tx_desc_ring[index].s.lock); if (odp_unlikely(nb_tx == 0)) { - if (__odp_errno != 0) + if (_odp_errno != 0) return -1; } else { if (odp_unlikely(tx_ts_idx && nb_tx >= tx_ts_idx)) @@ -1200,7 +1200,7 @@ static int netmap_promisc_mode_set(pktio_entry_t *pktio_entry, odp_bool_t enable) { if (pkt_priv(pktio_entry)->is_virtual) { - __odp_errno = ENOTSUP; + _odp_errno = ENOTSUP; return -1; } @@ -1273,7 +1273,7 @@ static int netmap_init_global(void) return 0; } -const pktio_if_ops_t netmap_pktio_ops = { +const pktio_if_ops_t _odp_netmap_pktio_ops = { .name = "netmap", .print = netmap_print, .init_global = netmap_init_global, diff --git a/platform/linux-generic/pktio/pcap.c b/platform/linux-generic/pktio/pcap.c index ffb6fab68..82635e59c 100644 --- a/platform/linux-generic/pktio/pcap.c +++ b/platform/linux-generic/pktio/pcap.c @@ -266,8 +266,8 @@ static int pcapif_recv_pkt(pktio_entry_t *pktio_entry, int index ODP_UNUSED, pkt_len = hdr->caplen; - ret = packet_alloc_multi(pcap->pool, pkt_len + frame_offset, - &pkt, 1); + ret = _odp_packet_alloc_multi(pcap->pool, pkt_len + frame_offset, + &pkt, 1); if (odp_unlikely(ret != 1)) break; diff --git a/platform/linux-generic/pktio/socket.c b/platform/linux-generic/pktio/socket.c index 29b6c9af6..eb4390e46 100644 --- a/platform/linux-generic/pktio/socket.c +++ b/platform/linux-generic/pktio/socket.c @@ -88,7 +88,7 @@ static int sock_close(pktio_entry_t *pktio_entry) pkt_sock_t *pkt_sock = pkt_priv(pktio_entry); if (pkt_sock->sockfd != -1 && close(pkt_sock->sockfd) != 0) { - __odp_errno = errno; + _odp_errno = errno; ODP_ERR("close(sockfd): %s\n", strerror(errno)); return -1; } @@ -120,7 +120,7 @@ static int sock_setup_pkt(pktio_entry_t *pktio_entry, const char *netdev, sockfd = socket(AF_PACKET, SOCK_RAW, htons(ETH_P_ALL)); if (sockfd == -1) { - __odp_errno = errno; + _odp_errno = errno; ODP_ERR("socket(): %s\n", strerror(errno)); goto error; } @@ -131,7 +131,7 @@ static int sock_setup_pkt(pktio_entry_t *pktio_entry, const char *netdev, snprintf(ethreq.ifr_name, IF_NAMESIZE, "%s", netdev); err = ioctl(sockfd, SIOCGIFINDEX, ðreq); if (err != 0) { - __odp_errno = errno; + _odp_errno = errno; ODP_ERR("ioctl(SIOCGIFINDEX): %s: \"%s\".\n", strerror(errno), ethreq.ifr_name); goto error; @@ -155,7 +155,7 @@ static int sock_setup_pkt(pktio_entry_t *pktio_entry, const char *netdev, sa_ll.sll_ifindex = if_idx; sa_ll.sll_protocol = htons(ETH_P_ALL); if (bind(sockfd, (struct sockaddr *)&sa_ll, sizeof(sa_ll)) < 0) { - __odp_errno = errno; + _odp_errno = errno; ODP_ERR("bind(to IF): %s\n", strerror(errno)); goto error; } @@ -230,7 +230,7 @@ static int sock_mmsg_recv(pktio_entry_t *pktio_entry, int index ODP_UNUSED, memset(msgvec, 0, sizeof(msgvec)); - nb_pkts = packet_alloc_multi(pool, alloc_len, pkt_table, num); + nb_pkts = _odp_packet_alloc_multi(pool, alloc_len, pkt_table, num); for (i = 0; i < nb_pkts; i++) { if (frame_offset) pull_head(packet_hdr(pkt_table[i]), frame_offset); @@ -454,7 +454,7 @@ static int sock_mmsg_send(pktio_entry_t *pktio_entry, int index ODP_UNUSED, ret = sendmmsg(sockfd, &msgvec[i], num - i, MSG_DONTWAIT); if (odp_unlikely(ret <= -1)) { if (i == 0 && SOCK_ERR_REPORT(errno)) { - __odp_errno = errno; + _odp_errno = errno; ODP_ERR("sendmmsg(): %s\n", strerror(errno)); odp_ticketlock_unlock(&pkt_sock->tx_lock); return -1; diff --git a/platform/linux-generic/pktio/socket_common.c b/platform/linux-generic/pktio/socket_common.c index 88c1471b0..ab0caeb81 100644 --- a/platform/linux-generic/pktio/socket_common.c +++ b/platform/linux-generic/pktio/socket_common.c @@ -57,7 +57,7 @@ int _odp_mac_addr_get_fd(int fd, const char *name, unsigned char mac_dst[]) snprintf(ethreq.ifr_name, IF_NAMESIZE, "%s", name); ret = ioctl(fd, SIOCGIFHWADDR, ðreq); if (ret != 0) { - __odp_errno = errno; + _odp_errno = errno; ODP_ERR("ioctl(SIOCGIFHWADDR): %s: \"%s\".\n", strerror(errno), ethreq.ifr_name); return -1; @@ -82,7 +82,7 @@ uint32_t _odp_mtu_get_fd(int fd, const char *name) snprintf(ifr.ifr_name, IF_NAMESIZE, "%s", name); ret = ioctl(fd, SIOCGIFMTU, &ifr); if (ret < 0) { - __odp_errno = errno; + _odp_errno = errno; ODP_ERR("ioctl(SIOCGIFMTU): %s: \"%s\".\n", strerror(errno), ifr.ifr_name); return 0; @@ -106,7 +106,7 @@ int _odp_mtu_set_fd(int fd, const char *name, int mtu) ret = ioctl(fd, SIOCSIFMTU, &ifr); if (ret < 0) { - __odp_errno = errno; + _odp_errno = errno; ODP_ERR("ioctl(SIOCSIFMTU): %s: \"%s\".\n", strerror(errno), ifr.ifr_name); return -1; @@ -127,7 +127,7 @@ int _odp_promisc_mode_set_fd(int fd, const char *name, int enable) snprintf(ifr.ifr_name, IF_NAMESIZE, "%s", name); ret = ioctl(fd, SIOCGIFFLAGS, &ifr); if (ret < 0) { - __odp_errno = errno; + _odp_errno = errno; ODP_DBG("ioctl(SIOCGIFFLAGS): %s: \"%s\".\n", strerror(errno), ifr.ifr_name); return -1; @@ -140,7 +140,7 @@ int _odp_promisc_mode_set_fd(int fd, const char *name, int enable) ret = ioctl(fd, SIOCSIFFLAGS, &ifr); if (ret < 0) { - __odp_errno = errno; + _odp_errno = errno; ODP_DBG("ioctl(SIOCSIFFLAGS): %s: \"%s\".\n", strerror(errno), ifr.ifr_name); return -1; @@ -161,7 +161,7 @@ int _odp_promisc_mode_get_fd(int fd, const char *name) snprintf(ifr.ifr_name, IF_NAMESIZE, "%s", name); ret = ioctl(fd, SIOCGIFFLAGS, &ifr); if (ret < 0) { - __odp_errno = errno; + _odp_errno = errno; ODP_DBG("ioctl(SIOCGIFFLAGS): %s: \"%s\".\n", strerror(errno), ifr.ifr_name); return -1; @@ -178,7 +178,7 @@ int _odp_link_status_fd(int fd, const char *name) snprintf(ifr.ifr_name, IF_NAMESIZE, "%s", name); ret = ioctl(fd, SIOCGIFFLAGS, &ifr); if (ret < 0) { - __odp_errno = errno; + _odp_errno = errno; ODP_DBG("ioctl(SIOCGIFFLAGS): %s: \"%s\".\n", strerror(errno), ifr.ifr_name); return ODP_PKTIO_LINK_STATUS_UNKNOWN; @@ -206,7 +206,7 @@ int _odp_link_info_fd(int fd, const char *name, odp_pktio_link_info_t *info) /* Link pause status */ ifr.ifr_data = (void *)&pcmd; if (ioctl(fd, SIOCETHTOOL, &ifr) && errno != EOPNOTSUPP) { - __odp_errno = errno; + _odp_errno = errno; ODP_ERR("ioctl(SIOCETHTOOL): %s: \"%s\".\n", strerror(errno), ifr.ifr_name); return -1; @@ -219,7 +219,7 @@ int _odp_link_info_fd(int fd, const char *name, odp_pktio_link_info_t *info) ifr.ifr_data = (void *)&ecmd_old; if (ioctl(fd, SIOCETHTOOL, &ifr) < 0) { - __odp_errno = errno; + _odp_errno = errno; ODP_ERR("ioctl(SIOCETHTOOL): %s: \"%s\".\n", strerror(errno), ifr.ifr_name); return -1; } @@ -277,7 +277,7 @@ int _odp_link_info_fd(int fd, const char *name, odp_pktio_link_info_t *info) *ecmd = hcmd; ifr.ifr_data = (void *)ecmd; if (ioctl(fd, SIOCETHTOOL, &ifr) < 0) { - __odp_errno = errno; + _odp_errno = errno; ODP_ERR("ioctl(SIOCETHTOOL): %s: \"%s\".\n", strerror(errno), ifr.ifr_name); return -1; diff --git a/platform/linux-generic/pktio/socket_mmap.c b/platform/linux-generic/pktio/socket_mmap.c index 05d0d8254..532f392fa 100644 --- a/platform/linux-generic/pktio/socket_mmap.c +++ b/platform/linux-generic/pktio/socket_mmap.c @@ -109,14 +109,14 @@ static int mmap_pkt_socket(void) int ret, sock = socket(PF_PACKET, SOCK_RAW, htons(ETH_P_ALL)); if (sock == -1) { - __odp_errno = errno; + _odp_errno = errno; ODP_ERR("socket(SOCK_RAW): %s\n", strerror(errno)); return -1; } ret = setsockopt(sock, SOL_PACKET, PACKET_VERSION, &ver, sizeof(ver)); if (ret == -1) { - __odp_errno = errno; + _odp_errno = errno; ODP_ERR("setsockopt(PACKET_VERSION): %s\n", strerror(errno)); close(sock); return -1; @@ -192,7 +192,7 @@ static inline unsigned pkt_mmap_v2_rx(pktio_entry_t *pktio_entry, vlan_len = 4; alloc_len = pkt_len + frame_offset + vlan_len; - ret = packet_alloc_multi(pool, alloc_len, &pkt, 1); + ret = _odp_packet_alloc_multi(pool, alloc_len, &pkt, 1); if (odp_unlikely(ret != 1)) { /* Stop receiving packets when pool is empty. Leave @@ -460,7 +460,7 @@ static int mmap_setup_ring(pkt_sock_mmap_t *pkt_sock, struct ring *ring, ret = setsockopt(sock, SOL_PACKET, type, &ring->req, sizeof(ring->req)); if (ret == -1) { - __odp_errno = errno; + _odp_errno = errno; ODP_ERR("setsockopt(pkt mmap): %s\n", strerror(errno)); return -1; } @@ -491,7 +491,7 @@ static int mmap_sock(pkt_sock_mmap_t *pkt_sock) MAP_SHARED | MAP_LOCKED | MAP_POPULATE, sock, 0); if (pkt_sock->mmap_base == MAP_FAILED) { - __odp_errno = errno; + _odp_errno = errno; ODP_ERR("mmap rx&tx buffer failed: %s\n", strerror(errno)); return -1; } @@ -547,7 +547,7 @@ static int mmap_bind_sock(pkt_sock_mmap_t *pkt_sock, const char *netdev) ret = bind(pkt_sock->sockfd, (struct sockaddr *)&pkt_sock->ll, sizeof(pkt_sock->ll)); if (ret == -1) { - __odp_errno = errno; + _odp_errno = errno; ODP_ERR("bind(to IF): %s\n", strerror(errno)); return -1; } @@ -567,7 +567,7 @@ static int sock_mmap_close(pktio_entry_t *entry) } if (pkt_sock->sockfd != -1 && close(pkt_sock->sockfd) != 0) { - __odp_errno = errno; + _odp_errno = errno; ODP_ERR("close(sockfd): %s\n", strerror(errno)); return -1; } @@ -641,7 +641,7 @@ static int sock_mmap_open(odp_pktio_t id ODP_UNUSED, if_idx = if_nametoindex(netdev); if (if_idx == 0) { - __odp_errno = errno; + _odp_errno = errno; ODP_ERR("if_nametoindex(): %s\n", strerror(errno)); goto error; } diff --git a/platform/linux-generic/pktio/stats/ethtool_stats.c b/platform/linux-generic/pktio/stats/ethtool_stats.c index 0bd4f2c61..8e94e03e7 100644 --- a/platform/linux-generic/pktio/stats/ethtool_stats.c +++ b/platform/linux-generic/pktio/stats/ethtool_stats.c @@ -48,13 +48,13 @@ static struct ethtool_gstrings *get_stringset(int fd, struct ifreq *ifr) drvinfo.cmd = ETHTOOL_GDRVINFO; ifr->ifr_data = (void *)&drvinfo; if (ioctl(fd, SIOCETHTOOL, ifr)) { - __odp_errno = errno; + _odp_errno = errno; ODP_ERR("Cannot get stats information\n"); return NULL; } len = *(uint32_t *)(void *)((char *)&drvinfo + drvinfo_offset); } else { - __odp_errno = errno; + _odp_errno = errno; return NULL; } @@ -74,7 +74,7 @@ static struct ethtool_gstrings *get_stringset(int fd, struct ifreq *ifr) strings->len = len; ifr->ifr_data = (void *)strings; if (ioctl(fd, SIOCETHTOOL, ifr)) { - __odp_errno = errno; + _odp_errno = errno; ODP_ERR("Cannot get stats information\n"); free(strings); return NULL; @@ -114,7 +114,7 @@ static int ethtool_stats(int fd, struct ifreq *ifr, odp_pktio_stats_t *stats) ifr->ifr_data = (void *)estats; err = ioctl(fd, SIOCETHTOOL, ifr); if (err < 0) { - __odp_errno = errno; + _odp_errno = errno; free(strings); free(estats); return -1; diff --git a/platform/linux-generic/pktio/stats/sysfs_stats.c b/platform/linux-generic/pktio/stats/sysfs_stats.c index 474586e19..45e005c74 100644 --- a/platform/linux-generic/pktio/stats/sysfs_stats.c +++ b/platform/linux-generic/pktio/stats/sysfs_stats.c @@ -19,7 +19,7 @@ static int sysfs_get_val(const char *fname, uint64_t *val) file = fopen(fname, "rt"); if (file == NULL) { - __odp_errno = errno; + _odp_errno = errno; /* do not print debug err if sysfs is not supported by * kernel driver. */ diff --git a/platform/linux-generic/pktio/tap.c b/platform/linux-generic/pktio/tap.c index cb85e9afd..e7c9417c3 100644 --- a/platform/linux-generic/pktio/tap.c +++ b/platform/linux-generic/pktio/tap.c @@ -90,7 +90,7 @@ static int mac_addr_set_fd(int fd, const char *name, ret = ioctl(fd, SIOCSIFHWADDR, ðreq); if (ret != 0) { - __odp_errno = errno; + _odp_errno = errno; ODP_ERR("ioctl(SIOCSIFHWADDR): %s: \"%s\".\n", strerror(errno), ethreq.ifr_name); return -1; @@ -121,7 +121,7 @@ static int tap_pktio_open(odp_pktio_t id ODP_UNUSED, fd = open("/dev/net/tun", O_RDWR); if (fd < 0) { - __odp_errno = errno; + _odp_errno = errno; ODP_ERR("failed to open /dev/net/tun: %s\n", strerror(errno)); return -1; } @@ -136,7 +136,7 @@ static int tap_pktio_open(odp_pktio_t id ODP_UNUSED, snprintf(ifr.ifr_name, IF_NAMESIZE, "%s", devname + 4); if (ioctl(fd, TUNSETIFF, (void *)&ifr) < 0) { - __odp_errno = errno; + _odp_errno = errno; ODP_ERR("%s: creating tap device failed: %s\n", ifr.ifr_name, strerror(errno)); goto tap_err; @@ -145,13 +145,13 @@ static int tap_pktio_open(odp_pktio_t id ODP_UNUSED, /* Set nonblocking mode on interface. */ flags = fcntl(fd, F_GETFL, 0); if (flags < 0) { - __odp_errno = errno; + _odp_errno = errno; ODP_ERR("fcntl(F_GETFL) failed: %s\n", strerror(errno)); goto tap_err; } if (fcntl(fd, F_SETFL, flags | O_NONBLOCK) < 0) { - __odp_errno = errno; + _odp_errno = errno; ODP_ERR("fcntl(F_SETFL) failed: %s\n", strerror(errno)); goto tap_err; } @@ -162,14 +162,14 @@ static int tap_pktio_open(odp_pktio_t id ODP_UNUSED, /* Create AF_INET socket for network interface related operations. */ skfd = socket(AF_INET, SOCK_DGRAM, 0); if (skfd < 0) { - __odp_errno = errno; + _odp_errno = errno; ODP_ERR("socket creation failed: %s\n", strerror(errno)); goto tap_err; } mtu = _odp_mtu_get_fd(skfd, devname + 4); if (mtu == 0) { - __odp_errno = errno; + _odp_errno = errno; ODP_ERR("_odp_mtu_get_fd failed: %s\n", strerror(errno)); goto sock_err; } @@ -201,7 +201,7 @@ static int tap_pktio_start(pktio_entry_t *pktio_entry) /* Up interface by default. */ if (ioctl(tap->skfd, SIOCGIFFLAGS, &ifr) < 0) { - __odp_errno = errno; + _odp_errno = errno; ODP_ERR("ioctl(SIOCGIFFLAGS) failed: %s\n", strerror(errno)); goto sock_err; } @@ -210,7 +210,7 @@ static int tap_pktio_start(pktio_entry_t *pktio_entry) ifr.ifr_flags |= IFF_RUNNING; if (ioctl(tap->skfd, SIOCSIFFLAGS, &ifr) < 0) { - __odp_errno = errno; + _odp_errno = errno; ODP_ERR("failed to come up: %s\n", strerror(errno)); goto sock_err; } @@ -232,7 +232,7 @@ static int tap_pktio_stop(pktio_entry_t *pktio_entry) /* Up interface by default. */ if (ioctl(tap->skfd, SIOCGIFFLAGS, &ifr) < 0) { - __odp_errno = errno; + _odp_errno = errno; ODP_ERR("ioctl(SIOCGIFFLAGS) failed: %s\n", strerror(errno)); goto sock_err; } @@ -241,7 +241,7 @@ static int tap_pktio_stop(pktio_entry_t *pktio_entry) ifr.ifr_flags &= ~IFF_RUNNING; if (ioctl(tap->skfd, SIOCSIFFLAGS, &ifr) < 0) { - __odp_errno = errno; + _odp_errno = errno; ODP_ERR("failed to come up: %s\n", strerror(errno)); goto sock_err; } @@ -258,13 +258,13 @@ static int tap_pktio_close(pktio_entry_t *pktio_entry) pkt_tap_t *tap = pkt_priv(pktio_entry); if (tap->fd != -1 && close(tap->fd) != 0) { - __odp_errno = errno; + _odp_errno = errno; ODP_ERR("close(tap->fd): %s\n", strerror(errno)); ret = -1; } if (tap->skfd != -1 && close(tap->skfd) != 0) { - __odp_errno = errno; + _odp_errno = errno; ODP_ERR("close(tap->skfd): %s\n", strerror(errno)); ret = -1; } @@ -289,8 +289,8 @@ static odp_packet_t pack_odp_pkt(pktio_entry_t *pktio_entry, const void *data, } } - num = packet_alloc_multi(pkt_priv(pktio_entry)->pool, - len + frame_offset, &pkt, 1); + num = _odp_packet_alloc_multi(pkt_priv(pktio_entry)->pool, + len + frame_offset, &pkt, 1); if (num != 1) return ODP_PACKET_INVALID; @@ -344,7 +344,7 @@ static int tap_pktio_recv(pktio_entry_t *pktio_entry, int index ODP_UNUSED, ts_val = odp_time_global(); if (retval < 0) { - __odp_errno = errno; + _odp_errno = errno; break; } @@ -374,7 +374,7 @@ static int tap_pktio_send_lockless(pktio_entry_t *pktio_entry, if (odp_unlikely(pkt_len > mtu)) { if (i == 0) { - __odp_errno = EMSGSIZE; + _odp_errno = EMSGSIZE; return -1; } break; @@ -391,7 +391,7 @@ static int tap_pktio_send_lockless(pktio_entry_t *pktio_entry, if (retval < 0) { if (i == 0 && SOCK_ERR_REPORT(errno)) { - __odp_errno = errno; + _odp_errno = errno; ODP_ERR("write(): %s\n", strerror(errno)); return -1; } @@ -399,7 +399,7 @@ static int tap_pktio_send_lockless(pktio_entry_t *pktio_entry, } else if ((uint32_t)retval != pkt_len) { ODP_ERR("sent partial ethernet packet\n"); if (i == 0) { - __odp_errno = EMSGSIZE; + _odp_errno = EMSGSIZE; return -1; } break; diff --git a/platform/linux-generic/test/Makefile.am b/platform/linux-generic/test/Makefile.am index d66f5ece3..1f7ae611e 100644 --- a/platform/linux-generic/test/Makefile.am +++ b/platform/linux-generic/test/Makefile.am @@ -89,3 +89,5 @@ clean-local: rm -f $(builddir)/$$f; \ done \ fi + +.NOTPARALLEL: diff --git a/platform/linux-generic/test/sched-basic.conf b/platform/linux-generic/test/sched-basic.conf new file mode 100644 index 000000000..57a8a772c --- /dev/null +++ b/platform/linux-generic/test/sched-basic.conf @@ -0,0 +1,8 @@ +# Mandatory fields +odp_implementation = "linux-generic" +config_file_version = "0.1.15" + +sched_basic: { + # Test scheduler with an odd spread value + prio_spread = 3 +} diff --git a/scripts/check-globals.sh b/scripts/check-globals.sh new file mode 100755 index 000000000..d198c50c8 --- /dev/null +++ b/scripts/check-globals.sh @@ -0,0 +1,37 @@ +#!/bin/bash -e +# +# Check that global symbols in a static library conform to a given regex. +# Only static library is checked, since libtool -export-symbols-regex +# takes care of dynamic libraries. +# +# Required variables: +# LIBTOOL Path to libtool. +# NM Path to nm. +# LIB Library directory. +# lib_LTLIBRARIES Library .la file. +# CHECK_GLOBALS_REGEX Global symbols matching this regex are accepted. +# + +tmpfile=$(mktemp) + +# get $objdir +$LIBTOOL --config > $tmpfile +. $tmpfile + +# get $old_library (static library name) +. $lib_LTLIBRARIES + +echo "$old_library: Checking global symbols, regex: $CHECK_GLOBALS_REGEX" + +# get a list of symbols that are global, are not undefined or weak, and +# do not match the regex +$NM -g --defined-only $LIB/$objdir/$old_library | \ + egrep " [uA-T] " | egrep -v "$CHECK_GLOBALS_REGEX" | tee $tmpfile + +num=$(cat $tmpfile | wc -l) +rm -f $tmpfile + +if [ "$num" != "0" ]; then + echo "$old_library: ($num non-matching symbols)" + false +fi diff --git a/scripts/checkpatch.pl b/scripts/checkpatch.pl index a8976b1e5..55fc74597 100755 --- a/scripts/checkpatch.pl +++ b/scripts/checkpatch.pl @@ -43,6 +43,8 @@ my $list_types = 0; my $fix = 0; my $fix_inplace = 0; my $root; +my $gitroot = $ENV{'GIT_DIR'}; +$gitroot = ".git" if !defined($gitroot); my %debug; my %camelcase = (); my %use_type = (); @@ -59,12 +61,13 @@ my $spelling_file = "$D/spelling.txt"; my $codespell = 0; my $codespellfile = "/usr/share/codespell/dictionary.txt"; my $conststructsfile = "$D/const_structs.checkpatch"; -my $typedefsfile = ""; +my $typedefsfile; my $color = "auto"; my $allow_c99_comments = 1; # Can be overridden by --ignore C99_COMMENT_TOLERANCE # git output parsing needs US English output, so first set backtick child process LANGUAGE my $git_command ='export LANGUAGE=en_US.UTF-8; git'; my $tabsize = 8; +my ${CONFIG_} = "CONFIG_"; sub help { my ($exitcode) = @_; @@ -127,6 +130,8 @@ Options: --typedefsfile Read additional types from this file --color[=WHEN] Use colors 'always', 'never', or only when output is a terminal ('auto'). Default is 'auto'. + --kconfig-prefix=WORD use WORD as a prefix for Kconfig symbols (default + ${CONFIG_}) -h, --help, --version display this help and exit When FILE is - read standard input. @@ -235,6 +240,7 @@ GetOptions( 'color=s' => \$color, 'no-color' => \$color, #keep old behaviors of -nocolor 'nocolor' => \$color, #keep old behaviors of -nocolor + 'kconfig-prefix=s' => \${CONFIG_}, 'h|help' => \$help, 'version' => \$help ) or help(1); @@ -376,6 +382,7 @@ our $InitAttribute = qr{$InitAttributeData|$InitAttributeConst|$InitAttributeIni # We need \b after 'init' otherwise 'initconst' will cause a false positive in a check our $Attribute = qr{ const| + volatile| __percpu| __nocast| __safe| @@ -480,7 +487,7 @@ our $logFunctions = qr{(?x: our $allocFunctions = qr{(?x: (?:(?:devm_)? - (?:kv|k|v)[czm]alloc(?:_node|_array)? | + (?:kv|k|v)[czm]alloc(?:_array)?(?:_node)? | kstrdup(?:_const)? | kmemdup(?:_nul)?) | (?:\w+)?alloc_skb(?:_ip_align)? | @@ -500,6 +507,88 @@ our $signature_tags = qr{(?xi: Cc: )}; +our $tracing_logging_tags = qr{(?xi: + [=-]*> | + <[=-]* | + \[ | + \] | + start | + called | + entered | + entry | + enter | + in | + inside | + here | + begin | + exit | + end | + done | + leave | + completed | + out | + return | + [\.\!:\s]* +)}; + +sub edit_distance_min { + my (@arr) = @_; + my $len = scalar @arr; + if ((scalar @arr) < 1) { + # if underflow, return + return; + } + my $min = $arr[0]; + for my $i (0 .. ($len-1)) { + if ($arr[$i] < $min) { + $min = $arr[$i]; + } + } + return $min; +} + +sub get_edit_distance { + my ($str1, $str2) = @_; + $str1 = lc($str1); + $str2 = lc($str2); + $str1 =~ s/-//g; + $str2 =~ s/-//g; + my $len1 = length($str1); + my $len2 = length($str2); + # two dimensional array storing minimum edit distance + my @distance; + for my $i (0 .. $len1) { + for my $j (0 .. $len2) { + if ($i == 0) { + $distance[$i][$j] = $j; + } elsif ($j == 0) { + $distance[$i][$j] = $i; + } elsif (substr($str1, $i-1, 1) eq substr($str2, $j-1, 1)) { + $distance[$i][$j] = $distance[$i - 1][$j - 1]; + } else { + my $dist1 = $distance[$i][$j - 1]; #insert distance + my $dist2 = $distance[$i - 1][$j]; # remove + my $dist3 = $distance[$i - 1][$j - 1]; #replace + $distance[$i][$j] = 1 + edit_distance_min($dist1, $dist2, $dist3); + } + } + } + return $distance[$len1][$len2]; +} + +sub find_standard_signature { + my ($sign_off) = @_; + my @standard_signature_tags = ( + 'Signed-off-by:', 'Co-developed-by:', 'Acked-by:', 'Tested-by:', + 'Reviewed-by:', 'Reported-by:', 'Suggested-by:' + ); + foreach my $signature (@standard_signature_tags) { + return $signature if (get_edit_distance($sign_off, $signature) <= 2); + } + + return ""; +} + our @typeListMisordered = ( qr{char\s+(?:un)?signed}, qr{int\s+(?:(?:un)?signed\s+)?short\s}, @@ -588,6 +677,8 @@ our @mode_permission_funcs = ( ["__ATTR", 2], ); +my $word_pattern = '\b[A-Z]?[a-z]{2,}\b'; + #Create a search pattern for all these functions to speed up a loop below our $mode_perms_search = ""; foreach my $entry (@mode_permission_funcs) { @@ -756,7 +847,7 @@ sub read_words { next; } - $$wordsRef .= '|' if ($$wordsRef ne ""); + $$wordsRef .= '|' if (defined $$wordsRef); $$wordsRef .= $line; } close($file); @@ -766,16 +857,18 @@ sub read_words { return 0; } -my $const_structs = ""; -read_words(\$const_structs, $conststructsfile) - or warn "No structs that should be const will be found - file '$conststructsfile': $!\n"; +my $const_structs; +if (show_type("CONST_STRUCT")) { + read_words(\$const_structs, $conststructsfile) + or warn "No structs that should be const will be found - file '$conststructsfile': $!\n"; +} -my $typeOtherTypedefs = ""; -if (length($typedefsfile)) { +if (defined($typedefsfile)) { + my $typeOtherTypedefs; read_words(\$typeOtherTypedefs, $typedefsfile) or warn "No additional types will be considered - file '$typedefsfile': $!\n"; + $typeTypedefs .= '|' . $typeOtherTypedefs if (defined $typeOtherTypedefs); } -$typeTypedefs .= '|' . $typeOtherTypedefs if ($typeOtherTypedefs ne ""); sub build_types { my $mods = "(?x: \n" . join("|\n ", (@modifierList, @modifierListFile)) . "\n)"; @@ -840,10 +933,16 @@ our $FuncArg = qr{$Typecast{0,1}($LvalOrFunc|$Constant|$String)}; our $declaration_macros = qr{(?x: (?:$Storage\s+)?(?:[A-Z_][A-Z0-9]*_){0,2}(?:DEFINE|DECLARE)(?:_[A-Z0-9]+){1,6}\s*\(| (?:$Storage\s+)?[HLP]?LIST_HEAD\s*\(| - (?:$Storage\s+)?${Type}\s+uninitialized_var\s*\(| (?:SKCIPHER_REQUEST|SHASH_DESC|AHASH_REQUEST)_ON_STACK\s*\( )}; +our %allow_repeated_words = ( + add => '', + added => '', + bad => '', + be => '', +); + sub deparenthesize { my ($string) = @_; return "" if (!defined($string)); @@ -901,7 +1000,7 @@ sub is_maintained_obsolete { sub is_SPDX_License_valid { my ($license) = @_; - return 1 if (!$tree || which("python") eq "" || !(-e "$root/scripts/spdxcheck.py") || !(-e "$root/.git")); + return 1 if (!$tree || which("python") eq "" || !(-e "$root/scripts/spdxcheck.py") || !(-e "$gitroot")); my $root_path = abs_path($root); my $status = `cd "$root_path"; echo "$license" | python scripts/spdxcheck.py -`; @@ -919,7 +1018,7 @@ sub seed_camelcase_includes { $camelcase_seeded = 1; - if (-e ".git") { + if (-e "$gitroot") { my $git_last_include_commit = `${git_command} log --no-merges --pretty=format:"%h%n" -1 -- include`; chomp $git_last_include_commit; $camelcase_cache = ".checkpatch-camelcase.git.$git_last_include_commit"; @@ -947,7 +1046,7 @@ sub seed_camelcase_includes { return; } - if (-e ".git") { + if (-e "$gitroot") { $files = `${git_command} ls-files "include/*.h"`; @include_files = split('\n', $files); } @@ -967,10 +1066,20 @@ sub seed_camelcase_includes { } } +sub git_is_single_file { + my ($filename) = @_; + + return 0 if ((which("git") eq "") || !(-e "$gitroot")); + + my $output = `${git_command} ls-files -- $filename 2>/dev/null`; + my $count = $output =~ tr/\n//; + return $count eq 1 && $output =~ m{^${filename}$}; +} + sub git_commit_info { my ($commit, $id, $desc) = @_; - return ($id, $desc) if ((which("git") eq "") || !(-e ".git")); + return ($id, $desc) if ((which("git") eq "") || !(-e "$gitroot")); my $output = `${git_command} log --no-color --format='%H %s' -1 $commit 2>&1`; $output =~ s/^\s*//gm; @@ -1009,7 +1118,7 @@ my $fixlinenr = -1; # If input is git commits, extract all commits from the commit expressions. # For example, HEAD-3 means we need check 'HEAD, HEAD~1, HEAD~2'. -die "$P: No git repository found\n" if ($git && !-e ".git"); +die "$P: No git repository found\n" if ($git && !-e "$gitroot"); if ($git) { my @commits = (); @@ -1040,6 +1149,9 @@ my $vname; $allow_c99_comments = !defined $ignore_type{"C99_COMMENT_TOLERANCE"}; for my $filename (@ARGV) { my $FILE; + my $is_git_file = git_is_single_file($filename); + my $oldfile = $file; + $file = 1 if ($is_git_file); if ($git) { open($FILE, '-|', "git format-patch -M --stdout -1 $filename") || die "$P: $filename: git format-patch failed - $!\n"; @@ -1084,6 +1196,7 @@ for my $filename (@ARGV) { @modifierListFile = (); @typeListFile = (); build_types(); + $file = $oldfile if ($is_git_file); } if (!$quiet) { @@ -1129,6 +1242,7 @@ sub parse_email { my ($formatted_email) = @_; my $name = ""; + my $quoted = ""; my $name_comment = ""; my $address = ""; my $comment = ""; @@ -1160,14 +1274,20 @@ sub parse_email { } } - $name = trim($name); - $name =~ s/^\"|\"$//g; - $name =~ s/(\s*\([^\)]+\))\s*//; - if (defined($1)) { - $name_comment = trim($1); + # Extract comments from names excluding quoted parts + # "John D. (Doe)" - Do not extract + if ($name =~ s/\"(.+)\"//) { + $quoted = $1; } + while ($name =~ s/\s*($balanced_parens)\s*/ /) { + $name_comment .= trim($1); + } + $name =~ s/^[ \"]+|[ \"]+$//g; + $name = trim("$quoted $name"); + $address = trim($address); $address =~ s/^\<|\>$//g; + $comment = trim($comment); if ($name =~ /[^\w \-]/i) { ##has "must quote" chars $name =~ s/(?<!\\)"/\\"/g; ##escape quotes @@ -1178,25 +1298,30 @@ sub parse_email { } sub format_email { - my ($name, $address) = @_; + my ($name, $name_comment, $address, $comment) = @_; my $formatted_email; - $name = trim($name); - $name =~ s/^\"|\"$//g; + $name =~ s/^[ \"]+|[ \"]+$//g; $address = trim($address); + $address =~ s/(?:\.|\,|\")+$//; ##trailing commas, dots or quotes if ($name =~ /[^\w \-]/i) { ##has "must quote" chars $name =~ s/(?<!\\)"/\\"/g; ##escape quotes $name = "\"$name\""; } + $name_comment = trim($name_comment); + $name_comment = " $name_comment" if ($name_comment ne ""); + $comment = trim($comment); + $comment = " $comment" if ($comment ne ""); + if ("$name" eq "") { $formatted_email = "$address"; } else { - $formatted_email = "$name <$address>"; + $formatted_email = "$name$name_comment <$address>"; } - + $formatted_email .= "$comment"; return $formatted_email; } @@ -1204,7 +1329,7 @@ sub reformat_email { my ($email) = @_; my ($email_name, $name_comment, $email_address, $comment) = parse_email($email); - return format_email($email_name, $email_address); + return format_email($email_name, $name_comment, $email_address, $comment); } sub same_email_addresses { @@ -1214,7 +1339,9 @@ sub same_email_addresses { my ($email2_name, $name2_comment, $email2_address, $comment2) = parse_email($email2); return $email1_name eq $email2_name && - $email1_address eq $email2_address; + $email1_address eq $email2_address && + $name1_comment eq $name2_comment && + $comment1 eq $comment2; } sub which { @@ -2326,6 +2453,15 @@ sub get_raw_comment { return $comment; } +sub exclude_global_initialisers { + my ($realfile) = @_; + + # Do not check for BPF programs (tools/testing/selftests/bpf/progs/*.c, samples/bpf/*_kern.c, *.bpf.c). + return $realfile =~ m@^tools/testing/selftests/bpf/progs/.*\.c$@ || + $realfile =~ m@^samples/bpf/.*_kern\.c$@ || + $realfile =~ m@/bpf/.*\.bpf\.c$@; +} + sub process { my $filename = shift; @@ -2344,6 +2480,7 @@ sub process { my $signoff = 0; my $author = ''; my $authorsignoff = 0; + my $author_sob = ''; my $is_patch = 0; my $is_binding_patch = -1; my $in_header_lines = $file ? 0 : 1; @@ -2407,7 +2544,7 @@ sub process { if ($rawline=~/^\+\+\+\s+(\S+)/) { $setup_docs = 0; - if ($1 =~ m@Documentation/admin-guide/kernel-parameters.rst$@) { + if ($1 =~ m@Documentation/admin-guide/kernel-parameters.txt$@) { $setup_docs = 1; } #next; @@ -2636,8 +2773,8 @@ sub process { # Check if the commit log has what seems like a diff which can confuse patch if ($in_commit_log && !$commit_log_has_diff && - (($line =~ m@^\s+diff\b.*a/[\w/]+@ && - $line =~ m@^\s+diff\b.*a/([\w/]+)\s+b/$1\b@) || + (($line =~ m@^\s+diff\b.*a/([\w/]+)@ && + $line =~ m@^\s+diff\b.*a/[\w/]+\s+b/$1\b@) || $line =~ m@^\s*(?:\-\-\-\s+a/|\+\+\+\s+b/)@ || $line =~ m/^\s*\@\@ \-\d+,\d+ \+\d+,\d+ \@\@/)) { ERROR("DIFF_IN_COMMIT_MSG", @@ -2658,6 +2795,10 @@ sub process { # Check the patch for a From: if (decode("MIME-Header", $line) =~ /^From:\s*(.*)/) { $author = $1; + my $curline = $linenr; + while(defined($rawlines[$curline]) && ($rawlines[$curline++] =~ /^[ \t]\s*(.*)/)) { + $author .= $1; + } $author = encode("utf8", $author) if ($line =~ /=\?utf-8\?/i); $author =~ s/"//g; $author = reformat_email($author); @@ -2667,9 +2808,37 @@ sub process { if ($line =~ /^\s*signed-off-by:\s*(.*)/i) { $signoff++; $in_commit_log = 0; - if ($author ne '') { + if ($author ne '' && $authorsignoff != 1) { if (same_email_addresses($1, $author)) { $authorsignoff = 1; + } else { + my $ctx = $1; + my ($email_name, $email_comment, $email_address, $comment1) = parse_email($ctx); + my ($author_name, $author_comment, $author_address, $comment2) = parse_email($author); + + if ($email_address eq $author_address && $email_name eq $author_name) { + $author_sob = $ctx; + $authorsignoff = 2; + } elsif ($email_address eq $author_address) { + $author_sob = $ctx; + $authorsignoff = 3; + } elsif ($email_name eq $author_name) { + $author_sob = $ctx; + $authorsignoff = 4; + + my $address1 = $email_address; + my $address2 = $author_address; + + if ($address1 =~ /(\S+)\+\S+(\@.*)/) { + $address1 = "$1$2"; + } + if ($address2 =~ /(\S+)\+\S+(\@.*)/) { + $address2 = "$1$2"; + } + if ($address1 eq $address2) { + $authorsignoff = 5; + } + } } } } @@ -2696,8 +2865,17 @@ sub process { my $ucfirst_sign_off = ucfirst(lc($sign_off)); if ($sign_off !~ /$signature_tags/) { - WARN("BAD_SIGN_OFF", - "Non-standard signature: $sign_off\n" . $herecurr); + my $suggested_signature = find_standard_signature($sign_off); + if ($suggested_signature eq "") { + WARN("BAD_SIGN_OFF", + "Non-standard signature: $sign_off\n" . $herecurr); + } else { + if (WARN("BAD_SIGN_OFF", + "Non-standard signature: '$sign_off' - perhaps '$suggested_signature'?\n" . $herecurr) && + $fix) { + $fixed[$fixlinenr] =~ s/$sign_off/$suggested_signature/; + } + } } if (defined $space_before && $space_before ne "") { if (WARN("BAD_SIGN_OFF", @@ -2726,7 +2904,7 @@ sub process { } my ($email_name, $name_comment, $email_address, $comment) = parse_email($email); - my $suggested_email = format_email(($email_name, $email_address)); + my $suggested_email = format_email(($email_name, $name_comment, $email_address, $comment)); if ($suggested_email eq "") { ERROR("BAD_SIGN_OFF", "Unrecognized email address: '$email'\n" . $herecurr); @@ -2737,8 +2915,76 @@ sub process { # Don't force email to have quotes # Allow just an angle bracketed address if (!same_email_addresses($email, $suggested_email)) { + if (WARN("BAD_SIGN_OFF", + "email address '$email' might be better as '$suggested_email'\n" . $herecurr) && + $fix) { + $fixed[$fixlinenr] =~ s/\Q$email\E/$suggested_email/; + } + } + + # Address part shouldn't have comments + my $stripped_address = $email_address; + $stripped_address =~ s/\([^\(\)]*\)//g; + if ($email_address ne $stripped_address) { + if (WARN("BAD_SIGN_OFF", + "address part of email should not have comments: '$email_address'\n" . $herecurr) && + $fix) { + $fixed[$fixlinenr] =~ s/\Q$email_address\E/$stripped_address/; + } + } + + # Only one name comment should be allowed + my $comment_count = () = $name_comment =~ /\([^\)]+\)/g; + if ($comment_count > 1) { WARN("BAD_SIGN_OFF", - "email address '$email' might be better as '$suggested_email$comment'\n" . $herecurr); + "Use a single name comment in email: '$email'\n" . $herecurr); + } + + + # stable@vger.kernel.org or stable@kernel.org shouldn't + # have an email name. In addition comments should strictly + # begin with a # + if ($email =~ /^.*stable\@(?:vger\.)?kernel\.org/i) { + if (($comment ne "" && $comment !~ /^#.+/) || + ($email_name ne "")) { + my $cur_name = $email_name; + my $new_comment = $comment; + $cur_name =~ s/[a-zA-Z\s\-\"]+//g; + + # Remove brackets enclosing comment text + # and # from start of comments to get comment text + $new_comment =~ s/^\((.*)\)$/$1/; + $new_comment =~ s/^\[(.*)\]$/$1/; + $new_comment =~ s/^[\s\#]+|\s+$//g; + + $new_comment = trim("$new_comment $cur_name") if ($cur_name ne $new_comment); + $new_comment = " # $new_comment" if ($new_comment ne ""); + my $new_email = "$email_address$new_comment"; + + if (WARN("BAD_STABLE_ADDRESS_STYLE", + "Invalid email format for stable: '$email', prefer '$new_email'\n" . $herecurr) && + $fix) { + $fixed[$fixlinenr] =~ s/\Q$email\E/$new_email/; + } + } + } elsif ($comment ne "" && $comment !~ /^(?:#.+|\(.+\))$/) { + my $new_comment = $comment; + + # Extract comment text from within brackets or + # c89 style /*...*/ comments + $new_comment =~ s/^\[(.*)\]$/$1/; + $new_comment =~ s/^\/\*(.*)\*\/$/$1/; + + $new_comment = trim($new_comment); + $new_comment =~ s/^[^\w]$//; # Single lettered comment with non word character is usually a typo + $new_comment = "($new_comment)" if ($new_comment ne ""); + my $new_email = format_email($email_name, $name_comment, $email_address, $new_comment); + + if (WARN("BAD_SIGN_OFF", + "Unexpected content after email: '$email', should be: '$new_email'\n" . $herecurr) && + $fix) { + $fixed[$fixlinenr] =~ s/\Q$email\E/$new_email/; + } } } @@ -2761,7 +3007,7 @@ sub process { } if (!defined $lines[$linenr]) { WARN("BAD_SIGN_OFF", - "Co-developed-by: must be immediately followed by Signed-off-by:\n" . "$here\n" . $rawline); + "Co-developed-by: must be immediately followed by Signed-off-by:\n" . "$here\n" . $rawline); } elsif ($rawlines[$linenr] !~ /^\s*signed-off-by:\s*(.*)/i) { WARN("BAD_SIGN_OFF", "Co-developed-by: must be immediately followed by Signed-off-by:\n" . "$here\n" . $rawline . "\n" .$rawlines[$linenr]); @@ -2781,8 +3027,11 @@ sub process { # Check for Gerrit Change-Ids not in any patch context if ($realfile eq '' && !$has_patch_separator && $line =~ /^\s*change-id:/i) { - ERROR("GERRIT_CHANGE_ID", - "Remove Gerrit Change-Id's before submitting upstream\n" . $herecurr); + if (ERROR("GERRIT_CHANGE_ID", + "Remove Gerrit Change-Id's before submitting upstream\n" . $herecurr) && + $fix) { + fix_delete_line($fixlinenr, $rawline); + } } # Check if the commit log is in a possible stack dump @@ -2804,8 +3053,8 @@ sub process { # file delta changes $line =~ /^\s*(?:[\w\.\-]+\/)++[\w\.\-]+:/ || # filename then : - $line =~ /^\s*(?:Fixes:|Link:)/i || - # A Fixes: or Link: line + $line =~ /^\s*(?:Fixes:|Link:|$signature_tags)/i || + # A Fixes: or Link: line or signature tag line $commit_log_possible_stack_dump)) { WARN("COMMIT_LOG_LONG_LINE", "Possible unwrapped commit description (prefer a maximum 75 chars per line)\n" . $herecurr); @@ -2818,6 +3067,15 @@ sub process { $commit_log_possible_stack_dump = 0; } +# Check for lines starting with a # + if ($in_commit_log && $line =~ /^#/) { + if (WARN("COMMIT_COMMENT_SYMBOL", + "Commit log lines starting with '#' are dropped by git as comments\n" . $herecurr) && + $fix) { + $fixed[$fixlinenr] =~ s/^/ /; + } + } + # Check for git id commit length and improperly formed commit descriptions if ($in_commit_log && !$commit_log_possible_stack_dump && $line !~ /^\s*(?:Link|Patchwork|http|https|BugLink|base-commit):/i && @@ -2958,15 +3216,18 @@ sub process { # Check for various typo / spelling mistakes if (defined($misspellings) && ($in_commit_log || $line =~ /^(?:\+|Subject:)/i)) { - while ($rawline =~ /(?:^|[^a-z@])($misspellings)(?:\b|$|[^a-z@])/gi) { + while ($rawline =~ /(?:^|[^\w\-'`])($misspellings)(?:[^\w\-'`]|$)/gi) { my $typo = $1; + my $blank = copy_spacing($rawline); + my $ptr = substr($blank, 0, $-[1]) . "^" x length($typo); + my $hereptr = "$hereline$ptr\n"; my $typo_fix = $spelling_fix{lc($typo)}; $typo_fix = ucfirst($typo_fix) if ($typo =~ /^[A-Z]/); $typo_fix = uc($typo_fix) if ($typo =~ /^[A-Z]+$/); my $msg_level = \&WARN; $msg_level = \&CHK if ($file); if (&{$msg_level}("TYPO_SPELLING", - "'$typo' may be misspelled - perhaps '$typo_fix'?\n" . $herecurr) && + "'$typo' may be misspelled - perhaps '$typo_fix'?\n" . $hereptr) && $fix) { $fixed[$fixlinenr] =~ s/(^|[^A-Za-z@])($typo)($|[^A-Za-z@])/$1$typo_fix$3/; } @@ -2984,6 +3245,60 @@ sub process { } } +# check for repeated words separated by a single space +# avoid false positive from list command eg, '-rw-r--r-- 1 root root' + if (($rawline =~ /^\+/ || $in_commit_log) && + $rawline !~ /[bcCdDlMnpPs\?-][rwxsStT-]{9}/) { + pos($rawline) = 1 if (!$in_commit_log); + while ($rawline =~ /\b($word_pattern) (?=($word_pattern))/g) { + + my $first = $1; + my $second = $2; + my $start_pos = $-[1]; + my $end_pos = $+[2]; + if ($first =~ /(?:struct|union|enum)/) { + pos($rawline) += length($first) + length($second) + 1; + next; + } + + next if (lc($first) ne lc($second)); + next if ($first eq 'long'); + + # check for character before and after the word matches + my $start_char = ''; + my $end_char = ''; + $start_char = substr($rawline, $start_pos - 1, 1) if ($start_pos > ($in_commit_log ? 0 : 1)); + $end_char = substr($rawline, $end_pos, 1) if ($end_pos < length($rawline)); + + next if ($start_char =~ /^\S$/); + next if (index(" \t.,;?!", $end_char) == -1); + + # avoid repeating hex occurrences like 'ff ff fe 09 ...' + if ($first =~ /\b[0-9a-f]{2,}\b/i) { + next if (!exists($allow_repeated_words{lc($first)})); + } + + if (WARN("REPEATED_WORD", + "Possible repeated word: '$first'\n" . $herecurr) && + $fix) { + $fixed[$fixlinenr] =~ s/\b$first $second\b/$first/; + } + } + + # if it's a repeated word on consecutive lines in a comment block + if ($prevline =~ /$;+\s*$/ && + $prevrawline =~ /($word_pattern)\s*$/) { + my $last_word = $1; + if ($rawline =~ /^\+\s*\*\s*$last_word /) { + if (WARN("REPEATED_WORD", + "Possible repeated word: '$last_word'\n" . $hereprev) && + $fix) { + $fixed[$fixlinenr] =~ s/(\+\s*\*\s*)$last_word /$1/; + } + } + } + } + # ignore non-hunk lines and lines being removed next if (!$hunk_line || $line =~ /^-/); @@ -3042,11 +3357,7 @@ sub process { if ($lines[$ln - 1] =~ /^\+\s*(?:bool|tristate|prompt)\s*["']/) { $is_start = 1; - } elsif ($lines[$ln - 1] =~ /^\+\s*(?:help|---help---)\s*$/) { - if ($lines[$ln - 1] =~ "---help---") { - WARN("CONFIG_DESCRIPTION", - "prefer 'help' over '---help---' for new help texts\n" . $herecurr); - } + } elsif ($lines[$ln - 1] =~ /^\+\s*(?:---)?help(?:---)?$/) { $length = -1; } @@ -3113,13 +3424,6 @@ sub process { } } -# discourage the use of boolean for type definition attributes of Kconfig options - if ($realfile =~ /Kconfig/ && - $line =~ /^\+\s*\bboolean\b/) { - WARN("CONFIG_TYPE_BOOLEAN", - "Use of boolean is deprecated, please use bool instead.\n" . $herecurr); - } - if (($realfile =~ /Makefile.*/ || $realfile =~ /Kbuild.*/) && ($line =~ /\+(EXTRA_[A-Z]+FLAGS).*/)) { my $flag = $1; @@ -3214,6 +3518,12 @@ sub process { } } +# check for embedded filenames + if ($rawline =~ /^\+.*\Q$realfile\E/) { + WARN("EMBEDDED_FILENAME", + "It's generally not useful to have the filename in the file\n" . $herecurr); + } + # check we are in a valid source file if not then ignore this hunk next if ($realfile !~ /\.(h|c|s|S|sh|dtsi|dts)$/); @@ -3291,8 +3601,18 @@ sub process { # check for adding lines without a newline. if ($line =~ /^\+/ && defined $lines[$linenr] && $lines[$linenr] =~ /^\\ No newline at end of file/) { - WARN("MISSING_EOF_NEWLINE", - "adding a line without newline at end of file\n" . $herecurr); + if (WARN("MISSING_EOF_NEWLINE", + "adding a line without newline at end of file\n" . $herecurr) && + $fix) { + fix_delete_line($fixlinenr+1, "No newline at end of file"); + } + } + +# check for .L prefix local symbols in .S files + if ($realfile =~ /\.S$/ && + $line =~ /^\+\s*(?:[A-Z]+_)?SYM_[A-Z]+_(?:START|END)(?:_[A-Z_]+)?\s*\(\s*\.L/) { + WARN("AVOID_L_PREFIX", + "Avoid using '.L' prefixed local symbol names for denoting a range of code via 'SYM_*_START/END' annotations; see Documentation/asm-annotations.rst\n" . $herecurr); } # check we are in a valid source file C or perl if not then ignore this hunk @@ -3326,14 +3646,28 @@ sub process { # check for assignments on the start of a line if ($sline =~ /^\+\s+($Assignment)[^=]/) { - CHK("ASSIGNMENT_CONTINUATIONS", - "Assignment operator '$1' should be on the previous line\n" . $hereprev); + my $operator = $1; + if (CHK("ASSIGNMENT_CONTINUATIONS", + "Assignment operator '$1' should be on the previous line\n" . $hereprev) && + $fix && $prevrawline =~ /^\+/) { + # add assignment operator to the previous line, remove from current line + $fixed[$fixlinenr - 1] .= " $operator"; + $fixed[$fixlinenr] =~ s/\Q$operator\E\s*//; + } } # check for && or || at the start of a line if ($rawline =~ /^\+\s*(&&|\|\|)/) { - CHK("LOGICAL_CONTINUATIONS", - "Logical continuations should be on the previous line\n" . $hereprev); + my $operator = $1; + if (CHK("LOGICAL_CONTINUATIONS", + "Logical continuations should be on the previous line\n" . $hereprev) && + $fix && $prevrawline =~ /^\+/) { + # insert logical operator at last non-comment, non-whitepsace char on previous line + $prevline =~ /[\s$;]*$/; + my $line_end = substr($prevrawline, $-[0]); + $fixed[$fixlinenr - 1] =~ s/\Q$line_end\E$/ $operator$line_end/; + $fixed[$fixlinenr] =~ s/\Q$operator\E\s*//; + } } # check indentation starts on a tab stop @@ -3401,7 +3735,7 @@ sub process { if ($realfile =~ m@^(drivers/net/|net/)@ && $prevrawline =~ /^\+[ \t]*\/\*[ \t]*$/ && $rawline =~ /^\+[ \t]*\*/ && - $realline > 2) { + $realline > 3) { # Do not warn about the initial copyright comment block after SPDX-License-Identifier WARN("NETWORKING_BLOCK_COMMENT_STYLE", "networking block comments don't use an empty /* line, use /* Comment...\n" . $hereprev); } @@ -3483,43 +3817,48 @@ sub process { } # check for missing blank lines after declarations - if ($sline =~ /^\+\s+\S/ && #Not at char 1 - # actual declarations - ($prevline =~ /^\+\s+$Declare\s*$Ident\s*[=,;:\[]/ || +# (declarations must have the same indentation and not be at the start of line) + if (($prevline =~ /\+(\s+)\S/) && $sline =~ /^\+$1\S/) { + # use temporaries + my $sl = $sline; + my $pl = $prevline; + # remove $Attribute/$Sparse uses to simplify comparisons + $sl =~ s/\b(?:$Attribute|$Sparse)\b//g; + $pl =~ s/\b(?:$Attribute|$Sparse)\b//g; + if (($pl =~ /^\+\s+$Declare\s*$Ident\s*[=,;:\[]/ || # function pointer declarations - $prevline =~ /^\+\s+$Declare\s*\(\s*\*\s*$Ident\s*\)\s*[=,;:\[\(]/ || + $pl =~ /^\+\s+$Declare\s*\(\s*\*\s*$Ident\s*\)\s*[=,;:\[\(]/ || # foo bar; where foo is some local typedef or #define - $prevline =~ /^\+\s+$Ident(?:\s+|\s*\*\s*)$Ident\s*[=,;\[]/ || + $pl =~ /^\+\s+$Ident(?:\s+|\s*\*\s*)$Ident\s*[=,;\[]/ || # known declaration macros - $prevline =~ /^\+\s+$declaration_macros/) && + $pl =~ /^\+\s+$declaration_macros/) && # for "else if" which can look like "$Ident $Ident" - !($prevline =~ /^\+\s+$c90_Keywords\b/ || + !($pl =~ /^\+\s+$c90_Keywords\b/ || # other possible extensions of declaration lines - $prevline =~ /(?:$Compare|$Assignment|$Operators)\s*$/ || + $pl =~ /(?:$Compare|$Assignment|$Operators)\s*$/ || # not starting a section or a macro "\" extended line - $prevline =~ /(?:\{\s*|\\)$/) && + $pl =~ /(?:\{\s*|\\)$/) && # looks like a declaration - !($sline =~ /^\+\s+$Declare\s*$Ident\s*[=,;:\[]/ || + !($sl =~ /^\+\s+$Declare\s*$Ident\s*[=,;:\[]/ || # function pointer declarations - $sline =~ /^\+\s+$Declare\s*\(\s*\*\s*$Ident\s*\)\s*[=,;:\[\(]/ || + $sl =~ /^\+\s+$Declare\s*\(\s*\*\s*$Ident\s*\)\s*[=,;:\[\(]/ || # foo bar; where foo is some local typedef or #define - $sline =~ /^\+\s+$Ident(?:\s+|\s*\*\s*)$Ident\s*[=,;\[]/ || + $sl =~ /^\+\s+$Ident(?:\s+|\s*\*\s*)$Ident\s*[=,;\[]/ || # known declaration macros - $sline =~ /^\+\s+$declaration_macros/ || + $sl =~ /^\+\s+$declaration_macros/ || # start of struct or union or enum - $sline =~ /^\+\s+(?:static\s+)?(?:const\s+)?(?:union|struct|enum|typedef)\b/ || + $sl =~ /^\+\s+(?:static\s+)?(?:const\s+)?(?:union|struct|enum|typedef)\b/ || # start or end of block or continuation of declaration - $sline =~ /^\+\s+(?:$|[\{\}\.\#\"\?\:\(\[])/ || + $sl =~ /^\+\s+(?:$|[\{\}\.\#\"\?\:\(\[])/ || # bitfield continuation - $sline =~ /^\+\s+$Ident\s*:\s*\d+\s*[,;]/ || + $sl =~ /^\+\s+$Ident\s*:\s*\d+\s*[,;]/ || # other possible extensions of declaration lines - $sline =~ /^\+\s+\(?\s*(?:$Compare|$Assignment|$Operators)/) && - # indentation of previous and current line are the same - (($prevline =~ /\+(\s+)\S/) && $sline =~ /^\+$1\S/)) { - if (WARN("LINE_SPACING", - "Missing a blank line after declarations\n" . $hereprev) && - $fix) { - fix_insert_line($fixlinenr, "\+"); + $sl =~ /^\+\s+\(?\s*(?:$Compare|$Assignment|$Operators)/)) { + if (WARN("LINE_SPACING", + "Missing a blank line after declarations\n" . $hereprev) && + $fix) { + fix_insert_line($fixlinenr, "\+"); + } } } @@ -3572,12 +3911,16 @@ sub process { } # check indentation of a line with a break; -# if the previous line is a goto or return and is indented the same # of tabs +# if the previous line is a goto, return or break +# and is indented the same # of tabs if ($sline =~ /^\+([\t]+)break\s*;\s*$/) { my $tabs = $1; - if ($prevline =~ /^\+$tabs(?:goto|return)\b/) { - WARN("UNNECESSARY_BREAK", - "break is not useful after a goto or return\n" . $hereprev); + if ($prevline =~ /^\+$tabs(goto|return|break)\b/) { + if (WARN("UNNECESSARY_BREAK", + "break is not useful after a $1\n" . $hereprev) && + $fix) { + fix_delete_line($fixlinenr, $rawline); + } } } @@ -3860,6 +4203,17 @@ sub process { #ignore lines not being added next if ($line =~ /^[^\+]/); +# check for self assignments used to avoid compiler warnings +# e.g.: int foo = foo, *bar = NULL; +# struct foo bar = *(&(bar)); + if ($line =~ /^\+\s*(?:$Declare)?([A-Za-z_][A-Za-z\d_]*)\s*=/) { + my $var = $1; + if ($line =~ /^\+\s*(?:$Declare)?$var\s*=\s*(?:$var|\*\s*\(?\s*&\s*\(?\s*$var\s*\)?\s*\)?)\s*[;,]/) { + WARN("SELF_ASSIGNMENT", + "Do not use self-assignments to avoid compiler warnings\n" . $herecurr); + } + } + # check for dereferences that span multiple lines if ($prevline =~ /^\+.*$Lval\s*(?:\.|->)\s*$/ && $line =~ /^\+\s*(?!\#\s*(?!define\s+|if))\s*$Lval/) { @@ -3975,8 +4329,7 @@ sub process { if (defined $realline_next && exists $lines[$realline_next - 1] && !defined $suppress_export{$realline_next} && - ($lines[$realline_next - 1] =~ /EXPORT_SYMBOL.*\((.*)\)/ || - $lines[$realline_next - 1] =~ /EXPORT_UNUSED_SYMBOL.*\((.*)\)/)) { + ($lines[$realline_next - 1] =~ /EXPORT_SYMBOL.*\((.*)\)/)) { # Handle definitions which produce identifiers with # a prefix: # XXX(foo); @@ -4003,8 +4356,7 @@ sub process { } if (!defined $suppress_export{$linenr} && $prevline =~ /^.\s*$/ && - ($line =~ /EXPORT_SYMBOL.*\((.*)\)/ || - $line =~ /EXPORT_UNUSED_SYMBOL.*\((.*)\)/)) { + ($line =~ /EXPORT_SYMBOL.*\((.*)\)/)) { #print "FOO B <$lines[$linenr - 1]>\n"; $suppress_export{$linenr} = 2; } @@ -4015,7 +4367,8 @@ sub process { } # check for global initialisers. - if ($line =~ /^\+$Type\s*$Ident(?:\s+$Modifier)*\s*=\s*($zero_initializer)\s*;/) { + if ($line =~ /^\+$Type\s*$Ident(?:\s+$Modifier)*\s*=\s*($zero_initializer)\s*;/ && + !exclude_global_initialisers($realfile)) { if (ERROR("GLOBAL_INITIALISERS", "do not initialise globals to $1\n" . $herecurr) && $fix) { @@ -4094,12 +4447,24 @@ sub process { } } +# check for const static or static <non ptr type> const declarations +# prefer 'static const <foo>' over 'const static <foo>' and 'static <foo> const' + if ($sline =~ /^\+\s*const\s+static\s+($Type)\b/ || + $sline =~ /^\+\s*static\s+($BasicType)\s+const\b/) { + if (WARN("STATIC_CONST", + "Move const after static - use 'static const $1'\n" . $herecurr) && + $fix) { + $fixed[$fixlinenr] =~ s/\bconst\s+static\b/static const/; + $fixed[$fixlinenr] =~ s/\bstatic\s+($BasicType)\s+const\b/static const $1/; + } + } + # check for non-global char *foo[] = {"bar", ...} declarations. if ($line =~ /^.\s+(?:static\s+|const\s+)?char\s+\*\s*\w+\s*\[\s*\]\s*=\s*\{/) { WARN("STATIC_CONST_CHAR_ARRAY", "char * array declaration might be better as static const\n" . $herecurr); - } + } # check for sizeof(foo)/sizeof(foo[0]) that could be ARRAY_SIZE(foo) if ($line =~ m@\bsizeof\s*\(\s*($Lval)\s*\)@) { @@ -4216,16 +4581,23 @@ sub process { "printk() should include KERN_<LEVEL> facility level\n" . $herecurr); } - if ($line =~ /\bprintk\s*\(\s*KERN_([A-Z]+)/) { - my $orig = $1; +# prefer variants of (subsystem|netdev|dev|pr)_<level> to printk(KERN_<LEVEL> + if ($line =~ /\b(printk(_once|_ratelimited)?)\s*\(\s*KERN_([A-Z]+)/) { + my $printk = $1; + my $modifier = $2; + my $orig = $3; + $modifier = "" if (!defined($modifier)); my $level = lc($orig); $level = "warn" if ($level eq "warning"); my $level2 = $level; $level2 = "dbg" if ($level eq "debug"); + $level .= $modifier; + $level2 .= $modifier; WARN("PREFER_PR_LEVEL", - "Prefer [subsystem eg: netdev]_$level2([subsystem]dev, ... then dev_$level2(dev, ... then pr_$level(... to printk(KERN_$orig ...\n" . $herecurr); + "Prefer [subsystem eg: netdev]_$level2([subsystem]dev, ... then dev_$level2(dev, ... then pr_$level(... to $printk(KERN_$orig ...\n" . $herecurr); } +# prefer dev_<level> to dev_printk(KERN_<LEVEL> if ($line =~ /\bdev_printk\s*\(\s*KERN_([A-Z]+)/) { my $orig = $1; my $level = lc($orig); @@ -4235,6 +4607,12 @@ sub process { "Prefer dev_$level(... to dev_printk(KERN_$orig, ...\n" . $herecurr); } +# trace_printk should not be used in production code. + if ($line =~ /\b(trace_printk|trace_puts|ftrace_vprintk)\s*\(/) { + WARN("TRACE_PRINTK", + "Do not use $1() in production code (this can be ignored if built only with a debug config option)\n" . $herecurr); + } + # ENOSYS means "bad syscall nr" and nothing else. This will have a small # number of false positives, but assembly files are not checked, so at # least the arch entry code will not trigger this warning. @@ -4265,7 +4643,7 @@ sub process { $fix) { fix_delete_line($fixlinenr, $rawline); my $fixed_line = $rawline; - $fixed_line =~ /(^..*$Type\s*$Ident\(.*\)\s*){(.*)$/; + $fixed_line =~ /(^..*$Type\s*$Ident\(.*\)\s*)\{(.*)$/; my $line1 = $1; my $line2 = $2; fix_insert_line($fixlinenr, ltrim($line1)); @@ -4676,7 +5054,7 @@ sub process { # A colon needs no spaces before when it is # terminating a case value or a label. } elsif ($opv eq ':C' || $opv eq ':L') { - if ($ctx =~ /Wx./) { + if ($ctx =~ /Wx./ and $realfile !~ m@.*\.lds\.h$@) { if (ERROR("SPACING", "space prohibited before that '$op' $at\n" . $hereptr)) { $good = rtrim($fix_elements[$n]) . trim($fix_elements[$n + 1]); @@ -4760,7 +5138,7 @@ sub process { ## $line !~ /^.\s*$Type\s+$Ident(?:\s*=[^,{]*)?\s*,\s*$Type\s*$Ident.*/) { ## ## # Remove any bracketed sections to ensure we do not -## # falsly report the parameters of functions. +## # falsely report the parameters of functions. ## my $ln = $line; ## while ($ln =~ s/\([^\(\)]*\)//g) { ## } @@ -4901,6 +5279,17 @@ sub process { } } +# check if a statement with a comma should be two statements like: +# foo = bar(), /* comma should be semicolon */ +# bar = baz(); + if (defined($stat) && + $stat =~ /^\+\s*(?:$Lval\s*$Assignment\s*)?$FuncArg\s*,\s*(?:$Lval\s*$Assignment\s*)?$FuncArg\s*;\s*$/) { + my $cnt = statement_rawlines($stat); + my $herectx = get_stat_here($linenr, $cnt, $here); + WARN("SUSPECT_COMMA_SEMICOLON", + "Possible comma where semicolon could be used\n" . $herectx); + } + # return is not a function if (defined($stat) && $stat =~ /^.\s*return(\s*)\(/s) { my $spacing = $1; @@ -4928,7 +5317,7 @@ sub process { $lines[$linenr - 3] !~ /^[ +]\s*$Ident\s*:/) { WARN("RETURN_VOID", "void function return statements are not generally useful\n" . $hereprev); - } + } # if statements using unnecessary parentheses - ie: if ((foo == bar)) if ($perl_version_ok && @@ -5021,8 +5410,30 @@ sub process { my ($s, $c) = ($stat, $cond); if ($c =~ /\bif\s*\(.*[^<>!=]=[^=].*/s) { - ERROR("ASSIGN_IN_IF", - "do not use assignment in if condition\n" . $herecurr); + if (ERROR("ASSIGN_IN_IF", + "do not use assignment in if condition\n" . $herecurr) && + $fix && $perl_version_ok) { + if ($rawline =~ /^\+(\s+)if\s*\(\s*(\!)?\s*\(\s*(($Lval)\s*=\s*$LvalOrFunc)\s*\)\s*(?:($Compare)\s*($FuncArg))?\s*\)\s*(\{)?\s*$/) { + my $space = $1; + my $not = $2; + my $statement = $3; + my $assigned = $4; + my $test = $8; + my $against = $9; + my $brace = $15; + fix_delete_line($fixlinenr, $rawline); + fix_insert_line($fixlinenr, "$space$statement;"); + my $newline = "${space}if ("; + $newline .= '!' if defined($not); + $newline .= '(' if (defined $not && defined($test) && defined($against)); + $newline .= "$assigned"; + $newline .= " $test $against" if (defined($test) && defined($against)); + $newline .= ')' if (defined $not && defined($test) && defined($against)); + $newline .= ')'; + $newline .= " {" if (defined($brace)); + fix_insert_line($fixlinenr + 1, $newline); + } + } } # Find out what is on the end of the line after the @@ -5143,6 +5554,8 @@ sub process { #CamelCase if ($var !~ /^$Constant$/ && $var =~ /[A-Z][a-z]|[a-z][A-Z]/ && +#Ignore some autogenerated defines and enum values + $var !~ /^(?:[A-Z]+_){1,5}[A-Z]{1,3}[a-z]/ && #Ignore Page<foo> variants $var !~ /^(?:Clear|Set|TestClear|TestSet|)Page[A-Z]/ && #ODP ignores @@ -5248,9 +5661,9 @@ sub process { $dstat =~ s/\s*$//s; # Flatten any parentheses and braces - while ($dstat =~ s/\([^\(\)]*\)/1/ || - $dstat =~ s/\{[^\{\}]*\}/1/ || - $dstat =~ s/.\[[^\[\]]*\]/1/) + while ($dstat =~ s/\([^\(\)]*\)/1u/ || + $dstat =~ s/\{[^\{\}]*\}/1u/ || + $dstat =~ s/.\[[^\[\]]*\]/1u/) { } @@ -5291,6 +5704,7 @@ sub process { $dstat !~ /^\.$Ident\s*=/ && # .foo = $dstat !~ /^(?:\#\s*$Ident|\#\s*$Constant)\s*$/ && # stringification #foo $dstat !~ /^do\s*$Constant\s*while\s*$Constant;?$/ && # do {...} while (...); // do {...} while (...) + $dstat !~ /^while\s*$Constant\s*$Constant\s*$/ && # while (...) {...} $dstat !~ /^for\s*$Constant$/ && # for (...) $dstat !~ /^for\s*$Constant\s+(?:$Ident|-?$Constant)$/ && # for (...) bar() $dstat !~ /^do\s*{/ && # do {... @@ -5609,6 +6023,17 @@ sub process { "Prefer using '\"%s...\", __func__' to using '$context_function', this function's name, in a string\n" . $herecurr); } +# check for unnecessary function tracing like uses +# This does not use $logFunctions because there are many instances like +# 'dprintk(FOO, "%s()\n", __func__);' which do not match $logFunctions + if ($rawline =~ /^\+.*\([^"]*"$tracing_logging_tags{0,3}%s(?:\s*\(\s*\)\s*)?$tracing_logging_tags{0,3}(?:\\n)?"\s*,\s*__func__\s*\)\s*;/) { + if (WARN("TRACING_LOGGING", + "Unnecessary ftrace-like logging - prefer using ftrace\n" . $herecurr) && + $fix) { + fix_delete_line($fixlinenr, $rawline); + } + } + # check for spaces before a quoted newline if ($rawline =~ /^.*\".*\s\\n/) { if (WARN("QUOTED_WHITESPACE_BEFORE_NEWLINE", @@ -5755,6 +6180,28 @@ sub process { "Avoid logging continuation uses where feasible\n" . $herecurr); } +# check for unnecessary use of %h[xudi] and %hh[xudi] in logging functions + if (defined $stat && + $line =~ /\b$logFunctions\s*\(/ && + index($stat, '"') >= 0) { + my $lc = $stat =~ tr@\n@@; + $lc = $lc + $linenr; + my $stat_real = get_stat_real($linenr, $lc); + pos($stat_real) = index($stat_real, '"'); + while ($stat_real =~ /[^\"%]*(%[\#\d\.\*\-]*(h+)[idux])/g) { + my $pspec = $1; + my $h = $2; + my $lineoff = substr($stat_real, 0, $-[1]) =~ tr@\n@@; + if (WARN("UNNECESSARY_MODIFIER", + "Integer promotion: Using '$h' in '$pspec' is unnecessary\n" . "$here\n$stat_real\n") && + $fix && $fixed[$fixlinenr + $lineoff] =~ /^\+/) { + my $nspec = $pspec; + $nspec =~ s/h//g; + $fixed[$fixlinenr + $lineoff] =~ s/\Q$pspec\E/$nspec/; + } + } + } + # check for mask then right shift without a parentheses if ($perl_version_ok && $line =~ /$LvalOrFunc\s*\&\s*($LvalOrFunc)\s*>>/ && @@ -5913,8 +6360,7 @@ sub process { my $barriers = qr{ mb| rmb| - wmb| - read_barrier_depends + wmb }x; my $barrier_stems = qr{ mb__before_atomic| @@ -5955,10 +6401,12 @@ sub process { } } -# check for smp_read_barrier_depends and read_barrier_depends - if (!$file && $line =~ /\b(smp_|)read_barrier_depends\s*\(/) { - WARN("READ_BARRIER_DEPENDS", - "$1read_barrier_depends should only be used in READ_ONCE or DEC Alpha code\n" . $herecurr); +# check for data_race without a comment. + if ($line =~ /\bdata_race\s*\(/) { + if (!ctx_has_comment($first_line, $linenr)) { + WARN("DATA_RACE", + "data_race without comment\n" . $herecurr); + } } # check of hardware specific defines @@ -6000,50 +6448,68 @@ sub process { } } -# Check for __attribute__ packed, prefer __packed +# Check for compiler attributes if ($realfile !~ m@\binclude/uapi/@ && - $line =~ /\b__attribute__\s*\(\s*\(.*\bpacked\b/) { - WARN("PREFER_PACKED", - "__packed is preferred over __attribute__((packed))\n" . $herecurr); - } - -# Check for __attribute__ aligned, prefer __aligned - if ($realfile !~ m@\binclude/uapi/@ && - $line =~ /\b__attribute__\s*\(\s*\(.*aligned/) { - WARN("PREFER_ALIGNED", - "__aligned(size) is preferred over __attribute__((aligned(size)))\n" . $herecurr); - } - -# Check for __attribute__ section, prefer __section - if ($realfile !~ m@\binclude/uapi/@ && - $line =~ /\b__attribute__\s*\(\s*\(.*_*section_*\s*\(\s*("[^"]*")/) { - my $old = substr($rawline, $-[1], $+[1] - $-[1]); - my $new = substr($old, 1, -1); - if (WARN("PREFER_SECTION", - "__section($new) is preferred over __attribute__((section($old)))\n" . $herecurr) && - $fix) { - $fixed[$fixlinenr] =~ s/\b__attribute__\s*\(\s*\(\s*_*section_*\s*\(\s*\Q$old\E\s*\)\s*\)\s*\)/__section($new)/; - } - } - -# Check for __attribute__ format(printf, prefer __printf - if ($realfile !~ m@\binclude/uapi/@ && - $line =~ /\b__attribute__\s*\(\s*\(\s*format\s*\(\s*printf/) { - if (WARN("PREFER_PRINTF", - "__printf(string-index, first-to-check) is preferred over __attribute__((format(printf, string-index, first-to-check)))\n" . $herecurr) && - $fix) { - $fixed[$fixlinenr] =~ s/\b__attribute__\s*\(\s*\(\s*format\s*\(\s*printf\s*,\s*(.*)\)\s*\)\s*\)/"__printf(" . trim($1) . ")"/ex; - + $rawline =~ /\b__attribute__\s*\(\s*($balanced_parens)\s*\)/) { + my $attr = $1; + $attr =~ s/\s*\(\s*(.*)\)\s*/$1/; + + my %attr_list = ( + "alias" => "__alias", + "aligned" => "__aligned", + "always_inline" => "__always_inline", + "assume_aligned" => "__assume_aligned", + "cold" => "__cold", + "const" => "__attribute_const__", + "copy" => "__copy", + "designated_init" => "__designated_init", + "externally_visible" => "__visible", + "format" => "printf|scanf", + "gnu_inline" => "__gnu_inline", + "malloc" => "__malloc", + "mode" => "__mode", + "no_caller_saved_registers" => "__no_caller_saved_registers", + "noclone" => "__noclone", + "noinline" => "noinline", + "nonstring" => "__nonstring", + "noreturn" => "__noreturn", + "packed" => "__packed", + "pure" => "__pure", + "section" => "__section", + "used" => "__used", + "weak" => "__weak" + ); + + while ($attr =~ /\s*(\w+)\s*(${balanced_parens})?/g) { + my $orig_attr = $1; + my $params = ''; + $params = $2 if defined($2); + my $curr_attr = $orig_attr; + $curr_attr =~ s/^[\s_]+|[\s_]+$//g; + if (exists($attr_list{$curr_attr})) { + my $new = $attr_list{$curr_attr}; + if ($curr_attr eq "format" && $params) { + $params =~ /^\s*\(\s*(\w+)\s*,\s*(.*)/; + $new = "__$1\($2"; + } else { + $new = "$new$params"; + } + if (WARN("PREFER_DEFINED_ATTRIBUTE_MACRO", + "Prefer $new over __attribute__(($orig_attr$params))\n" . $herecurr) && + $fix) { + my $remove = "\Q$orig_attr\E" . '\s*' . "\Q$params\E" . '(?:\s*,\s*)?'; + $fixed[$fixlinenr] =~ s/$remove//; + $fixed[$fixlinenr] =~ s/\b__attribute__/$new __attribute__/; + $fixed[$fixlinenr] =~ s/\}\Q$new\E/} $new/; + $fixed[$fixlinenr] =~ s/ __attribute__\s*\(\s*\(\s*\)\s*\)//; + } + } } - } -# Check for __attribute__ format(scanf, prefer __scanf - if ($realfile !~ m@\binclude/uapi/@ && - $line =~ /\b__attribute__\s*\(\s*\(\s*format\s*\(\s*scanf\b/) { - if (WARN("PREFER_SCANF", - "__scanf(string-index, first-to-check) is preferred over __attribute__((format(scanf, string-index, first-to-check)))\n" . $herecurr) && - $fix) { - $fixed[$fixlinenr] =~ s/\b__attribute__\s*\(\s*\(\s*format\s*\(\s*scanf\s*,\s*(.*)\)\s*\)\s*\)/"__scanf(" . trim($1) . ")"/ex; + # Check for __attribute__ unused, prefer __always_unused or __maybe_unused + if ($attr =~ /^_*unused/) { + WARN("PREFER_DEFINED_ATTRIBUTE_MACRO", + "__always_unused or __maybe_unused is preferred over __attribute__((__unused__))\n" . $herecurr); } } @@ -6079,18 +6545,18 @@ sub process { if ($line =~ /(\(\s*$C90_int_types\s*\)\s*)($Constant)\b/) { my $cast = $1; my $const = $2; + my $suffix = ""; + my $newconst = $const; + $newconst =~ s/${Int_type}$//; + $suffix .= 'U' if ($cast =~ /\bunsigned\b/); + if ($cast =~ /\blong\s+long\b/) { + $suffix .= 'LL'; + } elsif ($cast =~ /\blong\b/) { + $suffix .= 'L'; + } if (WARN("TYPECAST_INT_CONSTANT", - "Unnecessary typecast of c90 int constant\n" . $herecurr) && + "Unnecessary typecast of c90 int constant - '$cast$const' could be '$const$suffix'\n" . $herecurr) && $fix) { - my $suffix = ""; - my $newconst = $const; - $newconst =~ s/${Int_type}$//; - $suffix .= 'U' if ($cast =~ /\bunsigned\b/); - if ($cast =~ /\blong\s+long\b/) { - $suffix .= 'LL'; - } elsif ($cast =~ /\blong\b/) { - $suffix .= 'L'; - } $fixed[$fixlinenr] =~ s/\Q$cast\E$const\b/$newconst$suffix/; } } @@ -6239,6 +6705,12 @@ sub process { # } # } +# strlcpy uses that should likely be strscpy + if ($line =~ /\bstrlcpy\s*\(/) { + WARN("STRLCPY", + "Prefer strscpy over strlcpy - see: https://lore.kernel.org/r/CAHk-=wgfRnXz0W3D37d01q3JFkr_i_uTL=V6A6G1oUZcprmknw\@mail.gmail.com/\n" . $herecurr); + } + # typecasts on min/max could be min_t/max_t if ($perl_version_ok && defined $stat && @@ -6332,8 +6804,7 @@ sub process { if (defined $cond) { substr($s, 0, length($cond), ''); } - if ($s =~ /^\s*;/ && - $function_name ne 'uninitialized_var') + if ($s =~ /^\s*;/) { WARN("AVOID_EXTERNS", "externs should be avoided in .c files\n" . $herecurr); @@ -6352,17 +6823,13 @@ sub process { } # check for function declarations that have arguments without identifier names -# while avoiding uninitialized_var(x) if (defined $stat && - $stat =~ /^.\s*(?:extern\s+)?$Type\s*(?:($Ident)|\(\s*\*\s*$Ident\s*\))\s*\(\s*([^{]+)\s*\)\s*;/s && - (!defined($1) || - (defined($1) && $1 ne "uninitialized_var")) && - $2 ne "void") { - my $args = trim($2); + $stat =~ /^.\s*(?:extern\s+)?$Type\s*(?:$Ident|\(\s*\*\s*$Ident\s*\))\s*\(\s*([^{]+)\s*\)\s*;/s && + $1 ne "void") { + my $args = trim($1); while ($args =~ m/\s*($Type\s*(?:$Ident|\(\s*\*\s*$Ident?\s*\)\s*$balanced_parens)?)/g) { my $arg = trim($1); - if ($arg =~ /^$Type$/ && - $arg !~ /enum\s+$Ident$/) { + if ($arg =~ /^$Type$/ && $arg !~ /enum\s+$Ident$/) { WARN("FUNCTION_ARGUMENTS", "function definition argument '$arg' should also have an identifier name\n" . $herecurr); } @@ -6398,7 +6865,7 @@ sub process { if (!grep(/$name/, @setup_docs)) { CHK("UNDOCUMENTED_SETUP", - "__setup appears un-documented -- check Documentation/admin-guide/kernel-parameters.rst\n" . $herecurr); + "__setup appears un-documented -- check Documentation/admin-guide/kernel-parameters.txt\n" . $herecurr); } } @@ -6480,41 +6947,22 @@ sub process { } } +# check for IS_ENABLED() without CONFIG_<FOO> ($rawline for comments too) + if ($rawline =~ /\bIS_ENABLED\s*\(\s*(\w+)\s*\)/ && $1 !~ /^${CONFIG_}/) { + WARN("IS_ENABLED_CONFIG", + "IS_ENABLED($1) is normally used as IS_ENABLED(${CONFIG_}$1)\n" . $herecurr); + } + # check for #if defined CONFIG_<FOO> || defined CONFIG_<FOO>_MODULE - if ($line =~ /^\+\s*#\s*if\s+defined(?:\s*\(?\s*|\s+)(CONFIG_[A-Z_]+)\s*\)?\s*\|\|\s*defined(?:\s*\(?\s*|\s+)\1_MODULE\s*\)?\s*$/) { + if ($line =~ /^\+\s*#\s*if\s+defined(?:\s*\(?\s*|\s+)(${CONFIG_}[A-Z_]+)\s*\)?\s*\|\|\s*defined(?:\s*\(?\s*|\s+)\1_MODULE\s*\)?\s*$/) { my $config = $1; if (WARN("PREFER_IS_ENABLED", - "Prefer IS_ENABLED(<FOO>) to CONFIG_<FOO> || CONFIG_<FOO>_MODULE\n" . $herecurr) && + "Prefer IS_ENABLED(<FOO>) to ${CONFIG_}<FOO> || ${CONFIG_}<FOO>_MODULE\n" . $herecurr) && $fix) { $fixed[$fixlinenr] = "\+#if IS_ENABLED($config)"; } } -# check for case / default statements not preceded by break/fallthrough/switch - if ($line =~ /^.\s*(?:case\s+(?:$Ident|$Constant)\s*|default):/) { - my $has_break = 0; - my $has_statement = 0; - my $count = 0; - my $prevline = $linenr; - while ($prevline > 1 && ($file || $count < 3) && !$has_break) { - $prevline--; - my $rline = $rawlines[$prevline - 1]; - my $fline = $lines[$prevline - 1]; - last if ($fline =~ /^\@\@/); - next if ($fline =~ /^\-/); - next if ($fline =~ /^.(?:\s*(?:case\s+(?:$Ident|$Constant)[\s$;]*|default):[\s$;]*)*$/); - $has_break = 1 if ($rline =~ /fall[\s_-]*(through|thru)/i); - next if ($fline =~ /^.[\s$;]*$/); - $has_statement = 1; - $count++; - $has_break = 1 if ($fline =~ /\bswitch\b|\b(?:break\s*;[\s$;]*$|exit\s*\(\b|return\b|goto\b|continue\b)/); - } - if (!$has_break && $has_statement) { - WARN("MISSING_BREAK", - "Possible switch case/default not preceded by break or fallthrough comment\n" . $herecurr); - } - } - # check for /* fallthrough */ like comment, prefer fallthrough; my @fallthroughs = ( 'fallthrough', @@ -6630,7 +7078,8 @@ sub process { # check for various structs that are normally const (ops, kgdb, device_tree) # and avoid what seem like struct definitions 'struct foo {' - if ($line !~ /\bconst\b/ && + if (defined($const_structs) && + $line !~ /\bconst\b/ && $line =~ /\bstruct\s+($const_structs)\b(?!\s*\{)/) { WARN("CONST_STRUCT", "struct $1 should normally be const\n" . $herecurr); @@ -6638,12 +7087,14 @@ sub process { # use of NR_CPUS is usually wrong # ignore definitions of NR_CPUS and usage to define arrays as likely right +# ignore designated initializers using NR_CPUS if ($line =~ /\bNR_CPUS\b/ && $line !~ /^.\s*\s*#\s*if\b.*\bNR_CPUS\b/ && $line !~ /^.\s*\s*#\s*define\b.*\bNR_CPUS\b/ && $line !~ /^.\s*$Declare\s.*\[[^\]]*NR_CPUS[^\]]*\]/ && $line !~ /\[[^\]]*\.\.\.[^\]]*NR_CPUS[^\]]*\]/ && - $line !~ /\[[^\]]*NR_CPUS[^\]]*\.\.\.[^\]]*\]/) + $line !~ /\[[^\]]*NR_CPUS[^\]]*\.\.\.[^\]]*\]/ && + $line !~ /^.\s*\.\w+\s*=\s*.*\bNR_CPUS\b/) { WARN("NR_CPUS", "usage of NR_CPUS is often wrong - consider using cpu_possible(), num_possible_cpus(), for_each_possible_cpu(), etc\n" . $herecurr); @@ -6679,12 +7130,6 @@ sub process { } } -# check for mutex_trylock_recursive usage - if ($line =~ /mutex_trylock_recursive/) { - ERROR("LOCKING", - "recursive locking is bad, do not use this ever.\n" . $herecurr); - } - # check for lockdep_set_novalidate_class if ($line =~ /^.\s*lockdep_set_novalidate_class\s*\(/ || $line =~ /__lockdep_no_validate__\s*\)/ ) { @@ -6847,7 +7292,7 @@ sub process { exit(0); } - # This is not a patch, and we are are in 'no-patch' mode so + # This is not a patch, and we are in 'no-patch' mode so # just keep quiet. if (!$chk_patch && !$is_patch) { exit(0); @@ -6861,9 +7306,33 @@ sub process { if ($signoff == 0) { ERROR("MISSING_SIGN_OFF", "Missing Signed-off-by: line(s)\n"); - } elsif (!$authorsignoff) { - WARN("NO_AUTHOR_SIGN_OFF", - "Missing Signed-off-by: line by nominal patch author '$author'\n"); + } elsif ($authorsignoff != 1) { + # authorsignoff values: + # 0 -> missing sign off + # 1 -> sign off identical + # 2 -> names and addresses match, comments mismatch + # 3 -> addresses match, names different + # 4 -> names match, addresses different + # 5 -> names match, addresses excluding subaddress details (refer RFC 5233) match + + my $sob_msg = "'From: $author' != 'Signed-off-by: $author_sob'"; + + if ($authorsignoff == 0) { + ERROR("NO_AUTHOR_SIGN_OFF", + "Missing Signed-off-by: line by nominal patch author '$author'\n"); + } elsif ($authorsignoff == 2) { + CHK("FROM_SIGN_OFF_MISMATCH", + "From:/Signed-off-by: email comments mismatch: $sob_msg\n"); + } elsif ($authorsignoff == 3) { + WARN("FROM_SIGN_OFF_MISMATCH", + "From:/Signed-off-by: email name mismatch: $sob_msg\n"); + } elsif ($authorsignoff == 4) { + WARN("FROM_SIGN_OFF_MISMATCH", + "From:/Signed-off-by: email address mismatch: $sob_msg\n"); + } elsif ($authorsignoff == 5) { + WARN("FROM_SIGN_OFF_MISMATCH", + "From:/Signed-off-by: email subaddress mismatch: $sob_msg\n"); + } } } @@ -6944,4 +7413,4 @@ EOM } } return $clean; -}
\ No newline at end of file +} diff --git a/scripts/ci/coverity.sh b/scripts/ci/coverity.sh new file mode 100755 index 000000000..7272f8ed1 --- /dev/null +++ b/scripts/ci/coverity.sh @@ -0,0 +1,19 @@ +#!/bin/bash +set -e + +cd "$(dirname "$0")"/../.. +./bootstrap +./configure --enable-debug=full + +make clean + +cov-build --dir cov-int make -j $(nproc) + +tar czf odp-coverity.tgz cov-int + +curl --form token="${COVERITY_TOKEN}" \ + --form email="${COVERITY_EMAIL}" \ + --form file=@odp-coverity.tgz \ + --form version="${GITHUB_SHA}" \ + --form description="GitHub Actions ODP Coverity Build" \ + "https://scan.coverity.com/builds?project=${COVERITY_PROJECT}" diff --git a/scripts/ci/distcheck.sh b/scripts/ci/distcheck.sh index 9d45536f4..6cf49f8a7 100755 --- a/scripts/ci/distcheck.sh +++ b/scripts/ci/distcheck.sh @@ -16,4 +16,4 @@ export CI="true" # Additional configure flags for distcheck export DISTCHECK_CONFIGURE_FLAGS="${CONF}" -make distcheck +make -j $(nproc) distcheck diff --git a/scripts/spelling.txt b/scripts/spelling.txt index 6293995a4..2e3ba91a5 100644 --- a/scripts/spelling.txt +++ b/scripts/spelling.txt @@ -103,6 +103,7 @@ alloated||allocated allocatote||allocate allocatrd||allocated allocte||allocate +allocted||allocated allpication||application alocate||allocate alogirhtms||algorithms @@ -339,6 +340,7 @@ comppatible||compatible compres||compress compresion||compression comression||compression +comunicate||communicate comunication||communication conbination||combination conditionaly||conditionally @@ -466,6 +468,7 @@ developpment||development deveolpment||development devided||divided deviece||device +devision||division diable||disable dicline||decline dictionnary||dictionary @@ -479,6 +482,7 @@ difinition||definition digial||digital dimention||dimension dimesions||dimensions +diconnected||disconnected disgest||digest dispalying||displaying diplay||display @@ -518,6 +522,7 @@ downlads||downloads droped||dropped droput||dropout druing||during +dyanmic||dynamic dynmaic||dynamic eanable||enable eanble||enable @@ -542,6 +547,7 @@ encrupted||encrypted encrypiton||encryption encryptio||encryption endianess||endianness +enpoint||endpoint enhaced||enhanced enlightnment||enlightenment enqueing||enqueuing @@ -566,6 +572,7 @@ estbalishment||establishment etsablishment||establishment etsbalishment||establishment evalution||evaluation +exeeds||exceeds excecutable||executable exceded||exceeded exceds||exceeds @@ -574,6 +581,7 @@ excellant||excellent execeeded||exceeded execeeds||exceeds exeed||exceed +exeeds||exceeds exeuction||execution existance||existence existant||existent @@ -641,6 +649,7 @@ forwardig||forwarding frambuffer||framebuffer framming||framing framwork||framework +frequence||frequency frequncy||frequency frequancy||frequency frome||from @@ -683,10 +692,12 @@ handfull||handful hanlde||handle hanled||handled happend||happened +hardare||hardware harware||hardware havind||having heirarchically||hierarchically helpfull||helpful +heterogenous||heterogeneous hexdecimal||hexadecimal hybernate||hibernate hierachy||hierarchy @@ -731,6 +742,7 @@ inconsistant||inconsistent increas||increase incremeted||incremented incrment||increment +incuding||including inculde||include indendation||indentation indended||intended @@ -741,6 +753,7 @@ indiate||indicate indicat||indicate inexpect||inexpected inferface||interface +infinit||infinite infomation||information informatiom||information informations||information @@ -771,6 +784,7 @@ instace||instance instal||install instanciate||instantiate instanciated||instantiated +instuments||instruments insufficent||insufficient inteface||interface integreated||integrated @@ -869,12 +883,14 @@ mailformed||malformed malplaced||misplaced malplace||misplace managable||manageable +managament||management managment||management mangement||management manger||manager manoeuvering||maneuvering manufaucturing||manufacturing mappping||mapping +maping||mapping matchs||matches mathimatical||mathematical mathimatic||mathematic @@ -886,6 +902,7 @@ meetign||meeting memeory||memory memmber||member memoery||memory +memroy||memory ment||meant mergable||mergeable mesage||message @@ -999,6 +1016,7 @@ overlaping||overlapping overide||override overrided||overridden overriden||overridden +overrrun||overrun overun||overrun overwritting||overwriting overwriten||overwritten @@ -1035,6 +1053,7 @@ peforming||performing peice||piece pendantic||pedantic peprocessor||preprocessor +perfomance||performance perfoming||performing perfomring||performing periperal||peripheral @@ -1100,6 +1119,7 @@ prodecure||procedure progamming||programming progams||programs progess||progress +programable||programmable programers||programmers programm||program programms||programs @@ -1144,6 +1164,7 @@ recieved||received recieve||receive reciever||receiver recieves||receives +recieving||receiving recogniced||recognised recognizeable||recognizable recommanded||recommended @@ -1247,6 +1268,7 @@ searchs||searches secquence||sequence secund||second segement||segment +seleted||selected semaphone||semaphore senario||scenario senarios||scenarios @@ -1263,6 +1285,7 @@ seqeunce||sequence seqeuncer||sequencer seqeuencer||sequencer sequece||sequence +sequemce||sequence sequencial||sequential serivce||service serveral||several @@ -1333,6 +1356,7 @@ suble||subtle substract||subtract submited||submitted submition||submission +succeded||succeeded suceed||succeed succesfully||successfully succesful||successful @@ -1353,6 +1377,7 @@ supportin||supporting suppoted||supported suppported||supported suppport||support +supprot||support supress||suppress surpressed||suppressed surpresses||suppresses @@ -1401,6 +1426,7 @@ thresold||threshold throught||through trackling||tracking troughput||throughput +trys||tries thses||these tiggers||triggers tiggered||triggered @@ -1414,7 +1440,9 @@ traking||tracking tramsmitted||transmitted tramsmit||transmit tranasction||transaction +tranceiver||transceiver tranfer||transfer +tranmission||transmission transcevier||transceiver transciever||transceiver transferd||transferred @@ -1468,6 +1496,7 @@ unnecesary||unnecessary unneedingly||unnecessarily unnsupported||unsupported unmached||unmatched +unprecise||imprecise unregester||unregister unresgister||unregister unrgesiter||unregister @@ -1503,6 +1532,7 @@ varient||variant vaule||value verbse||verbose veify||verify +veriosn||version verisons||versions verison||version verson||version @@ -1533,4 +1563,4 @@ writeing||writing writting||writing wtih||with zombe||zombie -zomebie||zombie
\ No newline at end of file +zomebie||zombie diff --git a/test/miscellaneous/odp_api_from_cpp.cpp b/test/miscellaneous/odp_api_from_cpp.cpp index 7ed72ff17..f3297093a 100644 --- a/test/miscellaneous/odp_api_from_cpp.cpp +++ b/test/miscellaneous/odp_api_from_cpp.cpp @@ -4,8 +4,16 @@ int main(int argc ODP_UNUSED, const char *argv[] ODP_UNUSED) { + odp_instance_t inst; + + odp_init_global(&inst, NULL, NULL); + odp_init_local(inst, ODP_THREAD_WORKER); + std::cout << "\tODP API version: " << odp_version_api_str() << std::endl; std::cout << "\tODP implementation version: " << odp_version_impl_str() << std::endl; + odp_term_local(); + odp_term_global(inst); + return 0; } diff --git a/test/performance/Makefile.am b/test/performance/Makefile.am index 7566ab8a8..06866ae39 100644 --- a/test/performance/Makefile.am +++ b/test/performance/Makefile.am @@ -86,3 +86,5 @@ clean-local: rm -f $(builddir)/$$f; \ done \ fi + +.NOTPARALLEL: diff --git a/test/performance/odp_atomic_perf.c b/test/performance/odp_atomic_perf.c index 2ed88a5e8..887525f50 100644 --- a/test/performance/odp_atomic_perf.c +++ b/test/performance/odp_atomic_perf.c @@ -35,7 +35,8 @@ typedef int (*validate_fn_t)(void *val, void *out, uint32_t num_round, typedef enum { OP_32BIT, - OP_64BIT + OP_64BIT, + OP_128BIT } op_bit_t; /* Command line options */ @@ -51,6 +52,7 @@ typedef struct ODP_ALIGNED_CACHE test_atomic_t { union { odp_atomic_u32_t u32; odp_atomic_u64_t u64; + odp_atomic_u128_t u128; }; } test_atomic_t; @@ -73,6 +75,7 @@ struct test_global_t { union { odp_atomic_u32_t atomic_u32; odp_atomic_u64_t atomic_u64; + odp_atomic_u128_t atomic_u128; }; odp_cpumask_t cpumask; odph_thread_t thread_tbl[ODP_THREAD_COUNT_MAX]; @@ -81,6 +84,7 @@ struct test_global_t { union { uint32_t u32; uint64_t u64; + odp_u128_t u128; } output[ODP_THREAD_COUNT_MAX]; }; @@ -117,6 +121,25 @@ static inline void test_atomic_load_u64(void *val, void *out, uint32_t num_round *result = ret; } +static inline void test_atomic_load_u128(void *val, void *out, uint32_t num_round) +{ + odp_atomic_u128_t *atomic_val = val; + odp_u128_t *result = out; + odp_u128_t ret; + + ret.u64[0] = 0; + ret.u64[1] = 0; + + for (uint32_t i = 0; i < num_round; i++) { + odp_u128_t cur_val = odp_atomic_load_u128(atomic_val); + + ret.u64[0] += cur_val.u64[0]; + ret.u64[1] += cur_val.u64[1]; + } + + *result = ret; +} + static inline int validate_atomic_init_val_u32(void *val, void *out, uint32_t num_round, uint32_t num_worker ODP_UNUSED, int private ODP_UNUSED) @@ -128,8 +151,7 @@ static inline int validate_atomic_init_val_u32(void *val, void *out, uint32_t nu (*result != (uint32_t)INIT_VAL * num_round); } -static inline int validate_atomic_init_val_u64(void *val, void *out ODP_UNUSED, - uint32_t num_round ODP_UNUSED, +static inline int validate_atomic_init_val_u64(void *val, void *out, uint32_t num_round, uint32_t worker ODP_UNUSED, int private ODP_UNUSED) { odp_atomic_u64_t *atomic_val = val; @@ -139,6 +161,22 @@ static inline int validate_atomic_init_val_u64(void *val, void *out ODP_UNUSED, (*result != (uint64_t)INIT_VAL * num_round); } +static inline int validate_atomic_init_val_u128(void *val, void *out, uint32_t num_round, + uint32_t worker ODP_UNUSED, int private ODP_UNUSED) +{ + odp_u128_t atomic_val = odp_atomic_load_u128((odp_atomic_u128_t *)val); + odp_u128_t *result = out; + + if (atomic_val.u64[0] != INIT_VAL || atomic_val.u64[1] != INIT_VAL) + return -1; + + if (result->u64[0] != (uint64_t)INIT_VAL * num_round || + result->u64[1] != (uint64_t)INIT_VAL * num_round) + return -1; + + return 0; +} + static inline void test_atomic_store_u32(void *val, void *out ODP_UNUSED, uint32_t num_round) { odp_atomic_u32_t *atomic_val = val; @@ -157,6 +195,21 @@ static inline void test_atomic_store_u64(void *val, void *out ODP_UNUSED, uint32 odp_atomic_store_u64(atomic_val, new_val++); } +static inline void test_atomic_store_u128(void *val, void *out ODP_UNUSED, uint32_t num_round) +{ + odp_atomic_u128_t *atomic_val = val; + odp_u128_t new_val; + + new_val.u64[0] = INIT_VAL + 1; + new_val.u64[1] = INIT_VAL + 1; + + for (uint32_t i = 0; i < num_round; i++) { + odp_atomic_store_u128(atomic_val, new_val); + new_val.u64[0]++; + new_val.u64[1]++; + } +} + static inline int validate_atomic_num_round_u32(void *val, void *out ODP_UNUSED, uint32_t num_round, uint32_t worker ODP_UNUSED, int private ODP_UNUSED) { @@ -173,6 +226,16 @@ static inline int validate_atomic_num_round_u64(void *val, void *out ODP_UNUSED, return odp_atomic_load_u64(atomic_val) != ((uint64_t)INIT_VAL + num_round); } +static inline int validate_atomic_num_round_u128(void *val, void *out ODP_UNUSED, + uint32_t num_round, uint32_t worker ODP_UNUSED, + int private ODP_UNUSED) +{ + odp_u128_t atomic_val = odp_atomic_load_u128((odp_atomic_u128_t *)val); + + return (atomic_val.u64[0] != ((uint64_t)INIT_VAL + num_round) || + atomic_val.u64[1] != ((uint64_t)INIT_VAL + num_round)); +} + static inline void test_atomic_fetch_add_u32(void *val, void *out, uint32_t num_round) { odp_atomic_u32_t *atomic_val = val; @@ -470,6 +533,26 @@ static inline void test_atomic_cas_u64(void *val, void *out ODP_UNUSED, uint32_t } } +static inline void test_atomic_cas_u128(void *val, void *out ODP_UNUSED, uint32_t num_round) +{ + odp_atomic_u128_t *atomic_val = val; + odp_u128_t new_val; + odp_u128_t old_val; + + new_val.u64[0] = INIT_VAL + 1; + new_val.u64[1] = INIT_VAL + 1; + old_val.u64[0] = INIT_VAL; + old_val.u64[1] = INIT_VAL; + + for (uint32_t i = 0; i < num_round; i++) { + if (odp_atomic_cas_u128(atomic_val, &old_val, new_val)) { + old_val = new_val; + new_val.u64[0]++; + new_val.u64[1]++; + } + } +} + static inline int validate_atomic_cas_u32(void *val, void *out ODP_UNUSED, uint32_t num_round, uint32_t num_worker ODP_UNUSED, int private) { @@ -492,6 +575,19 @@ static inline int validate_atomic_cas_u64(void *val, void *out ODP_UNUSED, uint3 return result > ((uint64_t)INIT_VAL + num_round); } +static inline int validate_atomic_cas_u128(void *val, void *out ODP_UNUSED, uint32_t num_round, + uint32_t num_worker ODP_UNUSED, int private) +{ + odp_u128_t result = odp_atomic_load_u128((odp_atomic_u128_t *)val); + + if (private) + return (result.u64[0] != ((uint64_t)INIT_VAL + num_round) || + result.u64[1] != ((uint64_t)INIT_VAL + num_round)); + + return (result.u64[0] > ((uint64_t)INIT_VAL + num_round) || + result.u64[1] > ((uint64_t)INIT_VAL + num_round)); +} + static inline void test_atomic_xchg_u32(void *val, void *out, uint32_t num_round) { odp_atomic_u32_t *atomic_val = val; @@ -616,6 +712,26 @@ static inline void test_atomic_cas_acq_u64(void *val, void *out ODP_UNUSED, uint } } +static inline void test_atomic_cas_acq_u128(void *val, void *out ODP_UNUSED, uint32_t num_round) +{ + odp_atomic_u128_t *atomic_val = val; + odp_u128_t new_val; + odp_u128_t old_val; + + new_val.u64[0] = INIT_VAL + 1; + new_val.u64[1] = INIT_VAL + 1; + old_val.u64[0] = INIT_VAL; + old_val.u64[1] = INIT_VAL; + + for (uint32_t i = 0; i < num_round; i++) { + if (odp_atomic_cas_acq_u128(atomic_val, &old_val, new_val)) { + old_val = new_val; + new_val.u64[0]++; + new_val.u64[1]++; + } + } +} + static inline void test_atomic_cas_rel_u32(void *val, void *out ODP_UNUSED, uint32_t num_round) { odp_atomic_u32_t *atomic_val = val; @@ -640,6 +756,26 @@ static inline void test_atomic_cas_rel_u64(void *val, void *out ODP_UNUSED, uint } } +static inline void test_atomic_cas_rel_u128(void *val, void *out ODP_UNUSED, uint32_t num_round) +{ + odp_atomic_u128_t *atomic_val = val; + odp_u128_t new_val; + odp_u128_t old_val; + + new_val.u64[0] = INIT_VAL + 1; + new_val.u64[1] = INIT_VAL + 1; + old_val.u64[0] = INIT_VAL; + old_val.u64[1] = INIT_VAL; + + for (uint32_t i = 0; i < num_round; i++) { + if (odp_atomic_cas_rel_u128(atomic_val, &old_val, new_val)) { + old_val = new_val; + new_val.u64[0]++; + new_val.u64[1]++; + } + } +} + static inline void test_atomic_cas_acq_rel_u32(void *val, void *out ODP_UNUSED, uint32_t num_round) { odp_atomic_u32_t *atomic_val = val; @@ -664,6 +800,26 @@ static inline void test_atomic_cas_acq_rel_u64(void *val, void *out ODP_UNUSED, } } +static inline void test_atomic_cas_acq_rel_u128(void *val, void *out ODP_UNUSED, uint32_t num_round) +{ + odp_atomic_u128_t *atomic_val = val; + odp_u128_t new_val; + odp_u128_t old_val; + + new_val.u64[0] = INIT_VAL + 1; + new_val.u64[1] = INIT_VAL + 1; + old_val.u64[0] = INIT_VAL; + old_val.u64[1] = INIT_VAL; + + for (uint32_t i = 0; i < num_round; i++) { + if (odp_atomic_cas_acq_rel_u128(atomic_val, &old_val, new_val)) { + old_val = new_val; + new_val.u64[0]++; + new_val.u64[1]++; + } + } +} + static void print_usage(void) { printf("\n" @@ -707,6 +863,14 @@ static void print_info(test_options_t *test_options) printf(" odp_atomic_max_u64: %" PRIu32 "\n", atomic_ops.op.max); printf(" odp_atomic_cas_u64: %" PRIu32 "\n", atomic_ops.op.cas); printf(" odp_atomic_xchg_u64: %" PRIu32 "\n", atomic_ops.op.xchg); + + atomic_ops.all_bits = 0; + odp_atomic_lock_free_u128(&atomic_ops); + + printf(" odp_atomic_load_u128: %" PRIu32 "\n", atomic_ops.op.load); + printf(" odp_atomic_store_u128: %" PRIu32 "\n", atomic_ops.op.store); + printf(" odp_atomic_cas_u128: %" PRIu32 "\n", atomic_ops.op.cas); + printf("\n\n"); } @@ -808,12 +972,19 @@ static int set_num_cpu(test_global_t *global) static int init_test(test_global_t *global, const char *name, op_bit_t type) { + odp_u128_t init_val; + + init_val.u64[0] = INIT_VAL; + init_val.u64[1] = INIT_VAL; + printf("TEST: %s\n", name); if (type == OP_32BIT) odp_atomic_init_u32(&global->atomic_u32, INIT_VAL); else if (type == OP_64BIT) odp_atomic_init_u64(&global->atomic_u64, INIT_VAL); + else if (type == OP_128BIT) + odp_atomic_init_u128(&global->atomic_u128, init_val); else return -1; @@ -821,9 +992,13 @@ static int init_test(test_global_t *global, const char *name, op_bit_t type) if (type == OP_32BIT) { global->output[i].u32 = 0; odp_atomic_init_u32(&global->atomic_private[i].u32, INIT_VAL); - } else { + } else if (type == OP_64BIT) { global->output[i].u64 = 0; odp_atomic_init_u64(&global->atomic_private[i].u64, INIT_VAL); + } else { + global->output[i].u128.u64[0] = 0; + global->output[i].u128.u64[1] = 0; + odp_atomic_init_u128(&global->atomic_private[i].u128, init_val); } } return 0; @@ -844,20 +1019,29 @@ static int run_test(void *arg) void *out; uint32_t out_u32 = 0; uint64_t out_u64 = 0; + odp_u128_t out_u128; + + out_u128.u64[0] = 0; + out_u128.u64[1] = 0; if (type == OP_32BIT) { val = &global->atomic_u32; out = &out_u32; - } else { + } else if (type == OP_64BIT) { val = &global->atomic_u64; out = &out_u64; + } else { + val = &global->atomic_u128; + out = &out_u128; } if (global->test_options.private) { if (type == OP_32BIT) val = &global->atomic_private[idx].u32; - else + else if (type == OP_64BIT) val = &global->atomic_private[idx].u64; + else + val = &global->atomic_private[idx].u128; } /* Start all workers at the same time */ @@ -875,8 +1059,10 @@ static int run_test(void *arg) thread_ctx->nsec = nsec; if (type == OP_32BIT) global->output[idx].u32 = out_u32; - else + else if (type == OP_64BIT) global->output[idx].u64 = out_u64; + else + global->output[idx].u128 = out_u128; return 0; } @@ -933,11 +1119,16 @@ static int validate_results(test_global_t *global, validate_fn_t validate, op_bi val = &global->atomic_u32; if (private) val = &global->atomic_private[i].u32; - } else { + } else if (type == OP_64BIT) { out = &global->output[i].u64; val = &global->atomic_u64; if (private) val = &global->atomic_private[i].u64; + } else { + out = &global->output[i].u128; + val = &global->atomic_u128; + if (private) + val = &global->atomic_private[i].u128; } if (validate(val, out, num_round, num_cpu, private)) @@ -1078,7 +1269,19 @@ static test_case_t test_suite[] = { TEST_INFO("odp_atomic_cas_rel_u64", test_atomic_cas_rel_u64, validate_atomic_cas_u64, OP_64BIT), TEST_INFO("odp_atomic_cas_acq_rel_u64", test_atomic_cas_acq_rel_u64, - validate_atomic_cas_u64, OP_64BIT) + validate_atomic_cas_u64, OP_64BIT), + TEST_INFO("odp_atomic_load_u128", test_atomic_load_u128, + validate_atomic_init_val_u128, OP_128BIT), + TEST_INFO("odp_atomic_store_u128", test_atomic_store_u128, + validate_atomic_num_round_u128, OP_128BIT), + TEST_INFO("odp_atomic_cas_u128", test_atomic_cas_u128, + validate_atomic_cas_u128, OP_128BIT), + TEST_INFO("odp_atomic_cas_acq_u128", test_atomic_cas_acq_u128, + validate_atomic_cas_u128, OP_128BIT), + TEST_INFO("odp_atomic_cas_rel_u128", test_atomic_cas_rel_u128, + validate_atomic_cas_u128, OP_128BIT), + TEST_INFO("odp_atomic_cas_acq_rel_u128", test_atomic_cas_acq_rel_u128, + validate_atomic_cas_u128, OP_128BIT), }; int main(int argc, char **argv) diff --git a/test/performance/odp_crypto.c b/test/performance/odp_crypto.c index 07463ab68..36324622a 100644 --- a/test/performance/odp_crypto.c +++ b/test/performance/odp_crypto.c @@ -1071,6 +1071,16 @@ int main(int argc, char *argv[]) exit(EXIT_FAILURE); } + if (cargs.schedule && crypto_capa.queue_type_sched == 0) { + app_err("scheduled type completion queue not supported.\n"); + exit(EXIT_FAILURE); + } + + if (cargs.poll && crypto_capa.queue_type_plain == 0) { + app_err("plain type completion queue not supported.\n"); + exit(EXIT_FAILURE); + } + if (odp_pool_capability(&pool_capa)) { app_err("Pool capability request failed.\n"); exit(EXIT_FAILURE); diff --git a/test/performance/odp_l2fwd.c b/test/performance/odp_l2fwd.c index ca8502ef7..c7950cfd9 100644 --- a/test/performance/odp_l2fwd.c +++ b/test/performance/odp_l2fwd.c @@ -1,5 +1,5 @@ /* Copyright (c) 2014-2018, Linaro Limited - * Copyright (c) 2019-2020, Nokia + * Copyright (c) 2019-2021, Nokia * Copyright (c) 2020, Marvell * All rights reserved. * @@ -108,6 +108,7 @@ typedef struct { uint32_t packet_len; /* Maximum packet length supported */ uint32_t seg_len; /* Pool segment length */ int promisc_mode; /* Promiscuous mode enabled */ + int mtu; /* Interface MTU */ } appl_args_t; /* Statistics */ @@ -1054,6 +1055,38 @@ static int create_pktio(const char *dev, int idx, int num_rx, int num_tx, odp_po } } + if (gbl_args->appl.mtu) { + uint32_t maxlen_input = pktio_capa.maxlen.max_input ? gbl_args->appl.mtu : 0; + uint32_t maxlen_output = pktio_capa.maxlen.max_output ? gbl_args->appl.mtu : 0; + + if (!pktio_capa.set_op.op.maxlen) { + ODPH_ERR("Error: modifying interface MTU not supported %s\n", dev); + return -1; + } + + if (maxlen_input && + (maxlen_input < pktio_capa.maxlen.min_input || + maxlen_input > pktio_capa.maxlen.max_input)) { + ODPH_ERR("Error: unsupported MTU value %" PRIu32 " for %s " + "(min %" PRIu32 ", max %" PRIu32 ")\n", maxlen_input, dev, + pktio_capa.maxlen.min_input, pktio_capa.maxlen.max_input); + return -1; + } + if (maxlen_output && + (maxlen_output < pktio_capa.maxlen.min_output || + maxlen_output > pktio_capa.maxlen.max_output)) { + ODPH_ERR("Error: unsupported MTU value %" PRIu32 " for %s " + "(min %" PRIu32 ", max %" PRIu32 ")\n", maxlen_output, dev, + pktio_capa.maxlen.min_output, pktio_capa.maxlen.max_output); + return -1; + } + + if (odp_pktio_maxlen_set(pktio, maxlen_input, maxlen_output)) { + ODPH_ERR("Error: setting MTU failed %s\n", dev); + return -1; + } + } + odp_pktin_queue_param_init(&pktin_param); odp_pktout_queue_param_init(&pktout_param); @@ -1508,6 +1541,7 @@ static void usage(char *progname) " Default is num_pkts divided by vec_size.\n" " -x, --vec_size <num> Vector size (default %i).\n" " -z, --vec_tmo_ns <ns> Vector timeout in ns (default %llu ns).\n" + " -M, --mtu <len> Interface MTU in bytes.\n" " -P, --promisc_mode Enable promiscuous mode.\n" " -l, --packet_len <len> Maximum length of packets supported (default %d).\n" " -L, --seg_len <len> Packet pool segment length\n" @@ -1555,6 +1589,7 @@ static void parse_args(int argc, char *argv[], appl_args_t *appl_args) {"vec_size", required_argument, NULL, 'x'}, {"vec_tmo_ns", required_argument, NULL, 'z'}, {"vector_mode", no_argument, NULL, 'u'}, + {"mtu", required_argument, NULL, 'M'}, {"promisc_mode", no_argument, NULL, 'P'}, {"packet_len", required_argument, NULL, 'l'}, {"seg_len", required_argument, NULL, 'L'}, @@ -1563,7 +1598,7 @@ static void parse_args(int argc, char *argv[], appl_args_t *appl_args) {NULL, 0, NULL, 0} }; - static const char *shortopts = "+c:t:a:i:m:o:r:d:s:e:k:g:b:p:y:n:l:L:w:x:z:uPvh"; + static const char *shortopts = "+c:t:a:i:m:o:r:d:s:e:k:g:b:p:y:n:l:L:w:x:z:M:uPvh"; appl_args->time = 0; /* loop forever if time to run is 0 */ appl_args->accuracy = 1; /* get and print pps stats second */ @@ -1580,6 +1615,7 @@ static void parse_args(int argc, char *argv[], appl_args_t *appl_args) appl_args->num_pkt = 0; appl_args->packet_len = POOL_PKT_LEN; appl_args->seg_len = UINT32_MAX; + appl_args->mtu = 0; appl_args->promisc_mode = 0; appl_args->vector_mode = 0; appl_args->num_vec = 0; @@ -1731,6 +1767,9 @@ static void parse_args(int argc, char *argv[], appl_args_t *appl_args) case 'L': appl_args->seg_len = atoi(optarg); break; + case 'M': + appl_args->mtu = atoi(optarg); + break; case 'P': appl_args->promisc_mode = 1; break; @@ -1822,6 +1861,11 @@ static void print_info(appl_args_t *appl_args) else printf("PKTOUT_DIRECT\n"); + printf("MTU: "); + if (appl_args->mtu) + printf("%i bytes\n", appl_args->mtu); + else + printf("interface default\n"); printf("Promisc mode: %s\n", appl_args->promisc_mode ? "enabled" : "disabled"); printf("Burst size: %i\n", appl_args->burst_rx); diff --git a/test/performance/odp_packet_gen.c b/test/performance/odp_packet_gen.c index 8184eacad..b903eb200 100644 --- a/test/performance/odp_packet_gen.c +++ b/test/performance/odp_packet_gen.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2020, Nokia +/* Copyright (c) 2020-2021, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -57,6 +57,7 @@ typedef struct test_options_t { uint16_t udp_src; uint16_t udp_dst; uint32_t wait_sec; + uint32_t mtu; struct vlan_hdr { uint16_t tpid; @@ -162,6 +163,7 @@ static void print_usage(void) " into the number of bins (bins >= 2). Bin value of 0 means\n" " that each packet length is used. Comma-separated (no spaces).\n" " Overrides standard packet length option.\n" + " -M, --mtu <len> Interface MTU in bytes.\n" " -b, --burst_size Transmit burst size. Default: 8\n" " -x, --bursts Number of bursts per one transmit round. Default: 1\n" " -g, --gap Gap between transmit rounds in nsec. Default: 1000000\n" @@ -253,6 +255,7 @@ static int parse_options(int argc, char *argv[], test_global_t *global) {"udp_src", required_argument, NULL, 'o'}, {"udp_dst", required_argument, NULL, 'p'}, {"c_mode", required_argument, NULL, 'c'}, + {"mtu", required_argument, NULL, 'M'}, {"quit", required_argument, NULL, 'q'}, {"wait", required_argument, NULL, 'w'}, {"update_stat", required_argument, NULL, 'u'}, @@ -260,7 +263,7 @@ static int parse_options(int argc, char *argv[], test_global_t *global) {NULL, 0, NULL, 0} }; - static const char *shortopts = "+i:e:r:t:n:l:L:b:x:g:v:s:d:o:p:c:q:u:w:h"; + static const char *shortopts = "+i:e:r:t:n:l:L:M:b:x:g:v:s:d:o:p:c:q:u:w:h"; test_options->num_pktio = 0; test_options->num_rx = 1; @@ -285,6 +288,7 @@ static int parse_options(int argc, char *argv[], test_global_t *global) test_options->quit = 0; test_options->update_msec = 0; test_options->wait_sec = 0; + test_options->mtu = 0; for (i = 0; i < MAX_PKTIOS; i++) { memcpy(global->pktio[i].eth_dst.addr, default_eth_dst, 6); @@ -398,6 +402,9 @@ static int parse_options(int argc, char *argv[], test_global_t *global) test_options->rand_pkt_len_bins = val; test_options->use_rand_pkt_len = 1; break; + case 'M': + test_options->mtu = atoi(optarg); + break; case 'b': test_options->burst_size = atoi(optarg); break; @@ -631,6 +638,11 @@ static int open_pktios(test_global_t *global) test_options->rand_pkt_len_bins); else printf(" packet length %u bytes\n", pkt_len); + printf(" MTU: "); + if (test_options->mtu) + printf("%u bytes\n", test_options->mtu); + else + printf("interface default\n"); printf(" tx burst size %u\n", test_options->burst_size); printf(" tx bursts %u\n", test_options->bursts); printf(" tx burst gap %" PRIu64 " nsec\n", @@ -756,6 +768,41 @@ static int open_pktios(test_global_t *global) return -1; } + if (test_options->mtu) { + uint32_t maxlen_input = pktio_capa.maxlen.max_input ? test_options->mtu : 0; + uint32_t maxlen_output = pktio_capa.maxlen.max_output ? + test_options->mtu : 0; + + if (!pktio_capa.set_op.op.maxlen) { + ODPH_ERR("Error (%s): modifying interface MTU not supported.\n", + name); + return -1; + } + + if (maxlen_input && + (maxlen_input < pktio_capa.maxlen.min_input || + maxlen_input > pktio_capa.maxlen.max_input)) { + ODPH_ERR("Error (%s): unsupported MTU value %" PRIu32 " " + "(min %" PRIu32 ", max %" PRIu32 ")\n", name, maxlen_input, + pktio_capa.maxlen.min_input, pktio_capa.maxlen.max_input); + return -1; + } + if (maxlen_output && + (maxlen_output < pktio_capa.maxlen.min_output || + maxlen_output > pktio_capa.maxlen.max_output)) { + ODPH_ERR("Error (%s): unsupported MTU value %" PRIu32 " " + "(min %" PRIu32 ", max %" PRIu32 ")\n", name, + maxlen_output, pktio_capa.maxlen.min_output, + pktio_capa.maxlen.max_output); + return -1; + } + + if (odp_pktio_maxlen_set(pktio, maxlen_input, maxlen_output)) { + ODPH_ERR("Error (%s): setting MTU failed\n", name); + return -1; + } + } + odp_pktio_config_init(&pktio_config); pktio_config.parser.layer = ODP_PROTO_LAYER_ALL; diff --git a/test/validation/api/Makefile.am b/test/validation/api/Makefile.am index 8464ee027..9fb77bcf7 100644 --- a/test/validation/api/Makefile.am +++ b/test/validation/api/Makefile.am @@ -88,3 +88,5 @@ if test_installdir installcheck-local: $(DESTDIR)/$(testdir)/run-test.sh $(TESTNAME) endif + +.NOTPARALLEL: diff --git a/test/validation/api/classification/odp_classification_basic.c b/test/validation/api/classification/odp_classification_basic.c index c001445e7..d23093619 100644 --- a/test/validation/api/classification/odp_classification_basic.c +++ b/test/validation/api/classification/odp_classification_basic.c @@ -309,6 +309,9 @@ static void classification_test_pmr_composite_create(void) CU_ASSERT(odp_pmr_to_u64(pmr_composite) != odp_pmr_to_u64(ODP_PMR_INVALID)); + printf("\n"); + odp_cls_print_all(); + retval = odp_cls_pmr_destroy(pmr_composite); CU_ASSERT(retval == 0); diff --git a/test/validation/api/crypto/odp_crypto_test_inp.c b/test/validation/api/crypto/odp_crypto_test_inp.c index dd5813c61..7d6661be2 100644 --- a/test/validation/api/crypto/odp_crypto_test_inp.c +++ b/test/validation/api/crypto/odp_crypto_test_inp.c @@ -19,6 +19,8 @@ struct suite_context_s { odp_crypto_op_mode_t pref_mode; odp_pool_t pool; odp_queue_t queue; + odp_queue_type_t q_type; + odp_event_t (*compl_queue_deq)(void); }; static struct suite_context_s suite_context; @@ -208,9 +210,10 @@ static int alg_op(odp_packet_t pkt, odp_event_t event; odp_crypto_compl_t compl_event; - /* Poll completion queue for results */ + /* Get crypto completion event from compl_queue. */ + CU_ASSERT_FATAL(NULL != suite_context.compl_queue_deq); do { - event = odp_queue_deq(suite_context.queue); + event = suite_context.compl_queue_deq(); } while (event == ODP_EVENT_INVALID); CU_ASSERT(odp_event_is_valid(event) == 1); @@ -342,9 +345,10 @@ static int alg_packet_op_enq(odp_packet_t pkt, return rc; } - /* Poll completion queue for results */ + /* Get crypto completion event from compl_queue. */ + CU_ASSERT_FATAL(NULL != suite_context.compl_queue_deq); do { - event = odp_queue_deq(suite_context.queue); + event = suite_context.compl_queue_deq(); } while (event == ODP_EVENT_INVALID); CU_ASSERT(ODP_EVENT_PACKET == odp_event_type(event)); @@ -807,6 +811,15 @@ static int check_alg_support(odp_cipher_alg_t cipher, odp_auth_alg_t auth) return ODP_TEST_INACTIVE; } + if (suite_context.queue != ODP_QUEUE_INVALID) { + if (suite_context.q_type == ODP_QUEUE_TYPE_PLAIN && + capability.queue_type_plain == 0) + return ODP_TEST_INACTIVE; + if (suite_context.q_type == ODP_QUEUE_TYPE_SCHED && + capability.queue_type_sched == 0) + return ODP_TEST_INACTIVE; + } + if (suite_context.packet) { if (suite_context.op_mode == ODP_CRYPTO_SYNC && capability.sync_mode == ODP_SUPPORT_NO) @@ -2446,6 +2459,34 @@ static void crypto_test_check_alg_sha512(void) false); } +static odp_queue_t sched_compl_queue_create(void) +{ + odp_queue_param_t qparam; + + odp_queue_param_init(&qparam); + qparam.type = ODP_QUEUE_TYPE_SCHED; + qparam.sched.prio = ODP_SCHED_PRIO_DEFAULT; + qparam.sched.sync = ODP_SCHED_SYNC_PARALLEL; + qparam.sched.group = ODP_SCHED_GROUP_ALL; + + return odp_queue_create("crypto-out", &qparam); +} + +static odp_queue_t plain_compl_queue_create(void) +{ + return odp_queue_create("crypto-out", NULL); +} + +static odp_event_t sched_compl_queue_deq(void) +{ + return odp_schedule(NULL, ODP_SCHED_NO_WAIT); +} + +static odp_event_t plain_compl_queue_deq(void) +{ + return odp_queue_deq(suite_context.queue); +} + static int crypto_suite_sync_init(void) { suite_context.pool = odp_pool_lookup("packet_pool"); @@ -2457,16 +2498,45 @@ static int crypto_suite_sync_init(void) return 0; } -static int crypto_suite_async_init(void) +static int crypto_suite_async_plain_init(void) { + odp_queue_t out_queue; + suite_context.pool = odp_pool_lookup("packet_pool"); if (suite_context.pool == ODP_POOL_INVALID) return -1; - suite_context.queue = odp_queue_lookup("crypto-out"); - if (suite_context.queue == ODP_QUEUE_INVALID) + + out_queue = plain_compl_queue_create(); + if (ODP_QUEUE_INVALID == out_queue) { + fprintf(stderr, "Crypto outq creation failed.\n"); return -1; + } + suite_context.queue = out_queue; + suite_context.q_type = ODP_QUEUE_TYPE_PLAIN; + suite_context.compl_queue_deq = plain_compl_queue_deq; + suite_context.pref_mode = ODP_CRYPTO_ASYNC; + + return 0; +} +static int crypto_suite_async_sched_init(void) +{ + odp_queue_t out_queue; + + suite_context.pool = odp_pool_lookup("packet_pool"); + if (suite_context.pool == ODP_POOL_INVALID) + return -1; + + out_queue = sched_compl_queue_create(); + if (ODP_QUEUE_INVALID == out_queue) { + fprintf(stderr, "Crypto outq creation failed.\n"); + return -1; + } + suite_context.queue = out_queue; + suite_context.q_type = ODP_QUEUE_TYPE_SCHED; + suite_context.compl_queue_deq = sched_compl_queue_deq; suite_context.pref_mode = ODP_CRYPTO_ASYNC; + return 0; } @@ -2483,8 +2553,33 @@ static int crypto_suite_packet_sync_init(void) return 0; } -static int crypto_suite_packet_async_init(void) +static int crypto_suite_packet_async_plain_init(void) +{ + odp_queue_t out_queue; + + suite_context.packet = true; + suite_context.op_mode = ODP_CRYPTO_ASYNC; + + suite_context.pool = odp_pool_lookup("packet_pool"); + if (suite_context.pool == ODP_POOL_INVALID) + return -1; + + out_queue = plain_compl_queue_create(); + if (ODP_QUEUE_INVALID == out_queue) { + fprintf(stderr, "Crypto outq creation failed.\n"); + return -1; + } + suite_context.queue = out_queue; + suite_context.q_type = ODP_QUEUE_TYPE_PLAIN; + suite_context.compl_queue_deq = plain_compl_queue_deq; + + return 0; +} + +static int crypto_suite_packet_async_sched_init(void) { + odp_queue_t out_queue; + suite_context.packet = true; suite_context.op_mode = ODP_CRYPTO_ASYNC; @@ -2492,12 +2587,30 @@ static int crypto_suite_packet_async_init(void) if (suite_context.pool == ODP_POOL_INVALID) return -1; - suite_context.queue = odp_queue_lookup("crypto-out"); - if (suite_context.queue == ODP_QUEUE_INVALID) + out_queue = sched_compl_queue_create(); + if (ODP_QUEUE_INVALID == out_queue) { + fprintf(stderr, "Crypto outq creation failed.\n"); return -1; + } + suite_context.queue = out_queue; + suite_context.q_type = ODP_QUEUE_TYPE_SCHED; + suite_context.compl_queue_deq = sched_compl_queue_deq; + return 0; } +static int crypto_suite_term(void) +{ + if (ODP_QUEUE_INVALID != suite_context.queue) { + if (odp_queue_destroy(suite_context.queue)) + fprintf(stderr, "Crypto outq destroy failed.\n"); + } else { + fprintf(stderr, "Crypto outq not found.\n"); + } + + return odp_cunit_print_inactive(); +} + odp_testinfo_t crypto_suite[] = { ODP_TEST_INFO_CONDITIONAL(crypto_test_enc_alg_null, check_alg_null), @@ -2706,21 +2819,21 @@ odp_testinfo_t crypto_suite[] = { ODP_TEST_INFO_NULL, }; -/* Suite names */ -#define ODP_CRYPTO_SYNC_INP "odp_crypto_sync_inp" -#define ODP_CRYPTO_ASYNC_INP "odp_crypto_async_inp" -#define ODP_CRYPTO_PACKET_SYNC_INP "odp_crypto_packet_sync_inp" -#define ODP_CRYPTO_PACKET_ASYNC_INP "odp_crypto_packet_async_inp" - odp_suiteinfo_t crypto_suites[] = { - {ODP_CRYPTO_SYNC_INP, crypto_suite_sync_init, - NULL, crypto_suite}, - {ODP_CRYPTO_ASYNC_INP, crypto_suite_async_init, + {"odp_crypto_sync_inp", crypto_suite_sync_init, NULL, crypto_suite}, - {ODP_CRYPTO_PACKET_SYNC_INP, crypto_suite_packet_sync_init, - NULL, crypto_suite}, - {ODP_CRYPTO_PACKET_ASYNC_INP, crypto_suite_packet_async_init, + {"odp_crypto_async_plain_inp", crypto_suite_async_plain_init, + crypto_suite_term, crypto_suite}, + {"odp_crypto_async_sched_inp", crypto_suite_async_sched_init, + crypto_suite_term, crypto_suite}, + {"odp_crypto_packet_sync_inp", crypto_suite_packet_sync_init, NULL, crypto_suite}, + {"odp_crypto_packet_async_plain_inp", + crypto_suite_packet_async_plain_init, + crypto_suite_term, crypto_suite}, + {"odp_crypto_packet_async_sched_inp", + crypto_suite_packet_async_sched_init, + crypto_suite_term, crypto_suite}, ODP_SUITE_INFO_NULL, }; @@ -2728,9 +2841,7 @@ static int crypto_init(odp_instance_t *inst) { odp_pool_param_t params; odp_pool_t pool; - odp_queue_t out_queue; odp_pool_capability_t pool_capa; - odp_crypto_capability_t crypto_capa; odp_init_t init_param; odph_helper_options_t helper_options; @@ -2752,8 +2863,9 @@ static int crypto_init(odp_instance_t *inst) return -1; } - if (odp_crypto_capability(&crypto_capa)) { - fprintf(stderr, "error: odp_crypto_capability() failed.\n"); + /* Configure the scheduler. */ + if (odp_schedule_config(NULL)) { + fprintf(stderr, "odp_schedule_config() failed.\n"); return -1; } @@ -2786,11 +2898,6 @@ static int crypto_init(odp_instance_t *inst) fprintf(stderr, "Packet pool creation failed.\n"); return -1; } - out_queue = odp_queue_create("crypto-out", NULL); - if (ODP_QUEUE_INVALID == out_queue) { - fprintf(stderr, "Crypto outq creation failed.\n"); - return -1; - } return 0; } @@ -2798,15 +2905,6 @@ static int crypto_init(odp_instance_t *inst) static int crypto_term(odp_instance_t inst) { odp_pool_t pool; - odp_queue_t out_queue; - - out_queue = odp_queue_lookup("crypto-out"); - if (ODP_QUEUE_INVALID != out_queue) { - if (odp_queue_destroy(out_queue)) - fprintf(stderr, "Crypto outq destroy failed.\n"); - } else { - fprintf(stderr, "Crypto outq not found.\n"); - } pool = odp_pool_lookup("packet_pool"); if (ODP_POOL_INVALID != pool) { diff --git a/test/validation/api/ipsec/ipsec.c b/test/validation/api/ipsec/ipsec.c index 1458e9953..f36ca5769 100644 --- a/test/validation/api/ipsec/ipsec.c +++ b/test/validation/api/ipsec/ipsec.c @@ -1,5 +1,5 @@ /* Copyright (c) 2017-2018, Linaro Limited - * Copyright (c) 2019-2020, Nokia + * Copyright (c) 2018-2021, Nokia * Copyright (c) 2020, Marvell * All rights reserved. * @@ -16,9 +16,9 @@ #include "test_vectors.h" struct suite_context_s suite_context; +static odp_ipsec_capability_t capa; #define PKT_POOL_NUM 64 -#define PKT_POOL_LEN (1 * 1024) #define PACKET_USER_PTR ((void *)0x1212fefe) #define IPSEC_SA_CTX ((void *)0xfefefafa) @@ -124,11 +124,6 @@ int ipsec_check(odp_bool_t ah, odp_auth_alg_t auth, uint32_t auth_bits) { - odp_ipsec_capability_t capa; - - if (odp_ipsec_capability(&capa) < 0) - return ODP_TEST_INACTIVE; - if ((ODP_IPSEC_OP_MODE_SYNC == suite_context.inbound_op_mode && ODP_SUPPORT_NO == capa.op_mode_sync) || (ODP_IPSEC_OP_MODE_SYNC == suite_context.outbound_op_mode && @@ -288,9 +283,10 @@ void ipsec_sa_param_fill(odp_ipsec_sa_param_t *param, odp_ipsec_sa_param_init(param); param->dir = in ? ODP_IPSEC_DIR_INBOUND : ODP_IPSEC_DIR_OUTBOUND; - if (in) + if (in) { param->inbound.lookup_mode = ODP_IPSEC_LOOKUP_SPI; - + param->inbound.antireplay_ws = capa.max_antireplay_ws; + } param->proto = ah ? ODP_IPSEC_AH : ODP_IPSEC_ESP; @@ -321,6 +317,18 @@ void ipsec_sa_param_fill(odp_ipsec_sa_param_t *param, if (auth_key_extra) param->crypto.auth_key_extra = *auth_key_extra; + + /* + * Let's use arbitrary non-zero life time values to get life time + * checking code paths exercised. Let's not use very small values + * to avoid unexpected expiration with implementations that do + * not have packet-accurate life time checking but may report + * expiration a bit early. + */ + param->lifetime.soft_limit.bytes = 900 * 1000; + param->lifetime.hard_limit.bytes = 1000 * 1000; + param->lifetime.soft_limit.packets = 9000 * 1000; + param->lifetime.hard_limit.packets = 10000 * 1000; } void ipsec_sa_destroy(odp_ipsec_sa_t sa) @@ -817,6 +825,35 @@ static void ipsec_pkt_auth_err_set(odp_packet_t pkt) odp_packet_copy_from_mem(pkt, len - sizeof(data), sizeof(data), &data); } +static void ipsec_pkt_v4_check_udp_encap(odp_packet_t pkt) +{ + uint32_t l3_off = odp_packet_l3_offset(pkt); + odph_ipv4hdr_t ip; + + odp_packet_copy_to_mem(pkt, l3_off, sizeof(ip), &ip); + + CU_ASSERT(ip.proto == ODPH_IPPROTO_UDP); +} + +static void ipsec_pkt_v6_check_udp_encap(odp_packet_t pkt) +{ + uint32_t l3_off = odp_packet_l3_offset(pkt); + odph_ipv6hdr_ext_t ext; + odph_ipv6hdr_t ip; + uint8_t next_hdr; + + odp_packet_copy_to_mem(pkt, l3_off, sizeof(ip), &ip); + + next_hdr = ip.next_hdr; + + if (ip.next_hdr == ODPH_IPPROTO_HOPOPTS) { + odp_packet_copy_to_mem(pkt, l3_off + sizeof(ip), sizeof(ext), &ext); + next_hdr = ext.next_hdr; + } + + CU_ASSERT(next_hdr == ODPH_IPPROTO_UDP); +} + void ipsec_check_in_one(const ipsec_test_part *part, odp_ipsec_sa_t sa) { int num_out = part->num_pkt; @@ -961,6 +998,14 @@ void ipsec_check_out_in_one(const ipsec_test_part *part, if (part->flags.stats == IPSEC_TEST_STATS_AUTH_ERR) ipsec_pkt_auth_err_set(pkto[i]); + if (part->flags.udp_encap) { + if ((!part->flags.tunnel && part->flags.v6) || + (part->flags.tunnel && part->flags.tunnel_is_v6)) + ipsec_pkt_v6_check_udp_encap(pkto[i]); + else + ipsec_pkt_v4_check_udp_encap(pkto[i]); + } + pkt_in.len = odp_packet_len(pkto[i]); pkt_in.l2_offset = odp_packet_l2_offset(pkto[i]); pkt_in.l3_offset = odp_packet_l3_offset(pkto[i]); @@ -1049,19 +1094,19 @@ int ipsec_init(odp_instance_t *inst, odp_ipsec_op_mode_t mode) } odp_pool_param_init(¶ms); - params.pkt.seg_len = PKT_POOL_LEN; - params.pkt.len = PKT_POOL_LEN; + params.pkt.seg_len = MAX_PKT_LEN; + params.pkt.len = MAX_PKT_LEN; params.pkt.num = PKT_POOL_NUM; params.type = ODP_POOL_PACKET; if (pool_capa.pkt.max_seg_len && - PKT_POOL_LEN > pool_capa.pkt.max_seg_len) { + MAX_PKT_LEN > pool_capa.pkt.max_seg_len) { fprintf(stderr, "Warning: small packet segment length\n"); params.pkt.seg_len = pool_capa.pkt.max_seg_len; } if (pool_capa.pkt.max_len && - PKT_POOL_LEN > pool_capa.pkt.max_len) { + MAX_PKT_LEN > pool_capa.pkt.max_len) { fprintf(stderr, "Pool max packet length too small\n"); return -1; } @@ -1094,7 +1139,6 @@ int ipsec_init(odp_instance_t *inst, odp_ipsec_op_mode_t mode) int ipsec_config(odp_instance_t ODP_UNUSED inst) { - odp_ipsec_capability_t capa; odp_ipsec_config_t ipsec_config; if (odp_ipsec_capability(&capa) < 0) @@ -1118,6 +1162,7 @@ int ipsec_config(odp_instance_t ODP_UNUSED inst) return 0; odp_ipsec_config_init(&ipsec_config); + ipsec_config.max_num_sa = capa.max_num_sa; ipsec_config.inbound_mode = suite_context.inbound_op_mode; ipsec_config.outbound_mode = suite_context.outbound_op_mode; ipsec_config.outbound.all_chksum = ~0; diff --git a/test/validation/api/ipsec/ipsec.h b/test/validation/api/ipsec/ipsec.h index 3bbcb7b64..8b332b2f7 100644 --- a/test/validation/api/ipsec/ipsec.h +++ b/test/validation/api/ipsec/ipsec.h @@ -36,12 +36,14 @@ struct suite_context_s { extern struct suite_context_s suite_context; +#define MAX_PKT_LEN 1024 + typedef struct { uint32_t len; uint32_t l2_offset; uint32_t l3_offset; uint32_t l4_offset; - uint8_t data[256]; + uint8_t data[MAX_PKT_LEN]; } ipsec_test_packet; #define _ODP_PROTO_L3_TYPE_UNDEF ((odp_proto_l3_type_t)-1) @@ -60,6 +62,10 @@ typedef struct { odp_bool_t ah; odp_bool_t inline_hdr_in_packet; odp_bool_t test_sa_seq_num; + odp_bool_t v6; + odp_bool_t tunnel; + odp_bool_t tunnel_is_v6; + odp_bool_t udp_encap; enum ipsec_test_stats stats; } ipsec_test_flags; diff --git a/test/validation/api/ipsec/ipsec_test_out.c b/test/validation/api/ipsec/ipsec_test_out.c index 6f285d59a..5bc6b7a16 100644 --- a/test/validation/api/ipsec/ipsec_test_out.c +++ b/test/validation/api/ipsec/ipsec_test_out.c @@ -1,6 +1,6 @@ /* Copyright (c) 2017-2018, Linaro Limited * Copyright (c) 2020, Marvell - * Copyright (c) 2020, Nokia + * Copyright (c) 2020-2021, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -395,6 +395,18 @@ static void test_out_in_common(ipsec_test_flags *flags, const odp_crypto_key_t *cipher_key_extra, const odp_crypto_key_t *auth_key_extra) { + odp_ipsec_tunnel_param_t *tun_ptr = NULL; + odp_ipsec_tunnel_param_t tunnel; + uint32_t src_v4 = IPV4ADDR(10, 0, 111, 2); + uint32_t dst_v4 = IPV4ADDR(10, 0, 222, 2); + uint8_t src_v6[16] = { + 0x20, 0x01, 0x0d, 0xb8, 0x00, 0x00, 0x00, 0x00, + 0x02, 0x11, 0x43, 0xff, 0xfe, 0x4a, 0xd7, 0x0a, + }; + uint8_t dst_v6[16] = { + 0x20, 0x01, 0x0d, 0xb8, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x16, + }; odp_ipsec_sa_param_t param; odp_ipsec_stats_t stats; odp_ipsec_sa_t sa_out; @@ -407,22 +419,46 @@ static void test_out_in_common(ipsec_test_flags *flags, (auth == ODP_AUTH_ALG_NULL)) return; + if (flags->tunnel) { + if (flags->tunnel_is_v6) { + memset(&tunnel, 0, sizeof(odp_ipsec_tunnel_param_t)); + tunnel.type = ODP_IPSEC_TUNNEL_IPV6; + tunnel.ipv6.src_addr = &src_v6; + tunnel.ipv6.dst_addr = &dst_v6; + tunnel.ipv6.hlimit = 64; + tun_ptr = &tunnel; + } else { + memset(&tunnel, 0, sizeof(odp_ipsec_tunnel_param_t)); + tunnel.type = ODP_IPSEC_TUNNEL_IPV4; + tunnel.ipv4.src_addr = &src_v4; + tunnel.ipv4.dst_addr = &dst_v4; + tunnel.ipv4.ttl = 64; + tun_ptr = &tunnel; + } + } + ipsec_sa_param_fill(¶m, - false, flags->ah, 123, NULL, + false, flags->ah, 123, tun_ptr, cipher, cipher_key, auth, auth_key, cipher_key_extra, auth_key_extra); + if (flags->udp_encap) + param.opt.udp_encap = 1; + sa_out = odp_ipsec_sa_create(¶m); CU_ASSERT_NOT_EQUAL_FATAL(ODP_IPSEC_SA_INVALID, sa_out); ipsec_sa_param_fill(¶m, - true, flags->ah, 123, NULL, + true, flags->ah, 123, tun_ptr, cipher, cipher_key, auth, auth_key, cipher_key_extra, auth_key_extra); + if (flags->udp_encap) + param.opt.udp_encap = 1; + sa_in = odp_ipsec_sa_create(¶m); CU_ASSERT_NOT_EQUAL_FATAL(ODP_IPSEC_SA_INVALID, sa_in); @@ -446,6 +482,16 @@ static void test_out_in_common(ipsec_test_flags *flags, }, }; + if (flags->v6) { + test.pkt_in = &pkt_ipv6_icmp_0; + test.out[0].l3_type = ODP_PROTO_L3_TYPE_IPV6; + test.out[0].l4_type = ODP_PROTO_L4_TYPE_ICMPV6; + test.out[0].pkt_res = &pkt_ipv6_icmp_0; + test.in[0].l3_type = ODP_PROTO_L3_TYPE_IPV6; + test.in[0].l4_type = ODP_PROTO_L4_TYPE_ICMPV6; + test.in[0].pkt_res = &pkt_ipv6_icmp_0; + } + test.flags = *flags; if (flags->stats == IPSEC_TEST_STATS_PROTO_ERR) @@ -1495,6 +1541,167 @@ static void test_ipsec_stats(void) printf("\n "); } +static void test_udp_encap(void) +{ + ipsec_test_flags flags; + + memset(&flags, 0, sizeof(flags)); + flags.udp_encap = 1; + flags.tunnel = 0; + + printf("\n IPv4 Transport"); + flags.v6 = 0; + test_esp_out_in_all(&flags); + + printf("\n IPv6 Transport"); + flags.v6 = 1; + test_esp_out_in_all(&flags); + + flags.tunnel = 1; + + printf("\n IPv4-in-IPv4 Tunnel"); + flags.v6 = 0; + flags.tunnel_is_v6 = 0; + test_esp_out_in_all(&flags); + + printf("\n IPv4-in-IPv6 Tunnel"); + flags.v6 = 0; + flags.tunnel_is_v6 = 1; + test_esp_out_in_all(&flags); + + printf("\n IPv6-in-IPv4 Tunnel"); + flags.v6 = 1; + flags.tunnel_is_v6 = 0; + test_esp_out_in_all(&flags); + + printf("\n IPv6-in-IPv6 Tunnel"); + flags.v6 = 1; + flags.tunnel_is_v6 = 1; + test_esp_out_in_all(&flags); + + printf("\n "); +} + +static void test_max_num_sa(void) +{ + odp_ipsec_capability_t capa; + uint32_t sa_pairs; + odp_bool_t odd = false; + uint32_t n; + uint8_t cipher_key_data[128 / 8]; /* 128 bit key for AES */ + uint8_t auth_key_data[160 / 8]; /* 160 bit key for SHA-1 */ + odp_crypto_key_t cipher_key; + odp_crypto_key_t auth_key; + uint32_t tun_src; + uint32_t tun_dst; + odp_ipsec_tunnel_param_t tun = { + .type = ODP_IPSEC_TUNNEL_IPV4, + .ipv4.src_addr = &tun_src, + .ipv4.dst_addr = &tun_dst, + .ipv4.ttl = 64, + }; + odp_ipsec_sa_param_t param; + const uint32_t spi_start = 256; + odp_ipsec_sa_t sa_odd = ODP_IPSEC_SA_INVALID; + ipsec_test_part test = { + .pkt_in = &pkt_ipv4_icmp_0, + .flags = { + /* Test lookup now that we have lots of SAs */ + .lookup = 1, + }, + .num_pkt = 1, + .out = { + { .status.warn.all = 0, + .status.error.all = 0, + .l3_type = ODP_PROTO_L3_TYPE_IPV4, + .l4_type = ODP_PROTO_L4_TYPE_ICMPV4, + .pkt_res = &pkt_ipv4_icmp_0 }, + }, + .in = { + { .status.warn.all = 0, + .status.error.all = 0, + .l3_type = ODP_PROTO_L3_TYPE_IPV4, + .l4_type = ODP_PROTO_L4_TYPE_ICMPV4, + .pkt_res = &pkt_ipv4_icmp_0 }, + }, + }; + + CU_ASSERT_FATAL(odp_ipsec_capability(&capa) == 0); + sa_pairs = capa.max_num_sa / 2; + if (capa.max_num_sa > 2 && capa.max_num_sa % 2) + odd = true; + + odp_ipsec_sa_t sa_out[sa_pairs]; + odp_ipsec_sa_t sa_in[sa_pairs]; + + memset(cipher_key_data, 0xa5, sizeof(cipher_key_data)); + cipher_key.data = cipher_key_data; + cipher_key.length = sizeof(cipher_key_data); + + memset(auth_key_data, 0x5a, sizeof(auth_key_data)); + auth_key.data = auth_key_data; + auth_key.length = sizeof(auth_key_data); + + for (n = 0; n < sa_pairs; n++) { + /* Make keys unique */ + if (cipher_key.length > sizeof(n)) + memcpy(cipher_key.data, &n, sizeof(n)); + if (auth_key.length > sizeof(n)) + memcpy(auth_key.data, &n, sizeof(n)); + + /* These are for outbound SAs only */ + tun_src = 0x0a000000 + n; + tun_dst = 0x0a800000 + n; + + ipsec_sa_param_fill(¶m, + false, false, spi_start + n, &tun, + ODP_CIPHER_ALG_AES_CBC, &cipher_key, + ODP_AUTH_ALG_SHA1_HMAC, &auth_key, + NULL, NULL); + sa_out[n] = odp_ipsec_sa_create(¶m); + CU_ASSERT_FATAL(sa_out[n] != ODP_IPSEC_SA_INVALID); + + ipsec_sa_param_fill(¶m, + true, false, spi_start + n, &tun, + ODP_CIPHER_ALG_AES_CBC, &cipher_key, + ODP_AUTH_ALG_SHA1_HMAC, &auth_key, + NULL, NULL); + sa_in[n] = odp_ipsec_sa_create(¶m); + CU_ASSERT_FATAL(sa_in[n] != ODP_IPSEC_SA_INVALID); + } + + n = sa_pairs - 1; + if (odd) { + /* + * We have an odd number of max SAs. Let's create a similar + * SA as the last created outbound SA and test it against + * the last created inbound SA. + */ + tun_src = 0x0a000000 + n; + tun_dst = 0x0a800000 + n; + + ipsec_sa_param_fill(¶m, + false, false, spi_start + n, &tun, + ODP_CIPHER_ALG_AES_CBC, &cipher_key, + ODP_AUTH_ALG_SHA1_HMAC, &auth_key, + NULL, NULL); + sa_odd = odp_ipsec_sa_create(¶m); + CU_ASSERT_FATAL(sa_odd != ODP_IPSEC_SA_INVALID); + + ipsec_check_out_in_one(&test, sa_odd, sa_in[n]); + } + + for (n = 0; n < sa_pairs; n++) + ipsec_check_out_in_one(&test, sa_out[n], sa_in[n]); + + for (n = 0; n < sa_pairs; n++) { + ipsec_sa_destroy(sa_out[n]); + ipsec_sa_destroy(sa_in[n]); + } + if (odd) + ipsec_sa_destroy(sa_odd); +} + odp_testinfo_t ipsec_out_suite[] = { ODP_TEST_INFO(ipsec_test_capability), ODP_TEST_INFO(ipsec_test_default_values), @@ -1555,5 +1762,8 @@ odp_testinfo_t ipsec_out_suite[] = { is_out_mode_inline), ODP_TEST_INFO(test_ah_out_in_all), ODP_TEST_INFO(test_ipsec_stats), + ODP_TEST_INFO(test_udp_encap), + ODP_TEST_INFO_CONDITIONAL(test_max_num_sa, + ipsec_check_esp_aes_cbc_128_sha1), ODP_TEST_INFO_NULL, }; diff --git a/test/validation/api/pktio/lso.c b/test/validation/api/pktio/lso.c index e3ef57bf5..5d0596861 100644 --- a/test/validation/api/pktio/lso.c +++ b/test/validation/api/pktio/lso.c @@ -772,7 +772,6 @@ static void lso_send_ipv4(int use_opt) ODPH_DBG(" LSO segment[%i] payload: %u bytes\n", i, payload_len); - CU_ASSERT(odp_packet_has_ipv4(packet[i])); CU_ASSERT(odp_packet_has_ipfrag(packet[i])); CU_ASSERT(odp_packet_has_error(packet[i]) == 0); CU_ASSERT(payload_len <= IPV4_MAX_PAYLOAD); diff --git a/test/validation/api/pktio/pktio.c b/test/validation/api/pktio/pktio.c index 3f8df07f3..da87c7ebc 100644 --- a/test/validation/api/pktio/pktio.c +++ b/test/validation/api/pktio/pktio.c @@ -33,7 +33,6 @@ #define PKTIO_TS_INTERVAL (50 * ODP_TIME_MSEC_IN_NS) #define PKTIO_TS_MIN_RES 1000 #define PKTIO_TS_MAX_RES 10000000000 -#define PKTIO_TS_CMP_RES 1 #define PKTIO_SRC_MAC {1, 2, 3, 4, 5, 6} #define PKTIO_DST_MAC {6, 5, 4, 3, 2, 1} @@ -1654,6 +1653,7 @@ static void pktio_test_pktio_config(void) odp_pktio_t pktio; odp_pktio_capability_t capa; odp_pktio_config_t config; + const char *iface = iface_name[0]; pktio = create_pktio(0, ODP_PKTIN_MODE_DIRECT, ODP_PKTOUT_MODE_DIRECT); CU_ASSERT_FATAL(pktio != ODP_PKTIO_INVALID); @@ -1671,6 +1671,10 @@ static void pktio_test_pktio_config(void) CU_ASSERT_FATAL(odp_pktio_capability(pktio, &capa) == 0); + /* Loop interface supports loopback mode by definition */ + if (!strcmp(iface, "loop")) + CU_ASSERT(capa.config.enable_loop); + config = capa.config; /* Disable inbound_ipsec as it requires IPsec config to be done */ @@ -2310,7 +2314,7 @@ static void pktio_test_pktin_ts(void) odp_packet_t pkt_tbl[TX_BATCH_LEN]; uint32_t pkt_seq[TX_BATCH_LEN]; uint64_t ns1, ns2; - uint64_t res; + uint64_t res, res_ns; odp_time_t ts_prev; odp_time_t ts; int num_rx = 0; @@ -2351,9 +2355,11 @@ static void pktio_test_pktin_ts(void) ns1 = 100; ts = odp_pktio_ts_from_ns(pktio_tx, ns1); ns2 = odp_time_to_ns(ts); + res_ns = ODP_TIME_SEC_IN_NS / res; + if (ODP_TIME_SEC_IN_NS % res) + res_ns++; /* Allow some arithmetic tolerance */ - CU_ASSERT((ns2 <= (ns1 + PKTIO_TS_CMP_RES)) && - (ns2 >= (ns1 - PKTIO_TS_CMP_RES))); + CU_ASSERT((ns2 <= (ns1 + res_ns)) && (ns2 >= (ns1 - res_ns))); ret = create_packets(pkt_tbl, pkt_seq, TX_BATCH_LEN, pktio_tx, pktio_rx); @@ -2502,6 +2508,232 @@ static void pktio_test_pktout_ts(void) } } +static void pktio_test_pktout_compl(bool use_plain_queue) +{ + odp_pktio_t pktio[MAX_NUM_IFACES] = {ODP_PKTIO_INVALID}; + odp_queue_t compl_queue[TX_BATCH_LEN]; + odp_schedule_capability_t sched_capa; + odp_packet_t pkt_tbl[TX_BATCH_LEN]; + char queuename[ODP_QUEUE_NAME_LEN]; + odp_pktio_capability_t pktio_capa; + odp_queue_capability_t queue_capa; + uint16_t seq_found[TX_BATCH_LEN]; + odp_pktout_queue_t pktout_queue; + uint32_t pkt_seq[TX_BATCH_LEN]; + odp_pktio_t pktio_tx, pktio_rx; + odp_packet_tx_compl_t tx_compl; + odp_packet_tx_compl_opt_t opt; + pktio_info_t pktio_rx_info; + odp_pktio_config_t config; + odp_queue_param_t qparam; + int ret, i, num_rx = 0; + odp_event_t ev; + uint64_t wait; + + /* Create queues to receive PKTIO Tx completion events */ + CU_ASSERT_FATAL(!odp_schedule_capability(&sched_capa)); + CU_ASSERT_FATAL(!odp_queue_capability(&queue_capa)); + + for (i = 0; i < TX_BATCH_LEN; i++) { + sprintf(queuename, "TxComplQueue%u", i); + odp_queue_param_init(&qparam); + + if (use_plain_queue) { + qparam.type = ODP_QUEUE_TYPE_PLAIN; + } else { + qparam.type = ODP_QUEUE_TYPE_SCHED; + qparam.sched.prio = odp_schedule_default_prio(); + qparam.sched.sync = ODP_SCHED_SYNC_ATOMIC; + qparam.sched.group = ODP_SCHED_GROUP_ALL; + } + compl_queue[i] = odp_queue_create(queuename, &qparam); + CU_ASSERT_FATAL(compl_queue[i] != ODP_QUEUE_INVALID); + } + + memset(&pktout_queue, 0, sizeof(pktout_queue)); + CU_ASSERT_FATAL(num_ifaces >= 1); + + /* Open and configure interfaces */ + for (i = 0; i < num_ifaces; ++i) { + pktio[i] = create_pktio(i, ODP_PKTIN_MODE_DIRECT, + ODP_PKTOUT_MODE_DIRECT); + CU_ASSERT_FATAL(pktio[i] != ODP_PKTIO_INVALID); + + CU_ASSERT_FATAL(odp_pktio_capability(pktio[i], &pktio_capa) == 0); + + /* Configure Tx completion offload for PKTIO Tx */ + if (i == 0) { + CU_ASSERT_FATAL(pktio_capa.tx_compl.mode_all == 1); + if (use_plain_queue) { + /* CU_ASSERT needs these extra braces */ + CU_ASSERT_FATAL(pktio_capa.tx_compl.queue_type_plain != 0); + } else { + CU_ASSERT_FATAL(pktio_capa.tx_compl.queue_type_sched != 0); + } + + odp_pktio_config_init(&config); + config.pktout.bit.tx_compl_ena = 1; + CU_ASSERT_FATAL(odp_pktio_config(pktio[i], &config) == 0); + } + + CU_ASSERT_FATAL(odp_pktio_start(pktio[i]) == 0); + } + + for (i = 0; i < num_ifaces; i++) + _pktio_wait_linkup(pktio[i]); + + pktio_tx = pktio[0]; + pktio_rx = (num_ifaces > 1) ? pktio[1] : pktio_tx; + pktio_rx_info.id = pktio_rx; + pktio_rx_info.inq = ODP_QUEUE_INVALID; + pktio_rx_info.in_mode = ODP_PKTIN_MODE_DIRECT; + + ret = create_packets(pkt_tbl, pkt_seq, TX_BATCH_LEN, pktio_tx, pktio_rx); + CU_ASSERT_FATAL(ret == TX_BATCH_LEN); + + ret = odp_pktout_queue(pktio_tx, &pktout_queue, 1); + CU_ASSERT_FATAL(ret > 0); + + memset(&opt, 0, sizeof(opt)); + + /* Prepare batch of pkts with different tx completion queues */ + for (i = 0; i < TX_BATCH_LEN; i++) { + opt.queue = compl_queue[i]; + opt.mode = ODP_PACKET_TX_COMPL_ALL; + odp_packet_tx_compl_request(pkt_tbl[i], &opt); + + /* Set pkt sequence number as its user ptr */ + odp_packet_user_ptr_set(pkt_tbl[i], (const void *)&pkt_seq[i]); + } + + CU_ASSERT_FATAL(odp_pktout_send(pktout_queue, pkt_tbl, TX_BATCH_LEN) == TX_BATCH_LEN); + + num_rx = wait_for_packets(&pktio_rx_info, pkt_tbl, pkt_seq, TX_BATCH_LEN, TXRX_MODE_SINGLE, + ODP_TIME_SEC_IN_NS, false); + CU_ASSERT(num_rx == TX_BATCH_LEN); + for (i = 0; i < num_rx; i++) + odp_packet_free(pkt_tbl[i]); + + wait = odp_schedule_wait_time(ODP_TIME_SEC_IN_NS); + memset(seq_found, 0, sizeof(seq_found)); + + /* Receive Packet Tx completion events for all sent/dropped pkts */ + for (i = 0; i < TX_BATCH_LEN; i++) { + if (use_plain_queue) { + ev = odp_queue_deq(compl_queue[i]); + + /* Event validation */ + CU_ASSERT_FATAL(ev != ODP_EVENT_INVALID); + CU_ASSERT_FATAL(odp_event_type(ev) == ODP_EVENT_PACKET_TX_COMPL); + CU_ASSERT_FATAL(odp_packet_tx_compl_from_event(ev) != + ODP_PACKET_TX_COMPL_INVALID); + + tx_compl = odp_packet_tx_compl_from_event(ev); + CU_ASSERT_FATAL(odp_packet_tx_compl_to_event(tx_compl) == ev); + + /* User ptr should be same as packet's user ptr */ + CU_ASSERT(odp_packet_tx_compl_user_ptr(tx_compl) == + (const void *)&pkt_seq[i]); + + /* Alternatively call event free / compl free */ + if (i % 2) + odp_packet_tx_compl_free(tx_compl); + else + odp_event_free(ev); + } else { + odp_queue_t rcv_queue; + int j; + + ev = odp_schedule(&rcv_queue, wait); + + /* Event validation */ + CU_ASSERT_FATAL(ev != ODP_EVENT_INVALID); + CU_ASSERT_FATAL(odp_event_type(ev) == ODP_EVENT_PACKET_TX_COMPL); + CU_ASSERT_FATAL(odp_packet_tx_compl_from_event(ev) != + ODP_PACKET_TX_COMPL_INVALID); + + tx_compl = odp_packet_tx_compl_from_event(ev); + CU_ASSERT_FATAL(odp_packet_tx_compl_to_event(tx_compl) == ev); + + /* User ptr should be same as packet's user ptr i.e seq array ptr */ + for (j = 0; j < TX_BATCH_LEN; j++) { + if (!seq_found[j] && + ((const void *)&pkt_seq[j] == + odp_packet_tx_compl_user_ptr(tx_compl))) { + /* Mark that sequence number is found */ + seq_found[j] = 1; + + /* Receive queue validation */ + CU_ASSERT(rcv_queue == compl_queue[j]); + break; + } + } + /* Check that sequence number is found */ + CU_ASSERT(j < TX_BATCH_LEN); + + /* Alternatively call event free / compl free */ + if (i % 2) + odp_packet_tx_compl_free(tx_compl); + else + odp_event_free(ev); + } + } + + for (i = 0; i < TX_BATCH_LEN; i++) + odp_queue_destroy(compl_queue[i]); + + for (i = 0; i < num_ifaces; i++) { + CU_ASSERT_FATAL(odp_pktio_stop(pktio[i]) == 0); + CU_ASSERT_FATAL(odp_pktio_close(pktio[i]) == 0); + } +} + +static int pktio_check_pktout_compl(bool plain) +{ + odp_pktio_param_t pktio_param; + odp_pktio_capability_t capa; + odp_pktio_t pktio; + int ret; + + odp_pktio_param_init(&pktio_param); + pktio_param.in_mode = ODP_PKTIN_MODE_DIRECT; + pktio_param.out_mode = ODP_PKTOUT_MODE_DIRECT; + + pktio = odp_pktio_open(iface_name[0], pool[0], &pktio_param); + if (pktio == ODP_PKTIO_INVALID) + return ODP_TEST_INACTIVE; + + ret = odp_pktio_capability(pktio, &capa); + (void)odp_pktio_close(pktio); + + if (ret < 0 || !capa.tx_compl.mode_all || + (plain && !capa.tx_compl.queue_type_plain) || + (!plain && !capa.tx_compl.queue_type_sched)) + return ODP_TEST_INACTIVE; + + return ODP_TEST_ACTIVE; +} + +static int pktio_check_pktout_compl_plain_queue(void) +{ + return pktio_check_pktout_compl(true); +} + +static int pktio_check_pktout_compl_sched_queue(void) +{ + return pktio_check_pktout_compl(false); +} + +static void pktio_test_pktout_compl_plain_queue(void) +{ + pktio_test_pktout_compl(true); +} + +static void pktio_test_pktout_compl_sched_queue(void) +{ + pktio_test_pktout_compl(false); +} + static void pktio_test_chksum(void (*config_fn)(odp_pktio_t, odp_pktio_t), void (*prep_fn)(odp_packet_t pkt), void (*test_fn)(odp_packet_t pkt)) @@ -3467,6 +3699,111 @@ static void pktio_test_recv_maxlen_set(void) packet_len = PKT_LEN_NORMAL; } +static int pktio_check_pktout_aging_tmo(void) +{ + odp_pktio_param_t pktio_param; + odp_pktio_capability_t capa; + odp_pktio_t pktio; + int ret; + + odp_pktio_param_init(&pktio_param); + pktio_param.in_mode = ODP_PKTIN_MODE_DIRECT; + pktio_param.out_mode = ODP_PKTOUT_MODE_DIRECT; + + pktio = odp_pktio_open(iface_name[0], pool[0], &pktio_param); + if (pktio == ODP_PKTIO_INVALID) + return ODP_TEST_INACTIVE; + + ret = odp_pktio_capability(pktio, &capa); + (void)odp_pktio_close(pktio); + + if (ret < 0 || !capa.max_tx_aging_tmo_ns) + return ODP_TEST_INACTIVE; + + return ODP_TEST_ACTIVE; +} + +static void pktio_test_pktout_aging_tmo(void) +{ + odp_pktio_t pktio[MAX_NUM_IFACES] = {ODP_PKTIO_INVALID}; + odp_packet_t pkt_tbl[TX_BATCH_LEN]; + odp_pktio_capability_t pktio_capa; + odp_pktout_queue_t pktout_queue; + uint32_t pkt_seq[TX_BATCH_LEN]; + odp_pktio_t pktio_tx, pktio_rx; + pktio_info_t pktio_rx_info; + odp_pktio_config_t config; + int ret, i, num_rx = 0; + uint64_t tmo_0, tmo_1; + + /* Open and configure interfaces */ + for (i = 0; i < num_ifaces; ++i) { + pktio[i] = create_pktio(i, ODP_PKTIN_MODE_DIRECT, + ODP_PKTOUT_MODE_DIRECT); + CU_ASSERT_FATAL(pktio[i] != ODP_PKTIO_INVALID); + + CU_ASSERT_FATAL(odp_pktio_capability(pktio[i], &pktio_capa) == 0); + + /* Configure Tx aging for PKTIO Tx */ + if (i == 0) { + CU_ASSERT_FATAL(pktio_capa.max_tx_aging_tmo_ns > 0); + + odp_pktio_config_init(&config); + config.pktout.bit.aging_ena = 1; + CU_ASSERT_FATAL(odp_pktio_config(pktio[i], &config) == 0); + } + + CU_ASSERT_FATAL(odp_pktio_start(pktio[i]) == 0); + } + + for (i = 0; i < num_ifaces; i++) + _pktio_wait_linkup(pktio[i]); + + pktio_tx = pktio[0]; + pktio_rx = (num_ifaces > 1) ? pktio[1] : pktio_tx; + pktio_rx_info.id = pktio_rx; + pktio_rx_info.inq = ODP_QUEUE_INVALID; + pktio_rx_info.in_mode = ODP_PKTIN_MODE_DIRECT; + + ret = create_packets(pkt_tbl, pkt_seq, TX_BATCH_LEN, pktio_tx, + pktio_rx); + CU_ASSERT_FATAL(ret == TX_BATCH_LEN); + + ret = odp_pktout_queue(pktio_tx, &pktout_queue, 1); + CU_ASSERT_FATAL(ret > 0); + + /* Prepare packets with aging */ + for (i = 0; i < TX_BATCH_LEN; i++) { + /* Aging disabled by default */ + CU_ASSERT(odp_packet_aging_tmo(pkt_tbl[i]) == 0); + + /* Test tmo set relatively since we don't know about supported resolution */ + odp_packet_aging_tmo_set(pkt_tbl[i], pktio_capa.max_tx_aging_tmo_ns - 1); + tmo_0 = odp_packet_aging_tmo(pkt_tbl[i]); + + odp_packet_aging_tmo_set(pkt_tbl[i], pktio_capa.max_tx_aging_tmo_ns / 2); + tmo_1 = odp_packet_aging_tmo(pkt_tbl[i]); + CU_ASSERT(tmo_0 > tmo_1); + + /* Set max before transmitting */ + odp_packet_aging_tmo_set(pkt_tbl[i], pktio_capa.max_tx_aging_tmo_ns); + CU_ASSERT(odp_packet_aging_tmo(pkt_tbl[i]) != 0); + } + + CU_ASSERT_FATAL(odp_pktout_send(pktout_queue, pkt_tbl, TX_BATCH_LEN) == TX_BATCH_LEN); + + num_rx = wait_for_packets(&pktio_rx_info, pkt_tbl, pkt_seq, TX_BATCH_LEN, TXRX_MODE_SINGLE, + ODP_TIME_SEC_IN_NS, false); + CU_ASSERT(num_rx == TX_BATCH_LEN); + for (i = 0; i < num_rx; i++) + odp_packet_free(pkt_tbl[i]); + + for (i = 0; i < num_ifaces; i++) { + CU_ASSERT_FATAL(odp_pktio_stop(pktio[i]) == 0); + CU_ASSERT_FATAL(odp_pktio_close(pktio[i]) == 0); + } +} + static int pktio_suite_init(void) { int i; @@ -3654,6 +3991,12 @@ odp_testinfo_t pktio_suite_unsegmented[] = { pktio_check_chksum_out_sctp), ODP_TEST_INFO_CONDITIONAL(pktio_test_recv_maxlen_set, pktio_check_maxlen_set), + ODP_TEST_INFO_CONDITIONAL(pktio_test_pktout_aging_tmo, + pktio_check_pktout_aging_tmo), + ODP_TEST_INFO_CONDITIONAL(pktio_test_pktout_compl_plain_queue, + pktio_check_pktout_compl_plain_queue), + ODP_TEST_INFO_CONDITIONAL(pktio_test_pktout_compl_sched_queue, + pktio_check_pktout_compl_sched_queue), ODP_TEST_INFO_NULL }; diff --git a/test/validation/api/queue/queue.c b/test/validation/api/queue/queue.c index b5d594a9a..6f59cbf71 100644 --- a/test/validation/api/queue/queue.c +++ b/test/validation/api/queue/queue.c @@ -1,4 +1,5 @@ /* Copyright (c) 2014-2018, Linaro Limited + * Copyright (c) 2021, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -121,6 +122,19 @@ static int queue_suite_term(void) static void queue_test_capa(void) { odp_queue_capability_t capa; + + memset(&capa, 0, sizeof(odp_queue_capability_t)); + CU_ASSERT_FATAL(odp_queue_capability(&capa) == 0); + + CU_ASSERT(capa.max_queues > 0); + CU_ASSERT(capa.max_queues >= capa.plain.max_num); + CU_ASSERT(capa.max_queues >= capa.plain.lockfree.max_num); + CU_ASSERT(capa.max_queues >= capa.plain.waitfree.max_num); +} + +static void queue_test_max_plain(void) +{ + odp_queue_capability_t capa; odp_queue_param_t qparams; char name[ODP_QUEUE_NAME_LEN]; odp_queue_t queue[MAX_QUEUES]; @@ -600,6 +614,7 @@ static void queue_test_param(void) CU_ASSERT(qparams.sched.sync == ODP_SCHED_SYNC_PARALLEL); CU_ASSERT(qparams.sched.group == ODP_SCHED_GROUP_ALL); CU_ASSERT(qparams.sched.lock_count == 0); + CU_ASSERT(qparams.order == ODP_QUEUE_ORDER_KEEP); CU_ASSERT(qparams.nonblocking == ODP_BLOCKING); CU_ASSERT(qparams.context == NULL); CU_ASSERT(qparams.context_len == 0); @@ -612,7 +627,7 @@ static void queue_test_param(void) qparams.sched.group = ODP_SCHED_GROUP_WORKER; queue = odp_queue_create("test_queue", &qparams); - CU_ASSERT(ODP_QUEUE_INVALID != queue); + CU_ASSERT_FATAL(ODP_QUEUE_INVALID != queue); CU_ASSERT(odp_queue_to_u64(queue) != odp_queue_to_u64(ODP_QUEUE_INVALID)); CU_ASSERT(queue == odp_queue_lookup("test_queue")); @@ -631,7 +646,7 @@ static void queue_test_param(void) /* Create queue with no name */ odp_queue_param_init(&qparams); null_queue = odp_queue_create(NULL, &qparams); - CU_ASSERT(ODP_QUEUE_INVALID != null_queue); + CU_ASSERT_FATAL(ODP_QUEUE_INVALID != null_queue); CU_ASSERT(odp_queue_context(null_queue) == NULL); /* Plain type queue */ @@ -641,7 +656,7 @@ static void queue_test_param(void) qparams.context_len = sizeof(queue_context); queue = odp_queue_create("test_queue", &qparams); - CU_ASSERT(ODP_QUEUE_INVALID != queue); + CU_ASSERT_FATAL(ODP_QUEUE_INVALID != queue); CU_ASSERT(queue == odp_queue_lookup("test_queue")); CU_ASSERT(ODP_QUEUE_TYPE_PLAIN == odp_queue_type(queue)); CU_ASSERT(&queue_context == odp_queue_context(queue)); @@ -712,7 +727,7 @@ static void queue_test_info(void) /* Create a plain queue and set context */ q_plain = odp_queue_create(nq_plain, NULL); - CU_ASSERT(ODP_QUEUE_INVALID != q_plain); + CU_ASSERT_FATAL(ODP_QUEUE_INVALID != q_plain); CU_ASSERT(odp_queue_context_set(q_plain, q_plain_ctx, sizeof(q_plain_ctx)) == 0); @@ -730,7 +745,7 @@ static void queue_test_info(void) printf("\n Ordered locks NOT supported\n"); param.context = q_order_ctx; q_order = odp_queue_create(nq_order, ¶m); - CU_ASSERT(ODP_QUEUE_INVALID != q_order); + CU_ASSERT_FATAL(ODP_QUEUE_INVALID != q_order); /* Check info and call print for a plain queue */ CU_ASSERT(odp_queue_info(q_plain, &info) == 0); @@ -967,6 +982,7 @@ static void queue_test_mt_plain_nonblock_lf(void) odp_testinfo_t queue_suite[] = { ODP_TEST_INFO(queue_test_capa), ODP_TEST_INFO(queue_test_mode), + ODP_TEST_INFO(queue_test_max_plain), ODP_TEST_INFO(queue_test_burst), ODP_TEST_INFO(queue_test_burst_spmc), ODP_TEST_INFO(queue_test_burst_mpsc), diff --git a/test/validation/api/scheduler/scheduler.c b/test/validation/api/scheduler/scheduler.c index 4cbede9cd..8800350f2 100644 --- a/test/validation/api/scheduler/scheduler.c +++ b/test/validation/api/scheduler/scheduler.c @@ -1,5 +1,5 @@ /* Copyright (c) 2014-2018, Linaro Limited - * Copyright (c) 2019-2020, Nokia + * Copyright (c) 2019-2021, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -67,7 +67,8 @@ typedef struct { int buf_count; int buf_count_cpy; int queues_per_prio; - uint32_t num_queues; + odp_shm_t shm_glb; + odp_shm_t shm_args; odp_pool_t pool; odp_pool_t queue_ctx_pool; uint32_t max_sched_queue_size; @@ -163,6 +164,15 @@ static void scheduler_test_capa(void) CU_ASSERT(queue_capa.max_queues >= sched_capa.max_queues); } +static void sched_queue_param_init(odp_queue_param_t *param) +{ + odp_queue_param_init(param); + param->type = ODP_QUEUE_TYPE_SCHED; + param->sched.sync = ODP_SCHED_SYNC_PARALLEL; + param->sched.prio = odp_schedule_default_prio(); + param->sched.group = ODP_SCHED_GROUP_ALL; +} + static void scheduler_test_wait_time(void) { int i; @@ -179,11 +189,7 @@ static void scheduler_test_wait_time(void) wait_time = odp_schedule_wait_time(1); /* check ODP_SCHED_NO_WAIT */ - odp_queue_param_init(&qp); - qp.type = ODP_QUEUE_TYPE_SCHED; - qp.sched.sync = ODP_SCHED_SYNC_PARALLEL; - qp.sched.prio = odp_schedule_default_prio(); - qp.sched.group = ODP_SCHED_GROUP_ALL; + sched_queue_param_init(&qp); queue = odp_queue_create("dummy_queue", &qp); CU_ASSERT_FATAL(queue != ODP_QUEUE_INVALID); @@ -264,7 +270,6 @@ static void scheduler_test_queue_destroy(void) ODP_SCHED_SYNC_ATOMIC, ODP_SCHED_SYNC_ORDERED}; - odp_queue_param_init(&qp); odp_pool_param_init(¶ms); params.buf.size = 100; params.buf.align = 0; @@ -275,12 +280,10 @@ static void scheduler_test_queue_destroy(void) CU_ASSERT_FATAL(p != ODP_POOL_INVALID); - for (i = 0; i < 3; i++) { - qp.type = ODP_QUEUE_TYPE_SCHED; - qp.sched.prio = odp_schedule_default_prio(); - qp.sched.sync = sync[i]; - qp.sched.group = ODP_SCHED_GROUP_ALL; + sched_queue_param_init(&qp); + for (i = 0; i < 3; i++) { + qp.sched.sync = sync[i]; queue = odp_queue_create("sched_destroy_queue", &qp); CU_ASSERT_FATAL(queue != ODP_QUEUE_INVALID); @@ -342,12 +345,7 @@ static void scheduler_test_wait(void) CU_ASSERT_FATAL(p != ODP_POOL_INVALID); - odp_queue_param_init(&queue_param); - queue_param.type = ODP_QUEUE_TYPE_SCHED; - queue_param.sched.prio = odp_schedule_default_prio(); - queue_param.sched.sync = ODP_SCHED_SYNC_PARALLEL; - queue_param.sched.group = ODP_SCHED_GROUP_ALL; - + sched_queue_param_init(&queue_param); queue = odp_queue_create("sched_test_wait", &queue_param); CU_ASSERT_FATAL(queue != ODP_QUEUE_INVALID); @@ -464,11 +462,8 @@ static void scheduler_test_queue_size(void) j++; } - odp_queue_param_init(&queue_param); - queue_param.type = ODP_QUEUE_TYPE_SCHED; - queue_param.sched.prio = odp_schedule_default_prio(); + sched_queue_param_init(&queue_param); queue_param.sched.sync = sync[i]; - queue_param.sched.group = ODP_SCHED_GROUP_ALL; queue_param.size = queue_size; queue = odp_queue_create("test_queue_size", &queue_param); @@ -536,10 +531,6 @@ static void scheduler_test_full_queues(void) queue_size = default_config.queue_size; num_queues = default_config.num_queues; - /* Queues reserved by create_queues() in suite init */ - CU_ASSERT_FATAL(globals->num_queues < num_queues); - num_queues -= globals->num_queues; - odp_queue_t queue[num_queues]; CU_ASSERT_FATAL(!odp_pool_capability(&pool_capa)); @@ -576,11 +567,8 @@ static void scheduler_test_full_queues(void) /* Create and fill all queues */ for (j = 0; j < num_queues; j++) { - odp_queue_param_init(&queue_param); - queue_param.type = ODP_QUEUE_TYPE_SCHED; - queue_param.sched.prio = odp_schedule_default_prio(); + sched_queue_param_init(&queue_param); queue_param.sched.sync = sync[i]; - queue_param.sched.group = ODP_SCHED_GROUP_ALL; queue_param.size = events_per_queue; queue[j] = odp_queue_create("test_full_queues", @@ -641,6 +629,117 @@ static void scheduler_test_full_queues(void) CU_ASSERT(odp_pool_destroy(pool) == 0); } +static void scheduler_test_max_queues(odp_schedule_sync_t sync) +{ + odp_pool_t pool; + odp_pool_param_t pool_param; + odp_schedule_capability_t sched_capa; + odp_queue_param_t queue_param; + odp_buffer_t buf; + odp_event_t ev; + odp_queue_t src_queue; + uint64_t wait_time; + uint32_t i, src_idx; + uint32_t num_rounds = 4; + uint32_t num_queues = 64 * 1024; + + CU_ASSERT_FATAL(odp_schedule_capability(&sched_capa) == 0); + if (num_queues > sched_capa.max_queues) + num_queues = sched_capa.max_queues; + + CU_ASSERT_FATAL(num_queues > 0); + + odp_queue_t queue[num_queues]; + + odp_pool_param_init(&pool_param); + pool_param.type = ODP_POOL_BUFFER; + pool_param.buf.size = 100; + pool_param.buf.num = 1; + + pool = odp_pool_create("test_max_queues", &pool_param); + CU_ASSERT_FATAL(pool != ODP_POOL_INVALID); + + /* Ensure that scheduler is empty */ + drain_queues(); + + sched_queue_param_init(&queue_param); + queue_param.sched.sync = sync; + + for (i = 0; i < num_queues; i++) { + queue[i] = odp_queue_create("test_max_queues", &queue_param); + if (queue[i] == ODP_QUEUE_INVALID) + ODPH_ERR("Queue create failed %u/%u\n", i, num_queues); + + CU_ASSERT_FATAL(queue[i] != ODP_QUEUE_INVALID); + } + + buf = odp_buffer_alloc(pool); + CU_ASSERT_FATAL(buf != ODP_BUFFER_INVALID); + ev = odp_buffer_to_event(buf); + + CU_ASSERT_FATAL(odp_queue_enq(queue[0], ev) == 0); + + wait_time = odp_schedule_wait_time(500 * ODP_TIME_MSEC_IN_NS); + src_idx = 0; + + /* Send one event through all queues couple of times */ + for (i = 0; i < (num_rounds * num_queues); i++) { + uint32_t round = i / num_queues; + + ev = odp_schedule(&src_queue, wait_time); + if (ev == ODP_EVENT_INVALID) { + ODPH_ERR("Event was lost. Round %u, queue idx %u\n", round, src_idx); + CU_FAIL("Event was lost\n"); + break; + } + + CU_ASSERT(src_queue == queue[src_idx]); + + src_idx++; + if (src_idx == num_queues) + src_idx = 0; + + if (odp_queue_enq(queue[src_idx], ev)) { + ODPH_ERR("Enqueue failed. Round %u, queue idx %u\n", round, src_idx); + CU_FAIL("Enqueue failed\n") + odp_event_free(ev); + break; + } + } + + /* Free event and scheduling context */ + for (i = 0; i < 2; i++) { + ev = odp_schedule(NULL, wait_time); + + if (ev == ODP_EVENT_INVALID) + continue; + + odp_event_free(ev); + } + + CU_ASSERT(drain_queues() == 0); + + for (i = 0; i < num_queues; i++) + CU_ASSERT_FATAL(odp_queue_destroy(queue[i]) == 0); + + CU_ASSERT(odp_pool_destroy(pool) == 0); +} + +static void scheduler_test_max_queues_p(void) +{ + scheduler_test_max_queues(ODP_SCHED_SYNC_PARALLEL); +} + +static void scheduler_test_max_queues_a(void) +{ + scheduler_test_max_queues(ODP_SCHED_SYNC_ATOMIC); +} + +static void scheduler_test_max_queues_o(void) +{ + scheduler_test_max_queues(ODP_SCHED_SYNC_ORDERED); +} + static void scheduler_test_order_ignore(void) { odp_queue_capability_t queue_capa; @@ -687,11 +786,8 @@ static void scheduler_test_order_ignore(void) j++; } - odp_queue_param_init(&queue_param); - queue_param.type = ODP_QUEUE_TYPE_SCHED; - queue_param.sched.prio = odp_schedule_default_prio(); - queue_param.sched.sync = ODP_SCHED_SYNC_ORDERED; - queue_param.sched.group = ODP_SCHED_GROUP_ALL; + sched_queue_param_init(&queue_param); + queue_param.sched.sync = ODP_SCHED_SYNC_ORDERED; ordered = odp_queue_create("ordered", &queue_param); CU_ASSERT_FATAL(ordered != ODP_QUEUE_INVALID); @@ -784,9 +880,7 @@ static void scheduler_test_create_group(void) pool = odp_pool_create("create_group", &pool_params); CU_ASSERT_FATAL(pool != ODP_POOL_INVALID); - odp_queue_param_init(&qp); - qp.type = ODP_QUEUE_TYPE_SCHED; - qp.sched.prio = odp_schedule_default_prio(); + sched_queue_param_init(&qp); qp.sched.sync = ODP_SCHED_SYNC_ATOMIC; qp.sched.group = group; @@ -840,9 +934,7 @@ static void scheduler_test_create_max_groups(void) odp_thrmask_zero(&mask); odp_thrmask_set(&mask, thr_id); - odp_queue_param_init(&queue_param); - queue_param.type = ODP_QUEUE_TYPE_SCHED; - queue_param.sched.prio = odp_schedule_default_prio(); + sched_queue_param_init(&queue_param); queue_param.sched.sync = ODP_SCHED_SYNC_ATOMIC; for (i = 0; i < max_groups; i++) { @@ -986,9 +1078,7 @@ static void scheduler_test_groups(void) int num = NUM_GROUPS; int schedule_retries; - odp_queue_param_init(&qp); - qp.type = ODP_QUEUE_TYPE_SCHED; - qp.sched.prio = odp_schedule_default_prio(); + sched_queue_param_init(&qp); qp.sched.sync = sync[i]; qp.sched.group = mygrp1; @@ -1212,7 +1302,6 @@ static void chaos_run(unsigned int qtype) args->globals = globals; - odp_queue_param_init(&qp); odp_pool_param_init(¶ms); params.buf.size = sizeof(chaos_buf); params.buf.align = 0; @@ -1221,9 +1310,8 @@ static void chaos_run(unsigned int qtype) pool = odp_pool_create("sched_chaos_pool", ¶ms); CU_ASSERT_FATAL(pool != ODP_POOL_INVALID); - qp.type = ODP_QUEUE_TYPE_SCHED; - qp.sched.prio = odp_schedule_default_prio(); - qp.sched.group = ODP_SCHED_GROUP_ALL; + + sched_queue_param_init(&qp); for (i = 0; i < CHAOS_NUM_QUEUES; i++) { uint32_t ndx = (qtype == num_sync ? i % num_sync : qtype); @@ -1915,6 +2003,7 @@ static void scheduler_test_multi_1q_mt_a_excl(void) static void scheduler_test_pause_resume(void) { + odp_queue_param_t qp; odp_queue_t queue; odp_buffer_t buf; odp_event_t ev; @@ -1924,7 +2013,8 @@ static void scheduler_test_pause_resume(void) int local_bufs = 0; int ret; - queue = odp_queue_lookup("sched_0_0_n"); + sched_queue_param_init(&qp); + queue = odp_queue_create("pause_resume", &qp); CU_ASSERT_FATAL(queue != ODP_QUEUE_INVALID); pool = odp_pool_lookup(MSG_POOL_NAME); @@ -1972,10 +2062,12 @@ static void scheduler_test_pause_resume(void) } CU_ASSERT(drain_queues() == 0); + CU_ASSERT(odp_queue_destroy(queue) == 0); } static void scheduler_test_pause_enqueue(void) { + odp_queue_param_t qp; odp_queue_t queue; odp_buffer_t buf; odp_event_t ev; @@ -1986,7 +2078,8 @@ static void scheduler_test_pause_enqueue(void) int ret; int local_bufs; - queue = odp_queue_lookup("sched_0_0_n"); + sched_queue_param_init(&qp); + queue = odp_queue_create("pause_enqueue", &qp); CU_ASSERT_FATAL(queue != ODP_QUEUE_INVALID); pool = odp_pool_lookup(MSG_POOL_NAME); @@ -2070,11 +2163,13 @@ static void scheduler_test_pause_enqueue(void) /* Free all */ CU_ASSERT(drain_queues() == NUM_BUFS_PAUSE); + CU_ASSERT(odp_queue_destroy(queue) == 0); } /* Basic, single threaded ordered lock API testing */ static void scheduler_test_ordered_lock(void) { + odp_queue_param_t qp; odp_queue_t queue; odp_buffer_t buf; odp_event_t ev; @@ -2083,8 +2178,15 @@ static void scheduler_test_ordered_lock(void) int i; int ret; uint32_t lock_count; + odp_schedule_capability_t sched_capa; + + CU_ASSERT_FATAL(!odp_schedule_capability(&sched_capa)); - queue = odp_queue_lookup("sched_0_0_o"); + sched_queue_param_init(&qp); + qp.sched.sync = ODP_SCHED_SYNC_ORDERED; + qp.sched.lock_count = sched_capa.max_ordered_locks; + + queue = odp_queue_create("ordered_lock", &qp); CU_ASSERT_FATAL(queue != ODP_QUEUE_INVALID); CU_ASSERT_FATAL(odp_queue_type(queue) == ODP_QUEUE_TYPE_SCHED); CU_ASSERT_FATAL(odp_queue_sched_type(queue) == ODP_SCHED_SYNC_ORDERED); @@ -2093,6 +2195,7 @@ static void scheduler_test_ordered_lock(void) if (lock_count == 0) { printf(" NO ORDERED LOCKS. Ordered locks not tested.\n"); + CU_ASSERT(odp_queue_destroy(queue) == 0); return; } @@ -2124,6 +2227,7 @@ static void scheduler_test_ordered_lock(void) if (lock_count < 2) { printf(" ONLY ONE ORDERED LOCK. Unlock_lock not tested.\n"); CU_ASSERT(drain_queues() == BUFS_PER_QUEUE / 2); + CU_ASSERT(odp_queue_destroy(queue) == 0); return; } @@ -2140,6 +2244,7 @@ static void scheduler_test_ordered_lock(void) } CU_ASSERT(drain_queues() == 0); + CU_ASSERT(odp_queue_destroy(queue) == 0); } static int sched_and_plain_thread(void *arg) @@ -2258,8 +2363,7 @@ static void scheduler_test_sched_and_plain(odp_schedule_sync_t sync) queue_capa.plain.max_size < events_per_queue) events_per_queue = queue_capa.plain.max_size; - odp_queue_param_init(&queue_param); - queue_param.type = ODP_QUEUE_TYPE_SCHED; + sched_queue_param_init(&queue_param); queue_param.sched.sync = sync; queue_param.size = events_per_queue; if (sync == ODP_SCHED_SYNC_ORDERED) @@ -2556,105 +2660,6 @@ static int create_queues(test_globals_t *globals) } } } - globals->num_queues = prios * queues_per_prio * 4; - - return 0; -} - -static int scheduler_suite_init(void) -{ - odp_cpumask_t mask; - odp_shm_t shm; - odp_pool_t pool; - thread_args_t *args; - odp_pool_param_t params; - uint64_t num_flows; - odp_schedule_capability_t sched_capa; - odp_schedule_config_t sched_config; - - if (odp_schedule_capability(&sched_capa)) { - printf("odp_schedule_capability() failed\n"); - return -1; - } - - num_flows = 0; - odp_schedule_config_init(&sched_config); - - /* Enable flow aware scheduling */ - if (sched_capa.max_flow_id > 0) { - num_flows = MAX_FLOWS; - if ((MAX_FLOWS - 1) > sched_capa.max_flow_id) - num_flows = sched_capa.max_flow_id + 1; - - sched_config.max_flow_id = num_flows - 1; - } - - /* Configure the scheduler. All test cases share the config. */ - if (odp_schedule_config(&sched_config)) { - printf("odp_schedule_config() failed.\n"); - return -1; - } - - odp_pool_param_init(¶ms); - params.buf.size = BUF_SIZE; - params.buf.align = 0; - params.buf.num = MSG_POOL_SIZE; - params.type = ODP_POOL_BUFFER; - - pool = odp_pool_create(MSG_POOL_NAME, ¶ms); - - if (pool == ODP_POOL_INVALID) { - printf("Pool creation failed (msg).\n"); - return -1; - } - - shm = odp_shm_reserve(GLOBALS_SHM_NAME, - sizeof(test_globals_t), ODP_CACHE_LINE_SIZE, 0); - - if (shm == ODP_SHM_INVALID) { - printf("Shared memory reserve failed (globals).\n"); - return -1; - } - - globals = odp_shm_addr(shm); - - if (!globals) { - printf("Shared memory reserve failed (globals).\n"); - return -1; - } - - memset(globals, 0, sizeof(test_globals_t)); - - globals->num_flows = num_flows; - - globals->num_workers = odp_cpumask_default_worker(&mask, 0); - if (globals->num_workers > MAX_WORKERS) - globals->num_workers = MAX_WORKERS; - - shm = odp_shm_reserve(SHM_THR_ARGS_NAME, sizeof(thread_args_t), - ODP_CACHE_LINE_SIZE, 0); - - if (shm == ODP_SHM_INVALID) { - printf("Shared memory reserve failed (args).\n"); - return -1; - } - - args = odp_shm_addr(shm); - - if (!args) { - printf("Shared memory reserve failed (args).\n"); - return -1; - } - - memset(args, 0, sizeof(thread_args_t)); - - /* Barrier to sync test case execution */ - odp_barrier_init(&globals->barrier, globals->num_workers); - odp_ticketlock_init(&globals->lock); - odp_spinlock_init(&globals->atomic_lock); - - if (create_queues(globals) != 0) - return -1; return 0; } @@ -2711,34 +2716,6 @@ static int destroy_queues(void) return 0; } -static int scheduler_suite_term(void) -{ - odp_pool_t pool; - odp_shm_t shm; - - if (destroy_queues() != 0) { - ODPH_ERR("Failed to destroy queues\n"); - return -1; - } - - pool = odp_pool_lookup(MSG_POOL_NAME); - if (odp_pool_destroy(pool) != 0) - ODPH_ERR("Failed to destroy pool\n"); - - shm = odp_shm_lookup(SHM_THR_ARGS_NAME); - if (odp_shm_free(shm) != 0) - ODPH_ERR("Failed to free shm\n"); - - shm = odp_shm_lookup(GLOBALS_SHM_NAME); - if (odp_shm_free(shm) != 0) - ODPH_ERR("Failed to free shm\n"); - - if (odp_cunit_print_inactive()) - return -1; - - return 0; -} - static int check_flow_aware_support(void) { if (globals->num_flows == 0) { @@ -2792,11 +2769,8 @@ static void scheduler_test_flow_aware(void) memset(flow_stat, 0, sizeof(flow_stat)); flow_id = 0; - odp_queue_param_init(&queue_param); - queue_param.type = ODP_QUEUE_TYPE_SCHED; - queue_param.sched.prio = odp_schedule_default_prio(); + sched_queue_param_init(&queue_param); queue_param.sched.sync = sync[i]; - queue_param.sched.group = ODP_SCHED_GROUP_ALL; queue_param.size = queue_size; queue = odp_queue_create("test_flow_aware", &queue_param); @@ -2859,8 +2833,196 @@ static void scheduler_test_flow_aware(void) CU_ASSERT(odp_pool_destroy(pool) == 0); } -/* Default scheduler config */ -odp_testinfo_t scheduler_suite[] = { +static int scheduler_test_global_init(void) +{ + odp_cpumask_t mask; + odp_shm_t shm; + thread_args_t *args; + odp_pool_t pool; + odp_pool_param_t params; + uint64_t num_flows; + odp_schedule_capability_t sched_capa; + odp_schedule_config_t sched_config; + + shm = odp_shm_reserve(GLOBALS_SHM_NAME, + sizeof(test_globals_t), ODP_CACHE_LINE_SIZE, 0); + + if (shm == ODP_SHM_INVALID) { + printf("Shared memory reserve failed (globals).\n"); + return -1; + } + + globals = odp_shm_addr(shm); + + if (!globals) { + printf("Shared memory reserve failed (globals).\n"); + return -1; + } + + memset(globals, 0, sizeof(test_globals_t)); + globals->shm_glb = shm; + + globals->num_workers = odp_cpumask_default_worker(&mask, 0); + if (globals->num_workers > MAX_WORKERS) + globals->num_workers = MAX_WORKERS; + + shm = odp_shm_reserve(SHM_THR_ARGS_NAME, sizeof(thread_args_t), + ODP_CACHE_LINE_SIZE, 0); + + if (shm == ODP_SHM_INVALID) { + printf("Shared memory reserve failed (args).\n"); + return -1; + } + + args = odp_shm_addr(shm); + globals->shm_args = shm; + + if (!args) { + printf("Shared memory reserve failed (args).\n"); + return -1; + } + + memset(args, 0, sizeof(thread_args_t)); + + /* Barrier to sync test case execution */ + odp_barrier_init(&globals->barrier, globals->num_workers); + odp_ticketlock_init(&globals->lock); + odp_spinlock_init(&globals->atomic_lock); + + odp_pool_param_init(¶ms); + params.buf.size = BUF_SIZE; + params.buf.align = 0; + params.buf.num = MSG_POOL_SIZE; + params.type = ODP_POOL_BUFFER; + + pool = odp_pool_create(MSG_POOL_NAME, ¶ms); + + if (pool == ODP_POOL_INVALID) { + printf("Pool creation failed (msg).\n"); + return -1; + } + + globals->pool = pool; + + if (odp_schedule_capability(&sched_capa)) { + printf("odp_schedule_capability() failed\n"); + return -1; + } + + num_flows = 0; + odp_schedule_config_init(&sched_config); + + /* Enable flow aware scheduling */ + if (sched_capa.max_flow_id > 0) { + num_flows = MAX_FLOWS; + if ((MAX_FLOWS - 1) > sched_capa.max_flow_id) + num_flows = sched_capa.max_flow_id + 1; + + sched_config.max_flow_id = num_flows - 1; + } + + globals->num_flows = num_flows; + + /* Configure the scheduler. All test cases share the config. */ + if (odp_schedule_config(&sched_config)) { + printf("odp_schedule_config() failed.\n"); + return -1; + } + + return 0; +} + +static int scheduler_multi_suite_init(void) +{ + if (create_queues(globals) != 0) + return -1; + + return 0; +} + +static int scheduler_multi_suite_term(void) +{ + if (destroy_queues() != 0) { + ODPH_ERR("Failed to destroy queues\n"); + return -1; + } + + if (odp_cunit_print_inactive()) + return -1; + + return 0; +} + +static int scheduler_basic_suite_init(void) +{ + return 0; +} + +static int scheduler_basic_suite_term(void) +{ + if (odp_cunit_print_inactive()) + return -1; + + return 0; +} + +static int global_init(odp_instance_t *inst) +{ + odp_init_t init_param; + odph_helper_options_t helper_options; + + if (odph_options(&helper_options)) { + ODPH_ERR("odph_options() failed.\n"); + return -1; + } + + odp_init_param_init(&init_param); + init_param.mem_model = helper_options.mem_model; + + if (0 != odp_init_global(inst, &init_param, NULL)) { + ODPH_ERR("odp_init_global() failed.\n"); + return -1; + } + + if (0 != odp_init_local(*inst, ODP_THREAD_CONTROL)) { + ODPH_ERR("odp_init_local() failed.\n"); + return -1; + } + + if (scheduler_test_global_init()) { + ODPH_ERR("scheduler test global init failed\n"); + return -1; + } + + return 0; +} + +static int global_term(odp_instance_t inst) +{ + if (odp_pool_destroy(globals->pool)) + ODPH_ERR("Failed to destroy pool\n"); + + if (odp_shm_free(globals->shm_args)) + ODPH_ERR("Failed to free shm\n"); + + if (odp_shm_free(globals->shm_glb)) + ODPH_ERR("Failed to free shm\n"); + + if (odp_term_local()) { + ODPH_ERR("odp_term_local() failed.\n"); + return -1; + } + + if (odp_term_global(inst)) { + ODPH_ERR("odp_term_global() failed.\n"); + return -1; + } + + return 0; +} + +/* Basic scheduler test suite */ +odp_testinfo_t scheduler_basic_suite[] = { ODP_TEST_INFO(scheduler_test_init), ODP_TEST_INFO(scheduler_test_capa), ODP_TEST_INFO(scheduler_test_wait_time), @@ -2869,6 +3031,9 @@ odp_testinfo_t scheduler_suite[] = { ODP_TEST_INFO(scheduler_test_wait), ODP_TEST_INFO(scheduler_test_queue_size), ODP_TEST_INFO(scheduler_test_full_queues), + ODP_TEST_INFO(scheduler_test_max_queues_p), + ODP_TEST_INFO(scheduler_test_max_queues_a), + ODP_TEST_INFO(scheduler_test_max_queues_o), ODP_TEST_INFO(scheduler_test_order_ignore), ODP_TEST_INFO(scheduler_test_create_group), ODP_TEST_INFO(scheduler_test_create_max_groups), @@ -2883,6 +3048,12 @@ odp_testinfo_t scheduler_suite[] = { ODP_TEST_INFO(scheduler_test_ordered), ODP_TEST_INFO(scheduler_test_atomic_and_plain), ODP_TEST_INFO(scheduler_test_ordered_and_plain), + ODP_TEST_INFO_NULL +}; + +/* Scheduler test suite which runs events through hundreds of queues. Queues are created once + * in suite init phase. */ +odp_testinfo_t scheduler_multi_suite[] = { ODP_TEST_INFO(scheduler_test_chaos), ODP_TEST_INFO(scheduler_test_1q_1t_n), ODP_TEST_INFO(scheduler_test_1q_1t_a), @@ -2910,42 +3081,20 @@ odp_testinfo_t scheduler_suite[] = { ODP_TEST_INFO(scheduler_test_multi_mq_mt_prio_a), ODP_TEST_INFO(scheduler_test_multi_mq_mt_prio_o), ODP_TEST_INFO(scheduler_test_multi_1q_mt_a_excl), - ODP_TEST_INFO_NULL, + ODP_TEST_INFO_NULL }; odp_suiteinfo_t scheduler_suites[] = { - {"Scheduler", - scheduler_suite_init, scheduler_suite_term, scheduler_suite + {"Scheduler basic", + scheduler_basic_suite_init, scheduler_basic_suite_term, scheduler_basic_suite + }, + {"Scheduler multi", + scheduler_multi_suite_init, scheduler_multi_suite_term, scheduler_multi_suite }, + ODP_SUITE_INFO_NULL, }; -static int global_init(odp_instance_t *inst) -{ - odp_init_t init_param; - odph_helper_options_t helper_options; - - if (odph_options(&helper_options)) { - ODPH_ERR("odph_options() failed.\n"); - return -1; - } - - odp_init_param_init(&init_param); - init_param.mem_model = helper_options.mem_model; - - if (0 != odp_init_global(inst, &init_param, NULL)) { - ODPH_ERR("odp_init_global() failed.\n"); - return -1; - } - - if (0 != odp_init_local(*inst, ODP_THREAD_CONTROL)) { - ODPH_ERR("odp_init_local() failed.\n"); - return -1; - } - - return 0; -} - int main(int argc, char *argv[]) { int ret; @@ -2955,6 +3104,8 @@ int main(int argc, char *argv[]) return -1; odp_cunit_register_global_init(global_init); + odp_cunit_register_global_term(global_term); + ret = odp_cunit_register(scheduler_suites); if (ret == 0) diff --git a/test/validation/api/stash/stash.c b/test/validation/api/stash/stash.c index b8677d85f..c99d5a471 100644 --- a/test/validation/api/stash/stash.c +++ b/test/validation/api/stash/stash.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2020, Nokia +/* Copyright (c) 2020-2021, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -12,6 +12,11 @@ #define MAGIC_U16 0x25bf #define MAGIC_U8 0xab +#define VAL_U64 0x6b89f0742a672c34 +#define VAL_U32 0x713d847b +#define VAL_U16 0xb587 +#define VAL_U8 0x9d + #define NUM_U64 1024 #define NUM_U32 1024 #define NUM_U16 1024 @@ -361,37 +366,37 @@ static void stash_default_put(uint32_t size, int32_t burst) int32_t num; void *input, *output; uint64_t input_u64[burst]; - uint64_t output_u64[burst]; + uint64_t output_u64[burst + 2]; uint32_t input_u32[burst]; - uint32_t output_u32[burst]; + uint32_t output_u32[burst + 2]; uint16_t input_u16[burst]; - uint16_t output_u16[burst]; + uint16_t output_u16[burst + 2]; uint8_t input_u8[burst]; - uint8_t output_u8[burst]; + uint8_t output_u8[burst + 2]; if (size == sizeof(uint64_t)) { num = global.num_default.u64; input = input_u64; - output = output_u64; + output = &output_u64[1]; } else if (size == sizeof(uint32_t)) { num = global.num_default.u32; input = input_u32; - output = output_u32; + output = &output_u32[1]; } else if (size == sizeof(uint16_t)) { num = global.num_default.u16; input = input_u16; - output = output_u16; + output = &output_u16[1]; } else { num = global.num_default.u8; input = input_u8; - output = output_u8; + output = &output_u8[1]; } for (i = 0; i < burst; i++) { - input_u64[i] = MAGIC_U64; - input_u32[i] = MAGIC_U32; - input_u16[i] = MAGIC_U16; - input_u8[i] = MAGIC_U8; + input_u64[i] = VAL_U64; + input_u32[i] = VAL_U32; + input_u16[i] = VAL_U16; + input_u8[i] = VAL_U8; } odp_stash_param_init(¶m); @@ -427,21 +432,50 @@ static void stash_default_put(uint32_t size, int32_t burst) while (num_left) { memset(output, 0, burst * size); + /* Init first and last array element for under-/overflow checking */ + if (size == sizeof(uint64_t)) { + output_u64[0] = MAGIC_U64; + output_u64[burst + 1] = MAGIC_U64; + } else if (size == sizeof(uint32_t)) { + output_u32[0] = MAGIC_U32; + output_u32[burst + 1] = MAGIC_U32; + } else if (size == sizeof(uint16_t)) { + output_u16[0] = MAGIC_U16; + output_u16[burst + 1] = MAGIC_U16; + } else { + output_u8[0] = MAGIC_U8; + output_u8[burst + 1] = MAGIC_U8; + } + ret = odp_stash_get(stash, output, burst); CU_ASSERT_FATAL(ret >= 0); CU_ASSERT_FATAL(ret <= burst); + if (size == sizeof(uint64_t)) { + CU_ASSERT_FATAL(output_u64[0] == MAGIC_U64); + CU_ASSERT_FATAL(output_u64[burst + 1] == MAGIC_U64); + } else if (size == sizeof(uint32_t)) { + CU_ASSERT_FATAL(output_u32[0] == MAGIC_U32); + CU_ASSERT_FATAL(output_u32[burst + 1] == MAGIC_U32); + } else if (size == sizeof(uint16_t)) { + CU_ASSERT_FATAL(output_u16[0] == MAGIC_U16); + CU_ASSERT_FATAL(output_u16[burst + 1] == MAGIC_U16); + } else { + CU_ASSERT_FATAL(output_u8[0] == MAGIC_U8); + CU_ASSERT_FATAL(output_u8[burst + 1] == MAGIC_U8); + } + if (ret) { for (i = 0; i < ret; i++) { if (size == sizeof(uint64_t)) { /* CU_ASSERT needs brackets around it */ - CU_ASSERT(output_u64[i] == MAGIC_U64); + CU_ASSERT(output_u64[i + 1] == VAL_U64); } else if (size == sizeof(uint32_t)) { - CU_ASSERT(output_u32[i] == MAGIC_U32); + CU_ASSERT(output_u32[i + 1] == VAL_U32); } else if (size == sizeof(uint16_t)) { - CU_ASSERT(output_u16[i] == MAGIC_U16); + CU_ASSERT(output_u16[i + 1] == VAL_U16); } else { - CU_ASSERT(output_u8[i] == MAGIC_U8); + CU_ASSERT(output_u8[i + 1] == VAL_U8); } } @@ -468,30 +502,30 @@ static void stash_fifo_put(uint32_t size, int32_t burst) int32_t num; void *input, *output; uint64_t input_u64[burst]; - uint64_t output_u64[burst]; + uint64_t output_u64[burst + 2]; uint32_t input_u32[burst]; - uint32_t output_u32[burst]; + uint32_t output_u32[burst + 2]; uint16_t input_u16[burst]; - uint16_t output_u16[burst]; + uint16_t output_u16[burst + 2]; uint8_t input_u8[burst]; - uint8_t output_u8[burst]; + uint8_t output_u8[burst + 2]; if (size == sizeof(uint64_t)) { num = global.num_fifo.u64; input = input_u64; - output = output_u64; + output = &output_u64[1]; } else if (size == sizeof(uint32_t)) { num = global.num_fifo.u32; input = input_u32; - output = output_u32; + output = &output_u32[1]; } else if (size == sizeof(uint16_t)) { num = global.num_fifo.u16; input = input_u16; - output = output_u16; + output = &output_u16[1]; } else { num = global.num_fifo.u8; input = input_u8; - output = output_u8; + output = &output_u8[1]; } odp_stash_param_init(¶m); @@ -511,13 +545,13 @@ static void stash_fifo_put(uint32_t size, int32_t burst) while (num_left) { for (i = 0; i < burst; i++) { if (size == sizeof(uint64_t)) - input_u64[i] = MAGIC_U64 + num_left - i; + input_u64[i] = VAL_U64 + num_left - i; else if (size == sizeof(uint32_t)) - input_u32[i] = MAGIC_U32 + num_left - i; + input_u32[i] = VAL_U32 + num_left - i; else if (size == sizeof(uint16_t)) - input_u16[i] = MAGIC_U16 + num_left - i; + input_u16[i] = VAL_U16 + num_left - i; else - input_u8[i] = MAGIC_U8 + num_left - i; + input_u8[i] = VAL_U8 + num_left - i; } ret = odp_stash_put(stash, input, burst); @@ -538,28 +572,57 @@ static void stash_fifo_put(uint32_t size, int32_t burst) while (num_left) { memset(output, 0, burst * size); + /* Init first and last array element for under-/overflow checking */ + if (size == sizeof(uint64_t)) { + output_u64[0] = MAGIC_U64; + output_u64[burst + 1] = MAGIC_U64; + } else if (size == sizeof(uint32_t)) { + output_u32[0] = MAGIC_U32; + output_u32[burst + 1] = MAGIC_U32; + } else if (size == sizeof(uint16_t)) { + output_u16[0] = MAGIC_U16; + output_u16[burst + 1] = MAGIC_U16; + } else { + output_u8[0] = MAGIC_U8; + output_u8[burst + 1] = MAGIC_U8; + } + ret = odp_stash_get(stash, output, burst); CU_ASSERT_FATAL(ret >= 0); + if (size == sizeof(uint64_t)) { + CU_ASSERT_FATAL(output_u64[0] == MAGIC_U64); + CU_ASSERT_FATAL(output_u64[burst + 1] == MAGIC_U64); + } else if (size == sizeof(uint32_t)) { + CU_ASSERT_FATAL(output_u32[0] == MAGIC_U32); + CU_ASSERT_FATAL(output_u32[burst + 1] == MAGIC_U32); + } else if (size == sizeof(uint16_t)) { + CU_ASSERT_FATAL(output_u16[0] == MAGIC_U16); + CU_ASSERT_FATAL(output_u16[burst + 1] == MAGIC_U16); + } else { + CU_ASSERT_FATAL(output_u8[0] == MAGIC_U8); + CU_ASSERT_FATAL(output_u8[burst + 1] == MAGIC_U8); + } + if (ret) { CU_ASSERT_FATAL(ret <= burst); for (i = 0; i < ret; i++) { if (size == sizeof(uint64_t)) { - uint64_t val = MAGIC_U64 + num_left - i; + uint64_t val = VAL_U64 + num_left - i; - CU_ASSERT(output_u64[i] == val); + CU_ASSERT(output_u64[i + 1] == val); } else if (size == sizeof(uint32_t)) { - uint32_t val = MAGIC_U32 + num_left - i; + uint32_t val = VAL_U32 + num_left - i; - CU_ASSERT(output_u32[i] == val); + CU_ASSERT(output_u32[i + 1] == val); } else if (size == sizeof(uint16_t)) { - uint16_t val = MAGIC_U16 + num_left - i; + uint16_t val = VAL_U16 + num_left - i; - CU_ASSERT(output_u16[i] == val); + CU_ASSERT(output_u16[i + 1] == val); } else { - uint8_t val = MAGIC_U8 + num_left - i; + uint8_t val = VAL_U8 + num_left - i; - CU_ASSERT(output_u8[i] == val); + CU_ASSERT(output_u8[i + 1] == val); } } diff --git a/test/validation/api/system/system.c b/test/validation/api/system/system.c index d00f09f28..9beb89cf2 100644 --- a/test/validation/api/system/system.c +++ b/test/validation/api/system/system.c @@ -517,6 +517,14 @@ static void system_test_info(void) printf(" ODP_CPU_ARCH_ARMV8_5\n"); else if (info.cpu_isa_sw.arm == ODP_CPU_ARCH_ARMV8_6) printf(" ODP_CPU_ARCH_ARMV8_6\n"); + else if (info.cpu_isa_sw.arm == ODP_CPU_ARCH_ARMV8_7) + printf(" ODP_CPU_ARCH_ARMV8_7\n"); + else if (info.cpu_isa_sw.arm == ODP_CPU_ARCH_ARMV9_0) + printf(" ODP_CPU_ARCH_ARMV9_0\n"); + else if (info.cpu_isa_sw.arm == ODP_CPU_ARCH_ARMV9_1) + printf(" ODP_CPU_ARCH_ARMV9_1\n"); + else if (info.cpu_isa_sw.arm == ODP_CPU_ARCH_ARMV9_2) + printf(" ODP_CPU_ARCH_ARMV9_2\n"); else CU_FAIL("Unknown CPU ISA SW ARCH found!"); @@ -541,6 +549,14 @@ static void system_test_info(void) printf(" ODP_CPU_ARCH_ARMV8_5\n"); else if (info.cpu_isa_hw.arm == ODP_CPU_ARCH_ARMV8_6) printf(" ODP_CPU_ARCH_ARMV8_6\n"); + else if (info.cpu_isa_hw.arm == ODP_CPU_ARCH_ARMV8_7) + printf(" ODP_CPU_ARCH_ARMV8_7\n"); + else if (info.cpu_isa_hw.arm == ODP_CPU_ARCH_ARMV9_0) + printf(" ODP_CPU_ARCH_ARMV9_0\n"); + else if (info.cpu_isa_hw.arm == ODP_CPU_ARCH_ARMV9_1) + printf(" ODP_CPU_ARCH_ARMV9_1\n"); + else if (info.cpu_isa_hw.arm == ODP_CPU_ARCH_ARMV9_2) + printf(" ODP_CPU_ARCH_ARMV9_2\n"); else if (info.cpu_isa_hw.arm == ODP_CPU_ARCH_ARM_UNKNOWN) printf(" ODP_CPU_ARCH_ARM_UNKNOWN\n"); else diff --git a/test/validation/api/traffic_mngr/traffic_mngr.c b/test/validation/api/traffic_mngr/traffic_mngr.c index 1029a128b..13f39457d 100644 --- a/test/validation/api/traffic_mngr/traffic_mngr.c +++ b/test/validation/api/traffic_mngr/traffic_mngr.c @@ -2451,6 +2451,14 @@ static int traffic_mngr_check_shaper(void) odp_cpumask_t cpumask; int cpucount = odp_cpumask_all_available(&cpumask); +/* Skip the shaper test on arm64 systems */ +#if defined(__aarch64__) + printf("\nTemporarily skip shaper test which intermittently " + "fails on arm64 systems. Will be activated when issue " + "is resolved\n"); + return ODP_TEST_INACTIVE; +#endif + if (cpucount < 2) { ODPH_DBG("\nSkipping shaper test because cpucount = %d " "is less then min number 2 required\n", cpucount); |