diff options
40 files changed, 2033 insertions, 133 deletions
diff --git a/.travis.yml b/.travis.yml index b95a99516..d94a32486 100644 --- a/.travis.yml +++ b/.travis.yml @@ -46,23 +46,19 @@ env: # you need to generate a new one at https://codecov.io specific for your # repo. - CODECOV_TOKEN=8e1c0fd8-62ff-411e-a79f-5839f6662c11 - - OS=ubuntu_16.04 + - OS=ubuntu_18.04 - ARCH=x86_64 - CHECK=1 matrix: - CHECK=0 CONF="CFLAGS=-O3" - CHECK=0 CONF="CFLAGS=-O0 --enable-debug --enable-debug-print" - - CHECK=0 OS=ubuntu_18.04 CONF="CFLAGS=-O3" - - CHECK=0 OS=ubuntu_18.04 CONF="CFLAGS=-O0 --enable-debug --enable-debug-print" - CHECK=0 CONF="--enable-lto" - CHECK=0 CONF="--enable-lto --disable-abi-compat" - - CHECK=0 OS=ubuntu_18.04 CONF="--enable-lto" - - CHECK=0 OS=ubuntu_18.04 CONF="--enable-lto --disable-abi-compat" - CHECK=0 ARCH=arm64 - - CHECK=0 ARCH=armhf + - CHECK=0 ARCH=ppc64el - CHECK=0 ARCH=i386 - CHECK=0 ARCH=arm64 CONF="--disable-abi-compat" - - CHECK=0 ARCH=armhf CONF="--disable-abi-compat" + - CHECK=0 ARCH=ppc64el CONF="--disable-abi-compat" - CHECK=0 ARCH=i386 CONF="--disable-abi-compat" - CONF="" - CONF="--disable-abi-compat" @@ -73,7 +69,8 @@ env: - CHECK=0 CONF="--enable-pcapng-support" - CHECK=0 OS=centos_7 - CONF="--without-openssl --without-pcap" - - OS=ubuntu_18.04 + - OS=ubuntu_16.04 + - OS=ubuntu_20.04 matrix: exclude: @@ -86,9 +83,9 @@ matrix: - compiler: clang env: CHECK=0 CONF="--enable-lto --disable-abi-compat" - compiler: clang - env: CHECK=0 OS=ubuntu_18.04 CONF="--enable-lto" + env: CHECK=0 ARCH=ppc64el - compiler: clang - env: CHECK=0 OS=ubuntu_18.04 CONF="--enable-lto --disable-abi-compat" + env: CHECK=0 ARCH=ppc64el CONF="--disable-abi-compat" install: - sudo sysctl vm.nr_hugepages=1000 @@ -99,14 +96,14 @@ script: docker run --privileged -i -t -v `pwd`:/odp --shm-size 8g -e CC="${CC}" -e CONF="${CONF}" - ${DOCKER_NAMESPACE}/travis-odp-lng-${OS} /odp/scripts/ci/build_${ARCH}.sh ; + ${DOCKER_NAMESPACE}/travis-odp-${OS}-${ARCH} /odp/scripts/ci/build_${ARCH}.sh ; else echo "Running test" ; docker run --privileged -i -t -v `pwd`:/odp --shm-size 8g -e CC="${CC}" -e CONF="${CONF}" - ${DOCKER_NAMESPACE}/travis-odp-lng-${OS} /odp/scripts/ci/check.sh ; + ${DOCKER_NAMESPACE}/travis-odp-${OS}-${ARCH} /odp/scripts/ci/check.sh ; fi jobs: include: @@ -119,7 +116,7 @@ jobs: -v `pwd`:/odp --shm-size 8g -e CODECOV_TOKEN="${CODECOV_TOKEN}" -e CC="${CC}" - ${DOCKER_NAMESPACE}/travis-odp-lng-ubuntu_16.04 /odp/scripts/ci/coverage.sh + ${DOCKER_NAMESPACE}/travis-odp-${OS}-${ARCH} /odp/scripts/ci/coverage.sh - stage: test env: TEST=scheduler_sp compiler: gcc @@ -130,9 +127,9 @@ jobs: -e CC="${CC}" -e CONF="" -e ODP_SCHEDULER=sp - ${DOCKER_NAMESPACE}/travis-odp-lng-ubuntu_16.04 /odp/scripts/ci/check.sh + ${DOCKER_NAMESPACE}/travis-odp-${OS}-${ARCH} /odp/scripts/ci/check.sh - stage: test - env: TEST=dpdk_18.11 + env: TEST=dpdk_19.11 install: - true compiler: gcc @@ -142,7 +139,7 @@ jobs: -v `pwd`:/odp --shm-size 8g -e CC="${CC}" -e CONF="" - ${DOCKER_NAMESPACE}/travis-odp-lng-ubuntu_16.04-dpdk_18.11 /odp/scripts/ci/check.sh + ${DOCKER_NAMESPACE}/travis-odp-${OS}-${ARCH}-dpdk_19.11 /odp/scripts/ci/check.sh - stage: test env: TEST=distcheck canfail: yes @@ -153,7 +150,7 @@ jobs: -v `pwd`:/odp --shm-size 8g -e CC="${CC}" -e CONF="--enable-user-guides" - ${DOCKER_NAMESPACE}/travis-odp-lng-ubuntu_16.04 /odp/scripts/ci/distcheck.sh + ${DOCKER_NAMESPACE}/travis-odp-${OS}-${ARCH} /odp/scripts/ci/distcheck.sh - stage: test env: TEST=distcheck_nonabi canfail: yes @@ -164,7 +161,7 @@ jobs: -v `pwd`:/odp --shm-size 8g -e CC="${CC}" -e CONF="--enable-user-guides --disable-abi-compat" - ${DOCKER_NAMESPACE}/travis-odp-lng-ubuntu_16.04 /odp/scripts/ci/distcheck.sh + ${DOCKER_NAMESPACE}/travis-odp-${OS}-${ARCH} /odp/scripts/ci/distcheck.sh - stage: test env: TEST=out_of_tree compiler: gcc @@ -174,7 +171,7 @@ jobs: -v `pwd`:/odp --shm-size 8g -e CC="${CC}" -e CONF="" - ${DOCKER_NAMESPACE}/travis-odp-lng-ubuntu_16.04 /odp/scripts/ci/out_of_tree.sh + ${DOCKER_NAMESPACE}/travis-odp-${OS}-${ARCH} /odp/scripts/ci/out_of_tree.sh - stage: "build only" env: TEST=documentation compiler: gcc @@ -182,15 +179,14 @@ jobs: - if [ -z "${DOCKER_NAMESPACE}" ] ; then export DOCKER_NAMESPACE="opendataplane"; fi - docker run --privileged -i -t -v `pwd`:/odp --shm-size 8g -e CC="${CC}" - ${DOCKER_NAMESPACE}/travis-odp-lng-ubuntu_16.04 /odp/scripts/ci/doxygen.sh - + ${DOCKER_NAMESPACE}/travis-odp-${OS}-${ARCH} /odp/scripts/ci/doxygen.sh - stage: "build only" env: ARCH=x86_64 script: - if [ -z "${DOCKER_NAMESPACE}" ] ; then export DOCKER_NAMESPACE="opendataplane"; fi - docker run --privileged -i -t -v `pwd`:/odp --shm-size 8g -e CC="${CC}" - ${DOCKER_NAMESPACE}/travis-odp-lng-ubuntu_16.04 /odp/scripts/ci/build_${ARCH}.sh + ${DOCKER_NAMESPACE}/travis-odp-${OS}-${ARCH} /odp/scripts/ci/build_${ARCH}.sh - stage: "build only" env: ARCH=x86_64 compiler: clang @@ -198,7 +194,7 @@ jobs: - if [ -z "${DOCKER_NAMESPACE}" ] ; then export DOCKER_NAMESPACE="opendataplane"; fi - docker run --privileged -i -t -v `pwd`:/odp --shm-size 8g -e CC="${CC}" - ${DOCKER_NAMESPACE}/travis-odp-lng-ubuntu_16.04 /odp/scripts/ci/build_${ARCH}.sh + ${DOCKER_NAMESPACE}/travis-odp-${OS}-${ARCH} /odp/scripts/ci/build_${ARCH}.sh - stage: "build only" env: ARCH=arm64 install: @@ -207,14 +203,14 @@ jobs: - if [ -z "${DOCKER_NAMESPACE}" ] ; then export DOCKER_NAMESPACE="opendataplane"; fi - docker run --privileged -i -t -v `pwd`:/odp -e CC="${CC}" - ${DOCKER_NAMESPACE}/travis-odp-lng-ubuntu_16.04 /odp/scripts/ci/build_${ARCH}.sh + ${DOCKER_NAMESPACE}/travis-odp-${OS}-${ARCH} /odp/scripts/ci/build_${ARCH}.sh - stage: "build only" env: ARCH=i386 script: - if [ -z "${DOCKER_NAMESPACE}" ] ; then export DOCKER_NAMESPACE="opendataplane"; fi - docker run --privileged -i -t -v `pwd`:/odp --shm-size 8g -e CC="${CC}" - ${DOCKER_NAMESPACE}/travis-odp-lng-ubuntu_16.04 /odp/scripts/ci/build_${ARCH}.sh + ${DOCKER_NAMESPACE}/travis-odp-${OS}-${ARCH} /odp/scripts/ci/build_${ARCH}.sh - stage: test canfail: yes env: TEST=checkpatch diff --git a/example/Makefile.am b/example/Makefile.am index 28362f2c1..26723a9e4 100644 --- a/example/Makefile.am +++ b/example/Makefile.am @@ -3,7 +3,6 @@ SUBDIRS = classifier \ hello \ ipsec \ ipsec_api \ - ipfragreass \ ipsec_offload \ l2fwd_simple \ l3fwd \ @@ -15,3 +14,7 @@ SUBDIRS = classifier \ time \ timer \ traffic_mgmt + +if HAVE_DW_ATOMIC_CMP_EXC +SUBDIRS += ipfragreass +endif diff --git a/example/ipfragreass/Makefile.am b/example/ipfragreass/Makefile.am index b98e066f8..2cd61a39e 100644 --- a/example/ipfragreass/Makefile.am +++ b/example/ipfragreass/Makefile.am @@ -1,6 +1,6 @@ include $(top_srcdir)/example/Makefile.inc -LDADD += $(ATOMIC_LIBS) +LDADD += $(ATOMIC_LIBS) $(ATOMIC_LIBS_OPT) bin_PROGRAMS = odp_ipfragreass diff --git a/example/l3fwd/odp_l3fwd.c b/example/l3fwd/odp_l3fwd.c index 79613d672..0f8b8525c 100644 --- a/example/l3fwd/odp_l3fwd.c +++ b/example/l3fwd/odp_l3fwd.c @@ -803,17 +803,18 @@ static void setup_worker_qconf(app_args_t *args) in_queue_param.op_mode = ODP_PKTIO_OP_MT_UNSAFE; out_queue_param.op_mode = ODP_PKTIO_OP_MT_UNSAFE; - in_queue_param.hash_enable = 1; - in_queue_param.hash_proto.proto.ipv4 = 1; - in_queue_param.hash_proto.proto.ipv4_tcp = 1; - in_queue_param.hash_proto.proto.ipv4_udp = 1; - in_queue_param.num_queues = port->rxq_idx; if (port->rxq_idx > port->nb_rxq) { in_queue_param.num_queues = port->nb_rxq; in_queue_param.op_mode = ODP_PKTIO_OP_MT; } + in_queue_param.hash_enable = (in_queue_param.num_queues > 1) ? + 1 : 0; + in_queue_param.hash_proto.proto.ipv4 = 1; + in_queue_param.hash_proto.proto.ipv4_tcp = 1; + in_queue_param.hash_proto.proto.ipv4_udp = 1; + if (odp_pktin_queue_config(port->pktio, &in_queue_param)) ODPH_ABORT("Fail to config input queue for port %s\n", name); diff --git a/example/switch/odp_switch.c b/example/switch/odp_switch.c index ccb975f6c..bcc87680e 100644 --- a/example/switch/odp_switch.c +++ b/example/switch/odp_switch.c @@ -304,7 +304,7 @@ static int create_pktio(const char *dev, int idx, int num_rx, int num_tx, mode_tx = ODP_PKTIO_OP_MT; } - pktin_param.hash_enable = 1; + pktin_param.hash_enable = (num_rx > 1) ? 1 : 0; pktin_param.hash_proto.proto.ipv4_tcp = 1; pktin_param.hash_proto.proto.ipv4_udp = 1; pktin_param.num_queues = num_rx; diff --git a/m4/odp_atomic.m4 b/m4/odp_atomic.m4 index 413dcbda9..c0068b5e4 100644 --- a/m4/odp_atomic.m4 +++ b/m4/odp_atomic.m4 @@ -14,6 +14,30 @@ if test "x$use_libatomic" = "xyes"; then ATOMIC_LIBS="-latomic" fi AC_SUBST([ATOMIC_LIBS]) + +# Double wide __atomic_compare_exchange_n is required by ipfragreass example +use_libatomic_opt=no; +have_atomic_cmp_exc=yes; + +AC_CHECK_SIZEOF([void *]) +AC_PREPROC_IFELSE( + [AC_LANG_SOURCE([ + #if SIZEOF_VOID_P == 8 + #error + #endif + ])], [plat64=no], [plat64=yes]) + +if test "x$plat64" = "xyes"; then + ODP_ATOMIC_NEEDED_128BIT_CMP_EXC([use_libatomic_opt=yes], [have_atomic_cmp_exc=no]) +else + ODP_ATOMIC_NEEDED_64BIT_CMP_EXC([use_libatomic_opt=yes], [have_atomic_cmp_exc=no]) +fi + +if test "x$use_libatomic_opt" = "xyes"; then + ATOMIC_LIBS_OPT="-latomic" +fi +AC_SUBST([ATOMIC_LIBS_OPT]) +AM_CONDITIONAL([HAVE_DW_ATOMIC_CMP_EXC], [test x$have_atomic_cmp_exc = xyes]) ]) # ODP_ATOMIC # ODP_ATOMIC_BUILTINS @@ -93,3 +117,64 @@ if test "x$odp_cv_atomic_needed_128bit" = "xyes" ; then [AC_MSG_FAILURE([__atomic_exchange_16 is not available])]) fi ]) # ODP_ATOMIC_NEEDED_128BIT + +# ODP_ATOMIC_NEEDED_64BIT_CMP_EXC([ACTION_IF_NEEDED], [ACTION_IF_NOT_AVAILABLE]) +# ------------------------------------------------------------------------------ +# +AC_DEFUN([ODP_ATOMIC_NEEDED_64BIT_CMP_EXC], [dnl +AC_CACHE_CHECK([whether -latomic is needed for 64-bit atomic compare exchange], + [odp_cv_atomic_needed_64bit_cmp_exc], [dnl +AC_LINK_IFELSE( + [AC_LANG_SOURCE([[ + #include <stdint.h> + static uint64_t loc; + int main(void) + { + uint64_t exp = 0; + uint64_t = __atomic_compare_exchange_n(&loc, &exp, 1, 1, + __ATOMIC_ACQUIRE, + __ATOMIC_RELAXED); + return 0; + } + ]])], + [odp_cv_atomic_needed_64bit_cmp_exc=no], + [odp_cv_atomic_needed_64bit_cmp_exc=yes])]) + +if test "x$odp_cv_atomic_needed_64bit_cmp_exc" = "xyes" ; then + AC_CHECK_LIB( + [atomic], [__atomic_compare_exchange_8], + [m4_default([$1], [:])], + [m4_default([$2], [:])]) +fi + +]) # ODP_ATOMIC_NEEDED_64BIT_CMP_EXC + +# ODP_ATOMIC_NEEDED_128BIT_CMP_EXC([ACTION_IF_NEEDED], [ACTION_IF_NOT_AVAILABLE]) +# ------------------------------------------------------------------------------- +# +AC_DEFUN([ODP_ATOMIC_NEEDED_128BIT_CMP_EXC], [dnl +AC_CACHE_CHECK([whether -latomic is needed for 128-bit atomic compare exchange], + [odp_cv_atomic_needed_128bit_cmp_exc], [dnl +AC_LINK_IFELSE( + [AC_LANG_SOURCE([[ + #include <stdint.h> + static __int128 loc; + int main(void) + { + __int128 exp = 0; + __int128 = __atomic_compare_exchange_n(&loc, &exp, 1, 1, + __ATOMIC_ACQUIRE, + __ATOMIC_RELAXED); + return 0; + } + ]])], + [odp_cv_atomic_needed_128bit_cmp_exc=no], + [odp_cv_atomic_needed_128bit_cmp_exc=yes])]) + +if test "x$odp_cv_atomic_needed_128bit_cmp_exc" = "xyes" ; then + AC_CHECK_LIB( + [atomic], [__atomic_compare_exchange_16], + [m4_default([$1], [:])], + [m4_default([$2], [:])]) +fi +]) # ODP_ATOMIC_NEEDED_128BIT_CMP_EXC diff --git a/platform/linux-dpdk/include/odp/api/plat/packet_inlines.h b/platform/linux-dpdk/include/odp/api/plat/packet_inlines.h index 7eeb9108d..149ca1574 100644 --- a/platform/linux-dpdk/include/odp/api/plat/packet_inlines.h +++ b/platform/linux-dpdk/include/odp/api/plat/packet_inlines.h @@ -36,7 +36,12 @@ extern "C" { #include <rte_config.h> #include <rte_mbuf.h> +/* ppc64 rte_memcpy.h may overwrite bool with incompatible type */ #include <rte_memcpy.h> +#if defined(__PPC64__) && defined(bool) + #undef bool + #define bool _Bool +#endif /** @cond _ODP_HIDE_FROM_DOXYGEN_ */ diff --git a/platform/linux-dpdk/include/odp/api/plat/std_clib_inlines.h b/platform/linux-dpdk/include/odp/api/plat/std_clib_inlines.h index 932807fbb..aed77b780 100644 --- a/platform/linux-dpdk/include/odp/api/plat/std_clib_inlines.h +++ b/platform/linux-dpdk/include/odp/api/plat/std_clib_inlines.h @@ -15,7 +15,12 @@ #if defined(__clang__) #undef RTE_TOOLCHAIN_GCC #endif +/* ppc64 rte_memcpy.h may overwrite bool with incompatible type */ #include <rte_memcpy.h> +#if defined(__PPC64__) && defined(bool) + #undef bool + #define bool _Bool +#endif #ifndef _ODP_NO_INLINE /* Inline functions by default */ diff --git a/platform/linux-dpdk/include/odp_packet_internal.h b/platform/linux-dpdk/include/odp_packet_internal.h index 7edcc5dda..b7edb8a23 100644 --- a/platform/linux-dpdk/include/odp_packet_internal.h +++ b/platform/linux-dpdk/include/odp_packet_internal.h @@ -36,7 +36,8 @@ extern "C" { #if defined(__clang__) #undef RTE_TOOLCHAIN_GCC #endif -#include <rte_acl_osdep.h> +#include <rte_mbuf.h> +#include <rte_memory.h> /** Minimum segment length expected by packet_parse_common() */ #define PACKET_PARSE_SEG_LEN 96 diff --git a/platform/linux-dpdk/include/odp_packet_io_internal.h b/platform/linux-dpdk/include/odp_packet_io_internal.h index c33c8fea0..5bc3a3590 100644 --- a/platform/linux-dpdk/include/odp_packet_io_internal.h +++ b/platform/linux-dpdk/include/odp_packet_io_internal.h @@ -42,7 +42,7 @@ extern "C" { /* Forward declaration */ struct pktio_if_ops; -#define PKTIO_PRIVATE_SIZE 1728 +#define PKTIO_PRIVATE_SIZE 2048 struct pktio_entry { const struct pktio_if_ops *ops; /**< Implementation specific methods */ diff --git a/platform/linux-dpdk/include/odp_pool_internal.h b/platform/linux-dpdk/include/odp_pool_internal.h index 0102f16ca..847be28d6 100644 --- a/platform/linux-dpdk/include/odp_pool_internal.h +++ b/platform/linux-dpdk/include/odp_pool_internal.h @@ -33,6 +33,11 @@ extern "C" { #include <rte_config.h> #include <rte_mbuf.h> #include <rte_mempool.h> +/* ppc64 rte_memcpy.h may overwrite bool with incompatible type */ +#if defined(__PPC64__) && defined(bool) + #undef bool + #define bool _Bool +#endif /* Use ticketlock instead of spinlock */ #define POOL_USE_TICKETLOCK diff --git a/platform/linux-dpdk/libodp-linux.pc.in b/platform/linux-dpdk/libodp-linux.pc.in index 415054dbb..5b9421226 100644 --- a/platform/linux-dpdk/libodp-linux.pc.in +++ b/platform/linux-dpdk/libodp-linux.pc.in @@ -7,6 +7,6 @@ Name: libodp-linux Description: The ODP packet processing engine Version: @PKGCONFIG_VERSION@ Requires.private: libconfig@DPDK_PKG@ -Libs: -L${libdir} -lodp-linux -Libs.private: @DPDK_LIBS@ @OPENSSL_STATIC_LIBS@ @PTHREAD_LIBS@ @TIMER_LIBS@ -lpthread @ATOMIC_LIBS@ +Libs: -L${libdir} -lodp-linux @DPDK_LIBS_NON_ABI_COMPAT@ +Libs.private: @DPDK_LIBS_ABI_COMPAT@ @OPENSSL_STATIC_LIBS@ @PTHREAD_LIBS@ @TIMER_LIBS@ -lpthread @ATOMIC_LIBS@ Cflags: -I${includedir} @DPDK_CFLAGS@ diff --git a/platform/linux-dpdk/m4/configure.m4 b/platform/linux-dpdk/m4/configure.m4 index a957b0745..2b75f0049 100644 --- a/platform/linux-dpdk/m4/configure.m4 +++ b/platform/linux-dpdk/m4/configure.m4 @@ -37,6 +37,15 @@ AC_ARG_WITH([dpdk-path], ODP_DPDK([$DPDK_PATH], [], [AC_MSG_FAILURE([can't find DPDK])]) +# In non-abi-compat mode DPDK is exposed to the application +if test $ODP_ABI_COMPAT -eq 1; then + DPDK_LIBS_ABI_COMPAT=$DPDK_LIBS + AC_SUBST([DPDK_LIBS_ABI_COMPAT]) +else + DPDK_LIBS_NON_ABI_COMPAT=$DPDK_LIBS + AC_SUBST([DPDK_LIBS_NON_ABI_COMPAT]) +fi + case "${host}" in i?86* | x86*) DPDK_CFLAGS="${DPDK_CFLAGS} -msse4.2" diff --git a/platform/linux-dpdk/odp_crypto.c b/platform/linux-dpdk/odp_crypto.c index 9602d9c90..b0ff761cb 100644 --- a/platform/linux-dpdk/odp_crypto.c +++ b/platform/linux-dpdk/odp_crypto.c @@ -28,6 +28,7 @@ #include <rte_config.h> #include <rte_crypto.h> #include <rte_cryptodev.h> +#include <rte_malloc.h> #include <rte_version.h> #include <string.h> diff --git a/platform/linux-dpdk/odp_pool.c b/platform/linux-dpdk/odp_pool.c index 27fc8836f..1c5edf76d 100644 --- a/platform/linux-dpdk/odp_pool.c +++ b/platform/linux-dpdk/odp_pool.c @@ -30,6 +30,7 @@ #include <odp/api/plat/pool_inline_types.h> #include <rte_config.h> +#include <rte_errno.h> #include <rte_version.h> #ifdef POOL_USE_TICKETLOCK diff --git a/platform/linux-generic/include/odp_config_internal.h b/platform/linux-generic/include/odp_config_internal.h index 67011b0d4..8c0285ff0 100644 --- a/platform/linux-generic/include/odp_config_internal.h +++ b/platform/linux-generic/include/odp_config_internal.h @@ -97,7 +97,7 @@ extern "C" { /* * Maximum packet segment size including head- and tailrooms */ -#define CONFIG_PACKET_SEG_SIZE (8 * 1024) +#define CONFIG_PACKET_SEG_SIZE (60 * 1024) /* Maximum data length in a segment * diff --git a/platform/linux-generic/odp_ipsec.c b/platform/linux-generic/odp_ipsec.c index 7bb38c5c8..07c3a001d 100644 --- a/platform/linux-generic/odp_ipsec.c +++ b/platform/linux-generic/odp_ipsec.c @@ -67,25 +67,22 @@ int odp_ipsec_capability(odp_ipsec_capability_t *capa) return 0; } -/* This should be enough for all ciphers and auths. Currently used maximum is 3 - * capabilities */ -#define MAX_CAPS 10 - int odp_ipsec_cipher_capability(odp_cipher_alg_t cipher, odp_ipsec_cipher_capability_t capa[], int num) { - odp_crypto_cipher_capability_t crypto_capa[MAX_CAPS]; uint32_t req_iv_len; - int rc, i, out; + int rc, i, out, max_capa; + + max_capa = odp_crypto_cipher_capability(cipher, NULL, 0); + if (max_capa <= 0) + return max_capa; - rc = odp_crypto_cipher_capability(cipher, crypto_capa, MAX_CAPS); + odp_crypto_cipher_capability_t crypto_capa[max_capa]; + + rc = odp_crypto_cipher_capability(cipher, crypto_capa, max_capa); if (rc <= 0) return rc; - ODP_ASSERT(rc <= MAX_CAPS); - if (rc > MAX_CAPS) - rc = MAX_CAPS; - req_iv_len = _odp_ipsec_cipher_iv_len(cipher); for (i = 0, out = 0; i < rc; i++) { if (crypto_capa[i].iv_len != req_iv_len) @@ -102,18 +99,19 @@ int odp_ipsec_cipher_capability(odp_cipher_alg_t cipher, int odp_ipsec_auth_capability(odp_auth_alg_t auth, odp_ipsec_auth_capability_t capa[], int num) { - odp_crypto_auth_capability_t crypto_capa[MAX_CAPS]; uint32_t req_digest_len; - int rc, i, out; + int rc, i, out, max_capa; - rc = odp_crypto_auth_capability(auth, crypto_capa, MAX_CAPS); + max_capa = odp_crypto_auth_capability(auth, NULL, 0); + if (max_capa <= 0) + return max_capa; + + odp_crypto_auth_capability_t crypto_capa[max_capa]; + + rc = odp_crypto_auth_capability(auth, crypto_capa, max_capa); if (rc <= 0) return rc; - ODP_ASSERT(rc <= MAX_CAPS); - if (rc > MAX_CAPS) - rc = MAX_CAPS; - req_digest_len = _odp_ipsec_auth_digest_len(auth); for (i = 0, out = 0; i < rc; i++) { if (crypto_capa[i].digest_len != req_digest_len) diff --git a/platform/linux-generic/odp_timer.c b/platform/linux-generic/odp_timer.c index 3a82ee84b..5fa020118 100644 --- a/platform/linux-generic/odp_timer.c +++ b/platform/linux-generic/odp_timer.c @@ -307,9 +307,6 @@ static odp_timer_pool_t timer_pool_new(const char *name, tp_idx = i; timer_global->num_timer_pools++; - if (tp_idx > timer_global->highest_tp_idx) - timer_global->highest_tp_idx = tp_idx; - odp_ticketlock_unlock(&timer_global->lock); sz0 = ROUNDUP_CACHE_LINE(sizeof(timer_pool_t)); @@ -377,7 +374,10 @@ static odp_timer_pool_t timer_pool_new(const char *name, } tp->tp_idx = tp_idx; odp_spinlock_init(&tp->lock); + tp->start_time = odp_time_global(); + odp_ticketlock_lock(&timer_global->lock); + /* Inline timer scan may find the timer pool after this */ timer_global->timer_pool[tp_idx] = tp; if (timer_global->num_timer_pools == 1) @@ -391,13 +391,18 @@ static odp_timer_pool_t timer_pool_new(const char *name, } odp_ticketlock_unlock(&timer_global->lock); + if (!odp_global_rw->inline_timers) { if (tp->param.clk_src == ODP_CLOCK_CPU) itimer_init(tp); + } else { + /* Update the highest index for inline timer scan */ + odp_ticketlock_lock(&timer_global->lock); + if (tp_idx > timer_global->highest_tp_idx) + timer_global->highest_tp_idx = tp_idx; + odp_ticketlock_unlock(&timer_global->lock); } - tp->start_time = odp_time_global(); - return timer_pool_to_hdl(tp); } diff --git a/platform/linux-generic/pktio/pcap.c b/platform/linux-generic/pktio/pcap.c index 4332348b6..6ef998717 100644 --- a/platform/linux-generic/pktio/pcap.c +++ b/platform/linux-generic/pktio/pcap.c @@ -29,7 +29,8 @@ * doesn't exist it will be created, if it does exist it will * be overwritten. * loops the number of times to iterate through the input file, set - * to 0 to loop indefinitely. The default value is 1. + * to 0 to loop indefinitely. The default value is 1. Looping is + * only supported in thread mode (ODP_MEM_MODEL_THREAD). * * The total length of the string is limited by PKTIO_NAME_LEN. */ @@ -38,6 +39,7 @@ #include <odp_api.h> #include <odp/api/plat/packet_inlines.h> +#include <odp_global_data.h> #include <odp_packet_internal.h> #include <odp_packet_io_internal.h> @@ -199,6 +201,10 @@ static int _pcapif_reopen(pkt_pcap_t *pcap) { char errbuf[PCAP_ERRBUF_SIZE]; + /* Reopen causes pcap internal failure in process mode */ + if (odp_global_ro.init_param.mem_model == ODP_MEM_MODEL_PROCESS) + return 1; + if (pcap->loops != 0 && ++pcap->loop_cnt >= pcap->loops) return 1; diff --git a/platform/linux-generic/pktio/socket_mmap.c b/platform/linux-generic/pktio/socket_mmap.c index 97677bfce..1b1aa65f2 100644 --- a/platform/linux-generic/pktio/socket_mmap.c +++ b/platform/linux-generic/pktio/socket_mmap.c @@ -305,7 +305,7 @@ static inline int pkt_mmap_v2_tx(int sock, struct ring *ring, const odp_packet_t pkt_table[], uint32_t num) { - uint32_t i, pkt_len, num_tx; + uint32_t i, pkt_len, num_tx, tp_status; uint32_t first_frame_num, frame_num, next_frame_num, frame_count; int ret; uint8_t *buf; @@ -323,9 +323,10 @@ static inline int pkt_mmap_v2_tx(int sock, struct ring *ring, for (i = 0; i < num; i++) { tp_hdr[i] = next_ptr; + tp_status = tp_hdr[i]->tp_status & 0x7; - if (tp_hdr[i]->tp_status != TP_STATUS_AVAILABLE) { - if (tp_hdr[i]->tp_status == TP_STATUS_WRONG_FORMAT) { + if (tp_status != TP_STATUS_AVAILABLE) { + if (tp_status == TP_STATUS_WRONG_FORMAT) { ODP_ERR("Socket mmap: wrong format\n"); return -1; } @@ -359,7 +360,7 @@ static inline int pkt_mmap_v2_tx(int sock, struct ring *ring, ring->frame_num = frame_num; if (odp_unlikely(ret != total_len)) { - uint32_t tp_status, frame_sum; + uint32_t frame_sum; /* Returns -1 when nothing is sent (send() would block) */ if (ret < 0 && errno != EWOULDBLOCK) { @@ -372,7 +373,7 @@ static inline int pkt_mmap_v2_tx(int sock, struct ring *ring, * (TP_STATUS_AVAILABLE or TP_STATUS_SENDING). Assuming that * the rest will not be sent. */ for (i = 0; i < num; i++) { - tp_status = tp_hdr[i]->tp_status; + tp_status = tp_hdr[i]->tp_status & 0x7; if (tp_status == TP_STATUS_SEND_REQUEST) break; diff --git a/platform/linux-generic/test/validation/api/pktio/pktio_run_pcap.sh b/platform/linux-generic/test/validation/api/pktio/pktio_run_pcap.sh index e141bfd10..290bc81d5 100755 --- a/platform/linux-generic/test/validation/api/pktio/pktio_run_pcap.sh +++ b/platform/linux-generic/test/validation/api/pktio/pktio_run_pcap.sh @@ -28,6 +28,8 @@ else exit 1 fi +export ODP_PKTIO_TEST_DISABLE_START_STOP=1 + PCAP_FNAME=vald.pcap export ODP_PKTIO_IF0="pcap:out=${PCAP_FNAME}" export ODP_PKTIO_IF1="pcap:in=${PCAP_FNAME}" diff --git a/scripts/ci/build_arm64.sh b/scripts/ci/build_arm64.sh index 647dd29cf..abdc5acb8 100755 --- a/scripts/ci/build_arm64.sh +++ b/scripts/ci/build_arm64.sh @@ -11,4 +11,8 @@ else fi export CPPFLAGS="-I/usr/include/${TARGET_ARCH}/dpdk" +# Use target libraries +export PKG_CONFIG_PATH= +export PKG_CONFIG_LIBDIR=/usr/lib/${TARGET_ARCH}/pkgconfig + exec "$(dirname "$0")"/build.sh diff --git a/scripts/ci/build_armhf.sh b/scripts/ci/build_armhf.sh index aa42d7b36..36f54b047 100755 --- a/scripts/ci/build_armhf.sh +++ b/scripts/ci/build_armhf.sh @@ -9,8 +9,11 @@ else export CC="${TARGET_ARCH}-gcc" export CXX="${TARGET_ARCH}-g++" fi -export CPPFLAGS="-I/usr/include/${TARGET_ARCH}/dpdk" export CFLAGS="-march=armv7-a -mfpu=neon" export CXXFLAGS="-march=armv7-a" +# Use target libraries +export PKG_CONFIG_PATH= +export PKG_CONFIG_LIBDIR=/usr/lib/${TARGET_ARCH}/pkgconfig + exec "$(dirname "$0")"/build.sh diff --git a/scripts/ci/build_i386.sh b/scripts/ci/build_i386.sh index 565359679..e49558bcc 100755 --- a/scripts/ci/build_i386.sh +++ b/scripts/ci/build_i386.sh @@ -13,4 +13,8 @@ else fi export CPPFLAGS="-I/usr/include/i386-linux-gnu/dpdk" +# Use target libraries +export PKG_CONFIG_PATH= +export PKG_CONFIG_LIBDIR=/usr/lib/i386-linux-gnu/pkgconfig + exec "$(dirname "$0")"/build.sh diff --git a/scripts/ci/build_powerpc.sh b/scripts/ci/build_powerpc.sh deleted file mode 100755 index a213ee1d3..000000000 --- a/scripts/ci/build_powerpc.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash -set -e - -export TARGET_ARCH=powerpc-linux-gnu -if [ "${CC#clang}" != "${CC}" ] ; then - export CC="clang --target=${TARGET_ARCH}" - export CXX="clang++ --target=${TARGET_ARCH}" -else - export CC="${TARGET_ARCH}-gcc" - export CXX="${TARGET_ARCH}-g++" -fi -# No DPDK on PowerPC -export CONF="${CONF} --disable-dpdk" - -exec "$(dirname "$0")"/build.sh diff --git a/scripts/ci/build_ppc64el.sh b/scripts/ci/build_ppc64el.sh new file mode 100755 index 000000000..984481bb5 --- /dev/null +++ b/scripts/ci/build_ppc64el.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -e + +export TARGET_ARCH=powerpc64le-linux-gnu +if [ "${CC#clang}" != "${CC}" ] ; then + export CC="clang --target=${TARGET_ARCH}" + export CXX="clang++ --target=${TARGET_ARCH}" + # DPDK clang build broken + export CONF="${CONF} --disable-dpdk" +else + export CC="${TARGET_ARCH}-gcc" + export CXX="${TARGET_ARCH}-g++" +fi +export CPPFLAGS="-I/usr/include/${TARGET_ARCH}/dpdk" + +# Use target libraries +export PKG_CONFIG_PATH= +export PKG_CONFIG_LIBDIR=/usr/lib/${TARGET_ARCH}/pkgconfig + +exec "$(dirname "$0")"/build.sh diff --git a/test/performance/.gitignore b/test/performance/.gitignore index 41e18facc..a4e78b9af 100644 --- a/test/performance/.gitignore +++ b/test/performance/.gitignore @@ -6,6 +6,7 @@ odp_cpu_bench odp_crypto odp_ipsec odp_l2fwd +odp_packet_gen odp_pktio_ordered odp_pktio_perf odp_pool_perf diff --git a/test/performance/Makefile.am b/test/performance/Makefile.am index d0a93a253..3c75a79b8 100644 --- a/test/performance/Makefile.am +++ b/test/performance/Makefile.am @@ -12,6 +12,7 @@ EXECUTABLES = odp_bench_packet \ odp_sched_perf COMPILE_ONLY = odp_l2fwd \ + odp_packet_gen \ odp_pktio_ordered \ odp_sched_latency \ odp_sched_pktio \ @@ -19,6 +20,7 @@ COMPILE_ONLY = odp_l2fwd \ odp_timer_perf TESTSCRIPTS = odp_l2fwd_run.sh \ + odp_packet_gen_run.sh \ odp_sched_latency_run.sh \ odp_sched_pktio_run.sh \ odp_scheduling_run.sh @@ -39,6 +41,7 @@ odp_bench_packet_SOURCES = odp_bench_packet.c odp_cpu_bench_SOURCES = odp_cpu_bench.c odp_crypto_SOURCES = odp_crypto.c odp_ipsec_SOURCES = odp_ipsec.c +odp_packet_gen_SOURCES = odp_packet_gen.c odp_pktio_ordered_SOURCES = odp_pktio_ordered.c dummy_crc.h odp_sched_latency_SOURCES = odp_sched_latency.c odp_sched_pktio_SOURCES = odp_sched_pktio.c diff --git a/test/performance/odp_l2fwd.c b/test/performance/odp_l2fwd.c index fc1f04bd1..4bacc5309 100644 --- a/test/performance/odp_l2fwd.c +++ b/test/performance/odp_l2fwd.c @@ -759,7 +759,7 @@ static int create_pktio(const char *dev, int idx, int num_rx, int num_tx, mode_tx = ODP_PKTIO_OP_MT; } - pktin_param.hash_enable = 1; + pktin_param.hash_enable = (num_rx > 1) ? 1 : 0; pktin_param.hash_proto.proto.ipv4_udp = 1; pktin_param.num_queues = num_rx; pktin_param.op_mode = mode_rx; diff --git a/test/performance/odp_packet_gen.c b/test/performance/odp_packet_gen.c new file mode 100644 index 000000000..eaca511e6 --- /dev/null +++ b/test/performance/odp_packet_gen.c @@ -0,0 +1,1413 @@ +/* Copyright (c) 2020, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <stdio.h> +#include <string.h> +#include <stdint.h> +#include <inttypes.h> +#include <signal.h> +#include <stdlib.h> +#include <getopt.h> + +#include <odp_api.h> +#include <odp/helper/odph_api.h> + +#define MAX_PKTIOS 32 +#define MAX_PKTIO_NAME 255 +#define RX_THREAD 1 +#define TX_THREAD 2 +#define MAX_VLANS 4 + +/* Minimum number of packets to receive in CI test */ +#define MIN_RX_PACKETS_CI 800 + +typedef struct test_options_t { + uint64_t gap_nsec; + uint64_t quit; + uint32_t num_rx; + uint32_t num_tx; + uint32_t num_cpu; + uint32_t num_pktio; + uint32_t num_pkt; + uint32_t pkt_len; + uint32_t hdr_len; + uint32_t burst_size; + uint32_t bursts; + uint32_t num_vlan; + uint32_t ipv4_src; + uint32_t ipv4_dst; + uint16_t udp_src; + uint16_t udp_dst; + + struct vlan_hdr { + uint16_t tpid; + uint16_t tci; + } vlan[MAX_VLANS]; + + struct { + uint32_t udp_src; + uint32_t udp_dst; + } c_mode; + + char pktio_name[MAX_PKTIOS][MAX_PKTIO_NAME + 1]; + char ipv4_src_s[24]; + char ipv4_dst_s[24]; + +} test_options_t; + +typedef struct thread_arg_t { + void *global; + int tx_thr; + + /* pktout queue per pktio interface (per thread) */ + odp_pktout_queue_t pktout[MAX_PKTIOS]; + +} thread_arg_t; + +typedef struct ODP_ALIGNED_CACHE thread_stat_t { + uint64_t time_nsec; + uint64_t rx_timeouts; + uint64_t rx_packets; + uint64_t rx_bytes; + + uint64_t tx_timeouts; + uint64_t tx_packets; + uint64_t tx_drops; + + int thread_type; + +} thread_stat_t; + +typedef struct test_global_t { + test_options_t test_options; + odp_atomic_u32_t exit_test; + odp_barrier_t barrier; + odp_cpumask_t cpumask; + odp_pool_t pool; + uint64_t drained; + odph_thread_t thread_tbl[ODP_THREAD_COUNT_MAX]; + thread_stat_t stat[ODP_THREAD_COUNT_MAX]; + thread_arg_t thread_arg[ODP_THREAD_COUNT_MAX]; + + struct { + odph_ethaddr_t eth_src; + odph_ethaddr_t eth_dst; + odp_pktio_t pktio; + odp_pktout_queue_t pktout[ODP_THREAD_COUNT_MAX]; + int started; + + } pktio[MAX_PKTIOS]; + +} test_global_t; + +static test_global_t *test_global; + +static void print_usage(void) +{ + printf("\n" + "ODP packet generator\n" + "\n" + "Usage: odp_packet_gen [options]\n" + "\n" + " Mandatory:\n" + " -i, --interface <name> Packet IO interfaces. Comma-separated list of\n" + " interface names (no spaces) e.g. eth0,eth1.\n" + " At least one interface is required.\n" + "\n" + " Optional:\n" + " -e, --eth_dst <mac> Destination MAC address. Comma-separated list of\n" + " addresses (no spaces), one address per packet IO\n" + " interface e.g. AA:BB:CC:DD:EE:FF,11:22:33:44:55:66\n" + " Default per interface: 02:00:00:A0:B0:CX, where X = 0,1,...\n" + " -v, --vlan <tpid:tci> VLAN configuration. Comma-separated list of VLAN TPID:TCI\n" + " values in hexadecimal, starting from the outer most VLAN.\n" + " For example:\n" + " VLAN 200 (decimal): 8100:c8\n" + " Double tagged VLANs 1 and 2: 88a8:1,8100:2\n" + " -r, --num_rx Number of receive threads. Default: 1\n" + " -t, --num_tx Number of transmit threads. Default: 1\n" + " -n, --num_pkt Number of packets in the pool. Default: 1000\n" + " -l, --len Packet length. Default: 512\n" + " -b, --burst_size Transmit burst size. Default: 8\n" + " -x, --bursts Number of bursts per one transmit round. Default: 1\n" + " -g, --gap Gap between transmit rounds in nsec. Default: 1000000\n" + " Transmit packet rate per interface:\n" + " num_tx * burst_size * bursts * (10^9 / gap)\n" + " -s, --ipv4_src IPv4 source address. Default: 192.168.0.1\n" + " -d, --ipv4_dst IPv4 destination address. Default: 192.168.0.2\n" + " -o, --udp_src UDP source port. Default: 10000\n" + " -p, --udp_dst UDP destination port. Default: 20000\n" + " -c, --c_mode <counts> Counter mode for incrementing UDP port numbers.\n" + " Specify the number of port numbers used starting from\n" + " udp_src/udp_dst. Comma-serarated (no spaces) list of\n" + " count values: <udp_src count>,<udp_dst count>\n" + " Default value: 0,0\n" + " -q, --quit Quit after this many transmit rounds.\n" + " Default: 0 (don't quit)\n" + " -h, --help This help\n" + "\n"); +} + +static int parse_vlan(const char *str, test_global_t *global) +{ + struct vlan_hdr *vlan; + const char *start = str; + char *end; + int num_vlan = 0; + intptr_t str_len = strlen(str); + + while (num_vlan < MAX_VLANS) { + vlan = &global->test_options.vlan[num_vlan]; + + /* TPID in hexadecimal */ + end = NULL; + vlan->tpid = strtoul(start, &end, 16); + if (end < start) + break; + + /* Skip ':' */ + start = end + 1; + if (start - str >= str_len) + break; + + /* TCI in hexadecimal */ + end = NULL; + vlan->tci = strtoul(start, &end, 16); + if (end < start) + break; + + num_vlan++; + + /* Skip ',' or stop at the string end */ + start = end + 1; + if (start - str >= str_len) + break; + } + + return num_vlan; +} + +static int parse_options(int argc, char *argv[], test_global_t *global) +{ + int opt, i, len, str_len, long_index; + unsigned long int count; + uint32_t min_packets, num_tx_pkt; + char *name, *str, *end; + test_options_t *test_options = &global->test_options; + int ret = 0; + int help = 0; + uint8_t default_eth_dst[6] = {0x02, 0x00, 0x00, 0xa0, 0xb0, 0xc0}; + + static const struct option longopts[] = { + {"interface", required_argument, NULL, 'i'}, + {"eth_dst", required_argument, NULL, 'e'}, + {"num_rx", required_argument, NULL, 'r'}, + {"num_tx", required_argument, NULL, 't'}, + {"num_pkt", required_argument, NULL, 'n'}, + {"len", required_argument, NULL, 'l'}, + {"burst_size", required_argument, NULL, 'b'}, + {"bursts", required_argument, NULL, 'x'}, + {"gap", required_argument, NULL, 'g'}, + {"vlan", required_argument, NULL, 'v'}, + {"ipv4_src", required_argument, NULL, 's'}, + {"ipv4_dst", required_argument, NULL, 'd'}, + {"udp_src", required_argument, NULL, 'o'}, + {"udp_dst", required_argument, NULL, 'p'}, + {"c_mode", required_argument, NULL, 'c'}, + {"quit", required_argument, NULL, 'q'}, + {"help", no_argument, NULL, 'h'}, + {NULL, 0, NULL, 0} + }; + + static const char *shortopts = "+i:e:r:t:n:l:b:x:g:v:s:d:o:p:c:q:h"; + + test_options->num_pktio = 0; + test_options->num_rx = 1; + test_options->num_tx = 1; + test_options->num_pkt = 1000; + test_options->pkt_len = 512; + test_options->burst_size = 8; + test_options->bursts = 1; + test_options->gap_nsec = 1000000; + test_options->num_vlan = 0; + strncpy(test_options->ipv4_src_s, "192.168.0.1", + sizeof(test_options->ipv4_src_s) - 1); + strncpy(test_options->ipv4_dst_s, "192.168.0.2", + sizeof(test_options->ipv4_dst_s) - 1); + odph_ipv4_addr_parse(&test_options->ipv4_src, test_options->ipv4_src_s); + odph_ipv4_addr_parse(&test_options->ipv4_dst, test_options->ipv4_dst_s); + test_options->udp_src = 10000; + test_options->udp_dst = 20000; + test_options->c_mode.udp_src = 0; + test_options->c_mode.udp_dst = 0; + test_options->quit = 0; + + for (i = 0; i < MAX_PKTIOS; i++) { + memcpy(global->pktio[i].eth_dst.addr, default_eth_dst, 6); + global->pktio[i].eth_dst.addr[5] += i; + } + + while (1) { + opt = getopt_long(argc, argv, shortopts, longopts, &long_index); + + if (opt == -1) + break; + + switch (opt) { + case 'i': + i = 0; + str = optarg; + str_len = strlen(str); + + while (str_len > 0) { + len = strcspn(str, ","); + str_len -= len + 1; + + if (i == MAX_PKTIOS) { + printf("Error: Too many interfaces\n"); + ret = -1; + break; + } + + if (len > MAX_PKTIO_NAME) { + printf("Error: Too long interface name %s\n", + str); + ret = -1; + break; + } + + name = test_options->pktio_name[i]; + memcpy(name, str, len); + str += len + 1; + i++; + } + + test_options->num_pktio = i; + + break; + case 'e': + i = 0; + str = optarg; + str_len = strlen(str); + + while (str_len > 0) { + odph_ethaddr_t *dst = &global->pktio[i].eth_dst; + + len = strcspn(str, ","); + str_len -= len + 1; + + if (i == MAX_PKTIOS) { + printf("Error: Too many MAC addresses\n"); + ret = -1; + break; + } + + if (odph_eth_addr_parse(dst, str)) { + printf("Error: Bad MAC address: %s\n", + str); + ret = -1; + break; + } + + str += len + 1; + i++; + } + break; + case 'r': + test_options->num_rx = atoi(optarg); + break; + case 't': + test_options->num_tx = atoi(optarg); + break; + case 'n': + test_options->num_pkt = atoi(optarg); + break; + case 'l': + test_options->pkt_len = atoi(optarg); + break; + case 'b': + test_options->burst_size = atoi(optarg); + break; + case 'x': + test_options->bursts = atoi(optarg); + break; + case 'g': + test_options->gap_nsec = atoll(optarg); + break; + case 'v': + test_options->num_vlan = parse_vlan(optarg, global); + if (test_options->num_vlan == 0) { + printf("Error: Did not find any VLANs\n"); + ret = -1; + } + break; + case 's': + if (odph_ipv4_addr_parse(&test_options->ipv4_src, + optarg)) { + printf("Error: Bad IPv4 source address: %s\n", + optarg); + ret = -1; + } + strncpy(test_options->ipv4_src_s, optarg, + sizeof(test_options->ipv4_src_s) - 1); + break; + case 'd': + if (odph_ipv4_addr_parse(&test_options->ipv4_dst, + optarg)) { + printf("Error: Bad IPv4 destination address: %s\n", + optarg); + ret = -1; + } + strncpy(test_options->ipv4_dst_s, optarg, + sizeof(test_options->ipv4_dst_s) - 1); + break; + case 'c': + count = strtoul(optarg, &end, 0); + test_options->c_mode.udp_src = count; + + end++; + count = strtoul(end, NULL, 0); + test_options->c_mode.udp_dst = count; + break; + case 'q': + test_options->quit = atoll(optarg); + break; + case 'h': + /* fall through */ + default: + help = 1; + print_usage(); + ret = -1; + break; + } + } + + if (help == 0 && test_options->num_pktio == 0) { + printf("Error: At least one packet IO interface is needed.\n"); + printf(" Use -i <name> to specify interfaces.\n"); + ret = -1; + } + + if (test_options->num_rx < 1 || test_options->num_tx < 1) { + printf("Error: At least one rx and tx thread needed.\n"); + ret = -1; + } + + test_options->num_cpu = test_options->num_rx + test_options->num_tx; + + /* Pool needs to have enough packets for all tx side bursts and + * one rx side burst */ + num_tx_pkt = test_options->burst_size * test_options->bursts; + min_packets = (test_options->num_pktio * test_options->num_tx * + num_tx_pkt) + + (test_options->num_pktio * test_options->num_rx * + test_options->burst_size); + + if (test_options->num_pkt < min_packets) { + printf("Error: Pool needs to have at least %u packets\n", + min_packets); + ret = -1; + } + + if (test_options->gap_nsec) { + double gap_hz = 1000000000.0 / test_options->gap_nsec; + + if (gap_hz > (double)odp_time_local_res()) { + printf("\nWARNING: Burst gap exceeds time counter resolution " + "%" PRIu64 "\n\n", odp_time_local_res()); + } + } + + if (test_options->c_mode.udp_dst && + num_tx_pkt % test_options->c_mode.udp_dst) + printf("\nWARNING: Transmit packet count is not evenly divisible by UDP destination port count.\n\n"); + + if (test_options->c_mode.udp_src && + num_tx_pkt % test_options->c_mode.udp_src) + printf("\nWARNING: Transmit packet count is not evenly divisible by UDP source port count.\n\n"); + + test_options->hdr_len = ODPH_ETHHDR_LEN + + (test_options->num_vlan * ODPH_VLANHDR_LEN) + + ODPH_IPV4HDR_LEN + ODPH_UDPHDR_LEN; + + if (test_options->hdr_len >= test_options->pkt_len) { + printf("Error: Headers do not fit into packet length %u\n", + test_options->pkt_len); + ret = -1; + } + + return ret; +} + +static int set_num_cpu(test_global_t *global) +{ + int ret; + test_options_t *test_options = &global->test_options; + int num_cpu = test_options->num_cpu; + + /* One thread used for the main thread */ + if (num_cpu > ODP_THREAD_COUNT_MAX - 1) { + printf("Error: Too many threads. API supports max %i.\n", + ODP_THREAD_COUNT_MAX - 1); + return -1; + } + + ret = odp_cpumask_default_worker(&global->cpumask, num_cpu); + + if (ret != num_cpu) { + int cpu; + + /* Normally we want to use only worker threads */ + if (ret > 1) { + printf("Error: Too many workers. Maximum supported %i.\n", + ret); + return -1; + } + + /* When number of workers is very limited (e.g. ODP project CI), + * we try to use any CPUs available. */ + ret = odp_cpumask_all_available(&global->cpumask); + if (ret < num_cpu) { + printf("Error: Not enough CPUs. Maximum supported %i.\n", + ret); + return -1; + } + + /* Remove extra CPUs from the mask */ + cpu = odp_cpumask_first(&global->cpumask); + while (ret > num_cpu) { + odp_cpumask_clr(&global->cpumask, cpu); + cpu = odp_cpumask_first(&global->cpumask); + ret--; + } + } + + odp_barrier_init(&global->barrier, num_cpu + 1); + + return 0; +} + +static int open_pktios(test_global_t *global) +{ + odp_pool_capability_t pool_capa; + odp_pktio_capability_t pktio_capa; + odp_pool_param_t pool_param; + odp_pool_t pool; + odp_pktio_param_t pktio_param; + odp_pktio_t pktio; + odp_pktio_config_t pktio_config; + odp_pktin_queue_param_t pktin_param; + odp_pktout_queue_param_t pktout_param; + char *name; + uint32_t i, seg_len; + int j; + test_options_t *test_options = &global->test_options; + uint32_t num_rx = test_options->num_rx; + int num_tx = test_options->num_tx; + uint32_t num_pktio = test_options->num_pktio; + uint32_t num_pkt = test_options->num_pkt; + uint32_t pkt_len = test_options->pkt_len; + odp_pktout_queue_t pktout[num_tx]; + + printf("\nODP packet generator\n"); + printf(" quit test after %" PRIu64 " rounds\n", + test_options->quit); + printf(" num rx threads %u\n", num_rx); + printf(" num tx threads %i\n", num_tx); + printf(" num packets %u\n", num_pkt); + printf(" packet length %u bytes\n", pkt_len); + printf(" tx burst size %u\n", test_options->burst_size); + printf(" tx bursts %u\n", test_options->bursts); + printf(" tx burst gap %" PRIu64 " nsec\n", + test_options->gap_nsec); + printf(" clock resolution %" PRIu64 " Hz\n", odp_time_local_res()); + for (i = 0; i < test_options->num_vlan; i++) { + printf(" VLAN[%i] %x:%x\n", i, + test_options->vlan[i].tpid, test_options->vlan[i].tci); + } + printf(" IPv4 source %s\n", test_options->ipv4_src_s); + printf(" IPv4 destination %s\n", test_options->ipv4_dst_s); + printf(" UDP source %u\n", test_options->udp_src); + printf(" UDP destination %u\n", test_options->udp_dst); + printf(" UDP src count %u\n", test_options->c_mode.udp_src); + printf(" UDP dst count %u\n", test_options->c_mode.udp_dst); + printf(" num pktio %u\n", num_pktio); + + printf(" interfaces names: "); + for (i = 0; i < num_pktio; i++) { + if (i > 0) + printf(" "); + printf("%s\n", test_options->pktio_name[i]); + } + + printf(" destination MACs: "); + for (i = 0; i < num_pktio; i++) { + uint8_t *eth_dst = global->pktio[i].eth_dst.addr; + + if (i > 0) + printf(" "); + printf("%02x:%02x:%02x:%02x:%02x:%02x\n", + eth_dst[0], eth_dst[1], eth_dst[2], + eth_dst[3], eth_dst[4], eth_dst[5]); + } + printf("\n"); + + global->pool = ODP_POOL_INVALID; + + if (odp_pool_capability(&pool_capa)) { + printf("Error: Pool capability failed.\n"); + return -1; + } + + if (pool_capa.pkt.max_num && + num_pkt > pool_capa.pkt.max_num) { + printf("Error: Too many packets. Max %u supported.\n", + pool_capa.pkt.max_num); + return -1; + } + + if (pool_capa.pkt.max_len && + pkt_len > pool_capa.pkt.max_len) { + printf("Error: Too large packets. Max %u supported length.\n", + pool_capa.pkt.max_len); + return -1; + } + + seg_len = test_options->hdr_len; + if (pool_capa.pkt.max_seg_len && + seg_len > pool_capa.pkt.max_seg_len) { + printf("Error: Max segment length is too small %u\n", + pool_capa.pkt.max_seg_len); + return -1; + } + + /* Create pool */ + odp_pool_param_init(&pool_param); + pool_param.type = ODP_POOL_PACKET; + pool_param.pkt.num = num_pkt; + pool_param.pkt.len = pkt_len; + pool_param.pkt.seg_len = seg_len; + + pool = odp_pool_create("packet gen pool", &pool_param); + + if (pool == ODP_POOL_INVALID) { + printf("Error: Pool create failed.\n"); + return -1; + } + + global->pool = pool; + + odp_pktio_param_init(&pktio_param); + pktio_param.in_mode = ODP_PKTIN_MODE_SCHED; + pktio_param.out_mode = ODP_PKTOUT_MODE_DIRECT; + + for (i = 0; i < num_pktio; i++) + global->pktio[i].pktio = ODP_PKTIO_INVALID; + + /* Open and configure interfaces */ + for (i = 0; i < num_pktio; i++) { + name = test_options->pktio_name[i]; + pktio = odp_pktio_open(name, pool, &pktio_param); + + if (pktio == ODP_PKTIO_INVALID) { + printf("Error (%s): Pktio open failed.\n", name); + return -1; + } + + global->pktio[i].pktio = pktio; + + odp_pktio_print(pktio); + + if (odp_pktio_capability(pktio, &pktio_capa)) { + printf("Error (%s): Pktio capability failed.\n", name); + return -1; + } + + if (num_rx > pktio_capa.max_input_queues) { + printf("Error (%s): Too many RX threads. Interface supports max %u input queues.\n", + name, pktio_capa.max_input_queues); + return -1; + } + + if (num_tx > (int)pktio_capa.max_output_queues) { + printf("Error (%s): Too many TX threads. Interface supports max %u output queues.\n", + name, pktio_capa.max_output_queues); + return -1; + } + + if (odp_pktio_mac_addr(pktio, + &global->pktio[i].eth_src.addr, + ODPH_ETHADDR_LEN) != ODPH_ETHADDR_LEN) { + printf("Error (%s): MAC address read failed.\n", name); + return -1; + } + + odp_pktio_config_init(&pktio_config); + pktio_config.parser.layer = ODP_PROTO_LAYER_ALL; + + odp_pktio_config(pktio, &pktio_config); + + odp_pktin_queue_param_init(&pktin_param); + + pktin_param.queue_param.sched.prio = ODP_SCHED_PRIO_DEFAULT; + pktin_param.queue_param.sched.sync = ODP_SCHED_SYNC_PARALLEL; + pktin_param.queue_param.sched.group = ODP_SCHED_GROUP_ALL; + pktin_param.hash_enable = 1; + pktin_param.hash_proto.proto.ipv4_udp = 1; + pktin_param.num_queues = num_rx; + + if (odp_pktin_queue_config(pktio, &pktin_param)) { + printf("Error (%s): Pktin config failed.\n", name); + return -1; + } + + odp_pktout_queue_param_init(&pktout_param); + pktout_param.op_mode = ODP_PKTIO_OP_MT_UNSAFE; + pktout_param.num_queues = num_tx; + + if (odp_pktout_queue_config(pktio, &pktout_param)) { + printf("Error (%s): Pktout config failed.\n", name); + return -1; + } + + if (odp_pktout_queue(pktio, pktout, num_tx) != num_tx) { + printf("Error (%s): Pktout queue request failed.\n", + name); + return -1; + } + + for (j = 0; j < num_tx; j++) + global->pktio[i].pktout[j] = pktout[j]; + } + + return 0; +} + +static int start_pktios(test_global_t *global) +{ + uint32_t i; + test_options_t *test_options = &global->test_options; + uint32_t num_pktio = test_options->num_pktio; + + for (i = 0; i < num_pktio; i++) { + if (odp_pktio_start(global->pktio[i].pktio)) { + printf("Error (%s): Pktio start failed.\n", + test_options->pktio_name[i]); + + return -1; + } + + global->pktio[i].started = 1; + } + + return 0; +} + +static int stop_pktios(test_global_t *global) +{ + uint32_t i; + odp_pktio_t pktio; + int ret = 0; + test_options_t *test_options = &global->test_options; + uint32_t num_pktio = test_options->num_pktio; + + for (i = 0; i < num_pktio; i++) { + pktio = global->pktio[i].pktio; + + if (pktio == ODP_PKTIO_INVALID || global->pktio[i].started == 0) + continue; + + if (odp_pktio_stop(pktio)) { + printf("Error (%s): Pktio stop failed.\n", + test_options->pktio_name[i]); + ret = -1; + } + } + + return ret; +} + +static int close_pktios(test_global_t *global) +{ + uint32_t i; + odp_pktio_t pktio; + test_options_t *test_options = &global->test_options; + uint32_t num_pktio = test_options->num_pktio; + int ret = 0; + + for (i = 0; i < num_pktio; i++) { + pktio = global->pktio[i].pktio; + + if (pktio == ODP_PKTIO_INVALID) + continue; + + if (odp_pktio_close(pktio)) { + printf("Error (%s): Pktio close failed.\n", + test_options->pktio_name[i]); + ret = -1; + } + } + + if (global->pool != ODP_POOL_INVALID && + odp_pool_destroy(global->pool)) { + printf("Error: Pool destroy failed.\n"); + ret = -1; + } + + return ret; +} + +static int rx_thread(void *arg) +{ + int i, thr, num; + uint32_t exit_test; + odp_time_t t1, t2; + thread_arg_t *thread_arg = arg; + test_global_t *global = thread_arg->global; + uint64_t rx_timeouts = 0; + uint64_t rx_packets = 0; + uint64_t rx_bytes = 0; + uint64_t nsec = 0; + int ret = 0; + int clock_started = 0; + int paused = 0; + int max_num = 32; + odp_event_t ev[max_num]; + + thr = odp_thread_id(); + global->stat[thr].thread_type = RX_THREAD; + + /* Start all workers at the same time */ + odp_barrier_wait(&global->barrier); + + while (1) { + num = odp_schedule_multi_no_wait(NULL, ev, max_num); + + exit_test = odp_atomic_load_u32(&global->exit_test); + if (num == 0 && exit_test) { + if (paused == 0) { + odp_schedule_pause(); + paused = 1; + } else { + /* Exit schedule loop after schedule paused and + * no more packets received */ + break; + } + } + + if (num == 0) { + rx_timeouts++; + continue; + } + + if (!clock_started) { + t1 = odp_time_local(); + clock_started = 1; + } + + for (i = 0; i < num; i++) { + odp_packet_t pkt; + + pkt = odp_packet_from_event(ev[i]); + rx_bytes += odp_packet_len(pkt); + } + + odp_event_free_multi(ev, num); + rx_packets += num; + } + + t2 = odp_time_local(); + + if (clock_started) + nsec = odp_time_diff_ns(t2, t1); + + /* Update stats*/ + global->stat[thr].time_nsec = nsec; + global->stat[thr].rx_timeouts = rx_timeouts; + global->stat[thr].rx_packets = rx_packets; + global->stat[thr].rx_bytes = rx_bytes; + + return ret; +} + +static void drain_queues(test_global_t *global) +{ + odp_event_t ev; + uint64_t wait_time = odp_schedule_wait_time(100 * ODP_TIME_MSEC_IN_NS); + + while ((ev = odp_schedule(NULL, wait_time)) != ODP_EVENT_INVALID) { + global->drained++; + odp_event_free(ev); + } +} + +static int init_packets(test_global_t *global, int pktio, + odp_packet_t packet[], uint32_t num, uint16_t seq) +{ + odp_packet_t pkt; + uint32_t i, j, pkt_len, seg_len, payload_len, l2_len; + void *data; + uint8_t *u8; + odph_ethhdr_t *eth; + odph_vlanhdr_t *vlan; + odph_ipv4hdr_t *ip; + odph_udphdr_t *udp; + uint16_t tpid; + test_options_t *test_options = &global->test_options; + uint32_t num_vlan = test_options->num_vlan; + uint32_t hdr_len = test_options->hdr_len; + uint16_t udp_src = test_options->udp_src; + uint16_t udp_dst = test_options->udp_dst; + uint32_t udp_src_cnt = 0; + uint32_t udp_dst_cnt = 0; + + for (i = 0; i < num; i++) { + pkt = packet[i]; + pkt_len = odp_packet_len(pkt); + seg_len = odp_packet_seg_len(pkt); + data = odp_packet_data(pkt); + payload_len = pkt_len - hdr_len; + + if (seg_len < hdr_len) { + printf("Error: First segment too short %u\n", seg_len); + return -1; + } + + /* Ethernet */ + eth = data; + memcpy(eth->dst.addr, global->pktio[pktio].eth_dst.addr, 6); + memcpy(eth->src.addr, global->pktio[pktio].eth_src.addr, 6); + eth->type = odp_cpu_to_be_16(ODPH_ETHTYPE_IPV4); + l2_len = ODPH_ETHHDR_LEN; + + /* VLAN(s) */ + if (num_vlan) { + tpid = test_options->vlan[0].tpid; + eth->type = odp_cpu_to_be_16(tpid); + } + + for (j = 0; j < num_vlan; j++) { + vlan = (odph_vlanhdr_t *)((uint8_t *)data + l2_len); + vlan->tci = odp_cpu_to_be_16(test_options->vlan[j].tci); + if (j < num_vlan - 1) { + tpid = test_options->vlan[j + 1].tpid; + vlan->type = odp_cpu_to_be_16(tpid); + } + + l2_len += ODPH_VLANHDR_LEN; + } + + if (num_vlan) + vlan->type = odp_cpu_to_be_16(ODPH_ETHTYPE_IPV4); + + /* IPv4 */ + ip = (odph_ipv4hdr_t *)((uint8_t *)data + l2_len); + memset(ip, 0, ODPH_IPV4HDR_LEN); + ip->ver_ihl = ODPH_IPV4 << 4 | ODPH_IPV4HDR_IHL_MIN; + ip->tot_len = odp_cpu_to_be_16(pkt_len - l2_len); + ip->id = odp_cpu_to_be_16(seq + i); + ip->ttl = 64; + ip->proto = ODPH_IPPROTO_UDP; + ip->src_addr = odp_cpu_to_be_32(test_options->ipv4_src); + ip->dst_addr = odp_cpu_to_be_32(test_options->ipv4_dst); + ip->chksum = ~odp_chksum_ones_comp16(ip, ODPH_IPV4HDR_LEN); + + /* UDP */ + udp = (odph_udphdr_t *)((uint8_t *)data + l2_len + + ODPH_IPV4HDR_LEN); + memset(udp, 0, ODPH_UDPHDR_LEN); + udp->src_port = odp_cpu_to_be_16(udp_src); + udp->dst_port = odp_cpu_to_be_16(udp_dst); + udp->length = odp_cpu_to_be_16(payload_len + ODPH_UDPHDR_LEN); + udp->chksum = 0; + + /* Init UDP payload until the end of the first segment */ + u8 = data; + u8 += hdr_len; + for (j = 0; j < seg_len - hdr_len; j++) + u8[j] = j; + + /* Insert UDP checksum */ + odp_packet_l3_offset_set(pkt, l2_len); + odp_packet_l4_offset_set(pkt, l2_len + ODPH_IPV4HDR_LEN); + odp_packet_has_eth_set(pkt, 1); + odp_packet_has_ipv4_set(pkt, 1); + odp_packet_has_udp_set(pkt, 1); + udp->chksum = odph_ipv4_udp_chksum(pkt); + + /* Increment port numbers */ + if (test_options->c_mode.udp_src) { + udp_src_cnt++; + if (udp_src_cnt < test_options->c_mode.udp_src) { + udp_src++; + } else { + udp_src = test_options->udp_src; + udp_src_cnt = 0; + } + } + if (test_options->c_mode.udp_dst) { + udp_dst_cnt++; + if (udp_dst_cnt < test_options->c_mode.udp_dst) { + udp_dst++; + } else { + udp_dst = test_options->udp_dst; + udp_dst_cnt = 0; + } + } + } + + return 0; +} + +static inline int send_burst(odp_pktout_queue_t pktout, odp_packet_t pkt[], + int burst_size) { + int i, sent; + int num = burst_size; + odp_packet_t pkt_ref[burst_size]; + + for (i = 0; i < burst_size; i++) { + pkt_ref[i] = odp_packet_ref_static(pkt[i]); + + if (pkt_ref[i] == ODP_PACKET_INVALID) { + num = i; + break; + } + } + + if (odp_unlikely(num == 0)) + return 0; + + sent = odp_pktout_send(pktout, pkt_ref, num); + + if (odp_unlikely(sent < 0)) + sent = 0; + + if (odp_unlikely(sent != num)) { + uint32_t num_drop = num - sent; + + odp_packet_free_multi(&pkt_ref[sent], num_drop); + } + + return sent; +} + +static int tx_thread(void *arg) +{ + int i, thr, tx_thr, num_alloc; + uint32_t exit_test; + odp_time_t t1, t2, next_tmo; + uint64_t diff_ns, t1_nsec; + thread_arg_t *thread_arg = arg; + test_global_t *global = thread_arg->global; + test_options_t *test_options = &global->test_options; + odp_pool_t pool = global->pool; + uint64_t gap_nsec = test_options->gap_nsec; + uint64_t quit = test_options->quit; + uint64_t tx_timeouts = 0; + uint64_t tx_packets = 0; + uint64_t tx_drops = 0; + int ret = 0; + int burst_size = test_options->burst_size; + int bursts = test_options->bursts; + uint32_t num_tx = test_options->num_tx; + uint32_t pkt_len = test_options->pkt_len; + int num_pktio = test_options->num_pktio; + int num_pkt = num_pktio * bursts * burst_size; + odp_pktout_queue_t pktout[num_pktio]; + odp_packet_t pkt[num_pkt]; + + thr = odp_thread_id(); + tx_thr = thread_arg->tx_thr; + global->stat[thr].thread_type = TX_THREAD; + + num_alloc = odp_packet_alloc_multi(pool, pkt_len, pkt, num_pkt); + + if (num_alloc != num_pkt) { + printf("Error: alloc of %u packets failed (%i)\n", + num_pkt, thr); + ret = -1; + } + + /* Initialize packets per pktio interface */ + for (i = 0; ret == 0 && i < num_pktio; i++) { + int f = i * bursts * burst_size; + + pktout[i] = thread_arg->pktout[i]; + if (init_packets(global, i, &pkt[f], bursts * burst_size, f)) { + ret = -1; + break; + } + } + + /* Start all workers at the same time */ + odp_barrier_wait(&global->barrier); + + t1 = odp_time_local(); + + /* Start TX burst at different per thread offset */ + t1_nsec = odp_time_to_ns(t1) + gap_nsec + (tx_thr * gap_nsec / num_tx); + + while (ret == 0) { + exit_test = odp_atomic_load_u32(&global->exit_test); + if (exit_test) + break; + + if (quit && tx_timeouts >= quit) { + odp_atomic_inc_u32(&global->exit_test); + break; + } + + if (gap_nsec) { + uint64_t nsec = t1_nsec + tx_timeouts * gap_nsec; + + next_tmo = odp_time_local_from_ns(nsec); + odp_time_wait_until(next_tmo); + } + tx_timeouts++; + + /* Send bursts to each pktio */ + for (i = 0; i < num_pktio; i++) { + int sent, j; + int first = i * bursts * burst_size; + + for (j = 0; j < bursts; j++) { + sent = send_burst(pktout[i], + &pkt[first + j * burst_size], + burst_size); + + if (odp_unlikely(sent < 0)) { + ret = -1; + tx_drops += burst_size; + break; + } + + tx_packets += sent; + if (odp_unlikely(sent < burst_size)) + tx_drops += burst_size - sent; + } + } + } + + t2 = odp_time_local(); + diff_ns = odp_time_diff_ns(t2, t1); + + if (num_alloc > 0) + odp_packet_free_multi(pkt, num_alloc); + + /* Update stats */ + global->stat[thr].time_nsec = diff_ns; + global->stat[thr].tx_timeouts = tx_timeouts; + global->stat[thr].tx_packets = tx_packets; + global->stat[thr].tx_drops = tx_drops; + + return ret; +} + +static int start_workers(test_global_t *global, odp_instance_t instance) +{ + odph_thread_common_param_t thr_common; + int i, j, ret, tx_thr; + test_options_t *test_options = &global->test_options; + int num_pktio = test_options->num_pktio; + int num_rx = test_options->num_rx; + int num_cpu = test_options->num_cpu; + odph_thread_param_t thr_param[num_cpu]; + + memset(global->thread_tbl, 0, sizeof(global->thread_tbl)); + memset(thr_param, 0, sizeof(thr_param)); + memset(&thr_common, 0, sizeof(thr_common)); + + thr_common.instance = instance; + thr_common.cpumask = &global->cpumask; + + /* Receive threads */ + for (i = 0; i < num_rx; i++) { + thr_param[i].start = rx_thread; + thr_param[i].arg = &global->thread_arg[i]; + thr_param[i].thr_type = ODP_THREAD_WORKER; + } + + /* Transmit threads */ + tx_thr = 0; + for (i = num_rx; i < num_cpu; i++) { + for (j = 0; j < num_pktio; j++) { + odp_pktout_queue_t pktout; + + global->thread_arg[i].tx_thr = tx_thr; + + /* Dedicate a pktout queue per pktio interface + * (per TX thread) */ + pktout = global->pktio[j].pktout[tx_thr]; + global->thread_arg[i].pktout[j] = pktout; + } + + thr_param[i].start = tx_thread; + thr_param[i].arg = &global->thread_arg[i]; + thr_param[i].thr_type = ODP_THREAD_WORKER; + tx_thr++; + } + + ret = odph_thread_create(global->thread_tbl, &thr_common, thr_param, + num_cpu); + + if (ret != num_cpu) { + printf("Error: thread create failed %i\n", ret); + return -1; + } + + return 0; +} + +static int print_stat(test_global_t *global) +{ + int i, num_thr; + double rx_pkt_ave, rx_mbit_per_sec, tx_mbit_per_sec; + test_options_t *test_options = &global->test_options; + int num_rx = test_options->num_rx; + int num_tx = test_options->num_tx; + uint32_t pkt_len = test_options->pkt_len; + uint64_t rx_nsec_sum = 0; + uint64_t rx_pkt_sum = 0; + uint64_t rx_byte_sum = 0; + uint64_t rx_tmo_sum = 0; + uint64_t tx_nsec_sum = 0; + uint64_t tx_pkt_sum = 0; + uint64_t tx_drop_sum = 0; + uint64_t tx_tmo_sum = 0; + double rx_pkt_per_sec = 0.0; + double rx_byte_per_sec = 0.0; + double rx_pkt_len = 0.0; + double rx_sec = 0.0; + double tx_pkt_per_sec = 0.0; + double tx_sec = 0.0; + + printf("\nRESULTS PER THREAD\n"); + printf(" rx thread:\n"); + printf(" 1 2 3 4 5 6 7 8\n"); + printf(" ---------------------------------------------------------------------------------------\n"); + printf(" "); + + num_thr = 0; + for (i = 0; i < ODP_THREAD_COUNT_MAX; i++) { + if (global->stat[i].thread_type != RX_THREAD) + continue; + + if (num_thr && (num_thr % 8) == 0) + printf("\n "); + + printf("%10" PRIu64 " ", global->stat[i].rx_packets); + num_thr++; + } + + printf("\n\n"); + + printf(" tx thread:\n"); + printf(" 1 2 3 4 5 6 7 8\n"); + printf(" ---------------------------------------------------------------------------------------\n"); + printf(" "); + + num_thr = 0; + for (i = 0; i < ODP_THREAD_COUNT_MAX; i++) { + if (global->stat[i].thread_type != TX_THREAD) + continue; + + if (num_thr && (num_thr % 8) == 0) + printf("\n "); + + printf("%10" PRIu64 " ", global->stat[i].tx_packets); + num_thr++; + } + + printf("\n\n"); + + for (i = 0; i < ODP_THREAD_COUNT_MAX; i++) { + if (global->stat[i].thread_type == RX_THREAD) { + rx_tmo_sum += global->stat[i].rx_timeouts; + rx_pkt_sum += global->stat[i].rx_packets; + rx_byte_sum += global->stat[i].rx_bytes; + rx_nsec_sum += global->stat[i].time_nsec; + + } else if (global->stat[i].thread_type == TX_THREAD) { + tx_tmo_sum += global->stat[i].tx_timeouts; + tx_pkt_sum += global->stat[i].tx_packets; + tx_drop_sum += global->stat[i].tx_drops; + tx_nsec_sum += global->stat[i].time_nsec; + } + } + + rx_pkt_ave = (double)rx_pkt_sum / num_rx; + rx_sec = rx_nsec_sum / 1000000000.0; + tx_sec = tx_nsec_sum / 1000000000.0; + + /* Packets and bytes per thread per sec */ + if (rx_nsec_sum) { + rx_pkt_per_sec = (1000000000.0 * (double)rx_pkt_sum) / + (double)rx_nsec_sum; + + rx_byte_per_sec = 1000000000.0; + rx_byte_per_sec *= (rx_byte_sum + 24 * rx_pkt_sum); + rx_byte_per_sec /= (double)rx_nsec_sum; + } + + if (tx_nsec_sum) { + tx_pkt_per_sec = (1000000000.0 * (double)tx_pkt_sum) / + (double)tx_nsec_sum; + } + + /* Total Mbit/s */ + rx_mbit_per_sec = (num_rx * 8 * rx_byte_per_sec) / 1000000.0; + tx_mbit_per_sec = (num_tx * 8 * tx_pkt_per_sec * (pkt_len + 24)) + / 1000000.0; + + if (rx_pkt_sum) + rx_pkt_len = (double)rx_byte_sum / rx_pkt_sum; + + printf("TOTAL (%i rx and %i tx threads)\n", num_rx, num_tx); + printf(" rx timeouts: %" PRIu64 "\n", rx_tmo_sum); + printf(" rx time spent (sec): %.3f\n", rx_sec); + printf(" rx packets: %" PRIu64 "\n", rx_pkt_sum); + printf(" rx packets drained: %" PRIu64 "\n", global->drained); + printf(" rx packets per thr: %.1f\n", rx_pkt_ave); + printf(" rx packets per thr per sec: %.1f\n", rx_pkt_per_sec); + printf(" rx packets per sec: %.1f\n", num_rx * rx_pkt_per_sec); + printf(" rx ave packet len: %.1f\n", rx_pkt_len); + printf(" rx Mbit/s: %.1f\n", rx_mbit_per_sec); + printf("\n"); + printf(" tx timeouts: %" PRIu64 "\n", tx_tmo_sum); + printf(" tx time spent (sec): %.3f\n", tx_sec); + printf(" tx packets: %" PRIu64 "\n", tx_pkt_sum); + printf(" tx dropped packets: %" PRIu64 "\n", tx_drop_sum); + printf(" tx packets per thr per sec: %.1f\n", tx_pkt_per_sec); + printf(" tx packets per sec: %.1f\n", num_tx * tx_pkt_per_sec); + printf(" tx Mbit/s: %.1f\n", tx_mbit_per_sec); + printf("\n"); + + if (rx_pkt_sum < MIN_RX_PACKETS_CI) + return -1; + + return 0; +} + +static void sig_handler(int signo) +{ + (void)signo; + + if (test_global == NULL) + return; + + odp_atomic_add_u32(&test_global->exit_test, 1); +} + +int main(int argc, char **argv) +{ + odp_instance_t instance; + odp_init_t init; + test_global_t *global; + odp_shm_t shm; + int i; + int ret = 0; + + signal(SIGINT, sig_handler); + + /* List features not to be used */ + odp_init_param_init(&init); + init.not_used.feat.cls = 1; + init.not_used.feat.compress = 1; + init.not_used.feat.crypto = 1; + init.not_used.feat.ipsec = 1; + init.not_used.feat.timer = 1; + init.not_used.feat.tm = 1; + + /* Init ODP before calling anything else */ + if (odp_init_global(&instance, &init, NULL)) { + printf("Error: Global init failed.\n"); + return -1; + } + + /* Init this thread */ + if (odp_init_local(instance, ODP_THREAD_CONTROL)) { + printf("Error: Local init failed.\n"); + return -1; + } + + shm = odp_shm_reserve("packet_gen_global", sizeof(test_global_t), + ODP_CACHE_LINE_SIZE, 0); + + if (shm == ODP_SHM_INVALID) { + printf("Error: SHM reserve failed.\n"); + return -1; + } + + global = odp_shm_addr(shm); + test_global = global; + + memset(global, 0, sizeof(test_global_t)); + odp_atomic_init_u32(&global->exit_test, 0); + + for (i = 0; i < ODP_THREAD_COUNT_MAX; i++) + global->thread_arg[i].global = global; + + if (parse_options(argc, argv, global)) { + ret = -1; + goto term; + } + + odp_schedule_config(NULL); + + if (set_num_cpu(global)) { + ret = -1; + goto term; + } + + if (open_pktios(global)) { + ret = -1; + goto term; + } + + if (start_pktios(global)) { + ret = -1; + goto term; + } + + /* Start worker threads */ + start_workers(global, instance); + + /* Wait until workers have started. */ + odp_barrier_wait(&global->barrier); + + /* Wait workers to exit */ + odph_thread_join(global->thread_tbl, + global->test_options.num_cpu); + + if (stop_pktios(global)) + ret = -1; + + drain_queues(global); + + if (close_pktios(global)) + ret = -1; + + if (print_stat(global)) + ret = -2; + +term: + if (odp_shm_free(shm)) { + printf("Error: SHM free failed.\n"); + return -1; + } + + if (odp_term_local()) { + printf("Error: term local failed.\n"); + return -1; + } + + if (odp_term_global(instance)) { + printf("Error: term global failed.\n"); + return -1; + } + + return ret; +} diff --git a/test/performance/odp_packet_gen_run.sh b/test/performance/odp_packet_gen_run.sh new file mode 100755 index 000000000..eee7789b6 --- /dev/null +++ b/test/performance/odp_packet_gen_run.sh @@ -0,0 +1,74 @@ +#!/bin/sh +# +# Copyright (c) 2020, Nokia +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause +# + +# directory where test binaries have been built +TEST_DIR="${TEST_DIR:-$PWD}" + +# directory where test sources are, including scripts +TEST_SRC_DIR=$(dirname $0) + +PATH=$TEST_DIR:$PATH + +# exit codes expected by automake for skipped tests +TEST_SKIPPED=77 + +VALIDATION_TESTDIR=platform/$ODP_PLATFORM/test/validation +PLATFORM_VALIDATION=${TEST_SRC_DIR}/../../$VALIDATION_TESTDIR + +# Use installed pktio env or for make check take it from platform directory +if [ -f "./pktio_env" ]; then + . ./pktio_env +elif [ "$ODP_PLATFORM" = "" ]; then + echo "$0: error: ODP_PLATFORM must be defined" + # not skipped as this should never happen via "make check" + exit 1 +elif [ -f ${PLATFORM_VALIDATION}/api/pktio/pktio_env ]; then + . ${PLATFORM_VALIDATION}/api/pktio/pktio_env +else + echo "BUG: unable to find pktio_env!" + echo "pktio_env has to be in current directory or " + echo "in platform/\$ODP_PLATFORM/test." + echo "ODP_PLATFORM=\"$ODP_PLATFORM\"" + exit 1 +fi + +run_packet_gen() +{ + setup_pktio_env clean # install trap to call cleanup_pktio_env + + if [ $? -ne 0 ]; then + echo "setup_pktio_env error $?" + exit $TEST_SKIPPED + fi + + export ODP_PLATFORM_PARAMS="-m 256 --file-prefix="l2fwd" \ +--proc-type auto --no-pci --vdev net_pcap1,iface=$IF0 \ +--vdev net_pcap2,iface=$IF1" + + # Runs 500 * 10ms = 5 sec + # Sends 500 packets through both interfaces => total 1000 packets + odp_packet_gen${EXEEXT} -i 0,1 -b 1 -g 10000000 -q 500 + + ret=$? + + if [ $ret -eq -1 ]; then + echo "FAIL: test failed" + elif [ $ret -eq -2 ]; then + echo "FAIL: too few packets received" + fi + + cleanup_pktio_env + + exit $ret +} + +case "$1" in + setup) setup_pktio_env ;; + cleanup) cleanup_pktio_env ;; + *) run_packet_gen ;; +esac diff --git a/test/performance/odp_pktio_ordered.c b/test/performance/odp_pktio_ordered.c index 4ac6060b7..46e6c1861 100644 --- a/test/performance/odp_pktio_ordered.c +++ b/test/performance/odp_pktio_ordered.c @@ -642,7 +642,7 @@ static int create_pktio(const char *dev, int idx, int num_rx, int num_tx, mode_tx = ODP_PKTIO_OP_MT; } - pktin_param.hash_enable = 1; + pktin_param.hash_enable = (num_rx > 1) ? 1 : 0; pktin_param.hash_proto.proto.ipv4_udp = 1; pktin_param.num_queues = num_rx; pktin_param.op_mode = mode_rx; diff --git a/test/performance/odp_pktio_ordered_run.sh b/test/performance/odp_pktio_ordered_run.sh index 7cc8b2e5e..adbe50d87 100755 --- a/test/performance/odp_pktio_ordered_run.sh +++ b/test/performance/odp_pktio_ordered_run.sh @@ -11,7 +11,7 @@ TEST_DIR="${TEST_DIR:-$(dirname $0)}" DURATION=5 LOG=odp_pktio_ordered.log LOOPS=100000000 -PASS_PPS=350 +PASS_PPS=100 PCAP_IN=`find . ${TEST_SRC_DIR} $(dirname $0) -name udp64.pcap -print -quit` PCAP_OUT=/dev/null diff --git a/test/performance/odp_sched_latency.c b/test/performance/odp_sched_latency.c index c726074c5..241b3828a 100644 --- a/test/performance/odp_sched_latency.c +++ b/test/performance/odp_sched_latency.c @@ -54,6 +54,11 @@ ODP_STATIC_ASSERT(LO_PRIO_QUEUES <= MAX_QUEUES, "Too many LO priority queues"); #define HI_PRIO 0 #define LO_PRIO 1 +/* Test event forwarding mode */ +#define EVENT_FORWARD_RAND 0 +#define EVENT_FORWARD_INC 1 +#define EVENT_FORWARD_NONE 2 + /** Test event types */ typedef enum { WARM_UP, /**< Warm-up event */ @@ -75,6 +80,7 @@ typedef struct { typedef struct { unsigned int cpu_count; /**< CPU count */ odp_schedule_sync_t sync_type; /**< Scheduler sync type */ + int forward_mode; /**< Event forwarding mode */ int test_rounds; /**< Number of test rounds (millions) */ int warm_up_rounds; /**< Number of warm-up rounds */ struct { @@ -276,6 +282,11 @@ static void print_results(test_globals_t *globals) (stype == ODP_SCHED_SYNC_ATOMIC) ? "ATOMIC" : ((stype == ODP_SCHED_SYNC_ORDERED) ? "ORDERED" : "PARALLEL")); + printf(" Forwarding mode: %s\n", + (args->forward_mode == EVENT_FORWARD_RAND) ? "random" : + ((args->forward_mode == EVENT_FORWARD_INC) ? "incremental" : + "none")); + printf(" LO_PRIO queues: %i\n", args->prio[LO_PRIO].queues); if (args->prio[LO_PRIO].events_per_queue) printf(" LO_PRIO event per queue: %i\n", @@ -359,7 +370,7 @@ static int test_schedule(int thr, test_globals_t *globals) uint32_t i; test_event_t *event; test_stat_t *stats; - int dst_idx; + int dst_idx, change_queue; int warm_up_rounds = globals->args.warm_up_rounds; uint64_t test_rounds = globals->args.test_rounds * 1000000; @@ -367,6 +378,8 @@ static int test_schedule(int thr, test_globals_t *globals) globals->core_stat[thr].prio[HI_PRIO].min = UINT64_MAX; globals->core_stat[thr].prio[LO_PRIO].min = UINT64_MAX; + change_queue = globals->args.forward_mode != EVENT_FORWARD_NONE ? 1 : 0; + for (i = 0; i < test_rounds; i++) { ev = odp_schedule(&src_queue, ODP_SCHED_WAIT); @@ -399,8 +412,11 @@ static int test_schedule(int thr, test_globals_t *globals) stats->events++; } - /* Move event to next queue */ - dst_idx = event->src_idx[event->prio] + 1; + /* Move event to next queue if forwarding is enabled */ + if (change_queue) + dst_idx = event->src_idx[event->prio] + 1; + else + dst_idx = event->src_idx[event->prio]; if (dst_idx >= globals->args.prio[event->prio].queues) dst_idx = 0; event->src_idx[event->prio] = dst_idx; @@ -508,6 +524,10 @@ static void usage(void) "Optional OPTIONS:\n" " -c, --count <number> CPU count, 0=all available, default=1\n" " -d, --duration <number> Test duration in scheduling rounds (millions), default=%d, min=1\n" + " -f, --forward-mode <mode> Selection of target queue\n" + " 0: Random (default)\n" + " 1: Incremental\n" + " 2: Use source queue\n" " -l, --lo-prio-queues <number> Number of low priority scheduled queues\n" " -t, --hi-prio-queues <number> Number of high priority scheduled queues\n" " -m, --lo-prio-events-per-queue <number> Number of events per low priority queue\n" @@ -543,6 +563,7 @@ static void parse_args(int argc, char *argv[], test_args_t *args) static const struct option longopts[] = { {"count", required_argument, NULL, 'c'}, + {"forward-mode", required_argument, NULL, 'f'}, {"lo-prio-queues", required_argument, NULL, 'l'}, {"hi-prio-queues", required_argument, NULL, 't'}, {"lo-prio-events-per-queue", required_argument, NULL, 'm'}, @@ -556,9 +577,10 @@ static void parse_args(int argc, char *argv[], test_args_t *args) {NULL, 0, NULL, 0} }; - static const char *shortopts = "+c:d:s:l:t:m:n:o:p:rw:h"; + static const char *shortopts = "+c:d:f:s:l:t:m:n:o:p:rw:h"; args->cpu_count = 1; + args->forward_mode = EVENT_FORWARD_RAND; args->test_rounds = TEST_ROUNDS; args->warm_up_rounds = WARM_UP_ROUNDS; args->sync_type = ODP_SCHED_SYNC_PARALLEL; @@ -583,6 +605,9 @@ static void parse_args(int argc, char *argv[], test_args_t *args) case 'd': args->test_rounds = atoi(optarg); break; + case 'f': + args->forward_mode = atoi(optarg); + break; case 'l': args->prio[LO_PRIO].queues = atoi(optarg); break; @@ -645,6 +670,31 @@ static void parse_args(int argc, char *argv[], test_args_t *args) usage(); exit(EXIT_FAILURE); } + if (args->forward_mode > EVENT_FORWARD_NONE || + args->forward_mode < EVENT_FORWARD_RAND) { + printf("Invalid forwarding mode\n"); + usage(); + exit(EXIT_FAILURE); + } +} + +static void randomize_queues(odp_queue_t queues[], uint32_t num, uint64_t *seed) +{ + uint32_t i; + + for (i = 0; i < num; i++) { + uint32_t new_index; + odp_queue_t swap_queue; + odp_queue_t cur_queue = queues[i]; + + odp_random_test_data((uint8_t *)&new_index, sizeof(new_index), + seed); + new_index = new_index % num; + swap_queue = queues[new_index]; + + queues[new_index] = cur_queue; + queues[i] = swap_queue; + } } /** @@ -797,6 +847,12 @@ int main(int argc, char *argv[]) globals->queue[i][j] = queue; } + if (args.forward_mode == EVENT_FORWARD_RAND) { + uint64_t seed = i; + + randomize_queues(globals->queue[i], args.prio[i].queues, + &seed); + } } odp_barrier_init(&globals->barrier, num_workers); diff --git a/test/performance/odp_sched_perf.c b/test/performance/odp_sched_perf.c index b0f8f379a..f35a9bd90 100644 --- a/test/performance/odp_sched_perf.c +++ b/test/performance/odp_sched_perf.c @@ -18,6 +18,12 @@ #define MAX_QUEUES (256 * 1024) #define MAX_GROUPS 256 +/* Max time to wait for new events in nanoseconds */ +#define MAX_SCHED_WAIT_NS (10 * ODP_TIME_SEC_IN_NS) + +/* Scheduling round interval to check for MAX_SCHED_WAIT_NS */ +#define TIME_CHECK_INTERVAL (1024 * 1024) + /* Round up 'X' to a multiple of 'NUM' */ #define ROUNDUP(X, NUM) ((NUM) * (((X) + (NUM) - 1) / (NUM))) @@ -26,7 +32,7 @@ typedef struct test_options_t { uint32_t num_queue; uint32_t num_dummy; uint32_t num_event; - uint32_t num_round; + uint32_t num_sched; uint32_t num_group; uint32_t num_join; uint32_t max_burst; @@ -53,6 +59,7 @@ typedef struct test_stat_t { uint64_t cycles; uint64_t waits; uint64_t dummy_sum; + uint8_t failed; } test_stat_t; @@ -90,7 +97,7 @@ static void print_usage(void) " -q, --num_queue Number of queues. Default: 1.\n" " -d, --num_dummy Number of empty queues. Default: 0.\n" " -e, --num_event Number of events per queue. Default: 100.\n" - " -r, --num_round Number of rounds\n" + " -s, --num_sched Number of events to schedule per thread\n" " -g, --num_group Number of schedule groups. Round robins threads and queues into groups.\n" " 0: SCHED_GROUP_ALL (default)\n" " -j, --num_join Number of groups a thread joins. Threads are divide evenly into groups,\n" @@ -121,7 +128,7 @@ static int parse_options(int argc, char *argv[], test_options_t *test_options) {"num_queue", required_argument, NULL, 'q'}, {"num_dummy", required_argument, NULL, 'd'}, {"num_event", required_argument, NULL, 'e'}, - {"num_round", required_argument, NULL, 'r'}, + {"num_sched", required_argument, NULL, 's'}, {"num_group", required_argument, NULL, 'g'}, {"num_join", required_argument, NULL, 'j'}, {"burst", required_argument, NULL, 'b'}, @@ -136,13 +143,13 @@ static int parse_options(int argc, char *argv[], test_options_t *test_options) {NULL, 0, NULL, 0} }; - static const char *shortopts = "+c:q:d:e:r:g:j:b:t:f:w:k:l:n:m:h"; + static const char *shortopts = "+c:q:d:e:s:g:j:b:t:f:w:k:l:n:m:h"; test_options->num_cpu = 1; test_options->num_queue = 1; test_options->num_dummy = 0; test_options->num_event = 100; - test_options->num_round = 100000; + test_options->num_sched = 100000; test_options->num_group = 0; test_options->num_join = 0; test_options->max_burst = 100; @@ -173,8 +180,8 @@ static int parse_options(int argc, char *argv[], test_options_t *test_options) case 'e': test_options->num_event = atoi(optarg); break; - case 'r': - test_options->num_round = atoi(optarg); + case 's': + test_options->num_sched = atoi(optarg); break; case 'g': test_options->num_group = atoi(optarg); @@ -323,7 +330,7 @@ static int create_pool(test_global_t *global) uint32_t num_queue = test_options->num_queue; uint32_t num_dummy = test_options->num_dummy; uint32_t num_event = test_options->num_event; - uint32_t num_round = test_options->num_round; + uint32_t num_sched = test_options->num_sched; uint32_t max_burst = test_options->max_burst; uint32_t tot_queue = test_options->tot_queue; uint32_t tot_event = test_options->tot_event; @@ -342,7 +349,7 @@ static int create_pool(test_global_t *global) } printf("\nScheduler performance test\n"); - printf(" num rounds %u\n", num_round); + printf(" num sched %u\n", num_sched); printf(" num cpu %u\n", num_cpu); printf(" num queues %u\n", num_queue); printf(" num empty queues %u\n", num_dummy); @@ -711,14 +718,14 @@ static int test_sched(void *arg) int num, num_enq, ret, thr; uint32_t i, rounds; uint64_t c1, c2, cycles, nsec; - uint64_t events, enqueues, waits; - odp_time_t t1, t2; + uint64_t events, enqueues, waits, events_prev; + odp_time_t t1, t2, last_retry_ts; odp_queue_t queue; odp_queue_t *next; thread_arg_t *thread_arg = arg; test_global_t *global = thread_arg->global; test_options_t *test_options = &global->test_options; - uint32_t num_round = test_options->num_round; + uint32_t num_sched = test_options->num_sched; uint32_t max_burst = test_options->max_burst; uint32_t num_group = test_options->num_group; int forward = test_options->forward; @@ -730,6 +737,7 @@ static int test_sched(void *arg) uint32_t ctx_rw_words = test_options->ctx_rw_words; int touch_ctx = ctx_rd_words || ctx_rw_words; uint32_t ctx_offset = 0; + uint32_t sched_retries = 0; uint64_t data_sum = 0; uint64_t ctx_sum = 0; uint64_t wait_ns = test_options->wait_ns; @@ -775,6 +783,7 @@ static int test_sched(void *arg) enqueues = 0; events = 0; + events_prev = 0; waits = 0; ret = 0; @@ -783,12 +792,14 @@ static int test_sched(void *arg) t1 = odp_time_local(); c1 = odp_cpu_cycles(); + last_retry_ts = t1; - for (rounds = 0; rounds < num_round; rounds++) { + for (rounds = 0; events < num_sched; rounds++) { num = odp_schedule_multi(&queue, ODP_SCHED_NO_WAIT, ev, max_burst); if (odp_likely(num > 0)) { + sched_retries = 0; events += num; i = 0; @@ -822,6 +833,7 @@ static int test_sched(void *arg) if (num_enq < 0) { printf("Error: Enqueue failed. Round %u\n", rounds); + odp_event_free_multi(&ev[i], num); ret = -1; break; } @@ -835,6 +847,25 @@ static int test_sched(void *arg) break; continue; + } else if (num == 0) { + sched_retries++; + if (odp_unlikely(sched_retries > TIME_CHECK_INTERVAL)) { + odp_time_t cur_time = odp_time_local(); + + /* Measure time from the last received event and + * break if MAX_SCHED_WAIT_NS is exceeded */ + sched_retries = 0; + if (events_prev != events) { + events_prev = events; + last_retry_ts = cur_time; + } else if (odp_time_diff_ns(cur_time, + last_retry_ts) > + MAX_SCHED_WAIT_NS) { + printf("Error: scheduling timed out\n"); + ret = -1; + break; + } + } } /* <0 not specified as an error but checking anyway */ @@ -859,6 +890,7 @@ static int test_sched(void *arg) global->stat[thr].cycles = cycles; global->stat[thr].waits = waits; global->stat[thr].dummy_sum = data_sum + ctx_sum; + global->stat[thr].failed = ret; /* Pause scheduling before thread exit */ odp_schedule_pause(); @@ -974,6 +1006,10 @@ static void print_stat(test_global_t *global) /* Averages */ for (i = 0; i < ODP_THREAD_COUNT_MAX; i++) { + if (global->stat[i].failed) { + num_cpu--; + continue; + } rounds_sum += global->stat[i].rounds; enqueues_sum += global->stat[i].enqueues; events_sum += global->stat[i].events; @@ -982,7 +1018,7 @@ static void print_stat(test_global_t *global) waits_sum += global->stat[i].waits; } - if (rounds_sum == 0) { + if (rounds_sum == 0 || num_cpu <= 0) { printf("No results.\n"); return; } @@ -1006,8 +1042,13 @@ static void print_stat(test_global_t *global) if ((num % 10) == 0) printf("\n "); - printf("%6.1f ", (1000.0 * global->stat[i].events) / - global->stat[i].nsec); + if (global->stat[i].failed) + printf(" n/a "); + else + printf("%6.1f ", + (1000.0 * global->stat[i].events) / + global->stat[i].nsec); + num++; } } diff --git a/test/performance/odp_timer_perf.c b/test/performance/odp_timer_perf.c index 5a2317aab..7834d4e56 100644 --- a/test/performance/odp_timer_perf.c +++ b/test/performance/odp_timer_perf.c @@ -79,9 +79,9 @@ test_global_t test_global; static void print_usage(void) { printf("\n" - "Scheduler performance test\n" + "Timer performance test\n" "\n" - "Usage: odp_sched_perf [options]\n" + "Usage: odp_timer_perf [options]\n" "\n" " -c, --num_cpu Number of CPUs (worker threads). 0: all available CPUs. Default: 1\n" " -n, --num_tp Number of timer pools. Default: 1\n" diff --git a/test/validation/api/packet/packet.c b/test/validation/api/packet/packet.c index b7dfb59f3..6026ca32a 100644 --- a/test/validation/api/packet/packet.c +++ b/test/validation/api/packet/packet.c @@ -291,6 +291,7 @@ static void packet_test_alloc_free(void) params.pkt.seg_len = pool_capa.pkt.min_seg_len; params.pkt.len = pool_capa.pkt.min_seg_len; params.pkt.num = 1; + params.pkt.max_num = 1; pool = odp_pool_create("packet_pool_alloc", ¶ms); CU_ASSERT_FATAL(pool != ODP_POOL_INVALID); @@ -361,6 +362,7 @@ static void packet_test_alloc_free_multi(void) params.pkt.seg_len = pool_capa.pkt.min_seg_len; params.pkt.len = pool_capa.pkt.min_seg_len; params.pkt.num = num_pkt; + params.pkt.max_num = num_pkt; pool[0] = odp_pool_create("packet_pool_alloc_multi_0", ¶ms); pool[1] = odp_pool_create("packet_pool_alloc_multi_1", ¶ms); diff --git a/test/validation/api/pktio/pktio.c b/test/validation/api/pktio/pktio.c index 773d5bf2a..20c33b461 100644 --- a/test/validation/api/pktio/pktio.c +++ b/test/validation/api/pktio/pktio.c @@ -1181,13 +1181,8 @@ static void test_recv_tmo(recv_tmo_mode_e mode) memset(pkt_seq, 0, sizeof(pkt_seq)); - /* No packets sent yet, so should wait */ ns = 100 * ODP_TIME_MSEC_IN_NS; - ret = recv_packets_tmo(pktio_rx, &pkt_tbl[0], &pkt_seq[0], 1, mode, - odp_pktin_wait_time(ns), ns, 1); - CU_ASSERT(ret == 0); - ret = create_packets(pkt_tbl, pkt_seq, test_pkt_count, pktio_tx, pktio_rx); CU_ASSERT_FATAL(ret == test_pkt_count); @@ -1196,7 +1191,8 @@ static void test_recv_tmo(recv_tmo_mode_e mode) CU_ASSERT_FATAL(ret == test_pkt_count); ret = recv_packets_tmo(pktio_rx, &pkt_tbl[0], &pkt_seq[0], 1, mode, - odp_pktin_wait_time(UINT64_MAX), 0, 0); + odp_pktin_wait_time(10 * ODP_TIME_SEC_IN_NS), + 0, 0); CU_ASSERT_FATAL(ret == 1); ret = recv_packets_tmo(pktio_rx, &pkt_tbl[1], &pkt_seq[1], 1, mode, @@ -1758,13 +1754,6 @@ static void pktio_test_statistics_counters(void) CU_ASSERT(ret == 0); } - /* flush packets with magic number in pipes */ - for (i = 0; i < 1000; i++) { - ev = odp_schedule(NULL, wait); - if (ev != ODP_EVENT_INVALID) - odp_event_free(ev); - } - alloc = create_packets(tx_pkt, pkt_seq, 1000, pktio_tx, pktio_rx); ret = odp_pktio_stats_reset(pktio_tx); @@ -1835,6 +1824,13 @@ static void pktio_test_statistics_counters(void) } } +static int pktio_check_start_stop(void) +{ + if (getenv("ODP_PKTIO_TEST_DISABLE_START_STOP")) + return ODP_TEST_INACTIVE; + return ODP_TEST_ACTIVE; +} + static void pktio_test_start_stop(void) { odp_pktio_t pktio[MAX_NUM_IFACES]; @@ -2930,7 +2926,8 @@ odp_testinfo_t pktio_suite_unsegmented[] = { ODP_TEST_INFO(pktio_test_mtu), ODP_TEST_INFO(pktio_test_promisc), ODP_TEST_INFO(pktio_test_mac), - ODP_TEST_INFO(pktio_test_start_stop), + ODP_TEST_INFO_CONDITIONAL(pktio_test_start_stop, + pktio_check_start_stop), ODP_TEST_INFO(pktio_test_recv_on_wonly), ODP_TEST_INFO(pktio_test_send_on_ronly), ODP_TEST_INFO(pktio_test_plain_multi_event), diff --git a/test/validation/api/pool/pool.c b/test/validation/api/pool/pool.c index ec1b2ff5d..c4e937295 100644 --- a/test/validation/api/pool/pool.c +++ b/test/validation/api/pool/pool.c @@ -384,6 +384,45 @@ static void pool_test_pkt_max_num(void) CU_ASSERT(odp_pool_destroy(pool) == 0); } +static void pool_test_pkt_seg_len(void) +{ + uint32_t len = 1500; + uint32_t min_seg_len = 42; + uint32_t max_num = 10; + uint32_t num = 0; + uint32_t i; + odp_packet_t pkt_tbl[max_num]; + odp_pool_t pool; + odp_pool_param_t param; + + odp_pool_param_init(¶m); + + param.type = ODP_POOL_PACKET; + param.pkt.num = max_num; + param.pkt.len = len; + param.pkt.max_len = len; + param.pkt.seg_len = min_seg_len; + + pool = odp_pool_create("test_packet_seg_len", ¶m); + CU_ASSERT_FATAL(pool != ODP_POOL_INVALID); + + for (i = 0; i < max_num; i++) { + pkt_tbl[i] = odp_packet_alloc(pool, len); + + if (pkt_tbl[i] != ODP_PACKET_INVALID) + num++; + } + + CU_ASSERT(num == max_num); + + for (i = 0; i < num; i++) { + CU_ASSERT(odp_packet_seg_len(pkt_tbl[i]) >= min_seg_len); + odp_packet_free(pkt_tbl[i]); + } + + CU_ASSERT(odp_pool_destroy(pool) == 0); +} + static void pool_test_tmo_max_num(void) { odp_pool_t pool; @@ -537,6 +576,7 @@ odp_testinfo_t pool_suite[] = { ODP_TEST_INFO(pool_test_info_data_range), ODP_TEST_INFO(pool_test_buf_max_num), ODP_TEST_INFO(pool_test_pkt_max_num), + ODP_TEST_INFO(pool_test_pkt_seg_len), ODP_TEST_INFO(pool_test_tmo_max_num), ODP_TEST_INFO(pool_test_create_after_fork), ODP_TEST_INFO_NULL, diff --git a/test/validation/api/scheduler/scheduler.c b/test/validation/api/scheduler/scheduler.c index 90d30ff5b..3d5639ab2 100644 --- a/test/validation/api/scheduler/scheduler.c +++ b/test/validation/api/scheduler/scheduler.c @@ -1,5 +1,5 @@ /* Copyright (c) 2014-2018, Linaro Limited - * Copyright (c) 2019, Nokia + * Copyright (c) 2019-2020, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -10,6 +10,7 @@ #include <odp/helper/odph_api.h> #define MAX_ORDERED_LOCKS 2 +#define MAX_POOL_SIZE (1024 * 1024) #define MSG_POOL_SIZE (64 * 1024) #define QUEUES_PER_PRIO 16 #define BUF_SIZE 64 @@ -64,6 +65,7 @@ typedef struct { int buf_count; int buf_count_cpy; int queues_per_prio; + uint32_t num_queues; odp_pool_t pool; odp_pool_t queue_ctx_pool; uint32_t max_sched_queue_size; @@ -493,6 +495,138 @@ static void scheduler_test_queue_size(void) CU_ASSERT_FATAL(odp_pool_destroy(pool) == 0); } +static void scheduler_test_full_queues(void) +{ + odp_schedule_config_t default_config; + odp_pool_t pool; + odp_pool_capability_t pool_capa; + odp_pool_param_t pool_param; + odp_schedule_capability_t sched_capa; + odp_queue_param_t queue_param; + odp_event_t ev; + uint32_t i, j, k, num_bufs, events_per_queue, num_queues; + uint32_t queue_size = 2048; + int ret; + odp_schedule_sync_t sync[] = {ODP_SCHED_SYNC_PARALLEL, + ODP_SCHED_SYNC_ATOMIC, + ODP_SCHED_SYNC_ORDERED}; + + CU_ASSERT_FATAL(!odp_schedule_capability(&sched_capa)); + if (sched_capa.max_queue_size && queue_size > sched_capa.max_queue_size) + queue_size = sched_capa.max_queue_size; + + /* Scheduler has been already configured. Use default config as queue + * size and queue count. */ + odp_schedule_config_init(&default_config); + if (default_config.queue_size) + queue_size = default_config.queue_size; + num_queues = default_config.num_queues; + + /* Queues reserved by create_queues() in suite init */ + CU_ASSERT_FATAL(globals->num_queues < num_queues); + num_queues -= globals->num_queues; + + odp_queue_t queue[num_queues]; + + CU_ASSERT_FATAL(!odp_pool_capability(&pool_capa)); + num_bufs = num_queues * queue_size; + if (pool_capa.buf.max_num && num_bufs > pool_capa.buf.max_num) + num_bufs = pool_capa.buf.max_num; + if (num_bufs > MAX_POOL_SIZE) + num_bufs = MAX_POOL_SIZE; + events_per_queue = num_bufs / num_queues; + + /* Make sure there is at least one event for each queue */ + while (events_per_queue == 0) { + num_queues--; + events_per_queue = num_bufs / num_queues; + } + + odp_pool_param_init(&pool_param); + pool_param.buf.size = 100; + pool_param.buf.align = 0; + pool_param.buf.num = num_bufs; + pool_param.type = ODP_POOL_BUFFER; + + pool = odp_pool_create("test_full_queues", &pool_param); + CU_ASSERT_FATAL(pool != ODP_POOL_INVALID); + + /* Ensure that scheduler is empty */ + drain_queues(); + + /* Run test for each scheduler synchronization type */ + for (i = 0; i < 3; i++) { + uint64_t wait_time; + uint32_t num_enq = 0; + uint32_t num = 0; + + /* Create and fill all queues */ + for (j = 0; j < num_queues; j++) { + odp_queue_param_init(&queue_param); + queue_param.type = ODP_QUEUE_TYPE_SCHED; + queue_param.sched.prio = odp_schedule_default_prio(); + queue_param.sched.sync = sync[i]; + queue_param.sched.group = ODP_SCHED_GROUP_ALL; + queue_param.size = events_per_queue; + + queue[j] = odp_queue_create("test_full_queues", + &queue_param); + CU_ASSERT_FATAL(queue[j] != ODP_QUEUE_INVALID); + + for (k = 0; k < events_per_queue; k++) { + odp_buffer_t buf = odp_buffer_alloc(pool); + + CU_ASSERT(buf != ODP_BUFFER_INVALID); + if (buf == ODP_BUFFER_INVALID) + continue; + + ev = odp_buffer_to_event(buf); + ret = odp_queue_enq(queue[j], ev); + CU_ASSERT(ret == 0); + if (ret) { + odp_event_free(ev); + continue; + } + num_enq++; + } + } + /* Run normal scheduling rounds */ + wait_time = odp_schedule_wait_time(ODP_TIME_MSEC_IN_NS); + for (j = 0; j < num_bufs; j++) { + odp_queue_t src_queue; + + ev = odp_schedule(&src_queue, wait_time); + if (ev == ODP_EVENT_INVALID) + continue; + + ret = odp_queue_enq(src_queue, ev); + CU_ASSERT(ret == 0); + if (ret) { + odp_event_free(ev); + num_enq--; + } + } + /* Clean-up */ + wait_time = odp_schedule_wait_time(100 * ODP_TIME_MSEC_IN_NS); + for (j = 0; j < num_enq; j++) { + ev = odp_schedule(NULL, wait_time); + + if (ev == ODP_EVENT_INVALID) + continue; + + odp_event_free(ev); + num++; + } + + CU_ASSERT(num == num_enq); + CU_ASSERT(drain_queues() == 0); + + for (j = 0; j < num_queues; j++) + CU_ASSERT_FATAL(odp_queue_destroy(queue[j]) == 0); + } + CU_ASSERT(odp_pool_destroy(pool) == 0); +} + static void scheduler_test_order_ignore(void) { odp_queue_capability_t queue_capa; @@ -685,7 +819,7 @@ static void scheduler_test_groups(void) CU_ASSERT(lookup == ODP_SCHED_GROUP_INVALID); /* Now create it and verify we can find it */ - mygrp2 = odp_schedule_group_create("Test Group 2", &zeromask); + mygrp2 = odp_schedule_group_create("Test Group 2", &mymask); CU_ASSERT_FATAL(mygrp2 != ODP_SCHED_GROUP_INVALID); lookup = odp_schedule_group_lookup("Test Group 2"); @@ -694,15 +828,7 @@ static void scheduler_test_groups(void) /* Destroy group with no name */ CU_ASSERT_FATAL(odp_schedule_group_destroy(null_grp) == 0); - /* Verify we're not part of it */ - rc = odp_schedule_group_thrmask(mygrp2, &testmask); - CU_ASSERT(rc == 0); - CU_ASSERT(!odp_thrmask_isset(&testmask, thr_id)); - - /* Now join the group and verify we're part of it */ - rc = odp_schedule_group_join(mygrp2, &mymask); - CU_ASSERT(rc == 0); - + /* Verify we're part of group 2 */ rc = odp_schedule_group_thrmask(mygrp2, &testmask); CU_ASSERT(rc == 0); CU_ASSERT(odp_thrmask_isset(&testmask, thr_id)); @@ -711,6 +837,11 @@ static void scheduler_test_groups(void) rc = odp_schedule_group_leave(mygrp2, &mymask); CU_ASSERT(rc == 0); + /* Verify we're not part of group 2 anymore */ + rc = odp_schedule_group_thrmask(mygrp2, &testmask); + CU_ASSERT(rc == 0); + CU_ASSERT(!odp_thrmask_isset(&testmask, thr_id)); + /* Now verify scheduler adherence to groups */ odp_pool_param_init(¶ms); params.buf.size = 100; @@ -2200,6 +2331,7 @@ static int create_queues(test_globals_t *globals) } } } + globals->num_queues = prios * queues_per_prio * 4; return 0; } @@ -2510,6 +2642,7 @@ odp_testinfo_t scheduler_suite[] = { ODP_TEST_INFO(scheduler_test_queue_destroy), ODP_TEST_INFO(scheduler_test_wait), ODP_TEST_INFO(scheduler_test_queue_size), + ODP_TEST_INFO(scheduler_test_full_queues), ODP_TEST_INFO(scheduler_test_order_ignore), ODP_TEST_INFO(scheduler_test_groups), ODP_TEST_INFO(scheduler_test_pause_resume), |