diff options
Diffstat (limited to 'platform')
92 files changed, 1601 insertions, 985 deletions
diff --git a/platform/Makefile.inc b/platform/Makefile.inc index 475d38c37..c820727c2 100644 --- a/platform/Makefile.inc +++ b/platform/Makefile.inc @@ -17,9 +17,9 @@ lib_LTLIBRARIES = AM_LDFLAGS = -version-number '$(ODP_LIBSO_VERSION)' if ODP_ABI_COMPAT -AM_LDFLAGS += -export-symbols-regex '^(_deprecated_)?odp_' +AM_LDFLAGS += -export-symbols-regex '^(odp_|_deprecated_odp_)' else -AM_LDFLAGS += -export-symbols-regex '^(_deprecated_)?_?odp_' +AM_LDFLAGS += -export-symbols-regex '^(odp_|_odp_|_deprecated_odp_)' endif AM_CFLAGS = "-DODP_VERSION_BUILD=$(VERSION)" diff --git a/platform/linux-dpdk/Makefile.am b/platform/linux-dpdk/Makefile.am index 38a0258a0..e8c2531f0 100644 --- a/platform/linux-dpdk/Makefile.am +++ b/platform/linux-dpdk/Makefile.am @@ -151,7 +151,6 @@ BUILT_SOURCES = \ include/odp_libconfig_config.h __LIB__libodp_dpdk_la_SOURCES = \ - ../linux-generic/odp_atomic.c \ ../linux-generic/odp_barrier.c \ odp_buffer.c \ ../linux-generic/odp_chksum.c \ @@ -233,23 +232,29 @@ __LIB__libodp_dpdk_la_SOURCES += \ endif if ARCH_IS_ARM -__LIB__libodp_dpdk_la_SOURCES += arch/default/odp_cpu_cycles.c \ - arch/default/odp_global_time.c \ +__LIB__libodp_dpdk_la_SOURCES += arch/default/odp_atomic.c \ + arch/default/odp_cpu_cycles.c \ arch/default/odp_hash_crc32.c \ arch/arm/odp_sysinfo_parse.c odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_inlines.h \ arch/default/odp/api/abi/cpu_time.h \ arch/default/odp/api/abi/hash_crc32.h if !ODP_ABI_COMPAT -odpapiabiarchinclude_HEADERS += arch/arm/odp/api/abi/cpu.h +odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ + arch/default/odp/api/abi/atomic_inlines.h \ + arch/arm/odp/api/abi/cpu.h endif noinst_HEADERS += arch/arm/odp_atomic.h \ arch/arm/odp_cpu.h \ arch/arm/odp_cpu_idling.h \ - arch/arm/odp_llsc.h + arch/arm/odp_llsc.h \ + arch/default/odp_atomic.h \ + arch/default/odp_cpu.h \ + arch/default/odp_cpu_idling.h endif if ARCH_IS_AARCH64 -__LIB__libodp_dpdk_la_SOURCES += arch/default/odp_cpu_cycles.c \ +__LIB__libodp_dpdk_la_SOURCES += arch/aarch64/odp_atomic.c \ + arch/default/odp_cpu_cycles.c \ arch/aarch64/odp_global_time.c \ arch/default/odp_hash_crc32.c \ arch/aarch64/odp_sysinfo_parse.c @@ -257,7 +262,9 @@ odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_inlines.h \ arch/default/odp/api/abi/cpu_time.h \ arch/aarch64/odp/api/abi/hash_crc32.h if !ODP_ABI_COMPAT -odpapiabiarchinclude_HEADERS += arch/aarch64/odp/api/abi/atomic.h \ +odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ + arch/aarch64/odp/api/abi/atomic_inlines.h \ + arch/aarch64/odp/api/abi/atomic.h \ arch/aarch64/odp/api/abi/cpu.h endif noinst_HEADERS += arch/aarch64/odp_atomic.h \ @@ -266,7 +273,8 @@ noinst_HEADERS += arch/aarch64/odp_atomic.h \ arch/aarch64/odp_llsc.h endif if ARCH_IS_DEFAULT -__LIB__libodp_dpdk_la_SOURCES += arch/default/odp_cpu_cycles.c \ +__LIB__libodp_dpdk_la_SOURCES += arch/default/odp_atomic.c \ + arch/default/odp_cpu_cycles.c \ arch/default/odp_global_time.c \ arch/default/odp_hash_crc32.c \ arch/default/odp_sysinfo_parse.c @@ -274,13 +282,17 @@ odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_inlines.h \ arch/default/odp/api/abi/cpu_time.h \ arch/default/odp/api/abi/hash_crc32.h if !ODP_ABI_COMPAT -odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu.h +odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ + arch/default/odp/api/abi/atomic_inlines.h \ + arch/default/odp/api/abi/cpu.h endif -noinst_HEADERS += arch/default/odp_cpu.h \ +noinst_HEADERS += arch/default/odp_atomic.h \ + arch/default/odp_cpu.h \ arch/default/odp_cpu_idling.h endif if ARCH_IS_MIPS64 -__LIB__libodp_dpdk_la_SOURCES += arch/mips64/odp_cpu_cycles.c \ +__LIB__libodp_dpdk_la_SOURCES += arch/default/odp_atomic.c \ + arch/mips64/odp_cpu_cycles.c \ arch/default/odp_global_time.c \ arch/default/odp_hash_crc32.c \ arch/mips64/odp_sysinfo_parse.c @@ -288,13 +300,17 @@ odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_inlines.h \ arch/default/odp/api/abi/cpu_time.h \ arch/default/odp/api/abi/hash_crc32.h if !ODP_ABI_COMPAT -odpapiabiarchinclude_HEADERS += arch/mips64/odp/api/abi/cpu.h +odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ + arch/default/odp/api/abi/atomic_inlines.h \ + arch/mips64/odp/api/abi/cpu.h endif -noinst_HEADERS += arch/default/odp_cpu.h \ +noinst_HEADERS += arch/default/odp_atomic.h \ + arch/default/odp_cpu.h \ arch/default/odp_cpu_idling.h endif if ARCH_IS_POWERPC -__LIB__libodp_dpdk_la_SOURCES += arch/default/odp_cpu_cycles.c \ +__LIB__libodp_dpdk_la_SOURCES += arch/default/odp_atomic.c \ + arch/default/odp_cpu_cycles.c \ arch/default/odp_global_time.c \ arch/default/odp_hash_crc32.c \ arch/powerpc/odp_sysinfo_parse.c @@ -302,13 +318,17 @@ odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_inlines.h \ arch/default/odp/api/abi/cpu_time.h \ arch/default/odp/api/abi/hash_crc32.h if !ODP_ABI_COMPAT -odpapiabiarchinclude_HEADERS += arch/powerpc/odp/api/abi/cpu.h +odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ + arch/default/odp/api/abi/atomic_inlines.h \ + arch/powerpc/odp/api/abi/cpu.h endif -noinst_HEADERS += arch/default/odp_cpu.h \ +noinst_HEADERS += arch/default/odp_atomic.h \ + arch/default/odp_cpu.h \ arch/default/odp_cpu_idling.h endif if ARCH_IS_X86 -__LIB__libodp_dpdk_la_SOURCES += arch/x86/cpu_flags.c \ +__LIB__libodp_dpdk_la_SOURCES += arch/default/odp_atomic.c \ + arch/x86/cpu_flags.c \ arch/x86/odp_cpu_cycles.c \ arch/x86/odp_global_time.c \ arch/default/odp_hash_crc32.c \ @@ -318,10 +338,13 @@ odpapiabiarchinclude_HEADERS += arch/x86/odp/api/abi/cpu_inlines.h \ arch/x86/odp/api/abi/cpu_time.h \ arch/default/odp/api/abi/hash_crc32.h if !ODP_ABI_COMPAT -odpapiabiarchinclude_HEADERS += arch/x86/odp/api/abi/cpu.h +odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ + arch/default/odp/api/abi/atomic_inlines.h \ + arch/x86/odp/api/abi/cpu.h endif noinst_HEADERS += arch/x86/cpu_flags.h \ arch/x86/odp_cpu.h \ + arch/default/odp_atomic.h \ arch/default/odp_cpu.h \ arch/default/odp_cpu_idling.h endif @@ -332,3 +355,16 @@ __LIB__libodp_dpdk_la_LIBADD += $(LIBCONFIG_LIBS) __LIB__libodp_dpdk_la_LIBADD += $(DPDK_LIBS_LIBODP) __LIB__libodp_dpdk_la_LIBADD += $(PTHREAD_LIBS) __LIB__libodp_dpdk_la_LIBADD += $(TIMER_LIBS) + +CHECK_GLOBALS_REGEX = " (odp_|_odp_|_deprecated_odp_|miniz_|mz_|tdefl_|tinfl_|mp_hdlr_init_odp_pool_ops)" + +TESTS_ENVIRONMENT = \ + LIBTOOL="$(LIBTOOL)" \ + NM="$(NM)" \ + LIB="$(LIB)" \ + lib_LTLIBRARIES="$(lib_LTLIBRARIES)" \ + CHECK_GLOBALS_REGEX=$(CHECK_GLOBALS_REGEX) + +dist_check_SCRIPTS = check-globals.sh + +TESTS = $(dist_check_SCRIPTS) diff --git a/platform/linux-dpdk/Makefile.inc b/platform/linux-dpdk/Makefile.inc index 241eb5eac..6e153015b 100644 --- a/platform/linux-dpdk/Makefile.inc +++ b/platform/linux-dpdk/Makefile.inc @@ -3,4 +3,4 @@ AM_CXXFLAGS += $(DPDK_CFLAGS) LOG_COMPILER = $(top_builddir)/platform/linux-dpdk/test/wrapper-script.sh SH_LOG_COMPILER = $(LOG_COMPILER) -dist_check_SCRIPTS = $(top_builddir)/platform/linux-dpdk/test/wrapper-script.sh +EXTRA_DIST += $(top_builddir)/platform/linux-dpdk/test/wrapper-script.sh diff --git a/platform/linux-dpdk/check-globals.sh b/platform/linux-dpdk/check-globals.sh new file mode 120000 index 000000000..c999a29ef --- /dev/null +++ b/platform/linux-dpdk/check-globals.sh @@ -0,0 +1 @@ +../../scripts/check-globals.sh
\ No newline at end of file diff --git a/platform/linux-dpdk/include-abi/odp/api/abi/packet.h b/platform/linux-dpdk/include-abi/odp/api/abi/packet.h index e99f0d49f..3b55e2693 100644 --- a/platform/linux-dpdk/include-abi/odp/api/abi/packet.h +++ b/platform/linux-dpdk/include-abi/odp/api/abi/packet.h @@ -38,6 +38,10 @@ typedef ODP_HANDLE_T(odp_packet_vector_t); #define ODP_PACKET_VECTOR_INVALID _odp_cast_scalar(odp_packet_vector_t, 0) +typedef ODP_HANDLE_T(odp_packet_tx_compl_t); + +#define ODP_PACKET_TX_COMPL_INVALID _odp_cast_scalar(odp_packet_tx_compl_t, 0) + typedef uint8_t odp_proto_l2_type_t; #define ODP_PROTO_L2_TYPE_NONE 0 diff --git a/platform/linux-dpdk/include/odp_config_internal.h b/platform/linux-dpdk/include/odp_config_internal.h index fd3d075ec..73d2304c9 100644 --- a/platform/linux-dpdk/include/odp_config_internal.h +++ b/platform/linux-dpdk/include/odp_config_internal.h @@ -1,5 +1,5 @@ /* Copyright (c) 2016-2018, Linaro Limited - * Copyright (c) 2020, Nokia + * Copyright (c) 2020-2021, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -147,6 +147,13 @@ extern "C" { /* Maximum packet vector size */ #define CONFIG_PACKET_VECTOR_MAX_SIZE 256 +/* + * Maximum number of IPsec SAs. The actual maximum number can be further + * limited by the number of sessions supported by the crypto subsystem and + * is reported by odp_ipsec_capability(). + */ +#define CONFIG_IPSEC_MAX_NUM_SA 4000 + #ifdef __cplusplus } #endif diff --git a/platform/linux-dpdk/include/odp_errno_define.h b/platform/linux-dpdk/include/odp_errno_define.h index 4af5e1b98..9f9486267 100644 --- a/platform/linux-dpdk/include/odp_errno_define.h +++ b/platform/linux-dpdk/include/odp_errno_define.h @@ -19,7 +19,7 @@ extern "C" { #include <rte_errno.h> -#define __odp_errno (rte_errno) +#define _odp_errno (rte_errno) #ifdef __cplusplus } diff --git a/platform/linux-dpdk/include/odp_eventdev_internal.h b/platform/linux-dpdk/include/odp_eventdev_internal.h index 29e9d7a58..496a2238f 100644 --- a/platform/linux-dpdk/include/odp_eventdev_internal.h +++ b/platform/linux-dpdk/include/odp_eventdev_internal.h @@ -154,15 +154,15 @@ typedef struct { extern eventdev_global_t *_odp_eventdev_gbl; extern __thread eventdev_local_t _odp_eventdev_local; -int service_setup(uint32_t service_id); +int _odp_service_setup(uint32_t service_id); -int dummy_link_queues(uint8_t dev_id, uint8_t dummy_linked_queues[], int num); +int _odp_dummy_link_queues(uint8_t dev_id, uint8_t dummy_linked_queues[], int num); -int dummy_unlink_queues(uint8_t dev_id, uint8_t dummy_linked_queues[], int num); +int _odp_dummy_unlink_queues(uint8_t dev_id, uint8_t dummy_linked_queues[], int num); -void rx_adapter_port_stop(uint16_t port_id); +void _odp_rx_adapter_port_stop(uint16_t port_id); -int rx_adapter_close(void); +int _odp_rx_adapter_close(void); static inline uint8_t event_schedule_type(odp_schedule_sync_t sync) { diff --git a/platform/linux-dpdk/include/odp_packet_io_internal.h b/platform/linux-dpdk/include/odp_packet_io_internal.h index 631e36fb2..fe8d4770c 100644 --- a/platform/linux-dpdk/include/odp_packet_io_internal.h +++ b/platform/linux-dpdk/include/odp_packet_io_internal.h @@ -234,9 +234,9 @@ static inline void pktio_cls_enabled_set(pktio_entry_t *entry, int ena) entry->s.enabled.cls = !!ena; } -uint16_t dpdk_pktio_port_id(pktio_entry_t *entry); +uint16_t _odp_dpdk_pktio_port_id(pktio_entry_t *entry); -int input_pkts(pktio_entry_t *pktio_entry, odp_packet_t pkt_table[], int num); +int _odp_input_pkts(pktio_entry_t *pktio_entry, odp_packet_t pkt_table[], int num); static inline int _odp_pktio_tx_ts_enabled(pktio_entry_t *entry) { @@ -253,7 +253,7 @@ static inline void _odp_pktio_tx_ts_set(pktio_entry_t *entry) extern const pktio_if_ops_t _odp_loopback_pktio_ops; extern const pktio_if_ops_t _odp_null_pktio_ops; extern const pktio_if_ops_t _odp_dpdk_pktio_ops; -extern const pktio_if_ops_t * const pktio_if_ops[]; +extern const pktio_if_ops_t * const _odp_pktio_if_ops[]; /* Dummy function required by odp_pktin_recv_mq_tmo() */ static inline int diff --git a/platform/linux-dpdk/include/odp_pool_internal.h b/platform/linux-dpdk/include/odp_pool_internal.h index d0682ec43..0db8cf056 100644 --- a/platform/linux-dpdk/include/odp_pool_internal.h +++ b/platform/linux-dpdk/include/odp_pool_internal.h @@ -92,7 +92,7 @@ static inline pool_t *pool_entry_from_hdl(odp_pool_t pool_hdl) return &_odp_pool_glb->pool[_odp_typeval(pool_hdl) - 1]; } -static inline void buffer_free_multi(odp_buffer_hdr_t *buf_hdr[], int num) +static inline void _odp_buffer_free_multi(odp_buffer_hdr_t *buf_hdr[], int num) { int i; diff --git a/platform/linux-dpdk/libodp-dpdk.pc.in b/platform/linux-dpdk/libodp-dpdk.pc.in index 729191d5f..c3ee4f7b9 100644 --- a/platform/linux-dpdk/libodp-dpdk.pc.in +++ b/platform/linux-dpdk/libodp-dpdk.pc.in @@ -7,6 +7,6 @@ Name: lib@ODP_LIB_NAME@ Description: The ODP packet processing engine Version: @PKGCONFIG_VERSION@ Requires.private: libconfig -Libs: -L${libdir} -l@ODP_LIB_NAME@ @DPDK_LIBS_NON_ABI_COMPAT@ -Libs.private: @DPDK_LIBS_ABI_COMPAT@ @OPENSSL_STATIC_LIBS@ @PTHREAD_LIBS@ @TIMER_LIBS@ -lpthread @ATOMIC_LIBS@ +Libs: -L${libdir} -l@ODP_LIB_NAME@ @DPDK_LIBS_NON_ABI_COMPAT@ @ATOMIC_LIBS_NON_ABI_COMPAT@ +Libs.private: @DPDK_LIBS_ABI_COMPAT@ @OPENSSL_STATIC_LIBS@ @PTHREAD_LIBS@ @TIMER_LIBS@ -lpthread @ATOMIC_LIBS_ABI_COMPAT@ Cflags: -I${includedir} @DPDK_CFLAGS@ diff --git a/platform/linux-dpdk/m4/configure.m4 b/platform/linux-dpdk/m4/configure.m4 index b86bbbbb5..2006c0834 100644 --- a/platform/linux-dpdk/m4/configure.m4 +++ b/platform/linux-dpdk/m4/configure.m4 @@ -61,7 +61,7 @@ esac # Required for experimental rte_event_port_unlinks_in_progress() API DPDK_CFLAGS="${DPDK_CFLAGS} -DALLOW_EXPERIMENTAL_API" -AS_VAR_APPEND([PLAT_DEP_LIBS], ["${LIBCONFIG_LIBS} ${OPENSSL_LIBS} ${DPDK_LIBS_LT} ${LIBCLI_LIBS}"]) +AS_VAR_APPEND([PLAT_DEP_LIBS], ["${ATOMIC_LIBS} ${LIBCONFIG_LIBS} ${OPENSSL_LIBS} ${DPDK_LIBS_LT} ${LIBCLI_LIBS}"]) # Add text to the end of configure with platform specific settings. # Make sure it's aligned same as other lines in configure.ac. diff --git a/platform/linux-dpdk/odp_crypto.c b/platform/linux-dpdk/odp_crypto.c index 1ae54bb4a..96e6b00d3 100644 --- a/platform/linux-dpdk/odp_crypto.c +++ b/platform/linux-dpdk/odp_crypto.c @@ -623,6 +623,8 @@ int odp_crypto_capability(odp_crypto_capability_t *capability) capability->sync_mode = ODP_SUPPORT_YES; capability->async_mode = ODP_SUPPORT_PREFERRED; capability->max_sessions = MAX_SESSIONS; + capability->queue_type_plain = 1; + capability->queue_type_sched = 1; for (cdev_id = 0; cdev_id < cdev_count; cdev_id++) { struct rte_cryptodev_info dev_info; diff --git a/platform/linux-dpdk/odp_packet.c b/platform/linux-dpdk/odp_packet.c index e65a08d0e..d1b7558e5 100644 --- a/platform/linux-dpdk/odp_packet.c +++ b/platform/linux-dpdk/odp_packet.c @@ -61,18 +61,18 @@ const _odp_packet_inline_offset_t _odp_packet_inline ODP_ALIGNED_CACHE = { #include <odp/visibility_end.h> /* Catch if DPDK mbuf members sizes have changed */ -struct rte_mbuf dummy; -ODP_STATIC_ASSERT(sizeof(dummy.data_off) == sizeof(uint16_t), +struct rte_mbuf _odp_dummy_mbuf; +ODP_STATIC_ASSERT(sizeof(_odp_dummy_mbuf.data_off) == sizeof(uint16_t), "data_off should be uint16_t"); -ODP_STATIC_ASSERT(sizeof(dummy.pkt_len) == sizeof(uint32_t), +ODP_STATIC_ASSERT(sizeof(_odp_dummy_mbuf.pkt_len) == sizeof(uint32_t), "pkt_len should be uint32_t"); -ODP_STATIC_ASSERT(sizeof(dummy.data_len) == sizeof(uint16_t), +ODP_STATIC_ASSERT(sizeof(_odp_dummy_mbuf.data_len) == sizeof(uint16_t), "data_len should be uint16_t"); -ODP_STATIC_ASSERT(sizeof(dummy.nb_segs) == sizeof(uint16_t), +ODP_STATIC_ASSERT(sizeof(_odp_dummy_mbuf.nb_segs) == sizeof(uint16_t), "nb_segs should be uint16_t"); -ODP_STATIC_ASSERT(sizeof(dummy.hash.rss) == sizeof(uint32_t), +ODP_STATIC_ASSERT(sizeof(_odp_dummy_mbuf.hash.rss) == sizeof(uint32_t), "hash.rss should be uint32_t"); -ODP_STATIC_ASSERT(sizeof(dummy.ol_flags) == sizeof(uint64_t), +ODP_STATIC_ASSERT(sizeof(_odp_dummy_mbuf.ol_flags) == sizeof(uint64_t), "ol_flags should be uint64_t"); /* Check that invalid values are the same. Some versions of Clang have trouble @@ -118,7 +118,7 @@ static odp_packet_t packet_alloc(pool_t *pool, uint32_t len) struct rte_mbuf *mbuf = rte_pktmbuf_alloc(pool->rte_mempool); if (odp_unlikely(mbuf == NULL)) { - __odp_errno = ENOMEM; + _odp_errno = ENOMEM; return ODP_PACKET_INVALID; } pkt_hdr = (odp_packet_hdr_t *)mbuf; @@ -133,13 +133,13 @@ static odp_packet_t packet_alloc(pool_t *pool, uint32_t len) /* Check num_seg here so rte_pktmbuf_chain() always succeeds */ if (odp_unlikely(num_seg > RTE_MBUF_MAX_NB_SEGS)) { - __odp_errno = EOVERFLOW; + _odp_errno = EOVERFLOW; return ODP_PACKET_INVALID; } ret = rte_pktmbuf_alloc_bulk(pool->rte_mempool, mbufs, num_seg); if (odp_unlikely(ret)) { - __odp_errno = ENOMEM; + _odp_errno = ENOMEM; return ODP_PACKET_INVALID; } @@ -171,7 +171,7 @@ odp_packet_t odp_packet_alloc(odp_pool_t pool_hdl, uint32_t len) pool_t *pool = pool_entry_from_hdl(pool_hdl); if (odp_unlikely(pool->params.type != ODP_POOL_PACKET)) { - __odp_errno = EINVAL; + _odp_errno = EINVAL; return ODP_PACKET_INVALID; } @@ -188,7 +188,7 @@ int odp_packet_alloc_multi(odp_pool_t pool_hdl, uint32_t len, pool_t *pool = pool_entry_from_hdl(pool_hdl); if (odp_unlikely(pool->params.type != ODP_POOL_PACKET)) { - __odp_errno = EINVAL; + _odp_errno = EINVAL; return -1; } @@ -198,7 +198,7 @@ int odp_packet_alloc_multi(odp_pool_t pool_hdl, uint32_t len, for (i = 0; i < num; i++) { pkt[i] = packet_alloc(pool, len); if (odp_unlikely(pkt[i] == ODP_PACKET_INVALID)) - return (i == 0 && __odp_errno != ENOMEM) ? -1 : i; + return (i == 0 && _odp_errno != ENOMEM) ? -1 : i; } return i; } @@ -2321,3 +2321,56 @@ int odp_packet_payload_offset_set(odp_packet_t pkt, uint32_t offset) return 0; } + +void odp_packet_aging_tmo_set(odp_packet_t pkt, uint64_t tmo_ns) +{ + (void)pkt; + (void)tmo_ns; +} + +uint64_t odp_packet_aging_tmo(odp_packet_t pkt) +{ + (void)pkt; + return 0; +} + +int odp_packet_tx_compl_request(odp_packet_t pkt, const odp_packet_tx_compl_opt_t *opt) +{ + (void)pkt; + (void)opt; + + return -1; +} + +int odp_packet_has_tx_compl_request(odp_packet_t pkt) +{ + (void)pkt; + + return 0; +} + +odp_packet_tx_compl_t odp_packet_tx_compl_from_event(odp_event_t ev) +{ + (void)ev; + + return ODP_PACKET_TX_COMPL_INVALID; +} + +odp_event_t odp_packet_tx_compl_to_event(odp_packet_tx_compl_t tx_compl) +{ + (void)tx_compl; + + return ODP_EVENT_INVALID; +} + +void odp_packet_tx_compl_free(odp_packet_tx_compl_t tx_compl) +{ + (void)tx_compl; +} + +void *odp_packet_tx_compl_user_ptr(odp_packet_tx_compl_t tx_compl) +{ + (void)tx_compl; + + return NULL; +} diff --git a/platform/linux-dpdk/odp_packet_dpdk.c b/platform/linux-dpdk/odp_packet_dpdk.c index 4ef7bdc0c..7fd4de06d 100644 --- a/platform/linux-dpdk/odp_packet_dpdk.c +++ b/platform/linux-dpdk/odp_packet_dpdk.c @@ -129,7 +129,7 @@ static inline pkt_dpdk_t *pkt_priv(pktio_entry_t *pktio_entry) * Order matters. The first implementation to setup successfully * will be picked. * Array must be NULL terminated */ -const pktio_if_ops_t * const pktio_if_ops[] = { +const pktio_if_ops_t * const _odp_pktio_if_ops[] = { &_odp_loopback_pktio_ops, &_odp_dpdk_pktio_ops, &_odp_null_pktio_ops, @@ -140,7 +140,7 @@ extern void *pktio_entry_ptr[ODP_CONFIG_PKTIO_ENTRIES]; static uint32_t mtu_get_pkt_dpdk(pktio_entry_t *pktio_entry); -uint16_t dpdk_pktio_port_id(pktio_entry_t *pktio_entry) +uint16_t _odp_dpdk_pktio_port_id(pktio_entry_t *pktio_entry) { const pkt_dpdk_t *pkt_dpdk = pkt_priv(pktio_entry); @@ -693,7 +693,7 @@ static int close_pkt_dpdk(pktio_entry_t *pktio_entry) if (_odp_eventdev_gbl && _odp_eventdev_gbl->rx_adapter.status != RX_ADAPTER_INIT) - rx_adapter_port_stop(pkt_dpdk->port_id); + _odp_rx_adapter_port_stop(pkt_dpdk->port_id); else rte_eth_dev_stop(pkt_dpdk->port_id); @@ -873,7 +873,7 @@ static inline void prefetch_pkt(odp_packet_t pkt) odp_prefetch(&pkt_hdr->p); } -int input_pkts(pktio_entry_t *pktio_entry, odp_packet_t pkt_table[], int num) +int _odp_input_pkts(pktio_entry_t *pktio_entry, odp_packet_t pkt_table[], int num) { pkt_dpdk_t * const pkt_dpdk = pkt_priv(pktio_entry); uint16_t i; @@ -1012,10 +1012,10 @@ static int recv_pkt_dpdk(pktio_entry_t *pktio_entry, int index, odp_ticketlock_unlock(&pkt_dpdk->rx_lock[index]); /* Packets may also me received through eventdev, so don't add any - * processing here. Instead, perform all processing in input_pkts() + * processing here. Instead, perform all processing in _odp_input_pkts() * which is also called by eventdev. */ if (nb_rx) - return input_pkts(pktio_entry, pkt_table, nb_rx); + return _odp_input_pkts(pktio_entry, pkt_table, nb_rx); return 0; } @@ -1177,7 +1177,7 @@ static int send_pkt_dpdk(pktio_entry_t *pktio_entry, int index, return -1; if (odp_unlikely(mbuf->pkt_len > pkt_dpdk->mtu)) { - __odp_errno = EMSGSIZE; + _odp_errno = EMSGSIZE; return -1; } } else if (odp_unlikely(tx_ts_idx && pkts >= tx_ts_idx)) { diff --git a/platform/linux-dpdk/odp_pool.c b/platform/linux-dpdk/odp_pool.c index e36fff903..24c9c2aec 100644 --- a/platform/linux-dpdk/odp_pool.c +++ b/platform/linux-dpdk/odp_pool.c @@ -855,7 +855,7 @@ void odp_buffer_free(odp_buffer_t buf) void odp_buffer_free_multi(const odp_buffer_t buf[], int num) { - buffer_free_multi((odp_buffer_hdr_t **)(uintptr_t)buf, num); + _odp_buffer_free_multi((odp_buffer_hdr_t **)(uintptr_t)buf, num); } void odp_pool_print(odp_pool_t pool_hdl) diff --git a/platform/linux-dpdk/odp_queue_basic.c b/platform/linux-dpdk/odp_queue_basic.c index 4cf619965..4a4388b2b 100644 --- a/platform/linux-dpdk/odp_queue_basic.c +++ b/platform/linux-dpdk/odp_queue_basic.c @@ -144,6 +144,7 @@ static int queue_init_global(void) for (i = 0; i < CONFIG_MAX_QUEUES; i++) { /* init locks */ queue_entry_t *queue = qentry_from_index(i); + LOCK_INIT(queue); queue->s.index = i; queue->s.handle = (odp_queue_t)queue; @@ -370,6 +371,7 @@ static int queue_destroy(odp_queue_t handle) { int empty; queue_entry_t *queue; + queue = qentry_from_handle(handle); if (handle == ODP_QUEUE_INVALID) @@ -1168,7 +1170,7 @@ static odp_event_t queue_api_deq(odp_queue_t handle) } /* API functions */ -_odp_queue_api_fn_t queue_basic_api = { +_odp_queue_api_fn_t _odp_queue_basic_api = { .queue_create = queue_create, .queue_destroy = queue_destroy, .queue_lookup = queue_lookup, @@ -1192,7 +1194,7 @@ _odp_queue_api_fn_t queue_basic_api = { }; /* Functions towards internal components */ -queue_fn_t queue_basic_fn = { +queue_fn_t _odp_queue_basic_fn = { .init_global = queue_init_global, .term_global = queue_term_global, .init_local = queue_init_local, diff --git a/platform/linux-dpdk/odp_queue_eventdev.c b/platform/linux-dpdk/odp_queue_eventdev.c index 195a0fc68..5233031f6 100644 --- a/platform/linux-dpdk/odp_queue_eventdev.c +++ b/platform/linux-dpdk/odp_queue_eventdev.c @@ -185,7 +185,7 @@ static void print_dev_info(const struct rte_event_dev_info *info) info->event_dev_cap); } -int service_setup(uint32_t service_id) +int _odp_service_setup(uint32_t service_id) { uint32_t cores[RTE_MAX_LCORE]; uint32_t lcore = 0; @@ -315,7 +315,7 @@ static int queue_is_linked(uint8_t dev_id, uint8_t queue_id) } /* Dummy link all unlinked queues to port zero to pass evendev start */ -int dummy_link_queues(uint8_t dev_id, uint8_t dummy_linked_queues[], int num) +int _odp_dummy_link_queues(uint8_t dev_id, uint8_t dummy_linked_queues[], int num) { uint8_t priority = RTE_EVENT_DEV_PRIORITY_NORMAL; uint8_t queue_id; @@ -337,7 +337,7 @@ int dummy_link_queues(uint8_t dev_id, uint8_t dummy_linked_queues[], int num) } /* Remove dummy links to port zero */ -int dummy_unlink_queues(uint8_t dev_id, uint8_t dummy_linked_queues[], int num) +int _odp_dummy_unlink_queues(uint8_t dev_id, uint8_t dummy_linked_queues[], int num) { int i; @@ -465,8 +465,8 @@ static int init_event_dev(void) /* Eventdev requires that each queue is linked to at least one * port at startup. */ - num_dummy_links = dummy_link_queues(dev_id, dummy_links, - config.nb_event_queues); + num_dummy_links = _odp_dummy_link_queues(dev_id, dummy_links, + config.nb_event_queues); if (!(info.event_dev_cap & RTE_EVENT_DEV_CAP_DISTRIBUTED_SCHED)) { uint32_t service_id; @@ -476,7 +476,7 @@ static int init_event_dev(void) ODP_ERR("Unable to retrieve service ID\n"); return -1; } - if (service_setup(service_id)) { + if (_odp_service_setup(service_id)) { ODP_ERR("Failed to setup service core\n"); return -1; } @@ -489,7 +489,7 @@ static int init_event_dev(void) /* Unlink all ports from queues. Thread specific ports will be linked * when the application calls schedule/enqueue for the first time. */ - if (dummy_unlink_queues(dev_id, dummy_links, num_dummy_links)) { + if (_odp_dummy_unlink_queues(dev_id, dummy_links, num_dummy_links)) { rte_event_dev_stop(dev_id); rte_event_dev_close(dev_id); return -1; @@ -602,7 +602,7 @@ static int queue_term_global(void) UNLOCK(queue); } - if (rx_adapter_close()) + if (_odp_rx_adapter_close()) ret = -1; rte_event_dev_stop(_odp_eventdev_gbl->dev_id); @@ -1345,7 +1345,7 @@ static odp_event_t queue_api_deq(odp_queue_t handle) } /* API functions */ -_odp_queue_api_fn_t queue_eventdev_api = { +_odp_queue_api_fn_t _odp_queue_eventdev_api = { .queue_create = queue_create, .queue_destroy = queue_destroy, .queue_lookup = queue_lookup, @@ -1367,7 +1367,7 @@ _odp_queue_api_fn_t queue_eventdev_api = { }; /* Functions towards internal components */ -queue_fn_t queue_eventdev_fn = { +queue_fn_t _odp_queue_eventdev_fn = { .init_global = queue_init_global, .term_global = queue_term_global, .init_local = queue_init_local, diff --git a/platform/linux-dpdk/odp_queue_if.c b/platform/linux-dpdk/odp_queue_if.c index 310664fb8..3768dca54 100644 --- a/platform/linux-dpdk/odp_queue_if.c +++ b/platform/linux-dpdk/odp_queue_if.c @@ -23,11 +23,11 @@ const _odp_queue_api_fn_t *_odp_queue_api; #include <odp/visibility_end.h> -extern const _odp_queue_api_fn_t queue_basic_api; -extern const queue_fn_t queue_basic_fn; +extern const _odp_queue_api_fn_t _odp_queue_basic_api; +extern const queue_fn_t _odp_queue_basic_fn; -extern const _odp_queue_api_fn_t queue_eventdev_api; -extern const queue_fn_t queue_eventdev_fn; +extern const _odp_queue_api_fn_t _odp_queue_eventdev_api; +extern const queue_fn_t _odp_queue_eventdev_fn; const queue_fn_t *_odp_queue_fn; @@ -114,11 +114,11 @@ int _odp_queue_init_global(void) sched = _ODP_SCHEDULE_DEFAULT; if (!strcmp(sched, "basic") || !strcmp(sched, "sp")) { - _odp_queue_fn = &queue_basic_fn; - _odp_queue_api = &queue_basic_api; + _odp_queue_fn = &_odp_queue_basic_fn; + _odp_queue_api = &_odp_queue_basic_api; } else if (!strcmp(sched, "eventdev")) { - _odp_queue_fn = &queue_eventdev_fn; - _odp_queue_api = &queue_eventdev_api; + _odp_queue_fn = &_odp_queue_eventdev_fn; + _odp_queue_api = &_odp_queue_eventdev_api; } else { ODP_ABORT("Unknown scheduler specified via ODP_SCHEDULER\n"); return -1; diff --git a/platform/linux-dpdk/odp_schedule_eventdev.c b/platform/linux-dpdk/odp_schedule_eventdev.c index 4ab235d70..f7b6bbb8f 100644 --- a/platform/linux-dpdk/odp_schedule_eventdev.c +++ b/platform/linux-dpdk/odp_schedule_eventdev.c @@ -242,8 +242,8 @@ static int rx_adapter_add_queues(uint8_t rx_adapter_id, uint8_t port_id, int i; /* SW eventdev requires that all queues have ports linked */ - num_dummy_links = dummy_link_queues(_odp_eventdev_gbl->dev_id, dummy_links, - num_dummy_links); + num_dummy_links = _odp_dummy_link_queues(_odp_eventdev_gbl->dev_id, dummy_links, + num_dummy_links); for (i = 0; i < num_pktin; i++) { queue_entry_t *queue = qentry_from_handle(queues[i]); @@ -277,14 +277,14 @@ static int rx_adapter_add_queues(uint8_t rx_adapter_id, uint8_t port_id, break; } - if (dummy_unlink_queues(_odp_eventdev_gbl->dev_id, dummy_links, - num_dummy_links)) + if (_odp_dummy_unlink_queues(_odp_eventdev_gbl->dev_id, dummy_links, + num_dummy_links)) return -1; return ret; } -int rx_adapter_close(void) +int _odp_rx_adapter_close(void) { uint16_t port_id; uint8_t rx_adapter_id = _odp_eventdev_gbl->rx_adapter.id; @@ -308,7 +308,7 @@ int rx_adapter_close(void) return ret; } -void rx_adapter_port_stop(uint16_t port_id) +void _odp_rx_adapter_port_stop(uint16_t port_id) { uint8_t rx_adapter_id = _odp_eventdev_gbl->rx_adapter.id; @@ -415,7 +415,7 @@ static void schedule_pktio_start(int pktio_index, int num_pktin, int pktin_idx[], odp_queue_t queue[]) { pktio_entry_t *entry = get_pktio_entry(index_to_pktio(pktio_index)); - uint16_t port_id = dpdk_pktio_port_id(entry); + uint16_t port_id = _odp_dpdk_pktio_port_id(entry); uint8_t rx_adapter_id = _odp_eventdev_gbl->rx_adapter.id; /* All eventdev pktio devices should to be started before calling @@ -449,7 +449,7 @@ static void schedule_pktio_start(int pktio_index, int num_pktin, if (ret && ret != -ESRCH) { ODP_ABORT("Unable to retrieve service ID\n"); } else if (!ret) { - if (service_setup(service_id)) + if (_odp_service_setup(service_id)) ODP_ABORT("Unable to start RX service\n"); } @@ -539,7 +539,12 @@ static inline uint16_t event_input(struct rte_event ev[], odp_event_t out_ev[], uint16_t num_pkts = 0; uint16_t num_events = 0; uint16_t i; - uint8_t first_queue = ev[0].queue_id; + uint8_t first_queue; + + if (odp_unlikely(nb_events == 0)) + return 0; + + first_queue = ev[0].queue_id; for (i = 0; i < nb_events; i++) { struct rte_event *event = &ev[i]; @@ -568,8 +573,7 @@ static inline uint16_t event_input(struct rte_event ev[], odp_event_t out_ev[], if (num_pkts) { pktio_entry_t *entry = _odp_eventdev_gbl->pktio[pkt_table[0]->port]; - num_pkts = input_pkts(entry, (odp_packet_t *)pkt_table, - num_pkts); + num_pkts = _odp_input_pkts(entry, (odp_packet_t *)pkt_table, num_pkts); if (!odp_global_ro.init_param.not_used.feat.cls) num_pkts = classify_pkts((odp_packet_t *)pkt_table, @@ -722,10 +726,12 @@ static void schedule_resume(void) static void schedule_release_atomic(void) { + /* Nothing to do */ } static void schedule_release_ordered(void) { + /* Nothing to do */ } static uint64_t schedule_wait_time(uint64_t ns) @@ -784,9 +790,9 @@ static int schedule_thr_rem(odp_schedule_group_t group, int thr) return 0; } -/* This function is a no-op */ -static void schedule_prefetch(int num ODP_UNUSED) +static void schedule_prefetch(int num) { + (void)num; } static int schedule_num_prio(void) @@ -968,33 +974,41 @@ static int schedule_group_info(odp_schedule_group_t group, return ret; } -static void schedule_order_lock(uint32_t lock_index ODP_UNUSED) +static void schedule_order_lock(uint32_t lock_index) { + (void)lock_index; } -static void schedule_order_unlock(uint32_t lock_index ODP_UNUSED) +static void schedule_order_unlock(uint32_t lock_index) { + (void)lock_index; } -static void schedule_order_unlock_lock(uint32_t unlock_index ODP_UNUSED, - uint32_t lock_index ODP_UNUSED) +static void schedule_order_unlock_lock(uint32_t unlock_index, + uint32_t lock_index) { + (void)unlock_index; + (void)lock_index; } -static void schedule_order_lock_start(uint32_t lock_index ODP_UNUSED) +static void schedule_order_lock_start(uint32_t lock_index) { + (void)lock_index; } -static void schedule_order_lock_wait(uint32_t lock_index ODP_UNUSED) +static void schedule_order_lock_wait(uint32_t lock_index) { + (void)lock_index; } static void order_lock(void) { + /* Nothing to do */ } static void order_unlock(void) { + /* Nothing to do */ } static int schedule_capability(odp_schedule_capability_t *capa) diff --git a/platform/linux-dpdk/odp_system_info.c b/platform/linux-dpdk/odp_system_info.c index b6a723fd8..42c502fda 100644 --- a/platform/linux-dpdk/odp_system_info.c +++ b/platform/linux-dpdk/odp_system_info.c @@ -172,7 +172,8 @@ static char *get_hugepage_dir(uint64_t hugepage_sz) retval = strdup(tokens[MOUNTPT]); break; } - } else { /* there is an explicit page size, so check it */ + } else { + /* there is an explicit page size, so check it */ pagesz = rte_str_to_size(&pagesz_str[pagesize_opt_len]); if (pagesz == hugepage_sz) { retval = strdup(tokens[MOUNTPT]); diff --git a/platform/linux-dpdk/odp_timer.c b/platform/linux-dpdk/odp_timer.c index 0910d09ac..28b4b17a9 100644 --- a/platform/linux-dpdk/odp_timer.c +++ b/platform/linux-dpdk/odp_timer.c @@ -118,7 +118,7 @@ typedef struct timer_local_t { } timer_local_t; /* Points to timer global data */ -timer_global_t *timer_global; +static timer_global_t *timer_global; /* Timer thread local data */ static __thread timer_local_t timer_local; @@ -214,7 +214,7 @@ int _odp_timer_term_local(void) return 0; } -void _timer_run_inline(int dec) +void _odp_timer_run_inline(int dec) { int poll_interval = timer_global->poll_interval; odp_time_t now; @@ -428,6 +428,7 @@ odp_timer_pool_t odp_timer_pool_create(const char *name, void odp_timer_pool_start(void) { + /* Nothing to do */ } void odp_timer_pool_destroy(odp_timer_pool_t tp) diff --git a/platform/linux-dpdk/test/Makefile.am b/platform/linux-dpdk/test/Makefile.am index 6e4edf952..b21fd5314 100644 --- a/platform/linux-dpdk/test/Makefile.am +++ b/platform/linux-dpdk/test/Makefile.am @@ -56,3 +56,5 @@ clean-local: rm -f $(builddir)/$$f; \ done \ fi + +.NOTPARALLEL: diff --git a/platform/linux-dpdk/test/sched-basic.conf b/platform/linux-dpdk/test/sched-basic.conf new file mode 100644 index 000000000..c54089a0d --- /dev/null +++ b/platform/linux-dpdk/test/sched-basic.conf @@ -0,0 +1,8 @@ +# Mandatory fields +odp_implementation = "linux-dpdk" +config_file_version = "0.1.11" + +sched_basic: { + # Test scheduler with an odd spread value + prio_spread = 3 +} diff --git a/platform/linux-generic/Makefile.am b/platform/linux-generic/Makefile.am index 1d9a3bbdb..495ff1c88 100644 --- a/platform/linux-generic/Makefile.am +++ b/platform/linux-generic/Makefile.am @@ -161,7 +161,6 @@ BUILT_SOURCES = \ include/odp_libconfig_config.h __LIB__libodp_linux_la_SOURCES = \ - odp_atomic.c \ odp_barrier.c \ odp_buffer.c \ odp_chksum.c \ @@ -266,7 +265,8 @@ __LIB__libodp_linux_la_SOURCES += \ endif if ARCH_IS_ARM -__LIB__libodp_linux_la_SOURCES += arch/default/odp_cpu_cycles.c \ +__LIB__libodp_linux_la_SOURCES += arch/default/odp_atomic.c \ + arch/default/odp_cpu_cycles.c \ arch/default/odp_global_time.c \ arch/default/odp_hash_crc32.c \ arch/arm/odp_sysinfo_parse.c @@ -274,15 +274,21 @@ odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_inlines.h \ arch/default/odp/api/abi/cpu_time.h \ arch/default/odp/api/abi/hash_crc32.h if !ODP_ABI_COMPAT -odpapiabiarchinclude_HEADERS += arch/arm/odp/api/abi/cpu.h +odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ + arch/default/odp/api/abi/atomic_inlines.h \ + arch/arm/odp/api/abi/cpu.h endif noinst_HEADERS += arch/arm/odp_atomic.h \ arch/arm/odp_cpu.h \ arch/arm/odp_cpu_idling.h \ - arch/arm/odp_llsc.h + arch/arm/odp_llsc.h \ + arch/default/odp_atomic.h \ + arch/default/odp_cpu.h \ + arch/default/odp_cpu_idling.h endif if ARCH_IS_AARCH64 -__LIB__libodp_linux_la_SOURCES += arch/default/odp_cpu_cycles.c \ +__LIB__libodp_linux_la_SOURCES += arch/aarch64/odp_atomic.c \ + arch/default/odp_cpu_cycles.c \ arch/aarch64/odp_global_time.c \ arch/default/odp_hash_crc32.c \ arch/aarch64/odp_sysinfo_parse.c @@ -290,7 +296,9 @@ odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_inlines.h \ arch/default/odp/api/abi/cpu_time.h \ arch/aarch64/odp/api/abi/hash_crc32.h if !ODP_ABI_COMPAT -odpapiabiarchinclude_HEADERS += arch/aarch64/odp/api/abi/atomic.h \ +odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ + arch/aarch64/odp/api/abi/atomic_inlines.h \ + arch/aarch64/odp/api/abi/atomic.h \ arch/aarch64/odp/api/abi/cpu.h endif noinst_HEADERS += arch/aarch64/odp_atomic.h \ @@ -299,7 +307,8 @@ noinst_HEADERS += arch/aarch64/odp_atomic.h \ arch/aarch64/odp_llsc.h endif if ARCH_IS_DEFAULT -__LIB__libodp_linux_la_SOURCES += arch/default/odp_cpu_cycles.c \ +__LIB__libodp_linux_la_SOURCES += arch/default/odp_atomic.c \ + arch/default/odp_cpu_cycles.c \ arch/default/odp_global_time.c \ arch/default/odp_hash_crc32.c \ arch/default/odp_sysinfo_parse.c @@ -307,13 +316,17 @@ odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_inlines.h \ arch/default/odp/api/abi/cpu_time.h \ arch/default/odp/api/abi/hash_crc32.h if !ODP_ABI_COMPAT -odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu.h +odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ + arch/default/odp/api/abi/atomic_inlines.h \ + arch/default/odp/api/abi/cpu.h endif -noinst_HEADERS += arch/default/odp_cpu.h \ +noinst_HEADERS += arch/default/odp_atomic.h \ + arch/default/odp_cpu.h \ arch/default/odp_cpu_idling.h endif if ARCH_IS_MIPS64 -__LIB__libodp_linux_la_SOURCES += arch/mips64/odp_cpu_cycles.c \ +__LIB__libodp_linux_la_SOURCES += arch/default/odp_atomic.c \ + arch/mips64/odp_cpu_cycles.c \ arch/default/odp_global_time.c \ arch/default/odp_hash_crc32.c \ arch/mips64/odp_sysinfo_parse.c @@ -321,13 +334,17 @@ odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_inlines.h \ arch/default/odp/api/abi/cpu_time.h \ arch/default/odp/api/abi/hash_crc32.h if !ODP_ABI_COMPAT -odpapiabiarchinclude_HEADERS += arch/mips64/odp/api/abi/cpu.h +odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ + arch/default/odp/api/abi/atomic_inlines.h \ + arch/mips64/odp/api/abi/cpu.h endif -noinst_HEADERS += arch/default/odp_cpu.h \ +noinst_HEADERS += arch/default/odp_atomic.h \ + arch/default/odp_cpu.h \ arch/default/odp_cpu_idling.h endif if ARCH_IS_POWERPC -__LIB__libodp_linux_la_SOURCES += arch/default/odp_cpu_cycles.c \ +__LIB__libodp_linux_la_SOURCES += arch/default/odp_atomic.c \ + arch/default/odp_cpu_cycles.c \ arch/default/odp_global_time.c \ arch/default/odp_hash_crc32.c \ arch/powerpc/odp_sysinfo_parse.c @@ -335,13 +352,17 @@ odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_inlines.h \ arch/default/odp/api/abi/cpu_time.h \ arch/default/odp/api/abi/hash_crc32.h if !ODP_ABI_COMPAT -odpapiabiarchinclude_HEADERS += arch/powerpc/odp/api/abi/cpu.h +odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ + arch/default/odp/api/abi/atomic_inlines.h \ + arch/powerpc/odp/api/abi/cpu.h endif -noinst_HEADERS += arch/default/odp_cpu.h \ +noinst_HEADERS += arch/default/odp_atomic.h \ + arch/default/odp_cpu.h \ arch/default/odp_cpu_idling.h endif if ARCH_IS_X86 -__LIB__libodp_linux_la_SOURCES += arch/x86/cpu_flags.c \ +__LIB__libodp_linux_la_SOURCES += arch/default/odp_atomic.c \ + arch/x86/cpu_flags.c \ arch/x86/odp_cpu_cycles.c \ arch/x86/odp_global_time.c \ arch/default/odp_hash_crc32.c \ @@ -351,10 +372,13 @@ odpapiabiarchinclude_HEADERS += arch/x86/odp/api/abi/cpu_inlines.h \ arch/x86/odp/api/abi/cpu_time.h \ arch/default/odp/api/abi/hash_crc32.h if !ODP_ABI_COMPAT -odpapiabiarchinclude_HEADERS += arch/x86/odp/api/abi/cpu.h +odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ + arch/default/odp/api/abi/atomic_inlines.h \ + arch/x86/odp/api/abi/cpu.h endif noinst_HEADERS += arch/x86/cpu_flags.h \ arch/x86/odp_cpu.h \ + arch/default/odp_atomic.h \ arch/default/odp_cpu.h \ arch/default/odp_cpu_idling.h endif @@ -373,3 +397,16 @@ __LIB__libodp_linux_la_LIBADD += $(TIMER_LIBS) if ODP_PKTIO_PCAP __LIB__libodp_linux_la_LIBADD += $(PCAP_LIBS) endif + +CHECK_GLOBALS_REGEX = " (odp_|_odp_|_deprecated_odp_|miniz_|mz_|tdefl_|tinfl_|mp_hdlr_init_odp_pool_ops)" + +TESTS_ENVIRONMENT = \ + LIBTOOL="$(LIBTOOL)" \ + NM="$(NM)" \ + LIB="$(LIB)" \ + lib_LTLIBRARIES="$(lib_LTLIBRARIES)" \ + CHECK_GLOBALS_REGEX=$(CHECK_GLOBALS_REGEX) + +dist_check_SCRIPTS = check-globals.sh + +TESTS = $(dist_check_SCRIPTS) diff --git a/platform/linux-generic/arch/aarch64/odp/api/abi/atomic_inlines.h b/platform/linux-generic/arch/aarch64/odp/api/abi/atomic_inlines.h new file mode 100644 index 000000000..446f9cc50 --- /dev/null +++ b/platform/linux-generic/arch/aarch64/odp/api/abi/atomic_inlines.h @@ -0,0 +1,161 @@ +/* Copyright (c) 2021, ARM Limited + * Copyright (c) 2021, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_API_ABI_ATOMIC_INLINES_H_ +#define ODP_API_ABI_ATOMIC_INLINES_H_ + +#include <odp/api/atomic.h> + +#ifdef _ODP_LOCK_FREE_128BIT_ATOMICS + +/** + * @internal + * Helper macro for lockless atomic CAS operations on 128-bit integers + * @param[in,out] atom Pointer to the 128-bit atomic variable + * @param oper CAS operation + * @param old_val Old value + * @param new_val New value to be swapped + * @return 1 for success and 0 for fail + */ +#define ATOMIC_CAS_OP_128(atom, oper, old_val, new_val, val) \ +({ \ + odp_u128_t _val; \ + odp_atomic_u128_t *_atom = atom; \ + odp_u128_t *_old_val = old_val; \ + odp_u128_t _new_val = new_val; \ + odp_u128_t *ptr = (odp_u128_t *)(_atom); \ + register uint64_t old0 __asm__ ("x0"); \ + register uint64_t old1 __asm__ ("x1"); \ + register uint64_t new0 __asm__ ("x2"); \ + register uint64_t new1 __asm__ ("x3"); \ + old0 = (uint64_t)(_old_val)->u64[0]; \ + old1 = (uint64_t)(_old_val)->u64[1]; \ + new0 = (uint64_t)(_new_val).u64[0]; \ + new1 = (uint64_t)(_new_val).u64[1]; \ + __asm__ volatile(oper " %[old0], %[old1], %[new0], %[new1], [%[ptr]]" \ + : [old0] "+r" (old0), [old1] "+r" (old1) \ + : [new0] "r" (new0), [new1] "r" (new1), \ + [ptr] "r" (ptr) \ + : "memory"); \ + _val.u64[0] = old0; \ + _val.u64[1] = old1; \ + val = _val; \ +}) + +#define ATOMIC_CAS_OP_128_NO_ORDER(atom, old_value, new_value, val) \ + ATOMIC_CAS_OP_128(atom, "casp", old_value, new_value, val) + +#define ATOMIC_CAS_OP_128_ACQ(atom, old_value, new_value, val) \ + ATOMIC_CAS_OP_128(atom, "caspa", old_value, new_value, val) + +#define ATOMIC_CAS_OP_128_REL(atom, old_value, new_value, val) \ + ATOMIC_CAS_OP_128(atom, "caspl", old_value, new_value, val) + +#define ATOMIC_CAS_OP_128_ACQ_REL(atom, old_value, new_value, val) \ + ATOMIC_CAS_OP_128(atom, "caspal", old_value, new_value, val) + +static inline void _odp_atomic_init_u128(odp_atomic_u128_t *atom, odp_u128_t new_val) +{ + atom->v = new_val; +} + +static inline odp_u128_t _odp_atomic_load_u128(odp_atomic_u128_t *atom) +{ + odp_u128_t val, exp; + + exp.u64[0] = 0; + exp.u64[1] = 0; + ATOMIC_CAS_OP_128_NO_ORDER(atom, &exp, exp, val); + return val; +} + +static inline void _odp_atomic_store_u128(odp_atomic_u128_t *atom, odp_u128_t new_val) +{ + odp_u128_t old, val; + + old = atom->v; + + while (1) { + ATOMIC_CAS_OP_128_NO_ORDER(atom, &old, new_val, val); + + if ((val.u64[0] == old.u64[0]) && (val.u64[1] == old.u64[1])) + return; + + old = val; + } +} + +static inline int _odp_atomic_cas_u128(odp_atomic_u128_t *atom, odp_u128_t *old_val, + odp_u128_t new_val) +{ + odp_u128_t val; + + ATOMIC_CAS_OP_128_NO_ORDER(atom, old_val, new_val, val); + + if ((val.u64[0] == old_val->u64[0]) && (val.u64[1] == old_val->u64[1])) + return 1; + + old_val->u64[0] = val.u64[0]; + old_val->u64[1] = val.u64[1]; + + return 0; +} + +static inline int _odp_atomic_cas_acq_u128(odp_atomic_u128_t *atom, odp_u128_t *old_val, + odp_u128_t new_val) +{ + odp_u128_t val; + + ATOMIC_CAS_OP_128_ACQ(atom, old_val, new_val, val); + + if ((val.u64[0] == old_val->u64[0]) && (val.u64[1] == old_val->u64[1])) + return 1; + + old_val->u64[0] = val.u64[0]; + old_val->u64[1] = val.u64[1]; + + return 0; +} + +static inline int _odp_atomic_cas_rel_u128(odp_atomic_u128_t *atom, odp_u128_t *old_val, + odp_u128_t new_val) +{ + odp_u128_t val; + + ATOMIC_CAS_OP_128_REL(atom, old_val, new_val, val); + + if ((val.u64[0] == old_val->u64[0]) && (val.u64[1] == old_val->u64[1])) + return 1; + + old_val->u64[0] = val.u64[0]; + old_val->u64[1] = val.u64[1]; + + return 0; +} + +static inline int _odp_atomic_cas_acq_rel_u128(odp_atomic_u128_t *atom, odp_u128_t *old_val, + odp_u128_t new_val) +{ + odp_u128_t val; + + ATOMIC_CAS_OP_128_ACQ_REL(atom, old_val, new_val, val); + + if ((val.u64[0] == old_val->u64[0]) && (val.u64[1] == old_val->u64[1])) + return 1; + + old_val->u64[0] = val.u64[0]; + old_val->u64[1] = val.u64[1]; + + return 0; +} +#else /* _ODP_LOCK_FREE_128BIT_ATOMICS */ + +/* Use generic implementation */ +#include <odp/api/abi/atomic_generic.h> + +#endif +#endif diff --git a/platform/linux-generic/arch/aarch64/odp_atomic.c b/platform/linux-generic/arch/aarch64/odp_atomic.c new file mode 100644 index 000000000..c6b809768 --- /dev/null +++ b/platform/linux-generic/arch/aarch64/odp_atomic.c @@ -0,0 +1,56 @@ +/* Copyright (c) 2015-2018, Linaro Limited + * Copyright (c) 2021, ARM Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <odp/api/atomic.h> + +int odp_atomic_lock_free_u64(odp_atomic_op_t *atomic_op) +{ +#if __GCC_ATOMIC_LLONG_LOCK_FREE < 2 + /* All operations have locks */ + if (atomic_op) + atomic_op->all_bits = 0; + + return 0; +#else + /* All operations are lock-free */ + if (atomic_op) { + atomic_op->all_bits = ~((uint32_t)0); + atomic_op->op.init = 0; + } + + return 2; +#endif +} + +int odp_atomic_lock_free_u128(odp_atomic_op_t *atomic_op) +{ +#ifdef _ODP_LOCK_FREE_128BIT_ATOMICS + if (atomic_op) { + atomic_op->all_bits = 0; + atomic_op->op.load = 1; + atomic_op->op.store = 1; + atomic_op->op.cas = 1; + } + + return 2; +#elif defined(__SIZEOF_INT128__) + if (__atomic_is_lock_free(16, NULL)) { + if (atomic_op) { + atomic_op->all_bits = 0; + atomic_op->op.load = 1; + atomic_op->op.store = 1; + atomic_op->op.cas = 1; + } + return 2; + } +#endif + /* All operations have locks */ + if (atomic_op) + atomic_op->all_bits = 0; + + return 0; +} diff --git a/platform/linux-generic/arch/aarch64/odp_atomic.h b/platform/linux-generic/arch/aarch64/odp_atomic.h index dbeccebde..c7a28fc2c 100644 --- a/platform/linux-generic/arch/aarch64/odp_atomic.h +++ b/platform/linux-generic/arch/aarch64/odp_atomic.h @@ -12,6 +12,8 @@ #error This file should not be included directly, please include odp_cpu.h #endif +#include <limits.h> + #ifdef CONFIG_DMBSTR #define atomic_store_release(loc, val, ro) \ @@ -217,54 +219,108 @@ static inline __int128 __lockfree_load_16(__int128 *var, int mo) return old; } -#ifdef _ODP_LOCK_FREE_128BIT_ATOMICS +typedef unsigned __int128 _u128_t; -/** - * @internal - * Helper macro for lockless atomic CAS operations on 128-bit integers - * @param[in,out] atom Pointer to the 128-bit atomic variable - * @param oper CAS operation - * @param old_val Old value - * @param new_val New value to be swapped - * @return 1 for success and 0 for fail - */ -#define ATOMIC_CAS_OP_128(atom, oper, old_val, new_val, val) \ -({ \ - odp_u128_t _val; \ - odp_atomic_u128_t *_atom = atom; \ - odp_u128_t *_old_val = old_val; \ - odp_u128_t _new_val = new_val; \ - odp_u128_t *ptr = (odp_u128_t *)(_atom); \ - register uint64_t old0 __asm__ ("x0"); \ - register uint64_t old1 __asm__ ("x1"); \ - register uint64_t new0 __asm__ ("x2"); \ - register uint64_t new1 __asm__ ("x3"); \ - old0 = (uint64_t)(_old_val)->u64[0]; \ - old1 = (uint64_t)(_old_val)->u64[1]; \ - new0 = (uint64_t)(_new_val).u64[0]; \ - new1 = (uint64_t)(_new_val).u64[1]; \ - __asm__ volatile(oper " %[old0], %[old1], %[new0], %[new1], [%[ptr]]" \ - : [old0] "+r" (old0), [old1] "+r" (old1) \ - : [new0] "r" (new0), [new1] "r" (new1), \ - [ptr] "r" (ptr) \ - : "memory"); \ - _val.u64[0] = old0; \ - _val.u64[1] = old1; \ - val = _val; \ -}) - -#define ATOMIC_CAS_OP_128_NO_ORDER(atom, old_value, new_value, val) \ - ATOMIC_CAS_OP_128(atom, "casp", old_value, new_value, val) - -#define ATOMIC_CAS_OP_128_ACQ(atom, old_value, new_value, val) \ - ATOMIC_CAS_OP_128(atom, "caspa", old_value, new_value, val) - -#define ATOMIC_CAS_OP_128_REL(atom, old_value, new_value, val) \ - ATOMIC_CAS_OP_128(atom, "caspl", old_value, new_value, val) - -#define ATOMIC_CAS_OP_128_ACQ_REL(atom, old_value, new_value, val) \ - ATOMIC_CAS_OP_128(atom, "caspal", old_value, new_value, val) +static inline _u128_t lockfree_load_u128(_u128_t *atomic) +{ + return __lockfree_load_16((__int128 *)atomic, __ATOMIC_RELAXED); +} + +static inline int lockfree_cas_acq_rel_u128(_u128_t *atomic, + _u128_t old_val, + _u128_t new_val) +{ + return __lockfree_compare_exchange_16((__int128 *)atomic, + (__int128 *)&old_val, + new_val, + 0, + __ATOMIC_ACQ_REL, + __ATOMIC_RELAXED); +} + +static inline int lockfree_check_u128(void) +{ + return 1; +} + +/** Atomic bit set operations with memory ordering */ +#if defined(__SIZEOF_INT128__) && __SIZEOF_INT128__ == 16 +typedef __int128 bitset_t; +#define ATOM_BITSET_SIZE (CHAR_BIT * __SIZEOF_INT128__) + +#elif __GCC_ATOMIC_LLONG_LOCK_FREE == 2 && \ + __SIZEOF_LONG_LONG__ != __SIZEOF_LONG__ +typedef unsigned long long bitset_t; +#define ATOM_BITSET_SIZE (CHAR_BIT * __SIZEOF_LONG_LONG__) +#elif __GCC_ATOMIC_LONG_LOCK_FREE == 2 && __SIZEOF_LONG__ != __SIZEOF_INT__ +typedef unsigned long bitset_t; +#define ATOM_BITSET_SIZE (CHAR_BIT * __SIZEOF_LONG__) + +#elif __GCC_ATOMIC_INT_LOCK_FREE == 2 +typedef unsigned int bitset_t; +#define ATOM_BITSET_SIZE (CHAR_BIT * __SIZEOF_INT__) + +#else +/* Target does not support lock-free atomic operations */ +typedef unsigned int bitset_t; +#define ATOM_BITSET_SIZE (CHAR_BIT * __SIZEOF_INT__) #endif +#if ATOM_BITSET_SIZE <= 32 + +static inline bitset_t bitset_mask(uint32_t bit) +{ + return 1UL << bit; +} + +#elif ATOM_BITSET_SIZE <= 64 + +static inline bitset_t bitset_mask(uint32_t bit) +{ + return 1ULL << bit; +} + +#elif ATOM_BITSET_SIZE <= 128 + +static inline bitset_t bitset_mask(uint32_t bit) +{ + if (bit < 64) + return 1ULL << bit; + else + return (unsigned __int128)(1ULL << (bit - 64)) << 64; +} + +#else +#error Unsupported size of bit sets (ATOM_BITSET_SIZE) +#endif + +static inline bitset_t atom_bitset_load(bitset_t *bs, int mo) +{ + return __lockfree_load_16(bs, mo); +} + +static inline void atom_bitset_set(bitset_t *bs, uint32_t bit, int mo) +{ + (void)__lockfree_fetch_or_16(bs, bitset_mask(bit), mo); +} + +static inline void atom_bitset_clr(bitset_t *bs, uint32_t bit, int mo) +{ + (void)__lockfree_fetch_and_16(bs, ~bitset_mask(bit), mo); +} + +static inline bitset_t atom_bitset_xchg(bitset_t *bs, bitset_t neu, int mo) +{ + return __lockfree_exchange_16(bs, neu, mo); +} + +static inline bitset_t atom_bitset_cmpxchg(bitset_t *bs, bitset_t *old, + bitset_t neu, bool weak, + int mo_success, int mo_failure) +{ + return __lockfree_compare_exchange_16(bs, old, neu, weak, mo_success, + mo_failure); +} + #endif /* PLATFORM_LINUXGENERIC_ARCH_ARM_ODP_ATOMIC_H */ diff --git a/platform/linux-generic/arch/aarch64/odp_sysinfo_parse.c b/platform/linux-generic/arch/aarch64/odp_sysinfo_parse.c index 77e115756..ab64d501e 100644 --- a/platform/linux-generic/arch/aarch64/odp_sysinfo_parse.c +++ b/platform/linux-generic/arch/aarch64/odp_sysinfo_parse.c @@ -1,4 +1,5 @@ /* Copyright (c) 2018, Linaro Limited + * Copyright (c) 2020-2021, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -64,6 +65,7 @@ static void aarch64_part_info(char *str, int maxlen, odp_cpu_arch_arm_t *cpu_isa *cpu_isa = ODP_CPU_ARCH_ARM_UNKNOWN; if (implementer == 0x41) { + /* Part numbers are specified in Main ID Register (MIDR_EL1) documentation */ switch (part) { case 0xd02: snprintf(str, maxlen, "Cortex-A34"); @@ -105,6 +107,10 @@ static void aarch64_part_info(char *str, int maxlen, odp_cpu_arch_arm_t *cpu_isa snprintf(str, maxlen, "Cortex-A76"); *cpu_isa = ODP_CPU_ARCH_ARMV8_2; return; + case 0xd0c: + snprintf(str, maxlen, "Neoverse N1"); + *cpu_isa = ODP_CPU_ARCH_ARMV8_2; + return; case 0xd0e: snprintf(str, maxlen, "Cortex-A76AE"); *cpu_isa = ODP_CPU_ARCH_ARMV8_2; @@ -117,6 +123,18 @@ static void aarch64_part_info(char *str, int maxlen, odp_cpu_arch_arm_t *cpu_isa snprintf(str, maxlen, "Cortex-A78"); *cpu_isa = ODP_CPU_ARCH_ARMV8_2; return; + case 0xd42: + snprintf(str, maxlen, "Cortex-A78AE"); + *cpu_isa = ODP_CPU_ARCH_ARMV8_2; + return; + case 0xd4a: + snprintf(str, maxlen, "Neoverse E1"); + *cpu_isa = ODP_CPU_ARCH_ARMV8_2; + return; + case 0xd4b: + snprintf(str, maxlen, "Cortex-A78C"); + *cpu_isa = ODP_CPU_ARCH_ARMV8_2; + return; default: break; } @@ -195,6 +213,19 @@ static odp_cpu_arch_arm_t arm_isa_version(void) return ODP_CPU_ARCH_ARMV8_5; case 6: return ODP_CPU_ARCH_ARMV8_6; + case 7: + return ODP_CPU_ARCH_ARMV8_7; + default: + return ODP_CPU_ARCH_ARM_UNKNOWN; + } + } else if (major == 9) { + switch (minor) { + case 0: + return ODP_CPU_ARCH_ARMV9_0; + case 1: + return ODP_CPU_ARCH_ARMV9_1; + case 2: + return ODP_CPU_ARCH_ARMV9_2; default: return ODP_CPU_ARCH_ARM_UNKNOWN; } diff --git a/platform/linux-generic/arch/arm/odp_atomic.h b/platform/linux-generic/arch/arm/odp_atomic.h index d85a01841..bcc89ff73 100644 --- a/platform/linux-generic/arch/arm/odp_atomic.h +++ b/platform/linux-generic/arch/arm/odp_atomic.h @@ -1,5 +1,4 @@ -/* Copyright (c) 2017, ARM Limited. All rights reserved. - * +/* Copyright (c) 2017-2021, ARM Limited. All rights reserved. * Copyright (c) 2017-2018, Linaro Limited * All rights reserved. * @@ -13,6 +12,8 @@ #error This file should not be included directly, please include odp_cpu.h #endif +#include <limits.h> + #ifdef CONFIG_DMBSTR #define atomic_store_release(loc, val, ro) \ @@ -28,4 +29,80 @@ do { \ #endif /* CONFIG_DMBSTR */ +/** Atomic bit set operations with memory ordering */ +#if __GCC_ATOMIC_LLONG_LOCK_FREE == 2 && \ + __SIZEOF_LONG_LONG__ != __SIZEOF_LONG__ +typedef unsigned long long bitset_t; +#define ATOM_BITSET_SIZE (CHAR_BIT * __SIZEOF_LONG_LONG__) + +#elif __GCC_ATOMIC_LONG_LOCK_FREE == 2 && __SIZEOF_LONG__ != __SIZEOF_INT__ +typedef unsigned long bitset_t; +#define ATOM_BITSET_SIZE (CHAR_BIT * __SIZEOF_LONG__) + +#elif __GCC_ATOMIC_INT_LOCK_FREE == 2 +typedef unsigned int bitset_t; +#define ATOM_BITSET_SIZE (CHAR_BIT * __SIZEOF_INT__) + +#else +/* Target does not support lock-free atomic operations */ +typedef unsigned int bitset_t; +#define ATOM_BITSET_SIZE (CHAR_BIT * __SIZEOF_INT__) +#endif + +#if ATOM_BITSET_SIZE <= 32 + +static inline bitset_t bitset_mask(uint32_t bit) +{ + return 1UL << bit; +} + +#elif ATOM_BITSET_SIZE <= 64 + +static inline bitset_t bitset_mask(uint32_t bit) +{ + return 1ULL << bit; +} + +#elif ATOM_BITSET_SIZE <= 128 + +static inline bitset_t bitset_mask(uint32_t bit) +{ + if (bit < 64) + return 1ULL << bit; + else + return (unsigned __int128)(1ULL << (bit - 64)) << 64; +} + +#else +#error Unsupported size of bit sets (ATOM_BITSET_SIZE) +#endif + +static inline bitset_t atom_bitset_load(bitset_t *bs, int mo) +{ + return __atomic_load_n(bs, mo); +} + +static inline void atom_bitset_set(bitset_t *bs, uint32_t bit, int mo) +{ + (void)__atomic_fetch_or(bs, bitset_mask(bit), mo); +} + +static inline void atom_bitset_clr(bitset_t *bs, uint32_t bit, int mo) +{ + (void)__atomic_fetch_and(bs, ~bitset_mask(bit), mo); +} + +static inline bitset_t atom_bitset_xchg(bitset_t *bs, bitset_t neu, int mo) +{ + return __atomic_exchange_n(bs, neu, mo); +} + +static inline bitset_t atom_bitset_cmpxchg(bitset_t *bs, bitset_t *old, + bitset_t neu, bool weak, + int mo_success, int mo_failure) +{ + return __atomic_compare_exchange_n(bs, old, neu, weak, mo_success, + mo_failure); +} + #endif /* PLATFORM_LINUXGENERIC_ARCH_ARM_ODP_ATOMIC_H */ diff --git a/platform/linux-generic/arch/default/odp/api/abi/atomic_generic.h b/platform/linux-generic/arch/default/odp/api/abi/atomic_generic.h new file mode 100644 index 000000000..e274a4d64 --- /dev/null +++ b/platform/linux-generic/arch/default/odp/api/abi/atomic_generic.h @@ -0,0 +1,159 @@ +/* Copyright (c) 2021, ARM Limited + * Copyright (c) 2021, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_API_ABI_ATOMIC_GENERIC_H_ +#define ODP_API_ABI_ATOMIC_GENERIC_H_ + +#include <odp/api/atomic.h> + +#ifdef __SIZEOF_INT128__ + +static inline void _odp_atomic_init_u128(odp_atomic_u128_t *atom, odp_u128_t val) +{ + atom->v = val; +} + +static inline odp_u128_t _odp_atomic_load_u128(odp_atomic_u128_t *atom) +{ + odp_u128_t val; + + *(__int128_t *)&val = __atomic_load_n((__int128_t *)&atom->v, __ATOMIC_RELAXED); + return val; +} + +static inline void _odp_atomic_store_u128(odp_atomic_u128_t *atom, odp_u128_t val) +{ + __atomic_store_n((__int128_t *)&atom->v, *(__int128_t *)&val, __ATOMIC_RELAXED); +} + +static inline int _odp_atomic_cas_u128(odp_atomic_u128_t *atom, odp_u128_t *old_val, + odp_u128_t new_val) +{ + return __atomic_compare_exchange_n((__int128_t *)&atom->v, (__int128_t *)old_val, + *(__int128_t *)&new_val, 0 /* strong */, + __ATOMIC_RELAXED, __ATOMIC_RELAXED); +} + +static inline int _odp_atomic_cas_acq_u128(odp_atomic_u128_t *atom, odp_u128_t *old_val, + odp_u128_t new_val) +{ + return __atomic_compare_exchange_n((__int128_t *)&atom->v, (__int128_t *)old_val, + *(__int128_t *)&new_val, 0 /* strong */, + __ATOMIC_ACQUIRE, __ATOMIC_RELAXED); +} + +static inline int _odp_atomic_cas_rel_u128(odp_atomic_u128_t *atom, odp_u128_t *old_val, + odp_u128_t new_val) +{ + return __atomic_compare_exchange_n((__int128_t *)&atom->v, (__int128_t *)old_val, + *(__int128_t *)&new_val, 0 /* strong */, + __ATOMIC_RELEASE, __ATOMIC_RELAXED); +} + +static inline int _odp_atomic_cas_acq_rel_u128(odp_atomic_u128_t *atom, odp_u128_t *old_val, + odp_u128_t new_val) +{ + return __atomic_compare_exchange_n((__int128_t *)&atom->v, (__int128_t *)old_val, + *(__int128_t *)&new_val, 0 /* strong */, + __ATOMIC_ACQ_REL, __ATOMIC_RELAXED); +} + +#else /* Lock-based implementation */ + +/** + * @internal + * 128 bit store operation expression for the ATOMIC_OP macro + */ +#define ATOMIC_STORE_OP_128(new_val) \ +({ \ + (_atom)->v = (new_val); \ +}) + +/** + * @internal + * 128 bit CAS operation expression for the ATOMIC_OP macro + */ +#define ATOMIC_CAS_OP_128(ret_ptr, old_val, new_val) \ +({ \ + int *_ret_ptr = ret_ptr; \ + odp_u128_t *_old_val = old_val; \ + odp_u128_t _new_val = new_val; \ + if (((_atom)->v.u64[0] == (_old_val)->u64[0]) && \ + ((_atom)->v.u64[1] == (_old_val)->u64[1])) { \ + (_atom)->v = (_new_val); \ + *(_ret_ptr) = 1; \ + } else { \ + *(_ret_ptr) = 0; \ + } \ +}) + +/** + * @internal + * Helper macro for lock-based atomic operations on 128-bit integers + * @param[in,out] atom Pointer to the 128-bit atomic variable + * @param expr Expression used update the variable. + * @return The old value of the variable. + */ +#define ATOMIC_OP_128(atom, expr) \ +({ \ + odp_u128_t _old_val; \ + odp_atomic_u128_t *_atom = atom; \ + /* Loop while lock is already taken, stop when lock becomes clear */ \ + while (__atomic_test_and_set(&(_atom)->lock, __ATOMIC_ACQUIRE)) \ + (void)0; \ + _old_val = (_atom)->v; \ + (expr); /* Perform whatever update is desired */ \ + __atomic_clear(&(_atom)->lock, __ATOMIC_RELEASE); \ + _old_val; /* Return old value */ \ +}) + +static inline void _odp_atomic_init_u128(odp_atomic_u128_t *atom, odp_u128_t val) +{ + atom->v.u64[0] = val.u64[0]; + atom->v.u64[1] = val.u64[1]; + atom->lock = 0; +} + +static inline odp_u128_t _odp_atomic_load_u128(odp_atomic_u128_t *atom) +{ + return ATOMIC_OP_128(atom, (void)0); +} + +static inline void _odp_atomic_store_u128(odp_atomic_u128_t *atom, odp_u128_t val) +{ + ATOMIC_OP_128(atom, ATOMIC_STORE_OP_128(val)); +} + +static inline int _odp_atomic_cas_u128(odp_atomic_u128_t *atom, odp_u128_t *old_val, + odp_u128_t new_val) +{ + int ret; + + *old_val = ATOMIC_OP_128(atom, ATOMIC_CAS_OP_128(&ret, old_val, new_val)); + return ret; +} + +static inline int _odp_atomic_cas_acq_u128(odp_atomic_u128_t *atom, odp_u128_t *old_val, + odp_u128_t new_val) +{ + return _odp_atomic_cas_u128(atom, old_val, new_val); +} + +static inline int _odp_atomic_cas_rel_u128(odp_atomic_u128_t *atom, odp_u128_t *old_val, + odp_u128_t new_val) +{ + return _odp_atomic_cas_u128(atom, old_val, new_val); +} + +static inline int _odp_atomic_cas_acq_rel_u128(odp_atomic_u128_t *atom, odp_u128_t *old_val, + odp_u128_t new_val) +{ + return _odp_atomic_cas_u128(atom, old_val, new_val); +} +#endif + +#endif diff --git a/platform/linux-generic/arch/default/odp/api/abi/atomic_inlines.h b/platform/linux-generic/arch/default/odp/api/abi/atomic_inlines.h new file mode 100644 index 000000000..f1072d11f --- /dev/null +++ b/platform/linux-generic/arch/default/odp/api/abi/atomic_inlines.h @@ -0,0 +1,7 @@ +/* Copyright (c) 2021, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <odp/api/abi/atomic_generic.h> diff --git a/platform/linux-generic/arch/default/odp_atomic.c b/platform/linux-generic/arch/default/odp_atomic.c new file mode 100644 index 000000000..36fc5e8ea --- /dev/null +++ b/platform/linux-generic/arch/default/odp_atomic.c @@ -0,0 +1,47 @@ +/* Copyright (c) 2015-2018, Linaro Limited + * Copyright (c) 2021, ARM Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <odp/api/atomic.h> + +int odp_atomic_lock_free_u64(odp_atomic_op_t *atomic_op) +{ +#if __GCC_ATOMIC_LLONG_LOCK_FREE < 2 + /* All operations have locks */ + if (atomic_op) + atomic_op->all_bits = 0; + + return 0; +#else + /* All operations are lock-free */ + if (atomic_op) { + atomic_op->all_bits = ~((uint32_t)0); + atomic_op->op.init = 0; + } + + return 2; +#endif +} + +int odp_atomic_lock_free_u128(odp_atomic_op_t *atomic_op) +{ +#ifdef __SIZEOF_INT128__ + if (__atomic_is_lock_free(16, NULL)) { + if (atomic_op) { + atomic_op->all_bits = 0; + atomic_op->op.load = 1; + atomic_op->op.store = 1; + atomic_op->op.cas = 1; + } + return 2; + } +#endif + /* All operations have locks */ + if (atomic_op) + atomic_op->all_bits = 0; + + return 0; +} diff --git a/platform/linux-generic/arch/default/odp_atomic.h b/platform/linux-generic/arch/default/odp_atomic.h new file mode 100644 index 000000000..fc8260194 --- /dev/null +++ b/platform/linux-generic/arch/default/odp_atomic.h @@ -0,0 +1,114 @@ +/* Copyright (c) 2021, ARM Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_DEFAULT_ATOMIC_H_ +#define ODP_DEFAULT_ATOMIC_H_ + +#ifdef __SIZEOF_INT128__ + +typedef unsigned __int128 _u128_t; + +static inline _u128_t lockfree_load_u128(_u128_t *atomic) +{ + return __atomic_load_n(atomic, __ATOMIC_RELAXED); +} + +static inline int lockfree_cas_acq_rel_u128(_u128_t *atomic, + _u128_t old_val, + _u128_t new_val) +{ + return __atomic_compare_exchange_n(atomic, &old_val, new_val, + 0 /* strong */, + __ATOMIC_ACQ_REL, + __ATOMIC_RELAXED); +} + +static inline int lockfree_check_u128(void) +{ + return __atomic_is_lock_free(16, NULL); +} + +#endif + +#include <limits.h> + +/** Atomic bit set operations with memory ordering */ +#if __GCC_ATOMIC_LLONG_LOCK_FREE == 2 && \ + __SIZEOF_LONG_LONG__ != __SIZEOF_LONG__ +typedef unsigned long long bitset_t; +#define ATOM_BITSET_SIZE (CHAR_BIT * __SIZEOF_LONG_LONG__) + +#elif __GCC_ATOMIC_LONG_LOCK_FREE == 2 && __SIZEOF_LONG__ != __SIZEOF_INT__ +typedef unsigned long bitset_t; +#define ATOM_BITSET_SIZE (CHAR_BIT * __SIZEOF_LONG__) + +#elif __GCC_ATOMIC_INT_LOCK_FREE == 2 +typedef unsigned int bitset_t; +#define ATOM_BITSET_SIZE (CHAR_BIT * __SIZEOF_INT__) + +#else +/* Target does not support lock-free atomic operations */ +typedef unsigned int bitset_t; +#define ATOM_BITSET_SIZE (CHAR_BIT * __SIZEOF_INT__) +#endif + +#if ATOM_BITSET_SIZE <= 32 + +static inline bitset_t bitset_mask(uint32_t bit) +{ + return 1UL << bit; +} + +#elif ATOM_BITSET_SIZE <= 64 + +static inline bitset_t bitset_mask(uint32_t bit) +{ + return 1ULL << bit; +} + +#elif ATOM_BITSET_SIZE <= 128 + +static inline bitset_t bitset_mask(uint32_t bit) +{ + if (bit < 64) + return 1ULL << bit; + else + return (unsigned __int128)(1ULL << (bit - 64)) << 64; +} + +#else +#error Unsupported size of bit sets (ATOM_BITSET_SIZE) +#endif + +static inline bitset_t atom_bitset_load(bitset_t *bs, int mo) +{ + return __atomic_load_n(bs, mo); +} + +static inline void atom_bitset_set(bitset_t *bs, uint32_t bit, int mo) +{ + (void)__atomic_fetch_or(bs, bitset_mask(bit), mo); +} + +static inline void atom_bitset_clr(bitset_t *bs, uint32_t bit, int mo) +{ + (void)__atomic_fetch_and(bs, ~bitset_mask(bit), mo); +} + +static inline bitset_t atom_bitset_xchg(bitset_t *bs, bitset_t neu, int mo) +{ + return __atomic_exchange_n(bs, neu, mo); +} + +static inline bitset_t atom_bitset_cmpxchg(bitset_t *bs, bitset_t *old, + bitset_t neu, bool weak, + int mo_success, int mo_failure) +{ + return __atomic_compare_exchange_n(bs, old, neu, weak, mo_success, + mo_failure); +} + +#endif diff --git a/platform/linux-generic/arch/default/odp_cpu.h b/platform/linux-generic/arch/default/odp_cpu.h index d8bc125c8..821956819 100644 --- a/platform/linux-generic/arch/default/odp_cpu.h +++ b/platform/linux-generic/arch/default/odp_cpu.h @@ -20,6 +20,7 @@ #define atomic_store_release(loc, val, ro) \ __atomic_store_n(loc, val, __ATOMIC_RELEASE) +#include "odp_atomic.h" #include "odp_cpu_idling.h" #endif diff --git a/platform/linux-generic/check-globals.sh b/platform/linux-generic/check-globals.sh new file mode 120000 index 000000000..c999a29ef --- /dev/null +++ b/platform/linux-generic/check-globals.sh @@ -0,0 +1 @@ +../../scripts/check-globals.sh
\ No newline at end of file diff --git a/platform/linux-generic/include-abi/odp/api/abi/atomic.h b/platform/linux-generic/include-abi/odp/api/abi/atomic.h index 13c12a79f..7c11b0ab2 100644 --- a/platform/linux-generic/include-abi/odp/api/abi/atomic.h +++ b/platform/linux-generic/include-abi/odp/api/abi/atomic.h @@ -55,7 +55,7 @@ typedef struct ODP_ALIGNED(sizeof(uint64_t)) odp_atomic_u64_s { #endif -#ifdef _ODP_LOCK_FREE_128BIT_ATOMICS +#if defined(__SIZEOF_INT128__) || defined(_ODP_LOCK_FREE_128BIT_ATOMICS) /** * @internal diff --git a/platform/linux-generic/include-abi/odp/api/abi/event.h b/platform/linux-generic/include-abi/odp/api/abi/event.h index 27d750d16..1cbb81afe 100644 --- a/platform/linux-generic/include-abi/odp/api/abi/event.h +++ b/platform/linux-generic/include-abi/odp/api/abi/event.h @@ -34,7 +34,8 @@ typedef enum odp_event_type_t { ODP_EVENT_TIMEOUT = 3, ODP_EVENT_CRYPTO_COMPL = 4, ODP_EVENT_IPSEC_STATUS = 5, - ODP_EVENT_PACKET_VECTOR = 6 + ODP_EVENT_PACKET_VECTOR = 6, + ODP_EVENT_PACKET_TX_COMPL = 7 } odp_event_type_t; typedef enum odp_event_subtype_t { diff --git a/platform/linux-generic/include-abi/odp/api/abi/packet.h b/platform/linux-generic/include-abi/odp/api/abi/packet.h index 76ec97dc7..28e97637c 100644 --- a/platform/linux-generic/include-abi/odp/api/abi/packet.h +++ b/platform/linux-generic/include-abi/odp/api/abi/packet.h @@ -39,6 +39,10 @@ typedef ODP_HANDLE_T(odp_packet_vector_t); #define ODP_PACKET_VECTOR_INVALID _odp_cast_scalar(odp_packet_vector_t, 0) +typedef ODP_HANDLE_T(odp_packet_tx_compl_t); + +#define ODP_PACKET_TX_COMPL_INVALID _odp_cast_scalar(odp_packet_tx_compl_t, 0) + #define ODP_PACKET_OFFSET_INVALID 0xffff typedef uint8_t odp_proto_l2_type_t; diff --git a/platform/linux-generic/include/odp/api/plat/atomic_inlines.h b/platform/linux-generic/include/odp/api/plat/atomic_inlines.h index 5f0cba05e..4ab8bb411 100644 --- a/platform/linux-generic/include/odp/api/plat/atomic_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/atomic_inlines.h @@ -1,4 +1,5 @@ /* Copyright (c) 2016-2018, Linaro Limited + * Copyright (c) 2021, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -13,6 +14,8 @@ #ifndef _ODP_PLAT_ATOMIC_INLINES_H_ #define _ODP_PLAT_ATOMIC_INLINES_H_ +#include <odp/api/abi/atomic_inlines.h> + /** @cond _ODP_HIDE_FROM_DOXYGEN_ */ #ifndef _ODP_NO_INLINE @@ -82,6 +85,14 @@ #define odp_atomic_cas_acq_u32 __odp_atomic_cas_acq_u32 #define odp_atomic_cas_rel_u32 __odp_atomic_cas_rel_u32 #define odp_atomic_cas_acq_rel_u32 __odp_atomic_cas_acq_rel_u32 + #define odp_atomic_init_u128 __odp_atomic_init_u128 + #define odp_atomic_load_u128 __odp_atomic_load_u128 + #define odp_atomic_store_u128 __odp_atomic_store_u128 + #define odp_atomic_cas_u128 __odp_atomic_cas_u128 + #define odp_atomic_cas_acq_u128 __odp_atomic_cas_acq_u128 + #define odp_atomic_cas_rel_u128 __odp_atomic_cas_rel_u128 + #define odp_atomic_cas_acq_rel_u128 __odp_atomic_cas_acq_rel_u128 + #else #define _ODP_INLINE #endif @@ -530,6 +541,45 @@ _ODP_INLINE int odp_atomic_cas_acq_rel_u32(odp_atomic_u32_t *atom, __ATOMIC_RELAXED); } +_ODP_INLINE void odp_atomic_init_u128(odp_atomic_u128_t *atom, odp_u128_t val) +{ + _odp_atomic_init_u128(atom, val); +} + +_ODP_INLINE odp_u128_t odp_atomic_load_u128(odp_atomic_u128_t *atom) +{ + return _odp_atomic_load_u128(atom); +} + +_ODP_INLINE void odp_atomic_store_u128(odp_atomic_u128_t *atom, odp_u128_t val) +{ + _odp_atomic_store_u128(atom, val); +} + +_ODP_INLINE int odp_atomic_cas_u128(odp_atomic_u128_t *atom, + odp_u128_t *old_val, odp_u128_t new_val) +{ + return _odp_atomic_cas_u128(atom, old_val, new_val); +} + +_ODP_INLINE int odp_atomic_cas_acq_u128(odp_atomic_u128_t *atom, + odp_u128_t *old_val, odp_u128_t new_val) +{ + return _odp_atomic_cas_acq_u128(atom, old_val, new_val); +} + +_ODP_INLINE int odp_atomic_cas_rel_u128(odp_atomic_u128_t *atom, + odp_u128_t *old_val, odp_u128_t new_val) +{ + return _odp_atomic_cas_rel_u128(atom, old_val, new_val); +} + +_ODP_INLINE int odp_atomic_cas_acq_rel_u128(odp_atomic_u128_t *atom, + odp_u128_t *old_val, odp_u128_t new_val) +{ + return _odp_atomic_cas_acq_rel_u128(atom, old_val, new_val); +} + /** @endcond */ #endif diff --git a/platform/linux-generic/include/odp_bitset.h b/platform/linux-generic/include/odp_bitset.h index 5c10ef9e5..0931fb337 100644 --- a/platform/linux-generic/include/odp_bitset.h +++ b/platform/linux-generic/include/odp_bitset.h @@ -24,39 +24,8 @@ * (lock-free) max is 128 */ -/* Find a suitable data type that supports lock-free atomic operations */ -#if defined(__aarch64__) && defined(__SIZEOF_INT128__) && \ - __SIZEOF_INT128__ == 16 -#define LOCKFREE16 -typedef __int128 bitset_t; -#define ATOM_BITSET_SIZE (CHAR_BIT * __SIZEOF_INT128__) - -#elif __GCC_ATOMIC_LLONG_LOCK_FREE == 2 && \ - __SIZEOF_LONG_LONG__ != __SIZEOF_LONG__ -typedef unsigned long long bitset_t; -#define ATOM_BITSET_SIZE (CHAR_BIT * __SIZEOF_LONG_LONG__) - -#elif __GCC_ATOMIC_LONG_LOCK_FREE == 2 && __SIZEOF_LONG__ != __SIZEOF_INT__ -typedef unsigned long bitset_t; -#define ATOM_BITSET_SIZE (CHAR_BIT * __SIZEOF_LONG__) - -#elif __GCC_ATOMIC_INT_LOCK_FREE == 2 -typedef unsigned int bitset_t; -#define ATOM_BITSET_SIZE (CHAR_BIT * __SIZEOF_INT__) - -#else -/* Target does not support lock-free atomic operations */ -typedef unsigned int bitset_t; -#define ATOM_BITSET_SIZE (CHAR_BIT * __SIZEOF_INT__) -#endif - #if ATOM_BITSET_SIZE <= 32 -static inline bitset_t bitset_mask(uint32_t bit) -{ - return 1UL << bit; -} - /* Return first-bit-set with StdC ffs() semantics */ static inline uint32_t bitset_ffs(bitset_t b) { @@ -71,11 +40,6 @@ static inline bitset_t bitset_monitor(bitset_t *bs, int mo) #elif ATOM_BITSET_SIZE <= 64 -static inline bitset_t bitset_mask(uint32_t bit) -{ - return 1ULL << bit; -} - /* Return first-bit-set with StdC ffs() semantics */ static inline uint32_t bitset_ffs(bitset_t b) { @@ -90,14 +54,6 @@ static inline bitset_t bitset_monitor(bitset_t *bs, int mo) #elif ATOM_BITSET_SIZE <= 128 -static inline bitset_t bitset_mask(uint32_t bit) -{ - if (bit < 64) - return 1ULL << bit; - else - return (unsigned __int128)(1ULL << (bit - 64)) << 64; -} - /* Return first-bit-set with StdC ffs() semantics */ static inline uint32_t bitset_ffs(bitset_t b) { @@ -119,60 +75,6 @@ static inline bitset_t bitset_monitor(bitset_t *bs, int mo) #error Unsupported size of bit sets (ATOM_BITSET_SIZE) #endif -/* Atomic load with memory ordering */ -static inline bitset_t atom_bitset_load(bitset_t *bs, int mo) -{ -#ifdef LOCKFREE16 - return __lockfree_load_16(bs, mo); -#else - return __atomic_load_n(bs, mo); -#endif -} - -/* Atomic bit set with memory ordering */ -static inline void atom_bitset_set(bitset_t *bs, uint32_t bit, int mo) -{ -#ifdef LOCKFREE16 - (void)__lockfree_fetch_or_16(bs, bitset_mask(bit), mo); -#else - (void)__atomic_fetch_or(bs, bitset_mask(bit), mo); -#endif -} - -/* Atomic bit clear with memory ordering */ -static inline void atom_bitset_clr(bitset_t *bs, uint32_t bit, int mo) -{ -#ifdef LOCKFREE16 - (void)__lockfree_fetch_and_16(bs, ~bitset_mask(bit), mo); -#else - (void)__atomic_fetch_and(bs, ~bitset_mask(bit), mo); -#endif -} - -/* Atomic exchange with memory ordering */ -static inline bitset_t atom_bitset_xchg(bitset_t *bs, bitset_t neu, int mo) -{ -#ifdef LOCKFREE16 - return __lockfree_exchange_16(bs, neu, mo); -#else - return __atomic_exchange_n(bs, neu, mo); -#endif -} - -/* Atomic compare&exchange with memory ordering */ -static inline bitset_t atom_bitset_cmpxchg(bitset_t *bs, bitset_t *old, - bitset_t neu, bool weak, - int mo_success, int mo_failure) -{ -#ifdef LOCKFREE16 - return __lockfree_compare_exchange_16(bs, old, neu, weak, mo_success, - mo_failure); -#else - return __atomic_compare_exchange_n(bs, old, neu, weak, mo_success, - mo_failure); -#endif -} - /* Return a & ~b */ static inline bitset_t bitset_andn(bitset_t a, bitset_t b) { diff --git a/platform/linux-generic/include/odp_config_internal.h b/platform/linux-generic/include/odp_config_internal.h index 35f1c9142..1e8b390dd 100644 --- a/platform/linux-generic/include/odp_config_internal.h +++ b/platform/linux-generic/include/odp_config_internal.h @@ -1,5 +1,5 @@ /* Copyright (c) 2016-2018, Linaro Limited - * Copyright (c) 2019-2020, Nokia + * Copyright (c) 2019-2021, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -147,6 +147,13 @@ extern "C" { /* Enable pool statistics collection */ #define CONFIG_POOL_STATISTICS 1 +/* + * Maximum number of IPsec SAs. The actual maximum number can be further + * limited by the number of sessions supported by the crypto subsystem and + * is reported by odp_ipsec_capability(). + */ +#define CONFIG_IPSEC_MAX_NUM_SA 4000 + #ifdef __cplusplus } #endif diff --git a/platform/linux-generic/include/odp_errno_define.h b/platform/linux-generic/include/odp_errno_define.h index 570d37387..3f97618b0 100644 --- a/platform/linux-generic/include/odp_errno_define.h +++ b/platform/linux-generic/include/odp_errno_define.h @@ -17,7 +17,7 @@ extern "C" { #endif -extern __thread int __odp_errno; +extern __thread int _odp_errno; #ifdef __cplusplus } diff --git a/platform/linux-generic/include/odp_ipsec_internal.h b/platform/linux-generic/include/odp_ipsec_internal.h index 311c0e50c..2509d22ab 100644 --- a/platform/linux-generic/include/odp_ipsec_internal.h +++ b/platform/linux-generic/include/odp_ipsec_internal.h @@ -86,11 +86,6 @@ int _odp_ipsec_status_send(odp_queue_t queue, /* 32 is minimum required by the standard. We do not support more */ #define IPSEC_ANTIREPLAY_WS 32 -/** - * Maximum number of available SAs - */ -#define ODP_CONFIG_IPSEC_SAS 8 - struct ipsec_sa_s { odp_atomic_u32_t state ODP_ALIGNED_CACHE; @@ -240,6 +235,9 @@ uint32_t _odp_ipsec_cipher_iv_len(odp_cipher_alg_t cipher); /* Return digest length required for the cipher for IPsec use */ uint32_t _odp_ipsec_auth_digest_len(odp_auth_alg_t auth); +/* Return the maximum number of SAs supported by the implementation */ +uint32_t _odp_ipsec_max_num_sa(void); + /* * Get SA entry from handle without obtaining a reference */ diff --git a/platform/linux-generic/include/odp_packet_internal.h b/platform/linux-generic/include/odp_packet_internal.h index 8349df43a..4af4bf062 100644 --- a/platform/linux-generic/include/odp_packet_internal.h +++ b/platform/linux-generic/include/odp_packet_internal.h @@ -300,8 +300,8 @@ static inline void packet_set_len(odp_packet_hdr_t *pkt_hdr, uint32_t len) int _odp_packet_copy_md_to_packet(odp_packet_t srcpkt, odp_packet_t dstpkt); /* Packet alloc of pktios */ -int packet_alloc_multi(odp_pool_t pool_hdl, uint32_t len, - odp_packet_t pkt[], int max_num); +int _odp_packet_alloc_multi(odp_pool_t pool_hdl, uint32_t len, + odp_packet_t pkt[], int max_num); /* Perform packet parse up to a given protocol layer */ int _odp_packet_parse_layer(odp_packet_hdr_t *pkt_hdr, diff --git a/platform/linux-generic/include/odp_packet_io_internal.h b/platform/linux-generic/include/odp_packet_io_internal.h index c6b916240..07b0b4b69 100644 --- a/platform/linux-generic/include/odp_packet_io_internal.h +++ b/platform/linux-generic/include/odp_packet_io_internal.h @@ -265,8 +265,8 @@ static inline void _odp_pktio_tx_ts_set(pktio_entry_t *entry) odp_atomic_store_u64(&entry->s.tx_ts, ts_val.u64); } -extern const pktio_if_ops_t netmap_pktio_ops; -extern const pktio_if_ops_t dpdk_pktio_ops; +extern const pktio_if_ops_t _odp_netmap_pktio_ops; +extern const pktio_if_ops_t _odp_dpdk_pktio_ops; extern const pktio_if_ops_t _odp_sock_mmsg_pktio_ops; extern const pktio_if_ops_t _odp_sock_mmap_pktio_ops; extern const pktio_if_ops_t _odp_loopback_pktio_ops; @@ -276,7 +276,7 @@ extern const pktio_if_ops_t _odp_pcap_pktio_ops; extern const pktio_if_ops_t _odp_tap_pktio_ops; extern const pktio_if_ops_t _odp_null_pktio_ops; extern const pktio_if_ops_t _odp_ipc_pktio_ops; -extern const pktio_if_ops_t * const pktio_if_ops[]; +extern const pktio_if_ops_t * const _odp_pktio_if_ops[]; /** * Try interrupt-driven receive diff --git a/platform/linux-generic/include/odp_pool_internal.h b/platform/linux-generic/include/odp_pool_internal.h index 9deb63cd0..a0e4c5c65 100644 --- a/platform/linux-generic/include/odp_pool_internal.h +++ b/platform/linux-generic/include/odp_pool_internal.h @@ -160,8 +160,8 @@ static inline odp_buffer_hdr_t *buf_hdr_from_index_u32(uint32_t u32) return buf_hdr_from_index(pool, buffer_idx); } -int buffer_alloc_multi(pool_t *pool, odp_buffer_hdr_t *buf_hdr[], int num); -void buffer_free_multi(odp_buffer_hdr_t *buf_hdr[], int num_free); +int _odp_buffer_alloc_multi(pool_t *pool, odp_buffer_hdr_t *buf_hdr[], int num); +void _odp_buffer_free_multi(odp_buffer_hdr_t *buf_hdr[], int num_free); int _odp_buffer_is_valid(odp_buffer_t buf); #ifdef __cplusplus diff --git a/platform/linux-generic/include/odp_random_openssl_internal.h b/platform/linux-generic/include/odp_random_openssl_internal.h index 7784d560e..3205a2c32 100644 --- a/platform/linux-generic/include/odp_random_openssl_internal.h +++ b/platform/linux-generic/include/odp_random_openssl_internal.h @@ -16,7 +16,6 @@ extern "C" { #include <odp/api/random.h> odp_random_kind_t _odp_random_openssl_max_kind(void); -int32_t _odp_random_openssl_test_data(uint8_t *buf, uint32_t len, uint64_t *seed); int32_t _odp_random_openssl_data(uint8_t *buf, uint32_t len, odp_random_kind_t kind); int _odp_random_openssl_init_local(void); int _odp_random_openssl_term_local(void); diff --git a/platform/linux-generic/include/odp_timer_internal.h b/platform/linux-generic/include/odp_timer_internal.h index 0c2a0fe4b..23fd54bf9 100644 --- a/platform/linux-generic/include/odp_timer_internal.h +++ b/platform/linux-generic/include/odp_timer_internal.h @@ -40,13 +40,13 @@ typedef struct { /* A larger decrement value should be used after receiving events compared to * an 'empty' call. */ -void _timer_run_inline(int dec); +void _odp_timer_run_inline(int dec); /* Static inline wrapper to minimize modification of schedulers. */ static inline void timer_run(int dec) { if (odp_global_rw->inline_timers) - _timer_run_inline(dec); + _odp_timer_run_inline(dec); } #endif diff --git a/platform/linux-generic/libodp-linux.pc.in b/platform/linux-generic/libodp-linux.pc.in index a15ff2dbc..2b28414e9 100644 --- a/platform/linux-generic/libodp-linux.pc.in +++ b/platform/linux-generic/libodp-linux.pc.in @@ -7,6 +7,6 @@ Name: lib@ODP_LIB_NAME@ Description: The ODP packet processing engine Version: @PKGCONFIG_VERSION@ Requires.private: libconfig -Libs: -L${libdir} -l@ODP_LIB_NAME@ -Libs.private: @OPENSSL_STATIC_LIBS@ @DPDK_LIBS@ @PCAP_LIBS@ @PTHREAD_LIBS@ @TIMER_LIBS@ -lpthread @ATOMIC_LIBS@ +Libs: -L${libdir} -l@ODP_LIB_NAME@ @ATOMIC_LIBS_NON_ABI_COMPAT@ +Libs.private: @OPENSSL_STATIC_LIBS@ @DPDK_LIBS@ @PCAP_LIBS@ @PTHREAD_LIBS@ @TIMER_LIBS@ -lpthread @ATOMIC_LIBS_ABI_COMPAT@ Cflags: -I${includedir} diff --git a/platform/linux-generic/m4/configure.m4 b/platform/linux-generic/m4/configure.m4 index 924346c1f..2ebc23174 100644 --- a/platform/linux-generic/m4/configure.m4 +++ b/platform/linux-generic/m4/configure.m4 @@ -25,7 +25,7 @@ m4_include([platform/linux-generic/m4/odp_netmap.m4]) m4_include([platform/linux-generic/m4/odp_dpdk.m4]) ODP_SCHEDULER -AS_VAR_APPEND([PLAT_DEP_LIBS], ["${LIBCONFIG_LIBS} ${OPENSSL_LIBS} ${DPDK_LIBS_LT} ${LIBCLI_LIBS}"]) +AS_VAR_APPEND([PLAT_DEP_LIBS], ["${ATOMIC_LIBS} ${LIBCONFIG_LIBS} ${OPENSSL_LIBS} ${DPDK_LIBS_LT} ${LIBCLI_LIBS}"]) # Add text to the end of configure with platform specific settings. # Make sure it's aligned same as other lines in configure.ac. diff --git a/platform/linux-generic/odp_atomic.c b/platform/linux-generic/odp_atomic.c deleted file mode 100644 index 59253c645..000000000 --- a/platform/linux-generic/odp_atomic.c +++ /dev/null @@ -1,306 +0,0 @@ -/* Copyright (c) 2015-2018, Linaro Limited - * Copyright (c) 2021, ARM Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -#include <odp/api/atomic.h> -#include <odp_cpu.h> - -int odp_atomic_lock_free_u64(odp_atomic_op_t *atomic_op) -{ -#if __GCC_ATOMIC_LLONG_LOCK_FREE < 2 - /* All operations have locks */ - if (atomic_op) - atomic_op->all_bits = 0; - - return 0; -#else - /* All operations are lock-free */ - if (atomic_op) { - atomic_op->all_bits = ~((uint32_t)0); - atomic_op->op.init = 0; - } - - return 2; -#endif -} - -int odp_atomic_lock_free_u128(odp_atomic_op_t *atomic_op) -{ -#ifdef _ODP_LOCK_FREE_128BIT_ATOMICS - if (atomic_op) { - atomic_op->all_bits = 0; - atomic_op->op.load = 1; - atomic_op->op.store = 1; - atomic_op->op.cas = 1; - } - - return 2; -#else - /* All operations have locks */ - if (atomic_op) - atomic_op->all_bits = 0; - - return 0; -#endif -} - -#ifdef _ODP_LOCK_FREE_128BIT_ATOMICS - -static void __atomic_init_u128(odp_atomic_u128_t *atom, odp_u128_t new_val) -{ - odp_u128_t old, val; - - old = atom->v; - - while (1) { - ATOMIC_CAS_OP_128_NO_ORDER(atom, &old, new_val, val); - - if ((val.u64[0] == old.u64[0]) && (val.u64[1] == old.u64[1])) - return; - - old = val; - } -} - -static odp_u128_t __atomic_load_u128(odp_atomic_u128_t *atom) -{ - odp_u128_t val, exp; - - exp.u64[0] = 0; - exp.u64[1] = 0; - ATOMIC_CAS_OP_128_NO_ORDER(atom, &exp, exp, val); - return val; -} - -static void __atomic_store_u128(odp_atomic_u128_t *atom, odp_u128_t new_val) -{ - odp_u128_t old, val; - - old = atom->v; - - while (1) { - ATOMIC_CAS_OP_128_NO_ORDER(atom, &old, new_val, val); - - if ((val.u64[0] == old.u64[0]) && (val.u64[1] == old.u64[1])) - return; - - old = val; - } -} - -static int __atomic_cas_u128(odp_atomic_u128_t *atom, - odp_u128_t *old_val, odp_u128_t new_val) -{ - int ret = 0; - odp_u128_t val; - - ATOMIC_CAS_OP_128_NO_ORDER(atom, old_val, new_val, val); - - if ((val.u64[0] == old_val->u64[0]) && (val.u64[1] == old_val->u64[1])) - ret = 1; - - old_val->u64[0] = val.u64[0]; - old_val->u64[1] = val.u64[1]; - - return ret; -} - -static int __atomic_cas_acq_u128(odp_atomic_u128_t *atom, - odp_u128_t *old_val, odp_u128_t new_val) -{ - int ret = 0; - odp_u128_t val; - - ATOMIC_CAS_OP_128_ACQ(atom, old_val, new_val, val); - - if ((val.u64[0] == old_val->u64[0]) && (val.u64[1] == old_val->u64[1])) - ret = 1; - - old_val->u64[0] = val.u64[0]; - old_val->u64[1] = val.u64[1]; - - return ret; -} - -static int __atomic_cas_rel_u128(odp_atomic_u128_t *atom, - odp_u128_t *old_val, odp_u128_t new_val) -{ - int ret = 0; - odp_u128_t val; - - ATOMIC_CAS_OP_128_REL(atom, old_val, new_val, val); - - if ((val.u64[0] == old_val->u64[0]) && (val.u64[1] == old_val->u64[1])) - ret = 1; - - old_val->u64[0] = val.u64[0]; - old_val->u64[1] = val.u64[1]; - - return ret; -} - -static int __atomic_cas_acq_rel_u128(odp_atomic_u128_t *atom, - odp_u128_t *old_val, - odp_u128_t new_val) -{ - int ret = 0; - odp_u128_t val; - - ATOMIC_CAS_OP_128_ACQ_REL(atom, old_val, new_val, val); - - if ((val.u64[0] == old_val->u64[0]) && (val.u64[1] == old_val->u64[1])) - ret = 1; - - old_val->u64[0] = val.u64[0]; - old_val->u64[1] = val.u64[1]; - - return ret; -} - -#else /* Locked version */ - -/** - * @internal - * 128 bit store operation expression for the ATOMIC_OP macro - */ -#define ATOMIC_STORE_OP_128(new_val) \ -({ \ - (_atom)->v = (new_val); \ -}) - -/** - * @internal - * 128 bit CAS operation expression for the ATOMIC_OP macro - */ -#define ATOMIC_CAS_OP_128(ret_ptr, old_val, new_val) \ -({ \ - int *_ret_ptr = ret_ptr; \ - odp_u128_t *_old_val = old_val; \ - odp_u128_t _new_val = new_val; \ - if (((_atom)->v.u64[0] == (_old_val)->u64[0]) && \ - ((_atom)->v.u64[1] == (_old_val)->u64[1])) { \ - (_atom)->v = (_new_val); \ - *(_ret_ptr) = 1; \ - } else { \ - *(_ret_ptr) = 0; \ - } \ -}) - -/** - * @internal - * Helper macro for lock-based atomic operations on 128-bit integers - * @param[in,out] atom Pointer to the 128-bit atomic variable - * @param expr Expression used update the variable. - * @return The old value of the variable. - */ -#define ATOMIC_OP_128(atom, expr) \ -({ \ - odp_u128_t _old_val; \ - odp_atomic_u128_t *_atom = atom; \ - /* Loop while lock is already taken, stop when lock becomes clear */ \ - while (__atomic_test_and_set(&(_atom)->lock, __ATOMIC_ACQUIRE)) \ - (void)0; \ - _old_val = (_atom)->v; \ - (expr); /* Perform whatever update is desired */ \ - __atomic_clear(&(_atom)->lock, __ATOMIC_RELEASE); \ - _old_val; /* Return old value */ \ -}) - -static void __atomic_init_u128(odp_atomic_u128_t *atom, odp_u128_t val) -{ - atom->lock = 0; - ATOMIC_OP_128(atom, ATOMIC_STORE_OP_128(val)); -} - -static odp_u128_t __atomic_load_u128(odp_atomic_u128_t *atom) -{ - return ATOMIC_OP_128(atom, (void)0); -} - -static void __atomic_store_u128(odp_atomic_u128_t *atom, odp_u128_t val) -{ - ATOMIC_OP_128(atom, ATOMIC_STORE_OP_128(val)); -} - -static int __atomic_cas_u128(odp_atomic_u128_t *atom, - odp_u128_t *old_val, odp_u128_t new_val) -{ - int ret; - *old_val = ATOMIC_OP_128(atom, ATOMIC_CAS_OP_128(&ret, old_val, - new_val)); - return ret; -} - -static int __atomic_cas_acq_u128(odp_atomic_u128_t *atom, - odp_u128_t *old_val, - odp_u128_t new_val) -{ - int ret; - *old_val = ATOMIC_OP_128(atom, ATOMIC_CAS_OP_128(&ret, old_val, - new_val)); - return ret; -} - -static int __atomic_cas_rel_u128(odp_atomic_u128_t *atom, - odp_u128_t *old_val, - odp_u128_t new_val) -{ - int ret; - *old_val = ATOMIC_OP_128(atom, ATOMIC_CAS_OP_128(&ret, old_val, - new_val)); - return ret; -} - -static int __atomic_cas_acq_rel_u128(odp_atomic_u128_t *atom, - odp_u128_t *old_val, - odp_u128_t new_val) -{ - int ret; - *old_val = ATOMIC_OP_128(atom, ATOMIC_CAS_OP_128(&ret, old_val, - new_val)); - return ret; -} - -#endif - -void odp_atomic_init_u128(odp_atomic_u128_t *atom, odp_u128_t val) -{ - __atomic_init_u128(atom, val); -} - -odp_u128_t odp_atomic_load_u128(odp_atomic_u128_t *atom) -{ - return __atomic_load_u128(atom); -} - -void odp_atomic_store_u128(odp_atomic_u128_t *atom, odp_u128_t val) -{ - __atomic_store_u128(atom, val); -} - -int odp_atomic_cas_u128(odp_atomic_u128_t *atom, - odp_u128_t *old_val, odp_u128_t new_val) -{ - return __atomic_cas_u128(atom, old_val, new_val); -} - -int odp_atomic_cas_acq_u128(odp_atomic_u128_t *atom, - odp_u128_t *old_val, odp_u128_t new_val) -{ - return __atomic_cas_acq_u128(atom, old_val, new_val); -} - -int odp_atomic_cas_rel_u128(odp_atomic_u128_t *atom, - odp_u128_t *old_val, odp_u128_t new_val) -{ - return __atomic_cas_rel_u128(atom, old_val, new_val); -} - -int odp_atomic_cas_acq_rel_u128(odp_atomic_u128_t *atom, - odp_u128_t *old_val, odp_u128_t new_val) -{ - return __atomic_cas_acq_rel_u128(atom, old_val, new_val); -} diff --git a/platform/linux-generic/odp_classification.c b/platform/linux-generic/odp_classification.c index bfa410b44..9b31f99a8 100644 --- a/platform/linux-generic/odp_classification.c +++ b/platform/linux-generic/odp_classification.c @@ -120,12 +120,14 @@ int _odp_classification_init_global(void) for (i = 0; i < CLS_COS_MAX_ENTRY; i++) { /* init locks */ cos_t *cos = get_cos_entry_internal(_odp_cos_from_ndx(i)); + LOCK_INIT(&cos->s.lock); } for (i = 0; i < CLS_PMR_MAX_ENTRY; i++) { /* init locks */ pmr_t *pmr = get_pmr_entry_internal(_odp_pmr_from_ndx(i)); + LOCK_INIT(&pmr->s.lock); } @@ -290,8 +292,7 @@ odp_cos_t odp_cls_cos_create(const char *name, const odp_cls_cos_param_t *param) param->hash_proto); tbl_index = i * CLS_COS_QUEUE_MAX; for (j = 0; j < param->num_queue; j++) { - queue = odp_queue_create(NULL, &cos->s. - queue_param); + queue = odp_queue_create(NULL, &cos->s.queue_param); if (queue == ODP_QUEUE_INVALID) { /* unwind the queues */ _cls_queue_unwind(tbl_index, j); @@ -1781,3 +1782,101 @@ uint64_t odp_pmr_to_u64(odp_pmr_t hdl) { return _odp_pri(hdl); } + +static +void print_cos_ident(struct cos_s *cos) +{ + if (strlen(cos->name)) + ODP_PRINT("%s", cos->name); + + ODP_PRINT("(%" PRIu64 ")\n", + odp_cos_to_u64(_odp_cos_from_ndx(cos->index))); +} + +static +void print_queue_ident(odp_queue_t q) +{ + odp_queue_info_t info; + + if (!odp_queue_info(q, &info) && strlen(info.name)) + ODP_PRINT("%s", info.name); + else + ODP_PRINT("%" PRIx64, odp_queue_to_u64(q)); + + ODP_PRINT("\n"); +} + +static +void print_hex(const void *vp, int len) +{ + const uint8_t *p = vp; + + for (int i = 0; i < len; i++) + ODP_PRINT("%02x", *p++); +} + +static +void cls_print_cos(struct cos_s *cos) +{ + uint32_t num_rule = odp_atomic_load_u32(&cos->num_rule); + bool first = true; + + ODP_PRINT("cos: "); + print_cos_ident(cos); + ODP_PRINT(" queue: "); + print_queue_ident(cos->queue); + + for (uint32_t j = 0; j < num_rule; j++) { + struct pmr_s *pmr = &cos->pmr[j]->s; + + LOCK(&pmr->lock); + for (uint32_t k = 0; k < pmr->num_pmr; k++) { + pmr_term_value_t *v = &pmr->pmr_term_value[k]; + + if (first) + ODP_PRINT(" rules: "); + else + ODP_PRINT(" "); + + first = false; + + ODP_PRINT("%s: ", format_pmr_name(v->term)); + + if (v->term == ODP_PMR_CUSTOM_FRAME || + v->term == ODP_PMR_CUSTOM_L3) + ODP_PRINT("offset:%" PRIu32 " ", v->offset); + + if (v->range_term) { + ODP_PRINT("<range>"); + } else { + print_hex(v->match.value_u8, v->val_sz); + ODP_PRINT(" "); + print_hex(v->match.mask_u8, v->val_sz); + } + + ODP_PRINT(" -> "); + + if (pmr->mark) + ODP_PRINT("mark:%" PRIu16 " ", pmr->mark); + + print_cos_ident(&cos->linked_cos[j]->s); + } + UNLOCK(&pmr->lock); + } +} + +void odp_cls_print_all(void) +{ + ODP_PRINT("\n" + "Classifier info\n" + "---------------\n\n"); + + for (uint32_t i = 0; i < CLS_COS_MAX_ENTRY; i++) { + struct cos_s *cos = &cos_tbl->cos_entry[i].s; + + LOCK(&cos->lock); + if (cos->valid) + cls_print_cos(cos); + UNLOCK(&cos->lock); + } +} diff --git a/platform/linux-generic/odp_crypto_null.c b/platform/linux-generic/odp_crypto_null.c index 6dd98a2b7..cf9abc99f 100644 --- a/platform/linux-generic/odp_crypto_null.c +++ b/platform/linux-generic/odp_crypto_null.c @@ -121,6 +121,8 @@ int odp_crypto_capability(odp_crypto_capability_t *capa) capa->sync_mode = ODP_SUPPORT_PREFERRED; capa->async_mode = ODP_SUPPORT_YES; + capa->queue_type_plain = 1; + capa->queue_type_sched = 1; capa->ciphers.bit.null = 1; diff --git a/platform/linux-generic/odp_crypto_openssl.c b/platform/linux-generic/odp_crypto_openssl.c index 64984e201..07a91cc46 100644 --- a/platform/linux-generic/odp_crypto_openssl.c +++ b/platform/linux-generic/odp_crypto_openssl.c @@ -37,7 +37,7 @@ #define _ODP_HAVE_CHACHA20_POLY1305 0 #endif -#define MAX_SESSIONS 32 +#define MAX_SESSIONS 4000 #define AES_BLOCK_SIZE 16 #define AES_KEY_LENGTH 16 @@ -314,9 +314,9 @@ null_crypto_routine(odp_packet_t pkt ODP_UNUSED, } static void -null_crypto_init_routine(odp_crypto_generic_session_t *session ODP_UNUSED) +null_crypto_init_routine(odp_crypto_generic_session_t *session) { - return; + (void)session; } /* Mimic new OpenSSL 1.1.y API */ @@ -1849,6 +1849,8 @@ int odp_crypto_capability(odp_crypto_capability_t *capa) capa->sync_mode = ODP_SUPPORT_PREFERRED; capa->async_mode = ODP_SUPPORT_YES; + capa->queue_type_plain = 1; + capa->queue_type_sched = 1; capa->ciphers.bit.null = 1; capa->ciphers.bit.trides_cbc = 1; diff --git a/platform/linux-generic/odp_errno.c b/platform/linux-generic/odp_errno.c index 162300991..71fc2da77 100644 --- a/platform/linux-generic/odp_errno.c +++ b/platform/linux-generic/odp_errno.c @@ -9,24 +9,24 @@ #include <stdio.h> #include <odp_debug_internal.h> -__thread int __odp_errno; +__thread int _odp_errno; int odp_errno(void) { - return __odp_errno; + return _odp_errno; } void odp_errno_zero(void) { - __odp_errno = 0; + _odp_errno = 0; } void odp_errno_print(const char *str) { if (str != NULL) - ODP_PRINT("%s %s\n", str, strerror(__odp_errno)); + ODP_PRINT("%s %s\n", str, strerror(_odp_errno)); else - ODP_PRINT("%s\n", strerror(__odp_errno)); + ODP_PRINT("%s\n", strerror(_odp_errno)); } const char *odp_errno_str(int errnum) diff --git a/platform/linux-generic/odp_fdserver.c b/platform/linux-generic/odp_fdserver.c index 11e7602af..9e0d75de3 100644 --- a/platform/linux-generic/odp_fdserver.c +++ b/platform/linux-generic/odp_fdserver.c @@ -513,7 +513,7 @@ static int handle_request(int client_sock) break; case FD_SERVERSTOP_REQ: - FD_ODP_DBG("Stoping FD server\n"); + FD_ODP_DBG("Stopping FD server\n"); return 1; default: diff --git a/platform/linux-generic/odp_init.c b/platform/linux-generic/odp_init.c index 27390c13b..9e734f1a6 100644 --- a/platform/linux-generic/odp_init.c +++ b/platform/linux-generic/odp_init.c @@ -294,10 +294,10 @@ int odp_init_global(odp_instance_t *instance, const odp_init_t *params, const odp_platform_init_t *platform_params ODP_UNUSED) { + enum init_stage stage = NO_INIT; + memset(&odp_global_ro, 0, sizeof(odp_global_data_ro_t)); odp_global_ro.main_pid = getpid(); - - enum init_stage stage = NO_INIT; odp_global_ro.log_fn = odp_override_log; odp_global_ro.abort_fn = odp_override_abort; diff --git a/platform/linux-generic/odp_ipsec.c b/platform/linux-generic/odp_ipsec.c index f8746f812..766fd10b7 100644 --- a/platform/linux-generic/odp_ipsec.c +++ b/platform/linux-generic/odp_ipsec.c @@ -1,4 +1,5 @@ /* Copyright (c) 2017-2018, Linaro Limited + * Copyright (c) 2018-2021, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -121,7 +122,7 @@ int odp_ipsec_capability(odp_ipsec_capability_t *capa) capa->proto_ah = ODP_SUPPORT_YES; - capa->max_num_sa = ODP_CONFIG_IPSEC_SAS; + capa->max_num_sa = _odp_ipsec_max_num_sa(); capa->max_antireplay_ws = IPSEC_ANTIREPLAY_WS; @@ -215,7 +216,7 @@ void odp_ipsec_config_init(odp_ipsec_config_t *config) memset(config, 0, sizeof(odp_ipsec_config_t)); config->inbound_mode = ODP_IPSEC_OP_MODE_SYNC; config->outbound_mode = ODP_IPSEC_OP_MODE_SYNC; - config->max_num_sa = ODP_CONFIG_IPSEC_SAS; + config->max_num_sa = _odp_ipsec_max_num_sa(); config->inbound.default_queue = ODP_QUEUE_INVALID; config->inbound.lookup.min_spi = 0; config->inbound.lookup.max_spi = UINT32_MAX; @@ -224,7 +225,7 @@ void odp_ipsec_config_init(odp_ipsec_config_t *config) int odp_ipsec_config(const odp_ipsec_config_t *config) { - if (ODP_CONFIG_IPSEC_SAS < config->max_num_sa) + if (config->max_num_sa > _odp_ipsec_max_num_sa()) return -1; *ipsec_config = *config; @@ -1552,17 +1553,19 @@ static ipsec_sa_t *ipsec_out_single(odp_packet_t pkt, ipsec_out_checksums(pkt, &state); } } else { - if (state.is_ipv4) + if (state.is_ipv4) { rc = ipsec_out_tunnel_parse_ipv4(&state, ipsec_sa); - else if (state.is_ipv6) + } else if (state.is_ipv6) { rc = ipsec_out_tunnel_parse_ipv6(&state, ipsec_sa); - else if (opt->flag.tfc_dummy) { + } else if (opt->flag.tfc_dummy) { state.out_tunnel.ip_tos = 0; state.out_tunnel.ip_df = 0; state.out_tunnel.ip_flabel = 0; rc = 0; - } else + } else { rc = -1; + } + if (rc < 0) { status->error.alg = 1; goto exit; diff --git a/platform/linux-generic/odp_ipsec_sad.c b/platform/linux-generic/odp_ipsec_sad.c index 7cae32703..da8232b01 100644 --- a/platform/linux-generic/odp_ipsec_sad.c +++ b/platform/linux-generic/odp_ipsec_sad.c @@ -1,4 +1,5 @@ /* Copyright (c) 2017-2018, Linaro Limited + * Copyright (c) 2018-2021, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -9,6 +10,7 @@ #include <odp/api/random.h> #include <odp/api/shared_memory.h> +#include <odp_config_internal.h> #include <odp_init_internal.h> #include <odp_debug_internal.h> #include <odp_ipsec_internal.h> @@ -64,7 +66,7 @@ typedef struct sa_thread_local_s { odp_atomic_u32_t packet_quota; /* * Bytes that can be processed in this thread before looking at - * at the SA-global byte counter and checking hard and soft limits. + * the SA-global byte counter and checking hard and soft limits. */ uint32_t byte_quota; /* @@ -75,19 +77,20 @@ typedef struct sa_thread_local_s { } sa_thread_local_t; typedef struct ODP_ALIGNED_CACHE ipsec_thread_local_s { - sa_thread_local_t sa[ODP_CONFIG_IPSEC_SAS]; + sa_thread_local_t sa[CONFIG_IPSEC_MAX_NUM_SA]; uint16_t first_ipv4_id; /* first ID of current block of IDs */ uint16_t next_ipv4_id; /* next ID to be used */ } ipsec_thread_local_t; typedef struct ipsec_sa_table_t { - ipsec_sa_t ipsec_sa[ODP_CONFIG_IPSEC_SAS]; - ipsec_thread_local_t per_thread[ODP_THREAD_COUNT_MAX]; + ipsec_sa_t ipsec_sa[CONFIG_IPSEC_MAX_NUM_SA]; struct ODP_ALIGNED_CACHE { ring_mpmc_t ipv4_id_ring; uint32_t ipv4_id_data[IPV4_ID_RING_SIZE] ODP_ALIGNED_CACHE; } hot; + uint32_t max_num_sa; odp_shm_t shm; + ipsec_thread_local_t per_thread[]; } ipsec_sa_table_t; static ipsec_sa_table_t *ipsec_sa_tbl; @@ -122,8 +125,9 @@ static void init_sa_thread_local(ipsec_sa_t *sa) { sa_thread_local_t *sa_tl; int n; + int thread_count_max = odp_thread_count_max(); - for (n = 0; n < ODP_THREAD_COUNT_MAX; n++) { + for (n = 0; n < thread_count_max; n++) { sa_tl = &ipsec_sa_tbl->per_thread[n].sa[sa->ipsec_sa_idx]; odp_atomic_init_u32(&sa_tl->packet_quota, 0); sa_tl->byte_quota = 0; @@ -133,14 +137,28 @@ static void init_sa_thread_local(ipsec_sa_t *sa) int _odp_ipsec_sad_init_global(void) { + odp_crypto_capability_t crypto_capa; + uint32_t max_num_sa = CONFIG_IPSEC_MAX_NUM_SA; + uint64_t shm_size; + unsigned int thread_count_max = odp_thread_count_max(); odp_shm_t shm; unsigned i; if (odp_global_ro.disable.ipsec) return 0; + if (odp_crypto_capability(&crypto_capa)) { + ODP_ERR("odp_crypto_capability() failed\n"); + return -1; + } + if (max_num_sa > crypto_capa.max_sessions) + max_num_sa = crypto_capa.max_sessions; + + shm_size = sizeof(ipsec_sa_table_t) + + sizeof(ipsec_thread_local_t) * thread_count_max; + shm = odp_shm_reserve("_odp_ipsec_sa_table", - sizeof(ipsec_sa_table_t), + shm_size, ODP_CACHE_LINE_SIZE, 0); if (shm == ODP_SHM_INVALID) @@ -149,9 +167,10 @@ int _odp_ipsec_sad_init_global(void) ipsec_sa_tbl = odp_shm_addr(shm); memset(ipsec_sa_tbl, 0, sizeof(ipsec_sa_table_t)); ipsec_sa_tbl->shm = shm; + ipsec_sa_tbl->max_num_sa = max_num_sa; ring_mpmc_init(&ipsec_sa_tbl->hot.ipv4_id_ring); - for (i = 0; i < ODP_THREAD_COUNT_MAX; i++) { + for (i = 0; i < thread_count_max; i++) { /* * Make the current ID block fully used, forcing allocation * of a fresh block at first use. @@ -175,7 +194,7 @@ int _odp_ipsec_sad_init_global(void) 1); } - for (i = 0; i < ODP_CONFIG_IPSEC_SAS; i++) { + for (i = 0; i < ipsec_sa_tbl->max_num_sa; i++) { ipsec_sa_t *ipsec_sa = ipsec_sa_entry(i); ipsec_sa->ipsec_sa_hdl = ipsec_sa_index_to_handle(i); @@ -190,7 +209,7 @@ int _odp_ipsec_sad_init_global(void) int _odp_ipsec_sad_term_global(void) { - int i; + uint32_t i; ipsec_sa_t *ipsec_sa; int ret = 0; int rc = 0; @@ -198,7 +217,7 @@ int _odp_ipsec_sad_term_global(void) if (odp_global_ro.disable.ipsec) return 0; - for (i = 0; i < ODP_CONFIG_IPSEC_SAS; i++) { + for (i = 0; i < ipsec_sa_tbl->max_num_sa; i++) { ipsec_sa = ipsec_sa_entry(i); if (odp_atomic_load_u32(&ipsec_sa->state) != @@ -219,12 +238,17 @@ int _odp_ipsec_sad_term_global(void) return rc; } +uint32_t _odp_ipsec_max_num_sa(void) +{ + return ipsec_sa_tbl->max_num_sa; +} + static ipsec_sa_t *ipsec_sa_reserve(void) { - int i; + uint32_t i; ipsec_sa_t *ipsec_sa; - for (i = 0; i < ODP_CONFIG_IPSEC_SAS; i++) { + for (i = 0; i < ipsec_sa_tbl->max_num_sa; i++) { uint32_t state = IPSEC_SA_STATE_FREE; ipsec_sa = ipsec_sa_entry(i); @@ -443,7 +467,7 @@ odp_ipsec_sa_t odp_ipsec_sa_create(const odp_ipsec_sa_param_t *param) odp_atomic_init_u64(&ipsec_sa->hot.in.antireplay, 0); } else { ipsec_sa->lookup_mode = ODP_IPSEC_LOOKUP_DISABLED; - odp_atomic_store_u64(&ipsec_sa->hot.out.seq, 1); + odp_atomic_init_u64(&ipsec_sa->hot.out.seq, 1); ipsec_sa->out.frag_mode = param->outbound.frag_mode; ipsec_sa->out.mtu = param->outbound.mtu; } @@ -453,8 +477,8 @@ odp_ipsec_sa_t odp_ipsec_sa_create(const odp_ipsec_sa_param_t *param) ipsec_sa->copy_flabel = param->opt.copy_flabel; ipsec_sa->udp_encap = param->opt.udp_encap; - odp_atomic_store_u64(&ipsec_sa->hot.bytes, 0); - odp_atomic_store_u64(&ipsec_sa->hot.packets, 0); + odp_atomic_init_u64(&ipsec_sa->hot.bytes, 0); + odp_atomic_init_u64(&ipsec_sa->hot.packets, 0); ipsec_sa->soft_limit_bytes = param->lifetime.soft_limit.bytes; ipsec_sa->soft_limit_packets = param->lifetime.soft_limit.packets; ipsec_sa->hard_limit_bytes = param->lifetime.hard_limit.bytes; @@ -758,10 +782,10 @@ int odp_ipsec_sa_mtu_update(odp_ipsec_sa_t sa, uint32_t mtu) ipsec_sa_t *_odp_ipsec_sa_lookup(const ipsec_sa_lookup_t *lookup) { - int i; + uint32_t i; ipsec_sa_t *best = NULL; - for (i = 0; i < ODP_CONFIG_IPSEC_SAS; i++) { + for (i = 0; i < ipsec_sa_tbl->max_num_sa; i++) { ipsec_sa_t *ipsec_sa = ipsec_sa_entry(i); if (ipsec_sa_lock(ipsec_sa) < 0) @@ -922,7 +946,7 @@ int _odp_ipsec_sa_replay_update(ipsec_sa_t *ipsec_sa, uint32_t seq, uint16_t _odp_ipsec_sa_alloc_ipv4_id(ipsec_sa_t *ipsec_sa) { - (void) ipsec_sa; + (void)ipsec_sa; ipsec_thread_local_t *tl = &ipsec_sa_tbl->per_thread[odp_thread_id()]; uint32_t data; @@ -951,6 +975,7 @@ uint16_t _odp_ipsec_sa_alloc_ipv4_id(ipsec_sa_t *ipsec_sa) uint64_t _odp_ipsec_sa_stats_pkts(ipsec_sa_t *sa) { + int thread_count_max = odp_thread_count_max(); uint64_t tl_pkt_quota = 0; sa_thread_local_t *sa_tl; int n; @@ -966,7 +991,7 @@ uint64_t _odp_ipsec_sa_stats_pkts(ipsec_sa_t *sa) * need to be accounted for. */ - for (n = 0; n < ODP_THREAD_COUNT_MAX; n++) { + for (n = 0; n < thread_count_max; n++) { sa_tl = &ipsec_sa_tbl->per_thread[n].sa[sa->ipsec_sa_idx]; tl_pkt_quota += odp_atomic_load_u32(&sa_tl->packet_quota); } @@ -1012,7 +1037,6 @@ static void ipsec_in_sa_info(ipsec_sa_t *ipsec_sa, odp_ipsec_sa_info_t *sa_info) uint8_t *dst = sa_info->inbound.lookup_param.dst_addr; if (ipsec_sa->lookup_mode == ODP_IPSEC_LOOKUP_DSTADDR_SPI) { - if (ipsec_sa->param.inbound.lookup_param.ip_version == ODP_IPSEC_IPV4) memcpy(dst, &ipsec_sa->in.lookup_dst_ipv4, @@ -1020,8 +1044,8 @@ static void ipsec_in_sa_info(ipsec_sa_t *ipsec_sa, odp_ipsec_sa_info_t *sa_info) else memcpy(dst, &ipsec_sa->in.lookup_dst_ipv6, ODP_IPV6_ADDR_SIZE); - } + sa_info->param.inbound.lookup_param.dst_addr = dst; if (ipsec_sa->antireplay) { diff --git a/platform/linux-generic/odp_ishm.c b/platform/linux-generic/odp_ishm.c index 6b2d03d40..0014f3c34 100644 --- a/platform/linux-generic/odp_ishm.c +++ b/platform/linux-generic/odp_ishm.c @@ -393,7 +393,10 @@ static int hp_get_cached(uint64_t len) { int fd; - if (NULL == hpc || hpc->idx < 0 || len != hpc->len) + if (hpc == NULL) + return -1; + + if (hpc->idx < 0 || len != hpc->len) return -1; fd = hpc->fd[hpc->idx]; @@ -404,12 +407,17 @@ static int hp_get_cached(uint64_t len) static int hp_put_cached(int fd) { - if (NULL == hpc || odp_unlikely(++hpc->idx >= hpc->total)) { - hpc->idx--; + if (hpc == NULL) { + ODP_ERR("Bad hpc state\n"); + return -1; + } + + if (odp_unlikely((hpc->idx + 1) >= hpc->total)) { ODP_ERR("Trying to put more FD than allowed: %d\n", fd); return -1; } + hpc->idx++; hpc->fd[hpc->idx] = fd; return 0; @@ -428,7 +436,7 @@ static void *alloc_fragment(uintptr_t size, int block_index, intptr_t align, ishm_fragment_t *fragmnt; *best_fragmnt = NULL; ishm_fragment_t *rem_fragmnt; - uintptr_t border;/* possible start of new fragment (next alignement) */ + uintptr_t border;/* possible start of new fragment (next alignment) */ intptr_t left; /* room remaining after, if the segment is allocated */ uintptr_t remainder = odp_global_ro.shm_max_memory; @@ -1120,7 +1128,7 @@ int _odp_ishm_reserve(const char *name, uint64_t size, int fd, * can request more: If the user requirement exceeds the page * size then we have to make sure the block will be mapped at * the same address every where, otherwise alignment may be - * be wrong for some process */ + * wrong for some process */ hp_align = align; if (hp_align <= page_hp_size) hp_align = page_hp_size; @@ -1177,7 +1185,7 @@ int _odp_ishm_reserve(const char *name, uint64_t size, int fd, * can request more: If the user requirement exceeds the page * size then we have to make sure the block will be mapped at * the same address every where, otherwise alignment may be - * be wrong for some process */ + * wrong for some process */ if (align <= odp_sys_page_size()) align = odp_sys_page_size(); else @@ -2058,7 +2066,6 @@ int _odp_ishm_status(const char *title) "", len_total / 1024 / 1024, "", lost_total / 1024 / 1024); - /* display the virtual space allocations... : */ ODP_PRINT("\nishm virtual space:\n"); for (fragmnt = ishm_ftbl->used_fragmnts; @@ -2080,7 +2087,7 @@ int _odp_ishm_status(const char *title) /* some other sanity checks: */ if (fragmnt->prev != previous) - ODP_ERR("chaining error\n"); + ODP_ERR("chaining error\n"); if (fragmnt != ishm_ftbl->used_fragmnts) { if ((uintptr_t)fragmnt->start != last_address + 1) diff --git a/platform/linux-generic/odp_packet.c b/platform/linux-generic/odp_packet.c index 752ff8416..99b0b3ae2 100644 --- a/platform/linux-generic/odp_packet.c +++ b/platform/linux-generic/odp_packet.c @@ -405,11 +405,11 @@ static inline odp_packet_hdr_t *alloc_segments(pool_t *pool, int num) odp_packet_hdr_t *pkt_hdr[num]; int ret; - ret = buffer_alloc_multi(pool, (odp_buffer_hdr_t **)pkt_hdr, num); + ret = _odp_buffer_alloc_multi(pool, (odp_buffer_hdr_t **)pkt_hdr, num); if (odp_unlikely(ret != num)) { if (ret > 0) - buffer_free_multi((odp_buffer_hdr_t **)pkt_hdr, ret); + _odp_buffer_free_multi((odp_buffer_hdr_t **)pkt_hdr, ret); return NULL; } @@ -523,13 +523,12 @@ static inline void packet_free_multi(odp_buffer_hdr_t *hdr[], int num) /* Skip references and pack to be freed headers to array head */ if (odp_unlikely(num_ref)) hdr[i - num_ref] = hdr[i]; - } num -= num_ref; if (odp_likely(num)) - buffer_free_multi(hdr, num); + _odp_buffer_free_multi(hdr, num); } static inline void free_all_segments(odp_packet_hdr_t *pkt_hdr, int num) @@ -626,8 +625,8 @@ static inline int packet_alloc(pool_t *pool, uint32_t len, int max_pkt, odp_packet_hdr_t *hdr_next; odp_packet_hdr_t *hdr; - num_buf = buffer_alloc_multi(pool, (odp_buffer_hdr_t **)pkt_hdr, - max_buf); + num_buf = _odp_buffer_alloc_multi(pool, (odp_buffer_hdr_t **)pkt_hdr, + max_buf); /* Failed to allocate all segments */ if (odp_unlikely(num_buf != max_buf)) { @@ -640,7 +639,7 @@ static inline int packet_alloc(pool_t *pool, uint32_t len, int max_pkt, odp_buffer_hdr_t **p; p = (odp_buffer_hdr_t **)&pkt_hdr[num_buf - num_free]; - buffer_free_multi(p, num_free); + _odp_buffer_free_multi(p, num_free); } if (num == 0) @@ -673,8 +672,8 @@ static inline int packet_alloc(pool_t *pool, uint32_t len, int max_pkt, return num; } -int packet_alloc_multi(odp_pool_t pool_hdl, uint32_t len, - odp_packet_t pkt[], int max_num) +int _odp_packet_alloc_multi(odp_pool_t pool_hdl, uint32_t len, + odp_packet_t pkt[], int max_num) { pool_t *pool = pool_entry_from_hdl(pool_hdl); int num, num_seg; @@ -692,7 +691,7 @@ odp_packet_t odp_packet_alloc(odp_pool_t pool_hdl, uint32_t len) int num, num_seg; if (odp_unlikely(pool->params.type != ODP_POOL_PACKET)) { - __odp_errno = EINVAL; + _odp_errno = EINVAL; return ODP_PACKET_INVALID; } @@ -715,7 +714,7 @@ int odp_packet_alloc_multi(odp_pool_t pool_hdl, uint32_t len, int num, num_seg; if (odp_unlikely(pool->params.type != ODP_POOL_PACKET)) { - __odp_errno = EINVAL; + _odp_errno = EINVAL; return -1; } @@ -2880,3 +2879,56 @@ int odp_packet_payload_offset_set(odp_packet_t pkt, uint32_t offset) return 0; } + +void odp_packet_aging_tmo_set(odp_packet_t pkt, uint64_t tmo_ns) +{ + (void)pkt; + (void)tmo_ns; +} + +uint64_t odp_packet_aging_tmo(odp_packet_t pkt) +{ + (void)pkt; + return 0; +} + +int odp_packet_tx_compl_request(odp_packet_t pkt, const odp_packet_tx_compl_opt_t *opt) +{ + (void)pkt; + (void)opt; + + return -1; +} + +int odp_packet_has_tx_compl_request(odp_packet_t pkt) +{ + (void)pkt; + + return 0; +} + +odp_packet_tx_compl_t odp_packet_tx_compl_from_event(odp_event_t ev) +{ + (void)ev; + + return ODP_PACKET_TX_COMPL_INVALID; +} + +odp_event_t odp_packet_tx_compl_to_event(odp_packet_tx_compl_t tx_compl) +{ + (void)tx_compl; + + return ODP_EVENT_INVALID; +} + +void odp_packet_tx_compl_free(odp_packet_tx_compl_t tx_compl) +{ + (void)tx_compl; +} + +void *odp_packet_tx_compl_user_ptr(odp_packet_tx_compl_t tx_compl) +{ + (void)tx_compl; + + return NULL; +} diff --git a/platform/linux-generic/odp_packet_io.c b/platform/linux-generic/odp_packet_io.c index 442aa0d94..369c319c2 100644 --- a/platform/linux-generic/odp_packet_io.c +++ b/platform/linux-generic/odp_packet_io.c @@ -126,9 +126,9 @@ int _odp_pktio_init_global(void) _odp_pktio_entry_ptr[i] = pktio_entry; } - for (pktio_if = 0; pktio_if_ops[pktio_if]; ++pktio_if) { - if (pktio_if_ops[pktio_if]->init_global) - if (pktio_if_ops[pktio_if]->init_global()) { + for (pktio_if = 0; _odp_pktio_if_ops[pktio_if]; ++pktio_if) { + if (_odp_pktio_if_ops[pktio_if]->init_global) + if (_odp_pktio_if_ops[pktio_if]->init_global()) { ODP_ERR("failed to initialized pktio type %d", pktio_if); return -1; @@ -149,9 +149,9 @@ int _odp_pktio_init_local(void) { int pktio_if; - for (pktio_if = 0; pktio_if_ops[pktio_if]; ++pktio_if) { - if (pktio_if_ops[pktio_if]->init_local) - if (pktio_if_ops[pktio_if]->init_local()) { + for (pktio_if = 0; _odp_pktio_if_ops[pktio_if]; ++pktio_if) { + if (_odp_pktio_if_ops[pktio_if]->init_local) + if (_odp_pktio_if_ops[pktio_if]->init_local()) { ODP_ERR("failed to initialized pktio type %d", pktio_if); return -1; @@ -263,8 +263,8 @@ static const char *strip_pktio_type(const char *name, char *type_out) if_name++; /* Match if_type to enabled pktio devices */ - for (pktio_if = 0; pktio_if_ops[pktio_if]; pktio_if++) { - if (!strcmp(pktio_type, pktio_if_ops[pktio_if]->name)) { + for (pktio_if = 0; _odp_pktio_if_ops[pktio_if]; pktio_if++) { + if (!strcmp(pktio_type, _odp_pktio_if_ops[pktio_if]->name)) { if (type_out) strcpy(type_out, pktio_type); /* Some pktio devices expect device names to @@ -328,14 +328,14 @@ static odp_pktio_t setup_pktio_entry(const char *name, odp_pool_t pool, odp_pktio_config_init(&pktio_entry->s.config); - for (pktio_if = 0; pktio_if_ops[pktio_if]; ++pktio_if) { + for (pktio_if = 0; _odp_pktio_if_ops[pktio_if]; ++pktio_if) { /* Only use explicitly defined pktio type */ if (strlen(pktio_type) && - strcmp(pktio_if_ops[pktio_if]->name, pktio_type)) + strcmp(_odp_pktio_if_ops[pktio_if]->name, pktio_type)) continue; - ret = pktio_if_ops[pktio_if]->open(hdl, pktio_entry, if_name, - pool); + ret = _odp_pktio_if_ops[pktio_if]->open(hdl, pktio_entry, if_name, + pool); if (!ret) break; } @@ -352,7 +352,7 @@ static odp_pktio_t setup_pktio_entry(const char *name, odp_pool_t pool, snprintf(pktio_entry->s.full_name, sizeof(pktio_entry->s.full_name), "%s", name); pktio_entry->s.state = PKTIO_STATE_OPENED; - pktio_entry->s.ops = pktio_if_ops[pktio_if]; + pktio_entry->s.ops = _odp_pktio_if_ops[pktio_if]; unlock_entry(pktio_entry); return hdl; @@ -400,7 +400,7 @@ odp_pktio_t odp_pktio_open(const char *name, odp_pool_t pool, hdl = odp_pktio_lookup(name); if (hdl != ODP_PKTIO_INVALID) { /* interface is already open */ - __odp_errno = EEXIST; + _odp_errno = EEXIST; return ODP_PKTIO_INVALID; } @@ -1062,7 +1062,7 @@ static odp_buffer_hdr_t *pktin_dequeue(odp_queue_t queue) ODP_DBG("Interface %s dropped %i packets\n", entry->s.name, num - num_enq); - buffer_free_multi(&hdr_tbl[num_enq + 1], num - num_enq); + _odp_buffer_free_multi(&hdr_tbl[num_enq + 1], num - num_enq); } } @@ -1087,7 +1087,7 @@ static int pktin_deq_multi(odp_queue_t queue, odp_buffer_hdr_t *buf_hdr[], if (odp_unlikely(nbr > num)) ODP_ABORT("queue_deq_multi req: %d, returned %d\n", num, nbr); - /** queue already has number of requsted buffers, + /** queue already has number of requested buffers, * do not do receive in that case. */ if (nbr == num) @@ -1116,7 +1116,7 @@ static int pktin_deq_multi(odp_queue_t queue, odp_buffer_hdr_t *buf_hdr[], ODP_DBG("Interface %s dropped %i packets\n", entry->s.name, j - num_enq); - buffer_free_multi(&buf_hdr[num_enq], j - num_enq); + _odp_buffer_free_multi(&buf_hdr[num_enq], j - num_enq); } } @@ -1742,9 +1742,9 @@ int _odp_pktio_term_global(void) unlock_entry(pktio_entry); } - for (pktio_if = 0; pktio_if_ops[pktio_if]; ++pktio_if) { - if (pktio_if_ops[pktio_if]->term) - if (pktio_if_ops[pktio_if]->term()) + for (pktio_if = 0; _odp_pktio_if_ops[pktio_if]; ++pktio_if) { + if (_odp_pktio_if_ops[pktio_if]->term) + if (_odp_pktio_if_ops[pktio_if]->term()) ODP_ABORT("failed to terminate pktio type %d", pktio_if); } diff --git a/platform/linux-generic/odp_pool.c b/platform/linux-generic/odp_pool.c index 0b1d0a7f4..07da3d9cc 100644 --- a/platform/linux-generic/odp_pool.c +++ b/platform/linux-generic/odp_pool.c @@ -1056,7 +1056,7 @@ int odp_pool_info(odp_pool_t pool_hdl, odp_pool_info_t *info) return 0; } -int buffer_alloc_multi(pool_t *pool, odp_buffer_hdr_t *buf_hdr[], int max_num) +int _odp_buffer_alloc_multi(pool_t *pool, odp_buffer_hdr_t *buf_hdr[], int max_num) { uint32_t pool_idx = pool->pool_idx; pool_cache_t *cache = local.cache[pool_idx]; @@ -1171,7 +1171,7 @@ static inline void buffer_free_to_pool(pool_t *pool, odp_atomic_inc_u64(&pool->stats.cache_free_ops); } -void buffer_free_multi(odp_buffer_hdr_t *buf_hdr[], int num_total) +void _odp_buffer_free_multi(odp_buffer_hdr_t *buf_hdr[], int num_total) { pool_t *pool; int num; @@ -1210,7 +1210,7 @@ odp_buffer_t odp_buffer_alloc(odp_pool_t pool_hdl) ODP_ASSERT(ODP_POOL_INVALID != pool_hdl); pool = pool_entry_from_hdl(pool_hdl); - ret = buffer_alloc_multi(pool, (odp_buffer_hdr_t **)&buf, 1); + ret = _odp_buffer_alloc_multi(pool, (odp_buffer_hdr_t **)&buf, 1); if (odp_likely(ret == 1)) return buf; @@ -1226,17 +1226,17 @@ int odp_buffer_alloc_multi(odp_pool_t pool_hdl, odp_buffer_t buf[], int num) pool = pool_entry_from_hdl(pool_hdl); - return buffer_alloc_multi(pool, (odp_buffer_hdr_t **)buf, num); + return _odp_buffer_alloc_multi(pool, (odp_buffer_hdr_t **)buf, num); } void odp_buffer_free(odp_buffer_t buf) { - buffer_free_multi((odp_buffer_hdr_t **)&buf, 1); + _odp_buffer_free_multi((odp_buffer_hdr_t **)&buf, 1); } void odp_buffer_free_multi(const odp_buffer_t buf[], int num) { - buffer_free_multi((odp_buffer_hdr_t **)(uintptr_t)buf, num); + _odp_buffer_free_multi((odp_buffer_hdr_t **)(uintptr_t)buf, num); } int odp_pool_capability(odp_pool_capability_t *capa) diff --git a/platform/linux-generic/odp_queue_basic.c b/platform/linux-generic/odp_queue_basic.c index 2d2ed3c7f..8dc10467a 100644 --- a/platform/linux-generic/odp_queue_basic.c +++ b/platform/linux-generic/odp_queue_basic.c @@ -146,6 +146,7 @@ static int queue_init_global(void) for (i = 0; i < CONFIG_MAX_QUEUES; i++) { /* init locks */ queue_entry_t *queue = qentry_from_index(i); + LOCK_INIT(queue); queue->s.index = i; queue->s.handle = (odp_queue_t)queue; @@ -389,6 +390,7 @@ static int queue_destroy(odp_queue_t handle) { int empty; queue_entry_t *queue; + queue = qentry_from_handle(handle); if (handle == ODP_QUEUE_INVALID) diff --git a/platform/linux-generic/odp_queue_lf.c b/platform/linux-generic/odp_queue_lf.c index 82b95c34d..70d555ab5 100644 --- a/platform/linux-generic/odp_queue_lf.c +++ b/platform/linux-generic/odp_queue_lf.c @@ -23,62 +23,13 @@ typedef unsigned __int128 u128_t; -static inline void atomic_zero_u128(u128_t *atomic) +static inline void lockfree_zero_u128(u128_t *atomic) { __atomic_store_n(atomic, 0, __ATOMIC_RELAXED); } -#if defined(__aarch64__) -/* ARMv8 has atomic load-acq/store-rel instructions for a pair of - * 64bit of data. GCC atomic built-in for 128bits does not utilize these - * instructions but uses locks instead. Override GCC built-in for ARMv8. - */ #include <odp_cpu.h> -static inline int atomic_cas_acq_rel_u128(u128_t *atomic, u128_t old_val, - u128_t new_val) -{ - return __lockfree_compare_exchange_16((__int128 *)atomic, - (__int128 *)&old_val, - new_val, - 0, - __ATOMIC_ACQ_REL, - __ATOMIC_RELAXED); -} - -static inline u128_t atomic_load_u128(u128_t *atomic) -{ - return __lockfree_load_16((__int128 *)atomic, __ATOMIC_RELAXED); -} - -static inline int atomic_is_lockfree_u128(void) -{ - return 1; -} - -#else - -static inline u128_t atomic_load_u128(u128_t *atomic) -{ - return __atomic_load_n(atomic, __ATOMIC_RELAXED); -} - -static inline int atomic_cas_acq_rel_u128(u128_t *atomic, u128_t old_val, - u128_t new_val) -{ - return __atomic_compare_exchange_n(atomic, &old_val, new_val, - 0 /* strong */, - __ATOMIC_ACQ_REL, - __ATOMIC_RELAXED); -} - -static inline int atomic_is_lockfree_u128(void) -{ - return __atomic_is_lock_free(16, NULL); -} - -#endif - #else /* These definitions enable build in non 128 bit compatible systems. @@ -88,19 +39,19 @@ typedef struct ODP_ALIGNED(16) { uint64_t u64[2]; } u128_t; -static inline u128_t atomic_load_u128(u128_t *atomic) +static inline u128_t lockfree_load_u128(u128_t *atomic) { return *atomic; } -static inline void atomic_zero_u128(u128_t *atomic) +static inline void lockfree_zero_u128(u128_t *atomic) { atomic->u64[0] = 0; atomic->u64[1] = 0; } -static inline int atomic_cas_acq_rel_u128(u128_t *atomic, u128_t old_val, - u128_t new_val) +static inline int lockfree_cas_acq_rel_u128(u128_t *atomic, u128_t old_val, + u128_t new_val) { if (atomic->u64[0] == old_val.u64[0] && atomic->u64[1] == old_val.u64[1]) { @@ -112,7 +63,7 @@ static inline int atomic_cas_acq_rel_u128(u128_t *atomic, u128_t old_val, return 0; } -static inline int atomic_is_lockfree_u128(void) +static inline int lockfree_check_u128(void) { return 0; } @@ -187,7 +138,7 @@ static int queue_lf_enq(odp_queue_t handle, odp_buffer_hdr_t *buf_hdr) node = &queue_lf->node[idx]; idx = next_idx(idx); - node_val.u128 = atomic_load_u128(&node->u128); + node_val.u128 = lockfree_load_u128(&node->u128); if (node_val.s.counter == 0) { found = 1; @@ -200,7 +151,7 @@ static int queue_lf_enq(odp_queue_t handle, odp_buffer_hdr_t *buf_hdr) return -1; /* Try to insert data */ - if (atomic_cas_acq_rel_u128(&node->u128, node_val.u128, + if (lockfree_cas_acq_rel_u128(&node->u128, node_val.u128, new_val.u128)) return 0; } @@ -244,7 +195,7 @@ static odp_buffer_hdr_t *queue_lf_deq(odp_queue_t handle) * the lowest counter. */ for (i = 0; i < RING_LF_SIZE; i++) { node = &queue_lf->node[i]; - node_val.u128 = atomic_load_u128(&node->u128); + node_val.u128 = lockfree_load_u128(&node->u128); counter = node_val.s.counter; if (counter && counter < lowest) { @@ -265,7 +216,7 @@ static odp_buffer_hdr_t *queue_lf_deq(odp_queue_t handle) * values. */ for (i = 0; i < i_lowest; i++) { node = &queue_lf->node[i]; - node_val.u128 = atomic_load_u128(&node->u128); + node_val.u128 = lockfree_load_u128(&node->u128); counter = node_val.s.counter; if (counter && counter < lowest) { @@ -278,7 +229,7 @@ static odp_buffer_hdr_t *queue_lf_deq(odp_queue_t handle) buf_hdr = (void *)(uintptr_t)old_val.s.ptr; /* Try to remove data */ - if (atomic_cas_acq_rel_u128(&old->u128, old_val.u128, + if (lockfree_cas_acq_rel_u128(&old->u128, old_val.u128, new_val.u128)) return buf_hdr; } @@ -309,7 +260,7 @@ uint32_t _odp_queue_lf_init_global(uint32_t *queue_lf_size, int lockfree; /* 16 byte lockfree CAS operation is needed. */ - lockfree = atomic_is_lockfree_u128(); + lockfree = lockfree_check_u128(); ODP_DBG("\nLock-free queue init\n"); ODP_DBG(" u128 lock-free: %i\n\n", lockfree); @@ -359,7 +310,7 @@ static void init_queue(queue_lf_t *queue_lf) odp_atomic_init_u64(&queue_lf->enq_counter, 1); for (i = 0; i < RING_LF_SIZE; i++) - atomic_zero_u128(&queue_lf->node[i].u128); + lockfree_zero_u128(&queue_lf->node[i].u128); } void *_odp_queue_lf_create(queue_entry_t *queue) @@ -403,7 +354,7 @@ uint32_t _odp_queue_lf_length(void *queue_lf_ptr) uint32_t num = 0; for (i = 0; i < RING_LF_SIZE; i++) { - node_val.u128 = atomic_load_u128(&queue_lf->node[i].u128); + node_val.u128 = lockfree_load_u128(&queue_lf->node[i].u128); if (node_val.s.counter) num++; } diff --git a/platform/linux-generic/odp_queue_scalable.c b/platform/linux-generic/odp_queue_scalable.c index d70e174e0..248855be4 100644 --- a/platform/linux-generic/odp_queue_scalable.c +++ b/platform/linux-generic/odp_queue_scalable.c @@ -181,7 +181,6 @@ static int queue_init(queue_entry_t *queue, const char *name, sched_elem->schedq = _odp_sched_queue_add(param->sched.group, prio); ODP_ASSERT(sched_elem->schedq != NULL); - } return 0; @@ -218,9 +217,7 @@ static int queue_init_global(void) /* Add the reorder window size */ pool_size += sizeof(reorder_window_t) * CONFIG_MAX_QUEUES; - /* Choose min_alloc and max_alloc such that buddy allocator is - * is selected. - */ + /* Choose min_alloc and max_alloc such that buddy allocator is selected. */ min_alloc = 0; max_alloc = CONFIG_SCAL_QUEUE_SIZE * sizeof(odp_buffer_hdr_t *); queue_shm_pool = _odp_ishm_pool_create("queue_shm_pool", @@ -674,8 +671,7 @@ static int _queue_enq_multi(odp_queue_t handle, odp_buffer_hdr_t *buf_hdr[], static int _queue_enq(odp_queue_t handle, odp_buffer_hdr_t *buf_hdr) { - return odp_likely( - _queue_enq_multi(handle, &buf_hdr, 1) == 1) ? 0 : -1; + return odp_likely(_queue_enq_multi(handle, &buf_hdr, 1) == 1) ? 0 : -1; } static int queue_enq_multi(odp_queue_t handle, const odp_event_t ev[], int num) @@ -734,8 +730,7 @@ int _odp_queue_deq_sc(sched_elem_t *q, odp_event_t *evp, int num) mask = q->cons_mask; ring = q->cons_ring; do { - *evp++ = odp_buffer_to_event( - buf_from_buf_hdr(ring[old_read & mask])); + *evp++ = odp_buffer_to_event(buf_from_buf_hdr(ring[old_read & mask])); } while (++old_read != new_read); /* Signal producers that empty slots are available @@ -828,8 +823,7 @@ inline int _odp_queue_deq_mc(sched_elem_t *q, odp_event_t *evp, int num) ret = _odp_queue_deq(q, hdr_tbl, num); if (odp_likely(ret != 0)) { for (evt_idx = 0; evt_idx < num; evt_idx++) - evp[evt_idx] = odp_buffer_to_event( - buf_from_buf_hdr(hdr_tbl[evt_idx])); + evp[evt_idx] = odp_buffer_to_event(buf_from_buf_hdr(hdr_tbl[evt_idx])); } return ret; @@ -1024,11 +1018,11 @@ static void queue_print_all(void) odp_queue_op_mode_t enq_mode; odp_queue_op_mode_t deq_mode; odp_queue_order_t order; + odp_schedule_sync_t sync; + int prio; const char *bl_str; char type_c, enq_c, deq_c, order_c, sync_c; const int col_width = 24; - int prio = 0; - odp_schedule_sync_t sync = ODP_SCHED_SYNC_PARALLEL; ODP_PRINT("\nList of all queues\n"); ODP_PRINT("------------------\n"); @@ -1050,6 +1044,8 @@ static void queue_print_all(void) enq_mode = queue->s.param.enq_mode; deq_mode = queue->s.param.deq_mode; order = queue->s.param.order; + prio = queue->s.param.sched.prio; + sync = queue->s.param.sched.sync; UNLOCK(&queue->s.lock); diff --git a/platform/linux-generic/odp_random.c b/platform/linux-generic/odp_random.c index aae12a5d3..acae9663d 100644 --- a/platform/linux-generic/odp_random.c +++ b/platform/linux-generic/odp_random.c @@ -29,8 +29,6 @@ int32_t odp_random_data(uint8_t *buf, uint32_t len, odp_random_kind_t kind) int32_t odp_random_test_data(uint8_t *buf, uint32_t len, uint64_t *seed) { - if (_ODP_OPENSSL) - return _odp_random_openssl_test_data(buf, len, seed); return _odp_random_std_test_data(buf, len, seed); } diff --git a/platform/linux-generic/odp_random_openssl.c b/platform/linux-generic/odp_random_openssl.c index e1d45c4fe..a74f99bd4 100644 --- a/platform/linux-generic/odp_random_openssl.c +++ b/platform/linux-generic/odp_random_openssl.c @@ -36,27 +36,6 @@ int32_t _odp_random_openssl_data(uint8_t *buf, uint32_t len, return -1; } } - -int32_t _odp_random_openssl_test_data(uint8_t *buf, uint32_t len, - uint64_t *seed) -{ - union { - uint32_t rand_word; - uint8_t rand_byte[4]; - } u; - uint32_t i = 0, j; - uint32_t seed32 = (*seed) & 0xffffffff; - - while (i < len) { - u.rand_word = rand_r(&seed32); - - for (j = 0; j < 4 && i < len; j++, i++) - *buf++ = u.rand_byte[j]; - } - - *seed = seed32; - return len; -} #else /* Dummy functions for building without OpenSSL support */ odp_random_kind_t _odp_random_openssl_max_kind(void) @@ -70,13 +49,6 @@ int32_t _odp_random_openssl_data(uint8_t *buf ODP_UNUSED, { return -1; } - -int32_t _odp_random_openssl_test_data(uint8_t *buf ODP_UNUSED, - uint32_t len ODP_UNUSED, - uint64_t *seed ODP_UNUSED) -{ - return -1; -} #endif /* _ODP_OPENSSL */ int _odp_random_openssl_init_local(void) diff --git a/platform/linux-generic/odp_rwlock.c b/platform/linux-generic/odp_rwlock.c index 74f5307aa..03af7d26c 100644 --- a/platform/linux-generic/odp_rwlock.c +++ b/platform/linux-generic/odp_rwlock.c @@ -58,6 +58,7 @@ void odp_rwlock_write_lock(odp_rwlock_t *rwlock) while (is_locked == 0) { uint32_t zero = 0; + cnt = odp_atomic_load_u32(&rwlock->cnt); /* lock acquired, wait */ if (cnt != 0) { diff --git a/platform/linux-generic/odp_schedule_basic.c b/platform/linux-generic/odp_schedule_basic.c index 32a21442d..21f707be7 100644 --- a/platform/linux-generic/odp_schedule_basic.c +++ b/platform/linux-generic/odp_schedule_basic.c @@ -229,6 +229,7 @@ typedef struct { /* Scheduler interface config options (not used in fast path) */ schedule_config_t config_if; + uint32_t max_queues; } sched_global_t; @@ -399,6 +400,7 @@ static int schedule_init_global(void) odp_shm_t shm; int i, j, grp; int prefer_ratio; + uint32_t ring_size; ODP_DBG("Schedule init ... "); @@ -419,11 +421,23 @@ static int schedule_init_global(void) return -1; } + sched->shm = shm; prefer_ratio = sched->config.prefer_ratio; /* When num_spread == 1, only spread_tbl[0] is used. */ sched->max_spread = (sched->config.num_spread - 1) * prefer_ratio; - sched->shm = shm; + + ring_size = MAX_RING_SIZE / sched->config.num_spread; + ring_size = ROUNDUP_POWER2_U32(ring_size); + ODP_ASSERT(ring_size <= MAX_RING_SIZE); + sched->ring_mask = ring_size - 1; + + /* Each ring can hold in maximum ring_size-1 queues. Due to ring size round up, + * total capacity of rings may be larger than CONFIG_MAX_SCHED_QUEUES. */ + sched->max_queues = sched->ring_mask * sched->config.num_spread; + if (sched->max_queues > CONFIG_MAX_SCHED_QUEUES) + sched->max_queues = CONFIG_MAX_SCHED_QUEUES; + odp_spinlock_init(&sched->mask_lock); for (grp = 0; grp < NUM_SCHED_GRPS; grp++) { @@ -573,7 +587,6 @@ static inline int prio_level_from_api(int api_prio) static int schedule_create_queue(uint32_t queue_index, const odp_schedule_param_t *sched_param) { - uint32_t ring_size; int i; int prio = prio_level_from_api(sched_param->prio); uint8_t spread = spread_index(queue_index); @@ -620,11 +633,6 @@ static int schedule_create_queue(uint32_t queue_index, sched->queue[queue_index].pktio_index = 0; sched->queue[queue_index].pktin_index = 0; - ring_size = MAX_RING_SIZE / sched->config.num_spread; - ring_size = ROUNDUP_POWER2_U32(ring_size); - ODP_ASSERT(ring_size <= MAX_RING_SIZE); - sched->ring_mask = ring_size - 1; - odp_atomic_init_u64(&sched->order[queue_index].ctx, 0); odp_atomic_init_u64(&sched->order[queue_index].next_ctx, 0); @@ -762,7 +770,7 @@ static inline void ordered_stash_release(void) num_enq = 0; ODP_DBG("Dropped %i packets\n", num - num_enq); - buffer_free_multi(&buf_hdr[num_enq], num - num_enq); + _odp_buffer_free_multi(&buf_hdr[num_enq], num - num_enq); } } sched_local.ordered.stash_num = 0; @@ -822,7 +830,7 @@ static int schedule_term_local(void) static void schedule_config_init(odp_schedule_config_t *config) { - config->num_queues = CONFIG_MAX_SCHED_QUEUES; + config->num_queues = sched->max_queues; config->queue_size = _odp_queue_glb->config.max_queue_size; config->sched_group.all = sched->config_if.group_enable.all; config->sched_group.control = sched->config_if.group_enable.control; @@ -840,7 +848,6 @@ static void schedule_group_clear(odp_schedule_group_t group) grp_update_mask(group, &zero); sched->sched_grp[group].allocated = 0; - } static int schedule_config(const odp_schedule_config_t *config) @@ -999,7 +1006,7 @@ static inline int poll_pktin(uint32_t qi, int direct_recv, num_enq = 0; ODP_DBG("Dropped %i packets\n", num - num_enq); - buffer_free_multi(&b_hdr[num_enq], num - num_enq); + _odp_buffer_free_multi(&b_hdr[num_enq], num - num_enq); } return ret; @@ -1018,7 +1025,6 @@ static inline int do_schedule_grp(odp_queue_t *out_queue, odp_event_t out_ev[], /* Schedule events */ for (prio = 0; prio < NUM_PRIO; prio++) { - if (sched->prio_q_mask[prio] == 0) continue; @@ -1253,7 +1259,6 @@ static inline int schedule_loop(odp_queue_t *out_queue, uint64_t wait, int ret; while (1) { - ret = do_schedule(out_queue, out_ev, max_num); if (ret) { timer_run(2); @@ -1326,6 +1331,7 @@ static inline void order_lock(void) static void order_unlock(void) { + /* Nothing to do */ } static void schedule_order_lock(uint32_t lock_index) @@ -1621,9 +1627,9 @@ static int schedule_thr_rem(odp_schedule_group_t group, int thr) return 0; } -/* This function is a no-op */ -static void schedule_prefetch(int num ODP_UNUSED) +static void schedule_prefetch(int num) { + (void)num; } static int schedule_num_grps(void) @@ -1643,7 +1649,7 @@ static int schedule_capability(odp_schedule_capability_t *capa) capa->max_ordered_locks = schedule_max_ordered_locks(); capa->max_groups = schedule_num_grps(); capa->max_prios = schedule_num_prio(); - capa->max_queues = CONFIG_MAX_SCHED_QUEUES; + capa->max_queues = sched->max_queues; capa->max_queue_size = _odp_queue_glb->config.max_queue_size; capa->max_flow_id = BUF_HDR_MAX_FLOW_ID; diff --git a/platform/linux-generic/odp_schedule_scalable.c b/platform/linux-generic/odp_schedule_scalable.c index 2ba40256b..c9991c3f3 100644 --- a/platform/linux-generic/odp_schedule_scalable.c +++ b/platform/linux-generic/odp_schedule_scalable.c @@ -214,15 +214,13 @@ void _odp_sched_update_enq(sched_elem_t *q, uint32_t actual) ticket = nss.nxt_ticket++; /* Else queue already was non-empty. */ /* Attempt to update numevts counter and optionally take ticket. */ - } while (!__atomic_compare_exchange( - &q->qschst, &oss, &nss, - true, __ATOMIC_RELAXED, __ATOMIC_RELAXED)); + } while (!__atomic_compare_exchange(&q->qschst, &oss, &nss, + true, __ATOMIC_RELAXED, __ATOMIC_RELAXED)); if (odp_unlikely(ticket != TICKET_INVALID)) { /* Wait for our turn to update schedq. */ - if (odp_unlikely( - __atomic_load_n(&q->qschst.cur_ticket, - __ATOMIC_ACQUIRE) != ticket)) { + if (odp_unlikely(__atomic_load_n(&q->qschst.cur_ticket, + __ATOMIC_ACQUIRE) != ticket)) { sevl(); while (wfe() && monitor8(&q->qschst.cur_ticket, @@ -333,12 +331,9 @@ sched_update_deq(sched_elem_t *q, * the CAS operation */ nss.cur_ticket = _odp_sched_ts->ticket + 1; - } while (odp_unlikely(!__atomic_compare_exchange( - &q->qschst, - &oss, &nss, - true, - __ATOMIC_RELEASE, - __ATOMIC_RELAXED))); + } while (odp_unlikely(!__atomic_compare_exchange(&q->qschst, &oss, &nss, true, + __ATOMIC_RELEASE, + __ATOMIC_RELAXED))); return; } @@ -361,16 +356,14 @@ sched_update_deq(sched_elem_t *q, nss.wrr_budget = CONFIG_WRR_WEIGHT; } /* Attempt to update numevts and optionally take ticket. */ - } while (!__atomic_compare_exchange( - &q->qschst, &oss, &nss, - true, __ATOMIC_RELAXED, __ATOMIC_RELAXED)); + } while (!__atomic_compare_exchange(&q->qschst, &oss, &nss, + true, __ATOMIC_RELAXED, __ATOMIC_RELAXED)); if (odp_unlikely(ticket != TICKET_INVALID)) { ODP_ASSERT(q->qschst_type != ODP_SCHED_SYNC_ATOMIC); /* Wait for our turn to update schedq. */ - if (odp_unlikely( - __atomic_load_n(&q->qschst.cur_ticket, - __ATOMIC_ACQUIRE) != ticket)) { + if (odp_unlikely(__atomic_load_n(&q->qschst.cur_ticket, + __ATOMIC_ACQUIRE) != ticket)) { sevl(); while (wfe() && monitor8(&q->qschst.cur_ticket, @@ -728,8 +721,7 @@ static void pktio_start(int pktio_idx, elem->cons_type |= FLAG_PKTIN; /* Set pktin queue flag */ elem->pktio_idx = pktio_idx; elem->rx_queue = rxq; - elem->xoffset = sched_pktin_add(elem->sched_grp, - elem->sched_prio); + elem->xoffset = sched_pktin_add(elem->sched_grp, elem->sched_prio); ODP_ASSERT(elem->schedq != NULL); schedq_push(elem->schedq, elem); } @@ -899,8 +891,7 @@ static int _schedule(odp_queue_t *from, odp_event_t ev[], int num_evts) num = poll_pktin(atomq, ev, num_evts); if (odp_likely(num != 0)) { if (from) - *from = queue_get_handle( - (queue_entry_t *)atomq); + *from = queue_get_handle((queue_entry_t *)atomq); return num; } } else { @@ -932,8 +923,7 @@ dequeue_atomic: * scheduler. */ if (from) - *from = queue_get_handle( - (queue_entry_t *)atomq); + *from = queue_get_handle((queue_entry_t *)atomq); return num; } } @@ -1006,11 +996,10 @@ restart_same: * responsibility. */ /* The ticket taken below will signal producers */ - ts->ticket = __atomic_fetch_add( - &atomq->qschst.nxt_ticket, 1, __ATOMIC_RELAXED); - while (__atomic_load_n( - &atomq->qschst.cur_ticket, - __ATOMIC_ACQUIRE) != ts->ticket) { + ts->ticket = __atomic_fetch_add(&atomq->qschst.nxt_ticket, 1, + __ATOMIC_RELAXED); + while (__atomic_load_n(&atomq->qschst.cur_ticket, + __ATOMIC_ACQUIRE) != ts->ticket) { /* No need to use WFE, spinning here seems * very infrequent. */ @@ -1111,8 +1100,7 @@ restart_same: ts->rctx = rctx; if (from) - *from = queue_get_handle( - (queue_entry_t *)elem); + *from = queue_get_handle((queue_entry_t *)elem); return num; } #ifdef CONFIG_QSCHST_LOCK @@ -1571,8 +1559,7 @@ static int schedule_group_destroy(odp_schedule_group_t group) */ for (p = 0; p < ODP_SCHED_PRIO_NUM; p++) { if (sg->xcount[p] != 0) { - bitset_t wanted = atom_bitset_load( - &sg->thr_wanted, __ATOMIC_RELAXED); + bitset_t wanted = atom_bitset_load(&sg->thr_wanted, __ATOMIC_RELAXED); sevl(); while (wfe() && @@ -2167,6 +2154,7 @@ static void order_lock(void) */ static void order_unlock(void) { + /* Nothing to do */ } static uint32_t schedule_max_ordered_locks(void) diff --git a/platform/linux-generic/odp_schedule_scalable_ordered.c b/platform/linux-generic/odp_schedule_scalable_ordered.c index 239d18239..991be658e 100644 --- a/platform/linux-generic/odp_schedule_scalable_ordered.c +++ b/platform/linux-generic/odp_schedule_scalable_ordered.c @@ -332,15 +332,11 @@ int _odp_rctx_save(queue_entry_t *queue, odp_buffer_hdr_t *buf_hdr[], int num) /* No more space in current reorder context * Try to allocate another. */ - if (odp_unlikely( - bitset_is_null(ts->priv_rvec_free))) { - ts->priv_rvec_free = - atom_bitset_xchg( - &ts->rvec_free, - 0, - __ATOMIC_RELAXED); - if (odp_unlikely(bitset_is_null( - ts->priv_rvec_free))) + if (odp_unlikely(bitset_is_null(ts->priv_rvec_free))) { + ts->priv_rvec_free = atom_bitset_xchg(&ts->rvec_free, 0, + __ATOMIC_RELAXED); + + if (odp_unlikely(bitset_is_null(ts->priv_rvec_free))) /* Out of reorder contexts. * Return the number of events * stored so far. diff --git a/platform/linux-generic/odp_schedule_sp.c b/platform/linux-generic/odp_schedule_sp.c index cad391ed4..a91b9fbca 100644 --- a/platform/linux-generic/odp_schedule_sp.c +++ b/platform/linux-generic/odp_schedule_sp.c @@ -255,7 +255,6 @@ static int term_global(void) } odp_event_free(event); } - } } @@ -751,10 +750,12 @@ static void schedule_resume(void) static void schedule_release_atomic(void) { + /* Nothing to do */ } static void schedule_release_ordered(void) { + /* Nothing to do */ } static void schedule_prefetch(int num) @@ -1016,10 +1017,12 @@ static void schedule_order_lock_wait(uint32_t lock_index) static void order_lock(void) { + /* Nothing to do */ } static void order_unlock(void) { + /* Nothing to do */ } static int schedule_capability(odp_schedule_capability_t *capa) diff --git a/platform/linux-generic/odp_system_info.c b/platform/linux-generic/odp_system_info.c index cbbadf336..079210652 100644 --- a/platform/linux-generic/odp_system_info.c +++ b/platform/linux-generic/odp_system_info.c @@ -89,7 +89,6 @@ static int systemcpu_cache_line_size(void) } #endif - static uint64_t default_huge_page_size(void) { char str[1024]; @@ -248,11 +247,9 @@ static char *get_hugepage_dir(uint64_t hugepage_sz) retval = strdup(tokens[MOUNTPT]); break; } - } - /* there is an explicit page size, so check it */ - else { - pagesz = - str_to_size(&pagesz_str[pagesize_opt_len]); + } else { + /* there is an explicit page size, so check it */ + pagesz = str_to_size(&pagesz_str[pagesize_opt_len]); if (pagesz == hugepage_sz) { retval = strdup(tokens[MOUNTPT]); break; @@ -304,7 +301,6 @@ static int systemcpu(system_info_t *sysinfo) sysinfo->cpu_count = ret; - ret = systemcpu_cache_line_size(); if (ret == 0) { ODP_ERR("systemcpu_cache_line_size failed.\n"); diff --git a/platform/linux-generic/odp_timer.c b/platform/linux-generic/odp_timer.c index 8b2ba16c2..2db5bc7b1 100644 --- a/platform/linux-generic/odp_timer.c +++ b/platform/linux-generic/odp_timer.c @@ -510,7 +510,7 @@ static inline odp_timer_t timer_alloc(timer_pool_t *tp, odp_queue_t queue, const /* Add timer to queue */ _odp_queue_fn->timer_add(queue); } else { - __odp_errno = ENFILE; /* Reusing file table overflow */ + _odp_errno = ENFILE; /* Reusing file table overflow */ hdl = ODP_TIMER_INVALID; } odp_spinlock_unlock(&tp->lock); @@ -583,12 +583,9 @@ static bool timer_reset(uint32_t idx, uint64_t abs_tck, odp_buffer_t *tmo_buf, /* Atomic CAS will fail if we experienced torn reads, * retry update sequence until CAS succeeds */ - } while (!_odp_atomic_u128_cmp_xchg_mm( - (_odp_atomic_u128_t *)tb, - (_uint128_t *)&old, - (_uint128_t *)&new, - _ODP_MEMMODEL_RLS, - _ODP_MEMMODEL_RLX)); + } while (!_odp_atomic_u128_cmp_xchg_mm((_odp_atomic_u128_t *)tb, + (_uint128_t *)&old, (_uint128_t *)&new, + _ODP_MEMMODEL_RLS, _ODP_MEMMODEL_RLX)); #elif __GCC_ATOMIC_LLONG_LOCK_FREE >= 2 && \ defined __GCC_HAVE_SYNC_COMPARE_AND_SWAP_8 /* Target supports lock-free 64-bit CAS (and probably exchange) */ @@ -816,10 +813,9 @@ static inline void timer_expire(timer_pool_t *tp, uint32_t idx, uint64_t tick) new.exp_tck.v = exp_tck | TMO_INACTIVE; new.tmo_buf = ODP_BUFFER_INVALID; - int succ = _odp_atomic_u128_cmp_xchg_mm( - (_odp_atomic_u128_t *)tb, - (_uint128_t *)&old, (_uint128_t *)&new, - _ODP_MEMMODEL_RLS, _ODP_MEMMODEL_RLX); + int succ = _odp_atomic_u128_cmp_xchg_mm((_odp_atomic_u128_t *)tb, + (_uint128_t *)&old, (_uint128_t *)&new, + _ODP_MEMMODEL_RLS, _ODP_MEMMODEL_RLX); if (succ) tmo_buf = old.tmo_buf; /* Else CAS failed, something changed => skip timer @@ -966,7 +962,7 @@ static inline void timer_pool_scan_inline(int num, odp_time_t now) } } -void _timer_run_inline(int dec) +void _odp_timer_run_inline(int dec) { odp_time_t now; int num = timer_global->highest_tp_idx + 1; @@ -1284,19 +1280,19 @@ odp_timer_pool_t odp_timer_pool_create(const char *name, if ((param->res_ns && param->res_hz) || (param->res_ns == 0 && param->res_hz == 0)) { - __odp_errno = EINVAL; + _odp_errno = EINVAL; return ODP_TIMER_POOL_INVALID; } if (param->res_hz == 0 && param->res_ns < timer_global->highest_res_ns) { - __odp_errno = EINVAL; + _odp_errno = EINVAL; return ODP_TIMER_POOL_INVALID; } if (param->res_ns == 0 && param->res_hz > timer_global->highest_res_hz) { - __odp_errno = EINVAL; + _odp_errno = EINVAL; return ODP_TIMER_POOL_INVALID; } diff --git a/platform/linux-generic/odp_timer_wheel.c b/platform/linux-generic/odp_timer_wheel.c index 84640d215..71b3370b5 100644 --- a/platform/linux-generic/odp_timer_wheel.c +++ b/platform/linux-generic/odp_timer_wheel.c @@ -637,9 +637,7 @@ static int timer_current_wheel_update(timer_wheels_t *timer_wheels, if (timer_slot->user_data != 0) { rc = expired_timers_append(timer_wheels, timer_slot); if (rc < 0) - timer_wheels-> - expired_timers_ring-> - expired_ring_full_cnt++; + timer_wheels->expired_timers_ring->expired_ring_full_cnt++; timer_slot->user_data = 0; } @@ -954,9 +952,7 @@ void _odp_timer_wheel_stats_print(_odp_timer_wheel_t timer_wheel) ODP_PRINT("_odp_int_timer_wheel_stats current_ticks=%" PRIu64 "\n", timer_wheels->current_ticks); for (wheel_idx = 0; wheel_idx < 4; wheel_idx++) - _odp_int_timer_wheel_desc_print( - &timer_wheels->wheel_descs[wheel_idx], - wheel_idx); + _odp_int_timer_wheel_desc_print(&timer_wheels->wheel_descs[wheel_idx], wheel_idx); ODP_PRINT(" total timer_inserts=%" PRIu64 " timer_removes=%" PRIu64 " insert_fails=%" PRIu64 "\n", diff --git a/platform/linux-generic/odp_traffic_mngr.c b/platform/linux-generic/odp_traffic_mngr.c index e1aff6e8c..0ae107509 100644 --- a/platform/linux-generic/odp_traffic_mngr.c +++ b/platform/linux-generic/odp_traffic_mngr.c @@ -356,7 +356,6 @@ static void *alloc_entry_in_tbl(profile_tbl_t *profile_tbl, default: ODP_ERR("Invalid TM profile\n"); return NULL; - } } @@ -392,7 +391,6 @@ static void free_tbl_entry(profile_tbl_t *profile_tbl, default: ODP_ERR("Invalid TM profile\n"); return; - } } @@ -1238,10 +1236,9 @@ static odp_bool_t run_sched(tm_system_t *tm_system, * virtual finish time, just insert it into this * sched_state's list sorted by virtual finish times. */ - rc = _odp_sorted_list_insert( - tm_system->_odp_int_sorted_pool, - new_sched_state->sorted_list, - new_finish_time, new_pkt_desc->word); + rc = _odp_sorted_list_insert(tm_system->_odp_int_sorted_pool, + new_sched_state->sorted_list, + new_finish_time, new_pkt_desc->word); if (0 <= rc) { new_sched_state->sorted_list_cnt++; @@ -2067,8 +2064,7 @@ static void egress_vlan_marking(tm_vlan_marking_t *vlan_marking, * correctness rather then performance. */ split_hdr = hdr_len < (_ODP_ETHHDR_LEN + _ODP_VLANHDR_LEN); if (split_hdr) { - odp_packet_copy_to_mem(odp_pkt, _ODP_ETHHDR_LEN, - _ODP_VLANHDR_LEN, &vlan_hdr); + odp_packet_copy_to_mem(odp_pkt, _ODP_ETHHDR_LEN, _ODP_VLANHDR_LEN, &vlan_hdr); vlan_hdr_ptr = &vlan_hdr; } @@ -2082,8 +2078,7 @@ static void egress_vlan_marking(tm_vlan_marking_t *vlan_marking, vlan_hdr_ptr->tci = odp_cpu_to_be_16(new_tci); if (split_hdr) - odp_packet_copy_from_mem(odp_pkt, _ODP_ETHHDR_LEN, - _ODP_VLANHDR_LEN, &vlan_hdr); + odp_packet_copy_from_mem(odp_pkt, _ODP_ETHHDR_LEN, _ODP_VLANHDR_LEN, &vlan_hdr); } static void egress_ipv4_tos_marking(tm_tos_marking_t *tos_marking, @@ -2107,8 +2102,7 @@ static void egress_ipv4_tos_marking(tm_tos_marking_t *tos_marking, * correctness rather then performance. */ split_hdr = hdr_len < 12; if (split_hdr) { - odp_packet_copy_to_mem(odp_pkt, l3_offset, - _ODP_IPV4HDR_LEN, &ipv4_hdr); + odp_packet_copy_to_mem(odp_pkt, l3_offset, _ODP_IPV4HDR_LEN, &ipv4_hdr); ipv4_hdr_ptr = &ipv4_hdr; } @@ -2149,8 +2143,7 @@ static void egress_ipv4_tos_marking(tm_tos_marking_t *tos_marking, ipv4_hdr_ptr->tos = new_tos; ipv4_hdr_ptr->chksum = odp_cpu_to_be_16((~ones_compl_sum) & 0xFFFF); if (split_hdr) - odp_packet_copy_from_mem(odp_pkt, l3_offset, - _ODP_IPV4HDR_LEN, &ipv4_hdr); + odp_packet_copy_from_mem(odp_pkt, l3_offset, _ODP_IPV4HDR_LEN, &ipv4_hdr); } static void egress_ipv6_tc_marking(tm_tos_marking_t *tos_marking, @@ -2174,8 +2167,7 @@ static void egress_ipv6_tc_marking(tm_tos_marking_t *tos_marking, * correctness rather then performance. */ split_hdr = hdr_len < 4; if (split_hdr) { - odp_packet_copy_to_mem(odp_pkt, l3_offset, - _ODP_IPV6HDR_LEN, &ipv6_hdr); + odp_packet_copy_to_mem(odp_pkt, l3_offset, _ODP_IPV6HDR_LEN, &ipv6_hdr); ipv6_hdr_ptr = &ipv6_hdr; } @@ -2203,8 +2195,7 @@ static void egress_ipv6_tc_marking(tm_tos_marking_t *tos_marking, ipv6_hdr_ptr->ver_tc_flow = odp_cpu_to_be_32(new_ver_tc_flow); if (split_hdr) - odp_packet_copy_from_mem(odp_pkt, l3_offset, - _ODP_IPV6HDR_LEN, &ipv6_hdr); + odp_packet_copy_from_mem(odp_pkt, l3_offset, _ODP_IPV6HDR_LEN, &ipv6_hdr); } static void tm_egress_marking(tm_system_t *tm_system, odp_packet_t odp_pkt) @@ -2307,9 +2298,8 @@ static int tm_process_input_work_queue(tm_system_t *tm_system, /* If the tm_queue_obj already has a pkt to work with, * then just add this new pkt to the associated * _odp_int_pkt_queue. */ - (void)_odp_pkt_queue_append( - tm_system->_odp_int_queue_pool, - tm_queue_obj->_odp_int_pkt_queue, pkt); + (void)_odp_pkt_queue_append(tm_system->_odp_int_queue_pool, + tm_queue_obj->_odp_int_pkt_queue, pkt); tm_queue_obj->pkts_enqueued_cnt++; } else { /* If the tm_queue_obj doesn't have a pkt to work @@ -2749,6 +2739,7 @@ static int tm_thread_create(tm_system_group_t *tm_group) rc); return rc; } + static void _odp_tm_group_destroy(_odp_tm_group_t odp_tm_group) { tm_system_group_t *tm_group; @@ -2976,21 +2967,19 @@ odp_tm_t odp_tm_create(const char *name, odp_ticketlock_init(&tm_system->tm_system_lock); odp_atomic_init_u64(&tm_system->destroying, 0); - tm_system->_odp_int_sorted_pool = _odp_sorted_pool_create( - max_sorted_lists); + tm_system->_odp_int_sorted_pool = _odp_sorted_pool_create(max_sorted_lists); create_fail |= tm_system->_odp_int_sorted_pool == _ODP_INT_SORTED_POOL_INVALID; if (create_fail == 0) { - tm_system->_odp_int_queue_pool = _odp_queue_pool_create( - max_num_queues, max_queued_pkts); + tm_system->_odp_int_queue_pool = _odp_queue_pool_create(max_num_queues, + max_queued_pkts); create_fail |= tm_system->_odp_int_queue_pool == _ODP_INT_QUEUE_POOL_INVALID; } if (create_fail == 0) { - tm_system->_odp_int_timer_wheel = _odp_timer_wheel_create( - max_timers, tm_system); + tm_system->_odp_int_timer_wheel = _odp_timer_wheel_create(max_timers, tm_system); create_fail |= tm_system->_odp_int_timer_wheel == _ODP_INT_TIMER_WHEEL_INVALID; } @@ -3007,20 +2996,14 @@ odp_tm_t odp_tm_create(const char *name, if (create_fail) { _odp_int_name_tbl_delete(name_tbl_id); - if (tm_system->_odp_int_sorted_pool - != _ODP_INT_SORTED_POOL_INVALID) - _odp_sorted_pool_destroy( - tm_system->_odp_int_sorted_pool); + if (tm_system->_odp_int_sorted_pool != _ODP_INT_SORTED_POOL_INVALID) + _odp_sorted_pool_destroy(tm_system->_odp_int_sorted_pool); - if (tm_system->_odp_int_queue_pool != - _ODP_INT_QUEUE_POOL_INVALID) - _odp_queue_pool_destroy( - tm_system->_odp_int_queue_pool); + if (tm_system->_odp_int_queue_pool != _ODP_INT_QUEUE_POOL_INVALID) + _odp_queue_pool_destroy(tm_system->_odp_int_queue_pool); - if (tm_system->_odp_int_timer_wheel - != _ODP_INT_TIMER_WHEEL_INVALID) - _odp_timer_wheel_destroy( - tm_system->_odp_int_timer_wheel); + if (tm_system->_odp_int_timer_wheel != _ODP_INT_TIMER_WHEEL_INVALID) + _odp_timer_wheel_destroy(tm_system->_odp_int_timer_wheel); tm_system_free(tm_system); odp_ticketlock_unlock(&tm_glb->create_lock); @@ -3708,9 +3691,8 @@ odp_tm_node_t odp_tm_node_create(odp_tm_t odp_tm, const char *name, schedulers_obj = &tm_node_obj->schedulers_obj; schedulers_obj->num_priorities = num_priorities; for (priority = 0; priority < num_priorities; priority++) { - sorted_list = _odp_sorted_list_create( - tm_system->_odp_int_sorted_pool, - params->max_fanin); + sorted_list = _odp_sorted_list_create(tm_system->_odp_int_sorted_pool, + params->max_fanin); schedulers_obj->sched_states[priority].sorted_list = sorted_list; } @@ -4673,8 +4655,8 @@ int odp_tm_total_threshold_config(odp_tm_t odp_tm, return -1; odp_ticketlock_lock(&tm_glb->profile_lock); - tm_system->total_info.threshold_params = tm_get_profile_params( - thresholds_profile, TM_THRESHOLD_PROFILE); + tm_system->total_info.threshold_params = tm_get_profile_params(thresholds_profile, + TM_THRESHOLD_PROFILE); odp_ticketlock_unlock(&tm_glb->profile_lock); return 0; } diff --git a/platform/linux-generic/pktio/dpdk.c b/platform/linux-generic/pktio/dpdk.c index 97fb6ef1d..fb99d8ed5 100644 --- a/platform/linux-generic/pktio/dpdk.c +++ b/platform/linux-generic/pktio/dpdk.c @@ -432,8 +432,8 @@ static int pool_dequeue_bulk(struct rte_mempool *mp, void **obj_table, int pkts; int i; - pkts = packet_alloc_multi(pool, pool_entry->seg_len, packet_tbl, - num); + pkts = _odp_packet_alloc_multi(pool, pool_entry->seg_len, packet_tbl, + num); if (odp_unlikely(pkts != (int)num)) { if (pkts > 0) @@ -577,10 +577,10 @@ static inline int mbuf_to_pkt(pktio_entry_t *pktio_entry, /* Allocate maximum sized packets */ max_len = pkt_dpdk->data_room; - num = packet_alloc_multi(pool, max_len + frame_offset, - pkt_table, mbuf_num); + num = _odp_packet_alloc_multi(pool, max_len + frame_offset, + pkt_table, mbuf_num); if (num != mbuf_num) { - ODP_DBG("packet_alloc_multi() unable to allocate all packets: " + ODP_DBG("_odp_packet_alloc_multi() unable to allocate all packets: " "%d/%" PRIu16 " allocated\n", num, mbuf_num); for (i = num; i < mbuf_num; i++) rte_pktmbuf_free(mbuf_table[i]); @@ -805,7 +805,7 @@ static inline int pkt_to_mbuf(pktio_entry_t *pktio_entry, if (odp_unlikely(pkt_len > pkt_dpdk->mtu)) { if (i == 0) - __odp_errno = EMSGSIZE; + _odp_errno = EMSGSIZE; goto fail; } @@ -999,7 +999,7 @@ static inline int pkt_to_mbuf_zero(pktio_entry_t *pktio_entry, fail: if (i == 0) - __odp_errno = EMSGSIZE; + _odp_errno = EMSGSIZE; return i; } @@ -2063,8 +2063,8 @@ static int dpdk_send(pktio_entry_t *pktio_entry, int index, } } } - if (odp_unlikely(tx_pkts == 0 && __odp_errno != 0)) - return -1; + if (odp_unlikely(tx_pkts == 0 && _odp_errno != 0)) + return -1; } else { if (odp_unlikely(tx_pkts < mbufs)) { for (i = tx_pkts; i < mbufs; i++) @@ -2072,7 +2072,7 @@ static int dpdk_send(pktio_entry_t *pktio_entry, int index, } if (odp_unlikely(tx_pkts == 0)) { - if (__odp_errno != 0) + if (_odp_errno != 0) return -1; } else { odp_packet_free_multi(pkt_table, tx_pkts); @@ -2227,7 +2227,7 @@ static int dpdk_stats_reset(pktio_entry_t *pktio_entry) return 0; } -const pktio_if_ops_t dpdk_pktio_ops = { +const pktio_if_ops_t _odp_dpdk_pktio_ops = { .name = "dpdk", .init_global = dpdk_pktio_init_global, .init_local = dpdk_pktio_init_local, diff --git a/platform/linux-generic/pktio/io_ops.c b/platform/linux-generic/pktio/io_ops.c index cd85164e6..b5a08b58a 100644 --- a/platform/linux-generic/pktio/io_ops.c +++ b/platform/linux-generic/pktio/io_ops.c @@ -11,13 +11,13 @@ * Order matters. The first implementation to setup successfully * will be picked. * Array must be NULL terminated */ -const pktio_if_ops_t * const pktio_if_ops[] = { +const pktio_if_ops_t * const _odp_pktio_if_ops[] = { &_odp_loopback_pktio_ops, #ifdef _ODP_PKTIO_DPDK - &dpdk_pktio_ops, + &_odp_dpdk_pktio_ops, #endif #ifdef _ODP_PKTIO_NETMAP - &netmap_pktio_ops, + &_odp_netmap_pktio_ops, #endif #ifdef _ODP_PKTIO_PCAP &_odp_pcap_pktio_ops, diff --git a/platform/linux-generic/pktio/ipc.c b/platform/linux-generic/pktio/ipc.c index 845fd821d..e0c8cba3b 100644 --- a/platform/linux-generic/pktio/ipc.c +++ b/platform/linux-generic/pktio/ipc.c @@ -130,7 +130,7 @@ static ring_ptr_t *_ring_create(const char *name, uint32_t count, /* count must be a power of 2 */ if (!CHECK_IS_POWER2(count)) { ODP_ERR("Requested size is invalid, must be a power of 2\n"); - __odp_errno = EINVAL; + _odp_errno = EINVAL; return NULL; } @@ -145,7 +145,7 @@ static ring_ptr_t *_ring_create(const char *name, uint32_t count, ring_ptr_init(r); } else { - __odp_errno = ENOMEM; + _odp_errno = ENOMEM; ODP_ERR("Cannot reserve memory\n"); } diff --git a/platform/linux-generic/pktio/loop.c b/platform/linux-generic/pktio/loop.c index 1d5751eb8..d58c9a55c 100644 --- a/platform/linux-generic/pktio/loop.c +++ b/platform/linux-generic/pktio/loop.c @@ -311,7 +311,7 @@ static int loopback_send(pktio_entry_t *pktio_entry, int index ODP_UNUSED, if (odp_unlikely(pkt_len > pkt_loop->mtu)) { if (nb_tx == 0) { - __odp_errno = EMSGSIZE; + _odp_errno = EMSGSIZE; return -1; } break; @@ -431,6 +431,7 @@ static int loopback_init_capability(pktio_entry_t *pktio_entry) capa->maxlen.max_output = LOOP_MTU_MAX; odp_pktio_config_init(&capa->config); + capa->config.enable_loop = 1; capa->config.pktin.bit.ts_all = 1; capa->config.pktin.bit.ts_ptp = 1; capa->config.pktin.bit.ipv4_chksum = 1; diff --git a/platform/linux-generic/pktio/netmap.c b/platform/linux-generic/pktio/netmap.c index a6bebaa11..544503aa3 100644 --- a/platform/linux-generic/pktio/netmap.c +++ b/platform/linux-generic/pktio/netmap.c @@ -345,7 +345,7 @@ static int netmap_close(pktio_entry_t *pktio_entry) netmap_close_descriptors(pktio_entry); if (pkt_nm->sockfd != -1 && close(pkt_nm->sockfd) != 0) { - __odp_errno = errno; + _odp_errno = errno; ODP_ERR("close(sockfd): %s\n", strerror(errno)); return -1; } @@ -830,8 +830,8 @@ static inline int netmap_pkt_to_odp(pktio_entry_t *pktio_entry, /* Allocate maximum sized packets */ max_len = pkt_priv(pktio_entry)->mtu; - num = packet_alloc_multi(pool, max_len + frame_offset, - pkt_tbl, slot_num); + num = _odp_packet_alloc_multi(pool, max_len + frame_offset, + pkt_tbl, slot_num); for (i = 0; i < num; i++) { netmap_slot_t slot; @@ -1121,7 +1121,7 @@ static int netmap_send(pktio_entry_t *pktio_entry, int index, if (pkt_len > pkt_nm->mtu) { if (nb_tx == 0) - __odp_errno = EMSGSIZE; + _odp_errno = EMSGSIZE; break; } for (i = 0; i < NM_INJECT_RETRIES; i++) { @@ -1158,7 +1158,7 @@ static int netmap_send(pktio_entry_t *pktio_entry, int index, odp_ticketlock_unlock(&pkt_nm->tx_desc_ring[index].s.lock); if (odp_unlikely(nb_tx == 0)) { - if (__odp_errno != 0) + if (_odp_errno != 0) return -1; } else { if (odp_unlikely(tx_ts_idx && nb_tx >= tx_ts_idx)) @@ -1200,7 +1200,7 @@ static int netmap_promisc_mode_set(pktio_entry_t *pktio_entry, odp_bool_t enable) { if (pkt_priv(pktio_entry)->is_virtual) { - __odp_errno = ENOTSUP; + _odp_errno = ENOTSUP; return -1; } @@ -1273,7 +1273,7 @@ static int netmap_init_global(void) return 0; } -const pktio_if_ops_t netmap_pktio_ops = { +const pktio_if_ops_t _odp_netmap_pktio_ops = { .name = "netmap", .print = netmap_print, .init_global = netmap_init_global, diff --git a/platform/linux-generic/pktio/pcap.c b/platform/linux-generic/pktio/pcap.c index ffb6fab68..82635e59c 100644 --- a/platform/linux-generic/pktio/pcap.c +++ b/platform/linux-generic/pktio/pcap.c @@ -266,8 +266,8 @@ static int pcapif_recv_pkt(pktio_entry_t *pktio_entry, int index ODP_UNUSED, pkt_len = hdr->caplen; - ret = packet_alloc_multi(pcap->pool, pkt_len + frame_offset, - &pkt, 1); + ret = _odp_packet_alloc_multi(pcap->pool, pkt_len + frame_offset, + &pkt, 1); if (odp_unlikely(ret != 1)) break; diff --git a/platform/linux-generic/pktio/socket.c b/platform/linux-generic/pktio/socket.c index 29b6c9af6..eb4390e46 100644 --- a/platform/linux-generic/pktio/socket.c +++ b/platform/linux-generic/pktio/socket.c @@ -88,7 +88,7 @@ static int sock_close(pktio_entry_t *pktio_entry) pkt_sock_t *pkt_sock = pkt_priv(pktio_entry); if (pkt_sock->sockfd != -1 && close(pkt_sock->sockfd) != 0) { - __odp_errno = errno; + _odp_errno = errno; ODP_ERR("close(sockfd): %s\n", strerror(errno)); return -1; } @@ -120,7 +120,7 @@ static int sock_setup_pkt(pktio_entry_t *pktio_entry, const char *netdev, sockfd = socket(AF_PACKET, SOCK_RAW, htons(ETH_P_ALL)); if (sockfd == -1) { - __odp_errno = errno; + _odp_errno = errno; ODP_ERR("socket(): %s\n", strerror(errno)); goto error; } @@ -131,7 +131,7 @@ static int sock_setup_pkt(pktio_entry_t *pktio_entry, const char *netdev, snprintf(ethreq.ifr_name, IF_NAMESIZE, "%s", netdev); err = ioctl(sockfd, SIOCGIFINDEX, ðreq); if (err != 0) { - __odp_errno = errno; + _odp_errno = errno; ODP_ERR("ioctl(SIOCGIFINDEX): %s: \"%s\".\n", strerror(errno), ethreq.ifr_name); goto error; @@ -155,7 +155,7 @@ static int sock_setup_pkt(pktio_entry_t *pktio_entry, const char *netdev, sa_ll.sll_ifindex = if_idx; sa_ll.sll_protocol = htons(ETH_P_ALL); if (bind(sockfd, (struct sockaddr *)&sa_ll, sizeof(sa_ll)) < 0) { - __odp_errno = errno; + _odp_errno = errno; ODP_ERR("bind(to IF): %s\n", strerror(errno)); goto error; } @@ -230,7 +230,7 @@ static int sock_mmsg_recv(pktio_entry_t *pktio_entry, int index ODP_UNUSED, memset(msgvec, 0, sizeof(msgvec)); - nb_pkts = packet_alloc_multi(pool, alloc_len, pkt_table, num); + nb_pkts = _odp_packet_alloc_multi(pool, alloc_len, pkt_table, num); for (i = 0; i < nb_pkts; i++) { if (frame_offset) pull_head(packet_hdr(pkt_table[i]), frame_offset); @@ -454,7 +454,7 @@ static int sock_mmsg_send(pktio_entry_t *pktio_entry, int index ODP_UNUSED, ret = sendmmsg(sockfd, &msgvec[i], num - i, MSG_DONTWAIT); if (odp_unlikely(ret <= -1)) { if (i == 0 && SOCK_ERR_REPORT(errno)) { - __odp_errno = errno; + _odp_errno = errno; ODP_ERR("sendmmsg(): %s\n", strerror(errno)); odp_ticketlock_unlock(&pkt_sock->tx_lock); return -1; diff --git a/platform/linux-generic/pktio/socket_common.c b/platform/linux-generic/pktio/socket_common.c index 88c1471b0..ab0caeb81 100644 --- a/platform/linux-generic/pktio/socket_common.c +++ b/platform/linux-generic/pktio/socket_common.c @@ -57,7 +57,7 @@ int _odp_mac_addr_get_fd(int fd, const char *name, unsigned char mac_dst[]) snprintf(ethreq.ifr_name, IF_NAMESIZE, "%s", name); ret = ioctl(fd, SIOCGIFHWADDR, ðreq); if (ret != 0) { - __odp_errno = errno; + _odp_errno = errno; ODP_ERR("ioctl(SIOCGIFHWADDR): %s: \"%s\".\n", strerror(errno), ethreq.ifr_name); return -1; @@ -82,7 +82,7 @@ uint32_t _odp_mtu_get_fd(int fd, const char *name) snprintf(ifr.ifr_name, IF_NAMESIZE, "%s", name); ret = ioctl(fd, SIOCGIFMTU, &ifr); if (ret < 0) { - __odp_errno = errno; + _odp_errno = errno; ODP_ERR("ioctl(SIOCGIFMTU): %s: \"%s\".\n", strerror(errno), ifr.ifr_name); return 0; @@ -106,7 +106,7 @@ int _odp_mtu_set_fd(int fd, const char *name, int mtu) ret = ioctl(fd, SIOCSIFMTU, &ifr); if (ret < 0) { - __odp_errno = errno; + _odp_errno = errno; ODP_ERR("ioctl(SIOCSIFMTU): %s: \"%s\".\n", strerror(errno), ifr.ifr_name); return -1; @@ -127,7 +127,7 @@ int _odp_promisc_mode_set_fd(int fd, const char *name, int enable) snprintf(ifr.ifr_name, IF_NAMESIZE, "%s", name); ret = ioctl(fd, SIOCGIFFLAGS, &ifr); if (ret < 0) { - __odp_errno = errno; + _odp_errno = errno; ODP_DBG("ioctl(SIOCGIFFLAGS): %s: \"%s\".\n", strerror(errno), ifr.ifr_name); return -1; @@ -140,7 +140,7 @@ int _odp_promisc_mode_set_fd(int fd, const char *name, int enable) ret = ioctl(fd, SIOCSIFFLAGS, &ifr); if (ret < 0) { - __odp_errno = errno; + _odp_errno = errno; ODP_DBG("ioctl(SIOCSIFFLAGS): %s: \"%s\".\n", strerror(errno), ifr.ifr_name); return -1; @@ -161,7 +161,7 @@ int _odp_promisc_mode_get_fd(int fd, const char *name) snprintf(ifr.ifr_name, IF_NAMESIZE, "%s", name); ret = ioctl(fd, SIOCGIFFLAGS, &ifr); if (ret < 0) { - __odp_errno = errno; + _odp_errno = errno; ODP_DBG("ioctl(SIOCGIFFLAGS): %s: \"%s\".\n", strerror(errno), ifr.ifr_name); return -1; @@ -178,7 +178,7 @@ int _odp_link_status_fd(int fd, const char *name) snprintf(ifr.ifr_name, IF_NAMESIZE, "%s", name); ret = ioctl(fd, SIOCGIFFLAGS, &ifr); if (ret < 0) { - __odp_errno = errno; + _odp_errno = errno; ODP_DBG("ioctl(SIOCGIFFLAGS): %s: \"%s\".\n", strerror(errno), ifr.ifr_name); return ODP_PKTIO_LINK_STATUS_UNKNOWN; @@ -206,7 +206,7 @@ int _odp_link_info_fd(int fd, const char *name, odp_pktio_link_info_t *info) /* Link pause status */ ifr.ifr_data = (void *)&pcmd; if (ioctl(fd, SIOCETHTOOL, &ifr) && errno != EOPNOTSUPP) { - __odp_errno = errno; + _odp_errno = errno; ODP_ERR("ioctl(SIOCETHTOOL): %s: \"%s\".\n", strerror(errno), ifr.ifr_name); return -1; @@ -219,7 +219,7 @@ int _odp_link_info_fd(int fd, const char *name, odp_pktio_link_info_t *info) ifr.ifr_data = (void *)&ecmd_old; if (ioctl(fd, SIOCETHTOOL, &ifr) < 0) { - __odp_errno = errno; + _odp_errno = errno; ODP_ERR("ioctl(SIOCETHTOOL): %s: \"%s\".\n", strerror(errno), ifr.ifr_name); return -1; } @@ -277,7 +277,7 @@ int _odp_link_info_fd(int fd, const char *name, odp_pktio_link_info_t *info) *ecmd = hcmd; ifr.ifr_data = (void *)ecmd; if (ioctl(fd, SIOCETHTOOL, &ifr) < 0) { - __odp_errno = errno; + _odp_errno = errno; ODP_ERR("ioctl(SIOCETHTOOL): %s: \"%s\".\n", strerror(errno), ifr.ifr_name); return -1; diff --git a/platform/linux-generic/pktio/socket_mmap.c b/platform/linux-generic/pktio/socket_mmap.c index 05d0d8254..532f392fa 100644 --- a/platform/linux-generic/pktio/socket_mmap.c +++ b/platform/linux-generic/pktio/socket_mmap.c @@ -109,14 +109,14 @@ static int mmap_pkt_socket(void) int ret, sock = socket(PF_PACKET, SOCK_RAW, htons(ETH_P_ALL)); if (sock == -1) { - __odp_errno = errno; + _odp_errno = errno; ODP_ERR("socket(SOCK_RAW): %s\n", strerror(errno)); return -1; } ret = setsockopt(sock, SOL_PACKET, PACKET_VERSION, &ver, sizeof(ver)); if (ret == -1) { - __odp_errno = errno; + _odp_errno = errno; ODP_ERR("setsockopt(PACKET_VERSION): %s\n", strerror(errno)); close(sock); return -1; @@ -192,7 +192,7 @@ static inline unsigned pkt_mmap_v2_rx(pktio_entry_t *pktio_entry, vlan_len = 4; alloc_len = pkt_len + frame_offset + vlan_len; - ret = packet_alloc_multi(pool, alloc_len, &pkt, 1); + ret = _odp_packet_alloc_multi(pool, alloc_len, &pkt, 1); if (odp_unlikely(ret != 1)) { /* Stop receiving packets when pool is empty. Leave @@ -460,7 +460,7 @@ static int mmap_setup_ring(pkt_sock_mmap_t *pkt_sock, struct ring *ring, ret = setsockopt(sock, SOL_PACKET, type, &ring->req, sizeof(ring->req)); if (ret == -1) { - __odp_errno = errno; + _odp_errno = errno; ODP_ERR("setsockopt(pkt mmap): %s\n", strerror(errno)); return -1; } @@ -491,7 +491,7 @@ static int mmap_sock(pkt_sock_mmap_t *pkt_sock) MAP_SHARED | MAP_LOCKED | MAP_POPULATE, sock, 0); if (pkt_sock->mmap_base == MAP_FAILED) { - __odp_errno = errno; + _odp_errno = errno; ODP_ERR("mmap rx&tx buffer failed: %s\n", strerror(errno)); return -1; } @@ -547,7 +547,7 @@ static int mmap_bind_sock(pkt_sock_mmap_t *pkt_sock, const char *netdev) ret = bind(pkt_sock->sockfd, (struct sockaddr *)&pkt_sock->ll, sizeof(pkt_sock->ll)); if (ret == -1) { - __odp_errno = errno; + _odp_errno = errno; ODP_ERR("bind(to IF): %s\n", strerror(errno)); return -1; } @@ -567,7 +567,7 @@ static int sock_mmap_close(pktio_entry_t *entry) } if (pkt_sock->sockfd != -1 && close(pkt_sock->sockfd) != 0) { - __odp_errno = errno; + _odp_errno = errno; ODP_ERR("close(sockfd): %s\n", strerror(errno)); return -1; } @@ -641,7 +641,7 @@ static int sock_mmap_open(odp_pktio_t id ODP_UNUSED, if_idx = if_nametoindex(netdev); if (if_idx == 0) { - __odp_errno = errno; + _odp_errno = errno; ODP_ERR("if_nametoindex(): %s\n", strerror(errno)); goto error; } diff --git a/platform/linux-generic/pktio/stats/ethtool_stats.c b/platform/linux-generic/pktio/stats/ethtool_stats.c index 0bd4f2c61..8e94e03e7 100644 --- a/platform/linux-generic/pktio/stats/ethtool_stats.c +++ b/platform/linux-generic/pktio/stats/ethtool_stats.c @@ -48,13 +48,13 @@ static struct ethtool_gstrings *get_stringset(int fd, struct ifreq *ifr) drvinfo.cmd = ETHTOOL_GDRVINFO; ifr->ifr_data = (void *)&drvinfo; if (ioctl(fd, SIOCETHTOOL, ifr)) { - __odp_errno = errno; + _odp_errno = errno; ODP_ERR("Cannot get stats information\n"); return NULL; } len = *(uint32_t *)(void *)((char *)&drvinfo + drvinfo_offset); } else { - __odp_errno = errno; + _odp_errno = errno; return NULL; } @@ -74,7 +74,7 @@ static struct ethtool_gstrings *get_stringset(int fd, struct ifreq *ifr) strings->len = len; ifr->ifr_data = (void *)strings; if (ioctl(fd, SIOCETHTOOL, ifr)) { - __odp_errno = errno; + _odp_errno = errno; ODP_ERR("Cannot get stats information\n"); free(strings); return NULL; @@ -114,7 +114,7 @@ static int ethtool_stats(int fd, struct ifreq *ifr, odp_pktio_stats_t *stats) ifr->ifr_data = (void *)estats; err = ioctl(fd, SIOCETHTOOL, ifr); if (err < 0) { - __odp_errno = errno; + _odp_errno = errno; free(strings); free(estats); return -1; diff --git a/platform/linux-generic/pktio/stats/sysfs_stats.c b/platform/linux-generic/pktio/stats/sysfs_stats.c index 474586e19..45e005c74 100644 --- a/platform/linux-generic/pktio/stats/sysfs_stats.c +++ b/platform/linux-generic/pktio/stats/sysfs_stats.c @@ -19,7 +19,7 @@ static int sysfs_get_val(const char *fname, uint64_t *val) file = fopen(fname, "rt"); if (file == NULL) { - __odp_errno = errno; + _odp_errno = errno; /* do not print debug err if sysfs is not supported by * kernel driver. */ diff --git a/platform/linux-generic/pktio/tap.c b/platform/linux-generic/pktio/tap.c index cb85e9afd..e7c9417c3 100644 --- a/platform/linux-generic/pktio/tap.c +++ b/platform/linux-generic/pktio/tap.c @@ -90,7 +90,7 @@ static int mac_addr_set_fd(int fd, const char *name, ret = ioctl(fd, SIOCSIFHWADDR, ðreq); if (ret != 0) { - __odp_errno = errno; + _odp_errno = errno; ODP_ERR("ioctl(SIOCSIFHWADDR): %s: \"%s\".\n", strerror(errno), ethreq.ifr_name); return -1; @@ -121,7 +121,7 @@ static int tap_pktio_open(odp_pktio_t id ODP_UNUSED, fd = open("/dev/net/tun", O_RDWR); if (fd < 0) { - __odp_errno = errno; + _odp_errno = errno; ODP_ERR("failed to open /dev/net/tun: %s\n", strerror(errno)); return -1; } @@ -136,7 +136,7 @@ static int tap_pktio_open(odp_pktio_t id ODP_UNUSED, snprintf(ifr.ifr_name, IF_NAMESIZE, "%s", devname + 4); if (ioctl(fd, TUNSETIFF, (void *)&ifr) < 0) { - __odp_errno = errno; + _odp_errno = errno; ODP_ERR("%s: creating tap device failed: %s\n", ifr.ifr_name, strerror(errno)); goto tap_err; @@ -145,13 +145,13 @@ static int tap_pktio_open(odp_pktio_t id ODP_UNUSED, /* Set nonblocking mode on interface. */ flags = fcntl(fd, F_GETFL, 0); if (flags < 0) { - __odp_errno = errno; + _odp_errno = errno; ODP_ERR("fcntl(F_GETFL) failed: %s\n", strerror(errno)); goto tap_err; } if (fcntl(fd, F_SETFL, flags | O_NONBLOCK) < 0) { - __odp_errno = errno; + _odp_errno = errno; ODP_ERR("fcntl(F_SETFL) failed: %s\n", strerror(errno)); goto tap_err; } @@ -162,14 +162,14 @@ static int tap_pktio_open(odp_pktio_t id ODP_UNUSED, /* Create AF_INET socket for network interface related operations. */ skfd = socket(AF_INET, SOCK_DGRAM, 0); if (skfd < 0) { - __odp_errno = errno; + _odp_errno = errno; ODP_ERR("socket creation failed: %s\n", strerror(errno)); goto tap_err; } mtu = _odp_mtu_get_fd(skfd, devname + 4); if (mtu == 0) { - __odp_errno = errno; + _odp_errno = errno; ODP_ERR("_odp_mtu_get_fd failed: %s\n", strerror(errno)); goto sock_err; } @@ -201,7 +201,7 @@ static int tap_pktio_start(pktio_entry_t *pktio_entry) /* Up interface by default. */ if (ioctl(tap->skfd, SIOCGIFFLAGS, &ifr) < 0) { - __odp_errno = errno; + _odp_errno = errno; ODP_ERR("ioctl(SIOCGIFFLAGS) failed: %s\n", strerror(errno)); goto sock_err; } @@ -210,7 +210,7 @@ static int tap_pktio_start(pktio_entry_t *pktio_entry) ifr.ifr_flags |= IFF_RUNNING; if (ioctl(tap->skfd, SIOCSIFFLAGS, &ifr) < 0) { - __odp_errno = errno; + _odp_errno = errno; ODP_ERR("failed to come up: %s\n", strerror(errno)); goto sock_err; } @@ -232,7 +232,7 @@ static int tap_pktio_stop(pktio_entry_t *pktio_entry) /* Up interface by default. */ if (ioctl(tap->skfd, SIOCGIFFLAGS, &ifr) < 0) { - __odp_errno = errno; + _odp_errno = errno; ODP_ERR("ioctl(SIOCGIFFLAGS) failed: %s\n", strerror(errno)); goto sock_err; } @@ -241,7 +241,7 @@ static int tap_pktio_stop(pktio_entry_t *pktio_entry) ifr.ifr_flags &= ~IFF_RUNNING; if (ioctl(tap->skfd, SIOCSIFFLAGS, &ifr) < 0) { - __odp_errno = errno; + _odp_errno = errno; ODP_ERR("failed to come up: %s\n", strerror(errno)); goto sock_err; } @@ -258,13 +258,13 @@ static int tap_pktio_close(pktio_entry_t *pktio_entry) pkt_tap_t *tap = pkt_priv(pktio_entry); if (tap->fd != -1 && close(tap->fd) != 0) { - __odp_errno = errno; + _odp_errno = errno; ODP_ERR("close(tap->fd): %s\n", strerror(errno)); ret = -1; } if (tap->skfd != -1 && close(tap->skfd) != 0) { - __odp_errno = errno; + _odp_errno = errno; ODP_ERR("close(tap->skfd): %s\n", strerror(errno)); ret = -1; } @@ -289,8 +289,8 @@ static odp_packet_t pack_odp_pkt(pktio_entry_t *pktio_entry, const void *data, } } - num = packet_alloc_multi(pkt_priv(pktio_entry)->pool, - len + frame_offset, &pkt, 1); + num = _odp_packet_alloc_multi(pkt_priv(pktio_entry)->pool, + len + frame_offset, &pkt, 1); if (num != 1) return ODP_PACKET_INVALID; @@ -344,7 +344,7 @@ static int tap_pktio_recv(pktio_entry_t *pktio_entry, int index ODP_UNUSED, ts_val = odp_time_global(); if (retval < 0) { - __odp_errno = errno; + _odp_errno = errno; break; } @@ -374,7 +374,7 @@ static int tap_pktio_send_lockless(pktio_entry_t *pktio_entry, if (odp_unlikely(pkt_len > mtu)) { if (i == 0) { - __odp_errno = EMSGSIZE; + _odp_errno = EMSGSIZE; return -1; } break; @@ -391,7 +391,7 @@ static int tap_pktio_send_lockless(pktio_entry_t *pktio_entry, if (retval < 0) { if (i == 0 && SOCK_ERR_REPORT(errno)) { - __odp_errno = errno; + _odp_errno = errno; ODP_ERR("write(): %s\n", strerror(errno)); return -1; } @@ -399,7 +399,7 @@ static int tap_pktio_send_lockless(pktio_entry_t *pktio_entry, } else if ((uint32_t)retval != pkt_len) { ODP_ERR("sent partial ethernet packet\n"); if (i == 0) { - __odp_errno = EMSGSIZE; + _odp_errno = EMSGSIZE; return -1; } break; diff --git a/platform/linux-generic/test/Makefile.am b/platform/linux-generic/test/Makefile.am index d66f5ece3..1f7ae611e 100644 --- a/platform/linux-generic/test/Makefile.am +++ b/platform/linux-generic/test/Makefile.am @@ -89,3 +89,5 @@ clean-local: rm -f $(builddir)/$$f; \ done \ fi + +.NOTPARALLEL: diff --git a/platform/linux-generic/test/sched-basic.conf b/platform/linux-generic/test/sched-basic.conf new file mode 100644 index 000000000..57a8a772c --- /dev/null +++ b/platform/linux-generic/test/sched-basic.conf @@ -0,0 +1,8 @@ +# Mandatory fields +odp_implementation = "linux-generic" +config_file_version = "0.1.15" + +sched_basic: { + # Test scheduler with an odd spread value + prio_spread = 3 +} |