diff options
21 files changed, 140 insertions, 59 deletions
diff --git a/Makefile.inc b/Makefile.inc index c887b2ee2..f9c4cae4e 100644 --- a/Makefile.inc +++ b/Makefile.inc @@ -9,6 +9,7 @@ else ODP_INCLUDES += \ -I$(top_srcdir)/platform/@with_platform@/include \ -I$(top_srcdir)/platform/@with_platform@/arch/@ARCH_DIR@ \ + -I$(top_srcdir)/platform/@with_platform@/arch/default \ -I$(top_srcdir)/platform/@with_platform@/include-abi endif diff --git a/include/odp/api/spec/cpu.h b/include/odp/api/spec/cpu.h index d107f2838..f6a7e656b 100644 --- a/include/odp/api/spec/cpu.h +++ b/include/odp/api/spec/cpu.h @@ -161,14 +161,13 @@ uint64_t odp_cpu_cycles_max(void); uint64_t odp_cpu_cycles_resolution(void); /** - * @def odp_cpu_pause * Pause CPU execution for a short while * * This call is intended for tight loops which poll a shared resource. A short * pause within the loop may save energy and improve system performance as * CPU polling frequency is reduced. */ -/* void odp_cpu_pause(void); */ +void odp_cpu_pause(void); /** * @} diff --git a/include/odp/arch/x86_32-linux/odp/api/abi/cpu.h b/include/odp/arch/x86_32-linux/odp/api/abi/cpu.h index 523fa99e5..d7485c090 100644 --- a/include/odp/arch/x86_32-linux/odp/api/abi/cpu.h +++ b/include/odp/arch/x86_32-linux/odp/api/abi/cpu.h @@ -13,15 +13,6 @@ extern "C" { #define ODP_CACHE_LINE_SIZE 64 -static inline void odp_cpu_pause(void) -{ -#ifdef __SSE2__ - __asm__ __volatile__ ("pause"); -#else - __asm__ __volatile__ ("rep; nop"); -#endif -} - #ifdef __cplusplus } #endif diff --git a/include/odp/arch/x86_64-linux/odp/api/abi/cpu.h b/include/odp/arch/x86_64-linux/odp/api/abi/cpu.h index d4fcf8927..69b82eb05 100644 --- a/include/odp/arch/x86_64-linux/odp/api/abi/cpu.h +++ b/include/odp/arch/x86_64-linux/odp/api/abi/cpu.h @@ -13,11 +13,6 @@ extern "C" { #define ODP_CACHE_LINE_SIZE 64 -static inline void odp_cpu_pause(void) -{ - __asm__ __volatile__ ("pause"); -} - #ifdef __cplusplus } #endif diff --git a/platform/linux-generic/Makefile.am b/platform/linux-generic/Makefile.am index 6269e5783..218138f90 100644 --- a/platform/linux-generic/Makefile.am +++ b/platform/linux-generic/Makefile.am @@ -20,12 +20,16 @@ DISTCLEANFILES = include/odp_libconfig_config.h include/odp_libconfig_config.h: $(top_srcdir)/config/odp-$(with_platform).conf $(top_builddir)/config.status cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ +odpapiabiarchincludedir = $(archincludedir)/odp/api/abi +odpapiabiarchinclude_HEADERS = + if !ODP_ABI_COMPAT odpapiplatincludedir= $(includedir)/odp/api/plat odpapiplatinclude_HEADERS = \ include/odp/api/plat/atomic_inlines.h \ include/odp/api/plat/buffer_inline_types.h \ include/odp/api/plat/byteorder_inlines.h \ + include/odp/api/plat/cpu_inlines.h \ include/odp/api/plat/event_inlines.h \ include/odp/api/plat/packet_flag_inlines.h \ include/odp/api/plat/packet_inline_types.h \ @@ -38,8 +42,7 @@ odpapiplatinclude_HEADERS = \ include/odp/api/plat/thread_inlines.h \ include/odp/api/plat/ticketlock_inlines.h -odpapiabiarchincludedir = $(archincludedir)/odp/api/abi -odpapiabiarchinclude_HEADERS = \ +odpapiabiarchinclude_HEADERS += \ include-abi/odp/api/abi/align.h \ include-abi/odp/api/abi/atomic.h \ include-abi/odp/api/abi/barrier.h \ @@ -214,6 +217,7 @@ if ODP_ABI_COMPAT __LIB__libodp_linux_la_SOURCES += \ odp_atomic_api.c \ odp_byteorder_api.c \ + odp_cpu_api.c \ odp_event_api.c \ odp_packet_api.c \ odp_packet_flags_api.c \ @@ -228,32 +232,33 @@ if ARCH_IS_ARM __LIB__libodp_linux_la_SOURCES += arch/default/odp_cpu_cycles.c \ arch/default/odp_global_time.c \ arch/default/odp_sysinfo_parse.c +odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_inlines.h if !ODP_ABI_COMPAT odpapiabiarchinclude_HEADERS += arch/arm/odp/api/abi/cpu.h endif noinst_HEADERS += arch/arm/odp_atomic.h \ arch/arm/odp_cpu.h \ arch/arm/odp_cpu_idling.h \ - arch/default/odp_cpu_idling.h \ arch/arm/odp_llsc.h endif if ARCH_IS_AARCH64 __LIB__libodp_linux_la_SOURCES += arch/default/odp_cpu_cycles.c \ arch/aarch64/odp_global_time.c \ arch/default/odp_sysinfo_parse.c +odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_inlines.h if !ODP_ABI_COMPAT odpapiabiarchinclude_HEADERS += arch/aarch64/odp/api/abi/cpu.h endif noinst_HEADERS += arch/aarch64/odp_atomic.h \ arch/aarch64/odp_cpu.h \ arch/aarch64/odp_cpu_idling.h \ - arch/default/odp_cpu_idling.h \ arch/aarch64/odp_llsc.h endif if ARCH_IS_DEFAULT __LIB__libodp_linux_la_SOURCES += arch/default/odp_cpu_cycles.c \ arch/default/odp_global_time.c \ arch/default/odp_sysinfo_parse.c +odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_inlines.h if !ODP_ABI_COMPAT odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu.h endif @@ -264,6 +269,7 @@ if ARCH_IS_MIPS64 __LIB__libodp_linux_la_SOURCES += arch/mips64/odp_cpu_cycles.c \ arch/default/odp_global_time.c \ arch/mips64/odp_sysinfo_parse.c +odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_inlines.h if !ODP_ABI_COMPAT odpapiabiarchinclude_HEADERS += arch/mips64/odp/api/abi/cpu.h endif @@ -274,6 +280,7 @@ if ARCH_IS_POWERPC __LIB__libodp_linux_la_SOURCES += arch/default/odp_cpu_cycles.c \ arch/default/odp_global_time.c \ arch/powerpc/odp_sysinfo_parse.c +odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_inlines.h if !ODP_ABI_COMPAT odpapiabiarchinclude_HEADERS += arch/powerpc/odp/api/abi/cpu.h endif @@ -285,11 +292,12 @@ __LIB__libodp_linux_la_SOURCES += arch/x86/cpu_flags.c \ arch/x86/odp_cpu_cycles.c \ arch/x86/odp_global_time.c \ arch/x86/odp_sysinfo_parse.c +odpapiabiarchinclude_HEADERS += arch/x86/odp/api/abi/cpu_inlines.h if !ODP_ABI_COMPAT odpapiabiarchinclude_HEADERS += arch/x86/odp/api/abi/cpu.h endif -noinst_HEADERS += arch/x86/cpu_flags.h -noinst_HEADERS += arch/default/odp_cpu.h \ +noinst_HEADERS += arch/x86/cpu_flags.h \ + arch/default/odp_cpu.h \ arch/default/odp_cpu_idling.h endif diff --git a/platform/linux-generic/arch/default/odp/api/abi/cpu_inlines.h b/platform/linux-generic/arch/default/odp/api/abi/cpu_inlines.h new file mode 100644 index 000000000..f9c2bae9d --- /dev/null +++ b/platform/linux-generic/arch/default/odp/api/abi/cpu_inlines.h @@ -0,0 +1,15 @@ +/* Copyright (c) 2018, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_ARCH_CPU_INLINES_H_ +#define ODP_ARCH_CPU_INLINES_H_ + +#undef odp_cpu_pause +#undef odp_cpu_cycles +#undef odp_cpu_cycles_max +#undef odp_cpu_cycles_resolution + +#endif diff --git a/platform/linux-generic/arch/x86/odp/api/abi/cpu.h b/platform/linux-generic/arch/x86/odp/api/abi/cpu.h index 523fa99e5..9224af9a0 100644 --- a/platform/linux-generic/arch/x86/odp/api/abi/cpu.h +++ b/platform/linux-generic/arch/x86/odp/api/abi/cpu.h @@ -13,14 +13,8 @@ extern "C" { #define ODP_CACHE_LINE_SIZE 64 -static inline void odp_cpu_pause(void) -{ -#ifdef __SSE2__ - __asm__ __volatile__ ("pause"); -#else - __asm__ __volatile__ ("rep; nop"); -#endif -} +/* Inlined functions for non-ABI compat mode */ +#include <odp/api/plat/cpu_inlines.h> #ifdef __cplusplus } diff --git a/platform/linux-generic/arch/x86/odp/api/abi/cpu_inlines.h b/platform/linux-generic/arch/x86/odp/api/abi/cpu_inlines.h new file mode 100644 index 000000000..63be3da65 --- /dev/null +++ b/platform/linux-generic/arch/x86/odp/api/abi/cpu_inlines.h @@ -0,0 +1,48 @@ +/* Copyright (c) 2018, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_ARCH_CPU_INLINES_H_ +#define ODP_ARCH_CPU_INLINES_H_ + +#include <stdint.h> + +_ODP_INLINE void odp_cpu_pause(void) +{ +#ifdef __SSE2__ + __asm__ __volatile__ ("pause"); +#else + __asm__ __volatile__ ("rep; nop"); +#endif +} + +_ODP_INLINE uint64_t odp_cpu_cycles(void) +{ + union { + uint64_t tsc_64; + struct { + uint32_t lo_32; + uint32_t hi_32; + }; + } tsc; + + __asm__ __volatile__ ("rdtsc" : + "=a" (tsc.lo_32), + "=d" (tsc.hi_32) : : "memory"); + + return tsc.tsc_64; +} + +_ODP_INLINE uint64_t odp_cpu_cycles_max(void) +{ + return UINT64_MAX; +} + +_ODP_INLINE uint64_t odp_cpu_cycles_resolution(void) +{ + return 1; +} + +#endif diff --git a/platform/linux-generic/arch/x86/odp_cpu_cycles.c b/platform/linux-generic/arch/x86/odp_cpu_cycles.c index 4e50b6da0..c7d97a764 100644 --- a/platform/linux-generic/arch/x86/odp_cpu_cycles.c +++ b/platform/linux-generic/arch/x86/odp_cpu_cycles.c @@ -10,33 +10,6 @@ #include <odp_init_internal.h> #include <odp_debug_internal.h> -uint64_t odp_cpu_cycles(void) -{ - union { - uint64_t tsc_64; - struct { - uint32_t lo_32; - uint32_t hi_32; - }; - } tsc; - - __asm__ __volatile__ ("rdtsc" : - "=a" (tsc.lo_32), - "=d" (tsc.hi_32) : : "memory"); - - return tsc.tsc_64; -} - -uint64_t odp_cpu_cycles_max(void) -{ - return UINT64_MAX; -} - -uint64_t odp_cpu_cycles_resolution(void) -{ - return 1; -} - int _odp_cpu_cycles_init_global(void) { if (cpu_flags_has_rdtsc() == 0) { diff --git a/platform/linux-generic/arch/x86/odp_global_time.c b/platform/linux-generic/arch/x86/odp_global_time.c index 70a1a1a7a..da22d45a5 100644 --- a/platform/linux-generic/arch/x86/odp_global_time.c +++ b/platform/linux-generic/arch/x86/odp_global_time.c @@ -15,6 +15,9 @@ #include <odp_debug_internal.h> #include <odp_arch_time_internal.h> +/* Inlined API functions */ +#include <odp/api/plat/cpu_inlines.h> + uint64_t cpu_global_time(void) { return odp_cpu_cycles(); diff --git a/platform/linux-generic/include/odp/api/plat/cpu_inlines.h b/platform/linux-generic/include/odp/api/plat/cpu_inlines.h new file mode 100644 index 000000000..29b6ca7a1 --- /dev/null +++ b/platform/linux-generic/include/odp/api/plat/cpu_inlines.h @@ -0,0 +1,27 @@ +/* Copyright (c) 2018, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_PLAT_CPU_INLINES_H_ +#define ODP_PLAT_CPU_INLINES_H_ + +/** @cond _ODP_HIDE_FROM_DOXYGEN_ */ + +#ifndef _ODP_NO_INLINE + /* Inline functions by default */ + #define _ODP_INLINE static inline + #define odp_cpu_pause __odp_cpu_pause + #define odp_cpu_cycles __odp_cpu_cycles + #define odp_cpu_cycles_max __odp_cpu_cycles_max + #define odp_cpu_cycles_resolution __odp_cpu_cycles_resolution +#else + #define _ODP_INLINE +#endif + +#include <odp/api/abi/cpu_inlines.h> + +/** @endcond */ + +#endif diff --git a/platform/linux-generic/include/odp/api/plat/ticketlock_inlines.h b/platform/linux-generic/include/odp/api/plat/ticketlock_inlines.h index e84ed88ed..1d944bc13 100644 --- a/platform/linux-generic/include/odp/api/plat/ticketlock_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/ticketlock_inlines.h @@ -8,6 +8,7 @@ #define _ODP_PLAT_TICKETLOCK_INLINES_H_ #include <odp/api/atomic.h> +#include <odp/api/cpu.h> /** @cond _ODP_HIDE_FROM_DOXYGEN_ */ @@ -21,6 +22,7 @@ #define odp_ticketlock_is_locked __odp_ticketlock_is_locked /* Inline atomic functions */ #include <odp/api/plat/atomic_inlines.h> + #include <odp/api/plat/cpu_inlines.h> #else #undef _ODP_INLINE #define _ODP_INLINE diff --git a/platform/linux-generic/include/odp_ring_internal.h b/platform/linux-generic/include/odp_ring_internal.h index b47c41e5b..8a01d4619 100644 --- a/platform/linux-generic/include/odp_ring_internal.h +++ b/platform/linux-generic/include/odp_ring_internal.h @@ -15,6 +15,7 @@ extern "C" { #include <odp/api/hints.h> #include <odp_align_internal.h> #include <odp/api/plat/atomic_inlines.h> +#include <odp/api/plat/cpu_inlines.h> /* Ring empty, not a valid data value. */ #define RING_EMPTY ((uint32_t)-1) diff --git a/platform/linux-generic/odp_barrier.c b/platform/linux-generic/odp_barrier.c index ec35d0ff4..132ba370e 100644 --- a/platform/linux-generic/odp_barrier.c +++ b/platform/linux-generic/odp_barrier.c @@ -12,6 +12,7 @@ #include <odp/api/atomic.h> #include <odp/api/plat/atomic_inlines.h> #include <odp/api/plat/sync_inlines.h> +#include <odp/api/plat/cpu_inlines.h> void odp_barrier_init(odp_barrier_t *barrier, int count) { diff --git a/platform/linux-generic/odp_cpu_api.c b/platform/linux-generic/odp_cpu_api.c new file mode 100644 index 000000000..2e856ec25 --- /dev/null +++ b/platform/linux-generic/odp_cpu_api.c @@ -0,0 +1,13 @@ +/* Copyright (c) 2018, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include "config.h" + +#include <odp/api/cpu.h> + +/* Non-inlined functions for ABI compat mode */ +#define _ODP_NO_INLINE +#include <odp/api/plat/cpu_inlines.h> diff --git a/platform/linux-generic/odp_ipsec_sad.c b/platform/linux-generic/odp_ipsec_sad.c index 061b45f36..3b1686b86 100644 --- a/platform/linux-generic/odp_ipsec_sad.c +++ b/platform/linux-generic/odp_ipsec_sad.c @@ -7,7 +7,6 @@ #include "config.h" #include <odp/api/atomic.h> -#include <odp/api/plat/atomic_inlines.h> #include <odp/api/ipsec.h> #include <odp/api/random.h> #include <odp/api/shared_memory.h> @@ -16,6 +15,9 @@ #include <odp_debug_internal.h> #include <odp_ipsec_internal.h> +#include <odp/api/plat/atomic_inlines.h> +#include <odp/api/plat/cpu_inlines.h> + #include <string.h> #define IPSEC_SA_STATE_DISABLE 0x40000000 diff --git a/platform/linux-generic/odp_rwlock.c b/platform/linux-generic/odp_rwlock.c index 3a1086d21..e027a23b5 100644 --- a/platform/linux-generic/odp_rwlock.c +++ b/platform/linux-generic/odp_rwlock.c @@ -8,10 +8,12 @@ #include <stdbool.h> #include <odp/api/atomic.h> -#include <odp/api/plat/atomic_inlines.h> #include <odp/api/rwlock.h> #include <odp/api/cpu.h> +#include <odp/api/plat/atomic_inlines.h> +#include <odp/api/plat/cpu_inlines.h> + void odp_rwlock_init(odp_rwlock_t *rwlock) { odp_atomic_init_u32(&rwlock->cnt, 0); diff --git a/platform/linux-generic/odp_schedule_scalable.c b/platform/linux-generic/odp_schedule_scalable.c index 9a0dea475..ff24815c0 100644 --- a/platform/linux-generic/odp_schedule_scalable.c +++ b/platform/linux-generic/odp_schedule_scalable.c @@ -25,6 +25,7 @@ #include <odp_ishmpool_internal.h> #include <odp_align_internal.h> +#include <odp/api/plat/cpu_inlines.h> #include <odp_llqueue.h> #include <odp_queue_scalable_internal.h> #include <odp_schedule_if.h> diff --git a/platform/linux-generic/odp_schedule_scalable_ordered.c b/platform/linux-generic/odp_schedule_scalable_ordered.c index 2b3feaa13..f68a559d3 100644 --- a/platform/linux-generic/odp_schedule_scalable_ordered.c +++ b/platform/linux-generic/odp_schedule_scalable_ordered.c @@ -8,6 +8,7 @@ #include <config.h> #include <odp/api/shared_memory.h> +#include <odp/api/plat/cpu_inlines.h> #include <odp_queue_scalable_internal.h> #include <odp_schedule_if.h> #include <odp_bitset.h> diff --git a/platform/linux-generic/odp_spinlock.c b/platform/linux-generic/odp_spinlock.c index 2e51aabc8..54b7c713b 100644 --- a/platform/linux-generic/odp_spinlock.c +++ b/platform/linux-generic/odp_spinlock.c @@ -10,6 +10,8 @@ #include <odp/api/cpu.h> #include <odp_atomic_internal.h> +#include <odp/api/plat/cpu_inlines.h> + void odp_spinlock_init(odp_spinlock_t *spinlock) { _odp_atomic_flag_init(&spinlock->lock, 0); diff --git a/platform/linux-generic/pktio/ring.c b/platform/linux-generic/pktio/ring.c index 778c2fac2..518d940e1 100644 --- a/platform/linux-generic/pktio/ring.c +++ b/platform/linux-generic/pktio/ring.c @@ -80,6 +80,8 @@ #include <odp_packet_io_ring_internal.h> #include <odp_errno_define.h> +#include <odp/api/plat/cpu_inlines.h> + static TAILQ_HEAD(, _ring) odp_ring_list; #define RING_VAL_IS_POWER_2(x) ((((x) - 1) & (x)) == 0) |