Port c6cc6d3a "linux-gen: time: inline arch cpu time on x86"

Signed-off-by: Matias Elo <matias.elo@nokia.com>

3 files changed, 152 insertions, 95 deletions

diff --git a/platform/linux-dpdk/Makefile.am b/platform/linux-dpdk/Makefile.am
index e4438c1e3..ea15e34ff 100644
--- a/platform/linux-dpdk/Makefile.am
+++ b/platform/linux-dpdk/Makefile.am
@@ -40,7 +40,8 @@ odpapiplatinclude_HEADERS = \
 		  include/odp/api/plat/strong_types.h \
 		  include/odp/api/plat/sync_inlines.h \
 		  include/odp/api/plat/thread_inlines.h \
-		  include/odp/api/plat/ticketlock_inlines.h
+		  include/odp/api/plat/ticketlock_inlines.h \
+		  include/odp/api/plat/time_inlines.h
 
 odpapiabiarchinclude_HEADERS += \
 		  include-abi/odp/api/abi/align.h \
@@ -80,7 +81,6 @@ odpapiabiarchinclude_HEADERS += \
 endif
 
 noinst_HEADERS = \
-		  arch/odp_arch_time_internal.h \
 		  ${top_srcdir}/platform/linux-generic/include/odp_align_internal.h \
 		  ${top_srcdir}/platform/linux-generic/include/odp_atomic_internal.h \
 		  include/odp_buffer_internal.h \
@@ -201,14 +201,16 @@ __LIB__libodp_linux_la_SOURCES += \
 			   odp_std_clib_api.c \
 			   ../linux-generic/odp_sync_api.c \
 			   ../linux-generic/odp_thread_api.c \
-			   ../linux-generic/odp_ticketlock_api.c
+			   ../linux-generic/odp_ticketlock_api.c \
+			   ../linux-generic/odp_time_api.c
 endif
 
 if ARCH_IS_ARM
 __LIB__libodp_linux_la_SOURCES += arch/default/odp_cpu_cycles.c \
 				 arch/default/odp_global_time.c \
 				 arch/default/odp_sysinfo_parse.c
-odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_inlines.h
+odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_inlines.h \
+				arch/default/odp/api/abi/cpu_time.h
 if !ODP_ABI_COMPAT
 odpapiabiarchinclude_HEADERS += arch/arm/odp/api/abi/cpu.h
 endif
@@ -221,7 +223,8 @@ if ARCH_IS_AARCH64
 __LIB__libodp_linux_la_SOURCES += arch/default/odp_cpu_cycles.c \
 				 arch/aarch64/odp_global_time.c \
 				 arch/default/odp_sysinfo_parse.c
-odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_inlines.h
+odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_inlines.h \
+				arch/default/odp/api/abi/cpu_time.h
 if !ODP_ABI_COMPAT
 odpapiabiarchinclude_HEADERS += arch/aarch64/odp/api/abi/cpu.h
 endif
@@ -234,7 +237,8 @@ if ARCH_IS_DEFAULT
 __LIB__libodp_linux_la_SOURCES += arch/default/odp_cpu_cycles.c \
 				 arch/default/odp_global_time.c \
 				 arch/default/odp_sysinfo_parse.c
-odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_inlines.h
+odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_inlines.h \
+				arch/default/odp/api/abi/cpu_time.h
 if !ODP_ABI_COMPAT
 odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu.h
 endif
@@ -245,7 +249,8 @@ if ARCH_IS_MIPS64
 __LIB__libodp_linux_la_SOURCES += arch/mips64/odp_cpu_cycles.c \
 				 arch/default/odp_global_time.c \
 				 arch/mips64/odp_sysinfo_parse.c
-odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_inlines.h
+odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_inlines.h \
+				arch/default/odp/api/abi/cpu_time.h
 if !ODP_ABI_COMPAT
 odpapiabiarchinclude_HEADERS += arch/mips64/odp/api/abi/cpu.h
 endif
@@ -256,7 +261,8 @@ if ARCH_IS_POWERPC
 __LIB__libodp_linux_la_SOURCES += arch/default/odp_cpu_cycles.c \
 				 arch/default/odp_global_time.c \
 				 arch/powerpc/odp_sysinfo_parse.c
-odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_inlines.h
+odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_inlines.h \
+				arch/default/odp/api/abi/cpu_time.h
 if !ODP_ABI_COMPAT
 odpapiabiarchinclude_HEADERS += arch/powerpc/odp/api/abi/cpu.h
 endif
@@ -268,7 +274,9 @@ __LIB__libodp_linux_la_SOURCES += arch/x86/cpu_flags.c \
 				 arch/x86/odp_cpu_cycles.c \
 				 arch/x86/odp_global_time.c \
 				 arch/x86/odp_sysinfo_parse.c
-odpapiabiarchinclude_HEADERS += arch/x86/odp/api/abi/cpu_inlines.h
+odpapiabiarchinclude_HEADERS += arch/x86/odp/api/abi/cpu_inlines.h \
+				arch/x86/odp/api/abi/cpu_rdtsc.h \
+				arch/x86/odp/api/abi/cpu_time.h
 if !ODP_ABI_COMPAT
 odpapiabiarchinclude_HEADERS += arch/x86/odp/api/abi/cpu.h
 endif
diff --git a/platform/linux-dpdk/include/odp/api/plat/time_inlines.h b/platform/linux-dpdk/include/odp/api/plat/time_inlines.h
new file mode 100644
index 000000000..6c14d6c40
--- /dev/null
+++ b/platform/linux-dpdk/include/odp/api/plat/time_inlines.h
@@ -0,0 +1,84 @@
+/* Copyright (c) 2018, Linaro Limited
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier:     BSD-3-Clause
+ */
+
+#ifndef ODP_PLAT_TIME_INLINES_H_
+#define ODP_PLAT_TIME_INLINES_H_
+
+#include <stdint.h>
+
+#include <odp/api/align.h>
+
+#include <odp/api/abi/cpu_time.h>
+
+/** @cond _ODP_HIDE_FROM_DOXYGEN_ */
+
+#define _ODP_TIMESPEC_SIZE 16
+
+typedef odp_time_t (*time_cur_fn)(void);
+typedef uint64_t (*time_res_fn)(void);
+
+typedef struct time_handler_ {
+	time_cur_fn     time_cur;
+	time_res_fn     time_res;
+
+} time_handler_t;
+
+typedef struct _odp_time_global_t {
+	/* Storage space for struct timespec. Posix headers are not included
+	 * here to avoid application exposure. */
+	uint8_t ODP_ALIGNED(_ODP_TIMESPEC_SIZE) timespec[_ODP_TIMESPEC_SIZE];
+
+	int             use_hw;
+	uint64_t        hw_start;
+	uint64_t        hw_freq_hz;
+	/* DPDK specific */
+	time_handler_t  handler;
+
+} _odp_time_global_t;
+
+extern _odp_time_global_t _odp_time_glob;
+
+odp_time_t _odp_timespec_cur(void);
+
+static inline odp_time_t _odp_time_cur_gen(void)
+{
+	if (_odp_time_glob.use_hw) {
+		odp_time_t time;
+
+		time.count = _odp_cpu_global_time() - _odp_time_glob.hw_start;
+		return time;
+	}
+
+	return _odp_timespec_cur();
+}
+
+static inline odp_time_t _odp_time_cur(void)
+{
+	return _odp_time_glob.handler.time_cur();
+}
+
+#ifndef _ODP_NO_INLINE
+	/* Inline functions by default */
+	#define _ODP_INLINE static inline
+	#define odp_time_local   __odp_time_local
+	#define odp_time_global  __odp_time_global
+#else
+	#define _ODP_INLINE
+#endif
+
+_ODP_INLINE odp_time_t odp_time_local(void)
+{
+	return _odp_time_cur();
+}
+
+_ODP_INLINE odp_time_t odp_time_global(void)
+{
+	return _odp_time_cur();
+}
+
+/** @endcond */
+
+#endif
diff --git a/platform/linux-dpdk/odp_time.c b/platform/linux-dpdk/odp_time.c
index f8e94f2b7..0e01802d0 100644
--- a/platform/linux-dpdk/odp_time.c
+++ b/platform/linux-dpdk/odp_time.c
@@ -4,40 +4,29 @@
  * SPDX-License-Identifier:     BSD-3-Clause
  */
 
+#include "config.h"
+
 #include <odp_posix_extensions.h>
 
 #include <time.h>
+#include <string.h>
+#include <inttypes.h>
+
 #include <odp/api/time.h>
 #include <odp/api/hints.h>
 #include <odp_debug_internal.h>
 #include <odp_init_internal.h>
-#include <odp_arch_time_internal.h>
-#include <string.h>
-#include <inttypes.h>
+#include <odp/api/plat/time_inlines.h>
 
 #include <rte_config.h>
 #include <rte_cycles.h>
 
-typedef uint64_t (*time_to_ns_fn) (odp_time_t time);
-typedef odp_time_t (*time_cur_fn)(void);
-typedef odp_time_t (*time_from_ns_fn) (uint64_t ns);
-typedef uint64_t (*time_res_fn)(void);
-
-typedef struct time_handler_ {
-	time_cur_fn    time_cur;
-	time_res_fn     time_res;
-} time_handler_t;
+ODP_STATIC_ASSERT(_ODP_TIMESPEC_SIZE >= (sizeof(struct timespec)),
+		  "_ODP_TIMESPEC_SIZE too small");
 
-typedef struct time_global_t {
-	struct timespec spec_start;
-	int             use_hw;
-	uint64_t        hw_start;
-	uint64_t        hw_freq_hz;
-	/* DPDK specific */
-	time_handler_t  handler;
-} time_global_t;
+#include <odp/visibility_begin.h>
 
-static time_global_t global;
+_odp_time_global_t _odp_time_glob;
 
 /*
  * Posix timespec based functions
@@ -62,21 +51,26 @@ static inline uint64_t time_spec_diff_nsec(struct timespec *t2,
 	return nsec;
 }
 
-static inline odp_time_t time_spec_cur(void)
+odp_time_t _odp_timespec_cur(void)
 {
 	int ret;
 	odp_time_t time;
 	struct timespec sys_time;
+	struct timespec *start_time;
+
+	start_time = (struct timespec *)(uintptr_t)&_odp_time_glob.timespec;
 
 	ret = clock_gettime(CLOCK_MONOTONIC_RAW, &sys_time);
 	if (odp_unlikely(ret != 0))
 		ODP_ABORT("clock_gettime failed\n");
 
-	time.nsec = time_spec_diff_nsec(&sys_time, &global.spec_start);
+	time.nsec = time_spec_diff_nsec(&sys_time, start_time);
 
 	return time;
 }
 
+#include <odp/visibility_end.h>
+
 static inline uint64_t time_spec_res(void)
 {
 	int ret;
@@ -107,24 +101,15 @@ static inline odp_time_t time_spec_from_ns(uint64_t ns)
  * HW time counter based functions
  */
 
-static inline odp_time_t time_hw_cur(void)
-{
-	odp_time_t time;
-
-	time.count = cpu_global_time() - global.hw_start;
-
-	return time;
-}
-
 static inline uint64_t time_hw_res(void)
 {
-	return global.hw_freq_hz;
+	return _odp_time_glob.hw_freq_hz;
 }
 
 static inline uint64_t time_hw_to_ns(odp_time_t time)
 {
 	uint64_t nsec;
-	uint64_t freq_hz = global.hw_freq_hz;
+	uint64_t freq_hz = _odp_time_glob.hw_freq_hz;
 	uint64_t count = time.count;
 	uint64_t sec = 0;
 
@@ -142,7 +127,7 @@ static inline odp_time_t time_hw_from_ns(uint64_t ns)
 {
 	odp_time_t time;
 	uint64_t count;
-	uint64_t freq_hz = global.hw_freq_hz;
+	uint64_t freq_hz = _odp_time_glob.hw_freq_hz;
 	uint64_t sec = 0;
 
 	if (ns >= ODP_TIME_SEC_IN_NS) {
@@ -162,17 +147,9 @@ static inline odp_time_t time_hw_from_ns(uint64_t ns)
  * Common functions
  */
 
-static inline odp_time_t time_cur(void)
-{
-	if (global.use_hw)
-		return time_hw_cur();
-
-	return time_spec_cur();
-}
-
 static inline uint64_t time_res(void)
 {
-	if (global.use_hw)
+	if (_odp_time_glob.use_hw)
 		return time_hw_res();
 
 	return time_spec_res();
@@ -200,7 +177,7 @@ static inline odp_time_t time_sum(odp_time_t t1, odp_time_t t2)
 
 static inline uint64_t time_to_ns(odp_time_t time)
 {
-	if (global.use_hw)
+	if (_odp_time_glob.use_hw)
 		return time_hw_to_ns(time);
 
 	return time_spec_to_ns(time);
@@ -208,29 +185,24 @@ static inline uint64_t time_to_ns(odp_time_t time)
 
 static inline odp_time_t time_from_ns(uint64_t ns)
 {
-	if (global.use_hw)
+	if (_odp_time_glob.use_hw)
 		return time_hw_from_ns(ns);
 
 	return time_spec_from_ns(ns);
 }
 
-static inline uint64_t time_res_dpdk(void)
-{
-	return rte_get_timer_hz();
-}
-
 static inline odp_time_t time_cur_dpdk(void)
 {
 	odp_time_t time;
 
-	time.u64 = rte_get_timer_cycles() - global.hw_start;
+	time.u64 = rte_get_timer_cycles() - _odp_time_glob.hw_start;
 
 	return time;
 }
 
-static inline odp_time_t time_local(void)
+static inline uint64_t time_res_dpdk(void)
 {
-	return global.handler.time_cur();
+	return rte_get_timer_hz();
 }
 
 static inline void time_wait_until(odp_time_t time)
@@ -238,20 +210,10 @@ static inline void time_wait_until(odp_time_t time)
 	odp_time_t cur;
 
 	do {
-		cur = time_local();
+		cur = _odp_time_cur();
 	} while (time_cmp(time, cur) > 0);
 }
 
-odp_time_t odp_time_local(void)
-{
-	return time_local();
-}
-
-odp_time_t odp_time_global(void)
-{
-	return time_local();
-}
-
 odp_time_t odp_time_diff(odp_time_t t2, odp_time_t t1)
 {
 	odp_time_t time;
@@ -297,17 +259,17 @@ odp_time_t odp_time_sum(odp_time_t t1, odp_time_t t2)
 
 uint64_t odp_time_local_res(void)
 {
-	return global.handler.time_res();
+	return _odp_time_glob.handler.time_res();
 }
 
 uint64_t odp_time_global_res(void)
 {
-	return global.handler.time_res();
+	return _odp_time_glob.handler.time_res();
 }
 
 void odp_time_wait_ns(uint64_t ns)
 {
-	odp_time_t cur = time_local();
+	odp_time_t cur = _odp_time_cur();
 	odp_time_t wait = time_from_ns(ns);
 	odp_time_t end_time = time_sum(cur, wait);
 
@@ -365,43 +327,46 @@ static inline odp_bool_t is_dpdk_timer_cycles_support(void)
 
 int odp_time_init_global(void)
 {
+	struct timespec *timespec;
 	int ret = 0;
+	_odp_time_global_t *global = &_odp_time_glob;
 
-	memset(&global, 0, sizeof(time_global_t));
+	memset(global, 0, sizeof(_odp_time_global_t));
 
 	if (is_dpdk_timer_cycles_support()) {
-		global.handler.time_cur     = time_cur_dpdk;
-		global.handler.time_res     = time_res_dpdk;
-		global.hw_freq_hz  = time_res_dpdk();
-		global.use_hw = 1;
-		global.hw_start = rte_get_timer_cycles();
-		if (global.hw_start == 0)
+		_odp_time_glob.handler.time_cur = time_cur_dpdk;
+		_odp_time_glob.handler.time_res = time_res_dpdk;
+		_odp_time_glob.hw_freq_hz = time_res_dpdk();
+		_odp_time_glob.use_hw = 1;
+		_odp_time_glob.hw_start = rte_get_timer_cycles();
+		if (_odp_time_glob.hw_start == 0)
 			return -1;
 		else
 			return 0;
 	}
 
-	global.handler.time_cur     = time_cur;
-	global.handler.time_res     = time_res;
+	_odp_time_glob.handler.time_cur = _odp_time_cur_gen;
+	_odp_time_glob.handler.time_res = time_res;
 
-	if (cpu_has_global_time()) {
-		global.use_hw = 1;
-		global.hw_freq_hz  = cpu_global_time_freq();
+	if (_odp_cpu_has_global_time()) {
+		global->use_hw = 1;
+		global->hw_freq_hz  = _odp_cpu_global_time_freq();
 
-		if (global.hw_freq_hz == 0)
+		if (global->hw_freq_hz == 0)
 			return -1;
 
 		printf("HW time counter freq: %" PRIu64 " hz\n\n",
-		       global.hw_freq_hz);
+		       global->hw_freq_hz);
 
-		global.hw_start = cpu_global_time();
+		global->hw_start = _odp_cpu_global_time();
 		return 0;
 	}
 
-	global.spec_start.tv_sec  = 0;
-	global.spec_start.tv_nsec = 0;
+	timespec = (struct timespec *)(uintptr_t)global->timespec;
+	timespec->tv_sec  = 0;
+	timespec->tv_nsec = 0;
 
-	ret = clock_gettime(CLOCK_MONOTONIC_RAW, &global.spec_start);
+	ret = clock_gettime(CLOCK_MONOTONIC_RAW, timespec);
 
 	return ret;
 }