linux-gen: time: implement strict time stamp read functions

Implemented strict versions of local and global time stamp read
functions. Moved time register read code into inline functions
on ARMv8.

Signed-off-by: Petri Savolainen <petri.savolainen@nokia.com>
Reviewed-by: Matias Elo <matias.elo@nokia.com>
Reviewed-by: Jere Leppänen <jere.leppanen@nokia.com>

7 files changed, 104 insertions, 38 deletions

diff --git a/platform/linux-generic/Makefile.am b/platform/linux-generic/Makefile.am
index 2181abd26..e654381e6 100644
--- a/platform/linux-generic/Makefile.am
+++ b/platform/linux-generic/Makefile.am
@@ -293,7 +293,7 @@ __LIB__libodp_linux_la_SOURCES += arch/aarch64/odp_atomic.c \
 				  arch/default/odp_hash_crc32.c \
 				  arch/aarch64/odp_sysinfo_parse.c
 odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_inlines.h \
-				arch/default/odp/api/abi/cpu_time.h \
+				arch/aarch64/odp/api/abi/cpu_time.h \
 				arch/aarch64/odp/api/abi/hash_crc32.h
 if !ODP_ABI_COMPAT
 odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \
diff --git a/platform/linux-generic/arch/aarch64/odp/api/abi/cpu_time.h b/platform/linux-generic/arch/aarch64/odp/api/abi/cpu_time.h
new file mode 100644
index 000000000..781ee683c
--- /dev/null
+++ b/platform/linux-generic/arch/aarch64/odp/api/abi/cpu_time.h
@@ -0,0 +1,50 @@
+/* Copyright (c) 2021, Nokia
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier:     BSD-3-Clause
+ */
+
+#ifndef ODP_API_ABI_CPU_TIME_H_
+#define ODP_API_ABI_CPU_TIME_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <stdint.h>
+
+static inline uint64_t _odp_cpu_global_time(void)
+{
+	uint64_t cntvct;
+
+	__asm__ volatile("mrs %0, cntvct_el0" : "=r"(cntvct) : : "memory");
+
+	return cntvct;
+}
+
+static inline uint64_t _odp_cpu_global_time_strict(void)
+{
+	uint64_t cntvct;
+
+	__asm__ volatile("isb" ::: "memory");
+	__asm__ volatile("mrs %0, cntvct_el0" : "=r"(cntvct) : : "memory");
+
+	return cntvct;
+}
+
+static inline uint64_t _odp_cpu_global_time_freq(void)
+{
+	uint64_t cntfrq;
+
+	__asm__ volatile("mrs %0, cntfrq_el0" : "=r"(cntfrq) : : );
+
+	return cntfrq;
+}
+
+int _odp_cpu_has_global_time(void);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/platform/linux-generic/arch/aarch64/odp_global_time.c b/platform/linux-generic/arch/aarch64/odp_global_time.c
index fa59f5a15..53561b00c 100644
--- a/platform/linux-generic/arch/aarch64/odp_global_time.c
+++ b/platform/linux-generic/arch/aarch64/odp_global_time.c
@@ -4,33 +4,8 @@
  * SPDX-License-Identifier:     BSD-3-Clause
  */
 
-#include <odp_posix_extensions.h>
-
-#include <time.h>
-
-#include <odp_debug_internal.h>
 #include <odp/api/abi/cpu_time.h>
 
-#include <odp/visibility_begin.h>
-
-uint64_t _odp_cpu_global_time(void)
-{
-	uint64_t cntvct;
-
-	/*
-	 * To be consistent with other architectures, do not issue a
-	 * serializing instruction, e.g. ISB, before reading this
-	 * sys reg.
-	 */
-
-	/* Memory clobber to minimize optimization around load from sys reg. */
-	__asm__ volatile("mrs %0, cntvct_el0" : "=r"(cntvct) : : "memory");
-
-	return cntvct;
-}
-
-#include <odp/visibility_end.h>
-
 int _odp_cpu_has_global_time(void)
 {
 	uint64_t hz = _odp_cpu_global_time_freq();
@@ -48,12 +23,3 @@ int _odp_cpu_has_global_time(void)
 	 */
 	return hz >= 1000000 && hz <= 6000000000;
 }
-
-uint64_t _odp_cpu_global_time_freq(void)
-{
-	uint64_t cntfrq;
-
-	__asm__ volatile("mrs %0, cntfrq_el0" : "=r"(cntfrq) : : );
-
-	return cntfrq;
-}
diff --git a/platform/linux-generic/arch/default/odp/api/abi/cpu_time.h b/platform/linux-generic/arch/default/odp/api/abi/cpu_time.h
index 09138e70a..24e1c7d33 100644
--- a/platform/linux-generic/arch/default/odp/api/abi/cpu_time.h
+++ b/platform/linux-generic/arch/default/odp/api/abi/cpu_time.h
@@ -15,6 +15,7 @@ extern "C" {
 
 int _odp_cpu_has_global_time(void);
 uint64_t _odp_cpu_global_time(void);
+uint64_t _odp_cpu_global_time_strict(void);
 uint64_t _odp_cpu_global_time_freq(void);
 
 #ifdef __cplusplus
diff --git a/platform/linux-generic/arch/default/odp_global_time.c b/platform/linux-generic/arch/default/odp_global_time.c
index facffea7d..ee835413f 100644
--- a/platform/linux-generic/arch/default/odp_global_time.c
+++ b/platform/linux-generic/arch/default/odp_global_time.c
@@ -13,6 +13,11 @@ uint64_t _odp_cpu_global_time(void)
 	return 0;
 }
 
+uint64_t _odp_cpu_global_time_strict(void)
+{
+	return 0;
+}
+
 #include <odp/visibility_end.h>
 
 int _odp_cpu_has_global_time(void)
diff --git a/platform/linux-generic/arch/x86/odp/api/abi/cpu_time.h b/platform/linux-generic/arch/x86/odp/api/abi/cpu_time.h
index 05ff0db94..c74c4d606 100644
--- a/platform/linux-generic/arch/x86/odp/api/abi/cpu_time.h
+++ b/platform/linux-generic/arch/x86/odp/api/abi/cpu_time.h
@@ -19,6 +19,12 @@ static inline uint64_t _odp_cpu_global_time(void)
 	return _odp_cpu_rdtsc();
 }
 
+static inline uint64_t _odp_cpu_global_time_strict(void)
+{
+	__atomic_thread_fence(__ATOMIC_SEQ_CST);
+	return _odp_cpu_rdtsc();
+}
+
 int _odp_cpu_has_global_time(void);
 uint64_t _odp_cpu_global_time_freq(void);
 
diff --git a/platform/linux-generic/include/odp/api/plat/time_inlines.h b/platform/linux-generic/include/odp/api/plat/time_inlines.h
index 1ca495065..bb2913532 100644
--- a/platform/linux-generic/include/odp/api/plat/time_inlines.h
+++ b/platform/linux-generic/include/odp/api/plat/time_inlines.h
@@ -1,5 +1,5 @@
 /* Copyright (c) 2018, Linaro Limited
- * Copyright (c) 2020, Nokia
+ * Copyright (c) 2020-2021, Nokia
  * All rights reserved.
  *
  * SPDX-License-Identifier:     BSD-3-Clause
@@ -46,6 +46,18 @@ static inline odp_time_t _odp_time_cur(void)
 	return _odp_timespec_cur();
 }
 
+static inline odp_time_t _odp_time_cur_strict(void)
+{
+	if (_odp_time_glob.use_hw) {
+		odp_time_t time;
+
+		time.count = _odp_cpu_global_time_strict() - _odp_time_glob.hw_start;
+		return time;
+	}
+
+	return _odp_timespec_cur();
+}
+
 static inline uint64_t _odp_time_hw_to_ns(odp_time_t time)
 {
 	uint64_t nsec;
@@ -79,6 +91,12 @@ static inline uint64_t _odp_time_convert_to_ns(odp_time_t time)
 	#define odp_time_to_ns      __odp_time_to_ns
 	#define odp_time_local_ns   __odp_time_local_ns
 	#define odp_time_global_ns  __odp_time_global_ns
+
+	#define odp_time_local_strict      __odp_time_local_strict
+	#define odp_time_global_strict     __odp_time_global_strict
+	#define odp_time_local_strict_ns   __odp_time_local_strict_ns
+	#define odp_time_global_strict_ns  __odp_time_global_strict_ns
+
 	#define odp_time_cmp        __odp_time_cmp
 	#define odp_time_diff       __odp_time_diff
 	#define odp_time_sum        __odp_time_sum
@@ -97,9 +115,14 @@ _ODP_INLINE odp_time_t odp_time_global(void)
 	return _odp_time_cur();
 }
 
-_ODP_INLINE uint64_t odp_time_to_ns(odp_time_t time)
+_ODP_INLINE odp_time_t odp_time_local_strict(void)
 {
-	return _odp_time_convert_to_ns(time);
+	return _odp_time_cur_strict();
+}
+
+_ODP_INLINE odp_time_t odp_time_global_strict(void)
+{
+	return _odp_time_cur_strict();
 }
 
 _ODP_INLINE uint64_t odp_time_local_ns(void)
@@ -112,6 +135,21 @@ _ODP_INLINE uint64_t odp_time_global_ns(void)
 	return _odp_time_convert_to_ns(_odp_time_cur());
 }
 
+_ODP_INLINE uint64_t odp_time_local_strict_ns(void)
+{
+	return _odp_time_convert_to_ns(_odp_time_cur_strict());
+}
+
+_ODP_INLINE uint64_t odp_time_global_strict_ns(void)
+{
+	return _odp_time_convert_to_ns(_odp_time_cur_strict());
+}
+
+_ODP_INLINE uint64_t odp_time_to_ns(odp_time_t time)
+{
+	return _odp_time_convert_to_ns(time);
+}
+
 _ODP_INLINE int odp_time_cmp(odp_time_t t2, odp_time_t t1)
 {
 	if (odp_likely(t2.u64 > t1.u64))