linux-gen: sync: implement new memory barriers

Implement the new load and store synchronization barriers with
x86 and aarch64 instructions. The default implementation falls
back to the strongest GCC built-in (__sync_synchronize()), which
would be the strong enough on x86 (mfence), but too weak on
aarch64 (dmb).

Signed-off-by: Petri Savolainen <petri.savolainen@nokia.com>
Reviewed-by: Matias Elo <matias.elo@nokia.com>
Reviewed-by: Jere Leppänen <jere.leppanen@nokia.com>

5 files changed, 121 insertions, 4 deletions

diff --git a/platform/linux-generic/Makefile.am b/platform/linux-generic/Makefile.am
index 4c02c56fa..7afdbf620 100644
--- a/platform/linux-generic/Makefile.am
+++ b/platform/linux-generic/Makefile.am
@@ -339,6 +339,7 @@ odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \
 				arch/default/odp/api/abi/cpu_generic.h \
 				arch/arm/odp/api/abi/cpu_inlines.h \
 				arch/arm/odp/api/abi/cpu.h \
+				arch/default/odp/api/abi/sync_inlines.h \
 				arch/default/odp/api/abi/time_inlines.h
 endif
 noinst_HEADERS += arch/arm/odp_atomic.h \
@@ -367,6 +368,7 @@ odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \
 				arch/default/odp/api/abi/cpu_generic.h \
 				arch/aarch64/odp/api/abi/cpu_inlines.h \
 				arch/aarch64/odp/api/abi/cpu.h \
+				arch/aarch64/odp/api/abi/sync_inlines.h \
 				arch/common/odp/api/abi/time_cpu_inlines.h \
 				arch/aarch64/odp/api/abi/time_inlines.h
 endif
@@ -391,6 +393,7 @@ odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \
 				arch/default/odp/api/abi/cpu_generic.h \
 				arch/default/odp/api/abi/cpu_inlines.h \
 				arch/default/odp/api/abi/cpu.h \
+				arch/default/odp/api/abi/sync_inlines.h \
 				arch/default/odp/api/abi/time_inlines.h
 endif
 noinst_HEADERS += arch/default/odp_atomic.h \
@@ -412,6 +415,7 @@ odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \
 				arch/default/odp/api/abi/cpu_generic.h \
 				arch/default/odp/api/abi/cpu_inlines.h \
 				arch/powerpc/odp/api/abi/cpu.h \
+				arch/default/odp/api/abi/sync_inlines.h \
 				arch/default/odp/api/abi/time_inlines.h
 endif
 noinst_HEADERS += arch/default/odp_atomic.h \
@@ -436,6 +440,7 @@ odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \
 				arch/default/odp/api/abi/atomic_inlines.h \
 				arch/x86/odp/api/abi/cpu_inlines.h \
 				arch/x86/odp/api/abi/cpu.h \
+				arch/x86/odp/api/abi/sync_inlines.h \
 				arch/common/odp/api/abi/time_cpu_inlines.h \
 				arch/x86/odp/api/abi/time_inlines.h
 endif
diff --git a/platform/linux-generic/arch/aarch64/odp/api/abi/sync_inlines.h b/platform/linux-generic/arch/aarch64/odp/api/abi/sync_inlines.h
new file mode 100644
index 000000000..3d42e7dd8
--- /dev/null
+++ b/platform/linux-generic/arch/aarch64/odp/api/abi/sync_inlines.h
@@ -0,0 +1,31 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2023 Nokia
+ */
+
+#ifndef ODP_ARCH_SYNC_INLINES_H_
+#define ODP_ARCH_SYNC_INLINES_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+static inline void _odp_mb_sync(void)
+{
+	__asm__ volatile("dsb sy" ::: "memory");
+}
+
+static inline void _odp_mb_sync_load(void)
+{
+	__asm__ volatile("dsb ld" ::: "memory");
+}
+
+static inline void _odp_mb_sync_store(void)
+{
+	__asm__ volatile("dsb st" ::: "memory");
+}
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/platform/linux-generic/arch/default/odp/api/abi/sync_inlines.h b/platform/linux-generic/arch/default/odp/api/abi/sync_inlines.h
new file mode 100644
index 000000000..bfbb3039f
--- /dev/null
+++ b/platform/linux-generic/arch/default/odp/api/abi/sync_inlines.h
@@ -0,0 +1,31 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2023 Nokia
+ */
+
+#ifndef ODP_ARCH_SYNC_INLINES_H_
+#define ODP_ARCH_SYNC_INLINES_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+static inline void _odp_mb_sync(void)
+{
+	__sync_synchronize();
+}
+
+static inline void _odp_mb_sync_load(void)
+{
+	__sync_synchronize();
+}
+
+static inline void _odp_mb_sync_store(void)
+{
+	__sync_synchronize();
+}
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/platform/linux-generic/arch/x86/odp/api/abi/sync_inlines.h b/platform/linux-generic/arch/x86/odp/api/abi/sync_inlines.h
new file mode 100644
index 000000000..bebe6b571
--- /dev/null
+++ b/platform/linux-generic/arch/x86/odp/api/abi/sync_inlines.h
@@ -0,0 +1,31 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2023 Nokia
+ */
+
+#ifndef ODP_ARCH_SYNC_INLINES_H_
+#define ODP_ARCH_SYNC_INLINES_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+static inline void _odp_mb_sync(void)
+{
+	__asm__ volatile("mfence" ::: "memory");
+}
+
+static inline void _odp_mb_sync_load(void)
+{
+	__asm__ volatile("lfence" ::: "memory");
+}
+
+static inline void _odp_mb_sync_store(void)
+{
+	__asm__ volatile("sfence" ::: "memory");
+}
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/platform/linux-generic/include/odp/api/plat/sync_inlines.h b/platform/linux-generic/include/odp/api/plat/sync_inlines.h
index b6a96188c..b3a88b629 100644
--- a/platform/linux-generic/include/odp/api/plat/sync_inlines.h
+++ b/platform/linux-generic/include/odp/api/plat/sync_inlines.h
@@ -1,7 +1,6 @@
-/* Copyright (c) 2016-2018, Linaro Limited
- * All rights reserved.
- *
- * SPDX-License-Identifier:     BSD-3-Clause
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2016-2018 Linaro Limited
+ * Copyright (c) 2023 Nokia
  */
 
 /**
@@ -13,6 +12,8 @@
 #ifndef ODP_PLAT_SYNC_INLINE_H_
 #define ODP_PLAT_SYNC_INLINE_H_
 
+#include <odp/api/abi/sync_inlines.h>
+
 #ifdef __cplusplus
 extern "C" {
 #endif
@@ -25,6 +26,9 @@ extern "C" {
 	#define odp_mb_release __odp_mb_release
 	#define odp_mb_acquire __odp_mb_acquire
 	#define odp_mb_full __odp_mb_full
+	#define odp_mb_sync __odp_mb_sync
+	#define odp_mb_sync_load __odp_mb_sync_load
+	#define odp_mb_sync_store __odp_mb_sync_store
 #else
 	#define _ODP_INLINE
 #endif
@@ -44,6 +48,21 @@ _ODP_INLINE void odp_mb_full(void)
 	__atomic_thread_fence(__ATOMIC_SEQ_CST);
 }
 
+_ODP_INLINE void odp_mb_sync(void)
+{
+	_odp_mb_sync();
+}
+
+_ODP_INLINE void odp_mb_sync_load(void)
+{
+	_odp_mb_sync_load();
+}
+
+_ODP_INLINE void odp_mb_sync_store(void)
+{
+	_odp_mb_sync_store();
+}
+
 /** @endcond */
 
 #ifdef __cplusplus