From f42d23801f34dc0ca0aa0463bb92bf4fdc310d8e Mon Sep 17 00:00:00 2001 From: Petri Savolainen Date: Wed, 28 Jun 2023 10:48:24 +0300 Subject: linux-gen: sync: implement new memory barriers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implement the new load and store synchronization barriers with x86 and aarch64 instructions. The default implementation falls back to the strongest GCC built-in (__sync_synchronize()), which would be the strong enough on x86 (mfence), but too weak on aarch64 (dmb). Signed-off-by: Petri Savolainen Reviewed-by: Matias Elo Reviewed-by: Jere Leppänen --- platform/linux-generic/Makefile.am | 5 ++++ .../arch/aarch64/odp/api/abi/sync_inlines.h | 31 ++++++++++++++++++++++ .../arch/default/odp/api/abi/sync_inlines.h | 31 ++++++++++++++++++++++ .../arch/x86/odp/api/abi/sync_inlines.h | 31 ++++++++++++++++++++++ .../include/odp/api/plat/sync_inlines.h | 27 ++++++++++++++++--- 5 files changed, 121 insertions(+), 4 deletions(-) create mode 100644 platform/linux-generic/arch/aarch64/odp/api/abi/sync_inlines.h create mode 100644 platform/linux-generic/arch/default/odp/api/abi/sync_inlines.h create mode 100644 platform/linux-generic/arch/x86/odp/api/abi/sync_inlines.h diff --git a/platform/linux-generic/Makefile.am b/platform/linux-generic/Makefile.am index 4c02c56fa..7afdbf620 100644 --- a/platform/linux-generic/Makefile.am +++ b/platform/linux-generic/Makefile.am @@ -339,6 +339,7 @@ odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ arch/default/odp/api/abi/cpu_generic.h \ arch/arm/odp/api/abi/cpu_inlines.h \ arch/arm/odp/api/abi/cpu.h \ + arch/default/odp/api/abi/sync_inlines.h \ arch/default/odp/api/abi/time_inlines.h endif noinst_HEADERS += arch/arm/odp_atomic.h \ @@ -367,6 +368,7 @@ odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ arch/default/odp/api/abi/cpu_generic.h \ arch/aarch64/odp/api/abi/cpu_inlines.h \ arch/aarch64/odp/api/abi/cpu.h \ + arch/aarch64/odp/api/abi/sync_inlines.h \ arch/common/odp/api/abi/time_cpu_inlines.h \ arch/aarch64/odp/api/abi/time_inlines.h endif @@ -391,6 +393,7 @@ odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ arch/default/odp/api/abi/cpu_generic.h \ arch/default/odp/api/abi/cpu_inlines.h \ arch/default/odp/api/abi/cpu.h \ + arch/default/odp/api/abi/sync_inlines.h \ arch/default/odp/api/abi/time_inlines.h endif noinst_HEADERS += arch/default/odp_atomic.h \ @@ -412,6 +415,7 @@ odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ arch/default/odp/api/abi/cpu_generic.h \ arch/default/odp/api/abi/cpu_inlines.h \ arch/powerpc/odp/api/abi/cpu.h \ + arch/default/odp/api/abi/sync_inlines.h \ arch/default/odp/api/abi/time_inlines.h endif noinst_HEADERS += arch/default/odp_atomic.h \ @@ -436,6 +440,7 @@ odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ arch/default/odp/api/abi/atomic_inlines.h \ arch/x86/odp/api/abi/cpu_inlines.h \ arch/x86/odp/api/abi/cpu.h \ + arch/x86/odp/api/abi/sync_inlines.h \ arch/common/odp/api/abi/time_cpu_inlines.h \ arch/x86/odp/api/abi/time_inlines.h endif diff --git a/platform/linux-generic/arch/aarch64/odp/api/abi/sync_inlines.h b/platform/linux-generic/arch/aarch64/odp/api/abi/sync_inlines.h new file mode 100644 index 000000000..3d42e7dd8 --- /dev/null +++ b/platform/linux-generic/arch/aarch64/odp/api/abi/sync_inlines.h @@ -0,0 +1,31 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2023 Nokia + */ + +#ifndef ODP_ARCH_SYNC_INLINES_H_ +#define ODP_ARCH_SYNC_INLINES_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +static inline void _odp_mb_sync(void) +{ + __asm__ volatile("dsb sy" ::: "memory"); +} + +static inline void _odp_mb_sync_load(void) +{ + __asm__ volatile("dsb ld" ::: "memory"); +} + +static inline void _odp_mb_sync_store(void) +{ + __asm__ volatile("dsb st" ::: "memory"); +} + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/arch/default/odp/api/abi/sync_inlines.h b/platform/linux-generic/arch/default/odp/api/abi/sync_inlines.h new file mode 100644 index 000000000..bfbb3039f --- /dev/null +++ b/platform/linux-generic/arch/default/odp/api/abi/sync_inlines.h @@ -0,0 +1,31 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2023 Nokia + */ + +#ifndef ODP_ARCH_SYNC_INLINES_H_ +#define ODP_ARCH_SYNC_INLINES_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +static inline void _odp_mb_sync(void) +{ + __sync_synchronize(); +} + +static inline void _odp_mb_sync_load(void) +{ + __sync_synchronize(); +} + +static inline void _odp_mb_sync_store(void) +{ + __sync_synchronize(); +} + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/arch/x86/odp/api/abi/sync_inlines.h b/platform/linux-generic/arch/x86/odp/api/abi/sync_inlines.h new file mode 100644 index 000000000..bebe6b571 --- /dev/null +++ b/platform/linux-generic/arch/x86/odp/api/abi/sync_inlines.h @@ -0,0 +1,31 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2023 Nokia + */ + +#ifndef ODP_ARCH_SYNC_INLINES_H_ +#define ODP_ARCH_SYNC_INLINES_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +static inline void _odp_mb_sync(void) +{ + __asm__ volatile("mfence" ::: "memory"); +} + +static inline void _odp_mb_sync_load(void) +{ + __asm__ volatile("lfence" ::: "memory"); +} + +static inline void _odp_mb_sync_store(void) +{ + __asm__ volatile("sfence" ::: "memory"); +} + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/include/odp/api/plat/sync_inlines.h b/platform/linux-generic/include/odp/api/plat/sync_inlines.h index b6a96188c..b3a88b629 100644 --- a/platform/linux-generic/include/odp/api/plat/sync_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/sync_inlines.h @@ -1,7 +1,6 @@ -/* Copyright (c) 2016-2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2016-2018 Linaro Limited + * Copyright (c) 2023 Nokia */ /** @@ -13,6 +12,8 @@ #ifndef ODP_PLAT_SYNC_INLINE_H_ #define ODP_PLAT_SYNC_INLINE_H_ +#include + #ifdef __cplusplus extern "C" { #endif @@ -25,6 +26,9 @@ extern "C" { #define odp_mb_release __odp_mb_release #define odp_mb_acquire __odp_mb_acquire #define odp_mb_full __odp_mb_full + #define odp_mb_sync __odp_mb_sync + #define odp_mb_sync_load __odp_mb_sync_load + #define odp_mb_sync_store __odp_mb_sync_store #else #define _ODP_INLINE #endif @@ -44,6 +48,21 @@ _ODP_INLINE void odp_mb_full(void) __atomic_thread_fence(__ATOMIC_SEQ_CST); } +_ODP_INLINE void odp_mb_sync(void) +{ + _odp_mb_sync(); +} + +_ODP_INLINE void odp_mb_sync_load(void) +{ + _odp_mb_sync_load(); +} + +_ODP_INLINE void odp_mb_sync_store(void) +{ + _odp_mb_sync_store(); +} + /** @endcond */ #ifdef __cplusplus -- cgit v1.2.3