diff options
-rw-r--r-- | platform/linux-generic/Makefile.am | 3 | ||||
-rw-r--r-- | platform/linux-generic/arch/aarch64/odp/api/abi/atomic.h | 12 | ||||
-rw-r--r-- | platform/linux-generic/arch/aarch64/odp_atomic.h | 53 | ||||
-rw-r--r-- | platform/linux-generic/include-abi/odp/api/abi/atomic.h | 25 | ||||
-rw-r--r-- | platform/linux-generic/odp_atomic.c | 280 |
5 files changed, 370 insertions, 3 deletions
diff --git a/platform/linux-generic/Makefile.am b/platform/linux-generic/Makefile.am index 00ea764de..1d9a3bbdb 100644 --- a/platform/linux-generic/Makefile.am +++ b/platform/linux-generic/Makefile.am @@ -290,7 +290,8 @@ odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_inlines.h \ arch/default/odp/api/abi/cpu_time.h \ arch/aarch64/odp/api/abi/hash_crc32.h if !ODP_ABI_COMPAT -odpapiabiarchinclude_HEADERS += arch/aarch64/odp/api/abi/cpu.h +odpapiabiarchinclude_HEADERS += arch/aarch64/odp/api/abi/atomic.h \ + arch/aarch64/odp/api/abi/cpu.h endif noinst_HEADERS += arch/aarch64/odp_atomic.h \ arch/aarch64/odp_cpu.h \ diff --git a/platform/linux-generic/arch/aarch64/odp/api/abi/atomic.h b/platform/linux-generic/arch/aarch64/odp/api/abi/atomic.h new file mode 100644 index 000000000..d1dbf36b8 --- /dev/null +++ b/platform/linux-generic/arch/aarch64/odp/api/abi/atomic.h @@ -0,0 +1,12 @@ +/* Copyright (c) 2021, ARM Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifdef __ARM_FEATURE_ATOMICS +#define _ODP_LOCK_FREE_128BIT_ATOMICS +#endif + +#include <odp/api/abi-default/atomic.h> + diff --git a/platform/linux-generic/arch/aarch64/odp_atomic.h b/platform/linux-generic/arch/aarch64/odp_atomic.h index 8a0e7ce2b..dbeccebde 100644 --- a/platform/linux-generic/arch/aarch64/odp_atomic.h +++ b/platform/linux-generic/arch/aarch64/odp_atomic.h @@ -1,5 +1,4 @@ -/* Copyright (c) 2017, ARM Limited. All rights reserved. - * +/* Copyright (c) 2017-2021, ARM Limited * Copyright (c) 2017-2018, Linaro Limited * All rights reserved. * @@ -218,4 +217,54 @@ static inline __int128 __lockfree_load_16(__int128 *var, int mo) return old; } +#ifdef _ODP_LOCK_FREE_128BIT_ATOMICS + +/** + * @internal + * Helper macro for lockless atomic CAS operations on 128-bit integers + * @param[in,out] atom Pointer to the 128-bit atomic variable + * @param oper CAS operation + * @param old_val Old value + * @param new_val New value to be swapped + * @return 1 for success and 0 for fail + */ +#define ATOMIC_CAS_OP_128(atom, oper, old_val, new_val, val) \ +({ \ + odp_u128_t _val; \ + odp_atomic_u128_t *_atom = atom; \ + odp_u128_t *_old_val = old_val; \ + odp_u128_t _new_val = new_val; \ + odp_u128_t *ptr = (odp_u128_t *)(_atom); \ + register uint64_t old0 __asm__ ("x0"); \ + register uint64_t old1 __asm__ ("x1"); \ + register uint64_t new0 __asm__ ("x2"); \ + register uint64_t new1 __asm__ ("x3"); \ + old0 = (uint64_t)(_old_val)->u64[0]; \ + old1 = (uint64_t)(_old_val)->u64[1]; \ + new0 = (uint64_t)(_new_val).u64[0]; \ + new1 = (uint64_t)(_new_val).u64[1]; \ + __asm__ volatile(oper " %[old0], %[old1], %[new0], %[new1], [%[ptr]]" \ + : [old0] "+r" (old0), [old1] "+r" (old1) \ + : [new0] "r" (new0), [new1] "r" (new1), \ + [ptr] "r" (ptr) \ + : "memory"); \ + _val.u64[0] = old0; \ + _val.u64[1] = old1; \ + val = _val; \ +}) + +#define ATOMIC_CAS_OP_128_NO_ORDER(atom, old_value, new_value, val) \ + ATOMIC_CAS_OP_128(atom, "casp", old_value, new_value, val) + +#define ATOMIC_CAS_OP_128_ACQ(atom, old_value, new_value, val) \ + ATOMIC_CAS_OP_128(atom, "caspa", old_value, new_value, val) + +#define ATOMIC_CAS_OP_128_REL(atom, old_value, new_value, val) \ + ATOMIC_CAS_OP_128(atom, "caspl", old_value, new_value, val) + +#define ATOMIC_CAS_OP_128_ACQ_REL(atom, old_value, new_value, val) \ + ATOMIC_CAS_OP_128(atom, "caspal", old_value, new_value, val) + +#endif + #endif /* PLATFORM_LINUXGENERIC_ARCH_ARM_ODP_ATOMIC_H */ diff --git a/platform/linux-generic/include-abi/odp/api/abi/atomic.h b/platform/linux-generic/include-abi/odp/api/abi/atomic.h index 955b99370..13c12a79f 100644 --- a/platform/linux-generic/include-abi/odp/api/abi/atomic.h +++ b/platform/linux-generic/include-abi/odp/api/abi/atomic.h @@ -55,6 +55,31 @@ typedef struct ODP_ALIGNED(sizeof(uint64_t)) odp_atomic_u64_s { #endif +#ifdef _ODP_LOCK_FREE_128BIT_ATOMICS + +/** + * @internal + * Atomic 128-bit unsigned integer + */ +typedef struct ODP_ALIGNED(sizeof(odp_u128_t)) odp_atomic_u128_s { + odp_u128_t v; /**< Actual storage for the atomic variable */ +} odp_atomic_u128_t; + +#else + +/** + * @internal + * Atomic 128-bit unsigned integer + */ +typedef struct ODP_ALIGNED(sizeof(odp_u128_t)) odp_atomic_u128_s { + odp_u128_t v; /**< Actual storage for the atomic variable */ + /* Some architectures do not support lock-free operations on 128-bit + * data types. We use a spin lock to ensure atomicity. */ + char lock; /**< Spin lock (if needed) used to ensure atomic access */ +} odp_atomic_u128_t; + +#endif + /** @ingroup odp_atomic * @{ */ diff --git a/platform/linux-generic/odp_atomic.c b/platform/linux-generic/odp_atomic.c index 32ddc95c2..59253c645 100644 --- a/platform/linux-generic/odp_atomic.c +++ b/platform/linux-generic/odp_atomic.c @@ -1,10 +1,12 @@ /* Copyright (c) 2015-2018, Linaro Limited + * Copyright (c) 2021, ARM Limited * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause */ #include <odp/api/atomic.h> +#include <odp_cpu.h> int odp_atomic_lock_free_u64(odp_atomic_op_t *atomic_op) { @@ -24,3 +26,281 @@ int odp_atomic_lock_free_u64(odp_atomic_op_t *atomic_op) return 2; #endif } + +int odp_atomic_lock_free_u128(odp_atomic_op_t *atomic_op) +{ +#ifdef _ODP_LOCK_FREE_128BIT_ATOMICS + if (atomic_op) { + atomic_op->all_bits = 0; + atomic_op->op.load = 1; + atomic_op->op.store = 1; + atomic_op->op.cas = 1; + } + + return 2; +#else + /* All operations have locks */ + if (atomic_op) + atomic_op->all_bits = 0; + + return 0; +#endif +} + +#ifdef _ODP_LOCK_FREE_128BIT_ATOMICS + +static void __atomic_init_u128(odp_atomic_u128_t *atom, odp_u128_t new_val) +{ + odp_u128_t old, val; + + old = atom->v; + + while (1) { + ATOMIC_CAS_OP_128_NO_ORDER(atom, &old, new_val, val); + + if ((val.u64[0] == old.u64[0]) && (val.u64[1] == old.u64[1])) + return; + + old = val; + } +} + +static odp_u128_t __atomic_load_u128(odp_atomic_u128_t *atom) +{ + odp_u128_t val, exp; + + exp.u64[0] = 0; + exp.u64[1] = 0; + ATOMIC_CAS_OP_128_NO_ORDER(atom, &exp, exp, val); + return val; +} + +static void __atomic_store_u128(odp_atomic_u128_t *atom, odp_u128_t new_val) +{ + odp_u128_t old, val; + + old = atom->v; + + while (1) { + ATOMIC_CAS_OP_128_NO_ORDER(atom, &old, new_val, val); + + if ((val.u64[0] == old.u64[0]) && (val.u64[1] == old.u64[1])) + return; + + old = val; + } +} + +static int __atomic_cas_u128(odp_atomic_u128_t *atom, + odp_u128_t *old_val, odp_u128_t new_val) +{ + int ret = 0; + odp_u128_t val; + + ATOMIC_CAS_OP_128_NO_ORDER(atom, old_val, new_val, val); + + if ((val.u64[0] == old_val->u64[0]) && (val.u64[1] == old_val->u64[1])) + ret = 1; + + old_val->u64[0] = val.u64[0]; + old_val->u64[1] = val.u64[1]; + + return ret; +} + +static int __atomic_cas_acq_u128(odp_atomic_u128_t *atom, + odp_u128_t *old_val, odp_u128_t new_val) +{ + int ret = 0; + odp_u128_t val; + + ATOMIC_CAS_OP_128_ACQ(atom, old_val, new_val, val); + + if ((val.u64[0] == old_val->u64[0]) && (val.u64[1] == old_val->u64[1])) + ret = 1; + + old_val->u64[0] = val.u64[0]; + old_val->u64[1] = val.u64[1]; + + return ret; +} + +static int __atomic_cas_rel_u128(odp_atomic_u128_t *atom, + odp_u128_t *old_val, odp_u128_t new_val) +{ + int ret = 0; + odp_u128_t val; + + ATOMIC_CAS_OP_128_REL(atom, old_val, new_val, val); + + if ((val.u64[0] == old_val->u64[0]) && (val.u64[1] == old_val->u64[1])) + ret = 1; + + old_val->u64[0] = val.u64[0]; + old_val->u64[1] = val.u64[1]; + + return ret; +} + +static int __atomic_cas_acq_rel_u128(odp_atomic_u128_t *atom, + odp_u128_t *old_val, + odp_u128_t new_val) +{ + int ret = 0; + odp_u128_t val; + + ATOMIC_CAS_OP_128_ACQ_REL(atom, old_val, new_val, val); + + if ((val.u64[0] == old_val->u64[0]) && (val.u64[1] == old_val->u64[1])) + ret = 1; + + old_val->u64[0] = val.u64[0]; + old_val->u64[1] = val.u64[1]; + + return ret; +} + +#else /* Locked version */ + +/** + * @internal + * 128 bit store operation expression for the ATOMIC_OP macro + */ +#define ATOMIC_STORE_OP_128(new_val) \ +({ \ + (_atom)->v = (new_val); \ +}) + +/** + * @internal + * 128 bit CAS operation expression for the ATOMIC_OP macro + */ +#define ATOMIC_CAS_OP_128(ret_ptr, old_val, new_val) \ +({ \ + int *_ret_ptr = ret_ptr; \ + odp_u128_t *_old_val = old_val; \ + odp_u128_t _new_val = new_val; \ + if (((_atom)->v.u64[0] == (_old_val)->u64[0]) && \ + ((_atom)->v.u64[1] == (_old_val)->u64[1])) { \ + (_atom)->v = (_new_val); \ + *(_ret_ptr) = 1; \ + } else { \ + *(_ret_ptr) = 0; \ + } \ +}) + +/** + * @internal + * Helper macro for lock-based atomic operations on 128-bit integers + * @param[in,out] atom Pointer to the 128-bit atomic variable + * @param expr Expression used update the variable. + * @return The old value of the variable. + */ +#define ATOMIC_OP_128(atom, expr) \ +({ \ + odp_u128_t _old_val; \ + odp_atomic_u128_t *_atom = atom; \ + /* Loop while lock is already taken, stop when lock becomes clear */ \ + while (__atomic_test_and_set(&(_atom)->lock, __ATOMIC_ACQUIRE)) \ + (void)0; \ + _old_val = (_atom)->v; \ + (expr); /* Perform whatever update is desired */ \ + __atomic_clear(&(_atom)->lock, __ATOMIC_RELEASE); \ + _old_val; /* Return old value */ \ +}) + +static void __atomic_init_u128(odp_atomic_u128_t *atom, odp_u128_t val) +{ + atom->lock = 0; + ATOMIC_OP_128(atom, ATOMIC_STORE_OP_128(val)); +} + +static odp_u128_t __atomic_load_u128(odp_atomic_u128_t *atom) +{ + return ATOMIC_OP_128(atom, (void)0); +} + +static void __atomic_store_u128(odp_atomic_u128_t *atom, odp_u128_t val) +{ + ATOMIC_OP_128(atom, ATOMIC_STORE_OP_128(val)); +} + +static int __atomic_cas_u128(odp_atomic_u128_t *atom, + odp_u128_t *old_val, odp_u128_t new_val) +{ + int ret; + *old_val = ATOMIC_OP_128(atom, ATOMIC_CAS_OP_128(&ret, old_val, + new_val)); + return ret; +} + +static int __atomic_cas_acq_u128(odp_atomic_u128_t *atom, + odp_u128_t *old_val, + odp_u128_t new_val) +{ + int ret; + *old_val = ATOMIC_OP_128(atom, ATOMIC_CAS_OP_128(&ret, old_val, + new_val)); + return ret; +} + +static int __atomic_cas_rel_u128(odp_atomic_u128_t *atom, + odp_u128_t *old_val, + odp_u128_t new_val) +{ + int ret; + *old_val = ATOMIC_OP_128(atom, ATOMIC_CAS_OP_128(&ret, old_val, + new_val)); + return ret; +} + +static int __atomic_cas_acq_rel_u128(odp_atomic_u128_t *atom, + odp_u128_t *old_val, + odp_u128_t new_val) +{ + int ret; + *old_val = ATOMIC_OP_128(atom, ATOMIC_CAS_OP_128(&ret, old_val, + new_val)); + return ret; +} + +#endif + +void odp_atomic_init_u128(odp_atomic_u128_t *atom, odp_u128_t val) +{ + __atomic_init_u128(atom, val); +} + +odp_u128_t odp_atomic_load_u128(odp_atomic_u128_t *atom) +{ + return __atomic_load_u128(atom); +} + +void odp_atomic_store_u128(odp_atomic_u128_t *atom, odp_u128_t val) +{ + __atomic_store_u128(atom, val); +} + +int odp_atomic_cas_u128(odp_atomic_u128_t *atom, + odp_u128_t *old_val, odp_u128_t new_val) +{ + return __atomic_cas_u128(atom, old_val, new_val); +} + +int odp_atomic_cas_acq_u128(odp_atomic_u128_t *atom, + odp_u128_t *old_val, odp_u128_t new_val) +{ + return __atomic_cas_acq_u128(atom, old_val, new_val); +} + +int odp_atomic_cas_rel_u128(odp_atomic_u128_t *atom, + odp_u128_t *old_val, odp_u128_t new_val) +{ + return __atomic_cas_rel_u128(atom, old_val, new_val); +} + +int odp_atomic_cas_acq_rel_u128(odp_atomic_u128_t *atom, + odp_u128_t *old_val, odp_u128_t new_val) +{ + return __atomic_cas_acq_rel_u128(atom, old_val, new_val); +} |