linux-gen: atomic: add 128 bit atomic apis

Add 128 bit atomic compare-and-swap operations.

Signed-off-by: Govindarajan Mohandoss <govindarajan.mohandoss@arm.com>
Reviewed-by: Petri Savolainen <petri.savolainen@nokia.com>
Reviewed-by: Matias Elo <matias.elo@nokia.com>
Tested-by: Malvika Gupta <Malvika.Gupta@arm.com>

5 files changed, 370 insertions, 3 deletions

diff --git a/platform/linux-generic/Makefile.am b/platform/linux-generic/Makefile.am
index 00ea764de..1d9a3bbdb 100644
--- a/platform/linux-generic/Makefile.am
+++ b/platform/linux-generic/Makefile.am
@@ -290,7 +290,8 @@ odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_inlines.h \
 				arch/default/odp/api/abi/cpu_time.h \
 				arch/aarch64/odp/api/abi/hash_crc32.h
 if !ODP_ABI_COMPAT
-odpapiabiarchinclude_HEADERS += arch/aarch64/odp/api/abi/cpu.h
+odpapiabiarchinclude_HEADERS += arch/aarch64/odp/api/abi/atomic.h \
+		                arch/aarch64/odp/api/abi/cpu.h
 endif
 noinst_HEADERS += arch/aarch64/odp_atomic.h \
 		  arch/aarch64/odp_cpu.h \
diff --git a/platform/linux-generic/arch/aarch64/odp/api/abi/atomic.h b/platform/linux-generic/arch/aarch64/odp/api/abi/atomic.h
new file mode 100644
index 000000000..d1dbf36b8
--- /dev/null
+++ b/platform/linux-generic/arch/aarch64/odp/api/abi/atomic.h
@@ -0,0 +1,12 @@
+/* Copyright (c) 2021, ARM Limited
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier:     BSD-3-Clause
+ */
+
+#ifdef __ARM_FEATURE_ATOMICS
+#define _ODP_LOCK_FREE_128BIT_ATOMICS
+#endif
+
+#include <odp/api/abi-default/atomic.h>
+
diff --git a/platform/linux-generic/arch/aarch64/odp_atomic.h b/platform/linux-generic/arch/aarch64/odp_atomic.h
index 8a0e7ce2b..dbeccebde 100644
--- a/platform/linux-generic/arch/aarch64/odp_atomic.h
+++ b/platform/linux-generic/arch/aarch64/odp_atomic.h
@@ -1,5 +1,4 @@
-/* Copyright (c) 2017, ARM Limited. All rights reserved.
- *
+/* Copyright (c) 2017-2021, ARM Limited
  * Copyright (c) 2017-2018, Linaro Limited
  * All rights reserved.
  *
@@ -218,4 +217,54 @@ static inline __int128 __lockfree_load_16(__int128 *var, int mo)
 	return old;
 }
 
+#ifdef _ODP_LOCK_FREE_128BIT_ATOMICS
+
+/**
+ * @internal
+ * Helper macro for lockless atomic CAS operations on 128-bit integers
+ * @param[in,out] atom Pointer to the 128-bit atomic variable
+ * @param oper CAS operation
+ * @param old_val Old value
+ * @param new_val New value to be swapped
+ * @return 1 for success and 0 for fail
+ */
+#define ATOMIC_CAS_OP_128(atom, oper, old_val, new_val, val) \
+({ \
+	odp_u128_t _val; \
+	odp_atomic_u128_t *_atom = atom; \
+	odp_u128_t *_old_val = old_val; \
+	odp_u128_t _new_val = new_val; \
+	odp_u128_t *ptr = (odp_u128_t *)(_atom); \
+	register uint64_t old0 __asm__ ("x0"); \
+	register uint64_t old1 __asm__ ("x1"); \
+	register uint64_t new0 __asm__ ("x2"); \
+	register uint64_t new1 __asm__ ("x3"); \
+	old0 = (uint64_t)(_old_val)->u64[0]; \
+	old1 = (uint64_t)(_old_val)->u64[1]; \
+	new0 = (uint64_t)(_new_val).u64[0]; \
+	new1 = (uint64_t)(_new_val).u64[1]; \
+	__asm__ volatile(oper " %[old0], %[old1], %[new0], %[new1], [%[ptr]]" \
+			: [old0] "+r" (old0), [old1] "+r" (old1) \
+			: [new0] "r"  (new0), [new1] "r"  (new1), \
+			[ptr] "r" (ptr) \
+			: "memory"); \
+	_val.u64[0] = old0; \
+	_val.u64[1] = old1; \
+	val = _val; \
+})
+
+#define ATOMIC_CAS_OP_128_NO_ORDER(atom, old_value, new_value, val) \
+	ATOMIC_CAS_OP_128(atom, "casp", old_value, new_value, val)
+
+#define ATOMIC_CAS_OP_128_ACQ(atom, old_value, new_value, val) \
+	ATOMIC_CAS_OP_128(atom, "caspa", old_value, new_value, val)
+
+#define ATOMIC_CAS_OP_128_REL(atom, old_value, new_value, val) \
+	ATOMIC_CAS_OP_128(atom, "caspl", old_value, new_value, val)
+
+#define ATOMIC_CAS_OP_128_ACQ_REL(atom, old_value, new_value, val) \
+	ATOMIC_CAS_OP_128(atom, "caspal", old_value, new_value, val)
+
+#endif
+
 #endif  /* PLATFORM_LINUXGENERIC_ARCH_ARM_ODP_ATOMIC_H */
diff --git a/platform/linux-generic/include-abi/odp/api/abi/atomic.h b/platform/linux-generic/include-abi/odp/api/abi/atomic.h
index 955b99370..13c12a79f 100644
--- a/platform/linux-generic/include-abi/odp/api/abi/atomic.h
+++ b/platform/linux-generic/include-abi/odp/api/abi/atomic.h
@@ -55,6 +55,31 @@ typedef struct ODP_ALIGNED(sizeof(uint64_t)) odp_atomic_u64_s {
 
 #endif
 
+#ifdef _ODP_LOCK_FREE_128BIT_ATOMICS
+
+/**
+ * @internal
+ * Atomic 128-bit unsigned integer
+ */
+typedef struct ODP_ALIGNED(sizeof(odp_u128_t)) odp_atomic_u128_s {
+	odp_u128_t v; /**< Actual storage for the atomic variable */
+} odp_atomic_u128_t;
+
+#else
+
+/**
+ * @internal
+ * Atomic 128-bit unsigned integer
+ */
+typedef struct ODP_ALIGNED(sizeof(odp_u128_t)) odp_atomic_u128_s {
+	odp_u128_t v; /**< Actual storage for the atomic variable */
+	/* Some architectures do not support lock-free operations on 128-bit
+	 * data types. We use a spin lock to ensure atomicity. */
+	char lock; /**< Spin lock (if needed) used to ensure atomic access */
+} odp_atomic_u128_t;
+
+#endif
+
 /** @ingroup odp_atomic
  *  @{
  */
diff --git a/platform/linux-generic/odp_atomic.c b/platform/linux-generic/odp_atomic.c
index 32ddc95c2..59253c645 100644
--- a/platform/linux-generic/odp_atomic.c
+++ b/platform/linux-generic/odp_atomic.c
@@ -1,10 +1,12 @@
 /* Copyright (c) 2015-2018, Linaro Limited
+ * Copyright (c) 2021, ARM Limited
  * All rights reserved.
  *
  * SPDX-License-Identifier:     BSD-3-Clause
  */
 
 #include <odp/api/atomic.h>
+#include <odp_cpu.h>
 
 int odp_atomic_lock_free_u64(odp_atomic_op_t *atomic_op)
 {
@@ -24,3 +26,281 @@ int odp_atomic_lock_free_u64(odp_atomic_op_t *atomic_op)
 	return 2;
 #endif
 }
+
+int odp_atomic_lock_free_u128(odp_atomic_op_t *atomic_op)
+{
+#ifdef _ODP_LOCK_FREE_128BIT_ATOMICS
+	if (atomic_op) {
+		atomic_op->all_bits = 0;
+		atomic_op->op.load  = 1;
+		atomic_op->op.store = 1;
+		atomic_op->op.cas  = 1;
+	}
+
+	return 2;
+#else
+	/* All operations have locks */
+	if (atomic_op)
+		atomic_op->all_bits = 0;
+
+	return 0;
+#endif
+}
+
+#ifdef _ODP_LOCK_FREE_128BIT_ATOMICS
+
+static void __atomic_init_u128(odp_atomic_u128_t *atom, odp_u128_t new_val)
+{
+	odp_u128_t old, val;
+
+	old = atom->v;
+
+	while (1) {
+		ATOMIC_CAS_OP_128_NO_ORDER(atom, &old, new_val, val);
+
+		if ((val.u64[0] == old.u64[0]) && (val.u64[1] == old.u64[1]))
+			return;
+
+		old = val;
+	}
+}
+
+static odp_u128_t __atomic_load_u128(odp_atomic_u128_t *atom)
+{
+	odp_u128_t val, exp;
+
+	exp.u64[0] = 0;
+	exp.u64[1] = 0;
+	ATOMIC_CAS_OP_128_NO_ORDER(atom, &exp, exp, val);
+	return val;
+}
+
+static void __atomic_store_u128(odp_atomic_u128_t *atom, odp_u128_t new_val)
+{
+	odp_u128_t old, val;
+
+	old = atom->v;
+
+	while (1) {
+		ATOMIC_CAS_OP_128_NO_ORDER(atom, &old, new_val, val);
+
+		if ((val.u64[0] == old.u64[0]) && (val.u64[1] == old.u64[1]))
+			return;
+
+		old = val;
+	}
+}
+
+static int __atomic_cas_u128(odp_atomic_u128_t *atom,
+			     odp_u128_t *old_val, odp_u128_t new_val)
+{
+	int ret = 0;
+	odp_u128_t val;
+
+	ATOMIC_CAS_OP_128_NO_ORDER(atom, old_val, new_val, val);
+
+	if ((val.u64[0] == old_val->u64[0]) && (val.u64[1] == old_val->u64[1]))
+		ret = 1;
+
+	old_val->u64[0] = val.u64[0];
+	old_val->u64[1] = val.u64[1];
+
+	return ret;
+}
+
+static int __atomic_cas_acq_u128(odp_atomic_u128_t *atom,
+				 odp_u128_t *old_val, odp_u128_t new_val)
+{
+	int ret = 0;
+	odp_u128_t val;
+
+	ATOMIC_CAS_OP_128_ACQ(atom, old_val, new_val, val);
+
+	if ((val.u64[0] == old_val->u64[0]) && (val.u64[1] == old_val->u64[1]))
+		ret = 1;
+
+	old_val->u64[0] = val.u64[0];
+	old_val->u64[1] = val.u64[1];
+
+	return ret;
+}
+
+static int __atomic_cas_rel_u128(odp_atomic_u128_t *atom,
+				 odp_u128_t *old_val, odp_u128_t new_val)
+{
+	int ret = 0;
+	odp_u128_t val;
+
+	ATOMIC_CAS_OP_128_REL(atom, old_val, new_val, val);
+
+	if ((val.u64[0] == old_val->u64[0]) && (val.u64[1] == old_val->u64[1]))
+		ret = 1;
+
+	old_val->u64[0] = val.u64[0];
+	old_val->u64[1] = val.u64[1];
+
+	return ret;
+}
+
+static int __atomic_cas_acq_rel_u128(odp_atomic_u128_t *atom,
+				     odp_u128_t *old_val,
+				     odp_u128_t new_val)
+{
+	int ret = 0;
+	odp_u128_t val;
+
+	ATOMIC_CAS_OP_128_ACQ_REL(atom, old_val, new_val, val);
+
+	if ((val.u64[0] == old_val->u64[0]) && (val.u64[1] == old_val->u64[1]))
+		ret = 1;
+
+	old_val->u64[0] = val.u64[0];
+	old_val->u64[1] = val.u64[1];
+
+	return ret;
+}
+
+#else /* Locked version */
+
+/**
+ * @internal
+ * 128 bit store operation expression for the ATOMIC_OP macro
+ */
+#define ATOMIC_STORE_OP_128(new_val) \
+({ \
+	(_atom)->v = (new_val); \
+})
+
+/**
+ * @internal
+ * 128 bit CAS operation expression for the ATOMIC_OP macro
+ */
+#define ATOMIC_CAS_OP_128(ret_ptr, old_val, new_val) \
+({ \
+	int *_ret_ptr = ret_ptr; \
+	odp_u128_t *_old_val = old_val; \
+	odp_u128_t _new_val = new_val; \
+	if (((_atom)->v.u64[0] == (_old_val)->u64[0]) && \
+	    ((_atom)->v.u64[1] == (_old_val)->u64[1])) { \
+		(_atom)->v = (_new_val); \
+		*(_ret_ptr) = 1; \
+	} else { \
+		*(_ret_ptr) = 0; \
+	} \
+})
+
+/**
+ * @internal
+ * Helper macro for lock-based atomic operations on 128-bit integers
+ * @param[in,out] atom Pointer to the 128-bit atomic variable
+ * @param expr Expression used update the variable.
+ * @return The old value of the variable.
+ */
+#define ATOMIC_OP_128(atom, expr) \
+({ \
+	odp_u128_t _old_val; \
+	odp_atomic_u128_t *_atom = atom; \
+	/* Loop while lock is already taken, stop when lock becomes clear */ \
+	while (__atomic_test_and_set(&(_atom)->lock, __ATOMIC_ACQUIRE)) \
+		(void)0; \
+	_old_val = (_atom)->v; \
+	(expr); /* Perform whatever update is desired */ \
+	__atomic_clear(&(_atom)->lock, __ATOMIC_RELEASE); \
+	_old_val; /* Return old value */ \
+})
+
+static void __atomic_init_u128(odp_atomic_u128_t *atom, odp_u128_t val)
+{
+	atom->lock = 0;
+	ATOMIC_OP_128(atom, ATOMIC_STORE_OP_128(val));
+}
+
+static odp_u128_t __atomic_load_u128(odp_atomic_u128_t *atom)
+{
+	return ATOMIC_OP_128(atom, (void)0);
+}
+
+static void __atomic_store_u128(odp_atomic_u128_t *atom, odp_u128_t val)
+{
+	ATOMIC_OP_128(atom, ATOMIC_STORE_OP_128(val));
+}
+
+static int __atomic_cas_u128(odp_atomic_u128_t *atom,
+			     odp_u128_t *old_val, odp_u128_t new_val)
+{
+	int ret;
+	*old_val = ATOMIC_OP_128(atom, ATOMIC_CAS_OP_128(&ret, old_val,
+							 new_val));
+	return ret;
+}
+
+static int __atomic_cas_acq_u128(odp_atomic_u128_t *atom,
+				 odp_u128_t *old_val,
+				 odp_u128_t new_val)
+{
+	int ret;
+	*old_val = ATOMIC_OP_128(atom, ATOMIC_CAS_OP_128(&ret, old_val,
+							 new_val));
+	return ret;
+}
+
+static int __atomic_cas_rel_u128(odp_atomic_u128_t *atom,
+				 odp_u128_t *old_val,
+				 odp_u128_t new_val)
+{
+	int ret;
+	*old_val = ATOMIC_OP_128(atom, ATOMIC_CAS_OP_128(&ret, old_val,
+							 new_val));
+	return ret;
+}
+
+static int __atomic_cas_acq_rel_u128(odp_atomic_u128_t *atom,
+				     odp_u128_t *old_val,
+				     odp_u128_t new_val)
+{
+	int ret;
+	*old_val = ATOMIC_OP_128(atom, ATOMIC_CAS_OP_128(&ret, old_val,
+							 new_val));
+	return ret;
+}
+
+#endif
+
+void odp_atomic_init_u128(odp_atomic_u128_t *atom, odp_u128_t val)
+{
+	__atomic_init_u128(atom, val);
+}
+
+odp_u128_t odp_atomic_load_u128(odp_atomic_u128_t *atom)
+{
+	return __atomic_load_u128(atom);
+}
+
+void odp_atomic_store_u128(odp_atomic_u128_t *atom, odp_u128_t val)
+{
+	__atomic_store_u128(atom, val);
+}
+
+int odp_atomic_cas_u128(odp_atomic_u128_t *atom,
+			odp_u128_t *old_val, odp_u128_t new_val)
+{
+	return __atomic_cas_u128(atom, old_val, new_val);
+}
+
+int odp_atomic_cas_acq_u128(odp_atomic_u128_t *atom,
+			    odp_u128_t *old_val, odp_u128_t new_val)
+{
+	return __atomic_cas_acq_u128(atom, old_val, new_val);
+}
+
+int odp_atomic_cas_rel_u128(odp_atomic_u128_t *atom,
+			    odp_u128_t *old_val, odp_u128_t new_val)
+{
+	return __atomic_cas_rel_u128(atom, old_val, new_val);
+}
+
+int odp_atomic_cas_acq_rel_u128(odp_atomic_u128_t *atom,
+				odp_u128_t *old_val, odp_u128_t new_val)
+{
+	return __atomic_cas_acq_rel_u128(atom, old_val, new_val);
+}