linux-gen: atomic: reorganize architecture specific implementations

Move architecture specific atomic function implementations to separate
header files. Enables 128-bit atomic function inlining.

Signed-off-by: Matias Elo <matias.elo@nokia.com>
Reviewed-by: Petri Savolainen <petri.savolainen@nokia.com>

7 files changed, 364 insertions, 315 deletions

diff --git a/platform/linux-generic/Makefile.am b/platform/linux-generic/Makefile.am
index f563040f5..de98c473f 100644
--- a/platform/linux-generic/Makefile.am
+++ b/platform/linux-generic/Makefile.am
@@ -274,7 +274,9 @@ odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_inlines.h \
 				arch/default/odp/api/abi/cpu_time.h \
 				arch/default/odp/api/abi/hash_crc32.h
 if !ODP_ABI_COMPAT
-odpapiabiarchinclude_HEADERS += arch/arm/odp/api/abi/cpu.h
+odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \
+				arch/default/odp/api/abi/atomic_inlines.h \
+				arch/arm/odp/api/abi/cpu.h
 endif
 noinst_HEADERS += arch/arm/odp_atomic.h \
 		  arch/arm/odp_cpu.h \
@@ -290,7 +292,9 @@ odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_inlines.h \
 				arch/default/odp/api/abi/cpu_time.h \
 				arch/aarch64/odp/api/abi/hash_crc32.h
 if !ODP_ABI_COMPAT
-odpapiabiarchinclude_HEADERS += arch/aarch64/odp/api/abi/atomic.h \
+odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \
+				arch/aarch64/odp/api/abi/atomic_inlines.h \
+				arch/aarch64/odp/api/abi/atomic.h \
 		                arch/aarch64/odp/api/abi/cpu.h
 endif
 noinst_HEADERS += arch/aarch64/odp_atomic.h \
@@ -307,7 +311,9 @@ odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_inlines.h \
 				arch/default/odp/api/abi/cpu_time.h \
 				arch/default/odp/api/abi/hash_crc32.h
 if !ODP_ABI_COMPAT
-odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu.h
+odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \
+				arch/default/odp/api/abi/atomic_inlines.h \
+				arch/default/odp/api/abi/cpu.h
 endif
 noinst_HEADERS += arch/default/odp_cpu.h \
 		  arch/default/odp_cpu_idling.h
@@ -321,7 +327,9 @@ odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_inlines.h \
 				arch/default/odp/api/abi/cpu_time.h \
 				arch/default/odp/api/abi/hash_crc32.h
 if !ODP_ABI_COMPAT
-odpapiabiarchinclude_HEADERS += arch/mips64/odp/api/abi/cpu.h
+odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \
+				arch/default/odp/api/abi/atomic_inlines.h \
+				arch/mips64/odp/api/abi/cpu.h
 endif
 noinst_HEADERS += arch/default/odp_cpu.h \
 		  arch/default/odp_cpu_idling.h
@@ -335,7 +343,9 @@ odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_inlines.h \
 				arch/default/odp/api/abi/cpu_time.h \
 				arch/default/odp/api/abi/hash_crc32.h
 if !ODP_ABI_COMPAT
-odpapiabiarchinclude_HEADERS += arch/powerpc/odp/api/abi/cpu.h
+odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \
+				arch/default/odp/api/abi/atomic_inlines.h \
+				arch/powerpc/odp/api/abi/cpu.h
 endif
 noinst_HEADERS += arch/default/odp_cpu.h \
 		  arch/default/odp_cpu_idling.h
@@ -351,7 +361,9 @@ odpapiabiarchinclude_HEADERS += arch/x86/odp/api/abi/cpu_inlines.h \
 				arch/x86/odp/api/abi/cpu_time.h \
 				arch/default/odp/api/abi/hash_crc32.h
 if !ODP_ABI_COMPAT
-odpapiabiarchinclude_HEADERS += arch/x86/odp/api/abi/cpu.h
+odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \
+				arch/default/odp/api/abi/atomic_inlines.h \
+				arch/x86/odp/api/abi/cpu.h
 endif
 noinst_HEADERS += arch/x86/cpu_flags.h \
 		  arch/x86/odp_cpu.h \
diff --git a/platform/linux-generic/arch/aarch64/odp/api/abi/atomic_inlines.h b/platform/linux-generic/arch/aarch64/odp/api/abi/atomic_inlines.h
new file mode 100644
index 000000000..f2d739886
--- /dev/null
+++ b/platform/linux-generic/arch/aarch64/odp/api/abi/atomic_inlines.h
@@ -0,0 +1,176 @@
+/* Copyright (c) 2021, ARM Limited
+ * Copyright (c) 2021, Nokia
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier:     BSD-3-Clause
+ */
+
+#ifndef ODP_API_ABI_ATOMIC_INLINES_H_
+#define ODP_API_ABI_ATOMIC_INLINES_H_
+
+#include <odp/api/atomic.h>
+
+#ifdef _ODP_LOCK_FREE_128BIT_ATOMICS
+
+/**
+ * @internal
+ * Helper macro for lockless atomic CAS operations on 128-bit integers
+ * @param[in,out] atom Pointer to the 128-bit atomic variable
+ * @param oper CAS operation
+ * @param old_val Old value
+ * @param new_val New value to be swapped
+ * @return 1 for success and 0 for fail
+ */
+#define ATOMIC_CAS_OP_128(atom, oper, old_val, new_val, val) \
+({ \
+	odp_u128_t _val; \
+	odp_atomic_u128_t *_atom = atom; \
+	odp_u128_t *_old_val = old_val; \
+	odp_u128_t _new_val = new_val; \
+	odp_u128_t *ptr = (odp_u128_t *)(_atom); \
+	register uint64_t old0 __asm__ ("x0"); \
+	register uint64_t old1 __asm__ ("x1"); \
+	register uint64_t new0 __asm__ ("x2"); \
+	register uint64_t new1 __asm__ ("x3"); \
+	old0 = (uint64_t)(_old_val)->u64[0]; \
+	old1 = (uint64_t)(_old_val)->u64[1]; \
+	new0 = (uint64_t)(_new_val).u64[0]; \
+	new1 = (uint64_t)(_new_val).u64[1]; \
+	__asm__ volatile(oper " %[old0], %[old1], %[new0], %[new1], [%[ptr]]" \
+			: [old0] "+r" (old0), [old1] "+r" (old1) \
+			: [new0] "r"  (new0), [new1] "r"  (new1), \
+			[ptr] "r" (ptr) \
+			: "memory"); \
+	_val.u64[0] = old0; \
+	_val.u64[1] = old1; \
+	val = _val; \
+})
+
+#define ATOMIC_CAS_OP_128_NO_ORDER(atom, old_value, new_value, val) \
+	ATOMIC_CAS_OP_128(atom, "casp", old_value, new_value, val)
+
+#define ATOMIC_CAS_OP_128_ACQ(atom, old_value, new_value, val) \
+	ATOMIC_CAS_OP_128(atom, "caspa", old_value, new_value, val)
+
+#define ATOMIC_CAS_OP_128_REL(atom, old_value, new_value, val) \
+	ATOMIC_CAS_OP_128(atom, "caspl", old_value, new_value, val)
+
+#define ATOMIC_CAS_OP_128_ACQ_REL(atom, old_value, new_value, val) \
+	ATOMIC_CAS_OP_128(atom, "caspal", old_value, new_value, val)
+
+static inline void _odp_atomic_init_u128(odp_atomic_u128_t *atom, odp_u128_t new_val)
+{
+	odp_u128_t old, val;
+
+	old = atom->v;
+
+	while (1) {
+		ATOMIC_CAS_OP_128_NO_ORDER(atom, &old, new_val, val);
+
+		if ((val.u64[0] == old.u64[0]) && (val.u64[1] == old.u64[1]))
+			return;
+
+		old = val;
+	}
+}
+
+static inline odp_u128_t _odp_atomic_load_u128(odp_atomic_u128_t *atom)
+{
+	odp_u128_t val, exp;
+
+	exp.u64[0] = 0;
+	exp.u64[1] = 0;
+	ATOMIC_CAS_OP_128_NO_ORDER(atom, &exp, exp, val);
+	return val;
+}
+
+static inline void _odp_atomic_store_u128(odp_atomic_u128_t *atom, odp_u128_t new_val)
+{
+	odp_u128_t old, val;
+
+	old = atom->v;
+
+	while (1) {
+		ATOMIC_CAS_OP_128_NO_ORDER(atom, &old, new_val, val);
+
+		if ((val.u64[0] == old.u64[0]) && (val.u64[1] == old.u64[1]))
+			return;
+
+		old = val;
+	}
+}
+
+static inline int _odp_atomic_cas_u128(odp_atomic_u128_t *atom, odp_u128_t *old_val,
+				       odp_u128_t new_val)
+{
+	int ret = 0;
+	odp_u128_t val;
+
+	ATOMIC_CAS_OP_128_NO_ORDER(atom, old_val, new_val, val);
+
+	if ((val.u64[0] == old_val->u64[0]) && (val.u64[1] == old_val->u64[1]))
+		ret = 1;
+
+	old_val->u64[0] = val.u64[0];
+	old_val->u64[1] = val.u64[1];
+
+	return ret;
+}
+
+static inline int _odp_atomic_cas_acq_u128(odp_atomic_u128_t *atom, odp_u128_t *old_val,
+					   odp_u128_t new_val)
+{
+	int ret = 0;
+	odp_u128_t val;
+
+	ATOMIC_CAS_OP_128_ACQ(atom, old_val, new_val, val);
+
+	if ((val.u64[0] == old_val->u64[0]) && (val.u64[1] == old_val->u64[1]))
+		ret = 1;
+
+	old_val->u64[0] = val.u64[0];
+	old_val->u64[1] = val.u64[1];
+
+	return ret;
+}
+
+static inline int _odp_atomic_cas_rel_u128(odp_atomic_u128_t *atom, odp_u128_t *old_val,
+					   odp_u128_t new_val)
+{
+	int ret = 0;
+	odp_u128_t val;
+
+	ATOMIC_CAS_OP_128_REL(atom, old_val, new_val, val);
+
+	if ((val.u64[0] == old_val->u64[0]) && (val.u64[1] == old_val->u64[1]))
+		ret = 1;
+
+	old_val->u64[0] = val.u64[0];
+	old_val->u64[1] = val.u64[1];
+
+	return ret;
+}
+
+static inline int _odp_atomic_cas_acq_rel_u128(odp_atomic_u128_t *atom, odp_u128_t *old_val,
+					       odp_u128_t new_val)
+{
+	int ret = 0;
+	odp_u128_t val;
+
+	ATOMIC_CAS_OP_128_ACQ_REL(atom, old_val, new_val, val);
+
+	if ((val.u64[0] == old_val->u64[0]) && (val.u64[1] == old_val->u64[1]))
+		ret = 1;
+
+	old_val->u64[0] = val.u64[0];
+	old_val->u64[1] = val.u64[1];
+
+	return ret;
+}
+#else /* _ODP_LOCK_FREE_128BIT_ATOMICS */
+
+/* Use generic implementation */
+#include <odp/api/abi/atomic_generic.h>
+
+#endif
+#endif
diff --git a/platform/linux-generic/arch/aarch64/odp_atomic.h b/platform/linux-generic/arch/aarch64/odp_atomic.h
index dbeccebde..d6648a1f0 100644
--- a/platform/linux-generic/arch/aarch64/odp_atomic.h
+++ b/platform/linux-generic/arch/aarch64/odp_atomic.h
@@ -217,54 +217,4 @@ static inline __int128 __lockfree_load_16(__int128 *var, int mo)
 	return old;
 }
 
-#ifdef _ODP_LOCK_FREE_128BIT_ATOMICS
-
-/**
- * @internal
- * Helper macro for lockless atomic CAS operations on 128-bit integers
- * @param[in,out] atom Pointer to the 128-bit atomic variable
- * @param oper CAS operation
- * @param old_val Old value
- * @param new_val New value to be swapped
- * @return 1 for success and 0 for fail
- */
-#define ATOMIC_CAS_OP_128(atom, oper, old_val, new_val, val) \
-({ \
-	odp_u128_t _val; \
-	odp_atomic_u128_t *_atom = atom; \
-	odp_u128_t *_old_val = old_val; \
-	odp_u128_t _new_val = new_val; \
-	odp_u128_t *ptr = (odp_u128_t *)(_atom); \
-	register uint64_t old0 __asm__ ("x0"); \
-	register uint64_t old1 __asm__ ("x1"); \
-	register uint64_t new0 __asm__ ("x2"); \
-	register uint64_t new1 __asm__ ("x3"); \
-	old0 = (uint64_t)(_old_val)->u64[0]; \
-	old1 = (uint64_t)(_old_val)->u64[1]; \
-	new0 = (uint64_t)(_new_val).u64[0]; \
-	new1 = (uint64_t)(_new_val).u64[1]; \
-	__asm__ volatile(oper " %[old0], %[old1], %[new0], %[new1], [%[ptr]]" \
-			: [old0] "+r" (old0), [old1] "+r" (old1) \
-			: [new0] "r"  (new0), [new1] "r"  (new1), \
-			[ptr] "r" (ptr) \
-			: "memory"); \
-	_val.u64[0] = old0; \
-	_val.u64[1] = old1; \
-	val = _val; \
-})
-
-#define ATOMIC_CAS_OP_128_NO_ORDER(atom, old_value, new_value, val) \
-	ATOMIC_CAS_OP_128(atom, "casp", old_value, new_value, val)
-
-#define ATOMIC_CAS_OP_128_ACQ(atom, old_value, new_value, val) \
-	ATOMIC_CAS_OP_128(atom, "caspa", old_value, new_value, val)
-
-#define ATOMIC_CAS_OP_128_REL(atom, old_value, new_value, val) \
-	ATOMIC_CAS_OP_128(atom, "caspl", old_value, new_value, val)
-
-#define ATOMIC_CAS_OP_128_ACQ_REL(atom, old_value, new_value, val) \
-	ATOMIC_CAS_OP_128(atom, "caspal", old_value, new_value, val)
-
-#endif
-
 #endif  /* PLATFORM_LINUXGENERIC_ARCH_ARM_ODP_ATOMIC_H */
diff --git a/platform/linux-generic/arch/default/odp/api/abi/atomic_generic.h b/platform/linux-generic/arch/default/odp/api/abi/atomic_generic.h
new file mode 100644
index 000000000..0e73155e8
--- /dev/null
+++ b/platform/linux-generic/arch/default/odp/api/abi/atomic_generic.h
@@ -0,0 +1,113 @@
+/* Copyright (c) 2021, ARM Limited
+ * Copyright (c) 2021, Nokia
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier:     BSD-3-Clause
+ */
+
+#ifndef ODP_API_ABI_ATOMIC_GENERIC_H_
+#define ODP_API_ABI_ATOMIC_GENERIC_H_
+
+#include <odp/api/atomic.h>
+
+/**
+ * @internal
+ * 128 bit store operation expression for the ATOMIC_OP macro
+ */
+#define ATOMIC_STORE_OP_128(new_val) \
+({ \
+	(_atom)->v = (new_val); \
+})
+
+/**
+ * @internal
+ * 128 bit CAS operation expression for the ATOMIC_OP macro
+ */
+#define ATOMIC_CAS_OP_128(ret_ptr, old_val, new_val) \
+({ \
+	int *_ret_ptr = ret_ptr; \
+	odp_u128_t *_old_val = old_val; \
+	odp_u128_t _new_val = new_val; \
+	if (((_atom)->v.u64[0] == (_old_val)->u64[0]) && \
+	    ((_atom)->v.u64[1] == (_old_val)->u64[1])) { \
+		(_atom)->v = (_new_val); \
+		*(_ret_ptr) = 1; \
+	} else { \
+		*(_ret_ptr) = 0; \
+	} \
+})
+
+/**
+ * @internal
+ * Helper macro for lock-based atomic operations on 128-bit integers
+ * @param[in,out] atom Pointer to the 128-bit atomic variable
+ * @param expr Expression used update the variable.
+ * @return The old value of the variable.
+ */
+#define ATOMIC_OP_128(atom, expr) \
+({ \
+	odp_u128_t _old_val; \
+	odp_atomic_u128_t *_atom = atom; \
+	/* Loop while lock is already taken, stop when lock becomes clear */ \
+	while (__atomic_test_and_set(&(_atom)->lock, __ATOMIC_ACQUIRE)) \
+		(void)0; \
+	_old_val = (_atom)->v; \
+	(expr); /* Perform whatever update is desired */ \
+	__atomic_clear(&(_atom)->lock, __ATOMIC_RELEASE); \
+	_old_val; /* Return old value */ \
+})
+
+static inline void _odp_atomic_init_u128(odp_atomic_u128_t *atom, odp_u128_t val)
+{
+	atom->v.u64[0] = val.u64[0];
+	atom->v.u64[1] = val.u64[1];
+	atom->lock = 0;
+}
+
+static inline odp_u128_t _odp_atomic_load_u128(odp_atomic_u128_t *atom)
+{
+	return ATOMIC_OP_128(atom, (void)0);
+}
+
+static inline void _odp_atomic_store_u128(odp_atomic_u128_t *atom, odp_u128_t val)
+{
+	ATOMIC_OP_128(atom, ATOMIC_STORE_OP_128(val));
+}
+
+static inline int _odp_atomic_cas_u128(odp_atomic_u128_t *atom, odp_u128_t *old_val,
+				       odp_u128_t new_val)
+{
+	int ret;
+
+	*old_val = ATOMIC_OP_128(atom, ATOMIC_CAS_OP_128(&ret, old_val, new_val));
+	return ret;
+}
+
+static inline int _odp_atomic_cas_acq_u128(odp_atomic_u128_t *atom, odp_u128_t *old_val,
+					   odp_u128_t new_val)
+{
+	int ret;
+
+	*old_val = ATOMIC_OP_128(atom, ATOMIC_CAS_OP_128(&ret, old_val, new_val));
+	return ret;
+}
+
+static inline int _odp_atomic_cas_rel_u128(odp_atomic_u128_t *atom, odp_u128_t *old_val,
+					   odp_u128_t new_val)
+{
+	int ret;
+
+	*old_val = ATOMIC_OP_128(atom, ATOMIC_CAS_OP_128(&ret, old_val, new_val));
+	return ret;
+}
+
+static inline int _odp_atomic_cas_acq_rel_u128(odp_atomic_u128_t *atom, odp_u128_t *old_val,
+					       odp_u128_t new_val)
+{
+	int ret;
+
+	*old_val = ATOMIC_OP_128(atom, ATOMIC_CAS_OP_128(&ret, old_val, new_val));
+	return ret;
+}
+
+#endif
diff --git a/platform/linux-generic/arch/default/odp/api/abi/atomic_inlines.h b/platform/linux-generic/arch/default/odp/api/abi/atomic_inlines.h
new file mode 100644
index 000000000..f1072d11f
--- /dev/null
+++ b/platform/linux-generic/arch/default/odp/api/abi/atomic_inlines.h
@@ -0,0 +1,7 @@
+/* Copyright (c) 2021, Nokia
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier:     BSD-3-Clause
+ */
+
+#include <odp/api/abi/atomic_generic.h>
diff --git a/platform/linux-generic/include/odp/api/plat/atomic_inlines.h b/platform/linux-generic/include/odp/api/plat/atomic_inlines.h
index 5f0cba05e..4ab8bb411 100644
--- a/platform/linux-generic/include/odp/api/plat/atomic_inlines.h
+++ b/platform/linux-generic/include/odp/api/plat/atomic_inlines.h
@@ -1,4 +1,5 @@
 /* Copyright (c) 2016-2018, Linaro Limited
+ * Copyright (c) 2021, Nokia
  * All rights reserved.
  *
  * SPDX-License-Identifier: BSD-3-Clause
@@ -13,6 +14,8 @@
 #ifndef _ODP_PLAT_ATOMIC_INLINES_H_
 #define _ODP_PLAT_ATOMIC_INLINES_H_
 
+#include <odp/api/abi/atomic_inlines.h>
+
 /** @cond _ODP_HIDE_FROM_DOXYGEN_ */
 
 #ifndef _ODP_NO_INLINE
@@ -82,6 +85,14 @@
 	#define odp_atomic_cas_acq_u32 __odp_atomic_cas_acq_u32
 	#define odp_atomic_cas_rel_u32 __odp_atomic_cas_rel_u32
 	#define odp_atomic_cas_acq_rel_u32 __odp_atomic_cas_acq_rel_u32
+	#define odp_atomic_init_u128 __odp_atomic_init_u128
+	#define odp_atomic_load_u128 __odp_atomic_load_u128
+	#define odp_atomic_store_u128 __odp_atomic_store_u128
+	#define odp_atomic_cas_u128 __odp_atomic_cas_u128
+	#define odp_atomic_cas_acq_u128 __odp_atomic_cas_acq_u128
+	#define odp_atomic_cas_rel_u128 __odp_atomic_cas_rel_u128
+	#define odp_atomic_cas_acq_rel_u128 __odp_atomic_cas_acq_rel_u128
+
 #else
 	#define _ODP_INLINE
 #endif
@@ -530,6 +541,45 @@ _ODP_INLINE int odp_atomic_cas_acq_rel_u32(odp_atomic_u32_t *atom,
 					   __ATOMIC_RELAXED);
 }
 
+_ODP_INLINE void odp_atomic_init_u128(odp_atomic_u128_t *atom, odp_u128_t val)
+{
+	_odp_atomic_init_u128(atom, val);
+}
+
+_ODP_INLINE odp_u128_t odp_atomic_load_u128(odp_atomic_u128_t *atom)
+{
+	return _odp_atomic_load_u128(atom);
+}
+
+_ODP_INLINE void odp_atomic_store_u128(odp_atomic_u128_t *atom, odp_u128_t val)
+{
+	_odp_atomic_store_u128(atom, val);
+}
+
+_ODP_INLINE int odp_atomic_cas_u128(odp_atomic_u128_t *atom,
+				    odp_u128_t *old_val, odp_u128_t new_val)
+{
+	return _odp_atomic_cas_u128(atom, old_val, new_val);
+}
+
+_ODP_INLINE int odp_atomic_cas_acq_u128(odp_atomic_u128_t *atom,
+					odp_u128_t *old_val, odp_u128_t new_val)
+{
+	return _odp_atomic_cas_acq_u128(atom, old_val, new_val);
+}
+
+_ODP_INLINE int odp_atomic_cas_rel_u128(odp_atomic_u128_t *atom,
+					odp_u128_t *old_val, odp_u128_t new_val)
+{
+	return _odp_atomic_cas_rel_u128(atom, old_val, new_val);
+}
+
+_ODP_INLINE int odp_atomic_cas_acq_rel_u128(odp_atomic_u128_t *atom,
+					    odp_u128_t *old_val, odp_u128_t new_val)
+{
+	return _odp_atomic_cas_acq_rel_u128(atom, old_val, new_val);
+}
+
 /** @endcond */
 
 #endif
diff --git a/platform/linux-generic/odp_atomic.c b/platform/linux-generic/odp_atomic.c
index 59253c645..13ba2e5ed 100644
--- a/platform/linux-generic/odp_atomic.c
+++ b/platform/linux-generic/odp_atomic.c
@@ -6,7 +6,6 @@
  */
 
 #include <odp/api/atomic.h>
-#include <odp_cpu.h>
 
 int odp_atomic_lock_free_u64(odp_atomic_op_t *atomic_op)
 {
@@ -46,261 +45,3 @@ int odp_atomic_lock_free_u128(odp_atomic_op_t *atomic_op)
 	return 0;
 #endif
 }
-
-#ifdef _ODP_LOCK_FREE_128BIT_ATOMICS
-
-static void __atomic_init_u128(odp_atomic_u128_t *atom, odp_u128_t new_val)
-{
-	odp_u128_t old, val;
-
-	old = atom->v;
-
-	while (1) {
-		ATOMIC_CAS_OP_128_NO_ORDER(atom, &old, new_val, val);
-
-		if ((val.u64[0] == old.u64[0]) && (val.u64[1] == old.u64[1]))
-			return;
-
-		old = val;
-	}
-}
-
-static odp_u128_t __atomic_load_u128(odp_atomic_u128_t *atom)
-{
-	odp_u128_t val, exp;
-
-	exp.u64[0] = 0;
-	exp.u64[1] = 0;
-	ATOMIC_CAS_OP_128_NO_ORDER(atom, &exp, exp, val);
-	return val;
-}
-
-static void __atomic_store_u128(odp_atomic_u128_t *atom, odp_u128_t new_val)
-{
-	odp_u128_t old, val;
-
-	old = atom->v;
-
-	while (1) {
-		ATOMIC_CAS_OP_128_NO_ORDER(atom, &old, new_val, val);
-
-		if ((val.u64[0] == old.u64[0]) && (val.u64[1] == old.u64[1]))
-			return;
-
-		old = val;
-	}
-}
-
-static int __atomic_cas_u128(odp_atomic_u128_t *atom,
-			     odp_u128_t *old_val, odp_u128_t new_val)
-{
-	int ret = 0;
-	odp_u128_t val;
-
-	ATOMIC_CAS_OP_128_NO_ORDER(atom, old_val, new_val, val);
-
-	if ((val.u64[0] == old_val->u64[0]) && (val.u64[1] == old_val->u64[1]))
-		ret = 1;
-
-	old_val->u64[0] = val.u64[0];
-	old_val->u64[1] = val.u64[1];
-
-	return ret;
-}
-
-static int __atomic_cas_acq_u128(odp_atomic_u128_t *atom,
-				 odp_u128_t *old_val, odp_u128_t new_val)
-{
-	int ret = 0;
-	odp_u128_t val;
-
-	ATOMIC_CAS_OP_128_ACQ(atom, old_val, new_val, val);
-
-	if ((val.u64[0] == old_val->u64[0]) && (val.u64[1] == old_val->u64[1]))
-		ret = 1;
-
-	old_val->u64[0] = val.u64[0];
-	old_val->u64[1] = val.u64[1];
-
-	return ret;
-}
-
-static int __atomic_cas_rel_u128(odp_atomic_u128_t *atom,
-				 odp_u128_t *old_val, odp_u128_t new_val)
-{
-	int ret = 0;
-	odp_u128_t val;
-
-	ATOMIC_CAS_OP_128_REL(atom, old_val, new_val, val);
-
-	if ((val.u64[0] == old_val->u64[0]) && (val.u64[1] == old_val->u64[1]))
-		ret = 1;
-
-	old_val->u64[0] = val.u64[0];
-	old_val->u64[1] = val.u64[1];
-
-	return ret;
-}
-
-static int __atomic_cas_acq_rel_u128(odp_atomic_u128_t *atom,
-				     odp_u128_t *old_val,
-				     odp_u128_t new_val)
-{
-	int ret = 0;
-	odp_u128_t val;
-
-	ATOMIC_CAS_OP_128_ACQ_REL(atom, old_val, new_val, val);
-
-	if ((val.u64[0] == old_val->u64[0]) && (val.u64[1] == old_val->u64[1]))
-		ret = 1;
-
-	old_val->u64[0] = val.u64[0];
-	old_val->u64[1] = val.u64[1];
-
-	return ret;
-}
-
-#else /* Locked version */
-
-/**
- * @internal
- * 128 bit store operation expression for the ATOMIC_OP macro
- */
-#define ATOMIC_STORE_OP_128(new_val) \
-({ \
-	(_atom)->v = (new_val); \
-})
-
-/**
- * @internal
- * 128 bit CAS operation expression for the ATOMIC_OP macro
- */
-#define ATOMIC_CAS_OP_128(ret_ptr, old_val, new_val) \
-({ \
-	int *_ret_ptr = ret_ptr; \
-	odp_u128_t *_old_val = old_val; \
-	odp_u128_t _new_val = new_val; \
-	if (((_atom)->v.u64[0] == (_old_val)->u64[0]) && \
-	    ((_atom)->v.u64[1] == (_old_val)->u64[1])) { \
-		(_atom)->v = (_new_val); \
-		*(_ret_ptr) = 1; \
-	} else { \
-		*(_ret_ptr) = 0; \
-	} \
-})
-
-/**
- * @internal
- * Helper macro for lock-based atomic operations on 128-bit integers
- * @param[in,out] atom Pointer to the 128-bit atomic variable
- * @param expr Expression used update the variable.
- * @return The old value of the variable.
- */
-#define ATOMIC_OP_128(atom, expr) \
-({ \
-	odp_u128_t _old_val; \
-	odp_atomic_u128_t *_atom = atom; \
-	/* Loop while lock is already taken, stop when lock becomes clear */ \
-	while (__atomic_test_and_set(&(_atom)->lock, __ATOMIC_ACQUIRE)) \
-		(void)0; \
-	_old_val = (_atom)->v; \
-	(expr); /* Perform whatever update is desired */ \
-	__atomic_clear(&(_atom)->lock, __ATOMIC_RELEASE); \
-	_old_val; /* Return old value */ \
-})
-
-static void __atomic_init_u128(odp_atomic_u128_t *atom, odp_u128_t val)
-{
-	atom->lock = 0;
-	ATOMIC_OP_128(atom, ATOMIC_STORE_OP_128(val));
-}
-
-static odp_u128_t __atomic_load_u128(odp_atomic_u128_t *atom)
-{
-	return ATOMIC_OP_128(atom, (void)0);
-}
-
-static void __atomic_store_u128(odp_atomic_u128_t *atom, odp_u128_t val)
-{
-	ATOMIC_OP_128(atom, ATOMIC_STORE_OP_128(val));
-}
-
-static int __atomic_cas_u128(odp_atomic_u128_t *atom,
-			     odp_u128_t *old_val, odp_u128_t new_val)
-{
-	int ret;
-	*old_val = ATOMIC_OP_128(atom, ATOMIC_CAS_OP_128(&ret, old_val,
-							 new_val));
-	return ret;
-}
-
-static int __atomic_cas_acq_u128(odp_atomic_u128_t *atom,
-				 odp_u128_t *old_val,
-				 odp_u128_t new_val)
-{
-	int ret;
-	*old_val = ATOMIC_OP_128(atom, ATOMIC_CAS_OP_128(&ret, old_val,
-							 new_val));
-	return ret;
-}
-
-static int __atomic_cas_rel_u128(odp_atomic_u128_t *atom,
-				 odp_u128_t *old_val,
-				 odp_u128_t new_val)
-{
-	int ret;
-	*old_val = ATOMIC_OP_128(atom, ATOMIC_CAS_OP_128(&ret, old_val,
-							 new_val));
-	return ret;
-}
-
-static int __atomic_cas_acq_rel_u128(odp_atomic_u128_t *atom,
-				     odp_u128_t *old_val,
-				     odp_u128_t new_val)
-{
-	int ret;
-	*old_val = ATOMIC_OP_128(atom, ATOMIC_CAS_OP_128(&ret, old_val,
-							 new_val));
-	return ret;
-}
-
-#endif
-
-void odp_atomic_init_u128(odp_atomic_u128_t *atom, odp_u128_t val)
-{
-	__atomic_init_u128(atom, val);
-}
-
-odp_u128_t odp_atomic_load_u128(odp_atomic_u128_t *atom)
-{
-	return __atomic_load_u128(atom);
-}
-
-void odp_atomic_store_u128(odp_atomic_u128_t *atom, odp_u128_t val)
-{
-	__atomic_store_u128(atom, val);
-}
-
-int odp_atomic_cas_u128(odp_atomic_u128_t *atom,
-			odp_u128_t *old_val, odp_u128_t new_val)
-{
-	return __atomic_cas_u128(atom, old_val, new_val);
-}
-
-int odp_atomic_cas_acq_u128(odp_atomic_u128_t *atom,
-			    odp_u128_t *old_val, odp_u128_t new_val)
-{
-	return __atomic_cas_acq_u128(atom, old_val, new_val);
-}
-
-int odp_atomic_cas_rel_u128(odp_atomic_u128_t *atom,
-			    odp_u128_t *old_val, odp_u128_t new_val)
-{
-	return __atomic_cas_rel_u128(atom, old_val, new_val);
-}
-
-int odp_atomic_cas_acq_rel_u128(odp_atomic_u128_t *atom,
-				odp_u128_t *old_val, odp_u128_t new_val)
-{
-	return __atomic_cas_acq_rel_u128(atom, old_val, new_val);
-}