9 files changed, 93 insertions, 19 deletions
diff --git a/platform/linux-generic/arch/aarch64/odp/api/abi/atomic.h b/platform/linux-generic/arch/aarch64/odp/api/abi/atomic.h
new file mode 100644
index 000000000..d1dbf36b8
--- /dev/null
+++ b/platform/linux-generic/arch/aarch64/odp/api/abi/atomic.h
@@ -0,0 +1,12 @@
+/* Copyright (c) 2021, ARM Limited
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier:     BSD-3-Clause
+ */
+
+#ifdef __ARM_FEATURE_ATOMICS
+#define _ODP_LOCK_FREE_128BIT_ATOMICS
+#endif
+
+#include <odp/api/abi-default/atomic.h>
+
diff --git a/platform/linux-generic/arch/aarch64/odp_atomic.h b/platform/linux-generic/arch/aarch64/odp_atomic.h
index 8a0e7ce2b..dbeccebde 100644
--- a/platform/linux-generic/arch/aarch64/odp_atomic.h
+++ b/platform/linux-generic/arch/aarch64/odp_atomic.h
@@ -1,5 +1,4 @@
-/* Copyright (c) 2017, ARM Limited. All rights reserved.
- *
+/* Copyright (c) 2017-2021, ARM Limited
  * Copyright (c) 2017-2018, Linaro Limited
  * All rights reserved.
  *
@@ -218,4 +217,54 @@ static inline __int128 __lockfree_load_16(__int128 *var, int mo)
 	return old;
 }
 
+#ifdef _ODP_LOCK_FREE_128BIT_ATOMICS
+
+/**
+ * @internal
+ * Helper macro for lockless atomic CAS operations on 128-bit integers
+ * @param[in,out] atom Pointer to the 128-bit atomic variable
+ * @param oper CAS operation
+ * @param old_val Old value
+ * @param new_val New value to be swapped
+ * @return 1 for success and 0 for fail
+ */
+#define ATOMIC_CAS_OP_128(atom, oper, old_val, new_val, val) \
+({ \
+	odp_u128_t _val; \
+	odp_atomic_u128_t *_atom = atom; \
+	odp_u128_t *_old_val = old_val; \
+	odp_u128_t _new_val = new_val; \
+	odp_u128_t *ptr = (odp_u128_t *)(_atom); \
+	register uint64_t old0 __asm__ ("x0"); \
+	register uint64_t old1 __asm__ ("x1"); \
+	register uint64_t new0 __asm__ ("x2"); \
+	register uint64_t new1 __asm__ ("x3"); \
+	old0 = (uint64_t)(_old_val)->u64[0]; \
+	old1 = (uint64_t)(_old_val)->u64[1]; \
+	new0 = (uint64_t)(_new_val).u64[0]; \
+	new1 = (uint64_t)(_new_val).u64[1]; \
+	__asm__ volatile(oper " %[old0], %[old1], %[new0], %[new1], [%[ptr]]" \
+			: [old0] "+r" (old0), [old1] "+r" (old1) \
+			: [new0] "r"  (new0), [new1] "r"  (new1), \
+			[ptr] "r" (ptr) \
+			: "memory"); \
+	_val.u64[0] = old0; \
+	_val.u64[1] = old1; \
+	val = _val; \
+})
+
+#define ATOMIC_CAS_OP_128_NO_ORDER(atom, old_value, new_value, val) \
+	ATOMIC_CAS_OP_128(atom, "casp", old_value, new_value, val)
+
+#define ATOMIC_CAS_OP_128_ACQ(atom, old_value, new_value, val) \
+	ATOMIC_CAS_OP_128(atom, "caspa", old_value, new_value, val)
+
+#define ATOMIC_CAS_OP_128_REL(atom, old_value, new_value, val) \
+	ATOMIC_CAS_OP_128(atom, "caspl", old_value, new_value, val)
+
+#define ATOMIC_CAS_OP_128_ACQ_REL(atom, old_value, new_value, val) \
+	ATOMIC_CAS_OP_128(atom, "caspal", old_value, new_value, val)
+
+#endif
+
 #endif  /* PLATFORM_LINUXGENERIC_ARCH_ARM_ODP_ATOMIC_H */
diff --git a/platform/linux-generic/arch/aarch64/odp_cpu.h b/platform/linux-generic/arch/aarch64/odp_cpu.h
index 0da6e3c60..84bc4dffd 100644
--- a/platform/linux-generic/arch/aarch64/odp_cpu.h
+++ b/platform/linux-generic/arch/aarch64/odp_cpu.h
@@ -59,4 +59,10 @@ do {							     \
 #include "odp_atomic.h"
 #include "odp_cpu_idling.h"
 
+#ifdef __ARM_FEATURE_UNALIGNED
+#define _ODP_UNALIGNED 1
+#else
+#define _ODP_UNALIGNED 0
+#endif
+
 #endif  /* PLATFORM_LINUXGENERIC_ARCH_ARM_ODP_CPU_H */
diff --git a/platform/linux-generic/arch/aarch64/odp_cpu_idling.h b/platform/linux-generic/arch/aarch64/odp_cpu_idling.h
index 8528c8591..a6cea8c63 100644
--- a/platform/linux-generic/arch/aarch64/odp_cpu_idling.h
+++ b/platform/linux-generic/arch/aarch64/odp_cpu_idling.h
@@ -30,12 +30,6 @@ static inline int wfe(void)
 	return 1;
 }
 
-static inline void doze(void)
-{
-	/* When using WFE do not stall the pipeline using other means */
-	odp_cpu_pause();
-}
-
 #define monitor128(addr, mo) lld((addr), (mo))
 #define monitor64(addr, mo) ll64((addr), (mo))
 #define monitor32(addr, mo) ll32((addr), (mo))
diff --git a/platform/linux-generic/arch/arm/odp_cpu.h b/platform/linux-generic/arch/arm/odp_cpu.h
index d3d2de34f..82d47325f 100644
--- a/platform/linux-generic/arch/arm/odp_cpu.h
+++ b/platform/linux-generic/arch/arm/odp_cpu.h
@@ -52,4 +52,10 @@ static inline void _odp_dmb(void)
 #include "odp_atomic.h"
 #include "odp_cpu_idling.h"
 
+#ifdef __ARM_FEATURE_UNALIGNED
+#define _ODP_UNALIGNED 1
+#else
+#define _ODP_UNALIGNED 0
+#endif
+
 #endif  /* PLATFORM_LINUXGENERIC_ARCH_ARM_ODP_CPU_H */
diff --git a/platform/linux-generic/arch/arm/odp_cpu_idling.h b/platform/linux-generic/arch/arm/odp_cpu_idling.h
index 8528c8591..a6cea8c63 100644
--- a/platform/linux-generic/arch/arm/odp_cpu_idling.h
+++ b/platform/linux-generic/arch/arm/odp_cpu_idling.h
@@ -30,12 +30,6 @@ static inline int wfe(void)
 	return 1;
 }
 
-static inline void doze(void)
-{
-	/* When using WFE do not stall the pipeline using other means */
-	odp_cpu_pause();
-}
-
 #define monitor128(addr, mo) lld((addr), (mo))
 #define monitor64(addr, mo) ll64((addr), (mo))
 #define monitor32(addr, mo) ll32((addr), (mo))
diff --git a/platform/linux-generic/arch/default/odp_cpu.h b/platform/linux-generic/arch/default/odp_cpu.h
index 18dd968fb..d8bc125c8 100644
--- a/platform/linux-generic/arch/default/odp_cpu.h
+++ b/platform/linux-generic/arch/default/odp_cpu.h
@@ -9,6 +9,10 @@
 #ifndef ODP_DEFAULT_CPU_H_
 #define ODP_DEFAULT_CPU_H_
 
+#ifndef _ODP_UNALIGNED
+#define _ODP_UNALIGNED 0
+#endif
+
 /******************************************************************************
  * Atomics
  *****************************************************************************/
diff --git a/platform/linux-generic/arch/default/odp_cpu_idling.h b/platform/linux-generic/arch/default/odp_cpu_idling.h
index 70b8cae6e..9d23ad20d 100644
--- a/platform/linux-generic/arch/default/odp_cpu_idling.h
+++ b/platform/linux-generic/arch/default/odp_cpu_idling.h
@@ -28,9 +28,4 @@ static inline int wfe(void)
 #define monitor32(addr, mo) __atomic_load_n((addr), (mo))
 #define monitor8(addr, mo) __atomic_load_n((addr), (mo))
 
-static inline void doze(void)
-{
-	odp_cpu_pause();
-}
-
 #endif
diff --git a/platform/linux-generic/arch/x86/odp_cpu.h b/platform/linux-generic/arch/x86/odp_cpu.h
new file mode 100644
index 000000000..8f8f22daf
--- /dev/null
+++ b/platform/linux-generic/arch/x86/odp_cpu.h
@@ -0,0 +1,14 @@
+/* Copyright (c) 2021, Nokia
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+#ifndef ODP_X86_CPU_H_
+#define ODP_X86_CPU_H_
+
+#define _ODP_UNALIGNED 1
+
+#include <default/odp_cpu.h>
+
+#endif