1 files changed, 740 insertions, 183 deletions

diff --git a/test/validation/api/atomic/atomic.c b/test/validation/api/atomic/atomic.c
index 54bd5ee3e..d4b15cf32 100644
--- a/test/validation/api/atomic/atomic.c
+++ b/test/validation/api/atomic/atomic.c
@@ -1,4 +1,5 @@
 /* Copyright (c) 2014-2018, Linaro Limited
+ * Copyright (c) 2021-2022 Nokia
  * All rights reserved.
  *
  * SPDX-License-Identifier:	 BSD-3-Clause
@@ -11,14 +12,11 @@
 #include <odp_cunit_common.h>
 #include <unistd.h>
 
-#define VERBOSE			0
-#define MAX_ITERATIONS		1000
-
 #define ADD_SUB_CNT		5
 
-#define CNT			50000
-#define U32_INIT_VAL		(1UL << 28)
-#define U64_INIT_VAL		(1ULL << 33)
+#define CNT			100000ULL
+#define U32_INIT_VAL		(1UL << 31)
+#define U64_INIT_VAL		(1ULL << 63)
 #define U32_MAGIC		0xa23f65b2
 #define U64_MAGIC		0xf2e1c5430cb6a52e
 
@@ -26,9 +24,8 @@
 
 #define UNUSED			__attribute__((__unused__))
 
-#define CHECK_MAX_MIN		(1 << 0)
-#define CHECK_XCHG		(1 << 2)
-#define CHECK_CAS_128		(1 << 4)
+#define min(a, b) (a < b ? a : b)
+#define max(a, b) (a > b ? a : b)
 
 typedef __volatile uint32_t volatile_u32_t;
 typedef __volatile uint64_t volatile_u64_t;
@@ -45,59 +42,27 @@ typedef struct {
 	odp_atomic_u32_t a32u_xchg;
 
 	uint32_t g_num_threads;
-	uint32_t g_iterations;
-	uint32_t g_verbose;
 
 	odp_barrier_t global_barrier;
 } global_shared_mem_t;
 
-/* Per-thread memory */
-typedef struct {
-	global_shared_mem_t *global_mem;
-
-	int thread_id;
-	int thread_core;
-
-	volatile_u64_t delay_counter;
-} per_thread_mem_t;
-
 static odp_shm_t global_shm;
 static global_shared_mem_t *global_mem;
 
 /* Initialise per-thread memory */
-static per_thread_mem_t *thread_init(void)
+static void thread_init(void)
 {
 	global_shared_mem_t *global_mem;
-	per_thread_mem_t *per_thread_mem;
 	odp_shm_t global_shm;
-	uint32_t per_thread_mem_len;
-
-	per_thread_mem_len = sizeof(per_thread_mem_t);
-	per_thread_mem = malloc(per_thread_mem_len);
-	memset(per_thread_mem, 0, per_thread_mem_len);
-
-	per_thread_mem->delay_counter = 1;
-
-	per_thread_mem->thread_id = odp_thread_id();
-	per_thread_mem->thread_core = odp_cpu_id();
 
 	global_shm = odp_shm_lookup(GLOBAL_SHM_NAME);
 	global_mem = odp_shm_addr(global_shm);
 	CU_ASSERT_PTR_NOT_NULL(global_mem);
-
-	per_thread_mem->global_mem = global_mem;
-
-	return per_thread_mem;
-}
-
-static void thread_finalize(per_thread_mem_t *per_thread_mem)
-{
-	free(per_thread_mem);
 }
 
 static void test_atomic_inc_32(void)
 {
-	int i;
+	uint64_t i;
 
 	odp_barrier_wait(&global_mem->global_barrier);
 
@@ -107,7 +72,7 @@ static void test_atomic_inc_32(void)
 
 static void test_atomic_inc_64(void)
 {
-	int i;
+	uint64_t i;
 
 	odp_barrier_wait(&global_mem->global_barrier);
 
@@ -117,7 +82,7 @@ static void test_atomic_inc_64(void)
 
 static void test_atomic_dec_32(void)
 {
-	int i;
+	uint64_t i;
 
 	odp_barrier_wait(&global_mem->global_barrier);
 
@@ -127,7 +92,7 @@ static void test_atomic_dec_32(void)
 
 static void test_atomic_dec_64(void)
 {
-	int i;
+	uint64_t i;
 
 	odp_barrier_wait(&global_mem->global_barrier);
 
@@ -137,7 +102,7 @@ static void test_atomic_dec_64(void)
 
 static void test_atomic_fetch_inc_32(void)
 {
-	int i;
+	uint64_t i;
 
 	odp_barrier_wait(&global_mem->global_barrier);
 
@@ -147,7 +112,7 @@ static void test_atomic_fetch_inc_32(void)
 
 static void test_atomic_fetch_inc_64(void)
 {
-	int i;
+	uint64_t i;
 
 	odp_barrier_wait(&global_mem->global_barrier);
 
@@ -157,7 +122,7 @@ static void test_atomic_fetch_inc_64(void)
 
 static void test_atomic_fetch_dec_32(void)
 {
-	int i;
+	uint64_t i;
 
 	odp_barrier_wait(&global_mem->global_barrier);
 
@@ -167,7 +132,7 @@ static void test_atomic_fetch_dec_32(void)
 
 static void test_atomic_fetch_dec_64(void)
 {
-	int i;
+	uint64_t i;
 
 	odp_barrier_wait(&global_mem->global_barrier);
 
@@ -177,7 +142,7 @@ static void test_atomic_fetch_dec_64(void)
 
 static void test_atomic_add_32(void)
 {
-	int i;
+	uint64_t i;
 
 	odp_barrier_wait(&global_mem->global_barrier);
 
@@ -187,7 +152,7 @@ static void test_atomic_add_32(void)
 
 static void test_atomic_add_64(void)
 {
-	int i;
+	uint64_t i;
 
 	odp_barrier_wait(&global_mem->global_barrier);
 
@@ -197,7 +162,7 @@ static void test_atomic_add_64(void)
 
 static void test_atomic_sub_32(void)
 {
-	int i;
+	uint64_t i;
 
 	odp_barrier_wait(&global_mem->global_barrier);
 
@@ -207,7 +172,7 @@ static void test_atomic_sub_32(void)
 
 static void test_atomic_sub_64(void)
 {
-	int i;
+	uint64_t i;
 
 	odp_barrier_wait(&global_mem->global_barrier);
 
@@ -217,7 +182,7 @@ static void test_atomic_sub_64(void)
 
 static void test_atomic_fetch_add_32(void)
 {
-	int i;
+	uint64_t i;
 
 	odp_barrier_wait(&global_mem->global_barrier);
 
@@ -227,7 +192,7 @@ static void test_atomic_fetch_add_32(void)
 
 static void test_atomic_fetch_add_64(void)
 {
-	int i;
+	uint64_t i;
 
 	odp_barrier_wait(&global_mem->global_barrier);
 
@@ -237,7 +202,7 @@ static void test_atomic_fetch_add_64(void)
 
 static void test_atomic_fetch_sub_32(void)
 {
-	int i;
+	uint64_t i;
 
 	odp_barrier_wait(&global_mem->global_barrier);
 
@@ -247,7 +212,7 @@ static void test_atomic_fetch_sub_32(void)
 
 static void test_atomic_fetch_sub_64(void)
 {
-	int i;
+	uint64_t i;
 
 	odp_barrier_wait(&global_mem->global_barrier);
 
@@ -257,7 +222,7 @@ static void test_atomic_fetch_sub_64(void)
 
 static void test_atomic_min_32(void)
 {
-	int i;
+	uint64_t i;
 	uint32_t tmp;
 
 	odp_barrier_wait(&global_mem->global_barrier);
@@ -270,8 +235,7 @@ static void test_atomic_min_32(void)
 
 static void test_atomic_min_64(void)
 {
-	int i;
-	uint64_t tmp;
+	uint64_t i, tmp;
 
 	odp_barrier_wait(&global_mem->global_barrier);
 
@@ -283,7 +247,7 @@ static void test_atomic_min_64(void)
 
 static void test_atomic_max_32(void)
 {
-	int i;
+	uint64_t i;
 	uint32_t tmp;
 
 	odp_barrier_wait(&global_mem->global_barrier);
@@ -296,8 +260,7 @@ static void test_atomic_max_32(void)
 
 static void test_atomic_max_64(void)
 {
-	int i;
-	uint64_t tmp;
+	uint64_t i, tmp;
 
 	odp_barrier_wait(&global_mem->global_barrier);
 
@@ -309,23 +272,111 @@ static void test_atomic_max_64(void)
 
 static void test_atomic_cas_inc_32(void)
 {
-	int i;
-	uint32_t old;
+	uint64_t i, old_mismatch = 0;
+	uint32_t old, old_old;
 	odp_atomic_u32_t *a32u = &global_mem->a32u;
 
 	odp_barrier_wait(&global_mem->global_barrier);
 
 	for (i = 0; i < CNT; i++) {
 		old = odp_atomic_load_u32(a32u);
+		old_old = old;
 
-		while (odp_atomic_cas_u32(a32u, &old, old + 1) == 0)
-			;
+		while (odp_atomic_cas_u32(a32u, &old, old + 1) == 0) {
+			if (old == old_old)
+				old_mismatch++;
+
+			old_old = old;
+		}
+
+		if (old != old_old)
+			old_mismatch++;
+	}
+
+	CU_ASSERT(old_mismatch == 0);
+}
+
+static void test_atomic_cas_acq_inc_32(void)
+{
+	uint64_t i, old_mismatch = 0;
+	uint32_t old, old_old;
+	odp_atomic_u32_t *a32u = &global_mem->a32u;
+
+	odp_barrier_wait(&global_mem->global_barrier);
+
+	for (i = 0; i < CNT; i++) {
+		old = odp_atomic_load_u32(a32u);
+		old_old = old;
+
+		while (odp_atomic_cas_acq_u32(a32u, &old, old + 1) == 0) {
+			if (old == old_old)
+				old_mismatch++;
+
+			old_old = old;
+		}
+
+		if (old != old_old)
+			old_mismatch++;
+	}
+
+	CU_ASSERT(old_mismatch == 0);
+}
+
+static void test_atomic_cas_rel_inc_32(void)
+{
+	uint64_t i, old_mismatch = 0;
+	uint32_t old, old_old;
+	odp_atomic_u32_t *a32u = &global_mem->a32u;
+
+	odp_barrier_wait(&global_mem->global_barrier);
+
+	for (i = 0; i < CNT; i++) {
+		old = odp_atomic_load_u32(a32u);
+		old_old = old;
+
+		while (odp_atomic_cas_rel_u32(a32u, &old, old + 1) == 0) {
+			if (old == old_old)
+				old_mismatch++;
+
+			old_old = old;
+		}
+
+		if (old != old_old)
+			old_mismatch++;
+	}
+
+	CU_ASSERT(old_mismatch == 0);
+}
+
+static void test_atomic_cas_acq_rel_inc_32(void)
+{
+	uint64_t i, old_mismatch = 0;
+	uint32_t old, old_old;
+	odp_atomic_u32_t *a32u = &global_mem->a32u;
+
+	odp_barrier_wait(&global_mem->global_barrier);
+
+	for (i = 0; i < CNT; i++) {
+		old = odp_atomic_load_u32(a32u);
+		old_old = old;
+
+		while (odp_atomic_cas_acq_rel_u32(a32u, &old, old + 1) == 0) {
+			if (old == old_old)
+				old_mismatch++;
+
+			old_old = old;
+		}
+
+		if (old != old_old)
+			old_mismatch++;
 	}
+
+	CU_ASSERT(old_mismatch == 0);
 }
 
 static void test_atomic_cas_dec_32(void)
 {
-	int i;
+	uint64_t i;
 	uint32_t old;
 	odp_atomic_u32_t *a32u = &global_mem->a32u;
 
@@ -341,24 +392,107 @@ static void test_atomic_cas_dec_32(void)
 
 static void test_atomic_cas_inc_64(void)
 {
-	int i;
-	uint64_t old;
+	uint64_t i, old, old_old, old_mismatch = 0;
 	odp_atomic_u64_t *a64u = &global_mem->a64u;
 
 	odp_barrier_wait(&global_mem->global_barrier);
 
 	for (i = 0; i < CNT; i++) {
 		old = odp_atomic_load_u64(a64u);
+		old_old = old;
 
-		while (odp_atomic_cas_u64(a64u, &old, old + 1) == 0)
-			;
+		while (odp_atomic_cas_u64(a64u, &old, old + 1) == 0) {
+			if (old == old_old)
+				old_mismatch++;
+
+			old_old = old;
+		}
+
+		if (old != old_old)
+			old_mismatch++;
+	}
+
+	CU_ASSERT(old_mismatch == 0);
+}
+
+static void test_atomic_cas_acq_inc_64(void)
+{
+	uint64_t i, old, old_old, old_mismatch = 0;
+	odp_atomic_u64_t *a64u = &global_mem->a64u;
+
+	odp_barrier_wait(&global_mem->global_barrier);
+
+	for (i = 0; i < CNT; i++) {
+		old = odp_atomic_load_u64(a64u);
+		old_old = old;
+
+		while (odp_atomic_cas_acq_u64(a64u, &old, old + 1) == 0) {
+			if (old == old_old)
+				old_mismatch++;
+
+			old_old = old;
+		}
+
+		if (old != old_old)
+			old_mismatch++;
+	}
+
+	CU_ASSERT(old_mismatch == 0);
+}
+
+static void test_atomic_cas_rel_inc_64(void)
+{
+	uint64_t i, old, old_old, old_mismatch = 0;
+	odp_atomic_u64_t *a64u = &global_mem->a64u;
+
+	odp_barrier_wait(&global_mem->global_barrier);
+
+	for (i = 0; i < CNT; i++) {
+		old = odp_atomic_load_u64(a64u);
+		old_old = old;
+
+		while (odp_atomic_cas_rel_u64(a64u, &old, old + 1) == 0) {
+			if (old == old_old)
+				old_mismatch++;
+
+			old_old = old;
+		}
+
+		if (old != old_old)
+			old_mismatch++;
+	}
+
+	CU_ASSERT(old_mismatch == 0);
+}
+
+static void test_atomic_cas_acq_rel_inc_64(void)
+{
+	uint64_t i, old, old_old, old_mismatch = 0;
+	odp_atomic_u64_t *a64u = &global_mem->a64u;
+
+	odp_barrier_wait(&global_mem->global_barrier);
+
+	for (i = 0; i < CNT; i++) {
+		old = odp_atomic_load_u64(a64u);
+		old_old = old;
+
+		while (odp_atomic_cas_acq_rel_u64(a64u, &old, old + 1) == 0) {
+			if (old == old_old)
+				old_mismatch++;
+
+			old_old = old;
+		}
+
+		if (old != old_old)
+			old_mismatch++;
 	}
+
+	CU_ASSERT(old_mismatch == 0);
 }
 
 static void test_atomic_cas_dec_64(void)
 {
-	int i;
-	uint64_t old;
+	uint64_t i, old;
 	odp_atomic_u64_t *a64u = &global_mem->a64u;
 
 	odp_barrier_wait(&global_mem->global_barrier);
@@ -371,55 +505,59 @@ static void test_atomic_cas_dec_64(void)
 	}
 }
 
+#define BUF_SIZE (64 * 1024)
+
 static void test_atomic_xchg_32(void)
 {
 	uint32_t old, new;
-	int i;
-	odp_atomic_u32_t *a32u = &global_mem->a32u;
+	uint64_t i;
 	odp_atomic_u32_t *a32u_xchg = &global_mem->a32u_xchg;
+	uint8_t buf[BUF_SIZE];
+	uint64_t seed = odp_thread_id();
+	uint64_t count_old = 0, count_new = 0;
+
+	odp_random_test_data(buf, BUF_SIZE, &seed);
 
 	odp_barrier_wait(&global_mem->global_barrier);
 
 	for (i = 0; i < CNT; i++) {
-		new = odp_atomic_fetch_inc_u32(a32u);
+		new = buf[i & (BUF_SIZE - 1)];
 		old = odp_atomic_xchg_u32(a32u_xchg, new);
-
-		if (old & 0x1)
-			odp_atomic_xchg_u32(a32u_xchg, 0);
-		else
-			odp_atomic_xchg_u32(a32u_xchg, 1);
+		count_old += old;
+		count_new += new;
 	}
 
-	odp_atomic_sub_u32(a32u, CNT);
-	odp_atomic_xchg_u32(a32u_xchg, U32_MAGIC);
+	odp_atomic_add_u32(a32u_xchg, count_old);
+	odp_atomic_sub_u32(a32u_xchg, count_new);
 }
 
 static void test_atomic_xchg_64(void)
 {
 	uint64_t old, new;
-	int i;
-	odp_atomic_u64_t *a64u = &global_mem->a64u;
+	uint64_t i;
 	odp_atomic_u64_t *a64u_xchg = &global_mem->a64u_xchg;
+	uint8_t buf[BUF_SIZE];
+	uint64_t seed = odp_thread_id();
+	uint64_t count_old = 0, count_new = 0;
+
+	odp_random_test_data(buf, BUF_SIZE, &seed);
 
 	odp_barrier_wait(&global_mem->global_barrier);
 
 	for (i = 0; i < CNT; i++) {
-		new = odp_atomic_fetch_inc_u64(a64u);
+		new = buf[i & (BUF_SIZE - 1)];
 		old = odp_atomic_xchg_u64(a64u_xchg, new);
-
-		if (old & 0x1)
-			odp_atomic_xchg_u64(a64u_xchg, 0);
-		else
-			odp_atomic_xchg_u64(a64u_xchg, 1);
+		count_old += old;
+		count_new += new;
 	}
 
-	odp_atomic_sub_u64(a64u, CNT);
-	odp_atomic_xchg_u64(a64u_xchg, U64_MAGIC);
+	odp_atomic_add_u64(a64u_xchg, count_old);
+	odp_atomic_sub_u64(a64u_xchg, count_new);
 }
 
 static void test_atomic_non_relaxed_32(void)
 {
-	int i;
+	uint64_t i;
 	uint32_t tmp;
 	odp_atomic_u32_t *a32u = &global_mem->a32u;
 	odp_atomic_u32_t *a32u_min = &global_mem->a32u_min;
@@ -447,17 +585,14 @@ static void test_atomic_non_relaxed_32(void)
 			;
 
 		tmp = odp_atomic_load_u32(a32u_xchg);
-		/* finally set value for validation */
-		while (odp_atomic_cas_acq_rel_u32(a32u_xchg, &tmp, U32_MAGIC)
-		       == 0)
+		while (odp_atomic_cas_acq_rel_u32(a32u_xchg, &tmp, tmp + 1) == 0)
 			;
 	}
 }
 
 static void test_atomic_non_relaxed_64(void)
 {
-	int i;
-	uint64_t tmp;
+	uint64_t i, tmp;
 	odp_atomic_u64_t *a64u = &global_mem->a64u;
 	odp_atomic_u64_t *a64u_min = &global_mem->a64u_min;
 	odp_atomic_u64_t *a64u_max = &global_mem->a64u_max;
@@ -484,16 +619,15 @@ static void test_atomic_non_relaxed_64(void)
 			;
 
 		tmp = odp_atomic_load_u64(a64u_xchg);
-		/* finally set value for validation */
-		while (odp_atomic_cas_acq_rel_u64(a64u_xchg, &tmp, U64_MAGIC)
-		       == 0)
+		while (odp_atomic_cas_acq_rel_u64(a64u_xchg, &tmp, tmp + 1) == 0)
 			;
 	}
 }
 
 static void test_atomic_relaxed_128(void)
 {
-	int i, ret;
+	int ret;
+	uint64_t i;
 	odp_u128_t old, new;
 	odp_atomic_u128_t *a128u = &global_mem->a128u;
 
@@ -514,7 +648,8 @@ static void test_atomic_relaxed_128(void)
 
 static void test_atomic_non_relaxed_128_acq(void)
 {
-	int i, ret;
+	int ret;
+	uint64_t i;
 	odp_u128_t old, new;
 	odp_atomic_u128_t *a128u = &global_mem->a128u;
 
@@ -535,7 +670,8 @@ static void test_atomic_non_relaxed_128_acq(void)
 
 static void test_atomic_non_relaxed_128_rel(void)
 {
-	int i, ret;
+	int ret;
+	uint64_t i;
 	odp_u128_t old, new;
 	odp_atomic_u128_t *a128u = &global_mem->a128u;
 
@@ -556,7 +692,8 @@ static void test_atomic_non_relaxed_128_rel(void)
 
 static void test_atomic_non_relaxed_128_acq_rel(void)
 {
-	int i, ret;
+	int ret;
+	uint64_t i;
 	odp_u128_t old, new;
 	odp_atomic_u128_t *a128u = &global_mem->a128u;
 
@@ -623,6 +760,42 @@ static void test_atomic_fetch_add_sub_64(void)
 	test_atomic_fetch_sub_64();
 }
 
+static void test_atomic_inc_add_32(void)
+{
+	test_atomic_inc_32();
+	test_atomic_fetch_inc_32();
+	test_atomic_add_32();
+	test_atomic_fetch_add_32();
+	test_atomic_cas_inc_32();
+}
+
+static void test_atomic_inc_add_64(void)
+{
+	test_atomic_inc_64();
+	test_atomic_fetch_inc_64();
+	test_atomic_add_64();
+	test_atomic_fetch_add_64();
+	test_atomic_cas_inc_64();
+}
+
+static void test_atomic_dec_sub_32(void)
+{
+	test_atomic_dec_32();
+	test_atomic_fetch_dec_32();
+	test_atomic_sub_32();
+	test_atomic_fetch_sub_32();
+	test_atomic_cas_dec_32();
+}
+
+static void test_atomic_dec_sub_64(void)
+{
+	test_atomic_dec_64();
+	test_atomic_fetch_dec_64();
+	test_atomic_sub_64();
+	test_atomic_fetch_sub_64();
+	test_atomic_cas_dec_64();
+}
+
 static void test_atomic_max_min_32(void)
 {
 	test_atomic_max_32();
@@ -691,39 +864,105 @@ static void test_atomic_store(void)
 	odp_atomic_store_u128(&global_mem->a128u, a128u_tmp);
 }
 
-static void test_atomic_validate(int check)
+static void test_atomic_validate_init_val_32_64(void)
 {
 	CU_ASSERT(U32_INIT_VAL == odp_atomic_load_u32(&global_mem->a32u));
 	CU_ASSERT(U64_INIT_VAL == odp_atomic_load_u64(&global_mem->a64u));
+}
 
-	odp_u128_t a128u_tmp;
+static void test_atomic_validate_init_val_128(void)
+{
+	odp_u128_t a128u = odp_atomic_load_u128(&global_mem->a128u);
 
-	a128u_tmp = odp_atomic_load_u128(&global_mem->a128u);
+	CU_ASSERT(U64_INIT_VAL == a128u.u64[0]);
+	CU_ASSERT(U64_INIT_VAL == a128u.u64[1]);
+}
 
-	if (check & CHECK_CAS_128) {
-		uint64_t iterations = 0;
+static void test_atomic_validate_init_val(void)
+{
+	test_atomic_validate_init_val_32_64();
+	test_atomic_validate_init_val_128();
+}
 
-		iterations = a128u_tmp.u64[0] - a128u_tmp.u64[1];
-		CU_ASSERT(iterations == 4 * CNT * global_mem->g_num_threads);
-	} else {
-		CU_ASSERT(U64_INIT_VAL == a128u_tmp.u64[0]);
-		CU_ASSERT(U64_INIT_VAL == a128u_tmp.u64[1]);
-	}
+static void test_atomic_validate_inc_add(void)
+{
+	test_atomic_validate_init_val_128();
 
-	if (check & CHECK_MAX_MIN) {
-		CU_ASSERT(odp_atomic_load_u32(&global_mem->a32u_max) >
-			  odp_atomic_load_u32(&global_mem->a32u_min));
+	/* Two increment tests, one cas increment test and two add tests. */
+	const uint64_t total_count = CNT * (3 + 2 * ADD_SUB_CNT) * global_mem->g_num_threads;
+	const uint32_t a32u = U32_INIT_VAL + total_count;
 
-		CU_ASSERT(odp_atomic_load_u64(&global_mem->a64u_max) >
-			  odp_atomic_load_u64(&global_mem->a64u_min));
-	}
+	CU_ASSERT(a32u == odp_atomic_load_u32(&global_mem->a32u));
+	CU_ASSERT(U64_INIT_VAL + total_count == odp_atomic_load_u64(&global_mem->a64u));
+}
 
-	if (check & CHECK_XCHG) {
-		CU_ASSERT(odp_atomic_load_u32(&global_mem->a32u_xchg) ==
-				U32_MAGIC);
-		CU_ASSERT(odp_atomic_load_u64(&global_mem->a64u_xchg) ==
-				U64_MAGIC);
-	}
+static void test_atomic_validate_dec_sub(void)
+{
+	test_atomic_validate_init_val_128();
+
+	/* Two decrement tests, one cas decrement test and two sub tests. */
+	const uint64_t total_count = CNT * (3 + 2 * ADD_SUB_CNT) * global_mem->g_num_threads;
+	const uint32_t a32u = U32_INIT_VAL - total_count;
+
+	CU_ASSERT(a32u == odp_atomic_load_u32(&global_mem->a32u));
+	CU_ASSERT(U64_INIT_VAL - total_count == odp_atomic_load_u64(&global_mem->a64u));
+}
+
+static void test_atomic_validate_cas_inc_dec(void)
+{
+	test_atomic_validate_init_val_32_64();
+
+	odp_u128_t a128u = odp_atomic_load_u128(&global_mem->a128u);
+	const uint64_t iterations = a128u.u64[0] - a128u.u64[1];
+
+	CU_ASSERT(iterations == 4 * CNT * global_mem->g_num_threads);
+}
+
+static void test_atomic_validate_max_min(void)
+{
+	test_atomic_validate_init_val();
+
+	const uint64_t total_count = CNT * global_mem->g_num_threads;
+	/*
+	 * Max is the result of fetch_inc, so the final max value is total_count - 1. In
+	 * a long test, counter may overflow, in which case max is saturated at
+	 * UINT32_MAX, and min at 0.
+	 */
+	const uint32_t a32u_max = min(U32_INIT_VAL + total_count - 1, UINT32_MAX);
+	const uint32_t a32u_min = U32_INIT_VAL + total_count - 1 > UINT32_MAX ? 0 : U32_INIT_VAL;
+
+	CU_ASSERT(odp_atomic_load_u32(&global_mem->a32u_max) == a32u_max);
+	CU_ASSERT(odp_atomic_load_u32(&global_mem->a32u_min) == a32u_min);
+	CU_ASSERT(odp_atomic_load_u64(&global_mem->a64u_max) == U64_INIT_VAL + total_count - 1);
+	CU_ASSERT(odp_atomic_load_u64(&global_mem->a64u_min) == U64_INIT_VAL);
+}
+
+static void test_atomic_validate_xchg(void)
+{
+	test_atomic_validate_init_val();
+
+	CU_ASSERT(odp_atomic_load_u32(&global_mem->a32u_xchg) == U32_INIT_VAL);
+	CU_ASSERT(odp_atomic_load_u64(&global_mem->a64u_xchg) == U64_INIT_VAL);
+}
+
+static void test_atomic_validate_non_relaxed(void)
+{
+	test_atomic_validate_init_val();
+
+	const uint64_t total_count = CNT * global_mem->g_num_threads;
+	/* 3 increments per round. */
+	const uint32_t a32u = U32_INIT_VAL + 3 * total_count;
+	/* 1 increment per round. */
+	const uint32_t a32u_max = U32_INIT_VAL + total_count;
+	const uint32_t a32u_min = U32_INIT_VAL - total_count;
+
+	CU_ASSERT(odp_atomic_load_u32(&global_mem->a32u_xchg) == a32u);
+	CU_ASSERT(odp_atomic_load_u64(&global_mem->a64u_xchg) == U64_INIT_VAL + 3 * total_count);
+
+	CU_ASSERT(odp_atomic_load_u32(&global_mem->a32u_max) == a32u_max);
+	CU_ASSERT(odp_atomic_load_u32(&global_mem->a32u_min) == a32u_min);
+	CU_ASSERT(odp_atomic_load_u64(&global_mem->a64u_max) == U64_INIT_VAL + total_count);
+	CU_ASSERT(odp_atomic_load_u64(&global_mem->a64u_min) == U64_INIT_VAL - total_count);
 }
 
 static int atomic_init(odp_instance_t *inst)
@@ -762,8 +1001,6 @@ static int atomic_init(odp_instance_t *inst)
 	memset(global_mem, 0, sizeof(global_shared_mem_t));
 
 	global_mem->g_num_threads = MAX_WORKERS;
-	global_mem->g_iterations = MAX_ITERATIONS;
-	global_mem->g_verbose = VERBOSE;
 
 	workers_count = odp_cpumask_default_worker(&mask, 0);
 
@@ -810,121 +1047,238 @@ static int atomic_term(odp_instance_t inst)
 }
 
 /* Atomic tests */
-static int test_atomic_inc_dec_thread(void *arg UNUSED)
+
+static int test_atomic_inc_thread(void *arg UNUSED)
+{
+	thread_init();
+	test_atomic_inc_32();
+	test_atomic_inc_64();
+
+	return CU_get_number_of_failures();
+}
+
+static int test_atomic_dec_thread(void *arg UNUSED)
 {
-	per_thread_mem_t *per_thread_mem;
+	thread_init();
+	test_atomic_dec_32();
+	test_atomic_dec_64();
 
-	per_thread_mem = thread_init();
+	return CU_get_number_of_failures();
+}
+
+static int test_atomic_add_thread(void *arg UNUSED)
+{
+	thread_init();
+	test_atomic_add_32();
+	test_atomic_add_64();
+
+	return CU_get_number_of_failures();
+}
+
+static int test_atomic_sub_thread(void *arg UNUSED)
+{
+	thread_init();
+	test_atomic_sub_32();
+	test_atomic_sub_64();
+
+	return CU_get_number_of_failures();
+}
+
+static int test_atomic_fetch_inc_thread(void *arg UNUSED)
+{
+	thread_init();
+	test_atomic_fetch_inc_32();
+	test_atomic_fetch_inc_64();
+
+	return CU_get_number_of_failures();
+}
+
+static int test_atomic_fetch_dec_thread(void *arg UNUSED)
+{
+	thread_init();
+	test_atomic_fetch_dec_32();
+	test_atomic_fetch_dec_64();
+
+	return CU_get_number_of_failures();
+}
+
+static int test_atomic_fetch_add_thread(void *arg UNUSED)
+{
+	thread_init();
+	test_atomic_fetch_add_32();
+	test_atomic_fetch_add_64();
+
+	return CU_get_number_of_failures();
+}
+
+static int test_atomic_fetch_sub_thread(void *arg UNUSED)
+{
+	thread_init();
+	test_atomic_fetch_sub_32();
+	test_atomic_fetch_sub_64();
+
+	return CU_get_number_of_failures();
+}
+
+static int test_atomic_max_thread(void *arg UNUSED)
+{
+	thread_init();
+	test_atomic_max_32();
+	test_atomic_max_64();
+
+	return CU_get_number_of_failures();
+}
+
+static int test_atomic_min_thread(void *arg UNUSED)
+{
+	thread_init();
+	test_atomic_min_32();
+	test_atomic_min_64();
+
+	return CU_get_number_of_failures();
+}
+
+static int test_atomic_cas_thread(void *arg UNUSED)
+{
+	thread_init();
+	test_atomic_cas_inc_32();
+	test_atomic_cas_inc_64();
+	test_atomic_relaxed_128();
+
+	return CU_get_number_of_failures();
+}
+
+static int test_atomic_cas_acq_thread(void *arg UNUSED)
+{
+	thread_init();
+	test_atomic_cas_acq_inc_32();
+	test_atomic_cas_acq_inc_64();
+	test_atomic_non_relaxed_128_acq();
+
+	return CU_get_number_of_failures();
+}
+
+static int test_atomic_cas_rel_thread(void *arg UNUSED)
+{
+	thread_init();
+	test_atomic_cas_rel_inc_32();
+	test_atomic_cas_rel_inc_64();
+	test_atomic_non_relaxed_128_rel();
+
+	return CU_get_number_of_failures();
+}
+
+static int test_atomic_cas_acq_rel_thread(void *arg UNUSED)
+{
+	thread_init();
+	test_atomic_cas_acq_rel_inc_32();
+	test_atomic_cas_acq_rel_inc_64();
+	test_atomic_non_relaxed_128_acq_rel();
+
+	return CU_get_number_of_failures();
+}
+
+static int test_atomic_inc_dec_thread(void *arg UNUSED)
+{
+	thread_init();
 	test_atomic_inc_dec_32();
 	test_atomic_inc_dec_64();
 
-	thread_finalize(per_thread_mem);
-
 	return CU_get_number_of_failures();
 }
 
 static int test_atomic_add_sub_thread(void *arg UNUSED)
 {
-	per_thread_mem_t *per_thread_mem;
-
-	per_thread_mem = thread_init();
+	thread_init();
 	test_atomic_add_sub_32();
 	test_atomic_add_sub_64();
 
-	thread_finalize(per_thread_mem);
-
 	return CU_get_number_of_failures();
 }
 
 static int test_atomic_fetch_inc_dec_thread(void *arg UNUSED)
 {
-	per_thread_mem_t *per_thread_mem;
-
-	per_thread_mem = thread_init();
+	thread_init();
 	test_atomic_fetch_inc_dec_32();
 	test_atomic_fetch_inc_dec_64();
 
-	thread_finalize(per_thread_mem);
-
 	return CU_get_number_of_failures();
 }
 
 static int test_atomic_fetch_add_sub_thread(void *arg UNUSED)
 {
-	per_thread_mem_t *per_thread_mem;
-
-	per_thread_mem = thread_init();
+	thread_init();
 	test_atomic_fetch_add_sub_32();
 	test_atomic_fetch_add_sub_64();
 
-	thread_finalize(per_thread_mem);
+	return CU_get_number_of_failures();
+}
+
+static int test_atomic_inc_add_thread(void *arg UNUSED)
+{
+	thread_init();
+	test_atomic_inc_add_32();
+	test_atomic_inc_add_64();
 
 	return CU_get_number_of_failures();
 }
 
-static int test_atomic_max_min_thread(void *arg UNUSED)
+static int test_atomic_dec_sub_thread(void *arg UNUSED)
 {
-	per_thread_mem_t *per_thread_mem;
+	thread_init();
+	test_atomic_dec_sub_32();
+	test_atomic_dec_sub_64();
 
-	per_thread_mem = thread_init();
+	return CU_get_number_of_failures();
+}
+
+static int test_atomic_max_min_thread(void *arg UNUSED)
+{
+	thread_init();
 	test_atomic_max_min_32();
 	test_atomic_max_min_64();
 
-	thread_finalize(per_thread_mem);
-
 	return CU_get_number_of_failures();
 }
 
 static int test_atomic_cas_inc_dec_thread(void *arg UNUSED)
 {
-	per_thread_mem_t *per_thread_mem;
-
-	per_thread_mem = thread_init();
+	thread_init();
 	test_atomic_cas_inc_dec_32();
 	test_atomic_cas_inc_dec_64();
 	test_atomic_cas_inc_128();
 
-	thread_finalize(per_thread_mem);
-
 	return CU_get_number_of_failures();
 }
 
 static int test_atomic_xchg_thread(void *arg UNUSED)
 {
-	per_thread_mem_t *per_thread_mem;
-
-	per_thread_mem = thread_init();
+	thread_init();
 	test_atomic_xchg_32();
 	test_atomic_xchg_64();
 
-	thread_finalize(per_thread_mem);
-
 	return CU_get_number_of_failures();
 }
 
 static int test_atomic_non_relaxed_thread(void *arg UNUSED)
 {
-	per_thread_mem_t *per_thread_mem;
-
-	per_thread_mem = thread_init();
+	thread_init();
 	test_atomic_non_relaxed_32();
 	test_atomic_non_relaxed_64();
 
-	thread_finalize(per_thread_mem);
-
 	return CU_get_number_of_failures();
 }
 
-static void test_atomic_functional(int func_ptr(void *), int check)
+static void test_atomic_functional(int test_fn(void *), void validate_fn(void))
 {
 	pthrd_arg arg;
 
 	arg.numthrds = global_mem->g_num_threads;
 	test_atomic_init();
 	test_atomic_store();
-	odp_cunit_thread_create(func_ptr, &arg);
+	odp_cunit_thread_create(test_fn, &arg);
 	odp_cunit_thread_exit(&arg);
-	test_atomic_validate(check);
+	validate_fn();
 }
 
 static void test_atomic_op_lock_free_set(void)
@@ -1077,45 +1431,231 @@ static void test_atomic_op_lock_free_128(void)
 	}
 }
 
+static void atomic_test_atomic_init(void)
+{
+	uint64_t i;
+	odp_atomic_u128_t *a128u = &global_mem->a128u;
+
+	for (i = 0; i < CNT; i++) {
+		odp_u128_t a128u_tmp;
+
+		odp_atomic_init_u32(&global_mem->a32u, i);
+		odp_atomic_init_u64(&global_mem->a64u, i);
+		odp_atomic_init_u32(&global_mem->a32u_min, i);
+		odp_atomic_init_u32(&global_mem->a32u_max, i);
+		odp_atomic_init_u64(&global_mem->a64u_min, i);
+		odp_atomic_init_u64(&global_mem->a64u_max, i);
+		odp_atomic_init_u32(&global_mem->a32u_xchg, i);
+		odp_atomic_init_u64(&global_mem->a64u_xchg, i);
+
+		a128u_tmp.u64[0] = i;
+		a128u_tmp.u64[1] = i;
+		odp_atomic_init_u128(&global_mem->a128u, a128u_tmp);
+
+		CU_ASSERT(odp_atomic_load_u32(&global_mem->a32u) == i);
+		CU_ASSERT(odp_atomic_load_u64(&global_mem->a64u) == i);
+		CU_ASSERT(odp_atomic_load_u32(&global_mem->a32u_min) == i);
+		CU_ASSERT(odp_atomic_load_u32(&global_mem->a32u_max) == i);
+		CU_ASSERT(odp_atomic_load_u64(&global_mem->a64u_min) == i);
+		CU_ASSERT(odp_atomic_load_u64(&global_mem->a64u_max) == i);
+		CU_ASSERT(odp_atomic_load_u32(&global_mem->a32u_xchg) == i);
+		CU_ASSERT(odp_atomic_load_u64(&global_mem->a64u_xchg) == i);
+
+		a128u_tmp = odp_atomic_load_u128(a128u);
+		CU_ASSERT(a128u_tmp.u64[0] == i);
+		CU_ASSERT(a128u_tmp.u64[1] == i);
+	}
+}
+
+static void test_atomic_validate_inc(void)
+{
+	const uint64_t total_count = CNT * global_mem->g_num_threads;
+	const uint32_t a32u = U32_INIT_VAL + total_count;
+
+	CU_ASSERT(a32u == odp_atomic_load_u32(&global_mem->a32u));
+	CU_ASSERT(U64_INIT_VAL + total_count == odp_atomic_load_u64(&global_mem->a64u));
+}
+
+static void atomic_test_atomic_inc(void)
+{
+	test_atomic_functional(test_atomic_inc_thread, test_atomic_validate_inc);
+}
+
+static void test_atomic_validate_dec(void)
+{
+	const uint64_t total_count = CNT * global_mem->g_num_threads;
+	const uint32_t a32u = U32_INIT_VAL - total_count;
+
+	CU_ASSERT(a32u == odp_atomic_load_u32(&global_mem->a32u));
+	CU_ASSERT(U64_INIT_VAL - total_count == odp_atomic_load_u64(&global_mem->a64u));
+}
+
+static void atomic_test_atomic_dec(void)
+{
+	test_atomic_functional(test_atomic_dec_thread, test_atomic_validate_dec);
+}
+
+static void test_atomic_validate_add(void)
+{
+	const uint64_t total_count = CNT * ADD_SUB_CNT * global_mem->g_num_threads;
+	const uint32_t a32u = U32_INIT_VAL + total_count;
+
+	CU_ASSERT(a32u == odp_atomic_load_u32(&global_mem->a32u));
+	CU_ASSERT(U64_INIT_VAL + total_count == odp_atomic_load_u64(&global_mem->a64u));
+}
+
+static void atomic_test_atomic_add(void)
+{
+	test_atomic_functional(test_atomic_add_thread, test_atomic_validate_add);
+}
+
+static void test_atomic_validate_sub(void)
+{
+	const uint64_t total_count = CNT * ADD_SUB_CNT * global_mem->g_num_threads;
+	const uint32_t a32u = U32_INIT_VAL - total_count;
+
+	CU_ASSERT(a32u == odp_atomic_load_u32(&global_mem->a32u));
+	CU_ASSERT(U64_INIT_VAL - total_count == odp_atomic_load_u64(&global_mem->a64u));
+}
+
+static void atomic_test_atomic_sub(void)
+{
+	test_atomic_functional(test_atomic_sub_thread, test_atomic_validate_sub);
+}
+
+static void atomic_test_atomic_fetch_inc(void)
+{
+	test_atomic_functional(test_atomic_fetch_inc_thread, test_atomic_validate_inc);
+}
+
+static void atomic_test_atomic_fetch_dec(void)
+{
+	test_atomic_functional(test_atomic_fetch_dec_thread, test_atomic_validate_dec);
+}
+
+static void atomic_test_atomic_fetch_add(void)
+{
+	test_atomic_functional(test_atomic_fetch_add_thread, test_atomic_validate_add);
+}
+
+static void atomic_test_atomic_fetch_sub(void)
+{
+	test_atomic_functional(test_atomic_fetch_sub_thread, test_atomic_validate_sub);
+}
+
+static void test_atomic_validate_max(void)
+{
+	const uint64_t total_count = CNT * global_mem->g_num_threads - 1;
+	/* In a long test, counter may overflow, in which case max is saturated at UINT32_MAX. */
+	const uint32_t a32u_max = min(U32_INIT_VAL + total_count, UINT32_MAX);
+
+	CU_ASSERT(a32u_max == odp_atomic_load_u32(&global_mem->a32u_max));
+	CU_ASSERT(U64_INIT_VAL + total_count == odp_atomic_load_u64(&global_mem->a64u_max));
+}
+
+static void atomic_test_atomic_max(void)
+{
+	test_atomic_functional(test_atomic_max_thread, test_atomic_validate_max);
+}
+
+static void test_atomic_validate_min(void)
+{
+	const uint64_t total_count = CNT * global_mem->g_num_threads - 1;
+	/* In a long test, counter may underflow, in which case min is saturated at 0. */
+	const uint32_t a32u_min = max((int64_t)U32_INIT_VAL - (int64_t)total_count, 0);
+
+	CU_ASSERT(a32u_min == odp_atomic_load_u32(&global_mem->a32u_min));
+	CU_ASSERT(U64_INIT_VAL - total_count == odp_atomic_load_u64(&global_mem->a64u_min));
+}
+
+static void atomic_test_atomic_min(void)
+{
+	test_atomic_functional(test_atomic_min_thread, test_atomic_validate_min);
+}
+
+static void test_atomic_validate_cas_128(void)
+{
+	odp_u128_t a128u = odp_atomic_load_u128(&global_mem->a128u);
+	const uint64_t iterations = a128u.u64[0] - a128u.u64[1];
+
+	CU_ASSERT(iterations == CNT * global_mem->g_num_threads);
+}
+
+static void test_atomic_validate_cas(void)
+{
+	test_atomic_validate_inc();
+	test_atomic_validate_cas_128();
+}
+
+static void atomic_test_atomic_cas(void)
+{
+	test_atomic_functional(test_atomic_cas_thread, test_atomic_validate_cas);
+}
+
+static void atomic_test_atomic_cas_acq(void)
+{
+	test_atomic_functional(test_atomic_cas_acq_thread, test_atomic_validate_cas);
+}
+
+static void atomic_test_atomic_cas_rel(void)
+{
+	test_atomic_functional(test_atomic_cas_rel_thread, test_atomic_validate_cas);
+}
+
+static void atomic_test_atomic_cas_acq_rel(void)
+{
+	test_atomic_functional(test_atomic_cas_acq_rel_thread, test_atomic_validate_cas);
+}
+
 static void atomic_test_atomic_inc_dec(void)
 {
-	test_atomic_functional(test_atomic_inc_dec_thread, 0);
+	test_atomic_functional(test_atomic_inc_dec_thread, test_atomic_validate_init_val);
 }
 
 static void atomic_test_atomic_add_sub(void)
 {
-	test_atomic_functional(test_atomic_add_sub_thread, 0);
+	test_atomic_functional(test_atomic_add_sub_thread, test_atomic_validate_init_val);
 }
 
 static void atomic_test_atomic_fetch_inc_dec(void)
 {
-	test_atomic_functional(test_atomic_fetch_inc_dec_thread, 0);
+	test_atomic_functional(test_atomic_fetch_inc_dec_thread, test_atomic_validate_init_val);
 }
 
 static void atomic_test_atomic_fetch_add_sub(void)
 {
-	test_atomic_functional(test_atomic_fetch_add_sub_thread, 0);
+	test_atomic_functional(test_atomic_fetch_add_sub_thread, test_atomic_validate_init_val);
+}
+
+static void atomic_test_atomic_inc_add(void)
+{
+	test_atomic_functional(test_atomic_inc_add_thread, test_atomic_validate_inc_add);
+}
+
+static void atomic_test_atomic_dec_sub(void)
+{
+	test_atomic_functional(test_atomic_dec_sub_thread, test_atomic_validate_dec_sub);
 }
 
 static void atomic_test_atomic_max_min(void)
 {
-	test_atomic_functional(test_atomic_max_min_thread, CHECK_MAX_MIN);
+	test_atomic_functional(test_atomic_max_min_thread, test_atomic_validate_max_min);
 }
 
 static void atomic_test_atomic_cas_inc_dec(void)
 {
-	test_atomic_functional(test_atomic_cas_inc_dec_thread, CHECK_CAS_128);
+	test_atomic_functional(test_atomic_cas_inc_dec_thread, test_atomic_validate_cas_inc_dec);
 }
 
 static void atomic_test_atomic_xchg(void)
 {
-	test_atomic_functional(test_atomic_xchg_thread, CHECK_XCHG);
+	test_atomic_functional(test_atomic_xchg_thread, test_atomic_validate_xchg);
 }
 
 static void atomic_test_atomic_non_relaxed(void)
 {
 	test_atomic_functional(test_atomic_non_relaxed_thread,
-			       CHECK_MAX_MIN | CHECK_XCHG);
+			       test_atomic_validate_non_relaxed);
 }
 
 static void atomic_test_atomic_op_lock_free(void)
@@ -1126,10 +1666,27 @@ static void atomic_test_atomic_op_lock_free(void)
 }
 
 odp_testinfo_t atomic_suite_atomic[] = {
+	ODP_TEST_INFO(atomic_test_atomic_init),
+	ODP_TEST_INFO(atomic_test_atomic_inc),
+	ODP_TEST_INFO(atomic_test_atomic_dec),
+	ODP_TEST_INFO(atomic_test_atomic_add),
+	ODP_TEST_INFO(atomic_test_atomic_sub),
+	ODP_TEST_INFO(atomic_test_atomic_fetch_inc),
+	ODP_TEST_INFO(atomic_test_atomic_fetch_dec),
+	ODP_TEST_INFO(atomic_test_atomic_fetch_add),
+	ODP_TEST_INFO(atomic_test_atomic_fetch_sub),
+	ODP_TEST_INFO(atomic_test_atomic_max),
+	ODP_TEST_INFO(atomic_test_atomic_min),
+	ODP_TEST_INFO(atomic_test_atomic_cas),
+	ODP_TEST_INFO(atomic_test_atomic_cas_acq),
+	ODP_TEST_INFO(atomic_test_atomic_cas_rel),
+	ODP_TEST_INFO(atomic_test_atomic_cas_acq_rel),
 	ODP_TEST_INFO(atomic_test_atomic_inc_dec),
 	ODP_TEST_INFO(atomic_test_atomic_add_sub),
 	ODP_TEST_INFO(atomic_test_atomic_fetch_inc_dec),
 	ODP_TEST_INFO(atomic_test_atomic_fetch_add_sub),
+	ODP_TEST_INFO(atomic_test_atomic_inc_add),
+	ODP_TEST_INFO(atomic_test_atomic_dec_sub),
 	ODP_TEST_INFO(atomic_test_atomic_max_min),
 	ODP_TEST_INFO(atomic_test_atomic_cas_inc_dec),
 	ODP_TEST_INFO(atomic_test_atomic_xchg),