diff options
Diffstat (limited to 'test/validation/api/atomic/atomic.c')
-rw-r--r-- | test/validation/api/atomic/atomic.c | 923 |
1 files changed, 740 insertions, 183 deletions
diff --git a/test/validation/api/atomic/atomic.c b/test/validation/api/atomic/atomic.c index 54bd5ee3e..d4b15cf32 100644 --- a/test/validation/api/atomic/atomic.c +++ b/test/validation/api/atomic/atomic.c @@ -1,4 +1,5 @@ /* Copyright (c) 2014-2018, Linaro Limited + * Copyright (c) 2021-2022 Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -11,14 +12,11 @@ #include <odp_cunit_common.h> #include <unistd.h> -#define VERBOSE 0 -#define MAX_ITERATIONS 1000 - #define ADD_SUB_CNT 5 -#define CNT 50000 -#define U32_INIT_VAL (1UL << 28) -#define U64_INIT_VAL (1ULL << 33) +#define CNT 100000ULL +#define U32_INIT_VAL (1UL << 31) +#define U64_INIT_VAL (1ULL << 63) #define U32_MAGIC 0xa23f65b2 #define U64_MAGIC 0xf2e1c5430cb6a52e @@ -26,9 +24,8 @@ #define UNUSED __attribute__((__unused__)) -#define CHECK_MAX_MIN (1 << 0) -#define CHECK_XCHG (1 << 2) -#define CHECK_CAS_128 (1 << 4) +#define min(a, b) (a < b ? a : b) +#define max(a, b) (a > b ? a : b) typedef __volatile uint32_t volatile_u32_t; typedef __volatile uint64_t volatile_u64_t; @@ -45,59 +42,27 @@ typedef struct { odp_atomic_u32_t a32u_xchg; uint32_t g_num_threads; - uint32_t g_iterations; - uint32_t g_verbose; odp_barrier_t global_barrier; } global_shared_mem_t; -/* Per-thread memory */ -typedef struct { - global_shared_mem_t *global_mem; - - int thread_id; - int thread_core; - - volatile_u64_t delay_counter; -} per_thread_mem_t; - static odp_shm_t global_shm; static global_shared_mem_t *global_mem; /* Initialise per-thread memory */ -static per_thread_mem_t *thread_init(void) +static void thread_init(void) { global_shared_mem_t *global_mem; - per_thread_mem_t *per_thread_mem; odp_shm_t global_shm; - uint32_t per_thread_mem_len; - - per_thread_mem_len = sizeof(per_thread_mem_t); - per_thread_mem = malloc(per_thread_mem_len); - memset(per_thread_mem, 0, per_thread_mem_len); - - per_thread_mem->delay_counter = 1; - - per_thread_mem->thread_id = odp_thread_id(); - per_thread_mem->thread_core = odp_cpu_id(); global_shm = odp_shm_lookup(GLOBAL_SHM_NAME); global_mem = odp_shm_addr(global_shm); CU_ASSERT_PTR_NOT_NULL(global_mem); - - per_thread_mem->global_mem = global_mem; - - return per_thread_mem; -} - -static void thread_finalize(per_thread_mem_t *per_thread_mem) -{ - free(per_thread_mem); } static void test_atomic_inc_32(void) { - int i; + uint64_t i; odp_barrier_wait(&global_mem->global_barrier); @@ -107,7 +72,7 @@ static void test_atomic_inc_32(void) static void test_atomic_inc_64(void) { - int i; + uint64_t i; odp_barrier_wait(&global_mem->global_barrier); @@ -117,7 +82,7 @@ static void test_atomic_inc_64(void) static void test_atomic_dec_32(void) { - int i; + uint64_t i; odp_barrier_wait(&global_mem->global_barrier); @@ -127,7 +92,7 @@ static void test_atomic_dec_32(void) static void test_atomic_dec_64(void) { - int i; + uint64_t i; odp_barrier_wait(&global_mem->global_barrier); @@ -137,7 +102,7 @@ static void test_atomic_dec_64(void) static void test_atomic_fetch_inc_32(void) { - int i; + uint64_t i; odp_barrier_wait(&global_mem->global_barrier); @@ -147,7 +112,7 @@ static void test_atomic_fetch_inc_32(void) static void test_atomic_fetch_inc_64(void) { - int i; + uint64_t i; odp_barrier_wait(&global_mem->global_barrier); @@ -157,7 +122,7 @@ static void test_atomic_fetch_inc_64(void) static void test_atomic_fetch_dec_32(void) { - int i; + uint64_t i; odp_barrier_wait(&global_mem->global_barrier); @@ -167,7 +132,7 @@ static void test_atomic_fetch_dec_32(void) static void test_atomic_fetch_dec_64(void) { - int i; + uint64_t i; odp_barrier_wait(&global_mem->global_barrier); @@ -177,7 +142,7 @@ static void test_atomic_fetch_dec_64(void) static void test_atomic_add_32(void) { - int i; + uint64_t i; odp_barrier_wait(&global_mem->global_barrier); @@ -187,7 +152,7 @@ static void test_atomic_add_32(void) static void test_atomic_add_64(void) { - int i; + uint64_t i; odp_barrier_wait(&global_mem->global_barrier); @@ -197,7 +162,7 @@ static void test_atomic_add_64(void) static void test_atomic_sub_32(void) { - int i; + uint64_t i; odp_barrier_wait(&global_mem->global_barrier); @@ -207,7 +172,7 @@ static void test_atomic_sub_32(void) static void test_atomic_sub_64(void) { - int i; + uint64_t i; odp_barrier_wait(&global_mem->global_barrier); @@ -217,7 +182,7 @@ static void test_atomic_sub_64(void) static void test_atomic_fetch_add_32(void) { - int i; + uint64_t i; odp_barrier_wait(&global_mem->global_barrier); @@ -227,7 +192,7 @@ static void test_atomic_fetch_add_32(void) static void test_atomic_fetch_add_64(void) { - int i; + uint64_t i; odp_barrier_wait(&global_mem->global_barrier); @@ -237,7 +202,7 @@ static void test_atomic_fetch_add_64(void) static void test_atomic_fetch_sub_32(void) { - int i; + uint64_t i; odp_barrier_wait(&global_mem->global_barrier); @@ -247,7 +212,7 @@ static void test_atomic_fetch_sub_32(void) static void test_atomic_fetch_sub_64(void) { - int i; + uint64_t i; odp_barrier_wait(&global_mem->global_barrier); @@ -257,7 +222,7 @@ static void test_atomic_fetch_sub_64(void) static void test_atomic_min_32(void) { - int i; + uint64_t i; uint32_t tmp; odp_barrier_wait(&global_mem->global_barrier); @@ -270,8 +235,7 @@ static void test_atomic_min_32(void) static void test_atomic_min_64(void) { - int i; - uint64_t tmp; + uint64_t i, tmp; odp_barrier_wait(&global_mem->global_barrier); @@ -283,7 +247,7 @@ static void test_atomic_min_64(void) static void test_atomic_max_32(void) { - int i; + uint64_t i; uint32_t tmp; odp_barrier_wait(&global_mem->global_barrier); @@ -296,8 +260,7 @@ static void test_atomic_max_32(void) static void test_atomic_max_64(void) { - int i; - uint64_t tmp; + uint64_t i, tmp; odp_barrier_wait(&global_mem->global_barrier); @@ -309,23 +272,111 @@ static void test_atomic_max_64(void) static void test_atomic_cas_inc_32(void) { - int i; - uint32_t old; + uint64_t i, old_mismatch = 0; + uint32_t old, old_old; odp_atomic_u32_t *a32u = &global_mem->a32u; odp_barrier_wait(&global_mem->global_barrier); for (i = 0; i < CNT; i++) { old = odp_atomic_load_u32(a32u); + old_old = old; - while (odp_atomic_cas_u32(a32u, &old, old + 1) == 0) - ; + while (odp_atomic_cas_u32(a32u, &old, old + 1) == 0) { + if (old == old_old) + old_mismatch++; + + old_old = old; + } + + if (old != old_old) + old_mismatch++; + } + + CU_ASSERT(old_mismatch == 0); +} + +static void test_atomic_cas_acq_inc_32(void) +{ + uint64_t i, old_mismatch = 0; + uint32_t old, old_old; + odp_atomic_u32_t *a32u = &global_mem->a32u; + + odp_barrier_wait(&global_mem->global_barrier); + + for (i = 0; i < CNT; i++) { + old = odp_atomic_load_u32(a32u); + old_old = old; + + while (odp_atomic_cas_acq_u32(a32u, &old, old + 1) == 0) { + if (old == old_old) + old_mismatch++; + + old_old = old; + } + + if (old != old_old) + old_mismatch++; + } + + CU_ASSERT(old_mismatch == 0); +} + +static void test_atomic_cas_rel_inc_32(void) +{ + uint64_t i, old_mismatch = 0; + uint32_t old, old_old; + odp_atomic_u32_t *a32u = &global_mem->a32u; + + odp_barrier_wait(&global_mem->global_barrier); + + for (i = 0; i < CNT; i++) { + old = odp_atomic_load_u32(a32u); + old_old = old; + + while (odp_atomic_cas_rel_u32(a32u, &old, old + 1) == 0) { + if (old == old_old) + old_mismatch++; + + old_old = old; + } + + if (old != old_old) + old_mismatch++; + } + + CU_ASSERT(old_mismatch == 0); +} + +static void test_atomic_cas_acq_rel_inc_32(void) +{ + uint64_t i, old_mismatch = 0; + uint32_t old, old_old; + odp_atomic_u32_t *a32u = &global_mem->a32u; + + odp_barrier_wait(&global_mem->global_barrier); + + for (i = 0; i < CNT; i++) { + old = odp_atomic_load_u32(a32u); + old_old = old; + + while (odp_atomic_cas_acq_rel_u32(a32u, &old, old + 1) == 0) { + if (old == old_old) + old_mismatch++; + + old_old = old; + } + + if (old != old_old) + old_mismatch++; } + + CU_ASSERT(old_mismatch == 0); } static void test_atomic_cas_dec_32(void) { - int i; + uint64_t i; uint32_t old; odp_atomic_u32_t *a32u = &global_mem->a32u; @@ -341,24 +392,107 @@ static void test_atomic_cas_dec_32(void) static void test_atomic_cas_inc_64(void) { - int i; - uint64_t old; + uint64_t i, old, old_old, old_mismatch = 0; odp_atomic_u64_t *a64u = &global_mem->a64u; odp_barrier_wait(&global_mem->global_barrier); for (i = 0; i < CNT; i++) { old = odp_atomic_load_u64(a64u); + old_old = old; - while (odp_atomic_cas_u64(a64u, &old, old + 1) == 0) - ; + while (odp_atomic_cas_u64(a64u, &old, old + 1) == 0) { + if (old == old_old) + old_mismatch++; + + old_old = old; + } + + if (old != old_old) + old_mismatch++; + } + + CU_ASSERT(old_mismatch == 0); +} + +static void test_atomic_cas_acq_inc_64(void) +{ + uint64_t i, old, old_old, old_mismatch = 0; + odp_atomic_u64_t *a64u = &global_mem->a64u; + + odp_barrier_wait(&global_mem->global_barrier); + + for (i = 0; i < CNT; i++) { + old = odp_atomic_load_u64(a64u); + old_old = old; + + while (odp_atomic_cas_acq_u64(a64u, &old, old + 1) == 0) { + if (old == old_old) + old_mismatch++; + + old_old = old; + } + + if (old != old_old) + old_mismatch++; + } + + CU_ASSERT(old_mismatch == 0); +} + +static void test_atomic_cas_rel_inc_64(void) +{ + uint64_t i, old, old_old, old_mismatch = 0; + odp_atomic_u64_t *a64u = &global_mem->a64u; + + odp_barrier_wait(&global_mem->global_barrier); + + for (i = 0; i < CNT; i++) { + old = odp_atomic_load_u64(a64u); + old_old = old; + + while (odp_atomic_cas_rel_u64(a64u, &old, old + 1) == 0) { + if (old == old_old) + old_mismatch++; + + old_old = old; + } + + if (old != old_old) + old_mismatch++; + } + + CU_ASSERT(old_mismatch == 0); +} + +static void test_atomic_cas_acq_rel_inc_64(void) +{ + uint64_t i, old, old_old, old_mismatch = 0; + odp_atomic_u64_t *a64u = &global_mem->a64u; + + odp_barrier_wait(&global_mem->global_barrier); + + for (i = 0; i < CNT; i++) { + old = odp_atomic_load_u64(a64u); + old_old = old; + + while (odp_atomic_cas_acq_rel_u64(a64u, &old, old + 1) == 0) { + if (old == old_old) + old_mismatch++; + + old_old = old; + } + + if (old != old_old) + old_mismatch++; } + + CU_ASSERT(old_mismatch == 0); } static void test_atomic_cas_dec_64(void) { - int i; - uint64_t old; + uint64_t i, old; odp_atomic_u64_t *a64u = &global_mem->a64u; odp_barrier_wait(&global_mem->global_barrier); @@ -371,55 +505,59 @@ static void test_atomic_cas_dec_64(void) } } +#define BUF_SIZE (64 * 1024) + static void test_atomic_xchg_32(void) { uint32_t old, new; - int i; - odp_atomic_u32_t *a32u = &global_mem->a32u; + uint64_t i; odp_atomic_u32_t *a32u_xchg = &global_mem->a32u_xchg; + uint8_t buf[BUF_SIZE]; + uint64_t seed = odp_thread_id(); + uint64_t count_old = 0, count_new = 0; + + odp_random_test_data(buf, BUF_SIZE, &seed); odp_barrier_wait(&global_mem->global_barrier); for (i = 0; i < CNT; i++) { - new = odp_atomic_fetch_inc_u32(a32u); + new = buf[i & (BUF_SIZE - 1)]; old = odp_atomic_xchg_u32(a32u_xchg, new); - - if (old & 0x1) - odp_atomic_xchg_u32(a32u_xchg, 0); - else - odp_atomic_xchg_u32(a32u_xchg, 1); + count_old += old; + count_new += new; } - odp_atomic_sub_u32(a32u, CNT); - odp_atomic_xchg_u32(a32u_xchg, U32_MAGIC); + odp_atomic_add_u32(a32u_xchg, count_old); + odp_atomic_sub_u32(a32u_xchg, count_new); } static void test_atomic_xchg_64(void) { uint64_t old, new; - int i; - odp_atomic_u64_t *a64u = &global_mem->a64u; + uint64_t i; odp_atomic_u64_t *a64u_xchg = &global_mem->a64u_xchg; + uint8_t buf[BUF_SIZE]; + uint64_t seed = odp_thread_id(); + uint64_t count_old = 0, count_new = 0; + + odp_random_test_data(buf, BUF_SIZE, &seed); odp_barrier_wait(&global_mem->global_barrier); for (i = 0; i < CNT; i++) { - new = odp_atomic_fetch_inc_u64(a64u); + new = buf[i & (BUF_SIZE - 1)]; old = odp_atomic_xchg_u64(a64u_xchg, new); - - if (old & 0x1) - odp_atomic_xchg_u64(a64u_xchg, 0); - else - odp_atomic_xchg_u64(a64u_xchg, 1); + count_old += old; + count_new += new; } - odp_atomic_sub_u64(a64u, CNT); - odp_atomic_xchg_u64(a64u_xchg, U64_MAGIC); + odp_atomic_add_u64(a64u_xchg, count_old); + odp_atomic_sub_u64(a64u_xchg, count_new); } static void test_atomic_non_relaxed_32(void) { - int i; + uint64_t i; uint32_t tmp; odp_atomic_u32_t *a32u = &global_mem->a32u; odp_atomic_u32_t *a32u_min = &global_mem->a32u_min; @@ -447,17 +585,14 @@ static void test_atomic_non_relaxed_32(void) ; tmp = odp_atomic_load_u32(a32u_xchg); - /* finally set value for validation */ - while (odp_atomic_cas_acq_rel_u32(a32u_xchg, &tmp, U32_MAGIC) - == 0) + while (odp_atomic_cas_acq_rel_u32(a32u_xchg, &tmp, tmp + 1) == 0) ; } } static void test_atomic_non_relaxed_64(void) { - int i; - uint64_t tmp; + uint64_t i, tmp; odp_atomic_u64_t *a64u = &global_mem->a64u; odp_atomic_u64_t *a64u_min = &global_mem->a64u_min; odp_atomic_u64_t *a64u_max = &global_mem->a64u_max; @@ -484,16 +619,15 @@ static void test_atomic_non_relaxed_64(void) ; tmp = odp_atomic_load_u64(a64u_xchg); - /* finally set value for validation */ - while (odp_atomic_cas_acq_rel_u64(a64u_xchg, &tmp, U64_MAGIC) - == 0) + while (odp_atomic_cas_acq_rel_u64(a64u_xchg, &tmp, tmp + 1) == 0) ; } } static void test_atomic_relaxed_128(void) { - int i, ret; + int ret; + uint64_t i; odp_u128_t old, new; odp_atomic_u128_t *a128u = &global_mem->a128u; @@ -514,7 +648,8 @@ static void test_atomic_relaxed_128(void) static void test_atomic_non_relaxed_128_acq(void) { - int i, ret; + int ret; + uint64_t i; odp_u128_t old, new; odp_atomic_u128_t *a128u = &global_mem->a128u; @@ -535,7 +670,8 @@ static void test_atomic_non_relaxed_128_acq(void) static void test_atomic_non_relaxed_128_rel(void) { - int i, ret; + int ret; + uint64_t i; odp_u128_t old, new; odp_atomic_u128_t *a128u = &global_mem->a128u; @@ -556,7 +692,8 @@ static void test_atomic_non_relaxed_128_rel(void) static void test_atomic_non_relaxed_128_acq_rel(void) { - int i, ret; + int ret; + uint64_t i; odp_u128_t old, new; odp_atomic_u128_t *a128u = &global_mem->a128u; @@ -623,6 +760,42 @@ static void test_atomic_fetch_add_sub_64(void) test_atomic_fetch_sub_64(); } +static void test_atomic_inc_add_32(void) +{ + test_atomic_inc_32(); + test_atomic_fetch_inc_32(); + test_atomic_add_32(); + test_atomic_fetch_add_32(); + test_atomic_cas_inc_32(); +} + +static void test_atomic_inc_add_64(void) +{ + test_atomic_inc_64(); + test_atomic_fetch_inc_64(); + test_atomic_add_64(); + test_atomic_fetch_add_64(); + test_atomic_cas_inc_64(); +} + +static void test_atomic_dec_sub_32(void) +{ + test_atomic_dec_32(); + test_atomic_fetch_dec_32(); + test_atomic_sub_32(); + test_atomic_fetch_sub_32(); + test_atomic_cas_dec_32(); +} + +static void test_atomic_dec_sub_64(void) +{ + test_atomic_dec_64(); + test_atomic_fetch_dec_64(); + test_atomic_sub_64(); + test_atomic_fetch_sub_64(); + test_atomic_cas_dec_64(); +} + static void test_atomic_max_min_32(void) { test_atomic_max_32(); @@ -691,39 +864,105 @@ static void test_atomic_store(void) odp_atomic_store_u128(&global_mem->a128u, a128u_tmp); } -static void test_atomic_validate(int check) +static void test_atomic_validate_init_val_32_64(void) { CU_ASSERT(U32_INIT_VAL == odp_atomic_load_u32(&global_mem->a32u)); CU_ASSERT(U64_INIT_VAL == odp_atomic_load_u64(&global_mem->a64u)); +} - odp_u128_t a128u_tmp; +static void test_atomic_validate_init_val_128(void) +{ + odp_u128_t a128u = odp_atomic_load_u128(&global_mem->a128u); - a128u_tmp = odp_atomic_load_u128(&global_mem->a128u); + CU_ASSERT(U64_INIT_VAL == a128u.u64[0]); + CU_ASSERT(U64_INIT_VAL == a128u.u64[1]); +} - if (check & CHECK_CAS_128) { - uint64_t iterations = 0; +static void test_atomic_validate_init_val(void) +{ + test_atomic_validate_init_val_32_64(); + test_atomic_validate_init_val_128(); +} - iterations = a128u_tmp.u64[0] - a128u_tmp.u64[1]; - CU_ASSERT(iterations == 4 * CNT * global_mem->g_num_threads); - } else { - CU_ASSERT(U64_INIT_VAL == a128u_tmp.u64[0]); - CU_ASSERT(U64_INIT_VAL == a128u_tmp.u64[1]); - } +static void test_atomic_validate_inc_add(void) +{ + test_atomic_validate_init_val_128(); - if (check & CHECK_MAX_MIN) { - CU_ASSERT(odp_atomic_load_u32(&global_mem->a32u_max) > - odp_atomic_load_u32(&global_mem->a32u_min)); + /* Two increment tests, one cas increment test and two add tests. */ + const uint64_t total_count = CNT * (3 + 2 * ADD_SUB_CNT) * global_mem->g_num_threads; + const uint32_t a32u = U32_INIT_VAL + total_count; - CU_ASSERT(odp_atomic_load_u64(&global_mem->a64u_max) > - odp_atomic_load_u64(&global_mem->a64u_min)); - } + CU_ASSERT(a32u == odp_atomic_load_u32(&global_mem->a32u)); + CU_ASSERT(U64_INIT_VAL + total_count == odp_atomic_load_u64(&global_mem->a64u)); +} - if (check & CHECK_XCHG) { - CU_ASSERT(odp_atomic_load_u32(&global_mem->a32u_xchg) == - U32_MAGIC); - CU_ASSERT(odp_atomic_load_u64(&global_mem->a64u_xchg) == - U64_MAGIC); - } +static void test_atomic_validate_dec_sub(void) +{ + test_atomic_validate_init_val_128(); + + /* Two decrement tests, one cas decrement test and two sub tests. */ + const uint64_t total_count = CNT * (3 + 2 * ADD_SUB_CNT) * global_mem->g_num_threads; + const uint32_t a32u = U32_INIT_VAL - total_count; + + CU_ASSERT(a32u == odp_atomic_load_u32(&global_mem->a32u)); + CU_ASSERT(U64_INIT_VAL - total_count == odp_atomic_load_u64(&global_mem->a64u)); +} + +static void test_atomic_validate_cas_inc_dec(void) +{ + test_atomic_validate_init_val_32_64(); + + odp_u128_t a128u = odp_atomic_load_u128(&global_mem->a128u); + const uint64_t iterations = a128u.u64[0] - a128u.u64[1]; + + CU_ASSERT(iterations == 4 * CNT * global_mem->g_num_threads); +} + +static void test_atomic_validate_max_min(void) +{ + test_atomic_validate_init_val(); + + const uint64_t total_count = CNT * global_mem->g_num_threads; + /* + * Max is the result of fetch_inc, so the final max value is total_count - 1. In + * a long test, counter may overflow, in which case max is saturated at + * UINT32_MAX, and min at 0. + */ + const uint32_t a32u_max = min(U32_INIT_VAL + total_count - 1, UINT32_MAX); + const uint32_t a32u_min = U32_INIT_VAL + total_count - 1 > UINT32_MAX ? 0 : U32_INIT_VAL; + + CU_ASSERT(odp_atomic_load_u32(&global_mem->a32u_max) == a32u_max); + CU_ASSERT(odp_atomic_load_u32(&global_mem->a32u_min) == a32u_min); + CU_ASSERT(odp_atomic_load_u64(&global_mem->a64u_max) == U64_INIT_VAL + total_count - 1); + CU_ASSERT(odp_atomic_load_u64(&global_mem->a64u_min) == U64_INIT_VAL); +} + +static void test_atomic_validate_xchg(void) +{ + test_atomic_validate_init_val(); + + CU_ASSERT(odp_atomic_load_u32(&global_mem->a32u_xchg) == U32_INIT_VAL); + CU_ASSERT(odp_atomic_load_u64(&global_mem->a64u_xchg) == U64_INIT_VAL); +} + +static void test_atomic_validate_non_relaxed(void) +{ + test_atomic_validate_init_val(); + + const uint64_t total_count = CNT * global_mem->g_num_threads; + /* 3 increments per round. */ + const uint32_t a32u = U32_INIT_VAL + 3 * total_count; + /* 1 increment per round. */ + const uint32_t a32u_max = U32_INIT_VAL + total_count; + const uint32_t a32u_min = U32_INIT_VAL - total_count; + + CU_ASSERT(odp_atomic_load_u32(&global_mem->a32u_xchg) == a32u); + CU_ASSERT(odp_atomic_load_u64(&global_mem->a64u_xchg) == U64_INIT_VAL + 3 * total_count); + + CU_ASSERT(odp_atomic_load_u32(&global_mem->a32u_max) == a32u_max); + CU_ASSERT(odp_atomic_load_u32(&global_mem->a32u_min) == a32u_min); + CU_ASSERT(odp_atomic_load_u64(&global_mem->a64u_max) == U64_INIT_VAL + total_count); + CU_ASSERT(odp_atomic_load_u64(&global_mem->a64u_min) == U64_INIT_VAL - total_count); } static int atomic_init(odp_instance_t *inst) @@ -762,8 +1001,6 @@ static int atomic_init(odp_instance_t *inst) memset(global_mem, 0, sizeof(global_shared_mem_t)); global_mem->g_num_threads = MAX_WORKERS; - global_mem->g_iterations = MAX_ITERATIONS; - global_mem->g_verbose = VERBOSE; workers_count = odp_cpumask_default_worker(&mask, 0); @@ -810,121 +1047,238 @@ static int atomic_term(odp_instance_t inst) } /* Atomic tests */ -static int test_atomic_inc_dec_thread(void *arg UNUSED) + +static int test_atomic_inc_thread(void *arg UNUSED) +{ + thread_init(); + test_atomic_inc_32(); + test_atomic_inc_64(); + + return CU_get_number_of_failures(); +} + +static int test_atomic_dec_thread(void *arg UNUSED) { - per_thread_mem_t *per_thread_mem; + thread_init(); + test_atomic_dec_32(); + test_atomic_dec_64(); - per_thread_mem = thread_init(); + return CU_get_number_of_failures(); +} + +static int test_atomic_add_thread(void *arg UNUSED) +{ + thread_init(); + test_atomic_add_32(); + test_atomic_add_64(); + + return CU_get_number_of_failures(); +} + +static int test_atomic_sub_thread(void *arg UNUSED) +{ + thread_init(); + test_atomic_sub_32(); + test_atomic_sub_64(); + + return CU_get_number_of_failures(); +} + +static int test_atomic_fetch_inc_thread(void *arg UNUSED) +{ + thread_init(); + test_atomic_fetch_inc_32(); + test_atomic_fetch_inc_64(); + + return CU_get_number_of_failures(); +} + +static int test_atomic_fetch_dec_thread(void *arg UNUSED) +{ + thread_init(); + test_atomic_fetch_dec_32(); + test_atomic_fetch_dec_64(); + + return CU_get_number_of_failures(); +} + +static int test_atomic_fetch_add_thread(void *arg UNUSED) +{ + thread_init(); + test_atomic_fetch_add_32(); + test_atomic_fetch_add_64(); + + return CU_get_number_of_failures(); +} + +static int test_atomic_fetch_sub_thread(void *arg UNUSED) +{ + thread_init(); + test_atomic_fetch_sub_32(); + test_atomic_fetch_sub_64(); + + return CU_get_number_of_failures(); +} + +static int test_atomic_max_thread(void *arg UNUSED) +{ + thread_init(); + test_atomic_max_32(); + test_atomic_max_64(); + + return CU_get_number_of_failures(); +} + +static int test_atomic_min_thread(void *arg UNUSED) +{ + thread_init(); + test_atomic_min_32(); + test_atomic_min_64(); + + return CU_get_number_of_failures(); +} + +static int test_atomic_cas_thread(void *arg UNUSED) +{ + thread_init(); + test_atomic_cas_inc_32(); + test_atomic_cas_inc_64(); + test_atomic_relaxed_128(); + + return CU_get_number_of_failures(); +} + +static int test_atomic_cas_acq_thread(void *arg UNUSED) +{ + thread_init(); + test_atomic_cas_acq_inc_32(); + test_atomic_cas_acq_inc_64(); + test_atomic_non_relaxed_128_acq(); + + return CU_get_number_of_failures(); +} + +static int test_atomic_cas_rel_thread(void *arg UNUSED) +{ + thread_init(); + test_atomic_cas_rel_inc_32(); + test_atomic_cas_rel_inc_64(); + test_atomic_non_relaxed_128_rel(); + + return CU_get_number_of_failures(); +} + +static int test_atomic_cas_acq_rel_thread(void *arg UNUSED) +{ + thread_init(); + test_atomic_cas_acq_rel_inc_32(); + test_atomic_cas_acq_rel_inc_64(); + test_atomic_non_relaxed_128_acq_rel(); + + return CU_get_number_of_failures(); +} + +static int test_atomic_inc_dec_thread(void *arg UNUSED) +{ + thread_init(); test_atomic_inc_dec_32(); test_atomic_inc_dec_64(); - thread_finalize(per_thread_mem); - return CU_get_number_of_failures(); } static int test_atomic_add_sub_thread(void *arg UNUSED) { - per_thread_mem_t *per_thread_mem; - - per_thread_mem = thread_init(); + thread_init(); test_atomic_add_sub_32(); test_atomic_add_sub_64(); - thread_finalize(per_thread_mem); - return CU_get_number_of_failures(); } static int test_atomic_fetch_inc_dec_thread(void *arg UNUSED) { - per_thread_mem_t *per_thread_mem; - - per_thread_mem = thread_init(); + thread_init(); test_atomic_fetch_inc_dec_32(); test_atomic_fetch_inc_dec_64(); - thread_finalize(per_thread_mem); - return CU_get_number_of_failures(); } static int test_atomic_fetch_add_sub_thread(void *arg UNUSED) { - per_thread_mem_t *per_thread_mem; - - per_thread_mem = thread_init(); + thread_init(); test_atomic_fetch_add_sub_32(); test_atomic_fetch_add_sub_64(); - thread_finalize(per_thread_mem); + return CU_get_number_of_failures(); +} + +static int test_atomic_inc_add_thread(void *arg UNUSED) +{ + thread_init(); + test_atomic_inc_add_32(); + test_atomic_inc_add_64(); return CU_get_number_of_failures(); } -static int test_atomic_max_min_thread(void *arg UNUSED) +static int test_atomic_dec_sub_thread(void *arg UNUSED) { - per_thread_mem_t *per_thread_mem; + thread_init(); + test_atomic_dec_sub_32(); + test_atomic_dec_sub_64(); - per_thread_mem = thread_init(); + return CU_get_number_of_failures(); +} + +static int test_atomic_max_min_thread(void *arg UNUSED) +{ + thread_init(); test_atomic_max_min_32(); test_atomic_max_min_64(); - thread_finalize(per_thread_mem); - return CU_get_number_of_failures(); } static int test_atomic_cas_inc_dec_thread(void *arg UNUSED) { - per_thread_mem_t *per_thread_mem; - - per_thread_mem = thread_init(); + thread_init(); test_atomic_cas_inc_dec_32(); test_atomic_cas_inc_dec_64(); test_atomic_cas_inc_128(); - thread_finalize(per_thread_mem); - return CU_get_number_of_failures(); } static int test_atomic_xchg_thread(void *arg UNUSED) { - per_thread_mem_t *per_thread_mem; - - per_thread_mem = thread_init(); + thread_init(); test_atomic_xchg_32(); test_atomic_xchg_64(); - thread_finalize(per_thread_mem); - return CU_get_number_of_failures(); } static int test_atomic_non_relaxed_thread(void *arg UNUSED) { - per_thread_mem_t *per_thread_mem; - - per_thread_mem = thread_init(); + thread_init(); test_atomic_non_relaxed_32(); test_atomic_non_relaxed_64(); - thread_finalize(per_thread_mem); - return CU_get_number_of_failures(); } -static void test_atomic_functional(int func_ptr(void *), int check) +static void test_atomic_functional(int test_fn(void *), void validate_fn(void)) { pthrd_arg arg; arg.numthrds = global_mem->g_num_threads; test_atomic_init(); test_atomic_store(); - odp_cunit_thread_create(func_ptr, &arg); + odp_cunit_thread_create(test_fn, &arg); odp_cunit_thread_exit(&arg); - test_atomic_validate(check); + validate_fn(); } static void test_atomic_op_lock_free_set(void) @@ -1077,45 +1431,231 @@ static void test_atomic_op_lock_free_128(void) } } +static void atomic_test_atomic_init(void) +{ + uint64_t i; + odp_atomic_u128_t *a128u = &global_mem->a128u; + + for (i = 0; i < CNT; i++) { + odp_u128_t a128u_tmp; + + odp_atomic_init_u32(&global_mem->a32u, i); + odp_atomic_init_u64(&global_mem->a64u, i); + odp_atomic_init_u32(&global_mem->a32u_min, i); + odp_atomic_init_u32(&global_mem->a32u_max, i); + odp_atomic_init_u64(&global_mem->a64u_min, i); + odp_atomic_init_u64(&global_mem->a64u_max, i); + odp_atomic_init_u32(&global_mem->a32u_xchg, i); + odp_atomic_init_u64(&global_mem->a64u_xchg, i); + + a128u_tmp.u64[0] = i; + a128u_tmp.u64[1] = i; + odp_atomic_init_u128(&global_mem->a128u, a128u_tmp); + + CU_ASSERT(odp_atomic_load_u32(&global_mem->a32u) == i); + CU_ASSERT(odp_atomic_load_u64(&global_mem->a64u) == i); + CU_ASSERT(odp_atomic_load_u32(&global_mem->a32u_min) == i); + CU_ASSERT(odp_atomic_load_u32(&global_mem->a32u_max) == i); + CU_ASSERT(odp_atomic_load_u64(&global_mem->a64u_min) == i); + CU_ASSERT(odp_atomic_load_u64(&global_mem->a64u_max) == i); + CU_ASSERT(odp_atomic_load_u32(&global_mem->a32u_xchg) == i); + CU_ASSERT(odp_atomic_load_u64(&global_mem->a64u_xchg) == i); + + a128u_tmp = odp_atomic_load_u128(a128u); + CU_ASSERT(a128u_tmp.u64[0] == i); + CU_ASSERT(a128u_tmp.u64[1] == i); + } +} + +static void test_atomic_validate_inc(void) +{ + const uint64_t total_count = CNT * global_mem->g_num_threads; + const uint32_t a32u = U32_INIT_VAL + total_count; + + CU_ASSERT(a32u == odp_atomic_load_u32(&global_mem->a32u)); + CU_ASSERT(U64_INIT_VAL + total_count == odp_atomic_load_u64(&global_mem->a64u)); +} + +static void atomic_test_atomic_inc(void) +{ + test_atomic_functional(test_atomic_inc_thread, test_atomic_validate_inc); +} + +static void test_atomic_validate_dec(void) +{ + const uint64_t total_count = CNT * global_mem->g_num_threads; + const uint32_t a32u = U32_INIT_VAL - total_count; + + CU_ASSERT(a32u == odp_atomic_load_u32(&global_mem->a32u)); + CU_ASSERT(U64_INIT_VAL - total_count == odp_atomic_load_u64(&global_mem->a64u)); +} + +static void atomic_test_atomic_dec(void) +{ + test_atomic_functional(test_atomic_dec_thread, test_atomic_validate_dec); +} + +static void test_atomic_validate_add(void) +{ + const uint64_t total_count = CNT * ADD_SUB_CNT * global_mem->g_num_threads; + const uint32_t a32u = U32_INIT_VAL + total_count; + + CU_ASSERT(a32u == odp_atomic_load_u32(&global_mem->a32u)); + CU_ASSERT(U64_INIT_VAL + total_count == odp_atomic_load_u64(&global_mem->a64u)); +} + +static void atomic_test_atomic_add(void) +{ + test_atomic_functional(test_atomic_add_thread, test_atomic_validate_add); +} + +static void test_atomic_validate_sub(void) +{ + const uint64_t total_count = CNT * ADD_SUB_CNT * global_mem->g_num_threads; + const uint32_t a32u = U32_INIT_VAL - total_count; + + CU_ASSERT(a32u == odp_atomic_load_u32(&global_mem->a32u)); + CU_ASSERT(U64_INIT_VAL - total_count == odp_atomic_load_u64(&global_mem->a64u)); +} + +static void atomic_test_atomic_sub(void) +{ + test_atomic_functional(test_atomic_sub_thread, test_atomic_validate_sub); +} + +static void atomic_test_atomic_fetch_inc(void) +{ + test_atomic_functional(test_atomic_fetch_inc_thread, test_atomic_validate_inc); +} + +static void atomic_test_atomic_fetch_dec(void) +{ + test_atomic_functional(test_atomic_fetch_dec_thread, test_atomic_validate_dec); +} + +static void atomic_test_atomic_fetch_add(void) +{ + test_atomic_functional(test_atomic_fetch_add_thread, test_atomic_validate_add); +} + +static void atomic_test_atomic_fetch_sub(void) +{ + test_atomic_functional(test_atomic_fetch_sub_thread, test_atomic_validate_sub); +} + +static void test_atomic_validate_max(void) +{ + const uint64_t total_count = CNT * global_mem->g_num_threads - 1; + /* In a long test, counter may overflow, in which case max is saturated at UINT32_MAX. */ + const uint32_t a32u_max = min(U32_INIT_VAL + total_count, UINT32_MAX); + + CU_ASSERT(a32u_max == odp_atomic_load_u32(&global_mem->a32u_max)); + CU_ASSERT(U64_INIT_VAL + total_count == odp_atomic_load_u64(&global_mem->a64u_max)); +} + +static void atomic_test_atomic_max(void) +{ + test_atomic_functional(test_atomic_max_thread, test_atomic_validate_max); +} + +static void test_atomic_validate_min(void) +{ + const uint64_t total_count = CNT * global_mem->g_num_threads - 1; + /* In a long test, counter may underflow, in which case min is saturated at 0. */ + const uint32_t a32u_min = max((int64_t)U32_INIT_VAL - (int64_t)total_count, 0); + + CU_ASSERT(a32u_min == odp_atomic_load_u32(&global_mem->a32u_min)); + CU_ASSERT(U64_INIT_VAL - total_count == odp_atomic_load_u64(&global_mem->a64u_min)); +} + +static void atomic_test_atomic_min(void) +{ + test_atomic_functional(test_atomic_min_thread, test_atomic_validate_min); +} + +static void test_atomic_validate_cas_128(void) +{ + odp_u128_t a128u = odp_atomic_load_u128(&global_mem->a128u); + const uint64_t iterations = a128u.u64[0] - a128u.u64[1]; + + CU_ASSERT(iterations == CNT * global_mem->g_num_threads); +} + +static void test_atomic_validate_cas(void) +{ + test_atomic_validate_inc(); + test_atomic_validate_cas_128(); +} + +static void atomic_test_atomic_cas(void) +{ + test_atomic_functional(test_atomic_cas_thread, test_atomic_validate_cas); +} + +static void atomic_test_atomic_cas_acq(void) +{ + test_atomic_functional(test_atomic_cas_acq_thread, test_atomic_validate_cas); +} + +static void atomic_test_atomic_cas_rel(void) +{ + test_atomic_functional(test_atomic_cas_rel_thread, test_atomic_validate_cas); +} + +static void atomic_test_atomic_cas_acq_rel(void) +{ + test_atomic_functional(test_atomic_cas_acq_rel_thread, test_atomic_validate_cas); +} + static void atomic_test_atomic_inc_dec(void) { - test_atomic_functional(test_atomic_inc_dec_thread, 0); + test_atomic_functional(test_atomic_inc_dec_thread, test_atomic_validate_init_val); } static void atomic_test_atomic_add_sub(void) { - test_atomic_functional(test_atomic_add_sub_thread, 0); + test_atomic_functional(test_atomic_add_sub_thread, test_atomic_validate_init_val); } static void atomic_test_atomic_fetch_inc_dec(void) { - test_atomic_functional(test_atomic_fetch_inc_dec_thread, 0); + test_atomic_functional(test_atomic_fetch_inc_dec_thread, test_atomic_validate_init_val); } static void atomic_test_atomic_fetch_add_sub(void) { - test_atomic_functional(test_atomic_fetch_add_sub_thread, 0); + test_atomic_functional(test_atomic_fetch_add_sub_thread, test_atomic_validate_init_val); +} + +static void atomic_test_atomic_inc_add(void) +{ + test_atomic_functional(test_atomic_inc_add_thread, test_atomic_validate_inc_add); +} + +static void atomic_test_atomic_dec_sub(void) +{ + test_atomic_functional(test_atomic_dec_sub_thread, test_atomic_validate_dec_sub); } static void atomic_test_atomic_max_min(void) { - test_atomic_functional(test_atomic_max_min_thread, CHECK_MAX_MIN); + test_atomic_functional(test_atomic_max_min_thread, test_atomic_validate_max_min); } static void atomic_test_atomic_cas_inc_dec(void) { - test_atomic_functional(test_atomic_cas_inc_dec_thread, CHECK_CAS_128); + test_atomic_functional(test_atomic_cas_inc_dec_thread, test_atomic_validate_cas_inc_dec); } static void atomic_test_atomic_xchg(void) { - test_atomic_functional(test_atomic_xchg_thread, CHECK_XCHG); + test_atomic_functional(test_atomic_xchg_thread, test_atomic_validate_xchg); } static void atomic_test_atomic_non_relaxed(void) { test_atomic_functional(test_atomic_non_relaxed_thread, - CHECK_MAX_MIN | CHECK_XCHG); + test_atomic_validate_non_relaxed); } static void atomic_test_atomic_op_lock_free(void) @@ -1126,10 +1666,27 @@ static void atomic_test_atomic_op_lock_free(void) } odp_testinfo_t atomic_suite_atomic[] = { + ODP_TEST_INFO(atomic_test_atomic_init), + ODP_TEST_INFO(atomic_test_atomic_inc), + ODP_TEST_INFO(atomic_test_atomic_dec), + ODP_TEST_INFO(atomic_test_atomic_add), + ODP_TEST_INFO(atomic_test_atomic_sub), + ODP_TEST_INFO(atomic_test_atomic_fetch_inc), + ODP_TEST_INFO(atomic_test_atomic_fetch_dec), + ODP_TEST_INFO(atomic_test_atomic_fetch_add), + ODP_TEST_INFO(atomic_test_atomic_fetch_sub), + ODP_TEST_INFO(atomic_test_atomic_max), + ODP_TEST_INFO(atomic_test_atomic_min), + ODP_TEST_INFO(atomic_test_atomic_cas), + ODP_TEST_INFO(atomic_test_atomic_cas_acq), + ODP_TEST_INFO(atomic_test_atomic_cas_rel), + ODP_TEST_INFO(atomic_test_atomic_cas_acq_rel), ODP_TEST_INFO(atomic_test_atomic_inc_dec), ODP_TEST_INFO(atomic_test_atomic_add_sub), ODP_TEST_INFO(atomic_test_atomic_fetch_inc_dec), ODP_TEST_INFO(atomic_test_atomic_fetch_add_sub), + ODP_TEST_INFO(atomic_test_atomic_inc_add), + ODP_TEST_INFO(atomic_test_atomic_dec_sub), ODP_TEST_INFO(atomic_test_atomic_max_min), ODP_TEST_INFO(atomic_test_atomic_cas_inc_dec), ODP_TEST_INFO(atomic_test_atomic_xchg), |