diff options
Diffstat (limited to 'test/performance')
| -rw-r--r-- | test/performance/.gitignore | 1 | ||||
| -rw-r--r-- | test/performance/Makefile.am | 2 | ||||
| -rw-r--r-- | test/performance/odp_bench_timer.c | 871 | ||||
| -rw-r--r-- | test/performance/odp_dma_perf.c | 2021 | ||||
| -rwxr-xr-x | test/performance/odp_dma_perf_run.sh | 65 | ||||
| -rw-r--r-- | test/performance/odp_ipsec.c | 2 | ||||
| -rw-r--r-- | test/performance/odp_ipsecfwd.c | 88 | ||||
| -rw-r--r-- | test/performance/odp_l2fwd.c | 18 | ||||
| -rw-r--r-- | test/performance/odp_sched_pktio.c | 4 | ||||
| -rw-r--r-- | test/performance/odp_timer_perf.c | 10 |
10 files changed, 2208 insertions, 874 deletions
diff --git a/test/performance/.gitignore b/test/performance/.gitignore index b86699c91..087a163d8 100644 --- a/test/performance/.gitignore +++ b/test/performance/.gitignore @@ -5,6 +5,7 @@ odp_atomic_perf odp_bench_buffer odp_bench_misc odp_bench_packet +odp_bench_timer odp_cpu_bench odp_crc odp_crypto diff --git a/test/performance/Makefile.am b/test/performance/Makefile.am index 9dc83fd22..67d57590a 100644 --- a/test/performance/Makefile.am +++ b/test/performance/Makefile.am @@ -6,6 +6,7 @@ EXECUTABLES = odp_atomic_perf \ odp_bench_buffer \ odp_bench_misc \ odp_bench_packet \ + odp_bench_timer \ odp_crc \ odp_lock_perf \ odp_mem_perf \ @@ -60,6 +61,7 @@ odp_atomic_perf_SOURCES = odp_atomic_perf.c odp_bench_buffer_SOURCES = odp_bench_buffer.c odp_bench_misc_SOURCES = odp_bench_misc.c odp_bench_packet_SOURCES = odp_bench_packet.c +odp_bench_timer_SOURCES = odp_bench_timer.c odp_cpu_bench_SOURCES = odp_cpu_bench.c odp_crc_SOURCES = odp_crc.c odp_crypto_SOURCES = odp_crypto.c diff --git a/test/performance/odp_bench_timer.c b/test/performance/odp_bench_timer.c new file mode 100644 index 000000000..918d19e5d --- /dev/null +++ b/test/performance/odp_bench_timer.c @@ -0,0 +1,871 @@ +/* Copyright (c) 2023, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef _GNU_SOURCE +#define _GNU_SOURCE /* Needed for sigaction */ +#endif + +#include <odp_api.h> +#include <odp/helper/odph_api.h> + +#include <getopt.h> +#include <inttypes.h> +#include <signal.h> +#include <stdlib.h> +#include <unistd.h> + +/* Number of API function calls per test case */ +#define REPEAT_COUNT 1000 + +/* Default number of rounds per test case */ +#define ROUNDS 1000u + +/** User area size in bytes */ +#define UAREA_SIZE 8 + +/** Timer duration in nsec */ +#define TIMER_NSEC 50000000 + +#define BENCH_INFO(run, max, name) \ + {#run, run, max, name} + +/* Run benchmark, returns >0 on success */ +typedef int (*bench_run_fn_t)(void); + +/* Benchmark data */ +typedef struct { + /* Default test name */ + const char *name; + + /* Test function to run */ + bench_run_fn_t run; + + /* Test specific limit for rounds (tuning for slow implementation) */ + uint32_t max_rounds; + + /* Override default test name */ + const char *desc; + +} bench_info_t; + +typedef struct { + /* Command line options */ + struct { + /* Clock source to be used */ + int clk_src; + + /* Measure time vs CPU cycles */ + int time; + + /* Benchmark index to run indefinitely */ + int bench_idx; + + /* Rounds per test case */ + uint32_t rounds; + + } opt; + + odp_timer_pool_t timer_pool; + odp_timer_t timer; + odp_queue_t queue; + odp_pool_t pool; + odp_timeout_t timeout; + odp_event_t event; + uint64_t timer_nsec; + uint64_t tick; + uint64_t nsec; + double tick_hz; + int plain_queue; + + /* Benchmark functions */ + bench_info_t *bench; + + /* Number of benchmark functions */ + int num_bench; + + /* Break worker loop if set to 1 */ + odp_atomic_u32_t exit_thread; + + /* Test case input / output data */ + uint64_t a1[REPEAT_COUNT]; + odp_event_t ev[REPEAT_COUNT]; + odp_timeout_t tmo[REPEAT_COUNT]; + odp_timer_t tim[REPEAT_COUNT]; + + /* Dummy result */ + uint64_t dummy; + + /* Benchmark run failed */ + int bench_failed; + + /* CPU mask as string */ + char cpumask_str[ODP_CPUMASK_STR_SIZE]; + +} gbl_args_t; + +static gbl_args_t *gbl_args; + +static void sig_handler(int signo ODP_UNUSED) +{ + if (gbl_args == NULL) + return; + odp_atomic_store_u32(&gbl_args->exit_thread, 1); +} + +static int setup_sig_handler(void) +{ + struct sigaction action; + + memset(&action, 0, sizeof(action)); + action.sa_handler = sig_handler; + + /* No additional signals blocked. By default, the signal which triggered + * the handler is blocked. */ + if (sigemptyset(&action.sa_mask)) + return -1; + + if (sigaction(SIGINT, &action, NULL)) + return -1; + + return 0; +} + +/* Run given benchmark indefinitely */ +static void run_indef(gbl_args_t *args, int idx) +{ + const char *desc; + const bench_info_t *bench = &args->bench[idx]; + + desc = bench->desc != NULL ? bench->desc : bench->name; + + printf("Running odp_%s test indefinitely\n", desc); + + while (!odp_atomic_load_u32(&gbl_args->exit_thread)) { + int ret; + + ret = bench->run(); + + if (!ret) + ODPH_ABORT("Benchmark %s failed\n", desc); + } +} + +static int run_benchmarks(void *arg) +{ + int i, j; + uint64_t c1, c2; + odp_time_t t1, t2; + gbl_args_t *args = arg; + const int meas_time = args->opt.time; + + printf("\nAverage %s per function call\n", meas_time ? "time (nsec)" : "CPU cycles"); + printf("-------------------------------------------------\n"); + + /* Run each test twice. Results from the first warm-up round are ignored. */ + for (i = 0; i < 2; i++) { + uint64_t total = 0; + uint32_t round = 1; + + for (j = 0; j < gbl_args->num_bench; round++) { + int ret; + const char *desc; + const bench_info_t *bench = &args->bench[j]; + uint32_t max_rounds = args->opt.rounds; + + if (bench->max_rounds && max_rounds > bench->max_rounds) + max_rounds = bench->max_rounds; + + /* Run selected test indefinitely */ + if (args->opt.bench_idx) { + if ((j + 1) != args->opt.bench_idx) { + j++; + continue; + } + + run_indef(args, j); + return 0; + } + + desc = bench->desc != NULL ? bench->desc : bench->name; + + if (meas_time) + t1 = odp_time_local(); + else + c1 = odp_cpu_cycles(); + + ret = bench->run(); + + if (meas_time) + t2 = odp_time_local(); + else + c2 = odp_cpu_cycles(); + + if (!ret) { + ODPH_ERR("Benchmark odp_%s failed\n", desc); + args->bench_failed = -1; + return -1; + } + + if (meas_time) + total += odp_time_diff_ns(t2, t1); + else + total += odp_cpu_cycles_diff(c2, c1); + + for (i = 0; i < REPEAT_COUNT; i++) + args->dummy += args->a1[i]; + + if (round >= max_rounds) { + double result; + + /* Each benchmark runs internally REPEAT_COUNT times. */ + result = ((double)total) / (max_rounds * REPEAT_COUNT); + + /* No print from warm-up round */ + if (i > 0) + printf("[%02d] odp_%-26s: %12.2f\n", j + 1, desc, result); + + j++; + total = 0; + round = 1; + } + } + } + + /* Print dummy result to prevent compiler to optimize it away*/ + printf("\n(dummy result: 0x%" PRIx64 ")\n\n", args->dummy); + + return 0; +} + +static int timer_current_tick(void) +{ + int i; + odp_timer_pool_t timer_pool = gbl_args->timer_pool; + uint64_t *a1 = gbl_args->a1; + + for (i = 0; i < REPEAT_COUNT; i++) + a1[i] = odp_timer_current_tick(timer_pool); + + return i; +} + +static int timer_tick_to_ns(void) +{ + int i; + odp_timer_pool_t timer_pool = gbl_args->timer_pool; + uint64_t *a1 = gbl_args->a1; + uint64_t tick = gbl_args->tick; + + for (i = 0; i < REPEAT_COUNT; i++) + a1[i] = odp_timer_tick_to_ns(timer_pool, tick); + + return i; +} + +static int timer_ns_to_tick(void) +{ + int i; + odp_timer_pool_t timer_pool = gbl_args->timer_pool; + uint64_t *a1 = gbl_args->a1; + uint64_t nsec = gbl_args->nsec; + + for (i = 0; i < REPEAT_COUNT; i++) + a1[i] = odp_timer_ns_to_tick(timer_pool, nsec); + + return i; +} + +static int timeout_to_event(void) +{ + int i; + odp_event_t *ev = gbl_args->ev; + odp_timeout_t timeout = gbl_args->timeout; + + for (i = 0; i < REPEAT_COUNT; i++) + ev[i] = odp_timeout_to_event(timeout); + + gbl_args->dummy += odp_event_to_u64(ev[0]); + + return i; +} + +static int timeout_from_event(void) +{ + int i; + odp_event_t ev = gbl_args->event; + odp_timeout_t *tmo = gbl_args->tmo; + + for (i = 0; i < REPEAT_COUNT; i++) + tmo[i] = odp_timeout_from_event(ev); + + gbl_args->dummy += odp_timeout_to_u64(tmo[0]); + + return i; +} + +static int timeout_fresh(void) +{ + int i; + odp_timeout_t timeout = gbl_args->timeout; + uint64_t *a1 = gbl_args->a1; + + for (i = 0; i < REPEAT_COUNT; i++) + a1[i] = odp_timeout_fresh(timeout); + + return i; +} + +static int timeout_timer(void) +{ + int i; + odp_timeout_t timeout = gbl_args->timeout; + odp_timer_t *tim = gbl_args->tim; + + for (i = 0; i < REPEAT_COUNT; i++) + tim[i] = odp_timeout_timer(timeout); + + gbl_args->dummy += odp_timer_to_u64(tim[0]); + + return i; +} + +static int timeout_tick(void) +{ + int i; + odp_timeout_t timeout = gbl_args->timeout; + uint64_t *a1 = gbl_args->a1; + + for (i = 0; i < REPEAT_COUNT; i++) + a1[i] = odp_timeout_tick(timeout); + + return i; +} + +static int timeout_user_ptr(void) +{ + int i; + odp_timeout_t timeout = gbl_args->timeout; + uint64_t *a1 = gbl_args->a1; + + for (i = 0; i < REPEAT_COUNT; i++) + a1[i] = (uintptr_t)odp_timeout_user_ptr(timeout); + + return i; +} + +static int timeout_user_area(void) +{ + int i; + odp_timeout_t timeout = gbl_args->timeout; + uint64_t *a1 = gbl_args->a1; + + for (i = 0; i < REPEAT_COUNT; i++) + a1[i] = (uintptr_t)odp_timeout_user_area(timeout); + + return i; +} + +static int timeout_to_u64(void) +{ + int i; + odp_timeout_t timeout = gbl_args->timeout; + uint64_t *a1 = gbl_args->a1; + + for (i = 0; i < REPEAT_COUNT; i++) + a1[i] = odp_timeout_to_u64(timeout); + + return i; +} + +static int timer_to_u64(void) +{ + int i; + odp_timer_t timer = gbl_args->timer; + uint64_t *a1 = gbl_args->a1; + + for (i = 0; i < REPEAT_COUNT; i++) + a1[i] = odp_timer_to_u64(timer); + + return i; +} + +static int timer_pool_to_u64(void) +{ + int i; + odp_timer_pool_t tp = gbl_args->timer_pool; + uint64_t *a1 = gbl_args->a1; + + for (i = 0; i < REPEAT_COUNT; i++) + a1[i] = odp_timer_pool_to_u64(tp); + + return i; +} + +bench_info_t test_suite[] = { + BENCH_INFO(timer_current_tick, 0, NULL), + BENCH_INFO(timer_tick_to_ns, 0, NULL), + BENCH_INFO(timer_ns_to_tick, 0, NULL), + BENCH_INFO(timeout_to_event, 0, NULL), + BENCH_INFO(timeout_from_event, 0, NULL), + BENCH_INFO(timeout_fresh, 0, NULL), + BENCH_INFO(timeout_timer, 0, NULL), + BENCH_INFO(timeout_tick, 0, NULL), + BENCH_INFO(timeout_user_ptr, 0, NULL), + BENCH_INFO(timeout_user_area, 0, NULL), + BENCH_INFO(timeout_to_u64, 0, NULL), + BENCH_INFO(timer_to_u64, 0, NULL), + BENCH_INFO(timer_pool_to_u64, 0, NULL), +}; + +/* Print usage information */ +static void usage(void) +{ + printf("\n" + "ODP timer API micro benchmarks\n" + "\n" + "Options:\n" + " -s, --clk_src Clock source select (default 0):\n" + " 0: ODP_CLOCK_DEFAULT\n" + " 1: ODP_CLOCK_SRC_1, ...\n" + " -t, --time <opt> Time measurement. 0: measure CPU cycles (default), 1: measure time\n" + " -i, --index <idx> Benchmark index to run indefinitely.\n" + " -r, --rounds <num> Run each test case 'num' times (default %u).\n" + " -h, --help Display help and exit.\n\n" + "\n", ROUNDS); +} + +/* Parse command line arguments */ +static int parse_args(int argc, char *argv[]) +{ + int opt; + int long_index; + static const struct option longopts[] = { + {"clk_src", required_argument, NULL, 's'}, + {"time", required_argument, NULL, 't'}, + {"index", required_argument, NULL, 'i'}, + {"rounds", required_argument, NULL, 'r'}, + {"help", no_argument, NULL, 'h'}, + {NULL, 0, NULL, 0} + }; + + static const char *shortopts = "s:t:i:r:h"; + + gbl_args->opt.clk_src = ODP_CLOCK_DEFAULT; + gbl_args->opt.time = 0; /* Measure CPU cycles */ + gbl_args->opt.bench_idx = 0; /* Run all benchmarks */ + gbl_args->opt.rounds = ROUNDS; + + while (1) { + opt = getopt_long(argc, argv, shortopts, longopts, &long_index); + + if (opt == -1) + break; /* No more options */ + + switch (opt) { + case 's': + gbl_args->opt.clk_src = atoi(optarg); + break; + case 't': + gbl_args->opt.time = atoi(optarg); + break; + case 'i': + gbl_args->opt.bench_idx = atoi(optarg); + break; + case 'r': + gbl_args->opt.rounds = atoi(optarg); + break; + case 'h': + usage(); + return 1; + default: + ODPH_ERR("Bad option. Use -h for help.\n"); + return -1; + } + } + + if (gbl_args->opt.rounds < 1) { + ODPH_ERR("Invalid test cycle repeat count: %u\n", gbl_args->opt.rounds); + return -1; + } + + if (gbl_args->opt.bench_idx < 0 || gbl_args->opt.bench_idx > gbl_args->num_bench) { + ODPH_ERR("Bad bench index %i\n", gbl_args->opt.bench_idx); + return -1; + } + + optind = 1; /* Reset 'extern optind' from the getopt lib */ + + return 0; +} + +/* Print system and application info */ +static void print_info(void) +{ + odp_sys_info_print(); + + printf("\n" + "odp_bench_timer options\n" + "-----------------------\n"); + + printf("CPU mask: %s\n", gbl_args->cpumask_str); + printf("Clock source: %i\n", gbl_args->opt.clk_src); + printf("Measurement unit: %s\n", gbl_args->opt.time ? "nsec" : "CPU cycles"); + printf("Test rounds: %u\n", gbl_args->opt.rounds); + printf("Timer duration: %" PRIu64 " nsec\n", gbl_args->timer_nsec); + printf("Timer tick freq: %.2f Hz\n", gbl_args->tick_hz); + printf("\n"); +} + +static int create_timer(void) +{ + odp_pool_capability_t pool_capa; + odp_timer_capability_t timer_capa; + odp_timer_clk_src_t clk_src; + odp_timer_pool_param_t tp_param; + odp_timer_pool_t tp; + odp_pool_t pool; + odp_pool_param_t pool_param; + odp_timeout_t tmo; + odp_queue_param_t queue_param; + odp_queue_t queue; + odp_timer_t timer; + uint64_t t1, t2, diff, tick1, tick2; + + if (odp_pool_capability(&pool_capa)) { + ODPH_ERR("Pool capa failed\n"); + return -1; + } + + clk_src = gbl_args->opt.clk_src; + if (odp_timer_capability(clk_src, &timer_capa)) { + ODPH_ERR("Timer capa failed\n"); + return -1; + } + + odp_timer_pool_param_init(&tp_param); + tp_param.clk_src = clk_src; + tp_param.res_ns = timer_capa.max_res.res_ns; + tp_param.min_tmo = timer_capa.max_res.min_tmo; + tp_param.max_tmo = timer_capa.max_res.max_tmo; + tp_param.num_timers = 10; + + tp = odp_timer_pool_create("bench_timer", &tp_param); + + if (tp == ODP_TIMER_POOL_INVALID) { + ODPH_ERR("Timer pool create failed\n"); + return -1; + } + + gbl_args->timer_pool = tp; + + odp_timer_pool_start(); + + gbl_args->timer_nsec = TIMER_NSEC; + if (TIMER_NSEC < tp_param.min_tmo) + gbl_args->timer_nsec = tp_param.min_tmo; + else if (TIMER_NSEC > tp_param.max_tmo) + gbl_args->timer_nsec = tp_param.max_tmo; + + odp_pool_param_init(&pool_param); + pool_param.type = ODP_POOL_TIMEOUT; + pool_param.tmo.num = 10; + pool_param.tmo.uarea_size = UAREA_SIZE; + if (UAREA_SIZE > pool_capa.tmo.max_uarea_size) + pool_param.tmo.uarea_size = pool_capa.tmo.max_uarea_size; + + pool = odp_pool_create("bench_timer", &pool_param); + + if (pool == ODP_POOL_INVALID) { + ODPH_ERR("Timeout pool create failed\n"); + return -1; + } + + gbl_args->pool = pool; + + tmo = odp_timeout_alloc(pool); + + if (tmo == ODP_TIMEOUT_INVALID) { + ODPH_ERR("Timeout alloc failed\n"); + return -1; + } + + gbl_args->timeout = tmo; + gbl_args->tick = odp_timer_current_tick(tp); + gbl_args->nsec = odp_timer_tick_to_ns(tp, gbl_args->tick); + + /* Measure timer tick frequency for test information */ + t1 = odp_time_global_strict_ns(); + tick1 = odp_timer_current_tick(tp); + + odp_time_wait_ns(200 * ODP_TIME_MSEC_IN_NS); + + tick2 = odp_timer_current_tick(tp); + t2 = odp_time_global_strict_ns(); + diff = t2 - t1; + + if (diff) + gbl_args->tick_hz = (tick2 - tick1) / ((double)diff / ODP_TIME_SEC_IN_NS); + + odp_queue_param_init(&queue_param); + queue_param.type = ODP_QUEUE_TYPE_SCHED; + queue_param.sched.prio = odp_schedule_default_prio(); + queue_param.sched.sync = ODP_SCHED_SYNC_ATOMIC; + queue_param.sched.group = ODP_SCHED_GROUP_ALL; + + if (timer_capa.queue_type_sched == 0) { + queue_param.type = ODP_QUEUE_TYPE_PLAIN; + gbl_args->plain_queue = 1; + } + + queue = odp_queue_create("bench_timer", &queue_param); + if (queue == ODP_QUEUE_INVALID) { + ODPH_ERR("Queue create failed\n"); + return -1; + } + + gbl_args->queue = queue; + + timer = odp_timer_alloc(tp, queue, (void *)(uintptr_t)0xdeadbeef); + if (timer == ODP_TIMER_INVALID) { + ODPH_ERR("Timer alloc failed\n"); + return -1; + } + + gbl_args->timer = timer; + + return 0; +} + +static int wait_timer(void) +{ + odp_timer_start_t start_param; + odp_timer_t timer = gbl_args->timer; + odp_timer_pool_t tp = gbl_args->timer_pool; + uint64_t wait_nsec = 2 * gbl_args->timer_nsec; + uint64_t sched_wait = odp_schedule_wait_time(wait_nsec); + odp_event_t ev; + uint64_t start; + + start_param.tick_type = ODP_TIMER_TICK_REL; + start_param.tick = odp_timer_ns_to_tick(tp, gbl_args->timer_nsec); + start_param.tmo_ev = odp_timeout_to_event(gbl_args->timeout); + + if (odp_timer_start(timer, &start_param) != ODP_TIMER_SUCCESS) { + ODPH_ERR("Timer start failed\n"); + return -1; + } + + gbl_args->timeout = ODP_TIMEOUT_INVALID; + gbl_args->event = ODP_EVENT_INVALID; + + /* Wait for timeout */ + if (gbl_args->plain_queue) { + start = odp_time_global_ns(); + while (1) { + ev = odp_queue_deq(gbl_args->queue); + + if (ev != ODP_EVENT_INVALID) + break; + + if ((odp_time_global_ns() - start) > wait_nsec) { + ODPH_ERR("Timeout event missing\n"); + return -1; + } + } + + gbl_args->event = ev; + } else { + ev = odp_schedule(NULL, sched_wait); + + if (ev == ODP_EVENT_INVALID) { + ODPH_ERR("Timeout event missing\n"); + return -1; + } + + gbl_args->event = ev; + + /* Free schedule context */ + if (odp_schedule(NULL, ODP_SCHED_NO_WAIT) != ODP_EVENT_INVALID) { + ODPH_ERR("Extra timeout event\n"); + return -1; + } + } + + if (odp_event_type(gbl_args->event) != ODP_EVENT_TIMEOUT) { + ODPH_ERR("Bad event type\n"); + return -1; + } + + gbl_args->timeout = odp_timeout_from_event(gbl_args->event); + + return 0; +} + +int main(int argc, char *argv[]) +{ + odph_helper_options_t helper_options; + odph_thread_t worker_thread; + odph_thread_common_param_t thr_common; + odph_thread_param_t thr_param; + int cpu, i; + odp_shm_t shm; + odp_cpumask_t cpumask, default_mask; + odp_instance_t instance; + odp_init_t init_param; + int ret = 0; + + /* Let helper collect its own arguments (e.g. --odph_proc) */ + argc = odph_parse_options(argc, argv); + if (odph_options(&helper_options)) { + ODPH_ERR("Reading ODP helper options failed\n"); + exit(EXIT_FAILURE); + } + + odp_init_param_init(&init_param); + init_param.mem_model = helper_options.mem_model; + + /* Init ODP before calling anything else */ + if (odp_init_global(&instance, &init_param, NULL)) { + ODPH_ERR("Global init failed\n"); + exit(EXIT_FAILURE); + } + + /* Init this thread */ + if (odp_init_local(instance, ODP_THREAD_CONTROL)) { + ODPH_ERR("Local init failed\n"); + exit(EXIT_FAILURE); + } + + if (setup_sig_handler()) { + ODPH_ERR("Signal handler setup failed\n"); + exit(EXIT_FAILURE); + } + + odp_schedule_config(NULL); + + /* Reserve memory for args from shared mem */ + shm = odp_shm_reserve("shm_args", sizeof(gbl_args_t), ODP_CACHE_LINE_SIZE, 0); + if (shm == ODP_SHM_INVALID) { + ODPH_ERR("Shared mem reserve failed\n"); + exit(EXIT_FAILURE); + } + + gbl_args = odp_shm_addr(shm); + if (gbl_args == NULL) { + ODPH_ERR("Shared mem alloc failed\n"); + exit(EXIT_FAILURE); + } + + memset(gbl_args, 0, sizeof(gbl_args_t)); + odp_atomic_init_u32(&gbl_args->exit_thread, 0); + gbl_args->timer_pool = ODP_TIMER_POOL_INVALID; + gbl_args->timer = ODP_TIMER_INVALID; + gbl_args->queue = ODP_QUEUE_INVALID; + gbl_args->pool = ODP_POOL_INVALID; + gbl_args->timeout = ODP_TIMEOUT_INVALID; + + gbl_args->bench = test_suite; + gbl_args->num_bench = sizeof(test_suite) / sizeof(test_suite[0]); + + for (i = 0; i < REPEAT_COUNT; i++) { + gbl_args->a1[i] = i; + gbl_args->ev[i] = ODP_EVENT_INVALID; + gbl_args->tmo[i] = ODP_TIMEOUT_INVALID; + gbl_args->tim[i] = ODP_TIMER_INVALID; + } + + /* Parse and store the application arguments */ + ret = parse_args(argc, argv); + if (ret) + goto exit; + + /* Get default worker cpumask */ + if (odp_cpumask_default_worker(&default_mask, 1) != 1) { + ODPH_ERR("Unable to allocate worker thread\n"); + ret = -1; + goto exit; + } + + (void)odp_cpumask_to_str(&default_mask, gbl_args->cpumask_str, + sizeof(gbl_args->cpumask_str)); + + /* Create timer and other resources */ + ret = create_timer(); + if (ret) + goto exit; + + print_info(); + + /* Start one timer and wait for the timeout event. Timer expiration fills in + * timeout event metadata. */ + ret = wait_timer(); + if (ret) + goto exit; + + memset(&worker_thread, 0, sizeof(odph_thread_t)); + + /* Create worker thread */ + cpu = odp_cpumask_first(&default_mask); + + odp_cpumask_zero(&cpumask); + odp_cpumask_set(&cpumask, cpu); + + odph_thread_common_param_init(&thr_common); + thr_common.instance = instance; + thr_common.cpumask = &cpumask; + thr_common.share_param = 1; + + odph_thread_param_init(&thr_param); + thr_param.start = run_benchmarks; + thr_param.arg = gbl_args; + thr_param.thr_type = ODP_THREAD_WORKER; + + odph_thread_create(&worker_thread, &thr_common, &thr_param, 1); + + odph_thread_join(&worker_thread, 1); + + ret = gbl_args->bench_failed; + +exit: + if (gbl_args->timeout != ODP_TIMEOUT_INVALID) + odp_timeout_free(gbl_args->timeout); + + if (gbl_args->pool != ODP_POOL_INVALID) + odp_pool_destroy(gbl_args->pool); + + if (gbl_args->timer != ODP_TIMER_INVALID) + odp_timer_free(gbl_args->timer); + + if (gbl_args->timer_pool != ODP_TIMER_POOL_INVALID) + odp_timer_pool_destroy(gbl_args->timer_pool); + + if (gbl_args->queue != ODP_QUEUE_INVALID) { + if (odp_queue_destroy(gbl_args->queue)) { + ODPH_ERR("Queue destroy failed\n"); + exit(EXIT_FAILURE); + } + } + + if (odp_shm_free(shm)) { + ODPH_ERR("Shared mem free failed\n"); + exit(EXIT_FAILURE); + } + + if (odp_term_local()) { + ODPH_ERR("Local term failed\n"); + exit(EXIT_FAILURE); + } + + if (odp_term_global(instance)) { + ODPH_ERR("Global term failed\n"); + exit(EXIT_FAILURE); + } + + if (ret < 0) + return EXIT_FAILURE; + + return EXIT_SUCCESS; +} diff --git a/test/performance/odp_dma_perf.c b/test/performance/odp_dma_perf.c index 62899f913..be23f27ca 100644 --- a/test/performance/odp_dma_perf.c +++ b/test/performance/odp_dma_perf.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2021-2022, Nokia +/* Copyright (c) 2021-2023, Nokia * * All rights reserved. * @@ -9,266 +9,515 @@ #define _GNU_SOURCE #endif -#include <stdio.h> -#include <stdlib.h> -#include <string.h> #include <inttypes.h> +#include <stdlib.h> +#include <signal.h> +#include <stdint.h> +#include <unistd.h> #include <odp_api.h> #include <odp/helper/odph_api.h> #define EXIT_NOT_SUP 2 - -#define DEFAULT_SEG_SIZE 1024U -#define ROUNDS 1000000 -#define DEFAULT_WAIT_NS ODP_TIME_SEC_IN_NS -#define COMPL_DELIMITER "," -/* For now, a static maximum amount of input segments */ -#define MAX_NUM_IN_SEGS 64 -#define SHM_SRC "odp_dma_perf_shm_src" -#define SHM_DST "odp_dma_perf_shm_dst" - -#define TRS_TYPE_SYNC 0 -#define TRS_TYPE_ASYNC 1 - -#define GRN_ALL 0 -#define GRN_IND 1 - -#define TYPE_PKT 0 -#define TYPE_MEM 1 - -#define COMPL_MODE_POLL 0 -#define COMPL_MODE_EVENT 1 +#define PROG_NAME "odp_dma_perf" + +enum { + SYNC = 0U, + ASYNC +}; + +enum { + PACKET = 0U, + MEMORY +}; + +enum { + POLL = 0U, + EVENT +}; + +enum { + SINGLE = 0U, + MANY +}; + +#define DEF_TRS_TYPE SYNC +#define DEF_SEG_CNT 1U +#define DEF_LEN 1024U +#define DEF_SEG_TYPE PACKET +#define DEF_MODE POLL +#define DEF_INFLIGHT 1U +#define DEF_TIME 10U +#define DEF_WORKERS 1U +#define DEF_POLICY SINGLE + +#define MAX_SEGS 1024U +#define MAX_WORKERS 24 + +#define MIN(a, b) (((a) < (b)) ? (a) : (b)) +#define MAX(a, b) (((a) < (b)) ? (b) : (a)) #define GIGAS 1000000000 #define MEGAS 1000000 #define KILOS 1000 -#define RETRIES 1000U - -typedef struct test_config_t { - int trs_type; - int trs_grn; - int num_in_seg; - uint32_t seg_size; - int seg_type; - int num_rounds; - int dma_rounds; - uint64_t wait_ns; - - struct { - int num_modes; - uint32_t compl_mask; - int modes[MAX_NUM_IN_SEGS]; - } compl_modes; - +typedef enum { + PRS_OK, + PRS_NOK, + PRS_TERM, + PRS_NOT_SUP +} parse_result_t; + +typedef struct { + uint64_t completed; + uint64_t start_errs; + uint64_t poll_errs; + uint64_t scheduler_timeouts; + uint64_t transfer_errs; + uint64_t tot_tm; + uint64_t trs_tm; + uint64_t max_trs_tm; + uint64_t min_trs_tm; + uint64_t start_cc; + uint64_t max_start_cc; + uint64_t min_start_cc; + uint64_t wait_cc; + uint64_t max_wait_cc; + uint64_t min_wait_cc; + uint64_t trs_cc; + uint64_t max_trs_cc; + uint64_t min_trs_cc; + uint64_t start_cnt; + uint64_t wait_cnt; + uint64_t trs_poll_cnt; + uint64_t trs_cnt; +} stats_t; + +typedef struct { + odp_dma_transfer_param_t trs_param; + odp_dma_compl_param_t compl_param; + odp_ticketlock_t lock; + uint64_t trs_start_tm; + uint64_t trs_start_cc; + uint64_t trs_poll_cnt; + odp_bool_t is_running; +} trs_info_t; + +typedef struct ODP_ALIGNED_CACHE { struct { + trs_info_t infos[MAX_SEGS]; + odp_dma_seg_t src_seg[MAX_SEGS]; + odp_dma_seg_t dst_seg[MAX_SEGS]; + odp_dma_t handle; odp_pool_t pool; odp_queue_t compl_q; - odp_dma_t handle; - odp_dma_seg_t dst_seg; - odp_dma_seg_t src_seg[MAX_NUM_IN_SEGS]; - } dma_config; - - union { - struct { - odp_shm_t shm_src; - odp_shm_t shm_dst; - void *src; - void *dst; - }; - - struct { - odp_pool_t pool; - odp_packet_t pkts[MAX_NUM_IN_SEGS + 1]; - }; - } seg_config; + uint32_t num_in_segs; + uint32_t num_out_segs; + uint32_t src_seg_len; + uint32_t dst_seg_len; + uint32_t num_inflight; + uint8_t trs_type; + uint8_t compl_mode; + } dma; struct { - int (*setup_fn)(struct test_config_t *config); - void (*trs_base_fn)(struct test_config_t *config, - odp_dma_transfer_param_t *trs_params, uint32_t *trs_lengths); - void (*trs_dyn_fn)(struct test_config_t *config, uint32_t offset, uint32_t len); - int (*verify_fn)(const struct test_config_t *config); - void (*free_fn)(struct test_config_t *config); - int (*run_fn)(struct test_config_t *config); - } test_case_api; -} test_config_t; - -typedef struct compl_wait_entry_t { - int type; - odp_dma_transfer_id_t id; -} compl_wait_entry_t; - -static const int compl_mode_map[] = { ODP_DMA_COMPL_POLL, ODP_DMA_COMPL_EVENT }; - -static void set_option_defaults(test_config_t *config) + odp_packet_t src_pkt[MAX_SEGS]; + odp_packet_t dst_pkt[MAX_SEGS]; + odp_pool_t src_pool; + odp_pool_t dst_pool; + odp_shm_t src_shm; + odp_shm_t dst_shm; + void *src; + void *dst; + } seg; + + odp_schedule_group_t grp; +} sd_t; + +typedef struct prog_config_s prog_config_t; + +typedef struct ODP_ALIGNED_CACHE { + stats_t stats; + prog_config_t *prog_config; + sd_t *sd; +} thread_config_t; + +typedef struct { + /* Configure DMA session specific resources. */ + odp_bool_t (*session_cfg_fn)(sd_t *sd); + /* Setup transfer elements (memory/packet segments). */ + odp_bool_t (*setup_fn)(sd_t *sd); + /* Configure DMA transfers (segment addresses etc.). */ + void (*trs_fn)(sd_t *sd); + /* Configure transfer completion resources (transfer IDs, events etc.). */ + odp_bool_t (*compl_fn)(sd_t *sd); + /* Initiate required initial transfers. */ + odp_bool_t (*bootstrap_fn)(sd_t *sd); + /* Wait and handle finished transfer. */ + void (*wait_fn)(sd_t *sd, stats_t *stats); + /* Handle all unfinished transfers after main test has been stopped. */ + void (*drain_fn)(void); + /* Free any resources that might have been allocated during setup phase. */ + void (*free_fn)(const sd_t *sd); +} test_api_t; + +typedef struct prog_config_s { + odph_thread_t threads[MAX_WORKERS]; + thread_config_t thread_config[MAX_WORKERS]; + sd_t sds[MAX_WORKERS]; + test_api_t api; + odp_atomic_u32_t is_running; + odp_instance_t odp_instance; + odp_barrier_t init_barrier; + odp_barrier_t term_barrier; + odp_dma_compl_mode_t compl_mode_mask; + odp_pool_t src_pool; + odp_pool_t dst_pool; + uint32_t num_in_segs; + uint32_t num_out_segs; + uint32_t src_seg_len; + uint32_t dst_seg_len; + uint32_t num_inflight; + uint32_t time_sec; + uint32_t num_sessions; + int num_workers; + uint8_t trs_type; + uint8_t seg_type; + uint8_t compl_mode; + uint8_t policy; +} prog_config_t; + +static prog_config_t *prog_conf; + +static const int mode_map[] = { ODP_DMA_COMPL_POLL, ODP_DMA_COMPL_EVENT }; + +static void terminate(int signal ODP_UNUSED) { - memset(config, 0, sizeof(*config)); - config->num_in_seg = 1; - config->seg_size = DEFAULT_SEG_SIZE; - config->num_rounds = ROUNDS; - config->wait_ns = DEFAULT_WAIT_NS; - config->compl_modes.compl_mask = ODP_DMA_COMPL_SYNC; + odp_atomic_store_u32(&prog_conf->is_running, 0U); } -static void parse_completion_modes(test_config_t *config, const char *optarg) +static void init_config(prog_config_t *config) { - char *tmp_str = strdup(optarg); - char *tmp; - int mode; - uint32_t i = 0U; - - config->compl_modes.num_modes = 0; - - if (tmp_str == NULL) - return; + sd_t *sd; + trs_info_t *info; + stats_t *stats; - tmp = strtok(tmp_str, COMPL_DELIMITER); + memset(config, 0, sizeof(*config)); + config->compl_mode_mask |= ODP_DMA_COMPL_SYNC; + config->src_pool = ODP_POOL_INVALID; + config->dst_pool = ODP_POOL_INVALID; + config->num_in_segs = DEF_SEG_CNT; + config->num_out_segs = DEF_SEG_CNT; + config->src_seg_len = DEF_LEN; + config->num_inflight = DEF_INFLIGHT; + config->time_sec = DEF_TIME; + config->num_workers = DEF_WORKERS; + config->trs_type = DEF_TRS_TYPE; + config->seg_type = DEF_SEG_TYPE; + config->compl_mode = DEF_MODE; + config->policy = DEF_POLICY; + + for (uint32_t i = 0U; i < MAX_WORKERS; ++i) { + sd = &config->sds[i]; + stats = &config->thread_config[i].stats; + memset(sd, 0, sizeof(*sd)); + + for (uint32_t i = 0U; i < MAX_SEGS; ++i) { + info = &sd->dma.infos[i]; + info->compl_param.transfer_id = ODP_DMA_TRANSFER_ID_INVALID; + info->compl_param.event = ODP_EVENT_INVALID; + info->compl_param.queue = ODP_QUEUE_INVALID; + odp_ticketlock_init(&info->lock); + sd->seg.src_pkt[i] = ODP_PACKET_INVALID; + sd->seg.dst_pkt[i] = ODP_PACKET_INVALID; + } - while (tmp) { - mode = atoi(tmp); - config->compl_modes.modes[i] = mode; - config->compl_modes.compl_mask |= compl_mode_map[mode]; - ++i; - ++config->compl_modes.num_modes; - tmp = strtok(NULL, COMPL_DELIMITER); + sd->dma.handle = ODP_DMA_INVALID; + sd->dma.pool = ODP_POOL_INVALID; + sd->dma.compl_q = ODP_QUEUE_INVALID; + sd->seg.src_shm = ODP_SHM_INVALID; + sd->seg.dst_shm = ODP_SHM_INVALID; + sd->grp = ODP_SCHED_GROUP_INVALID; + stats->min_trs_tm = UINT64_MAX; + stats->min_start_cc = UINT64_MAX; + stats->min_wait_cc = UINT64_MAX; + stats->min_trs_cc = UINT64_MAX; } - - free(tmp_str); } static void print_usage(void) { printf("\n" - "DMA performance test. Transfers a set of source segments to a single destination\n" - "segment.\n" + "DMA performance test. Load DMA subsystem from several workers.\n" "\n" "Examples:\n" - " odp_dma_perf\n" - " odp_dma_perf -t 0 -g 1 -i 2\n" - " odp_dma_perf -t 1 -g 1 -i 4 -m 0,0,0,0\n" - " odp_dma_perf -t 1 -g 1 -i 7 -m 0,0,0,0,0,0,1 -T 1 -r 1000 -s 2048\n" + " " PROG_NAME "\n" + " " PROG_NAME " -s 10240\n" + " " PROG_NAME " -t 0 -i 1 -o 1 -s 51200 -S 1 -f 64 -T 10\n" + " " PROG_NAME " -t 1 -i 10 -o 10 -s 4096 -S 0 -m 1 -f 10 -c 4 -p 1\n" "\n" - "Usage: odp_dma_perf [options]\n" + "Usage: " PROG_NAME " [options]\n" "\n" - " -t, --trs_type Transfer type for test data. Synchronous by default.\n" + " -t, --trs_type Transfer type for test data. %u by default.\n" " Types:\n" " 0: synchronous\n" " 1: asynchronous\n" - " -g, --trs_grn Transfer granularity for source segments. All\n" - " segments are sent in one transfer by default.\n" - " Options:\n" - " 0: all segments in a single transfer\n" - " 1: individual transfers for segments\n" - " -i, --num_in_seg Number of input segments to transfer. 1 by\n" - " default. Maximum supported amount is %d.\n" - " -s, --in_seg_size Segment size for all input segments in bytes. 1024\n" - " bytes by default. Maximum allowed destination\n" - " segment size may limit this choice.\n" - " -T, --in_seg_type Input segment data type. Packet by default.\n" + " -i, --num_in_seg Number of input segments to transfer. 0 means the\n" + " maximum count supported by the implementation. %u by\n" + " default.\n" + " -o, --num_out_seg Number of output segments to transfer to. 0 means\n" + " the maximum count supported by the implementation.\n" + " %u by default.\n" + " -s, --in_seg_len Input segment length in bytes. 0 length means\n" + " maximum segment length supported by implementation.\n" + " The actual maximum might be limited by what type of\n" + " data is transferred (packet/memory).\n" + " %u by default.\n" + " -S, --in_seg_type Input segment data type. %u by default.\n" " Types:\n" " 0: packet\n" " 1: memory\n" - " -m, --compl_modes Completion mode(s) for transfers delimited by a\n" - " comma. Only applicable in asynchronous mode.\n" + " -m, --compl_mode Completion mode for transfers. %u by default.\n" " Modes:\n" " 0: poll\n" " 1: event\n" - " -r, --num_rounds Number of times to run the test scenario. %d by\n" + " -f, --max_in_flight Max transfers in-flight per session. 0 means the\n" + " maximum supported by tester/implementation. %u by\n" + " default.\n" + " -T, --time_sec Time in seconds to run. 0 means infinite. %u by\n" " default.\n" - " -w, --wait_nsec Number of nanoseconds to wait for completion events.\n" - " 1 second (1000000000) by default.\n" + " -c, --worker_count Amount of workers. %u by default.\n" + " -p, --policy DMA session policy. %u by default.\n" + " Policies:\n" + " 0: One session shared by workers\n" + " 1: One session per worker\n" " -h, --help This help.\n" - "\n", - MAX_NUM_IN_SEGS, ROUNDS); + "\n", DEF_TRS_TYPE, DEF_SEG_CNT, DEF_SEG_CNT, DEF_LEN, DEF_SEG_TYPE, DEF_MODE, + DEF_INFLIGHT, DEF_TIME, DEF_WORKERS, DEF_POLICY); } -static int check_completion_modes(test_config_t *config) +static parse_result_t check_options(prog_config_t *config) { - if (config->trs_type == TRS_TYPE_SYNC) - return 0; + int max_workers; + odp_dma_capability_t dma_capa; + uint32_t num_sessions, max_seg_len, max_trs, max_in, max_out, max_segs; + odp_schedule_capability_t sched_capa; + odp_pool_capability_t pool_capa; + odp_shm_capability_t shm_capa; + uint64_t shm_size = 0U; + + if (config->trs_type != SYNC && config->trs_type != ASYNC) { + ODPH_ERR("Invalid transfer type: %u\n", config->trs_type); + return PRS_NOK; + } + + if (config->seg_type != PACKET && config->seg_type != MEMORY) { + ODPH_ERR("Invalid segment type: %u\n", config->seg_type); + return PRS_NOK; + } + + max_workers = MIN(odp_thread_count_max() - 1, MAX_WORKERS); - if (config->compl_modes.num_modes > MAX_NUM_IN_SEGS) - return -1; + if (config->num_workers <= 0 || config->num_workers > max_workers) { + ODPH_ERR("Invalid thread count: %d (min: 1, max: %d)\n", config->num_workers, + max_workers); + return PRS_NOK; + } - if (config->trs_grn == GRN_IND && - config->num_in_seg != config->compl_modes.num_modes) - return -1; + if (config->policy != SINGLE && config->policy != MANY) { + ODPH_ERR("Invalid DMA session policy: %u\n", config->policy); + return PRS_NOK; + } - if (config->trs_grn == GRN_ALL && - config->compl_modes.num_modes != 1) - return -1; + if (odp_dma_capability(&dma_capa) < 0) { + ODPH_ERR("Error querying DMA capabilities\n"); + return PRS_NOK; + } - for (int i = 0; i < config->compl_modes.num_modes; ++i) { - if (config->compl_modes.modes[i] != COMPL_MODE_POLL && - config->compl_modes.modes[i] != COMPL_MODE_EVENT) - return -1; + num_sessions = config->policy == SINGLE ? 1 : config->num_workers; - config->compl_modes.modes[i] = compl_mode_map[config->compl_modes.modes[i]]; + if (num_sessions > dma_capa.max_sessions) { + ODPH_ERR("Not enough DMA sessions supported: %u (max: %u)\n", num_sessions, + dma_capa.max_sessions); + return PRS_NOT_SUP; } - return 0; -} + config->num_sessions = num_sessions; -static int check_options(test_config_t *config) -{ - if (config->trs_type != TRS_TYPE_SYNC && - config->trs_type != TRS_TYPE_ASYNC) { - ODPH_ERR("Invalid transfer type: %d.\n", config->trs_type); - return -1; + if (config->num_in_segs == 0U) + config->num_in_segs = dma_capa.max_src_segs; + + if (config->num_out_segs == 0U) + config->num_out_segs = dma_capa.max_dst_segs; + + if (config->num_in_segs > dma_capa.max_src_segs || + config->num_out_segs > dma_capa.max_dst_segs || + config->num_in_segs + config->num_out_segs > dma_capa.max_segs) { + ODPH_ERR("Unsupported segment count configuration, in: %u, out: %u (max in: %u, " + "max out: %u, max tot: %u)\n", config->num_in_segs, config->num_out_segs, + dma_capa.max_src_segs, dma_capa.max_dst_segs, dma_capa.max_segs); + return PRS_NOT_SUP; } - if (config->trs_grn != GRN_ALL && config->trs_grn != GRN_IND) { - ODPH_ERR("Invalid granularity: %d.\n", config->trs_grn); - return -1; + if (config->src_seg_len == 0U) + config->src_seg_len = dma_capa.max_seg_len; + + config->dst_seg_len = config->src_seg_len * config->num_in_segs / + config->num_out_segs + config->src_seg_len * + config->num_in_segs % config->num_out_segs; + + max_seg_len = MAX(config->src_seg_len, config->dst_seg_len); + + if (max_seg_len > dma_capa.max_seg_len) { + ODPH_ERR("Unsupported total DMA segment length: %u (max: %u)\n", max_seg_len, + dma_capa.max_seg_len); + return PRS_NOT_SUP; } - config->dma_rounds = config->trs_grn == GRN_IND ? config->num_in_seg : 1; + if (config->trs_type == ASYNC) { + if (config->compl_mode != POLL && config->compl_mode != EVENT) { + ODPH_ERR("Invalid completion mode: %u\n", config->compl_mode); + return PRS_NOK; + } + + if (config->compl_mode == POLL && (dma_capa.compl_mode_mask & ODP_DMA_COMPL_POLL) + == 0U) { + ODPH_ERR("Unsupported DMA completion mode, poll\n"); + return PRS_NOT_SUP; + } + + if (config->compl_mode == EVENT) { + if (config->num_sessions > dma_capa.pool.max_pools) { + ODPH_ERR("Unsupported amount of completion pools: %u (max: %u)\n", + config->num_sessions, dma_capa.pool.max_pools); + return PRS_NOT_SUP; + } + + if ((dma_capa.compl_mode_mask & ODP_DMA_COMPL_EVENT) == 0U) { + ODPH_ERR("Unsupported DMA completion mode, event\n"); + return PRS_NOT_SUP; + } + + if (dma_capa.queue_type_sched == 0) { + ODPH_ERR("Unsupported DMA queueing type, scheduled\n"); + return PRS_NOT_SUP; + } + + if (config->num_inflight > dma_capa.pool.max_num) { + ODPH_ERR("Unsupported amount of completion events: %u (max: %u)\n", + config->num_inflight, dma_capa.pool.max_num); + return PRS_NOT_SUP; + } + + if (odp_schedule_capability(&sched_capa) < 0) { + ODPH_ERR("Error querying scheduler capabilities\n"); + return PRS_NOK; + } + + if (config->num_sessions > sched_capa.max_groups - 3U) { + ODPH_ERR("Unsupported amount of scheduler groups: %u (max: %u)\n", + config->num_sessions, sched_capa.max_groups - 3U); + return PRS_NOT_SUP; + } + } - if (config->num_in_seg < 1 || config->num_in_seg > MAX_NUM_IN_SEGS) { - ODPH_ERR("Invalid number of input segments: %d.\n", config->num_in_seg); - return -1; + config->compl_mode_mask |= mode_map[config->compl_mode]; } - if (config->seg_type != TYPE_PKT && config->seg_type != TYPE_MEM) { - ODPH_ERR("Invalid input segment type: %d.\n", config->seg_type); - return -1; + max_trs = MIN(dma_capa.max_transfers, MAX_SEGS); + + if (config->num_inflight == 0U) + config->num_inflight = max_trs; + + if (config->num_inflight > max_trs) { + ODPH_ERR("Unsupported amount of in-flight DMA transfers: %u (max: %u)\n", + config->num_inflight, max_trs); + return PRS_NOT_SUP; } - if (check_completion_modes(config)) { - ODPH_ERR("Invalid completion modes.\n"); - return -1; + max_in = config->num_in_segs * config->num_inflight; + max_out = config->num_out_segs * config->num_inflight; + max_segs = MAX(max_in, max_out); + + if (max_segs > MAX_SEGS) { + ODPH_ERR("Unsupported input/output * inflight segment combination: %u (max: %u)\n", + max_segs, MAX_SEGS); + return PRS_NOT_SUP; } - if (config->num_rounds < 1) { - ODPH_ERR("Invalid number of rounds: %d.\n", config->num_rounds); - return -1; + if (config->seg_type == PACKET) { + if (odp_pool_capability(&pool_capa) < 0) { + ODPH_ERR("Error querying pool capabilities\n"); + return PRS_NOK; + } + + if (pool_capa.pkt.max_pools < 2U) { + ODPH_ERR("Unsupported amount of packet pools: 2 (max: %u)\n", + pool_capa.pkt.max_pools); + return PRS_NOT_SUP; + } + + if (pool_capa.pkt.max_len != 0U && max_seg_len > pool_capa.pkt.max_len) { + ODPH_ERR("Unsupported packet size: %u (max: %u)\n", max_seg_len, + pool_capa.pkt.max_len); + return PRS_NOT_SUP; + } + + if (pool_capa.pkt.max_num != 0U && + max_segs * num_sessions > pool_capa.pkt.max_num) { + ODPH_ERR("Unsupported amount of packet pool elements: %u (max: %u)\n", + max_segs * num_sessions, pool_capa.pkt.max_num); + return PRS_NOT_SUP; + } + } else { + /* If SHM implementation capabilities are very puny, program will have already + * failed when reserving memory for global program configuration. */ + if (odp_shm_capability(&shm_capa) < 0) { + ODPH_ERR("Error querying SHM capabilities\n"); + return PRS_NOK; + } + + /* One block for program configuration, one for source memory and one for + * destination memory. */ + if (shm_capa.max_blocks < 3U) { + ODPH_ERR("Unsupported amount of SHM blocks: 3 (max: %u)\n", + shm_capa.max_blocks); + return PRS_NOT_SUP; + } + + shm_size = (uint64_t)config->dst_seg_len * config->num_out_segs * + config->num_inflight; + + if (shm_capa.max_size != 0U && shm_size > shm_capa.max_size) { + ODPH_ERR("Unsupported total SHM block size: %" PRIu64 "" + " (max: %" PRIu64 ")\n", shm_size, shm_capa.max_size); + return PRS_NOT_SUP; + } } - return 0; + return PRS_OK; } -static int parse_options(int argc, char **argv, test_config_t *config) +static parse_result_t parse_options(int argc, char **argv, prog_config_t *config) { int opt, long_index; - static const struct option longopts[] = { { "trs_type", required_argument, NULL, 't' }, - { "trs_grn", required_argument, NULL, 'g' }, { "num_in_seg", required_argument, NULL, 'i' }, - { "in_seg_size", required_argument, NULL, 's' }, - { "in_seg_type", required_argument, NULL, 'T' }, - { "compl_modes", required_argument, NULL, 'm' }, - { "num_rounds", required_argument, NULL, 'r' }, - { "wait_nsec", required_argument, NULL, 'w' }, + { "num_out_seg", required_argument, NULL, 'o' }, + { "in_seg_len", required_argument, NULL, 's' }, + { "in_seg_type", required_argument, NULL, 'S' }, + { "compl_mode", required_argument, NULL, 'm' }, + { "max_in_flight", required_argument, NULL, 'f'}, + { "time_sec", required_argument, NULL, 'T' }, + { "worker_count", required_argument, NULL, 'c' }, + { "policy", required_argument, NULL, 'p' }, { "help", no_argument, NULL, 'h' }, { NULL, 0, NULL, 0 } }; + static const char *shortopts = "t:i:o:s:S:m:f:T:c:p:h"; - static const char *shortopts = "t:g:i:s:T:m:r:w:h"; - - set_option_defaults(config); + init_config(config); while (1) { opt = getopt_long(argc, argv, shortopts, longopts, &long_index); @@ -280,862 +529,1076 @@ static int parse_options(int argc, char **argv, test_config_t *config) case 't': config->trs_type = atoi(optarg); break; - case 'g': - config->trs_grn = atoi(optarg); - break; case 'i': - config->num_in_seg = atoi(optarg); + config->num_in_segs = atoi(optarg); + break; + case 'o': + config->num_out_segs = atoi(optarg); break; case 's': - config->seg_size = atoi(optarg); + config->src_seg_len = atoi(optarg); break; - case 'T': + case 'S': config->seg_type = atoi(optarg); break; case 'm': - parse_completion_modes(config, optarg); + config->compl_mode = atoi(optarg); break; - case 'r': - config->num_rounds = atoi(optarg); + case 'f': + config->num_inflight = atoi(optarg); break; - case 'w': - config->wait_ns = atoll(optarg); + case 'T': + config->time_sec = atoi(optarg); + break; + case 'c': + config->num_workers = atoi(optarg); + break; + case 'p': + config->policy = atoi(optarg); break; case 'h': + print_usage(); + return PRS_TERM; + case '?': default: print_usage(); - return -1; + return PRS_NOK; } } - if (check_options(config)) - return -1; - - return 0; + return check_options(config); } -static int check_shm_capabilities(const test_config_t *config) +static parse_result_t setup_program(int argc, char **argv, prog_config_t *config) { - odp_shm_capability_t capa; - - if (odp_shm_capability(&capa)) { - ODPH_ERR("Error querying SHM capabilities.\n"); - return -1; + struct sigaction action = { .sa_handler = terminate }; + + if (sigemptyset(&action.sa_mask) == -1 || sigaddset(&action.sa_mask, SIGINT) == -1 || + sigaddset(&action.sa_mask, SIGTERM) == -1 || + sigaddset(&action.sa_mask, SIGHUP) == -1 || sigaction(SIGINT, &action, NULL) == -1 || + sigaction(SIGTERM, &action, NULL) == -1 || sigaction(SIGHUP, &action, NULL) == -1) { + ODPH_ERR("Error installing signal handler\n"); + return PRS_NOK; } - if (capa.max_blocks < 2U) { - ODPH_ERR("Unsupported amount of SHM blocks.\n"); - return -1; - } - - if (capa.max_size != 0U && config->num_in_seg * config->seg_size > capa.max_size) { - ODPH_ERR("Unsupported total SHM block size.\n"); - return -1; - } - - if (capa.max_align != 0U && capa.max_align < ODP_CACHE_LINE_SIZE) { - ODPH_ERR("Unsupported SHM block alignment size.\n"); - return -1; - } - - return 0; + return parse_options(argc, argv, config); } -static int check_dma_capabilities(const test_config_t *config) +static odp_pool_t get_src_packet_pool(void) { - odp_dma_capability_t capa; - const int is_event = config->compl_modes.compl_mask & ODP_DMA_COMPL_EVENT; - uint32_t event_compl_count = 0U; - - if (odp_dma_capability(&capa)) { - ODPH_ERR("Error querying DMA capabilities.\n"); - return -1; - } - - if (capa.max_sessions == 0U) { - ODPH_ERR("DMA not supported.\n"); - return -1; - } - - if (config->trs_type == TRS_TYPE_ASYNC) { - if ((config->compl_modes.compl_mask & ODP_DMA_COMPL_POLL) && - (capa.compl_mode_mask & ODP_DMA_COMPL_POLL) == 0U) { - ODPH_ERR("Unsupported DMA completion mode, poll.\n"); - return -1; - } - - if (is_event && (capa.compl_mode_mask & ODP_DMA_COMPL_EVENT) == 0U) { - ODPH_ERR("Unsupported DMA completion mode, event.\n"); - return -1; - } - - if (is_event && capa.queue_type_sched == 0) { - ODPH_ERR("Unsupported DMA queueing type.\n"); - return -1; - } - - if (config->trs_grn == GRN_IND) { - if ((uint32_t)config->num_in_seg > capa.max_transfers) { - ODPH_ERR("Unsupported amount of in-flight DMA transfers.\n"); - return -1; - } - - for (int i = 0; i < config->compl_modes.num_modes; ++i) - if (config->compl_modes.modes[i] == ODP_DMA_COMPL_EVENT) - ++event_compl_count; - - if (event_compl_count > capa.pool.max_num) { - ODPH_ERR("Unsupported amount of completion events.\n"); - return -1; - } - } - } - - if (config->trs_grn == GRN_ALL) { - if ((uint32_t)config->num_in_seg > capa.max_src_segs) { - ODPH_ERR("Unsupported amount of DMA source segments.\n"); - return -1; - } - - if (config->num_in_seg + 1U > capa.max_segs) { - ODPH_ERR("Unsupported total amount of DMA segments.\n"); - return -1; - } - } - - if (config->trs_grn == GRN_IND && capa.max_segs < 2U) { - ODPH_ERR("Unsupported total amount of DMA segments.\n"); - return -1; - } + odp_pool_param_t param; - if (config->num_in_seg * config->seg_size > capa.max_seg_len) { - ODPH_ERR("Unsupported total DMA segment size.\n"); - return -1; - } + if (prog_conf->src_pool != ODP_POOL_INVALID) + return prog_conf->src_pool; - return 0; -} + odp_pool_param_init(¶m); + param.type = ODP_POOL_PACKET; + param.pkt.num = prog_conf->num_inflight * prog_conf->num_in_segs * prog_conf->num_sessions; + param.pkt.len = prog_conf->src_seg_len; + param.pkt.seg_len = prog_conf->src_seg_len; + prog_conf->src_pool = odp_pool_create(PROG_NAME "_src_pkts", ¶m); -static int check_capabilities(const test_config_t *config) -{ - return check_shm_capabilities(config) || - check_dma_capabilities(config); + return prog_conf->src_pool; } -static int configure_packets(test_config_t *config) +static odp_pool_t get_dst_packet_pool(void) { odp_pool_param_t param; - for (int i = 0; i < config->num_in_seg + 1; ++i) - config->seg_config.pkts[i] = ODP_PACKET_INVALID; + if (prog_conf->dst_pool != ODP_POOL_INVALID) + return prog_conf->dst_pool; odp_pool_param_init(¶m); param.type = ODP_POOL_PACKET; - /* Configured amount of input segments and one output segment */ - param.pkt.num = config->num_in_seg + 1U; - param.pkt.len = config->num_in_seg * config->seg_size; - config->seg_config.pool = odp_pool_create("odp_dma_perf_packets", ¶m); - - if (config->seg_config.pool == ODP_POOL_INVALID) { - ODPH_ERR("Error creating packet pool.\n"); - return -1; - } + param.pkt.num = prog_conf->num_inflight * prog_conf->num_out_segs * + prog_conf->num_sessions; + param.pkt.len = prog_conf->dst_seg_len; + param.pkt.seg_len = prog_conf->dst_seg_len; + prog_conf->dst_pool = odp_pool_create(PROG_NAME "_dst_pkts", ¶m); - return 0; + return prog_conf->dst_pool; } -static int allocate_packets(test_config_t *config) +static odp_bool_t configure_packets(sd_t *sd) { - for (int i = 0; i < config->num_in_seg; ++i) { - config->seg_config.pkts[i] = odp_packet_alloc(config->seg_config.pool, - config->seg_size); + sd->seg.src_pool = get_src_packet_pool(); - if (config->seg_config.pkts[i] == ODP_PACKET_INVALID) { - ODPH_ERR("Error allocating input test packets.\n"); - return -1; - } + if (sd->seg.src_pool == ODP_POOL_INVALID) { + ODPH_ERR("Error creating source packet pool\n"); + return false; } - config->seg_config.pkts[config->num_in_seg] = - odp_packet_alloc(config->seg_config.pool, config->num_in_seg * config->seg_size); + sd->seg.dst_pool = get_dst_packet_pool(); - if (config->seg_config.pkts[config->num_in_seg] == ODP_PACKET_INVALID) { - ODPH_ERR("Error allocating output test packet.\n"); - return -1; + if (sd->seg.dst_pool == ODP_POOL_INVALID) { + ODPH_ERR("Error creating destination packet pool\n"); + return false; } - return 0; + return true; } -static int populate_packets(test_config_t *config) +static odp_bool_t allocate_packets(sd_t *sd) { - for (int i = 0; i < config->num_in_seg; ++i) { - uint8_t data[odp_packet_len(config->seg_config.pkts[i])]; - - memset(data, i + 1, sizeof(data)); + for (uint32_t i = 0U; i < sd->dma.num_inflight * sd->dma.num_in_segs; ++i) { + sd->seg.src_pkt[i] = odp_packet_alloc(sd->seg.src_pool, sd->dma.src_seg_len); - if (odp_packet_copy_from_mem(config->seg_config.pkts[i], 0U, sizeof(data), data)) - return -1; + if (sd->seg.src_pkt[i] == ODP_PACKET_INVALID) { + ODPH_ERR("Error allocating source segment packets\n"); + return false; + } } - return 0; -} + for (uint32_t i = 0U; i < sd->dma.num_inflight * sd->dma.num_out_segs; ++i) { + sd->seg.dst_pkt[i] = odp_packet_alloc(sd->seg.dst_pool, sd->dma.dst_seg_len); -static int setup_packet_segments(test_config_t *config) -{ - return configure_packets(config) || - allocate_packets(config) || - populate_packets(config); -} - -static void configure_packet_dma_transfer_base(test_config_t *config, - odp_dma_transfer_param_t trs_params[], - uint32_t trs_lengths[]) -{ - memset(trs_lengths, 0, sizeof(*trs_lengths) * config->dma_rounds); - - for (int i = 0; i < config->num_in_seg; ++i) { - config->dma_config.src_seg[i].packet = config->seg_config.pkts[i]; - config->dma_config.src_seg[i].offset = 0U; - config->dma_config.src_seg[i].len = odp_packet_len(config->seg_config.pkts[i]); + if (sd->seg.dst_pkt[i] == ODP_PACKET_INVALID) { + ODPH_ERR("Error allocating destination segment packets\n"); + return false; + } } - config->dma_config.dst_seg.packet = config->seg_config.pkts[config->num_in_seg]; - - for (int i = 0; i < config->dma_rounds; ++i) { - odp_dma_transfer_param_init(&trs_params[i]); - trs_params[i].src_format = ODP_DMA_FORMAT_PACKET; - trs_params[i].dst_format = ODP_DMA_FORMAT_PACKET; - trs_params[i].num_src = config->trs_grn == GRN_IND ? 1 : config->num_in_seg; - trs_params[i].num_dst = 1U; - trs_params[i].src_seg = &config->dma_config.src_seg[i]; - trs_params[i].dst_seg = &config->dma_config.dst_seg; - trs_lengths[i] = config->trs_grn == GRN_IND ? - config->dma_config.src_seg[i].len : - config->num_in_seg * config->seg_size; - } + return true; } -static inline void configure_packet_dma_transfer_dynamic(test_config_t *config, uint32_t offset, - uint32_t len) +static odp_bool_t setup_packet_segments(sd_t *sd) { - config->dma_config.dst_seg.offset = offset; - config->dma_config.dst_seg.len = len; + return configure_packets(sd) && allocate_packets(sd); } -static int verify_packet_transfer(const test_config_t *config) +static void configure_packet_dma_transfer(sd_t *sd) { - uint32_t len, offset = 0U; - - for (int i = 0; i < config->num_in_seg; ++i) { - len = odp_packet_len(config->seg_config.pkts[i]); - uint8_t src_data[len]; - uint8_t dst_data[len]; - - if (odp_packet_copy_to_mem(config->seg_config.pkts[i], 0U, len, src_data) || - odp_packet_copy_to_mem(config->seg_config.pkts[config->num_in_seg], offset, - len, dst_data)) { - ODPH_ERR("Error verifying DMA transfer.\n"); - return -1; + odp_dma_seg_t *start_src_seg, *start_dst_seg, *seg; + uint32_t k = 0U, z = 0U, len; + odp_packet_t pkt; + odp_dma_transfer_param_t *param; + + for (uint32_t i = 0U; i < sd->dma.num_inflight; ++i) { + start_src_seg = &sd->dma.src_seg[k]; + start_dst_seg = &sd->dma.dst_seg[z]; + + for (uint32_t j = 0U; j < sd->dma.num_in_segs; ++j, ++k) { + pkt = sd->seg.src_pkt[k]; + seg = &start_src_seg[j]; + seg->packet = pkt; + seg->offset = 0U; + seg->len = sd->dma.src_seg_len; } - if (memcmp(src_data, dst_data, len)) { - ODPH_ERR("Error in DMA transfer, source and destination data do not match.\n"); - return -1; + len = sd->dma.num_in_segs * sd->dma.src_seg_len; + + for (uint32_t j = 0U; j < sd->dma.num_out_segs; ++j, ++z) { + pkt = sd->seg.dst_pkt[z]; + seg = &start_dst_seg[j]; + seg->packet = pkt; + seg->offset = 0U; + seg->len = MIN(len, sd->dma.dst_seg_len); + len -= sd->dma.dst_seg_len; } - offset += len; + param = &sd->dma.infos[i].trs_param; + odp_dma_transfer_param_init(param); + param->src_format = ODP_DMA_FORMAT_PACKET; + param->dst_format = ODP_DMA_FORMAT_PACKET; + param->num_src = sd->dma.num_in_segs; + param->num_dst = sd->dma.num_out_segs; + param->src_seg = start_src_seg; + param->dst_seg = start_dst_seg; } - - return 0; } -static void free_packets(test_config_t *config) +static void free_packets(const sd_t *sd) { - /* Configured amount of input segments and one output segment */ - for (int i = 0; i < config->num_in_seg + 1; ++i) - if (config->seg_config.pkts[i] != ODP_PACKET_INVALID) - odp_packet_free(config->seg_config.pkts[i]); + for (uint32_t i = 0U; i < sd->dma.num_inflight * sd->dma.num_in_segs; ++i) { + if (sd->seg.src_pkt[i] != ODP_PACKET_INVALID) + odp_packet_free(sd->seg.src_pkt[i]); + } - if (config->seg_config.pool != ODP_POOL_INVALID) - (void)odp_pool_destroy(config->seg_config.pool); + for (uint32_t i = 0U; i < sd->dma.num_inflight * sd->dma.num_out_segs; ++i) { + if (sd->seg.dst_pkt[i] != ODP_PACKET_INVALID) + odp_packet_free(sd->seg.dst_pkt[i]); + } } -static int allocate_memory(test_config_t *config) +static odp_bool_t allocate_memory(sd_t *sd) { - const uint64_t size = config->num_in_seg * (uint64_t)config->seg_size; + const uint64_t num_segs = (uint64_t)sd->dma.num_in_segs * sd->dma.num_inflight; - config->seg_config.shm_src = ODP_SHM_INVALID; - config->seg_config.shm_dst = ODP_SHM_INVALID; - config->seg_config.src = NULL; - config->seg_config.dst = NULL; + sd->seg.src_shm = odp_shm_reserve(PROG_NAME "_src_shm", sd->dma.src_seg_len * num_segs, + ODP_CACHE_LINE_SIZE, 0U); + sd->seg.dst_shm = odp_shm_reserve(PROG_NAME "_dst_shm", sd->dma.dst_seg_len * num_segs, + ODP_CACHE_LINE_SIZE, 0U); - config->seg_config.shm_src = odp_shm_reserve(SHM_SRC, size, ODP_CACHE_LINE_SIZE, 0); - config->seg_config.shm_dst = odp_shm_reserve(SHM_DST, size, ODP_CACHE_LINE_SIZE, 0); - - if (config->seg_config.shm_src == ODP_SHM_INVALID || - config->seg_config.shm_dst == ODP_SHM_INVALID) { - ODPH_ERR("Error allocating SHM block.\n"); - return -1; + if (sd->seg.src_shm == ODP_SHM_INVALID || sd->seg.dst_shm == ODP_SHM_INVALID) { + ODPH_ERR("Error allocating SHM block\n"); + return false; } - config->seg_config.src = odp_shm_addr(config->seg_config.shm_src); - config->seg_config.dst = odp_shm_addr(config->seg_config.shm_dst); + sd->seg.src = odp_shm_addr(sd->seg.src_shm); + sd->seg.dst = odp_shm_addr(sd->seg.dst_shm); - if (config->seg_config.src == NULL || config->seg_config.dst == NULL) { - ODPH_ERR("Error resolving SHM block address.\n"); - return -1; + if (sd->seg.src == NULL || sd->seg.dst == NULL) { + ODPH_ERR("Error resolving SHM block address\n"); + return false; } - return 0; + return true; } -static int populate_memory(test_config_t *config) +static odp_bool_t setup_memory_segments(sd_t *sd) { - uint8_t val; - uint8_t *addr; - - for (int i = 0; i < config->num_in_seg; ++i) { - val = 0U; - addr = (uint8_t *)config->seg_config.src + i * config->seg_size; - - for (uint32_t i = 0U; i < config->seg_size; ++i) - addr[i] = val++; - } - - return 0; + return allocate_memory(sd); } -static int setup_memory_segments(test_config_t *config) +static void configure_address_dma_transfer(sd_t *sd) { - return allocate_memory(config) || - populate_memory(config); -} + odp_dma_seg_t *start_src_seg, *start_dst_seg, *seg; + uint32_t k = 0U, z = 0U, len; + odp_dma_transfer_param_t *param; + + for (uint32_t i = 0U; i < sd->dma.num_inflight; ++i) { + start_src_seg = &sd->dma.src_seg[k]; + start_dst_seg = &sd->dma.dst_seg[z]; + + for (uint32_t j = 0U; j < sd->dma.num_in_segs; ++j, ++k) { + seg = &start_src_seg[j]; + seg->addr = (uint8_t *)sd->seg.src + k * sd->dma.src_seg_len; + seg->len = sd->dma.src_seg_len; + } -static void configure_address_dma_transfer_base(test_config_t *config, - odp_dma_transfer_param_t trs_params[], - uint32_t trs_lengths[]) -{ - memset(trs_lengths, 0, sizeof(*trs_lengths) * config->dma_rounds); + len = sd->dma.num_in_segs * sd->dma.src_seg_len; - for (int i = 0; i < config->num_in_seg; ++i) { - config->dma_config.src_seg[i].addr = - (uint8_t *)config->seg_config.src + i * config->seg_size; - config->dma_config.src_seg[i].len = config->seg_size; - } + for (uint32_t j = 0U; j < sd->dma.num_out_segs; ++j, ++z) { + seg = &start_dst_seg[j]; + seg->addr = (uint8_t *)sd->seg.dst + z * sd->dma.dst_seg_len; + seg->len = MIN(len, sd->dma.dst_seg_len); + len -= sd->dma.dst_seg_len; + } - config->dma_config.dst_seg.addr = config->seg_config.dst; - - for (int i = 0; i < config->dma_rounds; ++i) { - odp_dma_transfer_param_init(&trs_params[i]); - trs_params[i].src_format = ODP_DMA_FORMAT_ADDR; - trs_params[i].dst_format = ODP_DMA_FORMAT_ADDR; - trs_params[i].num_src = config->trs_grn == GRN_IND ? 1 : config->num_in_seg; - trs_params[i].num_dst = 1U; - trs_params[i].src_seg = &config->dma_config.src_seg[i]; - trs_params[i].dst_seg = &config->dma_config.dst_seg; - trs_lengths[i] = config->trs_grn == GRN_IND ? - config->dma_config.src_seg[i].len : - config->num_in_seg * config->seg_size; + param = &sd->dma.infos[i].trs_param; + odp_dma_transfer_param_init(param); + param->src_format = ODP_DMA_FORMAT_ADDR; + param->dst_format = ODP_DMA_FORMAT_ADDR; + param->num_src = sd->dma.num_in_segs; + param->num_dst = sd->dma.num_out_segs; + param->src_seg = start_src_seg; + param->dst_seg = start_dst_seg; } } -static inline void configure_address_dma_transfer_dynamic(test_config_t *config, uint32_t offset, - uint32_t len) +static void free_memory(const sd_t *sd) { - config->dma_config.dst_seg.addr = (uint8_t *)config->seg_config.dst + offset; - config->dma_config.dst_seg.len = len; + if (sd->seg.src_shm != ODP_SHM_INVALID) + (void)odp_shm_free(sd->seg.src_shm); + + if (sd->seg.dst_shm != ODP_SHM_INVALID) + (void)odp_shm_free(sd->seg.dst_shm); } -static int verify_memory_transfer(const test_config_t *config) +static void run_transfer(odp_dma_t handle, trs_info_t *info, stats_t *stats) { - if (memcmp(config->seg_config.src, config->seg_config.dst, - config->num_in_seg * config->seg_size)) { - ODPH_ERR("Error in DMA transfer, source and destination data do not match.\n"); - return -1; - } + uint64_t start_tm, end_tm, start_cc, end_cc, trs_tm, trs_cc, start_cc_diff; + odp_dma_result_t res; + int ret; - return 0; + start_tm = odp_time_local_strict_ns(); + start_cc = odp_cpu_cycles(); + ret = odp_dma_transfer(handle, &info->trs_param, &res); + end_cc = odp_cpu_cycles(); + end_tm = odp_time_local_strict_ns(); + + if (odp_unlikely(ret <= 0)) { + ++stats->start_errs; + } else { + trs_tm = end_tm - start_tm; + stats->max_trs_tm = MAX(trs_tm, stats->max_trs_tm); + stats->min_trs_tm = MIN(trs_tm, stats->min_trs_tm); + stats->trs_tm += trs_tm; + trs_cc = odp_cpu_cycles_diff(end_cc, start_cc); + stats->max_trs_cc = MAX(trs_cc, stats->max_trs_cc); + stats->min_trs_cc = MIN(trs_cc, stats->min_trs_cc); + stats->trs_cc += trs_cc; + ++stats->trs_cnt; + start_cc_diff = odp_cpu_cycles_diff(end_cc, start_cc); + stats->max_start_cc = MAX(start_cc_diff, stats->max_start_cc); + stats->min_start_cc = MIN(start_cc_diff, stats->min_start_cc); + stats->start_cc += start_cc_diff; + ++stats->start_cnt; + + if (odp_unlikely(!res.success)) + ++stats->transfer_errs; + else + ++stats->completed; + } } -static void free_memory(test_config_t *config) +static void run_transfers_mt_unsafe(sd_t *sd, stats_t *stats) { - if (config->seg_config.shm_src != ODP_SHM_INVALID) - (void)odp_shm_free(config->seg_config.shm_src); + const uint32_t count = sd->dma.num_inflight; + odp_dma_t handle = sd->dma.handle; + trs_info_t *infos = sd->dma.infos; - if (config->seg_config.shm_dst != ODP_SHM_INVALID) - (void)odp_shm_free(config->seg_config.shm_dst); + for (uint32_t i = 0U; i < count; ++i) + run_transfer(handle, &infos[i], stats); } -static void print_humanised_speed(uint64_t speed) +static void run_transfers_mt_safe(sd_t *sd, stats_t *stats) { - if (speed > GIGAS) - printf("%.2f GB/s\n", (double)speed / GIGAS); - else if (speed > MEGAS) - printf("%.2f MB/s\n", (double)speed / MEGAS); - else if (speed > KILOS) - printf("%.2f KB/s\n", (double)speed / KILOS); - else - printf("%" PRIu64 " B/s\n", speed); + const uint32_t count = sd->dma.num_inflight; + odp_dma_t handle = sd->dma.handle; + trs_info_t *infos = sd->dma.infos, *info; + + for (uint32_t i = 0U; i < count; ++i) { + info = &infos[i]; + + if (odp_ticketlock_trylock(&info->lock)) { + run_transfer(handle, info, stats); + odp_ticketlock_unlock(&info->lock); + } + } } -static void print_results(const test_config_t *config, uint64_t time, uint32_t retries) +static odp_bool_t configure_poll_compl(sd_t *sd) { - const int is_sync = config->trs_type == TRS_TYPE_SYNC; - const uint64_t avg_time = time / config->num_rounds; - uint64_t avg_speed = 0U; + odp_dma_compl_param_t *param; - printf("\n" - "=============================================\n\n" - "DMA transfer test done\n\n" - " mode: %s\n" - " granularity: %s\n" - " input segment count: %d\n" - " segment size: %u\n" - " segment type: %s\n", - is_sync ? "synchronous" : "asynchronous", - config->trs_grn == GRN_IND ? "individual" : "all", - config->num_in_seg, config->seg_size, - config->seg_type == TYPE_PKT ? "packet" : "memory"); - - if (!is_sync) { - printf(" completion modes in order: "); - - for (int i = 0; i < config->compl_modes.num_modes; ++i) - printf("%s", config->compl_modes.modes[i] == ODP_DMA_COMPL_POLL ? - "poll " : "event "); + for (uint32_t i = 0U; i < sd->dma.num_inflight; ++i) { + param = &sd->dma.infos[i].compl_param; - printf("\n"); - } + odp_dma_compl_param_init(param); + param->compl_mode = mode_map[sd->dma.compl_mode]; + param->transfer_id = odp_dma_transfer_id_alloc(sd->dma.handle); - if (avg_time > 0U) - avg_speed = config->num_in_seg * config->seg_size * ODP_TIME_SEC_IN_NS / avg_time; + if (param->transfer_id == ODP_DMA_TRANSFER_ID_INVALID) { + ODPH_ERR("Error allocating transfer ID\n"); + return false; + } + } - printf(" rounds run: %d\n" - " average time per transfer: %" PRIu64 " ns\n" - " average transfer speed: ", - config->num_rounds, avg_time); - print_humanised_speed(avg_speed); - printf(" retries with usec sleep: %u\n", retries); - printf("\n=============================================\n"); + return true; } -static int run_dma_sync(test_config_t *config) +static void poll_transfer(odp_dma_t handle, trs_info_t *info, stats_t *stats) { - odp_dma_transfer_param_t trs_params[config->dma_rounds]; - uint32_t trs_lengths[config->dma_rounds]; - odp_time_t start, end; - uint32_t num_rounds = config->num_rounds, offset, retries = 0U; - int done = 0; + uint64_t start_cc, end_cc, trs_tm, trs_cc, wait_cc, start_tm, start_cc_diff; + odp_dma_result_t res; + int ret; - config->test_case_api.trs_base_fn(config, trs_params, trs_lengths); - start = odp_time_local_strict(); + if (info->is_running) { + start_cc = odp_cpu_cycles(); + ret = odp_dma_transfer_done(handle, info->compl_param.transfer_id, &res); + end_cc = odp_cpu_cycles(); - while (num_rounds--) { - offset = 0U; + if (odp_unlikely(ret < 0)) { + ++stats->poll_errs; + return; + } - for (int i = 0; i < config->dma_rounds; ++i) { - config->test_case_api.trs_dyn_fn(config, offset, trs_lengths[i]); + ++info->trs_poll_cnt; + wait_cc = odp_cpu_cycles_diff(end_cc, start_cc); + stats->max_wait_cc = MAX(wait_cc, stats->max_wait_cc); + stats->min_wait_cc = MIN(wait_cc, stats->min_wait_cc); + stats->wait_cc += wait_cc; + ++stats->wait_cnt; + + if (ret == 0) + return; + + trs_tm = odp_time_global_strict_ns() - info->trs_start_tm; + stats->max_trs_tm = MAX(trs_tm, stats->max_trs_tm); + stats->min_trs_tm = MIN(trs_tm, stats->min_trs_tm); + stats->trs_tm += trs_tm; + trs_cc = odp_cpu_cycles_diff(odp_cpu_cycles(), info->trs_start_cc); + stats->max_trs_cc = MAX(trs_cc, stats->max_trs_cc); + stats->min_trs_cc = MIN(trs_cc, stats->min_trs_cc); + stats->trs_cc += trs_cc; + stats->trs_poll_cnt += info->trs_poll_cnt; + ++stats->trs_cnt; + + if (odp_unlikely(!res.success)) + ++stats->transfer_errs; + else + ++stats->completed; + + info->is_running = false; + } else { + start_tm = odp_time_global_strict_ns(); + start_cc = odp_cpu_cycles(); + ret = odp_dma_transfer_start(handle, &info->trs_param, &info->compl_param); + end_cc = odp_cpu_cycles(); + + if (odp_unlikely(ret <= 0)) { + ++stats->start_errs; + } else { + info->trs_start_tm = start_tm; + info->trs_start_cc = start_cc; + info->trs_poll_cnt = 0U; + start_cc_diff = odp_cpu_cycles_diff(end_cc, start_cc); + stats->max_start_cc = MAX(start_cc_diff, stats->max_start_cc); + stats->min_start_cc = MIN(start_cc_diff, stats->min_start_cc); + stats->start_cc += start_cc_diff; + ++stats->start_cnt; + info->is_running = true; + } + } +} - while (1) { - done = odp_dma_transfer(config->dma_config.handle, &trs_params[i], - NULL); +static void poll_transfers_mt_unsafe(sd_t *sd, stats_t *stats) +{ + const uint32_t count = sd->dma.num_inflight; + odp_dma_t handle = sd->dma.handle; + trs_info_t *infos = sd->dma.infos; - if (done > 0) - break; + for (uint32_t i = 0U; i < count; ++i) + poll_transfer(handle, &infos[i], stats); +} - if (done == 0 && retries++ < RETRIES) { - odp_time_wait_ns(1000U); - continue; - } +static void poll_transfers_mt_safe(sd_t *sd, stats_t *stats) +{ + const uint32_t count = sd->dma.num_inflight; + odp_dma_t handle = sd->dma.handle; + trs_info_t *infos = sd->dma.infos, *info; - ODPH_ERR("Error starting a sync DMA transfer.\n"); - return -1; - } + for (uint32_t i = 0U; i < count; ++i) { + info = &infos[i]; - offset += trs_lengths[i]; + if (odp_ticketlock_trylock(&info->lock)) { + poll_transfer(handle, info, stats); + odp_ticketlock_unlock(&info->lock); } } - - end = odp_time_local_strict(); - print_results(config, odp_time_diff_ns(end, start), retries); - return 0; } -static int configure_dma_event_completion(test_config_t *config) +static odp_bool_t configure_event_compl_session(sd_t *sd) { - int ret; + odp_thrmask_t zero; odp_dma_pool_param_t pool_param; odp_queue_param_t queue_param; - config->dma_config.pool = ODP_POOL_INVALID; - config->dma_config.compl_q = ODP_QUEUE_INVALID; - - ret = odp_schedule_config(NULL); + odp_thrmask_zero(&zero); + sd->grp = odp_schedule_group_create(PROG_NAME "_scd_grp", &zero); - if (ret < 0) { - ODPH_ERR("Error configuring scheduler.\n"); - return -1; + if (sd->grp == ODP_SCHED_GROUP_INVALID) { + ODPH_ERR("Error creating scheduler group for DMA session\n"); + return false; } odp_dma_pool_param_init(&pool_param); - pool_param.num = config->num_in_seg; - config->dma_config.pool = odp_dma_pool_create("odp_dma_perf_events", &pool_param); + pool_param.num = sd->dma.num_inflight; + sd->dma.pool = odp_dma_pool_create(PROG_NAME "_dma_evs", &pool_param); - if (config->dma_config.pool == ODP_POOL_INVALID) { - ODPH_ERR("Error creating DMA event completion pool.\n"); - return -1; + if (sd->dma.pool == ODP_POOL_INVALID) { + ODPH_ERR("Error creating DMA event completion pool\n"); + return false; } odp_queue_param_init(&queue_param); queue_param.type = ODP_QUEUE_TYPE_SCHED; queue_param.sched.sync = ODP_SCHED_SYNC_PARALLEL; queue_param.sched.prio = odp_schedule_default_prio(); - queue_param.sched.group = ODP_SCHED_GROUP_ALL; - config->dma_config.compl_q = odp_queue_create("odp_dma_perf_queue", &queue_param); + queue_param.sched.group = sd->grp; + sd->dma.compl_q = odp_queue_create(PROG_NAME, &queue_param); - if (config->dma_config.compl_q == ODP_QUEUE_INVALID) { - ODPH_ERR("Error creating DMA completion queue.\n"); - return -1; + if (sd->dma.compl_q == ODP_QUEUE_INVALID) { + ODPH_ERR("Error creating DMA completion queue\n"); + return false; } - return 0; + return true; } -static int configure_dma_completion_params(test_config_t *config, - odp_dma_compl_param_t compl_params[]) +static odp_bool_t configure_event_compl(sd_t *sd) { - odp_dma_compl_t compl_ev; + odp_dma_compl_param_t *param; + odp_dma_compl_t c_ev; - for (int i = 0; i < config->dma_rounds; ++i) - odp_dma_compl_param_init(&compl_params[i]); + for (uint32_t i = 0U; i < sd->dma.num_inflight; ++i) { + param = &sd->dma.infos[i].compl_param; - for (int i = 0; i < config->dma_rounds; ++i) { - if (config->compl_modes.modes[i] == ODP_DMA_COMPL_EVENT) { - compl_params[i].compl_mode = ODP_DMA_COMPL_EVENT; - compl_ev = odp_dma_compl_alloc(config->dma_config.pool); + odp_dma_compl_param_init(param); + param->compl_mode = mode_map[sd->dma.compl_mode]; + c_ev = odp_dma_compl_alloc(sd->dma.pool); - if (compl_ev == ODP_DMA_COMPL_INVALID) { - ODPH_ERR("Error creating DMA completion event.\n"); - return -1; - } - - compl_params[i].event = odp_dma_compl_to_event(compl_ev); - compl_params[i].queue = config->dma_config.compl_q; - } else if (config->compl_modes.modes[i] == ODP_DMA_COMPL_POLL) { - compl_params[i].compl_mode = ODP_DMA_COMPL_POLL; - compl_params[i].transfer_id = - odp_dma_transfer_id_alloc(config->dma_config.handle); - - if (compl_params[i].transfer_id == ODP_DMA_TRANSFER_ID_INVALID) { - ODPH_ERR("Error creating DMA transfer ID.\n"); - return -1; - } + if (c_ev == ODP_DMA_COMPL_INVALID) { + ODPH_ERR("Error allocating completion event\n"); + return false; } - compl_params[i].user_ptr = NULL; + param->event = odp_dma_compl_to_event(c_ev); + param->queue = sd->dma.compl_q; + param->user_ptr = &sd->dma.infos[i]; } - return 0; + return true; } -static void build_wait_list(const test_config_t *config, odp_dma_compl_param_t compl_params[], - compl_wait_entry_t list[]) +static odp_bool_t start_initial_transfers(sd_t *sd) { - int last_ev_idx, has_events = 0; + uint64_t start_tm, start_cc; + trs_info_t *info; + int ret; - memset(list, 0, sizeof(*list) * config->dma_rounds); + for (uint32_t i = 0U; i < sd->dma.num_inflight; ++i) { + info = &sd->dma.infos[i]; + start_tm = odp_time_global_strict_ns(); + start_cc = odp_cpu_cycles(); + ret = odp_dma_transfer_start(sd->dma.handle, &info->trs_param, &info->compl_param); - for (int i = 0, j = 0, k = 0; i < config->dma_rounds; ++i) { - if (config->compl_modes.modes[i] == ODP_DMA_COMPL_EVENT) { - compl_wait_entry_t entry = { .type = ODP_DMA_COMPL_EVENT }; + if (ret <= 0) { + ODPH_ERR("Error starting DMA transfer\n"); + return false; + } - list[j] = entry; - ++j; + info->trs_start_tm = start_tm; + info->trs_start_cc = start_cc; + } - for (; k < i; ++k) { - entry.type = ODP_DMA_COMPL_POLL; - entry.id = compl_params[k].transfer_id; - list[j++] = entry; - } + return true; +} - ++k; - last_ev_idx = i; - has_events = 1; - } - } +static void wait_compl_event(sd_t *sd, stats_t *stats) +{ + uint64_t start_cc, end_cc, wait_cc, trs_tm, trs_cc, start_tm, start_cc_diff; + odp_event_t ev; + odp_dma_result_t res; + trs_info_t *info; + int ret; - last_ev_idx = has_events ? last_ev_idx + 1 : 0; + start_cc = odp_cpu_cycles(); + ev = odp_schedule(NULL, odp_schedule_wait_time(ODP_TIME_SEC_IN_NS)); + end_cc = odp_cpu_cycles(); - for (int i = last_ev_idx; i < config->dma_rounds; ++i) { - compl_wait_entry_t entry = { .type = ODP_DMA_COMPL_POLL, - .id = compl_params[i].transfer_id }; - list[i] = entry; + if (odp_unlikely(ev == ODP_EVENT_INVALID)) { + ++stats->scheduler_timeouts; + return; + } + + odp_dma_compl_result(odp_dma_compl_from_event(ev), &res); + info = res.user_ptr; + trs_tm = odp_time_global_strict_ns() - info->trs_start_tm; + stats->max_trs_tm = MAX(trs_tm, stats->max_trs_tm); + stats->min_trs_tm = MIN(trs_tm, stats->min_trs_tm); + stats->trs_tm += trs_tm; + trs_cc = odp_cpu_cycles_diff(odp_cpu_cycles(), info->trs_start_cc); + stats->max_trs_cc = MAX(trs_cc, stats->max_trs_cc); + stats->min_trs_cc = MIN(trs_cc, stats->min_trs_cc); + stats->trs_cc += trs_cc; + ++stats->trs_cnt; + wait_cc = odp_cpu_cycles_diff(end_cc, start_cc); + stats->max_wait_cc = MAX(wait_cc, stats->max_wait_cc); + stats->min_wait_cc = MIN(wait_cc, stats->min_wait_cc); + stats->wait_cc += wait_cc; + ++stats->wait_cnt; + + if (odp_unlikely(!res.success)) + ++stats->transfer_errs; + else + ++stats->completed; + + start_tm = odp_time_global_strict_ns(); + start_cc = odp_cpu_cycles(); + ret = odp_dma_transfer_start(sd->dma.handle, &info->trs_param, &info->compl_param); + end_cc = odp_cpu_cycles(); + + if (odp_unlikely(ret <= 0)) { + ++stats->start_errs; + } else { + info->trs_start_tm = start_tm; + info->trs_start_cc = start_cc; + start_cc_diff = odp_cpu_cycles_diff(end_cc, start_cc); + stats->max_start_cc = MAX(start_cc_diff, stats->max_start_cc); + stats->min_start_cc = MIN(start_cc_diff, stats->min_start_cc); + stats->start_cc += start_cc_diff; + ++stats->start_cnt; } } -static inline int wait_dma_transfers_ready(test_config_t *config, compl_wait_entry_t list[]) +static void drain_compl_events(void) { odp_event_t ev; - const uint64_t wait_time = odp_schedule_wait_time(config->wait_ns); - uint64_t start, end; - int done = 0; - - for (int i = 0; i < config->dma_rounds; ++i) { - if (list[i].type == ODP_DMA_COMPL_EVENT) { - ev = odp_schedule(NULL, wait_time); - - if (ev == ODP_EVENT_INVALID) { - ODPH_ERR("Error waiting event completion.\n"); - return -1; - } - } else { - start = odp_time_local_ns(); - end = start + ODP_TIME_SEC_IN_NS; - while (1) { - done = odp_dma_transfer_done(config->dma_config.handle, list[i].id, - NULL); + while (true) { + ev = odp_schedule(NULL, odp_schedule_wait_time(ODP_TIME_SEC_IN_NS)); - if (done > 0) - break; + if (ev == ODP_EVENT_INVALID) + break; + } +} - if (done == 0 && odp_time_local_ns() < end) - continue; +static void setup_api(prog_config_t *config) +{ + if (config->seg_type == PACKET) { + config->api.setup_fn = setup_packet_segments; + config->api.trs_fn = configure_packet_dma_transfer; + config->api.free_fn = free_packets; + } else { + config->api.setup_fn = setup_memory_segments; + config->api.trs_fn = configure_address_dma_transfer; + config->api.free_fn = free_memory; + } - ODPH_ERR("Error waiting poll completion.\n"); - return -1; - } + if (config->trs_type == SYNC) { + config->api.compl_fn = NULL; + config->api.wait_fn = config->num_workers == 1 || config->policy == MANY ? + run_transfers_mt_unsafe : run_transfers_mt_safe; + config->api.drain_fn = NULL; + } else { + if (config->compl_mode == POLL) { + config->api.session_cfg_fn = NULL; + config->api.compl_fn = configure_poll_compl; + config->api.bootstrap_fn = NULL; + config->api.wait_fn = config->num_workers == 1 || config->policy == MANY ? + poll_transfers_mt_unsafe : poll_transfers_mt_safe; + config->api.drain_fn = NULL; + } else { + config->api.session_cfg_fn = configure_event_compl_session; + config->api.compl_fn = configure_event_compl; + config->api.bootstrap_fn = start_initial_transfers; + config->api.wait_fn = wait_compl_event; + config->api.drain_fn = drain_compl_events; } } - - return 0; } -static void free_dma_completion_events(test_config_t *config, odp_dma_compl_param_t compl_params[]) +static odp_bool_t setup_session_descriptors(prog_config_t *config) { - for (int i = 0; i < config->dma_rounds; ++i) - if (config->compl_modes.modes[i] == ODP_DMA_COMPL_EVENT && - compl_params[i].event != ODP_EVENT_INVALID) - odp_dma_compl_free(odp_dma_compl_from_event(compl_params[i].event)); + sd_t *sd; + const odp_dma_param_t dma_params = { + .direction = ODP_DMA_MAIN_TO_MAIN, + .type = ODP_DMA_TYPE_COPY, + .compl_mode_mask = config->compl_mode_mask, + .mt_mode = config->num_workers == 1 || config->policy == MANY ? + ODP_DMA_MT_SERIAL : ODP_DMA_MT_SAFE, + .order = ODP_DMA_ORDER_NONE }; + + for (uint32_t i = 0U; i < config->num_sessions; ++i) { + char name[ODP_DMA_NAME_LEN]; + + sd = &config->sds[i]; + sd->dma.num_in_segs = config->num_in_segs; + sd->dma.num_out_segs = config->num_out_segs; + sd->dma.src_seg_len = config->src_seg_len; + sd->dma.dst_seg_len = config->dst_seg_len; + sd->dma.num_inflight = config->num_inflight; + sd->dma.trs_type = config->trs_type; + sd->dma.compl_mode = config->compl_mode; + snprintf(name, sizeof(name), PROG_NAME "_dma_%u", i); + sd->dma.handle = odp_dma_create(name, &dma_params); + + if (sd->dma.handle == ODP_DMA_INVALID) { + ODPH_ERR("Error creating DMA session\n"); + return false; + } + + if (config->api.session_cfg_fn != NULL && !config->api.session_cfg_fn(sd)) + return false; + } + + return true; } -static void free_dma_transfer_ids(test_config_t *config, odp_dma_compl_param_t compl_params[]) +static odp_bool_t setup_data(prog_config_t *config) { - for (int i = 0; i < config->dma_rounds; ++i) - if (config->compl_modes.modes[i] == ODP_DMA_COMPL_POLL && - compl_params[i].transfer_id != ODP_DMA_TRANSFER_ID_INVALID) - odp_dma_transfer_id_free(config->dma_config.handle, - compl_params[i].transfer_id); + sd_t *sd; + + for (uint32_t i = 0U; i < config->num_sessions; ++i) { + sd = &config->sds[i]; + + if (!config->api.setup_fn(sd)) + return false; + + config->api.trs_fn(sd); + + if (config->api.compl_fn != NULL && !config->api.compl_fn(sd)) + return false; + } + + return true; } -static int run_dma_async_transfer(test_config_t *config) +static int transfer(void *args) { - odp_dma_transfer_param_t trs_params[config->dma_rounds]; - uint32_t trs_lengths[config->dma_rounds]; - odp_dma_compl_param_t compl_params[config->dma_rounds]; - int ret = 0, started; - compl_wait_entry_t compl_wait_list[config->dma_rounds]; - odp_time_t start, end; - uint32_t num_rounds = config->num_rounds, offset, retries = 0U; - - config->test_case_api.trs_base_fn(config, trs_params, trs_lengths); - - if (configure_dma_completion_params(config, compl_params)) { - ret = -1; - goto out_compl_evs; + thread_config_t *thr_config = args; + prog_config_t *prog_config = thr_config->prog_config; + sd_t *sd = thr_config->sd; + stats_t *stats = &thr_config->stats; + test_api_t *api = &prog_conf->api; + odp_thrmask_t mask; + uint64_t start_tm, end_tm; + + odp_barrier_wait(&prog_config->init_barrier); + + if (sd->grp != ODP_SCHED_GROUP_INVALID) { + odp_thrmask_zero(&mask); + odp_thrmask_set(&mask, odp_thread_id()); + + if (odp_schedule_group_join(sd->grp, &mask) < 0) { + ODPH_ERR("Error joining scheduler group\n"); + goto out; + } } - build_wait_list(config, compl_params, compl_wait_list); - start = odp_time_local_strict(); + start_tm = odp_time_local_strict_ns(); - while (num_rounds--) { - offset = 0U; + while (odp_atomic_load_u32(&prog_config->is_running)) + api->wait_fn(sd, stats); - for (int i = 0; i < config->dma_rounds; ++i) { - config->test_case_api.trs_dyn_fn(config, offset, trs_lengths[i]); + end_tm = odp_time_local_strict_ns(); + thr_config->stats.tot_tm = end_tm - start_tm; - while (1) { - started = odp_dma_transfer_start(config->dma_config.handle, - &trs_params[i], &compl_params[i]); + if (api->drain_fn != NULL) + api->drain_fn(); - if (started > 0) - break; +out: + odp_barrier_wait(&prog_config->term_barrier); - if (started == 0 && retries++ < RETRIES) { - odp_time_wait_ns(1000U); - continue; - } + return 0; +} - ODPH_ERR("Error starting an async DMA transfer.\n"); - ret = -1; - goto out_trs_ids; - } +static odp_bool_t setup_workers(prog_config_t *config) +{ + odp_cpumask_t cpumask; + int num_workers; + odph_thread_common_param_t thr_common; + odph_thread_param_t thr_params[config->num_workers], *thr_param; + thread_config_t *thr_config; + sd_t *sd; + + /* Barrier init count for control and worker. */ + odp_barrier_init(&config->init_barrier, config->num_workers + 1); + odp_barrier_init(&config->term_barrier, config->num_workers); + num_workers = odp_cpumask_default_worker(&cpumask, config->num_workers); + odph_thread_common_param_init(&thr_common); + thr_common.instance = config->odp_instance; + thr_common.cpumask = &cpumask; + + for (int i = 0; i < config->num_workers; ++i) { + thr_param = &thr_params[i]; + thr_config = &config->thread_config[i]; + sd = config->policy == SINGLE ? &config->sds[0U] : &config->sds[i]; + + odph_thread_param_init(thr_param); + thr_param->start = transfer; + thr_param->thr_type = ODP_THREAD_WORKER; + thr_config->prog_config = config; + thr_config->sd = sd; + thr_param->arg = thr_config; + } - offset += trs_lengths[i]; - } + num_workers = odph_thread_create(config->threads, &thr_common, thr_params, num_workers); - if (wait_dma_transfers_ready(config, compl_wait_list)) { - ODPH_ERR("Error finishing an async DMA transfer.\n"); - ret = -1; - goto out_trs_ids; - } + if (num_workers != config->num_workers) { + ODPH_ERR("Error configuring worker threads\n"); + return false; } - end = odp_time_local_strict(); - print_results(config, odp_time_diff_ns(end, start), retries); + for (uint32_t i = 0U; i < config->num_sessions; ++i) { + if (config->api.bootstrap_fn != NULL && !config->api.bootstrap_fn(&config->sds[i])) + return false; + } -out_compl_evs: - free_dma_completion_events(config, compl_params); + odp_barrier_wait(&config->init_barrier); -out_trs_ids: - free_dma_transfer_ids(config, compl_params); - return ret; + return true; } -static void free_dma_event_completion(test_config_t *config) +static odp_bool_t setup_test(prog_config_t *config) { - if (config->dma_config.compl_q != ODP_QUEUE_INVALID) - (void)odp_queue_destroy(config->dma_config.compl_q); + setup_api(config); - if (config->dma_config.pool != ODP_POOL_INVALID) - (void)odp_pool_destroy(config->dma_config.pool); + return setup_session_descriptors(config) && setup_data(config) && setup_workers(config); } -static int run_dma_async(test_config_t *config) +static void stop_test(prog_config_t *config) { - const int is_event_compl = config->compl_modes.compl_mask & ODP_DMA_COMPL_EVENT; - int ret = 0; - - if (is_event_compl) - if (configure_dma_event_completion(config)) { - ret = -1; - goto out; - } + (void)odph_thread_join(config->threads, config->num_workers); +} - if (run_dma_async_transfer(config)) - ret = -1; +static void teardown_data(const sd_t *sd, void (*free_fn)(const sd_t *sd)) +{ + const odp_dma_compl_param_t *compl_param; -out: - if (is_event_compl) - free_dma_event_completion(config); + for (uint32_t i = 0U; i < MAX_SEGS; ++i) { + compl_param = &sd->dma.infos[i].compl_param; - return ret; -} + if (compl_param->transfer_id != ODP_DMA_TRANSFER_ID_INVALID) + odp_dma_transfer_id_free(sd->dma.handle, compl_param->transfer_id); -static void setup_test_case_api(test_config_t *config) -{ - switch (config->seg_type) { - case TYPE_PKT: - config->test_case_api.setup_fn = setup_packet_segments; - config->test_case_api.trs_base_fn = configure_packet_dma_transfer_base; - config->test_case_api.trs_dyn_fn = configure_packet_dma_transfer_dynamic; - config->test_case_api.verify_fn = verify_packet_transfer; - config->test_case_api.free_fn = free_packets; - break; - case TYPE_MEM: - config->test_case_api.setup_fn = setup_memory_segments; - config->test_case_api.trs_base_fn = configure_address_dma_transfer_base; - config->test_case_api.trs_dyn_fn = configure_address_dma_transfer_dynamic; - config->test_case_api.verify_fn = verify_memory_transfer; - config->test_case_api.free_fn = free_memory; - break; - default: - break; + if (compl_param->event != ODP_EVENT_INVALID) + odp_event_free(compl_param->event); } - config->test_case_api.run_fn = config->trs_type == TRS_TYPE_SYNC ? - run_dma_sync : - run_dma_async; + free_fn(sd); } -static int configure_dma_session(test_config_t *config) +static void teardown_test(prog_config_t *config) { - const odp_dma_param_t params = { .direction = ODP_DMA_MAIN_TO_MAIN, - .type = ODP_DMA_TYPE_COPY, - .compl_mode_mask = config->compl_modes.compl_mask, - .mt_mode = ODP_DMA_MT_SERIAL, - .order = ODP_DMA_ORDER_NONE }; + sd_t *sd; + + for (uint32_t i = 0U; i < config->num_sessions; ++i) { + sd = &config->sds[i]; + teardown_data(sd, config->api.free_fn); - config->dma_config.handle = odp_dma_create("odp_dma_perf", ¶ms); + if (sd->dma.compl_q != ODP_QUEUE_INVALID) + (void)odp_queue_destroy(sd->dma.compl_q); - if (config->dma_config.handle == ODP_DMA_INVALID) { - ODPH_ERR("Error creating DMA session.\n"); - return -1; + if (sd->dma.pool != ODP_POOL_INVALID) + (void)odp_pool_destroy(sd->dma.pool); + + if (sd->grp != ODP_SCHED_GROUP_INVALID) + (void)odp_schedule_group_destroy(sd->grp); + + if (sd->dma.handle != ODP_DMA_INVALID) + (void)odp_dma_destroy(sd->dma.handle); } - return 0; + if (config->src_pool != ODP_POOL_INVALID) + (void)odp_pool_destroy(config->src_pool); + + if (config->dst_pool != ODP_POOL_INVALID) + (void)odp_pool_destroy(config->dst_pool); } -static void free_dma_session(test_config_t *config) +static void print_humanised(uint64_t value, const char *type) { - if (config->dma_config.handle != ODP_DMA_INVALID) - (void)odp_dma_destroy(config->dma_config.handle); + if (value > GIGAS) + printf("%.2f G%s\n", (double)value / GIGAS, type); + else if (value > MEGAS) + printf("%.2f M%s\n", (double)value / MEGAS, type); + else if (value > KILOS) + printf("%.2f K%s\n", (double)value / KILOS, type); + else + printf("%" PRIu64 " %s\n", value, type); +} + +static void print_stats(const prog_config_t *config) +{ + const stats_t *stats; + uint64_t data_cnt = config->num_in_segs * config->src_seg_len, tot_completed = 0U, + tot_tm = 0U, tot_trs_tm = 0U, tot_trs_cc = 0U, tot_trs_cnt = 0U, tot_min_tm = UINT64_MAX, + tot_max_tm = 0U, tot_min_cc = UINT64_MAX, tot_max_cc = 0U, avg_start_cc, avg_wait_cc, + avg_tot_tm; + + printf("\n======================\n\n" + "DMA performance test done\n\n" + " mode: %s\n" + " input segment count: %u\n" + " output segment count: %u\n" + " segment length: %u\n" + " segment type: %s\n" + " inflight count: %u\n" + " session policy: %s\n\n", + config->trs_type == SYNC ? "synchronous" : config->compl_mode == POLL ? + "asynchronous-poll" : "asynchronous-event", config->num_in_segs, + config->num_out_segs, config->src_seg_len, + config->seg_type == PACKET ? "packet" : "memory", config->num_inflight, + config->policy == SINGLE ? "shared" : "per-worker"); + + for (int i = 0; i < config->num_workers; ++i) { + stats = &config->thread_config[i].stats; + tot_completed += stats->completed; + tot_tm += stats->tot_tm; + tot_trs_tm += stats->trs_tm; + tot_trs_cc += stats->trs_cc; + tot_trs_cnt += stats->trs_cnt; + tot_min_tm = MIN(tot_min_tm, stats->min_trs_tm); + tot_max_tm = MAX(tot_max_tm, stats->max_trs_tm); + tot_min_cc = MIN(tot_min_cc, stats->min_trs_cc); + tot_max_cc = MAX(tot_max_cc, stats->max_trs_cc); + + printf(" worker %d:\n", i); + printf(" successful transfers: %" PRIu64 "\n" + " start errors: %" PRIu64 "\n", + stats->completed, stats->start_errs); + + if (config->trs_type == ASYNC) { + if (config->compl_mode == POLL) + printf(" poll errors: %" PRIu64 "\n", + stats->poll_errs); + else + printf(" scheduler timeouts: %" PRIu64 "\n", + stats->scheduler_timeouts); + } + + printf(" transfer errors: %" PRIu64 "\n" + " run time: %" PRIu64 " ns\n", + stats->transfer_errs, stats->tot_tm); + + if (config->policy == MANY) { + printf(" DMA session:\n" + " average time per transfer: %" PRIu64 " " + "(min: %" PRIu64 ", max: %" PRIu64 ") ns\n" + " average cycles per transfer: %" PRIu64 " " + "(min: %" PRIu64 ", max: %" PRIu64 ")\n" + " ops: ", + stats->trs_cnt > 0U ? stats->trs_tm / stats->trs_cnt : 0U, + stats->trs_cnt > 0U ? stats->min_trs_tm : 0U, + stats->trs_cnt > 0U ? stats->max_trs_tm : 0U, + stats->trs_cnt > 0U ? stats->trs_cc / stats->trs_cnt : 0U, + stats->trs_cnt > 0U ? stats->min_trs_cc : 0U, + stats->trs_cnt > 0U ? stats->max_trs_cc : 0U); + print_humanised(stats->completed / (stats->tot_tm / ODP_TIME_SEC_IN_NS), + "OPS"); + printf(" speed: "); + print_humanised(stats->completed * data_cnt / + (stats->tot_tm / ODP_TIME_SEC_IN_NS), "B/s"); + } + + avg_start_cc = stats->start_cnt > 0U ? stats->start_cc / stats->start_cnt : 0U; + printf(" average cycles breakdown:\n"); + + if (config->trs_type == SYNC) { + printf(" odp_dma_transfer(): %" PRIu64 " " + "(min: %" PRIu64 ", max: %" PRIu64 ")\n", avg_start_cc, + avg_start_cc > 0U ? stats->min_start_cc : 0U, + avg_start_cc > 0U ? stats->max_start_cc : 0U); + } else { + printf(" odp_dma_transfer_start(): %" PRIu64 " " + "(min: %" PRIu64 ", max: %" PRIu64 ")\n", avg_start_cc, + avg_start_cc > 0U ? stats->min_start_cc : 0U, + avg_start_cc > 0U ? stats->max_start_cc : 0U); + + avg_wait_cc = stats->wait_cnt > 0U ? stats->wait_cc / stats->wait_cnt : 0U; + + if (config->compl_mode == POLL) { + printf(" odp_dma_transfer_done(): %" PRIu64 "" + " (min: %" PRIu64 ", max: %" PRIu64 ", x %" PRIu64 "" + " per transfer)\n", avg_wait_cc, + avg_wait_cc > 0U ? stats->min_wait_cc : 0U, + avg_wait_cc > 0U ? stats->max_wait_cc : 0U, + stats->trs_cnt > 0U ? + stats->trs_poll_cnt / stats->trs_cnt : 0U); + } else { + printf(" odp_schedule(): %" PRIu64 " " + " (min: %" PRIu64 ", max: %" PRIu64 ")\n", avg_wait_cc, + avg_wait_cc > 0U ? stats->min_wait_cc : 0U, + avg_wait_cc > 0U ? stats->max_wait_cc : 0U); + } + } + + printf("\n"); + } + + avg_tot_tm = tot_tm / config->num_workers / ODP_TIME_SEC_IN_NS; + printf(" total:\n" + " average time per transfer: %" PRIu64 " (min: %" PRIu64 + ", max: %" PRIu64 ") ns\n" + " average cycles per transfer: %" PRIu64 " (min: %" PRIu64 + ", max: %" PRIu64 ")\n" + " ops: ", + tot_trs_cnt > 0U ? tot_trs_tm / tot_trs_cnt : 0U, + tot_trs_cnt > 0U ? tot_min_tm : 0U, + tot_trs_cnt > 0U ? tot_max_tm : 0U, + tot_trs_cnt > 0U ? tot_trs_cc / tot_trs_cnt : 0U, + tot_trs_cnt > 0U ? tot_min_cc : 0U, + tot_trs_cnt > 0U ? tot_max_cc : 0U); + print_humanised(avg_tot_tm > 0U ? tot_completed / avg_tot_tm : 0U, "OPS"); + printf(" speed: "); + print_humanised(avg_tot_tm > 0U ? tot_completed * data_cnt / avg_tot_tm : 0U, "B/s"); + printf("\n"); + printf("======================\n"); } int main(int argc, char **argv) { odph_helper_options_t odph_opts; - test_config_t test_config; + odp_init_t init_param; odp_instance_t odp_instance; + odp_shm_t shm_cfg = ODP_SHM_INVALID; + parse_result_t parse_res; int ret = EXIT_SUCCESS; argc = odph_parse_options(argc, argv); if (odph_options(&odph_opts)) { - ODPH_ERR("Error while reading ODP helper options, exiting.\n"); + ODPH_ERR("Error while reading ODP helper options, exiting\n"); exit(EXIT_FAILURE); } - if (parse_options(argc, argv, &test_config)) - exit(EXIT_FAILURE); + odp_init_param_init(&init_param); + init_param.mem_model = odph_opts.mem_model; - if (odp_init_global(&odp_instance, NULL, NULL)) { - ODPH_ERR("ODP global init failed, exiting.\n"); + if (odp_init_global(&odp_instance, &init_param, NULL)) { + ODPH_ERR("ODP global init failed, exiting\n"); exit(EXIT_FAILURE); } if (odp_init_local(odp_instance, ODP_THREAD_CONTROL)) { - ODPH_ERR("ODP local init failed, exiting.\n"); + ODPH_ERR("ODP local init failed, exiting\n"); exit(EXIT_FAILURE); } - if (check_capabilities(&test_config)) { - ODPH_ERR("Unsupported scenario attempted, exiting.\n"); - ret = EXIT_NOT_SUP; - goto out_odp; + shm_cfg = odp_shm_reserve(PROG_NAME "_cfg", sizeof(prog_config_t), ODP_CACHE_LINE_SIZE, + 0U); + + if (shm_cfg == ODP_SHM_INVALID) { + ODPH_ERR("Error reserving shared memory\n"); + ret = EXIT_FAILURE; + goto out; } - setup_test_case_api(&test_config); + prog_conf = odp_shm_addr(shm_cfg); - if (configure_dma_session(&test_config)) { + if (prog_conf == NULL) { + ODPH_ERR("Error resolving shared memory address\n"); ret = EXIT_FAILURE; - goto out_dma; + goto out; } - if (test_config.test_case_api.setup_fn(&test_config)) { + parse_res = setup_program(argc, argv, prog_conf); + + if (parse_res == PRS_NOK) { ret = EXIT_FAILURE; - goto out_test_case; + goto out; + } + + if (parse_res == PRS_TERM) { + ret = EXIT_SUCCESS; + goto out; } - if (test_config.test_case_api.run_fn(&test_config) || - test_config.test_case_api.verify_fn(&test_config)) + if (parse_res == PRS_NOT_SUP) { + ret = EXIT_NOT_SUP; + goto out; + } + + if (odp_schedule_config(NULL) < 0) { + ODPH_ERR("Error configuring scheduler\n"); ret = EXIT_FAILURE; + goto out; + } -out_test_case: - test_config.test_case_api.free_fn(&test_config); + prog_conf->odp_instance = odp_instance; + odp_atomic_init_u32(&prog_conf->is_running, 1U); -out_dma: - free_dma_session(&test_config); + if (!setup_test(prog_conf)) { + ret = EXIT_FAILURE; + goto out_test; + } + + if (prog_conf->time_sec) { + sleep(prog_conf->time_sec); + odp_atomic_store_u32(&prog_conf->is_running, 0U); + } + + stop_test(prog_conf); + print_stats(prog_conf); + +out_test: + /* Release all resources that have been allocated during 'setup_test()'. */ + teardown_test(prog_conf); + +out: + if (shm_cfg != ODP_SHM_INVALID) + (void)odp_shm_free(shm_cfg); -out_odp: if (odp_term_local()) { - ODPH_ERR("ODP local terminate failed, exiting.\n"); + ODPH_ERR("ODP local terminate failed, exiting\n"); exit(EXIT_FAILURE); } if (odp_term_global(odp_instance)) { - ODPH_ERR("ODP global terminate failed, exiting.\n"); + ODPH_ERR("ODP global terminate failed, exiting\n"); exit(EXIT_FAILURE); } diff --git a/test/performance/odp_dma_perf_run.sh b/test/performance/odp_dma_perf_run.sh index dc314d400..37bc4382f 100755 --- a/test/performance/odp_dma_perf_run.sh +++ b/test/performance/odp_dma_perf_run.sh @@ -1,14 +1,16 @@ #!/bin/sh # -# Copyright (c) 2022, Nokia +# Copyright (c) 2022-2023, Nokia # All rights reserved. # # SPDX-License-Identifier: BSD-3-Clause TEST_DIR="${TEST_DIR:-$(dirname $0)}" BIN_NAME=odp_dma_perf -SEG_SIZE=1024 -ROUNDS=1000 +SEGC=0 +SEGS=1024 +INFL=1 +TIME=1 TESTS_RUN=0 check_result() @@ -23,73 +25,24 @@ check_result() fi } -echo "odp_dma_perf: synchronous transfer 1" +echo "odp_dma_perf: synchronous transfer" echo "====================================" -${TEST_DIR}/${BIN_NAME}${EXEEXT} -t 0 -g 0 -i 6 -s $SEG_SIZE -T 0 -r $ROUNDS - -check_result $? - -echo "odp_dma_perf: synchronous transfer 2" -echo "====================================" - -${TEST_DIR}/${BIN_NAME}${EXEEXT} -t 0 -g 1 -i 6 -s $SEG_SIZE -T 0 -r $ROUNDS - -check_result $? - -echo "odp_dma_perf: synchronous transfer 3" -echo "====================================" - -${TEST_DIR}/${BIN_NAME}${EXEEXT} -t 0 -g 0 -i 6 -s $SEG_SIZE -T 1 -r $ROUNDS - -check_result $? - -echo "odp_dma_perf: synchronous transfer 4" -echo "====================================" - -${TEST_DIR}/${BIN_NAME}${EXEEXT} -t 0 -g 1 -i 6 -s $SEG_SIZE -T 1 -r $ROUNDS +${TEST_DIR}/${BIN_NAME}${EXEEXT} -t 0 -i $SEGC -o $SEGC -s $SEGS -S 0 -f $INFL -T $TIME check_result $? echo "odp_dma_perf: asynchronous transfer 1" echo "=====================================" -${TEST_DIR}/${BIN_NAME}${EXEEXT} -t 1 -g 0 -i 6 -s $SEG_SIZE -T 0 -m 0 -r $ROUNDS +${TEST_DIR}/${BIN_NAME}${EXEEXT} -t 1 -i $SEGC -o $SEGC -s $SEGS -S 1 -m 0 -f $INFL -T $TIME check_result $? echo "odp_dma_perf: asynchronous transfer 2" echo "=====================================" -${TEST_DIR}/${BIN_NAME}${EXEEXT} -t 1 -g 1 -i 6 -s $SEG_SIZE -T 0 -m 0,0,0,0,0,0 -r $ROUNDS - -check_result $? - -echo "odp_dma_perf: asynchronous transfer 3" -echo "=====================================" - -${TEST_DIR}/${BIN_NAME}${EXEEXT} -t 1 -g 1 -i 6 -s $SEG_SIZE -T 0 -m 0,0,0,0,0,1 -r $ROUNDS - -check_result $? - -echo "odp_dma_perf: asynchronous transfer 4" -echo "=====================================" - -${TEST_DIR}/${BIN_NAME}${EXEEXT} -t 1 -g 0 -i 6 -s $SEG_SIZE -T 1 -m 0 -r $ROUNDS - -check_result $? - -echo "odp_dma_perf: asynchronous transfer 5" -echo "=====================================" - -${TEST_DIR}/${BIN_NAME}${EXEEXT} -t 1 -g 1 -i 6 -s $SEG_SIZE -T 1 -m 0,0,0,0,0,0 -r $ROUNDS - -check_result $? - -echo "odp_dma_perf: asynchronous transfer 6" -echo "=====================================" - -${TEST_DIR}/${BIN_NAME}${EXEEXT} -t 1 -g 1 -i 6 -s $SEG_SIZE -T 1 -m 0,0,0,0,0,1 -r $ROUNDS +${TEST_DIR}/${BIN_NAME}${EXEEXT} -t 1 -i $SEGC -o $SEGC -s $SEGS -S 1 -m 1 -f $INFL -T $TIME check_result $? diff --git a/test/performance/odp_ipsec.c b/test/performance/odp_ipsec.c index 50e26b1c9..677e7762f 100644 --- a/test/performance/odp_ipsec.c +++ b/test/performance/odp_ipsec.c @@ -930,7 +930,7 @@ run_measure_one_config(ipsec_args_t *cargs, return -1; } - rc = odph_ipsec_alg_check(capa, config->crypto.cipher_alg, + rc = odph_ipsec_alg_check(&capa, config->crypto.cipher_alg, config->crypto.cipher_key.length, config->crypto.auth_alg, config->crypto.auth_key.length); diff --git a/test/performance/odp_ipsecfwd.c b/test/performance/odp_ipsecfwd.c index a6df747f3..16c745afa 100644 --- a/test/performance/odp_ipsecfwd.c +++ b/test/performance/odp_ipsecfwd.c @@ -135,6 +135,7 @@ typedef struct prog_config_s { fwd_entry_t fwd_entries[MAX_FWDS]; odp_queue_t sa_qs[MAX_SA_QUEUES]; pktio_t pktios[MAX_IFS]; + odp_atomic_u32_t is_running; sa_config_t default_cfg; ops_t ops; char *conf_file; @@ -181,7 +182,7 @@ typedef struct { uint8_t q_idx; } pkt_ifs_t; -static exposed_alg_t exposed_algs[] = { +static const exposed_alg_t exposed_algs[] = { ALG_ENTRY(ODP_CIPHER_ALG_NULL, CIPHER_TYPE), ALG_ENTRY(ODP_CIPHER_ALG_DES, CIPHER_TYPE), ALG_ENTRY(ODP_CIPHER_ALG_3DES_CBC, CIPHER_TYPE), @@ -208,9 +209,9 @@ static exposed_alg_t exposed_algs[] = { /* SPIs for in and out directions */ static odp_ipsec_sa_t *spi_to_sa_map[2U][MAX_SPIS]; -static odp_atomic_u32_t is_running; static const int ipsec_out_mark; static __thread pkt_ifs_t ifs; +static prog_config_t *prog_conf; static void init_config(prog_config_t *config) { @@ -226,7 +227,7 @@ static void init_config(prog_config_t *config) static void terminate(int signal ODP_UNUSED) { - odp_atomic_store_u32(&is_running, 0U); + odp_atomic_store_u32(&prog_conf->is_running, 0U); } static void parse_interfaces(prog_config_t *config, const char *optarg) @@ -1109,9 +1110,10 @@ static void parse_inbound(config_setting_t *cfg, sa_config_t *config) config->sa_param.inbound.lookup_mode = val; if (config_setting_lookup_string(cs, "lookup_dst_addr", &val_str) == CONFIG_TRUE) { - odph_ipv4_addr_parse(&config->lkp_dst_ip, val_str); - config->lkp_dst_ip = odp_cpu_to_be_32(config->lkp_dst_ip); - config->sa_param.inbound.lookup_param.dst_addr = &config->lkp_dst_ip; + if (odph_ipv4_addr_parse(&config->lkp_dst_ip, val_str) == 0) { + config->lkp_dst_ip = odp_cpu_to_be_32(config->lkp_dst_ip); + config->sa_param.inbound.lookup_param.dst_addr = &config->lkp_dst_ip; + } } if (config_setting_lookup_int(cs, "antireplay_ws", &val) == CONFIG_TRUE) @@ -1134,15 +1136,17 @@ static void parse_outbound(config_setting_t *cfg, sa_config_t *config) if (tunnel != NULL) { if (config_setting_lookup_string(tunnel, "src_addr", &val_str) == CONFIG_TRUE) { - odph_ipv4_addr_parse(&config->src_ip, val_str); - config->src_ip = odp_cpu_to_be_32(config->src_ip); - config->sa_param.outbound.tunnel.ipv4.src_addr = &config->src_ip; + if (odph_ipv4_addr_parse(&config->src_ip, val_str) == 0) { + config->src_ip = odp_cpu_to_be_32(config->src_ip); + config->sa_param.outbound.tunnel.ipv4.src_addr = &config->src_ip; + } } if (config_setting_lookup_string(tunnel, "dst_addr", &val_str) == CONFIG_TRUE) { - odph_ipv4_addr_parse(&config->dst_ip, val_str); - config->dst_ip = odp_cpu_to_be_32(config->dst_ip); - config->sa_param.outbound.tunnel.ipv4.dst_addr = &config->dst_ip; + if (odph_ipv4_addr_parse(&config->dst_ip, val_str) == 0) { + config->dst_ip = odp_cpu_to_be_32(config->dst_ip); + config->sa_param.outbound.tunnel.ipv4.dst_addr = &config->dst_ip; + } } if (config_setting_lookup_int(tunnel, "dscp", &val) == CONFIG_TRUE) @@ -1744,6 +1748,7 @@ static int process_packets(void *args) int thr_idx = odp_thread_id(); odp_event_t evs[MAX_BURST], ev; ops_t ops = config->prog_config->ops; + odp_atomic_u32_t *is_running = &config->prog_config->is_running; uint32_t cnt; odp_event_type_t type; odp_event_subtype_t subtype; @@ -1756,7 +1761,7 @@ static int process_packets(void *args) config->thr_idx = thr_idx; odp_barrier_wait(&config->prog_config->init_barrier); - while (odp_atomic_load_u32(&is_running)) { + while (odp_atomic_load_u32(is_running)) { int num_pkts_in = 0, num_pkts_ips = 0; /* TODO: Add possibility to configure scheduler and ipsec enq/deq burst sizes. */ cnt = ops.rx(config, evs, MAX_BURST); @@ -1943,12 +1948,24 @@ static void print_stats(const prog_config_t *config) int main(int argc, char **argv) { + odph_helper_options_t odph_opts; + odp_init_t init_param; odp_instance_t odp_instance; + odp_shm_t shm_cfg = ODP_SHM_INVALID; parse_result_t parse_res; - prog_config_t config; int ret = EXIT_SUCCESS; - if (odp_init_global(&odp_instance, NULL, NULL) < 0) { + argc = odph_parse_options(argc, argv); + + if (odph_options(&odph_opts) == -1) { + ODPH_ERR("Error while reading ODP helper options, exiting\n"); + exit(EXIT_FAILURE); + } + + odp_init_param_init(&init_param); + init_param.mem_model = odph_opts.mem_model; + + if (odp_init_global(&odp_instance, &init_param, NULL) < 0) { ODPH_ERR("ODP global init failed, exiting\n"); exit(EXIT_FAILURE); } @@ -1958,15 +1975,32 @@ int main(int argc, char **argv) exit(EXIT_FAILURE); } - init_config(&config); + shm_cfg = odp_shm_reserve(PROG_NAME "_cfg", sizeof(prog_config_t), ODP_CACHE_LINE_SIZE, + 0U); - if (!config.is_dir_rx && odp_schedule_config(NULL) < 0) { + if (shm_cfg == ODP_SHM_INVALID) { + ODPH_ERR("Error reserving shared memory\n"); + ret = EXIT_FAILURE; + goto out; + } + + prog_conf = odp_shm_addr(shm_cfg); + + if (prog_conf == NULL) { + ODPH_ERR("Error resolving shared memory address\n"); + ret = EXIT_FAILURE; + goto out; + } + + init_config(prog_conf); + + if (!prog_conf->is_dir_rx && odp_schedule_config(NULL) < 0) { ODPH_ERR("Error configuring scheduler\n"); ret = EXIT_FAILURE; goto out_test; } - parse_res = setup_program(argc, argv, &config); + parse_res = setup_program(argc, argv, prog_conf); if (parse_res == PRS_NOK) { ret = EXIT_FAILURE; @@ -1978,22 +2012,26 @@ int main(int argc, char **argv) goto out_test; } - config.odp_instance = odp_instance; - odp_atomic_init_u32(&is_running, 1U); + prog_conf->odp_instance = odp_instance; + odp_atomic_init_u32(&prog_conf->is_running, 1U); - if (!setup_test(&config)) { + if (!setup_test(prog_conf)) { ret = EXIT_FAILURE; goto out_test; } - while (odp_atomic_load_u32(&is_running)) + while (odp_atomic_load_u32(&prog_conf->is_running)) odp_cpu_pause(); - stop_test(&config); - print_stats(&config); + stop_test(prog_conf); + print_stats(prog_conf); out_test: - teardown_test(&config); + teardown_test(prog_conf); + +out: + if (shm_cfg != ODP_SHM_INVALID) + (void)odp_shm_free(shm_cfg); if (odp_term_local() < 0) { ODPH_ERR("ODP local terminate failed, exiting\n"); diff --git a/test/performance/odp_l2fwd.c b/test/performance/odp_l2fwd.c index ab36a4ebe..27b8d4821 100644 --- a/test/performance/odp_l2fwd.c +++ b/test/performance/odp_l2fwd.c @@ -1948,7 +1948,7 @@ static void create_groups(int num, odp_schedule_group_t *group) } } -static int set_vector_pool_params(odp_pool_param_t *params, odp_pool_capability_t pool_capa) +static int set_vector_pool_params(odp_pool_param_t *params, const odp_pool_capability_t *pool_capa) { uint32_t num_vec, vec_size; @@ -1957,14 +1957,14 @@ static int set_vector_pool_params(odp_pool_param_t *params, odp_pool_capability_ else vec_size = gbl_args->appl.vec_size; - ODPH_ASSERT(pool_capa.vector.max_size > 0); - if (vec_size > pool_capa.vector.max_size) { + ODPH_ASSERT(pool_capa->vector.max_size > 0); + if (vec_size > pool_capa->vector.max_size) { if (gbl_args->appl.vec_size == 0) { - vec_size = pool_capa.vector.max_size; + vec_size = pool_capa->vector.max_size; printf("\nWarning: Vector size reduced to %u\n\n", vec_size); } else { ODPH_ERR("Vector size too big %u. Maximum is %u.\n", - vec_size, pool_capa.vector.max_size); + vec_size, pool_capa->vector.max_size); return -1; } } @@ -1978,13 +1978,13 @@ static int set_vector_pool_params(odp_pool_param_t *params, odp_pool_capability_ num_vec = gbl_args->appl.num_vec; } - if (pool_capa.vector.max_num && num_vec > pool_capa.vector.max_num) { + if (pool_capa->vector.max_num && num_vec > pool_capa->vector.max_num) { if (gbl_args->appl.num_vec == 0) { - num_vec = pool_capa.vector.max_num; + num_vec = pool_capa->vector.max_num; printf("\nWarning: number of vectors reduced to %u\n\n", num_vec); } else { ODPH_ERR("Too many vectors (%u) per pool. Maximum is %u.\n", - num_vec, pool_capa.vector.max_num); + num_vec, pool_capa->vector.max_num); return -1; } } @@ -2197,7 +2197,7 @@ int main(int argc, char *argv[]) } odp_pool_param_init(¶ms); - if (set_vector_pool_params(¶ms, pool_capa)) + if (set_vector_pool_params(¶ms, &pool_capa)) return -1; gbl_args->vector_num = params.vector.num; diff --git a/test/performance/odp_sched_pktio.c b/test/performance/odp_sched_pktio.c index d82dce12f..1333e66e4 100644 --- a/test/performance/odp_sched_pktio.c +++ b/test/performance/odp_sched_pktio.c @@ -424,7 +424,7 @@ static int worker_thread_timers(void *arg) odp_queue_t queue; pktin_queue_context_t *queue_context; odp_timer_t timer; - odp_timer_set_t ret; + odp_timer_retval_t ret; odp_timer_start_t start_param; worker_arg_t *worker_arg = arg; test_global_t *test_global = worker_arg->test_global_ptr; @@ -1326,7 +1326,7 @@ static int start_timers(test_global_t *test_global) int i, j; odp_timeout_t timeout; odp_timer_t timer; - odp_timer_set_t ret; + odp_timer_retval_t ret; odp_timer_start_t start_param; uint64_t timeout_tick = test_global->timer.timeout_tick; int num_pktio = test_global->opt.num_pktio; diff --git a/test/performance/odp_timer_perf.c b/test/performance/odp_timer_perf.c index 3df9a875f..a7d98e68f 100644 --- a/test/performance/odp_timer_perf.c +++ b/test/performance/odp_timer_perf.c @@ -644,7 +644,7 @@ static void cancel_timers(test_global_t *global, uint32_t worker_idx) if (timer == ODP_TIMER_INVALID) continue; - if (odp_timer_cancel(timer, &ev) == 0) + if (odp_timer_cancel(timer, &ev) == ODP_TIMER_SUCCESS) odp_event_free(ev); } } @@ -753,8 +753,14 @@ static int set_cancel_mode_worker(void *arg) status = odp_timer_cancel(timer, &ev); num_cancel++; - if (status < 0) + if (odp_unlikely(status == ODP_TIMER_TOO_NEAR)) { continue; + } else if (odp_unlikely(status != ODP_TIMER_SUCCESS)) { + ODPH_ERR("Timer (%u/%u) cancel failed (ret %i)\n", i, j, + status); + ret = -1; + break; + } start_param.tick_type = ODP_TIMER_TICK_ABS; start_param.tick = tick + j * period_tick; |