diff options
Diffstat (limited to 'test/performance/odp_dma_perf.c')
| -rw-r--r-- | test/performance/odp_dma_perf.c | 1106 |
1 files changed, 1106 insertions, 0 deletions
diff --git a/test/performance/odp_dma_perf.c b/test/performance/odp_dma_perf.c new file mode 100644 index 000000000..26397bf49 --- /dev/null +++ b/test/performance/odp_dma_perf.c @@ -0,0 +1,1106 @@ +/* Copyright (c) 2021-2022, Nokia + * + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef _GNU_SOURCE +#define _GNU_SOURCE +#endif + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <inttypes.h> + +#include <odp_api.h> +#include <odp/helper/odph_api.h> + +#define EXIT_NOT_SUP 2 + +#define DEFAULT_SEG_SIZE 1024U +#define ROUNDS 1000000 +#define COMPL_DELIMITER "," +/* For now, a static maximum amount of input segments */ +#define MAX_NUM_IN_SEGS 64 +#define SHM_SRC "odp_dma_perf_shm_src" +#define SHM_DST "odp_dma_perf_shm_dst" + +#define TRS_TYPE_SYNC 0 +#define TRS_TYPE_ASYNC 1 + +#define GRN_ALL 0 +#define GRN_IND 1 + +#define TYPE_PKT 0 +#define TYPE_MEM 1 + +#define COMPL_MODE_POLL 0 +#define COMPL_MODE_EVENT 1 + +#define GIGAS 1000000000 +#define MEGAS 1000000 +#define KILOS 1000 + +typedef struct test_config_t { + int trs_type; + int trs_grn; + int num_in_seg; + uint32_t seg_size; + int seg_type; + int num_rounds; + int dma_rounds; + + struct { + int num_modes; + uint32_t compl_mask; + int modes[MAX_NUM_IN_SEGS]; + } compl_modes; + + struct { + odp_pool_t pool; + odp_queue_t compl_q; + odp_dma_t handle; + odp_dma_seg_t dst_seg; + odp_dma_seg_t src_seg[MAX_NUM_IN_SEGS]; + } dma_config; + + union { + struct { + odp_shm_t shm_src; + odp_shm_t shm_dst; + void *src; + void *dst; + }; + + struct { + odp_pool_t pool; + odp_packet_t pkts[MAX_NUM_IN_SEGS + 1]; + }; + } seg_config; + + struct { + int (*setup_fn)(struct test_config_t *config); + void (*trs_base_fn)(struct test_config_t *config, + odp_dma_transfer_param_t *trs_params, uint32_t *trs_lengths); + void (*trs_dyn_fn)(struct test_config_t *config, uint32_t offset, uint32_t len); + int (*verify_fn)(const struct test_config_t *config); + void (*free_fn)(struct test_config_t *config); + int (*run_fn)(struct test_config_t *config); + } test_case_api; +} test_config_t; + +typedef struct compl_wait_entry_t { + int type; + odp_dma_transfer_id_t id; +} compl_wait_entry_t; + +static const int compl_mode_map[] = { ODP_DMA_COMPL_POLL, ODP_DMA_COMPL_EVENT }; + +static void set_option_defaults(test_config_t *config) +{ + memset(config, 0, sizeof(*config)); + config->num_in_seg = 1; + config->seg_size = DEFAULT_SEG_SIZE; + config->num_rounds = ROUNDS; + config->compl_modes.compl_mask = ODP_DMA_COMPL_SYNC; +} + +static void parse_completion_modes(test_config_t *config, const char *optarg) +{ + char *tmp_str = strdup(optarg); + char *tmp = strtok(tmp_str, COMPL_DELIMITER); + int mode; + uint32_t i = 0U; + + config->compl_modes.num_modes = 0; + + if (tmp == NULL) { + free(tmp_str); + return; + } + + while (tmp) { + mode = atoi(tmp); + config->compl_modes.modes[i] = mode; + config->compl_modes.compl_mask |= compl_mode_map[mode]; + ++i; + ++config->compl_modes.num_modes; + tmp = strtok(NULL, COMPL_DELIMITER); + } + + free(tmp_str); +} + +static void print_usage(void) +{ + printf("\n" + "DMA performance test. Transfers a set of source segments to a single destination\n" + "segment.\n" + "\n" + "Examples:\n" + " odp_dma_perf\n" + " odp_dma_perf -t 0 -g 1 -i 2\n" + " odp_dma_perf -t 1 -g 1 -i 4 -m 0,0,0,0\n" + " odp_dma_perf -t 1 -g 1 -i 7 -m 0,0,0,0,0,0,1 -T 1 -r 1000 -s 2048\n" + "\n" + "Usage: odp_dma_perf [options]\n" + "\n" + " -t, --trs_type Transfer type for test data. Synchronous by default.\n" + " Types:\n" + " 0: synchronous\n" + " 1: asynchronous\n" + " -g, --trs_grn Transfer granularity for source segments. All\n" + " segments are sent in one transfer by default.\n" + " Options:\n" + " 0: all segments in a single transfer\n" + " 1: individual transfers for segments\n" + " -i, --num_in_seg Number of input segments to transfer. 1 by\n" + " default. Maximum supported amount is %d.\n" + " -s, --in_seg_size Segment size for all input segments in bytes. 1024\n" + " bytes by default. Maximum allowed destination\n" + " segment size may limit this choice.\n" + " -T, --in_seg_type Input segment data type. Packet by default.\n" + " Types:\n" + " 0: packet\n" + " 1: memory\n" + " -m, --compl_modes Completion mode(s) for transfers delimited by a\n" + " comma. Only applicable in asynchronous mode.\n" + " Modes:\n" + " 0: poll\n" + " 1: event\n" + " -r, --num_rounds Number of times to run the test scenario. %d by\n" + " default.\n" + " -h, --help This help.\n" + "\n", + MAX_NUM_IN_SEGS, ROUNDS); +} + +static int check_completion_modes(test_config_t *config) +{ + if (config->trs_type == TRS_TYPE_SYNC) + return 0; + + if (config->compl_modes.num_modes > MAX_NUM_IN_SEGS) + return -1; + + if (config->trs_grn == GRN_IND && + config->num_in_seg != config->compl_modes.num_modes) + return -1; + + if (config->trs_grn == GRN_ALL && + config->compl_modes.num_modes != 1) + return -1; + + for (int i = 0; i < config->compl_modes.num_modes; ++i) { + if (config->compl_modes.modes[i] != COMPL_MODE_POLL && + config->compl_modes.modes[i] != COMPL_MODE_EVENT) + return -1; + + config->compl_modes.modes[i] = compl_mode_map[config->compl_modes.modes[i]]; + } + + return 0; +} + +static int check_options(test_config_t *config) +{ + if (config->trs_type != TRS_TYPE_SYNC && + config->trs_type != TRS_TYPE_ASYNC) { + ODPH_ERR("Invalid transfer type: %d.\n", config->trs_type); + return -1; + } + + if (config->trs_grn != GRN_ALL && config->trs_grn != GRN_IND) { + ODPH_ERR("Invalid granularity: %d.\n", config->trs_grn); + return -1; + } + + config->dma_rounds = config->trs_grn == GRN_IND ? config->num_in_seg : 1; + + if (config->num_in_seg < 1 || config->num_in_seg > MAX_NUM_IN_SEGS) { + ODPH_ERR("Invalid number of input segments: %d.\n", config->num_in_seg); + return -1; + } + + if (config->seg_type != TYPE_PKT && config->seg_type != TYPE_MEM) { + ODPH_ERR("Invalid input segment type: %d.\n", config->seg_type); + return -1; + } + + if (check_completion_modes(config)) { + ODPH_ERR("Invalid completion modes.\n"); + return -1; + } + + if (config->num_rounds < 1) { + ODPH_ERR("Invalid number of rounds: %d.\n", config->num_rounds); + return -1; + } + + return 0; +} + +static int parse_options(int argc, char **argv, test_config_t *config) +{ + int opt, long_index; + + static const struct option longopts[] = { + { "trs_type", required_argument, NULL, 't' }, + { "trs_grn", required_argument, NULL, 'g' }, + { "num_in_seg", required_argument, NULL, 'i' }, + { "in_seg_size", required_argument, NULL, 's' }, + { "in_seg_type", required_argument, NULL, 'T' }, + { "compl_modes", required_argument, NULL, 'm' }, + { "num_rounds", required_argument, NULL, 'r' }, + { "help", no_argument, NULL, 'h' }, + { NULL, 0, NULL, 0 } + }; + + static const char *shortopts = "t:g:i:s:T:m:r:h"; + + set_option_defaults(config); + + while (1) { + opt = getopt_long(argc, argv, shortopts, longopts, &long_index); + + if (opt == -1) + break; + + switch (opt) { + case 't': + config->trs_type = atoi(optarg); + break; + case 'g': + config->trs_grn = atoi(optarg); + break; + case 'i': + config->num_in_seg = atoi(optarg); + break; + case 's': + config->seg_size = atoi(optarg); + break; + case 'T': + config->seg_type = atoi(optarg); + break; + case 'm': + parse_completion_modes(config, optarg); + break; + case 'r': + config->num_rounds = atoi(optarg); + break; + case 'h': + default: + print_usage(); + return -1; + } + } + + if (check_options(config)) + return -1; + + return 0; +} + +static int check_shm_capabilities(const test_config_t *config) +{ + odp_shm_capability_t capa; + + if (odp_shm_capability(&capa)) { + ODPH_ERR("Error querying SHM capabilities.\n"); + return -1; + } + + if (capa.max_blocks < 2U) { + ODPH_ERR("Unsupported amount of SHM blocks.\n"); + return -1; + } + + if (capa.max_size != 0U && config->num_in_seg * config->seg_size > capa.max_size) { + ODPH_ERR("Unsupported total SHM block size.\n"); + return -1; + } + + if (capa.max_align != 0U && capa.max_align < ODP_CACHE_LINE_SIZE) { + ODPH_ERR("Unsupported SHM block alignment size.\n"); + return -1; + } + + return 0; +} + +static int check_dma_capabilities(const test_config_t *config) +{ + odp_dma_capability_t capa; + const int is_event = config->compl_modes.compl_mask & ODP_DMA_COMPL_EVENT; + uint32_t event_compl_count = 0U; + + if (odp_dma_capability(&capa)) { + ODPH_ERR("Error querying DMA capabilities.\n"); + return -1; + } + + if (capa.max_sessions == 0U) { + ODPH_ERR("DMA not supported.\n"); + return -1; + } + + if (config->trs_type == TRS_TYPE_ASYNC) { + if ((config->compl_modes.compl_mask & ODP_DMA_COMPL_POLL) && + (capa.compl_mode_mask & ODP_DMA_COMPL_POLL) == 0U) { + ODPH_ERR("Unsupported DMA completion mode, poll.\n"); + return -1; + } + + if (is_event && (capa.compl_mode_mask & ODP_DMA_COMPL_EVENT) == 0U) { + ODPH_ERR("Unsupported DMA completion mode, event.\n"); + return -1; + } + + if (is_event && capa.queue_type_sched == 0) { + ODPH_ERR("Unsupported DMA queueing type.\n"); + return -1; + } + + if (config->trs_grn == GRN_IND) { + if ((uint32_t)config->num_in_seg > capa.max_transfers) { + ODPH_ERR("Unsupported amount of in-flight DMA transfers.\n"); + return -1; + } + + for (int i = 0; i < config->compl_modes.num_modes; ++i) + if (config->compl_modes.modes[i] == ODP_DMA_COMPL_EVENT) + ++event_compl_count; + + if (event_compl_count > capa.pool.max_num) { + ODPH_ERR("Unsupported amount of completion events.\n"); + return -1; + } + } + } + + if (config->trs_grn == GRN_ALL) { + if ((uint32_t)config->num_in_seg > capa.max_src_segs) { + ODPH_ERR("Unsupported amount of DMA source segments.\n"); + return -1; + } + + if (config->num_in_seg + 1U > capa.max_segs) { + ODPH_ERR("Unsupported total amount of DMA segments.\n"); + return -1; + } + } + + if (config->trs_grn == GRN_IND && capa.max_segs < 2U) { + ODPH_ERR("Unsupported total amount of DMA segments.\n"); + return -1; + } + + if (config->num_in_seg * config->seg_size > capa.max_seg_len) { + ODPH_ERR("Unsupported total DMA segment size.\n"); + return -1; + } + + return 0; +} + +static int check_capabilities(const test_config_t *config) +{ + return check_shm_capabilities(config) || + check_dma_capabilities(config); +} + +static int configure_packets(test_config_t *config) +{ + odp_pool_param_t param; + + for (int i = 0; i < config->num_in_seg + 1; ++i) + config->seg_config.pkts[i] = ODP_PACKET_INVALID; + + odp_pool_param_init(¶m); + param.type = ODP_POOL_PACKET; + /* Configured amount of input segments and one output segment */ + param.pkt.num = config->num_in_seg + 1U; + param.pkt.len = config->num_in_seg * config->seg_size; + config->seg_config.pool = odp_pool_create("odp_dma_perf_packets", ¶m); + + if (config->seg_config.pool == ODP_POOL_INVALID) { + ODPH_ERR("Error creating packet pool.\n"); + return -1; + } + + return 0; +} + +static int allocate_packets(test_config_t *config) +{ + for (int i = 0; i < config->num_in_seg; ++i) { + config->seg_config.pkts[i] = odp_packet_alloc(config->seg_config.pool, + config->seg_size); + + if (config->seg_config.pkts[i] == ODP_PACKET_INVALID) { + ODPH_ERR("Error allocating input test packets.\n"); + return -1; + } + } + + config->seg_config.pkts[config->num_in_seg] = + odp_packet_alloc(config->seg_config.pool, config->num_in_seg * config->seg_size); + + if (config->seg_config.pkts[config->num_in_seg] == ODP_PACKET_INVALID) { + ODPH_ERR("Error allocating output test packet.\n"); + return -1; + } + + return 0; +} + +static int populate_packets(test_config_t *config) +{ + for (int i = 0; i < config->num_in_seg; ++i) { + uint8_t data[odp_packet_len(config->seg_config.pkts[i])]; + + memset(data, i + 1, sizeof(data)); + + if (odp_packet_copy_from_mem(config->seg_config.pkts[i], 0U, sizeof(data), data)) + return -1; + } + + return 0; +} + +static int setup_packet_segments(test_config_t *config) +{ + return configure_packets(config) || + allocate_packets(config) || + populate_packets(config); +} + +static void configure_packet_dma_transfer_base(test_config_t *config, + odp_dma_transfer_param_t trs_params[], + uint32_t trs_lengths[]) +{ + memset(trs_lengths, 0, sizeof(*trs_lengths) * config->dma_rounds); + + for (int i = 0; i < config->num_in_seg; ++i) { + config->dma_config.src_seg[i].packet = config->seg_config.pkts[i]; + config->dma_config.src_seg[i].offset = 0U; + config->dma_config.src_seg[i].len = odp_packet_len(config->seg_config.pkts[i]); + } + + config->dma_config.dst_seg.packet = config->seg_config.pkts[config->num_in_seg]; + + for (int i = 0; i < config->dma_rounds; ++i) { + odp_dma_transfer_param_init(&trs_params[i]); + trs_params[i].src_format = ODP_DMA_FORMAT_PACKET; + trs_params[i].dst_format = ODP_DMA_FORMAT_PACKET; + trs_params[i].num_src = config->trs_grn == GRN_IND ? 1 : config->num_in_seg; + trs_params[i].num_dst = 1U; + trs_params[i].src_seg = &config->dma_config.src_seg[i]; + trs_params[i].dst_seg = &config->dma_config.dst_seg; + trs_lengths[i] = config->trs_grn == GRN_IND ? + config->dma_config.src_seg[i].len : + config->num_in_seg * config->seg_size; + } +} + +static inline void configure_packet_dma_transfer_dynamic(test_config_t *config, uint32_t offset, + uint32_t len) +{ + config->dma_config.dst_seg.offset = offset; + config->dma_config.dst_seg.len = len; +} + +static int verify_packet_transfer(const test_config_t *config) +{ + uint32_t len, offset = 0U; + + for (int i = 0; i < config->num_in_seg; ++i) { + len = odp_packet_len(config->seg_config.pkts[i]); + uint8_t src_data[len]; + uint8_t dst_data[len]; + + if (odp_packet_copy_to_mem(config->seg_config.pkts[i], 0U, len, src_data) || + odp_packet_copy_to_mem(config->seg_config.pkts[config->num_in_seg], offset, + len, dst_data)) { + ODPH_ERR("Error verifying DMA transfer.\n"); + return -1; + } + + if (memcmp(src_data, dst_data, len)) { + ODPH_ERR("Error in DMA transfer, source and destination data do not match.\n"); + return -1; + } + + offset += len; + } + + return 0; +} + +static void free_packets(test_config_t *config) +{ + /* Configured amount of input segments and one output segment */ + for (int i = 0; i < config->num_in_seg + 1; ++i) + if (config->seg_config.pkts[i] != ODP_PACKET_INVALID) + odp_packet_free(config->seg_config.pkts[i]); + + if (config->seg_config.pool != ODP_POOL_INVALID) + (void)odp_pool_destroy(config->seg_config.pool); +} + +static int allocate_memory(test_config_t *config) +{ + const uint64_t size = config->num_in_seg * (uint64_t)config->seg_size; + + config->seg_config.shm_src = ODP_SHM_INVALID; + config->seg_config.shm_dst = ODP_SHM_INVALID; + config->seg_config.src = NULL; + config->seg_config.dst = NULL; + + config->seg_config.shm_src = odp_shm_reserve(SHM_SRC, size, ODP_CACHE_LINE_SIZE, 0); + config->seg_config.shm_dst = odp_shm_reserve(SHM_DST, size, ODP_CACHE_LINE_SIZE, 0); + + if (config->seg_config.shm_src == ODP_SHM_INVALID || + config->seg_config.shm_dst == ODP_SHM_INVALID) { + ODPH_ERR("Error allocating SHM block.\n"); + return -1; + } + + config->seg_config.src = odp_shm_addr(config->seg_config.shm_src); + config->seg_config.dst = odp_shm_addr(config->seg_config.shm_dst); + + if (config->seg_config.src == NULL || config->seg_config.dst == NULL) { + ODPH_ERR("Error resolving SHM block address.\n"); + return -1; + } + + return 0; +} + +static int populate_memory(test_config_t *config) +{ + uint8_t val; + uint8_t *addr; + + for (int i = 0; i < config->num_in_seg; ++i) { + val = 0U; + addr = (uint8_t *)config->seg_config.src + i * config->seg_size; + + for (uint32_t i = 0U; i < config->seg_size; ++i) + addr[i] = val++; + } + + return 0; +} + +static int setup_memory_segments(test_config_t *config) +{ + return allocate_memory(config) || + populate_memory(config); +} + +static void configure_address_dma_transfer_base(test_config_t *config, + odp_dma_transfer_param_t trs_params[], + uint32_t trs_lengths[]) +{ + memset(trs_lengths, 0, sizeof(*trs_lengths) * config->dma_rounds); + + for (int i = 0; i < config->num_in_seg; ++i) { + config->dma_config.src_seg[i].addr = + (uint8_t *)config->seg_config.src + i * config->seg_size; + config->dma_config.src_seg[i].len = config->seg_size; + } + + config->dma_config.dst_seg.addr = config->seg_config.dst; + + for (int i = 0; i < config->dma_rounds; ++i) { + odp_dma_transfer_param_init(&trs_params[i]); + trs_params[i].src_format = ODP_DMA_FORMAT_ADDR; + trs_params[i].dst_format = ODP_DMA_FORMAT_ADDR; + trs_params[i].num_src = config->trs_grn == GRN_IND ? 1 : config->num_in_seg; + trs_params[i].num_dst = 1U; + trs_params[i].src_seg = &config->dma_config.src_seg[i]; + trs_params[i].dst_seg = &config->dma_config.dst_seg; + trs_lengths[i] = config->trs_grn == GRN_IND ? + config->dma_config.src_seg[i].len : + config->num_in_seg * config->seg_size; + } +} + +static inline void configure_address_dma_transfer_dynamic(test_config_t *config, uint32_t offset, + uint32_t len) +{ + config->dma_config.dst_seg.addr = (uint8_t *)config->seg_config.dst + offset; + config->dma_config.dst_seg.len = len; +} + +static int verify_memory_transfer(const test_config_t *config) +{ + if (memcmp(config->seg_config.src, config->seg_config.dst, + config->num_in_seg * config->seg_size)) { + ODPH_ERR("Error in DMA transfer, source and destination data do not match.\n"); + return -1; + } + + return 0; +} + +static void free_memory(test_config_t *config) +{ + if (config->seg_config.shm_src != ODP_SHM_INVALID) + (void)odp_shm_free(config->seg_config.shm_src); + + if (config->seg_config.shm_dst != ODP_SHM_INVALID) + (void)odp_shm_free(config->seg_config.shm_dst); +} + +static void print_humanised_speed(uint64_t speed) +{ + if (speed > GIGAS) + printf("%.2f GB/s\n", (double)speed / GIGAS); + else if (speed > MEGAS) + printf("%.2f MB/s\n", (double)speed / MEGAS); + else if (speed > KILOS) + printf("%.2f KB/s\n", (double)speed / KILOS); + else + printf("%" PRIu64 " B/s\n", speed); +} + +static void print_results(const test_config_t *config, uint64_t time) +{ + const int is_sync = config->trs_type == TRS_TYPE_SYNC; + const uint64_t avg_time = time / config->num_rounds; + uint64_t avg_speed = 0U; + + printf("\n" + "=============================================\n\n" + "DMA transfer test done\n\n" + " mode: %s\n" + " granularity: %s\n" + " input segment count: %d\n" + " segment size: %u\n" + " segment type: %s\n", + is_sync ? "synchronous" : "asynchronous", + config->trs_grn == GRN_IND ? "individual" : "all", + config->num_in_seg, config->seg_size, + config->seg_type == TYPE_PKT ? "packet" : "memory"); + + if (!is_sync) { + printf(" completion modes in order: "); + + for (int i = 0; i < config->compl_modes.num_modes; ++i) + printf("%s", config->compl_modes.modes[i] == ODP_DMA_COMPL_POLL ? + "poll " : "event "); + + printf("\n"); + } + + if (avg_time > 0U) + avg_speed = config->num_in_seg * config->seg_size * ODP_TIME_SEC_IN_NS / avg_time; + + printf(" rounds run: %d\n" + " average time per transfer: %" PRIu64 " ns\n" + " average transfer speed: ", + config->num_rounds, avg_time); + print_humanised_speed(avg_speed); + printf("\n=============================================\n"); +} + +static int run_dma_sync(test_config_t *config) +{ + odp_dma_transfer_param_t trs_params[config->dma_rounds]; + uint32_t trs_lengths[config->dma_rounds]; + odp_time_t start, end; + uint32_t num_rounds = config->num_rounds, offset; + + config->test_case_api.trs_base_fn(config, trs_params, trs_lengths); + start = odp_time_local_strict(); + + while (num_rounds--) { + offset = 0U; + + for (int i = 0; i < config->dma_rounds; ++i) { + config->test_case_api.trs_dyn_fn(config, offset, trs_lengths[i]); + + if (odp_dma_transfer(config->dma_config.handle, &trs_params[i], NULL) + <= 0) { + ODPH_ERR("Error starting a sync DMA transfer.\n"); + return -1; + } + + offset += trs_lengths[i]; + } + } + + end = odp_time_local_strict(); + print_results(config, odp_time_diff_ns(end, start)); + return 0; +} + +static int configure_dma_event_completion(test_config_t *config) +{ + int ret; + odp_dma_pool_param_t pool_param; + odp_queue_param_t queue_param; + + config->dma_config.pool = ODP_POOL_INVALID; + config->dma_config.compl_q = ODP_QUEUE_INVALID; + + ret = odp_schedule_config(NULL); + + if (ret < 0) { + ODPH_ERR("Error configuring scheduler.\n"); + return -1; + } + + odp_dma_pool_param_init(&pool_param); + pool_param.num = config->num_in_seg; + config->dma_config.pool = odp_dma_pool_create("odp_dma_perf_events", &pool_param); + + if (config->dma_config.pool == ODP_POOL_INVALID) { + ODPH_ERR("Error creating DMA event completion pool.\n"); + return -1; + } + + odp_queue_param_init(&queue_param); + queue_param.type = ODP_QUEUE_TYPE_SCHED; + queue_param.sched.sync = ODP_SCHED_SYNC_PARALLEL; + queue_param.sched.prio = odp_schedule_default_prio(); + queue_param.sched.group = ODP_SCHED_GROUP_ALL; + config->dma_config.compl_q = odp_queue_create("odp_dma_perf_queue", &queue_param); + + if (config->dma_config.compl_q == ODP_QUEUE_INVALID) { + ODPH_ERR("Error creating DMA completion queue.\n"); + return -1; + } + + return 0; +} + +static int configure_dma_completion_params(test_config_t *config, + odp_dma_compl_param_t compl_params[]) +{ + odp_dma_compl_t compl_ev; + + for (int i = 0; i < config->dma_rounds; ++i) + odp_dma_compl_param_init(&compl_params[i]); + + for (int i = 0; i < config->dma_rounds; ++i) { + if (config->compl_modes.modes[i] == ODP_DMA_COMPL_EVENT) { + compl_params[i].compl_mode = ODP_DMA_COMPL_EVENT; + compl_ev = odp_dma_compl_alloc(config->dma_config.pool); + + if (compl_ev == ODP_DMA_COMPL_INVALID) { + ODPH_ERR("Error creating DMA completion event.\n"); + return -1; + } + + compl_params[i].event = odp_dma_compl_to_event(compl_ev); + compl_params[i].queue = config->dma_config.compl_q; + } else if (config->compl_modes.modes[i] == ODP_DMA_COMPL_POLL) { + compl_params[i].compl_mode = ODP_DMA_COMPL_POLL; + compl_params[i].transfer_id = + odp_dma_transfer_id_alloc(config->dma_config.handle); + + if (compl_params[i].transfer_id == ODP_DMA_TRANSFER_ID_INVALID) { + ODPH_ERR("Error creating DMA transfer ID.\n"); + return -1; + } + } + + compl_params[i].user_ptr = NULL; + } + + return 0; +} + +static void build_wait_list(const test_config_t *config, odp_dma_compl_param_t compl_params[], + compl_wait_entry_t list[]) +{ + int last_ev_idx, has_events = 0; + + memset(list, 0, sizeof(*list) * config->dma_rounds); + + for (int i = 0, j = 0, k = 0; i < config->dma_rounds; ++i) { + if (config->compl_modes.modes[i] == ODP_DMA_COMPL_EVENT) { + compl_wait_entry_t entry = { .type = ODP_DMA_COMPL_EVENT }; + + list[j] = entry; + ++j; + + for (; k < i; ++k) { + entry.type = ODP_DMA_COMPL_POLL; + entry.id = compl_params[k].transfer_id; + list[j++] = entry; + } + + ++k; + last_ev_idx = i; + has_events = 1; + } + } + + last_ev_idx = has_events ? last_ev_idx + 1 : 0; + + for (int i = last_ev_idx; i < config->dma_rounds; ++i) { + compl_wait_entry_t entry = { .type = ODP_DMA_COMPL_POLL, + .id = compl_params[i].transfer_id }; + list[i] = entry; + } +} + +static inline int wait_dma_transfers_ready(test_config_t *config, compl_wait_entry_t list[]) +{ + odp_event_t ev; + const uint64_t wait_time = odp_schedule_wait_time(ODP_TIME_SEC_IN_NS * 5U); + int done = 0; + + for (int i = 0; i < config->dma_rounds; ++i) { + if (list[i].type == ODP_DMA_COMPL_EVENT) { + ev = odp_schedule(NULL, wait_time); + + if (ev == ODP_EVENT_INVALID) { + ODPH_ERR("Error waiting event completion.\n"); + return -1; + } + } else { + while (1) { + done = odp_dma_transfer_done(config->dma_config.handle, list[i].id, + NULL); + + if (done > 0) + break; + + if (done == 0) + continue; + + ODPH_ERR("Error waiting poll completion.\n"); + return -1; + } + } + } + + return 0; +} + +static void free_dma_completion_events(test_config_t *config, odp_dma_compl_param_t compl_params[]) +{ + for (int i = 0; i < config->dma_rounds; ++i) + if (config->compl_modes.modes[i] == ODP_DMA_COMPL_EVENT && + compl_params[i].event != ODP_EVENT_INVALID) + odp_dma_compl_free(odp_dma_compl_from_event(compl_params[i].event)); +} + +static void free_dma_transfer_ids(test_config_t *config, odp_dma_compl_param_t compl_params[]) +{ + for (int i = 0; i < config->dma_rounds; ++i) + if (config->compl_modes.modes[i] == ODP_DMA_COMPL_POLL && + compl_params[i].transfer_id != ODP_DMA_TRANSFER_ID_INVALID) + odp_dma_transfer_id_free(config->dma_config.handle, + compl_params[i].transfer_id); +} + +static int run_dma_async_transfer(test_config_t *config) +{ + odp_dma_transfer_param_t trs_params[config->dma_rounds]; + uint32_t trs_lengths[config->dma_rounds]; + odp_dma_compl_param_t compl_params[config->dma_rounds]; + int ret = 0; + compl_wait_entry_t compl_wait_list[config->dma_rounds]; + odp_time_t start, end; + uint32_t num_rounds = config->num_rounds, offset; + + config->test_case_api.trs_base_fn(config, trs_params, trs_lengths); + + if (configure_dma_completion_params(config, compl_params)) { + ret = -1; + goto out_compl_evs; + } + + build_wait_list(config, compl_params, compl_wait_list); + start = odp_time_local_strict(); + + while (num_rounds--) { + offset = 0U; + + for (int i = 0; i < config->dma_rounds; ++i) { + config->test_case_api.trs_dyn_fn(config, offset, trs_lengths[i]); + + if (odp_dma_transfer_start(config->dma_config.handle, &trs_params[i], + &compl_params[i]) <= 0) { + ODPH_ERR("Error starting an async DMA transfer.\n"); + ret = -1; + goto out_trs_ids; + } + + offset += trs_lengths[i]; + } + + if (wait_dma_transfers_ready(config, compl_wait_list)) { + ODPH_ERR("Error finishing an async DMA transfer.\n"); + ret = -1; + goto out_trs_ids; + } + } + + end = odp_time_local_strict(); + print_results(config, odp_time_diff_ns(end, start)); + +out_compl_evs: + free_dma_completion_events(config, compl_params); + +out_trs_ids: + free_dma_transfer_ids(config, compl_params); + return ret; +} + +static void free_dma_event_completion(test_config_t *config) +{ + if (config->dma_config.compl_q != ODP_QUEUE_INVALID) + (void)odp_queue_destroy(config->dma_config.compl_q); + + if (config->dma_config.pool != ODP_POOL_INVALID) + (void)odp_pool_destroy(config->dma_config.pool); +} + +static int run_dma_async(test_config_t *config) +{ + const int is_event_compl = config->compl_modes.compl_mask & ODP_DMA_COMPL_EVENT; + int ret = 0; + + if (is_event_compl) + if (configure_dma_event_completion(config)) { + ret = -1; + goto out; + } + + if (run_dma_async_transfer(config)) + ret = -1; + +out: + if (is_event_compl) + free_dma_event_completion(config); + + return ret; +} + +static void setup_test_case_api(test_config_t *config) +{ + switch (config->seg_type) { + case TYPE_PKT: + config->test_case_api.setup_fn = setup_packet_segments; + config->test_case_api.trs_base_fn = configure_packet_dma_transfer_base; + config->test_case_api.trs_dyn_fn = configure_packet_dma_transfer_dynamic; + config->test_case_api.verify_fn = verify_packet_transfer; + config->test_case_api.free_fn = free_packets; + break; + case TYPE_MEM: + config->test_case_api.setup_fn = setup_memory_segments; + config->test_case_api.trs_base_fn = configure_address_dma_transfer_base; + config->test_case_api.trs_dyn_fn = configure_address_dma_transfer_dynamic; + config->test_case_api.verify_fn = verify_memory_transfer; + config->test_case_api.free_fn = free_memory; + break; + default: + break; + } + + config->test_case_api.run_fn = config->trs_type == TRS_TYPE_SYNC ? + run_dma_sync : + run_dma_async; +} + +static int configure_dma_session(test_config_t *config) +{ + const odp_dma_param_t params = { .direction = ODP_DMA_MAIN_TO_MAIN, + .type = ODP_DMA_TYPE_COPY, + .compl_mode_mask = config->compl_modes.compl_mask, + .mt_mode = ODP_DMA_MT_SERIAL, + .order = ODP_DMA_ORDER_NONE }; + + config->dma_config.handle = odp_dma_create("odp_dma_perf", ¶ms); + + if (config->dma_config.handle == ODP_DMA_INVALID) { + ODPH_ERR("Error creating DMA session.\n"); + return -1; + } + + return 0; +} + +static void free_dma_session(test_config_t *config) +{ + if (config->dma_config.handle != ODP_DMA_INVALID) + (void)odp_dma_destroy(config->dma_config.handle); +} + +int main(int argc, char **argv) +{ + odph_helper_options_t odph_opts; + test_config_t test_config; + odp_instance_t odp_instance; + int ret = EXIT_SUCCESS; + + argc = odph_parse_options(argc, argv); + + if (odph_options(&odph_opts)) { + ODPH_ERR("Error while reading ODP helper options, exiting.\n"); + exit(EXIT_FAILURE); + } + + if (parse_options(argc, argv, &test_config)) + exit(EXIT_FAILURE); + + if (odp_init_global(&odp_instance, NULL, NULL)) { + ODPH_ERR("ODP global init failed, exiting.\n"); + exit(EXIT_FAILURE); + } + + if (odp_init_local(odp_instance, ODP_THREAD_CONTROL)) { + ODPH_ERR("ODP local init failed, exiting.\n"); + exit(EXIT_FAILURE); + } + + if (check_capabilities(&test_config)) { + ODPH_ERR("Unsupported scenario attempted, exiting.\n"); + ret = EXIT_NOT_SUP; + goto out_odp; + } + + setup_test_case_api(&test_config); + + if (configure_dma_session(&test_config)) { + ret = EXIT_FAILURE; + goto out_dma; + } + + if (test_config.test_case_api.setup_fn(&test_config)) { + ret = EXIT_FAILURE; + goto out_test_case; + } + + if (test_config.test_case_api.run_fn(&test_config) || + test_config.test_case_api.verify_fn(&test_config)) + ret = EXIT_FAILURE; + +out_test_case: + test_config.test_case_api.free_fn(&test_config); + +out_dma: + free_dma_session(&test_config); + +out_odp: + if (odp_term_local()) { + ODPH_ERR("ODP local terminate failed, exiting.\n"); + exit(EXIT_FAILURE); + } + + if (odp_term_global(odp_instance)) { + ODPH_ERR("ODP global terminate failed, exiting.\n"); + exit(EXIT_FAILURE); + } + + return ret; +} |