path: root/test/performance/odp_dma_perf.c

/* SPDX-License-Identifier: BSD-3-Clause
 * Copyright (c) 2021-2023 Nokia
 */

/**
 * DMA performance tester
 *
 * This tester application can be used to profile the performance of an ODP DMA implementation.
 * Tester workflow is simple and consists of issuing as many back-to-back DMA transfers as the
 * implementation allows and then recording key performance statistics (such as function overhead,
 * latencies etc.).
 */

#ifndef _GNU_SOURCE
#define _GNU_SOURCE
#endif

#include <inttypes.h>
#include <stdlib.h>
#include <signal.h>
#include <stdint.h>
#include <unistd.h>

#include <odp_api.h>
#include <odp/helper/odph_api.h>

#define EXIT_NOT_SUP 2
#define PROG_NAME "odp_dma_perf"

enum {
	SYNC_DMA = 0U,
	ASYNC_DMA,
	SW_COPY
};

enum {
	DENSE_PACKET = 0U,
	SPARSE_PACKET,
	DENSE_MEMORY,
	SPARSE_MEMORY
};

enum {
	POLL = 0U,
	EVENT
};

enum {
	SINGLE = 0U,
	MANY
};

#define DEF_TRS_TYPE SYNC_DMA
#define DEF_SEG_CNT 1U
#define DEF_LEN 1024U
#define DEF_SEG_TYPE DENSE_PACKET
#define DEF_MODE POLL
#define DEF_INFLIGHT 1U
#define DEF_TIME 10U
#define DEF_WORKERS 1U
#define DEF_POLICY SINGLE

#define MAX_SEGS 1024U
#define MAX_WORKERS 24
#define MAX_MEMORY (256U * 1024U * 1024U)

#define GIGAS 1000000000
#define MEGAS 1000000
#define KILOS 1000

#define DATA 0xAA

typedef enum {
	PRS_OK,
	PRS_NOK,
	PRS_TERM,
	PRS_NOT_SUP
} parse_result_t;

typedef struct {
	uint64_t completed;
	uint64_t start_errs;
	uint64_t poll_errs;
	uint64_t scheduler_timeouts;
	uint64_t transfer_errs;
	uint64_t data_errs;
	uint64_t tot_tm;
	uint64_t trs_tm;
	uint64_t max_trs_tm;
	uint64_t min_trs_tm;
	uint64_t start_cc;
	uint64_t max_start_cc;
	uint64_t min_start_cc;
	uint64_t wait_cc;
	uint64_t max_wait_cc;
	uint64_t min_wait_cc;
	uint64_t trs_cc;
	uint64_t max_trs_cc;
	uint64_t min_trs_cc;
	uint64_t start_cnt;
	uint64_t wait_cnt;
	uint64_t trs_poll_cnt;
	uint64_t trs_cnt;
} stats_t;

typedef struct {
	odp_dma_transfer_param_t trs_param;
	odp_dma_compl_param_t compl_param;
	odp_ticketlock_t lock;
	uint64_t trs_start_tm;
	uint64_t trs_start_cc;
	uint64_t trs_poll_cnt;
	odp_bool_t is_running;
} trs_info_t;

typedef struct sd_s sd_t;
typedef void (*ver_fn_t)(trs_info_t *info, stats_t *stats);

typedef struct ODP_ALIGNED_CACHE sd_s {
	struct {
		trs_info_t infos[MAX_SEGS];
		odp_dma_seg_t src_seg[MAX_SEGS];
		odp_dma_seg_t dst_seg[MAX_SEGS];
		odp_dma_t handle;
		odp_pool_t pool;
		odp_queue_t compl_q;
		uint32_t num_in_segs;
		uint32_t num_out_segs;
		uint32_t src_seg_len;
		uint32_t dst_seg_len;
		uint32_t num_inflight;
		uint8_t trs_type;
		uint8_t compl_mode;
	} dma;

	struct {
		odp_packet_t src_pkt[MAX_SEGS];
		odp_packet_t dst_pkt[MAX_SEGS];
		odp_pool_t src_pool;
		odp_pool_t dst_pool;
		odp_shm_t src_shm;
		odp_shm_t dst_shm;
		void *src;
		void *dst;
		void *src_high;
		void *dst_high;
		void *cur_src;
		void *cur_dst;
		uint64_t shm_size;
		uint8_t seg_type;
	} seg;

	odp_schedule_group_t grp;
	/* Prepare single transfer. */
	void (*prep_trs_fn)(sd_t *sd, trs_info_t *info);
	/* Verify single transfer. */
	ver_fn_t ver_fn;
} sd_t;

typedef struct prog_config_s prog_config_t;

typedef struct ODP_ALIGNED_CACHE {
	stats_t stats;
	prog_config_t *prog_config;
	sd_t *sd;
} thread_config_t;

typedef struct {
	/* Configure DMA session specific resources. */
	odp_bool_t (*session_cfg_fn)(sd_t *sd);
	/* Setup transfer elements (memory/packet segments). */
	odp_bool_t (*setup_fn)(sd_t *sd);
	/* Configure DMA transfers (segment addresses etc.). */
	void (*trs_fn)(sd_t *sd);
	/* Configure transfer completion resources (transfer IDs, events etc.). */
	odp_bool_t (*compl_fn)(sd_t *sd);
	/* Initiate required initial transfers. */
	odp_bool_t (*bootstrap_fn)(sd_t *sd);
	/* Wait and handle finished transfer. */
	void (*wait_fn)(sd_t *sd, stats_t *stats);
	/* Handle all unfinished transfers after main test has been stopped. */
	void (*drain_fn)(sd_t *sd);
	/* Free any resources that might have been allocated during setup phase. */
	void (*free_fn)(const sd_t *sd);
} test_api_t;

typedef struct prog_config_s {
	odph_thread_t threads[MAX_WORKERS];
	thread_config_t thread_config[MAX_WORKERS];
	sd_t sds[MAX_WORKERS];
	test_api_t api;
	odp_atomic_u32_t is_running;
	odp_instance_t odp_instance;
	odp_barrier_t init_barrier;
	odp_barrier_t term_barrier;
	odp_dma_compl_mode_t compl_mode_mask;
	odp_pool_t src_pool;
	odp_pool_t dst_pool;
	uint64_t shm_size;
	uint32_t num_in_segs;
	uint32_t num_out_segs;
	uint32_t src_seg_len;
	uint32_t dst_seg_len;
	uint32_t num_inflight;
	uint32_t time_sec;
	uint32_t num_sessions;
	uint32_t src_cache_size;
	uint32_t dst_cache_size;
	int num_workers;
	odp_bool_t is_verify;
	uint8_t trs_type;
	uint8_t seg_type;
	uint8_t compl_mode;
	uint8_t policy;
} prog_config_t;

static prog_config_t *prog_conf;

static const int mode_map[] = { ODP_DMA_COMPL_POLL, ODP_DMA_COMPL_EVENT };

static void terminate(int signal ODP_UNUSED)
{
	odp_atomic_store_u32(&prog_conf->is_running, 0U);
}

static void init_config(prog_config_t *config)
{
	sd_t *sd;
	trs_info_t *info;
	stats_t *stats;

	memset(config, 0, sizeof(*config));
	config->compl_mode_mask |= ODP_DMA_COMPL_SYNC;
	config->src_pool = ODP_POOL_INVALID;
	config->dst_pool = ODP_POOL_INVALID;
	config->num_in_segs = DEF_SEG_CNT;
	config->num_out_segs = DEF_SEG_CNT;
	config->src_seg_len = DEF_LEN;
	config->num_inflight = DEF_INFLIGHT;
	config->time_sec = DEF_TIME;
	config->num_workers = DEF_WORKERS;
	config->trs_type = DEF_TRS_TYPE;
	config->seg_type = DEF_SEG_TYPE;
	config->compl_mode = DEF_MODE;
	config->policy = DEF_POLICY;

	for (uint32_t i = 0U; i < MAX_WORKERS; ++i) {
		sd = &config->sds[i];
		stats = &config->thread_config[i].stats;
		memset(sd, 0, sizeof(*sd));

		for (uint32_t i = 0U; i < MAX_SEGS; ++i) {
			info = &sd->dma.infos[i];
			info->compl_param.transfer_id = ODP_DMA_TRANSFER_ID_INVALID;
			info->compl_param.event = ODP_EVENT_INVALID;
			info->compl_param.queue = ODP_QUEUE_INVALID;
			odp_ticketlock_init(&info->lock);
			sd->seg.src_pkt[i] = ODP_PACKET_INVALID;
			sd->seg.dst_pkt[i] = ODP_PACKET_INVALID;
		}

		sd->dma.handle = ODP_DMA_INVALID;
		sd->dma.pool = ODP_POOL_INVALID;
		sd->dma.compl_q = ODP_QUEUE_INVALID;
		sd->seg.src_shm = ODP_SHM_INVALID;
		sd->seg.dst_shm = ODP_SHM_INVALID;
		sd->grp = ODP_SCHED_GROUP_INVALID;
		stats->min_trs_tm = UINT64_MAX;
		stats->min_start_cc = UINT64_MAX;
		stats->min_wait_cc = UINT64_MAX;
		stats->min_trs_cc = UINT64_MAX;
	}
}

static void print_usage(void)
{
	printf("\n"
	       "DMA performance test. Load DMA subsystem from several workers.\n"
	       "\n"
	       "Usage: " PROG_NAME " [OPTIONS]\n"
	       "\n"
	       "  E.g. " PROG_NAME "\n"
	       "       " PROG_NAME " -s 10240\n"
	       "       " PROG_NAME " -t 0 -i 1 -o 1 -s 51200 -S 2 -f 64 -T 10\n"
	       "       " PROG_NAME " -t 1 -i 10 -o 10 -s 4096 -S 0 -m 1 -f 10 -c 4 -p 1\n"
	       "       " PROG_NAME " -t 2 -i 10 -o 1 -s 1024 -S 3 -f 10 -c 4 -p 1\n"
	       "\n"
	       "Optional OPTIONS:\n"
	       "\n"
	       "  -t, --trs_type      Transfer type for test data. %u by default.\n"
	       "                      Types:\n"
	       "                          0: synchronous DMA\n"
	       "                          1: asynchronous DMA\n"
	       "                          2: SW memory copy\n"
	       "  -i, --num_in_seg    Number of input segments to transfer. 0 means the maximum\n"
	       "                      count supported by the implementation. %u by default.\n"
	       "  -o, --num_out_seg   Number of output segments to transfer to. 0 means the\n"
	       "                      maximum count supported by the implementation. %u by\n"
	       "                      default.\n"
	       "  -s, --in_seg_len    Input segment length in bytes. 0 length means the maximum\n"
	       "                      segment length supported by the implementation. The actual\n"
	       "                      maximum might be limited by what type of data is\n"
	       "                      transferred (packet/memory). %u by default.\n"
	       "  -S, --in_seg_type   Input segment data type. Dense types can load the DMA\n"
	       "                      subsystem more heavily as transfer resources are\n"
	       "                      pre-configured. Sparse types might on the other hand\n"
	       "                      reflect application usage more precisely as transfer\n"
	       "                      resources are configured in runtime. %u by default.\n"
	       "                      Types:\n"
	       "                          0: dense packet\n"
	       "                          1: sparse packet\n"
	       "                          2: dense memory\n"
	       "                          3: sparse memory\n"
	       "  -m, --compl_mode    Completion mode for transfers. %u by default.\n"
	       "                      Modes:\n"
	       "                          0: poll\n"
	       "                          1: event\n"
	       "  -f, --max_in_flight Maximum transfers in-flight per session. 0 means the\n"
	       "                      maximum supported by the tester/implementation. %u by\n"
	       "                      default.\n"
	       "  -T, --time_sec      Time in seconds to run. 0 means infinite. %u by default.\n"
	       "  -c, --worker_count  Amount of workers. %u by default.\n"
	       "  -p, --policy        DMA session policy. %u by default.\n"
	       "                      Policies:\n"
	       "                          0: One session shared by workers\n"
	       "                          1: One session per worker\n"
	       "  -v, --verify        Verify transfers. Checks correctness of destination data\n"
	       "                      after successful transfers.\n"
	       "  -h, --help          This help.\n"
	       "\n", DEF_TRS_TYPE, DEF_SEG_CNT, DEF_SEG_CNT, DEF_LEN, DEF_SEG_TYPE, DEF_MODE,
	       DEF_INFLIGHT, DEF_TIME, DEF_WORKERS, DEF_POLICY);
}

static parse_result_t check_options(prog_config_t *config)
{
	int max_workers;
	odp_dma_capability_t dma_capa;
	uint32_t num_sessions, max_seg_len, max_trs, max_in, max_out, max_segs;
	odp_schedule_capability_t sched_capa;
	odp_pool_capability_t pool_capa;
	odp_shm_capability_t shm_capa;
	uint64_t shm_size = 0U;

	if (config->trs_type != SYNC_DMA && config->trs_type != ASYNC_DMA &&
	    config->trs_type != SW_COPY) {
		ODPH_ERR("Invalid transfer type: %u\n", config->trs_type);
		return PRS_NOK;
	}

	if (config->seg_type != DENSE_PACKET && config->seg_type != SPARSE_PACKET &&
	    config->seg_type != DENSE_MEMORY && config->seg_type != SPARSE_MEMORY) {
		ODPH_ERR("Invalid segment type: %u\n", config->seg_type);
		return PRS_NOK;
	}

	max_workers = ODPH_MIN(odp_thread_count_max() - 1, MAX_WORKERS);

	if (config->num_workers <= 0 || config->num_workers > max_workers) {
		ODPH_ERR("Invalid thread count: %d (min: 1, max: %d)\n", config->num_workers,
			 max_workers);
		return PRS_NOK;
	}

	if (config->policy != SINGLE && config->policy != MANY) {
		ODPH_ERR("Invalid DMA session policy: %u\n", config->policy);
		return PRS_NOK;
	}

	if (odp_dma_capability(&dma_capa) < 0) {
		ODPH_ERR("Error querying DMA capabilities\n");
		return PRS_NOK;
	}

	num_sessions = config->policy == SINGLE ? 1 : config->num_workers;

	if (num_sessions > dma_capa.max_sessions) {
		ODPH_ERR("Not enough DMA sessions supported: %u (max: %u)\n", num_sessions,
			 dma_capa.max_sessions);
		return PRS_NOT_SUP;
	}

	config->num_sessions = num_sessions;

	if (config->num_in_segs == 0U)
		config->num_in_segs = dma_capa.max_src_segs;

	if (config->num_out_segs == 0U)
		config->num_out_segs = dma_capa.max_dst_segs;

	if (config->num_in_segs > dma_capa.max_src_segs ||
	    config->num_out_segs > dma_capa.max_dst_segs ||
	    config->num_in_segs + config->num_out_segs > dma_capa.max_segs) {
		ODPH_ERR("Unsupported segment count configuration, in: %u, out: %u (max in: %u, "
			 "max out: %u, max tot: %u)\n", config->num_in_segs, config->num_out_segs,
			 dma_capa.max_src_segs, dma_capa.max_dst_segs, dma_capa.max_segs);
		return PRS_NOT_SUP;
	}

	if (config->src_seg_len == 0U)
		config->src_seg_len = dma_capa.max_seg_len;

	config->dst_seg_len = config->src_seg_len * config->num_in_segs /
			      config->num_out_segs + config->src_seg_len *
			      config->num_in_segs % config->num_out_segs;

	max_seg_len = ODPH_MAX(config->src_seg_len, config->dst_seg_len);

	if (max_seg_len > dma_capa.max_seg_len) {
		ODPH_ERR("Unsupported total DMA segment length: %u (max: %u)\n", max_seg_len,
			 dma_capa.max_seg_len);
		return PRS_NOT_SUP;
	}

	if (config->trs_type == ASYNC_DMA) {
		if (config->compl_mode != POLL && config->compl_mode != EVENT) {
			ODPH_ERR("Invalid completion mode: %u\n", config->compl_mode);
			return PRS_NOK;
		}

		if (config->compl_mode == POLL && (dma_capa.compl_mode_mask & ODP_DMA_COMPL_POLL)
		    == 0U) {
			ODPH_ERR("Unsupported DMA completion mode, poll\n");
			return PRS_NOT_SUP;
		}

		if (config->compl_mode == EVENT) {
			if (config->num_sessions > dma_capa.pool.max_pools) {
				ODPH_ERR("Unsupported amount of completion pools: %u (max: %u)\n",
					 config->num_sessions, dma_capa.pool.max_pools);
				return PRS_NOT_SUP;
			}

			if ((dma_capa.compl_mode_mask & ODP_DMA_COMPL_EVENT) == 0U) {
				ODPH_ERR("Unsupported DMA completion mode, event\n");
				return PRS_NOT_SUP;
			}

			if (dma_capa.queue_type_sched == 0) {
				ODPH_ERR("Unsupported DMA queueing type, scheduled\n");
				return PRS_NOT_SUP;
			}

			if (config->num_inflight > dma_capa.pool.max_num) {
				ODPH_ERR("Unsupported amount of completion events: %u (max: %u)\n",
					 config->num_inflight, dma_capa.pool.max_num);
				return PRS_NOT_SUP;
			}

			if (odp_schedule_capability(&sched_capa) < 0) {
				ODPH_ERR("Error querying scheduler capabilities\n");
				return PRS_NOK;
			}

			if (config->num_sessions > sched_capa.max_groups - 3U) {
				ODPH_ERR("Unsupported amount of scheduler groups: %u (max: %u)\n",
					 config->num_sessions, sched_capa.max_groups - 3U);
				return PRS_NOT_SUP;
			}
		}

		config->compl_mode_mask |= mode_map[config->compl_mode];
	}

	max_trs = ODPH_MIN(dma_capa.max_transfers, MAX_SEGS);

	if (config->num_inflight == 0U)
		config->num_inflight = max_trs;

	if (config->num_inflight > max_trs) {
		ODPH_ERR("Unsupported amount of in-flight DMA transfers: %u (max: %u)\n",
			 config->num_inflight, max_trs);
		return PRS_NOT_SUP;
	}

	max_in = config->num_in_segs * config->num_inflight;
	max_out = config->num_out_segs * config->num_inflight;
	max_segs = ODPH_MAX(max_in, max_out);

	if (max_segs > MAX_SEGS) {
		ODPH_ERR("Unsupported input/output * inflight segment combination: %u (max: %u)\n",
			 max_segs, MAX_SEGS);
		return PRS_NOT_SUP;
	}

	if (config->seg_type == DENSE_PACKET || config->seg_type == SPARSE_PACKET) {
		if (odp_pool_capability(&pool_capa) < 0) {
			ODPH_ERR("Error querying pool capabilities\n");
			return PRS_NOK;
		}

		if (pool_capa.pkt.max_pools < 2U) {
			ODPH_ERR("Unsupported amount of packet pools: 2 (max: %u)\n",
				 pool_capa.pkt.max_pools);
			return PRS_NOT_SUP;
		}

		if (pool_capa.pkt.max_len != 0U && max_seg_len > pool_capa.pkt.max_len) {
			ODPH_ERR("Unsupported packet size: %u (max: %u)\n", max_seg_len,
				 pool_capa.pkt.max_len);
			return PRS_NOT_SUP;
		}

		if (pool_capa.pkt.max_num != 0U &&
		    max_segs * num_sessions > pool_capa.pkt.max_num) {
			ODPH_ERR("Unsupported amount of packet pool elements: %u (max: %u)\n",
				 max_segs * num_sessions, pool_capa.pkt.max_num);
			return PRS_NOT_SUP;
		}

		config->src_cache_size = ODPH_MIN(ODPH_MAX(max_in, pool_capa.pkt.min_cache_size),
						  pool_capa.pkt.max_cache_size);
		config->dst_cache_size = ODPH_MIN(ODPH_MAX(max_out, pool_capa.pkt.min_cache_size),
						  pool_capa.pkt.max_cache_size);
	} else {
		/* If SHM implementation capabilities are very puny, program will have already
		 * failed when reserving memory for global program configuration. */
		if (odp_shm_capability(&shm_capa) < 0) {
			ODPH_ERR("Error querying SHM capabilities\n");
			return PRS_NOK;
		}

		/* One block for program configuration, one for source memory and one for
		 * destination memory. */
		if (shm_capa.max_blocks < 3U) {
			ODPH_ERR("Unsupported amount of SHM blocks: 3 (max: %u)\n",
				 shm_capa.max_blocks);
			return PRS_NOT_SUP;
		}

		shm_size = (uint64_t)config->dst_seg_len * config->num_out_segs *
			   config->num_inflight;

		if (shm_capa.max_size != 0U && shm_size > shm_capa.max_size) {
			ODPH_ERR("Unsupported total SHM block size: %" PRIu64 ""
				 " (max: %" PRIu64 ")\n", shm_size, shm_capa.max_size);
			return PRS_NOT_SUP;
		}

		if (config->seg_type == SPARSE_MEMORY && shm_size < MAX_MEMORY)
			shm_size = shm_capa.max_size != 0U ?
					ODPH_MIN(shm_capa.max_size, MAX_MEMORY) : MAX_MEMORY;

		config->shm_size = shm_size;
	}

	return PRS_OK;
}

static parse_result_t parse_options(int argc, char **argv, prog_config_t *config)
{
	int opt, long_index;
	static const struct option longopts[] = {
		{ "trs_type", required_argument, NULL, 't' },
		{ "num_in_seg", required_argument, NULL, 'i' },
		{ "num_out_seg", required_argument, NULL, 'o' },
		{ "in_seg_len", required_argument, NULL, 's' },
		{ "in_seg_type", required_argument, NULL, 'S' },
		{ "compl_mode", required_argument, NULL, 'm' },
		{ "max_in_flight", required_argument, NULL, 'f'},
		{ "time_sec", required_argument, NULL, 'T' },
		{ "worker_count", required_argument, NULL, 'c' },
		{ "policy", required_argument, NULL, 'p' },
		{ "verify", no_argument, NULL, 'v' },
		{ "help", no_argument, NULL, 'h' },
		{ NULL, 0, NULL, 0 }
	};
	static const char *shortopts = "t:i:o:s:S:m:f:T:c:p:vh";

	init_config(config);

	while (1) {
		opt = getopt_long(argc, argv, shortopts, longopts, &long_index);

		if (opt == -1)
			break;

		switch (opt) {
		case 't':
			config->trs_type = atoi(optarg);
			break;
		case 'i':
			config->num_in_segs = atoi(optarg);
			break;
		case 'o':
			config->num_out_segs = atoi(optarg);
			break;
		case 's':
			config->src_seg_len = atoi(optarg);
			break;
		case 'S':
			config->seg_type = atoi(optarg);
			break;
		case 'm':
			config->compl_mode = atoi(optarg);
			break;
		case 'f':
			config->num_inflight = atoi(optarg);
			break;
		case 'T':
			config->time_sec = atoi(optarg);
			break;
		case 'c':
			config->num_workers = atoi(optarg);
			break;
		case 'p':
			config->policy = atoi(optarg);
			break;
		case 'v':
			config->is_verify = true;
			break;
		case 'h':
			print_usage();
			return PRS_TERM;
		case '?':
		default:
			print_usage();
			return PRS_NOK;
		}
	}

	return check_options(config);
}

static parse_result_t setup_program(int argc, char **argv, prog_config_t *config)
{
	struct sigaction action = { .sa_handler = terminate };

	if (sigemptyset(&action.sa_mask) == -1 || sigaddset(&action.sa_mask, SIGINT) == -1 ||
	    sigaddset(&action.sa_mask, SIGTERM) == -1 ||
	    sigaddset(&action.sa_mask, SIGHUP) == -1 || sigaction(SIGINT, &action, NULL) == -1 ||
	    sigaction(SIGTERM, &action, NULL) == -1 || sigaction(SIGHUP, &action, NULL) == -1) {
		ODPH_ERR("Error installing signal handler\n");
		return PRS_NOK;
	}

	return parse_options(argc, argv, config);
}

static odp_pool_t get_src_packet_pool(void)
{
	odp_pool_param_t param;
	uint32_t num_pkts_per_worker = ODPH_MAX(prog_conf->num_inflight * prog_conf->num_in_segs,
						prog_conf->src_cache_size);

	if (prog_conf->src_pool != ODP_POOL_INVALID)
		return prog_conf->src_pool;

	odp_pool_param_init(&param);
	param.type = ODP_POOL_PACKET;
	param.pkt.num = num_pkts_per_worker * prog_conf->num_workers;
	param.pkt.len = prog_conf->src_seg_len;
	param.pkt.seg_len = prog_conf->src_seg_len;
	param.pkt.cache_size = prog_conf->src_cache_size;
	prog_conf->src_pool = odp_pool_create(PROG_NAME "_src_pkts", &param);

	return prog_conf->src_pool;
}

static odp_pool_t get_dst_packet_pool(void)
{
	odp_pool_param_t param;
	uint32_t num_pkts_per_worker = ODPH_MAX(prog_conf->num_inflight * prog_conf->num_out_segs,
						prog_conf->dst_cache_size);

	if (prog_conf->dst_pool != ODP_POOL_INVALID)
		return prog_conf->dst_pool;

	odp_pool_param_init(&param);
	param.type = ODP_POOL_PACKET;
	param.pkt.num = num_pkts_per_worker * prog_conf->num_workers;
	param.pkt.len = prog_conf->dst_seg_len;
	param.pkt.seg_len = prog_conf->dst_seg_len;
	param.pkt.cache_size = prog_conf->dst_cache_size;
	prog_conf->dst_pool = odp_pool_create(PROG_NAME "_dst_pkts", &param);

	return prog_conf->dst_pool;
}

static odp_bool_t configure_packets(sd_t *sd)
{
	sd->seg.src_pool = get_src_packet_pool();

	if (sd->seg.src_pool == ODP_POOL_INVALID) {
		ODPH_ERR("Error creating source packet pool\n");
		return false;
	}

	sd->seg.dst_pool = get_dst_packet_pool();

	if (sd->seg.dst_pool == ODP_POOL_INVALID) {
		ODPH_ERR("Error creating destination packet pool\n");
		return false;
	}

	return true;
}

static odp_bool_t allocate_packets(sd_t *sd)
{
	for (uint32_t i = 0U; i < sd->dma.num_inflight * sd->dma.num_in_segs; ++i) {
		sd->seg.src_pkt[i] = odp_packet_alloc(sd->seg.src_pool, sd->dma.src_seg_len);

		if (sd->seg.src_pkt[i] == ODP_PACKET_INVALID) {
			ODPH_ERR("Error allocating source segment packets\n");
			return false;
		}
	}

	for (uint32_t i = 0U; i < sd->dma.num_inflight * sd->dma.num_out_segs; ++i) {
		sd->seg.dst_pkt[i] = odp_packet_alloc(sd->seg.dst_pool, sd->dma.dst_seg_len);

		if (sd->seg.dst_pkt[i] == ODP_PACKET_INVALID) {
			ODPH_ERR("Error allocating destination segment packets\n");
			return false;
		}
	}

	return true;
}

static odp_bool_t setup_packet_segments(sd_t *sd)
{
	return configure_packets(sd) &&
	       (sd->seg.seg_type == DENSE_PACKET ? allocate_packets(sd) : true);
}

static inline void fill_data(uint8_t *data, uint32_t len)
{
	memset(data, DATA, len);
}

static void configure_packet_transfer(sd_t *sd)
{
	odp_dma_seg_t *start_src_seg, *start_dst_seg, *seg;
	uint32_t k = 0U, z = 0U, len;
	odp_packet_t pkt;
	odp_dma_transfer_param_t *param;

	for (uint32_t i = 0U; i < sd->dma.num_inflight; ++i) {
		start_src_seg = &sd->dma.src_seg[k];
		start_dst_seg = &sd->dma.dst_seg[z];

		for (uint32_t j = 0U; j < sd->dma.num_in_segs; ++j, ++k) {
			pkt = sd->seg.src_pkt[k];
			seg = &start_src_seg[j];
			seg->packet = pkt;
			seg->offset = 0U;
			seg->len = sd->dma.src_seg_len;

			if (seg->packet != ODP_PACKET_INVALID)
				fill_data(odp_packet_data(seg->packet), seg->len);
		}

		len = sd->dma.num_in_segs * sd->dma.src_seg_len;

		for (uint32_t j = 0U; j < sd->dma.num_out_segs; ++j, ++z) {
			pkt = sd->seg.dst_pkt[z];
			seg = &start_dst_seg[j];
			seg->packet = pkt;
			seg->offset = 0U;
			seg->len = ODPH_MIN(len, sd->dma.dst_seg_len);
			len -= sd->dma.dst_seg_len;
		}

		param = &sd->dma.infos[i].trs_param;
		odp_dma_transfer_param_init(param);
		param->src_format = ODP_DMA_FORMAT_PACKET;
		param->dst_format = ODP_DMA_FORMAT_PACKET;
		param->num_src = sd->dma.num_in_segs;
		param->num_dst = sd->dma.num_out_segs;
		param->src_seg = start_src_seg;
		param->dst_seg = start_dst_seg;
	}
}

static void free_packets(const sd_t *sd)
{
	for (uint32_t i = 0U; i < sd->dma.num_inflight * sd->dma.num_in_segs; ++i) {
		if (sd->seg.src_pkt[i] != ODP_PACKET_INVALID)
			odp_packet_free(sd->seg.src_pkt[i]);
	}

	for (uint32_t i = 0U; i < sd->dma.num_inflight * sd->dma.num_out_segs; ++i) {
		if (sd->seg.dst_pkt[i] != ODP_PACKET_INVALID)
			odp_packet_free(sd->seg.dst_pkt[i]);
	}
}

static odp_bool_t allocate_memory(sd_t *sd)
{
	sd->seg.src_shm = odp_shm_reserve(PROG_NAME "_src_shm", sd->seg.shm_size,
					  ODP_CACHE_LINE_SIZE, 0U);
	sd->seg.dst_shm = odp_shm_reserve(PROG_NAME "_dst_shm", sd->seg.shm_size,
					  ODP_CACHE_LINE_SIZE, 0U);

	if (sd->seg.src_shm == ODP_SHM_INVALID || sd->seg.dst_shm == ODP_SHM_INVALID) {
		ODPH_ERR("Error allocating SHM block\n");
		return false;
	}

	sd->seg.src = odp_shm_addr(sd->seg.src_shm);
	sd->seg.dst = odp_shm_addr(sd->seg.dst_shm);

	if (sd->seg.src == NULL || sd->seg.dst == NULL) {
		ODPH_ERR("Error resolving SHM block address\n");
		return false;
	}

	sd->seg.src_high = (uint8_t *)sd->seg.src + sd->seg.shm_size - sd->dma.src_seg_len;
	sd->seg.dst_high = (uint8_t *)sd->seg.dst + sd->seg.shm_size - sd->dma.dst_seg_len;
	sd->seg.cur_src = sd->seg.src;
	sd->seg.cur_dst = sd->seg.dst;

	return true;
}

static odp_bool_t setup_memory_segments(sd_t *sd)
{
	return allocate_memory(sd);
}

static void configure_address_transfer(sd_t *sd)
{
	odp_dma_seg_t *start_src_seg, *start_dst_seg, *seg;
	uint32_t k = 0U, z = 0U, len;
	odp_dma_transfer_param_t *param;

	for (uint32_t i = 0U; i < sd->dma.num_inflight; ++i) {
		start_src_seg = &sd->dma.src_seg[k];
		start_dst_seg = &sd->dma.dst_seg[z];

		for (uint32_t j = 0U; j < sd->dma.num_in_segs; ++j, ++k) {
			seg = &start_src_seg[j];
			seg->addr = sd->seg.seg_type == SPARSE_MEMORY ?
					NULL : (uint8_t *)sd->seg.src + k * sd->dma.src_seg_len;
			seg->len = sd->dma.src_seg_len;

			if (seg->addr != NULL)
				fill_data(seg->addr, seg->len);
		}

		len = sd->dma.num_in_segs * sd->dma.src_seg_len;

		for (uint32_t j = 0U; j < sd->dma.num_out_segs; ++j, ++z) {
			seg = &start_dst_seg[j];
			seg->addr = sd->seg.seg_type == SPARSE_MEMORY ?
					NULL : (uint8_t *)sd->seg.dst + z * sd->dma.dst_seg_len;
			seg->len = ODPH_MIN(len, sd->dma.dst_seg_len);
			len -= sd->dma.dst_seg_len;
		}

		param = &sd->dma.infos[i].trs_param;
		odp_dma_transfer_param_init(param);
		param->src_format = ODP_DMA_FORMAT_ADDR;
		param->dst_format = ODP_DMA_FORMAT_ADDR;
		param->num_src = sd->dma.num_in_segs;
		param->num_dst = sd->dma.num_out_segs;
		param->src_seg = start_src_seg;
		param->dst_seg = start_dst_seg;
	}
}

static void free_memory(const sd_t *sd)
{
	if (sd->seg.src_shm != ODP_SHM_INVALID)
		(void)odp_shm_free(sd->seg.src_shm);

	if (sd->seg.dst_shm != ODP_SHM_INVALID)
		(void)odp_shm_free(sd->seg.dst_shm);
}

static void run_transfer(odp_dma_t handle, trs_info_t *info, stats_t *stats, ver_fn_t ver_fn)
{
	uint64_t start_tm, end_tm, start_cc, end_cc, trs_tm, trs_cc, start_cc_diff;
	odp_dma_result_t res;
	int ret;

	start_tm = odp_time_local_strict_ns();
	start_cc = odp_cpu_cycles();
	ret = odp_dma_transfer(handle, &info->trs_param, &res);
	end_cc = odp_cpu_cycles();
	end_tm = odp_time_local_strict_ns();

	if (odp_unlikely(ret <= 0)) {
		++stats->start_errs;
	} else {
		trs_tm = end_tm - start_tm;
		stats->max_trs_tm = ODPH_MAX(trs_tm, stats->max_trs_tm);
		stats->min_trs_tm = ODPH_MIN(trs_tm, stats->min_trs_tm);
		stats->trs_tm += trs_tm;
		trs_cc = odp_cpu_cycles_diff(end_cc, start_cc);
		stats->max_trs_cc = ODPH_MAX(trs_cc, stats->max_trs_cc);
		stats->min_trs_cc = ODPH_MIN(trs_cc, stats->min_trs_cc);
		stats->trs_cc += trs_cc;
		++stats->trs_cnt;
		start_cc_diff = odp_cpu_cycles_diff(end_cc, start_cc);
		stats->max_start_cc = ODPH_MAX(start_cc_diff, stats->max_start_cc);
		stats->min_start_cc = ODPH_MIN(start_cc_diff, stats->min_start_cc);
		stats->start_cc += start_cc_diff;
		++stats->start_cnt;

		if (odp_unlikely(!res.success)) {
			++stats->transfer_errs;
		} else {
			++stats->completed;

			if (ver_fn != NULL)
				ver_fn(info, stats);
		}
	}
}

static void run_transfers_mt_unsafe(sd_t *sd, stats_t *stats)
{
	const uint32_t count = sd->dma.num_inflight;
	odp_dma_t handle = sd->dma.handle;
	trs_info_t *infos = sd->dma.infos, *info;

	for (uint32_t i = 0U; i < count; ++i) {
		info = &infos[i];

		if (sd->prep_trs_fn != NULL)
			sd->prep_trs_fn(sd, info);

		run_transfer(handle, info, stats, sd->ver_fn);
	}
}

static void run_transfers_mt_safe(sd_t *sd, stats_t *stats)
{
	const uint32_t count = sd->dma.num_inflight;
	odp_dma_t handle = sd->dma.handle;
	trs_info_t *infos = sd->dma.infos, *info;

	for (uint32_t i = 0U; i < count; ++i) {
		info = &infos[i];

		if (odp_ticketlock_trylock(&info->lock)) {
			if (sd->prep_trs_fn != NULL)
				sd->prep_trs_fn(sd, info);

			run_transfer(handle, info, stats, sd->ver_fn);
			odp_ticketlock_unlock(&info->lock);
		}
	}
}

static odp_bool_t configure_poll_compl(sd_t *sd)
{
	odp_dma_compl_param_t *param;

	for (uint32_t i = 0U; i < sd->dma.num_inflight; ++i) {
		param = &sd->dma.infos[i].compl_param;

		odp_dma_compl_param_init(param);
		param->compl_mode = mode_map[sd->dma.compl_mode];
		param->transfer_id = odp_dma_transfer_id_alloc(sd->dma.handle);

		if (param->transfer_id == ODP_DMA_TRANSFER_ID_INVALID) {
			ODPH_ERR("Error allocating transfer ID\n");
			return false;
		}
	}

	return true;
}

static void poll_transfer(sd_t *sd, trs_info_t *info, stats_t *stats)
{
	uint64_t start_cc, end_cc, trs_tm, trs_cc, wait_cc, start_tm, start_cc_diff;
	odp_dma_t handle = sd->dma.handle;
	odp_dma_result_t res;
	int ret;

	if (info->is_running) {
		start_cc = odp_cpu_cycles();
		ret = odp_dma_transfer_done(handle, info->compl_param.transfer_id, &res);
		end_cc = odp_cpu_cycles();

		if (odp_unlikely(ret < 0)) {
			++stats->poll_errs;
			return;
		}

		++info->trs_poll_cnt;
		wait_cc = odp_cpu_cycles_diff(end_cc, start_cc);
		stats->max_wait_cc = ODPH_MAX(wait_cc, stats->max_wait_cc);
		stats->min_wait_cc = ODPH_MIN(wait_cc, stats->min_wait_cc);
		stats->wait_cc += wait_cc;
		++stats->wait_cnt;

		if (ret == 0)
			return;

		trs_tm = odp_time_global_strict_ns() - info->trs_start_tm;
		stats->max_trs_tm = ODPH_MAX(trs_tm, stats->max_trs_tm);
		stats->min_trs_tm = ODPH_MIN(trs_tm, stats->min_trs_tm);
		stats->trs_tm += trs_tm;
		trs_cc = odp_cpu_cycles_diff(odp_cpu_cycles(), info->trs_start_cc);
		stats->max_trs_cc = ODPH_MAX(trs_cc, stats->max_trs_cc);
		stats->min_trs_cc = ODPH_MIN(trs_cc, stats->min_trs_cc);
		stats->trs_cc += trs_cc;
		stats->trs_poll_cnt += info->trs_poll_cnt;
		++stats->trs_cnt;

		if (odp_unlikely(!res.success)) {
			++stats->transfer_errs;
		} else {
			++stats->completed;

			if (sd->ver_fn != NULL)
				sd->ver_fn(info, stats);
		}

		info->is_running = false;
	} else {
		if (sd->prep_trs_fn != NULL)
			sd->prep_trs_fn(sd, info);

		start_tm = odp_time_global_strict_ns();
		start_cc = odp_cpu_cycles();
		ret = odp_dma_transfer_start(handle, &info->trs_param, &info->compl_param);
		end_cc = odp_cpu_cycles();

		if (odp_unlikely(ret <= 0)) {
			++stats->start_errs;
		} else {
			info->trs_start_tm = start_tm;
			info->trs_start_cc = start_cc;
			info->trs_poll_cnt = 0U;
			start_cc_diff = odp_cpu_cycles_diff(end_cc, start_cc);
			stats->max_start_cc = ODPH_MAX(start_cc_diff, stats->max_start_cc);
			stats->min_start_cc = ODPH_MIN(start_cc_diff, stats->min_start_cc);
			stats->start_cc += start_cc_diff;
			++stats->start_cnt;
			info->is_running = true;
		}
	}
}

static void poll_transfers_mt_unsafe(sd_t *sd, stats_t *stats)
{
	const uint32_t count = sd->dma.num_inflight;
	trs_info_t *infos = sd->dma.infos;

	for (uint32_t i = 0U; i < count; ++i)
		poll_transfer(sd, &infos[i], stats);
}

static void poll_transfers_mt_safe(sd_t *sd, stats_t *stats)
{
	const uint32_t count = sd->dma.num_inflight;
	trs_info_t *infos = sd->dma.infos, *info;

	for (uint32_t i = 0U; i < count; ++i) {
		info = &infos[i];

		if (odp_ticketlock_trylock(&info->lock)) {
			poll_transfer(sd, info, stats);
			odp_ticketlock_unlock(&info->lock);
		}
	}
}

static void drain_poll_transfers(sd_t *sd)
{
	const uint32_t count = sd->dma.num_inflight;
	trs_info_t *infos = sd->dma.infos, *info;
	odp_dma_t handle = sd->dma.handle;
	int rc;

	for (uint32_t i = 0U; i < count; ++i) {
		info = &infos[i];

		if (info->is_running) {
			do {
				rc = odp_dma_transfer_done(handle, info->compl_param.transfer_id,
							   NULL);
			} while (rc == 0);
		}
	}
}

static odp_bool_t configure_event_compl_session(sd_t *sd)
{
	odp_thrmask_t zero;
	odp_dma_pool_param_t pool_param;
	odp_queue_param_t queue_param;

	odp_thrmask_zero(&zero);
	sd->grp = odp_schedule_group_create(PROG_NAME "_scd_grp", &zero);

	if (sd->grp == ODP_SCHED_GROUP_INVALID) {
		ODPH_ERR("Error creating scheduler group for DMA session\n");
		return false;
	}

	odp_dma_pool_param_init(&pool_param);
	pool_param.num = sd->dma.num_inflight;
	sd->dma.pool = odp_dma_pool_create(PROG_NAME "_dma_evs", &pool_param);

	if (sd->dma.pool == ODP_POOL_INVALID) {
		ODPH_ERR("Error creating DMA event completion pool\n");
		return false;
	}

	odp_queue_param_init(&queue_param);
	queue_param.type = ODP_QUEUE_TYPE_SCHED;
	queue_param.sched.sync = ODP_SCHED_SYNC_PARALLEL;
	queue_param.sched.prio = odp_schedule_default_prio();
	queue_param.sched.group = sd->grp;
	sd->dma.compl_q = odp_queue_create(PROG_NAME, &queue_param);

	if (sd->dma.compl_q == ODP_QUEUE_INVALID) {
		ODPH_ERR("Error creating DMA completion queue\n");
		return false;
	}

	return true;
}

static odp_bool_t configure_event_compl(sd_t *sd)
{
	odp_dma_compl_param_t *param;
	odp_dma_compl_t c_ev;

	for (uint32_t i = 0U; i < sd->dma.num_inflight; ++i) {
		param = &sd->dma.infos[i].compl_param;

		odp_dma_compl_param_init(param);
		param->compl_mode = mode_map[sd->dma.compl_mode];
		c_ev = odp_dma_compl_alloc(sd->dma.pool);

		if (c_ev == ODP_DMA_COMPL_INVALID) {
			ODPH_ERR("Error allocating completion event\n");
			return false;
		}

		param->event = odp_dma_compl_to_event(c_ev);
		param->queue = sd->dma.compl_q;
		param->user_ptr = &sd->dma.infos[i];
	}

	return true;
}

static odp_bool_t start_initial_transfers(sd_t *sd)
{
	uint64_t start_tm, start_cc;
	trs_info_t *info;
	int ret;

	for (uint32_t i = 0U; i < sd->dma.num_inflight; ++i) {
		info = &sd->dma.infos[i];

		if (sd->prep_trs_fn != NULL)
			sd->prep_trs_fn(sd, info);

		start_tm = odp_time_global_strict_ns();
		start_cc = odp_cpu_cycles();
		ret = odp_dma_transfer_start(sd->dma.handle, &info->trs_param, &info->compl_param);

		if (ret <= 0) {
			ODPH_ERR("Error starting DMA transfer\n");
			return false;
		}

		info->trs_start_tm = start_tm;
		info->trs_start_cc = start_cc;
	}

	return true;
}

static void wait_compl_event(sd_t *sd, stats_t *stats)
{
	uint64_t start_cc, end_cc, wait_cc, trs_tm, trs_cc, start_tm, start_cc_diff;
	odp_event_t ev;
	odp_dma_result_t res;
	trs_info_t *info;
	int ret;

	start_cc = odp_cpu_cycles();
	ev = odp_schedule(NULL, odp_schedule_wait_time(ODP_TIME_SEC_IN_NS));
	end_cc = odp_cpu_cycles();

	if (odp_unlikely(ev == ODP_EVENT_INVALID)) {
		++stats->scheduler_timeouts;
		return;
	}

	odp_dma_compl_result(odp_dma_compl_from_event(ev), &res);
	info = res.user_ptr;
	trs_tm = odp_time_global_strict_ns() - info->trs_start_tm;
	stats->max_trs_tm = ODPH_MAX(trs_tm, stats->max_trs_tm);
	stats->min_trs_tm = ODPH_MIN(trs_tm, stats->min_trs_tm);
	stats->trs_tm += trs_tm;
	trs_cc = odp_cpu_cycles_diff(odp_cpu_cycles(), info->trs_start_cc);
	stats->max_trs_cc = ODPH_MAX(trs_cc, stats->max_trs_cc);
	stats->min_trs_cc = ODPH_MIN(trs_cc, stats->min_trs_cc);
	stats->trs_cc += trs_cc;
	++stats->trs_cnt;
	wait_cc = odp_cpu_cycles_diff(end_cc, start_cc);
	stats->max_wait_cc = ODPH_MAX(wait_cc, stats->max_wait_cc);
	stats->min_wait_cc = ODPH_MIN(wait_cc, stats->min_wait_cc);
	stats->wait_cc += wait_cc;
	++stats->wait_cnt;

	if (odp_unlikely(!res.success)) {
		++stats->transfer_errs;
	} else {
		++stats->completed;

		if (sd->ver_fn != NULL)
			sd->ver_fn(info, stats);
	}

	if (sd->prep_trs_fn != NULL)
		sd->prep_trs_fn(sd, info);

	start_tm = odp_time_global_strict_ns();
	start_cc = odp_cpu_cycles();
	ret = odp_dma_transfer_start(sd->dma.handle, &info->trs_param, &info->compl_param);
	end_cc = odp_cpu_cycles();

	if (odp_unlikely(ret <= 0)) {
		++stats->start_errs;
	} else {
		info->trs_start_tm = start_tm;
		info->trs_start_cc = start_cc;
		start_cc_diff = odp_cpu_cycles_diff(end_cc, start_cc);
		stats->max_start_cc = ODPH_MAX(start_cc_diff, stats->max_start_cc);
		stats->min_start_cc = ODPH_MIN(start_cc_diff, stats->min_start_cc);
		stats->start_cc += start_cc_diff;
		++stats->start_cnt;
	}
}

static void drain_compl_events(ODP_UNUSED sd_t *sd)
{
	odp_event_t ev;

	while (true) {
		ev = odp_schedule(NULL, odp_schedule_wait_time(ODP_TIME_SEC_IN_NS));

		if (ev == ODP_EVENT_INVALID)
			break;
	}
}

static void run_memcpy(trs_info_t *info, stats_t *stats, ver_fn_t ver_fn)
{
	uint64_t start_tm, end_tm, start_cc, end_cc, trs_tm, trs_cc, start_cc_diff;
	const odp_dma_transfer_param_t *param = &info->trs_param;
	uint32_t tot_len, src_len, dst_len, min_len, len, i = 0U, j = 0U, src_off = 0U,
	dst_off = 0U, src_rem, dst_rem;
	const odp_bool_t is_addr = param->src_format == ODP_DMA_FORMAT_ADDR;
	uint8_t *src_data, *dst_data;

	/* Test data is configured so that total source and total destination sizes always match,
	 * all source and all destination segments have the same size and in case of packets,
	 * there's always just a single segment. */
	tot_len = param->num_src * param->src_seg->len;
	src_len = param->src_seg->len;
	dst_len = param->dst_seg->len;
	min_len = ODPH_MIN(src_len, dst_len);
	len = min_len;
	start_tm = odp_time_local_strict_ns();
	start_cc = odp_cpu_cycles();

	while (tot_len > 0U) {
		if (is_addr) {
			src_data = param->src_seg[i].addr;
			dst_data = param->dst_seg[j].addr;
		} else {
			src_data = odp_packet_data(param->src_seg[i].packet);
			dst_data = odp_packet_data(param->dst_seg[j].packet);
		}

		memcpy(dst_data + dst_off, src_data + src_off, len);
		dst_off += len;
		src_off += len;
		src_rem = src_len - src_off;
		dst_rem = dst_len - dst_off;
		tot_len -= len;
		len = ODPH_MIN(ODPH_MAX(src_rem, dst_rem), min_len);

		if (dst_rem > 0U) {
			++i;
			src_off = 0U;
		} else {
			++j;
			dst_off = 0U;
		}
	}

	end_cc = odp_cpu_cycles();
	end_tm = odp_time_local_strict_ns();
	trs_tm = end_tm - start_tm;
	stats->max_trs_tm = ODPH_MAX(trs_tm, stats->max_trs_tm);
	stats->min_trs_tm = ODPH_MIN(trs_tm, stats->min_trs_tm);
	stats->trs_tm += trs_tm;
	trs_cc = odp_cpu_cycles_diff(end_cc, start_cc);
	stats->max_trs_cc = ODPH_MAX(trs_cc, stats->max_trs_cc);
	stats->min_trs_cc = ODPH_MIN(trs_cc, stats->min_trs_cc);
	stats->trs_cc += trs_cc;
	++stats->trs_cnt;
	start_cc_diff = odp_cpu_cycles_diff(end_cc, start_cc);
	stats->max_start_cc = ODPH_MAX(start_cc_diff, stats->max_start_cc);
	stats->min_start_cc = ODPH_MIN(start_cc_diff, stats->min_start_cc);
	stats->start_cc += start_cc_diff;
	++stats->start_cnt;
	++stats->completed;

	if (ver_fn != NULL)
		ver_fn(info, stats);
}

static void run_memcpy_mt_unsafe(sd_t *sd, stats_t *stats)
{
	const uint32_t count = sd->dma.num_inflight;
	trs_info_t *infos = sd->dma.infos, *info;

	for (uint32_t i = 0U; i < count; ++i) {
		info = &infos[i];

		if (sd->prep_trs_fn != NULL)
			sd->prep_trs_fn(sd, info);

		run_memcpy(info, stats, sd->ver_fn);
	}
}

static void run_memcpy_mt_safe(sd_t *sd, stats_t *stats)
{
	const uint32_t count = sd->dma.num_inflight;
	trs_info_t *infos = sd->dma.infos, *info;

	for (uint32_t i = 0U; i < count; ++i) {
		info = &infos[i];

		if (odp_ticketlock_trylock(&info->lock)) {
			if (sd->prep_trs_fn != NULL)
				sd->prep_trs_fn(sd, info);

			run_memcpy(info, stats, sd->ver_fn);
			odp_ticketlock_unlock(&info->lock);
		}
	}
}

static void setup_api(prog_config_t *config)
{
	if (config->seg_type == DENSE_PACKET || config->seg_type == SPARSE_PACKET) {
		config->api.setup_fn = setup_packet_segments;
		config->api.trs_fn = configure_packet_transfer;
		config->api.free_fn = free_packets;
	} else {
		config->api.setup_fn = setup_memory_segments;
		config->api.trs_fn = configure_address_transfer;
		config->api.free_fn = free_memory;
	}

	if (config->trs_type == SYNC_DMA) {
		config->api.session_cfg_fn = NULL;
		config->api.compl_fn = NULL;
		config->api.bootstrap_fn = NULL;
		config->api.wait_fn = config->num_workers == 1 || config->policy == MANY ?
					run_transfers_mt_unsafe : run_transfers_mt_safe;
		config->api.drain_fn = NULL;
	} else if (config->trs_type == ASYNC_DMA) {
		if (config->compl_mode == POLL) {
			config->api.session_cfg_fn = NULL;
			config->api.compl_fn = configure_poll_compl;
			config->api.bootstrap_fn = NULL;
			config->api.wait_fn = config->num_workers == 1 || config->policy == MANY ?
						poll_transfers_mt_unsafe : poll_transfers_mt_safe;
			config->api.drain_fn = drain_poll_transfers;
		} else {
			config->api.session_cfg_fn = configure_event_compl_session;
			config->api.compl_fn = configure_event_compl;
			config->api.bootstrap_fn = start_initial_transfers;
			config->api.wait_fn = wait_compl_event;
			config->api.drain_fn = drain_compl_events;
		}
	} else {
		config->api.session_cfg_fn = NULL;
		config->api.compl_fn = NULL;
		config->api.bootstrap_fn = NULL;
		config->api.wait_fn = config->num_workers == 1 || config->policy == MANY ?
					run_memcpy_mt_unsafe : run_memcpy_mt_safe;
		config->api.drain_fn = NULL;
	}
}

static void prepare_packet_transfer(sd_t *sd, trs_info_t *info)
{
	odp_dma_transfer_param_t *param = &info->trs_param;
	odp_dma_seg_t *seg;

	for (uint32_t i = 0U; i < param->num_src; ++i) {
		seg = &param->src_seg[i];

		if (odp_likely(seg->packet != ODP_PACKET_INVALID))
			odp_packet_free(seg->packet);

		seg->packet = odp_packet_alloc(sd->seg.src_pool, seg->len);

		if (odp_unlikely(seg->packet == ODP_PACKET_INVALID))
			/* There should always be enough packets. */
			ODPH_ABORT("Failed to allocate packet, aborting\n");

		fill_data(odp_packet_data(seg->packet), seg->len);
	}

	for (uint32_t i = 0U; i < param->num_dst; ++i) {
		seg = &param->dst_seg[i];

		if (odp_likely(seg->packet != ODP_PACKET_INVALID))
			odp_packet_free(seg->packet);

		seg->packet = odp_packet_alloc(sd->seg.dst_pool, seg->len);

		if (odp_unlikely(seg->packet == ODP_PACKET_INVALID))
			/* There should always be enough packets. */
			ODPH_ABORT("Failed to allocate packet, aborting\n");
	}
}

static void prepare_address_transfer(sd_t *sd, trs_info_t *info)
{
	odp_dma_transfer_param_t *param = &info->trs_param;
	uint8_t *addr = sd->seg.cur_src;
	odp_dma_seg_t *seg;

	for (uint32_t i = 0U; i < param->num_src; ++i) {
		seg = &param->src_seg[i];

		if (odp_unlikely(addr > (uint8_t *)sd->seg.src_high))
			addr = sd->seg.src;

		seg->addr = addr;
		addr += sd->dma.src_seg_len;
		fill_data(seg->addr, seg->len);
	}

	sd->seg.cur_src = addr + ODP_CACHE_LINE_SIZE;
	addr = sd->seg.cur_dst;

	for (uint32_t i = 0U; i < param->num_dst; ++i) {
		if (odp_unlikely(addr > (uint8_t *)sd->seg.dst_high))
			addr = sd->seg.dst;

		param->dst_seg[i].addr = addr;
		addr += sd->dma.dst_seg_len;
	}

	sd->seg.cur_dst = addr + ODP_CACHE_LINE_SIZE;
}

static void verify_transfer(trs_info_t *info, stats_t *stats)
{
	odp_dma_transfer_param_t *param = &info->trs_param;
	odp_dma_seg_t *seg;
	const odp_bool_t is_addr = param->dst_format == ODP_DMA_FORMAT_ADDR;
	uint8_t *data;

	for (uint32_t i = 0U; i < param->num_dst; ++i) {
		seg = &param->dst_seg[i];
		data = is_addr ? seg->addr : odp_packet_data(seg->packet);

		for (uint32_t j = 0U; j < seg->len; ++j)
			if (odp_unlikely(data[j] != DATA)) {
				++stats->data_errs;
				return;
			}
	}
}

static odp_bool_t setup_session_descriptors(prog_config_t *config)
{
	sd_t *sd;
	const odp_dma_param_t dma_params = {
		.direction = ODP_DMA_MAIN_TO_MAIN,
		.type = ODP_DMA_TYPE_COPY,
		.compl_mode_mask = config->compl_mode_mask,
		.mt_mode = config->num_workers == 1 || config->policy == MANY ?
				ODP_DMA_MT_SERIAL : ODP_DMA_MT_SAFE,
		.order = ODP_DMA_ORDER_NONE };

	for (uint32_t i = 0U; i < config->num_sessions; ++i) {
		char name[ODP_DMA_NAME_LEN];

		sd = &config->sds[i];
		sd->dma.num_in_segs = config->num_in_segs;
		sd->dma.num_out_segs = config->num_out_segs;
		sd->dma.src_seg_len = config->src_seg_len;
		sd->dma.dst_seg_len = config->dst_seg_len;
		sd->dma.num_inflight = config->num_inflight;
		sd->dma.trs_type = config->trs_type;
		sd->dma.compl_mode = config->compl_mode;
		snprintf(name, sizeof(name), PROG_NAME "_dma_%u", i);
		sd->dma.handle = odp_dma_create(name, &dma_params);

		if (sd->dma.handle == ODP_DMA_INVALID) {
			ODPH_ERR("Error creating DMA session\n");
			return false;
		}

		if (config->api.session_cfg_fn != NULL && !config->api.session_cfg_fn(sd))
			return false;

		sd->seg.shm_size = config->shm_size;
		sd->seg.seg_type = config->seg_type;
		sd->prep_trs_fn = config->seg_type == SPARSE_PACKET ? prepare_packet_transfer :
					config->seg_type == SPARSE_MEMORY ?
						prepare_address_transfer : NULL;
		sd->ver_fn = config->is_verify ? verify_transfer : NULL;
	}

	return true;
}

static odp_bool_t setup_data(prog_config_t *config)
{
	sd_t *sd;

	for (uint32_t i = 0U; i < config->num_sessions; ++i) {
		sd = &config->sds[i];

		if (!config->api.setup_fn(sd))
			return false;

		config->api.trs_fn(sd);

		if (config->api.compl_fn != NULL && !config->api.compl_fn(sd))
			return false;
	}

	return true;
}

static int transfer(void *args)
{
	thread_config_t *thr_config = args;
	prog_config_t *prog_config = thr_config->prog_config;
	sd_t *sd = thr_config->sd;
	stats_t *stats = &thr_config->stats;
	test_api_t *api = &prog_conf->api;
	odp_thrmask_t mask;
	uint64_t start_tm, end_tm;

	odp_barrier_wait(&prog_config->init_barrier);

	if (sd->grp != ODP_SCHED_GROUP_INVALID) {
		odp_thrmask_zero(&mask);
		odp_thrmask_set(&mask, odp_thread_id());

		if (odp_schedule_group_join(sd->grp, &mask) < 0) {
			ODPH_ERR("Error joining scheduler group\n");
			goto out;
		}
	}

	start_tm = odp_time_local_strict_ns();

	while (odp_atomic_load_u32(&prog_config->is_running))
		api->wait_fn(sd, stats);

	end_tm = odp_time_local_strict_ns();
	thr_config->stats.tot_tm = end_tm - start_tm;

	if (api->drain_fn != NULL)
		api->drain_fn(sd);

out:
	odp_barrier_wait(&prog_config->term_barrier);

	return 0;
}

static odp_bool_t setup_workers(prog_config_t *config)
{
	odp_cpumask_t cpumask;
	int num_workers;
	odph_thread_common_param_t thr_common;
	odph_thread_param_t thr_params[config->num_workers], *thr_param;
	thread_config_t *thr_config;
	sd_t *sd;

	/* Barrier init count for control and worker. */
	odp_barrier_init(&config->init_barrier, config->num_workers + 1);
	odp_barrier_init(&config->term_barrier, config->num_workers);
	num_workers = odp_cpumask_default_worker(&cpumask, config->num_workers);
	odph_thread_common_param_init(&thr_common);
	thr_common.instance = config->odp_instance;
	thr_common.cpumask = &cpumask;

	for (int i = 0; i < config->num_workers; ++i) {
		thr_param = &thr_params[i];
		thr_config = &config->thread_config[i];
		sd = config->policy == SINGLE ? &config->sds[0U] : &config->sds[i];

		odph_thread_param_init(thr_param);
		thr_param->start = transfer;
		thr_param->thr_type = ODP_THREAD_WORKER;
		thr_config->prog_config = config;
		thr_config->sd = sd;
		thr_param->arg = thr_config;
	}

	num_workers = odph_thread_create(config->threads, &thr_common, thr_params, num_workers);

	if (num_workers != config->num_workers) {
		ODPH_ERR("Error configuring worker threads\n");
		return false;
	}

	for (uint32_t i = 0U; i < config->num_sessions; ++i) {
		if (config->api.bootstrap_fn != NULL && !config->api.bootstrap_fn(&config->sds[i]))
			return false;
	}

	odp_barrier_wait(&config->init_barrier);

	return true;
}

static odp_bool_t setup_test(prog_config_t *config)
{
	setup_api(config);

	return setup_session_descriptors(config) && setup_data(config) && setup_workers(config);
}

static void stop_test(prog_config_t *config)
{
	(void)odph_thread_join(config->threads, config->num_workers);
}

static void teardown_data(const sd_t *sd, void (*free_fn)(const sd_t *sd))
{
	const odp_dma_compl_param_t *compl_param;

	for (uint32_t i = 0U; i < MAX_SEGS; ++i) {
		compl_param = &sd->dma.infos[i].compl_param;

		if (compl_param->transfer_id != ODP_DMA_TRANSFER_ID_INVALID)
			odp_dma_transfer_id_free(sd->dma.handle, compl_param->transfer_id);

		if (compl_param->event != ODP_EVENT_INVALID)
			odp_event_free(compl_param->event);
	}

	free_fn(sd);
}

static void teardown_test(prog_config_t *config)
{
	sd_t *sd;

	for (uint32_t i = 0U; i < config->num_sessions; ++i) {
		sd = &config->sds[i];
		teardown_data(sd, config->api.free_fn);

		if (sd->dma.compl_q != ODP_QUEUE_INVALID)
			(void)odp_queue_destroy(sd->dma.compl_q);

		if (sd->dma.pool != ODP_POOL_INVALID)
			(void)odp_pool_destroy(sd->dma.pool);

		if (sd->grp != ODP_SCHED_GROUP_INVALID)
			(void)odp_schedule_group_destroy(sd->grp);

		if (sd->dma.handle != ODP_DMA_INVALID)
			(void)odp_dma_destroy(sd->dma.handle);
	}

	if (config->src_pool != ODP_POOL_INVALID)
		(void)odp_pool_destroy(config->src_pool);

	if (config->dst_pool != ODP_POOL_INVALID)
		(void)odp_pool_destroy(config->dst_pool);
}

static void print_humanised(uint64_t value, const char *type)
{
	if (value > GIGAS)
		printf("%.2f G%s\n", (double)value / GIGAS, type);
	else if (value > MEGAS)
		printf("%.2f M%s\n", (double)value / MEGAS, type);
	else if (value > KILOS)
		printf("%.2f K%s\n", (double)value / KILOS, type);
	else
		printf("%" PRIu64 " %s\n", value, type);
}

static void print_stats(const prog_config_t *config)
{
	const stats_t *stats;
	uint64_t data_cnt = config->num_in_segs * config->src_seg_len, tot_completed = 0U,
	tot_tm = 0U, tot_trs_tm = 0U, tot_trs_cc = 0U, tot_trs_cnt = 0U, tot_min_tm = UINT64_MAX,
	tot_max_tm = 0U, tot_min_cc = UINT64_MAX, tot_max_cc = 0U, avg_start_cc, avg_wait_cc,
	avg_tot_tm;

	printf("\n======================\n\n"
	       "DMA performance test done\n\n"
	       "    mode:                 %s\n"
	       "    input segment count:  %u\n"
	       "    output segment count: %u\n"
	       "    segment length:       %u\n"
	       "    segment type:         %s\n"
	       "    inflight count:       %u\n"
	       "    session policy:       %s\n\n",
	       config->trs_type == SYNC_DMA ? "DMA synchronous" :
		config->trs_type == ASYNC_DMA && config->compl_mode == POLL ?
			"DMA asynchronous-poll" :
				config->trs_type == ASYNC_DMA && config->compl_mode == EVENT ?
					"DMA asynchronous-event" : "SW", config->num_in_segs,
	       config->num_out_segs, config->src_seg_len,
	       config->seg_type == DENSE_PACKET ? "dense packet" :
		config->seg_type == SPARSE_PACKET ? "sparse packet" :
			config->seg_type == DENSE_MEMORY ? "dense memory" : "sparse memory",
	       config->num_inflight, config->policy == SINGLE ? "shared" : "per-worker");

	for (int i = 0; i < config->num_workers; ++i) {
		stats = &config->thread_config[i].stats;
		tot_completed += stats->completed;
		tot_tm += stats->tot_tm;
		tot_trs_tm += stats->trs_tm;
		tot_trs_cc += stats->trs_cc;
		tot_trs_cnt += stats->trs_cnt;
		tot_min_tm = ODPH_MIN(tot_min_tm, stats->min_trs_tm);
		tot_max_tm = ODPH_MAX(tot_max_tm, stats->max_trs_tm);
		tot_min_cc = ODPH_MIN(tot_min_cc, stats->min_trs_cc);
		tot_max_cc = ODPH_MAX(tot_max_cc, stats->max_trs_cc);

		printf("    worker %d:\n", i);
		printf("        successful transfers: %" PRIu64 "\n"
		       "        start errors:         %" PRIu64 "\n",
		       stats->completed, stats->start_errs);

		if (config->trs_type == ASYNC_DMA) {
			if (config->compl_mode == POLL)
				printf("        poll errors:          %" PRIu64 "\n",
				       stats->poll_errs);
			else
				printf("        scheduler timeouts:   %" PRIu64 "\n",
				       stats->scheduler_timeouts);
		}

		printf("        transfer errors:      %" PRIu64 "\n", stats->transfer_errs);

		if (config->is_verify)
			printf("        data errors:          %" PRIu64 "\n", stats->data_errs);

		printf("        run time:             %" PRIu64 " ns\n", stats->tot_tm);

		if (config->policy == MANY) {
			printf("        session:\n"
			       "            average time per transfer:   %" PRIu64 " "
			       "(min: %" PRIu64 ", max: %" PRIu64 ") ns\n"
			       "            average cycles per transfer: %" PRIu64 " "
			       "(min: %" PRIu64 ", max: %" PRIu64 ")\n"
			       "            ops:                         ",
			       stats->trs_cnt > 0U ? stats->trs_tm / stats->trs_cnt : 0U,
			       stats->trs_cnt > 0U ? stats->min_trs_tm : 0U,
			       stats->trs_cnt > 0U ? stats->max_trs_tm : 0U,
			       stats->trs_cnt > 0U ? stats->trs_cc / stats->trs_cnt : 0U,
			       stats->trs_cnt > 0U ? stats->min_trs_cc : 0U,
			       stats->trs_cnt > 0U ? stats->max_trs_cc : 0U);
			print_humanised(stats->completed / (stats->tot_tm / ODP_TIME_SEC_IN_NS),
					"OPS");
			printf("            speed:                       ");
			print_humanised(stats->completed * data_cnt /
					(stats->tot_tm / ODP_TIME_SEC_IN_NS), "B/s");
		}

		avg_start_cc = stats->start_cnt > 0U ? stats->start_cc / stats->start_cnt : 0U;
		printf("        average cycles breakdown:\n");

		if (config->trs_type == SYNC_DMA) {
			printf("            odp_dma_transfer(): %" PRIu64 " "
			       "(min: %" PRIu64 ", max: %" PRIu64 ")\n", avg_start_cc,
			       avg_start_cc > 0U ? stats->min_start_cc : 0U,
			       avg_start_cc > 0U ? stats->max_start_cc : 0U);
		} else if (config->trs_type == SW_COPY) {
			printf("            memcpy(): %" PRIu64 " "
			       "(min: %" PRIu64 ", max: %" PRIu64 ")\n", avg_start_cc,
			       avg_start_cc > 0U ? stats->min_start_cc : 0U,
			       avg_start_cc > 0U ? stats->max_start_cc : 0U);
		} else {
			printf("            odp_dma_transfer_start(): %" PRIu64 " "
			       "(min: %" PRIu64 ", max: %" PRIu64 ")\n", avg_start_cc,
			       avg_start_cc > 0U ? stats->min_start_cc : 0U,
			       avg_start_cc > 0U ? stats->max_start_cc : 0U);

			avg_wait_cc = stats->wait_cnt > 0U ? stats->wait_cc / stats->wait_cnt : 0U;

			if (config->compl_mode == POLL) {
				printf("            odp_dma_transfer_done():  %" PRIu64 ""
				       " (min: %" PRIu64 ", max: %" PRIu64 ", x%" PRIu64 ""
				       " per transfer)\n", avg_wait_cc,
				       avg_wait_cc > 0U ? stats->min_wait_cc : 0U,
				       avg_wait_cc > 0U ? stats->max_wait_cc : 0U,
				       stats->trs_cnt > 0U ?
						stats->trs_poll_cnt / stats->trs_cnt : 0U);
			} else {
				printf("            odp_schedule():           %" PRIu64 " "
				       " (min: %" PRIu64 ", max: %" PRIu64 ")\n", avg_wait_cc,
				       avg_wait_cc > 0U ? stats->min_wait_cc : 0U,
				       avg_wait_cc > 0U ? stats->max_wait_cc : 0U);
			}
		}

		printf("\n");
	}

	avg_tot_tm = tot_tm / config->num_workers / ODP_TIME_SEC_IN_NS;
	printf("    total:\n"
	       "        average time per transfer:   %" PRIu64 " (min: %" PRIu64
	       ", max: %" PRIu64 ") ns\n"
	       "        average cycles per transfer: %" PRIu64 " (min: %" PRIu64
	       ", max: %" PRIu64 ")\n"
	       "        ops:                         ",
	       tot_trs_cnt > 0U ? tot_trs_tm / tot_trs_cnt : 0U,
	       tot_trs_cnt > 0U ? tot_min_tm : 0U,
	       tot_trs_cnt > 0U ? tot_max_tm : 0U,
	       tot_trs_cnt > 0U ? tot_trs_cc / tot_trs_cnt : 0U,
	       tot_trs_cnt > 0U ? tot_min_cc : 0U,
	       tot_trs_cnt > 0U ? tot_max_cc : 0U);
	print_humanised(avg_tot_tm > 0U ? tot_completed / avg_tot_tm : 0U, "OPS");
	printf("        speed:                       ");
	print_humanised(avg_tot_tm > 0U ? tot_completed * data_cnt / avg_tot_tm : 0U, "B/s");
	printf("\n");
	printf("======================\n");
}

int main(int argc, char **argv)
{
	odph_helper_options_t odph_opts;
	odp_init_t init_param;
	odp_instance_t odp_instance;
	odp_shm_t shm_cfg = ODP_SHM_INVALID;
	parse_result_t parse_res;
	int ret = EXIT_SUCCESS;

	argc = odph_parse_options(argc, argv);

	if (odph_options(&odph_opts)) {
		ODPH_ERR("Error while reading ODP helper options, exiting\n");
		exit(EXIT_FAILURE);
	}

	odp_init_param_init(&init_param);
	init_param.mem_model = odph_opts.mem_model;

	if (odp_init_global(&odp_instance, &init_param, NULL)) {
		ODPH_ERR("ODP global init failed, exiting\n");
		exit(EXIT_FAILURE);
	}

	if (odp_init_local(odp_instance, ODP_THREAD_CONTROL)) {
		ODPH_ERR("ODP local init failed, exiting\n");
		exit(EXIT_FAILURE);
	}

	shm_cfg = odp_shm_reserve(PROG_NAME "_cfg", sizeof(prog_config_t), ODP_CACHE_LINE_SIZE,
				  0U);

	if (shm_cfg == ODP_SHM_INVALID) {
		ODPH_ERR("Error reserving shared memory\n");
		ret = EXIT_FAILURE;
		goto out;
	}

	prog_conf = odp_shm_addr(shm_cfg);

	if (prog_conf == NULL) {
		ODPH_ERR("Error resolving shared memory address\n");
		ret = EXIT_FAILURE;
		goto out;
	}

	parse_res = setup_program(argc, argv, prog_conf);

	if (parse_res == PRS_NOK) {
		ret = EXIT_FAILURE;
		goto out;
	}

	if (parse_res == PRS_TERM) {
		ret = EXIT_SUCCESS;
		goto out;
	}

	if (parse_res == PRS_NOT_SUP) {
		ret = EXIT_NOT_SUP;
		goto out;
	}

	if (odp_schedule_config(NULL) < 0) {
		ODPH_ERR("Error configuring scheduler\n");
		ret = EXIT_FAILURE;
		goto out;
	}

	prog_conf->odp_instance = odp_instance;
	odp_atomic_init_u32(&prog_conf->is_running, 1U);

	if (!setup_test(prog_conf)) {
		ret = EXIT_FAILURE;
		goto out_test;
	}

	if (prog_conf->time_sec) {
		sleep(prog_conf->time_sec);
		odp_atomic_store_u32(&prog_conf->is_running, 0U);
	}

	stop_test(prog_conf);
	print_stats(prog_conf);

out_test:
	/* Release all resources that have been allocated during 'setup_test()'. */
	teardown_test(prog_conf);

out:
	if (shm_cfg != ODP_SHM_INVALID)
		(void)odp_shm_free(shm_cfg);

	if (odp_term_local()) {
		ODPH_ERR("ODP local terminate failed, exiting\n");
		exit(EXIT_FAILURE);
	}

	if (odp_term_global(odp_instance)) {
		ODPH_ERR("ODP global terminate failed, exiting\n");
		exit(EXIT_FAILURE);
	}

	return ret;
}