path: root/platform/linux-keystone2/odp_pool.c

/*
 * Copyright (c) 2014, Linaro Limited
 * Copyright (c) 2014, Texas Instruments Incorporated
 * All rights reserved.
 *
 * SPDX-License-Identifier:     BSD-3-Clause
 */

#include <odp/config.h>
#include <odp/std_types.h>
#include <odp/pool.h>
#include <odp/buffer.h>
#include <odp/packet.h>
#include <odp_timer_internal.h>
#include <odp_pool_internal.h>
#include <odp_buffer_internal.h>
#include <odp_align_internal.h>
#include <odp_internal.h>
#include <odp/config.h>
#include <odp/hints.h>
#include <odp/plat/debug.h>
#include <odp/plat/osal.h>

#include "cppi_flow.h"

#include <string.h>
#include <stdlib.h>

/**
 * @todo: Currently a number of HW descriptors is limited,
 *        so temporary limit max number of buffers per pool
 *        to be albe to run ODP example apps.
 *        Descriptor management have to be made more intelligent
 *        To remove this limitation.
 */
#define MAX_BUFS_PER_POOL	6144

typedef struct __buf_info_s {
	Qmss_QueueHnd queue;
	uint32_t buf_size;
	uint32_t buf_align;
	int buf_type;
	int pool_id;
} __buf_info_t;

static const char odp_pool_table_name[] = "odp_pool_entries";

static void _free_descriptors(Qmss_MemRegion mem_region, Qmss_QueueHnd queue);
static void _detach_buffers(Qmss_QueueHnd queue);
static int pool_free_cb(void *data);

shres_table_t *pool_tbl;

void odp_pool_param_init(odp_pool_param_t *params)
{
	memset(params, 0, sizeof(odp_pool_param_t));
}

int odp_pool_init_global(void)
{
	pool_tbl = shres_table_create(odp_pool_table_name, pool_entry_t,
				      ODP_CONFIG_POOLS, pool_free_cb);
	if (pool_tbl == SHRES_TABLE_INVALID)
		return -1;

	ODP_DBG("\nBuffer pool init global\n");
	ODP_DBG("  pool_entry_t size     %zu\n", sizeof(pool_entry_t));
	ODP_DBG("\n");
	return 0;
}

int odp_pool_term_global(void)
{
	if (shres_table_destroy(odp_pool_table_name) < 0)
		return -1;

	return 0;
}

static int pool_free_cb(void *data)
{
	pool_entry_t *entry = (pool_entry_t *)data;

	if (Qmss_getQueueEntryCount(entry->free_queue) != entry->num_bufs) {
		ODP_ERR("Not all buffers are returned to the pool\n");
		return -1;
	}

	_detach_buffers(entry->free_queue);
	_free_descriptors(odp_global->public_desc_memregion,
			  entry->free_queue);
	ODP_DBG("Pool \"%s\" destroyed\n", entry->name);
	return 0;
}

static void _free_descriptors(Qmss_MemRegion mem_region,
			     Qmss_QueueHnd queue)
{
	Qmss_QueueHnd mem_reg_queue = Qmss_getMemRegQueueHandle(mem_region);
	Qmss_Result result;

	while (Qmss_getQueueEntryCount(queue) != 0) {
		uintptr_t desc_phys = QMSS_DESC_PTR(Qmss_queuePopRaw(queue));
		ODP_AS_STR(desc_phys, "No descriptors in a pool free queue");

		/* Place back the descriptor into the memory region queue. */
		Qmss_queuePushDescSizeRaw(mem_reg_queue, (void *)desc_phys, 16);
	}
	result = Qmss_queueClose(queue);
	ODP_AS_STR(result >= 0, "Queue close failed");
}

static Qmss_QueueHnd _alloc_descriptors(Qmss_MemRegion mem_region,
				       uint32_t desc_num)
{
	Cppi_DescCfg desc_cfg;
	Qmss_QueueHnd queue;
	uint32_t num_allocated;

	/* Initialize the descriptor configuration */
	memset((void *)&desc_cfg, 0, sizeof(Cppi_DescCfg));

	/* Populate the descriptor configuration:  */
	desc_cfg.memRegion             = mem_region;
	desc_cfg.descNum               = desc_num;
	desc_cfg.destQueueNum          = QMSS_PARAM_NOT_SPECIFIED;
	desc_cfg.queueGroup            = 0;
	desc_cfg.queueType             = Qmss_QueueType_GENERAL_PURPOSE_QUEUE;
	desc_cfg.initDesc              = Cppi_InitDesc_INIT_DESCRIPTOR;
	desc_cfg.descType              = Cppi_DescType_HOST;
	desc_cfg.returnQueue.qMgr      = QMSS_PARAM_NOT_SPECIFIED;
	desc_cfg.returnQueue.qNum      = QMSS_PARAM_NOT_SPECIFIED;
	desc_cfg.epibPresent           = Cppi_EPIB_EPIB_PRESENT;
	desc_cfg.returnPushPolicy      = Qmss_Location_TAIL;
	desc_cfg.cfg.host.returnPolicy = Cppi_ReturnPolicy_RETURN_BUFFER;
	desc_cfg.cfg.host.psLocation   = Cppi_PSLoc_PS_IN_DESC;

	/* Initialize the descriptors and place them into a queue. */
	queue = Cppi_initDescriptor(&desc_cfg, &num_allocated);
	if (queue < 0) {
		ODP_ERR("Cppi_initDescriptor() failed: %i\n", (int32_t)queue);
		return queue;
	}

	if (num_allocated != desc_num) {
		ODP_ERR("Wrong number of descriptors allocated: requested: %u, allocated: %u\n",
			desc_num, num_allocated);
		_free_descriptors(mem_region, queue);
		return (Qmss_QueueHnd)QMSS_LLD_EBASE;
	}

	ODP_AS_STR(Qmss_getQueueEntryCount(queue) == desc_num,
		   "Wrong number of descriptors in queue");

	return queue;
}

static void _detach_buffers(Qmss_QueueHnd queue)
{
	uint32_t num_bufs = Qmss_getQueueEntryCount(queue);
	uint32_t i;

	for (i = 0; i < num_bufs; i++) {
		uint8_t *buf_ptr;
		odp_pa_t buf_phys;
		uint32_t buf_size;
		odp_pa_t desc_phys = QMSS_DESC_PTR(Qmss_queuePopRaw(queue));
		Cppi_HostDesc *desc = _odp_mem_phys_to_virt(desc_phys);
		ODP_AS_STR(desc != NULL, "No descriptors in a pool free queue");

		_cppi_desc_orig_info(desc, &buf_phys, &buf_size);
		buf_ptr = _odp_mem_phys_to_virt(buf_phys);
		if (buf_ptr) {
			hplib_vmMemFree(buf_ptr, buf_size, 0);
			_cppi_desc_orig_info_set(desc, 0, 0);
		}

		/* Push this descriptor into the corresponding queue */
		Qmss_queuePushDescSizeRaw(queue, (void *)desc_phys, 16);
	}
}

static int _attach_buffers(__buf_info_t *buf_info)
{
	uint32_t num_bufs = Qmss_getQueueEntryCount(buf_info->queue);
	uint32_t i;

	if (num_bufs == 0)
		return -1;

	for (i = 0; i < num_bufs; i++) {
		uint8_t *buf_ptr;
		odp_pa_t buf_phys;
		/* Pop a descriptor of the free queue. */
		odp_pa_t desc_phys =
			QMSS_DESC_PTR(Qmss_queuePopRaw(buf_info->queue));
		Cppi_HostDesc *desc = _odp_mem_phys_to_virt(desc_phys);
		ODP_AS_STR(desc != NULL, "No descriptors in a pool free queue");

		/* Allocate memory for the data buffer. */
		buf_ptr = hplib_vmMemAlloc(buf_info->buf_size,
					   buf_info->buf_align, 0);
		if (buf_ptr == NULL) {
			/* Return descriptor to a queue and detach buffers */
			Qmss_queuePushDescSizeRaw(buf_info->queue,
						  (void *)desc_phys, 16);
			_detach_buffers(buf_info->queue);
			return -1;
		}

		buf_phys = _odp_mem_virt_to_phys(buf_ptr);
		/* Set the original buffer information in the descriptor. */
		_cppi_desc_orig_info_set(desc, buf_phys, buf_info->buf_size);

		/* Set the data and payload length into the packet. */
		_cppi_desc_buf_info_set(desc, buf_phys, buf_info->buf_size);

		_cppi_desc_pkt_type_set(desc, buf_info->buf_type);
		_cppi_desc_pool_id_set(desc, buf_info->pool_id);

		/* Push this descriptor into the corresponding queue */
		Qmss_queuePushDescSizeRaw(buf_info->queue,
					  (void *)desc_phys, 16);
	}

	return 0;
}

static int _odp_buffer_pool_init(pool_entry_t *entry)
{
	__buf_info_t buf_info;

	buf_info.buf_align = entry->orig_params.buf.align;
	buf_info.buf_size  = entry->orig_params.buf.size;

	/* Set correct alignment based on input request */
	if (buf_info.buf_align == 0)
		buf_info.buf_align = ODP_CACHE_LINE_SIZE;
	else if (buf_info.buf_align < ODP_CONFIG_BUFFER_ALIGN_MIN)
		buf_info.buf_align = ODP_CONFIG_BUFFER_ALIGN_MIN;
	else if (buf_info.buf_align > ODP_CONFIG_BUFFER_ALIGN_MAX)
		return -1;

	if (buf_info.buf_size > ODP_CONFIG_PACKET_SEG_LEN_MAX)
		return -1;

	entry->headroom = _ODP_BUFHDR_SIZE;
	entry->tailroom = 0;

	buf_info.buf_size += entry->headroom;
	buf_info.buf_size  = ODP_CACHE_LINE_SIZE_ROUNDUP(buf_info.buf_size);

	entry->buf_size = buf_info.buf_size;
	entry->buf_align = buf_info.buf_align;

	buf_info.queue = entry->free_queue;
	buf_info.buf_type = entry->orig_params.type;
	buf_info.pool_id = shres_id(entry);
	if (_attach_buffers(&buf_info))
		return -1;

	return 0;
}

static int _odp_timeout_pool_init(pool_entry_t *entry)
{
	__buf_info_t buf_info;

	entry->tailroom = 0;
	entry->headroom = sizeof(odp_timeout_hdr_stride);

	entry->buf_size = entry->headroom;
	entry->buf_align = ODP_CACHE_LINE_SIZE;

	buf_info.queue = entry->free_queue;
	buf_info.buf_size = entry->buf_size;
	buf_info.buf_align = entry->buf_align;
	buf_info.buf_type = entry->orig_params.type;
	buf_info.pool_id = shres_id(entry);
	if (_attach_buffers(&buf_info))
		return -1;

	return 0;
}

static int _odp_packet_pool_init(pool_entry_t *entry)
{
	__buf_info_t buf_info;

	//ODP_AS_STR(entry->orig_params.pkt.len >= entry->orig_params.pkt.seg_len,
		   //"Packet length should be >= segment length");

	buf_info.buf_size  = entry->orig_params.pkt.len;
	if (buf_info.buf_size == 0)
		buf_info.buf_size = ODP_CONFIG_PACKET_SEG_LEN_MIN;
	else if (buf_info.buf_size > ODP_CONFIG_PACKET_SEG_LEN_MAX)
		return -1;

	entry->headroom = _ODP_PKTHDR_SIZE + ODP_CONFIG_PACKET_HEADROOM;
	entry->headroom += entry->orig_params.pkt.uarea_size;
	entry->tailroom = 0;

	buf_info.buf_size += entry->headroom;
	buf_info.buf_size = ODP_CACHE_LINE_SIZE_ROUNDUP(buf_info.buf_size);

	entry->buf_size = buf_info.buf_size;
	entry->buf_align = ODP_CACHE_LINE_SIZE;

	buf_info.queue = entry->free_queue;
	buf_info.buf_align = ODP_CACHE_LINE_SIZE;
	buf_info.buf_type = entry->orig_params.type;
	buf_info.pool_id = shres_id(entry);
	if (_attach_buffers(&buf_info))
		return -1;

	return 0;
}

odp_pool_t odp_pool_create(const char *name, odp_pool_param_t *params)
{
	uint32_t num_bufs;
	pool_entry_t *entry;
	static const char anon_name[] = "anonymous";
	int ret = 0;

	if (params == NULL)
		return ODP_POOL_INVALID;

	entry = shres_alloc(pool_tbl, typeof(*entry));
	if (!entry) {
		ODP_ERR("Couldn't allocate pool entry\n");
		return ODP_POOL_INVALID;
	}

	entry->flags.all = 0;

	if (name == NULL) {
		entry->name[0] = 0;
		name = anon_name;
	} else {
		strncpy(entry->name, name, ODP_POOL_NAME_LEN - 1);
		entry->name[ODP_POOL_NAME_LEN - 1] = 0;
		entry->flags.has_name = 1;
	}

	num_bufs = params->buf.num;
	if (num_bufs > MAX_BUFS_PER_POOL) {
		odp_pr_dbg("Limiting number of buffer in %s from %d to %d\n",
			   name, num_bufs, MAX_BUFS_PER_POOL);
		num_bufs = MAX_BUFS_PER_POOL;
	}

	entry->free_queue = _alloc_descriptors(
					odp_global->public_desc_memregion,
					num_bufs);
	if (entry->free_queue < 0)
		return ODP_POOL_INVALID;

	memcpy(&entry->orig_params, params, sizeof(entry->orig_params));
	entry->num_bufs = num_bufs;

	switch (params->type) {
	case ODP_POOL_BUFFER:
		ret = _odp_buffer_pool_init(entry);
		break;
	case ODP_POOL_PACKET:
		ret = _odp_packet_pool_init(entry);
		break;
	case ODP_POOL_TIMEOUT:
		ret = _odp_timeout_pool_init(entry);
		break;
	default:
		ODP_ERR("Wrong pool type\n");
		ret = -1;
	}

	ODP_DBG("Pool \"%s\" created\n", entry->name);

	if (ret) {
		_free_descriptors(odp_global->public_desc_memregion,
				  entry->free_queue);
		_shres_free(pool_tbl, &entry->entry_head);
		return ODP_POOL_INVALID;
	}

	return _odp_pool_from_entry(entry);
}

int odp_pool_destroy(odp_pool_t pool)
{
	if (pool == ODP_POOL_INVALID)
		return -1;

	pool_entry_t *entry = _odp_pool_entry(pool);
	if (shres_free(pool_tbl, entry) < 0)
		return -1;

	shres_lock(entry);
	if (shres_is_allocated(entry)) {
		shres_unlock(entry);
		ODP_ERR("Pool in use cannot be destroyed right now. It will be destroyed latter\n");
		return -1;
	}
	shres_unlock(entry);

	return 0;
}

odp_pool_t odp_pool_lookup(const char *name)
{
	pool_entry_t *entry;

	shres_table_lock(pool_tbl);
	shres_table_for_each_allocated_entry(pool_tbl, entry) {
		if (strcmp(name, entry->name) == 0) {
			shres_table_unlock(pool_tbl);
			return _odp_pool_from_entry(entry);
		}
	}
	shres_table_unlock(pool_tbl);

	return ODP_POOL_INVALID;
}

int odp_pool_info(odp_pool_t pool,
			 odp_pool_info_t *info)
{
	pool_entry_t *entry = _odp_pool_entry(pool);

	if (entry == NULL || info == NULL)
		return -1;

	info->name = entry->name;
	memcpy(&info->params, &entry->orig_params, sizeof(entry->orig_params));

	return 0;
}

static const char *_odp_buffer_type_string(int type)
{
	switch (type) {
	case ODP_POOL_BUFFER:
		return "buffer";
	case ODP_POOL_PACKET:
		return "packet";
	case ODP_POOL_TIMEOUT:
		return "timeout";
	default:
		return "invalid";
	}
}

static uint32_t _odp_pool_available_bufs(odp_pool_t pool)
{
	pool_entry_t *entry = _odp_pool_entry(pool);
	return Qmss_getQueueEntryCount(entry->free_queue);
}

void odp_pool_print(odp_pool_t pool)
{
	pool_entry_t *entry = _odp_pool_entry(pool);

	if (!entry)
		return;

	ODP_PRINT("Pool info\n");
	ODP_PRINT("---------\n");
	ODP_PRINT(" pool            %p\n", pool);
	ODP_PRINT(" pool_id         %i\n", _odp_pool_id(pool));
	ODP_PRINT(" name            %s\n",
		  entry->flags.has_name ? entry->name : "Unnamed Pool");
	ODP_PRINT(" pool type       %s\n",
		  _odp_buffer_type_string(entry->orig_params.type));
	ODP_PRINT(" buf size        %u requested, %u used\n",
		  entry->orig_params.buf.size, entry->buf_size);
	ODP_PRINT(" buf align       %u requested, %u used\n",
		  entry->orig_params.buf.align, entry->buf_align);
	ODP_PRINT(" num_bufs        %u requested, %u used\n",
		  entry->orig_params.buf.num, entry->num_bufs);
	ODP_PRINT(" bufs available  %u\n", _odp_pool_available_bufs(pool));
}

cppi_flow_entry_t *_odp_pool_cppi_flow_create(odp_pool_t pool, uint32_t headroom)
{
	pool_entry_t *entry = _odp_pool_entry(pool);

	return cppi_flow_create_no_dest(entry->free_queue, headroom);
}