linux-gen: dpdk: adapt to new memory source operations interface

Adapt DPDK packet IO to use the new `_odp_pool_mem_src_ops_t` interface.

Signed-off-by: Tuomas Taipale <tuomas.taipale@nokia.com>
Reviewed-by: Matias Elo <matias.elo@nokia.com>

5 files changed, 116 insertions, 88 deletions

diff --git a/platform/linux-generic/include/odp_packet_dpdk.h b/platform/linux-generic/include/odp_packet_dpdk.h
index b326000e6..7d42971be 100644
--- a/platform/linux-generic/include/odp_packet_dpdk.h
+++ b/platform/linux-generic/include/odp_packet_dpdk.h
@@ -26,16 +26,6 @@ struct rte_mbuf;
 #define PTYPE_UDP       0x20
 #define PTYPE_TCP       0x40
 
-/**
- * Calculate size of zero-copy DPDK packet pool object
- */
-uint32_t _odp_dpdk_pool_obj_size(pool_t *pool, uint32_t block_size);
-
-/**
- * Create zero-copy DPDK packet pool
- */
-int _odp_dpdk_pool_create(pool_t *pool);
-
 /** Packet parser using DPDK interface */
 int _odp_dpdk_packet_parse_common(packet_parser_t *pkt_hdr,
 				  const uint8_t *ptr,
diff --git a/platform/linux-generic/include/odp_pool_internal.h b/platform/linux-generic/include/odp_pool_internal.h
index 68f13472c..824aa9292 100644
--- a/platform/linux-generic/include/odp_pool_internal.h
+++ b/platform/linux-generic/include/odp_pool_internal.h
@@ -54,9 +54,6 @@ typedef struct ODP_ALIGNED_CACHE {
 } pool_ring_t;
 #pragma GCC diagnostic pop
 
-/* Callback function for pool destroy */
-typedef void (*pool_destroy_cb_fn)(void *pool);
-
 struct _odp_pool_mem_src_ops_t;
 
 typedef struct pool_t {
@@ -95,14 +92,8 @@ typedef struct pool_t {
 	uint8_t         *uarea_base_addr;
 	odp_pool_type_t  type_2;
 	odp_pool_ext_param_t ext_param;
-
-	/* Used by DPDK zero-copy pktio */
-	uint32_t         dpdk_elt_size;
 	uint32_t         skipped_blocks;
-	uint8_t          pool_in_use;
 	uint8_t          mem_from_huge_pages;
-	pool_destroy_cb_fn ext_destroy;
-	void            *ext_desc;
 	const struct _odp_pool_mem_src_ops_t *mem_src_ops;
 	/* Private area for memory source operations */
 	uint8_t mem_src_data[_ODP_POOL_MEM_SRC_DATA_SIZE] ODP_ALIGNED_CACHE;
diff --git a/platform/linux-generic/odp_pool.c b/platform/linux-generic/odp_pool.c
index b39028231..739665cb9 100644
--- a/platform/linux-generic/odp_pool.c
+++ b/platform/linux-generic/odp_pool.c
@@ -15,7 +15,6 @@
 #include <odp_pool_internal.h>
 #include <odp_init_internal.h>
 #include <odp_packet_internal.h>
-#include <odp_packet_dpdk.h>
 #include <odp_config_internal.h>
 #include <odp_debug_internal.h>
 #include <odp_macros_internal.h>
@@ -633,6 +632,26 @@ static int reserve_uarea(pool_t *pool, uint32_t uarea_size, uint32_t num_pkt, ui
 	return 0;
 }
 
+static void set_mem_src_ops(pool_t *pool)
+{
+	odp_bool_t is_active_found = false;
+
+	pool->mem_src_ops = NULL;
+
+	for (int i = 0; _odp_pool_mem_src_ops[i]; i++) {
+		if (!is_active_found) {
+			if (_odp_pool_mem_src_ops[i]->is_active()) {
+				is_active_found = true;
+				pool->mem_src_ops = _odp_pool_mem_src_ops[i];
+				ODP_DBG("Packet pool as a memory source for: %s\n",
+					pool->mem_src_ops->name);
+			}
+		} else if (_odp_pool_mem_src_ops[i]->is_active()) {
+			_odp_pool_mem_src_ops[i]->force_disable();
+		}
+	}
+}
+
 /* Create pool according to params. Actual type of the pool is type_2, which is recorded for pool
  * info calls. */
 odp_pool_t _odp_pool_create(const char *name, const odp_pool_param_t *params,
@@ -776,26 +795,29 @@ odp_pool_t _odp_pool_create(const char *name, const odp_pool_param_t *params,
 	pool->type_2 = type_2;
 	pool->params = *params;
 	pool->block_offset = 0;
+	set_mem_src_ops(pool);
 
 	if (type == ODP_POOL_PACKET) {
-		uint32_t dpdk_obj_size;
+		uint32_t adj_size;
 
 		hdr_size = _ODP_ROUNDUP_CACHE_LINE(sizeof(odp_packet_hdr_t));
 		block_size = hdr_size + align + headroom + seg_len + tailroom;
-		/* Calculate extra space required for storing DPDK objects and
-		 * mbuf headers. NOP if no DPDK pktio used or zero-copy mode is
-		 * disabled. */
-		dpdk_obj_size = _odp_dpdk_pool_obj_size(pool, block_size);
-		if (!dpdk_obj_size) {
-			ODP_ERR("Calculating DPDK mempool obj size failed\n");
-			return ODP_POOL_INVALID;
+		adj_size = block_size;
+
+		if (pool->mem_src_ops && pool->mem_src_ops->adjust_size) {
+			pool->mem_src_ops->adjust_size(pool->mem_src_data,  &adj_size,
+						       &pool->block_offset, &shmflags);
+
+			if (!adj_size) {
+				ODP_ERR("Calculating adjusted block size failed\n");
+				return ODP_POOL_INVALID;
+			}
 		}
-		if (dpdk_obj_size != block_size) {
-			shmflags |= ODP_SHM_HP;
-			block_size = dpdk_obj_size;
-		} else {
+
+		if (adj_size != block_size)
+			block_size = adj_size;
+		else
 			block_size = _ODP_ROUNDUP_CACHE_LINE(block_size);
-		}
 	} else {
 		/* Header size is rounded up to cache line size, so the
 		 * following data can be cache line aligned without extra
@@ -838,8 +860,6 @@ odp_pool_t _odp_pool_create(const char *name, const odp_pool_param_t *params,
 	pool->tailroom       = tailroom;
 	pool->block_size     = block_size;
 	pool->shm_size       = (num + num_extra) * (uint64_t)block_size;
-	pool->ext_desc       = NULL;
-	pool->ext_destroy    = NULL;
 
 	set_pool_cache_size(pool, cache_size);
 
@@ -866,9 +886,9 @@ odp_pool_t _odp_pool_create(const char *name, const odp_pool_param_t *params,
 	ring_ptr_init(&pool->ring->hdr);
 	init_buffers(pool);
 
-	/* Create zero-copy DPDK memory pool. NOP if zero-copy is disabled. */
-	if (type == ODP_POOL_PACKET && _odp_dpdk_pool_create(pool)) {
-		ODP_ERR("Creating DPDK packet pool failed\n");
+	if (type == ODP_POOL_PACKET && pool->mem_src_ops && pool->mem_src_ops->bind &&
+	    pool->mem_src_ops->bind(pool->mem_src_data, pool)) {
+		ODP_ERR("Binding pool as memory source failed\n");
 		goto error;
 	}
 
@@ -1083,12 +1103,8 @@ int odp_pool_destroy(odp_pool_t pool_hdl)
 		return -1;
 	}
 
-	/* Destroy external DPDK mempool */
-	if (pool->ext_destroy) {
-		pool->ext_destroy(pool->ext_desc);
-		pool->ext_destroy = NULL;
-		pool->ext_desc = NULL;
-	}
+	if (pool->type == ODP_POOL_PACKET && pool->mem_src_ops && pool->mem_src_ops->unbind)
+		pool->mem_src_ops->unbind(pool->mem_src_data);
 
 	/* Make sure local caches are empty */
 	for (i = 0; i < ODP_THREAD_COUNT_MAX; i++)
@@ -1461,6 +1477,8 @@ void odp_pool_print(odp_pool_t pool_hdl)
 	ODP_PRINT("  uarea base addr %p\n", (void *)pool->uarea_base_addr);
 	ODP_PRINT("  cache size      %u\n", pool->cache_size);
 	ODP_PRINT("  burst size      %u\n", pool->burst_size);
+	ODP_PRINT("  mem src         %s\n",
+		  pool->mem_src_ops ? pool->mem_src_ops->name : "(none)");
 	ODP_PRINT("\n");
 }
 
diff --git a/platform/linux-generic/odp_pool_mem_src_ops.c b/platform/linux-generic/odp_pool_mem_src_ops.c
index 14b85a77b..d9b810a6a 100644
--- a/platform/linux-generic/odp_pool_mem_src_ops.c
+++ b/platform/linux-generic/odp_pool_mem_src_ops.c
@@ -7,7 +7,12 @@
 #include <odp/autoheader_internal.h>
 #include <odp_pool_internal.h>
 
+extern const _odp_pool_mem_src_ops_t _odp_pool_dpdk_mem_src_ops;
+
 /* List of available ODP packet pool memory source operations. Array must be NULL terminated */
 const _odp_pool_mem_src_ops_t * const _odp_pool_mem_src_ops[] = {
+#ifdef _ODP_PKTIO_DPDK
+	&_odp_pool_dpdk_mem_src_ops,
+#endif
 	NULL
 };
diff --git a/platform/linux-generic/pktio/dpdk.c b/platform/linux-generic/pktio/dpdk.c
index 95aaf0f23..006344b48 100644
--- a/platform/linux-generic/pktio/dpdk.c
+++ b/platform/linux-generic/pktio/dpdk.c
@@ -24,6 +24,7 @@
 #include <odp/api/plat/time_inlines.h>
 
 #include <odp_packet_io_internal.h>
+#include <odp_pool_internal.h>
 #include <odp_classification_internal.h>
 #include <odp_socket_common.h>
 #include <odp_packet_dpdk.h>
@@ -153,6 +154,15 @@ typedef struct ODP_ALIGNED_CACHE {
 ODP_STATIC_ASSERT(PKTIO_PRIVATE_SIZE >= sizeof(pkt_dpdk_t),
 		  "PKTIO_PRIVATE_SIZE too small");
 
+typedef struct {
+	uint32_t dpdk_elt_size;
+	uint8_t pool_in_use;
+	struct rte_mempool *pkt_pool;
+} mem_src_data_t;
+
+ODP_STATIC_ASSERT(_ODP_POOL_MEM_SRC_DATA_SIZE >= sizeof(mem_src_data_t),
+		  "_ODP_POOL_MEM_SRC_DATA_SIZE too small");
+
 static inline struct rte_mbuf *mbuf_from_pkt_hdr(odp_packet_hdr_t *pkt_hdr)
 {
 	return ((struct rte_mbuf *)pkt_hdr) - 1;
@@ -168,6 +178,11 @@ static inline pkt_dpdk_t *pkt_priv(pktio_entry_t *pktio_entry)
 	return (pkt_dpdk_t *)(uintptr_t)(pktio_entry->s.pkt_priv);
 }
 
+static inline mem_src_data_t *mem_src_priv(uint8_t *data)
+{
+	return (mem_src_data_t *)data;
+}
+
 static int disable_pktio; /** !0 this pktio disabled, 0 enabled */
 
 static int dpdk_pktio_init(void);
@@ -320,13 +335,14 @@ static void pktmbuf_init(struct rte_mempool *mp, void *opaque_arg ODP_UNUSED,
  *  Create custom DPDK packet pool
  */
 static struct rte_mempool *mbuf_pool_create(const char *name,
-					    pool_t *pool_entry)
+					    pool_t *pool_entry,
+					    uint32_t dpdk_elt_size)
 {
 	odp_shm_info_t shm_info;
 	struct rte_mempool *mp = NULL;
 	struct rte_pktmbuf_pool_private mbp_priv;
 	struct rte_mempool_objsz sz;
-	unsigned int elt_size = pool_entry->dpdk_elt_size;
+	unsigned int elt_size = dpdk_elt_size;
 	unsigned int num = pool_entry->num, populated = 0;
 	uint32_t total_size;
 	uint64_t page_size, offset = 0, remainder = 0;
@@ -423,9 +439,10 @@ static int pool_enqueue(struct rte_mempool *mp,
 {
 	odp_packet_t pkt_tbl[num];
 	pool_t *pool_entry = (pool_t *)mp->pool_config;
+	mem_src_data_t *mem_src_data = mem_src_priv(pool_entry->mem_src_data);
 	unsigned i;
 
-	if (odp_unlikely(num == 0 || !pool_entry->pool_in_use))
+	if (odp_unlikely(num == 0 || !mem_src_data->pool_in_use))
 		return 0;
 
 	for (i = 0; i < num; i++) {
@@ -497,67 +514,72 @@ static void pool_free(struct rte_mempool *mp)
 	}
 }
 
-static void pool_destroy(void *pool)
+static void pool_destroy(uint8_t *data)
 {
-	struct rte_mempool *mp = (struct rte_mempool *)pool;
+	mem_src_data_t *mem_src_data = mem_src_priv(data);
 
-	if (mp != NULL) {
-		pool_t *pool_entry = (pool_t *)mp->pool_config;
-
-		pool_entry->pool_in_use = 0;
-		rte_mempool_free(mp);
+	if (mem_src_data->pkt_pool != NULL) {
+		mem_src_data->pool_in_use = 0;
+		rte_mempool_free(mem_src_data->pkt_pool);
 	}
+
+	mem_src_data->pkt_pool = NULL;
 }
 
-int _odp_dpdk_pool_create(pool_t *pool)
+static int pool_create(uint8_t *data, pool_t *pool)
 {
 	struct rte_mempool *pkt_pool;
 	char pool_name[RTE_MEMPOOL_NAMESIZE];
+	mem_src_data_t *mem_src_data = mem_src_priv(data);
+
+	mem_src_data->pkt_pool = NULL;
 
 	if (!_ODP_DPDK_ZERO_COPY)
 		return 0;
 
-	pool->pool_in_use = 0;
-
+	mem_src_data->pool_in_use = 0;
 	snprintf(pool_name, sizeof(pool_name),
 		 "dpdk_pktpool_%" PRIu32 "_%" PRIu32 "", odp_global_ro.main_pid,
 		 pool->pool_idx);
-	pkt_pool = mbuf_pool_create(pool_name, pool);
+	pkt_pool = mbuf_pool_create(pool_name, pool, mem_src_data->dpdk_elt_size);
 
 	if (pkt_pool == NULL) {
 		ODP_ERR("Creating external DPDK pool failed\n");
 		return -1;
 	}
 
-	pool->ext_desc = pkt_pool;
-	pool->ext_destroy = pool_destroy;
-	pool->pool_in_use = 1;
+	mem_src_data->pkt_pool = pkt_pool;
+	mem_src_data->pool_in_use = 1;
 
 	return 0;
 }
 
-uint32_t _odp_dpdk_pool_obj_size(pool_t *pool, uint32_t block_size)
+static void pool_obj_size(uint8_t *data, uint32_t *block_size, uint32_t *block_offset,
+			  uint32_t *flags)
 {
 	struct rte_mempool_objsz sz;
+	uint32_t size;
 	uint32_t total_size;
+	mem_src_data_t *mem_src_data = mem_src_priv(data);
 
 	if (!_ODP_DPDK_ZERO_COPY)
-		return block_size;
+		return;
 
 	if (odp_global_rw->dpdk_initialized == 0) {
 		if (dpdk_pktio_init()) {
 			ODP_ERR("Initializing DPDK failed\n");
-			return 0;
+			*block_size = 0;
+			return;
 		}
 		odp_global_rw->dpdk_initialized = 1;
 	}
 
-	block_size += sizeof(struct rte_mbuf);
-	total_size = rte_mempool_calc_obj_size(block_size, MEMPOOL_FLAGS, &sz);
-	pool->dpdk_elt_size = sz.elt_size;
-	pool->block_offset = sz.header_size + sizeof(struct rte_mbuf);
-
-	return total_size;
+	*flags |= ODP_SHM_HP;
+	size = *block_size + sizeof(struct rte_mbuf);
+	total_size = rte_mempool_calc_obj_size(size, MEMPOOL_FLAGS, &sz);
+	mem_src_data->dpdk_elt_size = sz.elt_size;
+	*block_size = total_size;
+	*block_offset = sz.header_size + sizeof(struct rte_mbuf);
 }
 
 static struct rte_mempool_ops odp_pool_ops = {
@@ -1735,7 +1757,9 @@ static int dpdk_open(odp_pktio_t id ODP_UNUSED,
 		pkt_dpdk->min_rx_burst = 0;
 
 	if (_ODP_DPDK_ZERO_COPY) {
-		pkt_pool = (struct rte_mempool *)pool_entry->ext_desc;
+		mem_src_data_t *mem_src_data = mem_src_priv(pool_entry->mem_src_data);
+
+		pkt_pool = mem_src_data->pkt_pool;
 	} else {
 		snprintf(pool_name, sizeof(pool_name), "pktpool_%s", netdev);
 		/* Check if the pool exists already */
@@ -2406,27 +2430,27 @@ const pktio_if_ops_t _odp_dpdk_pktio_ops = {
 	.output_queues_config = dpdk_output_queues_config
 };
 
-#else
-
-#include <stdint.h>
-
-#include <odp/api/hints.h>
-
-#include <odp_packet_dpdk.h>
-#include <odp_pool_internal.h>
-
-/*
- * Dummy functions for pool_create()
- */
-
-uint32_t _odp_dpdk_pool_obj_size(pool_t *pool ODP_UNUSED, uint32_t block_size)
+static odp_bool_t is_mem_src_active(void)
 {
-	return block_size;
+	return !disable_pktio && _ODP_DPDK_ZERO_COPY;
 }
 
-int _odp_dpdk_pool_create(pool_t *pool ODP_UNUSED)
+static void force_mem_src_disable(void)
 {
-	return 0;
+	if (_ODP_DPDK_ZERO_COPY)
+		disable_pktio = 1;
 }
 
+const _odp_pool_mem_src_ops_t _odp_pool_dpdk_mem_src_ops = {
+	.name = "dpdk_zc",
+	.is_active = is_mem_src_active,
+	.force_disable = force_mem_src_disable,
+	.adjust_size = pool_obj_size,
+	.bind = pool_create,
+	.unbind = pool_destroy
+};
+
+#else
+/* Avoid warning about empty translation unit */
+typedef int _odp_dummy;
 #endif /* _ODP_PKTIO_DPDK */