Port a735a7131 "linux-gen: event: add event validation support"

Port original commit from linux-generic.

Signed-off-by: Matias Elo <matias.elo@nokia.com>
Reviewed-by: Tuomas Taipale <tuomas.taipale@nokia.com>

12 files changed, 95 insertions, 26 deletions

diff --git a/platform/linux-dpdk/Makefile.am b/platform/linux-dpdk/Makefile.am
index 31148168b..7f45d7683 100644
--- a/platform/linux-dpdk/Makefile.am
+++ b/platform/linux-dpdk/Makefile.am
@@ -36,6 +36,7 @@ odpapiplatinclude_HEADERS = \
 		  include/odp/api/plat/debug_inlines.h \
 		  include/odp/api/plat/event_inlines.h \
 		  include/odp/api/plat/event_inline_types.h \
+		  include/odp/api/plat/event_validation_external.h \
 		  include/odp/api/plat/event_vector_inline_types.h \
 		  include/odp/api/plat/hash_inlines.h \
 		  include/odp/api/plat/ipsec_inlines.h \
@@ -144,6 +145,7 @@ noinst_HEADERS = \
 		  ${top_srcdir}/platform/linux-generic/include/odp_print_internal.h \
 		  include/odp_errno_define.h \
 		  include/odp_event_internal.h \
+		  ${top_srcdir}/platform/linux-generic/include/odp_event_validation_internal.h \
 		  ${top_srcdir}/platform/linux-generic/include/odp_packet_dpdk.h \
 		  ${top_srcdir}/platform/linux-generic/include/odp_pcapng.h \
 		  ${top_srcdir}/platform/linux-generic/include/odp_pkt_queue_internal.h \
@@ -200,6 +202,7 @@ __LIB__libodp_dpdk_la_SOURCES = \
 			   odp_crypto.c \
 			   odp_errno.c \
 			   odp_event.c \
+			   ../linux-generic/odp_event_validation.c \
 			   ../linux-generic/odp_hash_crc_gen.c \
 			   odp_init.c \
 			   ../linux-generic/odp_impl.c \
diff --git a/platform/linux-dpdk/include/odp/api/plat/buffer_inlines.h b/platform/linux-dpdk/include/odp/api/plat/buffer_inlines.h
index 37a3bf296..396d78559 100644
--- a/platform/linux-dpdk/include/odp/api/plat/buffer_inlines.h
+++ b/platform/linux-dpdk/include/odp/api/plat/buffer_inlines.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2019-2022, Nokia
+/* Copyright (c) 2019-2023, Nokia
  * All rights reserved.
  *
  * SPDX-License-Identifier:     BSD-3-Clause
@@ -15,6 +15,7 @@
 #include <odp/api/plat/buffer_inline_types.h>
 #include <odp/api/plat/debug_inlines.h>
 #include <odp/api/plat/event_inline_types.h>
+#include <odp/api/plat/event_validation_external.h>
 
 #include <rte_mbuf.h>
 #include <rte_mempool.h>
@@ -94,6 +95,8 @@ _ODP_INLINE void odp_buffer_free(odp_buffer_t buf)
 {
 	struct rte_mbuf *mbuf = (struct rte_mbuf *)buf;
 
+	_odp_buffer_validate(buf, _ODP_EV_BUFFER_FREE);
+
 	rte_mempool_put(mbuf->pool, mbuf);
 }
 
@@ -106,6 +109,8 @@ _ODP_INLINE void odp_buffer_free_multi(const odp_buffer_t buf[], int num)
 	if (odp_unlikely(num <= 0))
 		return;
 
+	_odp_buffer_validate_multi(buf, num, _ODP_EV_BUFFER_FREE_MULTI);
+
 	mbuf_tbl[0] = (struct rte_mbuf *)buf[0];
 	mp_pending = mbuf_tbl[0]->pool;
 	num_pending = 1;
@@ -133,6 +138,9 @@ _ODP_INLINE int odp_buffer_is_valid(odp_buffer_t buf)
 	if (odp_event_type(odp_buffer_to_event(buf)) != ODP_EVENT_BUFFER)
 		return 0;
 
+	if (odp_unlikely(_odp_buffer_validate(buf, _ODP_EV_BUFFER_IS_VALID)))
+		return 0;
+
 	return 1;
 }
 
diff --git a/platform/linux-dpdk/include/odp/api/plat/event_validation_external.h b/platform/linux-dpdk/include/odp/api/plat/event_validation_external.h
new file mode 120000
index 000000000..001662d8b
--- /dev/null
+++ b/platform/linux-dpdk/include/odp/api/plat/event_validation_external.h
@@ -0,0 +1 @@
+../../../../../linux-generic/include/odp/api/plat/event_validation_external.h
\ No newline at end of file
diff --git a/platform/linux-dpdk/include/odp/api/plat/packet_inlines.h b/platform/linux-dpdk/include/odp/api/plat/packet_inlines.h
index 4347afb88..4ae53c7b3 100644
--- a/platform/linux-dpdk/include/odp/api/plat/packet_inlines.h
+++ b/platform/linux-dpdk/include/odp/api/plat/packet_inlines.h
@@ -1,5 +1,5 @@
 /* Copyright (c) 2016-2018, Linaro Limited
- * Copyright (c) 2019-2022, Nokia
+ * Copyright (c) 2019-2023, Nokia
  * All rights reserved.
  *
  * SPDX-License-Identifier: BSD-3-Clause
@@ -25,6 +25,7 @@ extern "C" {
 #include <odp/api/time.h>
 
 #include <odp/api/plat/debug_inlines.h>
+#include <odp/api/plat/event_validation_external.h>
 #include <odp/api/plat/packet_io_inlines.h>
 #include <odp/api/plat/packet_inline_types.h>
 #include <odp/api/plat/pool_inline_types.h>
@@ -663,17 +664,23 @@ _ODP_INLINE odp_event_t odp_packet_tx_compl_to_event(odp_packet_tx_compl_t tx_co
 
 _ODP_INLINE void odp_packet_free(odp_packet_t pkt)
 {
+	_odp_packet_validate(pkt, _ODP_EV_PACKET_FREE);
+
 	rte_pktmbuf_free((struct rte_mbuf *)pkt);
 }
 
 _ODP_INLINE void odp_packet_free_multi(const odp_packet_t pkt[], int num)
 {
+	_odp_packet_validate_multi(pkt, num, _ODP_EV_PACKET_FREE_MULTI);
+
 	rte_pktmbuf_free_bulk((struct rte_mbuf **)(uintptr_t)pkt, (unsigned int)num);
 }
 
 _ODP_INLINE void odp_packet_free_sp(const odp_packet_t pkt[], int num)
 {
-	odp_packet_free_multi(pkt, num);
+	_odp_packet_validate_multi(pkt, num, _ODP_EV_PACKET_FREE_SP);
+
+	rte_pktmbuf_free_bulk((struct rte_mbuf **)(uintptr_t)pkt, (unsigned int)num);
 }
 
 _ODP_INLINE void *odp_packet_seg_data(odp_packet_t pkt ODP_UNUSED,
diff --git a/platform/linux-dpdk/include/odp_event_internal.h b/platform/linux-dpdk/include/odp_event_internal.h
index 320f42146..5ccf5a3ed 100644
--- a/platform/linux-dpdk/include/odp_event_internal.h
+++ b/platform/linux-dpdk/include/odp_event_internal.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2021-2022, Nokia
+/* Copyright (c) 2021-2023, Nokia
  * All rights reserved.
  *
  * SPDX-License-Identifier:     BSD-3-Clause
@@ -80,6 +80,13 @@ static inline void _odp_event_type_set(odp_event_t event, int ev)
 	_odp_event_hdr(event)->event_type = ev;
 }
 
+static inline uint64_t *_odp_event_endmark_get_ptr(odp_event_t event)
+{
+	struct rte_mbuf *mbuf = _odp_event_to_mbuf(event);
+
+	return (uint64_t *)((uint8_t *)mbuf->buf_addr + mbuf->buf_len);
+}
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/platform/linux-dpdk/include/odp_pool_internal.h b/platform/linux-dpdk/include/odp_pool_internal.h
index a5ffcf517..b82324a77 100644
--- a/platform/linux-dpdk/include/odp_pool_internal.h
+++ b/platform/linux-dpdk/include/odp_pool_internal.h
@@ -1,5 +1,5 @@
 /* Copyright (c) 2013-2018, Linaro Limited
- * Copyright (c) 2021-2022, Nokia
+ * Copyright (c) 2021-2023, Nokia
  * All rights reserved.
  *
  * SPDX-License-Identifier:     BSD-3-Clause
@@ -72,7 +72,7 @@ typedef struct ODP_ALIGNED_CACHE {
 	/* Everything under this mark is memset() to zero on pool create */
 	uint8_t			memset_mark;
 	struct rte_mempool	*rte_mempool;
-	uint32_t		seg_len;
+	uint32_t		seg_len; /* Initial packet segment length (excludes endmark) */
 	uint32_t		ext_head_offset;
 	uint32_t		num;
 	uint32_t		num_populated;
@@ -84,6 +84,7 @@ typedef struct ODP_ALIGNED_CACHE {
 	odp_shm_t		uarea_shm;
 	uint64_t		uarea_shm_size;
 	uint32_t		uarea_size;
+	uint32_t		trailer_size; /* Endmark size */
 	uint8_t			*uarea_base_addr;
 	char			name[ODP_POOL_NAME_LEN];
 
diff --git a/platform/linux-dpdk/m4/configure.m4 b/platform/linux-dpdk/m4/configure.m4
index 90d55b6f6..70295400f 100644
--- a/platform/linux-dpdk/m4/configure.m4
+++ b/platform/linux-dpdk/m4/configure.m4
@@ -5,11 +5,13 @@ ODP_VISIBILITY
 ODP_ATOMIC
 
 m4_include([platform/linux-dpdk/m4/odp_cpu.m4])
+m4_include([platform/linux-dpdk/m4/odp_event_validation.m4])
 m4_include([platform/linux-dpdk/m4/odp_libconfig.m4])
 m4_include([platform/linux-dpdk/m4/odp_openssl.m4])
 m4_include([platform/linux-dpdk/m4/odp_pcapng.m4])
 m4_include([platform/linux-dpdk/m4/odp_scheduler.m4])
 
+ODP_EVENT_VALIDATION
 ODP_PTHREAD
 ODP_SCHEDULER
 ODP_TIMER
@@ -68,6 +70,7 @@ AS_VAR_APPEND([PLAT_DEP_LIBS], ["${ATOMIC_LIBS} ${LIBCONFIG_LIBS} ${OPENSSL_LIBS
 # Add text to the end of configure with platform specific settings.
 # Make sure it's aligned same as other lines in configure.ac.
 AS_VAR_APPEND([PLAT_CFG_TEXT], ["
+	event_validation:       ${enable_event_validation}
 	openssl:                ${with_openssl}
 	openssl_rand:           ${openssl_rand}
 	pcap:			${have_pmd_pcap}
diff --git a/platform/linux-dpdk/m4/odp_event_validation.m4 b/platform/linux-dpdk/m4/odp_event_validation.m4
new file mode 120000
index 000000000..0d457c6ff
--- /dev/null
+++ b/platform/linux-dpdk/m4/odp_event_validation.m4
@@ -0,0 +1 @@
+../../linux-generic/m4/odp_event_validation.m4
\ No newline at end of file
diff --git a/platform/linux-dpdk/odp_event.c b/platform/linux-dpdk/odp_event.c
index 24853f83e..625b6eefe 100644
--- a/platform/linux-dpdk/odp_event.c
+++ b/platform/linux-dpdk/odp_event.c
@@ -1,5 +1,5 @@
 /* Copyright (c) 2015-2018, Linaro Limited
- * Copyright (c) 2020-2022, Nokia
+ * Copyright (c) 2020-2023, Nokia
  * All rights reserved.
  *
  * SPDX-License-Identifier:     BSD-3-Clause
@@ -18,6 +18,7 @@
 #include <odp_debug_internal.h>
 #include <odp_packet_internal.h>
 #include <odp_event_internal.h>
+#include <odp_event_validation_internal.h>
 #include <odp_event_vector_internal.h>
 
 /* Inlined API functions */
@@ -42,13 +43,15 @@ _odp_event_inline_offset ODP_ALIGNED_CACHE = {
 
 #include <odp/visibility_end.h>
 
-void odp_event_free(odp_event_t event)
+static inline void event_free(odp_event_t event, _odp_ev_id_t id)
 {
 	switch (odp_event_type(event)) {
 	case ODP_EVENT_BUFFER:
+		_odp_buffer_validate(odp_buffer_from_event(event), id);
 		odp_buffer_free(odp_buffer_from_event(event));
 		break;
 	case ODP_EVENT_PACKET:
+		_odp_packet_validate(odp_packet_from_event(event), id);
 		odp_packet_free(odp_packet_from_event(event));
 		break;
 	case ODP_EVENT_PACKET_VECTOR:
@@ -76,17 +79,21 @@ void odp_event_free(odp_event_t event)
 	}
 }
 
-void odp_event_free_multi(const odp_event_t event[], int num)
+void odp_event_free(odp_event_t event)
 {
-	int i;
+	event_free(event, _ODP_EV_EVENT_FREE);
+}
 
-	for (i = 0; i < num; i++)
-		odp_event_free(event[i]);
+void odp_event_free_multi(const odp_event_t event[], int num)
+{
+	for (int i = 0; i < num; i++)
+		event_free(event[i], _ODP_EV_EVENT_FREE_MULTI);
 }
 
 void odp_event_free_sp(const odp_event_t event[], int num)
 {
-	odp_event_free_multi(event, num);
+	for (int i = 0; i < num; i++)
+		event_free(event[i], _ODP_EV_EVENT_FREE_SP);
 }
 
 uint64_t odp_event_to_u64(odp_event_t hdl)
@@ -104,9 +111,9 @@ int odp_event_is_valid(odp_event_t event)
 
 	switch (odp_event_type(event)) {
 	case ODP_EVENT_BUFFER:
-		/* Fall through */
+		return !_odp_buffer_validate(odp_buffer_from_event(event), _ODP_EV_EVENT_IS_VALID);
 	case ODP_EVENT_PACKET:
-		/* Fall through */
+		return !_odp_packet_validate(odp_packet_from_event(event), _ODP_EV_EVENT_IS_VALID);
 	case ODP_EVENT_TIMEOUT:
 		/* Fall through */
 #if ODP_DEPRECATED_API
diff --git a/platform/linux-dpdk/odp_init.c b/platform/linux-dpdk/odp_init.c
index 329252f9c..b144835ea 100644
--- a/platform/linux-dpdk/odp_init.c
+++ b/platform/linux-dpdk/odp_init.c
@@ -1,5 +1,5 @@
 /* Copyright (c) 2013-2018, Linaro Limited
- * Copyright (c) 2019-2021, Nokia
+ * Copyright (c) 2019-2023, Nokia
  * All rights reserved.
  *
  * SPDX-License-Identifier:     BSD-3-Clause
@@ -41,6 +41,7 @@ enum init_stage {
 	HASH_INIT,
 	THREAD_INIT,
 	POOL_INIT,
+	EVENT_VALIDATION_INIT,
 	STASH_INIT,
 	QUEUE_INIT,
 	SCHED_INIT,
@@ -426,6 +427,13 @@ static int term_global(enum init_stage stage)
 		}
 		/* Fall through */
 
+	case EVENT_VALIDATION_INIT:
+		if (_odp_event_validation_term_global()) {
+			_ODP_ERR("ODP event validation term failed.\n");
+			rc = -1;
+		}
+		/* Fall through */
+
 	case POOL_INIT:
 		if (_odp_pool_term_global()) {
 			_ODP_ERR("ODP buffer pool term failed.\n");
@@ -590,6 +598,12 @@ int odp_init_global(odp_instance_t *instance,
 	}
 	stage = POOL_INIT;
 
+	if (_odp_event_validation_init_global()) {
+		_ODP_ERR("ODP event validation init failed.\n");
+		goto init_failed;
+	}
+	stage = EVENT_VALIDATION_INIT;
+
 	if (_odp_stash_init_global()) {
 		_ODP_ERR("ODP stash init failed.\n");
 		goto init_failed;
diff --git a/platform/linux-dpdk/odp_packet.c b/platform/linux-dpdk/odp_packet.c
index ba0b6c908..9b8755ce9 100644
--- a/platform/linux-dpdk/odp_packet.c
+++ b/platform/linux-dpdk/odp_packet.c
@@ -24,6 +24,7 @@
 #include <odp_chksum_internal.h>
 #include <odp_debug_internal.h>
 #include <odp_event_internal.h>
+#include <odp_event_validation_internal.h>
 #include <odp_macros_internal.h>
 #include <odp_packet_internal.h>
 #include <odp_packet_io_internal.h>
@@ -1145,6 +1146,9 @@ int odp_packet_is_valid(odp_packet_t pkt)
 	if (odp_event_type(ev) != ODP_EVENT_PACKET)
 		return 0;
 
+	if (odp_unlikely(_odp_packet_validate(pkt, _ODP_EV_PACKET_IS_VALID)))
+		return 0;
+
 	switch (odp_event_subtype(ev)) {
 	case ODP_EVENT_PACKET_BASIC:
 		/* Fall through */
diff --git a/platform/linux-dpdk/odp_pool.c b/platform/linux-dpdk/odp_pool.c
index b0df7997a..d36f197c0 100644
--- a/platform/linux-dpdk/odp_pool.c
+++ b/platform/linux-dpdk/odp_pool.c
@@ -1,5 +1,5 @@
 /* Copyright (c) 2013-2018, Linaro Limited
- * Copyright (c) 2019-2022, Nokia
+ * Copyright (c) 2019-2023, Nokia
  * All rights reserved.
  *
  * SPDX-License-Identifier:     BSD-3-Clause
@@ -16,6 +16,8 @@
 #include <odp_buffer_internal.h>
 #include <odp_config_internal.h>
 #include <odp_debug_internal.h>
+#include <odp_event_internal.h>
+#include <odp_event_validation_internal.h>
 #include <odp_event_vector_internal.h>
 #include <odp_init_internal.h>
 #include <odp_libconfig_internal.h>
@@ -380,6 +382,12 @@ odp_dpdk_mbuf_ctor(struct rte_mempool *mp,
 		pkt_hdr->uarea_addr = uarea;
 	}
 
+	/* Initialize data endmark */
+	if (mb_ctor_arg->type == ODP_POOL_BUFFER || mb_ctor_arg->type == ODP_POOL_PACKET) {
+		mb->buf_len -= _ODP_EV_ENDMARK_SIZE;
+		_odp_event_endmark_set(_odp_event_from_mbuf(mb));
+	}
+
 	/* Initialize event vector metadata */
 	if (mb_ctor_arg->type == ODP_POOL_VECTOR) {
 		odp_event_vector_hdr_t *vect_hdr = (void *)raw_mbuf;
@@ -679,6 +687,7 @@ odp_pool_t _odp_pool_create(const char *name, const odp_pool_param_t *params,
 	uint32_t buf_align, blk_size, headroom, tailroom, min_seg_len;
 	uint32_t max_len, min_align;
 	uint32_t uarea_size = 0;
+	uint32_t trailer_size = 0;
 	int8_t event_type;
 	char pool_name[ODP_POOL_NAME_LEN];
 	char rte_name[RTE_MEMPOOL_NAMESIZE];
@@ -709,7 +718,8 @@ odp_pool_t _odp_pool_create(const char *name, const odp_pool_param_t *params,
 		switch (type) {
 		case ODP_POOL_BUFFER:
 			buf_align = params->buf.align;
-			blk_size = params->buf.size;
+			trailer_size = _ODP_EV_ENDMARK_SIZE;
+			blk_size = params->buf.size + trailer_size;
 			cache_size = params->buf.cache_size;
 			uarea_size = params->buf.uarea_size;
 
@@ -734,6 +744,7 @@ odp_pool_t _odp_pool_create(const char *name, const odp_pool_param_t *params,
 		case ODP_POOL_PACKET:
 			headroom = RTE_PKTMBUF_HEADROOM;
 			tailroom = CONFIG_PACKET_TAILROOM;
+			trailer_size = _ODP_EV_ENDMARK_SIZE;
 			min_seg_len = CONFIG_PACKET_SEG_LEN_MIN;
 			min_align = ODP_CONFIG_BUFFER_ALIGN_MIN;
 			cache_size = params->pkt.cache_size;
@@ -743,16 +754,16 @@ odp_pool_t _odp_pool_create(const char *name, const odp_pool_param_t *params,
 				blk_size = params->pkt.seg_len;
 			if (params->pkt.len > blk_size)
 				blk_size = params->pkt.len;
-			/* Make sure at least one max len packet fits in the
-			 * pool.
-			 */
+
+			/* Make sure at least one max len packet fits in the pool */
 			max_len = 0;
 			if (params->pkt.max_len != 0)
 				max_len = params->pkt.max_len;
 			if ((max_len + blk_size) / blk_size > params->pkt.num)
 				blk_size = (max_len + params->pkt.num) /
 					params->pkt.num;
-			blk_size = _ODP_ROUNDUP_ALIGN(headroom + blk_size + tailroom, min_align);
+			blk_size = _ODP_ROUNDUP_ALIGN(headroom + blk_size + tailroom + trailer_size,
+						      min_align);
 
 			/* Segment size minus headroom might be rounded down by the driver (e.g.
 			 * ixgbe) to the nearest multiple of 1024. Round it up here to make sure the
@@ -827,7 +838,7 @@ odp_pool_t _odp_pool_create(const char *name, const odp_pool_param_t *params,
 			mp = rte_pktmbuf_pool_create(rte_name, num, cache_size,
 						     priv_size, data_room_size,
 						     rte_socket_id());
-			pool->seg_len = data_room_size;
+			pool->seg_len = data_room_size - trailer_size;
 		} else {
 			unsigned int priv_size;
 
@@ -855,6 +866,7 @@ odp_pool_t _odp_pool_create(const char *name, const odp_pool_param_t *params,
 		pool->type = type;
 		pool->type_2 = type_2;
 		pool->params = *params;
+		pool->trailer_size = trailer_size;
 
 		if (reserve_uarea(pool, uarea_size, num)) {
 			_ODP_ERR("User area SHM reserve failed\n");
@@ -1157,7 +1169,7 @@ int odp_pool_ext_capability(odp_pool_type_t type,
 	capa->pkt.max_num_buf = _odp_pool_glb->config.pkt_max_num;
 	capa->pkt.max_buf_size = MAX_SIZE;
 	capa->pkt.odp_header_size = SIZEOF_OBJHDR + sizeof(odp_packet_hdr_t);
-	capa->pkt.odp_trailer_size = 0;
+	capa->pkt.odp_trailer_size = _ODP_EV_ENDMARK_SIZE;
 	capa->pkt.min_mem_align = ODP_CACHE_LINE_SIZE;
 	capa->pkt.min_buf_align = ODP_CACHE_LINE_SIZE;
 	capa->pkt.min_head_align = EXT_MIN_HEAD_ALIGN;
@@ -1326,12 +1338,13 @@ odp_pool_t odp_pool_ext_create(const char *name,
 
 		pool->ext_param = *params;
 		pool->ext_head_offset = hdr_size;
+		pool->trailer_size = _ODP_EV_ENDMARK_SIZE;
 		pool->num = num;
 		pool->num_populated = 0;
 		pool->params.pkt.uarea_size = params->pkt.uarea_size;
 		pool->params.type = params->type;
 		pool->pool_ext = 1;
-		pool->seg_len = blk_size;
+		pool->seg_len = blk_size - pool->trailer_size;
 		pool->type = params->type;
 		strcpy(pool->name, pool_name);
 
@@ -1449,7 +1462,7 @@ int odp_pool_ext_populate(odp_pool_t pool_hdl, void *buf[], uint32_t buf_size,
 
 		mb_ctor_arg.seg_buf_offset = sizeof(odp_packet_hdr_t) +
 					     params->pkt.app_header_size;
-		mb_ctor_arg.seg_buf_size = pool->seg_len;
+		mb_ctor_arg.seg_buf_size = pool->seg_len + pool->trailer_size;
 		mb_ctor_arg.type = params->type;
 		mb_ctor_arg.event_type = pool->type;
 		mb_ctor_arg.pool = pool;