From 3f0a93e790849b69fc3a99649880f7c6821ebf4c Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Mon, 23 Jan 2023 10:43:01 +0200 Subject: Port a735a7131 "linux-gen: event: add event validation support" Port original commit from linux-generic. Signed-off-by: Matias Elo Reviewed-by: Tuomas Taipale --- platform/linux-dpdk/Makefile.am | 3 ++ .../include/odp/api/plat/buffer_inlines.h | 10 ++++++- .../odp/api/plat/event_validation_external.h | 1 + .../include/odp/api/plat/packet_inlines.h | 11 ++++++-- platform/linux-dpdk/include/odp_event_internal.h | 9 +++++- platform/linux-dpdk/include/odp_pool_internal.h | 5 ++-- platform/linux-dpdk/m4/configure.m4 | 3 ++ platform/linux-dpdk/m4/odp_event_validation.m4 | 1 + platform/linux-dpdk/odp_event.c | 25 ++++++++++------ platform/linux-dpdk/odp_init.c | 16 ++++++++++- platform/linux-dpdk/odp_packet.c | 4 +++ platform/linux-dpdk/odp_pool.c | 33 +++++++++++++++------- 12 files changed, 95 insertions(+), 26 deletions(-) create mode 120000 platform/linux-dpdk/include/odp/api/plat/event_validation_external.h create mode 120000 platform/linux-dpdk/m4/odp_event_validation.m4 diff --git a/platform/linux-dpdk/Makefile.am b/platform/linux-dpdk/Makefile.am index 31148168b..7f45d7683 100644 --- a/platform/linux-dpdk/Makefile.am +++ b/platform/linux-dpdk/Makefile.am @@ -36,6 +36,7 @@ odpapiplatinclude_HEADERS = \ include/odp/api/plat/debug_inlines.h \ include/odp/api/plat/event_inlines.h \ include/odp/api/plat/event_inline_types.h \ + include/odp/api/plat/event_validation_external.h \ include/odp/api/plat/event_vector_inline_types.h \ include/odp/api/plat/hash_inlines.h \ include/odp/api/plat/ipsec_inlines.h \ @@ -144,6 +145,7 @@ noinst_HEADERS = \ ${top_srcdir}/platform/linux-generic/include/odp_print_internal.h \ include/odp_errno_define.h \ include/odp_event_internal.h \ + ${top_srcdir}/platform/linux-generic/include/odp_event_validation_internal.h \ ${top_srcdir}/platform/linux-generic/include/odp_packet_dpdk.h \ ${top_srcdir}/platform/linux-generic/include/odp_pcapng.h \ ${top_srcdir}/platform/linux-generic/include/odp_pkt_queue_internal.h \ @@ -200,6 +202,7 @@ __LIB__libodp_dpdk_la_SOURCES = \ odp_crypto.c \ odp_errno.c \ odp_event.c \ + ../linux-generic/odp_event_validation.c \ ../linux-generic/odp_hash_crc_gen.c \ odp_init.c \ ../linux-generic/odp_impl.c \ diff --git a/platform/linux-dpdk/include/odp/api/plat/buffer_inlines.h b/platform/linux-dpdk/include/odp/api/plat/buffer_inlines.h index 37a3bf296..396d78559 100644 --- a/platform/linux-dpdk/include/odp/api/plat/buffer_inlines.h +++ b/platform/linux-dpdk/include/odp/api/plat/buffer_inlines.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2019-2022, Nokia +/* Copyright (c) 2019-2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -15,6 +15,7 @@ #include #include #include +#include #include #include @@ -94,6 +95,8 @@ _ODP_INLINE void odp_buffer_free(odp_buffer_t buf) { struct rte_mbuf *mbuf = (struct rte_mbuf *)buf; + _odp_buffer_validate(buf, _ODP_EV_BUFFER_FREE); + rte_mempool_put(mbuf->pool, mbuf); } @@ -106,6 +109,8 @@ _ODP_INLINE void odp_buffer_free_multi(const odp_buffer_t buf[], int num) if (odp_unlikely(num <= 0)) return; + _odp_buffer_validate_multi(buf, num, _ODP_EV_BUFFER_FREE_MULTI); + mbuf_tbl[0] = (struct rte_mbuf *)buf[0]; mp_pending = mbuf_tbl[0]->pool; num_pending = 1; @@ -133,6 +138,9 @@ _ODP_INLINE int odp_buffer_is_valid(odp_buffer_t buf) if (odp_event_type(odp_buffer_to_event(buf)) != ODP_EVENT_BUFFER) return 0; + if (odp_unlikely(_odp_buffer_validate(buf, _ODP_EV_BUFFER_IS_VALID))) + return 0; + return 1; } diff --git a/platform/linux-dpdk/include/odp/api/plat/event_validation_external.h b/platform/linux-dpdk/include/odp/api/plat/event_validation_external.h new file mode 120000 index 000000000..001662d8b --- /dev/null +++ b/platform/linux-dpdk/include/odp/api/plat/event_validation_external.h @@ -0,0 +1 @@ +../../../../../linux-generic/include/odp/api/plat/event_validation_external.h \ No newline at end of file diff --git a/platform/linux-dpdk/include/odp/api/plat/packet_inlines.h b/platform/linux-dpdk/include/odp/api/plat/packet_inlines.h index 4347afb88..4ae53c7b3 100644 --- a/platform/linux-dpdk/include/odp/api/plat/packet_inlines.h +++ b/platform/linux-dpdk/include/odp/api/plat/packet_inlines.h @@ -1,5 +1,5 @@ /* Copyright (c) 2016-2018, Linaro Limited - * Copyright (c) 2019-2022, Nokia + * Copyright (c) 2019-2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -25,6 +25,7 @@ extern "C" { #include #include +#include #include #include #include @@ -663,17 +664,23 @@ _ODP_INLINE odp_event_t odp_packet_tx_compl_to_event(odp_packet_tx_compl_t tx_co _ODP_INLINE void odp_packet_free(odp_packet_t pkt) { + _odp_packet_validate(pkt, _ODP_EV_PACKET_FREE); + rte_pktmbuf_free((struct rte_mbuf *)pkt); } _ODP_INLINE void odp_packet_free_multi(const odp_packet_t pkt[], int num) { + _odp_packet_validate_multi(pkt, num, _ODP_EV_PACKET_FREE_MULTI); + rte_pktmbuf_free_bulk((struct rte_mbuf **)(uintptr_t)pkt, (unsigned int)num); } _ODP_INLINE void odp_packet_free_sp(const odp_packet_t pkt[], int num) { - odp_packet_free_multi(pkt, num); + _odp_packet_validate_multi(pkt, num, _ODP_EV_PACKET_FREE_SP); + + rte_pktmbuf_free_bulk((struct rte_mbuf **)(uintptr_t)pkt, (unsigned int)num); } _ODP_INLINE void *odp_packet_seg_data(odp_packet_t pkt ODP_UNUSED, diff --git a/platform/linux-dpdk/include/odp_event_internal.h b/platform/linux-dpdk/include/odp_event_internal.h index 320f42146..5ccf5a3ed 100644 --- a/platform/linux-dpdk/include/odp_event_internal.h +++ b/platform/linux-dpdk/include/odp_event_internal.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2021-2022, Nokia +/* Copyright (c) 2021-2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -80,6 +80,13 @@ static inline void _odp_event_type_set(odp_event_t event, int ev) _odp_event_hdr(event)->event_type = ev; } +static inline uint64_t *_odp_event_endmark_get_ptr(odp_event_t event) +{ + struct rte_mbuf *mbuf = _odp_event_to_mbuf(event); + + return (uint64_t *)((uint8_t *)mbuf->buf_addr + mbuf->buf_len); +} + #ifdef __cplusplus } #endif diff --git a/platform/linux-dpdk/include/odp_pool_internal.h b/platform/linux-dpdk/include/odp_pool_internal.h index a5ffcf517..b82324a77 100644 --- a/platform/linux-dpdk/include/odp_pool_internal.h +++ b/platform/linux-dpdk/include/odp_pool_internal.h @@ -1,5 +1,5 @@ /* Copyright (c) 2013-2018, Linaro Limited - * Copyright (c) 2021-2022, Nokia + * Copyright (c) 2021-2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -72,7 +72,7 @@ typedef struct ODP_ALIGNED_CACHE { /* Everything under this mark is memset() to zero on pool create */ uint8_t memset_mark; struct rte_mempool *rte_mempool; - uint32_t seg_len; + uint32_t seg_len; /* Initial packet segment length (excludes endmark) */ uint32_t ext_head_offset; uint32_t num; uint32_t num_populated; @@ -84,6 +84,7 @@ typedef struct ODP_ALIGNED_CACHE { odp_shm_t uarea_shm; uint64_t uarea_shm_size; uint32_t uarea_size; + uint32_t trailer_size; /* Endmark size */ uint8_t *uarea_base_addr; char name[ODP_POOL_NAME_LEN]; diff --git a/platform/linux-dpdk/m4/configure.m4 b/platform/linux-dpdk/m4/configure.m4 index 90d55b6f6..70295400f 100644 --- a/platform/linux-dpdk/m4/configure.m4 +++ b/platform/linux-dpdk/m4/configure.m4 @@ -5,11 +5,13 @@ ODP_VISIBILITY ODP_ATOMIC m4_include([platform/linux-dpdk/m4/odp_cpu.m4]) +m4_include([platform/linux-dpdk/m4/odp_event_validation.m4]) m4_include([platform/linux-dpdk/m4/odp_libconfig.m4]) m4_include([platform/linux-dpdk/m4/odp_openssl.m4]) m4_include([platform/linux-dpdk/m4/odp_pcapng.m4]) m4_include([platform/linux-dpdk/m4/odp_scheduler.m4]) +ODP_EVENT_VALIDATION ODP_PTHREAD ODP_SCHEDULER ODP_TIMER @@ -68,6 +70,7 @@ AS_VAR_APPEND([PLAT_DEP_LIBS], ["${ATOMIC_LIBS} ${LIBCONFIG_LIBS} ${OPENSSL_LIBS # Add text to the end of configure with platform specific settings. # Make sure it's aligned same as other lines in configure.ac. AS_VAR_APPEND([PLAT_CFG_TEXT], [" + event_validation: ${enable_event_validation} openssl: ${with_openssl} openssl_rand: ${openssl_rand} pcap: ${have_pmd_pcap} diff --git a/platform/linux-dpdk/m4/odp_event_validation.m4 b/platform/linux-dpdk/m4/odp_event_validation.m4 new file mode 120000 index 000000000..0d457c6ff --- /dev/null +++ b/platform/linux-dpdk/m4/odp_event_validation.m4 @@ -0,0 +1 @@ +../../linux-generic/m4/odp_event_validation.m4 \ No newline at end of file diff --git a/platform/linux-dpdk/odp_event.c b/platform/linux-dpdk/odp_event.c index 24853f83e..625b6eefe 100644 --- a/platform/linux-dpdk/odp_event.c +++ b/platform/linux-dpdk/odp_event.c @@ -1,5 +1,5 @@ /* Copyright (c) 2015-2018, Linaro Limited - * Copyright (c) 2020-2022, Nokia + * Copyright (c) 2020-2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -18,6 +18,7 @@ #include #include #include +#include #include /* Inlined API functions */ @@ -42,13 +43,15 @@ _odp_event_inline_offset ODP_ALIGNED_CACHE = { #include -void odp_event_free(odp_event_t event) +static inline void event_free(odp_event_t event, _odp_ev_id_t id) { switch (odp_event_type(event)) { case ODP_EVENT_BUFFER: + _odp_buffer_validate(odp_buffer_from_event(event), id); odp_buffer_free(odp_buffer_from_event(event)); break; case ODP_EVENT_PACKET: + _odp_packet_validate(odp_packet_from_event(event), id); odp_packet_free(odp_packet_from_event(event)); break; case ODP_EVENT_PACKET_VECTOR: @@ -76,17 +79,21 @@ void odp_event_free(odp_event_t event) } } -void odp_event_free_multi(const odp_event_t event[], int num) +void odp_event_free(odp_event_t event) { - int i; + event_free(event, _ODP_EV_EVENT_FREE); +} - for (i = 0; i < num; i++) - odp_event_free(event[i]); +void odp_event_free_multi(const odp_event_t event[], int num) +{ + for (int i = 0; i < num; i++) + event_free(event[i], _ODP_EV_EVENT_FREE_MULTI); } void odp_event_free_sp(const odp_event_t event[], int num) { - odp_event_free_multi(event, num); + for (int i = 0; i < num; i++) + event_free(event[i], _ODP_EV_EVENT_FREE_SP); } uint64_t odp_event_to_u64(odp_event_t hdl) @@ -104,9 +111,9 @@ int odp_event_is_valid(odp_event_t event) switch (odp_event_type(event)) { case ODP_EVENT_BUFFER: - /* Fall through */ + return !_odp_buffer_validate(odp_buffer_from_event(event), _ODP_EV_EVENT_IS_VALID); case ODP_EVENT_PACKET: - /* Fall through */ + return !_odp_packet_validate(odp_packet_from_event(event), _ODP_EV_EVENT_IS_VALID); case ODP_EVENT_TIMEOUT: /* Fall through */ #if ODP_DEPRECATED_API diff --git a/platform/linux-dpdk/odp_init.c b/platform/linux-dpdk/odp_init.c index 329252f9c..b144835ea 100644 --- a/platform/linux-dpdk/odp_init.c +++ b/platform/linux-dpdk/odp_init.c @@ -1,5 +1,5 @@ /* Copyright (c) 2013-2018, Linaro Limited - * Copyright (c) 2019-2021, Nokia + * Copyright (c) 2019-2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -41,6 +41,7 @@ enum init_stage { HASH_INIT, THREAD_INIT, POOL_INIT, + EVENT_VALIDATION_INIT, STASH_INIT, QUEUE_INIT, SCHED_INIT, @@ -426,6 +427,13 @@ static int term_global(enum init_stage stage) } /* Fall through */ + case EVENT_VALIDATION_INIT: + if (_odp_event_validation_term_global()) { + _ODP_ERR("ODP event validation term failed.\n"); + rc = -1; + } + /* Fall through */ + case POOL_INIT: if (_odp_pool_term_global()) { _ODP_ERR("ODP buffer pool term failed.\n"); @@ -590,6 +598,12 @@ int odp_init_global(odp_instance_t *instance, } stage = POOL_INIT; + if (_odp_event_validation_init_global()) { + _ODP_ERR("ODP event validation init failed.\n"); + goto init_failed; + } + stage = EVENT_VALIDATION_INIT; + if (_odp_stash_init_global()) { _ODP_ERR("ODP stash init failed.\n"); goto init_failed; diff --git a/platform/linux-dpdk/odp_packet.c b/platform/linux-dpdk/odp_packet.c index ba0b6c908..9b8755ce9 100644 --- a/platform/linux-dpdk/odp_packet.c +++ b/platform/linux-dpdk/odp_packet.c @@ -24,6 +24,7 @@ #include #include #include +#include #include #include #include @@ -1145,6 +1146,9 @@ int odp_packet_is_valid(odp_packet_t pkt) if (odp_event_type(ev) != ODP_EVENT_PACKET) return 0; + if (odp_unlikely(_odp_packet_validate(pkt, _ODP_EV_PACKET_IS_VALID))) + return 0; + switch (odp_event_subtype(ev)) { case ODP_EVENT_PACKET_BASIC: /* Fall through */ diff --git a/platform/linux-dpdk/odp_pool.c b/platform/linux-dpdk/odp_pool.c index b0df7997a..d36f197c0 100644 --- a/platform/linux-dpdk/odp_pool.c +++ b/platform/linux-dpdk/odp_pool.c @@ -1,5 +1,5 @@ /* Copyright (c) 2013-2018, Linaro Limited - * Copyright (c) 2019-2022, Nokia + * Copyright (c) 2019-2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -16,6 +16,8 @@ #include #include #include +#include +#include #include #include #include @@ -380,6 +382,12 @@ odp_dpdk_mbuf_ctor(struct rte_mempool *mp, pkt_hdr->uarea_addr = uarea; } + /* Initialize data endmark */ + if (mb_ctor_arg->type == ODP_POOL_BUFFER || mb_ctor_arg->type == ODP_POOL_PACKET) { + mb->buf_len -= _ODP_EV_ENDMARK_SIZE; + _odp_event_endmark_set(_odp_event_from_mbuf(mb)); + } + /* Initialize event vector metadata */ if (mb_ctor_arg->type == ODP_POOL_VECTOR) { odp_event_vector_hdr_t *vect_hdr = (void *)raw_mbuf; @@ -679,6 +687,7 @@ odp_pool_t _odp_pool_create(const char *name, const odp_pool_param_t *params, uint32_t buf_align, blk_size, headroom, tailroom, min_seg_len; uint32_t max_len, min_align; uint32_t uarea_size = 0; + uint32_t trailer_size = 0; int8_t event_type; char pool_name[ODP_POOL_NAME_LEN]; char rte_name[RTE_MEMPOOL_NAMESIZE]; @@ -709,7 +718,8 @@ odp_pool_t _odp_pool_create(const char *name, const odp_pool_param_t *params, switch (type) { case ODP_POOL_BUFFER: buf_align = params->buf.align; - blk_size = params->buf.size; + trailer_size = _ODP_EV_ENDMARK_SIZE; + blk_size = params->buf.size + trailer_size; cache_size = params->buf.cache_size; uarea_size = params->buf.uarea_size; @@ -734,6 +744,7 @@ odp_pool_t _odp_pool_create(const char *name, const odp_pool_param_t *params, case ODP_POOL_PACKET: headroom = RTE_PKTMBUF_HEADROOM; tailroom = CONFIG_PACKET_TAILROOM; + trailer_size = _ODP_EV_ENDMARK_SIZE; min_seg_len = CONFIG_PACKET_SEG_LEN_MIN; min_align = ODP_CONFIG_BUFFER_ALIGN_MIN; cache_size = params->pkt.cache_size; @@ -743,16 +754,16 @@ odp_pool_t _odp_pool_create(const char *name, const odp_pool_param_t *params, blk_size = params->pkt.seg_len; if (params->pkt.len > blk_size) blk_size = params->pkt.len; - /* Make sure at least one max len packet fits in the - * pool. - */ + + /* Make sure at least one max len packet fits in the pool */ max_len = 0; if (params->pkt.max_len != 0) max_len = params->pkt.max_len; if ((max_len + blk_size) / blk_size > params->pkt.num) blk_size = (max_len + params->pkt.num) / params->pkt.num; - blk_size = _ODP_ROUNDUP_ALIGN(headroom + blk_size + tailroom, min_align); + blk_size = _ODP_ROUNDUP_ALIGN(headroom + blk_size + tailroom + trailer_size, + min_align); /* Segment size minus headroom might be rounded down by the driver (e.g. * ixgbe) to the nearest multiple of 1024. Round it up here to make sure the @@ -827,7 +838,7 @@ odp_pool_t _odp_pool_create(const char *name, const odp_pool_param_t *params, mp = rte_pktmbuf_pool_create(rte_name, num, cache_size, priv_size, data_room_size, rte_socket_id()); - pool->seg_len = data_room_size; + pool->seg_len = data_room_size - trailer_size; } else { unsigned int priv_size; @@ -855,6 +866,7 @@ odp_pool_t _odp_pool_create(const char *name, const odp_pool_param_t *params, pool->type = type; pool->type_2 = type_2; pool->params = *params; + pool->trailer_size = trailer_size; if (reserve_uarea(pool, uarea_size, num)) { _ODP_ERR("User area SHM reserve failed\n"); @@ -1157,7 +1169,7 @@ int odp_pool_ext_capability(odp_pool_type_t type, capa->pkt.max_num_buf = _odp_pool_glb->config.pkt_max_num; capa->pkt.max_buf_size = MAX_SIZE; capa->pkt.odp_header_size = SIZEOF_OBJHDR + sizeof(odp_packet_hdr_t); - capa->pkt.odp_trailer_size = 0; + capa->pkt.odp_trailer_size = _ODP_EV_ENDMARK_SIZE; capa->pkt.min_mem_align = ODP_CACHE_LINE_SIZE; capa->pkt.min_buf_align = ODP_CACHE_LINE_SIZE; capa->pkt.min_head_align = EXT_MIN_HEAD_ALIGN; @@ -1326,12 +1338,13 @@ odp_pool_t odp_pool_ext_create(const char *name, pool->ext_param = *params; pool->ext_head_offset = hdr_size; + pool->trailer_size = _ODP_EV_ENDMARK_SIZE; pool->num = num; pool->num_populated = 0; pool->params.pkt.uarea_size = params->pkt.uarea_size; pool->params.type = params->type; pool->pool_ext = 1; - pool->seg_len = blk_size; + pool->seg_len = blk_size - pool->trailer_size; pool->type = params->type; strcpy(pool->name, pool_name); @@ -1449,7 +1462,7 @@ int odp_pool_ext_populate(odp_pool_t pool_hdl, void *buf[], uint32_t buf_size, mb_ctor_arg.seg_buf_offset = sizeof(odp_packet_hdr_t) + params->pkt.app_header_size; - mb_ctor_arg.seg_buf_size = pool->seg_len; + mb_ctor_arg.seg_buf_size = pool->seg_len + pool->trailer_size; mb_ctor_arg.type = params->type; mb_ctor_arg.event_type = pool->type; mb_ctor_arg.pool = pool; -- cgit v1.2.3