From e2eb952cc47843f453edd01bc3b153feeff5ef68 Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Fri, 28 Oct 2022 18:02:29 +0300 Subject: linux-gen: crypto: inline event conversion functions Inline odp_crypto_packet_from_event() and odp_crypto_packet_to_event() function implementations. Signed-off-by: Matias Elo Reviewed-by: Tuomas Taipale --- .../include/odp/api/plat/crypto_inlines.h | 49 ++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 platform/linux-generic/include/odp/api/plat/crypto_inlines.h (limited to 'platform/linux-generic/include') diff --git a/platform/linux-generic/include/odp/api/plat/crypto_inlines.h b/platform/linux-generic/include/odp/api/plat/crypto_inlines.h new file mode 100644 index 000000000..cfea67283 --- /dev/null +++ b/platform/linux-generic/include/odp/api/plat/crypto_inlines.h @@ -0,0 +1,49 @@ +/* Copyright (c) 2022, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_PLAT_CRYPTO_INLINES_H_ +#define ODP_PLAT_CRYPTO_INLINES_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include + +#include + +/** @cond _ODP_HIDE_FROM_DOXYGEN_ */ + +#ifndef _ODP_NO_INLINE + /* Inline functions by default */ + #define _ODP_INLINE static inline + #define odp_crypto_packet_from_event __odp_crypto_packet_from_event + #define odp_crypto_packet_to_event __odp_crypto_packet_to_event +#else + #define _ODP_INLINE +#endif + +_ODP_INLINE odp_packet_t odp_crypto_packet_from_event(odp_event_t ev) +{ + _ODP_ASSERT(odp_event_type(ev) == ODP_EVENT_PACKET); + _ODP_ASSERT(odp_event_subtype(ev) == ODP_EVENT_PACKET_CRYPTO); + + return odp_packet_from_event(ev); +} + +_ODP_INLINE odp_event_t odp_crypto_packet_to_event(odp_packet_t pkt) +{ + return odp_packet_to_event(pkt); +} + +/** @endcond */ + +#ifdef __cplusplus +} +#endif + +#endif -- cgit v1.2.3 From 76e9963229adb40410d7e4439bac35ec92e42c46 Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Fri, 28 Oct 2022 18:15:36 +0300 Subject: linux-gen: crypto: inline odp_crypto_result() implementation Inline odp_crypto_result() function implementation. Signed-off-by: Matias Elo Reviewed-by: Tuomas Taipale --- .../linux-generic/include/odp/api/plat/crypto_inlines.h | 16 ++++++++++++++++ .../include/odp/api/plat/packet_inline_types.h | 1 + 2 files changed, 17 insertions(+) (limited to 'platform/linux-generic/include') diff --git a/platform/linux-generic/include/odp/api/plat/crypto_inlines.h b/platform/linux-generic/include/odp/api/plat/crypto_inlines.h index cfea67283..8e98d8580 100644 --- a/platform/linux-generic/include/odp/api/plat/crypto_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/crypto_inlines.h @@ -11,10 +11,12 @@ extern "C" { #endif +#include #include #include #include +#include /** @cond _ODP_HIDE_FROM_DOXYGEN_ */ @@ -23,6 +25,7 @@ extern "C" { #define _ODP_INLINE static inline #define odp_crypto_packet_from_event __odp_crypto_packet_from_event #define odp_crypto_packet_to_event __odp_crypto_packet_to_event + #define odp_crypto_result __odp_crypto_result #else #define _ODP_INLINE #endif @@ -40,6 +43,19 @@ _ODP_INLINE odp_event_t odp_crypto_packet_to_event(odp_packet_t pkt) return odp_packet_to_event(pkt); } +_ODP_INLINE int odp_crypto_result(odp_crypto_packet_result_t *result, odp_packet_t pkt) +{ + odp_crypto_packet_result_t *op_result; + + _ODP_ASSERT(odp_packet_subtype(pkt) == ODP_EVENT_PACKET_CRYPTO); + + op_result = _odp_pkt_get_ptr(pkt, odp_crypto_packet_result_t, crypto_op); + + *result = *op_result; + + return 0; +} + /** @endcond */ #ifdef __cplusplus diff --git a/platform/linux-generic/include/odp/api/plat/packet_inline_types.h b/platform/linux-generic/include/odp/api/plat/packet_inline_types.h index ae03457f9..6773b73ad 100644 --- a/platform/linux-generic/include/odp/api/plat/packet_inline_types.h +++ b/platform/linux-generic/include/odp/api/plat/packet_inline_types.h @@ -53,6 +53,7 @@ typedef struct _odp_packet_inline_offset_t { uint16_t subtype; uint16_t cls_mark; uint16_t ipsec_ctx; + uint16_t crypto_op; } _odp_packet_inline_offset_t; -- cgit v1.2.3 From a735a7131bceadb87c25b553446e4829923bcc1c Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Wed, 29 Jun 2022 16:36:22 +0300 Subject: linux-gen: event: add event validation support Add support for runtime event validation (initially buffer endmark checking). Event validation can be enabled during configure with '--enable-event-validation' [warn/abort] or with --enabled-debug=full. When event validation is enabled, endmarks are checked in: - odp_buffer_free() / odp_buffer_free_multi() - odp_buffer_is_valid() - odp_event_free() / odp_event_free_multi() / odp_event_free_sp() - odp_event_is_valid() - odp_packet_free() / odp_packet_free_multi() / odp_packet_free_sp() - odp_packet_is_valid() Signed-off-by: Matias Elo Reviewed-by: Tuomas Taipale --- .../odp/api/plat/event_validation_external.h | 93 ++++++++++++++++++++++ .../include/odp/api/plat/packet_inlines.h | 3 +- .../include/odp/api/plat/pool_inline_types.h | 1 + .../linux-generic/include/odp_event_internal.h | 7 +- .../include/odp_event_validation_internal.h | 52 ++++++++++++ platform/linux-generic/include/odp_init_internal.h | 3 + platform/linux-generic/include/odp_pool_internal.h | 1 + 7 files changed, 158 insertions(+), 2 deletions(-) create mode 100644 platform/linux-generic/include/odp/api/plat/event_validation_external.h create mode 100644 platform/linux-generic/include/odp_event_validation_internal.h (limited to 'platform/linux-generic/include') diff --git a/platform/linux-generic/include/odp/api/plat/event_validation_external.h b/platform/linux-generic/include/odp/api/plat/event_validation_external.h new file mode 100644 index 000000000..2d6c8613a --- /dev/null +++ b/platform/linux-generic/include/odp/api/plat/event_validation_external.h @@ -0,0 +1,93 @@ +/* Copyright (c) 2023, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +/** + * @file + * + * ODP event validation + * + * @warning These definitions are not part of ODP API, they are for + * implementation internal use only. + */ + +#ifndef ODP_EVENT_VALIDATION_EXTERNAL_H_ +#define ODP_EVENT_VALIDATION_EXTERNAL_H_ + +#include + +#include +#include +#include +#include + +/** @cond _ODP_HIDE_FROM_DOXYGEN_ */ + +#ifdef __cplusplus +extern "C" { +#endif + +/** Enumerations for identifying ODP API functions */ +typedef enum { + _ODP_EV_BUFFER_FREE = 0, + _ODP_EV_BUFFER_FREE_MULTI, + _ODP_EV_BUFFER_IS_VALID, + _ODP_EV_EVENT_FREE, + _ODP_EV_EVENT_FREE_MULTI, + _ODP_EV_EVENT_FREE_SP, + _ODP_EV_EVENT_IS_VALID, + _ODP_EV_PACKET_FREE, + _ODP_EV_PACKET_FREE_MULTI, + _ODP_EV_PACKET_FREE_SP, + _ODP_EV_PACKET_IS_VALID, + _ODP_EV_MAX +} _odp_ev_id_t; + +/* Implementation internal event validation functions */ +#if _ODP_EVENT_VALIDATION + +int _odp_buffer_validate(odp_buffer_t buf, _odp_ev_id_t ev_id); + +int _odp_buffer_validate_multi(const odp_buffer_t buf[], int num, _odp_ev_id_t ev_id); + +int _odp_packet_validate(odp_packet_t pkt, _odp_ev_id_t ev_id); + +int _odp_packet_validate_multi(const odp_packet_t pkt[], int num, _odp_ev_id_t ev_id); + +#else + +static inline int _odp_buffer_validate(odp_buffer_t buf ODP_UNUSED, _odp_ev_id_t ev_id ODP_UNUSED) +{ + return 0; +} + +static inline int _odp_buffer_validate_multi(const odp_buffer_t buf[] ODP_UNUSED, + int num ODP_UNUSED, + _odp_ev_id_t ev_id ODP_UNUSED) +{ + return 0; +} + +static inline int _odp_packet_validate(odp_packet_t pkt ODP_UNUSED, _odp_ev_id_t ev_id ODP_UNUSED) +{ + return 0; +} + +static inline int _odp_packet_validate_multi(const odp_packet_t pkt[] ODP_UNUSED, + int num ODP_UNUSED, + _odp_ev_id_t ev_id ODP_UNUSED) +{ + return 0; +} + +#endif /* _ODP_EVENT_VALIDATION */ + +#ifdef __cplusplus +} +#endif + +/** @endcond */ + +#endif diff --git a/platform/linux-generic/include/odp/api/plat/packet_inlines.h b/platform/linux-generic/include/odp/api/plat/packet_inlines.h index 01d47d837..93e95e21c 100644 --- a/platform/linux-generic/include/odp/api/plat/packet_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/packet_inlines.h @@ -632,7 +632,8 @@ _ODP_INLINE uint32_t odp_packet_buf_size(odp_packet_buf_t pkt_buf) odp_pool_t pool = _odp_pkt_get(pkt_buf, odp_pool_t, pool); return _odp_pool_get(pool, uint32_t, ext_pkt_buf_size) - - _odp_pool_get(pool, uint32_t, ext_head_offset); + _odp_pool_get(pool, uint32_t, ext_head_offset) - + _odp_pool_get(pool, uint32_t, trailer_size); } _ODP_INLINE void *odp_packet_buf_head(odp_packet_buf_t pkt_buf) diff --git a/platform/linux-generic/include/odp/api/plat/pool_inline_types.h b/platform/linux-generic/include/odp/api/plat/pool_inline_types.h index 02f59f982..fbff7eda7 100644 --- a/platform/linux-generic/include/odp/api/plat/pool_inline_types.h +++ b/platform/linux-generic/include/odp/api/plat/pool_inline_types.h @@ -30,6 +30,7 @@ typedef struct _odp_pool_inline_offset_t { uint16_t index; uint16_t seg_len; uint16_t uarea_size; + uint16_t trailer_size; uint16_t ext_head_offset; uint16_t ext_pkt_buf_size; diff --git a/platform/linux-generic/include/odp_event_internal.h b/platform/linux-generic/include/odp_event_internal.h index 5a29e926e..4bc28d708 100644 --- a/platform/linux-generic/include/odp_event_internal.h +++ b/platform/linux-generic/include/odp_event_internal.h @@ -53,7 +53,7 @@ typedef struct _odp_event_hdr_t { /* --- Mostly read only data --- */ - /* Initial buffer tail pointer */ + /* Initial buffer tail pointer and endmark location (if enabled) */ uint8_t *buf_end; /* Combined pool and event index */ @@ -85,6 +85,11 @@ static inline void _odp_event_type_set(odp_event_t event, int ev) _odp_event_hdr(event)->event_type = ev; } +static inline uint64_t *_odp_event_endmark_get_ptr(odp_event_t event) +{ + return (uint64_t *)(uintptr_t)_odp_event_hdr(event)->buf_end; +} + #ifdef __cplusplus } #endif diff --git a/platform/linux-generic/include/odp_event_validation_internal.h b/platform/linux-generic/include/odp_event_validation_internal.h new file mode 100644 index 000000000..f4ac16f31 --- /dev/null +++ b/platform/linux-generic/include/odp_event_validation_internal.h @@ -0,0 +1,52 @@ +/* Copyright (c) 2023, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_EVENT_VALIDATION_INTERNAL_H_ +#define ODP_EVENT_VALIDATION_INTERNAL_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include + +#include +#include + +#include + +#include + +#include + +#if _ODP_EVENT_VALIDATION + +#define _ODP_EV_ENDMARK_VAL 0xDEADBEEFDEADBEEF +#define _ODP_EV_ENDMARK_SIZE (sizeof(uint64_t)) + +static inline void _odp_event_endmark_set(odp_event_t event) +{ + uint64_t *endmark_ptr; + + endmark_ptr = _odp_event_endmark_get_ptr(event); + *endmark_ptr = _ODP_EV_ENDMARK_VAL; +} + +#else + +#define _ODP_EV_ENDMARK_VAL 0 +#define _ODP_EV_ENDMARK_SIZE 0 + +static inline void _odp_event_endmark_set(odp_event_t event ODP_UNUSED) +{ +} + +#endif + +#ifdef __cplusplus +} +#endif +#endif diff --git a/platform/linux-generic/include/odp_init_internal.h b/platform/linux-generic/include/odp_init_internal.h index 2a1039854..24e8346ad 100644 --- a/platform/linux-generic/include/odp_init_internal.h +++ b/platform/linux-generic/include/odp_init_internal.h @@ -33,6 +33,9 @@ int _odp_pool_init_local(void); int _odp_pool_term_global(void); int _odp_pool_term_local(void); +int _odp_event_validation_init_global(void); +int _odp_event_validation_term_global(void); + int _odp_queue_init_global(void); int _odp_queue_term_global(void); diff --git a/platform/linux-generic/include/odp_pool_internal.h b/platform/linux-generic/include/odp_pool_internal.h index 1c5b51c3d..c8d2168f3 100644 --- a/platform/linux-generic/include/odp_pool_internal.h +++ b/platform/linux-generic/include/odp_pool_internal.h @@ -87,6 +87,7 @@ typedef struct pool_t { uint32_t block_size; uint32_t block_offset; uint32_t num_populated; + uint32_t trailer_size; uint8_t *base_addr; uint8_t *max_addr; uint8_t *uarea_base_addr; -- cgit v1.2.3 From 85d0d642154a15ea46dd1a1fcbcde3c4bdeffa49 Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Wed, 28 Dec 2022 08:49:16 +0200 Subject: linux-gen: event: add event validation support for queue enq operations Add event validation support to following queue enqueue functions: - odp_queue_enq() - odp_queue_enq_multi() Signed-off-by: Matias Elo Reviewed-by: Tuomas Taipale --- .../include/odp/api/plat/event_validation_external.h | 18 ++++++++++++++++++ .../linux-generic/include/odp/api/plat/queue_inlines.h | 10 ++++++++++ 2 files changed, 28 insertions(+) (limited to 'platform/linux-generic/include') diff --git a/platform/linux-generic/include/odp/api/plat/event_validation_external.h b/platform/linux-generic/include/odp/api/plat/event_validation_external.h index 2d6c8613a..5a939402f 100644 --- a/platform/linux-generic/include/odp/api/plat/event_validation_external.h +++ b/platform/linux-generic/include/odp/api/plat/event_validation_external.h @@ -42,12 +42,18 @@ typedef enum { _ODP_EV_PACKET_FREE_MULTI, _ODP_EV_PACKET_FREE_SP, _ODP_EV_PACKET_IS_VALID, + _ODP_EV_QUEUE_ENQ, + _ODP_EV_QUEUE_ENQ_MULTI, _ODP_EV_MAX } _odp_ev_id_t; /* Implementation internal event validation functions */ #if _ODP_EVENT_VALIDATION +int _odp_event_validate(odp_event_t event, _odp_ev_id_t id); + +int _odp_event_validate_multi(const odp_event_t event[], int num, _odp_ev_id_t id); + int _odp_buffer_validate(odp_buffer_t buf, _odp_ev_id_t ev_id); int _odp_buffer_validate_multi(const odp_buffer_t buf[], int num, _odp_ev_id_t ev_id); @@ -58,6 +64,18 @@ int _odp_packet_validate_multi(const odp_packet_t pkt[], int num, _odp_ev_id_t e #else +static inline int _odp_event_validate(odp_event_t event ODP_UNUSED, _odp_ev_id_t ev_id ODP_UNUSED) +{ + return 0; +} + +static inline int _odp_event_validate_multi(const odp_event_t event[] ODP_UNUSED, + int num ODP_UNUSED, + _odp_ev_id_t ev_id ODP_UNUSED) +{ + return 0; +} + static inline int _odp_buffer_validate(odp_buffer_t buf ODP_UNUSED, _odp_ev_id_t ev_id ODP_UNUSED) { return 0; diff --git a/platform/linux-generic/include/odp/api/plat/queue_inlines.h b/platform/linux-generic/include/odp/api/plat/queue_inlines.h index 22673a887..609c0c9e4 100644 --- a/platform/linux-generic/include/odp/api/plat/queue_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/queue_inlines.h @@ -1,4 +1,5 @@ /* Copyright (c) 2018, Linaro Limited + * Copyright (c) 2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -7,6 +8,9 @@ #ifndef ODP_PLAT_QUEUE_INLINES_H_ #define ODP_PLAT_QUEUE_INLINES_H_ +#include + +#include #include /** @cond _ODP_HIDE_FROM_DOXYGEN_ */ @@ -37,12 +41,18 @@ _ODP_INLINE void *odp_queue_context(odp_queue_t handle) _ODP_INLINE int odp_queue_enq(odp_queue_t queue, odp_event_t ev) { + if (odp_unlikely(_odp_event_validate(ev, _ODP_EV_QUEUE_ENQ))) + return -1; + return _odp_queue_api->queue_enq(queue, ev); } _ODP_INLINE int odp_queue_enq_multi(odp_queue_t queue, const odp_event_t events[], int num) { + if (odp_unlikely(_odp_event_validate_multi(events, num, _ODP_EV_QUEUE_ENQ_MULTI))) + return -1; + return _odp_queue_api->queue_enq_multi(queue, events, num); } -- cgit v1.2.3 From 77d2986ac48a62b7c0493bbd666c33915969677b Mon Sep 17 00:00:00 2001 From: Tuomas Taipale Date: Tue, 27 Dec 2022 12:36:00 +0000 Subject: linux-gen: loop: add multi-queue support Add multi-queue support to loop packet I/O. This enables more realistic packet I/O application testing. RX and TX specific locking is removed as all data shared between transmitting and receiving threads is already multi-thread safe. Signed-off-by: Tuomas Taipale Reviewed-by: Matias Elo --- platform/linux-generic/include/odp_packet_io_internal.h | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) (limited to 'platform/linux-generic/include') diff --git a/platform/linux-generic/include/odp_packet_io_internal.h b/platform/linux-generic/include/odp_packet_io_internal.h index 954602959..187a3a76f 100644 --- a/platform/linux-generic/include/odp_packet_io_internal.h +++ b/platform/linux-generic/include/odp_packet_io_internal.h @@ -1,5 +1,5 @@ /* Copyright (c) 2013-2018, Linaro Limited - * Copyright (c) 2019-2022, Nokia + * Copyright (c) 2019-2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -68,12 +68,8 @@ struct pktio_if_ops; #define PKTIO_PRIVATE_SIZE 33792 #elif defined(_ODP_PKTIO_XDP) #define PKTIO_PRIVATE_SIZE 29696 -#elif defined(_ODP_PKTIO_DPDK) && ODP_CACHE_LINE_SIZE == 128 -#define PKTIO_PRIVATE_SIZE 4160 -#elif defined(_ODP_PKTIO_DPDK) -#define PKTIO_PRIVATE_SIZE 3968 #else -#define PKTIO_PRIVATE_SIZE 384 +#define PKTIO_PRIVATE_SIZE 9216 #endif typedef struct ODP_ALIGNED_CACHE { -- cgit v1.2.3 From 8aee6094ef983d1f537d738b8a87186fb61e7cdd Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Fri, 13 Jan 2023 16:23:30 +0200 Subject: linux-gen: stash: reserve all memory from a single shm block Use one common SHM block for all stash memory. This enables creating a large number of stashes without hitting the SHM block count limit, for the cost of having to pre-reserve the memory during global init. The amount of required memory can be adjusted with new configuration file options: - stash:max_num: maximum number of stashes - stash:max_num_obj: maximum number of objects in a stash Signed-off-by: Matias Elo Reviewed-by: Tuomas Taipale --- platform/linux-generic/include/odp_config_internal.h | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'platform/linux-generic/include') diff --git a/platform/linux-generic/include/odp_config_internal.h b/platform/linux-generic/include/odp_config_internal.h index d3d09abf4..e4f8d6d6d 100644 --- a/platform/linux-generic/include/odp_config_internal.h +++ b/platform/linux-generic/include/odp_config_internal.h @@ -1,5 +1,5 @@ /* Copyright (c) 2016-2018, Linaro Limited - * Copyright (c) 2019-2021, Nokia + * Copyright (c) 2019-2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -66,7 +66,7 @@ extern "C" { /* * Maximum number of stashes */ -#define CONFIG_MAX_STASHES 128 +#define CONFIG_MAX_STASHES 2048 /* * Maximum number of packet IO resources @@ -134,10 +134,10 @@ extern "C" { /* * Number of shared memory blocks reserved for implementation internal use. * - * Each stash requires one SHM block, each pool requires three blocks (buffers, - * ring, user area), and 20 blocks are reserved for per ODP module global data. + * Each pool requires three blocks (buffers, ring, user area), and 20 blocks + * are reserved for per ODP module global data. */ -#define CONFIG_INTERNAL_SHM_BLOCKS (CONFIG_MAX_STASHES + (ODP_CONFIG_POOLS * 3) + 20) +#define CONFIG_INTERNAL_SHM_BLOCKS ((ODP_CONFIG_POOLS * 3) + 20) /* * Maximum number of shared memory blocks. -- cgit v1.2.3 From 579d8105776ea23f097fdb894a50759b6055b8b4 Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Mon, 23 Jan 2023 13:01:26 +0200 Subject: api: buffer: split header files Split buffer API into separate header files for functions and types. This fixes circular dependency issues with inline headers. Signed-off-by: Matias Elo Reviewed-by: Tuomas Taipale --- platform/linux-generic/include/odp/api/plat/buffer_inlines.h | 5 ++--- .../linux-generic/include/odp/api/plat/event_validation_external.h | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) (limited to 'platform/linux-generic/include') diff --git a/platform/linux-generic/include/odp/api/plat/buffer_inlines.h b/platform/linux-generic/include/odp/api/plat/buffer_inlines.h index 34d4b5675..75ef36cf3 100644 --- a/platform/linux-generic/include/odp/api/plat/buffer_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/buffer_inlines.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2019-2022, Nokia +/* Copyright (c) 2019-2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -7,11 +7,10 @@ #ifndef ODP_PLAT_BUFFER_INLINES_H_ #define ODP_PLAT_BUFFER_INLINES_H_ +#include #include #include -#include - #include #include #include diff --git a/platform/linux-generic/include/odp/api/plat/event_validation_external.h b/platform/linux-generic/include/odp/api/plat/event_validation_external.h index 5a939402f..7f5c0364f 100644 --- a/platform/linux-generic/include/odp/api/plat/event_validation_external.h +++ b/platform/linux-generic/include/odp/api/plat/event_validation_external.h @@ -18,7 +18,7 @@ #include -#include +#include #include #include #include -- cgit v1.2.3 From 9079409fcd9754ce0086c5967fb5a7c32183c583 Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Mon, 16 Jan 2023 17:02:12 +0200 Subject: linux-gen: ring: add 64-bit variant of mpmc ring Add new 64-bit variant of implementation internal multi-producer multi- consumer ring. Signed-off-by: Matias Elo Reviewed-by: Tuomas Taipale --- .../include/odp_queue_basic_internal.h | 5 +- .../linux-generic/include/odp_ring_mpmc_internal.h | 115 ++++++++++++++------- .../include/odp_ring_mpmc_u32_internal.h | 25 +++++ .../include/odp_ring_mpmc_u64_internal.h | 25 +++++ 4 files changed, 133 insertions(+), 37 deletions(-) create mode 100644 platform/linux-generic/include/odp_ring_mpmc_u32_internal.h create mode 100644 platform/linux-generic/include/odp_ring_mpmc_u64_internal.h (limited to 'platform/linux-generic/include') diff --git a/platform/linux-generic/include/odp_queue_basic_internal.h b/platform/linux-generic/include/odp_queue_basic_internal.h index 830f50a9d..3cdcf8600 100644 --- a/platform/linux-generic/include/odp_queue_basic_internal.h +++ b/platform/linux-generic/include/odp_queue_basic_internal.h @@ -1,4 +1,5 @@ /* Copyright (c) 2013-2018, Linaro Limited + * Copyright (c) 2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -23,7 +24,7 @@ extern "C" { #include #include #include -#include +#include #include #include #include @@ -47,7 +48,7 @@ typedef struct ODP_ALIGNED_CACHE queue_entry_s { odp_queue_type_t type; /* MPMC ring (2 cache lines). */ - ring_mpmc_t ring_mpmc; + ring_mpmc_u32_t ring_mpmc; odp_ticketlock_t lock; union { diff --git a/platform/linux-generic/include/odp_ring_mpmc_internal.h b/platform/linux-generic/include/odp_ring_mpmc_internal.h index 6ed4dd4d1..37b7780b5 100644 --- a/platform/linux-generic/include/odp_ring_mpmc_internal.h +++ b/platform/linux-generic/include/odp_ring_mpmc_internal.h @@ -1,4 +1,5 @@ /* Copyright (c) 2018, Linaro Limited + * Copyright (c) 2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -19,6 +20,8 @@ extern "C" { #include #include +#include + /* Ring of uint32_t data * * Ring stores head and tail counters. Ring indexes are formed from these @@ -34,14 +37,22 @@ extern "C" { * r_tail r_head w_tail w_head * */ -typedef struct { + +struct ring_mpmc_common { odp_atomic_u32_t r_head ODP_ALIGNED_CACHE; odp_atomic_u32_t r_tail; odp_atomic_u32_t w_head ODP_ALIGNED_CACHE; odp_atomic_u32_t w_tail; +}; + +typedef struct ODP_ALIGNED_CACHE { + struct ring_mpmc_common r; +} ring_mpmc_u32_t; -} ring_mpmc_t; +typedef struct ODP_ALIGNED_CACHE { + struct ring_mpmc_common r; +} ring_mpmc_u64_t; static inline int ring_mpmc_cas_u32(odp_atomic_u32_t *atom, uint32_t *old_val, uint32_t new_val) @@ -52,21 +63,57 @@ static inline int ring_mpmc_cas_u32(odp_atomic_u32_t *atom, __ATOMIC_RELAXED); } +#endif /* End of include guards */ + +#undef _ring_mpmc_gen_t +#undef _ring_mpmc_data_t +#undef _RING_MPMC_INIT +#undef _RING_MPMC_DEQ_MULTI +#undef _RING_MPMC_ENQ_MULTI +#undef _RING_MPMC_IS_EMPTY +#undef _RING_MPMC_LEN + +/* This header should NOT be included directly. There are no include guards for + * the following types and function definitions! */ +#ifndef _ODP_RING_TYPE +#error Include type specific (u32/u64) ring header instead of this common file. +#endif + +#if _ODP_RING_TYPE == _ODP_RING_TYPE_U32 + #define _ring_mpmc_gen_t ring_mpmc_u32_t + #define _ring_mpmc_data_t uint32_t + + #define _RING_MPMC_INIT ring_mpmc_u32_init + #define _RING_MPMC_DEQ_MULTI ring_mpmc_u32_deq_multi + #define _RING_MPMC_ENQ_MULTI ring_mpmc_u32_enq_multi + #define _RING_MPMC_IS_EMPTY ring_mpmc_u32_is_empty + #define _RING_MPMC_LEN ring_mpmc_u32_len +#elif _ODP_RING_TYPE == _ODP_RING_TYPE_U64 + #define _ring_mpmc_gen_t ring_mpmc_u64_t + #define _ring_mpmc_data_t uint64_t + + #define _RING_MPMC_INIT ring_mpmc_u64_init + #define _RING_MPMC_DEQ_MULTI ring_mpmc_u64_deq_multi + #define _RING_MPMC_ENQ_MULTI ring_mpmc_u64_enq_multi + #define _RING_MPMC_IS_EMPTY ring_mpmc_u64_is_empty + #define _RING_MPMC_LEN ring_mpmc_u64_len +#endif + /* Initialize ring */ -static inline void ring_mpmc_init(ring_mpmc_t *ring) +static inline void _RING_MPMC_INIT(_ring_mpmc_gen_t *ring) { - odp_atomic_init_u32(&ring->w_head, 0); - odp_atomic_init_u32(&ring->w_tail, 0); - odp_atomic_init_u32(&ring->r_head, 0); - odp_atomic_init_u32(&ring->r_tail, 0); + odp_atomic_init_u32(&ring->r.w_head, 0); + odp_atomic_init_u32(&ring->r.w_tail, 0); + odp_atomic_init_u32(&ring->r.r_head, 0); + odp_atomic_init_u32(&ring->r.r_tail, 0); } /* Dequeue data from the ring head. Num is smaller than ring size. */ -static inline uint32_t ring_mpmc_deq_multi(ring_mpmc_t *ring, - uint32_t *ring_data, - uint32_t ring_mask, - uint32_t data[], - uint32_t num) +static inline uint32_t _RING_MPMC_DEQ_MULTI(_ring_mpmc_gen_t *ring, + _ring_mpmc_data_t *ring_data, + uint32_t ring_mask, + _ring_mpmc_data_t data[], + uint32_t num) { uint32_t old_head, new_head, w_tail, num_data, i; @@ -75,9 +122,9 @@ static inline uint32_t ring_mpmc_deq_multi(ring_mpmc_t *ring, * When CAS operation succeeds, this thread owns data between old * and new r_head. */ do { - old_head = odp_atomic_load_acq_u32(&ring->r_head); + old_head = odp_atomic_load_acq_u32(&ring->r.r_head); odp_prefetch(&ring_data[(old_head + 1) & ring_mask]); - w_tail = odp_atomic_load_acq_u32(&ring->w_tail); + w_tail = odp_atomic_load_acq_u32(&ring->r.w_tail); num_data = w_tail - old_head; /* Ring is empty */ @@ -90,7 +137,7 @@ static inline uint32_t ring_mpmc_deq_multi(ring_mpmc_t *ring, new_head = old_head + num; - } while (odp_unlikely(ring_mpmc_cas_u32(&ring->r_head, &old_head, + } while (odp_unlikely(ring_mpmc_cas_u32(&ring->r.r_head, &old_head, new_head) == 0)); /* Read data. This will not move above load acquire of r_head. */ @@ -98,21 +145,21 @@ static inline uint32_t ring_mpmc_deq_multi(ring_mpmc_t *ring, data[i] = ring_data[(old_head + 1 + i) & ring_mask]; /* Wait until other readers have updated the tail */ - while (odp_unlikely(odp_atomic_load_u32(&ring->r_tail) != old_head)) + while (odp_unlikely(odp_atomic_load_u32(&ring->r.r_tail) != old_head)) odp_cpu_pause(); /* Release the new reader tail, writers acquire it. */ - odp_atomic_store_rel_u32(&ring->r_tail, new_head); + odp_atomic_store_rel_u32(&ring->r.r_tail, new_head); return num; } /* Enqueue multiple data into the ring tail. Num is smaller than ring size. */ -static inline uint32_t ring_mpmc_enq_multi(ring_mpmc_t *ring, - uint32_t *ring_data, - uint32_t ring_mask, - const uint32_t data[], - uint32_t num) +static inline uint32_t _RING_MPMC_ENQ_MULTI(_ring_mpmc_gen_t *ring, + _ring_mpmc_data_t *ring_data, + uint32_t ring_mask, + const _ring_mpmc_data_t data[], + uint32_t num) { uint32_t old_head, new_head, r_tail, num_free, i; uint32_t size = ring_mask + 1; @@ -122,8 +169,8 @@ static inline uint32_t ring_mpmc_enq_multi(ring_mpmc_t *ring, * When CAS operation succeeds, this thread owns data between old * and new w_head. */ do { - r_tail = odp_atomic_load_acq_u32(&ring->r_tail); - old_head = odp_atomic_load_acq_u32(&ring->w_head); + r_tail = odp_atomic_load_acq_u32(&ring->r.r_tail); + old_head = odp_atomic_load_acq_u32(&ring->r.w_head); num_free = size - (old_head - r_tail); @@ -137,7 +184,7 @@ static inline uint32_t ring_mpmc_enq_multi(ring_mpmc_t *ring, new_head = old_head + num; - } while (odp_unlikely(ring_mpmc_cas_u32(&ring->w_head, &old_head, + } while (odp_unlikely(ring_mpmc_cas_u32(&ring->r.w_head, &old_head, new_head) == 0)); /* Write data. This will not move above load acquire of w_head. */ @@ -145,29 +192,29 @@ static inline uint32_t ring_mpmc_enq_multi(ring_mpmc_t *ring, ring_data[(old_head + 1 + i) & ring_mask] = data[i]; /* Wait until other writers have updated the tail */ - while (odp_unlikely(odp_atomic_load_u32(&ring->w_tail) != old_head)) + while (odp_unlikely(odp_atomic_load_u32(&ring->r.w_tail) != old_head)) odp_cpu_pause(); /* Release the new writer tail, readers acquire it. */ - odp_atomic_store_rel_u32(&ring->w_tail, new_head); + odp_atomic_store_rel_u32(&ring->r.w_tail, new_head); return num; } /* Check if ring is empty */ -static inline int ring_mpmc_is_empty(ring_mpmc_t *ring) +static inline int _RING_MPMC_IS_EMPTY(_ring_mpmc_gen_t *ring) { - uint32_t head = odp_atomic_load_u32(&ring->r_head); - uint32_t tail = odp_atomic_load_u32(&ring->w_tail); + uint32_t head = odp_atomic_load_u32(&ring->r.r_head); + uint32_t tail = odp_atomic_load_u32(&ring->r.w_tail); return head == tail; } /* Return current ring length */ -static inline uint32_t ring_mpmc_length(ring_mpmc_t *ring) +static inline uint32_t _RING_MPMC_LEN(_ring_mpmc_gen_t *ring) { - uint32_t head = odp_atomic_load_u32(&ring->r_head); - uint32_t tail = odp_atomic_load_u32(&ring->w_tail); + uint32_t head = odp_atomic_load_u32(&ring->r.r_head); + uint32_t tail = odp_atomic_load_u32(&ring->r.w_tail); return tail - head; } @@ -175,5 +222,3 @@ static inline uint32_t ring_mpmc_length(ring_mpmc_t *ring) #ifdef __cplusplus } #endif - -#endif diff --git a/platform/linux-generic/include/odp_ring_mpmc_u32_internal.h b/platform/linux-generic/include/odp_ring_mpmc_u32_internal.h new file mode 100644 index 000000000..4699b5b47 --- /dev/null +++ b/platform/linux-generic/include/odp_ring_mpmc_u32_internal.h @@ -0,0 +1,25 @@ +/* Copyright (c) 2023, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_RING_MPMC_U32_INTERNAL_H_ +#define ODP_RING_MPMC_U32_INTERNAL_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include + +#undef _ODP_RING_TYPE +#define _ODP_RING_TYPE _ODP_RING_TYPE_U32 + +#include + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/include/odp_ring_mpmc_u64_internal.h b/platform/linux-generic/include/odp_ring_mpmc_u64_internal.h new file mode 100644 index 000000000..e7bf31a94 --- /dev/null +++ b/platform/linux-generic/include/odp_ring_mpmc_u64_internal.h @@ -0,0 +1,25 @@ +/* Copyright (c) 2023, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_RING_MPMC_U64_INTERNAL_H_ +#define ODP_RING_MPMC_U64_INTERNAL_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include + +#undef _ODP_RING_TYPE +#define _ODP_RING_TYPE _ODP_RING_TYPE_U64 + +#include + +#ifdef __cplusplus +} +#endif + +#endif -- cgit v1.2.3 From bba00b595276b556564fa05ce1dff51f29417103 Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Thu, 19 Jan 2023 09:30:25 +0200 Subject: linux-gen: ring: add batch enqueue and dequeue operations to mpmc ring Add support for batch enqueue/dequeue operations to MPMC ring (read/write 0 or N objects). Unnecessary comments about 'num' parameter being smaller than ring size have been removed. Signed-off-by: Matias Elo Reviewed-by: Tuomas Taipale --- .../linux-generic/include/odp_ring_mpmc_internal.h | 95 +++++++++++++++++++++- 1 file changed, 93 insertions(+), 2 deletions(-) (limited to 'platform/linux-generic/include') diff --git a/platform/linux-generic/include/odp_ring_mpmc_internal.h b/platform/linux-generic/include/odp_ring_mpmc_internal.h index 37b7780b5..e35179267 100644 --- a/platform/linux-generic/include/odp_ring_mpmc_internal.h +++ b/platform/linux-generic/include/odp_ring_mpmc_internal.h @@ -70,6 +70,8 @@ static inline int ring_mpmc_cas_u32(odp_atomic_u32_t *atom, #undef _RING_MPMC_INIT #undef _RING_MPMC_DEQ_MULTI #undef _RING_MPMC_ENQ_MULTI +#undef _RING_MPMC_DEQ_BATCH +#undef _RING_MPMC_ENQ_BATCH #undef _RING_MPMC_IS_EMPTY #undef _RING_MPMC_LEN @@ -86,6 +88,8 @@ static inline int ring_mpmc_cas_u32(odp_atomic_u32_t *atom, #define _RING_MPMC_INIT ring_mpmc_u32_init #define _RING_MPMC_DEQ_MULTI ring_mpmc_u32_deq_multi #define _RING_MPMC_ENQ_MULTI ring_mpmc_u32_enq_multi + #define _RING_MPMC_DEQ_BATCH ring_mpmc_u32_deq_batch + #define _RING_MPMC_ENQ_BATCH ring_mpmc_u32_enq_batch #define _RING_MPMC_IS_EMPTY ring_mpmc_u32_is_empty #define _RING_MPMC_LEN ring_mpmc_u32_len #elif _ODP_RING_TYPE == _ODP_RING_TYPE_U64 @@ -95,6 +99,8 @@ static inline int ring_mpmc_cas_u32(odp_atomic_u32_t *atom, #define _RING_MPMC_INIT ring_mpmc_u64_init #define _RING_MPMC_DEQ_MULTI ring_mpmc_u64_deq_multi #define _RING_MPMC_ENQ_MULTI ring_mpmc_u64_enq_multi + #define _RING_MPMC_DEQ_BATCH ring_mpmc_u64_deq_batch + #define _RING_MPMC_ENQ_BATCH ring_mpmc_u64_enq_batch #define _RING_MPMC_IS_EMPTY ring_mpmc_u64_is_empty #define _RING_MPMC_LEN ring_mpmc_u64_len #endif @@ -108,7 +114,7 @@ static inline void _RING_MPMC_INIT(_ring_mpmc_gen_t *ring) odp_atomic_init_u32(&ring->r.r_tail, 0); } -/* Dequeue data from the ring head. Num is smaller than ring size. */ +/* Dequeue data from the ring head */ static inline uint32_t _RING_MPMC_DEQ_MULTI(_ring_mpmc_gen_t *ring, _ring_mpmc_data_t *ring_data, uint32_t ring_mask, @@ -154,7 +160,49 @@ static inline uint32_t _RING_MPMC_DEQ_MULTI(_ring_mpmc_gen_t *ring, return num; } -/* Enqueue multiple data into the ring tail. Num is smaller than ring size. */ +/* Dequeue num or 0 data from the ring head */ +static inline uint32_t _RING_MPMC_DEQ_BATCH(_ring_mpmc_gen_t *ring, + _ring_mpmc_data_t *ring_data, + uint32_t ring_mask, + _ring_mpmc_data_t data[], + uint32_t num) +{ + uint32_t old_head, new_head, w_tail, num_data, i; + + /* Load acquires ensure that w_tail load happens after r_head load, + * and thus r_head value is always behind or equal to w_tail value. + * When CAS operation succeeds, this thread owns data between old + * and new r_head. */ + do { + old_head = odp_atomic_load_acq_u32(&ring->r.r_head); + odp_prefetch(&ring_data[(old_head + 1) & ring_mask]); + w_tail = odp_atomic_load_acq_u32(&ring->r.w_tail); + num_data = w_tail - old_head; + + /* Not enough data available */ + if (num_data < num) + return 0; + + new_head = old_head + num; + + } while (odp_unlikely(ring_mpmc_cas_u32(&ring->r.r_head, &old_head, + new_head) == 0)); + + /* Read data. This will not move above load acquire of r_head. */ + for (i = 0; i < num; i++) + data[i] = ring_data[(old_head + 1 + i) & ring_mask]; + + /* Wait until other readers have updated the tail */ + while (odp_unlikely(odp_atomic_load_u32(&ring->r.r_tail) != old_head)) + odp_cpu_pause(); + + /* Release the new reader tail, writers acquire it. */ + odp_atomic_store_rel_u32(&ring->r.r_tail, new_head); + + return num; +} + +/* Enqueue multiple data into the ring tail */ static inline uint32_t _RING_MPMC_ENQ_MULTI(_ring_mpmc_gen_t *ring, _ring_mpmc_data_t *ring_data, uint32_t ring_mask, @@ -201,6 +249,49 @@ static inline uint32_t _RING_MPMC_ENQ_MULTI(_ring_mpmc_gen_t *ring, return num; } +/* Enqueue num or 0 data into the ring tail */ +static inline uint32_t _RING_MPMC_ENQ_BATCH(_ring_mpmc_gen_t *ring, + _ring_mpmc_data_t *ring_data, + uint32_t ring_mask, + const _ring_mpmc_data_t data[], + uint32_t num) +{ + uint32_t old_head, new_head, r_tail, num_free, i; + uint32_t size = ring_mask + 1; + + /* Load acquires ensure that w_head load happens after r_tail load, + * and thus r_tail value is always behind or equal to w_head value. + * When CAS operation succeeds, this thread owns data between old + * and new w_head. */ + do { + r_tail = odp_atomic_load_acq_u32(&ring->r.r_tail); + old_head = odp_atomic_load_acq_u32(&ring->r.w_head); + + num_free = size - (old_head - r_tail); + + /* Not enough free space available */ + if (num_free < num) + return 0; + + new_head = old_head + num; + + } while (odp_unlikely(ring_mpmc_cas_u32(&ring->r.w_head, &old_head, + new_head) == 0)); + + /* Write data. This will not move above load acquire of w_head. */ + for (i = 0; i < num; i++) + ring_data[(old_head + 1 + i) & ring_mask] = data[i]; + + /* Wait until other writers have updated the tail */ + while (odp_unlikely(odp_atomic_load_u32(&ring->r.w_tail) != old_head)) + odp_cpu_pause(); + + /* Release the new writer tail, readers acquire it. */ + odp_atomic_store_rel_u32(&ring->r.w_tail, new_head); + + return num; +} + /* Check if ring is empty */ static inline int _RING_MPMC_IS_EMPTY(_ring_mpmc_gen_t *ring) { -- cgit v1.2.3