diff options
Diffstat (limited to 'platform/linux-generic/include/odp/api')
15 files changed, 864 insertions, 22 deletions
diff --git a/platform/linux-generic/include/odp/api/plat/buffer_inline_types.h b/platform/linux-generic/include/odp/api/plat/buffer_inline_types.h new file mode 100644 index 000000000..9689ddd06 --- /dev/null +++ b/platform/linux-generic/include/odp/api/plat/buffer_inline_types.h @@ -0,0 +1,35 @@ +/* Copyright (c) 2022, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_PLAT_BUFFER_INLINE_TYPES_H_ +#define ODP_PLAT_BUFFER_INLINE_TYPES_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <stdint.h> + +/** @cond _ODP_HIDE_FROM_DOXYGEN_ */ + +/* Buffer header field accessors */ +#define _odp_buffer_get(buffer_hdr, cast, field) \ + (*(cast *)(uintptr_t)((uint8_t *)buffer_hdr + \ + _odp_buffer_inline_offset.field)) + +/* Buffer header field offsets for inline functions */ +typedef struct _odp_buffer_inline_offset_t { + uint16_t uarea_addr; + +} _odp_buffer_inline_offset_t; + +/** @endcond */ + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/include/odp/api/plat/buffer_inlines.h b/platform/linux-generic/include/odp/api/plat/buffer_inlines.h index 576eb846e..d876f60f4 100644 --- a/platform/linux-generic/include/odp/api/plat/buffer_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/buffer_inlines.h @@ -7,16 +7,21 @@ #ifndef ODP_PLAT_BUFFER_INLINES_H_ #define ODP_PLAT_BUFFER_INLINES_H_ -#include <odp/api/event_types.h> +#include <odp/api/event.h> #include <odp/api/pool_types.h> #include <odp/api/abi/buffer.h> +#include <odp/api/plat/buffer_inline_types.h> +#include <odp/api/plat/debug_inlines.h> #include <odp/api/plat/event_inline_types.h> +#include <odp/api/plat/pool_inline_types.h> /** @cond _ODP_HIDE_FROM_DOXYGEN_ */ extern const _odp_event_inline_offset_t _odp_event_inline_offset; +extern const _odp_buffer_inline_offset_t _odp_buffer_inline_offset; +extern const _odp_pool_inline_offset_t _odp_pool_inline; #ifndef _ODP_NO_INLINE /* Inline functions by default */ @@ -24,13 +29,17 @@ extern const _odp_event_inline_offset_t _odp_event_inline_offset; #define odp_buffer_from_event __odp_buffer_from_event #define odp_buffer_to_event __odp_buffer_to_event #define odp_buffer_addr __odp_buffer_addr + #define odp_buffer_size __odp_buffer_size #define odp_buffer_pool __odp_buffer_pool + #define odp_buffer_user_area __odp_buffer_user_area #else #define _ODP_INLINE #endif _ODP_INLINE odp_buffer_t odp_buffer_from_event(odp_event_t ev) { + _ODP_ASSERT(odp_event_type(ev) == ODP_EVENT_BUFFER); + return (odp_buffer_t)ev; } @@ -44,11 +53,23 @@ _ODP_INLINE void *odp_buffer_addr(odp_buffer_t buf) return _odp_event_hdr_field((odp_event_t)buf, void *, base_data); } +_ODP_INLINE uint32_t odp_buffer_size(odp_buffer_t buf) +{ + odp_pool_t pool = _odp_event_hdr_field(buf, odp_pool_t, pool); + + return _odp_pool_get(pool, uint32_t, seg_len); +} + _ODP_INLINE odp_pool_t odp_buffer_pool(odp_buffer_t buf) { return _odp_event_hdr_field(buf, odp_pool_t, pool); } +_ODP_INLINE void *odp_buffer_user_area(odp_buffer_t buf) +{ + return _odp_buffer_get(buf, void *, uarea_addr); +} + /** @endcond */ #endif diff --git a/platform/linux-generic/include/odp/api/plat/debug_inlines.h b/platform/linux-generic/include/odp/api/plat/debug_inlines.h new file mode 100644 index 000000000..bada48f9b --- /dev/null +++ b/platform/linux-generic/include/odp/api/plat/debug_inlines.h @@ -0,0 +1,102 @@ +/* Copyright (c) 2014-2018, Linaro Limited + * Copyright (c) 2020-2022, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +/** + * @file + * + * ODP Debug inlines + * + * @warning These definitions are not part of ODP API, they are for + * implementation internal use only. + */ + +#ifndef ODP_DEBUG_INLINES_H_ +#define ODP_DEBUG_INLINES_H_ + +#include <odp/autoheader_external.h> + +#include <odp/api/hints.h> +#include <odp/api/init.h> + +#include <odp/api/plat/thread_inline_types.h> + +/** @cond _ODP_HIDE_FROM_DOXYGEN_ */ + +#ifdef __cplusplus +extern "C" { +#endif + +/* Avoid "ISO C99 requires at least one argument for the "..." in a variadic + * macro" errors when building with 'pedantic' option. */ +#pragma GCC system_header + +extern odp_log_func_t ODP_PRINTF_FORMAT(2, 3) _odp_log_fn; +extern odp_abort_func_t _odp_abort_fn; + +#define _ODP_LOG_FN(level, fmt, ...) \ + do { \ + if (_odp_this_thread && _odp_this_thread->log_fn) \ + _odp_this_thread->log_fn(level, fmt, ##__VA_ARGS__); \ + else \ + _odp_log_fn(level, fmt, ##__VA_ARGS__); \ + } while (0) + +/** + * ODP LOG macro. + */ +#define _ODP_LOG(level, fmt, ...) \ + _ODP_LOG_FN(level, "%s:%d:%s():" fmt, __FILE__, \ + __LINE__, __func__, ##__VA_ARGS__) + +/** + * Runtime assertion-macro - aborts if 'cond' is false. + */ +#define _ODP_ASSERT(cond) \ + do { if ((ODP_DEBUG == 1) && (!(cond))) { \ + _ODP_ERR("%s\n", #cond); \ + _odp_abort_fn(); } \ + } while (0) + +/* + * Print debug message to log, if ODP_DEBUG_PRINT flag is set (ignores CONFIG_DEBUG_LEVEL). + */ +#define _ODP_DBG(fmt, ...) \ + do { \ + if (ODP_DEBUG_PRINT == 1) \ + _ODP_LOG(ODP_LOG_DBG, fmt, ##__VA_ARGS__);\ + } while (0) + +/** + * Log error message. + */ +#define _ODP_ERR(fmt, ...) \ + _ODP_LOG(ODP_LOG_ERR, fmt, ##__VA_ARGS__) + +/** + * Log abort message and then stop execution (by default call abort()). + * This function should not return. + */ +#define _ODP_ABORT(fmt, ...) \ + do { \ + _ODP_LOG(ODP_LOG_ABORT, fmt, ##__VA_ARGS__); \ + _odp_abort_fn(); \ + } while (0) + +/** + * Log print message when the application calls one of the ODP APIs + * specifically for dumping internal data. + */ +#define _ODP_PRINT(fmt, ...) \ + _ODP_LOG_FN(ODP_LOG_PRINT, fmt, ##__VA_ARGS__) + +#ifdef __cplusplus +} +#endif + +/** @endcond */ + +#endif diff --git a/platform/linux-generic/include/odp/api/plat/event_inlines.h b/platform/linux-generic/include/odp/api/plat/event_inlines.h index 27c2a82fd..dd8e9249c 100644 --- a/platform/linux-generic/include/odp/api/plat/event_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/event_inlines.h @@ -9,13 +9,15 @@ #define ODP_PLAT_EVENT_INLINES_H_ #include <odp/api/event_types.h> -#include <odp/api/packet.h> +#include <odp/api/packet_types.h> #include <odp/api/plat/event_inline_types.h> +#include <odp/api/plat/packet_inline_types.h> /** @cond _ODP_HIDE_FROM_DOXYGEN_ */ extern const _odp_event_inline_offset_t _odp_event_inline_offset; +extern const _odp_packet_inline_offset_t _odp_packet_inline; #ifndef _ODP_NO_INLINE /* Inline functions by default */ @@ -26,8 +28,6 @@ extern const _odp_event_inline_offset_t _odp_event_inline_offset; #define odp_event_types __odp_event_types #define odp_event_flow_id __odp_event_flow_id #define odp_event_flow_id_set __odp_event_flow_id_set - - #include <odp/api/plat/packet_inlines.h> #else #define _ODP_INLINE #endif @@ -67,7 +67,7 @@ _ODP_INLINE odp_event_subtype_t odp_event_subtype(odp_event_t event) if (__odp_event_type_get(event) != ODP_EVENT_PACKET) return ODP_EVENT_NO_SUBTYPE; - return odp_packet_subtype(odp_packet_from_event(event)); + return (odp_event_subtype_t)_odp_pkt_get((odp_packet_t)event, int8_t, subtype); } _ODP_INLINE odp_event_type_t odp_event_types(odp_event_t event, @@ -76,7 +76,7 @@ _ODP_INLINE odp_event_type_t odp_event_types(odp_event_t event, odp_event_type_t event_type = __odp_event_type_get(event); *subtype = event_type == ODP_EVENT_PACKET ? - odp_packet_subtype(odp_packet_from_event(event)) : + (odp_event_subtype_t)_odp_pkt_get((odp_packet_t)event, int8_t, subtype) : ODP_EVENT_NO_SUBTYPE; return event_type; diff --git a/platform/linux-generic/include/odp/api/plat/packet_inline_types.h b/platform/linux-generic/include/odp/api/plat/packet_inline_types.h index b00173aca..2ae0829c3 100644 --- a/platform/linux-generic/include/odp/api/plat/packet_inline_types.h +++ b/platform/linux-generic/include/odp/api/plat/packet_inline_types.h @@ -51,6 +51,7 @@ typedef struct _odp_packet_inline_offset_t { uint16_t input_flags; uint16_t flags; uint16_t subtype; + uint16_t cls_mark; } _odp_packet_inline_offset_t; diff --git a/platform/linux-generic/include/odp/api/plat/packet_inlines.h b/platform/linux-generic/include/odp/api/plat/packet_inlines.h index da4eabe2e..21d184ac5 100644 --- a/platform/linux-generic/include/odp/api/plat/packet_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/packet_inlines.h @@ -14,16 +14,13 @@ #ifndef _ODP_PLAT_PACKET_INLINES_H_ #define _ODP_PLAT_PACKET_INLINES_H_ +#include <odp/api/event.h> #include <odp/api/hints.h> #include <odp/api/packet_types.h> #include <odp/api/pool_types.h> #include <odp/api/time.h> -#include <odp/api/abi/buffer.h> -#include <odp/api/abi/event_types.h> -#include <odp/api/abi/packet.h> -#include <odp/api/abi/packet_io.h> - +#include <odp/api/plat/debug_inlines.h> #include <odp/api/plat/packet_io_inlines.h> #include <odp/api/plat/packet_inline_types.h> #include <odp/api/plat/pool_inline_types.h> @@ -48,6 +45,7 @@ #define odp_packet_input_index __odp_packet_input_index #define odp_packet_num_segs __odp_packet_num_segs #define odp_packet_user_ptr __odp_packet_user_ptr + #define odp_packet_user_ptr_set __odp_packet_user_ptr_set #define odp_packet_user_area __odp_packet_user_area #define odp_packet_user_area_size __odp_packet_user_area_size #define odp_packet_user_flag __odp_packet_user_flag @@ -55,11 +53,23 @@ #define odp_packet_l2_offset __odp_packet_l2_offset #define odp_packet_l3_offset __odp_packet_l3_offset #define odp_packet_l4_offset __odp_packet_l4_offset + #define odp_packet_l2_offset_set __odp_packet_l2_offset_set + #define odp_packet_l3_offset_set __odp_packet_l3_offset_set + #define odp_packet_l4_offset_set __odp_packet_l4_offset_set #define odp_packet_l2_ptr __odp_packet_l2_ptr #define odp_packet_l3_ptr __odp_packet_l3_ptr #define odp_packet_l4_ptr __odp_packet_l4_ptr + #define odp_packet_l2_type __odp_packet_l2_type + #define odp_packet_l3_type __odp_packet_l3_type + #define odp_packet_l4_type __odp_packet_l4_type + #define odp_packet_l3_chksum_status __odp_packet_l3_chksum_status + #define odp_packet_l4_chksum_status __odp_packet_l4_chksum_status + #define odp_packet_l3_chksum_insert __odp_packet_l3_chksum_insert + #define odp_packet_l4_chksum_insert __odp_packet_l4_chksum_insert #define odp_packet_flow_hash __odp_packet_flow_hash #define odp_packet_ts __odp_packet_ts + #define odp_packet_ts_set __odp_packet_ts_set + #define odp_packet_ts_request __odp_packet_ts_request #define odp_packet_head __odp_packet_head #define odp_packet_is_segmented __odp_packet_is_segmented #define odp_packet_first_seg __odp_packet_first_seg @@ -79,6 +89,7 @@ #define odp_packet_color __odp_packet_color #define odp_packet_drop_eligible __odp_packet_drop_eligible #define odp_packet_shaper_len_adjust __odp_packet_shaper_len_adjust + #define odp_packet_cls_mark __odp_packet_cls_mark #define odp_packet_buf_data_len __odp_packet_buf_data_len #define odp_packet_buf_size __odp_packet_buf_size #define odp_packet_buf_head __odp_packet_buf_head @@ -176,6 +187,20 @@ _ODP_INLINE void *odp_packet_user_ptr(odp_packet_t pkt) return _odp_pkt_get(pkt, void *, user_ptr); } +_ODP_INLINE void odp_packet_user_ptr_set(odp_packet_t pkt, const void *ptr) +{ + _odp_packet_flags_t *flags = _odp_pkt_get_ptr(pkt, _odp_packet_flags_t, flags); + const void **user_ptr = _odp_pkt_get_ptr(pkt, const void *, user_ptr); + + if (odp_unlikely(ptr == NULL)) { + flags->user_ptr_set = 0; + return; + } + + *user_ptr = ptr; + flags->user_ptr_set = 1; +} + _ODP_INLINE void *odp_packet_user_area(odp_packet_t pkt) { return _odp_pkt_get(pkt, void *, user_area); @@ -219,6 +244,42 @@ _ODP_INLINE uint32_t odp_packet_l4_offset(odp_packet_t pkt) return _odp_pkt_get(pkt, uint16_t, l4_offset); } +_ODP_INLINE int odp_packet_l2_offset_set(odp_packet_t pkt, uint32_t offset) +{ + uint16_t *l2_offset = _odp_pkt_get_ptr(pkt, uint16_t, l2_offset); + _odp_packet_input_flags_t *input_flags = _odp_pkt_get_ptr(pkt, _odp_packet_input_flags_t, + input_flags); + + if (odp_unlikely(offset >= odp_packet_len(pkt))) + return -1; + + input_flags->l2 = 1; + *l2_offset = (uint16_t)offset; + return 0; +} + +_ODP_INLINE int odp_packet_l3_offset_set(odp_packet_t pkt, uint32_t offset) +{ + uint16_t *l3_offset = _odp_pkt_get_ptr(pkt, uint16_t, l3_offset); + + if (odp_unlikely(offset >= odp_packet_len(pkt))) + return -1; + + *l3_offset = (uint16_t)offset; + return 0; +} + +_ODP_INLINE int odp_packet_l4_offset_set(odp_packet_t pkt, uint32_t offset) +{ + uint16_t *l4_offset = _odp_pkt_get_ptr(pkt, uint16_t, l4_offset); + + if (odp_unlikely(offset >= odp_packet_len(pkt))) + return -1; + + *l4_offset = (uint16_t)offset; + return 0; +} + _ODP_INLINE void *odp_packet_l2_ptr(odp_packet_t pkt, uint32_t *len) { uint32_t offset = odp_packet_l2_offset(pkt); @@ -273,6 +334,107 @@ _ODP_INLINE void *odp_packet_l4_ptr(odp_packet_t pkt, uint32_t *len) return data + offset; } +_ODP_INLINE odp_proto_l2_type_t odp_packet_l2_type(odp_packet_t pkt) +{ + _odp_packet_input_flags_t input_flags; + + input_flags.all = _odp_pkt_get(pkt, uint64_t, input_flags); + + return input_flags.eth ? ODP_PROTO_L2_TYPE_ETH : ODP_PROTO_L2_TYPE_NONE; +} + +_ODP_INLINE odp_proto_l3_type_t odp_packet_l3_type(odp_packet_t pkt) +{ + _odp_packet_input_flags_t input_flags; + + input_flags.all = _odp_pkt_get(pkt, uint64_t, input_flags); + + if (input_flags.ipv4) + return ODP_PROTO_L3_TYPE_IPV4; + else if (input_flags.ipv6) + return ODP_PROTO_L3_TYPE_IPV6; + else if (input_flags.arp) + return ODP_PROTO_L3_TYPE_ARP; + + return ODP_PROTO_L3_TYPE_NONE; +} + +_ODP_INLINE odp_proto_l4_type_t odp_packet_l4_type(odp_packet_t pkt) +{ + _odp_packet_input_flags_t input_flags; + + input_flags.all = _odp_pkt_get(pkt, uint64_t, input_flags); + + if (input_flags.tcp) + return ODP_PROTO_L4_TYPE_TCP; + else if (input_flags.udp) + return ODP_PROTO_L4_TYPE_UDP; + else if (input_flags.sctp) + return ODP_PROTO_L4_TYPE_SCTP; + else if (input_flags.ipsec_ah) + return ODP_PROTO_L4_TYPE_AH; + else if (input_flags.ipsec_esp) + return ODP_PROTO_L4_TYPE_ESP; + else if (input_flags.icmp && input_flags.ipv4) + return ODP_PROTO_L4_TYPE_ICMPV4; + else if (input_flags.icmp && input_flags.ipv6) + return ODP_PROTO_L4_TYPE_ICMPV6; + else if (input_flags.no_next_hdr) + return ODP_PROTO_L4_TYPE_NO_NEXT; + + return ODP_PROTO_L4_TYPE_NONE; +} + +_ODP_INLINE odp_packet_chksum_status_t odp_packet_l3_chksum_status(odp_packet_t pkt) +{ + _odp_packet_flags_t flags; + _odp_packet_input_flags_t input_flags; + + flags.all_flags = _odp_pkt_get(pkt, uint32_t, flags); + input_flags.all = _odp_pkt_get(pkt, uint64_t, input_flags); + + if (!input_flags.l3_chksum_done) + return ODP_PACKET_CHKSUM_UNKNOWN; + + if (flags.l3_chksum_err) + return ODP_PACKET_CHKSUM_BAD; + + return ODP_PACKET_CHKSUM_OK; +} + +_ODP_INLINE odp_packet_chksum_status_t odp_packet_l4_chksum_status(odp_packet_t pkt) +{ + _odp_packet_flags_t flags; + _odp_packet_input_flags_t input_flags; + + flags.all_flags = _odp_pkt_get(pkt, uint32_t, flags); + input_flags.all = _odp_pkt_get(pkt, uint64_t, input_flags); + + if (!input_flags.l4_chksum_done) + return ODP_PACKET_CHKSUM_UNKNOWN; + + if (flags.l4_chksum_err) + return ODP_PACKET_CHKSUM_BAD; + + return ODP_PACKET_CHKSUM_OK; +} + +_ODP_INLINE void odp_packet_l3_chksum_insert(odp_packet_t pkt, int insert) +{ + _odp_packet_flags_t *flags = _odp_pkt_get_ptr(pkt, _odp_packet_flags_t, flags); + + flags->l3_chksum_set = 1; + flags->l3_chksum = !!insert; +} + +_ODP_INLINE void odp_packet_l4_chksum_insert(odp_packet_t pkt, int insert) +{ + _odp_packet_flags_t *flags = _odp_pkt_get_ptr(pkt, _odp_packet_flags_t, flags); + + flags->l4_chksum_set = 1; + flags->l4_chksum = !!insert; +} + _ODP_INLINE uint32_t odp_packet_flow_hash(odp_packet_t pkt) { return _odp_pkt_get(pkt, uint32_t, flow_hash); @@ -283,6 +445,23 @@ _ODP_INLINE odp_time_t odp_packet_ts(odp_packet_t pkt) return _odp_pkt_get(pkt, odp_time_t, timestamp); } +_ODP_INLINE void odp_packet_ts_set(odp_packet_t pkt, odp_time_t timestamp) +{ + odp_time_t *ts = _odp_pkt_get_ptr(pkt, odp_time_t, timestamp); + _odp_packet_input_flags_t *input_flags = _odp_pkt_get_ptr(pkt, _odp_packet_input_flags_t, + input_flags); + + *ts = timestamp; + input_flags->timestamp = 1; +} + +_ODP_INLINE void odp_packet_ts_request(odp_packet_t pkt, int enable) +{ + _odp_packet_flags_t *flags = _odp_pkt_get_ptr(pkt, _odp_packet_flags_t, flags); + + flags->ts_set = !!enable; +} + _ODP_INLINE void *odp_packet_head(odp_packet_t pkt) { return (uint8_t *)odp_packet_data(pkt) - odp_packet_headroom(pkt); @@ -364,6 +543,8 @@ _ODP_INLINE int odp_packet_copy_to_mem(odp_packet_t pkt, uint32_t offset, _ODP_INLINE odp_packet_t odp_packet_from_event(odp_event_t ev) { + _ODP_ASSERT(odp_event_type(ev) == ODP_EVENT_PACKET); + return (odp_packet_t)ev; } @@ -398,6 +579,8 @@ _ODP_INLINE odp_event_subtype_t odp_packet_subtype(odp_packet_t pkt) _ODP_INLINE odp_packet_tx_compl_t odp_packet_tx_compl_from_event(odp_event_t ev) { + _ODP_ASSERT(odp_event_type(ev) == ODP_EVENT_PACKET_TX_COMPL); + return (odp_packet_tx_compl_t)(uintptr_t)ev; } @@ -433,6 +616,15 @@ _ODP_INLINE int8_t odp_packet_shaper_len_adjust(odp_packet_t pkt) return (int8_t)flags.shaper_len_adj; } +_ODP_INLINE uint64_t odp_packet_cls_mark(odp_packet_t pkt) +{ + _odp_packet_input_flags_t input_flags; + + input_flags.all = _odp_pkt_get(pkt, uint64_t, input_flags); + + return input_flags.cls_mark ? _odp_pkt_get(pkt, uint16_t, cls_mark) : 0; +} + _ODP_INLINE uint32_t odp_packet_buf_data_len(odp_packet_buf_t pkt_buf) { return _odp_pkt_get(pkt_buf, uint32_t, seg_len); diff --git a/platform/linux-generic/include/odp/api/plat/packet_vector_inlines.h b/platform/linux-generic/include/odp/api/plat/packet_vector_inlines.h index 76604dc4f..08fb07a62 100644 --- a/platform/linux-generic/include/odp/api/plat/packet_vector_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/packet_vector_inlines.h @@ -14,10 +14,11 @@ #ifndef _ODP_PLAT_PACKET_VECTOR_INLINES_H_ #define _ODP_PLAT_PACKET_VECTOR_INLINES_H_ -#include <odp/api/event_types.h> +#include <odp/api/event.h> #include <odp/api/packet_types.h> #include <odp/api/pool_types.h> +#include <odp/api/plat/debug_inlines.h> #include <odp/api/plat/event_vector_inline_types.h> #include <stdint.h> @@ -45,6 +46,8 @@ extern const _odp_event_vector_inline_offset_t _odp_event_vector_inline; _ODP_INLINE odp_packet_vector_t odp_packet_vector_from_event(odp_event_t ev) { + _ODP_ASSERT(odp_event_type(ev) == ODP_EVENT_PACKET_VECTOR); + return (odp_packet_vector_t)ev; } diff --git a/platform/linux-generic/include/odp/api/plat/rwlock_inlines.h b/platform/linux-generic/include/odp/api/plat/rwlock_inlines.h new file mode 100644 index 000000000..0bb9d8c6e --- /dev/null +++ b/platform/linux-generic/include/odp/api/plat/rwlock_inlines.h @@ -0,0 +1,105 @@ +/* Copyright (c) 2014-2018, Linaro Limited + * Copyright (c) 2022, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_PLAT_RWLOCK_INLINES_H_ +#define ODP_PLAT_RWLOCK_INLINES_H_ + +#include <odp/api/atomic.h> +#include <odp/api/cpu.h> + +#include <odp/api/abi/rwlock.h> + +#include <stdint.h> + +/** @cond _ODP_HIDE_FROM_DOXYGEN_ */ + +#ifndef _ODP_NO_INLINE + /* Inline functions by default */ + #define _ODP_INLINE static inline + #define odp_rwlock_init __odp_rwlock_init + #define odp_rwlock_read_lock __odp_rwlock_read_lock + #define odp_rwlock_read_trylock __odp_rwlock_read_trylock + #define odp_rwlock_read_unlock __odp_rwlock_read_unlock + #define odp_rwlock_write_lock __odp_rwlock_write_lock + #define odp_rwlock_write_trylock __odp_rwlock_write_trylock + #define odp_rwlock_write_unlock __odp_rwlock_write_unlock +#else + #undef _ODP_INLINE + #define _ODP_INLINE +#endif + +_ODP_INLINE void odp_rwlock_init(odp_rwlock_t *rwlock) +{ + odp_atomic_init_u32(&rwlock->cnt, 0); +} + +_ODP_INLINE void odp_rwlock_read_lock(odp_rwlock_t *rwlock) +{ + uint32_t cnt; + int is_locked = 0; + + while (is_locked == 0) { + cnt = odp_atomic_load_u32(&rwlock->cnt); + /* waiting for read lock */ + if ((int32_t)cnt < 0) { + odp_cpu_pause(); + continue; + } + is_locked = odp_atomic_cas_acq_u32(&rwlock->cnt, &cnt, cnt + 1); + } +} + +_ODP_INLINE int odp_rwlock_read_trylock(odp_rwlock_t *rwlock) +{ + uint32_t cnt = odp_atomic_load_u32(&rwlock->cnt); + + while (cnt != (uint32_t)-1) { + if (odp_atomic_cas_acq_u32(&rwlock->cnt, &cnt, cnt + 1)) + return 1; + } + + return 0; +} + +_ODP_INLINE void odp_rwlock_read_unlock(odp_rwlock_t *rwlock) +{ + odp_atomic_sub_rel_u32(&rwlock->cnt, 1); +} + +_ODP_INLINE void odp_rwlock_write_lock(odp_rwlock_t *rwlock) +{ + uint32_t cnt; + int is_locked = 0; + + while (is_locked == 0) { + uint32_t zero = 0; + + cnt = odp_atomic_load_u32(&rwlock->cnt); + /* lock acquired, wait */ + if (cnt != 0) { + odp_cpu_pause(); + continue; + } + is_locked = odp_atomic_cas_acq_u32(&rwlock->cnt, &zero, (uint32_t)-1); + } +} + +_ODP_INLINE int odp_rwlock_write_trylock(odp_rwlock_t *rwlock) +{ + uint32_t zero = 0; + + return odp_atomic_cas_acq_u32(&rwlock->cnt, &zero, (uint32_t)-1); +} + +_ODP_INLINE void odp_rwlock_write_unlock(odp_rwlock_t *rwlock) +{ + odp_atomic_store_rel_u32(&rwlock->cnt, 0); +} + +/** @endcond */ + +#endif diff --git a/platform/linux-generic/include/odp/api/plat/rwlock_recursive_inlines.h b/platform/linux-generic/include/odp/api/plat/rwlock_recursive_inlines.h new file mode 100644 index 000000000..21ad4be4a --- /dev/null +++ b/platform/linux-generic/include/odp/api/plat/rwlock_recursive_inlines.h @@ -0,0 +1,142 @@ +/* Copyright (c) 2013-2018, Linaro Limited + * Copyright (c) 2022, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_PLAT_RWLOCK_RECURSIVE_INLINES_H_ +#define ODP_PLAT_RWLOCK_RECURSIVE_INLINES_H_ + +#include <odp/api/rwlock.h> +#include <odp/api/thread.h> + +#include <odp/api/abi/rwlock_recursive.h> + +#include <odp/api/plat/debug_inlines.h> + +#include <stdint.h> +#include <string.h> + +/** @cond _ODP_HIDE_FROM_DOXYGEN_ */ + +#ifndef _ODP_NO_INLINE + /* Inline functions by default */ + #define _ODP_INLINE static inline + #define odp_rwlock_recursive_init __odp_rwlock_recursive_init + #define odp_rwlock_recursive_read_lock __odp_rwlock_recursive_read_lock + #define odp_rwlock_recursive_read_trylock __odp_rwlock_recursive_read_trylock + #define odp_rwlock_recursive_read_unlock __odp_rwlock_recursive_read_unlock + #define odp_rwlock_recursive_write_lock __odp_rwlock_recursive_write_lock + #define odp_rwlock_recursive_write_trylock __odp_rwlock_recursive_write_trylock + #define odp_rwlock_recursive_write_unlock __odp_rwlock_recursive_write_unlock +#else + #undef _ODP_INLINE + #define _ODP_INLINE +#endif + +_ODP_INLINE void odp_rwlock_recursive_init(odp_rwlock_recursive_t *rlock) +{ + memset(rlock, 0, sizeof(odp_rwlock_recursive_t)); + odp_rwlock_init(&rlock->lock); + rlock->wr_owner = -1; +} + +/* Multiple readers can recurse the lock concurrently */ +_ODP_INLINE void odp_rwlock_recursive_read_lock(odp_rwlock_recursive_t *rlock) +{ + int thr = odp_thread_id(); + + if (rlock->rd_cnt[thr]) { + _ODP_ASSERT(rlock->rd_cnt[thr] < UINT8_MAX); + rlock->rd_cnt[thr]++; + return; + } + + odp_rwlock_read_lock(&rlock->lock); + rlock->rd_cnt[thr] = 1; +} + +/* Multiple readers can recurse the lock concurrently */ +_ODP_INLINE int odp_rwlock_recursive_read_trylock(odp_rwlock_recursive_t *rlock) +{ + int thr = odp_thread_id(); + + if (rlock->rd_cnt[thr]) { + _ODP_ASSERT(rlock->rd_cnt[thr] < UINT8_MAX); + rlock->rd_cnt[thr]++; + return 1; + } + + if (odp_rwlock_read_trylock(&rlock->lock)) { + rlock->rd_cnt[thr] = 1; + return 1; + } + + return 0; +} + +_ODP_INLINE void odp_rwlock_recursive_read_unlock(odp_rwlock_recursive_t *rlock) +{ + int thr = odp_thread_id(); + + _ODP_ASSERT(rlock->rd_cnt[thr]); + rlock->rd_cnt[thr]--; + + if (rlock->rd_cnt[thr] > 0) + return; + + odp_rwlock_read_unlock(&rlock->lock); +} + +/* Only one writer can recurse the lock */ +_ODP_INLINE void odp_rwlock_recursive_write_lock(odp_rwlock_recursive_t *rlock) +{ + int thr = odp_thread_id(); + + if (rlock->wr_owner == thr) { + _ODP_ASSERT(rlock->wr_cnt < UINT32_MAX); + rlock->wr_cnt++; + return; + } + + odp_rwlock_write_lock(&rlock->lock); + rlock->wr_owner = thr; + rlock->wr_cnt = 1; +} + +/* Only one writer can recurse the lock */ +_ODP_INLINE int odp_rwlock_recursive_write_trylock(odp_rwlock_recursive_t *rlock) +{ + int thr = odp_thread_id(); + + if (rlock->wr_owner == thr) { + _ODP_ASSERT(rlock->wr_cnt < UINT32_MAX); + rlock->wr_cnt++; + return 1; + } + + if (odp_rwlock_write_trylock(&rlock->lock)) { + rlock->wr_owner = thr; + rlock->wr_cnt = 1; + return 1; + } + + return 0; +} + +_ODP_INLINE void odp_rwlock_recursive_write_unlock(odp_rwlock_recursive_t *rlock) +{ + _ODP_ASSERT(rlock->wr_cnt); + rlock->wr_cnt--; + + if (rlock->wr_cnt > 0) + return; + + rlock->wr_owner = -1; + odp_rwlock_write_unlock(&rlock->lock); +} + +/** @endcond */ + +#endif diff --git a/platform/linux-generic/include/odp/api/plat/schedule_inline_types.h b/platform/linux-generic/include/odp/api/plat/schedule_inline_types.h new file mode 100644 index 000000000..92089c9a0 --- /dev/null +++ b/platform/linux-generic/include/odp/api/plat/schedule_inline_types.h @@ -0,0 +1,68 @@ +/* Copyright (c) 2022, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_PLAT_SCHEDULE_INLINE_TYPES_H_ +#define ODP_PLAT_SCHEDULE_INLINE_TYPES_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <odp/api/event_types.h> +#include <odp/api/queue_types.h> +#include <odp/api/schedule_types.h> +#include <odp/api/thrmask.h> + +#include <stdint.h> + +/** @cond _ODP_HIDE_FROM_DOXYGEN_ */ + +/* Schedule API functions */ +typedef struct { + uint64_t (*schedule_wait_time)(uint64_t ns); + int (*schedule_capability)(odp_schedule_capability_t *capa); + void (*schedule_config_init)(odp_schedule_config_t *config); + int (*schedule_config)(const odp_schedule_config_t *config); + odp_event_t (*schedule)(odp_queue_t *from, uint64_t wait); + int (*schedule_multi)(odp_queue_t *from, uint64_t wait, odp_event_t events[], int num); + int (*schedule_multi_wait)(odp_queue_t *from, odp_event_t events[], int num); + int (*schedule_multi_no_wait)(odp_queue_t *from, odp_event_t events[], int num); + void (*schedule_pause)(void); + void (*schedule_resume)(void); + void (*schedule_release_atomic)(void); + void (*schedule_release_ordered)(void); + void (*schedule_prefetch)(int num); + int (*schedule_min_prio)(void); + int (*schedule_max_prio)(void); + int (*schedule_default_prio)(void); + int (*schedule_num_prio)(void); + odp_schedule_group_t (*schedule_group_create)(const char *name, const odp_thrmask_t *mask); + int (*schedule_group_destroy)(odp_schedule_group_t group); + odp_schedule_group_t (*schedule_group_lookup)(const char *name); + int (*schedule_group_join)(odp_schedule_group_t group, const odp_thrmask_t *mask); + int (*schedule_group_leave)(odp_schedule_group_t group, const odp_thrmask_t *mask); + int (*schedule_group_thrmask)(odp_schedule_group_t group, odp_thrmask_t *mask); + int (*schedule_group_info)(odp_schedule_group_t group, odp_schedule_group_info_t *info); + void (*schedule_order_lock)(uint32_t lock_index); + void (*schedule_order_unlock)(uint32_t lock_index); + void (*schedule_order_unlock_lock)(uint32_t unlock_index, uint32_t lock_index); + void (*schedule_order_lock_start)(uint32_t lock_index); + void (*schedule_order_lock_wait)(uint32_t lock_index); + void (*schedule_order_wait)(void); + void (*schedule_print)(void); + +} _odp_schedule_api_fn_t; + +/* Scheduler configuration status */ +int _odp_schedule_configured(void); + +/** @endcond */ + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/include/odp/api/plat/schedule_inlines.h b/platform/linux-generic/include/odp/api/plat/schedule_inlines.h new file mode 100644 index 000000000..733b068d9 --- /dev/null +++ b/platform/linux-generic/include/odp/api/plat/schedule_inlines.h @@ -0,0 +1,135 @@ +/* Copyright (c) 2022, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_PLAT_SCHEDULE_INLINES_H_ +#define ODP_PLAT_SCHEDULE_INLINES_H_ + +#include <odp/api/event_types.h> +#include <odp/api/queue_types.h> + +#include <odp/api/plat/debug_inlines.h> +#include <odp/api/plat/schedule_inline_types.h> + +#include <stdint.h> + +/** @cond _ODP_HIDE_FROM_DOXYGEN_ */ + +extern const _odp_schedule_api_fn_t *_odp_sched_api; + +#ifndef _ODP_NO_INLINE + /* Inline functions by default */ + #define _ODP_INLINE static inline + #define odp_schedule __odp_schedule + #define odp_schedule_multi __odp_schedule_multi + #define odp_schedule_multi_wait __odp_schedule_multi_wait + #define odp_schedule_multi_no_wait __odp_schedule_multi_no_wait + #define odp_schedule_wait_time __odp_schedule_wait_time + #define odp_schedule_pause __odp_schedule_pause + #define odp_schedule_resume __odp_schedule_resume + #define odp_schedule_release_atomic __odp_schedule_release_atomic + #define odp_schedule_release_ordered __odp_schedule_release_ordered + #define odp_schedule_prefetch __odp_schedule_prefetch + #define odp_schedule_order_lock __odp_schedule_order_lock + #define odp_schedule_order_unlock __odp_schedule_order_unlock + #define odp_schedule_order_unlock_lock __odp_schedule_order_unlock_lock + #define odp_schedule_order_lock_start __odp_schedule_order_lock_start + #define odp_schedule_order_lock_wait __odp_schedule_order_lock_wait + #define odp_schedule_order_wait __odp_schedule_order_wait +#else + #define _ODP_INLINE +#endif + +_ODP_INLINE odp_event_t odp_schedule(odp_queue_t *from, uint64_t wait) +{ + _ODP_ASSERT(_odp_schedule_configured()); + + return _odp_sched_api->schedule(from, wait); +} + +_ODP_INLINE int odp_schedule_multi(odp_queue_t *from, uint64_t wait, odp_event_t events[], int num) +{ + _ODP_ASSERT(_odp_schedule_configured()); + + return _odp_sched_api->schedule_multi(from, wait, events, num); +} + +_ODP_INLINE int odp_schedule_multi_wait(odp_queue_t *from, odp_event_t events[], int num) +{ + _ODP_ASSERT(_odp_schedule_configured()); + + return _odp_sched_api->schedule_multi_wait(from, events, num); +} + +_ODP_INLINE int odp_schedule_multi_no_wait(odp_queue_t *from, odp_event_t events[], int num) +{ + _ODP_ASSERT(_odp_schedule_configured()); + + return _odp_sched_api->schedule_multi_no_wait(from, events, num); +} + +_ODP_INLINE uint64_t odp_schedule_wait_time(uint64_t ns) +{ + return _odp_sched_api->schedule_wait_time(ns); +} + +_ODP_INLINE void odp_schedule_pause(void) +{ + _odp_sched_api->schedule_pause(); +} + +_ODP_INLINE void odp_schedule_resume(void) +{ + _odp_sched_api->schedule_resume(); +} + +_ODP_INLINE void odp_schedule_release_atomic(void) +{ + _odp_sched_api->schedule_release_atomic(); +} + +_ODP_INLINE void odp_schedule_release_ordered(void) +{ + _odp_sched_api->schedule_release_ordered(); +} + +_ODP_INLINE void odp_schedule_prefetch(int num) +{ + _odp_sched_api->schedule_prefetch(num); +} + +_ODP_INLINE void odp_schedule_order_lock(uint32_t lock_index) +{ + _odp_sched_api->schedule_order_lock(lock_index); +} + +_ODP_INLINE void odp_schedule_order_unlock(uint32_t lock_index) +{ + _odp_sched_api->schedule_order_unlock(lock_index); +} + +_ODP_INLINE void odp_schedule_order_unlock_lock(uint32_t unlock_index, uint32_t lock_index) +{ + _odp_sched_api->schedule_order_unlock_lock(unlock_index, lock_index); +} + +_ODP_INLINE void odp_schedule_order_lock_start(uint32_t lock_index) +{ + _odp_sched_api->schedule_order_lock_start(lock_index); +} + +_ODP_INLINE void odp_schedule_order_lock_wait(uint32_t lock_index) +{ + _odp_sched_api->schedule_order_lock_wait(lock_index); +} + +_ODP_INLINE void odp_schedule_order_wait(void) +{ + _odp_sched_api->schedule_order_wait(); +} + +/** @endcond */ + +#endif diff --git a/platform/linux-generic/include/odp/api/plat/spinlock_recursive_inlines.h b/platform/linux-generic/include/odp/api/plat/spinlock_recursive_inlines.h index 2dd846fe9..e795353f4 100644 --- a/platform/linux-generic/include/odp/api/plat/spinlock_recursive_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/spinlock_recursive_inlines.h @@ -13,6 +13,10 @@ #include <odp/api/abi/spinlock_recursive.h> +#include <odp/api/plat/debug_inlines.h> + +#include <stdint.h> + /** @cond _ODP_HIDE_FROM_DOXYGEN_ */ #ifndef _ODP_NO_INLINE @@ -43,6 +47,7 @@ _ODP_INLINE void odp_spinlock_recursive_lock(odp_spinlock_recursive_t *rlock) int thr = odp_thread_id(); if (rlock->owner == thr) { + _ODP_ASSERT(rlock->cnt < UINT32_MAX); rlock->cnt++; return; } @@ -57,6 +62,7 @@ _ODP_INLINE int odp_spinlock_recursive_trylock(odp_spinlock_recursive_t *rlock) int thr = odp_thread_id(); if (rlock->owner == thr) { + _ODP_ASSERT(rlock->cnt < UINT32_MAX); rlock->cnt++; return 1; } @@ -72,6 +78,7 @@ _ODP_INLINE int odp_spinlock_recursive_trylock(odp_spinlock_recursive_t *rlock) _ODP_INLINE void odp_spinlock_recursive_unlock(odp_spinlock_recursive_t *rlock) { + _ODP_ASSERT(rlock->cnt); rlock->cnt--; if (rlock->cnt > 0) diff --git a/platform/linux-generic/include/odp/api/plat/thread_inline_types.h b/platform/linux-generic/include/odp/api/plat/thread_inline_types.h new file mode 100644 index 000000000..c9a15b06b --- /dev/null +++ b/platform/linux-generic/include/odp/api/plat/thread_inline_types.h @@ -0,0 +1,36 @@ +/* Copyright (c) 2018-2018, Linaro Limited + * Copyright (c) 2022, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_PLAT_THREAD_INLINE_TYPES_H_ +#define ODP_PLAT_THREAD_INLINE_TYPES_H_ + +#include <odp/api/init.h> +#include <odp/api/spec/thread_types.h> + +#ifdef __cplusplus +extern "C" { +#endif + +/** @cond _ODP_HIDE_FROM_DOXYGEN_ */ + +typedef struct { + odp_log_func_t log_fn; + odp_thread_type_t type; + int thr; + int cpu; + +} _odp_thread_state_t; + +extern __thread _odp_thread_state_t *_odp_this_thread; + +/** @endcond */ + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/include/odp/api/plat/thread_inlines.h b/platform/linux-generic/include/odp/api/plat/thread_inlines.h index fc9275209..2f2e50b06 100644 --- a/platform/linux-generic/include/odp/api/plat/thread_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/thread_inlines.h @@ -9,22 +9,14 @@ #include <odp/api/init.h> +#include <odp/api/plat/thread_inline_types.h> + #ifdef __cplusplus extern "C" { #endif /** @cond _ODP_HIDE_FROM_DOXYGEN_ */ -typedef struct { - int thr; - int cpu; - odp_thread_type_t type; - odp_log_func_t log_fn; - -} _odp_thread_state_t; - -extern __thread _odp_thread_state_t *_odp_this_thread; - #ifndef _ODP_NO_INLINE /* Inline functions by default */ #define _ODP_INLINE static inline diff --git a/platform/linux-generic/include/odp/api/plat/timer_inlines.h b/platform/linux-generic/include/odp/api/plat/timer_inlines.h index 7642376d0..4f5620caf 100644 --- a/platform/linux-generic/include/odp/api/plat/timer_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/timer_inlines.h @@ -10,6 +10,7 @@ #include <odp/api/event.h> #include <odp/api/timer_types.h> +#include <odp/api/plat/debug_inlines.h> #include <odp/api/plat/timer_inline_types.h> #include <stdint.h> @@ -71,6 +72,8 @@ _ODP_INLINE uint64_t odp_timer_ns_to_tick(odp_timer_pool_t tp, uint64_t ns) _ODP_INLINE odp_timeout_t odp_timeout_from_event(odp_event_t ev) { + _ODP_ASSERT(odp_event_type(ev) == ODP_EVENT_TIMEOUT); + return (odp_timeout_t)ev; } |