diff options
Diffstat (limited to 'platform/linux-generic/include')
26 files changed, 901 insertions, 163 deletions
diff --git a/platform/linux-generic/include/odp/api/plat/buffer_inline_types.h b/platform/linux-generic/include/odp/api/plat/buffer_inline_types.h new file mode 100644 index 000000000..9689ddd06 --- /dev/null +++ b/platform/linux-generic/include/odp/api/plat/buffer_inline_types.h @@ -0,0 +1,35 @@ +/* Copyright (c) 2022, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_PLAT_BUFFER_INLINE_TYPES_H_ +#define ODP_PLAT_BUFFER_INLINE_TYPES_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <stdint.h> + +/** @cond _ODP_HIDE_FROM_DOXYGEN_ */ + +/* Buffer header field accessors */ +#define _odp_buffer_get(buffer_hdr, cast, field) \ + (*(cast *)(uintptr_t)((uint8_t *)buffer_hdr + \ + _odp_buffer_inline_offset.field)) + +/* Buffer header field offsets for inline functions */ +typedef struct _odp_buffer_inline_offset_t { + uint16_t uarea_addr; + +} _odp_buffer_inline_offset_t; + +/** @endcond */ + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/include/odp/api/plat/buffer_inlines.h b/platform/linux-generic/include/odp/api/plat/buffer_inlines.h index 576eb846e..d876f60f4 100644 --- a/platform/linux-generic/include/odp/api/plat/buffer_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/buffer_inlines.h @@ -7,16 +7,21 @@ #ifndef ODP_PLAT_BUFFER_INLINES_H_ #define ODP_PLAT_BUFFER_INLINES_H_ -#include <odp/api/event_types.h> +#include <odp/api/event.h> #include <odp/api/pool_types.h> #include <odp/api/abi/buffer.h> +#include <odp/api/plat/buffer_inline_types.h> +#include <odp/api/plat/debug_inlines.h> #include <odp/api/plat/event_inline_types.h> +#include <odp/api/plat/pool_inline_types.h> /** @cond _ODP_HIDE_FROM_DOXYGEN_ */ extern const _odp_event_inline_offset_t _odp_event_inline_offset; +extern const _odp_buffer_inline_offset_t _odp_buffer_inline_offset; +extern const _odp_pool_inline_offset_t _odp_pool_inline; #ifndef _ODP_NO_INLINE /* Inline functions by default */ @@ -24,13 +29,17 @@ extern const _odp_event_inline_offset_t _odp_event_inline_offset; #define odp_buffer_from_event __odp_buffer_from_event #define odp_buffer_to_event __odp_buffer_to_event #define odp_buffer_addr __odp_buffer_addr + #define odp_buffer_size __odp_buffer_size #define odp_buffer_pool __odp_buffer_pool + #define odp_buffer_user_area __odp_buffer_user_area #else #define _ODP_INLINE #endif _ODP_INLINE odp_buffer_t odp_buffer_from_event(odp_event_t ev) { + _ODP_ASSERT(odp_event_type(ev) == ODP_EVENT_BUFFER); + return (odp_buffer_t)ev; } @@ -44,11 +53,23 @@ _ODP_INLINE void *odp_buffer_addr(odp_buffer_t buf) return _odp_event_hdr_field((odp_event_t)buf, void *, base_data); } +_ODP_INLINE uint32_t odp_buffer_size(odp_buffer_t buf) +{ + odp_pool_t pool = _odp_event_hdr_field(buf, odp_pool_t, pool); + + return _odp_pool_get(pool, uint32_t, seg_len); +} + _ODP_INLINE odp_pool_t odp_buffer_pool(odp_buffer_t buf) { return _odp_event_hdr_field(buf, odp_pool_t, pool); } +_ODP_INLINE void *odp_buffer_user_area(odp_buffer_t buf) +{ + return _odp_buffer_get(buf, void *, uarea_addr); +} + /** @endcond */ #endif diff --git a/platform/linux-generic/include/odp/api/plat/debug_inlines.h b/platform/linux-generic/include/odp/api/plat/debug_inlines.h new file mode 100644 index 000000000..bada48f9b --- /dev/null +++ b/platform/linux-generic/include/odp/api/plat/debug_inlines.h @@ -0,0 +1,102 @@ +/* Copyright (c) 2014-2018, Linaro Limited + * Copyright (c) 2020-2022, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +/** + * @file + * + * ODP Debug inlines + * + * @warning These definitions are not part of ODP API, they are for + * implementation internal use only. + */ + +#ifndef ODP_DEBUG_INLINES_H_ +#define ODP_DEBUG_INLINES_H_ + +#include <odp/autoheader_external.h> + +#include <odp/api/hints.h> +#include <odp/api/init.h> + +#include <odp/api/plat/thread_inline_types.h> + +/** @cond _ODP_HIDE_FROM_DOXYGEN_ */ + +#ifdef __cplusplus +extern "C" { +#endif + +/* Avoid "ISO C99 requires at least one argument for the "..." in a variadic + * macro" errors when building with 'pedantic' option. */ +#pragma GCC system_header + +extern odp_log_func_t ODP_PRINTF_FORMAT(2, 3) _odp_log_fn; +extern odp_abort_func_t _odp_abort_fn; + +#define _ODP_LOG_FN(level, fmt, ...) \ + do { \ + if (_odp_this_thread && _odp_this_thread->log_fn) \ + _odp_this_thread->log_fn(level, fmt, ##__VA_ARGS__); \ + else \ + _odp_log_fn(level, fmt, ##__VA_ARGS__); \ + } while (0) + +/** + * ODP LOG macro. + */ +#define _ODP_LOG(level, fmt, ...) \ + _ODP_LOG_FN(level, "%s:%d:%s():" fmt, __FILE__, \ + __LINE__, __func__, ##__VA_ARGS__) + +/** + * Runtime assertion-macro - aborts if 'cond' is false. + */ +#define _ODP_ASSERT(cond) \ + do { if ((ODP_DEBUG == 1) && (!(cond))) { \ + _ODP_ERR("%s\n", #cond); \ + _odp_abort_fn(); } \ + } while (0) + +/* + * Print debug message to log, if ODP_DEBUG_PRINT flag is set (ignores CONFIG_DEBUG_LEVEL). + */ +#define _ODP_DBG(fmt, ...) \ + do { \ + if (ODP_DEBUG_PRINT == 1) \ + _ODP_LOG(ODP_LOG_DBG, fmt, ##__VA_ARGS__);\ + } while (0) + +/** + * Log error message. + */ +#define _ODP_ERR(fmt, ...) \ + _ODP_LOG(ODP_LOG_ERR, fmt, ##__VA_ARGS__) + +/** + * Log abort message and then stop execution (by default call abort()). + * This function should not return. + */ +#define _ODP_ABORT(fmt, ...) \ + do { \ + _ODP_LOG(ODP_LOG_ABORT, fmt, ##__VA_ARGS__); \ + _odp_abort_fn(); \ + } while (0) + +/** + * Log print message when the application calls one of the ODP APIs + * specifically for dumping internal data. + */ +#define _ODP_PRINT(fmt, ...) \ + _ODP_LOG_FN(ODP_LOG_PRINT, fmt, ##__VA_ARGS__) + +#ifdef __cplusplus +} +#endif + +/** @endcond */ + +#endif diff --git a/platform/linux-generic/include/odp/api/plat/event_inlines.h b/platform/linux-generic/include/odp/api/plat/event_inlines.h index 27c2a82fd..dd8e9249c 100644 --- a/platform/linux-generic/include/odp/api/plat/event_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/event_inlines.h @@ -9,13 +9,15 @@ #define ODP_PLAT_EVENT_INLINES_H_ #include <odp/api/event_types.h> -#include <odp/api/packet.h> +#include <odp/api/packet_types.h> #include <odp/api/plat/event_inline_types.h> +#include <odp/api/plat/packet_inline_types.h> /** @cond _ODP_HIDE_FROM_DOXYGEN_ */ extern const _odp_event_inline_offset_t _odp_event_inline_offset; +extern const _odp_packet_inline_offset_t _odp_packet_inline; #ifndef _ODP_NO_INLINE /* Inline functions by default */ @@ -26,8 +28,6 @@ extern const _odp_event_inline_offset_t _odp_event_inline_offset; #define odp_event_types __odp_event_types #define odp_event_flow_id __odp_event_flow_id #define odp_event_flow_id_set __odp_event_flow_id_set - - #include <odp/api/plat/packet_inlines.h> #else #define _ODP_INLINE #endif @@ -67,7 +67,7 @@ _ODP_INLINE odp_event_subtype_t odp_event_subtype(odp_event_t event) if (__odp_event_type_get(event) != ODP_EVENT_PACKET) return ODP_EVENT_NO_SUBTYPE; - return odp_packet_subtype(odp_packet_from_event(event)); + return (odp_event_subtype_t)_odp_pkt_get((odp_packet_t)event, int8_t, subtype); } _ODP_INLINE odp_event_type_t odp_event_types(odp_event_t event, @@ -76,7 +76,7 @@ _ODP_INLINE odp_event_type_t odp_event_types(odp_event_t event, odp_event_type_t event_type = __odp_event_type_get(event); *subtype = event_type == ODP_EVENT_PACKET ? - odp_packet_subtype(odp_packet_from_event(event)) : + (odp_event_subtype_t)_odp_pkt_get((odp_packet_t)event, int8_t, subtype) : ODP_EVENT_NO_SUBTYPE; return event_type; diff --git a/platform/linux-generic/include/odp/api/plat/packet_inline_types.h b/platform/linux-generic/include/odp/api/plat/packet_inline_types.h index b00173aca..2ae0829c3 100644 --- a/platform/linux-generic/include/odp/api/plat/packet_inline_types.h +++ b/platform/linux-generic/include/odp/api/plat/packet_inline_types.h @@ -51,6 +51,7 @@ typedef struct _odp_packet_inline_offset_t { uint16_t input_flags; uint16_t flags; uint16_t subtype; + uint16_t cls_mark; } _odp_packet_inline_offset_t; diff --git a/platform/linux-generic/include/odp/api/plat/packet_inlines.h b/platform/linux-generic/include/odp/api/plat/packet_inlines.h index da4eabe2e..21d184ac5 100644 --- a/platform/linux-generic/include/odp/api/plat/packet_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/packet_inlines.h @@ -14,16 +14,13 @@ #ifndef _ODP_PLAT_PACKET_INLINES_H_ #define _ODP_PLAT_PACKET_INLINES_H_ +#include <odp/api/event.h> #include <odp/api/hints.h> #include <odp/api/packet_types.h> #include <odp/api/pool_types.h> #include <odp/api/time.h> -#include <odp/api/abi/buffer.h> -#include <odp/api/abi/event_types.h> -#include <odp/api/abi/packet.h> -#include <odp/api/abi/packet_io.h> - +#include <odp/api/plat/debug_inlines.h> #include <odp/api/plat/packet_io_inlines.h> #include <odp/api/plat/packet_inline_types.h> #include <odp/api/plat/pool_inline_types.h> @@ -48,6 +45,7 @@ #define odp_packet_input_index __odp_packet_input_index #define odp_packet_num_segs __odp_packet_num_segs #define odp_packet_user_ptr __odp_packet_user_ptr + #define odp_packet_user_ptr_set __odp_packet_user_ptr_set #define odp_packet_user_area __odp_packet_user_area #define odp_packet_user_area_size __odp_packet_user_area_size #define odp_packet_user_flag __odp_packet_user_flag @@ -55,11 +53,23 @@ #define odp_packet_l2_offset __odp_packet_l2_offset #define odp_packet_l3_offset __odp_packet_l3_offset #define odp_packet_l4_offset __odp_packet_l4_offset + #define odp_packet_l2_offset_set __odp_packet_l2_offset_set + #define odp_packet_l3_offset_set __odp_packet_l3_offset_set + #define odp_packet_l4_offset_set __odp_packet_l4_offset_set #define odp_packet_l2_ptr __odp_packet_l2_ptr #define odp_packet_l3_ptr __odp_packet_l3_ptr #define odp_packet_l4_ptr __odp_packet_l4_ptr + #define odp_packet_l2_type __odp_packet_l2_type + #define odp_packet_l3_type __odp_packet_l3_type + #define odp_packet_l4_type __odp_packet_l4_type + #define odp_packet_l3_chksum_status __odp_packet_l3_chksum_status + #define odp_packet_l4_chksum_status __odp_packet_l4_chksum_status + #define odp_packet_l3_chksum_insert __odp_packet_l3_chksum_insert + #define odp_packet_l4_chksum_insert __odp_packet_l4_chksum_insert #define odp_packet_flow_hash __odp_packet_flow_hash #define odp_packet_ts __odp_packet_ts + #define odp_packet_ts_set __odp_packet_ts_set + #define odp_packet_ts_request __odp_packet_ts_request #define odp_packet_head __odp_packet_head #define odp_packet_is_segmented __odp_packet_is_segmented #define odp_packet_first_seg __odp_packet_first_seg @@ -79,6 +89,7 @@ #define odp_packet_color __odp_packet_color #define odp_packet_drop_eligible __odp_packet_drop_eligible #define odp_packet_shaper_len_adjust __odp_packet_shaper_len_adjust + #define odp_packet_cls_mark __odp_packet_cls_mark #define odp_packet_buf_data_len __odp_packet_buf_data_len #define odp_packet_buf_size __odp_packet_buf_size #define odp_packet_buf_head __odp_packet_buf_head @@ -176,6 +187,20 @@ _ODP_INLINE void *odp_packet_user_ptr(odp_packet_t pkt) return _odp_pkt_get(pkt, void *, user_ptr); } +_ODP_INLINE void odp_packet_user_ptr_set(odp_packet_t pkt, const void *ptr) +{ + _odp_packet_flags_t *flags = _odp_pkt_get_ptr(pkt, _odp_packet_flags_t, flags); + const void **user_ptr = _odp_pkt_get_ptr(pkt, const void *, user_ptr); + + if (odp_unlikely(ptr == NULL)) { + flags->user_ptr_set = 0; + return; + } + + *user_ptr = ptr; + flags->user_ptr_set = 1; +} + _ODP_INLINE void *odp_packet_user_area(odp_packet_t pkt) { return _odp_pkt_get(pkt, void *, user_area); @@ -219,6 +244,42 @@ _ODP_INLINE uint32_t odp_packet_l4_offset(odp_packet_t pkt) return _odp_pkt_get(pkt, uint16_t, l4_offset); } +_ODP_INLINE int odp_packet_l2_offset_set(odp_packet_t pkt, uint32_t offset) +{ + uint16_t *l2_offset = _odp_pkt_get_ptr(pkt, uint16_t, l2_offset); + _odp_packet_input_flags_t *input_flags = _odp_pkt_get_ptr(pkt, _odp_packet_input_flags_t, + input_flags); + + if (odp_unlikely(offset >= odp_packet_len(pkt))) + return -1; + + input_flags->l2 = 1; + *l2_offset = (uint16_t)offset; + return 0; +} + +_ODP_INLINE int odp_packet_l3_offset_set(odp_packet_t pkt, uint32_t offset) +{ + uint16_t *l3_offset = _odp_pkt_get_ptr(pkt, uint16_t, l3_offset); + + if (odp_unlikely(offset >= odp_packet_len(pkt))) + return -1; + + *l3_offset = (uint16_t)offset; + return 0; +} + +_ODP_INLINE int odp_packet_l4_offset_set(odp_packet_t pkt, uint32_t offset) +{ + uint16_t *l4_offset = _odp_pkt_get_ptr(pkt, uint16_t, l4_offset); + + if (odp_unlikely(offset >= odp_packet_len(pkt))) + return -1; + + *l4_offset = (uint16_t)offset; + return 0; +} + _ODP_INLINE void *odp_packet_l2_ptr(odp_packet_t pkt, uint32_t *len) { uint32_t offset = odp_packet_l2_offset(pkt); @@ -273,6 +334,107 @@ _ODP_INLINE void *odp_packet_l4_ptr(odp_packet_t pkt, uint32_t *len) return data + offset; } +_ODP_INLINE odp_proto_l2_type_t odp_packet_l2_type(odp_packet_t pkt) +{ + _odp_packet_input_flags_t input_flags; + + input_flags.all = _odp_pkt_get(pkt, uint64_t, input_flags); + + return input_flags.eth ? ODP_PROTO_L2_TYPE_ETH : ODP_PROTO_L2_TYPE_NONE; +} + +_ODP_INLINE odp_proto_l3_type_t odp_packet_l3_type(odp_packet_t pkt) +{ + _odp_packet_input_flags_t input_flags; + + input_flags.all = _odp_pkt_get(pkt, uint64_t, input_flags); + + if (input_flags.ipv4) + return ODP_PROTO_L3_TYPE_IPV4; + else if (input_flags.ipv6) + return ODP_PROTO_L3_TYPE_IPV6; + else if (input_flags.arp) + return ODP_PROTO_L3_TYPE_ARP; + + return ODP_PROTO_L3_TYPE_NONE; +} + +_ODP_INLINE odp_proto_l4_type_t odp_packet_l4_type(odp_packet_t pkt) +{ + _odp_packet_input_flags_t input_flags; + + input_flags.all = _odp_pkt_get(pkt, uint64_t, input_flags); + + if (input_flags.tcp) + return ODP_PROTO_L4_TYPE_TCP; + else if (input_flags.udp) + return ODP_PROTO_L4_TYPE_UDP; + else if (input_flags.sctp) + return ODP_PROTO_L4_TYPE_SCTP; + else if (input_flags.ipsec_ah) + return ODP_PROTO_L4_TYPE_AH; + else if (input_flags.ipsec_esp) + return ODP_PROTO_L4_TYPE_ESP; + else if (input_flags.icmp && input_flags.ipv4) + return ODP_PROTO_L4_TYPE_ICMPV4; + else if (input_flags.icmp && input_flags.ipv6) + return ODP_PROTO_L4_TYPE_ICMPV6; + else if (input_flags.no_next_hdr) + return ODP_PROTO_L4_TYPE_NO_NEXT; + + return ODP_PROTO_L4_TYPE_NONE; +} + +_ODP_INLINE odp_packet_chksum_status_t odp_packet_l3_chksum_status(odp_packet_t pkt) +{ + _odp_packet_flags_t flags; + _odp_packet_input_flags_t input_flags; + + flags.all_flags = _odp_pkt_get(pkt, uint32_t, flags); + input_flags.all = _odp_pkt_get(pkt, uint64_t, input_flags); + + if (!input_flags.l3_chksum_done) + return ODP_PACKET_CHKSUM_UNKNOWN; + + if (flags.l3_chksum_err) + return ODP_PACKET_CHKSUM_BAD; + + return ODP_PACKET_CHKSUM_OK; +} + +_ODP_INLINE odp_packet_chksum_status_t odp_packet_l4_chksum_status(odp_packet_t pkt) +{ + _odp_packet_flags_t flags; + _odp_packet_input_flags_t input_flags; + + flags.all_flags = _odp_pkt_get(pkt, uint32_t, flags); + input_flags.all = _odp_pkt_get(pkt, uint64_t, input_flags); + + if (!input_flags.l4_chksum_done) + return ODP_PACKET_CHKSUM_UNKNOWN; + + if (flags.l4_chksum_err) + return ODP_PACKET_CHKSUM_BAD; + + return ODP_PACKET_CHKSUM_OK; +} + +_ODP_INLINE void odp_packet_l3_chksum_insert(odp_packet_t pkt, int insert) +{ + _odp_packet_flags_t *flags = _odp_pkt_get_ptr(pkt, _odp_packet_flags_t, flags); + + flags->l3_chksum_set = 1; + flags->l3_chksum = !!insert; +} + +_ODP_INLINE void odp_packet_l4_chksum_insert(odp_packet_t pkt, int insert) +{ + _odp_packet_flags_t *flags = _odp_pkt_get_ptr(pkt, _odp_packet_flags_t, flags); + + flags->l4_chksum_set = 1; + flags->l4_chksum = !!insert; +} + _ODP_INLINE uint32_t odp_packet_flow_hash(odp_packet_t pkt) { return _odp_pkt_get(pkt, uint32_t, flow_hash); @@ -283,6 +445,23 @@ _ODP_INLINE odp_time_t odp_packet_ts(odp_packet_t pkt) return _odp_pkt_get(pkt, odp_time_t, timestamp); } +_ODP_INLINE void odp_packet_ts_set(odp_packet_t pkt, odp_time_t timestamp) +{ + odp_time_t *ts = _odp_pkt_get_ptr(pkt, odp_time_t, timestamp); + _odp_packet_input_flags_t *input_flags = _odp_pkt_get_ptr(pkt, _odp_packet_input_flags_t, + input_flags); + + *ts = timestamp; + input_flags->timestamp = 1; +} + +_ODP_INLINE void odp_packet_ts_request(odp_packet_t pkt, int enable) +{ + _odp_packet_flags_t *flags = _odp_pkt_get_ptr(pkt, _odp_packet_flags_t, flags); + + flags->ts_set = !!enable; +} + _ODP_INLINE void *odp_packet_head(odp_packet_t pkt) { return (uint8_t *)odp_packet_data(pkt) - odp_packet_headroom(pkt); @@ -364,6 +543,8 @@ _ODP_INLINE int odp_packet_copy_to_mem(odp_packet_t pkt, uint32_t offset, _ODP_INLINE odp_packet_t odp_packet_from_event(odp_event_t ev) { + _ODP_ASSERT(odp_event_type(ev) == ODP_EVENT_PACKET); + return (odp_packet_t)ev; } @@ -398,6 +579,8 @@ _ODP_INLINE odp_event_subtype_t odp_packet_subtype(odp_packet_t pkt) _ODP_INLINE odp_packet_tx_compl_t odp_packet_tx_compl_from_event(odp_event_t ev) { + _ODP_ASSERT(odp_event_type(ev) == ODP_EVENT_PACKET_TX_COMPL); + return (odp_packet_tx_compl_t)(uintptr_t)ev; } @@ -433,6 +616,15 @@ _ODP_INLINE int8_t odp_packet_shaper_len_adjust(odp_packet_t pkt) return (int8_t)flags.shaper_len_adj; } +_ODP_INLINE uint64_t odp_packet_cls_mark(odp_packet_t pkt) +{ + _odp_packet_input_flags_t input_flags; + + input_flags.all = _odp_pkt_get(pkt, uint64_t, input_flags); + + return input_flags.cls_mark ? _odp_pkt_get(pkt, uint16_t, cls_mark) : 0; +} + _ODP_INLINE uint32_t odp_packet_buf_data_len(odp_packet_buf_t pkt_buf) { return _odp_pkt_get(pkt_buf, uint32_t, seg_len); diff --git a/platform/linux-generic/include/odp/api/plat/packet_vector_inlines.h b/platform/linux-generic/include/odp/api/plat/packet_vector_inlines.h index 76604dc4f..08fb07a62 100644 --- a/platform/linux-generic/include/odp/api/plat/packet_vector_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/packet_vector_inlines.h @@ -14,10 +14,11 @@ #ifndef _ODP_PLAT_PACKET_VECTOR_INLINES_H_ #define _ODP_PLAT_PACKET_VECTOR_INLINES_H_ -#include <odp/api/event_types.h> +#include <odp/api/event.h> #include <odp/api/packet_types.h> #include <odp/api/pool_types.h> +#include <odp/api/plat/debug_inlines.h> #include <odp/api/plat/event_vector_inline_types.h> #include <stdint.h> @@ -45,6 +46,8 @@ extern const _odp_event_vector_inline_offset_t _odp_event_vector_inline; _ODP_INLINE odp_packet_vector_t odp_packet_vector_from_event(odp_event_t ev) { + _ODP_ASSERT(odp_event_type(ev) == ODP_EVENT_PACKET_VECTOR); + return (odp_packet_vector_t)ev; } diff --git a/platform/linux-generic/include/odp/api/plat/rwlock_inlines.h b/platform/linux-generic/include/odp/api/plat/rwlock_inlines.h new file mode 100644 index 000000000..0bb9d8c6e --- /dev/null +++ b/platform/linux-generic/include/odp/api/plat/rwlock_inlines.h @@ -0,0 +1,105 @@ +/* Copyright (c) 2014-2018, Linaro Limited + * Copyright (c) 2022, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_PLAT_RWLOCK_INLINES_H_ +#define ODP_PLAT_RWLOCK_INLINES_H_ + +#include <odp/api/atomic.h> +#include <odp/api/cpu.h> + +#include <odp/api/abi/rwlock.h> + +#include <stdint.h> + +/** @cond _ODP_HIDE_FROM_DOXYGEN_ */ + +#ifndef _ODP_NO_INLINE + /* Inline functions by default */ + #define _ODP_INLINE static inline + #define odp_rwlock_init __odp_rwlock_init + #define odp_rwlock_read_lock __odp_rwlock_read_lock + #define odp_rwlock_read_trylock __odp_rwlock_read_trylock + #define odp_rwlock_read_unlock __odp_rwlock_read_unlock + #define odp_rwlock_write_lock __odp_rwlock_write_lock + #define odp_rwlock_write_trylock __odp_rwlock_write_trylock + #define odp_rwlock_write_unlock __odp_rwlock_write_unlock +#else + #undef _ODP_INLINE + #define _ODP_INLINE +#endif + +_ODP_INLINE void odp_rwlock_init(odp_rwlock_t *rwlock) +{ + odp_atomic_init_u32(&rwlock->cnt, 0); +} + +_ODP_INLINE void odp_rwlock_read_lock(odp_rwlock_t *rwlock) +{ + uint32_t cnt; + int is_locked = 0; + + while (is_locked == 0) { + cnt = odp_atomic_load_u32(&rwlock->cnt); + /* waiting for read lock */ + if ((int32_t)cnt < 0) { + odp_cpu_pause(); + continue; + } + is_locked = odp_atomic_cas_acq_u32(&rwlock->cnt, &cnt, cnt + 1); + } +} + +_ODP_INLINE int odp_rwlock_read_trylock(odp_rwlock_t *rwlock) +{ + uint32_t cnt = odp_atomic_load_u32(&rwlock->cnt); + + while (cnt != (uint32_t)-1) { + if (odp_atomic_cas_acq_u32(&rwlock->cnt, &cnt, cnt + 1)) + return 1; + } + + return 0; +} + +_ODP_INLINE void odp_rwlock_read_unlock(odp_rwlock_t *rwlock) +{ + odp_atomic_sub_rel_u32(&rwlock->cnt, 1); +} + +_ODP_INLINE void odp_rwlock_write_lock(odp_rwlock_t *rwlock) +{ + uint32_t cnt; + int is_locked = 0; + + while (is_locked == 0) { + uint32_t zero = 0; + + cnt = odp_atomic_load_u32(&rwlock->cnt); + /* lock acquired, wait */ + if (cnt != 0) { + odp_cpu_pause(); + continue; + } + is_locked = odp_atomic_cas_acq_u32(&rwlock->cnt, &zero, (uint32_t)-1); + } +} + +_ODP_INLINE int odp_rwlock_write_trylock(odp_rwlock_t *rwlock) +{ + uint32_t zero = 0; + + return odp_atomic_cas_acq_u32(&rwlock->cnt, &zero, (uint32_t)-1); +} + +_ODP_INLINE void odp_rwlock_write_unlock(odp_rwlock_t *rwlock) +{ + odp_atomic_store_rel_u32(&rwlock->cnt, 0); +} + +/** @endcond */ + +#endif diff --git a/platform/linux-generic/include/odp/api/plat/rwlock_recursive_inlines.h b/platform/linux-generic/include/odp/api/plat/rwlock_recursive_inlines.h new file mode 100644 index 000000000..21ad4be4a --- /dev/null +++ b/platform/linux-generic/include/odp/api/plat/rwlock_recursive_inlines.h @@ -0,0 +1,142 @@ +/* Copyright (c) 2013-2018, Linaro Limited + * Copyright (c) 2022, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_PLAT_RWLOCK_RECURSIVE_INLINES_H_ +#define ODP_PLAT_RWLOCK_RECURSIVE_INLINES_H_ + +#include <odp/api/rwlock.h> +#include <odp/api/thread.h> + +#include <odp/api/abi/rwlock_recursive.h> + +#include <odp/api/plat/debug_inlines.h> + +#include <stdint.h> +#include <string.h> + +/** @cond _ODP_HIDE_FROM_DOXYGEN_ */ + +#ifndef _ODP_NO_INLINE + /* Inline functions by default */ + #define _ODP_INLINE static inline + #define odp_rwlock_recursive_init __odp_rwlock_recursive_init + #define odp_rwlock_recursive_read_lock __odp_rwlock_recursive_read_lock + #define odp_rwlock_recursive_read_trylock __odp_rwlock_recursive_read_trylock + #define odp_rwlock_recursive_read_unlock __odp_rwlock_recursive_read_unlock + #define odp_rwlock_recursive_write_lock __odp_rwlock_recursive_write_lock + #define odp_rwlock_recursive_write_trylock __odp_rwlock_recursive_write_trylock + #define odp_rwlock_recursive_write_unlock __odp_rwlock_recursive_write_unlock +#else + #undef _ODP_INLINE + #define _ODP_INLINE +#endif + +_ODP_INLINE void odp_rwlock_recursive_init(odp_rwlock_recursive_t *rlock) +{ + memset(rlock, 0, sizeof(odp_rwlock_recursive_t)); + odp_rwlock_init(&rlock->lock); + rlock->wr_owner = -1; +} + +/* Multiple readers can recurse the lock concurrently */ +_ODP_INLINE void odp_rwlock_recursive_read_lock(odp_rwlock_recursive_t *rlock) +{ + int thr = odp_thread_id(); + + if (rlock->rd_cnt[thr]) { + _ODP_ASSERT(rlock->rd_cnt[thr] < UINT8_MAX); + rlock->rd_cnt[thr]++; + return; + } + + odp_rwlock_read_lock(&rlock->lock); + rlock->rd_cnt[thr] = 1; +} + +/* Multiple readers can recurse the lock concurrently */ +_ODP_INLINE int odp_rwlock_recursive_read_trylock(odp_rwlock_recursive_t *rlock) +{ + int thr = odp_thread_id(); + + if (rlock->rd_cnt[thr]) { + _ODP_ASSERT(rlock->rd_cnt[thr] < UINT8_MAX); + rlock->rd_cnt[thr]++; + return 1; + } + + if (odp_rwlock_read_trylock(&rlock->lock)) { + rlock->rd_cnt[thr] = 1; + return 1; + } + + return 0; +} + +_ODP_INLINE void odp_rwlock_recursive_read_unlock(odp_rwlock_recursive_t *rlock) +{ + int thr = odp_thread_id(); + + _ODP_ASSERT(rlock->rd_cnt[thr]); + rlock->rd_cnt[thr]--; + + if (rlock->rd_cnt[thr] > 0) + return; + + odp_rwlock_read_unlock(&rlock->lock); +} + +/* Only one writer can recurse the lock */ +_ODP_INLINE void odp_rwlock_recursive_write_lock(odp_rwlock_recursive_t *rlock) +{ + int thr = odp_thread_id(); + + if (rlock->wr_owner == thr) { + _ODP_ASSERT(rlock->wr_cnt < UINT32_MAX); + rlock->wr_cnt++; + return; + } + + odp_rwlock_write_lock(&rlock->lock); + rlock->wr_owner = thr; + rlock->wr_cnt = 1; +} + +/* Only one writer can recurse the lock */ +_ODP_INLINE int odp_rwlock_recursive_write_trylock(odp_rwlock_recursive_t *rlock) +{ + int thr = odp_thread_id(); + + if (rlock->wr_owner == thr) { + _ODP_ASSERT(rlock->wr_cnt < UINT32_MAX); + rlock->wr_cnt++; + return 1; + } + + if (odp_rwlock_write_trylock(&rlock->lock)) { + rlock->wr_owner = thr; + rlock->wr_cnt = 1; + return 1; + } + + return 0; +} + +_ODP_INLINE void odp_rwlock_recursive_write_unlock(odp_rwlock_recursive_t *rlock) +{ + _ODP_ASSERT(rlock->wr_cnt); + rlock->wr_cnt--; + + if (rlock->wr_cnt > 0) + return; + + rlock->wr_owner = -1; + odp_rwlock_write_unlock(&rlock->lock); +} + +/** @endcond */ + +#endif diff --git a/platform/linux-generic/include/odp/api/plat/schedule_inline_types.h b/platform/linux-generic/include/odp/api/plat/schedule_inline_types.h new file mode 100644 index 000000000..92089c9a0 --- /dev/null +++ b/platform/linux-generic/include/odp/api/plat/schedule_inline_types.h @@ -0,0 +1,68 @@ +/* Copyright (c) 2022, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_PLAT_SCHEDULE_INLINE_TYPES_H_ +#define ODP_PLAT_SCHEDULE_INLINE_TYPES_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <odp/api/event_types.h> +#include <odp/api/queue_types.h> +#include <odp/api/schedule_types.h> +#include <odp/api/thrmask.h> + +#include <stdint.h> + +/** @cond _ODP_HIDE_FROM_DOXYGEN_ */ + +/* Schedule API functions */ +typedef struct { + uint64_t (*schedule_wait_time)(uint64_t ns); + int (*schedule_capability)(odp_schedule_capability_t *capa); + void (*schedule_config_init)(odp_schedule_config_t *config); + int (*schedule_config)(const odp_schedule_config_t *config); + odp_event_t (*schedule)(odp_queue_t *from, uint64_t wait); + int (*schedule_multi)(odp_queue_t *from, uint64_t wait, odp_event_t events[], int num); + int (*schedule_multi_wait)(odp_queue_t *from, odp_event_t events[], int num); + int (*schedule_multi_no_wait)(odp_queue_t *from, odp_event_t events[], int num); + void (*schedule_pause)(void); + void (*schedule_resume)(void); + void (*schedule_release_atomic)(void); + void (*schedule_release_ordered)(void); + void (*schedule_prefetch)(int num); + int (*schedule_min_prio)(void); + int (*schedule_max_prio)(void); + int (*schedule_default_prio)(void); + int (*schedule_num_prio)(void); + odp_schedule_group_t (*schedule_group_create)(const char *name, const odp_thrmask_t *mask); + int (*schedule_group_destroy)(odp_schedule_group_t group); + odp_schedule_group_t (*schedule_group_lookup)(const char *name); + int (*schedule_group_join)(odp_schedule_group_t group, const odp_thrmask_t *mask); + int (*schedule_group_leave)(odp_schedule_group_t group, const odp_thrmask_t *mask); + int (*schedule_group_thrmask)(odp_schedule_group_t group, odp_thrmask_t *mask); + int (*schedule_group_info)(odp_schedule_group_t group, odp_schedule_group_info_t *info); + void (*schedule_order_lock)(uint32_t lock_index); + void (*schedule_order_unlock)(uint32_t lock_index); + void (*schedule_order_unlock_lock)(uint32_t unlock_index, uint32_t lock_index); + void (*schedule_order_lock_start)(uint32_t lock_index); + void (*schedule_order_lock_wait)(uint32_t lock_index); + void (*schedule_order_wait)(void); + void (*schedule_print)(void); + +} _odp_schedule_api_fn_t; + +/* Scheduler configuration status */ +int _odp_schedule_configured(void); + +/** @endcond */ + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/include/odp/api/plat/schedule_inlines.h b/platform/linux-generic/include/odp/api/plat/schedule_inlines.h new file mode 100644 index 000000000..733b068d9 --- /dev/null +++ b/platform/linux-generic/include/odp/api/plat/schedule_inlines.h @@ -0,0 +1,135 @@ +/* Copyright (c) 2022, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_PLAT_SCHEDULE_INLINES_H_ +#define ODP_PLAT_SCHEDULE_INLINES_H_ + +#include <odp/api/event_types.h> +#include <odp/api/queue_types.h> + +#include <odp/api/plat/debug_inlines.h> +#include <odp/api/plat/schedule_inline_types.h> + +#include <stdint.h> + +/** @cond _ODP_HIDE_FROM_DOXYGEN_ */ + +extern const _odp_schedule_api_fn_t *_odp_sched_api; + +#ifndef _ODP_NO_INLINE + /* Inline functions by default */ + #define _ODP_INLINE static inline + #define odp_schedule __odp_schedule + #define odp_schedule_multi __odp_schedule_multi + #define odp_schedule_multi_wait __odp_schedule_multi_wait + #define odp_schedule_multi_no_wait __odp_schedule_multi_no_wait + #define odp_schedule_wait_time __odp_schedule_wait_time + #define odp_schedule_pause __odp_schedule_pause + #define odp_schedule_resume __odp_schedule_resume + #define odp_schedule_release_atomic __odp_schedule_release_atomic + #define odp_schedule_release_ordered __odp_schedule_release_ordered + #define odp_schedule_prefetch __odp_schedule_prefetch + #define odp_schedule_order_lock __odp_schedule_order_lock + #define odp_schedule_order_unlock __odp_schedule_order_unlock + #define odp_schedule_order_unlock_lock __odp_schedule_order_unlock_lock + #define odp_schedule_order_lock_start __odp_schedule_order_lock_start + #define odp_schedule_order_lock_wait __odp_schedule_order_lock_wait + #define odp_schedule_order_wait __odp_schedule_order_wait +#else + #define _ODP_INLINE +#endif + +_ODP_INLINE odp_event_t odp_schedule(odp_queue_t *from, uint64_t wait) +{ + _ODP_ASSERT(_odp_schedule_configured()); + + return _odp_sched_api->schedule(from, wait); +} + +_ODP_INLINE int odp_schedule_multi(odp_queue_t *from, uint64_t wait, odp_event_t events[], int num) +{ + _ODP_ASSERT(_odp_schedule_configured()); + + return _odp_sched_api->schedule_multi(from, wait, events, num); +} + +_ODP_INLINE int odp_schedule_multi_wait(odp_queue_t *from, odp_event_t events[], int num) +{ + _ODP_ASSERT(_odp_schedule_configured()); + + return _odp_sched_api->schedule_multi_wait(from, events, num); +} + +_ODP_INLINE int odp_schedule_multi_no_wait(odp_queue_t *from, odp_event_t events[], int num) +{ + _ODP_ASSERT(_odp_schedule_configured()); + + return _odp_sched_api->schedule_multi_no_wait(from, events, num); +} + +_ODP_INLINE uint64_t odp_schedule_wait_time(uint64_t ns) +{ + return _odp_sched_api->schedule_wait_time(ns); +} + +_ODP_INLINE void odp_schedule_pause(void) +{ + _odp_sched_api->schedule_pause(); +} + +_ODP_INLINE void odp_schedule_resume(void) +{ + _odp_sched_api->schedule_resume(); +} + +_ODP_INLINE void odp_schedule_release_atomic(void) +{ + _odp_sched_api->schedule_release_atomic(); +} + +_ODP_INLINE void odp_schedule_release_ordered(void) +{ + _odp_sched_api->schedule_release_ordered(); +} + +_ODP_INLINE void odp_schedule_prefetch(int num) +{ + _odp_sched_api->schedule_prefetch(num); +} + +_ODP_INLINE void odp_schedule_order_lock(uint32_t lock_index) +{ + _odp_sched_api->schedule_order_lock(lock_index); +} + +_ODP_INLINE void odp_schedule_order_unlock(uint32_t lock_index) +{ + _odp_sched_api->schedule_order_unlock(lock_index); +} + +_ODP_INLINE void odp_schedule_order_unlock_lock(uint32_t unlock_index, uint32_t lock_index) +{ + _odp_sched_api->schedule_order_unlock_lock(unlock_index, lock_index); +} + +_ODP_INLINE void odp_schedule_order_lock_start(uint32_t lock_index) +{ + _odp_sched_api->schedule_order_lock_start(lock_index); +} + +_ODP_INLINE void odp_schedule_order_lock_wait(uint32_t lock_index) +{ + _odp_sched_api->schedule_order_lock_wait(lock_index); +} + +_ODP_INLINE void odp_schedule_order_wait(void) +{ + _odp_sched_api->schedule_order_wait(); +} + +/** @endcond */ + +#endif diff --git a/platform/linux-generic/include/odp/api/plat/spinlock_recursive_inlines.h b/platform/linux-generic/include/odp/api/plat/spinlock_recursive_inlines.h index 2dd846fe9..e795353f4 100644 --- a/platform/linux-generic/include/odp/api/plat/spinlock_recursive_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/spinlock_recursive_inlines.h @@ -13,6 +13,10 @@ #include <odp/api/abi/spinlock_recursive.h> +#include <odp/api/plat/debug_inlines.h> + +#include <stdint.h> + /** @cond _ODP_HIDE_FROM_DOXYGEN_ */ #ifndef _ODP_NO_INLINE @@ -43,6 +47,7 @@ _ODP_INLINE void odp_spinlock_recursive_lock(odp_spinlock_recursive_t *rlock) int thr = odp_thread_id(); if (rlock->owner == thr) { + _ODP_ASSERT(rlock->cnt < UINT32_MAX); rlock->cnt++; return; } @@ -57,6 +62,7 @@ _ODP_INLINE int odp_spinlock_recursive_trylock(odp_spinlock_recursive_t *rlock) int thr = odp_thread_id(); if (rlock->owner == thr) { + _ODP_ASSERT(rlock->cnt < UINT32_MAX); rlock->cnt++; return 1; } @@ -72,6 +78,7 @@ _ODP_INLINE int odp_spinlock_recursive_trylock(odp_spinlock_recursive_t *rlock) _ODP_INLINE void odp_spinlock_recursive_unlock(odp_spinlock_recursive_t *rlock) { + _ODP_ASSERT(rlock->cnt); rlock->cnt--; if (rlock->cnt > 0) diff --git a/platform/linux-generic/include/odp/api/plat/thread_inline_types.h b/platform/linux-generic/include/odp/api/plat/thread_inline_types.h new file mode 100644 index 000000000..c9a15b06b --- /dev/null +++ b/platform/linux-generic/include/odp/api/plat/thread_inline_types.h @@ -0,0 +1,36 @@ +/* Copyright (c) 2018-2018, Linaro Limited + * Copyright (c) 2022, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_PLAT_THREAD_INLINE_TYPES_H_ +#define ODP_PLAT_THREAD_INLINE_TYPES_H_ + +#include <odp/api/init.h> +#include <odp/api/spec/thread_types.h> + +#ifdef __cplusplus +extern "C" { +#endif + +/** @cond _ODP_HIDE_FROM_DOXYGEN_ */ + +typedef struct { + odp_log_func_t log_fn; + odp_thread_type_t type; + int thr; + int cpu; + +} _odp_thread_state_t; + +extern __thread _odp_thread_state_t *_odp_this_thread; + +/** @endcond */ + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/include/odp/api/plat/thread_inlines.h b/platform/linux-generic/include/odp/api/plat/thread_inlines.h index fc9275209..2f2e50b06 100644 --- a/platform/linux-generic/include/odp/api/plat/thread_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/thread_inlines.h @@ -9,22 +9,14 @@ #include <odp/api/init.h> +#include <odp/api/plat/thread_inline_types.h> + #ifdef __cplusplus extern "C" { #endif /** @cond _ODP_HIDE_FROM_DOXYGEN_ */ -typedef struct { - int thr; - int cpu; - odp_thread_type_t type; - odp_log_func_t log_fn; - -} _odp_thread_state_t; - -extern __thread _odp_thread_state_t *_odp_this_thread; - #ifndef _ODP_NO_INLINE /* Inline functions by default */ #define _ODP_INLINE static inline diff --git a/platform/linux-generic/include/odp/api/plat/timer_inlines.h b/platform/linux-generic/include/odp/api/plat/timer_inlines.h index 7642376d0..4f5620caf 100644 --- a/platform/linux-generic/include/odp/api/plat/timer_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/timer_inlines.h @@ -10,6 +10,7 @@ #include <odp/api/event.h> #include <odp/api/timer_types.h> +#include <odp/api/plat/debug_inlines.h> #include <odp/api/plat/timer_inline_types.h> #include <stdint.h> @@ -71,6 +72,8 @@ _ODP_INLINE uint64_t odp_timer_ns_to_tick(odp_timer_pool_t tp, uint64_t ns) _ODP_INLINE odp_timeout_t odp_timeout_from_event(odp_event_t ev) { + _ODP_ASSERT(odp_event_type(ev) == ODP_EVENT_TIMEOUT); + return (odp_timeout_t)ev; } diff --git a/platform/linux-generic/include/odp_classification_datamodel.h b/platform/linux-generic/include/odp_classification_datamodel.h index 213db03db..00df57bef 100644 --- a/platform/linux-generic/include/odp_classification_datamodel.h +++ b/platform/linux-generic/include/odp_classification_datamodel.h @@ -132,24 +132,24 @@ typedef struct pmr_term_value { Class Of Service */ typedef struct ODP_ALIGNED_CACHE cos_s { + uint32_t valid; /* validity Flag */ + odp_atomic_u32_t num_rule; /* num of PMRs attached with this CoS */ + struct pmr_s *pmr[CLS_PMR_PER_COS_MAX]; /* Chained PMR */ + struct cos_s *linked_cos[CLS_PMR_PER_COS_MAX]; /* Chained CoS with PMR*/ + odp_bool_t stats_enable; odp_cos_action_t action; /* Action */ odp_queue_t queue; /* Associated Queue */ + uint32_t num_queue; odp_pool_t pool; /* Associated Buffer pool */ + uint8_t index; + bool queue_group; + odp_cls_hash_proto_t hash_proto; odp_pktin_vector_config_t vector; /* Packet vector config */ - struct pmr_s *pmr[CLS_PMR_PER_COS_MAX]; /* Chained PMR */ - struct cos_s *linked_cos[CLS_PMR_PER_COS_MAX]; /* Chained CoS with PMR*/ - uint32_t valid; /* validity Flag */ odp_cls_drop_t drop_policy; /* Associated Drop Policy */ size_t headroom; /* Headroom for this CoS */ odp_spinlock_t lock; /* cos lock */ - odp_atomic_u32_t num_rule; /* num of PMRs attached with this CoS */ - bool queue_group; - odp_cls_hash_proto_t hash_proto; - uint32_t num_queue; odp_queue_param_t queue_param; char name[ODP_COS_NAME_LEN]; /* name */ - uint8_t index; - odp_bool_t stats_enable; struct { odp_atomic_u64_t discards; odp_atomic_u64_t packets; @@ -159,13 +159,12 @@ typedef struct ODP_ALIGNED_CACHE cos_s { /* Pattern Matching Rule */ typedef struct ODP_ALIGNED_CACHE pmr_s { uint32_t valid; /* Validity Flag */ - odp_atomic_u32_t count; /* num of packets matching this rule */ uint32_t num_pmr; /* num of PMR Term Values*/ uint16_t mark; - odp_spinlock_t lock; /* pmr lock*/ - cos_t *src_cos; /* source CoS where PMR is attached */ pmr_term_value_t pmr_term_value[CLS_PMRTERM_MAX]; /* List of associated PMR Terms */ + odp_spinlock_t lock; /* pmr lock*/ + cos_t *src_cos; /* source CoS where PMR is attached */ } pmr_t; typedef struct ODP_ALIGNED_CACHE { diff --git a/platform/linux-generic/include/odp_classification_internal.h b/platform/linux-generic/include/odp_classification_internal.h index dd1f8aaf6..c3ecf4079 100644 --- a/platform/linux-generic/include/odp_classification_internal.h +++ b/platform/linux-generic/include/odp_classification_internal.h @@ -62,7 +62,7 @@ static inline void _odp_cos_queue_stats_add(cos_t *cos, odp_queue_t queue, int queue_idx = _odp_cos_queue_idx(cos, queue); if (odp_unlikely(queue_idx < 0)) { - ODP_ERR("Queue not attached to the CoS\n"); + _ODP_ERR("Queue not attached to the CoS\n"); return; } diff --git a/platform/linux-generic/include/odp_debug_internal.h b/platform/linux-generic/include/odp_debug_internal.h index 22dca3701..4184d5170 100644 --- a/platform/linux-generic/include/odp_debug_internal.h +++ b/platform/linux-generic/include/odp_debug_internal.h @@ -1,5 +1,5 @@ /* Copyright (c) 2014-2018, Linaro Limited - * Copyright (c) 2020, Nokia + * Copyright (c) 2020-2022, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -18,9 +18,10 @@ #define ODP_DEBUG_INTERNAL_H_ #include <odp/autoheader_external.h> + #include <odp/api/debug.h> -#include <odp_global_data.h> -#include <odp/api/plat/thread_inlines.h> + +#include <odp/api/plat/debug_inlines.h> #include <stdio.h> #include <stdlib.h> @@ -37,23 +38,6 @@ extern "C" { * level 0 to N. */ #define CONFIG_DEBUG_LEVEL 0 -#define _ODP_LOG_FN(level, fmt, ...) \ - do { \ - if (_odp_this_thread && _odp_this_thread->log_fn) \ - _odp_this_thread->log_fn(level, fmt, ##__VA_ARGS__); \ - else \ - odp_global_ro.log_fn(level, fmt, ##__VA_ARGS__); \ - } while (0) - -/** - * Runtime assertion-macro - aborts if 'cond' is false. - */ -#define ODP_ASSERT(cond) \ - do { if ((ODP_DEBUG == 1) && (!(cond))) { \ - ODP_ERR("%s\n", #cond); \ - odp_global_ro.abort_fn(); } \ - } while (0) - /** * This macro is used to indicate when a given function is not implemented */ @@ -61,14 +45,6 @@ extern "C" { _ODP_LOG_FN(ODP_LOG_UNIMPLEMENTED, \ "%s:%d:The function %s() is not implemented\n", \ __FILE__, __LINE__, __func__) -/* - * Print debug message to log, if ODP_DEBUG_PRINT flag is set (ignores CONFIG_DEBUG_LEVEL). - */ -#define ODP_DBG(fmt, ...) \ - do { \ - if (ODP_DEBUG_PRINT == 1) \ - ODP_LOG(ODP_LOG_DBG, fmt, ##__VA_ARGS__);\ - } while (0) /* * Print debug message to log, if ODP_DEBUG_PRINT flag is set and CONFIG_DEBUG_LEVEL is high enough. @@ -76,7 +52,7 @@ extern "C" { #define ODP_DBG_LVL(level, fmt, ...) \ do { \ if (ODP_DEBUG_PRINT == 1 && CONFIG_DEBUG_LEVEL >= (level)) \ - ODP_LOG(ODP_LOG_DBG, fmt, ##__VA_ARGS__);\ + _ODP_LOG(ODP_LOG_DBG, fmt, ##__VA_ARGS__);\ } while (0) /* @@ -88,36 +64,6 @@ extern "C" { _ODP_LOG_FN(ODP_LOG_DBG, fmt, ##__VA_ARGS__);\ } while (0) -/** - * Log error message. - */ -#define ODP_ERR(fmt, ...) \ - ODP_LOG(ODP_LOG_ERR, fmt, ##__VA_ARGS__) - -/** - * Log abort message and then stop execution (by default call abort()). - * This function should not return. - */ -#define ODP_ABORT(fmt, ...) \ - do { \ - ODP_LOG(ODP_LOG_ABORT, fmt, ##__VA_ARGS__); \ - odp_global_ro.abort_fn(); \ - } while (0) - -/** - * ODP LOG macro. - */ -#define ODP_LOG(level, fmt, ...) \ - _ODP_LOG_FN(level, "%s:%d:%s():" fmt, __FILE__, \ - __LINE__, __func__, ##__VA_ARGS__) - -/** - * Log print message when the application calls one of the ODP APIs - * specifically for dumping internal data. - */ -#define ODP_PRINT(fmt, ...) \ - _ODP_LOG_FN(ODP_LOG_PRINT, fmt, ##__VA_ARGS__) - #ifdef __cplusplus } #endif diff --git a/platform/linux-generic/include/odp_global_data.h b/platform/linux-generic/include/odp_global_data.h index 462b8d639..d4cc9cda4 100644 --- a/platform/linux-generic/include/odp_global_data.h +++ b/platform/linux-generic/include/odp_global_data.h @@ -61,8 +61,6 @@ typedef struct odp_global_data_ro_t { pid_t main_pid; pid_t fdserver_pid; char uid[UID_MAXLEN]; - odp_log_func_t ODP_PRINTF_FORMAT(2, 3) log_fn; - odp_abort_func_t abort_fn; system_info_t system_info; hugepage_info_t hugepage_info; odp_cpumask_t all_cpus; diff --git a/platform/linux-generic/include/odp_llqueue.h b/platform/linux-generic/include/odp_llqueue.h index 6340d111a..29810ebf3 100644 --- a/platform/linux-generic/include/odp_llqueue.h +++ b/platform/linux-generic/include/odp_llqueue.h @@ -84,7 +84,7 @@ static inline void llq_enqueue(struct llqueue *llq, struct llnode *node) { union llht old, neu; - ODP_ASSERT(node->next == NULL); + _ODP_ASSERT(node->next == NULL); node->next = SENTINEL; do { old.ui = lld(&llq->u.ui, __ATOMIC_RELAXED); @@ -93,7 +93,7 @@ static inline void llq_enqueue(struct llqueue *llq, struct llnode *node) } while (odp_unlikely(scd(&llq->u.ui, neu.ui, __ATOMIC_RELEASE))); if (old.st.tail != NULL) { /* List was not empty */ - ODP_ASSERT(old.st.tail->next == SENTINEL); + _ODP_ASSERT(old.st.tail->next == SENTINEL); old.st.tail->next = node; } } @@ -230,7 +230,7 @@ static inline void llqueue_init(struct llqueue *llq) static inline void llq_enqueue(struct llqueue *llq, struct llnode *node) { - ODP_ASSERT(node->next == NULL); + _ODP_ASSERT(node->next == NULL); node->next = SENTINEL; odp_spinlock_lock(&llq->lock); @@ -257,11 +257,11 @@ static inline struct llnode *llq_dequeue(struct llqueue *llq) if (llq->head != NULL) { node = llq->head; if (llq->head == llq->tail) { - ODP_ASSERT(node->next == SENTINEL); + _ODP_ASSERT(node->next == SENTINEL); llq->head = NULL; llq->tail = NULL; } else { - ODP_ASSERT(node->next != SENTINEL); + _ODP_ASSERT(node->next != SENTINEL); llq->head = node->next; } node->next = NULL; @@ -279,11 +279,11 @@ static inline odp_bool_t llq_dequeue_cond(struct llqueue *llq, if (odp_likely(llq->head != NULL && llq->head == node)) { success = true; if (llq->head == llq->tail) { - ODP_ASSERT(node->next == SENTINEL); + _ODP_ASSERT(node->next == SENTINEL); llq->head = NULL; llq->tail = NULL; } else { - ODP_ASSERT(node->next != SENTINEL); + _ODP_ASSERT(node->next != SENTINEL); llq->head = node->next; } node->next = NULL; @@ -302,7 +302,7 @@ static inline odp_bool_t llq_cond_rotate(struct llqueue *llq, if (odp_likely(llq->head == node)) { success = true; if (llq->tail != node) { - ODP_ASSERT(node->next != SENTINEL); + _ODP_ASSERT(node->next != SENTINEL); llq->head = node->next; llq->tail->next = node; llq->tail = node; diff --git a/platform/linux-generic/include/odp_packet_internal.h b/platform/linux-generic/include/odp_packet_internal.h index 253650b6d..a6a8c551c 100644 --- a/platform/linux-generic/include/odp_packet_internal.h +++ b/platform/linux-generic/include/odp_packet_internal.h @@ -342,8 +342,8 @@ static inline void _odp_packet_copy_md(odp_packet_hdr_t *dst_hdr, const uint32_t src_uarea_size = src_pool->param_uarea_size; const uint32_t dst_uarea_size = dst_pool->param_uarea_size; - ODP_ASSERT(dst_hdr->uarea_addr != NULL); - ODP_ASSERT(dst_uarea_size >= src_uarea_size); + _ODP_ASSERT(dst_hdr->uarea_addr != NULL); + _ODP_ASSERT(dst_uarea_size >= src_uarea_size); memcpy(dst_hdr->uarea_addr, src_hdr->uarea_addr, src_uarea_size); } else { @@ -351,7 +351,7 @@ static inline void _odp_packet_copy_md(odp_packet_hdr_t *dst_hdr, /* If user area exists, packets should always be from the same pool, so * user area pointers can simply be swapped. */ - ODP_ASSERT(dst_hdr->event_hdr.pool == src_hdr->event_hdr.pool); + _ODP_ASSERT(dst_hdr->event_hdr.pool == src_hdr->event_hdr.pool); src_hdr->uarea_addr = dst_hdr->uarea_addr; dst_hdr->uarea_addr = src_uarea; @@ -440,11 +440,6 @@ static inline int packet_hdr_has_l2(odp_packet_hdr_t *pkt_hdr) return pkt_hdr->p.input_flags.l2; } -static inline void packet_hdr_has_l2_set(odp_packet_hdr_t *pkt_hdr, int val) -{ - pkt_hdr->p.input_flags.l2 = val; -} - static inline int packet_hdr_has_eth(odp_packet_hdr_t *pkt_hdr) { return pkt_hdr->p.input_flags.eth; diff --git a/platform/linux-generic/include/odp_packet_io_internal.h b/platform/linux-generic/include/odp_packet_io_internal.h index 04a7ea286..844088ac1 100644 --- a/platform/linux-generic/include/odp_packet_io_internal.h +++ b/platform/linux-generic/include/odp_packet_io_internal.h @@ -268,8 +268,8 @@ static inline pktio_entry_t *get_pktio_entry(odp_pktio_t pktio) return NULL; if (odp_unlikely(_odp_typeval(pktio) > ODP_CONFIG_PKTIO_ENTRIES)) { - ODP_DBG("pktio limit %" PRIuPTR "/%d exceed\n", - _odp_typeval(pktio), ODP_CONFIG_PKTIO_ENTRIES); + _ODP_DBG("pktio limit %" PRIuPTR "/%d exceed\n", + _odp_typeval(pktio), ODP_CONFIG_PKTIO_ENTRIES); return NULL; } diff --git a/platform/linux-generic/include/odp_parse_internal.h b/platform/linux-generic/include/odp_parse_internal.h index c467abbcd..8b3fb480b 100644 --- a/platform/linux-generic/include/odp_parse_internal.h +++ b/platform/linux-generic/include/odp_parse_internal.h @@ -59,10 +59,13 @@ uint16_t _odp_parse_eth(packet_parser_t *prs, const uint8_t **parseptr, * * See _odp_packet_parse_common(). Requires up to PARSE_L3_L4_BYTES bytes of * contiguous packet data. + * + * - offset is the offset of the first byte of the data pointed to by parseptr + * - seg_end is the maximum offset that can be accessed plus one */ int _odp_packet_parse_common_l3_l4(packet_parser_t *prs, const uint8_t *parseptr, uint32_t offset, - uint32_t frame_len, uint32_t seg_len, + uint32_t frame_len, uint32_t seg_end, int layer, uint16_t ethtype, uint64_t *l4_part_sum, odp_pktin_config_opt_t opt); diff --git a/platform/linux-generic/include/odp_queue_scalable_internal.h b/platform/linux-generic/include/odp_queue_scalable_internal.h index f7fc94e71..dccc2aef9 100644 --- a/platform/linux-generic/include/odp_queue_scalable_internal.h +++ b/platform/linux-generic/include/odp_queue_scalable_internal.h @@ -67,7 +67,7 @@ static inline void *shm_pool_alloc_align(_odp_ishm_pool_t *pool, uint32_t size) void *addr; addr = _odp_ishm_pool_alloc(pool, _ODP_ROUNDUP_CACHE_LINE(size)); - ODP_ASSERT(((uintptr_t)addr & (ODP_CACHE_LINE_SIZE - 1)) == 0); + _ODP_ASSERT(((uintptr_t)addr & (ODP_CACHE_LINE_SIZE - 1)) == 0); return addr; } diff --git a/platform/linux-generic/include/odp_schedule_if.h b/platform/linux-generic/include/odp_schedule_if.h index d9ba165bc..c70fdcf7c 100644 --- a/platform/linux-generic/include/odp_schedule_if.h +++ b/platform/linux-generic/include/odp_schedule_if.h @@ -91,51 +91,6 @@ int _odp_sched_cb_pktin_poll(int pktio_index, int pktin_index, _odp_event_hdr_t *hdr_tbl[], int num); void _odp_sched_cb_pktio_stop_finalize(int pktio_index); -/* API functions */ -typedef struct { - uint64_t (*schedule_wait_time)(uint64_t ns); - int (*schedule_capability)(odp_schedule_capability_t *capa); - void (*schedule_config_init)(odp_schedule_config_t *config); - int (*schedule_config)(const odp_schedule_config_t *config); - odp_event_t (*schedule)(odp_queue_t *from, uint64_t wait); - int (*schedule_multi)(odp_queue_t *from, uint64_t wait, - odp_event_t events[], int num); - int (*schedule_multi_wait)(odp_queue_t *from, odp_event_t events[], - int num); - int (*schedule_multi_no_wait)(odp_queue_t *from, odp_event_t events[], - int num); - void (*schedule_pause)(void); - void (*schedule_resume)(void); - void (*schedule_release_atomic)(void); - void (*schedule_release_ordered)(void); - void (*schedule_prefetch)(int num); - int (*schedule_min_prio)(void); - int (*schedule_max_prio)(void); - int (*schedule_default_prio)(void); - int (*schedule_num_prio)(void); - odp_schedule_group_t (*schedule_group_create) - (const char *name, const odp_thrmask_t *mask); - int (*schedule_group_destroy)(odp_schedule_group_t group); - odp_schedule_group_t (*schedule_group_lookup)(const char *name); - int (*schedule_group_join)(odp_schedule_group_t group, - const odp_thrmask_t *mask); - int (*schedule_group_leave)(odp_schedule_group_t group, - const odp_thrmask_t *mask); - int (*schedule_group_thrmask)(odp_schedule_group_t group, - odp_thrmask_t *mask); - int (*schedule_group_info)(odp_schedule_group_t group, - odp_schedule_group_info_t *info); - void (*schedule_order_lock)(uint32_t lock_index); - void (*schedule_order_unlock)(uint32_t lock_index); - void (*schedule_order_unlock_lock)(uint32_t unlock_index, - uint32_t lock_index); - void (*schedule_order_lock_start)(uint32_t lock_index); - void (*schedule_order_lock_wait)(uint32_t lock_index); - void (*schedule_order_wait)(void); - void (*schedule_print)(void); - -} schedule_api_t; - #ifdef __cplusplus } #endif diff --git a/platform/linux-generic/include/odp_sysinfo_internal.h b/platform/linux-generic/include/odp_sysinfo_internal.h index 16e4ced84..0fef1aa32 100644 --- a/platform/linux-generic/include/odp_sysinfo_internal.h +++ b/platform/linux-generic/include/odp_sysinfo_internal.h @@ -27,9 +27,9 @@ static inline int _odp_dummy_cpuinfo(system_info_t *sysinfo) sysinfo->cpu_arch = ODP_CPU_ARCH_UNKNOWN; - ODP_DBG("Warning: use dummy values for freq and model string\n"); + _ODP_DBG("Warning: use dummy values for freq and model string\n"); for (i = 0; i < CONFIG_NUM_CPU_IDS; i++) { - ODP_PRINT("WARN: cpu[%i] uses default max frequency of " + _ODP_PRINT("WARN: cpu[%i] uses default max frequency of " "%" PRIu64 " Hz from config file\n", i, cpu_hz_max); sysinfo->cpu_hz_max[i] = cpu_hz_max; strcpy(sysinfo->model_str[i], "UNKNOWN"); |