diff options
Diffstat (limited to 'platform/linux-generic/include')
9 files changed, 304 insertions, 92 deletions
diff --git a/platform/linux-generic/include/odp/api/plat/debug_inlines.h b/platform/linux-generic/include/odp/api/plat/debug_inlines.h index 41af3dca4..0755b1fda 100644 --- a/platform/linux-generic/include/odp/api/plat/debug_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/debug_inlines.h @@ -1,5 +1,5 @@ /* Copyright (c) 2014-2018, Linaro Limited - * Copyright (c) 2020-2022, Nokia + * Copyright (c) 2020-2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -50,9 +50,9 @@ extern odp_abort_func_t _odp_abort_fn; /** * ODP LOG macro. */ -#define _ODP_LOG(level, fmt, ...) \ - _ODP_LOG_FN(level, "%s:%d:%s(): " fmt, __FILE__, \ - __LINE__, __func__, ##__VA_ARGS__) +#define _ODP_LOG(level, prefix, fmt, ...) \ + _ODP_LOG_FN(level, "%s: %s:%d:%s(): " fmt, prefix, \ + __FILE__, __LINE__, __func__, ##__VA_ARGS__) /** * Runtime assertion-macro - aborts if 'cond' is false. @@ -70,17 +70,27 @@ extern odp_abort_func_t _odp_abort_fn; do { \ if (ODP_DEBUG_PRINT == 1) \ __extension__ ({ \ - _ODP_LOG(ODP_LOG_DBG, ##__VA_ARGS__); \ + _ODP_LOG(ODP_LOG_DBG, "DBG", ##__VA_ARGS__); \ }); \ } while (0) /** + * Log warning message. + */ +#define _ODP_WARN(...) \ + do { \ + __extension__ ({ \ + _ODP_LOG(ODP_LOG_WARN, "WARN", ##__VA_ARGS__); \ + }); \ + } while (0) + +/** * Log error message. */ #define _ODP_ERR(...) \ do { \ __extension__ ({ \ - _ODP_LOG(ODP_LOG_ERR, ##__VA_ARGS__); \ + _ODP_LOG(ODP_LOG_ERR, "ERR", ##__VA_ARGS__); \ }); \ } while (0) @@ -91,7 +101,7 @@ extern odp_abort_func_t _odp_abort_fn; #define _ODP_ABORT(...) \ do { \ __extension__ ({ \ - _ODP_LOG(ODP_LOG_ABORT, ##__VA_ARGS__); \ + _ODP_LOG(ODP_LOG_ABORT, "ABORT", ##__VA_ARGS__); \ }); \ _odp_abort_fn(); \ } while (0) diff --git a/platform/linux-generic/include/odp/api/plat/dma_inlines.h b/platform/linux-generic/include/odp/api/plat/dma_inlines.h new file mode 100644 index 000000000..84b5fef5b --- /dev/null +++ b/platform/linux-generic/include/odp/api/plat/dma_inlines.h @@ -0,0 +1,135 @@ +/* Copyright (c) 2023, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_PLAT_DMA_INLINES_H_ +#define ODP_PLAT_DMA_INLINES_H_ + +#include <odp/api/buffer.h> +#include <odp/api/dma_types.h> +#include <odp/api/event_types.h> +#include <odp/api/hints.h> +#include <odp/api/pool_types.h> +#include <odp/api/queue_types.h> + +#include <odp/api/plat/debug_inlines.h> +#include <odp/api/plat/event_inline_types.h> + +#include <stdint.h> +#include <string.h> + +/** @cond _ODP_HIDE_FROM_DOXYGEN_ */ + +#ifndef _ODP_NO_INLINE + /* Inline functions by default */ + #define _ODP_INLINE static inline + #define odp_dma_compl_from_event __odp_dma_compl_from_event + #define odp_dma_compl_to_event __odp_dma_compl_to_event + #define odp_dma_compl_user_area __odp_dma_compl_user_area + #define odp_dma_compl_result __odp_dma_compl_result + #define odp_dma_transfer_param_init __odp_dma_transfer_param_init + #define odp_dma_compl_param_init __odp_dma_compl_param_init + #define odp_dma_compl_alloc __odp_dma_compl_alloc + #define odp_dma_compl_free __odp_dma_compl_free +#else + #define _ODP_INLINE +#endif + +_ODP_INLINE odp_dma_compl_t odp_dma_compl_from_event(odp_event_t ev) +{ + _ODP_ASSERT(_odp_event_hdr_field(ev, int8_t, event_type) == ODP_EVENT_DMA_COMPL); + + return (odp_dma_compl_t)(uintptr_t)ev; +} + +_ODP_INLINE odp_event_t odp_dma_compl_to_event(odp_dma_compl_t dma_compl) +{ + return (odp_event_t)(uintptr_t)dma_compl; +} + +_ODP_INLINE void *odp_dma_compl_user_area(odp_dma_compl_t dma_compl) +{ + return odp_buffer_user_area((odp_buffer_t)(uintptr_t)dma_compl); +} + +_ODP_INLINE int odp_dma_compl_result(odp_dma_compl_t dma_compl, odp_dma_result_t *result_out) +{ + odp_dma_result_t *result; + odp_buffer_t buf = (odp_buffer_t)(uintptr_t)dma_compl; + + if (odp_unlikely(dma_compl == ODP_DMA_COMPL_INVALID)) { + _ODP_ERR("Bad DMA compl handle\n"); + return -1; + } + + result = (odp_dma_result_t *)odp_buffer_addr(buf); + + if (result_out) + *result_out = *result; + + return result->success ? 0 : -1; +} + +_ODP_INLINE void odp_dma_transfer_param_init(odp_dma_transfer_param_t *trs_param) +{ + memset(trs_param, 0, sizeof(odp_dma_transfer_param_t)); + + trs_param->src_format = ODP_DMA_FORMAT_ADDR; + trs_param->dst_format = ODP_DMA_FORMAT_ADDR; + trs_param->num_src = 1; + trs_param->num_dst = 1; +} + +_ODP_INLINE void odp_dma_compl_param_init(odp_dma_compl_param_t *compl_param) +{ + memset(compl_param, 0, sizeof(odp_dma_compl_param_t)); + + compl_param->queue = ODP_QUEUE_INVALID; + compl_param->event = ODP_EVENT_INVALID; + compl_param->transfer_id = ODP_DMA_TRANSFER_ID_INVALID; +} + +_ODP_INLINE odp_dma_compl_t odp_dma_compl_alloc(odp_pool_t pool) +{ + odp_buffer_t buf; + odp_event_t ev; + odp_dma_result_t *result; + int8_t *ev_type; + + buf = odp_buffer_alloc(pool); + if (odp_unlikely(buf == ODP_BUFFER_INVALID)) + return ODP_DMA_COMPL_INVALID; + + result = (odp_dma_result_t *)odp_buffer_addr(buf); + memset(result, 0, sizeof(odp_dma_result_t)); + + ev = odp_buffer_to_event(buf); + ev_type = _odp_event_hdr_ptr(ev, int8_t, event_type); + *ev_type = ODP_EVENT_DMA_COMPL; + + return (odp_dma_compl_t)(uintptr_t)buf; +} + +_ODP_INLINE void odp_dma_compl_free(odp_dma_compl_t dma_compl) +{ + int8_t *ev_type; + odp_event_t ev; + odp_buffer_t buf = (odp_buffer_t)(uintptr_t)dma_compl; + + if (odp_unlikely(dma_compl == ODP_DMA_COMPL_INVALID)) { + _ODP_ERR("Bad DMA compl handle\n"); + return; + } + + ev = odp_buffer_to_event(buf); + ev_type = _odp_event_hdr_ptr(ev, int8_t, event_type); + *ev_type = ODP_EVENT_BUFFER; + + odp_buffer_free(buf); +} + +/** @endcond */ + +#endif diff --git a/platform/linux-generic/include/odp/api/plat/event_inlines.h b/platform/linux-generic/include/odp/api/plat/event_inlines.h index 4e3368ff0..2e7c7db5e 100644 --- a/platform/linux-generic/include/odp/api/plat/event_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/event_inlines.h @@ -9,12 +9,12 @@ #define ODP_PLAT_EVENT_INLINES_H_ #include <odp/api/buffer_types.h> -#include <odp/api/dma.h> #include <odp/api/event_types.h> #include <odp/api/packet_types.h> #include <odp/api/timer_types.h> #include <odp/api/plat/buffer_inline_types.h> +#include <odp/api/plat/debug_inlines.h> #include <odp/api/plat/event_inline_types.h> #include <odp/api/plat/event_vector_inline_types.h> #include <odp/api/plat/packet_inline_types.h> @@ -28,8 +28,10 @@ #define odp_event_type __odp_event_type #define odp_event_type_multi __odp_event_type_multi #define odp_event_user_area __odp_event_user_area + #define odp_event_user_area_and_flag __odp_event_user_area_and_flag #define odp_event_subtype __odp_event_subtype #define odp_event_types __odp_event_types + #define odp_event_types_multi __odp_event_types_multi #define odp_event_flow_id __odp_event_flow_id #define odp_event_flow_id_set __odp_event_flow_id_set #else @@ -72,6 +74,7 @@ _ODP_INLINE void *odp_event_user_area(odp_event_t event) switch (type) { case ODP_EVENT_BUFFER: + case ODP_EVENT_DMA_COMPL: return _odp_buffer_get((odp_buffer_t)event, void *, uarea_addr); case ODP_EVENT_PACKET: return _odp_pkt_get((odp_packet_t)event, void *, user_area); @@ -79,9 +82,47 @@ _ODP_INLINE void *odp_event_user_area(odp_event_t event) return _odp_event_vect_get((odp_packet_vector_t)event, void *, uarea_addr); case ODP_EVENT_TIMEOUT: return _odp_timeout_hdr_field((odp_timeout_t)event, void *, uarea_addr); + default: + return NULL; + } +} + +_ODP_INLINE void *odp_event_user_area_and_flag(odp_event_t event, int *flag) +{ + const odp_event_type_t type = __odp_event_type_get(event); + + _ODP_ASSERT(flag != NULL); + + switch (type) { + case ODP_EVENT_BUFFER: case ODP_EVENT_DMA_COMPL: - return odp_dma_compl_user_area((odp_dma_compl_t)event); + *flag = -1; + return _odp_buffer_get((odp_buffer_t)event, void *, uarea_addr); + case ODP_EVENT_PACKET: + { + _odp_packet_flags_t pkt_flags; + odp_packet_t pkt = (odp_packet_t)event; + + pkt_flags.all_flags = _odp_pkt_get(pkt, uint32_t, flags); + *flag = pkt_flags.user_flag; + + return _odp_pkt_get(pkt, void *, user_area); + } + case ODP_EVENT_PACKET_VECTOR: + { + _odp_event_vector_flags_t pktv_flags; + odp_packet_vector_t pktv = (odp_packet_vector_t)event; + + pktv_flags.all_flags = _odp_event_vect_get(pktv, uint32_t, flags); + *flag = pktv_flags.user_flag; + + return _odp_event_vect_get(pktv, void *, uarea_addr); + } + case ODP_EVENT_TIMEOUT: + *flag = -1; + return _odp_timeout_hdr_field((odp_timeout_t)event, void *, uarea_addr); default: + *flag = -1; return NULL; } } @@ -106,6 +147,22 @@ _ODP_INLINE odp_event_type_t odp_event_types(odp_event_t event, return event_type; } +_ODP_INLINE void odp_event_types_multi(const odp_event_t event[], odp_event_type_t type[], + odp_event_subtype_t subtype[], int num) +{ + for (int i = 0; i < num; i++) + type[i] = __odp_event_type_get(event[i]); + + if (subtype == NULL) + return; + + for (int i = 0; i < num; i++) { + subtype[i] = (type[i] == ODP_EVENT_PACKET) ? + (odp_event_subtype_t)_odp_pkt_get((odp_packet_t)event[i], int8_t, + subtype) : ODP_EVENT_NO_SUBTYPE; + } +} + _ODP_INLINE uint32_t odp_event_flow_id(odp_event_t event) { return _odp_event_hdr_field(event, uint8_t, flow_id); diff --git a/platform/linux-generic/include/odp/api/plat/sync_inlines.h b/platform/linux-generic/include/odp/api/plat/sync_inlines.h index b6a96188c..b3a88b629 100644 --- a/platform/linux-generic/include/odp/api/plat/sync_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/sync_inlines.h @@ -1,7 +1,6 @@ -/* Copyright (c) 2016-2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2016-2018 Linaro Limited + * Copyright (c) 2023 Nokia */ /** @@ -13,6 +12,8 @@ #ifndef ODP_PLAT_SYNC_INLINE_H_ #define ODP_PLAT_SYNC_INLINE_H_ +#include <odp/api/abi/sync_inlines.h> + #ifdef __cplusplus extern "C" { #endif @@ -25,6 +26,9 @@ extern "C" { #define odp_mb_release __odp_mb_release #define odp_mb_acquire __odp_mb_acquire #define odp_mb_full __odp_mb_full + #define odp_mb_sync __odp_mb_sync + #define odp_mb_sync_load __odp_mb_sync_load + #define odp_mb_sync_store __odp_mb_sync_store #else #define _ODP_INLINE #endif @@ -44,6 +48,21 @@ _ODP_INLINE void odp_mb_full(void) __atomic_thread_fence(__ATOMIC_SEQ_CST); } +_ODP_INLINE void odp_mb_sync(void) +{ + _odp_mb_sync(); +} + +_ODP_INLINE void odp_mb_sync_load(void) +{ + _odp_mb_sync_load(); +} + +_ODP_INLINE void odp_mb_sync_store(void) +{ + _odp_mb_sync_store(); +} + /** @endcond */ #ifdef __cplusplus diff --git a/platform/linux-generic/include/odp/api/plat/time_inlines.h b/platform/linux-generic/include/odp/api/plat/time_inlines.h index f8f4bee89..35a35c72e 100644 --- a/platform/linux-generic/include/odp/api/plat/time_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/time_inlines.h @@ -1,5 +1,5 @@ /* Copyright (c) 2018, Linaro Limited - * Copyright (c) 2020-2022, Nokia + * Copyright (c) 2020-2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -12,79 +12,12 @@ #include <odp/api/hints.h> #include <odp/api/time_types.h> -#include <odp/api/abi/cpu_time.h> +#include <odp/api/abi/time_inlines.h> #include <stdint.h> /** @cond _ODP_HIDE_FROM_DOXYGEN_ */ -#define _ODP_TIMESPEC_SIZE 16 -#define _ODP_TIME_GIGA_HZ 1000000000ULL - -typedef struct _odp_time_global_t { - /* Storage space for struct timespec. Posix headers are not included - * here to avoid application exposure. */ - uint8_t timespec[_ODP_TIMESPEC_SIZE] ODP_ALIGNED(_ODP_TIMESPEC_SIZE); - - int use_hw; - uint64_t hw_start; - uint64_t hw_freq_hz; - -} _odp_time_global_t; - -extern _odp_time_global_t _odp_time_glob; - -odp_time_t _odp_timespec_cur(void); - -static inline odp_time_t _odp_time_cur(void) -{ - if (_odp_time_glob.use_hw) { - odp_time_t time; - - time.count = _odp_cpu_global_time() - _odp_time_glob.hw_start; - return time; - } - - return _odp_timespec_cur(); -} - -static inline odp_time_t _odp_time_cur_strict(void) -{ - if (_odp_time_glob.use_hw) { - odp_time_t time; - - time.count = _odp_cpu_global_time_strict() - _odp_time_glob.hw_start; - return time; - } - - return _odp_timespec_cur(); -} - -static inline uint64_t _odp_time_hw_to_ns(odp_time_t time) -{ - uint64_t nsec; - uint64_t freq_hz = _odp_time_glob.hw_freq_hz; - uint64_t count = time.count; - uint64_t sec = 0; - - if (count >= freq_hz) { - sec = count / freq_hz; - count = count - sec * freq_hz; - } - - nsec = (_ODP_TIME_GIGA_HZ * count) / freq_hz; - - return (sec * _ODP_TIME_GIGA_HZ) + nsec; -} - -static inline uint64_t _odp_time_convert_to_ns(odp_time_t time) -{ - if (_odp_time_glob.use_hw) - return _odp_time_hw_to_ns(time); - - return time.nsec; -} - #ifndef _ODP_NO_INLINE /* Inline functions by default */ #define _ODP_INLINE static inline @@ -104,6 +37,14 @@ static inline uint64_t _odp_time_convert_to_ns(odp_time_t time) #define odp_time_diff_ns __odp_time_diff_ns #define odp_time_sum __odp_time_sum + #define odp_time_local_from_ns __odp_time_local_from_ns + #define odp_time_global_from_ns __odp_time_global_from_ns + + #define odp_time_local_res __odp_time_local_res + #define odp_time_global_res __odp_time_global_res + + #define odp_time_wait_ns __odp_time_wait_ns + #define odp_time_wait_until __odp_time_wait_until #else #define _ODP_INLINE #endif @@ -130,27 +71,27 @@ _ODP_INLINE odp_time_t odp_time_global_strict(void) _ODP_INLINE uint64_t odp_time_local_ns(void) { - return _odp_time_convert_to_ns(_odp_time_cur()); + return _odp_time_to_ns(_odp_time_cur()); } _ODP_INLINE uint64_t odp_time_global_ns(void) { - return _odp_time_convert_to_ns(_odp_time_cur()); + return _odp_time_to_ns(_odp_time_cur()); } _ODP_INLINE uint64_t odp_time_local_strict_ns(void) { - return _odp_time_convert_to_ns(_odp_time_cur_strict()); + return _odp_time_to_ns(_odp_time_cur_strict()); } _ODP_INLINE uint64_t odp_time_global_strict_ns(void) { - return _odp_time_convert_to_ns(_odp_time_cur_strict()); + return _odp_time_to_ns(_odp_time_cur_strict()); } _ODP_INLINE uint64_t odp_time_to_ns(odp_time_t time) { - return _odp_time_convert_to_ns(time); + return _odp_time_to_ns(time); } _ODP_INLINE int odp_time_cmp(odp_time_t t2, odp_time_t t1) @@ -191,6 +132,44 @@ _ODP_INLINE odp_time_t odp_time_sum(odp_time_t t1, odp_time_t t2) return time; } +_ODP_INLINE odp_time_t odp_time_local_from_ns(uint64_t ns) +{ + return _odp_time_from_ns(ns); +} + +_ODP_INLINE odp_time_t odp_time_global_from_ns(uint64_t ns) +{ + return _odp_time_from_ns(ns); +} + +_ODP_INLINE uint64_t odp_time_local_res(void) +{ + return _odp_time_res(); +} + +_ODP_INLINE uint64_t odp_time_global_res(void) +{ + return _odp_time_res(); +} + +_ODP_INLINE void odp_time_wait_until(odp_time_t time) +{ + odp_time_t cur; + + do { + cur = _odp_time_cur(); + } while (odp_time_cmp(time, cur) > 0); +} + +_ODP_INLINE void odp_time_wait_ns(uint64_t ns) +{ + odp_time_t cur = _odp_time_cur(); + odp_time_t wait = _odp_time_from_ns(ns); + odp_time_t end_time = odp_time_sum(cur, wait); + + odp_time_wait_until(end_time); +} + /** @endcond */ #endif diff --git a/platform/linux-generic/include/odp/api/plat/timer_inlines.h b/platform/linux-generic/include/odp/api/plat/timer_inlines.h index 648459c78..9ba0287e0 100644 --- a/platform/linux-generic/include/odp/api/plat/timer_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/timer_inlines.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2022, Nokia +/* Copyright (c) 2022-2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -27,6 +27,7 @@ #define odp_timer_tick_to_ns __odp_timer_tick_to_ns #define odp_timer_ns_to_tick __odp_timer_ns_to_tick #define odp_timeout_from_event __odp_timeout_from_event + #define odp_timeout_from_event_multi __odp_timeout_from_event_multi #define odp_timeout_to_event __odp_timeout_to_event #else #define _ODP_INLINE @@ -75,6 +76,15 @@ _ODP_INLINE odp_timeout_t odp_timeout_from_event(odp_event_t ev) return (odp_timeout_t)ev; } +_ODP_INLINE void odp_timeout_from_event_multi(odp_timeout_t tmo[], const odp_event_t ev[], int num) +{ + for (int i = 0; i < num; i++) { + _ODP_ASSERT(odp_event_type(ev[i]) == ODP_EVENT_TIMEOUT); + + tmo[i] = odp_timeout_from_event(ev[i]); + } +} + _ODP_INLINE odp_event_t odp_timeout_to_event(odp_timeout_t tmo) { return (odp_event_t)tmo; diff --git a/platform/linux-generic/include/odp_debug_internal.h b/platform/linux-generic/include/odp_debug_internal.h index 4c1fa8bbb..d1fc0d0ba 100644 --- a/platform/linux-generic/include/odp_debug_internal.h +++ b/platform/linux-generic/include/odp_debug_internal.h @@ -55,7 +55,7 @@ extern "C" { do { \ if (ODP_DEBUG_PRINT == 1 && CONFIG_DEBUG_LEVEL >= (level)) \ __extension__ ({ \ - _ODP_LOG(ODP_LOG_DBG, ##__VA_ARGS__); \ + _ODP_LOG(ODP_LOG_DBG, "DBG", ##__VA_ARGS__); \ }); \ } while (0) diff --git a/platform/linux-generic/include/odp_global_data.h b/platform/linux-generic/include/odp_global_data.h index d4cc9cda4..67b7572ef 100644 --- a/platform/linux-generic/include/odp_global_data.h +++ b/platform/linux-generic/include/odp_global_data.h @@ -1,4 +1,5 @@ /* Copyright (c) 2013-2018, Linaro Limited + * Copyright (c) 2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -34,7 +35,8 @@ typedef struct { uint64_t default_cpu_hz; uint64_t page_size; int cache_line_size; - odp_bool_t cpu_hz_static; + uint8_t cpu_hz_static; + uint8_t cpu_constant_tsc; odp_cpu_arch_t cpu_arch; odp_cpu_arch_isa_t cpu_isa_sw; odp_cpu_arch_isa_t cpu_isa_hw; diff --git a/platform/linux-generic/include/odp_sysinfo_internal.h b/platform/linux-generic/include/odp_sysinfo_internal.h index 0fef1aa32..c14cf78d9 100644 --- a/platform/linux-generic/include/odp_sysinfo_internal.h +++ b/platform/linux-generic/include/odp_sysinfo_internal.h @@ -27,9 +27,9 @@ static inline int _odp_dummy_cpuinfo(system_info_t *sysinfo) sysinfo->cpu_arch = ODP_CPU_ARCH_UNKNOWN; - _ODP_DBG("Warning: use dummy values for freq and model string\n"); + _ODP_WARN("Use dummy values for freq and model string\n"); for (i = 0; i < CONFIG_NUM_CPU_IDS; i++) { - _ODP_PRINT("WARN: cpu[%i] uses default max frequency of " + _ODP_WARN("CPU[%i] uses default max frequency of " "%" PRIu64 " Hz from config file\n", i, cpu_hz_max); sysinfo->cpu_hz_max[i] = cpu_hz_max; strcpy(sysinfo->model_str[i], "UNKNOWN"); |