diff options
Diffstat (limited to 'platform')
60 files changed, 1236 insertions, 1093 deletions
diff --git a/platform/linux-dpdk/Makefile.am b/platform/linux-dpdk/Makefile.am index 252b34fb2..ad608c4e1 100644 --- a/platform/linux-dpdk/Makefile.am +++ b/platform/linux-dpdk/Makefile.am @@ -33,6 +33,7 @@ odpapiplatinclude_HEADERS = \ include/odp/api/plat/byteorder_inlines.h \ include/odp/api/plat/cpu_inlines.h \ include/odp/api/plat/crypto_inlines.h \ + include/odp/api/plat/dma_inlines.h \ include/odp/api/plat/debug_inlines.h \ include/odp/api/plat/event_inlines.h \ include/odp/api/plat/event_inline_types.h \ @@ -78,6 +79,7 @@ odpapiabiarchinclude_HEADERS += \ include-abi/odp/api/abi/crypto.h \ include-abi/odp/api/abi/crypto_types.h \ include-abi/odp/api/abi/debug.h \ + include-abi/odp/api/abi/dma.h \ include-abi/odp/api/abi/dma_types.h \ include-abi/odp/api/abi/errno.h \ include-abi/odp/api/abi/event.h \ @@ -259,6 +261,7 @@ __LIB__libodp_dpdk_la_SOURCES += \ ../linux-generic/odp_byteorder_api.c \ ../linux-generic/odp_cpu_api.c \ ../linux-generic/odp_crypto_api.c \ + ../linux-generic/odp_dma_api.c \ ../linux-generic/odp_event_api.c \ ../linux-generic/odp_hash_api.c \ ../linux-generic/odp_ipsec_api.c \ @@ -283,18 +286,17 @@ endif if ARCH_IS_ARM __LIB__libodp_dpdk_la_SOURCES += arch/default/odp_atomic.c \ arch/default/odp_cpu_cycles.c \ - arch/default/odp_global_time.c \ arch/default/odp_hash_crc32.c \ arch/default/odp_random.c \ arch/arm/odp_sysinfo_parse.c -odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_time.h \ - arch/default/odp/api/abi/hash_crc32.h +odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/hash_crc32.h if !ODP_ABI_COMPAT odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ arch/default/odp/api/abi/atomic_inlines.h \ arch/default/odp/api/abi/cpu_generic.h \ arch/arm/odp/api/abi/cpu_inlines.h \ - arch/arm/odp/api/abi/cpu.h + arch/arm/odp/api/abi/cpu.h \ + arch/default/odp/api/abi/sync_inlines.h endif noinst_HEADERS += arch/arm/odp_atomic.h \ arch/arm/odp_cpu.h \ @@ -310,19 +312,18 @@ if ARCH_IS_AARCH64 __LIB__libodp_dpdk_la_SOURCES += arch/aarch64/odp_atomic.c \ arch/default/odp_cpu_cycles.c \ arch/aarch64/cpu_flags.c \ - arch/aarch64/odp_global_time.c \ arch/default/odp_hash_crc32.c \ arch/default/odp_random.c \ arch/aarch64/odp_sysinfo_parse.c -odpapiabiarchinclude_HEADERS += arch/aarch64/odp/api/abi/cpu_time.h \ - arch/aarch64/odp/api/abi/hash_crc32.h +odpapiabiarchinclude_HEADERS += arch/aarch64/odp/api/abi/hash_crc32.h if !ODP_ABI_COMPAT odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ arch/aarch64/odp/api/abi/atomic_inlines.h \ arch/aarch64/odp/api/abi/atomic.h \ arch/default/odp/api/abi/cpu_generic.h \ arch/aarch64/odp/api/abi/cpu_inlines.h \ - arch/aarch64/odp/api/abi/cpu.h + arch/aarch64/odp/api/abi/cpu.h \ + arch/aarch64/odp/api/abi/sync_inlines.h endif noinst_HEADERS += arch/aarch64/odp_atomic.h \ arch/aarch64/odp_cpu.h \ @@ -334,18 +335,17 @@ endif if ARCH_IS_DEFAULT __LIB__libodp_dpdk_la_SOURCES += arch/default/odp_atomic.c \ arch/default/odp_cpu_cycles.c \ - arch/default/odp_global_time.c \ arch/default/odp_hash_crc32.c \ arch/default/odp_random.c \ arch/default/odp_sysinfo_parse.c -odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_time.h \ - arch/default/odp/api/abi/hash_crc32.h +odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/hash_crc32.h if !ODP_ABI_COMPAT odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ arch/default/odp/api/abi/atomic_inlines.h \ arch/default/odp/api/abi/cpu_generic.h \ arch/default/odp/api/abi/cpu_inlines.h \ - arch/default/odp/api/abi/cpu.h + arch/default/odp/api/abi/cpu.h \ + arch/default/odp/api/abi/sync_inlines.h endif noinst_HEADERS += arch/default/odp_atomic.h \ arch/default/odp_cpu.h \ @@ -355,18 +355,17 @@ endif if ARCH_IS_POWERPC __LIB__libodp_dpdk_la_SOURCES += arch/default/odp_atomic.c \ arch/default/odp_cpu_cycles.c \ - arch/default/odp_global_time.c \ arch/default/odp_hash_crc32.c \ arch/default/odp_random.c \ arch/powerpc/odp_sysinfo_parse.c -odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_time.h \ - arch/default/odp/api/abi/hash_crc32.h +odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/hash_crc32.h if !ODP_ABI_COMPAT odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ arch/default/odp/api/abi/atomic_inlines.h \ arch/default/odp/api/abi/cpu_generic.h \ arch/default/odp/api/abi/cpu_inlines.h \ - arch/powerpc/odp/api/abi/cpu.h + arch/powerpc/odp/api/abi/cpu.h \ + arch/default/odp/api/abi/sync_inlines.h endif noinst_HEADERS += arch/default/odp_atomic.h \ arch/default/odp_cpu.h \ @@ -377,18 +376,18 @@ if ARCH_IS_X86 __LIB__libodp_dpdk_la_SOURCES += arch/default/odp_atomic.c \ arch/x86/cpu_flags.c \ arch/x86/odp_cpu_cycles.c \ - arch/x86/odp_global_time.c \ arch/default/odp_hash_crc32.c \ arch/default/odp_random.c \ arch/x86/odp_sysinfo_parse.c odpapiabiarchinclude_HEADERS += arch/x86/odp/api/abi/cpu_rdtsc.h \ - arch/x86/odp/api/abi/cpu_time.h \ - arch/x86/odp/api/abi/hash_crc32.h + arch/x86/odp/api/abi/hash_crc32.h \ + arch/x86/odp/api/abi/time_cpu.h if !ODP_ABI_COMPAT odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ arch/default/odp/api/abi/atomic_inlines.h \ arch/x86/odp/api/abi/cpu_inlines.h \ - arch/x86/odp/api/abi/cpu.h + arch/x86/odp/api/abi/cpu.h \ + arch/x86/odp/api/abi/sync_inlines.h endif noinst_HEADERS += arch/x86/cpu_flags.h \ arch/x86/odp_cpu.h \ diff --git a/platform/linux-dpdk/arch/aarch64/odp/api/abi/cpu_time.h b/platform/linux-dpdk/arch/aarch64/odp/api/abi/cpu_time.h deleted file mode 120000 index 52719241f..000000000 --- a/platform/linux-dpdk/arch/aarch64/odp/api/abi/cpu_time.h +++ /dev/null @@ -1 +0,0 @@ -../../../../../../linux-generic/arch/aarch64/odp/api/abi/cpu_time.h
\ No newline at end of file diff --git a/platform/linux-dpdk/arch/aarch64/odp/api/abi/sync_inlines.h b/platform/linux-dpdk/arch/aarch64/odp/api/abi/sync_inlines.h new file mode 120000 index 000000000..1281f2376 --- /dev/null +++ b/platform/linux-dpdk/arch/aarch64/odp/api/abi/sync_inlines.h @@ -0,0 +1 @@ +../../../../../../linux-generic/arch/aarch64/odp/api/abi/sync_inlines.h
\ No newline at end of file diff --git a/platform/linux-dpdk/arch/aarch64/odp_global_time.c b/platform/linux-dpdk/arch/aarch64/odp_global_time.c deleted file mode 120000 index 1b7fa3c24..000000000 --- a/platform/linux-dpdk/arch/aarch64/odp_global_time.c +++ /dev/null @@ -1 +0,0 @@ -../../../linux-generic/arch/aarch64/odp_global_time.c
\ No newline at end of file diff --git a/platform/linux-dpdk/arch/default/odp/api/abi/cpu_time.h b/platform/linux-dpdk/arch/default/odp/api/abi/cpu_time.h deleted file mode 120000 index 941d6ccca..000000000 --- a/platform/linux-dpdk/arch/default/odp/api/abi/cpu_time.h +++ /dev/null @@ -1 +0,0 @@ -../../../../../../linux-generic/arch/default/odp/api/abi/cpu_time.h
\ No newline at end of file diff --git a/platform/linux-dpdk/arch/default/odp/api/abi/sync_inlines.h b/platform/linux-dpdk/arch/default/odp/api/abi/sync_inlines.h new file mode 120000 index 000000000..328ff07d6 --- /dev/null +++ b/platform/linux-dpdk/arch/default/odp/api/abi/sync_inlines.h @@ -0,0 +1 @@ +../../../../../../linux-generic/arch/default/odp/api/abi/sync_inlines.h
\ No newline at end of file diff --git a/platform/linux-dpdk/arch/default/odp_global_time.c b/platform/linux-dpdk/arch/default/odp_global_time.c deleted file mode 120000 index 3f54b1478..000000000 --- a/platform/linux-dpdk/arch/default/odp_global_time.c +++ /dev/null @@ -1 +0,0 @@ -../../../linux-generic/arch/default/odp_global_time.c
\ No newline at end of file diff --git a/platform/linux-dpdk/arch/x86/odp/api/abi/cpu_time.h b/platform/linux-dpdk/arch/x86/odp/api/abi/cpu_time.h deleted file mode 120000 index 2fb280b8c..000000000 --- a/platform/linux-dpdk/arch/x86/odp/api/abi/cpu_time.h +++ /dev/null @@ -1 +0,0 @@ -../../../../../../linux-generic/arch/x86/odp/api/abi/cpu_time.h
\ No newline at end of file diff --git a/platform/linux-dpdk/arch/x86/odp/api/abi/sync_inlines.h b/platform/linux-dpdk/arch/x86/odp/api/abi/sync_inlines.h new file mode 120000 index 000000000..d5dba2679 --- /dev/null +++ b/platform/linux-dpdk/arch/x86/odp/api/abi/sync_inlines.h @@ -0,0 +1 @@ +../../../../../../linux-generic/arch/x86/odp/api/abi/sync_inlines.h
\ No newline at end of file diff --git a/platform/linux-dpdk/arch/x86/odp/api/abi/time_cpu.h b/platform/linux-dpdk/arch/x86/odp/api/abi/time_cpu.h new file mode 100644 index 000000000..cc313cff1 --- /dev/null +++ b/platform/linux-dpdk/arch/x86/odp/api/abi/time_cpu.h @@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2023 Nokia + */ + +#ifndef ODP_ARCH_TIME_CPU_H_ +#define ODP_ARCH_TIME_CPU_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +int _odp_time_cpu_global_freq_is_const(void); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-dpdk/arch/x86/odp_global_time.c b/platform/linux-dpdk/arch/x86/odp_global_time.c deleted file mode 120000 index 33a50e23e..000000000 --- a/platform/linux-dpdk/arch/x86/odp_global_time.c +++ /dev/null @@ -1 +0,0 @@ -../../../linux-generic/arch/x86/odp_global_time.c
\ No newline at end of file diff --git a/platform/linux-dpdk/include-abi/odp/api/abi/dma.h b/platform/linux-dpdk/include-abi/odp/api/abi/dma.h new file mode 120000 index 000000000..112a7b5a7 --- /dev/null +++ b/platform/linux-dpdk/include-abi/odp/api/abi/dma.h @@ -0,0 +1 @@ +../../../../../linux-generic/include-abi/odp/api/abi/dma.h
\ No newline at end of file diff --git a/platform/linux-dpdk/include/odp/api/plat/dma_inlines.h b/platform/linux-dpdk/include/odp/api/plat/dma_inlines.h new file mode 120000 index 000000000..5b60f374e --- /dev/null +++ b/platform/linux-dpdk/include/odp/api/plat/dma_inlines.h @@ -0,0 +1 @@ +../../../../../linux-generic/include/odp/api/plat/dma_inlines.h
\ No newline at end of file diff --git a/platform/linux-dpdk/include/odp/api/plat/time_inlines.h b/platform/linux-dpdk/include/odp/api/plat/time_inlines.h index 4b1320300..8e4eec59c 100644 --- a/platform/linux-dpdk/include/odp/api/plat/time_inlines.h +++ b/platform/linux-dpdk/include/odp/api/plat/time_inlines.h @@ -12,125 +12,116 @@ #include <odp/api/hints.h> #include <odp/api/time_types.h> -#include <odp/api/abi/cpu_time.h> +#include <rte_config.h> +#include <rte_atomic.h> +#include <rte_cycles.h> #include <stdint.h> /** @cond _ODP_HIDE_FROM_DOXYGEN_ */ -#define _ODP_TIMESPEC_SIZE 16 -#define _ODP_TIME_GIGA_HZ 1000000000ULL - -typedef odp_time_t (*time_cur_fn)(void); -typedef odp_time_t (*time_cur_strict_fn)(void); -typedef uint64_t (*time_res_fn)(void); - -typedef struct time_handler_ { - time_cur_fn time_cur; - time_cur_strict_fn time_cur_strict; - time_res_fn time_res; - -} time_handler_t; - typedef struct _odp_time_global_t { - /* Storage space for struct timespec. Posix headers are not included - * here to avoid application exposure. */ - uint8_t timespec[_ODP_TIMESPEC_SIZE] ODP_ALIGNED(_ODP_TIMESPEC_SIZE); - - int use_hw; - uint64_t hw_start; - uint64_t hw_freq_hz; - /* DPDK specific */ - time_handler_t handler; + uint64_t start_cycles; + uint64_t freq_hz; } _odp_time_global_t; extern _odp_time_global_t _odp_time_glob; -odp_time_t _odp_timespec_cur(void); - -static inline odp_time_t _odp_time_cur_gen(void) +static inline odp_time_t _odp_time_cur(void) { - if (_odp_time_glob.use_hw) { - odp_time_t time; + odp_time_t time; - time.count = _odp_cpu_global_time() - _odp_time_glob.hw_start; - return time; - } + time.u64 = rte_get_timer_cycles() - _odp_time_glob.start_cycles; - return _odp_timespec_cur(); + return time; } -static inline odp_time_t _odp_time_cur_gen_strict(void) +static inline odp_time_t _odp_time_cur_strict(void) { - if (_odp_time_glob.use_hw) { - odp_time_t time; - - time.count = _odp_cpu_global_time_strict() - _odp_time_glob.hw_start; - return time; - } + odp_time_t time; - return _odp_timespec_cur(); -} + rte_mb(); + time.u64 = rte_get_timer_cycles() - _odp_time_glob.start_cycles; -static inline odp_time_t _odp_time_cur(void) -{ - return _odp_time_glob.handler.time_cur(); -} - -static inline odp_time_t _odp_time_cur_strict(void) -{ - return _odp_time_glob.handler.time_cur_strict(); + return time; } -static inline uint64_t _odp_time_hw_to_ns(odp_time_t time) +static inline uint64_t _odp_time_to_ns(odp_time_t time) { uint64_t nsec; - uint64_t freq_hz = _odp_time_glob.hw_freq_hz; uint64_t count = time.count; uint64_t sec = 0; + const uint64_t freq_hz = _odp_time_glob.freq_hz; + const uint64_t giga_hz = 1000000000; if (count >= freq_hz) { sec = count / freq_hz; count = count - sec * freq_hz; } - nsec = (_ODP_TIME_GIGA_HZ * count) / freq_hz; + nsec = (giga_hz * count) / freq_hz; - return (sec * _ODP_TIME_GIGA_HZ) + nsec; + return (sec * giga_hz) + nsec; } -static inline uint64_t _odp_time_convert_to_ns(odp_time_t time) +static inline odp_time_t _odp_time_from_ns(uint64_t ns) { - if (_odp_time_glob.use_hw) - return _odp_time_hw_to_ns(time); + odp_time_t time; + uint64_t count; + uint64_t sec = 0; + const uint64_t freq_hz = _odp_time_glob.freq_hz; + + if (ns >= ODP_TIME_SEC_IN_NS) { + sec = ns / ODP_TIME_SEC_IN_NS; + ns = ns - sec * ODP_TIME_SEC_IN_NS; + } - return time.nsec; + count = sec * freq_hz; + count += (ns * freq_hz) / ODP_TIME_SEC_IN_NS; + + time.count = count; + + return time; } #ifndef _ODP_NO_INLINE /* Inline functions by default */ #define _ODP_INLINE static inline - #define odp_time_local __odp_time_local - #define odp_time_global __odp_time_global - #define odp_time_to_ns __odp_time_to_ns - #define odp_time_local_ns __odp_time_local_ns - #define odp_time_global_ns __odp_time_global_ns - - #define odp_time_local_strict __odp_time_local_strict - #define odp_time_global_strict __odp_time_global_strict - #define odp_time_local_strict_ns __odp_time_local_strict_ns - #define odp_time_global_strict_ns __odp_time_global_strict_ns - - #define odp_time_cmp __odp_time_cmp - #define odp_time_diff __odp_time_diff - #define odp_time_diff_ns __odp_time_diff_ns - #define odp_time_sum __odp_time_sum + #define odp_time_local_res __odp_time_local_res + #define odp_time_global_res __odp_time_global_res + #define odp_time_local __odp_time_local + #define odp_time_global __odp_time_global + #define odp_time_local_strict __odp_time_local_strict + #define odp_time_global_strict __odp_time_global_strict + #define odp_time_local_ns __odp_time_local_ns + #define odp_time_global_ns __odp_time_global_ns + #define odp_time_local_from_ns __odp_time_local_from_ns + #define odp_time_global_from_ns __odp_time_global_from_ns + #define odp_time_local_strict_ns __odp_time_local_strict_ns + #define odp_time_global_strict_ns __odp_time_global_strict_ns + #define odp_time_to_ns __odp_time_to_ns + #define odp_time_cmp __odp_time_cmp + #define odp_time_diff __odp_time_diff + #define odp_time_diff_ns __odp_time_diff_ns + #define odp_time_sum __odp_time_sum + #define odp_time_wait_ns __odp_time_wait_ns + #define odp_time_wait_until __odp_time_wait_until #else #define _ODP_INLINE #endif +_ODP_INLINE uint64_t odp_time_local_res(void) +{ + return _odp_time_glob.freq_hz; +} + +_ODP_INLINE uint64_t odp_time_global_res(void) +{ + return _odp_time_glob.freq_hz; +} + _ODP_INLINE odp_time_t odp_time_local(void) { return _odp_time_cur(); @@ -153,27 +144,37 @@ _ODP_INLINE odp_time_t odp_time_global_strict(void) _ODP_INLINE uint64_t odp_time_local_ns(void) { - return _odp_time_convert_to_ns(_odp_time_cur()); + return _odp_time_to_ns(_odp_time_cur()); } _ODP_INLINE uint64_t odp_time_global_ns(void) { - return _odp_time_convert_to_ns(_odp_time_cur()); + return _odp_time_to_ns(_odp_time_cur()); +} + +_ODP_INLINE odp_time_t odp_time_local_from_ns(uint64_t ns) +{ + return _odp_time_from_ns(ns); +} + +_ODP_INLINE odp_time_t odp_time_global_from_ns(uint64_t ns) +{ + return _odp_time_from_ns(ns); } _ODP_INLINE uint64_t odp_time_local_strict_ns(void) { - return _odp_time_convert_to_ns(_odp_time_cur_strict()); + return _odp_time_to_ns(_odp_time_cur_strict()); } _ODP_INLINE uint64_t odp_time_global_strict_ns(void) { - return _odp_time_convert_to_ns(_odp_time_cur_strict()); + return _odp_time_to_ns(_odp_time_cur_strict()); } _ODP_INLINE uint64_t odp_time_to_ns(odp_time_t time) { - return _odp_time_convert_to_ns(time); + return _odp_time_to_ns(time); } _ODP_INLINE int odp_time_cmp(odp_time_t t2, odp_time_t t1) @@ -214,6 +215,29 @@ _ODP_INLINE odp_time_t odp_time_sum(odp_time_t t1, odp_time_t t2) return time; } +static inline void _odp_time_wait_until(odp_time_t time) +{ + odp_time_t cur; + + do { + cur = _odp_time_cur(); + } while (odp_time_cmp(time, cur) > 0); +} + +_ODP_INLINE void odp_time_wait_ns(uint64_t ns) +{ + const odp_time_t cur = _odp_time_cur(); + const odp_time_t wait = _odp_time_from_ns(ns); + const odp_time_t end_time = odp_time_sum(cur, wait); + + _odp_time_wait_until(end_time); +} + +_ODP_INLINE void odp_time_wait_until(odp_time_t time) +{ + _odp_time_wait_until(time); +} + /** @endcond */ #endif diff --git a/platform/linux-dpdk/include/odp/api/plat/timer_inlines.h b/platform/linux-dpdk/include/odp/api/plat/timer_inlines.h index 357d4df06..66327acdb 100644 --- a/platform/linux-dpdk/include/odp/api/plat/timer_inlines.h +++ b/platform/linux-dpdk/include/odp/api/plat/timer_inlines.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2022, Nokia +/* Copyright (c) 2022-2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -25,6 +25,7 @@ #define odp_timeout_user_ptr __odp_timeout_user_ptr #define odp_timeout_user_area __odp_timeout_user_area #define odp_timeout_from_event __odp_timeout_from_event + #define odp_timeout_from_event_multi __odp_timeout_from_event_multi #define odp_timeout_to_event __odp_timeout_to_event #else #define _ODP_INLINE @@ -57,6 +58,15 @@ _ODP_INLINE odp_timeout_t odp_timeout_from_event(odp_event_t ev) return (odp_timeout_t)ev; } +_ODP_INLINE void odp_timeout_from_event_multi(odp_timeout_t tmo[], const odp_event_t ev[], int num) +{ + for (int i = 0; i < num; i++) { + _ODP_ASSERT(odp_event_type(ev[i]) == ODP_EVENT_TIMEOUT); + + tmo[i] = (odp_timeout_t)ev[i]; + } +} + _ODP_INLINE odp_event_t odp_timeout_to_event(odp_timeout_t tmo) { return (odp_event_t)tmo; diff --git a/platform/linux-dpdk/include/odp_pool_internal.h b/platform/linux-dpdk/include/odp_pool_internal.h index b82324a77..8512806f7 100644 --- a/platform/linux-dpdk/include/odp_pool_internal.h +++ b/platform/linux-dpdk/include/odp_pool_internal.h @@ -83,6 +83,7 @@ typedef struct ODP_ALIGNED_CACHE { odp_pool_ext_param_t ext_param; odp_shm_t uarea_shm; uint64_t uarea_shm_size; + uint32_t param_uarea_size; uint32_t uarea_size; uint32_t trailer_size; /* Endmark size */ uint8_t *uarea_base_addr; diff --git a/platform/linux-dpdk/odp_crypto.c b/platform/linux-dpdk/odp_crypto.c index 5761787f8..3269e3075 100644 --- a/platform/linux-dpdk/odp_crypto.c +++ b/platform/linux-dpdk/odp_crypto.c @@ -1431,6 +1431,16 @@ int odp_crypto_session_create(const odp_crypto_session_param_t *param, return -1; } + if (param->cipher_range_in_bits) { + *status = ODP_CRYPTO_SES_ERR_CIPHER; + *session_out = ODP_CRYPTO_SESSION_INVALID; + return -1; + } + if (param->auth_range_in_bits) { + *status = ODP_CRYPTO_SES_ERR_AUTH; + *session_out = ODP_CRYPTO_SESSION_INVALID; + return -1; + } if (param->auth_alg == ODP_AUTH_ALG_AES_GMAC && param->cipher_alg != ODP_CIPHER_ALG_NULL) { *status = ODP_CRYPTO_SES_ERR_ALG_COMBO; diff --git a/platform/linux-dpdk/odp_init.c b/platform/linux-dpdk/odp_init.c index c7ae1ea47..7e6aaab55 100644 --- a/platform/linux-dpdk/odp_init.c +++ b/platform/linux-dpdk/odp_init.c @@ -35,8 +35,8 @@ enum init_stage { LIBCONFIG_INIT, CPUMASK_INIT, CPU_CYCLES_INIT, - TIME_INIT, SYSINFO_INIT, + TIME_INIT, ISHM_INIT, FDSERVER_INIT, GLOBAL_RW_DATA_INIT, @@ -473,16 +473,16 @@ static int term_global(enum init_stage stage) } /* Fall through */ - case SYSINFO_INIT: - if (_odp_system_info_term()) { - _ODP_ERR("ODP system info term failed.\n"); + case TIME_INIT: + if (_odp_time_term_global()) { + _ODP_ERR("ODP time term failed.\n"); rc = -1; } /* Fall through */ - case TIME_INIT: - if (_odp_time_term_global()) { - _ODP_ERR("ODP time term failed.\n"); + case SYSINFO_INIT: + if (_odp_system_info_term()) { + _ODP_ERR("ODP system info term failed.\n"); rc = -1; } /* Fall through */ @@ -558,18 +558,18 @@ int odp_init_global(odp_instance_t *instance, return -1; } - if (_odp_time_init_global()) { - _ODP_ERR("ODP time init failed.\n"); - goto init_failed; - } - stage = TIME_INIT; - if (_odp_system_info_init()) { _ODP_ERR("ODP system_info init failed.\n"); goto init_failed; } stage = SYSINFO_INIT; + if (_odp_time_init_global()) { + _ODP_ERR("ODP time init failed.\n"); + goto init_failed; + } + stage = TIME_INIT; + if (_odp_shm_init_global(params)) { _ODP_ERR("ODP shm init failed.\n"); goto init_failed; diff --git a/platform/linux-dpdk/odp_pool.c b/platform/linux-dpdk/odp_pool.c index 83d75dd6c..fd27cd79e 100644 --- a/platform/linux-dpdk/odp_pool.c +++ b/platform/linux-dpdk/odp_pool.c @@ -263,6 +263,7 @@ int odp_pool_capability(odp_pool_capability_t *capa) capa->buf.max_size = MAX_SIZE; capa->buf.max_num = CONFIG_POOL_MAX_NUM; capa->buf.max_uarea_size = MAX_UAREA_SIZE; + capa->buf.uarea_persistence = true; capa->buf.min_cache_size = 0; capa->buf.max_cache_size = RTE_MEMPOOL_CACHE_MAX_SIZE; capa->buf.stats.all = supported_stats.all; @@ -279,6 +280,7 @@ int odp_pool_capability(odp_pool_capability_t *capa) capa->pkt.min_seg_len = CONFIG_PACKET_SEG_LEN_MIN; capa->pkt.max_seg_len = CONFIG_PACKET_MAX_SEG_LEN; capa->pkt.max_uarea_size = MAX_UAREA_SIZE; + capa->pkt.uarea_persistence = true; capa->pkt.min_cache_size = 0; capa->pkt.max_cache_size = RTE_MEMPOOL_CACHE_MAX_SIZE; capa->pkt.stats.all = supported_stats.all; @@ -287,6 +289,7 @@ int odp_pool_capability(odp_pool_capability_t *capa) capa->tmo.max_pools = max_pools; capa->tmo.max_num = CONFIG_POOL_MAX_NUM; capa->tmo.max_uarea_size = MAX_UAREA_SIZE; + capa->tmo.uarea_persistence = true; capa->tmo.min_cache_size = 0; capa->tmo.max_cache_size = RTE_MEMPOOL_CACHE_MAX_SIZE; capa->tmo.stats.all = supported_stats.all; @@ -295,6 +298,7 @@ int odp_pool_capability(odp_pool_capability_t *capa) capa->vector.max_pools = max_pools; capa->vector.max_num = CONFIG_POOL_MAX_NUM; capa->vector.max_uarea_size = MAX_UAREA_SIZE; + capa->vector.uarea_persistence = true; capa->vector.max_size = CONFIG_PACKET_VECTOR_MAX_SIZE; capa->vector.min_cache_size = 0; capa->vector.max_cache_size = RTE_MEMPOOL_CACHE_MAX_SIZE; @@ -550,6 +554,7 @@ static int reserve_uarea(pool_t *pool, uint32_t uarea_size, uint32_t num_pkt) pool->uarea_shm = ODP_SHM_INVALID; if (uarea_size == 0) { + pool->param_uarea_size = 0; pool->uarea_size = 0; pool->uarea_shm_size = 0; return 0; @@ -559,6 +564,7 @@ static int reserve_uarea(pool_t *pool, uint32_t uarea_size, uint32_t num_pkt) pool->pool_idx, pool->name); uarea_name[ODP_SHM_NAME_LEN - 1] = 0; + pool->param_uarea_size = uarea_size; pool->uarea_size = _ODP_ROUNDUP_CACHE_LINE(uarea_size); pool->uarea_shm_size = num_pkt * (uint64_t)pool->uarea_size; @@ -605,7 +611,8 @@ static void init_obj_priv_data(struct rte_mempool *mp ODP_UNUSED, void *arg, voi struct priv_data_t *priv_data = arg; struct rte_mbuf *mb = mbuf; _odp_event_hdr_t *event_hdr = (_odp_event_hdr_t *)mbuf; - void *uarea = priv_data->pool->uarea_base_addr + i * priv_data->pool->uarea_size; + pool_t *pool = priv_data->pool; + void *uarea = pool->uarea_base_addr + i * pool->uarea_size; void **obj_uarea; if (priv_data->type != ODP_POOL_PACKET) @@ -613,7 +620,7 @@ static void init_obj_priv_data(struct rte_mempool *mp ODP_UNUSED, void *arg, voi mb->data_off = 0; event_hdr->hdr.index = i; - event_hdr->hdr.pool = _odp_pool_handle(priv_data->pool); + event_hdr->hdr.pool = _odp_pool_handle(pool); event_hdr->hdr.type = priv_data->type; event_hdr->hdr.event_type = priv_data->event_type; @@ -636,6 +643,10 @@ static void init_obj_priv_data(struct rte_mempool *mp ODP_UNUSED, void *arg, voi *obj_uarea = uarea; + if (uarea && pool->params.uarea_init.init_fn) + pool->params.uarea_init.init_fn(uarea, pool->param_uarea_size, + pool->params.uarea_init.args, i); + if (priv_data->type == ODP_POOL_BUFFER || priv_data->type == ODP_POOL_PACKET) { mb->buf_len -= _ODP_EV_ENDMARK_SIZE; _odp_event_endmark_set(_odp_event_from_mbuf(mb)); @@ -751,13 +762,15 @@ odp_pool_t _odp_pool_create(const char *name, const odp_pool_param_t *params, return ODP_POOL_INVALID; } - rte_mempool_obj_iter(mp, init_obj_priv_data, &priv_data); pool->rte_mempool = mp; pool->seg_len = seg_size; pool->type_2 = type_2; pool->type = params->type; pool->params = *params; pool->trailer_size = trailer; + + rte_mempool_obj_iter(mp, init_obj_priv_data, &priv_data); + UNLOCK(&pool->lock); pool_hdl = _odp_pool_handle(pool); @@ -1097,6 +1110,7 @@ int odp_pool_ext_capability(odp_pool_type_t type, capa->pkt.max_headroom_size = RTE_PKTMBUF_HEADROOM; capa->pkt.max_segs_per_pkt = PKT_MAX_SEGS; capa->pkt.max_uarea_size = MAX_UAREA_SIZE; + capa->pkt.uarea_persistence = true; return 0; } @@ -1297,10 +1311,11 @@ static void init_ext_obj(struct rte_mempool *mp, void *arg, void *mbuf, unsigned { struct mbuf_ctor_arg *mb_ctor_arg = arg; struct rte_mbuf *mb = mbuf; - void *uarea = mb_ctor_arg->pool->uarea_base_addr + i * mb_ctor_arg->pool->uarea_size; + pool_t *pool = mb_ctor_arg->pool; + void *uarea = pool->uarea_base_addr + i * pool->uarea_size; _odp_event_hdr_t *event_hdr = (_odp_event_hdr_t *)mbuf; void **obj_uarea; - odp_pool_ext_param_t *p = &mb_ctor_arg->pool->ext_param; + odp_pool_ext_param_t *p = &pool->ext_param; uint32_t app_hdr_offset = sizeof(odp_packet_hdr_t); uint32_t app_hdr_size = p->pkt.app_header_size; uint32_t buf_size = p->pkt.buf_size; @@ -1333,7 +1348,7 @@ static void init_ext_obj(struct rte_mempool *mp, void *arg, void *mbuf, unsigned mb->next = NULL; /* Save index, might be useful for debugging purposes */ event_hdr->hdr.index = i; - event_hdr->hdr.pool = _odp_pool_handle(mb_ctor_arg->pool); + event_hdr->hdr.pool = _odp_pool_handle(pool); event_hdr->hdr.type = mb_ctor_arg->type; event_hdr->hdr.event_type = mb_ctor_arg->event_type; @@ -1356,6 +1371,10 @@ static void init_ext_obj(struct rte_mempool *mp, void *arg, void *mbuf, unsigned *obj_uarea = uarea; + if (uarea && pool->ext_param.uarea_init.init_fn) + pool->ext_param.uarea_init.init_fn(uarea, pool->param_uarea_size, + pool->ext_param.uarea_init.args, i); + if (mb_ctor_arg->type == ODP_POOL_BUFFER || mb_ctor_arg->type == ODP_POOL_PACKET) { mb->buf_len -= _ODP_EV_ENDMARK_SIZE; _odp_event_endmark_set(_odp_event_from_mbuf(mb)); diff --git a/platform/linux-dpdk/odp_system_info.c b/platform/linux-dpdk/odp_system_info.c index c76820e5d..886f7f216 100644 --- a/platform/linux-dpdk/odp_system_info.c +++ b/platform/linux-dpdk/odp_system_info.c @@ -59,8 +59,8 @@ static int read_cache_line_size(void) file = fopen(CACHE_LNSZ_FILE, "rt"); if (file == NULL) { /* File not found */ - _ODP_PRINT("WARN: unable to read host CPU cache line size. " - "Using ODP_CACHE_LINE_SIZE instead.\n"); + _ODP_WARN("Unable to read host CPU cache line size. " + "Using ODP_CACHE_LINE_SIZE instead.\n"); return ODP_CACHE_LINE_SIZE; } @@ -225,7 +225,7 @@ static int system_cache_line(system_info_t *sysinfo) sysinfo->cache_line_size = ret; if (ret != ODP_CACHE_LINE_SIZE) - _ODP_PRINT("WARN: host CPU cache line size and ODP_CACHE_LINE_SIZE don't match.\n"); + _ODP_WARN("Host CPU cache line size and ODP_CACHE_LINE_SIZE don't match.\n"); return 0; } diff --git a/platform/linux-dpdk/odp_time.c b/platform/linux-dpdk/odp_time.c index fbbc9358d..2fc25eb96 100644 --- a/platform/linux-dpdk/odp_time.c +++ b/platform/linux-dpdk/odp_time.c @@ -1,302 +1,43 @@ /* Copyright (c) 2013-2018, Linaro Limited - * Copyright (c) 2021-2022, Nokia + * Copyright (c) 2021-2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause */ -#include <odp_posix_extensions.h> - -#include <time.h> -#include <string.h> -#include <inttypes.h> +#include <odp/api/plat/time_inlines.h> -#include <odp/api/time.h> -#include <odp/api/hints.h> #include <odp_debug_internal.h> #include <odp_init_internal.h> -#include <odp/api/plat/time_inlines.h> #include <rte_config.h> -#include <rte_atomic.h> #include <rte_cycles.h> -ODP_STATIC_ASSERT(_ODP_TIMESPEC_SIZE >= (sizeof(struct timespec)), - "_ODP_TIMESPEC_SIZE too small"); +#include <string.h> #include <odp/visibility_begin.h> _odp_time_global_t _odp_time_glob; -/* - * Posix timespec based functions - */ - -static inline uint64_t time_spec_diff_nsec(struct timespec *t2, - struct timespec *t1) -{ - struct timespec diff; - uint64_t nsec; - - diff.tv_sec = t2->tv_sec - t1->tv_sec; - diff.tv_nsec = t2->tv_nsec - t1->tv_nsec; - - if (diff.tv_nsec < 0) { - diff.tv_nsec += ODP_TIME_SEC_IN_NS; - diff.tv_sec -= 1; - } - - nsec = (diff.tv_sec * ODP_TIME_SEC_IN_NS) + diff.tv_nsec; - - return nsec; -} - -odp_time_t _odp_timespec_cur(void) -{ - int ret; - odp_time_t time; - struct timespec sys_time; - struct timespec *start_time; - - start_time = (struct timespec *)(uintptr_t)&_odp_time_glob.timespec; - - ret = clock_gettime(CLOCK_MONOTONIC_RAW, &sys_time); - if (odp_unlikely(ret != 0)) - _ODP_ABORT("clock_gettime failed\n"); - - time.nsec = time_spec_diff_nsec(&sys_time, start_time); - - return time; -} - #include <odp/visibility_end.h> -static inline uint64_t time_spec_res(void) -{ - int ret; - struct timespec tres; - - ret = clock_getres(CLOCK_MONOTONIC_RAW, &tres); - if (odp_unlikely(ret != 0)) - _ODP_ABORT("clock_getres failed\n"); - - return ODP_TIME_SEC_IN_NS / (uint64_t)tres.tv_nsec; -} - -static inline odp_time_t time_spec_from_ns(uint64_t ns) -{ - odp_time_t time; - - time.nsec = ns; - - return time; -} - -/* - * HW time counter based functions - */ - -static inline uint64_t time_hw_res(void) -{ - return _odp_time_glob.hw_freq_hz; -} - -static inline odp_time_t time_hw_from_ns(uint64_t ns) -{ - odp_time_t time; - uint64_t count; - uint64_t freq_hz = _odp_time_glob.hw_freq_hz; - uint64_t sec = 0; - - if (ns >= ODP_TIME_SEC_IN_NS) { - sec = ns / ODP_TIME_SEC_IN_NS; - ns = ns - sec * ODP_TIME_SEC_IN_NS; - } - - count = sec * freq_hz; - count += (ns * freq_hz) / ODP_TIME_SEC_IN_NS; - - time.count = count; - - return time; -} - -/* - * Common functions - */ - -static inline uint64_t time_res(void) -{ - if (_odp_time_glob.use_hw) - return time_hw_res(); - - return time_spec_res(); -} - -static inline odp_time_t time_from_ns(uint64_t ns) -{ - if (_odp_time_glob.use_hw) - return time_hw_from_ns(ns); - - return time_spec_from_ns(ns); -} - -static inline odp_time_t time_cur_dpdk(void) -{ - odp_time_t time; - - time.u64 = rte_get_timer_cycles() - _odp_time_glob.hw_start; - - return time; -} - -static inline odp_time_t time_cur_dpdk_strict(void) -{ - odp_time_t time; - - rte_mb(); - time.u64 = rte_get_timer_cycles() - _odp_time_glob.hw_start; - - return time; -} - -static inline uint64_t time_res_dpdk(void) -{ - return rte_get_timer_hz(); -} - -static inline void time_wait_until(odp_time_t time) -{ - odp_time_t cur; - - do { - cur = _odp_time_cur(); - } while (odp_time_cmp(time, cur) > 0); -} - -odp_time_t odp_time_local_from_ns(uint64_t ns) -{ - return time_from_ns(ns); -} - -odp_time_t odp_time_global_from_ns(uint64_t ns) -{ - return time_from_ns(ns); -} - -uint64_t odp_time_local_res(void) -{ - return _odp_time_glob.handler.time_res(); -} - -uint64_t odp_time_global_res(void) -{ - return _odp_time_glob.handler.time_res(); -} - -void odp_time_wait_ns(uint64_t ns) -{ - odp_time_t cur = _odp_time_cur(); - odp_time_t wait = time_from_ns(ns); - odp_time_t end_time = odp_time_sum(cur, wait); - - time_wait_until(end_time); -} - -void odp_time_wait_until(odp_time_t time) -{ - time_wait_until(time); -} - -static odp_bool_t is_invariant_tsc_supported(void) -{ - FILE *file; - char *line = NULL; - size_t len = 0; - odp_bool_t nonstop_tsc = false; - odp_bool_t constant_tsc = false; - odp_bool_t ret = false; - - file = fopen("/proc/cpuinfo", "rt"); - while (getline(&line, &len, file) != -1) { - if (strstr(line, "flags") != NULL) { - if (strstr(line, "constant_tsc") != NULL) - constant_tsc = true; - if (strstr(line, "nonstop_tsc") != NULL) - nonstop_tsc = true; - - if (constant_tsc && nonstop_tsc) - ret = true; - else - ret = false; - - free(line); - fclose(file); - return ret; - } - } - free(line); - fclose(file); - return false; -} - -static inline odp_bool_t is_dpdk_timer_cycles_support(void) +int _odp_time_init_global(void) { - if (is_invariant_tsc_supported() == true) - return true; + memset(&_odp_time_glob, 0, sizeof(_odp_time_global_t)); #ifdef RTE_LIBEAL_USE_HPET - if (rte_eal_hpet_init(1) == 0) - return true; + if (rte_eal_hpet_init(1) != 0) + _ODP_WARN("HPET init failed. Using TSC time.\n"); #endif - return false; -} - -int _odp_time_init_global(void) -{ - struct timespec *timespec; - int ret = 0; - _odp_time_global_t *global = &_odp_time_glob; - - memset(global, 0, sizeof(_odp_time_global_t)); - - if (is_dpdk_timer_cycles_support()) { - _odp_time_glob.handler.time_cur = time_cur_dpdk; - _odp_time_glob.handler.time_cur_strict = time_cur_dpdk_strict; - _odp_time_glob.handler.time_res = time_res_dpdk; - _odp_time_glob.hw_freq_hz = time_res_dpdk(); - _odp_time_glob.use_hw = 1; - _odp_time_glob.hw_start = rte_get_timer_cycles(); - if (_odp_time_glob.hw_start == 0) - return -1; - else - return 0; - } - _odp_time_glob.handler.time_cur = _odp_time_cur_gen; - _odp_time_glob.handler.time_cur_strict = _odp_time_cur_gen_strict; - _odp_time_glob.handler.time_res = time_res; - - if (_odp_cpu_has_global_time()) { - global->use_hw = 1; - global->hw_freq_hz = _odp_cpu_global_time_freq(); - - if (global->hw_freq_hz == 0) - return -1; - - _ODP_PRINT("HW time counter freq: %" PRIu64 " hz\n\n", global->hw_freq_hz); - - global->hw_start = _odp_cpu_global_time(); - return 0; + _odp_time_glob.freq_hz = rte_get_timer_hz(); + _odp_time_glob.start_cycles = rte_get_timer_cycles(); + if (_odp_time_glob.start_cycles == 0) { + _ODP_ERR("Initializing start cycles failed.\n"); + return -1; } - timespec = (struct timespec *)(uintptr_t)global->timespec; - timespec->tv_sec = 0; - timespec->tv_nsec = 0; - - ret = clock_gettime(CLOCK_MONOTONIC_RAW, timespec); - - return ret; + return 0; } int _odp_time_term_global(void) diff --git a/platform/linux-generic/Makefile.am b/platform/linux-generic/Makefile.am index 42760f579..7afdbf620 100644 --- a/platform/linux-generic/Makefile.am +++ b/platform/linux-generic/Makefile.am @@ -10,6 +10,7 @@ AM_CPPFLAGS += -I$(top_builddir)/platform/$(with_platform)/include AM_CPPFLAGS += -I$(top_srcdir)/platform/$(with_platform)/arch AM_CPPFLAGS += -I$(top_srcdir)/platform/$(with_platform)/arch/@ARCH_DIR@ AM_CPPFLAGS += -I$(top_srcdir)/platform/$(with_platform)/arch/default +AM_CPPFLAGS += -I$(top_srcdir)/platform/$(with_platform)/arch/common AM_CPPFLAGS += $(OPENSSL_CPPFLAGS) @@ -34,6 +35,7 @@ odpapiplatinclude_HEADERS = \ include/odp/api/plat/byteorder_inlines.h \ include/odp/api/plat/cpu_inlines.h \ include/odp/api/plat/crypto_inlines.h \ + include/odp/api/plat/dma_inlines.h \ include/odp/api/plat/debug_inlines.h \ include/odp/api/plat/event_inlines.h \ include/odp/api/plat/event_inline_types.h \ @@ -79,6 +81,7 @@ odpapiabiarchinclude_HEADERS += \ include-abi/odp/api/abi/crypto.h \ include-abi/odp/api/abi/crypto_types.h \ include-abi/odp/api/abi/debug.h \ + include-abi/odp/api/abi/dma.h \ include-abi/odp/api/abi/dma_types.h \ include-abi/odp/api/abi/errno.h \ include-abi/odp/api/abi/event.h \ @@ -255,7 +258,6 @@ __LIB__libodp_linux_la_SOURCES = \ odp_pcapng.c \ odp_thread.c \ odp_thrmask.c \ - odp_time.c \ odp_timer.c \ odp_timer_wheel.c \ odp_traffic_mngr.c \ @@ -301,6 +303,7 @@ __LIB__libodp_linux_la_SOURCES += \ odp_byteorder_api.c \ odp_cpu_api.c \ odp_crypto_api.c \ + odp_dma_api.c \ odp_event_api.c \ odp_hash_api.c \ odp_ipsec_api.c \ @@ -325,18 +328,19 @@ endif if ARCH_IS_ARM __LIB__libodp_linux_la_SOURCES += arch/default/odp_atomic.c \ arch/default/odp_cpu_cycles.c \ - arch/default/odp_global_time.c \ arch/default/odp_hash_crc32.c \ arch/default/odp_random.c \ - arch/arm/odp_sysinfo_parse.c -odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_time.h \ - arch/default/odp/api/abi/hash_crc32.h + arch/arm/odp_sysinfo_parse.c \ + arch/default/odp_time.c +odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/hash_crc32.h if !ODP_ABI_COMPAT odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ arch/default/odp/api/abi/atomic_inlines.h \ arch/default/odp/api/abi/cpu_generic.h \ arch/arm/odp/api/abi/cpu_inlines.h \ - arch/arm/odp/api/abi/cpu.h + arch/arm/odp/api/abi/cpu.h \ + arch/default/odp/api/abi/sync_inlines.h \ + arch/default/odp/api/abi/time_inlines.h endif noinst_HEADERS += arch/arm/odp_atomic.h \ arch/arm/odp_cpu.h \ @@ -351,19 +355,22 @@ if ARCH_IS_AARCH64 __LIB__libodp_linux_la_SOURCES += arch/aarch64/odp_atomic.c \ arch/default/odp_cpu_cycles.c \ arch/aarch64/cpu_flags.c \ - arch/aarch64/odp_global_time.c \ arch/default/odp_hash_crc32.c \ arch/default/odp_random.c \ - arch/aarch64/odp_sysinfo_parse.c -odpapiabiarchinclude_HEADERS += arch/aarch64/odp/api/abi/cpu_time.h \ - arch/aarch64/odp/api/abi/hash_crc32.h + arch/aarch64/odp_sysinfo_parse.c \ + arch/common/odp_time_cpu.c +odpapiabiarchinclude_HEADERS += arch/aarch64/odp/api/abi/hash_crc32.h \ + arch/aarch64/odp/api/abi/time_cpu.h if !ODP_ABI_COMPAT odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ arch/aarch64/odp/api/abi/atomic_inlines.h \ arch/aarch64/odp/api/abi/atomic.h \ arch/default/odp/api/abi/cpu_generic.h \ arch/aarch64/odp/api/abi/cpu_inlines.h \ - arch/aarch64/odp/api/abi/cpu.h + arch/aarch64/odp/api/abi/cpu.h \ + arch/aarch64/odp/api/abi/sync_inlines.h \ + arch/common/odp/api/abi/time_cpu_inlines.h \ + arch/aarch64/odp/api/abi/time_inlines.h endif noinst_HEADERS += arch/aarch64/odp_atomic.h \ arch/aarch64/odp_cpu.h \ @@ -375,18 +382,19 @@ endif if ARCH_IS_DEFAULT __LIB__libodp_linux_la_SOURCES += arch/default/odp_atomic.c \ arch/default/odp_cpu_cycles.c \ - arch/default/odp_global_time.c \ arch/default/odp_hash_crc32.c \ arch/default/odp_random.c \ - arch/default/odp_sysinfo_parse.c -odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_time.h \ - arch/default/odp/api/abi/hash_crc32.h + arch/default/odp_sysinfo_parse.c \ + arch/default/odp_time.c +odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/hash_crc32.h if !ODP_ABI_COMPAT odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ arch/default/odp/api/abi/atomic_inlines.h \ arch/default/odp/api/abi/cpu_generic.h \ arch/default/odp/api/abi/cpu_inlines.h \ - arch/default/odp/api/abi/cpu.h + arch/default/odp/api/abi/cpu.h \ + arch/default/odp/api/abi/sync_inlines.h \ + arch/default/odp/api/abi/time_inlines.h endif noinst_HEADERS += arch/default/odp_atomic.h \ arch/default/odp_cpu.h \ @@ -396,18 +404,19 @@ endif if ARCH_IS_POWERPC __LIB__libodp_linux_la_SOURCES += arch/default/odp_atomic.c \ arch/default/odp_cpu_cycles.c \ - arch/default/odp_global_time.c \ arch/default/odp_hash_crc32.c \ arch/default/odp_random.c \ - arch/powerpc/odp_sysinfo_parse.c -odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_time.h \ - arch/default/odp/api/abi/hash_crc32.h + arch/powerpc/odp_sysinfo_parse.c \ + arch/default/odp_time.c +odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/hash_crc32.h if !ODP_ABI_COMPAT odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ arch/default/odp/api/abi/atomic_inlines.h \ arch/default/odp/api/abi/cpu_generic.h \ arch/default/odp/api/abi/cpu_inlines.h \ - arch/powerpc/odp/api/abi/cpu.h + arch/powerpc/odp/api/abi/cpu.h \ + arch/default/odp/api/abi/sync_inlines.h \ + arch/default/odp/api/abi/time_inlines.h endif noinst_HEADERS += arch/default/odp_atomic.h \ arch/default/odp_cpu.h \ @@ -418,18 +427,22 @@ if ARCH_IS_X86 __LIB__libodp_linux_la_SOURCES += arch/default/odp_atomic.c \ arch/x86/cpu_flags.c \ arch/x86/odp_cpu_cycles.c \ - arch/x86/odp_global_time.c \ arch/default/odp_hash_crc32.c \ arch/default/odp_random.c \ - arch/x86/odp_sysinfo_parse.c + arch/x86/odp_sysinfo_parse.c \ + arch/x86/odp_time_cpu.c \ + arch/common/odp_time_cpu.c odpapiabiarchinclude_HEADERS += arch/x86/odp/api/abi/cpu_rdtsc.h \ - arch/x86/odp/api/abi/cpu_time.h \ - arch/x86/odp/api/abi/hash_crc32.h + arch/x86/odp/api/abi/hash_crc32.h \ + arch/x86/odp/api/abi/time_cpu.h if !ODP_ABI_COMPAT odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ arch/default/odp/api/abi/atomic_inlines.h \ arch/x86/odp/api/abi/cpu_inlines.h \ - arch/x86/odp/api/abi/cpu.h + arch/x86/odp/api/abi/cpu.h \ + arch/x86/odp/api/abi/sync_inlines.h \ + arch/common/odp/api/abi/time_cpu_inlines.h \ + arch/x86/odp/api/abi/time_inlines.h endif noinst_HEADERS += arch/x86/cpu_flags.h \ arch/x86/odp_cpu.h \ diff --git a/platform/linux-generic/arch/aarch64/odp/api/abi/sync_inlines.h b/platform/linux-generic/arch/aarch64/odp/api/abi/sync_inlines.h new file mode 100644 index 000000000..3d42e7dd8 --- /dev/null +++ b/platform/linux-generic/arch/aarch64/odp/api/abi/sync_inlines.h @@ -0,0 +1,31 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2023 Nokia + */ + +#ifndef ODP_ARCH_SYNC_INLINES_H_ +#define ODP_ARCH_SYNC_INLINES_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +static inline void _odp_mb_sync(void) +{ + __asm__ volatile("dsb sy" ::: "memory"); +} + +static inline void _odp_mb_sync_load(void) +{ + __asm__ volatile("dsb ld" ::: "memory"); +} + +static inline void _odp_mb_sync_store(void) +{ + __asm__ volatile("dsb st" ::: "memory"); +} + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/arch/aarch64/odp/api/abi/cpu_time.h b/platform/linux-generic/arch/aarch64/odp/api/abi/time_cpu.h index 781ee683c..aba2799c7 100644 --- a/platform/linux-generic/arch/aarch64/odp/api/abi/cpu_time.h +++ b/platform/linux-generic/arch/aarch64/odp/api/abi/time_cpu.h @@ -4,8 +4,8 @@ * SPDX-License-Identifier: BSD-3-Clause */ -#ifndef ODP_API_ABI_CPU_TIME_H_ -#define ODP_API_ABI_CPU_TIME_H_ +#ifndef ODP_API_ABI_TIME_CPU_H_ +#define ODP_API_ABI_TIME_CPU_H_ #ifdef __cplusplus extern "C" { @@ -13,7 +13,7 @@ extern "C" { #include <stdint.h> -static inline uint64_t _odp_cpu_global_time(void) +static inline uint64_t _odp_time_cpu_global(void) { uint64_t cntvct; @@ -22,7 +22,7 @@ static inline uint64_t _odp_cpu_global_time(void) return cntvct; } -static inline uint64_t _odp_cpu_global_time_strict(void) +static inline uint64_t _odp_time_cpu_global_strict(void) { uint64_t cntvct; @@ -32,7 +32,7 @@ static inline uint64_t _odp_cpu_global_time_strict(void) return cntvct; } -static inline uint64_t _odp_cpu_global_time_freq(void) +static inline uint64_t _odp_time_cpu_global_freq(void) { uint64_t cntfrq; @@ -41,7 +41,10 @@ static inline uint64_t _odp_cpu_global_time_freq(void) return cntfrq; } -int _odp_cpu_has_global_time(void); +static inline int _odp_time_cpu_global_freq_is_const(void) +{ + return 1; +} #ifdef __cplusplus } diff --git a/platform/linux-generic/arch/aarch64/odp/api/abi/time_inlines.h b/platform/linux-generic/arch/aarch64/odp/api/abi/time_inlines.h new file mode 100644 index 000000000..331d1996f --- /dev/null +++ b/platform/linux-generic/arch/aarch64/odp/api/abi/time_inlines.h @@ -0,0 +1,7 @@ +/* Copyright (c) 2023, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <odp/api/abi/time_cpu_inlines.h> diff --git a/platform/linux-generic/arch/aarch64/odp_global_time.c b/platform/linux-generic/arch/aarch64/odp_global_time.c deleted file mode 100644 index 53561b00c..000000000 --- a/platform/linux-generic/arch/aarch64/odp_global_time.c +++ /dev/null @@ -1,25 +0,0 @@ -/* Copyright (c) 2015-2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -#include <odp/api/abi/cpu_time.h> - -int _odp_cpu_has_global_time(void) -{ - uint64_t hz = _odp_cpu_global_time_freq(); - - /* - * The system counter portion of the architected timer must - * provide a uniform view of system time to all processing - * elements in the system. This should hold true even for - * heterogeneous SoCs. - * - * Determine whether the system has 'global time' by checking - * whether a read of the architected timer frequency sys reg - * returns a sane value. Sane is considered to be within - * 1MHz and 6GHz (1us and .1667ns period). - */ - return hz >= 1000000 && hz <= 6000000000; -} diff --git a/platform/linux-generic/arch/aarch64/odp_sysinfo_parse.c b/platform/linux-generic/arch/aarch64/odp_sysinfo_parse.c index 25e3e4fc6..352bee9e9 100644 --- a/platform/linux-generic/arch/aarch64/odp_sysinfo_parse.c +++ b/platform/linux-generic/arch/aarch64/odp_sysinfo_parse.c @@ -336,9 +336,8 @@ int _odp_cpuinfo_parser(FILE *file, system_info_t *sysinfo) if (sysinfo->cpu_hz_max[id] == 0) { uint64_t hz = sysinfo->default_cpu_hz_max; - _ODP_PRINT("WARN: cpu[%i] uses default max " - "frequency of %" PRIu64 " Hz from " - "config file\n", id, hz); + _ODP_WARN("CPU[%i] uses default max frequency of %" PRIu64 " " + "Hz from config file\n", id, hz); sysinfo->cpu_hz_max[id] = hz; } diff --git a/platform/linux-generic/arch/common/odp/api/abi/time_cpu_inlines.h b/platform/linux-generic/arch/common/odp/api/abi/time_cpu_inlines.h new file mode 100644 index 000000000..c154c5f1a --- /dev/null +++ b/platform/linux-generic/arch/common/odp/api/abi/time_cpu_inlines.h @@ -0,0 +1,93 @@ +/* Copyright (c) 2013-2018, Linaro Limited + * Copyright (c) 2020-2023, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_ARCH_TIME_CPU_INLINES_H_ +#define ODP_ARCH_TIME_CPU_INLINES_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <odp/api/time_types.h> + +#include <odp/api/abi/time_cpu.h> + +#include <stdint.h> + +#define _ODP_TIME_GIGA_HZ 1000000000ULL + +typedef struct _odp_time_global_t { + uint64_t start_time; + uint64_t freq_hz; + +} _odp_time_global_t; + +extern _odp_time_global_t _odp_time_glob; + +static inline odp_time_t _odp_time_cur(void) +{ + odp_time_t time; + + time.count = _odp_time_cpu_global() - _odp_time_glob.start_time; + return time; +} + +static inline odp_time_t _odp_time_cur_strict(void) +{ + odp_time_t time; + + time.count = _odp_time_cpu_global_strict() - _odp_time_glob.start_time; + return time; +} + +static inline uint64_t _odp_time_to_ns(odp_time_t time) +{ + uint64_t nsec; + uint64_t freq_hz = _odp_time_glob.freq_hz; + uint64_t count = time.count; + uint64_t sec = 0; + + if (count >= freq_hz) { + sec = count / freq_hz; + count = count - sec * freq_hz; + } + + nsec = (_ODP_TIME_GIGA_HZ * count) / freq_hz; + + return (sec * _ODP_TIME_GIGA_HZ) + nsec; +} + +static inline odp_time_t _odp_time_from_ns(uint64_t ns) +{ + odp_time_t time; + uint64_t count; + uint64_t freq_hz = _odp_time_glob.freq_hz; + uint64_t sec = 0; + + if (ns >= ODP_TIME_SEC_IN_NS) { + sec = ns / ODP_TIME_SEC_IN_NS; + ns = ns - sec * ODP_TIME_SEC_IN_NS; + } + + count = sec * freq_hz; + count += (ns * freq_hz) / ODP_TIME_SEC_IN_NS; + + time.count = count; + + return time; +} + +static inline uint64_t _odp_time_res(void) +{ + return _odp_time_glob.freq_hz; +} + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/arch/common/odp_time_cpu.c b/platform/linux-generic/arch/common/odp_time_cpu.c new file mode 100644 index 000000000..bbfe82e21 --- /dev/null +++ b/platform/linux-generic/arch/common/odp_time_cpu.c @@ -0,0 +1,48 @@ +/* Copyright (c) 2013-2018, Linaro Limited + * Copyright (c) 2020-2023, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <odp/api/time_types.h> + +#include <odp/api/abi/time_cpu.h> +#include <odp/api/abi/time_cpu_inlines.h> + +#include <odp_debug_internal.h> +#include <odp_init_internal.h> + +#include <inttypes.h> +#include <stdint.h> +#include <string.h> + +#include <odp/visibility_begin.h> + +_odp_time_global_t _odp_time_glob; + +#include <odp/visibility_end.h> + +int _odp_time_init_global(void) +{ + _odp_time_global_t *global = &_odp_time_glob; + + memset(global, 0, sizeof(_odp_time_global_t)); + + if (!_odp_time_cpu_global_freq_is_const()) + return -1; + + global->freq_hz = _odp_time_cpu_global_freq(); + if (global->freq_hz == 0) + return -1; + + _ODP_PRINT("HW time counter freq: %" PRIu64 " hz\n\n", global->freq_hz); + + global->start_time = _odp_time_cpu_global(); + return 0; +} + +int _odp_time_term_global(void) +{ + return 0; +} diff --git a/platform/linux-generic/arch/default/odp/api/abi/cpu_time.h b/platform/linux-generic/arch/default/odp/api/abi/cpu_time.h deleted file mode 100644 index 24e1c7d33..000000000 --- a/platform/linux-generic/arch/default/odp/api/abi/cpu_time.h +++ /dev/null @@ -1,25 +0,0 @@ -/* Copyright (c) 2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -#ifndef ODP_ARCH_CPU_TIME_H_ -#define ODP_ARCH_CPU_TIME_H_ - -#ifdef __cplusplus -extern "C" { -#endif - -#include <stdint.h> - -int _odp_cpu_has_global_time(void); -uint64_t _odp_cpu_global_time(void); -uint64_t _odp_cpu_global_time_strict(void); -uint64_t _odp_cpu_global_time_freq(void); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/platform/linux-generic/arch/default/odp/api/abi/sync_inlines.h b/platform/linux-generic/arch/default/odp/api/abi/sync_inlines.h new file mode 100644 index 000000000..bfbb3039f --- /dev/null +++ b/platform/linux-generic/arch/default/odp/api/abi/sync_inlines.h @@ -0,0 +1,31 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2023 Nokia + */ + +#ifndef ODP_ARCH_SYNC_INLINES_H_ +#define ODP_ARCH_SYNC_INLINES_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +static inline void _odp_mb_sync(void) +{ + __sync_synchronize(); +} + +static inline void _odp_mb_sync_load(void) +{ + __sync_synchronize(); +} + +static inline void _odp_mb_sync_store(void) +{ + __sync_synchronize(); +} + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/arch/default/odp/api/abi/time_inlines.h b/platform/linux-generic/arch/default/odp/api/abi/time_inlines.h new file mode 100644 index 000000000..b38e52dac --- /dev/null +++ b/platform/linux-generic/arch/default/odp/api/abi/time_inlines.h @@ -0,0 +1,45 @@ +/* Copyright (c) 2013-2018, Linaro Limited + * Copyright (c) 2020-2023, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_ARCH_TIME_INLINES_H_ +#define ODP_ARCH_TIME_INLINES_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <odp/api/time_types.h> + +#include <stdint.h> + +odp_time_t _odp_time_cur(void); +uint64_t _odp_time_res(void); + +static inline odp_time_t _odp_time_cur_strict(void) +{ + return _odp_time_cur(); +} + +static inline uint64_t _odp_time_to_ns(odp_time_t time) +{ + return time.nsec; +} + +static inline odp_time_t _odp_time_from_ns(uint64_t ns) +{ + odp_time_t time; + + time.nsec = ns; + + return time; +} + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/arch/default/odp_global_time.c b/platform/linux-generic/arch/default/odp_global_time.c deleted file mode 100644 index ee835413f..000000000 --- a/platform/linux-generic/arch/default/odp_global_time.c +++ /dev/null @@ -1,31 +0,0 @@ -/* Copyright (c) 2015-2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -#include <odp/api/abi/cpu_time.h> - -#include <odp/visibility_begin.h> - -uint64_t _odp_cpu_global_time(void) -{ - return 0; -} - -uint64_t _odp_cpu_global_time_strict(void) -{ - return 0; -} - -#include <odp/visibility_end.h> - -int _odp_cpu_has_global_time(void) -{ - return 0; -} - -uint64_t _odp_cpu_global_time_freq(void) -{ - return 0; -} diff --git a/platform/linux-generic/arch/default/odp_time.c b/platform/linux-generic/arch/default/odp_time.c new file mode 100644 index 000000000..919a3ba68 --- /dev/null +++ b/platform/linux-generic/arch/default/odp_time.c @@ -0,0 +1,102 @@ +/* Copyright (c) 2013-2018, Linaro Limited + * Copyright (c) 2020-2023, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <odp_posix_extensions.h> + +#include <odp/api/align.h> +#include <odp/api/hints.h> +#include <odp/api/time_types.h> + +#include <odp/api/abi/time_inlines.h> + +#include <odp_debug_internal.h> +#include <odp_init_internal.h> + +#include <stdint.h> +#include <string.h> +#include <time.h> + +typedef struct _odp_time_global_t { + struct timespec start_time; + +} _odp_time_global_t; + +_odp_time_global_t _odp_time_glob; + +static inline uint64_t time_diff_nsec(struct timespec *t2, struct timespec *t1) +{ + struct timespec diff; + uint64_t nsec; + + diff.tv_sec = t2->tv_sec - t1->tv_sec; + diff.tv_nsec = t2->tv_nsec - t1->tv_nsec; + + if (diff.tv_nsec < 0) { + diff.tv_nsec += ODP_TIME_SEC_IN_NS; + diff.tv_sec -= 1; + } + + nsec = (diff.tv_sec * ODP_TIME_SEC_IN_NS) + diff.tv_nsec; + + return nsec; +} + +#include <odp/visibility_begin.h> + +odp_time_t _odp_time_cur(void) +{ + int ret; + odp_time_t time; + struct timespec sys_time; + struct timespec *start_time = &_odp_time_glob.start_time; + + ret = clock_gettime(CLOCK_MONOTONIC_RAW, &sys_time); + if (odp_unlikely(ret != 0)) + _ODP_ABORT("clock_gettime() failed\n"); + + time.nsec = time_diff_nsec(&sys_time, start_time); + + return time; +} + +uint64_t _odp_time_res(void) +{ + int ret; + struct timespec tres; + + ret = clock_getres(CLOCK_MONOTONIC_RAW, &tres); + if (odp_unlikely(ret != 0)) + _ODP_ABORT("clock_getres() failed\n"); + + return ODP_TIME_SEC_IN_NS / (uint64_t)tres.tv_nsec; +} + +#include <odp/visibility_end.h> + +int _odp_time_init_global(void) +{ + struct timespec *start_time; + int ret = 0; + _odp_time_global_t *global = &_odp_time_glob; + + memset(global, 0, sizeof(_odp_time_global_t)); + + start_time = &global->start_time; + start_time->tv_sec = 0; + start_time->tv_nsec = 0; + + ret = clock_gettime(CLOCK_MONOTONIC_RAW, start_time); + if (ret) + _ODP_ERR("clock_gettime() failed: %d\n", ret); + + return ret; +} + +int _odp_time_term_global(void) +{ + return 0; +} diff --git a/platform/linux-generic/arch/x86/cpu_flags.c b/platform/linux-generic/arch/x86/cpu_flags.c index 036645dbc..9211df002 100644 --- a/platform/linux-generic/arch/x86/cpu_flags.c +++ b/platform/linux-generic/arch/x86/cpu_flags.c @@ -1,45 +1,23 @@ /* Copyright (c) 2017-2018, Linaro Limited + * Copyright (c) 2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause */ -/*- - * BSD LICENSE - * - * Copyright(c) 2010-2015 Intel Corporation. All rights reserved. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2010-2015 Intel Corporation */ #include "cpu_flags.h" + +#include <odp/api/abi/time_cpu.h> + #include <odp_debug_internal.h> -#include <odp/api/abi/cpu_time.h> +#include <odp_global_data.h> + +#include <cpuid.h> +#include <errno.h> #include <stdio.h> #include <stdint.h> @@ -74,6 +52,7 @@ enum rte_cpu_flag_t { RTE_CPUFLAG_AVX, /**< AVX */ RTE_CPUFLAG_F16C, /**< F16C */ RTE_CPUFLAG_RDRAND, /**< RDRAND */ + RTE_CPUFLAG_HYPERVISOR, /**< Running in a VM */ /* (EAX 01h) EDX features */ RTE_CPUFLAG_FPU, /**< FPU */ @@ -130,6 +109,7 @@ enum rte_cpu_flag_t { RTE_CPUFLAG_INVPCID, /**< INVPCID */ RTE_CPUFLAG_RTM, /**< Transactional memory */ RTE_CPUFLAG_AVX512F, /**< AVX512F */ + RTE_CPUFLAG_RDSEED, /**< RDSEED instruction */ /* (EAX 80000001h) ECX features */ RTE_CPUFLAG_LAHF_SAHF, /**< LAHF_SAHF */ @@ -145,8 +125,29 @@ enum rte_cpu_flag_t { /* (EAX 80000007h) EDX features */ RTE_CPUFLAG_INVTSC, /**< INVTSC */ + RTE_CPUFLAG_AVX512DQ, /**< AVX512 Doubleword and Quadword */ + RTE_CPUFLAG_AVX512IFMA, /**< AVX512 Integer Fused Multiply-Add */ + RTE_CPUFLAG_AVX512CD, /**< AVX512 Conflict Detection*/ + RTE_CPUFLAG_AVX512BW, /**< AVX512 Byte and Word */ + RTE_CPUFLAG_AVX512VL, /**< AVX512 Vector Length */ + RTE_CPUFLAG_AVX512VBMI, /**< AVX512 Vector Bit Manipulation */ + RTE_CPUFLAG_AVX512VBMI2, /**< AVX512 Vector Bit Manipulation 2 */ + RTE_CPUFLAG_GFNI, /**< Galois Field New Instructions */ + RTE_CPUFLAG_VAES, /**< Vector AES */ + RTE_CPUFLAG_VPCLMULQDQ, /**< Vector Carry-less Multiply */ + RTE_CPUFLAG_AVX512VNNI, + /**< AVX512 Vector Neural Network Instructions */ + RTE_CPUFLAG_AVX512BITALG, /**< AVX512 Bit Algorithms */ + RTE_CPUFLAG_AVX512VPOPCNTDQ, /**< AVX512 Vector Popcount */ + RTE_CPUFLAG_CLDEMOTE, /**< Cache Line Demote */ + RTE_CPUFLAG_MOVDIRI, /**< Direct Store Instructions */ + RTE_CPUFLAG_MOVDIR64B, /**< Direct Store Instructions 64B */ + RTE_CPUFLAG_AVX512VP2INTERSECT, /**< AVX512 Two Register Intersection */ + + RTE_CPUFLAG_WAITPKG, /**< UMONITOR/UMWAIT/TPAUSE */ + /* The last item */ - RTE_CPUFLAG_NUMFLAGS, /**< This should always be the last! */ + RTE_CPUFLAG_NUMFLAGS, /**< This should always be the last! */ }; enum cpu_register_t { @@ -203,6 +204,7 @@ static const struct feature_entry cpu_feature_table[] = { FEAT_DEF(AVX, 0x00000001, 0, RTE_REG_ECX, 28) FEAT_DEF(F16C, 0x00000001, 0, RTE_REG_ECX, 29) FEAT_DEF(RDRAND, 0x00000001, 0, RTE_REG_ECX, 30) + FEAT_DEF(HYPERVISOR, 0x00000001, 0, RTE_REG_ECX, 31) FEAT_DEF(FPU, 0x00000001, 0, RTE_REG_EDX, 0) FEAT_DEF(VME, 0x00000001, 0, RTE_REG_EDX, 1) @@ -246,15 +248,36 @@ static const struct feature_entry cpu_feature_table[] = { FEAT_DEF(ENERGY_EFF, 0x00000006, 0, RTE_REG_ECX, 3) FEAT_DEF(FSGSBASE, 0x00000007, 0, RTE_REG_EBX, 0) - FEAT_DEF(BMI1, 0x00000007, 0, RTE_REG_EBX, 2) + FEAT_DEF(BMI1, 0x00000007, 0, RTE_REG_EBX, 3) FEAT_DEF(HLE, 0x00000007, 0, RTE_REG_EBX, 4) FEAT_DEF(AVX2, 0x00000007, 0, RTE_REG_EBX, 5) - FEAT_DEF(SMEP, 0x00000007, 0, RTE_REG_EBX, 6) - FEAT_DEF(BMI2, 0x00000007, 0, RTE_REG_EBX, 7) - FEAT_DEF(ERMS, 0x00000007, 0, RTE_REG_EBX, 8) + FEAT_DEF(SMEP, 0x00000007, 0, RTE_REG_EBX, 7) + FEAT_DEF(BMI2, 0x00000007, 0, RTE_REG_EBX, 8) + FEAT_DEF(ERMS, 0x00000007, 0, RTE_REG_EBX, 9) FEAT_DEF(INVPCID, 0x00000007, 0, RTE_REG_EBX, 10) FEAT_DEF(RTM, 0x00000007, 0, RTE_REG_EBX, 11) FEAT_DEF(AVX512F, 0x00000007, 0, RTE_REG_EBX, 16) + FEAT_DEF(AVX512DQ, 0x00000007, 0, RTE_REG_EBX, 17) + FEAT_DEF(RDSEED, 0x00000007, 0, RTE_REG_EBX, 18) + FEAT_DEF(AVX512IFMA, 0x00000007, 0, RTE_REG_EBX, 21) + FEAT_DEF(AVX512CD, 0x00000007, 0, RTE_REG_EBX, 28) + FEAT_DEF(AVX512BW, 0x00000007, 0, RTE_REG_EBX, 30) + FEAT_DEF(AVX512VL, 0x00000007, 0, RTE_REG_EBX, 31) + + FEAT_DEF(AVX512VBMI, 0x00000007, 0, RTE_REG_ECX, 1) + FEAT_DEF(WAITPKG, 0x00000007, 0, RTE_REG_ECX, 5) + FEAT_DEF(AVX512VBMI2, 0x00000007, 0, RTE_REG_ECX, 6) + FEAT_DEF(GFNI, 0x00000007, 0, RTE_REG_ECX, 8) + FEAT_DEF(VAES, 0x00000007, 0, RTE_REG_ECX, 9) + FEAT_DEF(VPCLMULQDQ, 0x00000007, 0, RTE_REG_ECX, 10) + FEAT_DEF(AVX512VNNI, 0x00000007, 0, RTE_REG_ECX, 11) + FEAT_DEF(AVX512BITALG, 0x00000007, 0, RTE_REG_ECX, 12) + FEAT_DEF(AVX512VPOPCNTDQ, 0x00000007, 0, RTE_REG_ECX, 14) + FEAT_DEF(CLDEMOTE, 0x00000007, 0, RTE_REG_ECX, 25) + FEAT_DEF(MOVDIRI, 0x00000007, 0, RTE_REG_ECX, 27) + FEAT_DEF(MOVDIR64B, 0x00000007, 0, RTE_REG_ECX, 28) + + FEAT_DEF(AVX512VP2INTERSECT, 0x00000007, 0, RTE_REG_EDX, 8) FEAT_DEF(LAHF_SAHF, 0x80000001, 0, RTE_REG_ECX, 0) FEAT_DEF(LZCNT, 0x80000001, 0, RTE_REG_ECX, 4) @@ -268,55 +291,30 @@ static const struct feature_entry cpu_feature_table[] = { FEAT_DEF(INVTSC, 0x80000007, 0, RTE_REG_EDX, 8) }; -/* - * Execute CPUID instruction and get contents of a specific register - * - * This function, when compiled with GCC, will generate architecture-neutral - * code, as per GCC manual. - */ -static void cpu_get_features(uint32_t leaf, uint32_t subleaf, - cpuid_registers_t out) -{ -#if defined(__i386__) && defined(__PIC__) - /* %ebx is a forbidden register if we compile with -fPIC or -fPIE */ - __asm__ __volatile__("movl %%ebx,%0 ; cpuid ; xchgl %%ebx,%0" - : "=r" (out[RTE_REG_EBX]), - "=a" (out[RTE_REG_EAX]), - "=c" (out[RTE_REG_ECX]), - "=d" (out[RTE_REG_EDX]) - : "a" (leaf), "c" (subleaf)); -#else - __asm__ __volatile__("cpuid" - : "=a" (out[RTE_REG_EAX]), - "=b" (out[RTE_REG_EBX]), - "=c" (out[RTE_REG_ECX]), - "=d" (out[RTE_REG_EDX]) - : "a" (leaf), "c" (subleaf)); -#endif -} - static int cpu_get_flag_enabled(enum rte_cpu_flag_t feature) { const struct feature_entry *feat; cpuid_registers_t regs; + unsigned int maxleaf; if (feature >= RTE_CPUFLAG_NUMFLAGS) /* Flag does not match anything in the feature tables */ - return -1; + return -ENOENT; feat = &cpu_feature_table[feature]; if (!feat->leaf) /* This entry in the table wasn't filled out! */ - return -1; + return -EFAULT; + + maxleaf = __get_cpuid_max(feat->leaf & 0x80000000, NULL); - cpu_get_features(feat->leaf & 0xffff0000, 0, regs); - if (((regs[RTE_REG_EAX] ^ feat->leaf) & 0xffff0000) || - regs[RTE_REG_EAX] < feat->leaf) + if (maxleaf < feat->leaf) return 0; - /* get the cpuid leaf containing the desired feature */ - cpu_get_features(feat->leaf, feat->subleaf, regs); + __cpuid_count(feat->leaf, feat->subleaf, + regs[RTE_REG_EAX], regs[RTE_REG_EBX], + regs[RTE_REG_ECX], regs[RTE_REG_EDX]); /* check if the feature is enabled */ return (regs[feat->reg] >> feat->bit) & 1; @@ -359,12 +357,16 @@ void _odp_cpu_flags_print_all(void) _ODP_PRINT("%s", str); } -int _odp_cpu_has_global_time(void) +int _odp_time_cpu_global_freq_is_const(void) { - if (cpu_get_flag_enabled(RTE_CPUFLAG_INVTSC) > 0) + if (odp_global_ro.system_info.cpu_constant_tsc || + cpu_get_flag_enabled(RTE_CPUFLAG_INVTSC) > 0) return 1; - return 0; + _ODP_ERR("WARN: assuming constant TSC based on CPU arch, but could not confirm from CPU " + "flags\n"); + + return 1; } int _odp_cpu_flags_has_rdtsc(void) diff --git a/platform/linux-generic/arch/x86/odp/api/abi/sync_inlines.h b/platform/linux-generic/arch/x86/odp/api/abi/sync_inlines.h new file mode 100644 index 000000000..bebe6b571 --- /dev/null +++ b/platform/linux-generic/arch/x86/odp/api/abi/sync_inlines.h @@ -0,0 +1,31 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2023 Nokia + */ + +#ifndef ODP_ARCH_SYNC_INLINES_H_ +#define ODP_ARCH_SYNC_INLINES_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +static inline void _odp_mb_sync(void) +{ + __asm__ volatile("mfence" ::: "memory"); +} + +static inline void _odp_mb_sync_load(void) +{ + __asm__ volatile("lfence" ::: "memory"); +} + +static inline void _odp_mb_sync_store(void) +{ + __asm__ volatile("sfence" ::: "memory"); +} + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/arch/x86/odp/api/abi/cpu_time.h b/platform/linux-generic/arch/x86/odp/api/abi/time_cpu.h index c74c4d606..baf79ad3f 100644 --- a/platform/linux-generic/arch/x86/odp/api/abi/cpu_time.h +++ b/platform/linux-generic/arch/x86/odp/api/abi/time_cpu.h @@ -4,8 +4,8 @@ * SPDX-License-Identifier: BSD-3-Clause */ -#ifndef ODP_ARCH_CPU_TIME_H_ -#define ODP_ARCH_CPU_TIME_H_ +#ifndef ODP_ARCH_TIME_CPU_H_ +#define ODP_ARCH_TIME_CPU_H_ #ifdef __cplusplus extern "C" { @@ -14,19 +14,19 @@ extern "C" { #include <stdint.h> #include <odp/api/abi/cpu_rdtsc.h> -static inline uint64_t _odp_cpu_global_time(void) +static inline uint64_t _odp_time_cpu_global(void) { return _odp_cpu_rdtsc(); } -static inline uint64_t _odp_cpu_global_time_strict(void) +static inline uint64_t _odp_time_cpu_global_strict(void) { __atomic_thread_fence(__ATOMIC_SEQ_CST); return _odp_cpu_rdtsc(); } -int _odp_cpu_has_global_time(void); -uint64_t _odp_cpu_global_time_freq(void); +int _odp_time_cpu_global_freq_is_const(void); +uint64_t _odp_time_cpu_global_freq(void); #ifdef __cplusplus } diff --git a/platform/linux-generic/arch/x86/odp/api/abi/time_inlines.h b/platform/linux-generic/arch/x86/odp/api/abi/time_inlines.h new file mode 100644 index 000000000..331d1996f --- /dev/null +++ b/platform/linux-generic/arch/x86/odp/api/abi/time_inlines.h @@ -0,0 +1,7 @@ +/* Copyright (c) 2023, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <odp/api/abi/time_cpu_inlines.h> diff --git a/platform/linux-generic/arch/x86/odp_sysinfo_parse.c b/platform/linux-generic/arch/x86/odp_sysinfo_parse.c index c74c52045..3cbdb2037 100644 --- a/platform/linux-generic/arch/x86/odp_sysinfo_parse.c +++ b/platform/linux-generic/arch/x86/odp_sysinfo_parse.c @@ -1,4 +1,5 @@ /* Copyright (c) 2016-2018, Linaro Limited + * Copyright (c) 2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -30,6 +31,11 @@ int _odp_cpuinfo_parser(FILE *file, system_info_t *sysinfo) strcpy(sysinfo->cpu_arch_str, "x86"); while (fgets(str, sizeof(str), file) != NULL && id < CONFIG_NUM_CPU_IDS) { + if (strstr(str, "flags") && strstr(str, "constant_tsc")) { + sysinfo->cpu_constant_tsc = 1; + continue; + } + pos = strstr(str, "model name"); if (pos) { freq_set = false; diff --git a/platform/linux-generic/arch/x86/odp_global_time.c b/platform/linux-generic/arch/x86/odp_time_cpu.c index 00e3a1ac1..aa00ac04e 100644 --- a/platform/linux-generic/arch/x86/odp_global_time.c +++ b/platform/linux-generic/arch/x86/odp_time_cpu.c @@ -6,17 +6,18 @@ #include <odp_posix_extensions.h> -#include <time.h> - #include <odp/api/hints.h> +#include <odp/api/time_types.h> + +#include <odp/api/abi/time_cpu.h> + #include <odp_debug_internal.h> -#include <odp/api/abi/cpu_time.h> -#define SEC_IN_NS 1000000000ULL +#include <time.h> /* Measure TSC frequency. Frequency information registers are defined for x86, * but those are often not enumerated. */ -uint64_t _odp_cpu_global_time_freq(void) +uint64_t _odp_time_cpu_global_freq(void) { struct timespec sleep, ts1, ts2; uint64_t t1, t2, ts_nsec, cycles, hz; @@ -29,35 +30,35 @@ uint64_t _odp_cpu_global_time_freq(void) sleep.tv_sec = 0; if (warm_up) - sleep.tv_nsec = SEC_IN_NS / 1000; + sleep.tv_nsec = ODP_TIME_SEC_IN_NS / 1000; else - sleep.tv_nsec = SEC_IN_NS / 4; + sleep.tv_nsec = ODP_TIME_SEC_IN_NS / 4; if (clock_gettime(CLOCK_MONOTONIC_RAW, &ts1)) { - _ODP_DBG("clock_gettime failed\n"); + _ODP_ERR("clock_gettime() failed\n"); return 0; } - t1 = _odp_cpu_global_time(); + t1 = _odp_time_cpu_global(); if (nanosleep(&sleep, NULL) < 0) { - _ODP_DBG("nanosleep failed\n"); + _ODP_ERR("nanosleep() failed\n"); return 0; } if (clock_gettime(CLOCK_MONOTONIC_RAW, &ts2)) { - _ODP_DBG("clock_gettime failed\n"); + _ODP_ERR("clock_gettime() failed\n"); return 0; } - t2 = _odp_cpu_global_time(); + t2 = _odp_time_cpu_global(); - ts_nsec = (ts2.tv_sec - ts1.tv_sec) * SEC_IN_NS; + ts_nsec = (ts2.tv_sec - ts1.tv_sec) * ODP_TIME_SEC_IN_NS; ts_nsec += ts2.tv_nsec - ts1.tv_nsec; cycles = t2 - t1; - hz = (cycles * SEC_IN_NS) / ts_nsec; + hz = (cycles * ODP_TIME_SEC_IN_NS) / ts_nsec; if (warm_up) warm_up = 0; diff --git a/platform/linux-generic/include-abi/odp/api/abi/dma.h b/platform/linux-generic/include-abi/odp/api/abi/dma.h new file mode 100644 index 000000000..60798143c --- /dev/null +++ b/platform/linux-generic/include-abi/odp/api/abi/dma.h @@ -0,0 +1,27 @@ +/* Copyright (c) 2023, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +/** + * @file + * + * ODP DMA + */ + +#ifndef ODP_API_ABI_DMA_H_ +#define ODP_API_ABI_DMA_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +/* Inlined functions for non-ABI compat mode */ +#include <odp/api/plat/dma_inlines.h> + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/include/odp/api/plat/debug_inlines.h b/platform/linux-generic/include/odp/api/plat/debug_inlines.h index 41af3dca4..0755b1fda 100644 --- a/platform/linux-generic/include/odp/api/plat/debug_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/debug_inlines.h @@ -1,5 +1,5 @@ /* Copyright (c) 2014-2018, Linaro Limited - * Copyright (c) 2020-2022, Nokia + * Copyright (c) 2020-2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -50,9 +50,9 @@ extern odp_abort_func_t _odp_abort_fn; /** * ODP LOG macro. */ -#define _ODP_LOG(level, fmt, ...) \ - _ODP_LOG_FN(level, "%s:%d:%s(): " fmt, __FILE__, \ - __LINE__, __func__, ##__VA_ARGS__) +#define _ODP_LOG(level, prefix, fmt, ...) \ + _ODP_LOG_FN(level, "%s: %s:%d:%s(): " fmt, prefix, \ + __FILE__, __LINE__, __func__, ##__VA_ARGS__) /** * Runtime assertion-macro - aborts if 'cond' is false. @@ -70,17 +70,27 @@ extern odp_abort_func_t _odp_abort_fn; do { \ if (ODP_DEBUG_PRINT == 1) \ __extension__ ({ \ - _ODP_LOG(ODP_LOG_DBG, ##__VA_ARGS__); \ + _ODP_LOG(ODP_LOG_DBG, "DBG", ##__VA_ARGS__); \ }); \ } while (0) /** + * Log warning message. + */ +#define _ODP_WARN(...) \ + do { \ + __extension__ ({ \ + _ODP_LOG(ODP_LOG_WARN, "WARN", ##__VA_ARGS__); \ + }); \ + } while (0) + +/** * Log error message. */ #define _ODP_ERR(...) \ do { \ __extension__ ({ \ - _ODP_LOG(ODP_LOG_ERR, ##__VA_ARGS__); \ + _ODP_LOG(ODP_LOG_ERR, "ERR", ##__VA_ARGS__); \ }); \ } while (0) @@ -91,7 +101,7 @@ extern odp_abort_func_t _odp_abort_fn; #define _ODP_ABORT(...) \ do { \ __extension__ ({ \ - _ODP_LOG(ODP_LOG_ABORT, ##__VA_ARGS__); \ + _ODP_LOG(ODP_LOG_ABORT, "ABORT", ##__VA_ARGS__); \ }); \ _odp_abort_fn(); \ } while (0) diff --git a/platform/linux-generic/include/odp/api/plat/dma_inlines.h b/platform/linux-generic/include/odp/api/plat/dma_inlines.h new file mode 100644 index 000000000..84b5fef5b --- /dev/null +++ b/platform/linux-generic/include/odp/api/plat/dma_inlines.h @@ -0,0 +1,135 @@ +/* Copyright (c) 2023, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_PLAT_DMA_INLINES_H_ +#define ODP_PLAT_DMA_INLINES_H_ + +#include <odp/api/buffer.h> +#include <odp/api/dma_types.h> +#include <odp/api/event_types.h> +#include <odp/api/hints.h> +#include <odp/api/pool_types.h> +#include <odp/api/queue_types.h> + +#include <odp/api/plat/debug_inlines.h> +#include <odp/api/plat/event_inline_types.h> + +#include <stdint.h> +#include <string.h> + +/** @cond _ODP_HIDE_FROM_DOXYGEN_ */ + +#ifndef _ODP_NO_INLINE + /* Inline functions by default */ + #define _ODP_INLINE static inline + #define odp_dma_compl_from_event __odp_dma_compl_from_event + #define odp_dma_compl_to_event __odp_dma_compl_to_event + #define odp_dma_compl_user_area __odp_dma_compl_user_area + #define odp_dma_compl_result __odp_dma_compl_result + #define odp_dma_transfer_param_init __odp_dma_transfer_param_init + #define odp_dma_compl_param_init __odp_dma_compl_param_init + #define odp_dma_compl_alloc __odp_dma_compl_alloc + #define odp_dma_compl_free __odp_dma_compl_free +#else + #define _ODP_INLINE +#endif + +_ODP_INLINE odp_dma_compl_t odp_dma_compl_from_event(odp_event_t ev) +{ + _ODP_ASSERT(_odp_event_hdr_field(ev, int8_t, event_type) == ODP_EVENT_DMA_COMPL); + + return (odp_dma_compl_t)(uintptr_t)ev; +} + +_ODP_INLINE odp_event_t odp_dma_compl_to_event(odp_dma_compl_t dma_compl) +{ + return (odp_event_t)(uintptr_t)dma_compl; +} + +_ODP_INLINE void *odp_dma_compl_user_area(odp_dma_compl_t dma_compl) +{ + return odp_buffer_user_area((odp_buffer_t)(uintptr_t)dma_compl); +} + +_ODP_INLINE int odp_dma_compl_result(odp_dma_compl_t dma_compl, odp_dma_result_t *result_out) +{ + odp_dma_result_t *result; + odp_buffer_t buf = (odp_buffer_t)(uintptr_t)dma_compl; + + if (odp_unlikely(dma_compl == ODP_DMA_COMPL_INVALID)) { + _ODP_ERR("Bad DMA compl handle\n"); + return -1; + } + + result = (odp_dma_result_t *)odp_buffer_addr(buf); + + if (result_out) + *result_out = *result; + + return result->success ? 0 : -1; +} + +_ODP_INLINE void odp_dma_transfer_param_init(odp_dma_transfer_param_t *trs_param) +{ + memset(trs_param, 0, sizeof(odp_dma_transfer_param_t)); + + trs_param->src_format = ODP_DMA_FORMAT_ADDR; + trs_param->dst_format = ODP_DMA_FORMAT_ADDR; + trs_param->num_src = 1; + trs_param->num_dst = 1; +} + +_ODP_INLINE void odp_dma_compl_param_init(odp_dma_compl_param_t *compl_param) +{ + memset(compl_param, 0, sizeof(odp_dma_compl_param_t)); + + compl_param->queue = ODP_QUEUE_INVALID; + compl_param->event = ODP_EVENT_INVALID; + compl_param->transfer_id = ODP_DMA_TRANSFER_ID_INVALID; +} + +_ODP_INLINE odp_dma_compl_t odp_dma_compl_alloc(odp_pool_t pool) +{ + odp_buffer_t buf; + odp_event_t ev; + odp_dma_result_t *result; + int8_t *ev_type; + + buf = odp_buffer_alloc(pool); + if (odp_unlikely(buf == ODP_BUFFER_INVALID)) + return ODP_DMA_COMPL_INVALID; + + result = (odp_dma_result_t *)odp_buffer_addr(buf); + memset(result, 0, sizeof(odp_dma_result_t)); + + ev = odp_buffer_to_event(buf); + ev_type = _odp_event_hdr_ptr(ev, int8_t, event_type); + *ev_type = ODP_EVENT_DMA_COMPL; + + return (odp_dma_compl_t)(uintptr_t)buf; +} + +_ODP_INLINE void odp_dma_compl_free(odp_dma_compl_t dma_compl) +{ + int8_t *ev_type; + odp_event_t ev; + odp_buffer_t buf = (odp_buffer_t)(uintptr_t)dma_compl; + + if (odp_unlikely(dma_compl == ODP_DMA_COMPL_INVALID)) { + _ODP_ERR("Bad DMA compl handle\n"); + return; + } + + ev = odp_buffer_to_event(buf); + ev_type = _odp_event_hdr_ptr(ev, int8_t, event_type); + *ev_type = ODP_EVENT_BUFFER; + + odp_buffer_free(buf); +} + +/** @endcond */ + +#endif diff --git a/platform/linux-generic/include/odp/api/plat/event_inlines.h b/platform/linux-generic/include/odp/api/plat/event_inlines.h index 4e3368ff0..2e7c7db5e 100644 --- a/platform/linux-generic/include/odp/api/plat/event_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/event_inlines.h @@ -9,12 +9,12 @@ #define ODP_PLAT_EVENT_INLINES_H_ #include <odp/api/buffer_types.h> -#include <odp/api/dma.h> #include <odp/api/event_types.h> #include <odp/api/packet_types.h> #include <odp/api/timer_types.h> #include <odp/api/plat/buffer_inline_types.h> +#include <odp/api/plat/debug_inlines.h> #include <odp/api/plat/event_inline_types.h> #include <odp/api/plat/event_vector_inline_types.h> #include <odp/api/plat/packet_inline_types.h> @@ -28,8 +28,10 @@ #define odp_event_type __odp_event_type #define odp_event_type_multi __odp_event_type_multi #define odp_event_user_area __odp_event_user_area + #define odp_event_user_area_and_flag __odp_event_user_area_and_flag #define odp_event_subtype __odp_event_subtype #define odp_event_types __odp_event_types + #define odp_event_types_multi __odp_event_types_multi #define odp_event_flow_id __odp_event_flow_id #define odp_event_flow_id_set __odp_event_flow_id_set #else @@ -72,6 +74,7 @@ _ODP_INLINE void *odp_event_user_area(odp_event_t event) switch (type) { case ODP_EVENT_BUFFER: + case ODP_EVENT_DMA_COMPL: return _odp_buffer_get((odp_buffer_t)event, void *, uarea_addr); case ODP_EVENT_PACKET: return _odp_pkt_get((odp_packet_t)event, void *, user_area); @@ -79,9 +82,47 @@ _ODP_INLINE void *odp_event_user_area(odp_event_t event) return _odp_event_vect_get((odp_packet_vector_t)event, void *, uarea_addr); case ODP_EVENT_TIMEOUT: return _odp_timeout_hdr_field((odp_timeout_t)event, void *, uarea_addr); + default: + return NULL; + } +} + +_ODP_INLINE void *odp_event_user_area_and_flag(odp_event_t event, int *flag) +{ + const odp_event_type_t type = __odp_event_type_get(event); + + _ODP_ASSERT(flag != NULL); + + switch (type) { + case ODP_EVENT_BUFFER: case ODP_EVENT_DMA_COMPL: - return odp_dma_compl_user_area((odp_dma_compl_t)event); + *flag = -1; + return _odp_buffer_get((odp_buffer_t)event, void *, uarea_addr); + case ODP_EVENT_PACKET: + { + _odp_packet_flags_t pkt_flags; + odp_packet_t pkt = (odp_packet_t)event; + + pkt_flags.all_flags = _odp_pkt_get(pkt, uint32_t, flags); + *flag = pkt_flags.user_flag; + + return _odp_pkt_get(pkt, void *, user_area); + } + case ODP_EVENT_PACKET_VECTOR: + { + _odp_event_vector_flags_t pktv_flags; + odp_packet_vector_t pktv = (odp_packet_vector_t)event; + + pktv_flags.all_flags = _odp_event_vect_get(pktv, uint32_t, flags); + *flag = pktv_flags.user_flag; + + return _odp_event_vect_get(pktv, void *, uarea_addr); + } + case ODP_EVENT_TIMEOUT: + *flag = -1; + return _odp_timeout_hdr_field((odp_timeout_t)event, void *, uarea_addr); default: + *flag = -1; return NULL; } } @@ -106,6 +147,22 @@ _ODP_INLINE odp_event_type_t odp_event_types(odp_event_t event, return event_type; } +_ODP_INLINE void odp_event_types_multi(const odp_event_t event[], odp_event_type_t type[], + odp_event_subtype_t subtype[], int num) +{ + for (int i = 0; i < num; i++) + type[i] = __odp_event_type_get(event[i]); + + if (subtype == NULL) + return; + + for (int i = 0; i < num; i++) { + subtype[i] = (type[i] == ODP_EVENT_PACKET) ? + (odp_event_subtype_t)_odp_pkt_get((odp_packet_t)event[i], int8_t, + subtype) : ODP_EVENT_NO_SUBTYPE; + } +} + _ODP_INLINE uint32_t odp_event_flow_id(odp_event_t event) { return _odp_event_hdr_field(event, uint8_t, flow_id); diff --git a/platform/linux-generic/include/odp/api/plat/sync_inlines.h b/platform/linux-generic/include/odp/api/plat/sync_inlines.h index b6a96188c..b3a88b629 100644 --- a/platform/linux-generic/include/odp/api/plat/sync_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/sync_inlines.h @@ -1,7 +1,6 @@ -/* Copyright (c) 2016-2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2016-2018 Linaro Limited + * Copyright (c) 2023 Nokia */ /** @@ -13,6 +12,8 @@ #ifndef ODP_PLAT_SYNC_INLINE_H_ #define ODP_PLAT_SYNC_INLINE_H_ +#include <odp/api/abi/sync_inlines.h> + #ifdef __cplusplus extern "C" { #endif @@ -25,6 +26,9 @@ extern "C" { #define odp_mb_release __odp_mb_release #define odp_mb_acquire __odp_mb_acquire #define odp_mb_full __odp_mb_full + #define odp_mb_sync __odp_mb_sync + #define odp_mb_sync_load __odp_mb_sync_load + #define odp_mb_sync_store __odp_mb_sync_store #else #define _ODP_INLINE #endif @@ -44,6 +48,21 @@ _ODP_INLINE void odp_mb_full(void) __atomic_thread_fence(__ATOMIC_SEQ_CST); } +_ODP_INLINE void odp_mb_sync(void) +{ + _odp_mb_sync(); +} + +_ODP_INLINE void odp_mb_sync_load(void) +{ + _odp_mb_sync_load(); +} + +_ODP_INLINE void odp_mb_sync_store(void) +{ + _odp_mb_sync_store(); +} + /** @endcond */ #ifdef __cplusplus diff --git a/platform/linux-generic/include/odp/api/plat/time_inlines.h b/platform/linux-generic/include/odp/api/plat/time_inlines.h index f8f4bee89..35a35c72e 100644 --- a/platform/linux-generic/include/odp/api/plat/time_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/time_inlines.h @@ -1,5 +1,5 @@ /* Copyright (c) 2018, Linaro Limited - * Copyright (c) 2020-2022, Nokia + * Copyright (c) 2020-2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -12,79 +12,12 @@ #include <odp/api/hints.h> #include <odp/api/time_types.h> -#include <odp/api/abi/cpu_time.h> +#include <odp/api/abi/time_inlines.h> #include <stdint.h> /** @cond _ODP_HIDE_FROM_DOXYGEN_ */ -#define _ODP_TIMESPEC_SIZE 16 -#define _ODP_TIME_GIGA_HZ 1000000000ULL - -typedef struct _odp_time_global_t { - /* Storage space for struct timespec. Posix headers are not included - * here to avoid application exposure. */ - uint8_t timespec[_ODP_TIMESPEC_SIZE] ODP_ALIGNED(_ODP_TIMESPEC_SIZE); - - int use_hw; - uint64_t hw_start; - uint64_t hw_freq_hz; - -} _odp_time_global_t; - -extern _odp_time_global_t _odp_time_glob; - -odp_time_t _odp_timespec_cur(void); - -static inline odp_time_t _odp_time_cur(void) -{ - if (_odp_time_glob.use_hw) { - odp_time_t time; - - time.count = _odp_cpu_global_time() - _odp_time_glob.hw_start; - return time; - } - - return _odp_timespec_cur(); -} - -static inline odp_time_t _odp_time_cur_strict(void) -{ - if (_odp_time_glob.use_hw) { - odp_time_t time; - - time.count = _odp_cpu_global_time_strict() - _odp_time_glob.hw_start; - return time; - } - - return _odp_timespec_cur(); -} - -static inline uint64_t _odp_time_hw_to_ns(odp_time_t time) -{ - uint64_t nsec; - uint64_t freq_hz = _odp_time_glob.hw_freq_hz; - uint64_t count = time.count; - uint64_t sec = 0; - - if (count >= freq_hz) { - sec = count / freq_hz; - count = count - sec * freq_hz; - } - - nsec = (_ODP_TIME_GIGA_HZ * count) / freq_hz; - - return (sec * _ODP_TIME_GIGA_HZ) + nsec; -} - -static inline uint64_t _odp_time_convert_to_ns(odp_time_t time) -{ - if (_odp_time_glob.use_hw) - return _odp_time_hw_to_ns(time); - - return time.nsec; -} - #ifndef _ODP_NO_INLINE /* Inline functions by default */ #define _ODP_INLINE static inline @@ -104,6 +37,14 @@ static inline uint64_t _odp_time_convert_to_ns(odp_time_t time) #define odp_time_diff_ns __odp_time_diff_ns #define odp_time_sum __odp_time_sum + #define odp_time_local_from_ns __odp_time_local_from_ns + #define odp_time_global_from_ns __odp_time_global_from_ns + + #define odp_time_local_res __odp_time_local_res + #define odp_time_global_res __odp_time_global_res + + #define odp_time_wait_ns __odp_time_wait_ns + #define odp_time_wait_until __odp_time_wait_until #else #define _ODP_INLINE #endif @@ -130,27 +71,27 @@ _ODP_INLINE odp_time_t odp_time_global_strict(void) _ODP_INLINE uint64_t odp_time_local_ns(void) { - return _odp_time_convert_to_ns(_odp_time_cur()); + return _odp_time_to_ns(_odp_time_cur()); } _ODP_INLINE uint64_t odp_time_global_ns(void) { - return _odp_time_convert_to_ns(_odp_time_cur()); + return _odp_time_to_ns(_odp_time_cur()); } _ODP_INLINE uint64_t odp_time_local_strict_ns(void) { - return _odp_time_convert_to_ns(_odp_time_cur_strict()); + return _odp_time_to_ns(_odp_time_cur_strict()); } _ODP_INLINE uint64_t odp_time_global_strict_ns(void) { - return _odp_time_convert_to_ns(_odp_time_cur_strict()); + return _odp_time_to_ns(_odp_time_cur_strict()); } _ODP_INLINE uint64_t odp_time_to_ns(odp_time_t time) { - return _odp_time_convert_to_ns(time); + return _odp_time_to_ns(time); } _ODP_INLINE int odp_time_cmp(odp_time_t t2, odp_time_t t1) @@ -191,6 +132,44 @@ _ODP_INLINE odp_time_t odp_time_sum(odp_time_t t1, odp_time_t t2) return time; } +_ODP_INLINE odp_time_t odp_time_local_from_ns(uint64_t ns) +{ + return _odp_time_from_ns(ns); +} + +_ODP_INLINE odp_time_t odp_time_global_from_ns(uint64_t ns) +{ + return _odp_time_from_ns(ns); +} + +_ODP_INLINE uint64_t odp_time_local_res(void) +{ + return _odp_time_res(); +} + +_ODP_INLINE uint64_t odp_time_global_res(void) +{ + return _odp_time_res(); +} + +_ODP_INLINE void odp_time_wait_until(odp_time_t time) +{ + odp_time_t cur; + + do { + cur = _odp_time_cur(); + } while (odp_time_cmp(time, cur) > 0); +} + +_ODP_INLINE void odp_time_wait_ns(uint64_t ns) +{ + odp_time_t cur = _odp_time_cur(); + odp_time_t wait = _odp_time_from_ns(ns); + odp_time_t end_time = odp_time_sum(cur, wait); + + odp_time_wait_until(end_time); +} + /** @endcond */ #endif diff --git a/platform/linux-generic/include/odp/api/plat/timer_inlines.h b/platform/linux-generic/include/odp/api/plat/timer_inlines.h index 648459c78..9ba0287e0 100644 --- a/platform/linux-generic/include/odp/api/plat/timer_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/timer_inlines.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2022, Nokia +/* Copyright (c) 2022-2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -27,6 +27,7 @@ #define odp_timer_tick_to_ns __odp_timer_tick_to_ns #define odp_timer_ns_to_tick __odp_timer_ns_to_tick #define odp_timeout_from_event __odp_timeout_from_event + #define odp_timeout_from_event_multi __odp_timeout_from_event_multi #define odp_timeout_to_event __odp_timeout_to_event #else #define _ODP_INLINE @@ -75,6 +76,15 @@ _ODP_INLINE odp_timeout_t odp_timeout_from_event(odp_event_t ev) return (odp_timeout_t)ev; } +_ODP_INLINE void odp_timeout_from_event_multi(odp_timeout_t tmo[], const odp_event_t ev[], int num) +{ + for (int i = 0; i < num; i++) { + _ODP_ASSERT(odp_event_type(ev[i]) == ODP_EVENT_TIMEOUT); + + tmo[i] = odp_timeout_from_event(ev[i]); + } +} + _ODP_INLINE odp_event_t odp_timeout_to_event(odp_timeout_t tmo) { return (odp_event_t)tmo; diff --git a/platform/linux-generic/include/odp_debug_internal.h b/platform/linux-generic/include/odp_debug_internal.h index 4c1fa8bbb..d1fc0d0ba 100644 --- a/platform/linux-generic/include/odp_debug_internal.h +++ b/platform/linux-generic/include/odp_debug_internal.h @@ -55,7 +55,7 @@ extern "C" { do { \ if (ODP_DEBUG_PRINT == 1 && CONFIG_DEBUG_LEVEL >= (level)) \ __extension__ ({ \ - _ODP_LOG(ODP_LOG_DBG, ##__VA_ARGS__); \ + _ODP_LOG(ODP_LOG_DBG, "DBG", ##__VA_ARGS__); \ }); \ } while (0) diff --git a/platform/linux-generic/include/odp_global_data.h b/platform/linux-generic/include/odp_global_data.h index d4cc9cda4..67b7572ef 100644 --- a/platform/linux-generic/include/odp_global_data.h +++ b/platform/linux-generic/include/odp_global_data.h @@ -1,4 +1,5 @@ /* Copyright (c) 2013-2018, Linaro Limited + * Copyright (c) 2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -34,7 +35,8 @@ typedef struct { uint64_t default_cpu_hz; uint64_t page_size; int cache_line_size; - odp_bool_t cpu_hz_static; + uint8_t cpu_hz_static; + uint8_t cpu_constant_tsc; odp_cpu_arch_t cpu_arch; odp_cpu_arch_isa_t cpu_isa_sw; odp_cpu_arch_isa_t cpu_isa_hw; diff --git a/platform/linux-generic/include/odp_sysinfo_internal.h b/platform/linux-generic/include/odp_sysinfo_internal.h index 0fef1aa32..c14cf78d9 100644 --- a/platform/linux-generic/include/odp_sysinfo_internal.h +++ b/platform/linux-generic/include/odp_sysinfo_internal.h @@ -27,9 +27,9 @@ static inline int _odp_dummy_cpuinfo(system_info_t *sysinfo) sysinfo->cpu_arch = ODP_CPU_ARCH_UNKNOWN; - _ODP_DBG("Warning: use dummy values for freq and model string\n"); + _ODP_WARN("Use dummy values for freq and model string\n"); for (i = 0; i < CONFIG_NUM_CPU_IDS; i++) { - _ODP_PRINT("WARN: cpu[%i] uses default max frequency of " + _ODP_WARN("CPU[%i] uses default max frequency of " "%" PRIu64 " Hz from config file\n", i, cpu_hz_max); sysinfo->cpu_hz_max[i] = cpu_hz_max; strcpy(sysinfo->model_str[i], "UNKNOWN"); diff --git a/platform/linux-generic/odp_crypto_openssl.c b/platform/linux-generic/odp_crypto_openssl.c index d4c7a3f1a..26c9ce7b7 100644 --- a/platform/linux-generic/odp_crypto_openssl.c +++ b/platform/linux-generic/odp_crypto_openssl.c @@ -55,7 +55,6 @@ * Keep sorted: first by key length, then by IV length */ static const odp_crypto_cipher_capability_t cipher_capa_null[] = { -{.key_len = 0, .iv_len = 0}, {.key_len = 0, .iv_len = 0, .bit_mode = 1} }; static const odp_crypto_cipher_capability_t cipher_capa_trides_cbc[] = { @@ -115,9 +114,7 @@ static const odp_crypto_cipher_capability_t cipher_capa_aes_eea2[] = { * Keep sorted: first by digest length, then by key length */ static const odp_crypto_auth_capability_t auth_capa_null[] = { -{.digest_len = 0, .key_len = 0, .aad_len = {.min = 0, .max = 0, .inc = 0}, - .bit_mode = 1}, -{.digest_len = 0, .key_len = 0, .aad_len = {.min = 0, .max = 0, .inc = 0} } }; +{.digest_len = 0, .key_len = 0, .aad_len = {.min = 0, .max = 0, .inc = 0}, .bit_mode = 1} }; static const odp_crypto_auth_capability_t auth_capa_md5_hmac[] = { {.digest_len = 12, .key_len = 16, .aad_len = {.min = 0, .max = 0, .inc = 0} }, @@ -217,8 +214,8 @@ struct odp_crypto_generic_session_t { odp_crypto_session_param_t p; odp_bool_t do_cipher_first; - uint8_t cipher_bit_mode : 1; - uint8_t cipher_range_used : 1; + uint8_t cipher_range_in_bits : 1; + uint8_t auth_range_in_bits : 1; uint8_t auth_range_used : 1; struct { @@ -1122,70 +1119,72 @@ static int process_cipher_param(odp_crypto_generic_session_t *session, return 0; } -static -odp_crypto_alg_err_t cipher_encrypt_bits(odp_packet_t pkt, - const odp_crypto_packet_op_param_t - *param, - odp_crypto_generic_session_t *session) +static odp_crypto_alg_err_t cipher_encrypt_bytes(odp_packet_t pkt, + const odp_crypto_packet_op_param_t *param, + odp_crypto_generic_session_t *session) { EVP_CIPHER_CTX *ctx = local.cipher_ctx[session->idx]; int dummy_len = 0; int cipher_len; - uint32_t in_len = (param->cipher_range.length + 7) / 8; + uint32_t in_len = param->cipher_range.length; + uint32_t offset = param->cipher_range.offset; uint8_t data[in_len]; int ret; - uint32_t offset; - - /* Range offset is in bits in bit mode but must be divisible by 8. */ - offset = param->cipher_range.offset / 8; EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, param->cipher_iv_ptr); - odp_packet_copy_to_mem(pkt, offset, in_len, data); - EVP_EncryptUpdate(ctx, data, &cipher_len, data, in_len); - ret = EVP_EncryptFinal_ex(ctx, data + cipher_len, &dummy_len); cipher_len += dummy_len; - odp_packet_copy_from_mem(pkt, offset, in_len, data); - return ret <= 0 ? ODP_CRYPTO_ALG_ERR_DATA_SIZE : ODP_CRYPTO_ALG_ERR_NONE; } -static -odp_crypto_alg_err_t cipher_decrypt_bits(odp_packet_t pkt, - const odp_crypto_packet_op_param_t - *param, - odp_crypto_generic_session_t *session) +static odp_crypto_alg_err_t cipher_decrypt_bytes(odp_packet_t pkt, + const odp_crypto_packet_op_param_t *param, + odp_crypto_generic_session_t *session) { EVP_CIPHER_CTX *ctx = local.cipher_ctx[session->idx]; int dummy_len = 0; int cipher_len; - uint32_t in_len = (param->cipher_range.length + 7) / 8; + uint32_t in_len = param->cipher_range.length; + uint32_t offset = param->cipher_range.offset; uint8_t data[in_len]; int ret; - uint32_t offset; - - /* Range offset is in bits in bit mode but must be divisible by 8. */ - offset = param->cipher_range.offset / 8; EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, param->cipher_iv_ptr); - odp_packet_copy_to_mem(pkt, offset, in_len, data); - EVP_DecryptUpdate(ctx, data, &cipher_len, data, in_len); - ret = EVP_DecryptFinal_ex(ctx, data + cipher_len, &dummy_len); cipher_len += dummy_len; - odp_packet_copy_from_mem(pkt, offset, in_len, data); - return ret <= 0 ? ODP_CRYPTO_ALG_ERR_DATA_SIZE : ODP_CRYPTO_ALG_ERR_NONE; } +static odp_crypto_alg_err_t cipher_encrypt_bits(odp_packet_t pkt, + const odp_crypto_packet_op_param_t *param, + odp_crypto_generic_session_t *session) +{ + odp_crypto_packet_op_param_t new_param = *param; + + new_param.cipher_range.offset /= 8; + new_param.cipher_range.length = (new_param.cipher_range.length + 7) / 8; + return cipher_encrypt_bytes(pkt, &new_param, session); +} + +static odp_crypto_alg_err_t cipher_decrypt_bits(odp_packet_t pkt, + const odp_crypto_packet_op_param_t *param, + odp_crypto_generic_session_t *session) +{ + odp_crypto_packet_op_param_t new_param = *param; + + new_param.cipher_range.offset /= 8; + new_param.cipher_range.length = (new_param.cipher_range.length + 7) / 8; + return cipher_decrypt_bytes(pkt, &new_param, session); +} + static int process_cipher_param_bits(odp_crypto_generic_session_t *session, const EVP_CIPHER *cipher) { @@ -1199,7 +1198,6 @@ static int process_cipher_param_bits(odp_crypto_generic_session_t *session, session->p.cipher_iv_len) return -1; - session->cipher_bit_mode = 1; session->cipher.evp_cipher = cipher; memcpy(session->cipher.key_data, session->p.cipher_key.data, @@ -1207,11 +1205,14 @@ static int process_cipher_param_bits(odp_crypto_generic_session_t *session, /* Set function */ if (ODP_CRYPTO_OP_ENCODE == session->p.op) { - session->cipher.func = cipher_encrypt_bits; session->cipher.init = cipher_encrypt_init; + session->cipher.func = session->cipher_range_in_bits ? cipher_encrypt_bits + : cipher_encrypt_bytes; + } else { - session->cipher.func = cipher_decrypt_bits; session->cipher.init = cipher_decrypt_init; + session->cipher.func = session->cipher_range_in_bits ? cipher_decrypt_bits + : cipher_decrypt_bytes; } return 0; @@ -1813,7 +1814,7 @@ int odp_crypto_capability(odp_crypto_capability_t *capa) #if _ODP_HAVE_CHACHA20_POLY1305 capa->ciphers.bit.chacha20_poly1305 = 1; #endif - capa->ciphers.bit.aes_eea2 = 1; + capa->ciphers.bit.aes_eea2 = 1; capa->auths.bit.null = 1; capa->auths.bit.md5_hmac = 1; @@ -2022,6 +2023,8 @@ odp_crypto_session_create(const odp_crypto_session_param_t *param, { int rc; odp_crypto_generic_session_t *session; + int cipher_bit_mode_supported = 0; + int auth_bit_mode_supported = 0; if (odp_global_ro.disable.crypto) { _ODP_ERR("Crypto is disabled\n"); @@ -2052,9 +2055,9 @@ odp_crypto_session_create(const odp_crypto_session_param_t *param, /* Copy parameters */ session->p = *param; - session->cipher_bit_mode = 0; + session->cipher_range_in_bits = !!param->cipher_range_in_bits; + session->auth_range_in_bits = !!param->auth_range_in_bits; session->auth_range_used = 1; - session->cipher_range_used = 1; if (session->p.cipher_iv_len > EVP_MAX_IV_LENGTH) { _ODP_DBG("Maximum IV length exceeded\n"); @@ -2079,7 +2082,7 @@ odp_crypto_session_create(const odp_crypto_session_param_t *param, case ODP_CIPHER_ALG_NULL: session->cipher.func = null_crypto_routine; session->cipher.init = null_crypto_init_routine; - session->cipher_range_used = 0; + cipher_bit_mode_supported = 1; rc = 0; break; case ODP_CIPHER_ALG_3DES_CBC: @@ -2184,11 +2187,15 @@ odp_crypto_session_create(const odp_crypto_session_param_t *param, EVP_aes_128_ctr()); else rc = -1; + cipher_bit_mode_supported = 1; break; default: rc = -1; } + if (session->cipher_range_in_bits && !cipher_bit_mode_supported) + rc = -1; + /* Check result */ if (rc) { *status = ODP_CRYPTO_SES_ERR_CIPHER; @@ -2200,7 +2207,7 @@ odp_crypto_session_create(const odp_crypto_session_param_t *param, case ODP_AUTH_ALG_NULL: session->auth.func = null_crypto_routine; session->auth.init = null_crypto_init_routine; - session->auth_range_used = 0; + auth_bit_mode_supported = 1; rc = 0; break; case ODP_AUTH_ALG_MD5_HMAC: @@ -2314,6 +2321,9 @@ odp_crypto_session_create(const odp_crypto_session_param_t *param, rc = -1; } + if (session->auth_range_in_bits && !auth_bit_mode_supported) + rc = -1; + /* Check result */ if (rc) { *status = ODP_CRYPTO_SES_ERR_AUTH; @@ -2632,12 +2642,16 @@ static void copy_ranges(odp_packet_t dst, int32_t shift = param->dst_offset_shift; int rc; - if (session->cipher_bit_mode) { + if (session->cipher_range_in_bits) { c_range.offset /= 8; c_range.length = (c_range.length + 7) / 8; } + if (session->auth_range_in_bits) { + a_range.offset /= 8; + a_range.length = (a_range.length + 7) / 8; + } - if (session->cipher_range_used) { + if (c_range.length > 0) { rc = odp_packet_copy_from_pkt(dst, c_range.offset + shift, src, c_range.offset, c_range.length); @@ -2646,7 +2660,7 @@ static void copy_ranges(odp_packet_t dst, return; } } - if (session->auth_range_used) { + if (session->auth_range_used && a_range.length > 0) { rc = odp_packet_copy_from_pkt(dst, a_range.offset + shift, src, a_range.offset, a_range.length); @@ -2663,12 +2677,13 @@ static int crypto_int_oop_encode(odp_packet_t pkt_in, const odp_crypto_packet_op_param_t *param) { odp_crypto_packet_op_param_t new_param = *param; - const uint32_t scale = session->cipher_bit_mode ? 8 : 1; + const uint32_t c_scale = session->cipher_range_in_bits ? 8 : 1; + const uint32_t a_scale = session->auth_range_in_bits ? 8 : 1; copy_ranges(*pkt_out, pkt_in, session, param); - new_param.cipher_range.offset += param->dst_offset_shift * scale; - new_param.auth_range.offset += param->dst_offset_shift; + new_param.cipher_range.offset += param->dst_offset_shift * c_scale; + new_param.auth_range.offset += param->dst_offset_shift * a_scale; return crypto_int(*pkt_out, pkt_out, &new_param); } diff --git a/platform/linux-generic/odp_dma.c b/platform/linux-generic/odp_dma.c index fa45b3b02..8ac18260c 100644 --- a/platform/linux-generic/odp_dma.c +++ b/platform/linux-generic/odp_dma.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2021-2022, Nokia +/* Copyright (c) 2021-2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -99,11 +99,12 @@ int odp_dma_capability(odp_dma_capability_t *capa) capa->queue_type_sched = 1; capa->queue_type_plain = 1; - capa->pool.max_pools = _odp_dma_glb->pool_capa.buf.max_pools; - capa->pool.max_num = _odp_dma_glb->pool_capa.buf.max_num; - capa->pool.max_uarea_size = _odp_dma_glb->pool_capa.buf.max_uarea_size; - capa->pool.min_cache_size = _odp_dma_glb->pool_capa.buf.min_cache_size; - capa->pool.max_cache_size = _odp_dma_glb->pool_capa.buf.max_cache_size; + capa->pool.max_pools = _odp_dma_glb->pool_capa.buf.max_pools; + capa->pool.max_num = _odp_dma_glb->pool_capa.buf.max_num; + capa->pool.max_uarea_size = _odp_dma_glb->pool_capa.buf.max_uarea_size; + capa->pool.uarea_persistence = _odp_dma_glb->pool_capa.buf.uarea_persistence; + capa->pool.min_cache_size = _odp_dma_glb->pool_capa.buf.min_cache_size; + capa->pool.max_cache_size = _odp_dma_glb->pool_capa.buf.max_cache_size; return 0; } @@ -321,16 +322,6 @@ odp_dma_t odp_dma_lookup(const char *name) return ODP_DMA_INVALID; } -void odp_dma_transfer_param_init(odp_dma_transfer_param_t *trs_param) -{ - memset(trs_param, 0, sizeof(odp_dma_transfer_param_t)); - - trs_param->src_format = ODP_DMA_FORMAT_ADDR; - trs_param->dst_format = ODP_DMA_FORMAT_ADDR; - trs_param->num_src = 1; - trs_param->num_dst = 1; -} - static uint32_t transfer_len(const odp_dma_transfer_param_t *trs_param) { uint32_t i; @@ -568,14 +559,6 @@ int odp_dma_transfer_multi(odp_dma_t dma, const odp_dma_transfer_param_t *trs_pa return i; } -void odp_dma_compl_param_init(odp_dma_compl_param_t *compl_param) -{ - memset(compl_param, 0, sizeof(odp_dma_compl_param_t)); - compl_param->queue = ODP_QUEUE_INVALID; - compl_param->event = ODP_EVENT_INVALID; - compl_param->transfer_id = ODP_DMA_TRANSFER_ID_INVALID; -} - odp_dma_transfer_id_t odp_dma_transfer_id_alloc(odp_dma_t dma) { int32_t num; @@ -756,83 +739,19 @@ odp_pool_t odp_dma_pool_create(const char *name, const odp_dma_pool_param_t *dma } odp_pool_param_init(&pool_param); - pool_param.type = ODP_POOL_BUFFER; - pool_param.buf.num = num; - pool_param.buf.uarea_size = uarea_size; - pool_param.buf.cache_size = cache_size; - pool_param.buf.size = sizeof(odp_dma_result_t); + pool_param.type = ODP_POOL_BUFFER; + pool_param.uarea_init.init_fn = dma_pool_param->uarea_init.init_fn; + pool_param.uarea_init.args = dma_pool_param->uarea_init.args; + pool_param.buf.num = num; + pool_param.buf.uarea_size = uarea_size; + pool_param.buf.cache_size = cache_size; + pool_param.buf.size = sizeof(odp_dma_result_t); pool = _odp_pool_create(name, &pool_param, ODP_POOL_DMA_COMPL); return pool; } -odp_dma_compl_t odp_dma_compl_alloc(odp_pool_t pool) -{ - odp_buffer_t buf; - odp_event_t ev; - odp_dma_result_t *result; - - buf = odp_buffer_alloc(pool); - - if (odp_unlikely(buf == ODP_BUFFER_INVALID)) - return ODP_DMA_COMPL_INVALID; - - result = odp_buffer_addr(buf); - memset(result, 0, sizeof(odp_dma_result_t)); - - ev = odp_buffer_to_event(buf); - _odp_event_type_set(ev, ODP_EVENT_DMA_COMPL); - - return (odp_dma_compl_t)(uintptr_t)buf; -} - -void odp_dma_compl_free(odp_dma_compl_t dma_compl) -{ - odp_event_t ev; - odp_buffer_t buf = (odp_buffer_t)(uintptr_t)dma_compl; - - if (odp_unlikely(dma_compl == ODP_DMA_COMPL_INVALID)) { - _ODP_ERR("Bad DMA compl handle\n"); - return; - } - - ev = odp_buffer_to_event(buf); - _odp_event_type_set(ev, ODP_EVENT_BUFFER); - - odp_buffer_free(buf); -} - -odp_dma_compl_t odp_dma_compl_from_event(odp_event_t ev) -{ - _ODP_ASSERT(odp_event_type(ev) == ODP_EVENT_DMA_COMPL); - - return (odp_dma_compl_t)(uintptr_t)ev; -} - -odp_event_t odp_dma_compl_to_event(odp_dma_compl_t dma_compl) -{ - return (odp_event_t)(uintptr_t)dma_compl; -} - -int odp_dma_compl_result(odp_dma_compl_t dma_compl, odp_dma_result_t *result_out) -{ - odp_dma_result_t *result; - odp_buffer_t buf = (odp_buffer_t)(uintptr_t)dma_compl; - - if (odp_unlikely(dma_compl == ODP_DMA_COMPL_INVALID)) { - _ODP_ERR("Bad DMA compl handle\n"); - return -1; - } - - result = odp_buffer_addr(buf); - - if (result_out) - *result_out = *result; - - return result->success ? 0 : -1; -} - uint64_t odp_dma_to_u64(odp_dma_t dma) { return (uint64_t)(uintptr_t)dma; @@ -843,11 +762,6 @@ uint64_t odp_dma_compl_to_u64(odp_dma_compl_t dma_compl) return (uint64_t)(uintptr_t)dma_compl; } -void *odp_dma_compl_user_area(odp_dma_compl_t dma_compl) -{ - return odp_buffer_user_area((odp_buffer_t)(uintptr_t)dma_compl); -} - void odp_dma_print(odp_dma_t dma) { dma_session_t *session = dma_session_from_handle(dma); diff --git a/platform/linux-generic/odp_dma_api.c b/platform/linux-generic/odp_dma_api.c new file mode 100644 index 000000000..1e1d5d91f --- /dev/null +++ b/platform/linux-generic/odp_dma_api.c @@ -0,0 +1,11 @@ +/* Copyright (c) 2023, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <odp/api/dma.h> + +/* Non-inlined functions for ABI compat mode */ +#define _ODP_NO_INLINE +#include <odp/api/plat/dma_inlines.h> diff --git a/platform/linux-generic/odp_init.c b/platform/linux-generic/odp_init.c index f1aae95bb..e6ea8bc0c 100644 --- a/platform/linux-generic/odp_init.c +++ b/platform/linux-generic/odp_init.c @@ -27,8 +27,8 @@ enum init_stage { LIBCONFIG_INIT, CPUMASK_INIT, CPU_CYCLES_INIT, - TIME_INIT, SYSINFO_INIT, + TIME_INIT, ISHM_INIT, FDSERVER_INIT, GLOBAL_RW_DATA_INIT, @@ -292,16 +292,16 @@ static int term_global(enum init_stage stage) } /* Fall through */ - case SYSINFO_INIT: - if (_odp_system_info_term()) { - _ODP_ERR("ODP system info term failed.\n"); + case TIME_INIT: + if (_odp_time_term_global()) { + _ODP_ERR("ODP time term failed.\n"); rc = -1; } /* Fall through */ - case TIME_INIT: - if (_odp_time_term_global()) { - _ODP_ERR("ODP time term failed.\n"); + case SYSINFO_INIT: + if (_odp_system_info_term()) { + _ODP_ERR("ODP system info term failed.\n"); rc = -1; } /* Fall through */ @@ -372,18 +372,18 @@ int odp_init_global(odp_instance_t *instance, } stage = CPU_CYCLES_INIT; - if (_odp_time_init_global()) { - _ODP_ERR("ODP time init failed.\n"); - goto init_failed; - } - stage = TIME_INIT; - if (_odp_system_info_init()) { _ODP_ERR("ODP system_info init failed.\n"); goto init_failed; } stage = SYSINFO_INIT; + if (_odp_time_init_global()) { + _ODP_ERR("ODP time init failed.\n"); + goto init_failed; + } + stage = TIME_INIT; + if (_odp_ishm_init_global(params)) { _ODP_ERR("ODP ishm init failed.\n"); goto init_failed; diff --git a/platform/linux-generic/odp_ishm.c b/platform/linux-generic/odp_ishm.c index 926e20a8a..3ef1894bc 100644 --- a/platform/linux-generic/odp_ishm.c +++ b/platform/linux-generic/odp_ishm.c @@ -1160,9 +1160,8 @@ int _odp_ishm_reserve(const char *name, uint64_t size, int fd, if (addr == NULL) { if (!huge_error_printed) { - _ODP_ERR("No huge pages, fall back to " - "normal pages. Check: " - "/proc/sys/vm/nr_hugepages.\n"); + _ODP_WARN("No huge pages, fall back to normal pages. " + "Check: /proc/sys/vm/nr_hugepages.\n"); huge_error_printed = 1; } } else { @@ -1274,8 +1273,8 @@ static void *reserve_single_va(uint64_t size, int *fd_out) } } if (fd < 0) - _ODP_ERR("No huge pages, fall back to normal pages. " - "Check: /proc/sys/vm/nr_hugepages.\n"); + _ODP_WARN("No huge pages, fall back to normal pages. " + "Check: /proc/sys/vm/nr_hugepages.\n"); ishm_tbl->single_va_huge = true; } diff --git a/platform/linux-generic/odp_pool.c b/platform/linux-generic/odp_pool.c index 672b92c02..e7b2398de 100644 --- a/platform/linux-generic/odp_pool.c +++ b/platform/linux-generic/odp_pool.c @@ -604,6 +604,14 @@ static void init_buffers(pool_t *pool) ring_ptr_enq(ring, mask, event_hdr); } pool->skipped_blocks = skipped_blocks; + + if (pool->uarea_size && pool->params.uarea_init.init_fn) { + for (uint32_t i = 0; i < pool->num; i++) { + uarea = &pool->uarea_base_addr[i * pool->uarea_size]; + pool->params.uarea_init.init_fn(uarea, pool->param_uarea_size, + pool->params.uarea_init.args, i); + } + } } static bool shm_is_from_huge_pages(odp_shm_t shm) @@ -1479,6 +1487,7 @@ int odp_pool_capability(odp_pool_capability_t *capa) capa->buf.max_size = MAX_SIZE; capa->buf.max_num = CONFIG_POOL_MAX_NUM; capa->buf.max_uarea_size = MAX_UAREA_SIZE; + capa->buf.uarea_persistence = true; capa->buf.min_cache_size = 0; capa->buf.max_cache_size = CONFIG_POOL_CACHE_MAX_SIZE; capa->buf.stats.all = supported_stats.all; @@ -1495,6 +1504,7 @@ int odp_pool_capability(odp_pool_capability_t *capa) capa->pkt.min_seg_len = CONFIG_PACKET_SEG_LEN_MIN; capa->pkt.max_seg_len = max_seg_len; capa->pkt.max_uarea_size = MAX_UAREA_SIZE; + capa->pkt.uarea_persistence = true; capa->pkt.min_cache_size = 0; capa->pkt.max_cache_size = CONFIG_POOL_CACHE_MAX_SIZE; capa->pkt.stats.all = supported_stats.all; @@ -1503,6 +1513,7 @@ int odp_pool_capability(odp_pool_capability_t *capa) capa->tmo.max_pools = max_pools; capa->tmo.max_num = CONFIG_POOL_MAX_NUM; capa->tmo.max_uarea_size = MAX_UAREA_SIZE; + capa->tmo.uarea_persistence = true; capa->tmo.min_cache_size = 0; capa->tmo.max_cache_size = CONFIG_POOL_CACHE_MAX_SIZE; capa->tmo.stats.all = supported_stats.all; @@ -1512,6 +1523,7 @@ int odp_pool_capability(odp_pool_capability_t *capa) capa->vector.max_num = CONFIG_POOL_MAX_NUM; capa->vector.max_size = CONFIG_PACKET_VECTOR_MAX_SIZE; capa->vector.max_uarea_size = MAX_UAREA_SIZE; + capa->vector.uarea_persistence = true; capa->vector.min_cache_size = 0; capa->vector.max_cache_size = CONFIG_POOL_CACHE_MAX_SIZE; capa->vector.stats.all = supported_stats.all; @@ -1869,6 +1881,7 @@ int odp_pool_ext_capability(odp_pool_type_t type, odp_pool_ext_capability_t *cap capa->pkt.max_headroom_size = CONFIG_PACKET_HEADROOM; capa->pkt.max_segs_per_pkt = PKT_MAX_SEGS; capa->pkt.max_uarea_size = MAX_UAREA_SIZE; + capa->pkt.uarea_persistence = true; return 0; } @@ -2096,8 +2109,18 @@ int odp_pool_ext_populate(odp_pool_t pool_hdl, void *buf[], uint32_t buf_size, u pool->base_addr = min_addr; pool->max_addr = max_addr; - if (flags & ODP_POOL_POPULATE_DONE) + if (flags & ODP_POOL_POPULATE_DONE) { pool->max_addr = max_addr + buf_size - 1; + if (pool->uarea_size && pool->ext_param.uarea_init.init_fn) { + for (i = 0; i < pool->num_populated; i++) { + uarea = &pool->uarea_base_addr[i * pool->uarea_size]; + pool->ext_param.uarea_init.init_fn(uarea, pool->param_uarea_size, + pool->ext_param.uarea_init.args, + i); + } + } + } + return 0; } diff --git a/platform/linux-generic/odp_system_info.c b/platform/linux-generic/odp_system_info.c index 4f1d92357..7864a3d17 100644 --- a/platform/linux-generic/odp_system_info.c +++ b/platform/linux-generic/odp_system_info.c @@ -55,7 +55,7 @@ static int read_cache_line_size(void) file = fopen(CACHE_LNSZ_FILE, "rt"); if (file == NULL) { /* File not found */ - _ODP_PRINT("WARN: unable to read host CPU cache line size. " + _ODP_WARN("Unable to read host CPU cache line size. " "Using ODP_CACHE_LINE_SIZE instead.\n"); return ODP_CACHE_LINE_SIZE; } @@ -299,7 +299,7 @@ static int system_cache_line(system_info_t *sysinfo) sysinfo->cache_line_size = ret; if (ret != ODP_CACHE_LINE_SIZE) - _ODP_PRINT("WARN: host CPU cache line size and ODP_CACHE_LINE_SIZE don't match.\n"); + _ODP_WARN("Host CPU cache line size and ODP_CACHE_LINE_SIZE don't match.\n"); return 0; } diff --git a/platform/linux-generic/odp_time.c b/platform/linux-generic/odp_time.c deleted file mode 100644 index 3498008f1..000000000 --- a/platform/linux-generic/odp_time.c +++ /dev/null @@ -1,215 +0,0 @@ -/* Copyright (c) 2013-2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -#include <odp_posix_extensions.h> - -#include <time.h> -#include <string.h> -#include <inttypes.h> - -#include <odp/api/time.h> -#include <odp/api/hints.h> -#include <odp_debug_internal.h> -#include <odp_init_internal.h> -#include <odp/api/plat/time_inlines.h> - -ODP_STATIC_ASSERT(_ODP_TIMESPEC_SIZE >= (sizeof(struct timespec)), - "_ODP_TIMESPEC_SIZE too small"); - -#include <odp/visibility_begin.h> - -_odp_time_global_t _odp_time_glob; - -/* - * Posix timespec based functions - */ - -static inline uint64_t time_spec_diff_nsec(struct timespec *t2, - struct timespec *t1) -{ - struct timespec diff; - uint64_t nsec; - - diff.tv_sec = t2->tv_sec - t1->tv_sec; - diff.tv_nsec = t2->tv_nsec - t1->tv_nsec; - - if (diff.tv_nsec < 0) { - diff.tv_nsec += ODP_TIME_SEC_IN_NS; - diff.tv_sec -= 1; - } - - nsec = (diff.tv_sec * ODP_TIME_SEC_IN_NS) + diff.tv_nsec; - - return nsec; -} - -odp_time_t _odp_timespec_cur(void) -{ - int ret; - odp_time_t time; - struct timespec sys_time; - struct timespec *start_time; - - start_time = (struct timespec *)(uintptr_t)&_odp_time_glob.timespec; - - ret = clock_gettime(CLOCK_MONOTONIC_RAW, &sys_time); - if (odp_unlikely(ret != 0)) - _ODP_ABORT("clock_gettime failed\n"); - - time.nsec = time_spec_diff_nsec(&sys_time, start_time); - - return time; -} - -#include <odp/visibility_end.h> - -static inline uint64_t time_spec_res(void) -{ - int ret; - struct timespec tres; - - ret = clock_getres(CLOCK_MONOTONIC_RAW, &tres); - if (odp_unlikely(ret != 0)) - _ODP_ABORT("clock_getres failed\n"); - - return ODP_TIME_SEC_IN_NS / (uint64_t)tres.tv_nsec; -} - -static inline odp_time_t time_spec_from_ns(uint64_t ns) -{ - odp_time_t time; - - time.nsec = ns; - - return time; -} - -/* - * HW time counter based functions - */ - -static inline uint64_t time_hw_res(void) -{ - return _odp_time_glob.hw_freq_hz; -} - -static inline odp_time_t time_hw_from_ns(uint64_t ns) -{ - odp_time_t time; - uint64_t count; - uint64_t freq_hz = _odp_time_glob.hw_freq_hz; - uint64_t sec = 0; - - if (ns >= ODP_TIME_SEC_IN_NS) { - sec = ns / ODP_TIME_SEC_IN_NS; - ns = ns - sec * ODP_TIME_SEC_IN_NS; - } - - count = sec * freq_hz; - count += (ns * freq_hz) / ODP_TIME_SEC_IN_NS; - - time.count = count; - - return time; -} - -/* - * Common functions - */ - -static inline uint64_t time_res(void) -{ - if (_odp_time_glob.use_hw) - return time_hw_res(); - - return time_spec_res(); -} - -static inline odp_time_t time_from_ns(uint64_t ns) -{ - if (_odp_time_glob.use_hw) - return time_hw_from_ns(ns); - - return time_spec_from_ns(ns); -} - -static inline void time_wait_until(odp_time_t time) -{ - odp_time_t cur; - - do { - cur = _odp_time_cur(); - } while (odp_time_cmp(time, cur) > 0); -} - -odp_time_t odp_time_local_from_ns(uint64_t ns) -{ - return time_from_ns(ns); -} - -odp_time_t odp_time_global_from_ns(uint64_t ns) -{ - return time_from_ns(ns); -} - -uint64_t odp_time_local_res(void) -{ - return time_res(); -} - -uint64_t odp_time_global_res(void) -{ - return time_res(); -} - -void odp_time_wait_ns(uint64_t ns) -{ - odp_time_t cur = _odp_time_cur(); - odp_time_t wait = time_from_ns(ns); - odp_time_t end_time = odp_time_sum(cur, wait); - - time_wait_until(end_time); -} - -void odp_time_wait_until(odp_time_t time) -{ - time_wait_until(time); -} - -int _odp_time_init_global(void) -{ - struct timespec *timespec; - int ret = 0; - _odp_time_global_t *global = &_odp_time_glob; - - memset(global, 0, sizeof(_odp_time_global_t)); - - if (_odp_cpu_has_global_time()) { - global->use_hw = 1; - global->hw_freq_hz = _odp_cpu_global_time_freq(); - - if (global->hw_freq_hz == 0) - return -1; - - _ODP_PRINT("HW time counter freq: %" PRIu64 " hz\n\n", global->hw_freq_hz); - - global->hw_start = _odp_cpu_global_time(); - return 0; - } - - timespec = (struct timespec *)(uintptr_t)global->timespec; - timespec->tv_sec = 0; - timespec->tv_nsec = 0; - - ret = clock_gettime(CLOCK_MONOTONIC_RAW, timespec); - - return ret; -} - -int _odp_time_term_global(void) -{ - return 0; -} diff --git a/platform/linux-generic/odp_weak.c b/platform/linux-generic/odp_weak.c index 9e1c9da6a..747886d4e 100644 --- a/platform/linux-generic/odp_weak.c +++ b/platform/linux-generic/odp_weak.c @@ -20,6 +20,7 @@ int odp_override_log(odp_log_level_t level, const char *fmt, ...) switch (level) { case ODP_LOG_ERR: case ODP_LOG_UNIMPLEMENTED: + case ODP_LOG_WARN: case ODP_LOG_ABORT: logfd = stderr; break; diff --git a/platform/linux-generic/pktio/socket_xdp.c b/platform/linux-generic/pktio/socket_xdp.c index 867483f76..599942657 100644 --- a/platform/linux-generic/pktio/socket_xdp.c +++ b/platform/linux-generic/pktio/socket_xdp.c @@ -309,8 +309,7 @@ static int sock_xdp_open(odp_pktio_t pktio, pktio_entry_t *pktio_entry, const ch if (!get_nic_queue_count(priv->helper_sock, devname, &priv->q_num_conf.drv_channels) || !get_nic_rss_indir_count(priv->helper_sock, devname, &priv->q_num_conf.drv_num_rss)) - _ODP_PRINT("Warning: Unable to query NIC queue count/RSS, manual cleanup" - " required\n"); + _ODP_WARN("Unable to query NIC queue count/RSS, manual cleanup required\n"); priv->is_shadow_q = is_shadow_q_driver(priv->helper_sock, pktio_entry->name); parse_options(priv->umem_info); @@ -543,8 +542,7 @@ static int sock_xdp_start(pktio_entry_t *pktio_entry) channels.combined = priv->q_num_conf.num_qs; if (!set_nic_queue_count(priv->helper_sock, pktio_entry->name, &channels)) - _ODP_PRINT("Warning: Unable to configure NIC queue count, manual configuration" - " required\n"); + _ODP_WARN("Unable to configure NIC queue count, manual configuration required\n"); if (priv->q_num_conf.num_in_conf_qs > 0U && !priv->is_shadow_q) { indir->indir_size = priv->q_num_conf.drv_num_rss; @@ -553,8 +551,7 @@ static int sock_xdp_start(pktio_entry_t *pktio_entry) indir->rss_config[i] = (i % priv->q_num_conf.num_in_conf_qs); if (!set_nic_rss_indir(priv->helper_sock, pktio_entry->name, indir)) - _ODP_PRINT("Warning: Unable to configure NIC RSS, manual configuration" - " required\n"); + _ODP_WARN("Unable to configure NIC RSS, manual configuration required\n"); } if (!create_sockets(priv, pktio_entry->name)) |