diff options
Diffstat (limited to 'platform/linux-dpdk')
21 files changed, 220 insertions, 399 deletions
diff --git a/platform/linux-dpdk/Makefile.am b/platform/linux-dpdk/Makefile.am index 252b34fb2..ad608c4e1 100644 --- a/platform/linux-dpdk/Makefile.am +++ b/platform/linux-dpdk/Makefile.am @@ -33,6 +33,7 @@ odpapiplatinclude_HEADERS = \ include/odp/api/plat/byteorder_inlines.h \ include/odp/api/plat/cpu_inlines.h \ include/odp/api/plat/crypto_inlines.h \ + include/odp/api/plat/dma_inlines.h \ include/odp/api/plat/debug_inlines.h \ include/odp/api/plat/event_inlines.h \ include/odp/api/plat/event_inline_types.h \ @@ -78,6 +79,7 @@ odpapiabiarchinclude_HEADERS += \ include-abi/odp/api/abi/crypto.h \ include-abi/odp/api/abi/crypto_types.h \ include-abi/odp/api/abi/debug.h \ + include-abi/odp/api/abi/dma.h \ include-abi/odp/api/abi/dma_types.h \ include-abi/odp/api/abi/errno.h \ include-abi/odp/api/abi/event.h \ @@ -259,6 +261,7 @@ __LIB__libodp_dpdk_la_SOURCES += \ ../linux-generic/odp_byteorder_api.c \ ../linux-generic/odp_cpu_api.c \ ../linux-generic/odp_crypto_api.c \ + ../linux-generic/odp_dma_api.c \ ../linux-generic/odp_event_api.c \ ../linux-generic/odp_hash_api.c \ ../linux-generic/odp_ipsec_api.c \ @@ -283,18 +286,17 @@ endif if ARCH_IS_ARM __LIB__libodp_dpdk_la_SOURCES += arch/default/odp_atomic.c \ arch/default/odp_cpu_cycles.c \ - arch/default/odp_global_time.c \ arch/default/odp_hash_crc32.c \ arch/default/odp_random.c \ arch/arm/odp_sysinfo_parse.c -odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_time.h \ - arch/default/odp/api/abi/hash_crc32.h +odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/hash_crc32.h if !ODP_ABI_COMPAT odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ arch/default/odp/api/abi/atomic_inlines.h \ arch/default/odp/api/abi/cpu_generic.h \ arch/arm/odp/api/abi/cpu_inlines.h \ - arch/arm/odp/api/abi/cpu.h + arch/arm/odp/api/abi/cpu.h \ + arch/default/odp/api/abi/sync_inlines.h endif noinst_HEADERS += arch/arm/odp_atomic.h \ arch/arm/odp_cpu.h \ @@ -310,19 +312,18 @@ if ARCH_IS_AARCH64 __LIB__libodp_dpdk_la_SOURCES += arch/aarch64/odp_atomic.c \ arch/default/odp_cpu_cycles.c \ arch/aarch64/cpu_flags.c \ - arch/aarch64/odp_global_time.c \ arch/default/odp_hash_crc32.c \ arch/default/odp_random.c \ arch/aarch64/odp_sysinfo_parse.c -odpapiabiarchinclude_HEADERS += arch/aarch64/odp/api/abi/cpu_time.h \ - arch/aarch64/odp/api/abi/hash_crc32.h +odpapiabiarchinclude_HEADERS += arch/aarch64/odp/api/abi/hash_crc32.h if !ODP_ABI_COMPAT odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ arch/aarch64/odp/api/abi/atomic_inlines.h \ arch/aarch64/odp/api/abi/atomic.h \ arch/default/odp/api/abi/cpu_generic.h \ arch/aarch64/odp/api/abi/cpu_inlines.h \ - arch/aarch64/odp/api/abi/cpu.h + arch/aarch64/odp/api/abi/cpu.h \ + arch/aarch64/odp/api/abi/sync_inlines.h endif noinst_HEADERS += arch/aarch64/odp_atomic.h \ arch/aarch64/odp_cpu.h \ @@ -334,18 +335,17 @@ endif if ARCH_IS_DEFAULT __LIB__libodp_dpdk_la_SOURCES += arch/default/odp_atomic.c \ arch/default/odp_cpu_cycles.c \ - arch/default/odp_global_time.c \ arch/default/odp_hash_crc32.c \ arch/default/odp_random.c \ arch/default/odp_sysinfo_parse.c -odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_time.h \ - arch/default/odp/api/abi/hash_crc32.h +odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/hash_crc32.h if !ODP_ABI_COMPAT odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ arch/default/odp/api/abi/atomic_inlines.h \ arch/default/odp/api/abi/cpu_generic.h \ arch/default/odp/api/abi/cpu_inlines.h \ - arch/default/odp/api/abi/cpu.h + arch/default/odp/api/abi/cpu.h \ + arch/default/odp/api/abi/sync_inlines.h endif noinst_HEADERS += arch/default/odp_atomic.h \ arch/default/odp_cpu.h \ @@ -355,18 +355,17 @@ endif if ARCH_IS_POWERPC __LIB__libodp_dpdk_la_SOURCES += arch/default/odp_atomic.c \ arch/default/odp_cpu_cycles.c \ - arch/default/odp_global_time.c \ arch/default/odp_hash_crc32.c \ arch/default/odp_random.c \ arch/powerpc/odp_sysinfo_parse.c -odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_time.h \ - arch/default/odp/api/abi/hash_crc32.h +odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/hash_crc32.h if !ODP_ABI_COMPAT odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ arch/default/odp/api/abi/atomic_inlines.h \ arch/default/odp/api/abi/cpu_generic.h \ arch/default/odp/api/abi/cpu_inlines.h \ - arch/powerpc/odp/api/abi/cpu.h + arch/powerpc/odp/api/abi/cpu.h \ + arch/default/odp/api/abi/sync_inlines.h endif noinst_HEADERS += arch/default/odp_atomic.h \ arch/default/odp_cpu.h \ @@ -377,18 +376,18 @@ if ARCH_IS_X86 __LIB__libodp_dpdk_la_SOURCES += arch/default/odp_atomic.c \ arch/x86/cpu_flags.c \ arch/x86/odp_cpu_cycles.c \ - arch/x86/odp_global_time.c \ arch/default/odp_hash_crc32.c \ arch/default/odp_random.c \ arch/x86/odp_sysinfo_parse.c odpapiabiarchinclude_HEADERS += arch/x86/odp/api/abi/cpu_rdtsc.h \ - arch/x86/odp/api/abi/cpu_time.h \ - arch/x86/odp/api/abi/hash_crc32.h + arch/x86/odp/api/abi/hash_crc32.h \ + arch/x86/odp/api/abi/time_cpu.h if !ODP_ABI_COMPAT odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ arch/default/odp/api/abi/atomic_inlines.h \ arch/x86/odp/api/abi/cpu_inlines.h \ - arch/x86/odp/api/abi/cpu.h + arch/x86/odp/api/abi/cpu.h \ + arch/x86/odp/api/abi/sync_inlines.h endif noinst_HEADERS += arch/x86/cpu_flags.h \ arch/x86/odp_cpu.h \ diff --git a/platform/linux-dpdk/arch/aarch64/odp/api/abi/cpu_time.h b/platform/linux-dpdk/arch/aarch64/odp/api/abi/cpu_time.h deleted file mode 120000 index 52719241f..000000000 --- a/platform/linux-dpdk/arch/aarch64/odp/api/abi/cpu_time.h +++ /dev/null @@ -1 +0,0 @@ -../../../../../../linux-generic/arch/aarch64/odp/api/abi/cpu_time.h
\ No newline at end of file diff --git a/platform/linux-dpdk/arch/aarch64/odp/api/abi/sync_inlines.h b/platform/linux-dpdk/arch/aarch64/odp/api/abi/sync_inlines.h new file mode 120000 index 000000000..1281f2376 --- /dev/null +++ b/platform/linux-dpdk/arch/aarch64/odp/api/abi/sync_inlines.h @@ -0,0 +1 @@ +../../../../../../linux-generic/arch/aarch64/odp/api/abi/sync_inlines.h
\ No newline at end of file diff --git a/platform/linux-dpdk/arch/aarch64/odp_global_time.c b/platform/linux-dpdk/arch/aarch64/odp_global_time.c deleted file mode 120000 index 1b7fa3c24..000000000 --- a/platform/linux-dpdk/arch/aarch64/odp_global_time.c +++ /dev/null @@ -1 +0,0 @@ -../../../linux-generic/arch/aarch64/odp_global_time.c
\ No newline at end of file diff --git a/platform/linux-dpdk/arch/default/odp/api/abi/cpu_time.h b/platform/linux-dpdk/arch/default/odp/api/abi/cpu_time.h deleted file mode 120000 index 941d6ccca..000000000 --- a/platform/linux-dpdk/arch/default/odp/api/abi/cpu_time.h +++ /dev/null @@ -1 +0,0 @@ -../../../../../../linux-generic/arch/default/odp/api/abi/cpu_time.h
\ No newline at end of file diff --git a/platform/linux-dpdk/arch/default/odp/api/abi/sync_inlines.h b/platform/linux-dpdk/arch/default/odp/api/abi/sync_inlines.h new file mode 120000 index 000000000..328ff07d6 --- /dev/null +++ b/platform/linux-dpdk/arch/default/odp/api/abi/sync_inlines.h @@ -0,0 +1 @@ +../../../../../../linux-generic/arch/default/odp/api/abi/sync_inlines.h
\ No newline at end of file diff --git a/platform/linux-dpdk/arch/default/odp_global_time.c b/platform/linux-dpdk/arch/default/odp_global_time.c deleted file mode 120000 index 3f54b1478..000000000 --- a/platform/linux-dpdk/arch/default/odp_global_time.c +++ /dev/null @@ -1 +0,0 @@ -../../../linux-generic/arch/default/odp_global_time.c
\ No newline at end of file diff --git a/platform/linux-dpdk/arch/x86/odp/api/abi/cpu_time.h b/platform/linux-dpdk/arch/x86/odp/api/abi/cpu_time.h deleted file mode 120000 index 2fb280b8c..000000000 --- a/platform/linux-dpdk/arch/x86/odp/api/abi/cpu_time.h +++ /dev/null @@ -1 +0,0 @@ -../../../../../../linux-generic/arch/x86/odp/api/abi/cpu_time.h
\ No newline at end of file diff --git a/platform/linux-dpdk/arch/x86/odp/api/abi/sync_inlines.h b/platform/linux-dpdk/arch/x86/odp/api/abi/sync_inlines.h new file mode 120000 index 000000000..d5dba2679 --- /dev/null +++ b/platform/linux-dpdk/arch/x86/odp/api/abi/sync_inlines.h @@ -0,0 +1 @@ +../../../../../../linux-generic/arch/x86/odp/api/abi/sync_inlines.h
\ No newline at end of file diff --git a/platform/linux-dpdk/arch/x86/odp/api/abi/time_cpu.h b/platform/linux-dpdk/arch/x86/odp/api/abi/time_cpu.h new file mode 100644 index 000000000..cc313cff1 --- /dev/null +++ b/platform/linux-dpdk/arch/x86/odp/api/abi/time_cpu.h @@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2023 Nokia + */ + +#ifndef ODP_ARCH_TIME_CPU_H_ +#define ODP_ARCH_TIME_CPU_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +int _odp_time_cpu_global_freq_is_const(void); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-dpdk/arch/x86/odp_global_time.c b/platform/linux-dpdk/arch/x86/odp_global_time.c deleted file mode 120000 index 33a50e23e..000000000 --- a/platform/linux-dpdk/arch/x86/odp_global_time.c +++ /dev/null @@ -1 +0,0 @@ -../../../linux-generic/arch/x86/odp_global_time.c
\ No newline at end of file diff --git a/platform/linux-dpdk/include-abi/odp/api/abi/dma.h b/platform/linux-dpdk/include-abi/odp/api/abi/dma.h new file mode 120000 index 000000000..112a7b5a7 --- /dev/null +++ b/platform/linux-dpdk/include-abi/odp/api/abi/dma.h @@ -0,0 +1 @@ +../../../../../linux-generic/include-abi/odp/api/abi/dma.h
\ No newline at end of file diff --git a/platform/linux-dpdk/include/odp/api/plat/dma_inlines.h b/platform/linux-dpdk/include/odp/api/plat/dma_inlines.h new file mode 120000 index 000000000..5b60f374e --- /dev/null +++ b/platform/linux-dpdk/include/odp/api/plat/dma_inlines.h @@ -0,0 +1 @@ +../../../../../linux-generic/include/odp/api/plat/dma_inlines.h
\ No newline at end of file diff --git a/platform/linux-dpdk/include/odp/api/plat/time_inlines.h b/platform/linux-dpdk/include/odp/api/plat/time_inlines.h index 4b1320300..8e4eec59c 100644 --- a/platform/linux-dpdk/include/odp/api/plat/time_inlines.h +++ b/platform/linux-dpdk/include/odp/api/plat/time_inlines.h @@ -12,125 +12,116 @@ #include <odp/api/hints.h> #include <odp/api/time_types.h> -#include <odp/api/abi/cpu_time.h> +#include <rte_config.h> +#include <rte_atomic.h> +#include <rte_cycles.h> #include <stdint.h> /** @cond _ODP_HIDE_FROM_DOXYGEN_ */ -#define _ODP_TIMESPEC_SIZE 16 -#define _ODP_TIME_GIGA_HZ 1000000000ULL - -typedef odp_time_t (*time_cur_fn)(void); -typedef odp_time_t (*time_cur_strict_fn)(void); -typedef uint64_t (*time_res_fn)(void); - -typedef struct time_handler_ { - time_cur_fn time_cur; - time_cur_strict_fn time_cur_strict; - time_res_fn time_res; - -} time_handler_t; - typedef struct _odp_time_global_t { - /* Storage space for struct timespec. Posix headers are not included - * here to avoid application exposure. */ - uint8_t timespec[_ODP_TIMESPEC_SIZE] ODP_ALIGNED(_ODP_TIMESPEC_SIZE); - - int use_hw; - uint64_t hw_start; - uint64_t hw_freq_hz; - /* DPDK specific */ - time_handler_t handler; + uint64_t start_cycles; + uint64_t freq_hz; } _odp_time_global_t; extern _odp_time_global_t _odp_time_glob; -odp_time_t _odp_timespec_cur(void); - -static inline odp_time_t _odp_time_cur_gen(void) +static inline odp_time_t _odp_time_cur(void) { - if (_odp_time_glob.use_hw) { - odp_time_t time; + odp_time_t time; - time.count = _odp_cpu_global_time() - _odp_time_glob.hw_start; - return time; - } + time.u64 = rte_get_timer_cycles() - _odp_time_glob.start_cycles; - return _odp_timespec_cur(); + return time; } -static inline odp_time_t _odp_time_cur_gen_strict(void) +static inline odp_time_t _odp_time_cur_strict(void) { - if (_odp_time_glob.use_hw) { - odp_time_t time; - - time.count = _odp_cpu_global_time_strict() - _odp_time_glob.hw_start; - return time; - } + odp_time_t time; - return _odp_timespec_cur(); -} + rte_mb(); + time.u64 = rte_get_timer_cycles() - _odp_time_glob.start_cycles; -static inline odp_time_t _odp_time_cur(void) -{ - return _odp_time_glob.handler.time_cur(); -} - -static inline odp_time_t _odp_time_cur_strict(void) -{ - return _odp_time_glob.handler.time_cur_strict(); + return time; } -static inline uint64_t _odp_time_hw_to_ns(odp_time_t time) +static inline uint64_t _odp_time_to_ns(odp_time_t time) { uint64_t nsec; - uint64_t freq_hz = _odp_time_glob.hw_freq_hz; uint64_t count = time.count; uint64_t sec = 0; + const uint64_t freq_hz = _odp_time_glob.freq_hz; + const uint64_t giga_hz = 1000000000; if (count >= freq_hz) { sec = count / freq_hz; count = count - sec * freq_hz; } - nsec = (_ODP_TIME_GIGA_HZ * count) / freq_hz; + nsec = (giga_hz * count) / freq_hz; - return (sec * _ODP_TIME_GIGA_HZ) + nsec; + return (sec * giga_hz) + nsec; } -static inline uint64_t _odp_time_convert_to_ns(odp_time_t time) +static inline odp_time_t _odp_time_from_ns(uint64_t ns) { - if (_odp_time_glob.use_hw) - return _odp_time_hw_to_ns(time); + odp_time_t time; + uint64_t count; + uint64_t sec = 0; + const uint64_t freq_hz = _odp_time_glob.freq_hz; + + if (ns >= ODP_TIME_SEC_IN_NS) { + sec = ns / ODP_TIME_SEC_IN_NS; + ns = ns - sec * ODP_TIME_SEC_IN_NS; + } - return time.nsec; + count = sec * freq_hz; + count += (ns * freq_hz) / ODP_TIME_SEC_IN_NS; + + time.count = count; + + return time; } #ifndef _ODP_NO_INLINE /* Inline functions by default */ #define _ODP_INLINE static inline - #define odp_time_local __odp_time_local - #define odp_time_global __odp_time_global - #define odp_time_to_ns __odp_time_to_ns - #define odp_time_local_ns __odp_time_local_ns - #define odp_time_global_ns __odp_time_global_ns - - #define odp_time_local_strict __odp_time_local_strict - #define odp_time_global_strict __odp_time_global_strict - #define odp_time_local_strict_ns __odp_time_local_strict_ns - #define odp_time_global_strict_ns __odp_time_global_strict_ns - - #define odp_time_cmp __odp_time_cmp - #define odp_time_diff __odp_time_diff - #define odp_time_diff_ns __odp_time_diff_ns - #define odp_time_sum __odp_time_sum + #define odp_time_local_res __odp_time_local_res + #define odp_time_global_res __odp_time_global_res + #define odp_time_local __odp_time_local + #define odp_time_global __odp_time_global + #define odp_time_local_strict __odp_time_local_strict + #define odp_time_global_strict __odp_time_global_strict + #define odp_time_local_ns __odp_time_local_ns + #define odp_time_global_ns __odp_time_global_ns + #define odp_time_local_from_ns __odp_time_local_from_ns + #define odp_time_global_from_ns __odp_time_global_from_ns + #define odp_time_local_strict_ns __odp_time_local_strict_ns + #define odp_time_global_strict_ns __odp_time_global_strict_ns + #define odp_time_to_ns __odp_time_to_ns + #define odp_time_cmp __odp_time_cmp + #define odp_time_diff __odp_time_diff + #define odp_time_diff_ns __odp_time_diff_ns + #define odp_time_sum __odp_time_sum + #define odp_time_wait_ns __odp_time_wait_ns + #define odp_time_wait_until __odp_time_wait_until #else #define _ODP_INLINE #endif +_ODP_INLINE uint64_t odp_time_local_res(void) +{ + return _odp_time_glob.freq_hz; +} + +_ODP_INLINE uint64_t odp_time_global_res(void) +{ + return _odp_time_glob.freq_hz; +} + _ODP_INLINE odp_time_t odp_time_local(void) { return _odp_time_cur(); @@ -153,27 +144,37 @@ _ODP_INLINE odp_time_t odp_time_global_strict(void) _ODP_INLINE uint64_t odp_time_local_ns(void) { - return _odp_time_convert_to_ns(_odp_time_cur()); + return _odp_time_to_ns(_odp_time_cur()); } _ODP_INLINE uint64_t odp_time_global_ns(void) { - return _odp_time_convert_to_ns(_odp_time_cur()); + return _odp_time_to_ns(_odp_time_cur()); +} + +_ODP_INLINE odp_time_t odp_time_local_from_ns(uint64_t ns) +{ + return _odp_time_from_ns(ns); +} + +_ODP_INLINE odp_time_t odp_time_global_from_ns(uint64_t ns) +{ + return _odp_time_from_ns(ns); } _ODP_INLINE uint64_t odp_time_local_strict_ns(void) { - return _odp_time_convert_to_ns(_odp_time_cur_strict()); + return _odp_time_to_ns(_odp_time_cur_strict()); } _ODP_INLINE uint64_t odp_time_global_strict_ns(void) { - return _odp_time_convert_to_ns(_odp_time_cur_strict()); + return _odp_time_to_ns(_odp_time_cur_strict()); } _ODP_INLINE uint64_t odp_time_to_ns(odp_time_t time) { - return _odp_time_convert_to_ns(time); + return _odp_time_to_ns(time); } _ODP_INLINE int odp_time_cmp(odp_time_t t2, odp_time_t t1) @@ -214,6 +215,29 @@ _ODP_INLINE odp_time_t odp_time_sum(odp_time_t t1, odp_time_t t2) return time; } +static inline void _odp_time_wait_until(odp_time_t time) +{ + odp_time_t cur; + + do { + cur = _odp_time_cur(); + } while (odp_time_cmp(time, cur) > 0); +} + +_ODP_INLINE void odp_time_wait_ns(uint64_t ns) +{ + const odp_time_t cur = _odp_time_cur(); + const odp_time_t wait = _odp_time_from_ns(ns); + const odp_time_t end_time = odp_time_sum(cur, wait); + + _odp_time_wait_until(end_time); +} + +_ODP_INLINE void odp_time_wait_until(odp_time_t time) +{ + _odp_time_wait_until(time); +} + /** @endcond */ #endif diff --git a/platform/linux-dpdk/include/odp/api/plat/timer_inlines.h b/platform/linux-dpdk/include/odp/api/plat/timer_inlines.h index 357d4df06..66327acdb 100644 --- a/platform/linux-dpdk/include/odp/api/plat/timer_inlines.h +++ b/platform/linux-dpdk/include/odp/api/plat/timer_inlines.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2022, Nokia +/* Copyright (c) 2022-2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -25,6 +25,7 @@ #define odp_timeout_user_ptr __odp_timeout_user_ptr #define odp_timeout_user_area __odp_timeout_user_area #define odp_timeout_from_event __odp_timeout_from_event + #define odp_timeout_from_event_multi __odp_timeout_from_event_multi #define odp_timeout_to_event __odp_timeout_to_event #else #define _ODP_INLINE @@ -57,6 +58,15 @@ _ODP_INLINE odp_timeout_t odp_timeout_from_event(odp_event_t ev) return (odp_timeout_t)ev; } +_ODP_INLINE void odp_timeout_from_event_multi(odp_timeout_t tmo[], const odp_event_t ev[], int num) +{ + for (int i = 0; i < num; i++) { + _ODP_ASSERT(odp_event_type(ev[i]) == ODP_EVENT_TIMEOUT); + + tmo[i] = (odp_timeout_t)ev[i]; + } +} + _ODP_INLINE odp_event_t odp_timeout_to_event(odp_timeout_t tmo) { return (odp_event_t)tmo; diff --git a/platform/linux-dpdk/include/odp_pool_internal.h b/platform/linux-dpdk/include/odp_pool_internal.h index b82324a77..8512806f7 100644 --- a/platform/linux-dpdk/include/odp_pool_internal.h +++ b/platform/linux-dpdk/include/odp_pool_internal.h @@ -83,6 +83,7 @@ typedef struct ODP_ALIGNED_CACHE { odp_pool_ext_param_t ext_param; odp_shm_t uarea_shm; uint64_t uarea_shm_size; + uint32_t param_uarea_size; uint32_t uarea_size; uint32_t trailer_size; /* Endmark size */ uint8_t *uarea_base_addr; diff --git a/platform/linux-dpdk/odp_crypto.c b/platform/linux-dpdk/odp_crypto.c index 5761787f8..3269e3075 100644 --- a/platform/linux-dpdk/odp_crypto.c +++ b/platform/linux-dpdk/odp_crypto.c @@ -1431,6 +1431,16 @@ int odp_crypto_session_create(const odp_crypto_session_param_t *param, return -1; } + if (param->cipher_range_in_bits) { + *status = ODP_CRYPTO_SES_ERR_CIPHER; + *session_out = ODP_CRYPTO_SESSION_INVALID; + return -1; + } + if (param->auth_range_in_bits) { + *status = ODP_CRYPTO_SES_ERR_AUTH; + *session_out = ODP_CRYPTO_SESSION_INVALID; + return -1; + } if (param->auth_alg == ODP_AUTH_ALG_AES_GMAC && param->cipher_alg != ODP_CIPHER_ALG_NULL) { *status = ODP_CRYPTO_SES_ERR_ALG_COMBO; diff --git a/platform/linux-dpdk/odp_init.c b/platform/linux-dpdk/odp_init.c index c7ae1ea47..7e6aaab55 100644 --- a/platform/linux-dpdk/odp_init.c +++ b/platform/linux-dpdk/odp_init.c @@ -35,8 +35,8 @@ enum init_stage { LIBCONFIG_INIT, CPUMASK_INIT, CPU_CYCLES_INIT, - TIME_INIT, SYSINFO_INIT, + TIME_INIT, ISHM_INIT, FDSERVER_INIT, GLOBAL_RW_DATA_INIT, @@ -473,16 +473,16 @@ static int term_global(enum init_stage stage) } /* Fall through */ - case SYSINFO_INIT: - if (_odp_system_info_term()) { - _ODP_ERR("ODP system info term failed.\n"); + case TIME_INIT: + if (_odp_time_term_global()) { + _ODP_ERR("ODP time term failed.\n"); rc = -1; } /* Fall through */ - case TIME_INIT: - if (_odp_time_term_global()) { - _ODP_ERR("ODP time term failed.\n"); + case SYSINFO_INIT: + if (_odp_system_info_term()) { + _ODP_ERR("ODP system info term failed.\n"); rc = -1; } /* Fall through */ @@ -558,18 +558,18 @@ int odp_init_global(odp_instance_t *instance, return -1; } - if (_odp_time_init_global()) { - _ODP_ERR("ODP time init failed.\n"); - goto init_failed; - } - stage = TIME_INIT; - if (_odp_system_info_init()) { _ODP_ERR("ODP system_info init failed.\n"); goto init_failed; } stage = SYSINFO_INIT; + if (_odp_time_init_global()) { + _ODP_ERR("ODP time init failed.\n"); + goto init_failed; + } + stage = TIME_INIT; + if (_odp_shm_init_global(params)) { _ODP_ERR("ODP shm init failed.\n"); goto init_failed; diff --git a/platform/linux-dpdk/odp_pool.c b/platform/linux-dpdk/odp_pool.c index 83d75dd6c..fd27cd79e 100644 --- a/platform/linux-dpdk/odp_pool.c +++ b/platform/linux-dpdk/odp_pool.c @@ -263,6 +263,7 @@ int odp_pool_capability(odp_pool_capability_t *capa) capa->buf.max_size = MAX_SIZE; capa->buf.max_num = CONFIG_POOL_MAX_NUM; capa->buf.max_uarea_size = MAX_UAREA_SIZE; + capa->buf.uarea_persistence = true; capa->buf.min_cache_size = 0; capa->buf.max_cache_size = RTE_MEMPOOL_CACHE_MAX_SIZE; capa->buf.stats.all = supported_stats.all; @@ -279,6 +280,7 @@ int odp_pool_capability(odp_pool_capability_t *capa) capa->pkt.min_seg_len = CONFIG_PACKET_SEG_LEN_MIN; capa->pkt.max_seg_len = CONFIG_PACKET_MAX_SEG_LEN; capa->pkt.max_uarea_size = MAX_UAREA_SIZE; + capa->pkt.uarea_persistence = true; capa->pkt.min_cache_size = 0; capa->pkt.max_cache_size = RTE_MEMPOOL_CACHE_MAX_SIZE; capa->pkt.stats.all = supported_stats.all; @@ -287,6 +289,7 @@ int odp_pool_capability(odp_pool_capability_t *capa) capa->tmo.max_pools = max_pools; capa->tmo.max_num = CONFIG_POOL_MAX_NUM; capa->tmo.max_uarea_size = MAX_UAREA_SIZE; + capa->tmo.uarea_persistence = true; capa->tmo.min_cache_size = 0; capa->tmo.max_cache_size = RTE_MEMPOOL_CACHE_MAX_SIZE; capa->tmo.stats.all = supported_stats.all; @@ -295,6 +298,7 @@ int odp_pool_capability(odp_pool_capability_t *capa) capa->vector.max_pools = max_pools; capa->vector.max_num = CONFIG_POOL_MAX_NUM; capa->vector.max_uarea_size = MAX_UAREA_SIZE; + capa->vector.uarea_persistence = true; capa->vector.max_size = CONFIG_PACKET_VECTOR_MAX_SIZE; capa->vector.min_cache_size = 0; capa->vector.max_cache_size = RTE_MEMPOOL_CACHE_MAX_SIZE; @@ -550,6 +554,7 @@ static int reserve_uarea(pool_t *pool, uint32_t uarea_size, uint32_t num_pkt) pool->uarea_shm = ODP_SHM_INVALID; if (uarea_size == 0) { + pool->param_uarea_size = 0; pool->uarea_size = 0; pool->uarea_shm_size = 0; return 0; @@ -559,6 +564,7 @@ static int reserve_uarea(pool_t *pool, uint32_t uarea_size, uint32_t num_pkt) pool->pool_idx, pool->name); uarea_name[ODP_SHM_NAME_LEN - 1] = 0; + pool->param_uarea_size = uarea_size; pool->uarea_size = _ODP_ROUNDUP_CACHE_LINE(uarea_size); pool->uarea_shm_size = num_pkt * (uint64_t)pool->uarea_size; @@ -605,7 +611,8 @@ static void init_obj_priv_data(struct rte_mempool *mp ODP_UNUSED, void *arg, voi struct priv_data_t *priv_data = arg; struct rte_mbuf *mb = mbuf; _odp_event_hdr_t *event_hdr = (_odp_event_hdr_t *)mbuf; - void *uarea = priv_data->pool->uarea_base_addr + i * priv_data->pool->uarea_size; + pool_t *pool = priv_data->pool; + void *uarea = pool->uarea_base_addr + i * pool->uarea_size; void **obj_uarea; if (priv_data->type != ODP_POOL_PACKET) @@ -613,7 +620,7 @@ static void init_obj_priv_data(struct rte_mempool *mp ODP_UNUSED, void *arg, voi mb->data_off = 0; event_hdr->hdr.index = i; - event_hdr->hdr.pool = _odp_pool_handle(priv_data->pool); + event_hdr->hdr.pool = _odp_pool_handle(pool); event_hdr->hdr.type = priv_data->type; event_hdr->hdr.event_type = priv_data->event_type; @@ -636,6 +643,10 @@ static void init_obj_priv_data(struct rte_mempool *mp ODP_UNUSED, void *arg, voi *obj_uarea = uarea; + if (uarea && pool->params.uarea_init.init_fn) + pool->params.uarea_init.init_fn(uarea, pool->param_uarea_size, + pool->params.uarea_init.args, i); + if (priv_data->type == ODP_POOL_BUFFER || priv_data->type == ODP_POOL_PACKET) { mb->buf_len -= _ODP_EV_ENDMARK_SIZE; _odp_event_endmark_set(_odp_event_from_mbuf(mb)); @@ -751,13 +762,15 @@ odp_pool_t _odp_pool_create(const char *name, const odp_pool_param_t *params, return ODP_POOL_INVALID; } - rte_mempool_obj_iter(mp, init_obj_priv_data, &priv_data); pool->rte_mempool = mp; pool->seg_len = seg_size; pool->type_2 = type_2; pool->type = params->type; pool->params = *params; pool->trailer_size = trailer; + + rte_mempool_obj_iter(mp, init_obj_priv_data, &priv_data); + UNLOCK(&pool->lock); pool_hdl = _odp_pool_handle(pool); @@ -1097,6 +1110,7 @@ int odp_pool_ext_capability(odp_pool_type_t type, capa->pkt.max_headroom_size = RTE_PKTMBUF_HEADROOM; capa->pkt.max_segs_per_pkt = PKT_MAX_SEGS; capa->pkt.max_uarea_size = MAX_UAREA_SIZE; + capa->pkt.uarea_persistence = true; return 0; } @@ -1297,10 +1311,11 @@ static void init_ext_obj(struct rte_mempool *mp, void *arg, void *mbuf, unsigned { struct mbuf_ctor_arg *mb_ctor_arg = arg; struct rte_mbuf *mb = mbuf; - void *uarea = mb_ctor_arg->pool->uarea_base_addr + i * mb_ctor_arg->pool->uarea_size; + pool_t *pool = mb_ctor_arg->pool; + void *uarea = pool->uarea_base_addr + i * pool->uarea_size; _odp_event_hdr_t *event_hdr = (_odp_event_hdr_t *)mbuf; void **obj_uarea; - odp_pool_ext_param_t *p = &mb_ctor_arg->pool->ext_param; + odp_pool_ext_param_t *p = &pool->ext_param; uint32_t app_hdr_offset = sizeof(odp_packet_hdr_t); uint32_t app_hdr_size = p->pkt.app_header_size; uint32_t buf_size = p->pkt.buf_size; @@ -1333,7 +1348,7 @@ static void init_ext_obj(struct rte_mempool *mp, void *arg, void *mbuf, unsigned mb->next = NULL; /* Save index, might be useful for debugging purposes */ event_hdr->hdr.index = i; - event_hdr->hdr.pool = _odp_pool_handle(mb_ctor_arg->pool); + event_hdr->hdr.pool = _odp_pool_handle(pool); event_hdr->hdr.type = mb_ctor_arg->type; event_hdr->hdr.event_type = mb_ctor_arg->event_type; @@ -1356,6 +1371,10 @@ static void init_ext_obj(struct rte_mempool *mp, void *arg, void *mbuf, unsigned *obj_uarea = uarea; + if (uarea && pool->ext_param.uarea_init.init_fn) + pool->ext_param.uarea_init.init_fn(uarea, pool->param_uarea_size, + pool->ext_param.uarea_init.args, i); + if (mb_ctor_arg->type == ODP_POOL_BUFFER || mb_ctor_arg->type == ODP_POOL_PACKET) { mb->buf_len -= _ODP_EV_ENDMARK_SIZE; _odp_event_endmark_set(_odp_event_from_mbuf(mb)); diff --git a/platform/linux-dpdk/odp_system_info.c b/platform/linux-dpdk/odp_system_info.c index c76820e5d..886f7f216 100644 --- a/platform/linux-dpdk/odp_system_info.c +++ b/platform/linux-dpdk/odp_system_info.c @@ -59,8 +59,8 @@ static int read_cache_line_size(void) file = fopen(CACHE_LNSZ_FILE, "rt"); if (file == NULL) { /* File not found */ - _ODP_PRINT("WARN: unable to read host CPU cache line size. " - "Using ODP_CACHE_LINE_SIZE instead.\n"); + _ODP_WARN("Unable to read host CPU cache line size. " + "Using ODP_CACHE_LINE_SIZE instead.\n"); return ODP_CACHE_LINE_SIZE; } @@ -225,7 +225,7 @@ static int system_cache_line(system_info_t *sysinfo) sysinfo->cache_line_size = ret; if (ret != ODP_CACHE_LINE_SIZE) - _ODP_PRINT("WARN: host CPU cache line size and ODP_CACHE_LINE_SIZE don't match.\n"); + _ODP_WARN("Host CPU cache line size and ODP_CACHE_LINE_SIZE don't match.\n"); return 0; } diff --git a/platform/linux-dpdk/odp_time.c b/platform/linux-dpdk/odp_time.c index fbbc9358d..2fc25eb96 100644 --- a/platform/linux-dpdk/odp_time.c +++ b/platform/linux-dpdk/odp_time.c @@ -1,302 +1,43 @@ /* Copyright (c) 2013-2018, Linaro Limited - * Copyright (c) 2021-2022, Nokia + * Copyright (c) 2021-2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause */ -#include <odp_posix_extensions.h> - -#include <time.h> -#include <string.h> -#include <inttypes.h> +#include <odp/api/plat/time_inlines.h> -#include <odp/api/time.h> -#include <odp/api/hints.h> #include <odp_debug_internal.h> #include <odp_init_internal.h> -#include <odp/api/plat/time_inlines.h> #include <rte_config.h> -#include <rte_atomic.h> #include <rte_cycles.h> -ODP_STATIC_ASSERT(_ODP_TIMESPEC_SIZE >= (sizeof(struct timespec)), - "_ODP_TIMESPEC_SIZE too small"); +#include <string.h> #include <odp/visibility_begin.h> _odp_time_global_t _odp_time_glob; -/* - * Posix timespec based functions - */ - -static inline uint64_t time_spec_diff_nsec(struct timespec *t2, - struct timespec *t1) -{ - struct timespec diff; - uint64_t nsec; - - diff.tv_sec = t2->tv_sec - t1->tv_sec; - diff.tv_nsec = t2->tv_nsec - t1->tv_nsec; - - if (diff.tv_nsec < 0) { - diff.tv_nsec += ODP_TIME_SEC_IN_NS; - diff.tv_sec -= 1; - } - - nsec = (diff.tv_sec * ODP_TIME_SEC_IN_NS) + diff.tv_nsec; - - return nsec; -} - -odp_time_t _odp_timespec_cur(void) -{ - int ret; - odp_time_t time; - struct timespec sys_time; - struct timespec *start_time; - - start_time = (struct timespec *)(uintptr_t)&_odp_time_glob.timespec; - - ret = clock_gettime(CLOCK_MONOTONIC_RAW, &sys_time); - if (odp_unlikely(ret != 0)) - _ODP_ABORT("clock_gettime failed\n"); - - time.nsec = time_spec_diff_nsec(&sys_time, start_time); - - return time; -} - #include <odp/visibility_end.h> -static inline uint64_t time_spec_res(void) -{ - int ret; - struct timespec tres; - - ret = clock_getres(CLOCK_MONOTONIC_RAW, &tres); - if (odp_unlikely(ret != 0)) - _ODP_ABORT("clock_getres failed\n"); - - return ODP_TIME_SEC_IN_NS / (uint64_t)tres.tv_nsec; -} - -static inline odp_time_t time_spec_from_ns(uint64_t ns) -{ - odp_time_t time; - - time.nsec = ns; - - return time; -} - -/* - * HW time counter based functions - */ - -static inline uint64_t time_hw_res(void) -{ - return _odp_time_glob.hw_freq_hz; -} - -static inline odp_time_t time_hw_from_ns(uint64_t ns) -{ - odp_time_t time; - uint64_t count; - uint64_t freq_hz = _odp_time_glob.hw_freq_hz; - uint64_t sec = 0; - - if (ns >= ODP_TIME_SEC_IN_NS) { - sec = ns / ODP_TIME_SEC_IN_NS; - ns = ns - sec * ODP_TIME_SEC_IN_NS; - } - - count = sec * freq_hz; - count += (ns * freq_hz) / ODP_TIME_SEC_IN_NS; - - time.count = count; - - return time; -} - -/* - * Common functions - */ - -static inline uint64_t time_res(void) -{ - if (_odp_time_glob.use_hw) - return time_hw_res(); - - return time_spec_res(); -} - -static inline odp_time_t time_from_ns(uint64_t ns) -{ - if (_odp_time_glob.use_hw) - return time_hw_from_ns(ns); - - return time_spec_from_ns(ns); -} - -static inline odp_time_t time_cur_dpdk(void) -{ - odp_time_t time; - - time.u64 = rte_get_timer_cycles() - _odp_time_glob.hw_start; - - return time; -} - -static inline odp_time_t time_cur_dpdk_strict(void) -{ - odp_time_t time; - - rte_mb(); - time.u64 = rte_get_timer_cycles() - _odp_time_glob.hw_start; - - return time; -} - -static inline uint64_t time_res_dpdk(void) -{ - return rte_get_timer_hz(); -} - -static inline void time_wait_until(odp_time_t time) -{ - odp_time_t cur; - - do { - cur = _odp_time_cur(); - } while (odp_time_cmp(time, cur) > 0); -} - -odp_time_t odp_time_local_from_ns(uint64_t ns) -{ - return time_from_ns(ns); -} - -odp_time_t odp_time_global_from_ns(uint64_t ns) -{ - return time_from_ns(ns); -} - -uint64_t odp_time_local_res(void) -{ - return _odp_time_glob.handler.time_res(); -} - -uint64_t odp_time_global_res(void) -{ - return _odp_time_glob.handler.time_res(); -} - -void odp_time_wait_ns(uint64_t ns) -{ - odp_time_t cur = _odp_time_cur(); - odp_time_t wait = time_from_ns(ns); - odp_time_t end_time = odp_time_sum(cur, wait); - - time_wait_until(end_time); -} - -void odp_time_wait_until(odp_time_t time) -{ - time_wait_until(time); -} - -static odp_bool_t is_invariant_tsc_supported(void) -{ - FILE *file; - char *line = NULL; - size_t len = 0; - odp_bool_t nonstop_tsc = false; - odp_bool_t constant_tsc = false; - odp_bool_t ret = false; - - file = fopen("/proc/cpuinfo", "rt"); - while (getline(&line, &len, file) != -1) { - if (strstr(line, "flags") != NULL) { - if (strstr(line, "constant_tsc") != NULL) - constant_tsc = true; - if (strstr(line, "nonstop_tsc") != NULL) - nonstop_tsc = true; - - if (constant_tsc && nonstop_tsc) - ret = true; - else - ret = false; - - free(line); - fclose(file); - return ret; - } - } - free(line); - fclose(file); - return false; -} - -static inline odp_bool_t is_dpdk_timer_cycles_support(void) +int _odp_time_init_global(void) { - if (is_invariant_tsc_supported() == true) - return true; + memset(&_odp_time_glob, 0, sizeof(_odp_time_global_t)); #ifdef RTE_LIBEAL_USE_HPET - if (rte_eal_hpet_init(1) == 0) - return true; + if (rte_eal_hpet_init(1) != 0) + _ODP_WARN("HPET init failed. Using TSC time.\n"); #endif - return false; -} - -int _odp_time_init_global(void) -{ - struct timespec *timespec; - int ret = 0; - _odp_time_global_t *global = &_odp_time_glob; - - memset(global, 0, sizeof(_odp_time_global_t)); - - if (is_dpdk_timer_cycles_support()) { - _odp_time_glob.handler.time_cur = time_cur_dpdk; - _odp_time_glob.handler.time_cur_strict = time_cur_dpdk_strict; - _odp_time_glob.handler.time_res = time_res_dpdk; - _odp_time_glob.hw_freq_hz = time_res_dpdk(); - _odp_time_glob.use_hw = 1; - _odp_time_glob.hw_start = rte_get_timer_cycles(); - if (_odp_time_glob.hw_start == 0) - return -1; - else - return 0; - } - _odp_time_glob.handler.time_cur = _odp_time_cur_gen; - _odp_time_glob.handler.time_cur_strict = _odp_time_cur_gen_strict; - _odp_time_glob.handler.time_res = time_res; - - if (_odp_cpu_has_global_time()) { - global->use_hw = 1; - global->hw_freq_hz = _odp_cpu_global_time_freq(); - - if (global->hw_freq_hz == 0) - return -1; - - _ODP_PRINT("HW time counter freq: %" PRIu64 " hz\n\n", global->hw_freq_hz); - - global->hw_start = _odp_cpu_global_time(); - return 0; + _odp_time_glob.freq_hz = rte_get_timer_hz(); + _odp_time_glob.start_cycles = rte_get_timer_cycles(); + if (_odp_time_glob.start_cycles == 0) { + _ODP_ERR("Initializing start cycles failed.\n"); + return -1; } - timespec = (struct timespec *)(uintptr_t)global->timespec; - timespec->tv_sec = 0; - timespec->tv_nsec = 0; - - ret = clock_gettime(CLOCK_MONOTONIC_RAW, timespec); - - return ret; + return 0; } int _odp_time_term_global(void) |