From 209396c6ce988f75919d0778b248a4deb874cc6b Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Thu, 8 Jun 2023 17:03:46 +0300 Subject: linux-gen: cpu: update x86 CPU flags code Update x86 CPU flags code from DPDK v23.03. DPDK copyright has been updated to the currently used format. Signed-off-by: Matias Elo Reviewed-by: Petri Savolainen --- platform/linux-generic/arch/x86/cpu_flags.c | 135 ++++++++++++++-------------- 1 file changed, 65 insertions(+), 70 deletions(-) (limited to 'platform') diff --git a/platform/linux-generic/arch/x86/cpu_flags.c b/platform/linux-generic/arch/x86/cpu_flags.c index 036645dbc..13753febb 100644 --- a/platform/linux-generic/arch/x86/cpu_flags.c +++ b/platform/linux-generic/arch/x86/cpu_flags.c @@ -1,45 +1,20 @@ /* Copyright (c) 2017-2018, Linaro Limited + * Copyright (c) 2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause */ -/*- - * BSD LICENSE - * - * Copyright(c) 2010-2015 Intel Corporation. All rights reserved. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2010-2015 Intel Corporation */ #include "cpu_flags.h" #include #include + +#include +#include #include #include @@ -74,6 +49,7 @@ enum rte_cpu_flag_t { RTE_CPUFLAG_AVX, /**< AVX */ RTE_CPUFLAG_F16C, /**< F16C */ RTE_CPUFLAG_RDRAND, /**< RDRAND */ + RTE_CPUFLAG_HYPERVISOR, /**< Running in a VM */ /* (EAX 01h) EDX features */ RTE_CPUFLAG_FPU, /**< FPU */ @@ -130,6 +106,7 @@ enum rte_cpu_flag_t { RTE_CPUFLAG_INVPCID, /**< INVPCID */ RTE_CPUFLAG_RTM, /**< Transactional memory */ RTE_CPUFLAG_AVX512F, /**< AVX512F */ + RTE_CPUFLAG_RDSEED, /**< RDSEED instruction */ /* (EAX 80000001h) ECX features */ RTE_CPUFLAG_LAHF_SAHF, /**< LAHF_SAHF */ @@ -145,8 +122,29 @@ enum rte_cpu_flag_t { /* (EAX 80000007h) EDX features */ RTE_CPUFLAG_INVTSC, /**< INVTSC */ + RTE_CPUFLAG_AVX512DQ, /**< AVX512 Doubleword and Quadword */ + RTE_CPUFLAG_AVX512IFMA, /**< AVX512 Integer Fused Multiply-Add */ + RTE_CPUFLAG_AVX512CD, /**< AVX512 Conflict Detection*/ + RTE_CPUFLAG_AVX512BW, /**< AVX512 Byte and Word */ + RTE_CPUFLAG_AVX512VL, /**< AVX512 Vector Length */ + RTE_CPUFLAG_AVX512VBMI, /**< AVX512 Vector Bit Manipulation */ + RTE_CPUFLAG_AVX512VBMI2, /**< AVX512 Vector Bit Manipulation 2 */ + RTE_CPUFLAG_GFNI, /**< Galois Field New Instructions */ + RTE_CPUFLAG_VAES, /**< Vector AES */ + RTE_CPUFLAG_VPCLMULQDQ, /**< Vector Carry-less Multiply */ + RTE_CPUFLAG_AVX512VNNI, + /**< AVX512 Vector Neural Network Instructions */ + RTE_CPUFLAG_AVX512BITALG, /**< AVX512 Bit Algorithms */ + RTE_CPUFLAG_AVX512VPOPCNTDQ, /**< AVX512 Vector Popcount */ + RTE_CPUFLAG_CLDEMOTE, /**< Cache Line Demote */ + RTE_CPUFLAG_MOVDIRI, /**< Direct Store Instructions */ + RTE_CPUFLAG_MOVDIR64B, /**< Direct Store Instructions 64B */ + RTE_CPUFLAG_AVX512VP2INTERSECT, /**< AVX512 Two Register Intersection */ + + RTE_CPUFLAG_WAITPKG, /**< UMONITOR/UMWAIT/TPAUSE */ + /* The last item */ - RTE_CPUFLAG_NUMFLAGS, /**< This should always be the last! */ + RTE_CPUFLAG_NUMFLAGS, /**< This should always be the last! */ }; enum cpu_register_t { @@ -203,6 +201,7 @@ static const struct feature_entry cpu_feature_table[] = { FEAT_DEF(AVX, 0x00000001, 0, RTE_REG_ECX, 28) FEAT_DEF(F16C, 0x00000001, 0, RTE_REG_ECX, 29) FEAT_DEF(RDRAND, 0x00000001, 0, RTE_REG_ECX, 30) + FEAT_DEF(HYPERVISOR, 0x00000001, 0, RTE_REG_ECX, 31) FEAT_DEF(FPU, 0x00000001, 0, RTE_REG_EDX, 0) FEAT_DEF(VME, 0x00000001, 0, RTE_REG_EDX, 1) @@ -246,15 +245,36 @@ static const struct feature_entry cpu_feature_table[] = { FEAT_DEF(ENERGY_EFF, 0x00000006, 0, RTE_REG_ECX, 3) FEAT_DEF(FSGSBASE, 0x00000007, 0, RTE_REG_EBX, 0) - FEAT_DEF(BMI1, 0x00000007, 0, RTE_REG_EBX, 2) + FEAT_DEF(BMI1, 0x00000007, 0, RTE_REG_EBX, 3) FEAT_DEF(HLE, 0x00000007, 0, RTE_REG_EBX, 4) FEAT_DEF(AVX2, 0x00000007, 0, RTE_REG_EBX, 5) - FEAT_DEF(SMEP, 0x00000007, 0, RTE_REG_EBX, 6) - FEAT_DEF(BMI2, 0x00000007, 0, RTE_REG_EBX, 7) - FEAT_DEF(ERMS, 0x00000007, 0, RTE_REG_EBX, 8) + FEAT_DEF(SMEP, 0x00000007, 0, RTE_REG_EBX, 7) + FEAT_DEF(BMI2, 0x00000007, 0, RTE_REG_EBX, 8) + FEAT_DEF(ERMS, 0x00000007, 0, RTE_REG_EBX, 9) FEAT_DEF(INVPCID, 0x00000007, 0, RTE_REG_EBX, 10) FEAT_DEF(RTM, 0x00000007, 0, RTE_REG_EBX, 11) FEAT_DEF(AVX512F, 0x00000007, 0, RTE_REG_EBX, 16) + FEAT_DEF(AVX512DQ, 0x00000007, 0, RTE_REG_EBX, 17) + FEAT_DEF(RDSEED, 0x00000007, 0, RTE_REG_EBX, 18) + FEAT_DEF(AVX512IFMA, 0x00000007, 0, RTE_REG_EBX, 21) + FEAT_DEF(AVX512CD, 0x00000007, 0, RTE_REG_EBX, 28) + FEAT_DEF(AVX512BW, 0x00000007, 0, RTE_REG_EBX, 30) + FEAT_DEF(AVX512VL, 0x00000007, 0, RTE_REG_EBX, 31) + + FEAT_DEF(AVX512VBMI, 0x00000007, 0, RTE_REG_ECX, 1) + FEAT_DEF(WAITPKG, 0x00000007, 0, RTE_REG_ECX, 5) + FEAT_DEF(AVX512VBMI2, 0x00000007, 0, RTE_REG_ECX, 6) + FEAT_DEF(GFNI, 0x00000007, 0, RTE_REG_ECX, 8) + FEAT_DEF(VAES, 0x00000007, 0, RTE_REG_ECX, 9) + FEAT_DEF(VPCLMULQDQ, 0x00000007, 0, RTE_REG_ECX, 10) + FEAT_DEF(AVX512VNNI, 0x00000007, 0, RTE_REG_ECX, 11) + FEAT_DEF(AVX512BITALG, 0x00000007, 0, RTE_REG_ECX, 12) + FEAT_DEF(AVX512VPOPCNTDQ, 0x00000007, 0, RTE_REG_ECX, 14) + FEAT_DEF(CLDEMOTE, 0x00000007, 0, RTE_REG_ECX, 25) + FEAT_DEF(MOVDIRI, 0x00000007, 0, RTE_REG_ECX, 27) + FEAT_DEF(MOVDIR64B, 0x00000007, 0, RTE_REG_ECX, 28) + + FEAT_DEF(AVX512VP2INTERSECT, 0x00000007, 0, RTE_REG_EDX, 8) FEAT_DEF(LAHF_SAHF, 0x80000001, 0, RTE_REG_ECX, 0) FEAT_DEF(LZCNT, 0x80000001, 0, RTE_REG_ECX, 4) @@ -268,55 +288,30 @@ static const struct feature_entry cpu_feature_table[] = { FEAT_DEF(INVTSC, 0x80000007, 0, RTE_REG_EDX, 8) }; -/* - * Execute CPUID instruction and get contents of a specific register - * - * This function, when compiled with GCC, will generate architecture-neutral - * code, as per GCC manual. - */ -static void cpu_get_features(uint32_t leaf, uint32_t subleaf, - cpuid_registers_t out) -{ -#if defined(__i386__) && defined(__PIC__) - /* %ebx is a forbidden register if we compile with -fPIC or -fPIE */ - __asm__ __volatile__("movl %%ebx,%0 ; cpuid ; xchgl %%ebx,%0" - : "=r" (out[RTE_REG_EBX]), - "=a" (out[RTE_REG_EAX]), - "=c" (out[RTE_REG_ECX]), - "=d" (out[RTE_REG_EDX]) - : "a" (leaf), "c" (subleaf)); -#else - __asm__ __volatile__("cpuid" - : "=a" (out[RTE_REG_EAX]), - "=b" (out[RTE_REG_EBX]), - "=c" (out[RTE_REG_ECX]), - "=d" (out[RTE_REG_EDX]) - : "a" (leaf), "c" (subleaf)); -#endif -} - static int cpu_get_flag_enabled(enum rte_cpu_flag_t feature) { const struct feature_entry *feat; cpuid_registers_t regs; + unsigned int maxleaf; if (feature >= RTE_CPUFLAG_NUMFLAGS) /* Flag does not match anything in the feature tables */ - return -1; + return -ENOENT; feat = &cpu_feature_table[feature]; if (!feat->leaf) /* This entry in the table wasn't filled out! */ - return -1; + return -EFAULT; + + maxleaf = __get_cpuid_max(feat->leaf & 0x80000000, NULL); - cpu_get_features(feat->leaf & 0xffff0000, 0, regs); - if (((regs[RTE_REG_EAX] ^ feat->leaf) & 0xffff0000) || - regs[RTE_REG_EAX] < feat->leaf) + if (maxleaf < feat->leaf) return 0; - /* get the cpuid leaf containing the desired feature */ - cpu_get_features(feat->leaf, feat->subleaf, regs); + __cpuid_count(feat->leaf, feat->subleaf, + regs[RTE_REG_EAX], regs[RTE_REG_EBX], + regs[RTE_REG_ECX], regs[RTE_REG_EDX]); /* check if the feature is enabled */ return (regs[feat->reg] >> feat->bit) & 1; -- cgit v1.2.3 From d158bbd99e4cd9e64393a18737021bbbed2f862d Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Mon, 12 Jun 2023 17:05:51 +0300 Subject: linux-gen: dma: inline completion event conversion functions Add necessary header files and inline implementations of odp_dma_compl_from_event() and odp_dma_compl_to_event(). Signed-off-by: Matias Elo Reviewed-by: Tuomas Taipale --- platform/linux-generic/Makefile.am | 3 ++ .../linux-generic/include-abi/odp/api/abi/dma.h | 27 ++++++++++++++ .../include/odp/api/plat/dma_inlines.h | 43 ++++++++++++++++++++++ platform/linux-generic/odp_dma.c | 12 ------ platform/linux-generic/odp_dma_api.c | 11 ++++++ 5 files changed, 84 insertions(+), 12 deletions(-) create mode 100644 platform/linux-generic/include-abi/odp/api/abi/dma.h create mode 100644 platform/linux-generic/include/odp/api/plat/dma_inlines.h create mode 100644 platform/linux-generic/odp_dma_api.c (limited to 'platform') diff --git a/platform/linux-generic/Makefile.am b/platform/linux-generic/Makefile.am index 42760f579..6565e18da 100644 --- a/platform/linux-generic/Makefile.am +++ b/platform/linux-generic/Makefile.am @@ -34,6 +34,7 @@ odpapiplatinclude_HEADERS = \ include/odp/api/plat/byteorder_inlines.h \ include/odp/api/plat/cpu_inlines.h \ include/odp/api/plat/crypto_inlines.h \ + include/odp/api/plat/dma_inlines.h \ include/odp/api/plat/debug_inlines.h \ include/odp/api/plat/event_inlines.h \ include/odp/api/plat/event_inline_types.h \ @@ -79,6 +80,7 @@ odpapiabiarchinclude_HEADERS += \ include-abi/odp/api/abi/crypto.h \ include-abi/odp/api/abi/crypto_types.h \ include-abi/odp/api/abi/debug.h \ + include-abi/odp/api/abi/dma.h \ include-abi/odp/api/abi/dma_types.h \ include-abi/odp/api/abi/errno.h \ include-abi/odp/api/abi/event.h \ @@ -301,6 +303,7 @@ __LIB__libodp_linux_la_SOURCES += \ odp_byteorder_api.c \ odp_cpu_api.c \ odp_crypto_api.c \ + odp_dma_api.c \ odp_event_api.c \ odp_hash_api.c \ odp_ipsec_api.c \ diff --git a/platform/linux-generic/include-abi/odp/api/abi/dma.h b/platform/linux-generic/include-abi/odp/api/abi/dma.h new file mode 100644 index 000000000..60798143c --- /dev/null +++ b/platform/linux-generic/include-abi/odp/api/abi/dma.h @@ -0,0 +1,27 @@ +/* Copyright (c) 2023, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +/** + * @file + * + * ODP DMA + */ + +#ifndef ODP_API_ABI_DMA_H_ +#define ODP_API_ABI_DMA_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +/* Inlined functions for non-ABI compat mode */ +#include + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/include/odp/api/plat/dma_inlines.h b/platform/linux-generic/include/odp/api/plat/dma_inlines.h new file mode 100644 index 000000000..9d9f74654 --- /dev/null +++ b/platform/linux-generic/include/odp/api/plat/dma_inlines.h @@ -0,0 +1,43 @@ +/* Copyright (c) 2023, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_PLAT_DMA_INLINES_H_ +#define ODP_PLAT_DMA_INLINES_H_ + +#include +#include + +#include +#include + +#include + +/** @cond _ODP_HIDE_FROM_DOXYGEN_ */ + +#ifndef _ODP_NO_INLINE + /* Inline functions by default */ + #define _ODP_INLINE static inline + #define odp_dma_compl_from_event __odp_dma_compl_from_event + #define odp_dma_compl_to_event __odp_dma_compl_to_event +#else + #define _ODP_INLINE +#endif + +_ODP_INLINE odp_dma_compl_t odp_dma_compl_from_event(odp_event_t ev) +{ + _ODP_ASSERT(_odp_event_hdr_field(ev, int8_t, event_type) == ODP_EVENT_DMA_COMPL); + + return (odp_dma_compl_t)(uintptr_t)ev; +} + +_ODP_INLINE odp_event_t odp_dma_compl_to_event(odp_dma_compl_t dma_compl) +{ + return (odp_event_t)(uintptr_t)dma_compl; +} + +/** @endcond */ + +#endif diff --git a/platform/linux-generic/odp_dma.c b/platform/linux-generic/odp_dma.c index fa45b3b02..2a0a56849 100644 --- a/platform/linux-generic/odp_dma.c +++ b/platform/linux-generic/odp_dma.c @@ -803,18 +803,6 @@ void odp_dma_compl_free(odp_dma_compl_t dma_compl) odp_buffer_free(buf); } -odp_dma_compl_t odp_dma_compl_from_event(odp_event_t ev) -{ - _ODP_ASSERT(odp_event_type(ev) == ODP_EVENT_DMA_COMPL); - - return (odp_dma_compl_t)(uintptr_t)ev; -} - -odp_event_t odp_dma_compl_to_event(odp_dma_compl_t dma_compl) -{ - return (odp_event_t)(uintptr_t)dma_compl; -} - int odp_dma_compl_result(odp_dma_compl_t dma_compl, odp_dma_result_t *result_out) { odp_dma_result_t *result; diff --git a/platform/linux-generic/odp_dma_api.c b/platform/linux-generic/odp_dma_api.c new file mode 100644 index 000000000..1e1d5d91f --- /dev/null +++ b/platform/linux-generic/odp_dma_api.c @@ -0,0 +1,11 @@ +/* Copyright (c) 2023, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include + +/* Non-inlined functions for ABI compat mode */ +#define _ODP_NO_INLINE +#include -- cgit v1.2.3 From 53a1ae930a2346867d34f5dabf45de5aabab3864 Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Mon, 12 Jun 2023 17:28:46 +0300 Subject: linux-gen: dma: inline small completion event functions Inline implementations of odp_dma_compl_user_area() and odp_dma_compl_result(). odp_event_user_area() implementation has been modified to avoid circular dependency. Signed-off-by: Matias Elo Reviewed-by: Tuomas Taipale --- .../include/odp/api/plat/dma_inlines.h | 28 ++++++++++++++++++++++ .../include/odp/api/plat/event_inlines.h | 4 +--- platform/linux-generic/odp_dma.c | 23 ------------------ 3 files changed, 29 insertions(+), 26 deletions(-) (limited to 'platform') diff --git a/platform/linux-generic/include/odp/api/plat/dma_inlines.h b/platform/linux-generic/include/odp/api/plat/dma_inlines.h index 9d9f74654..74d04c8b0 100644 --- a/platform/linux-generic/include/odp/api/plat/dma_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/dma_inlines.h @@ -7,13 +7,16 @@ #ifndef ODP_PLAT_DMA_INLINES_H_ #define ODP_PLAT_DMA_INLINES_H_ +#include #include #include +#include #include #include #include +#include /** @cond _ODP_HIDE_FROM_DOXYGEN_ */ @@ -22,6 +25,8 @@ #define _ODP_INLINE static inline #define odp_dma_compl_from_event __odp_dma_compl_from_event #define odp_dma_compl_to_event __odp_dma_compl_to_event + #define odp_dma_compl_user_area __odp_dma_compl_user_area + #define odp_dma_compl_result __odp_dma_compl_result #else #define _ODP_INLINE #endif @@ -38,6 +43,29 @@ _ODP_INLINE odp_event_t odp_dma_compl_to_event(odp_dma_compl_t dma_compl) return (odp_event_t)(uintptr_t)dma_compl; } +_ODP_INLINE void *odp_dma_compl_user_area(odp_dma_compl_t dma_compl) +{ + return odp_buffer_user_area((odp_buffer_t)(uintptr_t)dma_compl); +} + +_ODP_INLINE int odp_dma_compl_result(odp_dma_compl_t dma_compl, odp_dma_result_t *result_out) +{ + odp_dma_result_t *result; + odp_buffer_t buf = (odp_buffer_t)(uintptr_t)dma_compl; + + if (odp_unlikely(dma_compl == ODP_DMA_COMPL_INVALID)) { + _ODP_ERR("Bad DMA compl handle\n"); + return -1; + } + + result = (odp_dma_result_t *)odp_buffer_addr(buf); + + if (result_out) + *result_out = *result; + + return result->success ? 0 : -1; +} + /** @endcond */ #endif diff --git a/platform/linux-generic/include/odp/api/plat/event_inlines.h b/platform/linux-generic/include/odp/api/plat/event_inlines.h index 4e3368ff0..a353ab0b8 100644 --- a/platform/linux-generic/include/odp/api/plat/event_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/event_inlines.h @@ -9,7 +9,6 @@ #define ODP_PLAT_EVENT_INLINES_H_ #include -#include #include #include #include @@ -72,6 +71,7 @@ _ODP_INLINE void *odp_event_user_area(odp_event_t event) switch (type) { case ODP_EVENT_BUFFER: + case ODP_EVENT_DMA_COMPL: return _odp_buffer_get((odp_buffer_t)event, void *, uarea_addr); case ODP_EVENT_PACKET: return _odp_pkt_get((odp_packet_t)event, void *, user_area); @@ -79,8 +79,6 @@ _ODP_INLINE void *odp_event_user_area(odp_event_t event) return _odp_event_vect_get((odp_packet_vector_t)event, void *, uarea_addr); case ODP_EVENT_TIMEOUT: return _odp_timeout_hdr_field((odp_timeout_t)event, void *, uarea_addr); - case ODP_EVENT_DMA_COMPL: - return odp_dma_compl_user_area((odp_dma_compl_t)event); default: return NULL; } diff --git a/platform/linux-generic/odp_dma.c b/platform/linux-generic/odp_dma.c index 2a0a56849..ac3f6eb7c 100644 --- a/platform/linux-generic/odp_dma.c +++ b/platform/linux-generic/odp_dma.c @@ -803,24 +803,6 @@ void odp_dma_compl_free(odp_dma_compl_t dma_compl) odp_buffer_free(buf); } -int odp_dma_compl_result(odp_dma_compl_t dma_compl, odp_dma_result_t *result_out) -{ - odp_dma_result_t *result; - odp_buffer_t buf = (odp_buffer_t)(uintptr_t)dma_compl; - - if (odp_unlikely(dma_compl == ODP_DMA_COMPL_INVALID)) { - _ODP_ERR("Bad DMA compl handle\n"); - return -1; - } - - result = odp_buffer_addr(buf); - - if (result_out) - *result_out = *result; - - return result->success ? 0 : -1; -} - uint64_t odp_dma_to_u64(odp_dma_t dma) { return (uint64_t)(uintptr_t)dma; @@ -831,11 +813,6 @@ uint64_t odp_dma_compl_to_u64(odp_dma_compl_t dma_compl) return (uint64_t)(uintptr_t)dma_compl; } -void *odp_dma_compl_user_area(odp_dma_compl_t dma_compl) -{ - return odp_buffer_user_area((odp_buffer_t)(uintptr_t)dma_compl); -} - void odp_dma_print(odp_dma_t dma) { dma_session_t *session = dma_session_from_handle(dma); -- cgit v1.2.3 From 6b7ba3932ace5550004471c5c9acee6a42cdbd92 Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Mon, 12 Jun 2023 17:30:54 +0300 Subject: linux-gen: dma: inline parameter init functions Inline implementations of fast path parameter init functions odp_dma_transfer_param_init() and odp_dma_compl_param_init(). Signed-off-by: Matias Elo Reviewed-by: Tuomas Taipale --- .../include/odp/api/plat/dma_inlines.h | 22 ++++++++++++++++++++++ platform/linux-generic/odp_dma.c | 18 ------------------ 2 files changed, 22 insertions(+), 18 deletions(-) (limited to 'platform') diff --git a/platform/linux-generic/include/odp/api/plat/dma_inlines.h b/platform/linux-generic/include/odp/api/plat/dma_inlines.h index 74d04c8b0..6c41015b9 100644 --- a/platform/linux-generic/include/odp/api/plat/dma_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/dma_inlines.h @@ -11,6 +11,7 @@ #include #include #include +#include #include #include @@ -27,6 +28,8 @@ #define odp_dma_compl_to_event __odp_dma_compl_to_event #define odp_dma_compl_user_area __odp_dma_compl_user_area #define odp_dma_compl_result __odp_dma_compl_result + #define odp_dma_transfer_param_init __odp_dma_transfer_param_init + #define odp_dma_compl_param_init __odp_dma_compl_param_init #else #define _ODP_INLINE #endif @@ -66,6 +69,25 @@ _ODP_INLINE int odp_dma_compl_result(odp_dma_compl_t dma_compl, odp_dma_result_t return result->success ? 0 : -1; } +_ODP_INLINE void odp_dma_transfer_param_init(odp_dma_transfer_param_t *trs_param) +{ + memset(trs_param, 0, sizeof(odp_dma_transfer_param_t)); + + trs_param->src_format = ODP_DMA_FORMAT_ADDR; + trs_param->dst_format = ODP_DMA_FORMAT_ADDR; + trs_param->num_src = 1; + trs_param->num_dst = 1; +} + +_ODP_INLINE void odp_dma_compl_param_init(odp_dma_compl_param_t *compl_param) +{ + memset(compl_param, 0, sizeof(odp_dma_compl_param_t)); + + compl_param->queue = ODP_QUEUE_INVALID; + compl_param->event = ODP_EVENT_INVALID; + compl_param->transfer_id = ODP_DMA_TRANSFER_ID_INVALID; +} + /** @endcond */ #endif diff --git a/platform/linux-generic/odp_dma.c b/platform/linux-generic/odp_dma.c index ac3f6eb7c..b7e513db9 100644 --- a/platform/linux-generic/odp_dma.c +++ b/platform/linux-generic/odp_dma.c @@ -321,16 +321,6 @@ odp_dma_t odp_dma_lookup(const char *name) return ODP_DMA_INVALID; } -void odp_dma_transfer_param_init(odp_dma_transfer_param_t *trs_param) -{ - memset(trs_param, 0, sizeof(odp_dma_transfer_param_t)); - - trs_param->src_format = ODP_DMA_FORMAT_ADDR; - trs_param->dst_format = ODP_DMA_FORMAT_ADDR; - trs_param->num_src = 1; - trs_param->num_dst = 1; -} - static uint32_t transfer_len(const odp_dma_transfer_param_t *trs_param) { uint32_t i; @@ -568,14 +558,6 @@ int odp_dma_transfer_multi(odp_dma_t dma, const odp_dma_transfer_param_t *trs_pa return i; } -void odp_dma_compl_param_init(odp_dma_compl_param_t *compl_param) -{ - memset(compl_param, 0, sizeof(odp_dma_compl_param_t)); - compl_param->queue = ODP_QUEUE_INVALID; - compl_param->event = ODP_EVENT_INVALID; - compl_param->transfer_id = ODP_DMA_TRANSFER_ID_INVALID; -} - odp_dma_transfer_id_t odp_dma_transfer_id_alloc(odp_dma_t dma) { int32_t num; -- cgit v1.2.3 From 6990f9a52f511a693a771cea050de7e7eb590e4d Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Mon, 12 Jun 2023 17:32:28 +0300 Subject: linux-gen: dma: inline completion event alloc/free functions Inline implementations of odp_dma_compl_alloc() and odp_dma_compl_free() functions. Signed-off-by: Matias Elo Reviewed-by: Tuomas Taipale --- .../include/odp/api/plat/dma_inlines.h | 42 ++++++++++++++++++++++ platform/linux-generic/odp_dma.c | 36 ------------------- 2 files changed, 42 insertions(+), 36 deletions(-) (limited to 'platform') diff --git a/platform/linux-generic/include/odp/api/plat/dma_inlines.h b/platform/linux-generic/include/odp/api/plat/dma_inlines.h index 6c41015b9..84b5fef5b 100644 --- a/platform/linux-generic/include/odp/api/plat/dma_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/dma_inlines.h @@ -11,6 +11,7 @@ #include #include #include +#include #include #include @@ -30,6 +31,8 @@ #define odp_dma_compl_result __odp_dma_compl_result #define odp_dma_transfer_param_init __odp_dma_transfer_param_init #define odp_dma_compl_param_init __odp_dma_compl_param_init + #define odp_dma_compl_alloc __odp_dma_compl_alloc + #define odp_dma_compl_free __odp_dma_compl_free #else #define _ODP_INLINE #endif @@ -88,6 +91,45 @@ _ODP_INLINE void odp_dma_compl_param_init(odp_dma_compl_param_t *compl_param) compl_param->transfer_id = ODP_DMA_TRANSFER_ID_INVALID; } +_ODP_INLINE odp_dma_compl_t odp_dma_compl_alloc(odp_pool_t pool) +{ + odp_buffer_t buf; + odp_event_t ev; + odp_dma_result_t *result; + int8_t *ev_type; + + buf = odp_buffer_alloc(pool); + if (odp_unlikely(buf == ODP_BUFFER_INVALID)) + return ODP_DMA_COMPL_INVALID; + + result = (odp_dma_result_t *)odp_buffer_addr(buf); + memset(result, 0, sizeof(odp_dma_result_t)); + + ev = odp_buffer_to_event(buf); + ev_type = _odp_event_hdr_ptr(ev, int8_t, event_type); + *ev_type = ODP_EVENT_DMA_COMPL; + + return (odp_dma_compl_t)(uintptr_t)buf; +} + +_ODP_INLINE void odp_dma_compl_free(odp_dma_compl_t dma_compl) +{ + int8_t *ev_type; + odp_event_t ev; + odp_buffer_t buf = (odp_buffer_t)(uintptr_t)dma_compl; + + if (odp_unlikely(dma_compl == ODP_DMA_COMPL_INVALID)) { + _ODP_ERR("Bad DMA compl handle\n"); + return; + } + + ev = odp_buffer_to_event(buf); + ev_type = _odp_event_hdr_ptr(ev, int8_t, event_type); + *ev_type = ODP_EVENT_BUFFER; + + odp_buffer_free(buf); +} + /** @endcond */ #endif diff --git a/platform/linux-generic/odp_dma.c b/platform/linux-generic/odp_dma.c index b7e513db9..eef7e5c02 100644 --- a/platform/linux-generic/odp_dma.c +++ b/platform/linux-generic/odp_dma.c @@ -749,42 +749,6 @@ odp_pool_t odp_dma_pool_create(const char *name, const odp_dma_pool_param_t *dma return pool; } -odp_dma_compl_t odp_dma_compl_alloc(odp_pool_t pool) -{ - odp_buffer_t buf; - odp_event_t ev; - odp_dma_result_t *result; - - buf = odp_buffer_alloc(pool); - - if (odp_unlikely(buf == ODP_BUFFER_INVALID)) - return ODP_DMA_COMPL_INVALID; - - result = odp_buffer_addr(buf); - memset(result, 0, sizeof(odp_dma_result_t)); - - ev = odp_buffer_to_event(buf); - _odp_event_type_set(ev, ODP_EVENT_DMA_COMPL); - - return (odp_dma_compl_t)(uintptr_t)buf; -} - -void odp_dma_compl_free(odp_dma_compl_t dma_compl) -{ - odp_event_t ev; - odp_buffer_t buf = (odp_buffer_t)(uintptr_t)dma_compl; - - if (odp_unlikely(dma_compl == ODP_DMA_COMPL_INVALID)) { - _ODP_ERR("Bad DMA compl handle\n"); - return; - } - - ev = odp_buffer_to_event(buf); - _odp_event_type_set(ev, ODP_EVENT_BUFFER); - - odp_buffer_free(buf); -} - uint64_t odp_dma_to_u64(odp_dma_t dma) { return (uint64_t)(uintptr_t)dma; -- cgit v1.2.3 From 72683c2804d9a3c3e63d92c45d1e8cab95ef8be8 Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Tue, 13 Jun 2023 16:55:57 +0300 Subject: linux-gen: time: refactor implementation Refactor time codebase to enable more optimized architecture specific implementations. After this commit x86 and aarch64 architectures are assumed to always have support for HW time. The fallback to POSIX time functions has been removed from these architectures for improved performance. Signed-off-by: Matias Elo Reviewed-by: Petri Savolainen --- platform/linux-generic/Makefile.am | 57 +++--- .../arch/aarch64/odp/api/abi/cpu_time.h | 50 ----- .../arch/aarch64/odp/api/abi/time_cpu.h | 53 +++++ .../arch/aarch64/odp/api/abi/time_inlines.h | 7 + .../linux-generic/arch/aarch64/odp_global_time.c | 25 --- .../arch/common/odp/api/abi/time_cpu_inlines.h | 93 +++++++++ platform/linux-generic/arch/common/odp_time_cpu.c | 48 +++++ .../arch/default/odp/api/abi/cpu_time.h | 25 --- .../arch/default/odp/api/abi/time_inlines.h | 45 +++++ .../linux-generic/arch/default/odp_global_time.c | 31 --- platform/linux-generic/arch/default/odp_time.c | 102 ++++++++++ platform/linux-generic/arch/x86/cpu_flags.c | 9 +- .../linux-generic/arch/x86/odp/api/abi/cpu_time.h | 35 ---- .../linux-generic/arch/x86/odp/api/abi/time_cpu.h | 35 ++++ .../arch/x86/odp/api/abi/time_inlines.h | 7 + platform/linux-generic/arch/x86/odp_global_time.c | 69 ------- platform/linux-generic/arch/x86/odp_time_cpu.c | 70 +++++++ .../include/odp/api/plat/time_inlines.h | 127 +++++------- platform/linux-generic/odp_time.c | 215 --------------------- 19 files changed, 550 insertions(+), 553 deletions(-) delete mode 100644 platform/linux-generic/arch/aarch64/odp/api/abi/cpu_time.h create mode 100644 platform/linux-generic/arch/aarch64/odp/api/abi/time_cpu.h create mode 100644 platform/linux-generic/arch/aarch64/odp/api/abi/time_inlines.h delete mode 100644 platform/linux-generic/arch/aarch64/odp_global_time.c create mode 100644 platform/linux-generic/arch/common/odp/api/abi/time_cpu_inlines.h create mode 100644 platform/linux-generic/arch/common/odp_time_cpu.c delete mode 100644 platform/linux-generic/arch/default/odp/api/abi/cpu_time.h create mode 100644 platform/linux-generic/arch/default/odp/api/abi/time_inlines.h delete mode 100644 platform/linux-generic/arch/default/odp_global_time.c create mode 100644 platform/linux-generic/arch/default/odp_time.c delete mode 100644 platform/linux-generic/arch/x86/odp/api/abi/cpu_time.h create mode 100644 platform/linux-generic/arch/x86/odp/api/abi/time_cpu.h create mode 100644 platform/linux-generic/arch/x86/odp/api/abi/time_inlines.h delete mode 100644 platform/linux-generic/arch/x86/odp_global_time.c create mode 100644 platform/linux-generic/arch/x86/odp_time_cpu.c delete mode 100644 platform/linux-generic/odp_time.c (limited to 'platform') diff --git a/platform/linux-generic/Makefile.am b/platform/linux-generic/Makefile.am index 6565e18da..4c02c56fa 100644 --- a/platform/linux-generic/Makefile.am +++ b/platform/linux-generic/Makefile.am @@ -10,6 +10,7 @@ AM_CPPFLAGS += -I$(top_builddir)/platform/$(with_platform)/include AM_CPPFLAGS += -I$(top_srcdir)/platform/$(with_platform)/arch AM_CPPFLAGS += -I$(top_srcdir)/platform/$(with_platform)/arch/@ARCH_DIR@ AM_CPPFLAGS += -I$(top_srcdir)/platform/$(with_platform)/arch/default +AM_CPPFLAGS += -I$(top_srcdir)/platform/$(with_platform)/arch/common AM_CPPFLAGS += $(OPENSSL_CPPFLAGS) @@ -257,7 +258,6 @@ __LIB__libodp_linux_la_SOURCES = \ odp_pcapng.c \ odp_thread.c \ odp_thrmask.c \ - odp_time.c \ odp_timer.c \ odp_timer_wheel.c \ odp_traffic_mngr.c \ @@ -328,18 +328,18 @@ endif if ARCH_IS_ARM __LIB__libodp_linux_la_SOURCES += arch/default/odp_atomic.c \ arch/default/odp_cpu_cycles.c \ - arch/default/odp_global_time.c \ arch/default/odp_hash_crc32.c \ arch/default/odp_random.c \ - arch/arm/odp_sysinfo_parse.c -odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_time.h \ - arch/default/odp/api/abi/hash_crc32.h + arch/arm/odp_sysinfo_parse.c \ + arch/default/odp_time.c +odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/hash_crc32.h if !ODP_ABI_COMPAT odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ arch/default/odp/api/abi/atomic_inlines.h \ arch/default/odp/api/abi/cpu_generic.h \ arch/arm/odp/api/abi/cpu_inlines.h \ - arch/arm/odp/api/abi/cpu.h + arch/arm/odp/api/abi/cpu.h \ + arch/default/odp/api/abi/time_inlines.h endif noinst_HEADERS += arch/arm/odp_atomic.h \ arch/arm/odp_cpu.h \ @@ -354,19 +354,21 @@ if ARCH_IS_AARCH64 __LIB__libodp_linux_la_SOURCES += arch/aarch64/odp_atomic.c \ arch/default/odp_cpu_cycles.c \ arch/aarch64/cpu_flags.c \ - arch/aarch64/odp_global_time.c \ arch/default/odp_hash_crc32.c \ arch/default/odp_random.c \ - arch/aarch64/odp_sysinfo_parse.c -odpapiabiarchinclude_HEADERS += arch/aarch64/odp/api/abi/cpu_time.h \ - arch/aarch64/odp/api/abi/hash_crc32.h + arch/aarch64/odp_sysinfo_parse.c \ + arch/common/odp_time_cpu.c +odpapiabiarchinclude_HEADERS += arch/aarch64/odp/api/abi/hash_crc32.h \ + arch/aarch64/odp/api/abi/time_cpu.h if !ODP_ABI_COMPAT odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ arch/aarch64/odp/api/abi/atomic_inlines.h \ arch/aarch64/odp/api/abi/atomic.h \ arch/default/odp/api/abi/cpu_generic.h \ arch/aarch64/odp/api/abi/cpu_inlines.h \ - arch/aarch64/odp/api/abi/cpu.h + arch/aarch64/odp/api/abi/cpu.h \ + arch/common/odp/api/abi/time_cpu_inlines.h \ + arch/aarch64/odp/api/abi/time_inlines.h endif noinst_HEADERS += arch/aarch64/odp_atomic.h \ arch/aarch64/odp_cpu.h \ @@ -378,18 +380,18 @@ endif if ARCH_IS_DEFAULT __LIB__libodp_linux_la_SOURCES += arch/default/odp_atomic.c \ arch/default/odp_cpu_cycles.c \ - arch/default/odp_global_time.c \ arch/default/odp_hash_crc32.c \ arch/default/odp_random.c \ - arch/default/odp_sysinfo_parse.c -odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_time.h \ - arch/default/odp/api/abi/hash_crc32.h + arch/default/odp_sysinfo_parse.c \ + arch/default/odp_time.c +odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/hash_crc32.h if !ODP_ABI_COMPAT odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ arch/default/odp/api/abi/atomic_inlines.h \ arch/default/odp/api/abi/cpu_generic.h \ arch/default/odp/api/abi/cpu_inlines.h \ - arch/default/odp/api/abi/cpu.h + arch/default/odp/api/abi/cpu.h \ + arch/default/odp/api/abi/time_inlines.h endif noinst_HEADERS += arch/default/odp_atomic.h \ arch/default/odp_cpu.h \ @@ -399,18 +401,18 @@ endif if ARCH_IS_POWERPC __LIB__libodp_linux_la_SOURCES += arch/default/odp_atomic.c \ arch/default/odp_cpu_cycles.c \ - arch/default/odp_global_time.c \ arch/default/odp_hash_crc32.c \ arch/default/odp_random.c \ - arch/powerpc/odp_sysinfo_parse.c -odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_time.h \ - arch/default/odp/api/abi/hash_crc32.h + arch/powerpc/odp_sysinfo_parse.c \ + arch/default/odp_time.c +odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/hash_crc32.h if !ODP_ABI_COMPAT odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ arch/default/odp/api/abi/atomic_inlines.h \ arch/default/odp/api/abi/cpu_generic.h \ arch/default/odp/api/abi/cpu_inlines.h \ - arch/powerpc/odp/api/abi/cpu.h + arch/powerpc/odp/api/abi/cpu.h \ + arch/default/odp/api/abi/time_inlines.h endif noinst_HEADERS += arch/default/odp_atomic.h \ arch/default/odp_cpu.h \ @@ -421,18 +423,21 @@ if ARCH_IS_X86 __LIB__libodp_linux_la_SOURCES += arch/default/odp_atomic.c \ arch/x86/cpu_flags.c \ arch/x86/odp_cpu_cycles.c \ - arch/x86/odp_global_time.c \ arch/default/odp_hash_crc32.c \ arch/default/odp_random.c \ - arch/x86/odp_sysinfo_parse.c + arch/x86/odp_sysinfo_parse.c \ + arch/x86/odp_time_cpu.c \ + arch/common/odp_time_cpu.c odpapiabiarchinclude_HEADERS += arch/x86/odp/api/abi/cpu_rdtsc.h \ - arch/x86/odp/api/abi/cpu_time.h \ - arch/x86/odp/api/abi/hash_crc32.h + arch/x86/odp/api/abi/hash_crc32.h \ + arch/x86/odp/api/abi/time_cpu.h if !ODP_ABI_COMPAT odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ arch/default/odp/api/abi/atomic_inlines.h \ arch/x86/odp/api/abi/cpu_inlines.h \ - arch/x86/odp/api/abi/cpu.h + arch/x86/odp/api/abi/cpu.h \ + arch/common/odp/api/abi/time_cpu_inlines.h \ + arch/x86/odp/api/abi/time_inlines.h endif noinst_HEADERS += arch/x86/cpu_flags.h \ arch/x86/odp_cpu.h \ diff --git a/platform/linux-generic/arch/aarch64/odp/api/abi/cpu_time.h b/platform/linux-generic/arch/aarch64/odp/api/abi/cpu_time.h deleted file mode 100644 index 781ee683c..000000000 --- a/platform/linux-generic/arch/aarch64/odp/api/abi/cpu_time.h +++ /dev/null @@ -1,50 +0,0 @@ -/* Copyright (c) 2021, Nokia - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -#ifndef ODP_API_ABI_CPU_TIME_H_ -#define ODP_API_ABI_CPU_TIME_H_ - -#ifdef __cplusplus -extern "C" { -#endif - -#include - -static inline uint64_t _odp_cpu_global_time(void) -{ - uint64_t cntvct; - - __asm__ volatile("mrs %0, cntvct_el0" : "=r"(cntvct) : : "memory"); - - return cntvct; -} - -static inline uint64_t _odp_cpu_global_time_strict(void) -{ - uint64_t cntvct; - - __asm__ volatile("isb" ::: "memory"); - __asm__ volatile("mrs %0, cntvct_el0" : "=r"(cntvct) : : "memory"); - - return cntvct; -} - -static inline uint64_t _odp_cpu_global_time_freq(void) -{ - uint64_t cntfrq; - - __asm__ volatile("mrs %0, cntfrq_el0" : "=r"(cntfrq) : : ); - - return cntfrq; -} - -int _odp_cpu_has_global_time(void); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/platform/linux-generic/arch/aarch64/odp/api/abi/time_cpu.h b/platform/linux-generic/arch/aarch64/odp/api/abi/time_cpu.h new file mode 100644 index 000000000..aba2799c7 --- /dev/null +++ b/platform/linux-generic/arch/aarch64/odp/api/abi/time_cpu.h @@ -0,0 +1,53 @@ +/* Copyright (c) 2021, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_API_ABI_TIME_CPU_H_ +#define ODP_API_ABI_TIME_CPU_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include + +static inline uint64_t _odp_time_cpu_global(void) +{ + uint64_t cntvct; + + __asm__ volatile("mrs %0, cntvct_el0" : "=r"(cntvct) : : "memory"); + + return cntvct; +} + +static inline uint64_t _odp_time_cpu_global_strict(void) +{ + uint64_t cntvct; + + __asm__ volatile("isb" ::: "memory"); + __asm__ volatile("mrs %0, cntvct_el0" : "=r"(cntvct) : : "memory"); + + return cntvct; +} + +static inline uint64_t _odp_time_cpu_global_freq(void) +{ + uint64_t cntfrq; + + __asm__ volatile("mrs %0, cntfrq_el0" : "=r"(cntfrq) : : ); + + return cntfrq; +} + +static inline int _odp_time_cpu_global_freq_is_const(void) +{ + return 1; +} + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/arch/aarch64/odp/api/abi/time_inlines.h b/platform/linux-generic/arch/aarch64/odp/api/abi/time_inlines.h new file mode 100644 index 000000000..331d1996f --- /dev/null +++ b/platform/linux-generic/arch/aarch64/odp/api/abi/time_inlines.h @@ -0,0 +1,7 @@ +/* Copyright (c) 2023, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include diff --git a/platform/linux-generic/arch/aarch64/odp_global_time.c b/platform/linux-generic/arch/aarch64/odp_global_time.c deleted file mode 100644 index 53561b00c..000000000 --- a/platform/linux-generic/arch/aarch64/odp_global_time.c +++ /dev/null @@ -1,25 +0,0 @@ -/* Copyright (c) 2015-2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -#include - -int _odp_cpu_has_global_time(void) -{ - uint64_t hz = _odp_cpu_global_time_freq(); - - /* - * The system counter portion of the architected timer must - * provide a uniform view of system time to all processing - * elements in the system. This should hold true even for - * heterogeneous SoCs. - * - * Determine whether the system has 'global time' by checking - * whether a read of the architected timer frequency sys reg - * returns a sane value. Sane is considered to be within - * 1MHz and 6GHz (1us and .1667ns period). - */ - return hz >= 1000000 && hz <= 6000000000; -} diff --git a/platform/linux-generic/arch/common/odp/api/abi/time_cpu_inlines.h b/platform/linux-generic/arch/common/odp/api/abi/time_cpu_inlines.h new file mode 100644 index 000000000..c154c5f1a --- /dev/null +++ b/platform/linux-generic/arch/common/odp/api/abi/time_cpu_inlines.h @@ -0,0 +1,93 @@ +/* Copyright (c) 2013-2018, Linaro Limited + * Copyright (c) 2020-2023, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_ARCH_TIME_CPU_INLINES_H_ +#define ODP_ARCH_TIME_CPU_INLINES_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include + +#include + +#include + +#define _ODP_TIME_GIGA_HZ 1000000000ULL + +typedef struct _odp_time_global_t { + uint64_t start_time; + uint64_t freq_hz; + +} _odp_time_global_t; + +extern _odp_time_global_t _odp_time_glob; + +static inline odp_time_t _odp_time_cur(void) +{ + odp_time_t time; + + time.count = _odp_time_cpu_global() - _odp_time_glob.start_time; + return time; +} + +static inline odp_time_t _odp_time_cur_strict(void) +{ + odp_time_t time; + + time.count = _odp_time_cpu_global_strict() - _odp_time_glob.start_time; + return time; +} + +static inline uint64_t _odp_time_to_ns(odp_time_t time) +{ + uint64_t nsec; + uint64_t freq_hz = _odp_time_glob.freq_hz; + uint64_t count = time.count; + uint64_t sec = 0; + + if (count >= freq_hz) { + sec = count / freq_hz; + count = count - sec * freq_hz; + } + + nsec = (_ODP_TIME_GIGA_HZ * count) / freq_hz; + + return (sec * _ODP_TIME_GIGA_HZ) + nsec; +} + +static inline odp_time_t _odp_time_from_ns(uint64_t ns) +{ + odp_time_t time; + uint64_t count; + uint64_t freq_hz = _odp_time_glob.freq_hz; + uint64_t sec = 0; + + if (ns >= ODP_TIME_SEC_IN_NS) { + sec = ns / ODP_TIME_SEC_IN_NS; + ns = ns - sec * ODP_TIME_SEC_IN_NS; + } + + count = sec * freq_hz; + count += (ns * freq_hz) / ODP_TIME_SEC_IN_NS; + + time.count = count; + + return time; +} + +static inline uint64_t _odp_time_res(void) +{ + return _odp_time_glob.freq_hz; +} + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/arch/common/odp_time_cpu.c b/platform/linux-generic/arch/common/odp_time_cpu.c new file mode 100644 index 000000000..bbfe82e21 --- /dev/null +++ b/platform/linux-generic/arch/common/odp_time_cpu.c @@ -0,0 +1,48 @@ +/* Copyright (c) 2013-2018, Linaro Limited + * Copyright (c) 2020-2023, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include + +#include +#include + +#include +#include + +#include +#include +#include + +#include + +_odp_time_global_t _odp_time_glob; + +#include + +int _odp_time_init_global(void) +{ + _odp_time_global_t *global = &_odp_time_glob; + + memset(global, 0, sizeof(_odp_time_global_t)); + + if (!_odp_time_cpu_global_freq_is_const()) + return -1; + + global->freq_hz = _odp_time_cpu_global_freq(); + if (global->freq_hz == 0) + return -1; + + _ODP_PRINT("HW time counter freq: %" PRIu64 " hz\n\n", global->freq_hz); + + global->start_time = _odp_time_cpu_global(); + return 0; +} + +int _odp_time_term_global(void) +{ + return 0; +} diff --git a/platform/linux-generic/arch/default/odp/api/abi/cpu_time.h b/platform/linux-generic/arch/default/odp/api/abi/cpu_time.h deleted file mode 100644 index 24e1c7d33..000000000 --- a/platform/linux-generic/arch/default/odp/api/abi/cpu_time.h +++ /dev/null @@ -1,25 +0,0 @@ -/* Copyright (c) 2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -#ifndef ODP_ARCH_CPU_TIME_H_ -#define ODP_ARCH_CPU_TIME_H_ - -#ifdef __cplusplus -extern "C" { -#endif - -#include - -int _odp_cpu_has_global_time(void); -uint64_t _odp_cpu_global_time(void); -uint64_t _odp_cpu_global_time_strict(void); -uint64_t _odp_cpu_global_time_freq(void); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/platform/linux-generic/arch/default/odp/api/abi/time_inlines.h b/platform/linux-generic/arch/default/odp/api/abi/time_inlines.h new file mode 100644 index 000000000..b38e52dac --- /dev/null +++ b/platform/linux-generic/arch/default/odp/api/abi/time_inlines.h @@ -0,0 +1,45 @@ +/* Copyright (c) 2013-2018, Linaro Limited + * Copyright (c) 2020-2023, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_ARCH_TIME_INLINES_H_ +#define ODP_ARCH_TIME_INLINES_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include + +#include + +odp_time_t _odp_time_cur(void); +uint64_t _odp_time_res(void); + +static inline odp_time_t _odp_time_cur_strict(void) +{ + return _odp_time_cur(); +} + +static inline uint64_t _odp_time_to_ns(odp_time_t time) +{ + return time.nsec; +} + +static inline odp_time_t _odp_time_from_ns(uint64_t ns) +{ + odp_time_t time; + + time.nsec = ns; + + return time; +} + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/arch/default/odp_global_time.c b/platform/linux-generic/arch/default/odp_global_time.c deleted file mode 100644 index ee835413f..000000000 --- a/platform/linux-generic/arch/default/odp_global_time.c +++ /dev/null @@ -1,31 +0,0 @@ -/* Copyright (c) 2015-2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -#include - -#include - -uint64_t _odp_cpu_global_time(void) -{ - return 0; -} - -uint64_t _odp_cpu_global_time_strict(void) -{ - return 0; -} - -#include - -int _odp_cpu_has_global_time(void) -{ - return 0; -} - -uint64_t _odp_cpu_global_time_freq(void) -{ - return 0; -} diff --git a/platform/linux-generic/arch/default/odp_time.c b/platform/linux-generic/arch/default/odp_time.c new file mode 100644 index 000000000..919a3ba68 --- /dev/null +++ b/platform/linux-generic/arch/default/odp_time.c @@ -0,0 +1,102 @@ +/* Copyright (c) 2013-2018, Linaro Limited + * Copyright (c) 2020-2023, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include + +#include +#include +#include + +#include + +#include +#include + +#include +#include +#include + +typedef struct _odp_time_global_t { + struct timespec start_time; + +} _odp_time_global_t; + +_odp_time_global_t _odp_time_glob; + +static inline uint64_t time_diff_nsec(struct timespec *t2, struct timespec *t1) +{ + struct timespec diff; + uint64_t nsec; + + diff.tv_sec = t2->tv_sec - t1->tv_sec; + diff.tv_nsec = t2->tv_nsec - t1->tv_nsec; + + if (diff.tv_nsec < 0) { + diff.tv_nsec += ODP_TIME_SEC_IN_NS; + diff.tv_sec -= 1; + } + + nsec = (diff.tv_sec * ODP_TIME_SEC_IN_NS) + diff.tv_nsec; + + return nsec; +} + +#include + +odp_time_t _odp_time_cur(void) +{ + int ret; + odp_time_t time; + struct timespec sys_time; + struct timespec *start_time = &_odp_time_glob.start_time; + + ret = clock_gettime(CLOCK_MONOTONIC_RAW, &sys_time); + if (odp_unlikely(ret != 0)) + _ODP_ABORT("clock_gettime() failed\n"); + + time.nsec = time_diff_nsec(&sys_time, start_time); + + return time; +} + +uint64_t _odp_time_res(void) +{ + int ret; + struct timespec tres; + + ret = clock_getres(CLOCK_MONOTONIC_RAW, &tres); + if (odp_unlikely(ret != 0)) + _ODP_ABORT("clock_getres() failed\n"); + + return ODP_TIME_SEC_IN_NS / (uint64_t)tres.tv_nsec; +} + +#include + +int _odp_time_init_global(void) +{ + struct timespec *start_time; + int ret = 0; + _odp_time_global_t *global = &_odp_time_glob; + + memset(global, 0, sizeof(_odp_time_global_t)); + + start_time = &global->start_time; + start_time->tv_sec = 0; + start_time->tv_nsec = 0; + + ret = clock_gettime(CLOCK_MONOTONIC_RAW, start_time); + if (ret) + _ODP_ERR("clock_gettime() failed: %d\n", ret); + + return ret; +} + +int _odp_time_term_global(void) +{ + return 0; +} diff --git a/platform/linux-generic/arch/x86/cpu_flags.c b/platform/linux-generic/arch/x86/cpu_flags.c index 13753febb..482c25739 100644 --- a/platform/linux-generic/arch/x86/cpu_flags.c +++ b/platform/linux-generic/arch/x86/cpu_flags.c @@ -11,7 +11,7 @@ #include "cpu_flags.h" #include -#include +#include #include #include @@ -354,12 +354,15 @@ void _odp_cpu_flags_print_all(void) _ODP_PRINT("%s", str); } -int _odp_cpu_has_global_time(void) +int _odp_time_cpu_global_freq_is_const(void) { if (cpu_get_flag_enabled(RTE_CPUFLAG_INVTSC) > 0) return 1; - return 0; + _ODP_ERR("WARN: assuming constant TSC based on CPU arch, but could not confirm from CPU " + "flags\n"); + + return 1; } int _odp_cpu_flags_has_rdtsc(void) diff --git a/platform/linux-generic/arch/x86/odp/api/abi/cpu_time.h b/platform/linux-generic/arch/x86/odp/api/abi/cpu_time.h deleted file mode 100644 index c74c4d606..000000000 --- a/platform/linux-generic/arch/x86/odp/api/abi/cpu_time.h +++ /dev/null @@ -1,35 +0,0 @@ -/* Copyright (c) 2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -#ifndef ODP_ARCH_CPU_TIME_H_ -#define ODP_ARCH_CPU_TIME_H_ - -#ifdef __cplusplus -extern "C" { -#endif - -#include -#include - -static inline uint64_t _odp_cpu_global_time(void) -{ - return _odp_cpu_rdtsc(); -} - -static inline uint64_t _odp_cpu_global_time_strict(void) -{ - __atomic_thread_fence(__ATOMIC_SEQ_CST); - return _odp_cpu_rdtsc(); -} - -int _odp_cpu_has_global_time(void); -uint64_t _odp_cpu_global_time_freq(void); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/platform/linux-generic/arch/x86/odp/api/abi/time_cpu.h b/platform/linux-generic/arch/x86/odp/api/abi/time_cpu.h new file mode 100644 index 000000000..baf79ad3f --- /dev/null +++ b/platform/linux-generic/arch/x86/odp/api/abi/time_cpu.h @@ -0,0 +1,35 @@ +/* Copyright (c) 2018, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_ARCH_TIME_CPU_H_ +#define ODP_ARCH_TIME_CPU_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include + +static inline uint64_t _odp_time_cpu_global(void) +{ + return _odp_cpu_rdtsc(); +} + +static inline uint64_t _odp_time_cpu_global_strict(void) +{ + __atomic_thread_fence(__ATOMIC_SEQ_CST); + return _odp_cpu_rdtsc(); +} + +int _odp_time_cpu_global_freq_is_const(void); +uint64_t _odp_time_cpu_global_freq(void); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/arch/x86/odp/api/abi/time_inlines.h b/platform/linux-generic/arch/x86/odp/api/abi/time_inlines.h new file mode 100644 index 000000000..331d1996f --- /dev/null +++ b/platform/linux-generic/arch/x86/odp/api/abi/time_inlines.h @@ -0,0 +1,7 @@ +/* Copyright (c) 2023, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include diff --git a/platform/linux-generic/arch/x86/odp_global_time.c b/platform/linux-generic/arch/x86/odp_global_time.c deleted file mode 100644 index 00e3a1ac1..000000000 --- a/platform/linux-generic/arch/x86/odp_global_time.c +++ /dev/null @@ -1,69 +0,0 @@ -/* Copyright (c) 2015-2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -#include - -#include - -#include -#include -#include - -#define SEC_IN_NS 1000000000ULL - -/* Measure TSC frequency. Frequency information registers are defined for x86, - * but those are often not enumerated. */ -uint64_t _odp_cpu_global_time_freq(void) -{ - struct timespec sleep, ts1, ts2; - uint64_t t1, t2, ts_nsec, cycles, hz; - int i; - uint64_t avg = 0; - int rounds = 3; - int warm_up = 1; - - for (i = 0; i < rounds; i++) { - sleep.tv_sec = 0; - - if (warm_up) - sleep.tv_nsec = SEC_IN_NS / 1000; - else - sleep.tv_nsec = SEC_IN_NS / 4; - - if (clock_gettime(CLOCK_MONOTONIC_RAW, &ts1)) { - _ODP_DBG("clock_gettime failed\n"); - return 0; - } - - t1 = _odp_cpu_global_time(); - - if (nanosleep(&sleep, NULL) < 0) { - _ODP_DBG("nanosleep failed\n"); - return 0; - } - - if (clock_gettime(CLOCK_MONOTONIC_RAW, &ts2)) { - _ODP_DBG("clock_gettime failed\n"); - return 0; - } - - t2 = _odp_cpu_global_time(); - - ts_nsec = (ts2.tv_sec - ts1.tv_sec) * SEC_IN_NS; - ts_nsec += ts2.tv_nsec - ts1.tv_nsec; - - cycles = t2 - t1; - - hz = (cycles * SEC_IN_NS) / ts_nsec; - - if (warm_up) - warm_up = 0; - else - avg += hz; - } - - return avg / (rounds - 1); -} diff --git a/platform/linux-generic/arch/x86/odp_time_cpu.c b/platform/linux-generic/arch/x86/odp_time_cpu.c new file mode 100644 index 000000000..aa00ac04e --- /dev/null +++ b/platform/linux-generic/arch/x86/odp_time_cpu.c @@ -0,0 +1,70 @@ +/* Copyright (c) 2015-2018, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include + +#include +#include + +#include + +#include + +#include + +/* Measure TSC frequency. Frequency information registers are defined for x86, + * but those are often not enumerated. */ +uint64_t _odp_time_cpu_global_freq(void) +{ + struct timespec sleep, ts1, ts2; + uint64_t t1, t2, ts_nsec, cycles, hz; + int i; + uint64_t avg = 0; + int rounds = 3; + int warm_up = 1; + + for (i = 0; i < rounds; i++) { + sleep.tv_sec = 0; + + if (warm_up) + sleep.tv_nsec = ODP_TIME_SEC_IN_NS / 1000; + else + sleep.tv_nsec = ODP_TIME_SEC_IN_NS / 4; + + if (clock_gettime(CLOCK_MONOTONIC_RAW, &ts1)) { + _ODP_ERR("clock_gettime() failed\n"); + return 0; + } + + t1 = _odp_time_cpu_global(); + + if (nanosleep(&sleep, NULL) < 0) { + _ODP_ERR("nanosleep() failed\n"); + return 0; + } + + if (clock_gettime(CLOCK_MONOTONIC_RAW, &ts2)) { + _ODP_ERR("clock_gettime() failed\n"); + return 0; + } + + t2 = _odp_time_cpu_global(); + + ts_nsec = (ts2.tv_sec - ts1.tv_sec) * ODP_TIME_SEC_IN_NS; + ts_nsec += ts2.tv_nsec - ts1.tv_nsec; + + cycles = t2 - t1; + + hz = (cycles * ODP_TIME_SEC_IN_NS) / ts_nsec; + + if (warm_up) + warm_up = 0; + else + avg += hz; + } + + return avg / (rounds - 1); +} diff --git a/platform/linux-generic/include/odp/api/plat/time_inlines.h b/platform/linux-generic/include/odp/api/plat/time_inlines.h index f8f4bee89..35a35c72e 100644 --- a/platform/linux-generic/include/odp/api/plat/time_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/time_inlines.h @@ -1,5 +1,5 @@ /* Copyright (c) 2018, Linaro Limited - * Copyright (c) 2020-2022, Nokia + * Copyright (c) 2020-2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -12,79 +12,12 @@ #include #include -#include +#include #include /** @cond _ODP_HIDE_FROM_DOXYGEN_ */ -#define _ODP_TIMESPEC_SIZE 16 -#define _ODP_TIME_GIGA_HZ 1000000000ULL - -typedef struct _odp_time_global_t { - /* Storage space for struct timespec. Posix headers are not included - * here to avoid application exposure. */ - uint8_t timespec[_ODP_TIMESPEC_SIZE] ODP_ALIGNED(_ODP_TIMESPEC_SIZE); - - int use_hw; - uint64_t hw_start; - uint64_t hw_freq_hz; - -} _odp_time_global_t; - -extern _odp_time_global_t _odp_time_glob; - -odp_time_t _odp_timespec_cur(void); - -static inline odp_time_t _odp_time_cur(void) -{ - if (_odp_time_glob.use_hw) { - odp_time_t time; - - time.count = _odp_cpu_global_time() - _odp_time_glob.hw_start; - return time; - } - - return _odp_timespec_cur(); -} - -static inline odp_time_t _odp_time_cur_strict(void) -{ - if (_odp_time_glob.use_hw) { - odp_time_t time; - - time.count = _odp_cpu_global_time_strict() - _odp_time_glob.hw_start; - return time; - } - - return _odp_timespec_cur(); -} - -static inline uint64_t _odp_time_hw_to_ns(odp_time_t time) -{ - uint64_t nsec; - uint64_t freq_hz = _odp_time_glob.hw_freq_hz; - uint64_t count = time.count; - uint64_t sec = 0; - - if (count >= freq_hz) { - sec = count / freq_hz; - count = count - sec * freq_hz; - } - - nsec = (_ODP_TIME_GIGA_HZ * count) / freq_hz; - - return (sec * _ODP_TIME_GIGA_HZ) + nsec; -} - -static inline uint64_t _odp_time_convert_to_ns(odp_time_t time) -{ - if (_odp_time_glob.use_hw) - return _odp_time_hw_to_ns(time); - - return time.nsec; -} - #ifndef _ODP_NO_INLINE /* Inline functions by default */ #define _ODP_INLINE static inline @@ -104,6 +37,14 @@ static inline uint64_t _odp_time_convert_to_ns(odp_time_t time) #define odp_time_diff_ns __odp_time_diff_ns #define odp_time_sum __odp_time_sum + #define odp_time_local_from_ns __odp_time_local_from_ns + #define odp_time_global_from_ns __odp_time_global_from_ns + + #define odp_time_local_res __odp_time_local_res + #define odp_time_global_res __odp_time_global_res + + #define odp_time_wait_ns __odp_time_wait_ns + #define odp_time_wait_until __odp_time_wait_until #else #define _ODP_INLINE #endif @@ -130,27 +71,27 @@ _ODP_INLINE odp_time_t odp_time_global_strict(void) _ODP_INLINE uint64_t odp_time_local_ns(void) { - return _odp_time_convert_to_ns(_odp_time_cur()); + return _odp_time_to_ns(_odp_time_cur()); } _ODP_INLINE uint64_t odp_time_global_ns(void) { - return _odp_time_convert_to_ns(_odp_time_cur()); + return _odp_time_to_ns(_odp_time_cur()); } _ODP_INLINE uint64_t odp_time_local_strict_ns(void) { - return _odp_time_convert_to_ns(_odp_time_cur_strict()); + return _odp_time_to_ns(_odp_time_cur_strict()); } _ODP_INLINE uint64_t odp_time_global_strict_ns(void) { - return _odp_time_convert_to_ns(_odp_time_cur_strict()); + return _odp_time_to_ns(_odp_time_cur_strict()); } _ODP_INLINE uint64_t odp_time_to_ns(odp_time_t time) { - return _odp_time_convert_to_ns(time); + return _odp_time_to_ns(time); } _ODP_INLINE int odp_time_cmp(odp_time_t t2, odp_time_t t1) @@ -191,6 +132,44 @@ _ODP_INLINE odp_time_t odp_time_sum(odp_time_t t1, odp_time_t t2) return time; } +_ODP_INLINE odp_time_t odp_time_local_from_ns(uint64_t ns) +{ + return _odp_time_from_ns(ns); +} + +_ODP_INLINE odp_time_t odp_time_global_from_ns(uint64_t ns) +{ + return _odp_time_from_ns(ns); +} + +_ODP_INLINE uint64_t odp_time_local_res(void) +{ + return _odp_time_res(); +} + +_ODP_INLINE uint64_t odp_time_global_res(void) +{ + return _odp_time_res(); +} + +_ODP_INLINE void odp_time_wait_until(odp_time_t time) +{ + odp_time_t cur; + + do { + cur = _odp_time_cur(); + } while (odp_time_cmp(time, cur) > 0); +} + +_ODP_INLINE void odp_time_wait_ns(uint64_t ns) +{ + odp_time_t cur = _odp_time_cur(); + odp_time_t wait = _odp_time_from_ns(ns); + odp_time_t end_time = odp_time_sum(cur, wait); + + odp_time_wait_until(end_time); +} + /** @endcond */ #endif diff --git a/platform/linux-generic/odp_time.c b/platform/linux-generic/odp_time.c deleted file mode 100644 index 3498008f1..000000000 --- a/platform/linux-generic/odp_time.c +++ /dev/null @@ -1,215 +0,0 @@ -/* Copyright (c) 2013-2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -#include - -#include -#include -#include - -#include -#include -#include -#include -#include - -ODP_STATIC_ASSERT(_ODP_TIMESPEC_SIZE >= (sizeof(struct timespec)), - "_ODP_TIMESPEC_SIZE too small"); - -#include - -_odp_time_global_t _odp_time_glob; - -/* - * Posix timespec based functions - */ - -static inline uint64_t time_spec_diff_nsec(struct timespec *t2, - struct timespec *t1) -{ - struct timespec diff; - uint64_t nsec; - - diff.tv_sec = t2->tv_sec - t1->tv_sec; - diff.tv_nsec = t2->tv_nsec - t1->tv_nsec; - - if (diff.tv_nsec < 0) { - diff.tv_nsec += ODP_TIME_SEC_IN_NS; - diff.tv_sec -= 1; - } - - nsec = (diff.tv_sec * ODP_TIME_SEC_IN_NS) + diff.tv_nsec; - - return nsec; -} - -odp_time_t _odp_timespec_cur(void) -{ - int ret; - odp_time_t time; - struct timespec sys_time; - struct timespec *start_time; - - start_time = (struct timespec *)(uintptr_t)&_odp_time_glob.timespec; - - ret = clock_gettime(CLOCK_MONOTONIC_RAW, &sys_time); - if (odp_unlikely(ret != 0)) - _ODP_ABORT("clock_gettime failed\n"); - - time.nsec = time_spec_diff_nsec(&sys_time, start_time); - - return time; -} - -#include - -static inline uint64_t time_spec_res(void) -{ - int ret; - struct timespec tres; - - ret = clock_getres(CLOCK_MONOTONIC_RAW, &tres); - if (odp_unlikely(ret != 0)) - _ODP_ABORT("clock_getres failed\n"); - - return ODP_TIME_SEC_IN_NS / (uint64_t)tres.tv_nsec; -} - -static inline odp_time_t time_spec_from_ns(uint64_t ns) -{ - odp_time_t time; - - time.nsec = ns; - - return time; -} - -/* - * HW time counter based functions - */ - -static inline uint64_t time_hw_res(void) -{ - return _odp_time_glob.hw_freq_hz; -} - -static inline odp_time_t time_hw_from_ns(uint64_t ns) -{ - odp_time_t time; - uint64_t count; - uint64_t freq_hz = _odp_time_glob.hw_freq_hz; - uint64_t sec = 0; - - if (ns >= ODP_TIME_SEC_IN_NS) { - sec = ns / ODP_TIME_SEC_IN_NS; - ns = ns - sec * ODP_TIME_SEC_IN_NS; - } - - count = sec * freq_hz; - count += (ns * freq_hz) / ODP_TIME_SEC_IN_NS; - - time.count = count; - - return time; -} - -/* - * Common functions - */ - -static inline uint64_t time_res(void) -{ - if (_odp_time_glob.use_hw) - return time_hw_res(); - - return time_spec_res(); -} - -static inline odp_time_t time_from_ns(uint64_t ns) -{ - if (_odp_time_glob.use_hw) - return time_hw_from_ns(ns); - - return time_spec_from_ns(ns); -} - -static inline void time_wait_until(odp_time_t time) -{ - odp_time_t cur; - - do { - cur = _odp_time_cur(); - } while (odp_time_cmp(time, cur) > 0); -} - -odp_time_t odp_time_local_from_ns(uint64_t ns) -{ - return time_from_ns(ns); -} - -odp_time_t odp_time_global_from_ns(uint64_t ns) -{ - return time_from_ns(ns); -} - -uint64_t odp_time_local_res(void) -{ - return time_res(); -} - -uint64_t odp_time_global_res(void) -{ - return time_res(); -} - -void odp_time_wait_ns(uint64_t ns) -{ - odp_time_t cur = _odp_time_cur(); - odp_time_t wait = time_from_ns(ns); - odp_time_t end_time = odp_time_sum(cur, wait); - - time_wait_until(end_time); -} - -void odp_time_wait_until(odp_time_t time) -{ - time_wait_until(time); -} - -int _odp_time_init_global(void) -{ - struct timespec *timespec; - int ret = 0; - _odp_time_global_t *global = &_odp_time_glob; - - memset(global, 0, sizeof(_odp_time_global_t)); - - if (_odp_cpu_has_global_time()) { - global->use_hw = 1; - global->hw_freq_hz = _odp_cpu_global_time_freq(); - - if (global->hw_freq_hz == 0) - return -1; - - _ODP_PRINT("HW time counter freq: %" PRIu64 " hz\n\n", global->hw_freq_hz); - - global->hw_start = _odp_cpu_global_time(); - return 0; - } - - timespec = (struct timespec *)(uintptr_t)global->timespec; - timespec->tv_sec = 0; - timespec->tv_nsec = 0; - - ret = clock_gettime(CLOCK_MONOTONIC_RAW, timespec); - - return ret; -} - -int _odp_time_term_global(void) -{ - return 0; -} -- cgit v1.2.3 From af1a174d919465bf262a0dc5c91edaa1d3688fc1 Mon Sep 17 00:00:00 2001 From: Tuomas Taipale Date: Mon, 19 Jun 2023 12:58:32 +0000 Subject: linux-gen: pktio: move packet vector cast helper Move `packet_vector_to_event_hdr()` to `odp_event_vector_internal.h` where it is reusable also elsewhere. Rename it accordingly. Signed-off-by: Tuomas Taipale Reviewed-by: Matias Elo --- platform/linux-generic/include/odp_event_vector_internal.h | 10 +++++++++- platform/linux-generic/odp_packet_io.c | 7 +------ 2 files changed, 10 insertions(+), 7 deletions(-) (limited to 'platform') diff --git a/platform/linux-generic/include/odp_event_vector_internal.h b/platform/linux-generic/include/odp_event_vector_internal.h index 55e33b913..39f9daf04 100644 --- a/platform/linux-generic/include/odp_event_vector_internal.h +++ b/platform/linux-generic/include/odp_event_vector_internal.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2020-2021, Nokia +/* Copyright (c) 2020-2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -57,6 +57,14 @@ static inline odp_event_vector_hdr_t *_odp_packet_vector_hdr(odp_packet_vector_t return (odp_event_vector_hdr_t *)(uintptr_t)pktv; } +/** + * Return the event header + */ +static inline _odp_event_hdr_t *_odp_packet_vector_to_event_hdr(odp_packet_vector_t pktv) +{ + return (_odp_event_hdr_t *)(uintptr_t)&_odp_packet_vector_hdr(pktv)->event_hdr; +} + /** * Free packet vector and contained packets */ diff --git a/platform/linux-generic/odp_packet_io.c b/platform/linux-generic/odp_packet_io.c index ef33b6d77..c4cd87afc 100644 --- a/platform/linux-generic/odp_packet_io.c +++ b/platform/linux-generic/odp_packet_io.c @@ -72,11 +72,6 @@ static inline pktio_entry_t *pktio_entry_by_index(int index) return _odp_pktio_entry_ptr[index]; } -static inline _odp_event_hdr_t *packet_vector_to_event_hdr(odp_packet_vector_t pktv) -{ - return (_odp_event_hdr_t *)(uintptr_t)&_odp_packet_vector_hdr(pktv)->event_hdr; -} - static int read_config_file(pktio_global_t *pktio_glb) { const char *str; @@ -826,7 +821,7 @@ static inline int pktin_recv_buf(pktio_entry_t *entry, int pktin_index, if (odp_unlikely(pktv == ODP_PACKET_VECTOR_INVALID)) return 0; - event_hdrs[0] = packet_vector_to_event_hdr(pktv); + event_hdrs[0] = _odp_packet_vector_to_event_hdr(pktv); return 1; } -- cgit v1.2.3 From aa53b41739ab1df3c64aac3302615b6f04c72fb3 Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Wed, 21 Jun 2023 09:11:56 +0300 Subject: linux-gen: cpu: detect constant_tsc from cpuinfo INVTSC x86 CPU flag may not be always available with VMs. So, try also to find 'constant_tsc' flag from /proc/cpuinfo. Signed-off-by: Matias Elo Reviewed-by: Petri Savolainen --- platform/linux-generic/arch/x86/cpu_flags.c | 8 +++++-- .../linux-generic/arch/x86/odp_sysinfo_parse.c | 6 +++++ platform/linux-generic/include/odp_global_data.h | 4 +++- platform/linux-generic/odp_init.c | 26 +++++++++++----------- 4 files changed, 28 insertions(+), 16 deletions(-) (limited to 'platform') diff --git a/platform/linux-generic/arch/x86/cpu_flags.c b/platform/linux-generic/arch/x86/cpu_flags.c index 482c25739..9211df002 100644 --- a/platform/linux-generic/arch/x86/cpu_flags.c +++ b/platform/linux-generic/arch/x86/cpu_flags.c @@ -10,9 +10,12 @@ */ #include "cpu_flags.h" -#include + #include +#include +#include + #include #include #include @@ -356,7 +359,8 @@ void _odp_cpu_flags_print_all(void) int _odp_time_cpu_global_freq_is_const(void) { - if (cpu_get_flag_enabled(RTE_CPUFLAG_INVTSC) > 0) + if (odp_global_ro.system_info.cpu_constant_tsc || + cpu_get_flag_enabled(RTE_CPUFLAG_INVTSC) > 0) return 1; _ODP_ERR("WARN: assuming constant TSC based on CPU arch, but could not confirm from CPU " diff --git a/platform/linux-generic/arch/x86/odp_sysinfo_parse.c b/platform/linux-generic/arch/x86/odp_sysinfo_parse.c index c74c52045..3cbdb2037 100644 --- a/platform/linux-generic/arch/x86/odp_sysinfo_parse.c +++ b/platform/linux-generic/arch/x86/odp_sysinfo_parse.c @@ -1,4 +1,5 @@ /* Copyright (c) 2016-2018, Linaro Limited + * Copyright (c) 2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -30,6 +31,11 @@ int _odp_cpuinfo_parser(FILE *file, system_info_t *sysinfo) strcpy(sysinfo->cpu_arch_str, "x86"); while (fgets(str, sizeof(str), file) != NULL && id < CONFIG_NUM_CPU_IDS) { + if (strstr(str, "flags") && strstr(str, "constant_tsc")) { + sysinfo->cpu_constant_tsc = 1; + continue; + } + pos = strstr(str, "model name"); if (pos) { freq_set = false; diff --git a/platform/linux-generic/include/odp_global_data.h b/platform/linux-generic/include/odp_global_data.h index d4cc9cda4..67b7572ef 100644 --- a/platform/linux-generic/include/odp_global_data.h +++ b/platform/linux-generic/include/odp_global_data.h @@ -1,4 +1,5 @@ /* Copyright (c) 2013-2018, Linaro Limited + * Copyright (c) 2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -34,7 +35,8 @@ typedef struct { uint64_t default_cpu_hz; uint64_t page_size; int cache_line_size; - odp_bool_t cpu_hz_static; + uint8_t cpu_hz_static; + uint8_t cpu_constant_tsc; odp_cpu_arch_t cpu_arch; odp_cpu_arch_isa_t cpu_isa_sw; odp_cpu_arch_isa_t cpu_isa_hw; diff --git a/platform/linux-generic/odp_init.c b/platform/linux-generic/odp_init.c index f1aae95bb..e6ea8bc0c 100644 --- a/platform/linux-generic/odp_init.c +++ b/platform/linux-generic/odp_init.c @@ -27,8 +27,8 @@ enum init_stage { LIBCONFIG_INIT, CPUMASK_INIT, CPU_CYCLES_INIT, - TIME_INIT, SYSINFO_INIT, + TIME_INIT, ISHM_INIT, FDSERVER_INIT, GLOBAL_RW_DATA_INIT, @@ -292,16 +292,16 @@ static int term_global(enum init_stage stage) } /* Fall through */ - case SYSINFO_INIT: - if (_odp_system_info_term()) { - _ODP_ERR("ODP system info term failed.\n"); + case TIME_INIT: + if (_odp_time_term_global()) { + _ODP_ERR("ODP time term failed.\n"); rc = -1; } /* Fall through */ - case TIME_INIT: - if (_odp_time_term_global()) { - _ODP_ERR("ODP time term failed.\n"); + case SYSINFO_INIT: + if (_odp_system_info_term()) { + _ODP_ERR("ODP system info term failed.\n"); rc = -1; } /* Fall through */ @@ -372,18 +372,18 @@ int odp_init_global(odp_instance_t *instance, } stage = CPU_CYCLES_INIT; - if (_odp_time_init_global()) { - _ODP_ERR("ODP time init failed.\n"); - goto init_failed; - } - stage = TIME_INIT; - if (_odp_system_info_init()) { _ODP_ERR("ODP system_info init failed.\n"); goto init_failed; } stage = SYSINFO_INIT; + if (_odp_time_init_global()) { + _ODP_ERR("ODP time init failed.\n"); + goto init_failed; + } + stage = TIME_INIT; + if (_odp_ishm_init_global(params)) { _ODP_ERR("ODP ishm init failed.\n"); goto init_failed; -- cgit v1.2.3 From 14bc06f58928d4ad39e9c37d634d782f1307ddaf Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Thu, 22 Jun 2023 11:22:52 +0300 Subject: linux-gen: event: implement odp_event_user_area_and_flag() function Add inline implementation for the new odp_event_user_area_and_flag() function. Signed-off-by: Matias Elo Reviewed-by: Petri Savolainen --- .../include/odp/api/plat/event_inlines.h | 42 ++++++++++++++++++++++ 1 file changed, 42 insertions(+) (limited to 'platform') diff --git a/platform/linux-generic/include/odp/api/plat/event_inlines.h b/platform/linux-generic/include/odp/api/plat/event_inlines.h index a353ab0b8..437ff35ae 100644 --- a/platform/linux-generic/include/odp/api/plat/event_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/event_inlines.h @@ -14,6 +14,7 @@ #include #include +#include #include #include #include @@ -27,6 +28,7 @@ #define odp_event_type __odp_event_type #define odp_event_type_multi __odp_event_type_multi #define odp_event_user_area __odp_event_user_area + #define odp_event_user_area_and_flag __odp_event_user_area_and_flag #define odp_event_subtype __odp_event_subtype #define odp_event_types __odp_event_types #define odp_event_flow_id __odp_event_flow_id @@ -84,6 +86,46 @@ _ODP_INLINE void *odp_event_user_area(odp_event_t event) } } +_ODP_INLINE void *odp_event_user_area_and_flag(odp_event_t event, int *flag) +{ + const odp_event_type_t type = __odp_event_type_get(event); + + _ODP_ASSERT(flag != NULL); + + switch (type) { + case ODP_EVENT_BUFFER: + case ODP_EVENT_DMA_COMPL: + *flag = -1; + return _odp_buffer_get((odp_buffer_t)event, void *, uarea_addr); + case ODP_EVENT_PACKET: + { + _odp_packet_flags_t pkt_flags; + odp_packet_t pkt = (odp_packet_t)event; + + pkt_flags.all_flags = _odp_pkt_get(pkt, uint32_t, flags); + *flag = pkt_flags.user_flag; + + return _odp_pkt_get(pkt, void *, user_area); + } + case ODP_EVENT_PACKET_VECTOR: + { + _odp_event_vector_flags_t pktv_flags; + odp_packet_vector_t pktv = (odp_packet_vector_t)event; + + pktv_flags.all_flags = _odp_event_vect_get(pktv, uint32_t, flags); + *flag = pktv_flags.user_flag; + + return _odp_event_vect_get(pktv, void *, uarea_addr); + } + case ODP_EVENT_TIMEOUT: + *flag = -1; + return _odp_timeout_hdr_field((odp_timeout_t)event, void *, uarea_addr); + default: + *flag = -1; + return NULL; + } +} + _ODP_INLINE odp_event_subtype_t odp_event_subtype(odp_event_t event) { if (__odp_event_type_get(event) != ODP_EVENT_PACKET) -- cgit v1.2.3 From 009ee2e6801e07e6240e502e9193af57a0473d18 Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Fri, 16 Jun 2023 14:57:07 +0300 Subject: linux-gen: timer: implement odp_timeout_from_event_multi() function Add inline implementation for the new odp_timeout_from_event_multi() function. Signed-off-by: Matias Elo Reviewed-by: Tuomas Taipale Reviewed-by: Petri Savolainen --- platform/linux-generic/include/odp/api/plat/timer_inlines.h | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) (limited to 'platform') diff --git a/platform/linux-generic/include/odp/api/plat/timer_inlines.h b/platform/linux-generic/include/odp/api/plat/timer_inlines.h index 648459c78..9ba0287e0 100644 --- a/platform/linux-generic/include/odp/api/plat/timer_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/timer_inlines.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2022, Nokia +/* Copyright (c) 2022-2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -27,6 +27,7 @@ #define odp_timer_tick_to_ns __odp_timer_tick_to_ns #define odp_timer_ns_to_tick __odp_timer_ns_to_tick #define odp_timeout_from_event __odp_timeout_from_event + #define odp_timeout_from_event_multi __odp_timeout_from_event_multi #define odp_timeout_to_event __odp_timeout_to_event #else #define _ODP_INLINE @@ -75,6 +76,15 @@ _ODP_INLINE odp_timeout_t odp_timeout_from_event(odp_event_t ev) return (odp_timeout_t)ev; } +_ODP_INLINE void odp_timeout_from_event_multi(odp_timeout_t tmo[], const odp_event_t ev[], int num) +{ + for (int i = 0; i < num; i++) { + _ODP_ASSERT(odp_event_type(ev[i]) == ODP_EVENT_TIMEOUT); + + tmo[i] = odp_timeout_from_event(ev[i]); + } +} + _ODP_INLINE odp_event_t odp_timeout_to_event(odp_timeout_t tmo) { return (odp_event_t)tmo; -- cgit v1.2.3 From 38e378e559b5ce17f723d0e3c4aefbbca0bd2b13 Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Fri, 16 Jun 2023 16:47:32 +0300 Subject: linux-gen: event: implement odp_event_types_multi() function Add inline implementation for the new odp_event_types_multi() function. Signed-off-by: Matias Elo Reviewed-by: Petri Savolainen --- .../linux-generic/include/odp/api/plat/event_inlines.h | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) (limited to 'platform') diff --git a/platform/linux-generic/include/odp/api/plat/event_inlines.h b/platform/linux-generic/include/odp/api/plat/event_inlines.h index 437ff35ae..2e7c7db5e 100644 --- a/platform/linux-generic/include/odp/api/plat/event_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/event_inlines.h @@ -31,6 +31,7 @@ #define odp_event_user_area_and_flag __odp_event_user_area_and_flag #define odp_event_subtype __odp_event_subtype #define odp_event_types __odp_event_types + #define odp_event_types_multi __odp_event_types_multi #define odp_event_flow_id __odp_event_flow_id #define odp_event_flow_id_set __odp_event_flow_id_set #else @@ -146,6 +147,22 @@ _ODP_INLINE odp_event_type_t odp_event_types(odp_event_t event, return event_type; } +_ODP_INLINE void odp_event_types_multi(const odp_event_t event[], odp_event_type_t type[], + odp_event_subtype_t subtype[], int num) +{ + for (int i = 0; i < num; i++) + type[i] = __odp_event_type_get(event[i]); + + if (subtype == NULL) + return; + + for (int i = 0; i < num; i++) { + subtype[i] = (type[i] == ODP_EVENT_PACKET) ? + (odp_event_subtype_t)_odp_pkt_get((odp_packet_t)event[i], int8_t, + subtype) : ODP_EVENT_NO_SUBTYPE; + } +} + _ODP_INLINE uint32_t odp_event_flow_id(odp_event_t event) { return _odp_event_hdr_field(event, uint8_t, flow_id); -- cgit v1.2.3 From 0cad9dda5aa4c565c14aec0d0bf13da3401da7e1 Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Fri, 16 Jun 2023 13:03:18 +0300 Subject: linux-gen: debug: add support for new warning log level Add support for new ODP_LOG_WARN log level. New matching implementation internal _ODP_WARN() macro has been added. Signed-off-by: Matias Elo Reviewed-by: Tuomas Taipale Reviewed-by: Petri Savolainen --- platform/linux-generic/include/odp/api/plat/debug_inlines.h | 12 +++++++++++- platform/linux-generic/odp_weak.c | 1 + 2 files changed, 12 insertions(+), 1 deletion(-) (limited to 'platform') diff --git a/platform/linux-generic/include/odp/api/plat/debug_inlines.h b/platform/linux-generic/include/odp/api/plat/debug_inlines.h index 41af3dca4..e11dba97b 100644 --- a/platform/linux-generic/include/odp/api/plat/debug_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/debug_inlines.h @@ -1,5 +1,5 @@ /* Copyright (c) 2014-2018, Linaro Limited - * Copyright (c) 2020-2022, Nokia + * Copyright (c) 2020-2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -74,6 +74,16 @@ extern odp_abort_func_t _odp_abort_fn; }); \ } while (0) +/** + * Log warning message. + */ +#define _ODP_WARN(...) \ + do { \ + __extension__ ({ \ + _ODP_LOG(ODP_LOG_WARN, ##__VA_ARGS__); \ + }); \ + } while (0) + /** * Log error message. */ diff --git a/platform/linux-generic/odp_weak.c b/platform/linux-generic/odp_weak.c index 9e1c9da6a..747886d4e 100644 --- a/platform/linux-generic/odp_weak.c +++ b/platform/linux-generic/odp_weak.c @@ -20,6 +20,7 @@ int odp_override_log(odp_log_level_t level, const char *fmt, ...) switch (level) { case ODP_LOG_ERR: case ODP_LOG_UNIMPLEMENTED: + case ODP_LOG_WARN: case ODP_LOG_ABORT: logfd = stderr; break; -- cgit v1.2.3 From fcb53096f0b1f7def9832796c1d968f8f503eb7a Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Tue, 20 Jun 2023 11:21:51 +0300 Subject: linux-gen: debug: add message types to logs Include the type of message when writing implementation internal logs. Signed-off-by: Matias Elo Reviewed-by: Tuomas Taipale Reviewed-by: Petri Savolainen --- .../linux-generic/include/odp/api/plat/debug_inlines.h | 14 +++++++------- platform/linux-generic/include/odp_debug_internal.h | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) (limited to 'platform') diff --git a/platform/linux-generic/include/odp/api/plat/debug_inlines.h b/platform/linux-generic/include/odp/api/plat/debug_inlines.h index e11dba97b..0755b1fda 100644 --- a/platform/linux-generic/include/odp/api/plat/debug_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/debug_inlines.h @@ -50,9 +50,9 @@ extern odp_abort_func_t _odp_abort_fn; /** * ODP LOG macro. */ -#define _ODP_LOG(level, fmt, ...) \ - _ODP_LOG_FN(level, "%s:%d:%s(): " fmt, __FILE__, \ - __LINE__, __func__, ##__VA_ARGS__) +#define _ODP_LOG(level, prefix, fmt, ...) \ + _ODP_LOG_FN(level, "%s: %s:%d:%s(): " fmt, prefix, \ + __FILE__, __LINE__, __func__, ##__VA_ARGS__) /** * Runtime assertion-macro - aborts if 'cond' is false. @@ -70,7 +70,7 @@ extern odp_abort_func_t _odp_abort_fn; do { \ if (ODP_DEBUG_PRINT == 1) \ __extension__ ({ \ - _ODP_LOG(ODP_LOG_DBG, ##__VA_ARGS__); \ + _ODP_LOG(ODP_LOG_DBG, "DBG", ##__VA_ARGS__); \ }); \ } while (0) @@ -80,7 +80,7 @@ extern odp_abort_func_t _odp_abort_fn; #define _ODP_WARN(...) \ do { \ __extension__ ({ \ - _ODP_LOG(ODP_LOG_WARN, ##__VA_ARGS__); \ + _ODP_LOG(ODP_LOG_WARN, "WARN", ##__VA_ARGS__); \ }); \ } while (0) @@ -90,7 +90,7 @@ extern odp_abort_func_t _odp_abort_fn; #define _ODP_ERR(...) \ do { \ __extension__ ({ \ - _ODP_LOG(ODP_LOG_ERR, ##__VA_ARGS__); \ + _ODP_LOG(ODP_LOG_ERR, "ERR", ##__VA_ARGS__); \ }); \ } while (0) @@ -101,7 +101,7 @@ extern odp_abort_func_t _odp_abort_fn; #define _ODP_ABORT(...) \ do { \ __extension__ ({ \ - _ODP_LOG(ODP_LOG_ABORT, ##__VA_ARGS__); \ + _ODP_LOG(ODP_LOG_ABORT, "ABORT", ##__VA_ARGS__); \ }); \ _odp_abort_fn(); \ } while (0) diff --git a/platform/linux-generic/include/odp_debug_internal.h b/platform/linux-generic/include/odp_debug_internal.h index 4c1fa8bbb..d1fc0d0ba 100644 --- a/platform/linux-generic/include/odp_debug_internal.h +++ b/platform/linux-generic/include/odp_debug_internal.h @@ -55,7 +55,7 @@ extern "C" { do { \ if (ODP_DEBUG_PRINT == 1 && CONFIG_DEBUG_LEVEL >= (level)) \ __extension__ ({ \ - _ODP_LOG(ODP_LOG_DBG, ##__VA_ARGS__); \ + _ODP_LOG(ODP_LOG_DBG, "DBG", ##__VA_ARGS__); \ }); \ } while (0) -- cgit v1.2.3 From 2fdf01b70877a19a620e4c87daf6551a6b771ffa Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Fri, 16 Jun 2023 13:23:15 +0300 Subject: linux-gen: debug: utilize new _ODP_WARN() macro Use new _ODP_WARN() macro for logging implementation internal warnings. The warning messages have been made more consistent. Signed-off-by: Matias Elo Reviewed-by: Tuomas Taipale Reviewed-by: Petri Savolainen --- platform/linux-generic/arch/aarch64/odp_sysinfo_parse.c | 5 ++--- platform/linux-generic/include/odp_sysinfo_internal.h | 4 ++-- platform/linux-generic/odp_ishm.c | 9 ++++----- platform/linux-generic/odp_system_info.c | 4 ++-- platform/linux-generic/pktio/socket_xdp.c | 9 +++------ 5 files changed, 13 insertions(+), 18 deletions(-) (limited to 'platform') diff --git a/platform/linux-generic/arch/aarch64/odp_sysinfo_parse.c b/platform/linux-generic/arch/aarch64/odp_sysinfo_parse.c index 25e3e4fc6..352bee9e9 100644 --- a/platform/linux-generic/arch/aarch64/odp_sysinfo_parse.c +++ b/platform/linux-generic/arch/aarch64/odp_sysinfo_parse.c @@ -336,9 +336,8 @@ int _odp_cpuinfo_parser(FILE *file, system_info_t *sysinfo) if (sysinfo->cpu_hz_max[id] == 0) { uint64_t hz = sysinfo->default_cpu_hz_max; - _ODP_PRINT("WARN: cpu[%i] uses default max " - "frequency of %" PRIu64 " Hz from " - "config file\n", id, hz); + _ODP_WARN("CPU[%i] uses default max frequency of %" PRIu64 " " + "Hz from config file\n", id, hz); sysinfo->cpu_hz_max[id] = hz; } diff --git a/platform/linux-generic/include/odp_sysinfo_internal.h b/platform/linux-generic/include/odp_sysinfo_internal.h index 0fef1aa32..c14cf78d9 100644 --- a/platform/linux-generic/include/odp_sysinfo_internal.h +++ b/platform/linux-generic/include/odp_sysinfo_internal.h @@ -27,9 +27,9 @@ static inline int _odp_dummy_cpuinfo(system_info_t *sysinfo) sysinfo->cpu_arch = ODP_CPU_ARCH_UNKNOWN; - _ODP_DBG("Warning: use dummy values for freq and model string\n"); + _ODP_WARN("Use dummy values for freq and model string\n"); for (i = 0; i < CONFIG_NUM_CPU_IDS; i++) { - _ODP_PRINT("WARN: cpu[%i] uses default max frequency of " + _ODP_WARN("CPU[%i] uses default max frequency of " "%" PRIu64 " Hz from config file\n", i, cpu_hz_max); sysinfo->cpu_hz_max[i] = cpu_hz_max; strcpy(sysinfo->model_str[i], "UNKNOWN"); diff --git a/platform/linux-generic/odp_ishm.c b/platform/linux-generic/odp_ishm.c index 926e20a8a..3ef1894bc 100644 --- a/platform/linux-generic/odp_ishm.c +++ b/platform/linux-generic/odp_ishm.c @@ -1160,9 +1160,8 @@ int _odp_ishm_reserve(const char *name, uint64_t size, int fd, if (addr == NULL) { if (!huge_error_printed) { - _ODP_ERR("No huge pages, fall back to " - "normal pages. Check: " - "/proc/sys/vm/nr_hugepages.\n"); + _ODP_WARN("No huge pages, fall back to normal pages. " + "Check: /proc/sys/vm/nr_hugepages.\n"); huge_error_printed = 1; } } else { @@ -1274,8 +1273,8 @@ static void *reserve_single_va(uint64_t size, int *fd_out) } } if (fd < 0) - _ODP_ERR("No huge pages, fall back to normal pages. " - "Check: /proc/sys/vm/nr_hugepages.\n"); + _ODP_WARN("No huge pages, fall back to normal pages. " + "Check: /proc/sys/vm/nr_hugepages.\n"); ishm_tbl->single_va_huge = true; } diff --git a/platform/linux-generic/odp_system_info.c b/platform/linux-generic/odp_system_info.c index 4f1d92357..7864a3d17 100644 --- a/platform/linux-generic/odp_system_info.c +++ b/platform/linux-generic/odp_system_info.c @@ -55,7 +55,7 @@ static int read_cache_line_size(void) file = fopen(CACHE_LNSZ_FILE, "rt"); if (file == NULL) { /* File not found */ - _ODP_PRINT("WARN: unable to read host CPU cache line size. " + _ODP_WARN("Unable to read host CPU cache line size. " "Using ODP_CACHE_LINE_SIZE instead.\n"); return ODP_CACHE_LINE_SIZE; } @@ -299,7 +299,7 @@ static int system_cache_line(system_info_t *sysinfo) sysinfo->cache_line_size = ret; if (ret != ODP_CACHE_LINE_SIZE) - _ODP_PRINT("WARN: host CPU cache line size and ODP_CACHE_LINE_SIZE don't match.\n"); + _ODP_WARN("Host CPU cache line size and ODP_CACHE_LINE_SIZE don't match.\n"); return 0; } diff --git a/platform/linux-generic/pktio/socket_xdp.c b/platform/linux-generic/pktio/socket_xdp.c index 867483f76..599942657 100644 --- a/platform/linux-generic/pktio/socket_xdp.c +++ b/platform/linux-generic/pktio/socket_xdp.c @@ -309,8 +309,7 @@ static int sock_xdp_open(odp_pktio_t pktio, pktio_entry_t *pktio_entry, const ch if (!get_nic_queue_count(priv->helper_sock, devname, &priv->q_num_conf.drv_channels) || !get_nic_rss_indir_count(priv->helper_sock, devname, &priv->q_num_conf.drv_num_rss)) - _ODP_PRINT("Warning: Unable to query NIC queue count/RSS, manual cleanup" - " required\n"); + _ODP_WARN("Unable to query NIC queue count/RSS, manual cleanup required\n"); priv->is_shadow_q = is_shadow_q_driver(priv->helper_sock, pktio_entry->name); parse_options(priv->umem_info); @@ -543,8 +542,7 @@ static int sock_xdp_start(pktio_entry_t *pktio_entry) channels.combined = priv->q_num_conf.num_qs; if (!set_nic_queue_count(priv->helper_sock, pktio_entry->name, &channels)) - _ODP_PRINT("Warning: Unable to configure NIC queue count, manual configuration" - " required\n"); + _ODP_WARN("Unable to configure NIC queue count, manual configuration required\n"); if (priv->q_num_conf.num_in_conf_qs > 0U && !priv->is_shadow_q) { indir->indir_size = priv->q_num_conf.drv_num_rss; @@ -553,8 +551,7 @@ static int sock_xdp_start(pktio_entry_t *pktio_entry) indir->rss_config[i] = (i % priv->q_num_conf.num_in_conf_qs); if (!set_nic_rss_indir(priv->helper_sock, pktio_entry->name, indir)) - _ODP_PRINT("Warning: Unable to configure NIC RSS, manual configuration" - " required\n"); + _ODP_WARN("Unable to configure NIC RSS, manual configuration required\n"); } if (!create_sockets(priv, pktio_entry->name)) -- cgit v1.2.3 From 21ae3c664b368c52ef28914188c3abaa85a3ac16 Mon Sep 17 00:00:00 2001 From: Janne Peltonen Date: Tue, 4 Jul 2023 11:04:11 +0300 Subject: linux-gen: crypto: temporarily disable bit mode and EEA2 support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit To make it possible to change the API, implementation and validation tests in separate commits without breaking the tests at any point, temporarily disable bit mode and EEA2 support. Signed-off-by: Janne Peltonen Reviewed-by: Jere Leppänen Reviewed-by: Anoob Joseph --- platform/linux-generic/odp_crypto_openssl.c | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) (limited to 'platform') diff --git a/platform/linux-generic/odp_crypto_openssl.c b/platform/linux-generic/odp_crypto_openssl.c index d4c7a3f1a..41a726163 100644 --- a/platform/linux-generic/odp_crypto_openssl.c +++ b/platform/linux-generic/odp_crypto_openssl.c @@ -55,8 +55,7 @@ * Keep sorted: first by key length, then by IV length */ static const odp_crypto_cipher_capability_t cipher_capa_null[] = { -{.key_len = 0, .iv_len = 0}, -{.key_len = 0, .iv_len = 0, .bit_mode = 1} }; +{.key_len = 0, .iv_len = 0, .bit_mode = 0} }; static const odp_crypto_cipher_capability_t cipher_capa_trides_cbc[] = { {.key_len = 24, .iv_len = 8} }; @@ -107,7 +106,7 @@ static const odp_crypto_cipher_capability_t cipher_capa_chacha20_poly1305[] = { #endif static const odp_crypto_cipher_capability_t cipher_capa_aes_eea2[] = { -{.key_len = 16, .iv_len = 16, .bit_mode = 1} }; +{.key_len = 16, .iv_len = 16, .bit_mode = 0} }; /* * Authentication algorithm capabilities @@ -115,9 +114,7 @@ static const odp_crypto_cipher_capability_t cipher_capa_aes_eea2[] = { * Keep sorted: first by digest length, then by key length */ static const odp_crypto_auth_capability_t auth_capa_null[] = { -{.digest_len = 0, .key_len = 0, .aad_len = {.min = 0, .max = 0, .inc = 0}, - .bit_mode = 1}, -{.digest_len = 0, .key_len = 0, .aad_len = {.min = 0, .max = 0, .inc = 0} } }; +{.digest_len = 0, .key_len = 0, .aad_len = {.min = 0, .max = 0, .inc = 0}, .bit_mode = 0} }; static const odp_crypto_auth_capability_t auth_capa_md5_hmac[] = { {.digest_len = 12, .key_len = 16, .aad_len = {.min = 0, .max = 0, .inc = 0} }, @@ -1813,7 +1810,7 @@ int odp_crypto_capability(odp_crypto_capability_t *capa) #if _ODP_HAVE_CHACHA20_POLY1305 capa->ciphers.bit.chacha20_poly1305 = 1; #endif - capa->ciphers.bit.aes_eea2 = 1; + capa->ciphers.bit.aes_eea2 = 0; capa->auths.bit.null = 1; capa->auths.bit.md5_hmac = 1; -- cgit v1.2.3 From 7c863c0c3b80b8c3b4e345291b0d91baedd7073d Mon Sep 17 00:00:00 2001 From: Janne Peltonen Date: Tue, 4 Jul 2023 11:04:13 +0300 Subject: linux-gen: crypto: implement configurable bit mode MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add session creation time bit mode configuration for the algorithm implementations that have supported bit mode, i.e. the null algorithms and EEA2. Signed-off-by: Janne Peltonen Reviewed-by: Jere Leppänen Reviewed-by: Anoob Joseph --- platform/linux-generic/odp_crypto_openssl.c | 100 +++++++++++++++++----------- 1 file changed, 61 insertions(+), 39 deletions(-) (limited to 'platform') diff --git a/platform/linux-generic/odp_crypto_openssl.c b/platform/linux-generic/odp_crypto_openssl.c index 41a726163..d0b1da6cd 100644 --- a/platform/linux-generic/odp_crypto_openssl.c +++ b/platform/linux-generic/odp_crypto_openssl.c @@ -214,7 +214,8 @@ struct odp_crypto_generic_session_t { odp_crypto_session_param_t p; odp_bool_t do_cipher_first; - uint8_t cipher_bit_mode : 1; + uint8_t cipher_range_in_bits : 1; + uint8_t auth_range_in_bits : 1; uint8_t cipher_range_used : 1; uint8_t auth_range_used : 1; @@ -1119,70 +1120,72 @@ static int process_cipher_param(odp_crypto_generic_session_t *session, return 0; } -static -odp_crypto_alg_err_t cipher_encrypt_bits(odp_packet_t pkt, - const odp_crypto_packet_op_param_t - *param, - odp_crypto_generic_session_t *session) +static odp_crypto_alg_err_t cipher_encrypt_bytes(odp_packet_t pkt, + const odp_crypto_packet_op_param_t *param, + odp_crypto_generic_session_t *session) { EVP_CIPHER_CTX *ctx = local.cipher_ctx[session->idx]; int dummy_len = 0; int cipher_len; - uint32_t in_len = (param->cipher_range.length + 7) / 8; + uint32_t in_len = param->cipher_range.length; + uint32_t offset = param->cipher_range.offset; uint8_t data[in_len]; int ret; - uint32_t offset; - - /* Range offset is in bits in bit mode but must be divisible by 8. */ - offset = param->cipher_range.offset / 8; EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, param->cipher_iv_ptr); - odp_packet_copy_to_mem(pkt, offset, in_len, data); - EVP_EncryptUpdate(ctx, data, &cipher_len, data, in_len); - ret = EVP_EncryptFinal_ex(ctx, data + cipher_len, &dummy_len); cipher_len += dummy_len; - odp_packet_copy_from_mem(pkt, offset, in_len, data); - return ret <= 0 ? ODP_CRYPTO_ALG_ERR_DATA_SIZE : ODP_CRYPTO_ALG_ERR_NONE; } -static -odp_crypto_alg_err_t cipher_decrypt_bits(odp_packet_t pkt, - const odp_crypto_packet_op_param_t - *param, - odp_crypto_generic_session_t *session) +static odp_crypto_alg_err_t cipher_decrypt_bytes(odp_packet_t pkt, + const odp_crypto_packet_op_param_t *param, + odp_crypto_generic_session_t *session) { EVP_CIPHER_CTX *ctx = local.cipher_ctx[session->idx]; int dummy_len = 0; int cipher_len; - uint32_t in_len = (param->cipher_range.length + 7) / 8; + uint32_t in_len = param->cipher_range.length; + uint32_t offset = param->cipher_range.offset; uint8_t data[in_len]; int ret; - uint32_t offset; - - /* Range offset is in bits in bit mode but must be divisible by 8. */ - offset = param->cipher_range.offset / 8; EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, param->cipher_iv_ptr); - odp_packet_copy_to_mem(pkt, offset, in_len, data); - EVP_DecryptUpdate(ctx, data, &cipher_len, data, in_len); - ret = EVP_DecryptFinal_ex(ctx, data + cipher_len, &dummy_len); cipher_len += dummy_len; - odp_packet_copy_from_mem(pkt, offset, in_len, data); - return ret <= 0 ? ODP_CRYPTO_ALG_ERR_DATA_SIZE : ODP_CRYPTO_ALG_ERR_NONE; } +static odp_crypto_alg_err_t cipher_encrypt_bits(odp_packet_t pkt, + const odp_crypto_packet_op_param_t *param, + odp_crypto_generic_session_t *session) +{ + odp_crypto_packet_op_param_t new_param = *param; + + new_param.cipher_range.offset /= 8; + new_param.cipher_range.length = (new_param.cipher_range.length + 7) / 8; + return cipher_encrypt_bytes(pkt, &new_param, session); +} + +static odp_crypto_alg_err_t cipher_decrypt_bits(odp_packet_t pkt, + const odp_crypto_packet_op_param_t *param, + odp_crypto_generic_session_t *session) +{ + odp_crypto_packet_op_param_t new_param = *param; + + new_param.cipher_range.offset /= 8; + new_param.cipher_range.length = (new_param.cipher_range.length + 7) / 8; + return cipher_decrypt_bytes(pkt, &new_param, session); +} + static int process_cipher_param_bits(odp_crypto_generic_session_t *session, const EVP_CIPHER *cipher) { @@ -1196,7 +1199,6 @@ static int process_cipher_param_bits(odp_crypto_generic_session_t *session, session->p.cipher_iv_len) return -1; - session->cipher_bit_mode = 1; session->cipher.evp_cipher = cipher; memcpy(session->cipher.key_data, session->p.cipher_key.data, @@ -1204,11 +1206,14 @@ static int process_cipher_param_bits(odp_crypto_generic_session_t *session, /* Set function */ if (ODP_CRYPTO_OP_ENCODE == session->p.op) { - session->cipher.func = cipher_encrypt_bits; session->cipher.init = cipher_encrypt_init; + session->cipher.func = session->cipher_range_in_bits ? cipher_encrypt_bits + : cipher_encrypt_bytes; + } else { - session->cipher.func = cipher_decrypt_bits; session->cipher.init = cipher_decrypt_init; + session->cipher.func = session->cipher_range_in_bits ? cipher_decrypt_bits + : cipher_decrypt_bytes; } return 0; @@ -2019,6 +2024,8 @@ odp_crypto_session_create(const odp_crypto_session_param_t *param, { int rc; odp_crypto_generic_session_t *session; + int cipher_bit_mode_supported = 0; + int auth_bit_mode_supported = 0; if (odp_global_ro.disable.crypto) { _ODP_ERR("Crypto is disabled\n"); @@ -2049,7 +2056,8 @@ odp_crypto_session_create(const odp_crypto_session_param_t *param, /* Copy parameters */ session->p = *param; - session->cipher_bit_mode = 0; + session->cipher_range_in_bits = !!param->cipher_range_in_bits; + session->auth_range_in_bits = !!param->auth_range_in_bits; session->auth_range_used = 1; session->cipher_range_used = 1; @@ -2077,6 +2085,7 @@ odp_crypto_session_create(const odp_crypto_session_param_t *param, session->cipher.func = null_crypto_routine; session->cipher.init = null_crypto_init_routine; session->cipher_range_used = 0; + cipher_bit_mode_supported = 1; rc = 0; break; case ODP_CIPHER_ALG_3DES_CBC: @@ -2181,11 +2190,15 @@ odp_crypto_session_create(const odp_crypto_session_param_t *param, EVP_aes_128_ctr()); else rc = -1; + cipher_bit_mode_supported = 1; break; default: rc = -1; } + if (session->cipher_range_in_bits && !cipher_bit_mode_supported) + rc = -1; + /* Check result */ if (rc) { *status = ODP_CRYPTO_SES_ERR_CIPHER; @@ -2198,6 +2211,7 @@ odp_crypto_session_create(const odp_crypto_session_param_t *param, session->auth.func = null_crypto_routine; session->auth.init = null_crypto_init_routine; session->auth_range_used = 0; + auth_bit_mode_supported = 1; rc = 0; break; case ODP_AUTH_ALG_MD5_HMAC: @@ -2311,6 +2325,9 @@ odp_crypto_session_create(const odp_crypto_session_param_t *param, rc = -1; } + if (session->auth_range_in_bits && !auth_bit_mode_supported) + rc = -1; + /* Check result */ if (rc) { *status = ODP_CRYPTO_SES_ERR_AUTH; @@ -2629,10 +2646,14 @@ static void copy_ranges(odp_packet_t dst, int32_t shift = param->dst_offset_shift; int rc; - if (session->cipher_bit_mode) { + if (session->cipher_range_in_bits) { c_range.offset /= 8; c_range.length = (c_range.length + 7) / 8; } + if (session->auth_range_in_bits) { + a_range.offset /= 8; + a_range.length = (a_range.length + 7) / 8; + } if (session->cipher_range_used) { rc = odp_packet_copy_from_pkt(dst, c_range.offset + shift, @@ -2660,12 +2681,13 @@ static int crypto_int_oop_encode(odp_packet_t pkt_in, const odp_crypto_packet_op_param_t *param) { odp_crypto_packet_op_param_t new_param = *param; - const uint32_t scale = session->cipher_bit_mode ? 8 : 1; + const uint32_t c_scale = session->cipher_range_in_bits ? 8 : 1; + const uint32_t a_scale = session->auth_range_in_bits ? 8 : 1; copy_ranges(*pkt_out, pkt_in, session, param); - new_param.cipher_range.offset += param->dst_offset_shift * scale; - new_param.auth_range.offset += param->dst_offset_shift; + new_param.cipher_range.offset += param->dst_offset_shift * c_scale; + new_param.auth_range.offset += param->dst_offset_shift * a_scale; return crypto_int(*pkt_out, pkt_out, &new_param); } -- cgit v1.2.3 From 688e9d92a9b289285f4f7f7aceafa4aa8215f515 Mon Sep 17 00:00:00 2001 From: Janne Peltonen Date: Tue, 4 Jul 2023 11:04:16 +0300 Subject: linux-gen: crypto: re-enable bit mode and EEA2 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Now with the API, validation tests and implementation updated, expose again the bit mode capabilities and the EEA2 algorithm that were temporarily disabled. Signed-off-by: Janne Peltonen Reviewed-by: Jere Leppänen Reviewed-by: Anoob Joseph --- platform/linux-generic/odp_crypto_openssl.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'platform') diff --git a/platform/linux-generic/odp_crypto_openssl.c b/platform/linux-generic/odp_crypto_openssl.c index d0b1da6cd..3f24bd9a3 100644 --- a/platform/linux-generic/odp_crypto_openssl.c +++ b/platform/linux-generic/odp_crypto_openssl.c @@ -55,7 +55,7 @@ * Keep sorted: first by key length, then by IV length */ static const odp_crypto_cipher_capability_t cipher_capa_null[] = { -{.key_len = 0, .iv_len = 0, .bit_mode = 0} }; +{.key_len = 0, .iv_len = 0, .bit_mode = 1} }; static const odp_crypto_cipher_capability_t cipher_capa_trides_cbc[] = { {.key_len = 24, .iv_len = 8} }; @@ -106,7 +106,7 @@ static const odp_crypto_cipher_capability_t cipher_capa_chacha20_poly1305[] = { #endif static const odp_crypto_cipher_capability_t cipher_capa_aes_eea2[] = { -{.key_len = 16, .iv_len = 16, .bit_mode = 0} }; +{.key_len = 16, .iv_len = 16, .bit_mode = 1} }; /* * Authentication algorithm capabilities @@ -114,7 +114,7 @@ static const odp_crypto_cipher_capability_t cipher_capa_aes_eea2[] = { * Keep sorted: first by digest length, then by key length */ static const odp_crypto_auth_capability_t auth_capa_null[] = { -{.digest_len = 0, .key_len = 0, .aad_len = {.min = 0, .max = 0, .inc = 0}, .bit_mode = 0} }; +{.digest_len = 0, .key_len = 0, .aad_len = {.min = 0, .max = 0, .inc = 0}, .bit_mode = 1} }; static const odp_crypto_auth_capability_t auth_capa_md5_hmac[] = { {.digest_len = 12, .key_len = 16, .aad_len = {.min = 0, .max = 0, .inc = 0} }, @@ -1815,7 +1815,7 @@ int odp_crypto_capability(odp_crypto_capability_t *capa) #if _ODP_HAVE_CHACHA20_POLY1305 capa->ciphers.bit.chacha20_poly1305 = 1; #endif - capa->ciphers.bit.aes_eea2 = 0; + capa->ciphers.bit.aes_eea2 = 1; capa->auths.bit.null = 1; capa->auths.bit.md5_hmac = 1; -- cgit v1.2.3 From 310d4e8d227e6abaa46eec903d0b05ded0cd241d Mon Sep 17 00:00:00 2001 From: Janne Peltonen Date: Tue, 4 Jul 2023 11:04:18 +0300 Subject: linux-gen: crypto: copy the ranges of null algorithms in OOP op type MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Do not treat the null cipher and auth algorithms as special cases for the purpose of copying packet data in the OOP operation type but copy their respective cipher and auth ranges as with any non-null algorithms. Signed-off-by: Janne Peltonen Reviewed-by: Jere Leppänen Reviewed-by: Anoob Joseph --- platform/linux-generic/odp_crypto_openssl.c | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) (limited to 'platform') diff --git a/platform/linux-generic/odp_crypto_openssl.c b/platform/linux-generic/odp_crypto_openssl.c index 3f24bd9a3..26c9ce7b7 100644 --- a/platform/linux-generic/odp_crypto_openssl.c +++ b/platform/linux-generic/odp_crypto_openssl.c @@ -216,7 +216,6 @@ struct odp_crypto_generic_session_t { odp_bool_t do_cipher_first; uint8_t cipher_range_in_bits : 1; uint8_t auth_range_in_bits : 1; - uint8_t cipher_range_used : 1; uint8_t auth_range_used : 1; struct { @@ -2059,7 +2058,6 @@ odp_crypto_session_create(const odp_crypto_session_param_t *param, session->cipher_range_in_bits = !!param->cipher_range_in_bits; session->auth_range_in_bits = !!param->auth_range_in_bits; session->auth_range_used = 1; - session->cipher_range_used = 1; if (session->p.cipher_iv_len > EVP_MAX_IV_LENGTH) { _ODP_DBG("Maximum IV length exceeded\n"); @@ -2084,7 +2082,6 @@ odp_crypto_session_create(const odp_crypto_session_param_t *param, case ODP_CIPHER_ALG_NULL: session->cipher.func = null_crypto_routine; session->cipher.init = null_crypto_init_routine; - session->cipher_range_used = 0; cipher_bit_mode_supported = 1; rc = 0; break; @@ -2210,7 +2207,6 @@ odp_crypto_session_create(const odp_crypto_session_param_t *param, case ODP_AUTH_ALG_NULL: session->auth.func = null_crypto_routine; session->auth.init = null_crypto_init_routine; - session->auth_range_used = 0; auth_bit_mode_supported = 1; rc = 0; break; @@ -2655,7 +2651,7 @@ static void copy_ranges(odp_packet_t dst, a_range.length = (a_range.length + 7) / 8; } - if (session->cipher_range_used) { + if (c_range.length > 0) { rc = odp_packet_copy_from_pkt(dst, c_range.offset + shift, src, c_range.offset, c_range.length); @@ -2664,7 +2660,7 @@ static void copy_ranges(odp_packet_t dst, return; } } - if (session->auth_range_used) { + if (session->auth_range_used && a_range.length > 0) { rc = odp_packet_copy_from_pkt(dst, a_range.offset + shift, src, a_range.offset, a_range.length); -- cgit v1.2.3 From 3a8f63d29bab98a706a5fdfde2ec01ac616ce01e Mon Sep 17 00:00:00 2001 From: Tuomas Taipale Date: Mon, 26 Jun 2023 10:40:06 +0000 Subject: linux-gen: implement pool user area initialization Add pool, external pool and dma specific implementations for user area initialization. Signed-off-by: Tuomas Taipale Reviewed-by: Petri Savolainen --- platform/linux-generic/odp_dma.c | 25 ++++++++++++++----------- platform/linux-generic/odp_pool.c | 25 ++++++++++++++++++++++++- 2 files changed, 38 insertions(+), 12 deletions(-) (limited to 'platform') diff --git a/platform/linux-generic/odp_dma.c b/platform/linux-generic/odp_dma.c index eef7e5c02..8ac18260c 100644 --- a/platform/linux-generic/odp_dma.c +++ b/platform/linux-generic/odp_dma.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2021-2022, Nokia +/* Copyright (c) 2021-2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -99,11 +99,12 @@ int odp_dma_capability(odp_dma_capability_t *capa) capa->queue_type_sched = 1; capa->queue_type_plain = 1; - capa->pool.max_pools = _odp_dma_glb->pool_capa.buf.max_pools; - capa->pool.max_num = _odp_dma_glb->pool_capa.buf.max_num; - capa->pool.max_uarea_size = _odp_dma_glb->pool_capa.buf.max_uarea_size; - capa->pool.min_cache_size = _odp_dma_glb->pool_capa.buf.min_cache_size; - capa->pool.max_cache_size = _odp_dma_glb->pool_capa.buf.max_cache_size; + capa->pool.max_pools = _odp_dma_glb->pool_capa.buf.max_pools; + capa->pool.max_num = _odp_dma_glb->pool_capa.buf.max_num; + capa->pool.max_uarea_size = _odp_dma_glb->pool_capa.buf.max_uarea_size; + capa->pool.uarea_persistence = _odp_dma_glb->pool_capa.buf.uarea_persistence; + capa->pool.min_cache_size = _odp_dma_glb->pool_capa.buf.min_cache_size; + capa->pool.max_cache_size = _odp_dma_glb->pool_capa.buf.max_cache_size; return 0; } @@ -738,11 +739,13 @@ odp_pool_t odp_dma_pool_create(const char *name, const odp_dma_pool_param_t *dma } odp_pool_param_init(&pool_param); - pool_param.type = ODP_POOL_BUFFER; - pool_param.buf.num = num; - pool_param.buf.uarea_size = uarea_size; - pool_param.buf.cache_size = cache_size; - pool_param.buf.size = sizeof(odp_dma_result_t); + pool_param.type = ODP_POOL_BUFFER; + pool_param.uarea_init.init_fn = dma_pool_param->uarea_init.init_fn; + pool_param.uarea_init.args = dma_pool_param->uarea_init.args; + pool_param.buf.num = num; + pool_param.buf.uarea_size = uarea_size; + pool_param.buf.cache_size = cache_size; + pool_param.buf.size = sizeof(odp_dma_result_t); pool = _odp_pool_create(name, &pool_param, ODP_POOL_DMA_COMPL); diff --git a/platform/linux-generic/odp_pool.c b/platform/linux-generic/odp_pool.c index 672b92c02..e7b2398de 100644 --- a/platform/linux-generic/odp_pool.c +++ b/platform/linux-generic/odp_pool.c @@ -604,6 +604,14 @@ static void init_buffers(pool_t *pool) ring_ptr_enq(ring, mask, event_hdr); } pool->skipped_blocks = skipped_blocks; + + if (pool->uarea_size && pool->params.uarea_init.init_fn) { + for (uint32_t i = 0; i < pool->num; i++) { + uarea = &pool->uarea_base_addr[i * pool->uarea_size]; + pool->params.uarea_init.init_fn(uarea, pool->param_uarea_size, + pool->params.uarea_init.args, i); + } + } } static bool shm_is_from_huge_pages(odp_shm_t shm) @@ -1479,6 +1487,7 @@ int odp_pool_capability(odp_pool_capability_t *capa) capa->buf.max_size = MAX_SIZE; capa->buf.max_num = CONFIG_POOL_MAX_NUM; capa->buf.max_uarea_size = MAX_UAREA_SIZE; + capa->buf.uarea_persistence = true; capa->buf.min_cache_size = 0; capa->buf.max_cache_size = CONFIG_POOL_CACHE_MAX_SIZE; capa->buf.stats.all = supported_stats.all; @@ -1495,6 +1504,7 @@ int odp_pool_capability(odp_pool_capability_t *capa) capa->pkt.min_seg_len = CONFIG_PACKET_SEG_LEN_MIN; capa->pkt.max_seg_len = max_seg_len; capa->pkt.max_uarea_size = MAX_UAREA_SIZE; + capa->pkt.uarea_persistence = true; capa->pkt.min_cache_size = 0; capa->pkt.max_cache_size = CONFIG_POOL_CACHE_MAX_SIZE; capa->pkt.stats.all = supported_stats.all; @@ -1503,6 +1513,7 @@ int odp_pool_capability(odp_pool_capability_t *capa) capa->tmo.max_pools = max_pools; capa->tmo.max_num = CONFIG_POOL_MAX_NUM; capa->tmo.max_uarea_size = MAX_UAREA_SIZE; + capa->tmo.uarea_persistence = true; capa->tmo.min_cache_size = 0; capa->tmo.max_cache_size = CONFIG_POOL_CACHE_MAX_SIZE; capa->tmo.stats.all = supported_stats.all; @@ -1512,6 +1523,7 @@ int odp_pool_capability(odp_pool_capability_t *capa) capa->vector.max_num = CONFIG_POOL_MAX_NUM; capa->vector.max_size = CONFIG_PACKET_VECTOR_MAX_SIZE; capa->vector.max_uarea_size = MAX_UAREA_SIZE; + capa->vector.uarea_persistence = true; capa->vector.min_cache_size = 0; capa->vector.max_cache_size = CONFIG_POOL_CACHE_MAX_SIZE; capa->vector.stats.all = supported_stats.all; @@ -1869,6 +1881,7 @@ int odp_pool_ext_capability(odp_pool_type_t type, odp_pool_ext_capability_t *cap capa->pkt.max_headroom_size = CONFIG_PACKET_HEADROOM; capa->pkt.max_segs_per_pkt = PKT_MAX_SEGS; capa->pkt.max_uarea_size = MAX_UAREA_SIZE; + capa->pkt.uarea_persistence = true; return 0; } @@ -2096,8 +2109,18 @@ int odp_pool_ext_populate(odp_pool_t pool_hdl, void *buf[], uint32_t buf_size, u pool->base_addr = min_addr; pool->max_addr = max_addr; - if (flags & ODP_POOL_POPULATE_DONE) + if (flags & ODP_POOL_POPULATE_DONE) { pool->max_addr = max_addr + buf_size - 1; + if (pool->uarea_size && pool->ext_param.uarea_init.init_fn) { + for (i = 0; i < pool->num_populated; i++) { + uarea = &pool->uarea_base_addr[i * pool->uarea_size]; + pool->ext_param.uarea_init.init_fn(uarea, pool->param_uarea_size, + pool->ext_param.uarea_init.args, + i); + } + } + } + return 0; } -- cgit v1.2.3 From f42d23801f34dc0ca0aa0463bb92bf4fdc310d8e Mon Sep 17 00:00:00 2001 From: Petri Savolainen Date: Wed, 28 Jun 2023 10:48:24 +0300 Subject: linux-gen: sync: implement new memory barriers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implement the new load and store synchronization barriers with x86 and aarch64 instructions. The default implementation falls back to the strongest GCC built-in (__sync_synchronize()), which would be the strong enough on x86 (mfence), but too weak on aarch64 (dmb). Signed-off-by: Petri Savolainen Reviewed-by: Matias Elo Reviewed-by: Jere Leppänen --- platform/linux-generic/Makefile.am | 5 ++++ .../arch/aarch64/odp/api/abi/sync_inlines.h | 31 ++++++++++++++++++++++ .../arch/default/odp/api/abi/sync_inlines.h | 31 ++++++++++++++++++++++ .../arch/x86/odp/api/abi/sync_inlines.h | 31 ++++++++++++++++++++++ .../include/odp/api/plat/sync_inlines.h | 27 ++++++++++++++++--- 5 files changed, 121 insertions(+), 4 deletions(-) create mode 100644 platform/linux-generic/arch/aarch64/odp/api/abi/sync_inlines.h create mode 100644 platform/linux-generic/arch/default/odp/api/abi/sync_inlines.h create mode 100644 platform/linux-generic/arch/x86/odp/api/abi/sync_inlines.h (limited to 'platform') diff --git a/platform/linux-generic/Makefile.am b/platform/linux-generic/Makefile.am index 4c02c56fa..7afdbf620 100644 --- a/platform/linux-generic/Makefile.am +++ b/platform/linux-generic/Makefile.am @@ -339,6 +339,7 @@ odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ arch/default/odp/api/abi/cpu_generic.h \ arch/arm/odp/api/abi/cpu_inlines.h \ arch/arm/odp/api/abi/cpu.h \ + arch/default/odp/api/abi/sync_inlines.h \ arch/default/odp/api/abi/time_inlines.h endif noinst_HEADERS += arch/arm/odp_atomic.h \ @@ -367,6 +368,7 @@ odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ arch/default/odp/api/abi/cpu_generic.h \ arch/aarch64/odp/api/abi/cpu_inlines.h \ arch/aarch64/odp/api/abi/cpu.h \ + arch/aarch64/odp/api/abi/sync_inlines.h \ arch/common/odp/api/abi/time_cpu_inlines.h \ arch/aarch64/odp/api/abi/time_inlines.h endif @@ -391,6 +393,7 @@ odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ arch/default/odp/api/abi/cpu_generic.h \ arch/default/odp/api/abi/cpu_inlines.h \ arch/default/odp/api/abi/cpu.h \ + arch/default/odp/api/abi/sync_inlines.h \ arch/default/odp/api/abi/time_inlines.h endif noinst_HEADERS += arch/default/odp_atomic.h \ @@ -412,6 +415,7 @@ odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ arch/default/odp/api/abi/cpu_generic.h \ arch/default/odp/api/abi/cpu_inlines.h \ arch/powerpc/odp/api/abi/cpu.h \ + arch/default/odp/api/abi/sync_inlines.h \ arch/default/odp/api/abi/time_inlines.h endif noinst_HEADERS += arch/default/odp_atomic.h \ @@ -436,6 +440,7 @@ odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ arch/default/odp/api/abi/atomic_inlines.h \ arch/x86/odp/api/abi/cpu_inlines.h \ arch/x86/odp/api/abi/cpu.h \ + arch/x86/odp/api/abi/sync_inlines.h \ arch/common/odp/api/abi/time_cpu_inlines.h \ arch/x86/odp/api/abi/time_inlines.h endif diff --git a/platform/linux-generic/arch/aarch64/odp/api/abi/sync_inlines.h b/platform/linux-generic/arch/aarch64/odp/api/abi/sync_inlines.h new file mode 100644 index 000000000..3d42e7dd8 --- /dev/null +++ b/platform/linux-generic/arch/aarch64/odp/api/abi/sync_inlines.h @@ -0,0 +1,31 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2023 Nokia + */ + +#ifndef ODP_ARCH_SYNC_INLINES_H_ +#define ODP_ARCH_SYNC_INLINES_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +static inline void _odp_mb_sync(void) +{ + __asm__ volatile("dsb sy" ::: "memory"); +} + +static inline void _odp_mb_sync_load(void) +{ + __asm__ volatile("dsb ld" ::: "memory"); +} + +static inline void _odp_mb_sync_store(void) +{ + __asm__ volatile("dsb st" ::: "memory"); +} + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/arch/default/odp/api/abi/sync_inlines.h b/platform/linux-generic/arch/default/odp/api/abi/sync_inlines.h new file mode 100644 index 000000000..bfbb3039f --- /dev/null +++ b/platform/linux-generic/arch/default/odp/api/abi/sync_inlines.h @@ -0,0 +1,31 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2023 Nokia + */ + +#ifndef ODP_ARCH_SYNC_INLINES_H_ +#define ODP_ARCH_SYNC_INLINES_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +static inline void _odp_mb_sync(void) +{ + __sync_synchronize(); +} + +static inline void _odp_mb_sync_load(void) +{ + __sync_synchronize(); +} + +static inline void _odp_mb_sync_store(void) +{ + __sync_synchronize(); +} + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/arch/x86/odp/api/abi/sync_inlines.h b/platform/linux-generic/arch/x86/odp/api/abi/sync_inlines.h new file mode 100644 index 000000000..bebe6b571 --- /dev/null +++ b/platform/linux-generic/arch/x86/odp/api/abi/sync_inlines.h @@ -0,0 +1,31 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2023 Nokia + */ + +#ifndef ODP_ARCH_SYNC_INLINES_H_ +#define ODP_ARCH_SYNC_INLINES_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +static inline void _odp_mb_sync(void) +{ + __asm__ volatile("mfence" ::: "memory"); +} + +static inline void _odp_mb_sync_load(void) +{ + __asm__ volatile("lfence" ::: "memory"); +} + +static inline void _odp_mb_sync_store(void) +{ + __asm__ volatile("sfence" ::: "memory"); +} + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/include/odp/api/plat/sync_inlines.h b/platform/linux-generic/include/odp/api/plat/sync_inlines.h index b6a96188c..b3a88b629 100644 --- a/platform/linux-generic/include/odp/api/plat/sync_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/sync_inlines.h @@ -1,7 +1,6 @@ -/* Copyright (c) 2016-2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2016-2018 Linaro Limited + * Copyright (c) 2023 Nokia */ /** @@ -13,6 +12,8 @@ #ifndef ODP_PLAT_SYNC_INLINE_H_ #define ODP_PLAT_SYNC_INLINE_H_ +#include + #ifdef __cplusplus extern "C" { #endif @@ -25,6 +26,9 @@ extern "C" { #define odp_mb_release __odp_mb_release #define odp_mb_acquire __odp_mb_acquire #define odp_mb_full __odp_mb_full + #define odp_mb_sync __odp_mb_sync + #define odp_mb_sync_load __odp_mb_sync_load + #define odp_mb_sync_store __odp_mb_sync_store #else #define _ODP_INLINE #endif @@ -44,6 +48,21 @@ _ODP_INLINE void odp_mb_full(void) __atomic_thread_fence(__ATOMIC_SEQ_CST); } +_ODP_INLINE void odp_mb_sync(void) +{ + _odp_mb_sync(); +} + +_ODP_INLINE void odp_mb_sync_load(void) +{ + _odp_mb_sync_load(); +} + +_ODP_INLINE void odp_mb_sync_store(void) +{ + _odp_mb_sync_store(); +} + /** @endcond */ #ifdef __cplusplus -- cgit v1.2.3 From ec336bda1f9c1773944a1807e927d8ee69af778a Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Wed, 9 Aug 2023 09:23:14 +0300 Subject: Port d158bbd99 "linux-gen: dma: inline completion event conversion functions" Port original commit from linux-generic. Signed-off-by: Matias Elo Reviewed-by: Tuomas Taipale --- platform/linux-dpdk/Makefile.am | 3 +++ platform/linux-dpdk/include-abi/odp/api/abi/dma.h | 1 + platform/linux-dpdk/include/odp/api/plat/dma_inlines.h | 1 + 3 files changed, 5 insertions(+) create mode 120000 platform/linux-dpdk/include-abi/odp/api/abi/dma.h create mode 120000 platform/linux-dpdk/include/odp/api/plat/dma_inlines.h (limited to 'platform') diff --git a/platform/linux-dpdk/Makefile.am b/platform/linux-dpdk/Makefile.am index 252b34fb2..febc6e8eb 100644 --- a/platform/linux-dpdk/Makefile.am +++ b/platform/linux-dpdk/Makefile.am @@ -33,6 +33,7 @@ odpapiplatinclude_HEADERS = \ include/odp/api/plat/byteorder_inlines.h \ include/odp/api/plat/cpu_inlines.h \ include/odp/api/plat/crypto_inlines.h \ + include/odp/api/plat/dma_inlines.h \ include/odp/api/plat/debug_inlines.h \ include/odp/api/plat/event_inlines.h \ include/odp/api/plat/event_inline_types.h \ @@ -78,6 +79,7 @@ odpapiabiarchinclude_HEADERS += \ include-abi/odp/api/abi/crypto.h \ include-abi/odp/api/abi/crypto_types.h \ include-abi/odp/api/abi/debug.h \ + include-abi/odp/api/abi/dma.h \ include-abi/odp/api/abi/dma_types.h \ include-abi/odp/api/abi/errno.h \ include-abi/odp/api/abi/event.h \ @@ -259,6 +261,7 @@ __LIB__libodp_dpdk_la_SOURCES += \ ../linux-generic/odp_byteorder_api.c \ ../linux-generic/odp_cpu_api.c \ ../linux-generic/odp_crypto_api.c \ + ../linux-generic/odp_dma_api.c \ ../linux-generic/odp_event_api.c \ ../linux-generic/odp_hash_api.c \ ../linux-generic/odp_ipsec_api.c \ diff --git a/platform/linux-dpdk/include-abi/odp/api/abi/dma.h b/platform/linux-dpdk/include-abi/odp/api/abi/dma.h new file mode 120000 index 000000000..112a7b5a7 --- /dev/null +++ b/platform/linux-dpdk/include-abi/odp/api/abi/dma.h @@ -0,0 +1 @@ +../../../../../linux-generic/include-abi/odp/api/abi/dma.h \ No newline at end of file diff --git a/platform/linux-dpdk/include/odp/api/plat/dma_inlines.h b/platform/linux-dpdk/include/odp/api/plat/dma_inlines.h new file mode 120000 index 000000000..5b60f374e --- /dev/null +++ b/platform/linux-dpdk/include/odp/api/plat/dma_inlines.h @@ -0,0 +1 @@ +../../../../../linux-generic/include/odp/api/plat/dma_inlines.h \ No newline at end of file -- cgit v1.2.3 From af8f6a307c781f98ea58b6a7b1ac47b74ae58d54 Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Wed, 9 Aug 2023 09:51:11 +0300 Subject: Port aa53b4173 "linux-gen: cpu: detect constant_tsc from cpuinfo" Port original commit from linux-generic. Signed-off-by: Matias Elo Reviewed-by: Tuomas Taipale --- platform/linux-dpdk/odp_init.c | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) (limited to 'platform') diff --git a/platform/linux-dpdk/odp_init.c b/platform/linux-dpdk/odp_init.c index c7ae1ea47..7e6aaab55 100644 --- a/platform/linux-dpdk/odp_init.c +++ b/platform/linux-dpdk/odp_init.c @@ -35,8 +35,8 @@ enum init_stage { LIBCONFIG_INIT, CPUMASK_INIT, CPU_CYCLES_INIT, - TIME_INIT, SYSINFO_INIT, + TIME_INIT, ISHM_INIT, FDSERVER_INIT, GLOBAL_RW_DATA_INIT, @@ -473,16 +473,16 @@ static int term_global(enum init_stage stage) } /* Fall through */ - case SYSINFO_INIT: - if (_odp_system_info_term()) { - _ODP_ERR("ODP system info term failed.\n"); + case TIME_INIT: + if (_odp_time_term_global()) { + _ODP_ERR("ODP time term failed.\n"); rc = -1; } /* Fall through */ - case TIME_INIT: - if (_odp_time_term_global()) { - _ODP_ERR("ODP time term failed.\n"); + case SYSINFO_INIT: + if (_odp_system_info_term()) { + _ODP_ERR("ODP system info term failed.\n"); rc = -1; } /* Fall through */ @@ -558,18 +558,18 @@ int odp_init_global(odp_instance_t *instance, return -1; } - if (_odp_time_init_global()) { - _ODP_ERR("ODP time init failed.\n"); - goto init_failed; - } - stage = TIME_INIT; - if (_odp_system_info_init()) { _ODP_ERR("ODP system_info init failed.\n"); goto init_failed; } stage = SYSINFO_INIT; + if (_odp_time_init_global()) { + _ODP_ERR("ODP time init failed.\n"); + goto init_failed; + } + stage = TIME_INIT; + if (_odp_shm_init_global(params)) { _ODP_ERR("ODP shm init failed.\n"); goto init_failed; -- cgit v1.2.3 From 72e001d3dee103dc8ba780bb819a18ba22503dd4 Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Wed, 9 Aug 2023 09:54:32 +0300 Subject: Port 009ee2e68 "linux-gen: timer: implement odp_timeout_from_event_multi() function" Port original commit from linux-generic. Signed-off-by: Matias Elo Reviewed-by: Tuomas Taipale --- platform/linux-dpdk/include/odp/api/plat/timer_inlines.h | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) (limited to 'platform') diff --git a/platform/linux-dpdk/include/odp/api/plat/timer_inlines.h b/platform/linux-dpdk/include/odp/api/plat/timer_inlines.h index 357d4df06..66327acdb 100644 --- a/platform/linux-dpdk/include/odp/api/plat/timer_inlines.h +++ b/platform/linux-dpdk/include/odp/api/plat/timer_inlines.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2022, Nokia +/* Copyright (c) 2022-2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -25,6 +25,7 @@ #define odp_timeout_user_ptr __odp_timeout_user_ptr #define odp_timeout_user_area __odp_timeout_user_area #define odp_timeout_from_event __odp_timeout_from_event + #define odp_timeout_from_event_multi __odp_timeout_from_event_multi #define odp_timeout_to_event __odp_timeout_to_event #else #define _ODP_INLINE @@ -57,6 +58,15 @@ _ODP_INLINE odp_timeout_t odp_timeout_from_event(odp_event_t ev) return (odp_timeout_t)ev; } +_ODP_INLINE void odp_timeout_from_event_multi(odp_timeout_t tmo[], const odp_event_t ev[], int num) +{ + for (int i = 0; i < num; i++) { + _ODP_ASSERT(odp_event_type(ev[i]) == ODP_EVENT_TIMEOUT); + + tmo[i] = (odp_timeout_t)ev[i]; + } +} + _ODP_INLINE odp_event_t odp_timeout_to_event(odp_timeout_t tmo) { return (odp_event_t)tmo; -- cgit v1.2.3 From 15f79efa9022b5076388d21ea4587d911afab963 Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Wed, 9 Aug 2023 10:14:09 +0300 Subject: Port 2fdf01b70 "linux-gen: debug: utilize new _ODP_WARN() macro" Port original commit from linux-generic. Signed-off-by: Matias Elo Reviewed-by: Tuomas Taipale --- platform/linux-dpdk/odp_system_info.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'platform') diff --git a/platform/linux-dpdk/odp_system_info.c b/platform/linux-dpdk/odp_system_info.c index c76820e5d..886f7f216 100644 --- a/platform/linux-dpdk/odp_system_info.c +++ b/platform/linux-dpdk/odp_system_info.c @@ -59,8 +59,8 @@ static int read_cache_line_size(void) file = fopen(CACHE_LNSZ_FILE, "rt"); if (file == NULL) { /* File not found */ - _ODP_PRINT("WARN: unable to read host CPU cache line size. " - "Using ODP_CACHE_LINE_SIZE instead.\n"); + _ODP_WARN("Unable to read host CPU cache line size. " + "Using ODP_CACHE_LINE_SIZE instead.\n"); return ODP_CACHE_LINE_SIZE; } @@ -225,7 +225,7 @@ static int system_cache_line(system_info_t *sysinfo) sysinfo->cache_line_size = ret; if (ret != ODP_CACHE_LINE_SIZE) - _ODP_PRINT("WARN: host CPU cache line size and ODP_CACHE_LINE_SIZE don't match.\n"); + _ODP_WARN("Host CPU cache line size and ODP_CACHE_LINE_SIZE don't match.\n"); return 0; } -- cgit v1.2.3 From a5e4d9b2368454413f774e1bac879a130b7609d8 Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Wed, 9 Aug 2023 10:24:06 +0300 Subject: Port f42d23801 "linux-gen: sync: implement new memory barriers" Port original commit from linux-generic. Signed-off-by: Matias Elo Reviewed-by: Tuomas Taipale --- platform/linux-dpdk/Makefile.am | 15 ++++++++++----- .../linux-dpdk/arch/aarch64/odp/api/abi/sync_inlines.h | 1 + .../linux-dpdk/arch/default/odp/api/abi/sync_inlines.h | 1 + platform/linux-dpdk/arch/x86/odp/api/abi/sync_inlines.h | 1 + 4 files changed, 13 insertions(+), 5 deletions(-) create mode 120000 platform/linux-dpdk/arch/aarch64/odp/api/abi/sync_inlines.h create mode 120000 platform/linux-dpdk/arch/default/odp/api/abi/sync_inlines.h create mode 120000 platform/linux-dpdk/arch/x86/odp/api/abi/sync_inlines.h (limited to 'platform') diff --git a/platform/linux-dpdk/Makefile.am b/platform/linux-dpdk/Makefile.am index febc6e8eb..a11ccd193 100644 --- a/platform/linux-dpdk/Makefile.am +++ b/platform/linux-dpdk/Makefile.am @@ -297,7 +297,8 @@ odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ arch/default/odp/api/abi/atomic_inlines.h \ arch/default/odp/api/abi/cpu_generic.h \ arch/arm/odp/api/abi/cpu_inlines.h \ - arch/arm/odp/api/abi/cpu.h + arch/arm/odp/api/abi/cpu.h \ + arch/default/odp/api/abi/sync_inlines.h endif noinst_HEADERS += arch/arm/odp_atomic.h \ arch/arm/odp_cpu.h \ @@ -325,7 +326,8 @@ odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ arch/aarch64/odp/api/abi/atomic.h \ arch/default/odp/api/abi/cpu_generic.h \ arch/aarch64/odp/api/abi/cpu_inlines.h \ - arch/aarch64/odp/api/abi/cpu.h + arch/aarch64/odp/api/abi/cpu.h \ + arch/aarch64/odp/api/abi/sync_inlines.h endif noinst_HEADERS += arch/aarch64/odp_atomic.h \ arch/aarch64/odp_cpu.h \ @@ -348,7 +350,8 @@ odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ arch/default/odp/api/abi/atomic_inlines.h \ arch/default/odp/api/abi/cpu_generic.h \ arch/default/odp/api/abi/cpu_inlines.h \ - arch/default/odp/api/abi/cpu.h + arch/default/odp/api/abi/cpu.h \ + arch/default/odp/api/abi/sync_inlines.h endif noinst_HEADERS += arch/default/odp_atomic.h \ arch/default/odp_cpu.h \ @@ -369,7 +372,8 @@ odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ arch/default/odp/api/abi/atomic_inlines.h \ arch/default/odp/api/abi/cpu_generic.h \ arch/default/odp/api/abi/cpu_inlines.h \ - arch/powerpc/odp/api/abi/cpu.h + arch/powerpc/odp/api/abi/cpu.h \ + arch/default/odp/api/abi/sync_inlines.h endif noinst_HEADERS += arch/default/odp_atomic.h \ arch/default/odp_cpu.h \ @@ -391,7 +395,8 @@ if !ODP_ABI_COMPAT odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ arch/default/odp/api/abi/atomic_inlines.h \ arch/x86/odp/api/abi/cpu_inlines.h \ - arch/x86/odp/api/abi/cpu.h + arch/x86/odp/api/abi/cpu.h \ + arch/x86/odp/api/abi/sync_inlines.h endif noinst_HEADERS += arch/x86/cpu_flags.h \ arch/x86/odp_cpu.h \ diff --git a/platform/linux-dpdk/arch/aarch64/odp/api/abi/sync_inlines.h b/platform/linux-dpdk/arch/aarch64/odp/api/abi/sync_inlines.h new file mode 120000 index 000000000..1281f2376 --- /dev/null +++ b/platform/linux-dpdk/arch/aarch64/odp/api/abi/sync_inlines.h @@ -0,0 +1 @@ +../../../../../../linux-generic/arch/aarch64/odp/api/abi/sync_inlines.h \ No newline at end of file diff --git a/platform/linux-dpdk/arch/default/odp/api/abi/sync_inlines.h b/platform/linux-dpdk/arch/default/odp/api/abi/sync_inlines.h new file mode 120000 index 000000000..328ff07d6 --- /dev/null +++ b/platform/linux-dpdk/arch/default/odp/api/abi/sync_inlines.h @@ -0,0 +1 @@ +../../../../../../linux-generic/arch/default/odp/api/abi/sync_inlines.h \ No newline at end of file diff --git a/platform/linux-dpdk/arch/x86/odp/api/abi/sync_inlines.h b/platform/linux-dpdk/arch/x86/odp/api/abi/sync_inlines.h new file mode 120000 index 000000000..d5dba2679 --- /dev/null +++ b/platform/linux-dpdk/arch/x86/odp/api/abi/sync_inlines.h @@ -0,0 +1 @@ +../../../../../../linux-generic/arch/x86/odp/api/abi/sync_inlines.h \ No newline at end of file -- cgit v1.2.3 From 69592b88fe7743bf9084d39a6863eeb7807dbb06 Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Wed, 9 Aug 2023 16:06:41 +0300 Subject: Port 3a8f63d29 "linux-gen: implement pool user area initialization" Port original commit from linux-generic. Signed-off-by: Matias Elo Reviewed-by: Tuomas Taipale --- platform/linux-dpdk/include/odp_pool_internal.h | 1 + platform/linux-dpdk/odp_pool.c | 31 ++++++++++++++++++++----- 2 files changed, 26 insertions(+), 6 deletions(-) (limited to 'platform') diff --git a/platform/linux-dpdk/include/odp_pool_internal.h b/platform/linux-dpdk/include/odp_pool_internal.h index b82324a77..8512806f7 100644 --- a/platform/linux-dpdk/include/odp_pool_internal.h +++ b/platform/linux-dpdk/include/odp_pool_internal.h @@ -83,6 +83,7 @@ typedef struct ODP_ALIGNED_CACHE { odp_pool_ext_param_t ext_param; odp_shm_t uarea_shm; uint64_t uarea_shm_size; + uint32_t param_uarea_size; uint32_t uarea_size; uint32_t trailer_size; /* Endmark size */ uint8_t *uarea_base_addr; diff --git a/platform/linux-dpdk/odp_pool.c b/platform/linux-dpdk/odp_pool.c index 83d75dd6c..fd27cd79e 100644 --- a/platform/linux-dpdk/odp_pool.c +++ b/platform/linux-dpdk/odp_pool.c @@ -263,6 +263,7 @@ int odp_pool_capability(odp_pool_capability_t *capa) capa->buf.max_size = MAX_SIZE; capa->buf.max_num = CONFIG_POOL_MAX_NUM; capa->buf.max_uarea_size = MAX_UAREA_SIZE; + capa->buf.uarea_persistence = true; capa->buf.min_cache_size = 0; capa->buf.max_cache_size = RTE_MEMPOOL_CACHE_MAX_SIZE; capa->buf.stats.all = supported_stats.all; @@ -279,6 +280,7 @@ int odp_pool_capability(odp_pool_capability_t *capa) capa->pkt.min_seg_len = CONFIG_PACKET_SEG_LEN_MIN; capa->pkt.max_seg_len = CONFIG_PACKET_MAX_SEG_LEN; capa->pkt.max_uarea_size = MAX_UAREA_SIZE; + capa->pkt.uarea_persistence = true; capa->pkt.min_cache_size = 0; capa->pkt.max_cache_size = RTE_MEMPOOL_CACHE_MAX_SIZE; capa->pkt.stats.all = supported_stats.all; @@ -287,6 +289,7 @@ int odp_pool_capability(odp_pool_capability_t *capa) capa->tmo.max_pools = max_pools; capa->tmo.max_num = CONFIG_POOL_MAX_NUM; capa->tmo.max_uarea_size = MAX_UAREA_SIZE; + capa->tmo.uarea_persistence = true; capa->tmo.min_cache_size = 0; capa->tmo.max_cache_size = RTE_MEMPOOL_CACHE_MAX_SIZE; capa->tmo.stats.all = supported_stats.all; @@ -295,6 +298,7 @@ int odp_pool_capability(odp_pool_capability_t *capa) capa->vector.max_pools = max_pools; capa->vector.max_num = CONFIG_POOL_MAX_NUM; capa->vector.max_uarea_size = MAX_UAREA_SIZE; + capa->vector.uarea_persistence = true; capa->vector.max_size = CONFIG_PACKET_VECTOR_MAX_SIZE; capa->vector.min_cache_size = 0; capa->vector.max_cache_size = RTE_MEMPOOL_CACHE_MAX_SIZE; @@ -550,6 +554,7 @@ static int reserve_uarea(pool_t *pool, uint32_t uarea_size, uint32_t num_pkt) pool->uarea_shm = ODP_SHM_INVALID; if (uarea_size == 0) { + pool->param_uarea_size = 0; pool->uarea_size = 0; pool->uarea_shm_size = 0; return 0; @@ -559,6 +564,7 @@ static int reserve_uarea(pool_t *pool, uint32_t uarea_size, uint32_t num_pkt) pool->pool_idx, pool->name); uarea_name[ODP_SHM_NAME_LEN - 1] = 0; + pool->param_uarea_size = uarea_size; pool->uarea_size = _ODP_ROUNDUP_CACHE_LINE(uarea_size); pool->uarea_shm_size = num_pkt * (uint64_t)pool->uarea_size; @@ -605,7 +611,8 @@ static void init_obj_priv_data(struct rte_mempool *mp ODP_UNUSED, void *arg, voi struct priv_data_t *priv_data = arg; struct rte_mbuf *mb = mbuf; _odp_event_hdr_t *event_hdr = (_odp_event_hdr_t *)mbuf; - void *uarea = priv_data->pool->uarea_base_addr + i * priv_data->pool->uarea_size; + pool_t *pool = priv_data->pool; + void *uarea = pool->uarea_base_addr + i * pool->uarea_size; void **obj_uarea; if (priv_data->type != ODP_POOL_PACKET) @@ -613,7 +620,7 @@ static void init_obj_priv_data(struct rte_mempool *mp ODP_UNUSED, void *arg, voi mb->data_off = 0; event_hdr->hdr.index = i; - event_hdr->hdr.pool = _odp_pool_handle(priv_data->pool); + event_hdr->hdr.pool = _odp_pool_handle(pool); event_hdr->hdr.type = priv_data->type; event_hdr->hdr.event_type = priv_data->event_type; @@ -636,6 +643,10 @@ static void init_obj_priv_data(struct rte_mempool *mp ODP_UNUSED, void *arg, voi *obj_uarea = uarea; + if (uarea && pool->params.uarea_init.init_fn) + pool->params.uarea_init.init_fn(uarea, pool->param_uarea_size, + pool->params.uarea_init.args, i); + if (priv_data->type == ODP_POOL_BUFFER || priv_data->type == ODP_POOL_PACKET) { mb->buf_len -= _ODP_EV_ENDMARK_SIZE; _odp_event_endmark_set(_odp_event_from_mbuf(mb)); @@ -751,13 +762,15 @@ odp_pool_t _odp_pool_create(const char *name, const odp_pool_param_t *params, return ODP_POOL_INVALID; } - rte_mempool_obj_iter(mp, init_obj_priv_data, &priv_data); pool->rte_mempool = mp; pool->seg_len = seg_size; pool->type_2 = type_2; pool->type = params->type; pool->params = *params; pool->trailer_size = trailer; + + rte_mempool_obj_iter(mp, init_obj_priv_data, &priv_data); + UNLOCK(&pool->lock); pool_hdl = _odp_pool_handle(pool); @@ -1097,6 +1110,7 @@ int odp_pool_ext_capability(odp_pool_type_t type, capa->pkt.max_headroom_size = RTE_PKTMBUF_HEADROOM; capa->pkt.max_segs_per_pkt = PKT_MAX_SEGS; capa->pkt.max_uarea_size = MAX_UAREA_SIZE; + capa->pkt.uarea_persistence = true; return 0; } @@ -1297,10 +1311,11 @@ static void init_ext_obj(struct rte_mempool *mp, void *arg, void *mbuf, unsigned { struct mbuf_ctor_arg *mb_ctor_arg = arg; struct rte_mbuf *mb = mbuf; - void *uarea = mb_ctor_arg->pool->uarea_base_addr + i * mb_ctor_arg->pool->uarea_size; + pool_t *pool = mb_ctor_arg->pool; + void *uarea = pool->uarea_base_addr + i * pool->uarea_size; _odp_event_hdr_t *event_hdr = (_odp_event_hdr_t *)mbuf; void **obj_uarea; - odp_pool_ext_param_t *p = &mb_ctor_arg->pool->ext_param; + odp_pool_ext_param_t *p = &pool->ext_param; uint32_t app_hdr_offset = sizeof(odp_packet_hdr_t); uint32_t app_hdr_size = p->pkt.app_header_size; uint32_t buf_size = p->pkt.buf_size; @@ -1333,7 +1348,7 @@ static void init_ext_obj(struct rte_mempool *mp, void *arg, void *mbuf, unsigned mb->next = NULL; /* Save index, might be useful for debugging purposes */ event_hdr->hdr.index = i; - event_hdr->hdr.pool = _odp_pool_handle(mb_ctor_arg->pool); + event_hdr->hdr.pool = _odp_pool_handle(pool); event_hdr->hdr.type = mb_ctor_arg->type; event_hdr->hdr.event_type = mb_ctor_arg->event_type; @@ -1356,6 +1371,10 @@ static void init_ext_obj(struct rte_mempool *mp, void *arg, void *mbuf, unsigned *obj_uarea = uarea; + if (uarea && pool->ext_param.uarea_init.init_fn) + pool->ext_param.uarea_init.init_fn(uarea, pool->param_uarea_size, + pool->ext_param.uarea_init.args, i); + if (mb_ctor_arg->type == ODP_POOL_BUFFER || mb_ctor_arg->type == ODP_POOL_PACKET) { mb->buf_len -= _ODP_EV_ENDMARK_SIZE; _odp_event_endmark_set(_odp_event_from_mbuf(mb)); -- cgit v1.2.3 From cf8d53ba3f8eca93934790cb5d5d97414a5df539 Mon Sep 17 00:00:00 2001 From: Janne Peltonen Date: Fri, 11 Aug 2023 11:59:40 +0300 Subject: linux-dpdk: crypto: fail bit mode session creation when not supported ODP-DPDK does not support bit mode with any crypto algorithm. Fail session creation when a bit mode session is requested. Signed-off-by: Janne Peltonen Reviewed-by: Tuomas Taipale --- platform/linux-dpdk/odp_crypto.c | 10 ++++++++++ 1 file changed, 10 insertions(+) (limited to 'platform') diff --git a/platform/linux-dpdk/odp_crypto.c b/platform/linux-dpdk/odp_crypto.c index 5761787f8..3269e3075 100644 --- a/platform/linux-dpdk/odp_crypto.c +++ b/platform/linux-dpdk/odp_crypto.c @@ -1431,6 +1431,16 @@ int odp_crypto_session_create(const odp_crypto_session_param_t *param, return -1; } + if (param->cipher_range_in_bits) { + *status = ODP_CRYPTO_SES_ERR_CIPHER; + *session_out = ODP_CRYPTO_SESSION_INVALID; + return -1; + } + if (param->auth_range_in_bits) { + *status = ODP_CRYPTO_SES_ERR_AUTH; + *session_out = ODP_CRYPTO_SESSION_INVALID; + return -1; + } if (param->auth_alg == ODP_AUTH_ALG_AES_GMAC && param->cipher_alg != ODP_CIPHER_ALG_NULL) { *status = ODP_CRYPTO_SES_ERR_ALG_COMBO; -- cgit v1.2.3 From 7a200d709b64e743c002dd9737cd71f54ccc54ac Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Fri, 9 Jun 2023 11:11:10 +0300 Subject: linux-dpdk: time: refactor implementation Simplify and optimize performance of time implementation by supporting only DPDK time API. Signed-off-by: Matias Elo Reviewed-by: Tuomas Taipale --- platform/linux-dpdk/Makefile.am | 21 +- .../linux-dpdk/arch/aarch64/odp/api/abi/cpu_time.h | 1 - platform/linux-dpdk/arch/aarch64/odp_global_time.c | 1 - .../linux-dpdk/arch/default/odp/api/abi/cpu_time.h | 1 - platform/linux-dpdk/arch/default/odp_global_time.c | 1 - .../linux-dpdk/arch/x86/odp/api/abi/cpu_time.h | 1 - .../linux-dpdk/arch/x86/odp/api/abi/time_cpu.h | 18 ++ platform/linux-dpdk/arch/x86/odp_global_time.c | 1 - .../linux-dpdk/include/odp/api/plat/time_inlines.h | 180 +++++++------ platform/linux-dpdk/odp_time.c | 285 +-------------------- 10 files changed, 139 insertions(+), 371 deletions(-) delete mode 120000 platform/linux-dpdk/arch/aarch64/odp/api/abi/cpu_time.h delete mode 120000 platform/linux-dpdk/arch/aarch64/odp_global_time.c delete mode 120000 platform/linux-dpdk/arch/default/odp/api/abi/cpu_time.h delete mode 120000 platform/linux-dpdk/arch/default/odp_global_time.c delete mode 120000 platform/linux-dpdk/arch/x86/odp/api/abi/cpu_time.h create mode 100644 platform/linux-dpdk/arch/x86/odp/api/abi/time_cpu.h delete mode 120000 platform/linux-dpdk/arch/x86/odp_global_time.c (limited to 'platform') diff --git a/platform/linux-dpdk/Makefile.am b/platform/linux-dpdk/Makefile.am index a11ccd193..ad608c4e1 100644 --- a/platform/linux-dpdk/Makefile.am +++ b/platform/linux-dpdk/Makefile.am @@ -286,12 +286,10 @@ endif if ARCH_IS_ARM __LIB__libodp_dpdk_la_SOURCES += arch/default/odp_atomic.c \ arch/default/odp_cpu_cycles.c \ - arch/default/odp_global_time.c \ arch/default/odp_hash_crc32.c \ arch/default/odp_random.c \ arch/arm/odp_sysinfo_parse.c -odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_time.h \ - arch/default/odp/api/abi/hash_crc32.h +odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/hash_crc32.h if !ODP_ABI_COMPAT odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ arch/default/odp/api/abi/atomic_inlines.h \ @@ -314,12 +312,10 @@ if ARCH_IS_AARCH64 __LIB__libodp_dpdk_la_SOURCES += arch/aarch64/odp_atomic.c \ arch/default/odp_cpu_cycles.c \ arch/aarch64/cpu_flags.c \ - arch/aarch64/odp_global_time.c \ arch/default/odp_hash_crc32.c \ arch/default/odp_random.c \ arch/aarch64/odp_sysinfo_parse.c -odpapiabiarchinclude_HEADERS += arch/aarch64/odp/api/abi/cpu_time.h \ - arch/aarch64/odp/api/abi/hash_crc32.h +odpapiabiarchinclude_HEADERS += arch/aarch64/odp/api/abi/hash_crc32.h if !ODP_ABI_COMPAT odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ arch/aarch64/odp/api/abi/atomic_inlines.h \ @@ -339,12 +335,10 @@ endif if ARCH_IS_DEFAULT __LIB__libodp_dpdk_la_SOURCES += arch/default/odp_atomic.c \ arch/default/odp_cpu_cycles.c \ - arch/default/odp_global_time.c \ arch/default/odp_hash_crc32.c \ arch/default/odp_random.c \ arch/default/odp_sysinfo_parse.c -odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_time.h \ - arch/default/odp/api/abi/hash_crc32.h +odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/hash_crc32.h if !ODP_ABI_COMPAT odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ arch/default/odp/api/abi/atomic_inlines.h \ @@ -361,12 +355,10 @@ endif if ARCH_IS_POWERPC __LIB__libodp_dpdk_la_SOURCES += arch/default/odp_atomic.c \ arch/default/odp_cpu_cycles.c \ - arch/default/odp_global_time.c \ arch/default/odp_hash_crc32.c \ arch/default/odp_random.c \ arch/powerpc/odp_sysinfo_parse.c -odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_time.h \ - arch/default/odp/api/abi/hash_crc32.h +odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/hash_crc32.h if !ODP_ABI_COMPAT odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ arch/default/odp/api/abi/atomic_inlines.h \ @@ -384,13 +376,12 @@ if ARCH_IS_X86 __LIB__libodp_dpdk_la_SOURCES += arch/default/odp_atomic.c \ arch/x86/cpu_flags.c \ arch/x86/odp_cpu_cycles.c \ - arch/x86/odp_global_time.c \ arch/default/odp_hash_crc32.c \ arch/default/odp_random.c \ arch/x86/odp_sysinfo_parse.c odpapiabiarchinclude_HEADERS += arch/x86/odp/api/abi/cpu_rdtsc.h \ - arch/x86/odp/api/abi/cpu_time.h \ - arch/x86/odp/api/abi/hash_crc32.h + arch/x86/odp/api/abi/hash_crc32.h \ + arch/x86/odp/api/abi/time_cpu.h if !ODP_ABI_COMPAT odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ arch/default/odp/api/abi/atomic_inlines.h \ diff --git a/platform/linux-dpdk/arch/aarch64/odp/api/abi/cpu_time.h b/platform/linux-dpdk/arch/aarch64/odp/api/abi/cpu_time.h deleted file mode 120000 index 52719241f..000000000 --- a/platform/linux-dpdk/arch/aarch64/odp/api/abi/cpu_time.h +++ /dev/null @@ -1 +0,0 @@ -../../../../../../linux-generic/arch/aarch64/odp/api/abi/cpu_time.h \ No newline at end of file diff --git a/platform/linux-dpdk/arch/aarch64/odp_global_time.c b/platform/linux-dpdk/arch/aarch64/odp_global_time.c deleted file mode 120000 index 1b7fa3c24..000000000 --- a/platform/linux-dpdk/arch/aarch64/odp_global_time.c +++ /dev/null @@ -1 +0,0 @@ -../../../linux-generic/arch/aarch64/odp_global_time.c \ No newline at end of file diff --git a/platform/linux-dpdk/arch/default/odp/api/abi/cpu_time.h b/platform/linux-dpdk/arch/default/odp/api/abi/cpu_time.h deleted file mode 120000 index 941d6ccca..000000000 --- a/platform/linux-dpdk/arch/default/odp/api/abi/cpu_time.h +++ /dev/null @@ -1 +0,0 @@ -../../../../../../linux-generic/arch/default/odp/api/abi/cpu_time.h \ No newline at end of file diff --git a/platform/linux-dpdk/arch/default/odp_global_time.c b/platform/linux-dpdk/arch/default/odp_global_time.c deleted file mode 120000 index 3f54b1478..000000000 --- a/platform/linux-dpdk/arch/default/odp_global_time.c +++ /dev/null @@ -1 +0,0 @@ -../../../linux-generic/arch/default/odp_global_time.c \ No newline at end of file diff --git a/platform/linux-dpdk/arch/x86/odp/api/abi/cpu_time.h b/platform/linux-dpdk/arch/x86/odp/api/abi/cpu_time.h deleted file mode 120000 index 2fb280b8c..000000000 --- a/platform/linux-dpdk/arch/x86/odp/api/abi/cpu_time.h +++ /dev/null @@ -1 +0,0 @@ -../../../../../../linux-generic/arch/x86/odp/api/abi/cpu_time.h \ No newline at end of file diff --git a/platform/linux-dpdk/arch/x86/odp/api/abi/time_cpu.h b/platform/linux-dpdk/arch/x86/odp/api/abi/time_cpu.h new file mode 100644 index 000000000..cc313cff1 --- /dev/null +++ b/platform/linux-dpdk/arch/x86/odp/api/abi/time_cpu.h @@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2023 Nokia + */ + +#ifndef ODP_ARCH_TIME_CPU_H_ +#define ODP_ARCH_TIME_CPU_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +int _odp_time_cpu_global_freq_is_const(void); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-dpdk/arch/x86/odp_global_time.c b/platform/linux-dpdk/arch/x86/odp_global_time.c deleted file mode 120000 index 33a50e23e..000000000 --- a/platform/linux-dpdk/arch/x86/odp_global_time.c +++ /dev/null @@ -1 +0,0 @@ -../../../linux-generic/arch/x86/odp_global_time.c \ No newline at end of file diff --git a/platform/linux-dpdk/include/odp/api/plat/time_inlines.h b/platform/linux-dpdk/include/odp/api/plat/time_inlines.h index 4b1320300..8e4eec59c 100644 --- a/platform/linux-dpdk/include/odp/api/plat/time_inlines.h +++ b/platform/linux-dpdk/include/odp/api/plat/time_inlines.h @@ -12,125 +12,116 @@ #include #include -#include +#include +#include +#include #include /** @cond _ODP_HIDE_FROM_DOXYGEN_ */ -#define _ODP_TIMESPEC_SIZE 16 -#define _ODP_TIME_GIGA_HZ 1000000000ULL - -typedef odp_time_t (*time_cur_fn)(void); -typedef odp_time_t (*time_cur_strict_fn)(void); -typedef uint64_t (*time_res_fn)(void); - -typedef struct time_handler_ { - time_cur_fn time_cur; - time_cur_strict_fn time_cur_strict; - time_res_fn time_res; - -} time_handler_t; - typedef struct _odp_time_global_t { - /* Storage space for struct timespec. Posix headers are not included - * here to avoid application exposure. */ - uint8_t timespec[_ODP_TIMESPEC_SIZE] ODP_ALIGNED(_ODP_TIMESPEC_SIZE); - - int use_hw; - uint64_t hw_start; - uint64_t hw_freq_hz; - /* DPDK specific */ - time_handler_t handler; + uint64_t start_cycles; + uint64_t freq_hz; } _odp_time_global_t; extern _odp_time_global_t _odp_time_glob; -odp_time_t _odp_timespec_cur(void); - -static inline odp_time_t _odp_time_cur_gen(void) +static inline odp_time_t _odp_time_cur(void) { - if (_odp_time_glob.use_hw) { - odp_time_t time; + odp_time_t time; - time.count = _odp_cpu_global_time() - _odp_time_glob.hw_start; - return time; - } + time.u64 = rte_get_timer_cycles() - _odp_time_glob.start_cycles; - return _odp_timespec_cur(); + return time; } -static inline odp_time_t _odp_time_cur_gen_strict(void) +static inline odp_time_t _odp_time_cur_strict(void) { - if (_odp_time_glob.use_hw) { - odp_time_t time; - - time.count = _odp_cpu_global_time_strict() - _odp_time_glob.hw_start; - return time; - } + odp_time_t time; - return _odp_timespec_cur(); -} + rte_mb(); + time.u64 = rte_get_timer_cycles() - _odp_time_glob.start_cycles; -static inline odp_time_t _odp_time_cur(void) -{ - return _odp_time_glob.handler.time_cur(); -} - -static inline odp_time_t _odp_time_cur_strict(void) -{ - return _odp_time_glob.handler.time_cur_strict(); + return time; } -static inline uint64_t _odp_time_hw_to_ns(odp_time_t time) +static inline uint64_t _odp_time_to_ns(odp_time_t time) { uint64_t nsec; - uint64_t freq_hz = _odp_time_glob.hw_freq_hz; uint64_t count = time.count; uint64_t sec = 0; + const uint64_t freq_hz = _odp_time_glob.freq_hz; + const uint64_t giga_hz = 1000000000; if (count >= freq_hz) { sec = count / freq_hz; count = count - sec * freq_hz; } - nsec = (_ODP_TIME_GIGA_HZ * count) / freq_hz; + nsec = (giga_hz * count) / freq_hz; - return (sec * _ODP_TIME_GIGA_HZ) + nsec; + return (sec * giga_hz) + nsec; } -static inline uint64_t _odp_time_convert_to_ns(odp_time_t time) +static inline odp_time_t _odp_time_from_ns(uint64_t ns) { - if (_odp_time_glob.use_hw) - return _odp_time_hw_to_ns(time); + odp_time_t time; + uint64_t count; + uint64_t sec = 0; + const uint64_t freq_hz = _odp_time_glob.freq_hz; + + if (ns >= ODP_TIME_SEC_IN_NS) { + sec = ns / ODP_TIME_SEC_IN_NS; + ns = ns - sec * ODP_TIME_SEC_IN_NS; + } - return time.nsec; + count = sec * freq_hz; + count += (ns * freq_hz) / ODP_TIME_SEC_IN_NS; + + time.count = count; + + return time; } #ifndef _ODP_NO_INLINE /* Inline functions by default */ #define _ODP_INLINE static inline - #define odp_time_local __odp_time_local - #define odp_time_global __odp_time_global - #define odp_time_to_ns __odp_time_to_ns - #define odp_time_local_ns __odp_time_local_ns - #define odp_time_global_ns __odp_time_global_ns - - #define odp_time_local_strict __odp_time_local_strict - #define odp_time_global_strict __odp_time_global_strict - #define odp_time_local_strict_ns __odp_time_local_strict_ns - #define odp_time_global_strict_ns __odp_time_global_strict_ns - - #define odp_time_cmp __odp_time_cmp - #define odp_time_diff __odp_time_diff - #define odp_time_diff_ns __odp_time_diff_ns - #define odp_time_sum __odp_time_sum + #define odp_time_local_res __odp_time_local_res + #define odp_time_global_res __odp_time_global_res + #define odp_time_local __odp_time_local + #define odp_time_global __odp_time_global + #define odp_time_local_strict __odp_time_local_strict + #define odp_time_global_strict __odp_time_global_strict + #define odp_time_local_ns __odp_time_local_ns + #define odp_time_global_ns __odp_time_global_ns + #define odp_time_local_from_ns __odp_time_local_from_ns + #define odp_time_global_from_ns __odp_time_global_from_ns + #define odp_time_local_strict_ns __odp_time_local_strict_ns + #define odp_time_global_strict_ns __odp_time_global_strict_ns + #define odp_time_to_ns __odp_time_to_ns + #define odp_time_cmp __odp_time_cmp + #define odp_time_diff __odp_time_diff + #define odp_time_diff_ns __odp_time_diff_ns + #define odp_time_sum __odp_time_sum + #define odp_time_wait_ns __odp_time_wait_ns + #define odp_time_wait_until __odp_time_wait_until #else #define _ODP_INLINE #endif +_ODP_INLINE uint64_t odp_time_local_res(void) +{ + return _odp_time_glob.freq_hz; +} + +_ODP_INLINE uint64_t odp_time_global_res(void) +{ + return _odp_time_glob.freq_hz; +} + _ODP_INLINE odp_time_t odp_time_local(void) { return _odp_time_cur(); @@ -153,27 +144,37 @@ _ODP_INLINE odp_time_t odp_time_global_strict(void) _ODP_INLINE uint64_t odp_time_local_ns(void) { - return _odp_time_convert_to_ns(_odp_time_cur()); + return _odp_time_to_ns(_odp_time_cur()); } _ODP_INLINE uint64_t odp_time_global_ns(void) { - return _odp_time_convert_to_ns(_odp_time_cur()); + return _odp_time_to_ns(_odp_time_cur()); +} + +_ODP_INLINE odp_time_t odp_time_local_from_ns(uint64_t ns) +{ + return _odp_time_from_ns(ns); +} + +_ODP_INLINE odp_time_t odp_time_global_from_ns(uint64_t ns) +{ + return _odp_time_from_ns(ns); } _ODP_INLINE uint64_t odp_time_local_strict_ns(void) { - return _odp_time_convert_to_ns(_odp_time_cur_strict()); + return _odp_time_to_ns(_odp_time_cur_strict()); } _ODP_INLINE uint64_t odp_time_global_strict_ns(void) { - return _odp_time_convert_to_ns(_odp_time_cur_strict()); + return _odp_time_to_ns(_odp_time_cur_strict()); } _ODP_INLINE uint64_t odp_time_to_ns(odp_time_t time) { - return _odp_time_convert_to_ns(time); + return _odp_time_to_ns(time); } _ODP_INLINE int odp_time_cmp(odp_time_t t2, odp_time_t t1) @@ -214,6 +215,29 @@ _ODP_INLINE odp_time_t odp_time_sum(odp_time_t t1, odp_time_t t2) return time; } +static inline void _odp_time_wait_until(odp_time_t time) +{ + odp_time_t cur; + + do { + cur = _odp_time_cur(); + } while (odp_time_cmp(time, cur) > 0); +} + +_ODP_INLINE void odp_time_wait_ns(uint64_t ns) +{ + const odp_time_t cur = _odp_time_cur(); + const odp_time_t wait = _odp_time_from_ns(ns); + const odp_time_t end_time = odp_time_sum(cur, wait); + + _odp_time_wait_until(end_time); +} + +_ODP_INLINE void odp_time_wait_until(odp_time_t time) +{ + _odp_time_wait_until(time); +} + /** @endcond */ #endif diff --git a/platform/linux-dpdk/odp_time.c b/platform/linux-dpdk/odp_time.c index fbbc9358d..2fc25eb96 100644 --- a/platform/linux-dpdk/odp_time.c +++ b/platform/linux-dpdk/odp_time.c @@ -1,302 +1,43 @@ /* Copyright (c) 2013-2018, Linaro Limited - * Copyright (c) 2021-2022, Nokia + * Copyright (c) 2021-2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause */ -#include - -#include -#include -#include +#include -#include -#include #include #include -#include #include -#include #include -ODP_STATIC_ASSERT(_ODP_TIMESPEC_SIZE >= (sizeof(struct timespec)), - "_ODP_TIMESPEC_SIZE too small"); +#include #include _odp_time_global_t _odp_time_glob; -/* - * Posix timespec based functions - */ - -static inline uint64_t time_spec_diff_nsec(struct timespec *t2, - struct timespec *t1) -{ - struct timespec diff; - uint64_t nsec; - - diff.tv_sec = t2->tv_sec - t1->tv_sec; - diff.tv_nsec = t2->tv_nsec - t1->tv_nsec; - - if (diff.tv_nsec < 0) { - diff.tv_nsec += ODP_TIME_SEC_IN_NS; - diff.tv_sec -= 1; - } - - nsec = (diff.tv_sec * ODP_TIME_SEC_IN_NS) + diff.tv_nsec; - - return nsec; -} - -odp_time_t _odp_timespec_cur(void) -{ - int ret; - odp_time_t time; - struct timespec sys_time; - struct timespec *start_time; - - start_time = (struct timespec *)(uintptr_t)&_odp_time_glob.timespec; - - ret = clock_gettime(CLOCK_MONOTONIC_RAW, &sys_time); - if (odp_unlikely(ret != 0)) - _ODP_ABORT("clock_gettime failed\n"); - - time.nsec = time_spec_diff_nsec(&sys_time, start_time); - - return time; -} - #include -static inline uint64_t time_spec_res(void) -{ - int ret; - struct timespec tres; - - ret = clock_getres(CLOCK_MONOTONIC_RAW, &tres); - if (odp_unlikely(ret != 0)) - _ODP_ABORT("clock_getres failed\n"); - - return ODP_TIME_SEC_IN_NS / (uint64_t)tres.tv_nsec; -} - -static inline odp_time_t time_spec_from_ns(uint64_t ns) -{ - odp_time_t time; - - time.nsec = ns; - - return time; -} - -/* - * HW time counter based functions - */ - -static inline uint64_t time_hw_res(void) -{ - return _odp_time_glob.hw_freq_hz; -} - -static inline odp_time_t time_hw_from_ns(uint64_t ns) -{ - odp_time_t time; - uint64_t count; - uint64_t freq_hz = _odp_time_glob.hw_freq_hz; - uint64_t sec = 0; - - if (ns >= ODP_TIME_SEC_IN_NS) { - sec = ns / ODP_TIME_SEC_IN_NS; - ns = ns - sec * ODP_TIME_SEC_IN_NS; - } - - count = sec * freq_hz; - count += (ns * freq_hz) / ODP_TIME_SEC_IN_NS; - - time.count = count; - - return time; -} - -/* - * Common functions - */ - -static inline uint64_t time_res(void) -{ - if (_odp_time_glob.use_hw) - return time_hw_res(); - - return time_spec_res(); -} - -static inline odp_time_t time_from_ns(uint64_t ns) -{ - if (_odp_time_glob.use_hw) - return time_hw_from_ns(ns); - - return time_spec_from_ns(ns); -} - -static inline odp_time_t time_cur_dpdk(void) -{ - odp_time_t time; - - time.u64 = rte_get_timer_cycles() - _odp_time_glob.hw_start; - - return time; -} - -static inline odp_time_t time_cur_dpdk_strict(void) -{ - odp_time_t time; - - rte_mb(); - time.u64 = rte_get_timer_cycles() - _odp_time_glob.hw_start; - - return time; -} - -static inline uint64_t time_res_dpdk(void) -{ - return rte_get_timer_hz(); -} - -static inline void time_wait_until(odp_time_t time) -{ - odp_time_t cur; - - do { - cur = _odp_time_cur(); - } while (odp_time_cmp(time, cur) > 0); -} - -odp_time_t odp_time_local_from_ns(uint64_t ns) -{ - return time_from_ns(ns); -} - -odp_time_t odp_time_global_from_ns(uint64_t ns) -{ - return time_from_ns(ns); -} - -uint64_t odp_time_local_res(void) -{ - return _odp_time_glob.handler.time_res(); -} - -uint64_t odp_time_global_res(void) -{ - return _odp_time_glob.handler.time_res(); -} - -void odp_time_wait_ns(uint64_t ns) -{ - odp_time_t cur = _odp_time_cur(); - odp_time_t wait = time_from_ns(ns); - odp_time_t end_time = odp_time_sum(cur, wait); - - time_wait_until(end_time); -} - -void odp_time_wait_until(odp_time_t time) -{ - time_wait_until(time); -} - -static odp_bool_t is_invariant_tsc_supported(void) -{ - FILE *file; - char *line = NULL; - size_t len = 0; - odp_bool_t nonstop_tsc = false; - odp_bool_t constant_tsc = false; - odp_bool_t ret = false; - - file = fopen("/proc/cpuinfo", "rt"); - while (getline(&line, &len, file) != -1) { - if (strstr(line, "flags") != NULL) { - if (strstr(line, "constant_tsc") != NULL) - constant_tsc = true; - if (strstr(line, "nonstop_tsc") != NULL) - nonstop_tsc = true; - - if (constant_tsc && nonstop_tsc) - ret = true; - else - ret = false; - - free(line); - fclose(file); - return ret; - } - } - free(line); - fclose(file); - return false; -} - -static inline odp_bool_t is_dpdk_timer_cycles_support(void) +int _odp_time_init_global(void) { - if (is_invariant_tsc_supported() == true) - return true; + memset(&_odp_time_glob, 0, sizeof(_odp_time_global_t)); #ifdef RTE_LIBEAL_USE_HPET - if (rte_eal_hpet_init(1) == 0) - return true; + if (rte_eal_hpet_init(1) != 0) + _ODP_WARN("HPET init failed. Using TSC time.\n"); #endif - return false; -} - -int _odp_time_init_global(void) -{ - struct timespec *timespec; - int ret = 0; - _odp_time_global_t *global = &_odp_time_glob; - - memset(global, 0, sizeof(_odp_time_global_t)); - - if (is_dpdk_timer_cycles_support()) { - _odp_time_glob.handler.time_cur = time_cur_dpdk; - _odp_time_glob.handler.time_cur_strict = time_cur_dpdk_strict; - _odp_time_glob.handler.time_res = time_res_dpdk; - _odp_time_glob.hw_freq_hz = time_res_dpdk(); - _odp_time_glob.use_hw = 1; - _odp_time_glob.hw_start = rte_get_timer_cycles(); - if (_odp_time_glob.hw_start == 0) - return -1; - else - return 0; - } - _odp_time_glob.handler.time_cur = _odp_time_cur_gen; - _odp_time_glob.handler.time_cur_strict = _odp_time_cur_gen_strict; - _odp_time_glob.handler.time_res = time_res; - - if (_odp_cpu_has_global_time()) { - global->use_hw = 1; - global->hw_freq_hz = _odp_cpu_global_time_freq(); - - if (global->hw_freq_hz == 0) - return -1; - - _ODP_PRINT("HW time counter freq: %" PRIu64 " hz\n\n", global->hw_freq_hz); - - global->hw_start = _odp_cpu_global_time(); - return 0; + _odp_time_glob.freq_hz = rte_get_timer_hz(); + _odp_time_glob.start_cycles = rte_get_timer_cycles(); + if (_odp_time_glob.start_cycles == 0) { + _ODP_ERR("Initializing start cycles failed.\n"); + return -1; } - timespec = (struct timespec *)(uintptr_t)global->timespec; - timespec->tv_sec = 0; - timespec->tv_nsec = 0; - - ret = clock_gettime(CLOCK_MONOTONIC_RAW, timespec); - - return ret; + return 0; } int _odp_time_term_global(void) -- cgit v1.2.3