diff options
Diffstat (limited to 'platform/linux-generic/arch')
18 files changed, 509 insertions, 184 deletions
diff --git a/platform/linux-generic/arch/aarch64/odp/api/abi/sync_inlines.h b/platform/linux-generic/arch/aarch64/odp/api/abi/sync_inlines.h new file mode 100644 index 000000000..3d42e7dd8 --- /dev/null +++ b/platform/linux-generic/arch/aarch64/odp/api/abi/sync_inlines.h @@ -0,0 +1,31 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2023 Nokia + */ + +#ifndef ODP_ARCH_SYNC_INLINES_H_ +#define ODP_ARCH_SYNC_INLINES_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +static inline void _odp_mb_sync(void) +{ + __asm__ volatile("dsb sy" ::: "memory"); +} + +static inline void _odp_mb_sync_load(void) +{ + __asm__ volatile("dsb ld" ::: "memory"); +} + +static inline void _odp_mb_sync_store(void) +{ + __asm__ volatile("dsb st" ::: "memory"); +} + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/arch/aarch64/odp/api/abi/cpu_time.h b/platform/linux-generic/arch/aarch64/odp/api/abi/time_cpu.h index 781ee683c..aba2799c7 100644 --- a/platform/linux-generic/arch/aarch64/odp/api/abi/cpu_time.h +++ b/platform/linux-generic/arch/aarch64/odp/api/abi/time_cpu.h @@ -4,8 +4,8 @@ * SPDX-License-Identifier: BSD-3-Clause */ -#ifndef ODP_API_ABI_CPU_TIME_H_ -#define ODP_API_ABI_CPU_TIME_H_ +#ifndef ODP_API_ABI_TIME_CPU_H_ +#define ODP_API_ABI_TIME_CPU_H_ #ifdef __cplusplus extern "C" { @@ -13,7 +13,7 @@ extern "C" { #include <stdint.h> -static inline uint64_t _odp_cpu_global_time(void) +static inline uint64_t _odp_time_cpu_global(void) { uint64_t cntvct; @@ -22,7 +22,7 @@ static inline uint64_t _odp_cpu_global_time(void) return cntvct; } -static inline uint64_t _odp_cpu_global_time_strict(void) +static inline uint64_t _odp_time_cpu_global_strict(void) { uint64_t cntvct; @@ -32,7 +32,7 @@ static inline uint64_t _odp_cpu_global_time_strict(void) return cntvct; } -static inline uint64_t _odp_cpu_global_time_freq(void) +static inline uint64_t _odp_time_cpu_global_freq(void) { uint64_t cntfrq; @@ -41,7 +41,10 @@ static inline uint64_t _odp_cpu_global_time_freq(void) return cntfrq; } -int _odp_cpu_has_global_time(void); +static inline int _odp_time_cpu_global_freq_is_const(void) +{ + return 1; +} #ifdef __cplusplus } diff --git a/platform/linux-generic/arch/aarch64/odp/api/abi/time_inlines.h b/platform/linux-generic/arch/aarch64/odp/api/abi/time_inlines.h new file mode 100644 index 000000000..331d1996f --- /dev/null +++ b/platform/linux-generic/arch/aarch64/odp/api/abi/time_inlines.h @@ -0,0 +1,7 @@ +/* Copyright (c) 2023, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <odp/api/abi/time_cpu_inlines.h> diff --git a/platform/linux-generic/arch/aarch64/odp_global_time.c b/platform/linux-generic/arch/aarch64/odp_global_time.c deleted file mode 100644 index 53561b00c..000000000 --- a/platform/linux-generic/arch/aarch64/odp_global_time.c +++ /dev/null @@ -1,25 +0,0 @@ -/* Copyright (c) 2015-2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -#include <odp/api/abi/cpu_time.h> - -int _odp_cpu_has_global_time(void) -{ - uint64_t hz = _odp_cpu_global_time_freq(); - - /* - * The system counter portion of the architected timer must - * provide a uniform view of system time to all processing - * elements in the system. This should hold true even for - * heterogeneous SoCs. - * - * Determine whether the system has 'global time' by checking - * whether a read of the architected timer frequency sys reg - * returns a sane value. Sane is considered to be within - * 1MHz and 6GHz (1us and .1667ns period). - */ - return hz >= 1000000 && hz <= 6000000000; -} diff --git a/platform/linux-generic/arch/aarch64/odp_sysinfo_parse.c b/platform/linux-generic/arch/aarch64/odp_sysinfo_parse.c index 25e3e4fc6..352bee9e9 100644 --- a/platform/linux-generic/arch/aarch64/odp_sysinfo_parse.c +++ b/platform/linux-generic/arch/aarch64/odp_sysinfo_parse.c @@ -336,9 +336,8 @@ int _odp_cpuinfo_parser(FILE *file, system_info_t *sysinfo) if (sysinfo->cpu_hz_max[id] == 0) { uint64_t hz = sysinfo->default_cpu_hz_max; - _ODP_PRINT("WARN: cpu[%i] uses default max " - "frequency of %" PRIu64 " Hz from " - "config file\n", id, hz); + _ODP_WARN("CPU[%i] uses default max frequency of %" PRIu64 " " + "Hz from config file\n", id, hz); sysinfo->cpu_hz_max[id] = hz; } diff --git a/platform/linux-generic/arch/common/odp/api/abi/time_cpu_inlines.h b/platform/linux-generic/arch/common/odp/api/abi/time_cpu_inlines.h new file mode 100644 index 000000000..c154c5f1a --- /dev/null +++ b/platform/linux-generic/arch/common/odp/api/abi/time_cpu_inlines.h @@ -0,0 +1,93 @@ +/* Copyright (c) 2013-2018, Linaro Limited + * Copyright (c) 2020-2023, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_ARCH_TIME_CPU_INLINES_H_ +#define ODP_ARCH_TIME_CPU_INLINES_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <odp/api/time_types.h> + +#include <odp/api/abi/time_cpu.h> + +#include <stdint.h> + +#define _ODP_TIME_GIGA_HZ 1000000000ULL + +typedef struct _odp_time_global_t { + uint64_t start_time; + uint64_t freq_hz; + +} _odp_time_global_t; + +extern _odp_time_global_t _odp_time_glob; + +static inline odp_time_t _odp_time_cur(void) +{ + odp_time_t time; + + time.count = _odp_time_cpu_global() - _odp_time_glob.start_time; + return time; +} + +static inline odp_time_t _odp_time_cur_strict(void) +{ + odp_time_t time; + + time.count = _odp_time_cpu_global_strict() - _odp_time_glob.start_time; + return time; +} + +static inline uint64_t _odp_time_to_ns(odp_time_t time) +{ + uint64_t nsec; + uint64_t freq_hz = _odp_time_glob.freq_hz; + uint64_t count = time.count; + uint64_t sec = 0; + + if (count >= freq_hz) { + sec = count / freq_hz; + count = count - sec * freq_hz; + } + + nsec = (_ODP_TIME_GIGA_HZ * count) / freq_hz; + + return (sec * _ODP_TIME_GIGA_HZ) + nsec; +} + +static inline odp_time_t _odp_time_from_ns(uint64_t ns) +{ + odp_time_t time; + uint64_t count; + uint64_t freq_hz = _odp_time_glob.freq_hz; + uint64_t sec = 0; + + if (ns >= ODP_TIME_SEC_IN_NS) { + sec = ns / ODP_TIME_SEC_IN_NS; + ns = ns - sec * ODP_TIME_SEC_IN_NS; + } + + count = sec * freq_hz; + count += (ns * freq_hz) / ODP_TIME_SEC_IN_NS; + + time.count = count; + + return time; +} + +static inline uint64_t _odp_time_res(void) +{ + return _odp_time_glob.freq_hz; +} + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/arch/common/odp_time_cpu.c b/platform/linux-generic/arch/common/odp_time_cpu.c new file mode 100644 index 000000000..bbfe82e21 --- /dev/null +++ b/platform/linux-generic/arch/common/odp_time_cpu.c @@ -0,0 +1,48 @@ +/* Copyright (c) 2013-2018, Linaro Limited + * Copyright (c) 2020-2023, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <odp/api/time_types.h> + +#include <odp/api/abi/time_cpu.h> +#include <odp/api/abi/time_cpu_inlines.h> + +#include <odp_debug_internal.h> +#include <odp_init_internal.h> + +#include <inttypes.h> +#include <stdint.h> +#include <string.h> + +#include <odp/visibility_begin.h> + +_odp_time_global_t _odp_time_glob; + +#include <odp/visibility_end.h> + +int _odp_time_init_global(void) +{ + _odp_time_global_t *global = &_odp_time_glob; + + memset(global, 0, sizeof(_odp_time_global_t)); + + if (!_odp_time_cpu_global_freq_is_const()) + return -1; + + global->freq_hz = _odp_time_cpu_global_freq(); + if (global->freq_hz == 0) + return -1; + + _ODP_PRINT("HW time counter freq: %" PRIu64 " hz\n\n", global->freq_hz); + + global->start_time = _odp_time_cpu_global(); + return 0; +} + +int _odp_time_term_global(void) +{ + return 0; +} diff --git a/platform/linux-generic/arch/default/odp/api/abi/cpu_time.h b/platform/linux-generic/arch/default/odp/api/abi/cpu_time.h deleted file mode 100644 index 24e1c7d33..000000000 --- a/platform/linux-generic/arch/default/odp/api/abi/cpu_time.h +++ /dev/null @@ -1,25 +0,0 @@ -/* Copyright (c) 2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -#ifndef ODP_ARCH_CPU_TIME_H_ -#define ODP_ARCH_CPU_TIME_H_ - -#ifdef __cplusplus -extern "C" { -#endif - -#include <stdint.h> - -int _odp_cpu_has_global_time(void); -uint64_t _odp_cpu_global_time(void); -uint64_t _odp_cpu_global_time_strict(void); -uint64_t _odp_cpu_global_time_freq(void); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/platform/linux-generic/arch/default/odp/api/abi/sync_inlines.h b/platform/linux-generic/arch/default/odp/api/abi/sync_inlines.h new file mode 100644 index 000000000..bfbb3039f --- /dev/null +++ b/platform/linux-generic/arch/default/odp/api/abi/sync_inlines.h @@ -0,0 +1,31 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2023 Nokia + */ + +#ifndef ODP_ARCH_SYNC_INLINES_H_ +#define ODP_ARCH_SYNC_INLINES_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +static inline void _odp_mb_sync(void) +{ + __sync_synchronize(); +} + +static inline void _odp_mb_sync_load(void) +{ + __sync_synchronize(); +} + +static inline void _odp_mb_sync_store(void) +{ + __sync_synchronize(); +} + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/arch/default/odp/api/abi/time_inlines.h b/platform/linux-generic/arch/default/odp/api/abi/time_inlines.h new file mode 100644 index 000000000..b38e52dac --- /dev/null +++ b/platform/linux-generic/arch/default/odp/api/abi/time_inlines.h @@ -0,0 +1,45 @@ +/* Copyright (c) 2013-2018, Linaro Limited + * Copyright (c) 2020-2023, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_ARCH_TIME_INLINES_H_ +#define ODP_ARCH_TIME_INLINES_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <odp/api/time_types.h> + +#include <stdint.h> + +odp_time_t _odp_time_cur(void); +uint64_t _odp_time_res(void); + +static inline odp_time_t _odp_time_cur_strict(void) +{ + return _odp_time_cur(); +} + +static inline uint64_t _odp_time_to_ns(odp_time_t time) +{ + return time.nsec; +} + +static inline odp_time_t _odp_time_from_ns(uint64_t ns) +{ + odp_time_t time; + + time.nsec = ns; + + return time; +} + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/arch/default/odp_global_time.c b/platform/linux-generic/arch/default/odp_global_time.c deleted file mode 100644 index ee835413f..000000000 --- a/platform/linux-generic/arch/default/odp_global_time.c +++ /dev/null @@ -1,31 +0,0 @@ -/* Copyright (c) 2015-2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -#include <odp/api/abi/cpu_time.h> - -#include <odp/visibility_begin.h> - -uint64_t _odp_cpu_global_time(void) -{ - return 0; -} - -uint64_t _odp_cpu_global_time_strict(void) -{ - return 0; -} - -#include <odp/visibility_end.h> - -int _odp_cpu_has_global_time(void) -{ - return 0; -} - -uint64_t _odp_cpu_global_time_freq(void) -{ - return 0; -} diff --git a/platform/linux-generic/arch/default/odp_time.c b/platform/linux-generic/arch/default/odp_time.c new file mode 100644 index 000000000..919a3ba68 --- /dev/null +++ b/platform/linux-generic/arch/default/odp_time.c @@ -0,0 +1,102 @@ +/* Copyright (c) 2013-2018, Linaro Limited + * Copyright (c) 2020-2023, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <odp_posix_extensions.h> + +#include <odp/api/align.h> +#include <odp/api/hints.h> +#include <odp/api/time_types.h> + +#include <odp/api/abi/time_inlines.h> + +#include <odp_debug_internal.h> +#include <odp_init_internal.h> + +#include <stdint.h> +#include <string.h> +#include <time.h> + +typedef struct _odp_time_global_t { + struct timespec start_time; + +} _odp_time_global_t; + +_odp_time_global_t _odp_time_glob; + +static inline uint64_t time_diff_nsec(struct timespec *t2, struct timespec *t1) +{ + struct timespec diff; + uint64_t nsec; + + diff.tv_sec = t2->tv_sec - t1->tv_sec; + diff.tv_nsec = t2->tv_nsec - t1->tv_nsec; + + if (diff.tv_nsec < 0) { + diff.tv_nsec += ODP_TIME_SEC_IN_NS; + diff.tv_sec -= 1; + } + + nsec = (diff.tv_sec * ODP_TIME_SEC_IN_NS) + diff.tv_nsec; + + return nsec; +} + +#include <odp/visibility_begin.h> + +odp_time_t _odp_time_cur(void) +{ + int ret; + odp_time_t time; + struct timespec sys_time; + struct timespec *start_time = &_odp_time_glob.start_time; + + ret = clock_gettime(CLOCK_MONOTONIC_RAW, &sys_time); + if (odp_unlikely(ret != 0)) + _ODP_ABORT("clock_gettime() failed\n"); + + time.nsec = time_diff_nsec(&sys_time, start_time); + + return time; +} + +uint64_t _odp_time_res(void) +{ + int ret; + struct timespec tres; + + ret = clock_getres(CLOCK_MONOTONIC_RAW, &tres); + if (odp_unlikely(ret != 0)) + _ODP_ABORT("clock_getres() failed\n"); + + return ODP_TIME_SEC_IN_NS / (uint64_t)tres.tv_nsec; +} + +#include <odp/visibility_end.h> + +int _odp_time_init_global(void) +{ + struct timespec *start_time; + int ret = 0; + _odp_time_global_t *global = &_odp_time_glob; + + memset(global, 0, sizeof(_odp_time_global_t)); + + start_time = &global->start_time; + start_time->tv_sec = 0; + start_time->tv_nsec = 0; + + ret = clock_gettime(CLOCK_MONOTONIC_RAW, start_time); + if (ret) + _ODP_ERR("clock_gettime() failed: %d\n", ret); + + return ret; +} + +int _odp_time_term_global(void) +{ + return 0; +} diff --git a/platform/linux-generic/arch/x86/cpu_flags.c b/platform/linux-generic/arch/x86/cpu_flags.c index 036645dbc..9211df002 100644 --- a/platform/linux-generic/arch/x86/cpu_flags.c +++ b/platform/linux-generic/arch/x86/cpu_flags.c @@ -1,45 +1,23 @@ /* Copyright (c) 2017-2018, Linaro Limited + * Copyright (c) 2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause */ -/*- - * BSD LICENSE - * - * Copyright(c) 2010-2015 Intel Corporation. All rights reserved. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2010-2015 Intel Corporation */ #include "cpu_flags.h" + +#include <odp/api/abi/time_cpu.h> + #include <odp_debug_internal.h> -#include <odp/api/abi/cpu_time.h> +#include <odp_global_data.h> + +#include <cpuid.h> +#include <errno.h> #include <stdio.h> #include <stdint.h> @@ -74,6 +52,7 @@ enum rte_cpu_flag_t { RTE_CPUFLAG_AVX, /**< AVX */ RTE_CPUFLAG_F16C, /**< F16C */ RTE_CPUFLAG_RDRAND, /**< RDRAND */ + RTE_CPUFLAG_HYPERVISOR, /**< Running in a VM */ /* (EAX 01h) EDX features */ RTE_CPUFLAG_FPU, /**< FPU */ @@ -130,6 +109,7 @@ enum rte_cpu_flag_t { RTE_CPUFLAG_INVPCID, /**< INVPCID */ RTE_CPUFLAG_RTM, /**< Transactional memory */ RTE_CPUFLAG_AVX512F, /**< AVX512F */ + RTE_CPUFLAG_RDSEED, /**< RDSEED instruction */ /* (EAX 80000001h) ECX features */ RTE_CPUFLAG_LAHF_SAHF, /**< LAHF_SAHF */ @@ -145,8 +125,29 @@ enum rte_cpu_flag_t { /* (EAX 80000007h) EDX features */ RTE_CPUFLAG_INVTSC, /**< INVTSC */ + RTE_CPUFLAG_AVX512DQ, /**< AVX512 Doubleword and Quadword */ + RTE_CPUFLAG_AVX512IFMA, /**< AVX512 Integer Fused Multiply-Add */ + RTE_CPUFLAG_AVX512CD, /**< AVX512 Conflict Detection*/ + RTE_CPUFLAG_AVX512BW, /**< AVX512 Byte and Word */ + RTE_CPUFLAG_AVX512VL, /**< AVX512 Vector Length */ + RTE_CPUFLAG_AVX512VBMI, /**< AVX512 Vector Bit Manipulation */ + RTE_CPUFLAG_AVX512VBMI2, /**< AVX512 Vector Bit Manipulation 2 */ + RTE_CPUFLAG_GFNI, /**< Galois Field New Instructions */ + RTE_CPUFLAG_VAES, /**< Vector AES */ + RTE_CPUFLAG_VPCLMULQDQ, /**< Vector Carry-less Multiply */ + RTE_CPUFLAG_AVX512VNNI, + /**< AVX512 Vector Neural Network Instructions */ + RTE_CPUFLAG_AVX512BITALG, /**< AVX512 Bit Algorithms */ + RTE_CPUFLAG_AVX512VPOPCNTDQ, /**< AVX512 Vector Popcount */ + RTE_CPUFLAG_CLDEMOTE, /**< Cache Line Demote */ + RTE_CPUFLAG_MOVDIRI, /**< Direct Store Instructions */ + RTE_CPUFLAG_MOVDIR64B, /**< Direct Store Instructions 64B */ + RTE_CPUFLAG_AVX512VP2INTERSECT, /**< AVX512 Two Register Intersection */ + + RTE_CPUFLAG_WAITPKG, /**< UMONITOR/UMWAIT/TPAUSE */ + /* The last item */ - RTE_CPUFLAG_NUMFLAGS, /**< This should always be the last! */ + RTE_CPUFLAG_NUMFLAGS, /**< This should always be the last! */ }; enum cpu_register_t { @@ -203,6 +204,7 @@ static const struct feature_entry cpu_feature_table[] = { FEAT_DEF(AVX, 0x00000001, 0, RTE_REG_ECX, 28) FEAT_DEF(F16C, 0x00000001, 0, RTE_REG_ECX, 29) FEAT_DEF(RDRAND, 0x00000001, 0, RTE_REG_ECX, 30) + FEAT_DEF(HYPERVISOR, 0x00000001, 0, RTE_REG_ECX, 31) FEAT_DEF(FPU, 0x00000001, 0, RTE_REG_EDX, 0) FEAT_DEF(VME, 0x00000001, 0, RTE_REG_EDX, 1) @@ -246,15 +248,36 @@ static const struct feature_entry cpu_feature_table[] = { FEAT_DEF(ENERGY_EFF, 0x00000006, 0, RTE_REG_ECX, 3) FEAT_DEF(FSGSBASE, 0x00000007, 0, RTE_REG_EBX, 0) - FEAT_DEF(BMI1, 0x00000007, 0, RTE_REG_EBX, 2) + FEAT_DEF(BMI1, 0x00000007, 0, RTE_REG_EBX, 3) FEAT_DEF(HLE, 0x00000007, 0, RTE_REG_EBX, 4) FEAT_DEF(AVX2, 0x00000007, 0, RTE_REG_EBX, 5) - FEAT_DEF(SMEP, 0x00000007, 0, RTE_REG_EBX, 6) - FEAT_DEF(BMI2, 0x00000007, 0, RTE_REG_EBX, 7) - FEAT_DEF(ERMS, 0x00000007, 0, RTE_REG_EBX, 8) + FEAT_DEF(SMEP, 0x00000007, 0, RTE_REG_EBX, 7) + FEAT_DEF(BMI2, 0x00000007, 0, RTE_REG_EBX, 8) + FEAT_DEF(ERMS, 0x00000007, 0, RTE_REG_EBX, 9) FEAT_DEF(INVPCID, 0x00000007, 0, RTE_REG_EBX, 10) FEAT_DEF(RTM, 0x00000007, 0, RTE_REG_EBX, 11) FEAT_DEF(AVX512F, 0x00000007, 0, RTE_REG_EBX, 16) + FEAT_DEF(AVX512DQ, 0x00000007, 0, RTE_REG_EBX, 17) + FEAT_DEF(RDSEED, 0x00000007, 0, RTE_REG_EBX, 18) + FEAT_DEF(AVX512IFMA, 0x00000007, 0, RTE_REG_EBX, 21) + FEAT_DEF(AVX512CD, 0x00000007, 0, RTE_REG_EBX, 28) + FEAT_DEF(AVX512BW, 0x00000007, 0, RTE_REG_EBX, 30) + FEAT_DEF(AVX512VL, 0x00000007, 0, RTE_REG_EBX, 31) + + FEAT_DEF(AVX512VBMI, 0x00000007, 0, RTE_REG_ECX, 1) + FEAT_DEF(WAITPKG, 0x00000007, 0, RTE_REG_ECX, 5) + FEAT_DEF(AVX512VBMI2, 0x00000007, 0, RTE_REG_ECX, 6) + FEAT_DEF(GFNI, 0x00000007, 0, RTE_REG_ECX, 8) + FEAT_DEF(VAES, 0x00000007, 0, RTE_REG_ECX, 9) + FEAT_DEF(VPCLMULQDQ, 0x00000007, 0, RTE_REG_ECX, 10) + FEAT_DEF(AVX512VNNI, 0x00000007, 0, RTE_REG_ECX, 11) + FEAT_DEF(AVX512BITALG, 0x00000007, 0, RTE_REG_ECX, 12) + FEAT_DEF(AVX512VPOPCNTDQ, 0x00000007, 0, RTE_REG_ECX, 14) + FEAT_DEF(CLDEMOTE, 0x00000007, 0, RTE_REG_ECX, 25) + FEAT_DEF(MOVDIRI, 0x00000007, 0, RTE_REG_ECX, 27) + FEAT_DEF(MOVDIR64B, 0x00000007, 0, RTE_REG_ECX, 28) + + FEAT_DEF(AVX512VP2INTERSECT, 0x00000007, 0, RTE_REG_EDX, 8) FEAT_DEF(LAHF_SAHF, 0x80000001, 0, RTE_REG_ECX, 0) FEAT_DEF(LZCNT, 0x80000001, 0, RTE_REG_ECX, 4) @@ -268,55 +291,30 @@ static const struct feature_entry cpu_feature_table[] = { FEAT_DEF(INVTSC, 0x80000007, 0, RTE_REG_EDX, 8) }; -/* - * Execute CPUID instruction and get contents of a specific register - * - * This function, when compiled with GCC, will generate architecture-neutral - * code, as per GCC manual. - */ -static void cpu_get_features(uint32_t leaf, uint32_t subleaf, - cpuid_registers_t out) -{ -#if defined(__i386__) && defined(__PIC__) - /* %ebx is a forbidden register if we compile with -fPIC or -fPIE */ - __asm__ __volatile__("movl %%ebx,%0 ; cpuid ; xchgl %%ebx,%0" - : "=r" (out[RTE_REG_EBX]), - "=a" (out[RTE_REG_EAX]), - "=c" (out[RTE_REG_ECX]), - "=d" (out[RTE_REG_EDX]) - : "a" (leaf), "c" (subleaf)); -#else - __asm__ __volatile__("cpuid" - : "=a" (out[RTE_REG_EAX]), - "=b" (out[RTE_REG_EBX]), - "=c" (out[RTE_REG_ECX]), - "=d" (out[RTE_REG_EDX]) - : "a" (leaf), "c" (subleaf)); -#endif -} - static int cpu_get_flag_enabled(enum rte_cpu_flag_t feature) { const struct feature_entry *feat; cpuid_registers_t regs; + unsigned int maxleaf; if (feature >= RTE_CPUFLAG_NUMFLAGS) /* Flag does not match anything in the feature tables */ - return -1; + return -ENOENT; feat = &cpu_feature_table[feature]; if (!feat->leaf) /* This entry in the table wasn't filled out! */ - return -1; + return -EFAULT; + + maxleaf = __get_cpuid_max(feat->leaf & 0x80000000, NULL); - cpu_get_features(feat->leaf & 0xffff0000, 0, regs); - if (((regs[RTE_REG_EAX] ^ feat->leaf) & 0xffff0000) || - regs[RTE_REG_EAX] < feat->leaf) + if (maxleaf < feat->leaf) return 0; - /* get the cpuid leaf containing the desired feature */ - cpu_get_features(feat->leaf, feat->subleaf, regs); + __cpuid_count(feat->leaf, feat->subleaf, + regs[RTE_REG_EAX], regs[RTE_REG_EBX], + regs[RTE_REG_ECX], regs[RTE_REG_EDX]); /* check if the feature is enabled */ return (regs[feat->reg] >> feat->bit) & 1; @@ -359,12 +357,16 @@ void _odp_cpu_flags_print_all(void) _ODP_PRINT("%s", str); } -int _odp_cpu_has_global_time(void) +int _odp_time_cpu_global_freq_is_const(void) { - if (cpu_get_flag_enabled(RTE_CPUFLAG_INVTSC) > 0) + if (odp_global_ro.system_info.cpu_constant_tsc || + cpu_get_flag_enabled(RTE_CPUFLAG_INVTSC) > 0) return 1; - return 0; + _ODP_ERR("WARN: assuming constant TSC based on CPU arch, but could not confirm from CPU " + "flags\n"); + + return 1; } int _odp_cpu_flags_has_rdtsc(void) diff --git a/platform/linux-generic/arch/x86/odp/api/abi/sync_inlines.h b/platform/linux-generic/arch/x86/odp/api/abi/sync_inlines.h new file mode 100644 index 000000000..bebe6b571 --- /dev/null +++ b/platform/linux-generic/arch/x86/odp/api/abi/sync_inlines.h @@ -0,0 +1,31 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2023 Nokia + */ + +#ifndef ODP_ARCH_SYNC_INLINES_H_ +#define ODP_ARCH_SYNC_INLINES_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +static inline void _odp_mb_sync(void) +{ + __asm__ volatile("mfence" ::: "memory"); +} + +static inline void _odp_mb_sync_load(void) +{ + __asm__ volatile("lfence" ::: "memory"); +} + +static inline void _odp_mb_sync_store(void) +{ + __asm__ volatile("sfence" ::: "memory"); +} + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/arch/x86/odp/api/abi/cpu_time.h b/platform/linux-generic/arch/x86/odp/api/abi/time_cpu.h index c74c4d606..baf79ad3f 100644 --- a/platform/linux-generic/arch/x86/odp/api/abi/cpu_time.h +++ b/platform/linux-generic/arch/x86/odp/api/abi/time_cpu.h @@ -4,8 +4,8 @@ * SPDX-License-Identifier: BSD-3-Clause */ -#ifndef ODP_ARCH_CPU_TIME_H_ -#define ODP_ARCH_CPU_TIME_H_ +#ifndef ODP_ARCH_TIME_CPU_H_ +#define ODP_ARCH_TIME_CPU_H_ #ifdef __cplusplus extern "C" { @@ -14,19 +14,19 @@ extern "C" { #include <stdint.h> #include <odp/api/abi/cpu_rdtsc.h> -static inline uint64_t _odp_cpu_global_time(void) +static inline uint64_t _odp_time_cpu_global(void) { return _odp_cpu_rdtsc(); } -static inline uint64_t _odp_cpu_global_time_strict(void) +static inline uint64_t _odp_time_cpu_global_strict(void) { __atomic_thread_fence(__ATOMIC_SEQ_CST); return _odp_cpu_rdtsc(); } -int _odp_cpu_has_global_time(void); -uint64_t _odp_cpu_global_time_freq(void); +int _odp_time_cpu_global_freq_is_const(void); +uint64_t _odp_time_cpu_global_freq(void); #ifdef __cplusplus } diff --git a/platform/linux-generic/arch/x86/odp/api/abi/time_inlines.h b/platform/linux-generic/arch/x86/odp/api/abi/time_inlines.h new file mode 100644 index 000000000..331d1996f --- /dev/null +++ b/platform/linux-generic/arch/x86/odp/api/abi/time_inlines.h @@ -0,0 +1,7 @@ +/* Copyright (c) 2023, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <odp/api/abi/time_cpu_inlines.h> diff --git a/platform/linux-generic/arch/x86/odp_sysinfo_parse.c b/platform/linux-generic/arch/x86/odp_sysinfo_parse.c index c74c52045..3cbdb2037 100644 --- a/platform/linux-generic/arch/x86/odp_sysinfo_parse.c +++ b/platform/linux-generic/arch/x86/odp_sysinfo_parse.c @@ -1,4 +1,5 @@ /* Copyright (c) 2016-2018, Linaro Limited + * Copyright (c) 2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -30,6 +31,11 @@ int _odp_cpuinfo_parser(FILE *file, system_info_t *sysinfo) strcpy(sysinfo->cpu_arch_str, "x86"); while (fgets(str, sizeof(str), file) != NULL && id < CONFIG_NUM_CPU_IDS) { + if (strstr(str, "flags") && strstr(str, "constant_tsc")) { + sysinfo->cpu_constant_tsc = 1; + continue; + } + pos = strstr(str, "model name"); if (pos) { freq_set = false; diff --git a/platform/linux-generic/arch/x86/odp_global_time.c b/platform/linux-generic/arch/x86/odp_time_cpu.c index 00e3a1ac1..aa00ac04e 100644 --- a/platform/linux-generic/arch/x86/odp_global_time.c +++ b/platform/linux-generic/arch/x86/odp_time_cpu.c @@ -6,17 +6,18 @@ #include <odp_posix_extensions.h> -#include <time.h> - #include <odp/api/hints.h> +#include <odp/api/time_types.h> + +#include <odp/api/abi/time_cpu.h> + #include <odp_debug_internal.h> -#include <odp/api/abi/cpu_time.h> -#define SEC_IN_NS 1000000000ULL +#include <time.h> /* Measure TSC frequency. Frequency information registers are defined for x86, * but those are often not enumerated. */ -uint64_t _odp_cpu_global_time_freq(void) +uint64_t _odp_time_cpu_global_freq(void) { struct timespec sleep, ts1, ts2; uint64_t t1, t2, ts_nsec, cycles, hz; @@ -29,35 +30,35 @@ uint64_t _odp_cpu_global_time_freq(void) sleep.tv_sec = 0; if (warm_up) - sleep.tv_nsec = SEC_IN_NS / 1000; + sleep.tv_nsec = ODP_TIME_SEC_IN_NS / 1000; else - sleep.tv_nsec = SEC_IN_NS / 4; + sleep.tv_nsec = ODP_TIME_SEC_IN_NS / 4; if (clock_gettime(CLOCK_MONOTONIC_RAW, &ts1)) { - _ODP_DBG("clock_gettime failed\n"); + _ODP_ERR("clock_gettime() failed\n"); return 0; } - t1 = _odp_cpu_global_time(); + t1 = _odp_time_cpu_global(); if (nanosleep(&sleep, NULL) < 0) { - _ODP_DBG("nanosleep failed\n"); + _ODP_ERR("nanosleep() failed\n"); return 0; } if (clock_gettime(CLOCK_MONOTONIC_RAW, &ts2)) { - _ODP_DBG("clock_gettime failed\n"); + _ODP_ERR("clock_gettime() failed\n"); return 0; } - t2 = _odp_cpu_global_time(); + t2 = _odp_time_cpu_global(); - ts_nsec = (ts2.tv_sec - ts1.tv_sec) * SEC_IN_NS; + ts_nsec = (ts2.tv_sec - ts1.tv_sec) * ODP_TIME_SEC_IN_NS; ts_nsec += ts2.tv_nsec - ts1.tv_nsec; cycles = t2 - t1; - hz = (cycles * SEC_IN_NS) / ts_nsec; + hz = (cycles * ODP_TIME_SEC_IN_NS) / ts_nsec; if (warm_up) warm_up = 0; |