diff options
Diffstat (limited to 'platform/linux-generic')
60 files changed, 1873 insertions, 382 deletions
diff --git a/platform/linux-generic/Makefile.am b/platform/linux-generic/Makefile.am index e763c0abc..8763606ad 100644 --- a/platform/linux-generic/Makefile.am +++ b/platform/linux-generic/Makefile.am @@ -143,6 +143,7 @@ noinst_HEADERS = \ include/odp_ring_spsc_internal.h \ include/odp_ring_st_internal.h \ include/odp_ring_u32_internal.h \ + include/odp_ring_u64_internal.h \ include/odp_schedule_if.h \ include/odp_schedule_scalable_config.h \ include/odp_schedule_scalable.h \ @@ -295,6 +296,7 @@ endif if ARCH_IS_AARCH64 __LIB__libodp_linux_la_SOURCES += arch/aarch64/odp_atomic.c \ arch/default/odp_cpu_cycles.c \ + arch/aarch64/cpu_flags.c \ arch/aarch64/odp_global_time.c \ arch/default/odp_hash_crc32.c \ arch/aarch64/odp_sysinfo_parse.c @@ -310,6 +312,7 @@ odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ endif noinst_HEADERS += arch/aarch64/odp_atomic.h \ arch/aarch64/odp_cpu.h \ + arch/aarch64/cpu_flags.h \ arch/aarch64/odp_cpu_idling.h \ arch/aarch64/odp_llsc.h endif diff --git a/platform/linux-generic/arch/aarch64/cpu_flags.c b/platform/linux-generic/arch/aarch64/cpu_flags.c new file mode 100644 index 000000000..d70e26271 --- /dev/null +++ b/platform/linux-generic/arch/aarch64/cpu_flags.c @@ -0,0 +1,976 @@ +/* Copyright (c) 2018, Linaro Limited + * Copyright (c) 2020-2021, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <string.h> +#include <stdlib.h> +#include <sys/auxv.h> +#include <asm/hwcap.h> + +#include <odp/api/hints.h> +#include <odp_debug_internal.h> +#include "cpu_flags.h" + +typedef struct { + const char *feat_flag; + const unsigned int hwcap_field; + bool valid; +} hwcap_feat_flag_t; + +static hwcap_feat_flag_t hwcap_flags[] = { + { + /* Floating-point Extensions */ + .feat_flag = "FEAT_FP", +#ifdef HWCAP_FP + .hwcap_field = HWCAP_FP, + .valid = 1, +#endif + }, + + { + /* Floating-point Extensions */ + .feat_flag = "FEAT_FP", +#ifdef HWCAP_ASIMD + .hwcap_field = HWCAP_ASIMD, + .valid = 1, +#endif + }, + + { + /* Generic Timer is configured to generate events at approx. 10KHz */ + .feat_flag = "HWCAP_EVTSTRM", +#ifdef HWCAP_EVTSTRM + .hwcap_field = HWCAP_EVTSTRM, + .valid = 1, +#endif + }, + + { + /* Advanced SIMD AES Instructions */ + .feat_flag = "FEAT_AES", +#ifdef HWCAP_AES + .hwcap_field = HWCAP_AES, + .valid = 1, +#endif + }, + + { + /* Advanced SIMD PMULL Instructions */ + .feat_flag = "FEAT_PMULL", +#ifdef HWCAP_PMULL + .hwcap_field = HWCAP_PMULL, + .valid = 1, +#endif + }, + + { + /* Advanced SIMD SHA1 Instructions */ + .feat_flag = "FEAT_SHA1", +#ifdef HWCAP_SHA1 + .hwcap_field = HWCAP_SHA1, + .valid = 1, +#endif + }, + + { + /* Advanced SIMD SHA256 Instructions */ + .feat_flag = "FEAT_SHA256", +#ifdef HWCAP_SHA2 + .hwcap_field = HWCAP_SHA2, + .valid = 1, +#endif + }, + + { + /* CRC32 Instructions */ + .feat_flag = "FEAT_CRC32", +#ifdef HWCAP_CRC32 + .hwcap_field = HWCAP_CRC32, + .valid = 1, +#endif + }, + + { + /* Large System Extensions */ + .feat_flag = "FEAT_LSE", +#ifdef HWCAP_ATOMICS + .hwcap_field = HWCAP_ATOMICS, + .valid = 1, +#endif + }, + + { + /* Half-precision Floating-point Data Processing Instructions */ + .feat_flag = "FEAT_FP16", +#ifdef HWCAP_FPHP + .hwcap_field = HWCAP_FPHP, + .valid = 1, +#endif + }, + + { + /* Half-precision Floating-point Data Processing Instructions */ + .feat_flag = "FEAT_FP16", +#ifdef HWCAP_ASIMDHP + .hwcap_field = HWCAP_ASIMDHP, + .valid = 1, +#endif + }, + + { + /* Availability of EL0 Access to certain ID Registers */ + .feat_flag = "HWCAP_CPUID", +#ifdef HWCAP_CPUID + .hwcap_field = HWCAP_CPUID, + .valid = 1, +#endif + }, + + { + /* Rounding Double Multiply Accumulate Extensions */ + .feat_flag = "FEAT_RDM", +#ifdef HWCAP_ASIMDRDM + .hwcap_field = HWCAP_ASIMDRDM, + .valid = 1, +#endif + }, + + { + /* JavaScript FJCVTS Conversion Instructions */ + .feat_flag = "FEAT_JSCVT", +#ifdef HWCAP_JSCVT + .hwcap_field = HWCAP_JSCVT, + .valid = 1, +#endif + }, + + { + /* Floating-point FCMLA and FCADD Instructions */ + .feat_flag = "FEAT_FCMA", +#ifdef HWCAP_FCMA + .hwcap_field = HWCAP_FCMA, + .valid = 1, +#endif + }, + + { + /* Load-acquire RCpc Instructions */ + .feat_flag = "FEAT_LRCPC", +#ifdef HWCAP_LRCPC + .hwcap_field = HWCAP_LRCPC, + .valid = 1, +#endif + }, + + { + /* DC CVAP Instructions */ + .feat_flag = "FEAT_DPB", +#ifdef HWCAP_DCPOP + .hwcap_field = HWCAP_DCPOP, + .valid = 1, +#endif + }, + + { + /* Advanced SIMD EOR3, RAX1, XAR, and BCAX Instructions */ + .feat_flag = "FEAT_SHA3", +#ifdef HWCAP_SHA3 + .hwcap_field = HWCAP_SHA3, + .valid = 1, +#endif + }, + + { + /* Advanced SIMD SM3 Instructions */ + .feat_flag = "FEAT_SM3", +#ifdef HWCAP_SM3 + .hwcap_field = HWCAP_SM3, + .valid = 1, +#endif + }, + + { + /* Advanced SIMD SM4 Instructions */ + .feat_flag = "FEAT_SM4", +#ifdef HWCAP_SM4 + .hwcap_field = HWCAP_SM4, + .valid = 1, +#endif + }, + + { + /* Advanced SIMD Int8 Dot Product Instructions */ + .feat_flag = "FEAT_DotProd", +#ifdef HWCAP_ASIMDDP + .hwcap_field = HWCAP_ASIMDDP, + .valid = 1, +#endif + }, + + { + /* Advanced SIMD SHA512 Instructions */ + .feat_flag = "FEAT_SHA512", +#ifdef HWCAP_SHA512 + .hwcap_field = HWCAP_SHA512, + .valid = 1, +#endif + }, + + { + /* Scalable Vector Extensions */ + .feat_flag = "FEAT_SVE", +#ifdef HWCAP_SVE + .hwcap_field = HWCAP_SVE, + .valid = 1, +#endif + }, + + { + /* Half-precision Floating-point FMLAL Instructions */ + .feat_flag = "FEAT_FHM", +#ifdef HWCAP_ASIMDFHM + .hwcap_field = HWCAP_ASIMDFHM, + .valid = 1, +#endif + }, + + { + /* Data Independent Timing Instructions */ + .feat_flag = "FEAT_DIT", +#ifdef HWCAP_DIT + .hwcap_field = HWCAP_DIT, + .valid = 1, +#endif + }, + + { + /* Large System Extensions Version 2 */ + .feat_flag = "FEAT_LSE2", +#ifdef HWCAP_USCAT + .hwcap_field = HWCAP_USCAT, + .valid = 1, +#endif + }, + + { + /* Load-acquire RCpc Instructions Version 2 */ + .feat_flag = "FEAT_LRCPC2", +#ifdef HWCAP_ILRCPC + .hwcap_field = HWCAP_ILRCPC, + .valid = 1, +#endif + }, + + { + /* Condition Flag Manipulation Extensions */ + .feat_flag = "FEAT_FlagM", +#ifdef HWCAP_FLAGM + .hwcap_field = HWCAP_FLAGM, + .valid = 1, +#endif + }, + + { + /* Speculative Store Bypass Safe Instructions */ + .feat_flag = "FEAT_SSBS2", +#ifdef HWCAP_SSBS + .hwcap_field = HWCAP_SSBS, + .valid = 1, +#endif + }, + + { + /* Speculation Barrier Instructions */ + .feat_flag = "FEAT_SB", +#ifdef HWCAP_SB + .hwcap_field = HWCAP_SB, + .valid = 1, +#endif + }, + + { + /* Pointer Authentication Extensions */ + .feat_flag = "FEAT_PAuth", +#ifdef HWCAP_PACA + .hwcap_field = HWCAP_PACA, + .valid = 1, +#endif + }, + + { + /* Generic Authentication Extensions */ + .feat_flag = "HWCAP_PACG", +#ifdef HWCAP_PACG + .hwcap_field = HWCAP_PACG, + .valid = 1, +#endif + } +}; + +static hwcap_feat_flag_t hwcap2_flags[] = { + { + /* DC CVADP instructions */ + .feat_flag = "FEAT_DPB2", +#ifdef HWCAP2_DCPODP + .hwcap_field = HWCAP2_DCPODP, + .valid = 1, +#endif + }, + + { + /* Scalable Vector Extensions Version 2 */ + .feat_flag = "FEAT_SVE2", +#ifdef HWCAP2_SVE2 + .hwcap_field = HWCAP2_SVE2, + .valid = 1, +#endif + }, + + { + /* SVE AES Instructions */ + .feat_flag = "FEAT_SVE_AES", +#ifdef HWCAP2_SVEAES + .hwcap_field = HWCAP2_SVEAES, + .valid = 1, +#endif + }, + + { + /* SVE PMULL Instructions */ + .feat_flag = "FEAT_SVE_PMULL128", +#ifdef HWCAP2_SVEPMULL + .hwcap_field = HWCAP2_SVEPMULL, + .valid = 1, +#endif + }, + + { + /* SVE Bit Permute Instructions */ + .feat_flag = "FEAT_SVE_BitPerm", +#ifdef HWCAP2_SVEBITPERM + .hwcap_field = HWCAP2_SVEBITPERM, + .valid = 1, +#endif + }, + + { + /* SVE SHA-3 Instructions */ + .feat_flag = "FEAT_SVE_SHA3", +#ifdef HWCAP2_SVESHA3 + .hwcap_field = HWCAP2_SVESHA3, + .valid = 1, +#endif + }, + + { + /* SVE SM4 Instructions */ + .feat_flag = "FEAT_SVE_SM4", +#ifdef HWCAP2_SVESM4 + .hwcap_field = HWCAP2_SVESM4, + .valid = 1, +#endif + }, + + { + /* Condition Flag Manipulation Extensions Version 2 */ + .feat_flag = "FEAT_FlagM2", +#ifdef HWCAP2_FLAGM2 + .hwcap_field = HWCAP2_FLAGM2, + .valid = 1, +#endif + }, + + { + /* FRINT32Z, FRINT32X, FRINT64Z, and FRINT64X instructions */ + .feat_flag = "FEAT_FRINTTS", +#ifdef HWCAP2_FRINT + .hwcap_field = HWCAP2_FRINT, + .valid = 1, +#endif + }, + + { + /* SVE Int8 Matrix Multiplication Instructions */ + .feat_flag = "FEAT_I8MM", +#ifdef HWCAP2_SVEI8MM + .hwcap_field = HWCAP2_SVEI8MM, + .valid = 1, +#endif + }, + + { + /* SVE Single-precision Floating-point Matrix Multiply Instructions */ + .feat_flag = "FEAT_F32MM", +#ifdef HWCAP2_SVEF32MM + .hwcap_field = HWCAP2_SVEF32MM, + .valid = 1, +#endif + }, + + { + /* SVE Double-precision Floating-point Matrix Multiply Instructions */ + .feat_flag = "FEAT_F64MM", +#ifdef HWCAP2_SVEF64MM + .hwcap_field = HWCAP2_SVEF64MM, + .valid = 1, +#endif + }, + + { + /* SVE BFloat16 Instructions */ + .feat_flag = "FEAT_BF16", +#ifdef HWCAP2_SVEBF16 + .hwcap_field = HWCAP2_SVEBF16, + .valid = 1, +#endif + }, + + { + /* Advanced SIMD and Floating-point Int8 Matrix Multiplication Instructions */ + .feat_flag = "FEAT_I8MM", +#ifdef HWCAP2_I8MM + .hwcap_field = HWCAP2_I8MM, + .valid = 1, +#endif + }, + + { + /* Advanced SIMD and Floating-point BFloat16 Instructions */ + .feat_flag = "FEAT_BF16", +#ifdef HWCAP2_BF16 + .hwcap_field = HWCAP2_BF16, + .valid = 1, +#endif + }, + + { + /* Data Gathering Hint Extensions */ + .feat_flag = "FEAT_DGH", +#ifdef HWCAP2_DGH + .hwcap_field = HWCAP2_DGH, + .valid = 1, +#endif + }, + + { + /* Random Number Generation Extensions */ + .feat_flag = "FEAT_RNG", +#ifdef HWCAP2_RNG + .hwcap_field = HWCAP2_RNG, + .valid = 1, +#endif + }, + + { + /* Branch Target Identification Extensions */ + .feat_flag = "FEAT_BTI", +#ifdef HWCAP2_BTI + .hwcap_field = HWCAP2_BTI, + .valid = 1, +#endif + }, + + { + /* Full Memory Tagging Extensions */ + .feat_flag = "FEAT_MTE2", +#ifdef HWCAP2_MTE + .hwcap_field = HWCAP2_MTE, + .valid = 1, +#endif + } +}; + +static void _odp_sys_info_print_acle_flags(void) +{ + const char *ndef = "n/a"; + + /* Avoid compiler warning about unused variable */ + (void)ndef; + + /* See ARM C Language Extensions documentation for details */ + ODP_PRINT("ARM FEATURES:\n"); + + ODP_PRINT(" __ARM_ALIGN_MAX_PWR "); +#ifdef __ARM_ALIGN_MAX_PWR + ODP_PRINT("%i\n", __ARM_ALIGN_MAX_PWR); +#else + ODP_PRINT("%s\n", ndef); +#endif + + ODP_PRINT(" __ARM_ALIGN_MAX_STACK_PWR "); +#ifdef __ARM_ALIGN_MAX_STACK_PWR + ODP_PRINT("%i\n", __ARM_ALIGN_MAX_STACK_PWR); +#else + ODP_PRINT("%s\n", ndef); +#endif + + ODP_PRINT(" __ARM_ARCH "); +#ifdef __ARM_ARCH + ODP_PRINT("%i\n", __ARM_ARCH); +#else + ODP_PRINT("%s\n", ndef); +#endif + + ODP_PRINT(" __ARM_ARCH_ISA_A64 "); +#ifdef __ARM_ARCH_ISA_A64 + ODP_PRINT("%i\n", __ARM_ARCH_ISA_A64); +#else + ODP_PRINT("%s\n", ndef); +#endif + + ODP_PRINT(" __ARM_BIG_ENDIAN "); +#ifdef __ARM_BIG_ENDIAN + ODP_PRINT("%i\n", __ARM_BIG_ENDIAN); +#else + ODP_PRINT("%s\n", ndef); +#endif + + ODP_PRINT(" __ARM_BF16_FORMAT_ALTERNATIVE "); +#ifdef __ARM_BF16_FORMAT_ALTERNATIVE + ODP_PRINT("%i\n", __ARM_BF16_FORMAT_ALTERNATIVE); +#else + ODP_PRINT("%s\n", ndef); +#endif + + ODP_PRINT(" __ARM_FEATURE_ATOMICS "); +#ifdef __ARM_FEATURE_ATOMICS + ODP_PRINT("%i\n", __ARM_FEATURE_ATOMICS); +#else + ODP_PRINT("%s\n", ndef); +#endif + + ODP_PRINT(" __ARM_FEATURE_BF16 "); +#ifdef __ARM_FEATURE_BF16 + ODP_PRINT("%i\n", __ARM_FEATURE_BF16); +#else + ODP_PRINT("%s\n", ndef); +#endif + + ODP_PRINT(" __ARM_FEATURE_BTI_DEFAULT "); +#ifdef __ARM_FEATURE_BTI_DEFAULT + ODP_PRINT("%i\n", __ARM_FEATURE_BTI_DEFAULT); +#else + ODP_PRINT("%s\n", ndef); +#endif + + ODP_PRINT(" __ARM_FEATURE_CDE "); +#ifdef __ARM_FEATURE_CDE + ODP_PRINT("%i\n", __ARM_FEATURE_CDE); +#else + ODP_PRINT("%s\n", ndef); +#endif + + ODP_PRINT(" __ARM_FEATURE_CDE_COPROC "); +#ifdef __ARM_FEATURE_CDE_COPROC + ODP_PRINT("0x%X\n", __ARM_FEATURE_CDE_COPROC); +#else + ODP_PRINT("%s\n", ndef); +#endif + + ODP_PRINT(" __ARM_FEATURE_CLZ "); +#ifdef __ARM_FEATURE_CLZ + ODP_PRINT("%i\n", __ARM_FEATURE_CLZ); +#else + ODP_PRINT("%s\n", ndef); +#endif + + ODP_PRINT(" __ARM_FEATURE_COMPLEX "); +#ifdef __ARM_FEATURE_COMPLEX + ODP_PRINT("%i\n", __ARM_FEATURE_COMPLEX); +#else + ODP_PRINT("%s\n", ndef); +#endif + + ODP_PRINT(" __ARM_FEATURE_COPROC "); +#ifdef __ARM_FEATURE_COPROC + ODP_PRINT("0x%X\n", __ARM_FEATURE_COPROC); +#else + ODP_PRINT("%s\n", ndef); +#endif + + ODP_PRINT(" __ARM_FEATURE_CRC32 "); +#ifdef __ARM_FEATURE_CRC32 + ODP_PRINT("%i\n", __ARM_FEATURE_CRC32); +#else + ODP_PRINT("%s\n", ndef); +#endif + + ODP_PRINT(" __ARM_FEATURE_CRYPTO "); +#ifdef __ARM_FEATURE_CRYPTO + ODP_PRINT("%i\n", __ARM_FEATURE_CRYPTO); +#else + ODP_PRINT("%s\n", ndef); +#endif + + ODP_PRINT(" __ARM_FEATURE_DIRECTED_ROUNDING "); +#ifdef __ARM_FEATURE_DIRECTED_ROUNDING + ODP_PRINT("%i\n", __ARM_FEATURE_DIRECTED_ROUNDING); +#else + ODP_PRINT("%s\n", ndef); +#endif + + ODP_PRINT(" __ARM_FEATURE_DOTPROD "); +#ifdef __ARM_FEATURE_DOTPROD + ODP_PRINT("%i\n", __ARM_FEATURE_DOTPROD); +#else + ODP_PRINT("%s\n", ndef); +#endif + + ODP_PRINT(" __ARM_FEATURE_FMA "); +#ifdef __ARM_FEATURE_FMA + ODP_PRINT("%i\n", __ARM_FEATURE_FMA); +#else + ODP_PRINT("%s\n", ndef); +#endif + + ODP_PRINT(" __ARM_FEATURE_FP16_FML "); +#ifdef __ARM_FEATURE_FP16_FML + ODP_PRINT("%i\n", __ARM_FEATURE_FP16_FML); +#else + ODP_PRINT("%s\n", ndef); +#endif + + ODP_PRINT(" __ARM_FEATURE_FRINT "); +#ifdef __ARM_FEATURE_FRINT + ODP_PRINT("%i\n", __ARM_FEATURE_FRINT); +#else + ODP_PRINT("%s\n", ndef); +#endif + + ODP_PRINT(" __ARM_FEATURE_IDIV "); +#ifdef __ARM_FEATURE_IDIV + ODP_PRINT("%i\n", __ARM_FEATURE_IDIV); +#else + ODP_PRINT("%s\n", ndef); +#endif + + ODP_PRINT(" __ARM_FEATURE_JCVT "); +#ifdef __ARM_FEATURE_JCVT + ODP_PRINT("%i\n", __ARM_FEATURE_JCVT); +#else + ODP_PRINT("%s\n", ndef); +#endif + + ODP_PRINT(" __ARM_FEATURE_MATMUL_INT8 "); +#ifdef __ARM_FEATURE_MATMUL_INT8 + ODP_PRINT("%i\n", __ARM_FEATURE_MATMUL_INT8); +#else + ODP_PRINT("%s\n", ndef); +#endif + + ODP_PRINT(" __ARM_FEATURE_MEMORY_TAGGING "); +#ifdef __ARM_FEATURE_MEMORY_TAGGING + ODP_PRINT("%i\n", __ARM_FEATURE_MEMORY_TAGGING); +#else + ODP_PRINT("%s\n", ndef); +#endif + + ODP_PRINT(" __ARM_FEATURE_MVE "); +#ifdef __ARM_FEATURE_MVE + ODP_PRINT("0x%X\n", __ARM_FEATURE_MVE); +#else + ODP_PRINT("%s\n", ndef); +#endif + + ODP_PRINT(" __ARM_FEATURE_NUMERIC_MAXMIN "); +#ifdef __ARM_FEATURE_NUMERIC_MAXMIN + ODP_PRINT("%i\n", __ARM_FEATURE_NUMERIC_MAXMIN); +#else + ODP_PRINT("%s\n", ndef); +#endif + + ODP_PRINT(" __ARM_FEATURE_PAC_DEFAULT "); +#ifdef __ARM_FEATURE_PAC_DEFAULT + ODP_PRINT("0x%X\n", __ARM_FEATURE_PAC_DEFAULT); +#else + ODP_PRINT("%s\n", ndef); +#endif + + ODP_PRINT(" __ARM_FEATURE_QRDMX "); +#ifdef __ARM_FEATURE_QRDMX + ODP_PRINT("%i\n", __ARM_FEATURE_QRDMX); +#else + ODP_PRINT("%s\n", ndef); +#endif + + ODP_PRINT(" __ARM_FEATURE_RNG "); +#ifdef __ARM_FEATURE_RNG + ODP_PRINT("%i\n", __ARM_FEATURE_RNG); +#else + ODP_PRINT("%s\n", ndef); +#endif + + ODP_PRINT(" __ARM_FEATURE_SHA3 "); +#ifdef __ARM_FEATURE_SHA3 + ODP_PRINT("%i\n", __ARM_FEATURE_SHA3); +#else + ODP_PRINT("%s\n", ndef); +#endif + + ODP_PRINT(" __ARM_FEATURE_SHA512 "); +#ifdef __ARM_FEATURE_SHA512 + ODP_PRINT("%i\n", __ARM_FEATURE_SHA512); +#else + ODP_PRINT("%s\n", ndef); +#endif + + ODP_PRINT(" __ARM_FEATURE_SM3 "); +#ifdef __ARM_FEATURE_SM3 + ODP_PRINT("%i\n", __ARM_FEATURE_SM3); +#else + ODP_PRINT("%s\n", ndef); +#endif + + ODP_PRINT(" __ARM_FEATURE_SM4 "); +#ifdef __ARM_FEATURE_SM4 + ODP_PRINT("%i\n", __ARM_FEATURE_SM4); +#else + ODP_PRINT("%s\n", ndef); +#endif + + ODP_PRINT(" __ARM_FEATURE_TME "); +#ifdef __ARM_FEATURE_TME + ODP_PRINT("%i\n", __ARM_FEATURE_TME); +#else + ODP_PRINT("%s\n", ndef); +#endif + + ODP_PRINT(" __ARM_FEATURE_UNALIGNED "); +#ifdef __ARM_FEATURE_UNALIGNED + ODP_PRINT("%i\n", __ARM_FEATURE_UNALIGNED); +#else + ODP_PRINT("%s\n", ndef); +#endif + + ODP_PRINT(" __ARM_FP "); +#ifdef __ARM_FP + ODP_PRINT("0x%X\n", __ARM_FP); +#else + ODP_PRINT("%s\n", ndef); +#endif + + ODP_PRINT(" __ARM_FP_FAST "); +#ifdef __ARM_FP_FAST + ODP_PRINT("%i\n", __ARM_FP_FAST); +#else + ODP_PRINT("%s\n", ndef); +#endif + + ODP_PRINT(" __ARM_FP_FENV_ROUNDING "); +#ifdef __ARM_FP_FENV_ROUNDING + ODP_PRINT("%i\n", __ARM_FP_FENV_ROUNDING); +#else + ODP_PRINT("%s\n", ndef); +#endif + + ODP_PRINT(" __ARM_FP16_ARGS "); +#ifdef __ARM_FP16_ARGS + ODP_PRINT("%i\n", __ARM_FP16_ARGS); +#else + ODP_PRINT("%s\n", ndef); +#endif + + ODP_PRINT(" __ARM_FP16_FORMAT_ALTERNATIVE "); +#ifdef __ARM_FP16_FORMAT_ALTERNATIVE + ODP_PRINT("%i\n", __ARM_FP16_FORMAT_ALTERNATIVE); +#else + ODP_PRINT("%s\n", ndef); +#endif + + ODP_PRINT(" __ARM_FP16_FORMAT_IEEE "); +#ifdef __ARM_FP16_FORMAT_IEEE + ODP_PRINT("%i\n", __ARM_FP16_FORMAT_IEEE); +#else + ODP_PRINT("%s\n", ndef); +#endif + + ODP_PRINT(" __ARM_NEON "); +#ifdef __ARM_NEON + ODP_PRINT("%i\n", __ARM_NEON); +#else + ODP_PRINT("%s\n", ndef); +#endif + + ODP_PRINT(" __ARM_NEON_FP "); +#ifdef __ARM_NEON_FP + ODP_PRINT("0x%X\n", __ARM_NEON_FP); +#else + ODP_PRINT("%s\n", ndef); +#endif + + ODP_PRINT(" __ARM_PCS_AAPCS64 "); +#ifdef __ARM_PCS_AAPCS64 + ODP_PRINT("%i\n", __ARM_PCS_AAPCS64); +#else + ODP_PRINT("%s\n", ndef); +#endif + + ODP_PRINT(" __ARM_ROPI "); +#ifdef __ARM_ROPI + ODP_PRINT("%i\n", __ARM_ROPI); +#else + ODP_PRINT("%s\n", ndef); +#endif + + ODP_PRINT(" __ARM_RWPI "); +#ifdef __ARM_RWPI + ODP_PRINT("%i\n", __ARM_RWPI); +#else + ODP_PRINT("%s\n", ndef); +#endif + + ODP_PRINT(" __ARM_SIZEOF_MINIMAL_ENUM "); +#ifdef __ARM_SIZEOF_MINIMAL_ENUM + ODP_PRINT("%i\n", __ARM_SIZEOF_MINIMAL_ENUM); +#else + ODP_PRINT("%s\n", ndef); +#endif + + ODP_PRINT(" __ARM_SIZEOF_WCHAR_T "); +#ifdef __ARM_SIZEOF_WCHAR_T + ODP_PRINT("%i\n", __ARM_SIZEOF_WCHAR_T); +#else + ODP_PRINT("%s\n", ndef); +#endif + + ODP_PRINT(" ARM ISA version: "); +#if defined(__ARM_ARCH) + if (__ARM_ARCH < 8) { + ODP_PRINT("v%i\n", __ARM_ARCH); + } else if (__ARM_ARCH == 8) { + /* Actually, this checks for new NEON instructions in + * v8.1, but is currently the only way to distinguish + * v8.0 and >=v8.1. */ + #ifdef __ARM_FEATURE_QRDMX + ODP_PRINT("v8.1 or higher\n"); + #else + ODP_PRINT("v8.0\n"); + #endif + } else { + /* ACLE 2018 defines that from v8.1 onwards the value includes + * the minor version number: __ARM_ARCH = X * 100 + Y + * E.g. for Armv8.1 __ARM_ARCH = 801 */ + int major = __ARM_ARCH / 100; + int minor = __ARM_ARCH - (major * 100); + + ODP_PRINT("v%i.%i\n", major, minor); + } +#else + ODP_PRINT("%s\n", ndef); +#endif + + ODP_PRINT("\n"); +} + +static int check_hwcap_duplicates(unsigned int hwcap_field) +{ + int ret = 0; + + /* FP and AdvSIMD fields of the AArch64 Processor + * Feature Register 0 must have the same value and are + * defined by the same feature flag. Print the flag + * only once. */ +#ifdef HWCAP_ASIMD + if (hwcap_field == HWCAP_ASIMD) + ret = 1; +#endif +#ifdef HWCAP_ASIMDHP + if (hwcap_field == HWCAP_ASIMDHP) + ret = 1; +#endif + + return ret; +} + +static void _odp_sys_info_print_hwcap_flags(void) +{ + unsigned long hwcaps, hwcaps2; + unsigned int size, size2; + + ODP_PRINT("ARM FEATURES SUPPORTED BY HARDWARE:\n"); + + /* Print supported hardware flags via AT_HWCAP entry of the hwcaps + * auxiliary vector. */ + hwcaps = getauxval(AT_HWCAP); + size = sizeof(hwcap_flags) / sizeof(hwcap_feat_flag_t); + for (unsigned int i = 0; i < size; i++) { + if (hwcap_flags[i].valid) { + if (check_hwcap_duplicates(hwcap_flags[i].hwcap_field)) { + hwcaps = hwcaps >> 1; + continue; + } + + if (hwcaps & 0x01) + ODP_PRINT("%s ", hwcap_flags[i].feat_flag); + hwcaps = hwcaps >> 1; + } + } + + /* Print supported hardware flags via AT_HWCAP2 entry of the hwcaps + * auxiliary vector. */ + hwcaps2 = getauxval(AT_HWCAP2); + size2 = sizeof(hwcap2_flags) / sizeof(hwcap_feat_flag_t); + for (unsigned long i = 0; i < size2; i++) { + if (hwcap2_flags[i].valid) { + if (hwcaps2 & 0x01) + ODP_PRINT("%s ", hwcap2_flags[i].feat_flag); + hwcaps2 = hwcaps2 >> 1; + } + } + + ODP_PRINT("\n"); + + /* Re-initialize hwcaps and hwcaps2 */ + hwcaps = 0; + hwcaps2 = 0; + + ODP_PRINT("\nARM FEATURES NOT SUPPORTED BY HARDWARE:\n"); + + hwcaps = getauxval(AT_HWCAP); + for (unsigned long i = 0; i < size; i++) { + if (hwcap_flags[i].valid) { + if (check_hwcap_duplicates(hwcap_flags[i].hwcap_field)) { + hwcaps = hwcaps >> 1; + continue; + } + + if (!(hwcaps & 0x01)) + ODP_PRINT("%s ", hwcap_flags[i].feat_flag); + hwcaps = hwcaps >> 1; + } + } + + hwcaps2 = getauxval(AT_HWCAP2); + for (unsigned long i = 0; i < size2; i++) { + if (hwcap2_flags[i].valid) { + if (!(hwcaps2 & 0x01)) + ODP_PRINT("%s ", hwcap2_flags[i].feat_flag); + hwcaps2 = hwcaps2 >> 1; + } + } + + ODP_PRINT("\n"); + + ODP_PRINT("\nARM FEATURES NOT SUPPORTED BY KERNEL:\n"); + + for (unsigned long i = 0; i < size; i++) { + if (!hwcap_flags[i].valid) + ODP_PRINT("%s ", hwcap_flags[i].feat_flag); + } + + for (unsigned long i = 0; i < size2; i++) { + if (!hwcap2_flags[i].valid) + ODP_PRINT("%s ", hwcap2_flags[i].feat_flag); + } + + ODP_PRINT("\n\n"); +} + +void _odp_cpu_flags_print_all(void) +{ + _odp_sys_info_print_acle_flags(); + _odp_sys_info_print_hwcap_flags(); +} diff --git a/platform/linux-generic/arch/aarch64/cpu_flags.h b/platform/linux-generic/arch/aarch64/cpu_flags.h new file mode 100644 index 000000000..177b1c44f --- /dev/null +++ b/platform/linux-generic/arch/aarch64/cpu_flags.h @@ -0,0 +1,20 @@ +/* Copyright (c) 2021, ARM Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_PLAT_CPU_FLAGS_H_ +#define ODP_PLAT_CPU_FLAGS_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +void _odp_cpu_flags_print_all(void); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/arch/aarch64/odp/api/abi/atomic_inlines.h b/platform/linux-generic/arch/aarch64/odp/api/abi/atomic_inlines.h index 446f9cc50..f530afd4f 100644 --- a/platform/linux-generic/arch/aarch64/odp/api/abi/atomic_inlines.h +++ b/platform/linux-generic/arch/aarch64/odp/api/abi/atomic_inlines.h @@ -22,7 +22,7 @@ * @return 1 for success and 0 for fail */ #define ATOMIC_CAS_OP_128(atom, oper, old_val, new_val, val) \ -({ \ +__extension__ ({ \ odp_u128_t _val; \ odp_atomic_u128_t *_atom = atom; \ odp_u128_t *_old_val = old_val; \ @@ -152,7 +152,96 @@ static inline int _odp_atomic_cas_acq_rel_u128(odp_atomic_u128_t *atom, odp_u128 return 0; } -#else /* _ODP_LOCK_FREE_128BIT_ATOMICS */ + +static inline void _odp_atomic_add_u32(odp_atomic_u32_t *atom, uint32_t val) +{ + __asm__ volatile("stadd %w[val], %[atom]" + : [atom] "+Q" (atom->v) + : [val] "r" (val)); +} + +static inline void _odp_atomic_sub_u32(odp_atomic_u32_t *atom, uint32_t val) +{ + int32_t neg_val = -val; + + __asm__ volatile("stadd %w[neg_val], %[atom]" + : [atom] "+Q" (atom->v) + : [neg_val] "r" (neg_val)); +} + +static inline void _odp_atomic_inc_u32(odp_atomic_u32_t *atom) +{ + _odp_atomic_add_u32(atom, 1); +} + +static inline void _odp_atomic_dec_u32(odp_atomic_u32_t *atom) +{ + _odp_atomic_sub_u32(atom, 1); +} + +static inline void _odp_atomic_add_u64(odp_atomic_u64_t *atom, uint64_t val) +{ + __asm__ volatile("stadd %[val], %[atom]" + : [atom] "+Q" (atom->v) + : [val] "r" (val)); +} + +static inline void _odp_atomic_sub_u64(odp_atomic_u64_t *atom, uint64_t val) +{ + int64_t neg_val = -val; + + __asm__ volatile("stadd %[neg_val], %[atom]" + : [atom] "+Q" (atom->v) + : [neg_val] "r" (neg_val)); +} + +static inline void _odp_atomic_inc_u64(odp_atomic_u64_t *atom) +{ + _odp_atomic_add_u64(atom, 1); +} + +static inline void _odp_atomic_dec_u64(odp_atomic_u64_t *atom) +{ + _odp_atomic_sub_u64(atom, 1); +} + +static inline void _odp_atomic_add_rel_u32(odp_atomic_u32_t *atom, uint32_t val) +{ + __asm__ volatile("staddl %w[val], %[atom]" + : [atom] "+Q" (atom->v) + : [val] "r" (val) + : "memory"); +} + +static inline void _odp_atomic_sub_rel_u32(odp_atomic_u32_t *atom, uint32_t val) +{ + int32_t neg_val = -val; + + __asm__ volatile("staddl %w[neg_val], %[atom]" + : [atom] "+Q" (atom->v) + : [neg_val] "r" (neg_val) + : "memory"); +} + +static inline void _odp_atomic_add_rel_u64(odp_atomic_u64_t *atom, uint64_t val) +{ + __asm__ volatile("staddl %[val], %[atom]" + : [atom] "+Q" (atom->v) + : [val] "r" (val) + : "memory"); +} + +static inline void _odp_atomic_sub_rel_u64(odp_atomic_u64_t *atom, uint64_t val) +{ + int64_t neg_val = -val; + + __asm__ volatile("staddl %[neg_val], %[atom]" + : [atom] "+Q" (atom->v) + : [neg_val] "r" (neg_val) + : "memory"); +} + +#else /* !_ODP_LOCK_FREE_128BIT_ATOMICS */ /* Use generic implementation */ #include <odp/api/abi/atomic_generic.h> diff --git a/platform/linux-generic/arch/aarch64/odp_atomic.h b/platform/linux-generic/arch/aarch64/odp_atomic.h index c7a28fc2c..445d91e28 100644 --- a/platform/linux-generic/arch/aarch64/odp_atomic.h +++ b/platform/linux-generic/arch/aarch64/odp_atomic.h @@ -36,6 +36,10 @@ do { \ #define LL_MO(mo) (HAS_ACQ((mo)) ? __ATOMIC_ACQUIRE : __ATOMIC_RELAXED) #define SC_MO(mo) (HAS_RLS((mo)) ? __ATOMIC_RELEASE : __ATOMIC_RELAXED) +/* Prevent warnings about ISO C not supporting __int128 */ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wpedantic" + #ifndef __ARM_FEATURE_QRDMX /* Feature only available in v8.1a and beyond */ static inline bool __lockfree_compare_exchange_16(register __int128 *var, __int128 *exp, @@ -219,7 +223,7 @@ static inline __int128 __lockfree_load_16(__int128 *var, int mo) return old; } -typedef unsigned __int128 _u128_t; +__extension__ typedef unsigned __int128 _u128_t; static inline _u128_t lockfree_load_u128(_u128_t *atomic) { @@ -291,6 +295,8 @@ static inline bitset_t bitset_mask(uint32_t bit) return (unsigned __int128)(1ULL << (bit - 64)) << 64; } +#pragma GCC diagnostic pop + #else #error Unsupported size of bit sets (ATOM_BITSET_SIZE) #endif diff --git a/platform/linux-generic/arch/aarch64/odp_llsc.h b/platform/linux-generic/arch/aarch64/odp_llsc.h index cf55be132..2561451ca 100644 --- a/platform/linux-generic/arch/aarch64/odp_llsc.h +++ b/platform/linux-generic/arch/aarch64/odp_llsc.h @@ -115,11 +115,11 @@ static inline uint32_t sc(uint64_t *var, uint64_t neu, int mm) #define sc64(a, b, c) sc((a), (b), (c)) union i128 { - __int128 i128; + __extension__ __int128 i128; int64_t i64[2]; }; -static inline __int128 lld(__int128 *var, int mm) +__extension__ static inline __int128 lld(__int128 *var, int mm) { union i128 old; @@ -139,10 +139,12 @@ static inline __int128 lld(__int128 *var, int mm) } /* Return 0 on success, 1 on failure */ -static inline uint32_t scd(__int128 *var, __int128 neu, int mm) +__extension__ static inline uint32_t scd(__int128 *var, __int128 neu, int mm) { uint32_t ret; +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wpedantic" if (mm == __ATOMIC_RELEASE) __asm__ volatile("stlxp %w0, %1, %2, [%3]" : "=&r" (ret) @@ -159,6 +161,7 @@ static inline uint32_t scd(__int128 *var, __int128 neu, int mm) : ); else ODP_ABORT(); +#pragma GCC diagnostic pop return ret; } diff --git a/platform/linux-generic/arch/aarch64/odp_sysinfo_parse.c b/platform/linux-generic/arch/aarch64/odp_sysinfo_parse.c index ab64d501e..dbb6d43af 100644 --- a/platform/linux-generic/arch/aarch64/odp_sysinfo_parse.c +++ b/platform/linux-generic/arch/aarch64/odp_sysinfo_parse.c @@ -13,6 +13,7 @@ #include <odp_global_data.h> #include <odp_sysinfo_internal.h> #include <odp_debug_internal.h> +#include "cpu_flags.h" #define TMP_STR_LEN 64 @@ -334,139 +335,7 @@ int _odp_cpuinfo_parser(FILE *file, system_info_t *sysinfo) void _odp_sys_info_print_arch(void) { - const char *ndef = "n/a"; - - /* Avoid compiler warning about unused variable */ - (void)ndef; - - /* See ARM C Language Extensions documentation for details */ - ODP_PRINT("ARM FEATURES:\n"); - - ODP_PRINT(" __ARM_ARCH "); -#ifdef __ARM_ARCH - ODP_PRINT("%i\n", __ARM_ARCH); -#else - ODP_PRINT("%s\n", ndef); -#endif - - ODP_PRINT(" __ARM_ARCH_ISA_A64 "); -#ifdef __ARM_ARCH_ISA_A64 - ODP_PRINT("%i\n", __ARM_ARCH_ISA_A64); -#else - ODP_PRINT("%s\n", ndef); -#endif - - ODP_PRINT(" __ARM_NEON "); -#ifdef __ARM_NEON - ODP_PRINT("%i\n", __ARM_NEON); -#else - ODP_PRINT("%s\n", ndef); -#endif - - ODP_PRINT(" __ARM_FEATURE_ATOMICS "); -#ifdef __ARM_FEATURE_ATOMICS - ODP_PRINT("%i\n", __ARM_FEATURE_ATOMICS); -#else - ODP_PRINT("%s\n", ndef); -#endif - - ODP_PRINT(" __ARM_FEATURE_UNALIGNED "); -#ifdef __ARM_FEATURE_UNALIGNED - ODP_PRINT("%i\n", __ARM_FEATURE_UNALIGNED); -#else - ODP_PRINT("%s\n", ndef); -#endif - - ODP_PRINT(" __ARM_FEATURE_IDIV "); -#ifdef __ARM_FEATURE_IDIV - ODP_PRINT("%i\n", __ARM_FEATURE_IDIV); -#else - ODP_PRINT("%s\n", ndef); -#endif - - ODP_PRINT(" __ARM_FEATURE_QRDMX "); -#ifdef __ARM_FEATURE_QRDMX - ODP_PRINT("%i\n", __ARM_FEATURE_QRDMX); -#else - ODP_PRINT("%s\n", ndef); -#endif - - ODP_PRINT(" __ARM_FEATURE_DOTPROD "); -#ifdef __ARM_FEATURE_DOTPROD - ODP_PRINT("%i\n", __ARM_FEATURE_DOTPROD); -#else - ODP_PRINT("%s\n", ndef); -#endif - - ODP_PRINT(" __ARM_FEATURE_CRYPTO "); -#ifdef __ARM_FEATURE_CRYPTO - ODP_PRINT("%i\n", __ARM_FEATURE_CRYPTO); -#else - ODP_PRINT("%s\n", ndef); -#endif - - ODP_PRINT(" __ARM_FEATURE_SHA512 "); -#ifdef __ARM_FEATURE_SHA512 - ODP_PRINT("%i\n", __ARM_FEATURE_SHA512); -#else - ODP_PRINT("%s\n", ndef); -#endif - - ODP_PRINT(" __ARM_FEATURE_SHA3 "); -#ifdef __ARM_FEATURE_SHA3 - ODP_PRINT("%i\n", __ARM_FEATURE_SHA3); -#else - ODP_PRINT("%s\n", ndef); -#endif - - ODP_PRINT(" __ARM_FEATURE_SM3 "); -#ifdef __ARM_FEATURE_SM3 - ODP_PRINT("%i\n", __ARM_FEATURE_SM3); -#else - ODP_PRINT("%s\n", ndef); -#endif - - ODP_PRINT(" __ARM_FEATURE_SM4 "); -#ifdef __ARM_FEATURE_SM4 - ODP_PRINT("%i\n", __ARM_FEATURE_SM4); -#else - ODP_PRINT("%s\n", ndef); -#endif - - ODP_PRINT(" __ARM_FEATURE_CRC32 "); -#ifdef __ARM_FEATURE_CRC32 - ODP_PRINT("%i\n", __ARM_FEATURE_CRC32); -#else - ODP_PRINT("%s\n", ndef); -#endif - - ODP_PRINT(" ARM ISA version: "); -#if defined(__ARM_ARCH) - if (__ARM_ARCH < 8) { - ODP_PRINT("v%i\n", __ARM_ARCH); - } else if (__ARM_ARCH == 8) { - /* Actually, this checks for new NEON instructions in - * v8.1, but is currently the only way to distinguish - * v8.0 and >=v8.1. */ - #ifdef __ARM_FEATURE_QRDMX - ODP_PRINT("v8.1 or higher\n"); - #else - ODP_PRINT("v8.0\n"); - #endif - } else { - /* ACLE 2018 defines that from v8.1 onwards the value includes - * the minor version number: __ARM_ARCH = X * 100 + Y - * E.g. for Armv8.1 __ARM_ARCH = 801 */ - int major = __ARM_ARCH / 100; - int minor = __ARM_ARCH - (major * 100); - - ODP_PRINT("v%i.%i\n", major, minor); - } -#else - ODP_PRINT("%s\n", ndef); -#endif - - ODP_PRINT("\n"); + _odp_cpu_flags_print_all(); } uint64_t odp_cpu_arch_hz_current(int id ODP_UNUSED) diff --git a/platform/linux-generic/arch/default/odp/api/abi/atomic_generic.h b/platform/linux-generic/arch/default/odp/api/abi/atomic_generic.h index e274a4d64..aec8b9f00 100644 --- a/platform/linux-generic/arch/default/odp/api/abi/atomic_generic.h +++ b/platform/linux-generic/arch/default/odp/api/abi/atomic_generic.h @@ -10,6 +10,68 @@ #include <odp/api/atomic.h> +static inline void _odp_atomic_add_u32(odp_atomic_u32_t *atom, uint32_t val) +{ + (void)__atomic_fetch_add(&atom->v, val, __ATOMIC_RELAXED); +} + +static inline void _odp_atomic_sub_u32(odp_atomic_u32_t *atom, uint32_t val) +{ + (void)__atomic_fetch_sub(&atom->v, val, __ATOMIC_RELAXED); +} + +static inline void _odp_atomic_inc_u32(odp_atomic_u32_t *atom) +{ + (void)__atomic_fetch_add(&atom->v, 1, __ATOMIC_RELAXED); +} + +static inline void _odp_atomic_dec_u32(odp_atomic_u32_t *atom) +{ + (void)__atomic_fetch_sub(&atom->v, 1, __ATOMIC_RELAXED); +} + +static inline void _odp_atomic_add_rel_u32(odp_atomic_u32_t *atom, uint32_t val) +{ + (void)__atomic_fetch_add(&atom->v, val, __ATOMIC_RELEASE); +} + +static inline void _odp_atomic_sub_rel_u32(odp_atomic_u32_t *atom, uint32_t val) +{ + (void)__atomic_fetch_sub(&atom->v, val, __ATOMIC_RELEASE); +} + +static inline void _odp_atomic_add_u64(odp_atomic_u64_t *atom, uint64_t val) +{ + (void)__atomic_fetch_add(&atom->v, val, __ATOMIC_RELAXED); +} + +static inline void _odp_atomic_sub_u64(odp_atomic_u64_t *atom, uint64_t val) +{ + (void)__atomic_fetch_sub(&atom->v, val, __ATOMIC_RELAXED); +} + +static inline void _odp_atomic_inc_u64(odp_atomic_u64_t *atom) +{ + (void)__atomic_fetch_add(&atom->v, 1, __ATOMIC_RELAXED); +} + +static inline void _odp_atomic_dec_u64(odp_atomic_u64_t *atom) +{ + (void)__atomic_fetch_sub(&atom->v, 1, __ATOMIC_RELAXED); +} + +#ifndef ODP_ATOMIC_U64_LOCK +static inline void _odp_atomic_add_rel_u64(odp_atomic_u64_t *atom, uint64_t val) +{ + (void)__atomic_fetch_add(&atom->v, val, __ATOMIC_RELEASE); +} + +static inline void _odp_atomic_sub_rel_u64(odp_atomic_u64_t *atom, uint64_t val) +{ + (void)__atomic_fetch_sub(&atom->v, val, __ATOMIC_RELEASE); +} +#endif + #ifdef __SIZEOF_INT128__ static inline void _odp_atomic_init_u128(odp_atomic_u128_t *atom, odp_u128_t val) @@ -78,7 +140,7 @@ static inline int _odp_atomic_cas_acq_rel_u128(odp_atomic_u128_t *atom, odp_u128 * 128 bit CAS operation expression for the ATOMIC_OP macro */ #define ATOMIC_CAS_OP_128(ret_ptr, old_val, new_val) \ -({ \ +__extension__ ({ \ int *_ret_ptr = ret_ptr; \ odp_u128_t *_old_val = old_val; \ odp_u128_t _new_val = new_val; \ @@ -99,7 +161,7 @@ static inline int _odp_atomic_cas_acq_rel_u128(odp_atomic_u128_t *atom, odp_u128 * @return The old value of the variable. */ #define ATOMIC_OP_128(atom, expr) \ -({ \ +__extension__ ({ \ odp_u128_t _old_val; \ odp_atomic_u128_t *_atom = atom; \ /* Loop while lock is already taken, stop when lock becomes clear */ \ diff --git a/platform/linux-generic/arch/default/odp_atomic.h b/platform/linux-generic/arch/default/odp_atomic.h index fc8260194..7650d2b52 100644 --- a/platform/linux-generic/arch/default/odp_atomic.h +++ b/platform/linux-generic/arch/default/odp_atomic.h @@ -9,7 +9,7 @@ #ifdef __SIZEOF_INT128__ -typedef unsigned __int128 _u128_t; +__extension__ typedef unsigned __int128 _u128_t; static inline _u128_t lockfree_load_u128(_u128_t *atomic) { diff --git a/platform/linux-generic/include/odp/api/plat/atomic_inlines.h b/platform/linux-generic/include/odp/api/plat/atomic_inlines.h index a9da70890..5ce4bfc28 100644 --- a/platform/linux-generic/include/odp/api/plat/atomic_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/atomic_inlines.h @@ -100,7 +100,7 @@ _ODP_INLINE uint32_t odp_atomic_fetch_add_u32(odp_atomic_u32_t *atom, _ODP_INLINE void odp_atomic_add_u32(odp_atomic_u32_t *atom, uint32_t val) { - (void)__atomic_fetch_add(&atom->v, val, __ATOMIC_RELAXED); + _odp_atomic_add_u32(atom, val); } _ODP_INLINE uint32_t odp_atomic_fetch_sub_u32(odp_atomic_u32_t *atom, @@ -111,7 +111,7 @@ _ODP_INLINE uint32_t odp_atomic_fetch_sub_u32(odp_atomic_u32_t *atom, _ODP_INLINE void odp_atomic_sub_u32(odp_atomic_u32_t *atom, uint32_t val) { - (void)__atomic_fetch_sub(&atom->v, val, __ATOMIC_RELAXED); + _odp_atomic_sub_u32(atom, val); } _ODP_INLINE uint32_t odp_atomic_fetch_inc_u32(odp_atomic_u32_t *atom) @@ -121,7 +121,7 @@ _ODP_INLINE uint32_t odp_atomic_fetch_inc_u32(odp_atomic_u32_t *atom) _ODP_INLINE void odp_atomic_inc_u32(odp_atomic_u32_t *atom) { - (void)__atomic_fetch_add(&atom->v, 1, __ATOMIC_RELAXED); + _odp_atomic_inc_u32(atom); } _ODP_INLINE uint32_t odp_atomic_fetch_dec_u32(odp_atomic_u32_t *atom) @@ -131,7 +131,7 @@ _ODP_INLINE uint32_t odp_atomic_fetch_dec_u32(odp_atomic_u32_t *atom) _ODP_INLINE void odp_atomic_dec_u32(odp_atomic_u32_t *atom) { - (void)__atomic_fetch_sub(&atom->v, 1, __ATOMIC_RELAXED); + _odp_atomic_dec_u32(atom); } _ODP_INLINE int odp_atomic_cas_u32(odp_atomic_u32_t *atom, uint32_t *old_val, @@ -180,7 +180,7 @@ _ODP_INLINE void odp_atomic_min_u32(odp_atomic_u32_t *atom, uint32_t new_min) * CAS operation expression for the ATOMIC_OP macro */ #define ATOMIC_CAS_OP(ret_ptr, old_val, new_val) \ -({ \ +__extension__ ({ \ if (atom->v == (old_val)) { \ atom->v = (new_val); \ *(ret_ptr) = 1; \ @@ -197,7 +197,7 @@ _ODP_INLINE void odp_atomic_min_u32(odp_atomic_u32_t *atom, uint32_t new_min) * @return The old value of the variable. */ #define ATOMIC_OP(atom, expr) \ -({ \ +__extension__ ({ \ uint64_t _old_val; \ /* Loop while lock is already taken, stop when lock becomes clear */ \ while (__atomic_test_and_set(&(atom)->lock, __ATOMIC_ACQUIRE)) \ @@ -350,7 +350,7 @@ _ODP_INLINE uint64_t odp_atomic_fetch_add_u64(odp_atomic_u64_t *atom, _ODP_INLINE void odp_atomic_add_u64(odp_atomic_u64_t *atom, uint64_t val) { - (void)__atomic_fetch_add(&atom->v, val, __ATOMIC_RELAXED); + _odp_atomic_add_u64(atom, val); } _ODP_INLINE uint64_t odp_atomic_fetch_sub_u64(odp_atomic_u64_t *atom, @@ -361,7 +361,7 @@ _ODP_INLINE uint64_t odp_atomic_fetch_sub_u64(odp_atomic_u64_t *atom, _ODP_INLINE void odp_atomic_sub_u64(odp_atomic_u64_t *atom, uint64_t val) { - (void)__atomic_fetch_sub(&atom->v, val, __ATOMIC_RELAXED); + _odp_atomic_sub_u64(atom, val); } _ODP_INLINE uint64_t odp_atomic_fetch_inc_u64(odp_atomic_u64_t *atom) @@ -371,7 +371,7 @@ _ODP_INLINE uint64_t odp_atomic_fetch_inc_u64(odp_atomic_u64_t *atom) _ODP_INLINE void odp_atomic_inc_u64(odp_atomic_u64_t *atom) { - (void)__atomic_fetch_add(&atom->v, 1, __ATOMIC_RELAXED); + _odp_atomic_inc_u64(atom); } _ODP_INLINE uint64_t odp_atomic_fetch_dec_u64(odp_atomic_u64_t *atom) @@ -381,7 +381,7 @@ _ODP_INLINE uint64_t odp_atomic_fetch_dec_u64(odp_atomic_u64_t *atom) _ODP_INLINE void odp_atomic_dec_u64(odp_atomic_u64_t *atom) { - (void)__atomic_fetch_sub(&atom->v, 1, __ATOMIC_RELAXED); + _odp_atomic_dec_u64(atom); } _ODP_INLINE int odp_atomic_cas_u64(odp_atomic_u64_t *atom, uint64_t *old_val, @@ -411,12 +411,12 @@ _ODP_INLINE void odp_atomic_store_rel_u64(odp_atomic_u64_t *atom, uint64_t val) _ODP_INLINE void odp_atomic_add_rel_u64(odp_atomic_u64_t *atom, uint64_t val) { - (void)__atomic_fetch_add(&atom->v, val, __ATOMIC_RELEASE); + _odp_atomic_add_rel_u64(atom, val); } _ODP_INLINE void odp_atomic_sub_rel_u64(odp_atomic_u64_t *atom, uint64_t val) { - (void)__atomic_fetch_sub(&atom->v, val, __ATOMIC_RELEASE); + _odp_atomic_sub_rel_u64(atom, val); } _ODP_INLINE int odp_atomic_cas_acq_u64(odp_atomic_u64_t *atom, @@ -485,12 +485,12 @@ _ODP_INLINE void odp_atomic_store_rel_u32(odp_atomic_u32_t *atom, uint32_t val) _ODP_INLINE void odp_atomic_add_rel_u32(odp_atomic_u32_t *atom, uint32_t val) { - (void)__atomic_fetch_add(&atom->v, val, __ATOMIC_RELEASE); + _odp_atomic_add_rel_u32(atom, val); } _ODP_INLINE void odp_atomic_sub_rel_u32(odp_atomic_u32_t *atom, uint32_t val) { - (void)__atomic_fetch_sub(&atom->v, val, __ATOMIC_RELEASE); + _odp_atomic_sub_rel_u32(atom, val); } _ODP_INLINE int odp_atomic_cas_acq_u32(odp_atomic_u32_t *atom, diff --git a/platform/linux-generic/include/odp_atomic_internal.h b/platform/linux-generic/include/odp_atomic_internal.h index 81280b1fa..d98e0e6be 100644 --- a/platform/linux-generic/include/odp_atomic_internal.h +++ b/platform/linux-generic/include/odp_atomic_internal.h @@ -138,7 +138,7 @@ static inline void _odp_atomic_flag_clear(_odp_atomic_flag_t *flag) #ifdef ODP_ATOMIC_U128 /** An unsigned 128-bit (16-byte) scalar type */ -typedef __int128 _uint128_t; +__extension__ typedef __int128 _uint128_t; /** Atomic 128-bit type */ typedef struct ODP_ALIGNED(16) { diff --git a/platform/linux-generic/include/odp_buffer_internal.h b/platform/linux-generic/include/odp_buffer_internal.h index e7bc78d6e..dec85f9d3 100644 --- a/platform/linux-generic/include/odp_buffer_internal.h +++ b/platform/linux-generic/include/odp_buffer_internal.h @@ -83,7 +83,7 @@ struct ODP_ALIGNED_CACHE odp_buffer_hdr_t { uint8_t flow_id; /* Data or next header */ - uint8_t data[0]; + uint8_t data[]; }; /* Buffer header size is critical for performance. Ensure that it does not accidentally diff --git a/platform/linux-generic/include/odp_config_internal.h b/platform/linux-generic/include/odp_config_internal.h index 40d2639f1..899b261bd 100644 --- a/platform/linux-generic/include/odp_config_internal.h +++ b/platform/linux-generic/include/odp_config_internal.h @@ -54,6 +54,11 @@ extern "C" { #define CONFIG_QUEUE_MAX_ORD_LOCKS 2 /* + * Maximum number of stashes + */ +#define CONFIG_MAX_STASHES 128 + +/* * Maximum number of packet IO resources */ #define ODP_CONFIG_PKTIO_ENTRIES 64 @@ -119,10 +124,10 @@ extern "C" { /* * Number of shared memory blocks reserved for implementation internal use. * - * Each pool requires three SHM blocks (buffers, ring, user area). 20 blocks are - * reserved for per ODP module global data. + * Each stash requires one SHM block, each pool requires three blocks (buffers, + * ring, user area), and 20 blocks are reserved for per ODP module global data. */ -#define CONFIG_INTERNAL_SHM_BLOCKS ((ODP_CONFIG_POOLS * 3) + 20) +#define CONFIG_INTERNAL_SHM_BLOCKS (CONFIG_MAX_STASHES + (ODP_CONFIG_POOLS * 3) + 20) /* * Maximum number of shared memory blocks. diff --git a/platform/linux-generic/include/odp_debug_internal.h b/platform/linux-generic/include/odp_debug_internal.h index 5027cfe4a..c5c1890c3 100644 --- a/platform/linux-generic/include/odp_debug_internal.h +++ b/platform/linux-generic/include/odp_debug_internal.h @@ -29,6 +29,10 @@ extern "C" { #endif +/* Avoid "ISO C99 requires at least one argument for the "..." in a variadic + * macro" errors when building with 'pedantic' option. */ +#pragma GCC system_header + /* Debug level configure option. Zero is the highest level. Value of N prints debug messages from * level 0 to N. */ #define CONFIG_DEBUG_LEVEL 0 diff --git a/platform/linux-generic/include/odp_event_vector_internal.h b/platform/linux-generic/include/odp_event_vector_internal.h index d3322eff5..2d51801df 100644 --- a/platform/linux-generic/include/odp_event_vector_internal.h +++ b/platform/linux-generic/include/odp_event_vector_internal.h @@ -17,6 +17,8 @@ #include <odp/api/packet.h> #include <odp_buffer_internal.h> +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wpedantic" /** * Internal event vector header */ @@ -28,9 +30,10 @@ typedef struct { uint32_t size; /* Vector of packet handles */ - odp_packet_t packet[0]; + odp_packet_t packet[]; } odp_event_vector_hdr_t; +#pragma GCC diagnostic pop /** * Return the vector header diff --git a/platform/linux-generic/include/odp_ipsec_internal.h b/platform/linux-generic/include/odp_ipsec_internal.h index 66d85d119..cc224e4cc 100644 --- a/platform/linux-generic/include/odp_ipsec_internal.h +++ b/platform/linux-generic/include/odp_ipsec_internal.h @@ -84,6 +84,8 @@ int _odp_ipsec_status_send(odp_queue_t queue, #define IPSEC_MAX_SALT_LEN 4 /**< Maximum salt length in bytes */ +#define IPSEC_SEQ_HI_LEN 4 /**< ESN Higher bits length in bytes */ + /* The minimum supported AR window size */ #define IPSEC_AR_WIN_SIZE_MIN 32 @@ -175,6 +177,8 @@ struct ipsec_sa_s { unsigned copy_flabel : 1; unsigned aes_ctr_iv : 1; unsigned udp_encap : 1; + unsigned esn : 1; + unsigned insert_seq_hi : 1; /* Only for outbound */ unsigned use_counter_iv : 1; @@ -264,8 +268,14 @@ typedef struct odp_ipsec_sa_lookup_s { /** IPSEC AAD */ typedef struct ODP_PACKED { - odp_u32be_t spi; /**< Security Parameter Index */ - odp_u32be_t seq_no; /**< Sequence Number */ + /**< Security Parameter Index */ + odp_u32be_t spi; + + /**< Sequence Number */ + union { + odp_u32be_t seq_no; + odp_u64be_t seq_no64; + }; } ipsec_aad_t; /* Return IV length required for the cipher for IPsec use */ @@ -317,14 +327,14 @@ int _odp_ipsec_sa_lifetime_update(ipsec_sa_t *ipsec_sa, uint32_t len, * * @retval <0 if the packet falls out of window */ -int _odp_ipsec_sa_replay_precheck(ipsec_sa_t *ipsec_sa, uint32_t seq, +int _odp_ipsec_sa_replay_precheck(ipsec_sa_t *ipsec_sa, uint64_t seq, odp_ipsec_op_status_t *status); /* Run check on sequence number of the packet and update window if necessary. * * @retval <0 if the packet falls out of window */ -int _odp_ipsec_sa_replay_update(ipsec_sa_t *ipsec_sa, uint32_t seq, +int _odp_ipsec_sa_replay_update(ipsec_sa_t *ipsec_sa, uint64_t seq, odp_ipsec_op_status_t *status); /** diff --git a/platform/linux-generic/include/odp_llqueue.h b/platform/linux-generic/include/odp_llqueue.h index cc4b84ecd..e9cf9945e 100644 --- a/platform/linux-generic/include/odp_llqueue.h +++ b/platform/linux-generic/include/odp_llqueue.h @@ -49,7 +49,7 @@ static odp_bool_t llq_on_queue(struct llnode *node); typedef uint64_t dintptr_t; #endif #if __SIZEOF_PTRDIFF_T__ == 8 -typedef __int128 dintptr_t; +__extension__ typedef __int128 dintptr_t; #endif struct llnode { diff --git a/platform/linux-generic/include/odp_macros_internal.h b/platform/linux-generic/include/odp_macros_internal.h index 229772373..997e0fd5b 100644 --- a/platform/linux-generic/include/odp_macros_internal.h +++ b/platform/linux-generic/include/odp_macros_internal.h @@ -22,14 +22,14 @@ extern "C" { #define ARRAY_SIZE(x) (sizeof(x) / sizeof((x)[0])) #define MIN(a, b) \ - ({ \ + __extension__ ({ \ __typeof__(a) tmp_a = (a); \ __typeof__(b) tmp_b = (b); \ tmp_a < tmp_b ? tmp_a : tmp_b; \ }) #define MAX(a, b) \ - ({ \ + __extension__ ({ \ __typeof__(a) tmp_a = (a); \ __typeof__(b) tmp_b = (b); \ tmp_a > tmp_b ? tmp_a : tmp_b; \ @@ -39,7 +39,7 @@ extern "C" { ((type *)(void *)(((char *)pointer) - offsetof(type, member))) #define DIV_ROUND_UP(a, b) \ - ({ \ + __extension__ ({ \ __typeof__(a) tmp_a = (a); \ __typeof__(b) tmp_b = (b); \ ODP_STATIC_ASSERT(__builtin_constant_p(b), ""); \ diff --git a/platform/linux-generic/include/odp_packet_internal.h b/platform/linux-generic/include/odp_packet_internal.h index 497ea4aee..62f8aea25 100644 --- a/platform/linux-generic/include/odp_packet_internal.h +++ b/platform/linux-generic/include/odp_packet_internal.h @@ -73,6 +73,8 @@ typedef struct { ODP_STATIC_ASSERT(PKT_MAX_SEGS < UINT16_MAX, "PACKET_MAX_SEGS_ERROR"); +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wpedantic" /** * Internal Packet header * @@ -148,9 +150,10 @@ typedef struct odp_packet_hdr_t { }; /* Packet data storage */ - uint8_t data[0]; + uint8_t data[]; } odp_packet_hdr_t; +#pragma GCC diagnostic pop /* Packet header size is critical for performance. Ensure that it does not accidentally * grow over 256 bytes when cache line size is 64 bytes (or less). With larger cache line sizes, diff --git a/platform/linux-generic/include/odp_pool_internal.h b/platform/linux-generic/include/odp_pool_internal.h index dc9f0d207..001bdfc37 100644 --- a/platform/linux-generic/include/odp_pool_internal.h +++ b/platform/linux-generic/include/odp_pool_internal.h @@ -35,6 +35,8 @@ typedef struct ODP_ALIGNED_CACHE pool_cache_t { } pool_cache_t; +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wpedantic" /* Buffer header ring */ typedef struct ODP_ALIGNED_CACHE { /* Ring header */ @@ -44,9 +46,10 @@ typedef struct ODP_ALIGNED_CACHE { odp_buffer_hdr_t *buf_hdr[CONFIG_POOL_MAX_NUM + 1]; /* Index to pointer look-up table for external memory pool */ - odp_buffer_hdr_t *buf_hdr_by_index[0]; + odp_buffer_hdr_t *buf_hdr_by_index[]; } pool_ring_t; +#pragma GCC diagnostic pop /* Callback function for pool destroy */ typedef void (*pool_destroy_cb_fn)(void *pool); diff --git a/platform/linux-generic/include/odp_ring_common.h b/platform/linux-generic/include/odp_ring_common.h index 88e6bf880..a2d9e4be5 100644 --- a/platform/linux-generic/include/odp_ring_common.h +++ b/platform/linux-generic/include/odp_ring_common.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2019, Nokia +/* Copyright (c) 2019-2021, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -12,7 +12,8 @@ extern "C" { #endif #define _ODP_RING_TYPE_U32 1 -#define _ODP_RING_TYPE_PTR 2 +#define _ODP_RING_TYPE_U64 2 +#define _ODP_RING_TYPE_PTR 3 #ifdef __cplusplus } diff --git a/platform/linux-generic/include/odp_ring_internal.h b/platform/linux-generic/include/odp_ring_internal.h index 6ac6d0ee3..d11e81bf2 100644 --- a/platform/linux-generic/include/odp_ring_internal.h +++ b/platform/linux-generic/include/odp_ring_internal.h @@ -1,5 +1,5 @@ /* Copyright (c) 2016-2018, Linaro Limited - * Copyright (c) 2019, Nokia + * Copyright (c) 2019-2021, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -45,12 +45,17 @@ struct ring_common { typedef struct ODP_ALIGNED_CACHE { struct ring_common r; - uint32_t data[0]; + uint32_t data[]; } ring_u32_t; typedef struct ODP_ALIGNED_CACHE { struct ring_common r; - void *data[0]; + uint64_t data[]; +} ring_u64_t; + +typedef struct ODP_ALIGNED_CACHE { + struct ring_common r; + void *data[]; } ring_ptr_t; /* 32-bit CAS with memory order selection */ @@ -87,6 +92,16 @@ static inline int cas_mo_u32(odp_atomic_u32_t *atom, uint32_t *old_val, #define _RING_ENQ ring_u32_enq #define _RING_ENQ_MULTI ring_u32_enq_multi #define _RING_LEN ring_u32_len +#elif _ODP_RING_TYPE == _ODP_RING_TYPE_U64 + #define _ring_gen_t ring_u64_t + #define _ring_data_t uint64_t + + #define _RING_INIT ring_u64_init + #define _RING_DEQ ring_u64_deq + #define _RING_DEQ_MULTI ring_u64_deq_multi + #define _RING_ENQ ring_u64_enq + #define _RING_ENQ_MULTI ring_u64_enq_multi + #define _RING_LEN ring_u64_len #elif _ODP_RING_TYPE == _ODP_RING_TYPE_PTR #define _ring_gen_t ring_ptr_t #define _ring_data_t void * diff --git a/platform/linux-generic/include/odp_ring_u64_internal.h b/platform/linux-generic/include/odp_ring_u64_internal.h new file mode 100644 index 000000000..4b4c7b1b5 --- /dev/null +++ b/platform/linux-generic/include/odp_ring_u64_internal.h @@ -0,0 +1,25 @@ +/* Copyright (c) 2021, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_RING_U64_INTERNAL_H_ +#define ODP_RING_U64_INTERNAL_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <odp_ring_common.h> + +#undef _ODP_RING_TYPE +#define _ODP_RING_TYPE _ODP_RING_TYPE_U64 + +#include <odp_ring_internal.h> + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/include/odp_timer_internal.h b/platform/linux-generic/include/odp_timer_internal.h index 23fd54bf9..435fa8b70 100644 --- a/platform/linux-generic/include/odp_timer_internal.h +++ b/platform/linux-generic/include/odp_timer_internal.h @@ -20,6 +20,8 @@ #include <odp/api/timer.h> #include <odp_global_data.h> +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wpedantic" /** * Internal Timeout header */ @@ -37,6 +39,7 @@ typedef struct { odp_timer_t timer; } odp_timeout_hdr_t; +#pragma GCC diagnostic pop /* A larger decrement value should be used after receiving events compared to * an 'empty' call. */ diff --git a/platform/linux-generic/include/protocols/ipsec.h b/platform/linux-generic/include/protocols/ipsec.h index 7838076d4..eabd2d1d9 100644 --- a/platform/linux-generic/include/protocols/ipsec.h +++ b/platform/linux-generic/include/protocols/ipsec.h @@ -34,7 +34,7 @@ extern "C" { typedef struct ODP_PACKED { odp_u32be_t spi; /**< Security Parameter Index */ odp_u32be_t seq_no; /**< Sequence Number */ - uint8_t iv[0]; /**< Initialization vector */ + uint8_t iv[]; /**< Initialization vector */ } _odp_esphdr_t; /** @internal Compile time assert */ @@ -47,7 +47,7 @@ ODP_STATIC_ASSERT(sizeof(_odp_esphdr_t) == _ODP_ESPHDR_LEN, typedef struct ODP_PACKED { uint8_t pad_len; /**< Padding length (0-255) */ uint8_t next_header; /**< Next header protocol */ - uint8_t icv[0]; /**< Integrity Check Value (optional) */ + uint8_t icv[]; /**< Integrity Check Value (optional) */ } _odp_esptrl_t; /** @internal Compile time assert */ @@ -63,7 +63,7 @@ typedef struct ODP_PACKED { odp_u16be_t pad; /**< Padding (must be 0) */ odp_u32be_t spi; /**< Security Parameter Index */ odp_u32be_t seq_no; /**< Sequence Number */ - uint8_t icv[0]; /**< Integrity Check Value */ + uint8_t icv[]; /**< Integrity Check Value */ } _odp_ahhdr_t; /** @internal Compile time assert */ diff --git a/platform/linux-generic/m4/configure.m4 b/platform/linux-generic/m4/configure.m4 index 01593baea..ffac70414 100644 --- a/platform/linux-generic/m4/configure.m4 +++ b/platform/linux-generic/m4/configure.m4 @@ -35,6 +35,11 @@ AS_VAR_APPEND([PLAT_CFG_TEXT], [" pcapng: ${have_pcapng} default_config_path: ${default_config_path}"]) +# Ignore Clang specific errors about fields with variable sized type not at the +# end of a struct. This style is used by e.g. odp_packet_hdr_t and +# odp_timeout_hdr_t. +ODP_CHECK_CFLAG([-Wno-error=gnu-variable-sized-type-not-at-end]) + AC_CONFIG_COMMANDS_PRE([dnl AM_CONDITIONAL([PLATFORM_IS_LINUX_GENERIC], [test "${with_platform}" = "linux-generic"]) diff --git a/platform/linux-generic/odp_buffer.c b/platform/linux-generic/odp_buffer.c index 4f06ce2f0..fed113923 100644 --- a/platform/linux-generic/odp_buffer.c +++ b/platform/linux-generic/odp_buffer.c @@ -52,7 +52,7 @@ void odp_buffer_print(odp_buffer_t buf) len += snprintf(&str[len], n - len, "Buffer\n------\n"); len += snprintf(&str[len], n - len, " pool index %u\n", hdr->index.pool); len += snprintf(&str[len], n - len, " buffer index %u\n", hdr->index.buffer); - len += snprintf(&str[len], n - len, " addr %p\n", hdr->base_data); + len += snprintf(&str[len], n - len, " addr %p\n", (void *)hdr->base_data); len += snprintf(&str[len], n - len, " size %u\n", odp_buffer_size(buf)); str[len] = 0; diff --git a/platform/linux-generic/odp_comp.c b/platform/linux-generic/odp_comp.c index 579bdc03b..5c286caed 100644 --- a/platform/linux-generic/odp_comp.c +++ b/platform/linux-generic/odp_comp.c @@ -134,8 +134,8 @@ static void process_input(odp_packet_t pkt_out, out_data = odp_packet_offset(pkt_out, start, &out_len, &cur_seg); ODP_DBG("out_data %p seg_data_ptr %p out_len %d seg %p\n", - out_data, odp_packet_seg_data(pkt_out, cur_seg), - out_len, cur_seg); + (void *)out_data, odp_packet_seg_data(pkt_out, cur_seg), + out_len, (void *)cur_seg); if (0 == out_len) { /* there are no more segments */ @@ -156,8 +156,8 @@ static void process_input(odp_packet_t pkt_out, ODP_DBG("next_in %p, avail_in %d next_out %p" " avail_out %d, sync %d\n", - streamp->next_in, streamp->avail_in, - streamp->next_out, + (const void *)streamp->next_in, streamp->avail_in, + (void *)streamp->next_out, streamp->avail_out, sync); @@ -259,7 +259,7 @@ static int deflate_comp(odp_packet_t pkt_in, &in_len, &in_seg); ODP_DBG("data %p in_len %d seg %p len %d\n", - data, in_len, in_seg, len); + (void *)data, in_len, (void *)in_seg, len); if (in_len > len) in_len = len; diff --git a/platform/linux-generic/odp_crypto_null.c b/platform/linux-generic/odp_crypto_null.c index aa8e8a2b3..8bd3433f0 100644 --- a/platform/linux-generic/odp_crypto_null.c +++ b/platform/linux-generic/odp_crypto_null.c @@ -77,7 +77,7 @@ struct odp_crypto_global_s { /* These flags are cleared at alloc_session() */ uint8_t ctx_valid[ODP_THREAD_COUNT_MAX][MAX_SESSIONS]; - odp_ticketlock_t openssl_lock[0]; + odp_ticketlock_t openssl_lock[]; }; static odp_crypto_global_t *global; diff --git a/platform/linux-generic/odp_crypto_openssl.c b/platform/linux-generic/odp_crypto_openssl.c index d8276b4b4..6e702acab 100644 --- a/platform/linux-generic/odp_crypto_openssl.c +++ b/platform/linux-generic/odp_crypto_openssl.c @@ -245,7 +245,7 @@ struct odp_crypto_global_s { /* These flags are cleared at alloc_session() */ uint8_t ctx_valid[ODP_THREAD_COUNT_MAX][MAX_SESSIONS]; - odp_ticketlock_t openssl_lock[0]; + odp_ticketlock_t openssl_lock[]; }; static odp_crypto_global_t *global; diff --git a/platform/linux-generic/odp_fdserver.c b/platform/linux-generic/odp_fdserver.c index 3103ddd5c..8bb12b64a 100644 --- a/platform/linux-generic/odp_fdserver.c +++ b/platform/linux-generic/odp_fdserver.c @@ -68,13 +68,8 @@ #define MAP_ANONYMOUS MAP_ANON #endif -#define FD_ODP_DEBUG_PRINT 0 - -#define FD_ODP_DBG(fmt, ...) \ - do { \ - if (FD_ODP_DEBUG_PRINT == 1) \ - ODP_DBG(fmt, ##__VA_ARGS__);\ - } while (0) +/* Debug level for the FD server */ +#define FD_DBG 3 /* define the tables of file descriptors handled by this server: */ #define FDSERVER_MAX_ENTRIES 256 @@ -287,8 +282,8 @@ int _odp_fdserver_register_fd(fd_server_context_e context, uint64_t key, int command; int fd; - FD_ODP_DBG("FD client register: pid=%d key=%" PRIu64 ", fd=%d\n", - getpid(), key, fd_to_send); + ODP_DBG_LVL(FD_DBG, "FD client register: pid=%d key=%" PRIu64 ", fd=%d\n", + getpid(), key, fd_to_send); s_sock = get_socket(); if (s_sock < 0) @@ -326,8 +321,8 @@ int _odp_fdserver_deregister_fd(fd_server_context_e context, uint64_t key) int command; int fd; - FD_ODP_DBG("FD client deregister: pid=%d key=%" PRIu64 "\n", - getpid(), key); + ODP_DBG_LVL(FD_DBG, "FD client deregister: pid=%d key=%" PRIu64 "\n", + getpid(), key); s_sock = get_socket(); if (s_sock < 0) @@ -399,7 +394,7 @@ static int stop_server(void) int s_sock; /* server socket */ int res; - FD_ODP_DBG("FD sending server stop request\n"); + ODP_DBG_LVL(FD_DBG, "FD sending server stop request\n"); s_sock = get_socket(); if (s_sock < 0) @@ -446,8 +441,8 @@ static int handle_request(int client_sock) fd_table[fd_table_nb_entries].context = context; fd_table[fd_table_nb_entries].key = key; fd_table[fd_table_nb_entries++].fd = fd; - FD_ODP_DBG("storing {ctx=%d, key=%" PRIu64 "}->fd=%d\n", - context, key, fd); + ODP_DBG_LVL(FD_DBG, "storing {ctx=%d, key=%" PRIu64 "}->fd=%d\n", + context, key, fd); } else { ODP_ERR("FD table full\n"); send_fdserver_msg(client_sock, FD_REGISTER_NACK, @@ -499,8 +494,8 @@ static int handle_request(int client_sock) for (i = 0; i < fd_table_nb_entries; i++) { if ((fd_table[i].context == context) && (fd_table[i].key == key)) { - FD_ODP_DBG("drop {ctx=%d," - " key=%" PRIu64 "}->fd=%d\n", + ODP_DBG_LVL(FD_DBG, "drop {ctx=%d," + " key=%" PRIu64 "}->fd=%d\n", context, key, fd_table[i].fd); close(fd_table[i].fd); fd_table[i] = fd_table[--fd_table_nb_entries]; @@ -517,7 +512,7 @@ static int handle_request(int client_sock) break; case FD_SERVERSTOP_REQ: - FD_ODP_DBG("Stopping FD server\n"); + ODP_DBG_LVL(FD_DBG, "Stopping FD server\n"); return 1; default: diff --git a/platform/linux-generic/odp_init.c b/platform/linux-generic/odp_init.c index d5530c1f1..18646dc08 100644 --- a/platform/linux-generic/odp_init.c +++ b/platform/linux-generic/odp_init.c @@ -630,3 +630,10 @@ void odp_log_thread_fn_set(odp_log_func_t func) { _odp_this_thread->log_fn = func; } + +int odp_instance(odp_instance_t *instance) +{ + *instance = (odp_instance_t)odp_global_ro.main_pid; + + return 0; +} diff --git a/platform/linux-generic/odp_ipsec.c b/platform/linux-generic/odp_ipsec.c index 09596b502..3bd524c3c 100644 --- a/platform/linux-generic/odp_ipsec.c +++ b/platform/linux-generic/odp_ipsec.c @@ -357,7 +357,7 @@ typedef struct { struct { uint16_t hdr_len; uint16_t trl_len; - odp_u32be_t seq_no; + uint64_t seq_no; } in; odp_u32be_t ipv4_addr; uint8_t ipv6_addr[_ODP_IPV6ADDR_LEN]; @@ -379,6 +379,40 @@ typedef struct { uint8_t iv[IPSEC_MAX_IV_LEN]; } ipsec_state_t; +/* + * Computes 64-bit seq number according to RFC4303 A2 + */ +static inline uint64_t ipsec_compute_esn(ipsec_sa_t *ipsec_sa, uint32_t seq) +{ + uint32_t wintop_h, wintop_l, winbot_l, ws; + uint64_t seq64 = 0, wintop = 0; + + wintop = odp_atomic_load_u64(&ipsec_sa->hot.in.wintop_seq); + wintop_l = wintop & 0xffffffff; + wintop_h = wintop >> 32; + + ws = ipsec_sa->in.ar.win_size; + winbot_l = wintop_l - ws + 1; + + /* case A: window is within one sequence number subspace */ + if (wintop_l >= (ws - 1)) { + if (seq < winbot_l) + wintop_h++; + /* case B: window spans two sequence number subspaces */ + } else { + if (seq >= winbot_l) + wintop_h--; + } + + seq64 = ((uint64_t)wintop_h << 32) | seq; + return seq64; +} + +static inline uint32_t ipsec_get_seqh_len(ipsec_sa_t *ipsec_sa) +{ + return ipsec_sa->insert_seq_hi * IPSEC_SEQ_HI_LEN; +} + static int ipsec_parse_ipv4(ipsec_state_t *state, odp_packet_t pkt) { _odp_ipv4hdr_t ipv4hdr; @@ -569,17 +603,41 @@ static int ipsec_in_esp(odp_packet_t *pkt, param->auth_iv_ptr = state->iv; state->esp.aad.spi = esp.spi; - state->esp.aad.seq_no = esp.seq_no; state->in.seq_no = odp_be_to_cpu_32(esp.seq_no); + if (ipsec_sa->esn) { + state->in.seq_no = ipsec_compute_esn(ipsec_sa, state->in.seq_no); + state->esp.aad.seq_no64 = odp_cpu_to_be_64(state->in.seq_no); + } else { + state->esp.aad.seq_no = esp.seq_no; + } param->aad_ptr = (uint8_t *)&state->esp.aad; + /* Insert high-order bits of ESN before the ICV for ICV check + * with non-combined mode algorithms. + */ + if (ipsec_sa->insert_seq_hi) { + uint32_t inb_seqh = odp_cpu_to_be_32(state->in.seq_no >> 32); + uint32_t icv_offset = odp_packet_len(*pkt) - ipsec_sa->icv_len; + + if (odp_packet_extend_tail(pkt, IPSEC_SEQ_HI_LEN, NULL, NULL) < 0) { + status->error.alg = 1; + ODP_ERR("odp_packet_extend_tail failed\n"); + return -1; + } + odp_packet_move_data(*pkt, icv_offset + IPSEC_SEQ_HI_LEN, icv_offset, + ipsec_sa->icv_len); + odp_packet_copy_from_mem(*pkt, icv_offset, IPSEC_SEQ_HI_LEN, &inb_seqh); + } + param->auth_range.offset = ipsec_offset; param->auth_range.length = state->ip_tot_len - - state->ip_hdr_len - + state->ip_hdr_len + + ipsec_get_seqh_len(ipsec_sa) - ipsec_sa->icv_len; param->hash_result_offset = state->ip_offset + - state->ip_tot_len - + state->ip_tot_len + + ipsec_get_seqh_len(ipsec_sa) - ipsec_sa->icv_len; state->stats_length = param->cipher_range.length; @@ -681,6 +739,23 @@ static int ipsec_in_ah(odp_packet_t *pkt, } state->in.seq_no = odp_be_to_cpu_32(ah.seq_no); + if (ipsec_sa->esn) + state->in.seq_no = ipsec_compute_esn(ipsec_sa, state->in.seq_no); + + /* ESN higher 32 bits are included at the end of the packet data + * for inbound ICV computation. + */ + if (ipsec_sa->insert_seq_hi) { + uint32_t inb_seqh = odp_cpu_to_be_32(state->in.seq_no >> 32); + uint32_t seqh_offset = odp_packet_len(*pkt); + + if (odp_packet_extend_tail(pkt, IPSEC_SEQ_HI_LEN, NULL, NULL) < 0) { + status->error.alg = 1; + ODP_ERR("odp_packet_extend_tail failed\n"); + return -1; + } + odp_packet_copy_from_mem(*pkt, seqh_offset, IPSEC_SEQ_HI_LEN, &inb_seqh); + } param->auth_range.offset = state->ip_offset; param->auth_range.length = state->ip_tot_len; @@ -688,6 +763,7 @@ static int ipsec_in_ah(odp_packet_t *pkt, ipsec_sa->esp_iv_len; state->stats_length = param->auth_range.length; + param->auth_range.length += ipsec_get_seqh_len(ipsec_sa); return 0; } @@ -728,31 +804,35 @@ static int ipsec_in_ah_post(odp_packet_t pkt, static void ipsec_sa_err_stats_update(ipsec_sa_t *sa, odp_ipsec_op_status_t *status) { + odp_ipsec_op_status_t err_status; + if (odp_likely(ODP_IPSEC_OK == status->error.all)) return; if (NULL == sa) return; - if (status->error.proto) + err_status = *status; + + if (err_status.error.proto) odp_atomic_inc_u64(&sa->stats.proto_err); - if (status->error.auth) + if (err_status.error.auth) odp_atomic_inc_u64(&sa->stats.auth_err); - if (status->error.antireplay) + if (err_status.error.antireplay) odp_atomic_inc_u64(&sa->stats.antireplay_err); - if (status->error.alg) + if (err_status.error.alg) odp_atomic_inc_u64(&sa->stats.alg_err); - if (status->error.mtu) + if (err_status.error.mtu) odp_atomic_inc_u64(&sa->stats.mtu_err); - if (status->error.hard_exp_bytes) + if (err_status.error.hard_exp_bytes) odp_atomic_inc_u64(&sa->stats.hard_exp_bytes_err); - if (status->error.hard_exp_packets) + if (err_status.error.hard_exp_packets) odp_atomic_inc_u64(&sa->stats.hard_exp_pkts_err); } @@ -876,7 +956,8 @@ static ipsec_sa_t *ipsec_in_single(odp_packet_t pkt, goto post_lifetime_err_cnt_update; } - if (odp_packet_trunc_tail(&pkt, state.in.trl_len, NULL, NULL) < 0) { + if (odp_packet_trunc_tail(&pkt, state.in.trl_len + ipsec_get_seqh_len(ipsec_sa), + NULL, NULL) < 0) { status->error.alg = 1; goto post_lifetime_err_cnt_update; } @@ -1280,10 +1361,13 @@ static int ipsec_out_esp(odp_packet_t *pkt, memset(&esp, 0, sizeof(esp)); esp.spi = odp_cpu_to_be_32(ipsec_sa->spi); + state->esp.aad.spi = esp.spi; esp.seq_no = odp_cpu_to_be_32(seq_no & 0xffffffff); - state->esp.aad.spi = esp.spi; - state->esp.aad.seq_no = esp.seq_no; + if (ipsec_sa->esn) + state->esp.aad.seq_no64 = odp_cpu_to_be_64(seq_no); + else + state->esp.aad.seq_no = esp.seq_no; param->aad_ptr = (uint8_t *)&state->esp.aad; @@ -1359,6 +1443,22 @@ static int ipsec_out_esp(odp_packet_t *pkt, esptrl_offset, _ODP_ESPTRL_LEN, &esptrl); + /* Outbound ICV computation includes ESN higher 32 bits as part of ESP + * implicit trailer for individual algo's. + */ + if (ipsec_sa->insert_seq_hi) { + uint32_t outb_seqh = odp_cpu_to_be_32(seq_no >> 32); + + if (odp_packet_extend_tail(pkt, IPSEC_SEQ_HI_LEN, NULL, NULL) < 0) { + status->error.alg = 1; + ODP_ERR("odp_packet_extend_tail failed\n"); + return -1; + } + odp_packet_copy_from_mem(*pkt, + esptrl_offset + _ODP_ESPTRL_LEN, + IPSEC_SEQ_HI_LEN, &outb_seqh); + } + if (odp_unlikely(state->ip_tot_len < state->ip_hdr_len + hdr_len + ipsec_sa->icv_len)) { status->error.proto = 1; @@ -1373,10 +1473,12 @@ static int ipsec_out_esp(odp_packet_t *pkt, param->auth_range.offset = ipsec_offset; param->auth_range.length = state->ip_tot_len - - state->ip_hdr_len - + state->ip_hdr_len + + ipsec_get_seqh_len(ipsec_sa) - ipsec_sa->icv_len; param->hash_result_offset = state->ip_offset + - state->ip_tot_len - + state->ip_tot_len + + ipsec_get_seqh_len(ipsec_sa) - ipsec_sa->icv_len; state->stats_length = param->cipher_range.length; @@ -1384,10 +1486,27 @@ static int ipsec_out_esp(odp_packet_t *pkt, return 0; } -static void ipsec_out_esp_post(ipsec_state_t *state, odp_packet_t pkt) +static int ipsec_out_esp_post(ipsec_state_t *state, odp_packet_t *pkt, + ipsec_sa_t *ipsec_sa) { if (state->is_ipv4) - _odp_packet_ipv4_chksum_insert(pkt); + _odp_packet_ipv4_chksum_insert(*pkt); + + /* Remove the high order ESN bits that were added in the packet for ICV + * computation. + */ + if (ipsec_sa->insert_seq_hi) { + uint32_t icv_offset = odp_packet_len(*pkt) - ipsec_sa->icv_len; + + odp_packet_move_data(*pkt, icv_offset - IPSEC_SEQ_HI_LEN, icv_offset, + ipsec_sa->icv_len); + if (odp_packet_trunc_tail(pkt, IPSEC_SEQ_HI_LEN, NULL, NULL) < 0) { + ODP_ERR("odp_packet_trunc_tail failed\n"); + return -1; + } + } + + return 0; } static int ipsec_out_ah(odp_packet_t *pkt, @@ -1479,32 +1598,62 @@ static int ipsec_out_ah(odp_packet_t *pkt, 0, hdr_len - _ODP_AHHDR_LEN - ipsec_sa->esp_iv_len); + /* ESN higher 32 bits are included at the end of the packet data + * for outbound ICV computation. + */ + if (ipsec_sa->insert_seq_hi) { + uint32_t outb_seqh = odp_cpu_to_be_32(seq_no >> 32); + uint32_t seqh_offset = odp_packet_len(*pkt); + + if (odp_packet_extend_tail(pkt, IPSEC_SEQ_HI_LEN, NULL, NULL) < 0) { + status->error.alg = 1; + ODP_ERR("odp_packet_extend_tail failed\n"); + return -1; + } + odp_packet_copy_from_mem(*pkt, + seqh_offset, IPSEC_SEQ_HI_LEN, &outb_seqh); + } + param->auth_range.offset = state->ip_offset; param->auth_range.length = state->ip_tot_len; param->hash_result_offset = ipsec_offset + _ODP_AHHDR_LEN + ipsec_sa->esp_iv_len; state->stats_length = param->auth_range.length; + param->auth_range.length += ipsec_get_seqh_len(ipsec_sa); return 0; } -static void ipsec_out_ah_post(ipsec_state_t *state, odp_packet_t pkt) +static int ipsec_out_ah_post(ipsec_state_t *state, odp_packet_t *pkt, + ipsec_sa_t *ipsec_sa) { if (state->is_ipv4) { - _odp_ipv4hdr_t *ipv4hdr = odp_packet_l3_ptr(pkt, NULL); + _odp_ipv4hdr_t *ipv4hdr = odp_packet_l3_ptr(*pkt, NULL); ipv4hdr->ttl = state->ah_ipv4.ttl; ipv4hdr->tos = state->ah_ipv4.tos; ipv4hdr->frag_offset = state->ah_ipv4.frag_offset; - _odp_packet_ipv4_chksum_insert(pkt); + _odp_packet_ipv4_chksum_insert(*pkt); } else { - _odp_ipv6hdr_t *ipv6hdr = odp_packet_l3_ptr(pkt, NULL); + _odp_ipv6hdr_t *ipv6hdr = odp_packet_l3_ptr(*pkt, NULL); ipv6hdr->ver_tc_flow = state->ah_ipv6.ver_tc_flow; ipv6hdr->hop_limit = state->ah_ipv6.hop_limit; } + + /* Remove the high order ESN bits that were added in the packet for ICV + * computation. + */ + if (ipsec_sa->insert_seq_hi) { + if (odp_packet_trunc_tail(pkt, IPSEC_SEQ_HI_LEN, NULL, NULL) < 0) { + ODP_ERR("odp_packet_trunc_tail failed\n"); + return -1; + } + } + + return 0; } #define OL_TX_CHKSUM_PKT(_cfg, _proto, _ovr_set, _ovr) \ @@ -1714,9 +1863,14 @@ static ipsec_sa_t *ipsec_out_single(odp_packet_t pkt, /* Finalize the IP header */ if (ODP_IPSEC_ESP == ipsec_sa->proto) - ipsec_out_esp_post(&state, pkt); + rc = ipsec_out_esp_post(&state, &pkt, ipsec_sa); else if (ODP_IPSEC_AH == ipsec_sa->proto) - ipsec_out_ah_post(&state, pkt); + rc = ipsec_out_ah_post(&state, &pkt, ipsec_sa); + + if (rc < 0) { + status->error.alg = 1; + goto post_lifetime_err_cnt_update; + } goto exit; diff --git a/platform/linux-generic/odp_ipsec_events.c b/platform/linux-generic/odp_ipsec_events.c index bb786614a..a199ffdf3 100644 --- a/platform/linux-generic/odp_ipsec_events.c +++ b/platform/linux-generic/odp_ipsec_events.c @@ -18,12 +18,15 @@ #include <odp/api/plat/event_inlines.h> #include <odp/api/plat/queue_inlines.h> +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wpedantic" typedef struct { /* common buffer header */ odp_buffer_hdr_t buf_hdr; odp_ipsec_status_t status; } ipsec_status_hdr_t; +#pragma GCC diagnostic pop static odp_pool_t ipsec_status_pool = ODP_POOL_INVALID; diff --git a/platform/linux-generic/odp_ipsec_sad.c b/platform/linux-generic/odp_ipsec_sad.c index 0eea57a10..756370516 100644 --- a/platform/linux-generic/odp_ipsec_sad.c +++ b/platform/linux-generic/odp_ipsec_sad.c @@ -503,10 +503,8 @@ odp_ipsec_sa_t odp_ipsec_sa_create(const odp_ipsec_sa_param_t *param) } ipsec_sa->mode = param->mode; ipsec_sa->flags = 0; - if (param->opt.esn) { - ODP_ERR("ESN is not supported!\n"); - return ODP_IPSEC_SA_INVALID; - } + ipsec_sa->esn = param->opt.esn; + if (ODP_IPSEC_DIR_INBOUND == param->dir) { ipsec_sa->lookup_mode = param->inbound.lookup_mode; if (ODP_IPSEC_LOOKUP_DSTADDR_SPI == ipsec_sa->lookup_mode) { @@ -640,6 +638,11 @@ odp_ipsec_sa_t odp_ipsec_sa_create(const odp_ipsec_sa_param_t *param) goto error; ipsec_sa->salt_length = 0; + /* ESN higher 32 bits flag. + * This flag is set for individual algo's. + * This flag is reset for combined mode algo's and ODP_AUTH_ALG_NULL. + */ + ipsec_sa->insert_seq_hi = (ipsec_sa->esn) ? 1 : 0; switch (crypto_param.cipher_alg) { case ODP_CIPHER_ALG_NULL: @@ -700,10 +703,17 @@ odp_ipsec_sa_t odp_ipsec_sa_create(const odp_ipsec_sa_param_t *param) case ODP_AUTH_ALG_AES128_GCM: #endif case ODP_AUTH_ALG_AES_GCM: - crypto_param.auth_aad_len = sizeof(ipsec_aad_t); + case ODP_AUTH_ALG_AES_CCM: + if (ipsec_sa->esn) { + crypto_param.auth_aad_len = 12; + ipsec_sa->insert_seq_hi = 0; + } else { + crypto_param.auth_aad_len = 8; + } break; case ODP_AUTH_ALG_AES_GMAC: - if (ODP_CIPHER_ALG_NULL != crypto_param.cipher_alg) + if ((ODP_CIPHER_ALG_NULL != crypto_param.cipher_alg) || + ipsec_sa->esn) goto error; ipsec_sa->use_counter_iv = 1; ipsec_sa->esp_iv_len = 8; @@ -713,7 +723,12 @@ odp_ipsec_sa_t odp_ipsec_sa_create(const odp_ipsec_sa_param_t *param) salt_param = ¶m->crypto.auth_key_extra; break; case ODP_AUTH_ALG_CHACHA20_POLY1305: - crypto_param.auth_aad_len = sizeof(ipsec_aad_t); + if (ipsec_sa->esn) { + crypto_param.auth_aad_len = 12; + ipsec_sa->insert_seq_hi = 0; + } else { + crypto_param.auth_aad_len = 8; + } break; default: break; @@ -721,6 +736,10 @@ odp_ipsec_sa_t odp_ipsec_sa_create(const odp_ipsec_sa_param_t *param) ipsec_sa->icv_len = crypto_param.auth_digest_len; + /* For ODP_AUTH_ALG_NULL */ + if (!ipsec_sa->icv_len) + ipsec_sa->insert_seq_hi = 0; + if (ipsec_sa->salt_length) { if (ipsec_sa->salt_length > IPSEC_MAX_SALT_LEN) { ODP_ERR("IPSEC_MAX_SALT_LEN too small\n"); @@ -947,30 +966,37 @@ static uint64_t ipsec_sa_antireplay_max_seq(ipsec_sa_t *ipsec_sa) { uint64_t max_seq = 0; - max_seq = odp_atomic_load_u64(&ipsec_sa->hot.in.wintop_seq) & 0xffffffff; + max_seq = odp_atomic_load_u64(&ipsec_sa->hot.in.wintop_seq); + if (!ipsec_sa->esn) + max_seq &= 0xffffffff; return max_seq; } -int _odp_ipsec_sa_replay_precheck(ipsec_sa_t *ipsec_sa, uint32_t seq, +int _odp_ipsec_sa_replay_precheck(ipsec_sa_t *ipsec_sa, uint64_t seq, odp_ipsec_op_status_t *status) { /* Try to be as quick as possible, we will discard packets later */ - if (ipsec_sa->antireplay && ((seq + ipsec_sa->in.ar.win_size) <= - (odp_atomic_load_u64(&ipsec_sa->hot.in.wintop_seq) & 0xffffffff))) { - status->error.antireplay = 1; - return -1; - } + if (ipsec_sa->antireplay) { + uint64_t wintop = odp_atomic_load_u64(&ipsec_sa->hot.in.wintop_seq); + + if (!ipsec_sa->esn) + wintop &= 0xffffffff; + if ((seq + ipsec_sa->in.ar.win_size) <= wintop) { + status->error.antireplay = 1; + return -1; + } + } return 0; } -static inline int ipsec_wslarge_replay_update(ipsec_sa_t *ipsec_sa, uint32_t seq, +static inline int ipsec_wslarge_replay_update(ipsec_sa_t *ipsec_sa, uint64_t seq, odp_ipsec_op_status_t *status) { - uint32_t bucket, wintop_bucket, new_bucket; - uint32_t bkt_diff, bkt_cnt, top_seq; - uint64_t bit = 0; + uint64_t bucket, wintop_bucket, new_bucket; + uint64_t bkt_diff, bkt_cnt; + uint64_t bit = 0, top_seq; odp_spinlock_lock(&ipsec_sa->hot.in.lock); @@ -1052,14 +1078,14 @@ static inline int ipsec_ws32_replay_update(ipsec_sa_t *ipsec_sa, uint32_t seq, return 0; } -int _odp_ipsec_sa_replay_update(ipsec_sa_t *ipsec_sa, uint32_t seq, +int _odp_ipsec_sa_replay_update(ipsec_sa_t *ipsec_sa, uint64_t seq, odp_ipsec_op_status_t *status) { int ret; /* Window update for ws equal to 32 */ - if (ipsec_sa->in.ar.win_size == IPSEC_AR_WIN_SIZE_MIN) - ret = ipsec_ws32_replay_update(ipsec_sa, seq, status); + if ((!ipsec_sa->esn) && (ipsec_sa->in.ar.win_size == IPSEC_AR_WIN_SIZE_MIN)) + ret = ipsec_ws32_replay_update(ipsec_sa, (seq & 0xffffffff), status); else ret = ipsec_wslarge_replay_update(ipsec_sa, seq, status); diff --git a/platform/linux-generic/odp_ishm.c b/platform/linux-generic/odp_ishm.c index ebc931658..a10b9d5df 100644 --- a/platform/linux-generic/odp_ishm.c +++ b/platform/linux-generic/odp_ishm.c @@ -1621,17 +1621,18 @@ int _odp_ishm_cleanup_files(const char *dirpath) char userdir[PATH_MAX]; char prefix[PATH_MAX]; char *fullpath; - int d_len = strlen(dirpath); + int d_len; int p_len; int f_len; snprintf(userdir, PATH_MAX, "%s/%s", dirpath, odp_global_ro.uid); + d_len = strlen(userdir); dir = opendir(userdir); if (!dir) { /* ok if the dir does not exist. no much to delete then! */ ODP_DBG("opendir failed for %s: %s\n", - dirpath, strerror(errno)); + userdir, strerror(errno)); return 0; } snprintf(prefix, PATH_MAX, _ODP_FILES_FMT, odp_global_ro.main_pid); @@ -1645,7 +1646,7 @@ int _odp_ishm_cleanup_files(const char *dirpath) return -1; } snprintf(fullpath, PATH_MAX, "%s/%s", - dirpath, e->d_name); + userdir, e->d_name); ODP_DBG("deleting obsolete file: %s\n", fullpath); if (unlink(fullpath)) ODP_ERR("unlink failed for %s: %s\n", diff --git a/platform/linux-generic/odp_name_table.c b/platform/linux-generic/odp_name_table.c index 9c1814cae..b9ba6e3fe 100644 --- a/platform/linux-generic/odp_name_table.c +++ b/platform/linux-generic/odp_name_table.c @@ -70,7 +70,7 @@ typedef struct ODP_ALIGNED_CACHE { uint32_t num_avail_to_add; uint32_t base_id; name_tbl_entry_t *free_list_head; - name_tbl_entry_t entries[0]; + name_tbl_entry_t entries[]; } name_tbl_t; typedef struct { diff --git a/platform/linux-generic/odp_packet.c b/platform/linux-generic/odp_packet.c index c6a50bf84..ed5d81952 100644 --- a/platform/linux-generic/odp_packet.c +++ b/platform/linux-generic/odp_packet.c @@ -63,13 +63,22 @@ const _odp_packet_inline_offset_t _odp_packet_inline ODP_ALIGNED_CACHE = { #include <odp/visibility_end.h> -/* Check that invalid values are the same. Some versions of Clang have trouble - * with the strong type casting, and complain that these invalid values are not - * integral constants. */ +/* Check that invalid values are the same. Some versions of Clang and pedantic + * build have trouble with the strong type casting, and complain that these + * invalid values are not integral constants. + * + * Invalid values are required to be equal for _odp_buffer_is_valid() to work + * properly. */ #ifndef __clang__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wpedantic" ODP_STATIC_ASSERT(ODP_PACKET_INVALID == 0, "Packet invalid not 0"); ODP_STATIC_ASSERT(ODP_BUFFER_INVALID == 0, "Buffer invalid not 0"); ODP_STATIC_ASSERT(ODP_EVENT_INVALID == 0, "Event invalid not 0"); +ODP_STATIC_ASSERT(ODP_PACKET_VECTOR_INVALID == 0, "Packet vector invalid not 0"); +ODP_STATIC_ASSERT(ODP_PACKET_TX_COMPL_INVALID == 0, "Packet TX completion invalid not 0"); +ODP_STATIC_ASSERT(ODP_TIMEOUT_INVALID == 0, "Timeout invalid not 0"); +#pragma GCC diagnostic pop #endif static inline odp_buffer_t packet_to_buffer(odp_packet_t pkt) @@ -2592,7 +2601,7 @@ static int packet_l4_chksum(odp_packet_hdr_t *pkt_hdr, } } - return pkt_hdr->p.flags.all_flags != 0; + return pkt_hdr->p.flags.all.error != 0; } /** diff --git a/platform/linux-generic/odp_packet_vector.c b/platform/linux-generic/odp_packet_vector.c index d97bb96a1..6f0ee201a 100644 --- a/platform/linux-generic/odp_packet_vector.c +++ b/platform/linux-generic/odp_packet_vector.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2020, Nokia +/* Copyright (c) 2020-2021, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -30,34 +30,33 @@ const _odp_event_vector_inline_offset_t _odp_event_vector_inline ODP_ALIGNED_CAC #include <odp/visibility_end.h> -static inline odp_event_vector_hdr_t *event_vector_hdr_from_buffer(odp_buffer_t buf) +odp_packet_vector_t odp_packet_vector_alloc(odp_pool_t pool_hdl) { - return (odp_event_vector_hdr_t *)(uintptr_t)buf; -} + odp_packet_vector_t pktv; + pool_t *pool; + int ret; -odp_packet_vector_t odp_packet_vector_alloc(odp_pool_t pool) -{ - odp_buffer_t buf; + ODP_ASSERT(pool_hdl != ODP_POOL_INVALID); + + pool = pool_entry_from_hdl(pool_hdl); - ODP_ASSERT(pool_entry_from_hdl(pool)->type == ODP_POOL_VECTOR); + ODP_ASSERT(pool->type == ODP_POOL_VECTOR); - buf = odp_buffer_alloc(pool); - if (odp_unlikely(buf == ODP_BUFFER_INVALID)) - return ODP_PACKET_VECTOR_INVALID; + ret = _odp_buffer_alloc_multi(pool, (odp_buffer_hdr_t **)&pktv, 1); - ODP_ASSERT(event_vector_hdr_from_buffer(buf)->size == 0); + if (odp_likely(ret == 1)) + return pktv; - return odp_packet_vector_from_event(odp_buffer_to_event(buf)); + return ODP_PACKET_VECTOR_INVALID; } void odp_packet_vector_free(odp_packet_vector_t pktv) { odp_event_vector_hdr_t *pktv_hdr = _odp_packet_vector_hdr(pktv); - odp_event_t ev = odp_packet_vector_to_event(pktv); pktv_hdr->size = 0; - odp_buffer_free(odp_buffer_from_event(ev)); + _odp_buffer_free_multi((odp_buffer_hdr_t **)&pktv_hdr, 1); } int odp_packet_vector_valid(odp_packet_vector_t pktv) @@ -103,7 +102,7 @@ void odp_packet_vector_print(odp_packet_vector_t pktv) len += snprintf(&str[len], n - len, "Packet Vector\n"); len += snprintf(&str[len], n - len, - " handle %p\n", pktv); + " handle %p\n", (void *)pktv); len += snprintf(&str[len], n - len, " size %" PRIu32 "\n", pktv_hdr->size); @@ -114,7 +113,7 @@ void odp_packet_vector_print(odp_packet_vector_t pktv) str_len = snprintf(seg_str, max_len, " packet %p len %" PRIu32 "\n", - pkt, odp_packet_len(pkt)); + (void *)pkt, odp_packet_len(pkt)); /* Prevent print buffer overflow */ if (n - len - str_len < 10) { diff --git a/platform/linux-generic/odp_pcapng.c b/platform/linux-generic/odp_pcapng.c index ce950760e..6bdd1fd69 100644 --- a/platform/linux-generic/odp_pcapng.c +++ b/platform/linux-generic/odp_pcapng.c @@ -577,4 +577,7 @@ int _odp_pcapng_write_pkts(pktio_entry_t *entry, int qidx, return len; } -#endif /* _ODP_PCAPNG */ +#else /* _ODP_PCAPNG */ +/* Avoid warning about empty translation unit */ +typedef int _odp_dummy; +#endif diff --git a/platform/linux-generic/odp_pkt_queue.c b/platform/linux-generic/odp_pkt_queue.c index 556e034f8..ad70f0e0a 100644 --- a/platform/linux-generic/odp_pkt_queue.c +++ b/platform/linux-generic/odp_pkt_queue.c @@ -26,19 +26,12 @@ typedef struct ODP_ALIGNED_CACHE { odp_packet_t pkts[NUM_PKTS]; } queue_blk_t; +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wpedantic" typedef struct ODP_ALIGNED_CACHE { queue_blk_t blks[0]; } queue_blks_t; - -/* The queue_num_tbl is used to map from a queue_num to a queue_num_desc. - * The reason is based on the assumption that usually only a small fraction - * of the max_num_queues will have more than 1 pkt associated with it. This - * way the active queue_desc's can be dynamically allocated and freed according - * to the actual usage pattern. - */ -typedef struct { - uint32_t queue_num_to_blk_idx[0]; -} queue_num_tbl_t; +#pragma GCC diagnostic pop typedef struct { uint32_t num_blks; diff --git a/platform/linux-generic/odp_pool.c b/platform/linux-generic/odp_pool.c index 9c60a9458..49d2e74f5 100644 --- a/platform/linux-generic/odp_pool.c +++ b/platform/linux-generic/odp_pool.c @@ -669,6 +669,7 @@ static odp_pool_t pool_create(const char *name, const odp_pool_param_t *params, return ODP_POOL_INVALID; } + num = params->pkt.num; seg_len = CONFIG_PACKET_MAX_SEG_LEN; max_len = _odp_pool_glb->config.pkt_max_len; @@ -691,9 +692,19 @@ static odp_pool_t pool_create(const char *name, const odp_pool_param_t *params, return ODP_POOL_INVALID; } + /* Multiple segments required per 'params->pkt.len' packet */ + if (params->pkt.len > seg_len) + num *= (params->pkt.len + seg_len - 1) / seg_len; + + /* Make sure 'params->pkt.max_num' limitation holds */ + if (params->pkt.max_num && num > params->pkt.max_num) { + ODP_ERR("Pool 'max_num' parameter too small (%u/%u)\n", + params->pkt.max_num, num); + return ODP_POOL_INVALID; + } + headroom = CONFIG_PACKET_HEADROOM; tailroom = CONFIG_PACKET_TAILROOM; - num = params->pkt.num; uarea_size = params->pkt.uarea_size; cache_size = params->pkt.cache_size; break; @@ -853,12 +864,15 @@ static int check_params(const odp_pool_param_t *params) uint32_t cache_size, num; int num_threads = odp_global_ro.init_param.num_control + odp_global_ro.init_param.num_worker; + int cur_threads = odp_thread_count(); if (!params || odp_pool_capability(&capa) < 0) return -1; num = 0; cache_size = 0; + if (num_threads < cur_threads) + num_threads = cur_threads; switch (params->type) { case ODP_POOL_BUFFER: @@ -1270,6 +1284,9 @@ odp_buffer_t odp_buffer_alloc(odp_pool_t pool_hdl) ODP_ASSERT(ODP_POOL_INVALID != pool_hdl); pool = pool_entry_from_hdl(pool_hdl); + + ODP_ASSERT(pool->type == ODP_POOL_BUFFER); + ret = _odp_buffer_alloc_multi(pool, (odp_buffer_hdr_t **)&buf, 1); if (odp_likely(ret == 1)) @@ -1286,6 +1303,8 @@ int odp_buffer_alloc_multi(odp_pool_t pool_hdl, odp_buffer_t buf[], int num) pool = pool_entry_from_hdl(pool_hdl); + ODP_ASSERT(pool->type == ODP_POOL_BUFFER); + return _odp_buffer_alloc_multi(pool, (odp_buffer_hdr_t **)buf, num); } @@ -1392,15 +1411,65 @@ void odp_pool_print(odp_pool_t pool_hdl) ODP_PRINT(" block size %u\n", pool->block_size); ODP_PRINT(" uarea size %u\n", pool->uarea_size); ODP_PRINT(" shm size %" PRIu64 "\n", pool->shm_size); - ODP_PRINT(" base addr %p\n", pool->base_addr); - ODP_PRINT(" max addr %p\n", pool->max_addr); + ODP_PRINT(" base addr %p\n", (void *)pool->base_addr); + ODP_PRINT(" max addr %p\n", (void *)pool->max_addr); ODP_PRINT(" uarea shm size %" PRIu64 "\n", pool->uarea_shm_size); - ODP_PRINT(" uarea base addr %p\n", pool->uarea_base_addr); + ODP_PRINT(" uarea base addr %p\n", (void *)pool->uarea_base_addr); ODP_PRINT(" cache size %u\n", pool->cache_size); ODP_PRINT(" burst size %u\n", pool->burst_size); ODP_PRINT("\n"); } +void odp_pool_print_all(void) +{ + uint64_t available; + uint32_t i, index, tot, cache_size, seg_len; + uint32_t buf_len = 0; + uint8_t type, ext; + const int col_width = 24; + const char *name; + char type_c; + + ODP_PRINT("\nList of all pools\n"); + ODP_PRINT("-----------------\n"); + ODP_PRINT(" idx %-*s type free tot cache buf_len ext\n", col_width, "name"); + + for (i = 0; i < ODP_CONFIG_POOLS; i++) { + pool_t *pool = pool_entry(i); + + LOCK(&pool->lock); + + if (!pool->reserved) { + UNLOCK(&pool->lock); + continue; + } + + available = ring_ptr_len(&pool->ring->hdr); + cache_size = pool->cache_size; + ext = pool->pool_ext; + index = pool->pool_idx; + name = pool->name; + tot = pool->num; + type = pool->type; + seg_len = pool->seg_len; + + UNLOCK(&pool->lock); + + if (type == ODP_POOL_BUFFER || type == ODP_POOL_PACKET) + buf_len = seg_len; + + type_c = (type == ODP_POOL_BUFFER) ? 'B' : + (type == ODP_POOL_PACKET) ? 'P' : + (type == ODP_POOL_TIMEOUT) ? 'T' : + (type == ODP_POOL_VECTOR) ? 'V' : '-'; + + ODP_PRINT("%4u %-*s %c %6" PRIu64 " %6" PRIu32 " %6" PRIu32 " %8" PRIu32 " " + "%" PRIu8 "\n", index, col_width, name, type_c, available, tot, + cache_size, buf_len, ext); + } + ODP_PRINT("\n"); +} + odp_pool_t odp_buffer_pool(odp_buffer_t buf) { pool_t *pool = pool_from_buf(buf); diff --git a/platform/linux-generic/odp_queue_basic.c b/platform/linux-generic/odp_queue_basic.c index e35f01f7d..fe4d90930 100644 --- a/platform/linux-generic/odp_queue_basic.c +++ b/platform/linux-generic/odp_queue_basic.c @@ -701,7 +701,7 @@ static void queue_print(odp_queue_t handle) } ODP_PRINT("\nQueue info\n"); ODP_PRINT("----------\n"); - ODP_PRINT(" handle %p\n", queue->s.handle); + ODP_PRINT(" handle %p\n", (void *)queue->s.handle); ODP_PRINT(" index %" PRIu32 "\n", queue_id); ODP_PRINT(" name %s\n", queue->s.name); ODP_PRINT(" enq mode %s\n", diff --git a/platform/linux-generic/odp_queue_lf.c b/platform/linux-generic/odp_queue_lf.c index 206172bb8..f1b265970 100644 --- a/platform/linux-generic/odp_queue_lf.c +++ b/platform/linux-generic/odp_queue_lf.c @@ -21,7 +21,7 @@ #ifdef __SIZEOF_INT128__ -typedef unsigned __int128 u128_t; +__extension__ typedef unsigned __int128 u128_t; static inline void lockfree_zero_u128(u128_t *atomic) { diff --git a/platform/linux-generic/odp_queue_scalable.c b/platform/linux-generic/odp_queue_scalable.c index ad7807bf9..916cb6739 100644 --- a/platform/linux-generic/odp_queue_scalable.c +++ b/platform/linux-generic/odp_queue_scalable.c @@ -29,17 +29,11 @@ #include <odp_ishmpool_internal.h> #include <odp/api/plat/queue_inline_types.h> #include <odp_global_data.h> +#include <odp_macros_internal.h> #include <string.h> #include <inttypes.h> -#define MIN(a, b) \ - ({ \ - __typeof__(a) tmp_a = (a); \ - __typeof__(b) tmp_b = (b); \ - tmp_a < tmp_b ? tmp_a : tmp_b; \ - }) - #define LOCK(a) odp_ticketlock_lock(a) #define UNLOCK(a) odp_ticketlock_unlock(a) #define LOCK_INIT(a) odp_ticketlock_init(a) @@ -745,7 +739,7 @@ int _odp_queue_deq_sc(sched_elem_t *q, odp_event_t *evp, int num) return actual; } -inline int _odp_queue_deq(sched_elem_t *q, odp_buffer_hdr_t *buf_hdr[], int num) +int _odp_queue_deq(sched_elem_t *q, odp_buffer_hdr_t *buf_hdr[], int num) { int actual; ringidx_t old_read; @@ -812,7 +806,7 @@ inline int _odp_queue_deq(sched_elem_t *q, odp_buffer_hdr_t *buf_hdr[], int num) return actual; } -inline int _odp_queue_deq_mc(sched_elem_t *q, odp_event_t *evp, int num) +int _odp_queue_deq_mc(sched_elem_t *q, odp_event_t *evp, int num) { int ret, evt_idx; odp_buffer_hdr_t *hdr_tbl[QUEUE_MULTI_MAX]; @@ -965,7 +959,7 @@ static void queue_print(odp_queue_t handle) } ODP_PRINT("\nQueue info\n"); ODP_PRINT("----------\n"); - ODP_PRINT(" handle %p\n", queue->s.handle); + ODP_PRINT(" handle %p\n", (void *)queue->s.handle); ODP_PRINT(" index %" PRIu32 "\n", queue->s.index); ODP_PRINT(" name %s\n", queue->s.name); ODP_PRINT(" enq mode %s\n", diff --git a/platform/linux-generic/odp_schedule_basic.c b/platform/linux-generic/odp_schedule_basic.c index eb17e95eb..f86914722 100644 --- a/platform/linux-generic/odp_schedule_basic.c +++ b/platform/linux-generic/odp_schedule_basic.c @@ -195,6 +195,8 @@ typedef struct ODP_ALIGNED_CACHE { } sched_local_t; +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wpedantic" /* Priority queue */ typedef struct ODP_ALIGNED_CACHE { /* Ring header */ @@ -204,6 +206,7 @@ typedef struct ODP_ALIGNED_CACHE { uint32_t queue_index[MAX_RING_SIZE]; /* overlaps with ring.data[] */ } prio_queue_t; +#pragma GCC diagnostic pop /* Order context of a queue */ typedef struct ODP_ALIGNED_CACHE { @@ -247,8 +250,10 @@ typedef struct { } queue[CONFIG_MAX_SCHED_QUEUES]; /* Scheduler priority queues */ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wpedantic" prio_queue_t prio_q[NUM_SCHED_GRPS][NUM_PRIO][MAX_SPREAD]; - +#pragma GCC diagnostic pop uint32_t prio_q_count[NUM_SCHED_GRPS][NUM_PRIO][MAX_SPREAD]; odp_thrmask_t mask_all; @@ -1960,7 +1965,7 @@ static int schedule_num_grps(void) static void schedule_get_config(schedule_config_t *config) { *config = sched->config_if; -}; +} static int schedule_capability(odp_schedule_capability_t *capa) { diff --git a/platform/linux-generic/odp_schedule_if.c b/platform/linux-generic/odp_schedule_if.c index 528c4065e..3b908ea87 100644 --- a/platform/linux-generic/odp_schedule_if.c +++ b/platform/linux-generic/odp_schedule_if.c @@ -95,27 +95,27 @@ int odp_schedule_multi_no_wait(odp_queue_t *from, odp_event_t events[], int num) void odp_schedule_pause(void) { - return _odp_sched_api->schedule_pause(); + _odp_sched_api->schedule_pause(); } void odp_schedule_resume(void) { - return _odp_sched_api->schedule_resume(); + _odp_sched_api->schedule_resume(); } void odp_schedule_release_atomic(void) { - return _odp_sched_api->schedule_release_atomic(); + _odp_sched_api->schedule_release_atomic(); } void odp_schedule_release_ordered(void) { - return _odp_sched_api->schedule_release_ordered(); + _odp_sched_api->schedule_release_ordered(); } void odp_schedule_prefetch(int num) { - return _odp_sched_api->schedule_prefetch(num); + _odp_sched_api->schedule_prefetch(num); } int odp_schedule_min_prio(void) @@ -180,12 +180,12 @@ int odp_schedule_group_info(odp_schedule_group_t group, void odp_schedule_order_lock(uint32_t lock_index) { - return _odp_sched_api->schedule_order_lock(lock_index); + _odp_sched_api->schedule_order_lock(lock_index); } void odp_schedule_order_unlock(uint32_t lock_index) { - return _odp_sched_api->schedule_order_unlock(lock_index); + _odp_sched_api->schedule_order_unlock(lock_index); } void odp_schedule_order_unlock_lock(uint32_t unlock_index, uint32_t lock_index) diff --git a/platform/linux-generic/odp_schedule_sp.c b/platform/linux-generic/odp_schedule_sp.c index 4f6ce736d..470075cea 100644 --- a/platform/linux-generic/odp_schedule_sp.c +++ b/platform/linux-generic/odp_schedule_sp.c @@ -81,6 +81,8 @@ typedef struct ODP_ALIGNED_CACHE sched_cmd_t { sizeof(struct sched_cmd_s)]; } sched_cmd_t; +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wpedantic" typedef struct ODP_ALIGNED_CACHE { /* Ring header */ ring_u32_t ring; @@ -89,6 +91,7 @@ typedef struct ODP_ALIGNED_CACHE { uint32_t ring_idx[RING_SIZE]; /* overlaps with ring.data[] */ } prio_queue_t; +#pragma GCC diagnostic pop typedef struct thr_group_t { /* A generation counter for fast comparison if groups have changed */ @@ -123,7 +126,11 @@ typedef struct ODP_ALIGNED_CACHE sched_group_t { typedef struct { sched_cmd_t queue_cmd[NUM_QUEUE]; sched_cmd_t pktio_cmd[NUM_PKTIO]; + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wpedantic" prio_queue_t prio_queue[NUM_GROUP][NUM_PRIO]; +#pragma GCC diagnostic pop sched_group_t sched_group; odp_shm_t shm; /* Scheduler interface config options (not used in fast path) */ @@ -1056,7 +1063,7 @@ static void schedule_print(void) static void get_config(schedule_config_t *config) { *config = sched_global->config_if; -}; +} /* Fill in scheduler interface */ const schedule_fn_t _odp_schedule_sp_fn = { diff --git a/platform/linux-generic/odp_sorted_list.c b/platform/linux-generic/odp_sorted_list.c index 042818c83..8f998238d 100644 --- a/platform/linux-generic/odp_sorted_list.c +++ b/platform/linux-generic/odp_sorted_list.c @@ -28,9 +28,12 @@ typedef struct { uint32_t pad; } sorted_list_desc_t; +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wpedantic" typedef struct { sorted_list_desc_t descs[0]; } sorted_list_descs_t; +#pragma GCC diagnostic pop typedef struct { uint64_t total_inserts; diff --git a/platform/linux-generic/odp_stash.c b/platform/linux-generic/odp_stash.c index a1bf5b63d..57cf747f1 100644 --- a/platform/linux-generic/odp_stash.c +++ b/platform/linux-generic/odp_stash.c @@ -4,6 +4,7 @@ * SPDX-License-Identifier: BSD-3-Clause */ +#include <stdint.h> #include <stdio.h> #include <string.h> @@ -12,16 +13,18 @@ #include <odp/api/stash.h> #include <odp/api/plat/strong_types.h> +#include <odp_config_internal.h> #include <odp_debug_internal.h> #include <odp_global_data.h> #include <odp_init_internal.h> -#include <odp_ring_ptr_internal.h> #include <odp_ring_u32_internal.h> +#include <odp_ring_u64_internal.h> -#define MAX_STASHES 32 #define MAX_RING_SIZE (1024 * 1024) #define MIN_RING_SIZE 64 +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wpedantic" typedef struct stash_t { char name[ODP_STASH_NAME_LEN]; odp_shm_t shm; @@ -32,23 +35,24 @@ typedef struct stash_t { /* Ring header followed by variable sized data (object handles) */ union { struct ODP_ALIGNED_CACHE { - ring_ptr_t hdr; - uintptr_t data[0]; - } ring_ptr; - - struct ODP_ALIGNED_CACHE { ring_u32_t hdr; - uint32_t data[0]; + uint32_t data[]; } ring_u32; + + struct ODP_ALIGNED_CACHE { + ring_u64_t hdr; + uint64_t data[]; + } ring_u64; }; } stash_t; +#pragma GCC diagnostic pop typedef struct stash_global_t { odp_ticketlock_t lock; odp_shm_t shm; - uint8_t stash_reserved[MAX_STASHES]; - stash_t *stash[MAX_STASHES]; + uint8_t stash_reserved[CONFIG_MAX_STASHES]; + stash_t *stash[CONFIG_MAX_STASHES]; } stash_global_t; @@ -106,10 +110,10 @@ int odp_stash_capability(odp_stash_capability_t *capa, odp_stash_type_t type) (void)type; memset(capa, 0, sizeof(odp_stash_capability_t)); - capa->max_stashes_any_type = MAX_STASHES; - capa->max_stashes = MAX_STASHES; + capa->max_stashes_any_type = CONFIG_MAX_STASHES; + capa->max_stashes = CONFIG_MAX_STASHES; capa->max_num_obj = MAX_RING_SIZE; - capa->max_obj_size = sizeof(uintptr_t); + capa->max_obj_size = sizeof(uint64_t); return 0; } @@ -129,7 +133,7 @@ static int reserve_index(void) odp_ticketlock_lock(&stash_global->lock); - for (i = 0; i < MAX_STASHES; i++) { + for (i = 0; i < CONFIG_MAX_STASHES; i++) { if (stash_global->stash_reserved[i] == 0) { index = i; stash_global->stash_reserved[i] = 1; @@ -157,7 +161,7 @@ odp_stash_t odp_stash_create(const char *name, const odp_stash_param_t *param) odp_shm_t shm; stash_t *stash; uint64_t i, ring_size, shm_size; - int ring_ptr, index; + int ring_u64, index; char shm_name[ODP_STASH_NAME_LEN + 8]; if (odp_global_ro.disable.stash) { @@ -165,7 +169,7 @@ odp_stash_t odp_stash_create(const char *name, const odp_stash_param_t *param) return ODP_STASH_INVALID; } - if (param->obj_size > sizeof(uintptr_t)) { + if (param->obj_size > sizeof(uint64_t)) { ODP_ERR("Too large object handle.\n"); return ODP_STASH_INVALID; } @@ -187,9 +191,9 @@ odp_stash_t odp_stash_create(const char *name, const odp_stash_param_t *param) return ODP_STASH_INVALID; } - ring_ptr = 0; + ring_u64 = 0; if (param->obj_size > sizeof(uint32_t)) - ring_ptr = 1; + ring_u64 = 1; ring_size = param->num_obj; @@ -202,8 +206,8 @@ odp_stash_t odp_stash_create(const char *name, const odp_stash_param_t *param) memset(shm_name, 0, sizeof(shm_name)); snprintf(shm_name, sizeof(shm_name) - 1, "_stash_%s", name); - if (ring_ptr) - shm_size = sizeof(stash_t) + (ring_size * sizeof(uintptr_t)); + if (ring_u64) + shm_size = sizeof(stash_t) + (ring_size * sizeof(uint64_t)); else shm_size = sizeof(stash_t) + (ring_size * sizeof(uint32_t)); @@ -218,11 +222,11 @@ odp_stash_t odp_stash_create(const char *name, const odp_stash_param_t *param) stash = odp_shm_addr(shm); memset(stash, 0, sizeof(stash_t)); - if (ring_ptr) { - ring_ptr_init(&stash->ring_ptr.hdr); + if (ring_u64) { + ring_u64_init(&stash->ring_u64.hdr); for (i = 0; i < ring_size; i++) - stash->ring_ptr.data[i] = 0; + stash->ring_u64.data[i] = 0; } else { ring_u32_init(&stash->ring_u32.hdr); @@ -284,7 +288,7 @@ odp_stash_t odp_stash_lookup(const char *name) odp_ticketlock_lock(&stash_global->lock); - for (i = 0; i < MAX_STASHES; i++) { + for (i = 0; i < CONFIG_MAX_STASHES; i++) { stash = stash_global->stash[i]; if (stash && strcmp(stash->name, name) == 0) { @@ -311,11 +315,11 @@ int32_t odp_stash_put(odp_stash_t st, const void *obj, int32_t num) obj_size = stash->obj_size; - if (obj_size == sizeof(uintptr_t)) { - ring_ptr_t *ring_ptr = &stash->ring_ptr.hdr; + if (obj_size == sizeof(uint64_t)) { + ring_u64_t *ring_u64 = &stash->ring_u64.hdr; - ring_ptr_enq_multi(ring_ptr, stash->ring_mask, - (void *)(uintptr_t)obj, num); + ring_u64_enq_multi(ring_u64, stash->ring_mask, + (uint64_t *)(uintptr_t)obj, num); return num; } @@ -354,6 +358,55 @@ int32_t odp_stash_put(odp_stash_t st, const void *obj, int32_t num) return -1; } +int32_t odp_stash_put_u32(odp_stash_t st, const uint32_t u32[], int32_t num) +{ + stash_t *stash = (stash_t *)(uintptr_t)st; + + if (odp_unlikely(st == ODP_STASH_INVALID)) + return -1; + + ODP_ASSERT(stash->obj_size == sizeof(uint32_t)); + + ring_u32_enq_multi(&stash->ring_u32.hdr, stash->ring_mask, + (uint32_t *)(uintptr_t)u32, num); + return num; +} + +int32_t odp_stash_put_u64(odp_stash_t st, const uint64_t u64[], int32_t num) +{ + stash_t *stash = (stash_t *)(uintptr_t)st; + + if (odp_unlikely(st == ODP_STASH_INVALID)) + return -1; + + ODP_ASSERT(stash->obj_size == sizeof(uint64_t)); + + ring_u64_enq_multi(&stash->ring_u64.hdr, stash->ring_mask, + (uint64_t *)(uintptr_t)u64, num); + return num; +} + +int32_t odp_stash_put_ptr(odp_stash_t st, const uintptr_t ptr[], int32_t num) +{ + stash_t *stash = (stash_t *)(uintptr_t)st; + + if (odp_unlikely(st == ODP_STASH_INVALID)) + return -1; + + ODP_ASSERT(stash->obj_size == sizeof(uintptr_t)); + + if (sizeof(uintptr_t) == sizeof(uint32_t)) + ring_u32_enq_multi(&stash->ring_u32.hdr, stash->ring_mask, + (uint32_t *)(uintptr_t)ptr, num); + else if (sizeof(uintptr_t) == sizeof(uint64_t)) + ring_u64_enq_multi(&stash->ring_u64.hdr, stash->ring_mask, + (uint64_t *)(uintptr_t)ptr, num); + else + return -1; + + return num; +} + int32_t odp_stash_get(odp_stash_t st, void *obj, int32_t num) { stash_t *stash; @@ -367,10 +420,10 @@ int32_t odp_stash_get(odp_stash_t st, void *obj, int32_t num) obj_size = stash->obj_size; - if (obj_size == sizeof(uintptr_t)) { - ring_ptr_t *ring_ptr = &stash->ring_ptr.hdr; + if (obj_size == sizeof(uint64_t)) { + ring_u64_t *ring_u64 = &stash->ring_u64.hdr; - return ring_ptr_deq_multi(ring_ptr, stash->ring_mask, obj, num); + return ring_u64_deq_multi(ring_u64, stash->ring_mask, obj, num); } if (obj_size == sizeof(uint32_t)) { @@ -410,6 +463,52 @@ int32_t odp_stash_get(odp_stash_t st, void *obj, int32_t num) return -1; } +int32_t odp_stash_get_u32(odp_stash_t st, uint32_t u32[], int32_t num) +{ + stash_t *stash = (stash_t *)(uintptr_t)st; + + if (odp_unlikely(st == ODP_STASH_INVALID)) + return -1; + + ODP_ASSERT(stash->obj_size == sizeof(uint32_t)); + + return ring_u32_deq_multi(&stash->ring_u32.hdr, stash->ring_mask, u32, + num); +} + +int32_t odp_stash_get_u64(odp_stash_t st, uint64_t u64[], int32_t num) +{ + stash_t *stash = (stash_t *)(uintptr_t)st; + + if (odp_unlikely(st == ODP_STASH_INVALID)) + return -1; + + ODP_ASSERT(stash->obj_size == sizeof(uint64_t)); + + return ring_u64_deq_multi(&stash->ring_u64.hdr, stash->ring_mask, u64, + num); +} + +int32_t odp_stash_get_ptr(odp_stash_t st, uintptr_t ptr[], int32_t num) +{ + stash_t *stash = (stash_t *)(uintptr_t)st; + + if (odp_unlikely(st == ODP_STASH_INVALID)) + return -1; + + ODP_ASSERT(stash->obj_size == sizeof(uintptr_t)); + + if (sizeof(uintptr_t) == sizeof(uint32_t)) + return ring_u32_deq_multi(&stash->ring_u32.hdr, + stash->ring_mask, + (uint32_t *)(uintptr_t)ptr, num); + else if (sizeof(uintptr_t) == sizeof(uint64_t)) + return ring_u64_deq_multi(&stash->ring_u64.hdr, + stash->ring_mask, + (uint64_t *)(uintptr_t)ptr, num); + return -1; +} + int odp_stash_flush_cache(odp_stash_t st) { if (odp_unlikely(st == ODP_STASH_INVALID)) diff --git a/platform/linux-generic/odp_time.c b/platform/linux-generic/odp_time.c index 8d528727f..d908f02cb 100644 --- a/platform/linux-generic/odp_time.c +++ b/platform/linux-generic/odp_time.c @@ -185,7 +185,7 @@ void odp_time_wait_ns(uint64_t ns) void odp_time_wait_until(odp_time_t time) { - return time_wait_until(time); + time_wait_until(time); } int _odp_time_init_global(void) diff --git a/platform/linux-generic/odp_timer.c b/platform/linux-generic/odp_timer.c index f1ca28346..a28d31245 100644 --- a/platform/linux-generic/odp_timer.c +++ b/platform/linux-generic/odp_timer.c @@ -244,7 +244,7 @@ static inline timer_pool_t *handle_to_tp(odp_timer_t hdl) if (odp_likely(tp != NULL)) return timer_global->timer_pool[tp_idx]; } - ODP_ABORT("Invalid timer handle %p\n", hdl); + ODP_ABORT("Invalid timer handle %p\n", (void *)hdl); } static inline uint32_t handle_to_idx(odp_timer_t hdl, @@ -255,7 +255,7 @@ static inline uint32_t handle_to_idx(odp_timer_t hdl, __builtin_prefetch(&tp->tick_buf[idx], 0, 0); if (odp_likely(idx < odp_atomic_load_u32(&tp->high_wm))) return idx; - ODP_ABORT("Invalid timer handle %p\n", hdl); + ODP_ABORT("Invalid timer handle %p\n", (void *)hdl); } static inline odp_timer_t tp_idx_to_handle(timer_pool_t *tp, @@ -1496,20 +1496,29 @@ void *odp_timeout_user_ptr(odp_timeout_t tmo) return (void *)(uintptr_t)timeout_hdr(tmo)->user_ptr; } -odp_timeout_t odp_timeout_alloc(odp_pool_t pool) +odp_timeout_t odp_timeout_alloc(odp_pool_t pool_hdl) { - odp_buffer_t buf = odp_buffer_alloc(pool); + odp_timeout_t tmo; + pool_t *pool; + int ret; + + ODP_ASSERT(pool_hdl != ODP_POOL_INVALID); + + pool = pool_entry_from_hdl(pool_hdl); + + ODP_ASSERT(pool->type == ODP_POOL_TIMEOUT); - if (odp_unlikely(buf == ODP_BUFFER_INVALID)) - return ODP_TIMEOUT_INVALID; - return odp_timeout_from_event(odp_buffer_to_event(buf)); + ret = _odp_buffer_alloc_multi(pool, (odp_buffer_hdr_t **)&tmo, 1); + + if (odp_likely(ret == 1)) + return tmo; + + return ODP_TIMEOUT_INVALID; } void odp_timeout_free(odp_timeout_t tmo) { - odp_event_t ev = odp_timeout_to_event(tmo); - - odp_buffer_free(odp_buffer_from_event(ev)); + _odp_buffer_free_multi((odp_buffer_hdr_t **)&tmo, 1); } void odp_timer_pool_print(odp_timer_pool_t timer_pool) @@ -1525,7 +1534,7 @@ void odp_timer_pool_print(odp_timer_pool_t timer_pool) ODP_PRINT("\nTimer pool info\n"); ODP_PRINT("---------------\n"); - ODP_PRINT(" timer pool %p\n", tp); + ODP_PRINT(" timer pool %p\n", (void *)tp); ODP_PRINT(" tp index %u\n", tp->tp_idx); ODP_PRINT(" num timers %u\n", tp->num_alloc); ODP_PRINT(" num tp %i\n", timer_global->num_timer_pools); @@ -1550,7 +1559,7 @@ void odp_timer_print(odp_timer_t timer) ODP_PRINT("\nTimer info\n"); ODP_PRINT("----------\n"); - ODP_PRINT(" timer pool %p\n", tp); + ODP_PRINT(" timer pool %p\n", (void *)tp); ODP_PRINT(" timer index %u\n", idx); ODP_PRINT(" dest queue 0x%" PRIx64 "\n", odp_queue_to_u64(tim->queue)); ODP_PRINT(" user ptr %p\n", tim->user_ptr); @@ -1580,7 +1589,7 @@ void odp_timeout_print(odp_timeout_t tmo) ODP_PRINT("\nTimeout info\n"); ODP_PRINT("------------\n"); ODP_PRINT(" tmo handle 0x%" PRIx64 "\n", odp_timeout_to_u64(tmo)); - ODP_PRINT(" timer pool %p\n", tp); + ODP_PRINT(" timer pool %p\n", (void *)tp); ODP_PRINT(" timer index %u\n", idx); ODP_PRINT(" expiration %" PRIu64 "\n", tmo_hdr->expiration); ODP_PRINT(" user ptr %p\n", tmo_hdr->user_ptr); diff --git a/platform/linux-generic/odp_timer_wheel.c b/platform/linux-generic/odp_timer_wheel.c index 71b3370b5..c50d3a13d 100644 --- a/platform/linux-generic/odp_timer_wheel.c +++ b/platform/linux-generic/odp_timer_wheel.c @@ -94,9 +94,12 @@ typedef union { timer_blk_t *timer_blk_list; } current_timer_slot_t; +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wpedantic" typedef struct { current_timer_slot_t slots[0]; } current_wheel_t; +#pragma GCC diagnostic pop typedef struct { uint32_t count; @@ -105,7 +108,7 @@ typedef struct { uint32_t head_idx; uint32_t tail_idx; uint32_t max_idx; - current_timer_slot_t entries[0]; + current_timer_slot_t entries[]; } expired_ring_t; typedef struct { @@ -130,9 +133,12 @@ typedef union { /* Each general_timer_slot is 16 bytes long. */ list_entry_t list_entry; } general_timer_slot_t; +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wpedantic" typedef struct { general_timer_slot_t slots[0]; } general_wheel_t; +#pragma GCC diagnostic pop typedef struct { /* Note that rev stands for revolution - one complete sweep through diff --git a/platform/linux-generic/pktio/dpdk.c b/platform/linux-generic/pktio/dpdk.c index 9abba6292..36c1ec06e 100644 --- a/platform/linux-generic/pktio/dpdk.c +++ b/platform/linux-generic/pktio/dpdk.c @@ -559,7 +559,7 @@ static struct rte_mempool_ops odp_pool_ops = { .get_count = pool_get_count }; -MEMPOOL_REGISTER_OPS(odp_pool_ops); +MEMPOOL_REGISTER_OPS(odp_pool_ops) static inline int mbuf_to_pkt(pktio_entry_t *pktio_entry, odp_packet_t pkt_table[], diff --git a/platform/linux-generic/pktio/dpdk_parse.c b/platform/linux-generic/pktio/dpdk_parse.c index c1b75f126..0984f06c5 100644 --- a/platform/linux-generic/pktio/dpdk_parse.c +++ b/platform/linux-generic/pktio/dpdk_parse.c @@ -510,4 +510,7 @@ int _odp_dpdk_packet_parse_common(packet_parser_t *prs, const uint8_t *ptr, pktin_cfg); } -#endif /* _ODP_PKTIO_DPDK */ +#else /* _ODP_PKTIO_DPDK */ +/* Avoid warning about empty translation unit */ +typedef int _odp_dummy; +#endif diff --git a/platform/linux-generic/pktio/ipc.c b/platform/linux-generic/pktio/ipc.c index ad633e231..b89252303 100644 --- a/platform/linux-generic/pktio/ipc.c +++ b/platform/linux-generic/pktio/ipc.c @@ -1,5 +1,5 @@ /* Copyright (c) 2015-2018, Linaro Limited - * Copyright (c) 2019, Nokia + * Copyright (c) 2019-2021, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -15,17 +15,13 @@ #include <errno.h> #include <fcntl.h> +#include <stdint.h> #include <sys/mman.h> #include <sys/stat.h> #include <unistd.h> -#define IPC_ODP_DEBUG_PRINT 0 - -#define IPC_ODP_DBG(fmt, ...) \ - do { \ - if (IPC_ODP_DEBUG_PRINT == 1) \ - ODP_DBG(fmt, ##__VA_ARGS__);\ - } while (0) +/* Debug level for IPC */ +#define IPC_DBG 3 /* Burst size for IPC free operations */ #define IPC_BURST_SIZE 32 @@ -112,7 +108,7 @@ static inline pkt_ipc_t *pkt_priv(pktio_entry_t *pktio_entry) } /* MAC address for the "ipc" interface */ -static const char pktio_ipc_mac[] = {0x12, 0x12, 0x12, 0x12, 0x12, 0x12}; +static const uint8_t pktio_ipc_mac[] = {0x12, 0x12, 0x12, 0x12, 0x12, 0x12}; static odp_shm_t _ipc_map_remote_pool(const char *name, int pid); @@ -221,7 +217,7 @@ static int _ipc_master_start(pktio_entry_t *pktio_entry) odp_atomic_store_u32(&pktio_ipc->ready, 1); - IPC_ODP_DBG("%s started.\n", pktio_entry->s.name); + ODP_DBG_LVL(IPC_DBG, "%s started.\n", pktio_entry->s.name); return 0; } @@ -382,7 +378,7 @@ static odp_shm_t _ipc_map_remote_pool(const char *name, int pid) return ODP_SHM_INVALID; } - IPC_ODP_DBG("Mapped remote pool %s to local %s\n", name, rname); + ODP_DBG_LVL(IPC_DBG, "Mapped remote pool %s to local %s\n", name, rname); return shm; } @@ -547,7 +543,7 @@ static int ipc_pktio_open(odp_pktio_t id ODP_UNUSED, pktio_ipc->type = PKTIO_TYPE_IPC_SLAVE; snprintf(name, sizeof(name), "ipc:%s_info", tail); - IPC_ODP_DBG("lookup for name %s for pid %d\n", name, pid); + ODP_DBG_LVL(IPC_DBG, "lookup for name %s for pid %d\n", name, pid); shm = odp_shm_import(name, pid, name); if (ODP_SHM_INVALID == shm) return -1; @@ -639,7 +635,7 @@ static int ipc_pktio_recv_lockless(pktio_entry_t *pktio_entry, ready = odp_atomic_load_u32(&pktio_ipc->ready); if (odp_unlikely(!ready)) { - IPC_ODP_DBG("start pktio is missing before usage?\n"); + ODP_DBG_LVL(IPC_DBG, "start pktio is missing before usage?\n"); return 0; } @@ -692,7 +688,7 @@ static int ipc_pktio_recv_lockless(pktio_entry_t *pktio_entry, * packet ordering store such packets in local * cache. */ - IPC_ODP_DBG("unable to allocate packet %d/%d\n", + ODP_DBG_LVL(IPC_DBG, "unable to allocate packet %d/%d\n", i, pkts); break; } @@ -742,7 +738,7 @@ static int ipc_pktio_recv_lockless(pktio_entry_t *pktio_entry, ring_ptr_enq_multi(r_p, ring_mask, ipcbufs_p, pkts); for (i = 0; i < pkts; i++) { - IPC_ODP_DBG("%d/%d send to be free packet offset %" PRIuPTR "\n", + ODP_DBG_LVL(IPC_DBG, "%d/%d send to be free packet offset %" PRIuPTR "\n", i, pkts, offsets[i]); } @@ -819,11 +815,11 @@ static int ipc_pktio_send_lockless(pktio_entry_t *pktio_entry, (uint8_t *)odp_shm_addr(pool->shm); /* compile all function code even if ipc disabled with config */ - IPC_ODP_DBG("%d/%d send packet %" PRIu64 ", pool %" PRIu64 "," + ODP_DBG_LVL(IPC_DBG, "%d/%d send packet %" PRIu64 ", pool %" PRIu64 "," "phdr = %p, offset %td, sendoff %" PRIxPTR ", addr %p iaddr " "%p\n", i, num, odp_packet_to_u64(pkt), odp_pool_to_u64(pool_hdl), - pkt_hdr, (uint8_t *)pkt_hdr->seg_data - + (void *)pkt_hdr, (uint8_t *)pkt_hdr->seg_data - (uint8_t *)odp_shm_addr(pool->shm), offsets[i], odp_shm_addr(pool->shm), odp_shm_addr(ipc_pool->shm)); diff --git a/platform/linux-generic/pktio/loop.c b/platform/linux-generic/pktio/loop.c index 889a270ea..666e7ce90 100644 --- a/platform/linux-generic/pktio/loop.c +++ b/platform/linux-generic/pktio/loop.c @@ -49,7 +49,7 @@ static inline pkt_loop_t *pkt_priv(pktio_entry_t *pktio_entry) } /* MAC address for the "loop" interface */ -static const char pktio_loop_mac[] = {0x02, 0xe9, 0x34, 0x80, 0x73, 0x01}; +static const uint8_t pktio_loop_mac[] = {0x02, 0xe9, 0x34, 0x80, 0x73, 0x01}; static int loopback_stats_reset(pktio_entry_t *pktio_entry); static int loopback_init_capability(pktio_entry_t *pktio_entry); diff --git a/platform/linux-generic/pktio/netmap.c b/platform/linux-generic/pktio/netmap.c index 18d9b3a33..b45cdbe47 100644 --- a/platform/linux-generic/pktio/netmap.c +++ b/platform/linux-generic/pktio/netmap.c @@ -1331,5 +1331,7 @@ const pktio_if_ops_t _odp_netmap_pktio_ops = { .send = netmap_send, .fd_set = netmap_fd_set }; - -#endif /* _ODP_PKTIO_NETMAP */ +#else /* _ODP_PKTIO_NETMAP */ +/* Avoid warning about empty translation unit */ +typedef int _odp_dummy; +#endif diff --git a/platform/linux-generic/pktio/null.c b/platform/linux-generic/pktio/null.c index cbcde323c..8a421986b 100644 --- a/platform/linux-generic/pktio/null.c +++ b/platform/linux-generic/pktio/null.c @@ -7,6 +7,8 @@ #include <odp_api.h> #include <odp_packet_io_internal.h> +#include <stdint.h> + typedef struct { int promisc; /**< whether promiscuous mode is on */ } pkt_null_t; @@ -116,7 +118,7 @@ static uint32_t null_mtu_get(pktio_entry_t *pktio_entry ODP_UNUSED) return PKTIO_NULL_MTU; } -static const char null_mac[] = {0x02, 0xe9, 0x34, 0x80, 0x73, 0x05}; +static const uint8_t null_mac[] = {0x02, 0xe9, 0x34, 0x80, 0x73, 0x05}; static int null_mac_addr_get(pktio_entry_t *pktio_entry ODP_UNUSED, void *mac_addr) diff --git a/platform/linux-generic/pktio/pcap.c b/platform/linux-generic/pktio/pcap.c index bf4c87c02..d42840da9 100644 --- a/platform/linux-generic/pktio/pcap.c +++ b/platform/linux-generic/pktio/pcap.c @@ -50,6 +50,7 @@ #include <errno.h> #include <pcap/pcap.h> #include <pcap/bpf.h> +#include <stdint.h> typedef struct { char *fname_rx; /**< name of pcap file for rx */ @@ -75,7 +76,7 @@ static inline pkt_pcap_t *pkt_priv(pktio_entry_t *pktio_entry) #define PKTIO_PCAP_MTU_MIN (68 + _ODP_ETHHDR_LEN) #define PKTIO_PCAP_MTU_MAX (64 * 1024) -static const char pcap_mac[] = {0x02, 0xe9, 0x34, 0x80, 0x73, 0x04}; +static const uint8_t pcap_mac[] = {0x02, 0xe9, 0x34, 0x80, 0x73, 0x04}; static int pcapif_stats_reset(pktio_entry_t *pktio_entry); |