diff options
Diffstat (limited to 'platform/linux-generic')
31 files changed, 996 insertions, 812 deletions
diff --git a/platform/linux-generic/Makefile.am b/platform/linux-generic/Makefile.am index 70ab7ab7c..1d9a3bbdb 100644 --- a/platform/linux-generic/Makefile.am +++ b/platform/linux-generic/Makefile.am @@ -96,6 +96,7 @@ noinst_HEADERS = \ include/odp_atomic_internal.h \ include/odp_bitset.h \ include/odp_buffer_internal.h \ + include/odp_chksum_internal.h \ include/odp_classification_datamodel.h \ include/odp_classification_internal.h \ include/odp_config_internal.h \ @@ -289,7 +290,8 @@ odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_inlines.h \ arch/default/odp/api/abi/cpu_time.h \ arch/aarch64/odp/api/abi/hash_crc32.h if !ODP_ABI_COMPAT -odpapiabiarchinclude_HEADERS += arch/aarch64/odp/api/abi/cpu.h +odpapiabiarchinclude_HEADERS += arch/aarch64/odp/api/abi/atomic.h \ + arch/aarch64/odp/api/abi/cpu.h endif noinst_HEADERS += arch/aarch64/odp_atomic.h \ arch/aarch64/odp_cpu.h \ @@ -352,6 +354,7 @@ if !ODP_ABI_COMPAT odpapiabiarchinclude_HEADERS += arch/x86/odp/api/abi/cpu.h endif noinst_HEADERS += arch/x86/cpu_flags.h \ + arch/x86/odp_cpu.h \ arch/default/odp_cpu.h \ arch/default/odp_cpu_idling.h endif diff --git a/platform/linux-generic/arch/aarch64/odp/api/abi/atomic.h b/platform/linux-generic/arch/aarch64/odp/api/abi/atomic.h new file mode 100644 index 000000000..d1dbf36b8 --- /dev/null +++ b/platform/linux-generic/arch/aarch64/odp/api/abi/atomic.h @@ -0,0 +1,12 @@ +/* Copyright (c) 2021, ARM Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifdef __ARM_FEATURE_ATOMICS +#define _ODP_LOCK_FREE_128BIT_ATOMICS +#endif + +#include <odp/api/abi-default/atomic.h> + diff --git a/platform/linux-generic/arch/aarch64/odp_atomic.h b/platform/linux-generic/arch/aarch64/odp_atomic.h index 8a0e7ce2b..dbeccebde 100644 --- a/platform/linux-generic/arch/aarch64/odp_atomic.h +++ b/platform/linux-generic/arch/aarch64/odp_atomic.h @@ -1,5 +1,4 @@ -/* Copyright (c) 2017, ARM Limited. All rights reserved. - * +/* Copyright (c) 2017-2021, ARM Limited * Copyright (c) 2017-2018, Linaro Limited * All rights reserved. * @@ -218,4 +217,54 @@ static inline __int128 __lockfree_load_16(__int128 *var, int mo) return old; } +#ifdef _ODP_LOCK_FREE_128BIT_ATOMICS + +/** + * @internal + * Helper macro for lockless atomic CAS operations on 128-bit integers + * @param[in,out] atom Pointer to the 128-bit atomic variable + * @param oper CAS operation + * @param old_val Old value + * @param new_val New value to be swapped + * @return 1 for success and 0 for fail + */ +#define ATOMIC_CAS_OP_128(atom, oper, old_val, new_val, val) \ +({ \ + odp_u128_t _val; \ + odp_atomic_u128_t *_atom = atom; \ + odp_u128_t *_old_val = old_val; \ + odp_u128_t _new_val = new_val; \ + odp_u128_t *ptr = (odp_u128_t *)(_atom); \ + register uint64_t old0 __asm__ ("x0"); \ + register uint64_t old1 __asm__ ("x1"); \ + register uint64_t new0 __asm__ ("x2"); \ + register uint64_t new1 __asm__ ("x3"); \ + old0 = (uint64_t)(_old_val)->u64[0]; \ + old1 = (uint64_t)(_old_val)->u64[1]; \ + new0 = (uint64_t)(_new_val).u64[0]; \ + new1 = (uint64_t)(_new_val).u64[1]; \ + __asm__ volatile(oper " %[old0], %[old1], %[new0], %[new1], [%[ptr]]" \ + : [old0] "+r" (old0), [old1] "+r" (old1) \ + : [new0] "r" (new0), [new1] "r" (new1), \ + [ptr] "r" (ptr) \ + : "memory"); \ + _val.u64[0] = old0; \ + _val.u64[1] = old1; \ + val = _val; \ +}) + +#define ATOMIC_CAS_OP_128_NO_ORDER(atom, old_value, new_value, val) \ + ATOMIC_CAS_OP_128(atom, "casp", old_value, new_value, val) + +#define ATOMIC_CAS_OP_128_ACQ(atom, old_value, new_value, val) \ + ATOMIC_CAS_OP_128(atom, "caspa", old_value, new_value, val) + +#define ATOMIC_CAS_OP_128_REL(atom, old_value, new_value, val) \ + ATOMIC_CAS_OP_128(atom, "caspl", old_value, new_value, val) + +#define ATOMIC_CAS_OP_128_ACQ_REL(atom, old_value, new_value, val) \ + ATOMIC_CAS_OP_128(atom, "caspal", old_value, new_value, val) + +#endif + #endif /* PLATFORM_LINUXGENERIC_ARCH_ARM_ODP_ATOMIC_H */ diff --git a/platform/linux-generic/arch/aarch64/odp_cpu.h b/platform/linux-generic/arch/aarch64/odp_cpu.h index 0da6e3c60..84bc4dffd 100644 --- a/platform/linux-generic/arch/aarch64/odp_cpu.h +++ b/platform/linux-generic/arch/aarch64/odp_cpu.h @@ -59,4 +59,10 @@ do { \ #include "odp_atomic.h" #include "odp_cpu_idling.h" +#ifdef __ARM_FEATURE_UNALIGNED +#define _ODP_UNALIGNED 1 +#else +#define _ODP_UNALIGNED 0 +#endif + #endif /* PLATFORM_LINUXGENERIC_ARCH_ARM_ODP_CPU_H */ diff --git a/platform/linux-generic/arch/aarch64/odp_cpu_idling.h b/platform/linux-generic/arch/aarch64/odp_cpu_idling.h index 8528c8591..a6cea8c63 100644 --- a/platform/linux-generic/arch/aarch64/odp_cpu_idling.h +++ b/platform/linux-generic/arch/aarch64/odp_cpu_idling.h @@ -30,12 +30,6 @@ static inline int wfe(void) return 1; } -static inline void doze(void) -{ - /* When using WFE do not stall the pipeline using other means */ - odp_cpu_pause(); -} - #define monitor128(addr, mo) lld((addr), (mo)) #define monitor64(addr, mo) ll64((addr), (mo)) #define monitor32(addr, mo) ll32((addr), (mo)) diff --git a/platform/linux-generic/arch/arm/odp_cpu.h b/platform/linux-generic/arch/arm/odp_cpu.h index d3d2de34f..82d47325f 100644 --- a/platform/linux-generic/arch/arm/odp_cpu.h +++ b/platform/linux-generic/arch/arm/odp_cpu.h @@ -52,4 +52,10 @@ static inline void _odp_dmb(void) #include "odp_atomic.h" #include "odp_cpu_idling.h" +#ifdef __ARM_FEATURE_UNALIGNED +#define _ODP_UNALIGNED 1 +#else +#define _ODP_UNALIGNED 0 +#endif + #endif /* PLATFORM_LINUXGENERIC_ARCH_ARM_ODP_CPU_H */ diff --git a/platform/linux-generic/arch/arm/odp_cpu_idling.h b/platform/linux-generic/arch/arm/odp_cpu_idling.h index 8528c8591..a6cea8c63 100644 --- a/platform/linux-generic/arch/arm/odp_cpu_idling.h +++ b/platform/linux-generic/arch/arm/odp_cpu_idling.h @@ -30,12 +30,6 @@ static inline int wfe(void) return 1; } -static inline void doze(void) -{ - /* When using WFE do not stall the pipeline using other means */ - odp_cpu_pause(); -} - #define monitor128(addr, mo) lld((addr), (mo)) #define monitor64(addr, mo) ll64((addr), (mo)) #define monitor32(addr, mo) ll32((addr), (mo)) diff --git a/platform/linux-generic/arch/default/odp_cpu.h b/platform/linux-generic/arch/default/odp_cpu.h index 18dd968fb..d8bc125c8 100644 --- a/platform/linux-generic/arch/default/odp_cpu.h +++ b/platform/linux-generic/arch/default/odp_cpu.h @@ -9,6 +9,10 @@ #ifndef ODP_DEFAULT_CPU_H_ #define ODP_DEFAULT_CPU_H_ +#ifndef _ODP_UNALIGNED +#define _ODP_UNALIGNED 0 +#endif + /****************************************************************************** * Atomics *****************************************************************************/ diff --git a/platform/linux-generic/arch/default/odp_cpu_idling.h b/platform/linux-generic/arch/default/odp_cpu_idling.h index 70b8cae6e..9d23ad20d 100644 --- a/platform/linux-generic/arch/default/odp_cpu_idling.h +++ b/platform/linux-generic/arch/default/odp_cpu_idling.h @@ -28,9 +28,4 @@ static inline int wfe(void) #define monitor32(addr, mo) __atomic_load_n((addr), (mo)) #define monitor8(addr, mo) __atomic_load_n((addr), (mo)) -static inline void doze(void) -{ - odp_cpu_pause(); -} - #endif diff --git a/platform/linux-generic/arch/x86/odp_cpu.h b/platform/linux-generic/arch/x86/odp_cpu.h new file mode 100644 index 000000000..8f8f22daf --- /dev/null +++ b/platform/linux-generic/arch/x86/odp_cpu.h @@ -0,0 +1,14 @@ +/* Copyright (c) 2021, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_X86_CPU_H_ +#define ODP_X86_CPU_H_ + +#define _ODP_UNALIGNED 1 + +#include <default/odp_cpu.h> + +#endif diff --git a/platform/linux-generic/include-abi/odp/api/abi/atomic.h b/platform/linux-generic/include-abi/odp/api/abi/atomic.h index 955b99370..13c12a79f 100644 --- a/platform/linux-generic/include-abi/odp/api/abi/atomic.h +++ b/platform/linux-generic/include-abi/odp/api/abi/atomic.h @@ -55,6 +55,31 @@ typedef struct ODP_ALIGNED(sizeof(uint64_t)) odp_atomic_u64_s { #endif +#ifdef _ODP_LOCK_FREE_128BIT_ATOMICS + +/** + * @internal + * Atomic 128-bit unsigned integer + */ +typedef struct ODP_ALIGNED(sizeof(odp_u128_t)) odp_atomic_u128_s { + odp_u128_t v; /**< Actual storage for the atomic variable */ +} odp_atomic_u128_t; + +#else + +/** + * @internal + * Atomic 128-bit unsigned integer + */ +typedef struct ODP_ALIGNED(sizeof(odp_u128_t)) odp_atomic_u128_s { + odp_u128_t v; /**< Actual storage for the atomic variable */ + /* Some architectures do not support lock-free operations on 128-bit + * data types. We use a spin lock to ensure atomicity. */ + char lock; /**< Spin lock (if needed) used to ensure atomic access */ +} odp_atomic_u128_t; + +#endif + /** @ingroup odp_atomic * @{ */ diff --git a/platform/linux-generic/include/odp/api/plat/queue_inline_types.h b/platform/linux-generic/include/odp/api/plat/queue_inline_types.h index bbff4b412..e59c7f55a 100644 --- a/platform/linux-generic/include/odp/api/plat/queue_inline_types.h +++ b/platform/linux-generic/include/odp/api/plat/queue_inline_types.h @@ -51,6 +51,8 @@ typedef struct { void (*queue_param_init)(odp_queue_param_t *param); int (*queue_info)(odp_queue_t queue, odp_queue_info_t *info); void (*queue_print)(odp_queue_t queue); + void (*queue_print_all)(void); + } _odp_queue_api_fn_t; /** @endcond */ diff --git a/platform/linux-generic/include/odp_atomic_internal.h b/platform/linux-generic/include/odp_atomic_internal.h index 5ab4a89af..81280b1fa 100644 --- a/platform/linux-generic/include/odp_atomic_internal.h +++ b/platform/linux-generic/include/odp_atomic_internal.h @@ -27,13 +27,6 @@ extern "C" { #endif /** - * Pointer atomic type - */ -typedef struct ODP_ALIGNED(sizeof(void *)) { - void *v; /**< Actual storage for the atomic variable */ -} _odp_atomic_ptr_t; - -/** * Atomic flag (boolean) type * @Note this is not the same as a plain boolean type. * _odp_atomic_flag_t is guaranteed to be able to operate on atomically. @@ -58,562 +51,9 @@ typedef enum { _ODP_MEMMODEL_RLS = __ATOMIC_RELEASE, /** Acquire&release memory ordering, synchronize with acquire loads and release * stores in another (one other) thread */ - _ODP_MEMMODEL_ACQ_RLS = __ATOMIC_ACQ_REL, -/** Sequential consistent memory ordering, synchronize with acquire loads and - * release stores in all threads */ - _ODP_MEMMODEL_SC = __ATOMIC_SEQ_CST -} _odp_memmodel_t; - -/***************************************************************************** - * Operations on 32-bit atomics - * _odp_atomic_u32_load_mm - return current value - * _odp_atomic_u32_store_mm - no return value - * _odp_atomic_u32_xchg_mm - return old value - * _odp_atomic_u32_cmp_xchg_strong_mm - return bool - * _odp_atomic_u32_fetch_add_mm - return old value - * _odp_atomic_u32_add_mm - no return value - * _odp_atomic_u32_fetch_sub_mm - return old value - * _odp_atomic_u32_sub_mm - no return value - *****************************************************************************/ - -/** - * Atomic load of 32-bit atomic variable - * - * @param atom Pointer to a 32-bit atomic variable - * @param mmodel Memory ordering associated with the load operation - * - * @return Value of the variable - */ -static inline uint32_t _odp_atomic_u32_load_mm(const odp_atomic_u32_t *atom, - _odp_memmodel_t mmodel) -{ - return __atomic_load_n(&atom->v, mmodel); -} - -/** - * Atomic store to 32-bit atomic variable - * - * @param[out] atom Pointer to a 32-bit atomic variable - * @param val Value to store in the atomic variable - * @param mmodel Memory order associated with the store operation - */ -static inline void _odp_atomic_u32_store_mm(odp_atomic_u32_t *atom, - uint32_t val, - _odp_memmodel_t mmodel) -{ - __atomic_store_n(&atom->v, val, mmodel); -} - -/** - * Atomic exchange (swap) of 32-bit atomic variable - * - * @param[in,out] atom Pointer to a 32-bit atomic variable - * @param val New value to store in the atomic variable - * @param mmodel Memory order associated with the exchange operation - * - * @return Old value of the variable - */ -static inline uint32_t _odp_atomic_u32_xchg_mm(odp_atomic_u32_t *atom, - uint32_t val, - _odp_memmodel_t mmodel) - -{ - return __atomic_exchange_n(&atom->v, val, mmodel); -} - -/** - * Atomic compare and exchange (swap) of 32-bit atomic variable - * "Strong" semantics, will not fail spuriously. - * - * @param[in,out] atom Pointer to a 32-bit atomic variable - * @param[in,out] exp Pointer to expected value (updated on failure) - * @param val New value to write - * @param success Memory order associated with a successful compare-and-swap - * operation - * @param failure Memory order associated with a failed compare-and-swap - * operation - * - * @retval 1 exchange successul - * @retval 0 exchange failed and '*exp' updated with current value - */ -static inline int _odp_atomic_u32_cmp_xchg_strong_mm(odp_atomic_u32_t *atom, - uint32_t *exp, - uint32_t val, - _odp_memmodel_t success, - _odp_memmodel_t failure) -{ - return __atomic_compare_exchange_n(&atom->v, exp, val, - false/*strong*/, success, failure); -} - -/** - * Atomic fetch and add of 32-bit atomic variable - * - * @param[in,out] atom Pointer to a 32-bit atomic variable - * @param val Value to add to the atomic variable - * @param mmodel Memory order associated with the add operation - * - * @return Value of the atomic variable before the addition - */ -static inline uint32_t _odp_atomic_u32_fetch_add_mm(odp_atomic_u32_t *atom, - uint32_t val, - _odp_memmodel_t mmodel) -{ - return __atomic_fetch_add(&atom->v, val, mmodel); -} - -/** - * Atomic add of 32-bit atomic variable - * - * @param[in,out] atom Pointer to a 32-bit atomic variable - * @param val Value to add to the atomic variable - * @param mmodel Memory order associated with the add operation - */ -static inline void _odp_atomic_u32_add_mm(odp_atomic_u32_t *atom, uint32_t val, - _odp_memmodel_t mmodel) + _ODP_MEMMODEL_ACQ_RLS = __ATOMIC_ACQ_REL -{ - (void)__atomic_fetch_add(&atom->v, val, mmodel); -} - -/** - * Atomic fetch and subtract of 32-bit atomic variable - * - * @param[in,out] atom Pointer to a 32-bit atomic variable - * @param val Value to subtract from the atomic variable - * @param mmodel Memory order associated with the subtract operation - * - * @return Value of the atomic variable before the subtraction - */ -static inline uint32_t _odp_atomic_u32_fetch_sub_mm(odp_atomic_u32_t *atom, - uint32_t val, - _odp_memmodel_t mmodel) -{ - return __atomic_fetch_sub(&atom->v, val, mmodel); -} - -/** - * Atomic subtract of 32-bit atomic variable - * - * @param[in,out] atom Pointer to a 32-bit atomic variable - * @param val Value to subtract from the atomic variable - * @param mmodel Memory order associated with the subtract operation - */ -static inline void _odp_atomic_u32_sub_mm(odp_atomic_u32_t *atom, uint32_t val, - _odp_memmodel_t mmodel) - -{ - (void)__atomic_fetch_sub(&atom->v, val, mmodel); -} - -/***************************************************************************** - * Operations on 64-bit atomics - * _odp_atomic_u64_load_mm - return current value - * _odp_atomic_u64_store_mm - no return value - * _odp_atomic_u64_xchg_mm - return old value - * _odp_atomic_u64_cmp_xchg_strong_mm - return bool - * _odp_atomic_u64_fetch_add_mm - return old value - * _odp_atomic_u64_add_mm - no return value - * _odp_atomic_u64_fetch_sub_mm - return old value - * _odp_atomic_u64_sub_mm - no return value - *****************************************************************************/ - -/* Check if the compiler support lock-less atomic operations on 64-bit types */ -#ifdef ODP_ATOMIC_U64_LOCK -/** - * @internal - * Helper macro for lock-based atomic operations on 64-bit integers - * @param[in,out] atom Pointer to the 64-bit atomic variable - * @param expr Expression used update the variable. - * @param mm Memory order to use. - * @return The old value of the variable. - */ -#define ATOMIC_OP_MM(atom, expr, mm) \ -({ \ - uint64_t old_val; \ - /* Loop while lock is already taken, stop when lock becomes clear */ \ - while (__atomic_test_and_set(&(atom)->lock, \ - (mm) == _ODP_MEMMODEL_SC ? \ - __ATOMIC_SEQ_CST : __ATOMIC_ACQUIRE)) \ - (void)0; \ - old_val = (atom)->v; \ - (expr); /* Perform whatever update is desired */ \ - __atomic_clear(&(atom)->lock, \ - (mm) == _ODP_MEMMODEL_SC ? \ - __ATOMIC_SEQ_CST : __ATOMIC_RELEASE); \ - old_val; /* Return old value */ \ -}) - -/** - * Atomic load of 64-bit atomic variable - * - * @param atom Pointer to a 64-bit atomic variable - * @param mmodel Memory order associated with the load operation - * - * @return Value of the variable - */ -static inline uint64_t _odp_atomic_u64_load_mm(odp_atomic_u64_t *atom, - _odp_memmodel_t mmodel) -{ - return ATOMIC_OP_MM(atom, (void)0, mmodel); -} - -/** - * Atomic store to 64-bit atomic variable - * - * @param[out] atom Pointer to a 64-bit atomic variable - * @param val Value to write to the atomic variable - * @param mmodel Memory order associated with the store operation - */ -static inline void _odp_atomic_u64_store_mm(odp_atomic_u64_t *atom, - uint64_t val, - _odp_memmodel_t mmodel) -{ - (void)ATOMIC_OP_MM(atom, atom->v = val, mmodel); -} - -/** - * Atomic exchange (swap) of 64-bit atomic variable - * - * @param[in,out] atom Pointer to a 64-bit atomic variable - * @param val New value to write to the atomic variable - * @param mmodel Memory order associated with the exchange operation - * - * @return Old value of variable - */ -static inline uint64_t _odp_atomic_u64_xchg_mm(odp_atomic_u64_t *atom, - uint64_t val, - _odp_memmodel_t mmodel) - -{ - return ATOMIC_OP_MM(atom, atom->v = val, mmodel); -} - -/** - * Atomic compare and exchange (swap) of 64-bit atomic variable - * "Strong" semantics, will not fail spuriously. - * - * @param[in,out] atom Pointer to a 64-bit atomic variable - * @param[in,out] exp Pointer to expected value (updated on failure) - * @param val New value to write - * @param success Memory order associated with a successful compare-and-swap - * operation - * @param failure Memory order associated with a failed compare-and-swap - * operation - * - * @retval 1 exchange successful - * @retval 0 exchange failed and '*exp' updated with current value - */ -static inline int _odp_atomic_u64_cmp_xchg_strong_mm(odp_atomic_u64_t *atom, - uint64_t *exp, - uint64_t val, - _odp_memmodel_t success, - _odp_memmodel_t failure) -{ - /* Possibly we are a bit pessimistic with the memory models */ - odp_bool_t ret_succ; - /* Loop while lock is already taken, stop when lock becomes clear */ - while (__atomic_test_and_set(&(atom)->lock, - (success) == _ODP_MEMMODEL_SC ? - __ATOMIC_SEQ_CST : __ATOMIC_ACQUIRE)) - (void)0; - if (atom->v == *exp) { - atom->v = val; - ret_succ = 1; - } else { - *exp = atom->v; - ret_succ = 0; - } - __atomic_clear(&(atom)->lock, - (ret_succ ? success : failure) == _ODP_MEMMODEL_SC ? - __ATOMIC_SEQ_CST : __ATOMIC_RELEASE); - return ret_succ; -} - -/** - * Atomic fetch and add of 64-bit atomic variable - * - * @param[in,out] atom Pointer to a 64-bit atomic variable - * @param val Value to add to the atomic variable - * @param mmodel Memory order associated with the add operation - * - * @return Value of the atomic variable before the addition - */ -static inline uint64_t _odp_atomic_u64_fetch_add_mm(odp_atomic_u64_t *atom, - uint64_t val, - _odp_memmodel_t mmodel) -{ - return ATOMIC_OP_MM(atom, atom->v += val, mmodel); -} - -/** - * Atomic add of 64-bit atomic variable - * - * @param[in,out] atom Pointer to a 64-bit atomic variable - * @param val Value to add to the atomic variable - * @param mmodel Memory order associated with the add operation. - */ -static inline void _odp_atomic_u64_add_mm(odp_atomic_u64_t *atom, - uint64_t val, - _odp_memmodel_t mmodel) - -{ - (void)ATOMIC_OP_MM(atom, atom->v += val, mmodel); -} - -/** - * Atomic fetch and subtract of 64-bit atomic variable - * - * @param[in,out] atom Pointer to a 64-bit atomic variable - * @param val Value to subtract from the atomic variable - * @param mmodel Memory order associated with the subtract operation - * - * @return Value of the atomic variable before the subtraction - */ -static inline uint64_t _odp_atomic_u64_fetch_sub_mm(odp_atomic_u64_t *atom, - uint64_t val, - _odp_memmodel_t mmodel) -{ - return ATOMIC_OP_MM(atom, atom->v -= val, mmodel); -} - -/** - * Atomic subtract of 64-bit atomic variable - * - * @param[in,out] atom Pointer to a 64-bit atomic variable - * @param val Value to subtract from the atomic variable - * @param mmodel Memory order associated with the subtract operation - */ -static inline void _odp_atomic_u64_sub_mm(odp_atomic_u64_t *atom, - uint64_t val, - _odp_memmodel_t mmodel) - -{ - (void)ATOMIC_OP_MM(atom, atom->v -= val, mmodel); -} - -#undef ATOMIC_OP_MM - -#else /* ! ODP_ATOMIC_U64_LOCK */ - -/** - * Atomic load of 64-bit atomic variable - * - * @param atom Pointer to a 64-bit atomic variable - * @param mmodel Memory order associated with the load operation - * - * @return Value of the variable - */ -static inline uint64_t _odp_atomic_u64_load_mm(odp_atomic_u64_t *atom, - _odp_memmodel_t mmodel) -{ - return __atomic_load_n(&atom->v, mmodel); -} - -/** - * Atomic store to 64-bit atomic variable - * - * @param[out] atom Pointer to a 64-bit atomic variable - * @param val Value to write to the atomic variable - * @param mmodel Memory order associated with the store operation - */ -static inline void _odp_atomic_u64_store_mm(odp_atomic_u64_t *atom, - uint64_t val, - _odp_memmodel_t mmodel) -{ - __atomic_store_n(&atom->v, val, mmodel); -} - -/** - * Atomic exchange (swap) of 64-bit atomic variable - * - * @param[in,out] atom Pointer to a 64-bit atomic variable - * @param val New value to write to the atomic variable - * @param mmodel Memory order associated with the exchange operation - * - * @return Old value of variable - */ -static inline uint64_t _odp_atomic_u64_xchg_mm(odp_atomic_u64_t *atom, - uint64_t val, - _odp_memmodel_t mmodel) - -{ - return __atomic_exchange_n(&atom->v, val, mmodel); -} - -/** - * Atomic compare and exchange (swap) of 64-bit atomic variable - * "Strong" semantics, will not fail spuriously. - * - * @param[in,out] atom Pointer to a 64-bit atomic variable - * @param[in,out] exp Pointer to expected value (updated on failure) - * @param val New value to write - * @param success Memory order associated with a successful compare-and-swap - * operation - * @param failure Memory order associated with a failed compare-and-swap - * operation - * - * @retval 1 exchange successful - * @retval 0 exchange failed and '*exp' updated with current value - */ -static inline int _odp_atomic_u64_cmp_xchg_strong_mm(odp_atomic_u64_t *atom, - uint64_t *exp, - uint64_t val, - _odp_memmodel_t success, - _odp_memmodel_t failure) -{ - return __atomic_compare_exchange_n(&atom->v, exp, val, - false/*strong*/, success, failure); -} - -/** - * Atomic fetch and add of 64-bit atomic variable - * - * @param[in,out] atom Pointer to a 64-bit atomic variable - * @param val Value to add to the atomic variable - * @param mmodel Memory order associated with the add operation - * - * @return Value of the atomic variable before the addition - */ -static inline uint64_t _odp_atomic_u64_fetch_add_mm(odp_atomic_u64_t *atom, - uint64_t val, - _odp_memmodel_t mmodel) -{ - return __atomic_fetch_add(&atom->v, val, mmodel); -} - -/** - * Atomic add of 64-bit atomic variable - * - * @param[in,out] atom Pointer to a 64-bit atomic variable - * @param val Value to add to the atomic variable - * @param mmodel Memory order associated with the add operation. - */ -static inline void _odp_atomic_u64_add_mm(odp_atomic_u64_t *atom, - uint64_t val, - _odp_memmodel_t mmodel) - -{ - (void)__atomic_fetch_add(&atom->v, val, mmodel); -} - -/** - * Atomic fetch and subtract of 64-bit atomic variable - * - * @param[in,out] atom Pointer to a 64-bit atomic variable - * @param val Value to subtract from the atomic variable - * @param mmodel Memory order associated with the subtract operation - * - * @return Value of the atomic variable before the subtraction - */ -static inline uint64_t _odp_atomic_u64_fetch_sub_mm(odp_atomic_u64_t *atom, - uint64_t val, - _odp_memmodel_t mmodel) -{ - return __atomic_fetch_sub(&atom->v, val, mmodel); -} - -/** - * Atomic subtract of 64-bit atomic variable - * - * @param[in,out] atom Pointer to a 64-bit atomic variable - * @param val Value to subtract from the atomic variable - * @param mmodel Memory order associated with the subtract operation - */ -static inline void _odp_atomic_u64_sub_mm(odp_atomic_u64_t *atom, - uint64_t val, - _odp_memmodel_t mmodel) - -{ - (void)__atomic_fetch_sub(&atom->v, val, mmodel); -} - -#endif /* ! ODP_ATOMIC_U64_LOCK */ - -/***************************************************************************** - * Operations on pointer atomics - * _odp_atomic_ptr_init - no return value - * _odp_atomic_ptr_load - return current value - * _odp_atomic_ptr_store - no return value - * _odp_atomic_ptr_xchg - return old value - *****************************************************************************/ - -/** - * Initialization of pointer atomic variable - * - * @param[out] atom Pointer to a pointer atomic variable - * @param val Value to initialize the variable with - */ -static inline void _odp_atomic_ptr_init(_odp_atomic_ptr_t *atom, void *val) -{ - __atomic_store_n(&atom->v, val, __ATOMIC_RELAXED); -} - -/** - * Atomic load of pointer atomic variable - * - * @param atom Pointer to a pointer atomic variable - * @param mmodel Memory order associated with the load operation - * - * @return Value of the variable - */ -static inline void *_odp_atomic_ptr_load(const _odp_atomic_ptr_t *atom, - _odp_memmodel_t mmodel) -{ - return __atomic_load_n(&atom->v, mmodel); -} - -/** - * Atomic store to pointer atomic variable - * - * @param[out] atom Pointer to a pointer atomic variable - * @param val Value to write to the atomic variable - * @param mmodel Memory order associated with the store operation - */ -static inline void _odp_atomic_ptr_store(_odp_atomic_ptr_t *atom, void *val, - _odp_memmodel_t mmodel) -{ - __atomic_store_n(&atom->v, val, mmodel); -} - -/** - * Atomic exchange (swap) of pointer atomic variable - * - * @param[in,out] atom Pointer to a pointer atomic variable - * @param val New value to write - * @param mmodel Memory order associated with the exchange operation - * - * @return Old value of variable - */ -static inline void *_odp_atomic_ptr_xchg(_odp_atomic_ptr_t *atom, void *val, - _odp_memmodel_t mmodel) -{ - return __atomic_exchange_n(&atom->v, val, mmodel); -} - -/** - * Atomic compare and exchange (swap) of pointer atomic variable - * "Strong" semantics, will not fail spuriously. - * - * @param[in,out] atom Pointer to a pointer atomic variable - * @param[in,out] exp Pointer to expected value (updated on failure) - * @param val New value to write - * @param success Memory order associated with a successful compare-and-swap - * operation - * @param failure Memory order associated with a failed compare-and-swap - * operation - * - * @retval 1 exchange successful - * @retval 0 exchange failed and '*exp' updated with current value - */ -static inline int _odp_atomic_ptr_cmp_xchg_strong(_odp_atomic_ptr_t *atom, - void **exp, void *val, - _odp_memmodel_t success, - _odp_memmodel_t failure) -{ - return __atomic_compare_exchange_n(&atom->v, exp, val, - false/*strong*/, success, failure); -} +} _odp_memmodel_t; /***************************************************************************** * Operations on flag atomics diff --git a/platform/linux-generic/include/odp_chksum_internal.h b/platform/linux-generic/include/odp_chksum_internal.h new file mode 100644 index 000000000..5a134ae2d --- /dev/null +++ b/platform/linux-generic/include/odp_chksum_internal.h @@ -0,0 +1,189 @@ +/* Copyright (c) 2020, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_CHKSUM_INTERNAL_H_ +#define ODP_CHKSUM_INTERNAL_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <odp/api/hints.h> +#include <odp/api/byteorder.h> +#include <odp_cpu.h> +#include <stdint.h> + +/* + * Compute the final Internet checksum (RFC 1071) based on a partial + * sum. A partial sum can be obtained e.g. by calling + * chksum_partial(). + */ +static inline uint16_t chksum_finalize(uint64_t sum) +{ + sum = (sum >> 32) + (sum & 0xffffffff); + sum = (sum >> 16) + (sum & 0xffff); + /* + * The final & 0xffff is intentionally omitted, the extra bits + * are discarded by the implicit cast to the return type. + */ + return (sum >> 16) + sum; +} + +/* + * Compute a partial checksum. Several partial checksums may be summed + * together. The final checksum may be obtained by calling + * chksum_finalize(). Parameter offset is the offset of this segment + * of data from the start of IP header. + * + * This implementation + * + * - Accepts unaligned data. + * + * - Accepts data at any byte offset from the start of IP header, + * including odd offsets. + * + * - Uses unaligned memory access only if available. + * + * - Is optimized (for skylake, cn96, a53) by trial and error. + * + * The following did not improve performance (in synthetic tests): + * + * - 2 or 4 sub-sums in the main loop (to break dependency chains). + * + * - Aligning to 8 bytes instead of 4 (for ldp instruction). This + * makes the main loop faster on a53 (only), but the extra + * conditional branch has its cost. + * + * - __builtin_assume_aligned(). + */ +static uint64_t chksum_partial(const void *addr, uint32_t len, uint32_t offset) +{ + const uint8_t *b; + const uint16_t *w; + const uint32_t *d; + uint64_t sum = 0; + + /* + * Offset is either even or odd, the rest of it doesn't + * matter. + */ + offset &= 1; + + if (_ODP_UNALIGNED) { + /* + * We have efficient unaligned access. Just read + * dwords starting at the given address. + */ + d = (const uint32_t *)addr; + } else { + /* + * We must avoid unaligned access, so align to 4 bytes + * by summing up the first up to 3 bytes. + */ + b = (const uint8_t *)addr; + + if (odp_unlikely((uintptr_t)b & 1) && len >= 1) { + /* + * Align to 2 bytes by handling an odd + * byte. Since addr is unaligned, the first + * byte goes into the second byte of the sum. + */ + sum += odp_cpu_to_be_16(*b++); + len -= 1; + + /* An odd byte negates the effect of offset. */ + offset ^= 1; + } + + /* + * This cast increases alignment, but it's OK, since + * we've made sure that the pointer value is aligned. + */ + w = (const uint16_t *)(uintptr_t)b; + + if ((uintptr_t)w & 2 && len >= 2) { + /* Align bytes by handling an odd word. */ + sum += *w++; + len -= 2; + } + + /* Increases alignment. */ + d = (const uint32_t *)(uintptr_t)w; + } + + while (len >= 32) { + /* 8 dwords or 32 bytes per round. */ + + sum += *d++; + sum += *d++; + sum += *d++; + sum += *d++; + + sum += *d++; + sum += *d++; + sum += *d++; + sum += *d++; + + len -= 32; + } + + /* Last up to 7 dwords. */ + switch (len >> 2) { + case 7: + sum += *d++; + /* FALLTHROUGH */ + case 6: + sum += *d++; + /* FALLTHROUGH */ + case 5: + sum += *d++; + /* FALLTHROUGH */ + case 4: + sum += *d++; + /* FALLTHROUGH */ + case 3: + sum += *d++; + /* FALLTHROUGH */ + case 2: + sum += *d++; + /* FALLTHROUGH */ + case 1: + sum += *d++; + /* FALLTHROUGH */ + default: + break; + } + + len &= 3; + + w = (const uint16_t *)d; + if (len > 1) { + /* Last word. */ + sum += *w++; + len -= 2; + } + + if (len) { + /* Last byte. */ + b = (const uint8_t *)w; + sum += odp_cpu_to_be_16((uint16_t)*b << 8); + } + + /* + * If offset is odd, our sum is byte-flipped and we need to + * flip odd and even bytes. + */ + if (odp_unlikely(offset)) + sum = ((sum & 0xff00ff00ff00ff) << 8) | ((sum & 0xff00ff00ff00ff00) >> 8); + + return sum; +} + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/include/odp_timer_internal.h b/platform/linux-generic/include/odp_timer_internal.h index 17ebf2684..0c2a0fe4b 100644 --- a/platform/linux-generic/include/odp_timer_internal.h +++ b/platform/linux-generic/include/odp_timer_internal.h @@ -29,10 +29,13 @@ typedef struct { /* Requested expiration time */ uint64_t expiration; + /* User ptr inherited from parent timer */ - void *user_ptr; + const void *user_ptr; + /* Parent timer */ odp_timer_t timer; + } odp_timeout_hdr_t; /* A larger decrement value should be used after receiving events compared to diff --git a/platform/linux-generic/m4/configure.m4 b/platform/linux-generic/m4/configure.m4 index d4aeb455f..924346c1f 100644 --- a/platform/linux-generic/m4/configure.m4 +++ b/platform/linux-generic/m4/configure.m4 @@ -25,7 +25,7 @@ m4_include([platform/linux-generic/m4/odp_netmap.m4]) m4_include([platform/linux-generic/m4/odp_dpdk.m4]) ODP_SCHEDULER -AS_VAR_APPEND([PLAT_DEP_LIBS], ["${LIBCONFIG_LIBS} ${OPENSSL_LIBS} ${DPDK_LIBS_LT}"]) +AS_VAR_APPEND([PLAT_DEP_LIBS], ["${LIBCONFIG_LIBS} ${OPENSSL_LIBS} ${DPDK_LIBS_LT} ${LIBCLI_LIBS}"]) # Add text to the end of configure with platform specific settings. # Make sure it's aligned same as other lines in configure.ac. diff --git a/platform/linux-generic/odp_atomic.c b/platform/linux-generic/odp_atomic.c index 32ddc95c2..59253c645 100644 --- a/platform/linux-generic/odp_atomic.c +++ b/platform/linux-generic/odp_atomic.c @@ -1,10 +1,12 @@ /* Copyright (c) 2015-2018, Linaro Limited + * Copyright (c) 2021, ARM Limited * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause */ #include <odp/api/atomic.h> +#include <odp_cpu.h> int odp_atomic_lock_free_u64(odp_atomic_op_t *atomic_op) { @@ -24,3 +26,281 @@ int odp_atomic_lock_free_u64(odp_atomic_op_t *atomic_op) return 2; #endif } + +int odp_atomic_lock_free_u128(odp_atomic_op_t *atomic_op) +{ +#ifdef _ODP_LOCK_FREE_128BIT_ATOMICS + if (atomic_op) { + atomic_op->all_bits = 0; + atomic_op->op.load = 1; + atomic_op->op.store = 1; + atomic_op->op.cas = 1; + } + + return 2; +#else + /* All operations have locks */ + if (atomic_op) + atomic_op->all_bits = 0; + + return 0; +#endif +} + +#ifdef _ODP_LOCK_FREE_128BIT_ATOMICS + +static void __atomic_init_u128(odp_atomic_u128_t *atom, odp_u128_t new_val) +{ + odp_u128_t old, val; + + old = atom->v; + + while (1) { + ATOMIC_CAS_OP_128_NO_ORDER(atom, &old, new_val, val); + + if ((val.u64[0] == old.u64[0]) && (val.u64[1] == old.u64[1])) + return; + + old = val; + } +} + +static odp_u128_t __atomic_load_u128(odp_atomic_u128_t *atom) +{ + odp_u128_t val, exp; + + exp.u64[0] = 0; + exp.u64[1] = 0; + ATOMIC_CAS_OP_128_NO_ORDER(atom, &exp, exp, val); + return val; +} + +static void __atomic_store_u128(odp_atomic_u128_t *atom, odp_u128_t new_val) +{ + odp_u128_t old, val; + + old = atom->v; + + while (1) { + ATOMIC_CAS_OP_128_NO_ORDER(atom, &old, new_val, val); + + if ((val.u64[0] == old.u64[0]) && (val.u64[1] == old.u64[1])) + return; + + old = val; + } +} + +static int __atomic_cas_u128(odp_atomic_u128_t *atom, + odp_u128_t *old_val, odp_u128_t new_val) +{ + int ret = 0; + odp_u128_t val; + + ATOMIC_CAS_OP_128_NO_ORDER(atom, old_val, new_val, val); + + if ((val.u64[0] == old_val->u64[0]) && (val.u64[1] == old_val->u64[1])) + ret = 1; + + old_val->u64[0] = val.u64[0]; + old_val->u64[1] = val.u64[1]; + + return ret; +} + +static int __atomic_cas_acq_u128(odp_atomic_u128_t *atom, + odp_u128_t *old_val, odp_u128_t new_val) +{ + int ret = 0; + odp_u128_t val; + + ATOMIC_CAS_OP_128_ACQ(atom, old_val, new_val, val); + + if ((val.u64[0] == old_val->u64[0]) && (val.u64[1] == old_val->u64[1])) + ret = 1; + + old_val->u64[0] = val.u64[0]; + old_val->u64[1] = val.u64[1]; + + return ret; +} + +static int __atomic_cas_rel_u128(odp_atomic_u128_t *atom, + odp_u128_t *old_val, odp_u128_t new_val) +{ + int ret = 0; + odp_u128_t val; + + ATOMIC_CAS_OP_128_REL(atom, old_val, new_val, val); + + if ((val.u64[0] == old_val->u64[0]) && (val.u64[1] == old_val->u64[1])) + ret = 1; + + old_val->u64[0] = val.u64[0]; + old_val->u64[1] = val.u64[1]; + + return ret; +} + +static int __atomic_cas_acq_rel_u128(odp_atomic_u128_t *atom, + odp_u128_t *old_val, + odp_u128_t new_val) +{ + int ret = 0; + odp_u128_t val; + + ATOMIC_CAS_OP_128_ACQ_REL(atom, old_val, new_val, val); + + if ((val.u64[0] == old_val->u64[0]) && (val.u64[1] == old_val->u64[1])) + ret = 1; + + old_val->u64[0] = val.u64[0]; + old_val->u64[1] = val.u64[1]; + + return ret; +} + +#else /* Locked version */ + +/** + * @internal + * 128 bit store operation expression for the ATOMIC_OP macro + */ +#define ATOMIC_STORE_OP_128(new_val) \ +({ \ + (_atom)->v = (new_val); \ +}) + +/** + * @internal + * 128 bit CAS operation expression for the ATOMIC_OP macro + */ +#define ATOMIC_CAS_OP_128(ret_ptr, old_val, new_val) \ +({ \ + int *_ret_ptr = ret_ptr; \ + odp_u128_t *_old_val = old_val; \ + odp_u128_t _new_val = new_val; \ + if (((_atom)->v.u64[0] == (_old_val)->u64[0]) && \ + ((_atom)->v.u64[1] == (_old_val)->u64[1])) { \ + (_atom)->v = (_new_val); \ + *(_ret_ptr) = 1; \ + } else { \ + *(_ret_ptr) = 0; \ + } \ +}) + +/** + * @internal + * Helper macro for lock-based atomic operations on 128-bit integers + * @param[in,out] atom Pointer to the 128-bit atomic variable + * @param expr Expression used update the variable. + * @return The old value of the variable. + */ +#define ATOMIC_OP_128(atom, expr) \ +({ \ + odp_u128_t _old_val; \ + odp_atomic_u128_t *_atom = atom; \ + /* Loop while lock is already taken, stop when lock becomes clear */ \ + while (__atomic_test_and_set(&(_atom)->lock, __ATOMIC_ACQUIRE)) \ + (void)0; \ + _old_val = (_atom)->v; \ + (expr); /* Perform whatever update is desired */ \ + __atomic_clear(&(_atom)->lock, __ATOMIC_RELEASE); \ + _old_val; /* Return old value */ \ +}) + +static void __atomic_init_u128(odp_atomic_u128_t *atom, odp_u128_t val) +{ + atom->lock = 0; + ATOMIC_OP_128(atom, ATOMIC_STORE_OP_128(val)); +} + +static odp_u128_t __atomic_load_u128(odp_atomic_u128_t *atom) +{ + return ATOMIC_OP_128(atom, (void)0); +} + +static void __atomic_store_u128(odp_atomic_u128_t *atom, odp_u128_t val) +{ + ATOMIC_OP_128(atom, ATOMIC_STORE_OP_128(val)); +} + +static int __atomic_cas_u128(odp_atomic_u128_t *atom, + odp_u128_t *old_val, odp_u128_t new_val) +{ + int ret; + *old_val = ATOMIC_OP_128(atom, ATOMIC_CAS_OP_128(&ret, old_val, + new_val)); + return ret; +} + +static int __atomic_cas_acq_u128(odp_atomic_u128_t *atom, + odp_u128_t *old_val, + odp_u128_t new_val) +{ + int ret; + *old_val = ATOMIC_OP_128(atom, ATOMIC_CAS_OP_128(&ret, old_val, + new_val)); + return ret; +} + +static int __atomic_cas_rel_u128(odp_atomic_u128_t *atom, + odp_u128_t *old_val, + odp_u128_t new_val) +{ + int ret; + *old_val = ATOMIC_OP_128(atom, ATOMIC_CAS_OP_128(&ret, old_val, + new_val)); + return ret; +} + +static int __atomic_cas_acq_rel_u128(odp_atomic_u128_t *atom, + odp_u128_t *old_val, + odp_u128_t new_val) +{ + int ret; + *old_val = ATOMIC_OP_128(atom, ATOMIC_CAS_OP_128(&ret, old_val, + new_val)); + return ret; +} + +#endif + +void odp_atomic_init_u128(odp_atomic_u128_t *atom, odp_u128_t val) +{ + __atomic_init_u128(atom, val); +} + +odp_u128_t odp_atomic_load_u128(odp_atomic_u128_t *atom) +{ + return __atomic_load_u128(atom); +} + +void odp_atomic_store_u128(odp_atomic_u128_t *atom, odp_u128_t val) +{ + __atomic_store_u128(atom, val); +} + +int odp_atomic_cas_u128(odp_atomic_u128_t *atom, + odp_u128_t *old_val, odp_u128_t new_val) +{ + return __atomic_cas_u128(atom, old_val, new_val); +} + +int odp_atomic_cas_acq_u128(odp_atomic_u128_t *atom, + odp_u128_t *old_val, odp_u128_t new_val) +{ + return __atomic_cas_acq_u128(atom, old_val, new_val); +} + +int odp_atomic_cas_rel_u128(odp_atomic_u128_t *atom, + odp_u128_t *old_val, odp_u128_t new_val) +{ + return __atomic_cas_rel_u128(atom, old_val, new_val); +} + +int odp_atomic_cas_acq_rel_u128(odp_atomic_u128_t *atom, + odp_u128_t *old_val, odp_u128_t new_val) +{ + return __atomic_cas_acq_rel_u128(atom, old_val, new_val); +} diff --git a/platform/linux-generic/odp_chksum.c b/platform/linux-generic/odp_chksum.c index d018e33a3..a0336893e 100644 --- a/platform/linux-generic/odp_chksum.c +++ b/platform/linux-generic/odp_chksum.c @@ -6,31 +6,9 @@ #include <odp/api/chksum.h> #include <odp/api/std_types.h> +#include <odp_chksum_internal.h> -/* Simple implementation of ones complement sum. - * Based on RFC1071 and its errata. - */ uint16_t odp_chksum_ones_comp16(const void *p, uint32_t len) { - uint32_t sum = 0; - const uint16_t *data = p; - - while (len > 1) { - sum += *data++; - len -= 2; - } - - /* Add left-over byte, if any */ - if (len > 0) { - uint16_t left_over = 0; - - *(uint8_t *)&left_over = *(const uint8_t *)data; - sum += left_over; - } - - /* Fold 32-bit sum to 16 bits */ - while (sum >> 16) - sum = (sum & 0xffff) + (sum >> 16); - - return sum; + return chksum_finalize(chksum_partial(p, len, 0)); } diff --git a/platform/linux-generic/odp_classification.c b/platform/linux-generic/odp_classification.c index 5f452c598..bfa410b44 100644 --- a/platform/linux-generic/odp_classification.c +++ b/platform/linux-generic/odp_classification.c @@ -691,8 +691,7 @@ static int pmr_create_term(pmr_term_value_t *value, return -1; } - memset(&value->match.value, 0, MAX_PMR_TERM_SIZE); - memset(&value->match.mask, 0, MAX_PMR_TERM_SIZE); + memset(&value->match, 0, sizeof(value->match)); memcpy(&value->match.value, param->match.value, param->val_sz); memcpy(&value->match.mask, param->match.mask, param->val_sz); diff --git a/platform/linux-generic/odp_hash_crc_gen.c b/platform/linux-generic/odp_hash_crc_gen.c index 93ebe0e95..109d15420 100644 --- a/platform/linux-generic/odp_hash_crc_gen.c +++ b/platform/linux-generic/odp_hash_crc_gen.c @@ -66,15 +66,15 @@ static inline uint8_t reflect_u8(uint8_t byte) { uint8_t u8[8]; - u8[0] = (byte & (0x1 << 7)) >> 7; - u8[1] = (byte & (0x1 << 6)) >> 5; - u8[2] = (byte & (0x1 << 5)) >> 3; - u8[3] = (byte & (0x1 << 4)) >> 1; + u8[0] = (byte & (0x1u << 7)) >> 7; + u8[1] = (byte & (0x1u << 6)) >> 5; + u8[2] = (byte & (0x1u << 5)) >> 3; + u8[3] = (byte & (0x1u << 4)) >> 1; - u8[4] = (byte & (0x1 << 3)) << 1; - u8[5] = (byte & (0x1 << 2)) << 3; - u8[6] = (byte & (0x1 << 1)) << 5; - u8[7] = (byte & 0x1) << 7; + u8[4] = (byte & (0x1u << 3)) << 1; + u8[5] = (byte & (0x1u << 2)) << 3; + u8[6] = (byte & (0x1u << 1)) << 5; + u8[7] = (byte & 0x1u) << 7; return u8[0] | u8[1] | u8[2] | u8[3] | u8[4] | u8[5] | u8[6] | u8[7]; } @@ -84,10 +84,10 @@ static inline uint32_t reflect_u32(uint32_t u32) { uint8_t u8[4]; - u8[0] = reflect_u8((u32 & 0xff000000) >> 24); - u8[1] = reflect_u8((u32 & 0x00ff0000) >> 16); - u8[2] = reflect_u8((u32 & 0x0000ff00) >> 8); - u8[3] = reflect_u8(u32 & 0xff); + u8[0] = reflect_u8((u32 & 0xff000000u) >> 24); + u8[1] = reflect_u8((u32 & 0x00ff0000u) >> 16); + u8[2] = reflect_u8((u32 & 0x0000ff00u) >> 8); + u8[3] = reflect_u8(u32 & 0xffu); return (u8[3] << 24) | (u8[2] << 16) | (u8[1] << 8) | u8[0]; } @@ -97,9 +97,9 @@ static inline uint32_t reflect_u24(uint32_t u32) { uint8_t u8[4]; - u8[0] = reflect_u8((u32 & 0xff0000) >> 16); - u8[1] = reflect_u8((u32 & 0x00ff00) >> 8); - u8[2] = reflect_u8(u32 & 0xff); + u8[0] = reflect_u8((u32 & 0xff0000u) >> 16); + u8[1] = reflect_u8((u32 & 0x00ff00u) >> 8); + u8[2] = reflect_u8(u32 & 0xffu); return (u8[2] << 16) | (u8[1] << 8) | u8[0]; } @@ -109,8 +109,8 @@ static inline uint32_t reflect_u16(uint32_t u32) { uint8_t u8[4]; - u8[0] = reflect_u8((u32 & 0xff00) >> 8); - u8[1] = reflect_u8(u32 & 0xff); + u8[0] = reflect_u8((u32 & 0xff00u) >> 8); + u8[1] = reflect_u8(u32 & 0xffu); return (u8[1] << 8) | u8[0]; } @@ -128,8 +128,8 @@ static inline void crc_table_gen(uint32_t poly, int reflect, int width) crc_table->reflect = reflect; shift = width - 8; - mask = 0xffffffff >> (32 - width); - msb = 0x1 << (width - 1); + mask = 0xffffffffu >> (32 - width); + msb = 0x1u << (width - 1); if (reflect) { if (width == 32) @@ -145,7 +145,7 @@ static inline void crc_table_gen(uint32_t poly, int reflect, int width) crc = i; for (bit = 0; bit < 8; bit++) { - if (crc & 0x1) + if (crc & 0x1u) crc = poly ^ (crc >> 1); else crc = crc >> 1; @@ -173,7 +173,7 @@ static inline uint32_t crc_calc(const uint8_t *data, uint32_t data_len, uint32_t mask; shift = width - 8; - mask = 0xffffffff >> (32 - width); + mask = 0xffffffffu >> (32 - width); crc = init_val; @@ -181,7 +181,7 @@ static inline uint32_t crc_calc(const uint8_t *data, uint32_t data_len, byte = data[i]; if (reflect) { - crc = crc_table->crc[(crc ^ byte) & 0xff] ^ (crc >> 8); + crc = crc_table->crc[(crc ^ byte) & 0xffu] ^ (crc >> 8); } else { crc = crc_table->crc[(crc >> shift) ^ byte] ^ (crc << 8); diff --git a/platform/linux-generic/odp_ipsec.c b/platform/linux-generic/odp_ipsec.c index f0bbaf167..f8746f812 100644 --- a/platform/linux-generic/odp_ipsec.c +++ b/platform/linux-generic/odp_ipsec.c @@ -136,6 +136,7 @@ int odp_ipsec_capability(odp_ipsec_capability_t *capa) capa->max_queues = queue_capa.max_queues; capa->inline_ipsec_tm = ODP_SUPPORT_NO; + capa->test.sa_operations.seq_num = 1; return 0; } @@ -2040,6 +2041,26 @@ err: return in_pkt; } +int odp_ipsec_test_sa_update(odp_ipsec_sa_t sa, + odp_ipsec_test_sa_operation_t sa_op, + const odp_ipsec_test_sa_param_t *sa_param) +{ + ipsec_sa_t *ipsec_sa; + + ipsec_sa = _odp_ipsec_sa_entry_from_hdl(sa); + ODP_ASSERT(NULL != ipsec_sa); + + switch (sa_op) { + case ODP_IPSEC_TEST_SA_UPDATE_SEQ_NUM: + odp_atomic_store_u64(&ipsec_sa->hot.out.seq, sa_param->seq_num); + break; + default: + return -1; + } + + return 0; +} + int odp_ipsec_result(odp_ipsec_packet_result_t *result, odp_packet_t packet) { odp_ipsec_packet_result_t *res; @@ -2142,7 +2163,7 @@ void odp_ipsec_print(void) { ODP_PRINT("\nIPSEC print\n"); ODP_PRINT("-----------\n"); - ODP_PRINT(" max number of SA %u\n", ipsec_config->max_num_sa); + ODP_PRINT(" max number of SA %u\n\n", ipsec_config->max_num_sa); } void odp_ipsec_sa_print(odp_ipsec_sa_t sa) @@ -2151,5 +2172,5 @@ void odp_ipsec_sa_print(odp_ipsec_sa_t sa) ODP_PRINT("\nIPSEC SA print\n"); ODP_PRINT("--------------\n"); - ODP_PRINT(" SPI %u\n", ipsec_sa->spi); + ODP_PRINT(" SPI %u\n\n", ipsec_sa->spi); } diff --git a/platform/linux-generic/odp_ipsec_sad.c b/platform/linux-generic/odp_ipsec_sad.c index 0e021361c..7cae32703 100644 --- a/platform/linux-generic/odp_ipsec_sad.c +++ b/platform/linux-generic/odp_ipsec_sad.c @@ -318,6 +318,8 @@ void odp_ipsec_sa_param_init(odp_ipsec_sa_param_t *param) { memset(param, 0, sizeof(odp_ipsec_sa_param_t)); param->dest_queue = ODP_QUEUE_INVALID; + param->outbound.tunnel.ipv4.ttl = 255; + param->outbound.tunnel.ipv6.hlimit = 255; } /* Return IV length required for the cipher for IPsec use */ @@ -1007,8 +1009,9 @@ static void ipsec_out_sa_info(ipsec_sa_t *ipsec_sa, odp_ipsec_sa_info_t *sa_info static void ipsec_in_sa_info(ipsec_sa_t *ipsec_sa, odp_ipsec_sa_info_t *sa_info) { - if (ipsec_sa->param.mode == ODP_IPSEC_MODE_TUNNEL) { - uint8_t *dst = sa_info->inbound.lookup_param.dst_addr; + uint8_t *dst = sa_info->inbound.lookup_param.dst_addr; + + if (ipsec_sa->lookup_mode == ODP_IPSEC_LOOKUP_DSTADDR_SPI) { if (ipsec_sa->param.inbound.lookup_param.ip_version == ODP_IPSEC_IPV4) @@ -1018,8 +1021,8 @@ static void ipsec_in_sa_info(ipsec_sa_t *ipsec_sa, odp_ipsec_sa_info_t *sa_info) memcpy(dst, &ipsec_sa->in.lookup_dst_ipv6, ODP_IPV6_ADDR_SIZE); - sa_info->param.inbound.lookup_param.dst_addr = dst; } + sa_info->param.inbound.lookup_param.dst_addr = dst; if (ipsec_sa->antireplay) { sa_info->inbound.antireplay_ws = IPSEC_ANTIREPLAY_WS; diff --git a/platform/linux-generic/odp_packet.c b/platform/linux-generic/odp_packet.c index 2fe7fe67d..752ff8416 100644 --- a/platform/linux-generic/odp_packet.c +++ b/platform/linux-generic/odp_packet.c @@ -11,6 +11,7 @@ #include <odp/api/plat/packet_inlines.h> #include <odp_packet_internal.h> #include <odp_debug_internal.h> +#include <odp_chksum_internal.h> #include <odp_errno_define.h> #include <odp/api/hints.h> #include <odp/api/byteorder.h> @@ -1785,73 +1786,12 @@ int _odp_packet_copy_md_to_packet(odp_packet_t srcpkt, odp_packet_t dstpkt) return dst_uarea_size < src_uarea_size; } -/* Simple implementation of ones complement sum. - * Based on RFC1071 and its errata. - */ -typedef union { - uint16_t w; - uint8_t b[2]; -} swap_buf_t; - -static uint32_t segment_sum16_32(const uint8_t *p, - uint32_t len, - uint32_t offset) - +static uint64_t packet_sum_partial(odp_packet_hdr_t *pkt_hdr, + uint32_t l3_offset, + uint32_t offset, + uint32_t len) { - uint32_t sum = 0; - - /* Include second part of 16-bit short word split between segments */ - if (len > 0 && (offset % 2)) { - swap_buf_t sw; - - sw.b[0] = 0; - sw.b[1] = *p++; - sum = sw.w; - len--; - } - - /* - * If pointer is 16-bit aligned, we can do fast path calculation. - * If it is not, we sum hi and lo bytes separately and then sum them. - */ - if ((uintptr_t)p % 2) { - uint32_t sum1 = 0, sum2 = 0; - - while (len > 1) { - sum1 += *p++; - sum2 += *p++; - len -= 2; - } -#if (ODP_BYTE_ORDER == ODP_BIG_ENDIAN) - sum += sum2 + (sum1 << 8); -#else - sum += sum1 + (sum2 << 8); -#endif - } else { - while (len > 1) { - sum += *(const uint16_t *)(uintptr_t)p; - p += 2; - len -= 2; - } - } - - /* Add left-over byte, if any */ - if (len > 0) { - swap_buf_t sw; - - sw.b[0] = *p; - sw.b[1] = 0; - sum += sw.w; - } - - return sum; -} - -static uint32_t packet_sum16_32(odp_packet_hdr_t *pkt_hdr, - uint32_t offset, - uint32_t len) -{ - uint32_t sum = 0; + uint64_t sum = 0; if (offset + len > pkt_hdr->frame_len) return 0; @@ -1863,7 +1803,7 @@ static uint32_t packet_sum16_32(odp_packet_hdr_t *pkt_hdr, if (seglen > len) seglen = len; - sum += segment_sum16_32(mapaddr, seglen, offset); + sum += chksum_partial(mapaddr, seglen, offset - l3_offset); len -= seglen; offset += seglen; } @@ -1871,20 +1811,14 @@ static uint32_t packet_sum16_32(odp_packet_hdr_t *pkt_hdr, return sum; } -static uint16_t packet_sum_ones_comp16(odp_packet_hdr_t *pkt_hdr, - uint32_t offset, - uint32_t len, - uint32_t l4_part_sum) +static inline uint16_t packet_sum(odp_packet_hdr_t *pkt_hdr, + uint32_t l3_offset, + uint32_t offset, + uint32_t len, + uint64_t sum) { - uint32_t sum = l4_part_sum; - - sum += packet_sum16_32(pkt_hdr, offset, len); - - /* Not more than two additions */ - sum = (sum & 0xffff) + (sum >> 16); - sum = (sum & 0xffff) + (sum >> 16); - - return sum; + sum += packet_sum_partial(pkt_hdr, l3_offset, offset, len); + return chksum_finalize(sum); } static uint32_t packet_sum_crc32c(odp_packet_hdr_t *pkt_hdr, @@ -1999,7 +1933,7 @@ error: static inline uint8_t parse_ipv4(packet_parser_t *prs, const uint8_t **parseptr, uint32_t *offset, uint32_t frame_len, odp_proto_chksums_t chksums, - uint32_t *l4_part_sum) + uint64_t *l4_part_sum) { const _odp_ipv4hdr_t *ipv4 = (const _odp_ipv4hdr_t *)*parseptr; uint32_t dstaddr = odp_be_to_cpu_32(ipv4->dst_addr); @@ -2017,7 +1951,7 @@ static inline uint8_t parse_ipv4(packet_parser_t *prs, const uint8_t **parseptr, if (chksums.chksum.ipv4) { prs->input_flags.l3_chksum_done = 1; - if (odp_chksum_ones_comp16(ipv4, ihl * 4) != 0xffff) { + if (chksum_finalize(chksum_partial(ipv4, ihl * 4, 0)) != 0xffff) { prs->flags.ip_err = 1; prs->flags.l3_chksum_err = 1; return 0; @@ -2028,8 +1962,8 @@ static inline uint8_t parse_ipv4(packet_parser_t *prs, const uint8_t **parseptr, *parseptr += ihl * 4; if (chksums.chksum.udp || chksums.chksum.tcp) - *l4_part_sum = segment_sum16_32((const uint8_t *)&ipv4->src_addr, - 2 * _ODP_IPV4ADDR_LEN, 0); + *l4_part_sum = chksum_partial((const uint8_t *)&ipv4->src_addr, + 2 * _ODP_IPV4ADDR_LEN, 0); if (odp_unlikely(ihl > _ODP_IPV4HDR_IHL_MIN)) prs->input_flags.ipopt = 1; @@ -2059,7 +1993,7 @@ static inline uint8_t parse_ipv6(packet_parser_t *prs, const uint8_t **parseptr, uint32_t *offset, uint32_t frame_len, uint32_t seg_len, odp_proto_chksums_t chksums, - uint32_t *l4_part_sum) + uint64_t *l4_part_sum) { const _odp_ipv6hdr_t *ipv6 = (const _odp_ipv6hdr_t *)*parseptr; const _odp_ipv6hdr_ext_t *ipv6ext; @@ -2083,8 +2017,8 @@ static inline uint8_t parse_ipv6(packet_parser_t *prs, const uint8_t **parseptr, *parseptr += sizeof(_odp_ipv6hdr_t); if (chksums.chksum.udp || chksums.chksum.tcp) - *l4_part_sum = segment_sum16_32((const uint8_t *)&ipv6->src_addr, - 2 * _ODP_IPV6ADDR_LEN, 0); + *l4_part_sum = chksum_partial((const uint8_t *)&ipv6->src_addr, + 2 * _ODP_IPV6ADDR_LEN, 0); /* Skip past any IPv6 extension headers */ if (ipv6->next_hdr == _ODP_IPPROTO_HOPOPTS || @@ -2127,7 +2061,7 @@ static inline uint8_t parse_ipv6(packet_parser_t *prs, const uint8_t **parseptr, static inline void parse_tcp(packet_parser_t *prs, const uint8_t **parseptr, uint16_t tcp_len, odp_proto_chksums_t chksums, - uint32_t *l4_part_sum) + uint64_t *l4_part_sum) { const _odp_tcphdr_t *tcp = (const _odp_tcphdr_t *)*parseptr; uint32_t len = tcp->hl * 4; @@ -2153,7 +2087,7 @@ static inline void parse_tcp(packet_parser_t *prs, const uint8_t **parseptr, */ static inline void parse_udp(packet_parser_t *prs, const uint8_t **parseptr, odp_proto_chksums_t chksums, - uint32_t *l4_part_sum) + uint64_t *l4_part_sum) { const _odp_udphdr_t *udp = (const _odp_udphdr_t *)*parseptr; uint32_t udplen = odp_be_to_cpu_16(udp->length); @@ -2200,7 +2134,7 @@ static inline void parse_udp(packet_parser_t *prs, const uint8_t **parseptr, static inline void parse_sctp(packet_parser_t *prs, const uint8_t **parseptr, uint16_t sctp_len, odp_proto_chksums_t chksums, - uint32_t *l4_part_sum) + uint64_t *l4_part_sum) { if (odp_unlikely(sctp_len < sizeof(_odp_sctphdr_t))) { prs->flags.sctp_err = 1; @@ -2228,7 +2162,7 @@ int packet_parse_common_l3_l4(packet_parser_t *prs, const uint8_t *parseptr, uint32_t frame_len, uint32_t seg_len, int layer, uint16_t ethtype, odp_proto_chksums_t chksums, - uint32_t *l4_part_sum) + uint64_t *l4_part_sum) { uint8_t ip_proto; @@ -2341,7 +2275,7 @@ int _odp_packet_parse_common(packet_parser_t *prs, const uint8_t *ptr, uint32_t offset; uint16_t ethtype; const uint8_t *parseptr; - uint32_t l4_part_sum; + uint64_t l4_part_sum; parseptr = ptr; offset = 0; @@ -2378,7 +2312,7 @@ static inline int packet_ipv4_chksum(odp_packet_t pkt, if (odp_unlikely(res < 0)) return res; - *chksum = ~odp_chksum_ones_comp16(buf, nleft); + *chksum = ~chksum_finalize(chksum_partial(buf, nleft, 0)); return 0; } @@ -2426,7 +2360,7 @@ static int _odp_packet_tcp_udp_chksum_insert(odp_packet_t pkt, uint16_t proto) { odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); uint32_t zero = 0; - uint32_t sum; + uint64_t sum; uint16_t l3_ver; uint16_t chksum; uint32_t chksum_offset; @@ -2439,15 +2373,17 @@ static int _odp_packet_tcp_udp_chksum_insert(odp_packet_t pkt, uint16_t proto) odp_packet_copy_to_mem(pkt, pkt_hdr->p.l3_offset, 2, &l3_ver); if (_ODP_IPV4HDR_VER(l3_ver) == _ODP_IPV4) - sum = packet_sum16_32(pkt_hdr, - pkt_hdr->p.l3_offset + - _ODP_IPV4ADDR_OFFSSET, - 2 * _ODP_IPV4ADDR_LEN); + sum = packet_sum_partial(pkt_hdr, + pkt_hdr->p.l3_offset, + pkt_hdr->p.l3_offset + + _ODP_IPV4ADDR_OFFSSET, + 2 * _ODP_IPV4ADDR_LEN); else - sum = packet_sum16_32(pkt_hdr, - pkt_hdr->p.l3_offset + - _ODP_IPV6ADDR_OFFSSET, - 2 * _ODP_IPV6ADDR_LEN); + sum = packet_sum_partial(pkt_hdr, + pkt_hdr->p.l3_offset, + pkt_hdr->p.l3_offset + + _ODP_IPV6ADDR_OFFSSET, + 2 * _ODP_IPV6ADDR_LEN); #if ODP_BYTE_ORDER == ODP_BIG_ENDIAN sum += proto; #else @@ -2459,24 +2395,22 @@ static int _odp_packet_tcp_udp_chksum_insert(odp_packet_t pkt, uint16_t proto) pkt_hdr->p.l4_offset); chksum_offset = pkt_hdr->p.l4_offset + _ODP_UDP_CSUM_OFFSET; } else { - sum += packet_sum16_32(pkt_hdr, - pkt_hdr->p.l4_offset + - _ODP_UDP_LEN_OFFSET, - 2); + sum += packet_sum_partial(pkt_hdr, + pkt_hdr->p.l3_offset, + pkt_hdr->p.l4_offset + + _ODP_UDP_LEN_OFFSET, + 2); chksum_offset = pkt_hdr->p.l4_offset + _ODP_UDP_CSUM_OFFSET; } odp_packet_copy_from_mem(pkt, chksum_offset, 2, &zero); - sum += packet_sum16_32(pkt_hdr, - pkt_hdr->p.l4_offset, - pkt_hdr->frame_len - - pkt_hdr->p.l4_offset); - - /* Not more than two additions */ - sum = (sum & 0xffff) + (sum >> 16); - sum = (sum & 0xffff) + (sum >> 16); + sum += packet_sum_partial(pkt_hdr, + pkt_hdr->p.l3_offset, + pkt_hdr->p.l4_offset, + pkt_hdr->frame_len - + pkt_hdr->p.l4_offset); - chksum = ~sum; + chksum = ~chksum_finalize(sum); if (proto == _ODP_IPPROTO_UDP && chksum == 0) chksum = 0xffff; @@ -2538,18 +2472,19 @@ int _odp_packet_sctp_chksum_insert(odp_packet_t pkt) static int packet_l4_chksum(odp_packet_hdr_t *pkt_hdr, odp_proto_chksums_t chksums, - uint32_t l4_part_sum) + uint64_t l4_part_sum) { /* UDP chksum == 0 case is covered in parse_udp() */ if (chksums.chksum.udp && pkt_hdr->p.input_flags.udp && !pkt_hdr->p.input_flags.ipfrag && !pkt_hdr->p.input_flags.udp_chksum_zero) { - uint16_t sum = ~packet_sum_ones_comp16(pkt_hdr, - pkt_hdr->p.l4_offset, - pkt_hdr->frame_len - - pkt_hdr->p.l4_offset, - l4_part_sum); + uint16_t sum = ~packet_sum(pkt_hdr, + pkt_hdr->p.l3_offset, + pkt_hdr->p.l4_offset, + pkt_hdr->frame_len - + pkt_hdr->p.l4_offset, + l4_part_sum); pkt_hdr->p.input_flags.l4_chksum_done = 1; if (sum != 0) { @@ -2562,11 +2497,12 @@ static int packet_l4_chksum(odp_packet_hdr_t *pkt_hdr, if (chksums.chksum.tcp && pkt_hdr->p.input_flags.tcp && !pkt_hdr->p.input_flags.ipfrag) { - uint16_t sum = ~packet_sum_ones_comp16(pkt_hdr, - pkt_hdr->p.l4_offset, - pkt_hdr->frame_len - - pkt_hdr->p.l4_offset, - l4_part_sum); + uint16_t sum = ~packet_sum(pkt_hdr, + pkt_hdr->p.l3_offset, + pkt_hdr->p.l4_offset, + pkt_hdr->frame_len - + pkt_hdr->p.l4_offset, + l4_part_sum); pkt_hdr->p.input_flags.l4_chksum_done = 1; if (sum != 0) { @@ -2613,7 +2549,7 @@ int _odp_packet_parse_layer(odp_packet_hdr_t *pkt_hdr, const uint8_t *base = packet_data(pkt_hdr); uint32_t offset = 0; uint16_t ethtype; - uint32_t l4_part_sum = 0; + uint64_t l4_part_sum = 0; int rc; if (odp_unlikely(layer == ODP_PROTO_LAYER_NONE)) @@ -2649,7 +2585,7 @@ int odp_packet_parse(odp_packet_t pkt, uint32_t offset, odp_proto_layer_t layer = param->last_layer; int ret; uint16_t ethtype; - uint32_t l4_part_sum = 0; + uint64_t l4_part_sum = 0; if (proto == ODP_PROTO_NONE || layer == ODP_PROTO_LAYER_NONE) return -1; diff --git a/platform/linux-generic/odp_packet_io.c b/platform/linux-generic/odp_packet_io.c index 9363b315a..442aa0d94 100644 --- a/platform/linux-generic/odp_packet_io.c +++ b/platform/linux-generic/odp_packet_io.c @@ -1656,8 +1656,7 @@ void odp_pktio_print(odp_pktio_t hdl) return; } - len += snprintf(&str[len], n - len, - "pktio\n"); + len += snprintf(&str[len], n - len, "Pktio info\n----------\n"); len += snprintf(&str[len], n - len, " name %s\n", entry->s.name); len += snprintf(&str[len], n - len, diff --git a/platform/linux-generic/odp_queue_basic.c b/platform/linux-generic/odp_queue_basic.c index 0ffad0807..2d2ed3c7f 100644 --- a/platform/linux-generic/odp_queue_basic.c +++ b/platform/linux-generic/odp_queue_basic.c @@ -767,6 +767,97 @@ static void queue_print(odp_queue_t handle) UNLOCK(queue); } +static void queue_print_all(void) +{ + uint32_t i, index, len, max_len; + const char *name; + int status; + odp_queue_type_t type; + odp_nonblocking_t blocking; + odp_queue_op_mode_t enq_mode; + odp_queue_op_mode_t deq_mode; + odp_queue_order_t order; + const char *status_str; + const char *bl_str; + char type_c, enq_c, deq_c, order_c, sync_c; + const int col_width = 24; + int prio = 0; + odp_schedule_sync_t sync = ODP_SCHED_SYNC_PARALLEL; + + ODP_PRINT("\nList of all queues\n"); + ODP_PRINT("------------------\n"); + ODP_PRINT(" idx %-*s type stat blk enq deq ord len max_len sync prio\n", col_width, "name"); + + for (i = 0; i < CONFIG_MAX_QUEUES; i++) { + queue_entry_t *queue = qentry_from_index(i); + + if (queue->s.status < QUEUE_STATUS_READY) + continue; + + LOCK(queue); + + status = queue->s.status; + index = queue->s.index; + name = queue->s.name; + type = queue->s.type; + blocking = queue->s.param.nonblocking; + enq_mode = queue->s.param.enq_mode; + deq_mode = queue->s.param.deq_mode; + order = queue->s.param.order; + + if (queue->s.queue_lf) { + len = _odp_queue_lf_length(queue->s.queue_lf); + max_len = _odp_queue_lf_max_length(); + } else if (queue->s.spsc) { + len = ring_spsc_length(&queue->s.ring_spsc); + max_len = queue->s.ring_mask + 1; + } else if (type == ODP_QUEUE_TYPE_SCHED) { + len = ring_st_length(&queue->s.ring_st); + max_len = queue->s.ring_mask + 1; + prio = queue->s.param.sched.prio; + sync = queue->s.param.sched.sync; + } else { + len = ring_mpmc_length(&queue->s.ring_mpmc); + max_len = queue->s.ring_mask + 1; + } + + UNLOCK(queue); + + if (status < QUEUE_STATUS_READY) + continue; + + status_str = (status == QUEUE_STATUS_READY) ? "R" : + ((status == QUEUE_STATUS_SCHED) ? "S" : "NS"); + + type_c = (type == ODP_QUEUE_TYPE_PLAIN) ? 'P' : 'S'; + + bl_str = (blocking == ODP_BLOCKING) ? "B" : + ((blocking == ODP_NONBLOCKING_LF) ? "LF" : "WF"); + + enq_c = (enq_mode == ODP_QUEUE_OP_MT) ? 'S' : + ((enq_mode == ODP_QUEUE_OP_MT_UNSAFE) ? 'U' : 'D'); + + deq_c = (deq_mode == ODP_QUEUE_OP_MT) ? 'S' : + ((deq_mode == ODP_QUEUE_OP_MT_UNSAFE) ? 'U' : 'D'); + + order_c = (order == ODP_QUEUE_ORDER_KEEP) ? 'K' : 'I'; + + ODP_PRINT("%4u %-*s %c %2s %2s", index, col_width, name, type_c, + status_str, bl_str); + ODP_PRINT(" %c %c %c %6u %6u", enq_c, deq_c, order_c, len, max_len); + + if (type == ODP_QUEUE_TYPE_SCHED) { + sync_c = (sync == ODP_SCHED_SYNC_PARALLEL) ? 'P' : + ((sync == ODP_SCHED_SYNC_ATOMIC) ? 'A' : 'O'); + ODP_PRINT(" %c %4i", sync_c, prio); + } + + ODP_PRINT("\n"); + } + + ODP_PRINT("\n"); +} + static inline int _sched_queue_enq_multi(odp_queue_t handle, odp_buffer_hdr_t *buf_hdr[], int num) { @@ -1137,7 +1228,9 @@ _odp_queue_api_fn_t _odp_queue_basic_api = { .queue_to_u64 = queue_to_u64, .queue_param_init = queue_param_init, .queue_info = queue_info, - .queue_print = queue_print + .queue_print = queue_print, + .queue_print_all = queue_print_all + }; /* Functions towards internal components */ diff --git a/platform/linux-generic/odp_queue_if.c b/platform/linux-generic/odp_queue_if.c index d4b1c550c..9ebd5db86 100644 --- a/platform/linux-generic/odp_queue_if.c +++ b/platform/linux-generic/odp_queue_if.c @@ -88,7 +88,7 @@ uint64_t odp_queue_to_u64(odp_queue_t hdl) void odp_queue_param_init(odp_queue_param_t *param) { - return _odp_queue_api->queue_param_init(param); + _odp_queue_api->queue_param_init(param); } int odp_queue_info(odp_queue_t queue, odp_queue_info_t *info) @@ -98,7 +98,12 @@ int odp_queue_info(odp_queue_t queue, odp_queue_info_t *info) void odp_queue_print(odp_queue_t queue) { - return _odp_queue_api->queue_print(queue); + _odp_queue_api->queue_print(queue); +} + +void odp_queue_print_all(void) +{ + _odp_queue_api->queue_print_all(); } int _odp_queue_init_global(void) diff --git a/platform/linux-generic/odp_queue_scalable.c b/platform/linux-generic/odp_queue_scalable.c index 02bf9fbc2..d70e174e0 100644 --- a/platform/linux-generic/odp_queue_scalable.c +++ b/platform/linux-generic/odp_queue_scalable.c @@ -15,6 +15,7 @@ #include <odp/api/sync.h> #include <odp/api/plat/sync_inlines.h> #include <odp/api/traffic_mngr.h> +#include <odp/api/cpu.h> #include <odp_config_internal.h> #include <odp_debug_internal.h> @@ -465,7 +466,7 @@ static int queue_destroy(odp_queue_t handle) sevl(); while (wfe() && monitor32((uint32_t *)&q->qschst.numevts, __ATOMIC_RELAXED) != 0) - doze(); + odp_cpu_pause(); } if (q->schedq != NULL) { @@ -573,7 +574,7 @@ static inline int _odp_queue_enq(sched_elem_t *q, sevl(); while (wfe() && monitor32(&q->cons_write, __ATOMIC_RELAXED) != old_write) - doze(); + odp_cpu_pause(); } /* Signal consumers that events are available (release events) @@ -803,7 +804,7 @@ inline int _odp_queue_deq(sched_elem_t *q, odp_buffer_hdr_t *buf_hdr[], int num) sevl(); while (wfe() && monitor32(&q->prod_read, __ATOMIC_RELAXED) != old_read) - doze(); + odp_cpu_pause(); } /* Signal producers that empty slots are available @@ -1013,6 +1014,76 @@ static void queue_print(odp_queue_t handle) UNLOCK(&queue->s.lock); } +static void queue_print_all(void) +{ + uint32_t i, index; + const char *name; + int status; + odp_queue_type_t type; + odp_nonblocking_t blocking; + odp_queue_op_mode_t enq_mode; + odp_queue_op_mode_t deq_mode; + odp_queue_order_t order; + const char *bl_str; + char type_c, enq_c, deq_c, order_c, sync_c; + const int col_width = 24; + int prio = 0; + odp_schedule_sync_t sync = ODP_SCHED_SYNC_PARALLEL; + + ODP_PRINT("\nList of all queues\n"); + ODP_PRINT("------------------\n"); + ODP_PRINT(" idx %-*s type blk enq deq ord sync prio\n", col_width, "name"); + + for (i = 0; i < CONFIG_MAX_QUEUES; i++) { + queue_entry_t *queue = &queue_tbl->queue[i]; + + if (queue->s.status != QUEUE_STATUS_READY) + continue; + + LOCK(&queue->s.lock); + + status = queue->s.status; + index = queue->s.index; + name = queue->s.name; + type = queue->s.type; + blocking = queue->s.param.nonblocking; + enq_mode = queue->s.param.enq_mode; + deq_mode = queue->s.param.deq_mode; + order = queue->s.param.order; + + UNLOCK(&queue->s.lock); + + if (status != QUEUE_STATUS_READY) + continue; + + type_c = (type == ODP_QUEUE_TYPE_PLAIN) ? 'P' : 'S'; + + bl_str = (blocking == ODP_BLOCKING) ? "B" : + ((blocking == ODP_NONBLOCKING_LF) ? "LF" : "WF"); + + enq_c = (enq_mode == ODP_QUEUE_OP_MT) ? 'S' : + ((enq_mode == ODP_QUEUE_OP_MT_UNSAFE) ? 'U' : 'D'); + + deq_c = (deq_mode == ODP_QUEUE_OP_MT) ? 'S' : + ((deq_mode == ODP_QUEUE_OP_MT_UNSAFE) ? 'U' : 'D'); + + order_c = (order == ODP_QUEUE_ORDER_KEEP) ? 'K' : 'I'; + + ODP_PRINT("%4u %-*s %c %2s", index, col_width, name, type_c, bl_str); + ODP_PRINT(" %c %c %c", enq_c, deq_c, order_c); + + if (type == ODP_QUEUE_TYPE_SCHED) { + sync_c = (sync == ODP_SCHED_SYNC_PARALLEL) ? 'P' : + ((sync == ODP_SCHED_SYNC_ATOMIC) ? 'A' : 'O'); + ODP_PRINT(" %c %4i", sync_c, prio); + } + + ODP_PRINT("\n"); + } + + ODP_PRINT("\n"); +} + static uint64_t queue_to_u64(odp_queue_t hdl) { return _odp_pri(hdl); @@ -1099,7 +1170,8 @@ _odp_queue_api_fn_t _odp_queue_scalable_api = { .queue_to_u64 = queue_to_u64, .queue_param_init = queue_param_init, .queue_info = queue_info, - .queue_print = queue_print + .queue_print = queue_print, + .queue_print_all = queue_print_all }; /* Functions towards internal components */ diff --git a/platform/linux-generic/odp_schedule_basic.c b/platform/linux-generic/odp_schedule_basic.c index 0396123f3..32a21442d 100644 --- a/platform/linux-generic/odp_schedule_basic.c +++ b/platform/linux-generic/odp_schedule_basic.c @@ -1633,7 +1633,7 @@ static int schedule_num_grps(void) static void schedule_get_config(schedule_config_t *config) { - *config = *(&sched->config_if); + *config = sched->config_if; }; static int schedule_capability(odp_schedule_capability_t *capa) diff --git a/platform/linux-generic/odp_schedule_scalable.c b/platform/linux-generic/odp_schedule_scalable.c index 4da5c378d..2ba40256b 100644 --- a/platform/linux-generic/odp_schedule_scalable.c +++ b/platform/linux-generic/odp_schedule_scalable.c @@ -227,7 +227,7 @@ void _odp_sched_update_enq(sched_elem_t *q, uint32_t actual) while (wfe() && monitor8(&q->qschst.cur_ticket, __ATOMIC_ACQUIRE) != ticket) - doze(); + odp_cpu_pause(); } /* Enqueue at end of scheduler queue */ /* We are here because of empty-to-non-empty transition @@ -375,7 +375,7 @@ sched_update_deq(sched_elem_t *q, while (wfe() && monitor8(&q->qschst.cur_ticket, __ATOMIC_ACQUIRE) != ticket) - doze(); + odp_cpu_pause(); } /* We are here because of non-empty-to-empty transition or * WRR budget exhausted @@ -502,7 +502,7 @@ static inline void sched_update_popd(sched_elem_t *elem) sevl(); while (wfe() && monitor8(&elem->qschst.cur_ticket, __ATOMIC_ACQUIRE) != ticket) - doze(); + odp_cpu_pause(); } sched_update_popd_sc(elem); atomic_store_release(&elem->qschst.cur_ticket, ticket + 1, @@ -1074,7 +1074,7 @@ restart_same: while (wfe() && monitor32(&rwin->turn, __ATOMIC_ACQUIRE) != sn) - doze(); + odp_cpu_pause(); } #ifdef CONFIG_QSCHST_LOCK LOCK(&elem->qschlock); @@ -1162,7 +1162,7 @@ static void schedule_order_lock(uint32_t lock_index) while (wfe() && monitor32(&rctx->rwin->olock[lock_index], __ATOMIC_ACQUIRE) != rctx->sn) - doze(); + odp_cpu_pause(); } } @@ -1579,7 +1579,7 @@ static int schedule_group_destroy(odp_schedule_group_t group) !bitset_is_eql(wanted, bitset_monitor(&sg->thr_actual[p], __ATOMIC_RELAXED))) - doze(); + odp_cpu_pause(); } /* Else ignore because no ODP queues on this prio */ } @@ -2157,7 +2157,7 @@ static void order_lock(void) */ while (wfe() && monitor32(&rwin->hc.head, __ATOMIC_ACQUIRE) != sn) - doze(); + odp_cpu_pause(); } } diff --git a/platform/linux-generic/odp_schedule_sp.c b/platform/linux-generic/odp_schedule_sp.c index 90ba101cd..cad391ed4 100644 --- a/platform/linux-generic/odp_schedule_sp.c +++ b/platform/linux-generic/odp_schedule_sp.c @@ -1037,7 +1037,7 @@ static int schedule_capability(odp_schedule_capability_t *capa) static void get_config(schedule_config_t *config) { - *config = *(&sched_global->config_if); + *config = sched_global->config_if; }; /* Fill in scheduler interface */ diff --git a/platform/linux-generic/odp_timer.c b/platform/linux-generic/odp_timer.c index ce33c6787..8b2ba16c2 100644 --- a/platform/linux-generic/odp_timer.c +++ b/platform/linux-generic/odp_timer.c @@ -26,6 +26,7 @@ #include <odp/api/align.h> #include <odp_align_internal.h> #include <odp/api/atomic.h> +#include <odp/api/plat/atomic_inlines.h> #include <odp_atomic_internal.h> #include <odp/api/buffer.h> #include <odp/api/cpu.h> @@ -106,7 +107,7 @@ ODP_STATIC_ASSERT(sizeof(tick_buf_t) == 16, "sizeof(tick_buf_t) == 16"); #endif typedef struct { - void *user_ptr; + const void *user_ptr; odp_queue_t queue;/* Used for free list when timer is free */ } _odp_timer_t; @@ -186,10 +187,7 @@ static __thread timer_local_t timer_local; static void itimer_init(timer_pool_t *tp); static void itimer_fini(timer_pool_t *tp); -static void timer_init(_odp_timer_t *tim, - tick_buf_t *tb, - odp_queue_t _q, - void *_up) +static void timer_init(_odp_timer_t *tim, tick_buf_t *tb, odp_queue_t _q, const void *_up) { tim->queue = _q; tim->user_ptr = _up; @@ -200,7 +198,7 @@ static void timer_init(_odp_timer_t *tim, #if __GCC_ATOMIC_LLONG_LOCK_FREE < 2 tb->exp_tck.v = TMO_INACTIVE; #else - _odp_atomic_u64_store_mm(&tb->exp_tck, TMO_INACTIVE, _ODP_MEMMODEL_RLS); + odp_atomic_store_rel_u64(&tb->exp_tck, TMO_INACTIVE); #endif } @@ -487,9 +485,7 @@ static void odp_timer_pool_del(timer_pool_t *tp) ODP_ABORT("Failed to free shared memory (%d)\n", rc); } -static inline odp_timer_t timer_alloc(timer_pool_t *tp, - odp_queue_t queue, - void *user_ptr) +static inline odp_timer_t timer_alloc(timer_pool_t *tp, odp_queue_t queue, const void *user_ptr) { odp_timer_t hdl; @@ -504,13 +500,12 @@ static inline odp_timer_t timer_alloc(timer_pool_t *tp, tp->first_free = get_next_free(tim); /* Initialize timer */ timer_init(tim, &tp->tick_buf[idx], queue, user_ptr); - if (odp_unlikely(tp->num_alloc > - odp_atomic_load_u32(&tp->high_wm))) + if (odp_unlikely(tp->num_alloc > odp_atomic_load_u32(&tp->high_wm))) { /* Update high_wm last with release model to * ensure timer initialization is visible */ - _odp_atomic_u32_store_mm(&tp->high_wm, - tp->num_alloc, - _ODP_MEMMODEL_RLS); + odp_atomic_store_rel_u32(&tp->high_wm, tp->num_alloc); + } + hdl = tp_idx_to_handle(tp, idx); /* Add timer to queue */ _odp_queue_fn->timer_add(queue); @@ -602,8 +597,8 @@ static bool timer_reset(uint32_t idx, uint64_t abs_tck, odp_buffer_t *tmo_buf, uint64_t old; /* Swap in new expiration tick, get back old tick which * will indicate active/inactive timer state */ - old = _odp_atomic_u64_xchg_mm(&tb->exp_tck, abs_tck, - _ODP_MEMMODEL_RLX); + old = odp_atomic_xchg_u64(&tb->exp_tck, abs_tck); + if ((old & TMO_INACTIVE) != 0) { /* Timer was inactive (cancelled or expired), * we can't reset a timer without a timeout buffer. @@ -615,12 +610,7 @@ static bool timer_reset(uint32_t idx, uint64_t abs_tck, odp_buffer_t *tmo_buf, * reset or cancelled the timer. Without any * synchronization between the threads, we have a * data race and the behavior is undefined */ - (void)_odp_atomic_u64_cmp_xchg_strong_mm( - &tb->exp_tck, - &abs_tck, - old, - _ODP_MEMMODEL_RLX, - _ODP_MEMMODEL_RLX); + (void)odp_atomic_cas_u64(&tb->exp_tck, &abs_tck, old); success = false; } #else /* Target supports neither 128-bit nor 64-bit CAS => use lock */ @@ -806,7 +796,7 @@ static inline void timer_expire(timer_pool_t *tp, uint32_t idx, uint64_t tick) uint64_t exp_tck; #ifdef ODP_ATOMIC_U128 /* Atomic re-read for correctness */ - exp_tck = _odp_atomic_u64_load_mm(&tb->exp_tck, _ODP_MEMMODEL_RLX); + exp_tck = odp_atomic_load_u64(&tb->exp_tck); /* Re-check exp_tck */ if (odp_likely(exp_tck <= tick)) { /* Attempt to grab timeout buffer, replace with inactive timer @@ -888,8 +878,7 @@ static inline void timer_expire(timer_pool_t *tp, uint32_t idx, uint64_t tick) static inline void timer_pool_scan(timer_pool_t *tp, uint64_t tick) { tick_buf_t *array = &tp->tick_buf[0]; - uint32_t high_wm = _odp_atomic_u32_load_mm(&tp->high_wm, - _ODP_MEMMODEL_ACQ); + uint32_t high_wm = odp_atomic_load_acq_u32(&tp->high_wm); uint32_t i; ODP_ASSERT(high_wm <= tp->param.num_timers); @@ -1364,9 +1353,7 @@ uint64_t odp_timer_pool_to_u64(odp_timer_pool_t tpid) return _odp_pri(tpid); } -odp_timer_t odp_timer_alloc(odp_timer_pool_t tpid, - odp_queue_t queue, - void *user_ptr) +odp_timer_t odp_timer_alloc(odp_timer_pool_t tpid, odp_queue_t queue, const void *user_ptr) { timer_pool_t *tp = timer_pool_from_hdl(tpid); @@ -1497,7 +1484,7 @@ uint64_t odp_timeout_tick(odp_timeout_t tmo) void *odp_timeout_user_ptr(odp_timeout_t tmo) { - return timeout_hdr(tmo)->user_ptr; + return (void *)(uintptr_t)timeout_hdr(tmo)->user_ptr; } odp_timeout_t odp_timeout_alloc(odp_pool_t pool) @@ -1516,6 +1503,81 @@ void odp_timeout_free(odp_timeout_t tmo) odp_buffer_free(odp_buffer_from_event(ev)); } +void odp_timer_pool_print(odp_timer_pool_t timer_pool) +{ + timer_pool_t *tp; + + if (timer_pool == ODP_TIMER_POOL_INVALID) { + ODP_ERR("Bad timer pool handle\n"); + return; + } + + tp = timer_pool_from_hdl(timer_pool); + + ODP_PRINT("\nTimer pool info\n"); + ODP_PRINT("---------------\n"); + ODP_PRINT(" timer pool %p\n", tp); + ODP_PRINT(" tp index %u\n", tp->tp_idx); + ODP_PRINT(" num timers %u\n", tp->num_alloc); + ODP_PRINT(" num tp %i\n", timer_global->num_timer_pools); + ODP_PRINT(" inline timers %i\n", timer_global->use_inline_timers); + ODP_PRINT("\n"); +} + +void odp_timer_print(odp_timer_t timer) +{ + timer_pool_t *tp; + uint32_t idx; + _odp_timer_t *tim; + + if (timer == ODP_TIMER_INVALID) { + ODP_ERR("Bad timer handle\n"); + return; + } + + tp = handle_to_tp(timer); + idx = handle_to_idx(timer, tp); + tim = &tp->timers[idx]; + + ODP_PRINT("\nTimer info\n"); + ODP_PRINT("----------\n"); + ODP_PRINT(" timer pool %p\n", tp); + ODP_PRINT(" timer index %u\n", idx); + ODP_PRINT(" dest queue 0x%" PRIx64 "\n", odp_queue_to_u64(tim->queue)); + ODP_PRINT(" user ptr %p\n", tim->user_ptr); + ODP_PRINT("\n"); +} + +void odp_timeout_print(odp_timeout_t tmo) +{ + const odp_timeout_hdr_t *tmo_hdr; + odp_timer_t timer; + timer_pool_t *tp = NULL; + uint32_t idx = 0; + + if (tmo == ODP_TIMEOUT_INVALID) { + ODP_ERR("Bad timeout handle\n"); + return; + } + + tmo_hdr = timeout_hdr(tmo); + timer = tmo_hdr->timer; + + if (timer != ODP_TIMER_INVALID) { + tp = handle_to_tp(timer); + idx = handle_to_idx(timer, tp); + } + + ODP_PRINT("\nTimeout info\n"); + ODP_PRINT("------------\n"); + ODP_PRINT(" tmo handle 0x%" PRIx64 "\n", odp_timeout_to_u64(tmo)); + ODP_PRINT(" timer pool %p\n", tp); + ODP_PRINT(" timer index %u\n", idx); + ODP_PRINT(" expiration %" PRIu64 "\n", tmo_hdr->expiration); + ODP_PRINT(" user ptr %p\n", tmo_hdr->user_ptr); + ODP_PRINT("\n"); +} + int _odp_timer_init_global(const odp_init_t *params) { odp_shm_t shm; |