diff options
Diffstat (limited to 'platform/linux-generic/include')
22 files changed, 527 insertions, 128 deletions
diff --git a/platform/linux-generic/include/odp/atomic.h b/platform/linux-generic/include/odp/atomic.h index 10cf361b4..e262f4851 100644 --- a/platform/linux-generic/include/odp/atomic.h +++ b/platform/linux-generic/include/odp/atomic.h @@ -84,6 +84,45 @@ static inline void odp_atomic_dec_u32(odp_atomic_u32_t *atom) (void)__atomic_fetch_sub(&atom->v, 1, __ATOMIC_RELAXED); } +static inline int odp_atomic_cas_u32(odp_atomic_u32_t *atom, uint32_t *old_val, + uint32_t new_val) +{ + return __atomic_compare_exchange_n(&atom->v, old_val, new_val, + 0 /* strong */, + __ATOMIC_RELAXED, + __ATOMIC_RELAXED); +} + +static inline uint32_t odp_atomic_xchg_u32(odp_atomic_u32_t *atom, + uint32_t new_val) +{ + return __atomic_exchange_n(&atom->v, new_val, __ATOMIC_RELAXED); +} + +static inline void odp_atomic_max_u32(odp_atomic_u32_t *atom, uint32_t new_max) +{ + uint32_t old_val; + + old_val = odp_atomic_load_u32(atom); + + while (new_max > old_val) { + if (odp_atomic_cas_u32(atom, &old_val, new_max)) + break; + } +} + +static inline void odp_atomic_min_u32(odp_atomic_u32_t *atom, uint32_t new_min) +{ + uint32_t old_val; + + old_val = odp_atomic_load_u32(atom); + + while (new_min < old_val) { + if (odp_atomic_cas_u32(atom, &old_val, new_min)) + break; + } +} + static inline void odp_atomic_init_u64(odp_atomic_u64_t *atom, uint64_t val) { atom->v = val; @@ -185,6 +224,189 @@ static inline void odp_atomic_dec_u64(odp_atomic_u64_t *atom) #endif } +static inline int odp_atomic_cas_u64(odp_atomic_u64_t *atom, uint64_t *old_val, + uint64_t new_val) +{ +#if __GCC_ATOMIC_LLONG_LOCK_FREE < 2 + int ret; + *old_val = ATOMIC_OP(atom, ATOMIC_CAS_OP(&ret, *old_val, new_val)); + return ret; +#else + return __atomic_compare_exchange_n(&atom->v, old_val, new_val, + 0 /* strong */, + __ATOMIC_RELAXED, + __ATOMIC_RELAXED); +#endif +} + +static inline uint64_t odp_atomic_xchg_u64(odp_atomic_u64_t *atom, + uint64_t new_val) +{ +#if __GCC_ATOMIC_LLONG_LOCK_FREE < 2 + return ATOMIC_OP(atom, atom->v = new_val); +#else + return __atomic_exchange_n(&atom->v, new_val, __ATOMIC_RELAXED); +#endif +} + +static inline void odp_atomic_max_u64(odp_atomic_u64_t *atom, uint64_t new_max) +{ + uint64_t old_val; + + old_val = odp_atomic_load_u64(atom); + + while (new_max > old_val) { + if (odp_atomic_cas_u64(atom, &old_val, new_max)) + break; + } +} + +static inline void odp_atomic_min_u64(odp_atomic_u64_t *atom, uint64_t new_min) +{ + uint64_t old_val; + + old_val = odp_atomic_load_u64(atom); + + while (new_min < old_val) { + if (odp_atomic_cas_u64(atom, &old_val, new_min)) + break; + } +} + +static inline uint32_t odp_atomic_load_acq_u32(odp_atomic_u32_t *atom) +{ + return __atomic_load_n(&atom->v, __ATOMIC_ACQUIRE); +} + +static inline void odp_atomic_store_rel_u32(odp_atomic_u32_t *atom, + uint32_t val) +{ + __atomic_store_n(&atom->v, val, __ATOMIC_RELEASE); +} + +static inline void odp_atomic_add_rel_u32(odp_atomic_u32_t *atom, + uint32_t val) +{ + (void)__atomic_fetch_add(&atom->v, val, __ATOMIC_RELEASE); +} + +static inline void odp_atomic_sub_rel_u32(odp_atomic_u32_t *atom, + uint32_t val) +{ + (void)__atomic_fetch_sub(&atom->v, val, __ATOMIC_RELEASE); +} + +static inline int odp_atomic_cas_acq_u32(odp_atomic_u32_t *atom, + uint32_t *old_val, uint32_t new_val) +{ + return __atomic_compare_exchange_n(&atom->v, old_val, new_val, + 0 /* strong */, + __ATOMIC_ACQUIRE, + __ATOMIC_RELAXED); +} + +static inline int odp_atomic_cas_rel_u32(odp_atomic_u32_t *atom, + uint32_t *old_val, uint32_t new_val) +{ + return __atomic_compare_exchange_n(&atom->v, old_val, new_val, + 0 /* strong */, + __ATOMIC_RELEASE, + __ATOMIC_RELAXED); +} + +static inline int odp_atomic_cas_acq_rel_u32(odp_atomic_u32_t *atom, + uint32_t *old_val, + uint32_t new_val) +{ + return __atomic_compare_exchange_n(&atom->v, old_val, new_val, + 0 /* strong */, + __ATOMIC_ACQ_REL, + __ATOMIC_RELAXED); +} + +static inline uint64_t odp_atomic_load_acq_u64(odp_atomic_u64_t *atom) +{ +#if __GCC_ATOMIC_LLONG_LOCK_FREE < 2 + return ATOMIC_OP(atom, (void)0); +#else + return __atomic_load_n(&atom->v, __ATOMIC_ACQUIRE); +#endif +} + +static inline void odp_atomic_store_rel_u64(odp_atomic_u64_t *atom, + uint64_t val) +{ +#if __GCC_ATOMIC_LLONG_LOCK_FREE < 2 + (void)ATOMIC_OP(atom, atom->v = val); +#else + __atomic_store_n(&atom->v, val, __ATOMIC_RELEASE); +#endif +} + +static inline void odp_atomic_add_rel_u64(odp_atomic_u64_t *atom, uint64_t val) +{ +#if __GCC_ATOMIC_LLONG_LOCK_FREE < 2 + (void)ATOMIC_OP(atom, atom->v += val); +#else + (void)__atomic_fetch_add(&atom->v, val, __ATOMIC_RELEASE); +#endif +} + +static inline void odp_atomic_sub_rel_u64(odp_atomic_u64_t *atom, uint64_t val) +{ +#if __GCC_ATOMIC_LLONG_LOCK_FREE < 2 + (void)ATOMIC_OP(atom, atom->v -= val); +#else + (void)__atomic_fetch_sub(&atom->v, val, __ATOMIC_RELEASE); +#endif +} + +static inline int odp_atomic_cas_acq_u64(odp_atomic_u64_t *atom, + uint64_t *old_val, uint64_t new_val) +{ +#if __GCC_ATOMIC_LLONG_LOCK_FREE < 2 + int ret; + *old_val = ATOMIC_OP(atom, ATOMIC_CAS_OP(&ret, *old_val, new_val)); + return ret; +#else + return __atomic_compare_exchange_n(&atom->v, old_val, new_val, + 0 /* strong */, + __ATOMIC_ACQUIRE, + __ATOMIC_RELAXED); +#endif +} + +static inline int odp_atomic_cas_rel_u64(odp_atomic_u64_t *atom, + uint64_t *old_val, uint64_t new_val) +{ +#if __GCC_ATOMIC_LLONG_LOCK_FREE < 2 + int ret; + *old_val = ATOMIC_OP(atom, ATOMIC_CAS_OP(&ret, *old_val, new_val)); + return ret; +#else + return __atomic_compare_exchange_n(&atom->v, old_val, new_val, + 0 /* strong */, + __ATOMIC_RELEASE, + __ATOMIC_RELAXED); +#endif +} + +static inline int odp_atomic_cas_acq_rel_u64(odp_atomic_u64_t *atom, + uint64_t *old_val, + uint64_t new_val) +{ +#if __GCC_ATOMIC_LLONG_LOCK_FREE < 2 + int ret; + *old_val = ATOMIC_OP(atom, ATOMIC_CAS_OP(&ret, *old_val, new_val)); + return ret; +#else + return __atomic_compare_exchange_n(&atom->v, old_val, new_val, + 0 /* strong */, + __ATOMIC_ACQ_REL, + __ATOMIC_RELAXED); +#endif +} + /** * @} */ diff --git a/platform/linux-generic/include/odp/byteorder.h b/platform/linux-generic/include/odp/byteorder.h index 7fc7dc512..6c94556b0 100644 --- a/platform/linux-generic/include/odp/byteorder.h +++ b/platform/linux-generic/include/odp/byteorder.h @@ -25,7 +25,7 @@ extern "C" { * @{ */ -static inline uint16_t odp_be_to_cpu_16(uint16be_t be16) +static inline uint16_t odp_be_to_cpu_16(odp_u16be_t be16) { #if ODP_BYTE_ORDER == ODP_LITTLE_ENDIAN return __odp_builtin_bswap16((__odp_force uint16_t)be16); @@ -34,7 +34,7 @@ static inline uint16_t odp_be_to_cpu_16(uint16be_t be16) #endif } -static inline uint32_t odp_be_to_cpu_32(uint32be_t be32) +static inline uint32_t odp_be_to_cpu_32(odp_u32be_t be32) { #if ODP_BYTE_ORDER == ODP_LITTLE_ENDIAN return __builtin_bswap32((__odp_force uint32_t)be32); @@ -43,7 +43,7 @@ static inline uint32_t odp_be_to_cpu_32(uint32be_t be32) #endif } -static inline uint64_t odp_be_to_cpu_64(uint64be_t be64) +static inline uint64_t odp_be_to_cpu_64(odp_u64be_t be64) { #if ODP_BYTE_ORDER == ODP_LITTLE_ENDIAN return __builtin_bswap64((__odp_force uint64_t)be64); @@ -53,35 +53,35 @@ static inline uint64_t odp_be_to_cpu_64(uint64be_t be64) } -static inline uint16be_t odp_cpu_to_be_16(uint16_t cpu16) +static inline odp_u16be_t odp_cpu_to_be_16(uint16_t cpu16) { #if ODP_BYTE_ORDER == ODP_LITTLE_ENDIAN - return (__odp_force uint16be_t)__odp_builtin_bswap16(cpu16); + return (__odp_force odp_u16be_t)__odp_builtin_bswap16(cpu16); #else - return (__odp_force uint16be_t)cpu16; + return (__odp_force odp_u16be_t)cpu16; #endif } -static inline uint32be_t odp_cpu_to_be_32(uint32_t cpu32) +static inline odp_u32be_t odp_cpu_to_be_32(uint32_t cpu32) { #if ODP_BYTE_ORDER == ODP_LITTLE_ENDIAN - return (__odp_force uint32be_t)__builtin_bswap32(cpu32); + return (__odp_force odp_u32be_t)__builtin_bswap32(cpu32); #else - return (__odp_force uint32be_t)cpu32; + return (__odp_force odp_u32be_t)cpu32; #endif } -static inline uint64be_t odp_cpu_to_be_64(uint64_t cpu64) +static inline odp_u64be_t odp_cpu_to_be_64(uint64_t cpu64) { #if ODP_BYTE_ORDER == ODP_LITTLE_ENDIAN - return (__odp_force uint64be_t)__builtin_bswap64(cpu64); + return (__odp_force odp_u64be_t)__builtin_bswap64(cpu64); #else - return (__odp_force uint64be_t)cpu64; + return (__odp_force odp_u64be_t)cpu64; #endif } -static inline uint16_t odp_le_to_cpu_16(uint16le_t le16) +static inline uint16_t odp_le_to_cpu_16(odp_u16le_t le16) { #if ODP_BYTE_ORDER == ODP_LITTLE_ENDIAN return (__odp_force uint16_t)le16; @@ -90,7 +90,7 @@ static inline uint16_t odp_le_to_cpu_16(uint16le_t le16) #endif } -static inline uint32_t odp_le_to_cpu_32(uint32le_t le32) +static inline uint32_t odp_le_to_cpu_32(odp_u32le_t le32) { #if ODP_BYTE_ORDER == ODP_LITTLE_ENDIAN return (__odp_force uint32_t)le32; @@ -99,7 +99,7 @@ static inline uint32_t odp_le_to_cpu_32(uint32le_t le32) #endif } -static inline uint64_t odp_le_to_cpu_64(uint64le_t le64) +static inline uint64_t odp_le_to_cpu_64(odp_u64le_t le64) { #if ODP_BYTE_ORDER == ODP_LITTLE_ENDIAN return (__odp_force uint64_t)le64; @@ -109,30 +109,30 @@ static inline uint64_t odp_le_to_cpu_64(uint64le_t le64) } -static inline uint16le_t odp_cpu_to_le_16(uint16_t cpu16) +static inline odp_u16le_t odp_cpu_to_le_16(uint16_t cpu16) { #if ODP_BYTE_ORDER == ODP_LITTLE_ENDIAN - return (__odp_force uint16le_t)cpu16; + return (__odp_force odp_u16le_t)cpu16; #else - return (__odp_force uint16le_t)__odp_builtin_bswap16(cpu16); + return (__odp_force odp_u16le_t)__odp_builtin_bswap16(cpu16); #endif } -static inline uint32le_t odp_cpu_to_le_32(uint32_t cpu32) +static inline odp_u32le_t odp_cpu_to_le_32(uint32_t cpu32) { #if ODP_BYTE_ORDER == ODP_LITTLE_ENDIAN - return (__odp_force uint32le_t)cpu32; + return (__odp_force odp_u32le_t)cpu32; #else - return (__odp_force uint32le_t)__builtin_bswap32(cpu32); + return (__odp_force odp_u32le_t)__builtin_bswap32(cpu32); #endif } -static inline uint64le_t odp_cpu_to_le_64(uint64_t cpu64) +static inline odp_u64le_t odp_cpu_to_le_64(uint64_t cpu64) { #if ODP_BYTE_ORDER == ODP_LITTLE_ENDIAN - return (__odp_force uint64le_t)cpu64; + return (__odp_force odp_u64le_t)cpu64; #else - return (__odp_force uint64le_t)__builtin_bswap64(cpu64); + return (__odp_force odp_u64le_t)__builtin_bswap64(cpu64); #endif } diff --git a/platform/linux-generic/include/odp/cpu.h b/platform/linux-generic/include/odp/cpu.h index b5b532005..b98507dd5 100644 --- a/platform/linux-generic/include/odp/cpu.h +++ b/platform/linux-generic/include/odp/cpu.h @@ -17,6 +17,8 @@ extern "C" { #endif +#include <odp/cpu_arch.h> + #include <odp/api/cpu.h> #ifdef __cplusplus diff --git a/platform/linux-generic/include/odp/init.h b/platform/linux-generic/include/odp/init.h index 950a4f829..3233e36de 100644 --- a/platform/linux-generic/include/odp/init.h +++ b/platform/linux-generic/include/odp/init.h @@ -17,6 +17,8 @@ extern "C" { #endif +#include <odp/plat/init_types.h> + /** @ingroup odp_initialization * @{ */ diff --git a/platform/linux-generic/include/odp/plat/atomic_types.h b/platform/linux-generic/include/odp/plat/atomic_types.h index 0fe15ed13..bc0bd8bfe 100644 --- a/platform/linux-generic/include/odp/plat/atomic_types.h +++ b/platform/linux-generic/include/odp/plat/atomic_types.h @@ -43,6 +43,21 @@ struct odp_atomic_u32_s { } ODP_ALIGNED(sizeof(uint32_t)); /* Enforce alignement! */; #if __GCC_ATOMIC_LLONG_LOCK_FREE < 2 + +/** + * @internal + * CAS operation expression for the ATOMIC_OP macro + */ +#define ATOMIC_CAS_OP(ret_ptr, old_val, new_val) \ +({ \ + if (atom->v == (old_val)) { \ + atom->v = (new_val); \ + *(ret_ptr) = 1; \ + } else { \ + *(ret_ptr) = 0; \ + } \ +}) + /** * @internal * Helper macro for lock-based atomic operations on 64-bit integers @@ -52,14 +67,14 @@ struct odp_atomic_u32_s { */ #define ATOMIC_OP(atom, expr) \ ({ \ - uint64_t old_val; \ + uint64_t _old_val; \ /* Loop while lock is already taken, stop when lock becomes clear */ \ while (__atomic_test_and_set(&(atom)->lock, __ATOMIC_ACQUIRE)) \ (void)0; \ - old_val = (atom)->v; \ + _old_val = (atom)->v; \ (expr); /* Perform whatever update is desired */ \ __atomic_clear(&(atom)->lock, __ATOMIC_RELEASE); \ - old_val; /* Return old value */ \ + _old_val; /* Return old value */ \ }) #endif diff --git a/platform/linux-generic/include/odp/plat/byteorder_types.h b/platform/linux-generic/include/odp/plat/byteorder_types.h index cf917b1a2..0a8e4096e 100644 --- a/platform/linux-generic/include/odp/plat/byteorder_types.h +++ b/platform/linux-generic/include/odp/plat/byteorder_types.h @@ -67,17 +67,17 @@ extern "C" { #define ODP_BYTE_ORDER ODP_BIG_ENDIAN #endif -typedef uint16_t __odp_bitwise uint16le_t; -typedef uint16_t __odp_bitwise uint16be_t; +typedef uint16_t __odp_bitwise odp_u16le_t; +typedef uint16_t __odp_bitwise odp_u16be_t; -typedef uint32_t __odp_bitwise uint32le_t; -typedef uint32_t __odp_bitwise uint32be_t; +typedef uint32_t __odp_bitwise odp_u32le_t; +typedef uint32_t __odp_bitwise odp_u32be_t; -typedef uint64_t __odp_bitwise uint64le_t; -typedef uint64_t __odp_bitwise uint64be_t; +typedef uint64_t __odp_bitwise odp_u64le_t; +typedef uint64_t __odp_bitwise odp_u64be_t; -typedef uint16_t __odp_bitwise uint16sum_t; -typedef uint32_t __odp_bitwise uint32sum_t; +typedef uint16_t __odp_bitwise odp_u16sum_t; +typedef uint32_t __odp_bitwise odp_u32sum_t; /** * @} diff --git a/platform/linux-generic/include/odp/plat/init_types.h b/platform/linux-generic/include/odp/plat/init_types.h new file mode 100644 index 000000000..b240c93ca --- /dev/null +++ b/platform/linux-generic/include/odp/plat/init_types.h @@ -0,0 +1,30 @@ +/* Copyright (c) 2015, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +/** + * @file + * + * ODP initialization. + */ + +#ifndef ODP_INIT_TYPES_H_ +#define ODP_INIT_TYPES_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * @internal platform specific data + */ +typedef struct odp_platform_init_t { +} odp_platform_init_t; + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/include/odp/plat/packet_io_types.h b/platform/linux-generic/include/odp/plat/packet_io_types.h index 3cc64c6d2..934d7de97 100644 --- a/platform/linux-generic/include/odp/plat/packet_io_types.h +++ b/platform/linux-generic/include/odp/plat/packet_io_types.h @@ -21,16 +21,26 @@ extern "C" { #include <odp/std_types.h> #include <odp/plat/strong_types.h> -/** @addtogroup odp_packet_io ODP PACKET IO +/** @addtogroup odp_packet_io * Operations on a packet. * @{ */ typedef ODP_HANDLE_T(odp_pktio_t); -#define ODP_PKTIO_INVALID _odp_cast_scalar(odp_pktio_t, 0) +/** @internal */ +typedef struct odp_pktin_queue_t { + odp_pktio_t pktio; /**< @internal pktio handle */ + int index; /**< @internal pktio queue index */ +} odp_pktin_queue_t; + +/** @internal */ +typedef struct odp_pktout_queue_t { + odp_pktio_t pktio; /**< @internal pktio handle */ + int index; /**< @internal pktio queue index */ +} odp_pktout_queue_t; -#define ODP_PKTIO_ANY _odp_cast_scalar(odp_pktio_t, ~0) +#define ODP_PKTIO_INVALID _odp_cast_scalar(odp_pktio_t, 0) #define ODP_PKTIO_MACADDR_MAXSIZE 16 diff --git a/platform/linux-generic/include/odp/plat/queue_types.h b/platform/linux-generic/include/odp/plat/queue_types.h index a7df15576..40a53e5e6 100644 --- a/platform/linux-generic/include/odp/plat/queue_types.h +++ b/platform/linux-generic/include/odp/plat/queue_types.h @@ -33,14 +33,6 @@ typedef ODP_HANDLE_T(odp_queue_group_t); #define ODP_QUEUE_NAME_LEN 32 - -typedef int odp_queue_type_t; - -#define ODP_QUEUE_TYPE_SCHED 0 -#define ODP_QUEUE_TYPE_POLL 1 -#define ODP_QUEUE_TYPE_PKTIN 2 -#define ODP_QUEUE_TYPE_PKTOUT 3 - /** Get printable format of odp_queue_t */ static inline uint64_t odp_queue_to_u64(odp_queue_t hdl) { diff --git a/platform/linux-generic/include/odp/plat/schedule_types.h b/platform/linux-generic/include/odp/plat/schedule_types.h index 21fcbb84c..a4a352c04 100644 --- a/platform/linux-generic/include/odp/plat/schedule_types.h +++ b/platform/linux-generic/include/odp/plat/schedule_types.h @@ -37,7 +37,7 @@ typedef int odp_schedule_prio_t; typedef int odp_schedule_sync_t; -#define ODP_SCHED_SYNC_NONE 0 +#define ODP_SCHED_SYNC_PARALLEL 0 #define ODP_SCHED_SYNC_ATOMIC 1 #define ODP_SCHED_SYNC_ORDERED 2 diff --git a/platform/linux-generic/include/odp/std_clib.h b/platform/linux-generic/include/odp/std_clib.h index c939c48e9..11c59bec2 100644 --- a/platform/linux-generic/include/odp/std_clib.h +++ b/platform/linux-generic/include/odp/std_clib.h @@ -23,6 +23,11 @@ static inline void *odp_memset(void *ptr, int value, size_t num) return memset(ptr, value, num); } +static inline int odp_memcmp(const void *ptr1, const void *ptr2, size_t num) +{ + return memcmp(ptr1, ptr2, num); +} + #ifdef __cplusplus } #endif diff --git a/platform/linux-generic/include/odp_atomic_internal.h b/platform/linux-generic/include/odp_atomic_internal.h index ce62368e1..ff3813f73 100644 --- a/platform/linux-generic/include/odp_atomic_internal.h +++ b/platform/linux-generic/include/odp_atomic_internal.h @@ -64,12 +64,6 @@ typedef enum { _ODP_MEMMODEL_SC = __ATOMIC_SEQ_CST } _odp_memmodel_t; -/** - * Insert a full memory barrier (fence) in the compiler and instruction - * sequence. - */ -#define _ODP_FULL_BARRIER() __atomic_thread_fence(__ATOMIC_SEQ_CST) - /***************************************************************************** * Operations on 32-bit atomics * _odp_atomic_u32_load_mm - return current value diff --git a/platform/linux-generic/include/odp_classification_datamodel.h b/platform/linux-generic/include/odp_classification_datamodel.h index 5b6520266..27d8a526c 100644 --- a/platform/linux-generic/include/odp_classification_datamodel.h +++ b/platform/linux-generic/include/odp_classification_datamodel.h @@ -54,7 +54,7 @@ Stores the Term and Value mapping for a PMR. The maximum size of value currently supported in 64 bits **/ typedef struct pmr_term_value { - odp_pmr_term_e term; /* PMR Term */ + odp_pmr_term_t term; /* PMR Term */ uint64_t val; /**< Value to be matched */ uint64_t mask; /**< Masked set of bits to be matched */ uint32_t offset; /**< Offset if term == ODP_PMR_CUSTOM_FRAME */ diff --git a/platform/linux-generic/include/odp_classification_inlines.h b/platform/linux-generic/include/odp_classification_inlines.h index 5f0b564f7..96cf77ee2 100644 --- a/platform/linux-generic/include/odp_classification_inlines.h +++ b/platform/linux-generic/include/odp_classification_inlines.h @@ -154,11 +154,28 @@ static inline int verify_pmr_udp_sport(const uint8_t *pkt_addr, return 0; } -static inline int verify_pmr_dmac(const uint8_t *pkt_addr ODP_UNUSED, - odp_packet_hdr_t *pkt_hdr ODP_UNUSED, - pmr_term_value_t *term_value ODP_UNUSED) +static inline int verify_pmr_dmac(const uint8_t *pkt_addr, + odp_packet_hdr_t *pkt_hdr, + pmr_term_value_t *term_value) { - ODP_UNIMPLEMENTED(); + uint64_t dmac = 0; + uint64_t dmac_be = 0; + const odph_ethhdr_t *eth; + + if (!pkt_hdr->input_flags.eth) + return 0; + + eth = (const odph_ethhdr_t *)(pkt_addr + pkt_hdr->l2_offset); + memcpy(&dmac_be, eth->dst.addr, ODPH_ETHADDR_LEN); + dmac = odp_be_to_cpu_64(dmac_be); + /* since we are converting a 48 bit ethernet address from BE to cpu + format using odp_be_to_cpu_64() the last 16 bits needs to be right + shifted */ + if (dmac_be != dmac) + dmac = dmac >> (64 - (ODPH_ETHADDR_LEN * 8)); + + if (term_value->val == (dmac & term_value->mask)) + return 1; return 0; } diff --git a/platform/linux-generic/include/odp_internal.h b/platform/linux-generic/include/odp_internal.h index b22f95698..e75154a51 100644 --- a/platform/linux-generic/include/odp_internal.h +++ b/platform/linux-generic/include/odp_internal.h @@ -20,16 +20,20 @@ extern "C" { #include <odp/init.h> #include <odp/thread.h> +#include <stdio.h> extern __thread int __odp_errno; +#define MAX_CPU_NUMBER 128 + typedef struct { - uint64_t cpu_hz; + uint64_t cpu_hz_max[MAX_CPU_NUMBER]; uint64_t huge_page_size; uint64_t page_size; int cache_line_size; int cpu_count; - char model_str[128]; + char cpu_arch_str[128]; + char model_str[MAX_CPU_NUMBER][128]; } odp_system_info_t; struct odp_global_data_s { @@ -103,6 +107,9 @@ int odp_time_term_global(void); void _odp_flush_caches(void); +int odp_cpuinfo_parser(FILE *file, odp_system_info_t *sysinfo); +uint64_t odp_cpu_hz_current(int id); + #ifdef __cplusplus } #endif diff --git a/platform/linux-generic/include/odp_packet_io_internal.h b/platform/linux-generic/include/odp_packet_io_internal.h index 3ab0bc86f..f871f0a76 100644 --- a/platform/linux-generic/include/odp_packet_io_internal.h +++ b/platform/linux-generic/include/odp_packet_io_internal.h @@ -20,9 +20,6 @@ extern "C" { #include <odp/spinlock.h> #include <odp/ticketlock.h> -#include <odp_packet_socket.h> -#include <odp_packet_netmap.h> -#include <odp_packet_tap.h> #include <odp_classification_datamodel.h> #include <odp_align_internal.h> #include <odp_debug_internal.h> @@ -31,8 +28,16 @@ extern "C" { #include <odp/hints.h> #include <net/if.h> +#define PKTIO_MAX_QUEUES 64 +#include <odp_packet_socket.h> +#include <odp_packet_netmap.h> +#include <odp_packet_tap.h> + #define PKTIO_NAME_LEN 256 +#define PKTIN_INVALID ((odp_pktin_queue_t) {ODP_PKTIO_INVALID, 0}) +#define PKTOUT_INVALID ((odp_pktout_queue_t) {ODP_PKTIO_INVALID, 0}) + /** Determine if a socket read/write error should be reported. Transient errors * that simply require the caller to retry are ignored, the _send/_recv APIs * are non-blocking and it is the caller's responsibility to retry if the @@ -70,7 +75,6 @@ struct pktio_entry { int taken; /**< is entry taken(1) or free(0) */ int cls_enabled; /**< is classifier enabled */ odp_pktio_t handle; /**< pktio handle */ - odp_queue_t inq_default; /**< default input queue, if set */ odp_queue_t outq_default; /**< default out queue */ union { pkt_loop_t pkt_loop; /**< Using loopback for IO */ @@ -88,10 +92,30 @@ struct pktio_entry { STATE_STOP } state; classifier_t cls; /**< classifier linked with this pktio*/ + odp_pktio_stats_t stats; /**< statistic counters for pktio */ + enum { + STATS_SYSFS = 0, + STATS_ETHTOOL, + STATS_UNSUPPORTED + } stats_type; char name[PKTIO_NAME_LEN]; /**< name of pktio provided to pktio_open() */ odp_pktio_t id; odp_pktio_param_t param; + + /* Storage for queue handles + * Multi-queue support is pktio driver specific */ + unsigned num_in_queue; + unsigned num_out_queue; + + struct { + odp_queue_t queue; + odp_pktin_queue_t pktin; + } in_queue[PKTIO_MAX_QUEUES]; + + struct { + odp_pktout_queue_t pktout; + } out_queue[PKTIO_MAX_QUEUES]; }; typedef union { @@ -107,6 +131,7 @@ typedef struct { int is_free(pktio_entry_t *entry); typedef struct pktio_if_ops { + const char *name; int (*init)(void); int (*term)(void); int (*open)(odp_pktio_t pktio, pktio_entry_t *pktio_entry, @@ -114,6 +139,8 @@ typedef struct pktio_if_ops { int (*close)(pktio_entry_t *pktio_entry); int (*start)(pktio_entry_t *pktio_entry); int (*stop)(pktio_entry_t *pktio_entry); + int (*stats)(pktio_entry_t *pktio_entry, odp_pktio_stats_t *stats); + int (*stats_reset)(pktio_entry_t *pktio_entry); int (*recv)(pktio_entry_t *pktio_entry, odp_packet_t pkt_table[], unsigned len); int (*send)(pktio_entry_t *pktio_entry, odp_packet_t pkt_table[], @@ -122,6 +149,22 @@ typedef struct pktio_if_ops { int (*promisc_mode_set)(pktio_entry_t *pktio_entry, int enable); int (*promisc_mode_get)(pktio_entry_t *pktio_entry); int (*mac_get)(pktio_entry_t *pktio_entry, void *mac_addr); + int (*link_status)(pktio_entry_t *pktio_entry); + int (*capability)(pktio_entry_t *pktio_entry, + odp_pktio_capability_t *capa); + int (*input_queues_config)(pktio_entry_t *pktio_entry, + const odp_pktin_queue_param_t *param); + int (*output_queues_config)(pktio_entry_t *pktio_entry, + const odp_pktout_queue_param_t *p); + int (*in_queues)(pktio_entry_t *entry, odp_queue_t queues[], int num); + int (*pktin_queues)(pktio_entry_t *entry, odp_pktin_queue_t queues[], + int num); + int (*pktout_queues)(pktio_entry_t *entry, odp_pktout_queue_t queues[], + int num); + int (*recv_queue)(pktio_entry_t *entry, int index, + odp_packet_t packets[], int num); + int (*send_queue)(pktio_entry_t *entry, int index, + odp_packet_t packets[], int num); } pktio_if_ops_t; int _odp_packet_cls_enq(pktio_entry_t *pktio_entry, const uint8_t *base, @@ -158,7 +201,25 @@ static inline void pktio_cls_enabled_set(pktio_entry_t *entry, int ena) entry->s.cls_enabled = ena; } -int pktin_poll(pktio_entry_t *entry); +int pktin_poll(pktio_entry_t *entry, int num_queue, int index[]); + +/* + * Dummy single queue implementations of multi-queue API + */ +int single_capability(odp_pktio_capability_t *capa); +int single_input_queues_config(pktio_entry_t *entry, + const odp_pktin_queue_param_t *param); +int single_output_queues_config(pktio_entry_t *entry, + const odp_pktout_queue_param_t *param); +int single_in_queues(pktio_entry_t *entry, odp_queue_t queues[], int num); +int single_pktin_queues(pktio_entry_t *entry, odp_pktin_queue_t queues[], + int num); +int single_pktout_queues(pktio_entry_t *entry, odp_pktout_queue_t queues[], + int num); +int single_recv_queue(pktio_entry_t *entry, int index, odp_packet_t packets[], + int num); +int single_send_queue(pktio_entry_t *entry, int index, odp_packet_t packets[], + int num); extern const pktio_if_ops_t netmap_pktio_ops; extern const pktio_if_ops_t sock_mmsg_pktio_ops; @@ -170,6 +231,13 @@ extern const pktio_if_ops_t pcap_pktio_ops; extern const pktio_if_ops_t tap_pktio_ops; extern const pktio_if_ops_t * const pktio_if_ops[]; +int sysfs_stats(pktio_entry_t *pktio_entry, + odp_pktio_stats_t *stats); +int sock_stats_fd(pktio_entry_t *pktio_entry, + odp_pktio_stats_t *stats, + int fd); +int sock_stats_reset_fd(pktio_entry_t *pktio_entry, int fd); + #ifdef __cplusplus } #endif diff --git a/platform/linux-generic/include/odp_packet_netmap.h b/platform/linux-generic/include/odp_packet_netmap.h index 0577dfe20..26a8da1eb 100644 --- a/platform/linux-generic/include/odp_packet_netmap.h +++ b/platform/linux-generic/include/odp_packet_netmap.h @@ -7,19 +7,55 @@ #ifndef ODP_PACKET_NETMAP_H #define ODP_PACKET_NETMAP_H +#include <odp/align.h> +#include <odp/debug.h> +#include <odp/packet_io.h> #include <odp/pool.h> +#include <odp/ticketlock.h> +#include <odp_align_internal.h> #include <linux/if_ether.h> +#include <net/if.h> + +#define NM_MAX_DESC 32 + +/** Ring for mapping pktin/pktout queues to netmap descriptors */ +struct netmap_ring_t { + unsigned first; /**< Index of first netmap descriptor */ + unsigned last; /**< Index of last netmap descriptor */ + unsigned num; /**< Number of netmap descriptors */ + /** Netmap metadata for the device */ + struct nm_desc *desc[NM_MAX_DESC]; + unsigned cur; /**< Index of current netmap descriptor */ + odp_ticketlock_t lock; /**< Queue lock */ +}; + +typedef union { + struct netmap_ring_t s; + uint8_t pad[ODP_CACHE_LINE_SIZE_ROUNDUP(sizeof(struct netmap_ring_t))]; +} netmap_ring_t ODP_ALIGNED_CACHE; /** Packet socket using netmap mmaped rings for both Rx and Tx */ typedef struct { odp_pool_t pool; /**< pool to alloc packets from */ size_t max_frame_len; /**< buf_size - sizeof(pkt_hdr) */ - struct nm_desc *rx_desc; /**< netmap meta-data for the device */ - struct nm_desc *tx_desc; /**< netmap meta-data for the device */ uint32_t if_flags; /**< interface flags */ + uint32_t mtu; /**< maximum transmission unit */ int sockfd; /**< control socket */ unsigned char if_mac[ETH_ALEN]; /**< eth mac address */ + char nm_name[IF_NAMESIZE + 7]; /**< netmap:<ifname> */ + odp_pktio_capability_t capa; /**< interface capabilities */ + unsigned cur_rx_queue; /**< current pktin queue */ + uint32_t num_rx_rings; /**< number of nm rx rings */ + uint32_t num_tx_rings; /**< number of nm tx rings */ + unsigned num_rx_desc_rings; /**< number of rx descriptor rings */ + unsigned num_tx_desc_rings; /**< number of tx descriptor rings */ + odp_bool_t lockless_rx; /**< no locking for rx */ + odp_bool_t lockless_tx; /**< no locking for tx */ + /** mapping of pktin queues to netmap rx descriptors */ + netmap_ring_t rx_desc_ring[PKTIO_MAX_QUEUES]; + /** mapping of pktout queues to netmap tx descriptors */ + netmap_ring_t tx_desc_ring[PKTIO_MAX_QUEUES]; } pkt_netmap_t; #endif diff --git a/platform/linux-generic/include/odp_packet_socket.h b/platform/linux-generic/include/odp_packet_socket.h index 1eaafb7e5..a7797d1c0 100644 --- a/platform/linux-generic/include/odp_packet_socket.h +++ b/platform/linux-generic/include/odp_packet_socket.h @@ -18,6 +18,7 @@ #include <odp/debug.h> #include <odp/pool.h> #include <odp/packet.h> +#include <odp/packet_io.h> #include <linux/version.h> @@ -116,4 +117,60 @@ int promisc_mode_set_fd(int fd, const char *name, int enable); */ int promisc_mode_get_fd(int fd, const char *name); +/** + * Return link status of a packet socket (up/down) + */ +int link_status_fd(int fd, const char *name); + +/** + * Get enabled RSS hash protocols of a packet socket + * + * @param fd Socket file descriptor + * @param name Interface name + * @param hash_proto[out] Hash protocols + * + * @returns Number enabled hash protocols + */ +int rss_conf_get_fd(int fd, const char *name, + odp_pktin_hash_proto_t *hash_proto); + +/** + * Get supported RSS hash protocols of a packet socket + * + * Can be both read and modified. + * + * @param fd Socket file descriptor + * @param name Interface name + * @param hash_proto[out] Hash protocols + * + * @returns Number of supported hash protocols + */ +int rss_conf_get_supported_fd(int fd, const char *name, + odp_pktin_hash_proto_t *hash_proto); + +/** + * Set RSS hash protocols of a packet socket + * + * @param fd Socket file descriptor + * @param name Interface name + * @param hash_proto Hash protocols + * + * @retval 0 on success + * @retval <0 on failure + */ +int rss_conf_set_fd(int fd, const char *name, + const odp_pktin_hash_proto_t *proto); + +/** + * Print enabled RSS hash protocols + * + * @param hash_proto Hash protocols + */ +void rss_conf_print(const odp_pktin_hash_proto_t *hash_proto); + +/** + * Get ethtool statistics of a packet socket + */ +int ethtool_stats_get_fd(int fd, const char *name, odp_pktio_stats_t *stats); + #endif diff --git a/platform/linux-generic/include/odp_pool_internal.h b/platform/linux-generic/include/odp_pool_internal.h index ae4836cc8..fdac6067e 100644 --- a/platform/linux-generic/include/odp_pool_internal.h +++ b/platform/linux-generic/include/odp_pool_internal.h @@ -28,7 +28,6 @@ extern "C" { #include <odp/debug.h> #include <odp/shared_memory.h> #include <odp/atomic.h> -#include <odp_atomic_internal.h> #include <odp/thread.h> #include <string.h> diff --git a/platform/linux-generic/include/odp_queue_internal.h b/platform/linux-generic/include/odp_queue_internal.h index 1cc0ed26e..2d349466f 100644 --- a/platform/linux-generic/include/odp_queue_internal.h +++ b/platform/linux-generic/include/odp_queue_internal.h @@ -76,7 +76,7 @@ struct queue_entry_s { odp_event_t cmd_ev; odp_queue_type_t type; odp_queue_param_t param; - odp_pktio_t pktin; + odp_pktin_queue_t pktin; odp_pktio_t pktout; char name[ODP_QUEUE_NAME_LEN]; uint64_t order_in; diff --git a/platform/linux-generic/include/odp_schedule_internal.h b/platform/linux-generic/include/odp_schedule_internal.h index 6b301cd18..08683941a 100644 --- a/platform/linux-generic/include/odp_schedule_internal.h +++ b/platform/linux-generic/include/odp_schedule_internal.h @@ -23,7 +23,8 @@ extern "C" { int schedule_queue_init(queue_entry_t *qe); void schedule_queue_destroy(queue_entry_t *qe); int schedule_queue(const queue_entry_t *qe); -int schedule_pktio_start(odp_pktio_t pktio, int prio); +void schedule_pktio_start(odp_pktio_t pktio, int num_in_queue, + int in_queue_idx[]); void odp_schedule_release_context(void); #ifdef __cplusplus diff --git a/platform/linux-generic/include/odp_spin_internal.h b/platform/linux-generic/include/odp_spin_internal.h deleted file mode 100644 index 29c524fb1..000000000 --- a/platform/linux-generic/include/odp_spin_internal.h +++ /dev/null @@ -1,58 +0,0 @@ -/* Copyright (c) 2013, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - - - -#ifndef ODP_SPIN_INTERNAL_H_ -#define ODP_SPIN_INTERNAL_H_ - -#ifdef __cplusplus -extern "C" { -#endif - - -/** - * Spin loop for ODP internal use - */ -static inline void odp_spin(void) -{ -#if defined __x86_64__ || defined __i386__ - -#ifdef __SSE2__ - __asm__ __volatile__ ("pause"); -#else - __asm__ __volatile__ ("rep; nop"); -#endif - -#elif defined __arm__ - -#if __ARM_ARCH == 7 - __asm__ __volatile__ ("nop"); - __asm__ __volatile__ ("nop"); - __asm__ __volatile__ ("nop"); - __asm__ __volatile__ ("nop"); -#endif - -#elif defined __OCTEON__ - - __asm__ __volatile__ ("nop"); - __asm__ __volatile__ ("nop"); - __asm__ __volatile__ ("nop"); - __asm__ __volatile__ ("nop"); - __asm__ __volatile__ ("nop"); - __asm__ __volatile__ ("nop"); - __asm__ __volatile__ ("nop"); - __asm__ __volatile__ ("nop"); - -#endif -} - - -#ifdef __cplusplus -} -#endif - -#endif |