diff options
Diffstat (limited to 'platform/linux-generic/include')
12 files changed, 296 insertions, 75 deletions
diff --git a/platform/linux-generic/include/odp/api/plat/buffer_inline_types.h b/platform/linux-generic/include/odp/api/plat/buffer_inline_types.h deleted file mode 100644 index f59df6705..000000000 --- a/platform/linux-generic/include/odp/api/plat/buffer_inline_types.h +++ /dev/null @@ -1,36 +0,0 @@ -/* Copyright (c) 2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -#ifndef ODP_PLAT_BUFFER_INLINE_TYPES_H_ -#define ODP_PLAT_BUFFER_INLINE_TYPES_H_ - -#ifdef __cplusplus -extern "C" { -#endif - -#include <stdint.h> - -/** @cond _ODP_HIDE_FROM_DOXYGEN_ */ - -/* Buffer header field accessor */ -#define _odp_buf_hdr_field(buf_hdr, cast, field) \ - (*(cast *)(uintptr_t)((uint8_t *)buf_hdr + \ - _odp_buffer_inline_offset.field)) - -/* Buffer header field offsets for inline functions */ -typedef struct _odp_buffer_inline_offset_t { - uint16_t event_type; - uint16_t base_data; - -} _odp_buffer_inline_offset_t; - -/** @endcond */ - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/platform/linux-generic/include/odp/api/plat/buffer_inlines.h b/platform/linux-generic/include/odp/api/plat/buffer_inlines.h index 9e0ae3f50..3da402a83 100644 --- a/platform/linux-generic/include/odp/api/plat/buffer_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/buffer_inlines.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2019, Nokia +/* Copyright (c) 2019-2022, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -10,11 +10,11 @@ #include <odp/api/abi/buffer.h> #include <odp/api/abi/event_types.h> -#include <odp/api/plat/buffer_inline_types.h> +#include <odp/api/plat/event_inline_types.h> /** @cond _ODP_HIDE_FROM_DOXYGEN_ */ -extern const _odp_buffer_inline_offset_t _odp_buffer_inline_offset; +extern const _odp_event_inline_offset_t _odp_event_inline_offset; #ifndef _ODP_NO_INLINE /* Inline functions by default */ @@ -38,7 +38,7 @@ _ODP_INLINE odp_event_t odp_buffer_to_event(odp_buffer_t buf) _ODP_INLINE void *odp_buffer_addr(odp_buffer_t buf) { - return _odp_buf_hdr_field(buf, void *, base_data); + return _odp_event_hdr_field((odp_event_t)buf, void *, base_data); } /** @endcond */ diff --git a/platform/linux-generic/include/odp/api/plat/event_inline_types.h b/platform/linux-generic/include/odp/api/plat/event_inline_types.h new file mode 100644 index 000000000..c2727ec6f --- /dev/null +++ b/platform/linux-generic/include/odp/api/plat/event_inline_types.h @@ -0,0 +1,41 @@ +/* Copyright (c) 2018, Linaro Limited + * Copyright (c) 2022, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_PLAT_EVENT_INLINE_TYPES_H_ +#define ODP_PLAT_EVENT_INLINE_TYPES_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <stdint.h> + +/** @cond _ODP_HIDE_FROM_DOXYGEN_ */ + +/* Event header field accessors */ +#define _odp_event_hdr_field(event_hdr, cast, field) \ + (*(cast *)(uintptr_t)((uint8_t *)event_hdr + \ + _odp_event_inline_offset.field)) +#define _odp_event_hdr_ptr(event_hdr, cast, field) \ + ((cast *)(uintptr_t)((uint8_t *)event_hdr + \ + _odp_event_inline_offset.field)) + +/* Event header field offsets for inline functions */ +typedef struct _odp_event_inline_offset_t { + uint16_t event_type; + uint16_t base_data; + uint16_t flow_id; + +} _odp_event_inline_offset_t; + +/** @endcond */ + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/include/odp/api/plat/event_inlines.h b/platform/linux-generic/include/odp/api/plat/event_inlines.h index 43ee5e09a..27c2a82fd 100644 --- a/platform/linux-generic/include/odp/api/plat/event_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/event_inlines.h @@ -1,4 +1,5 @@ /* Copyright (c) 2018, Linaro Limited + * Copyright (c) 2022, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -7,18 +8,26 @@ #ifndef ODP_PLAT_EVENT_INLINES_H_ #define ODP_PLAT_EVENT_INLINES_H_ -#include <odp/api/abi/buffer.h> -#include <odp/api/plat/buffer_inline_types.h> +#include <odp/api/event_types.h> +#include <odp/api/packet.h> + +#include <odp/api/plat/event_inline_types.h> /** @cond _ODP_HIDE_FROM_DOXYGEN_ */ -extern const _odp_buffer_inline_offset_t _odp_buffer_inline_offset; +extern const _odp_event_inline_offset_t _odp_event_inline_offset; #ifndef _ODP_NO_INLINE /* Inline functions by default */ #define _ODP_INLINE static inline #define odp_event_type __odp_event_type #define odp_event_type_multi __odp_event_type_multi + #define odp_event_subtype __odp_event_subtype + #define odp_event_types __odp_event_types + #define odp_event_flow_id __odp_event_flow_id + #define odp_event_flow_id_set __odp_event_flow_id_set + + #include <odp/api/plat/packet_inlines.h> #else #define _ODP_INLINE #endif @@ -26,9 +35,8 @@ extern const _odp_buffer_inline_offset_t _odp_buffer_inline_offset; static inline odp_event_type_t __odp_event_type_get(odp_event_t event) { int8_t type; - odp_buffer_t buf = (odp_buffer_t)event; - type = _odp_buf_hdr_field(buf, int8_t, event_type); + type = _odp_event_hdr_field(event, int8_t, event_type); return (odp_event_type_t)type; } @@ -54,6 +62,38 @@ _ODP_INLINE int odp_event_type_multi(const odp_event_t event[], int num, return i; } +_ODP_INLINE odp_event_subtype_t odp_event_subtype(odp_event_t event) +{ + if (__odp_event_type_get(event) != ODP_EVENT_PACKET) + return ODP_EVENT_NO_SUBTYPE; + + return odp_packet_subtype(odp_packet_from_event(event)); +} + +_ODP_INLINE odp_event_type_t odp_event_types(odp_event_t event, + odp_event_subtype_t *subtype) +{ + odp_event_type_t event_type = __odp_event_type_get(event); + + *subtype = event_type == ODP_EVENT_PACKET ? + odp_packet_subtype(odp_packet_from_event(event)) : + ODP_EVENT_NO_SUBTYPE; + + return event_type; +} + +_ODP_INLINE uint32_t odp_event_flow_id(odp_event_t event) +{ + return _odp_event_hdr_field(event, uint8_t, flow_id); +} + +_ODP_INLINE void odp_event_flow_id_set(odp_event_t event, uint32_t id) +{ + uint8_t *flow_id = _odp_event_hdr_ptr(event, uint8_t, flow_id); + + *flow_id = (uint8_t)id; +} + /** @endcond */ #endif diff --git a/platform/linux-generic/include/odp/api/plat/spinlock_inlines.h b/platform/linux-generic/include/odp/api/plat/spinlock_inlines.h new file mode 100644 index 000000000..a04c43f88 --- /dev/null +++ b/platform/linux-generic/include/odp/api/plat/spinlock_inlines.h @@ -0,0 +1,65 @@ +/* Copyright (c) 2013-2018, Linaro Limited + * Copyright (c) 2022, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_PLAT_SPINLOCK_INLINES_H_ +#define ODP_PLAT_SPINLOCK_INLINES_H_ + +#include <odp/api/cpu.h> + +#include <odp/api/abi/spinlock.h> + +/** @cond _ODP_HIDE_FROM_DOXYGEN_ */ + +#ifndef _ODP_NO_INLINE + /* Inline functions by default */ + #define _ODP_INLINE static inline + #define odp_spinlock_init __odp_spinlock_init + #define odp_spinlock_lock __odp_spinlock_lock + #define odp_spinlock_trylock __odp_spinlock_trylock + #define odp_spinlock_unlock __odp_spinlock_unlock + #define odp_spinlock_is_locked __odp_spinlock_is_locked + + #include <odp/api/plat/cpu_inlines.h> +#else + #undef _ODP_INLINE + #define _ODP_INLINE +#endif + +_ODP_INLINE void odp_spinlock_init(odp_spinlock_t *spinlock) +{ + __atomic_clear(&spinlock->lock, __ATOMIC_RELAXED); +} + +_ODP_INLINE void odp_spinlock_lock(odp_spinlock_t *spinlock) +{ + /* While the lock is already taken... */ + while (__atomic_test_and_set(&spinlock->lock, __ATOMIC_ACQUIRE)) + /* ...spin reading the flag (relaxed MM), + * the loop will exit when the lock becomes available + * and we will retry the TAS operation above */ + while (__atomic_load_n(&spinlock->lock, __ATOMIC_RELAXED)) + odp_cpu_pause(); +} + +_ODP_INLINE int odp_spinlock_trylock(odp_spinlock_t *spinlock) +{ + return (__atomic_test_and_set(&spinlock->lock, __ATOMIC_ACQUIRE) == 0); +} + +_ODP_INLINE void odp_spinlock_unlock(odp_spinlock_t *spinlock) +{ + __atomic_clear(&spinlock->lock, __ATOMIC_RELEASE); +} + +_ODP_INLINE int odp_spinlock_is_locked(odp_spinlock_t *spinlock) +{ + return __atomic_load_n(&spinlock->lock, __ATOMIC_RELAXED) != 0; +} + +/** @endcond */ + +#endif diff --git a/platform/linux-generic/include/odp/api/plat/spinlock_recursive_inlines.h b/platform/linux-generic/include/odp/api/plat/spinlock_recursive_inlines.h new file mode 100644 index 000000000..2dd846fe9 --- /dev/null +++ b/platform/linux-generic/include/odp/api/plat/spinlock_recursive_inlines.h @@ -0,0 +1,91 @@ +/* Copyright (c) 2013-2018, Linaro Limited + * Copyright (c) 2022, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_PLAT_SPINLOCK_RECURSIVE_INLINES_H_ +#define ODP_PLAT_SPINLOCK_RECURSIVE_INLINES_H_ + +#include <odp/api/spinlock.h> +#include <odp/api/thread.h> + +#include <odp/api/abi/spinlock_recursive.h> + +/** @cond _ODP_HIDE_FROM_DOXYGEN_ */ + +#ifndef _ODP_NO_INLINE + /* Inline functions by default */ + #define _ODP_INLINE static inline + #define odp_spinlock_recursive_init __odp_spinlock_recursive_init + #define odp_spinlock_recursive_lock __odp_spinlock_recursive_lock + #define odp_spinlock_recursive_trylock __odp_spinlock_recursive_trylock + #define odp_spinlock_recursive_unlock __odp_spinlock_recursive_unlock + #define odp_spinlock_recursive_is_locked __odp_spinlock_recursive_is_locked + + #include <odp/api/plat/spinlock_inlines.h> + #include <odp/api/plat/thread_inlines.h> +#else + #undef _ODP_INLINE + #define _ODP_INLINE +#endif + +_ODP_INLINE void odp_spinlock_recursive_init(odp_spinlock_recursive_t *rlock) +{ + odp_spinlock_init(&rlock->lock); + rlock->owner = -1; + rlock->cnt = 0; +} + +_ODP_INLINE void odp_spinlock_recursive_lock(odp_spinlock_recursive_t *rlock) +{ + int thr = odp_thread_id(); + + if (rlock->owner == thr) { + rlock->cnt++; + return; + } + + odp_spinlock_lock(&rlock->lock); + rlock->owner = thr; + rlock->cnt = 1; +} + +_ODP_INLINE int odp_spinlock_recursive_trylock(odp_spinlock_recursive_t *rlock) +{ + int thr = odp_thread_id(); + + if (rlock->owner == thr) { + rlock->cnt++; + return 1; + } + + if (odp_spinlock_trylock(&rlock->lock)) { + rlock->owner = thr; + rlock->cnt = 1; + return 1; + } + + return 0; +} + +_ODP_INLINE void odp_spinlock_recursive_unlock(odp_spinlock_recursive_t *rlock) +{ + rlock->cnt--; + + if (rlock->cnt > 0) + return; + + rlock->owner = -1; + odp_spinlock_unlock(&rlock->lock); +} + +_ODP_INLINE int odp_spinlock_recursive_is_locked(odp_spinlock_recursive_t *rlock) +{ + return odp_thread_id() == rlock->owner ? 1 : odp_spinlock_is_locked(&rlock->lock); +} + +/** @endcond */ + +#endif diff --git a/platform/linux-generic/include/odp_buffer_internal.h b/platform/linux-generic/include/odp_buffer_internal.h index 5841720ef..8625fc5dd 100644 --- a/platform/linux-generic/include/odp_buffer_internal.h +++ b/platform/linux-generic/include/odp_buffer_internal.h @@ -50,20 +50,6 @@ static inline odp_buffer_hdr_t *_odp_buf_hdr(odp_buffer_t buf) return (odp_buffer_hdr_t *)(uintptr_t)buf; } -static inline uint32_t event_flow_id(odp_event_t ev) -{ - odp_buffer_hdr_t *buf_hdr = (odp_buffer_hdr_t *)(uintptr_t)ev; - - return buf_hdr->event_hdr.flow_id; -} - -static inline void event_flow_id_set(odp_event_t ev, uint32_t flow_id) -{ - odp_buffer_hdr_t *buf_hdr = (odp_buffer_hdr_t *)(uintptr_t)ev; - - buf_hdr->event_hdr.flow_id = flow_id; -} - #ifdef __cplusplus } #endif diff --git a/platform/linux-generic/include/odp_config_internal.h b/platform/linux-generic/include/odp_config_internal.h index 872d6f6d5..d3d09abf4 100644 --- a/platform/linux-generic/include/odp_config_internal.h +++ b/platform/linux-generic/include/odp_config_internal.h @@ -179,6 +179,16 @@ extern "C" { */ #define CONFIG_IPSEC_MAX_NUM_SA 4000 +/* + * Use 128-bit atomics for timer implementation (if available) + * + * On some platforms 128-bit atomic operations may be available, but the + * implementation of used 128-bit GCC built-in functions (e.g. + * __atomic_compare_exchange_n) utilizes expensive locking. Set to zero to use + * ODP lock based implementation instead. + */ +#define CONFIG_TIMER_128BIT_ATOMICS 1 + #ifdef __cplusplus } #endif diff --git a/platform/linux-generic/include/odp_event_internal.h b/platform/linux-generic/include/odp_event_internal.h index 92f201b01..60788fd52 100644 --- a/platform/linux-generic/include/odp_event_internal.h +++ b/platform/linux-generic/include/odp_event_internal.h @@ -79,11 +79,6 @@ static inline _odp_event_hdr_t *_odp_event_hdr(odp_event_t event) return (_odp_event_hdr_t *)(uintptr_t)event; } -static inline odp_event_type_t _odp_event_type(odp_event_t event) -{ - return _odp_event_hdr(event)->event_type; -} - static inline void _odp_event_type_set(odp_event_t event, int ev) { _odp_event_hdr(event)->event_type = ev; diff --git a/platform/linux-generic/include/odp_packet_internal.h b/platform/linux-generic/include/odp_packet_internal.h index e54d88f6a..977b3dda0 100644 --- a/platform/linux-generic/include/odp_packet_internal.h +++ b/platform/linux-generic/include/odp_packet_internal.h @@ -478,7 +478,7 @@ int _odp_packet_udp_chksum_insert(odp_packet_t pkt); int _odp_packet_sctp_chksum_insert(odp_packet_t pkt); int _odp_packet_l4_chksum(odp_packet_hdr_t *pkt_hdr, - odp_proto_chksums_t chksums, uint64_t l4_part_sum); + odp_pktin_config_opt_t opt, uint64_t l4_part_sum); #ifdef __cplusplus } diff --git a/platform/linux-generic/include/odp_packet_io_internal.h b/platform/linux-generic/include/odp_packet_io_internal.h index ca9f083da..0505d4378 100644 --- a/platform/linux-generic/include/odp_packet_io_internal.h +++ b/platform/linux-generic/include/odp_packet_io_internal.h @@ -65,12 +65,16 @@ struct pktio_if_ops; #if defined(_ODP_PKTIO_NETMAP) #define PKTIO_PRIVATE_SIZE 74752 +#elif defined(_ODP_PKTIO_XDP) && ODP_CACHE_LINE_SIZE == 128 +#define PKTIO_PRIVATE_SIZE 33792 +#elif defined(_ODP_PKTIO_XDP) +#define PKTIO_PRIVATE_SIZE 29696 #elif defined(_ODP_PKTIO_DPDK) && ODP_CACHE_LINE_SIZE == 128 #define PKTIO_PRIVATE_SIZE 10240 #elif defined(_ODP_PKTIO_DPDK) #define PKTIO_PRIVATE_SIZE 5632 #else -#define PKTIO_PRIVATE_SIZE 512 +#define PKTIO_PRIVATE_SIZE 384 #endif struct pktio_entry { @@ -124,14 +128,13 @@ struct pktio_entry { classifier_t cls; /**< classifier linked with this pktio*/ /* Driver level statistics counters */ odp_pktio_stats_t stats; - /* Statistics counters used outside drivers */ + /* Statistics counters used also outside drivers */ struct { odp_atomic_u64_t in_discards; odp_atomic_u64_t out_discards; } stats_extra; /* Latest Tx timestamp */ odp_atomic_u64_t tx_ts; - odp_proto_chksums_t in_chksums; /**< Checksums validation settings */ pktio_stats_type_t stats_type; char name[PKTIO_NAME_LEN]; /**< name of pktio provided to internal pktio_open() calls */ @@ -357,6 +360,27 @@ int _odp_lso_create_packets(odp_packet_t packet, const odp_packet_lso_opt_t *lso void _odp_pktio_allocate_and_send_tx_compl_events(const pktio_entry_t *entry, const odp_packet_t packets[], int num); +static inline int _odp_pktio_packet_to_pool(odp_packet_t *pkt, + odp_packet_hdr_t **pkt_hdr, + odp_pool_t new_pool) +{ + odp_packet_t new_pkt; + + if (odp_likely(new_pool == odp_packet_pool(*pkt))) + return 0; + + new_pkt = odp_packet_copy(*pkt, new_pool); + + if (odp_unlikely(new_pkt == ODP_PACKET_INVALID)) + return 1; + + odp_packet_free(*pkt); + *pkt = new_pkt; + *pkt_hdr = packet_hdr(new_pkt); + + return 0; +} + #ifdef __cplusplus } #endif diff --git a/platform/linux-generic/include/odp_parse_internal.h b/platform/linux-generic/include/odp_parse_internal.h index 22d8c2cf6..c467abbcd 100644 --- a/platform/linux-generic/include/odp_parse_internal.h +++ b/platform/linux-generic/include/odp_parse_internal.h @@ -64,7 +64,6 @@ int _odp_packet_parse_common_l3_l4(packet_parser_t *prs, const uint8_t *parseptr, uint32_t offset, uint32_t frame_len, uint32_t seg_len, int layer, uint16_t ethtype, - odp_proto_chksums_t chksums, uint64_t *l4_part_sum, odp_pktin_config_opt_t opt); @@ -77,17 +76,18 @@ int _odp_packet_parse_common_l3_l4(packet_parser_t *prs, * Returns 0 on success, 1 on packet errors, and -1 if the packet should be * dropped. */ -static inline int _odp_packet_parse_common(packet_parser_t *prs, +static inline int _odp_packet_parse_common(odp_packet_hdr_t *pkt_hdr, const uint8_t *ptr, uint32_t frame_len, uint32_t seg_len, int layer, - odp_proto_chksums_t chksums, - uint64_t *l4_part_sum, odp_pktin_config_opt_t opt) { + int r; uint32_t offset; uint16_t ethtype; const uint8_t *parseptr; + packet_parser_t *prs = &pkt_hdr->p; + uint64_t l4_part_sum = 0; parseptr = ptr; offset = 0; @@ -100,9 +100,14 @@ static inline int _odp_packet_parse_common(packet_parser_t *prs, ethtype = _odp_parse_eth(prs, &parseptr, &offset, frame_len); - return _odp_packet_parse_common_l3_l4(prs, parseptr, offset, frame_len, - seg_len, layer, ethtype, chksums, - l4_part_sum, opt); + r = _odp_packet_parse_common_l3_l4(prs, parseptr, offset, frame_len, + seg_len, layer, ethtype, + &l4_part_sum, opt); + + if (!r && layer >= ODP_PROTO_LAYER_L4) + r = _odp_packet_l4_chksum(pkt_hdr, opt, l4_part_sum); + + return r; } #ifdef __cplusplus |