diff options
author | Matias Elo <matias.elo@nokia.com> | 2022-07-01 11:44:35 +0300 |
---|---|---|
committer | GitHub <noreply@github.com> | 2022-07-01 11:44:35 +0300 |
commit | 3b9d1514e55955953786211f1fdec8e97f3688c7 (patch) | |
tree | 1d8ffcc8573670792ec788fd4a006103ff5e8a8e /platform | |
parent | 58fe883d5e43dc315d4751433f73c5d8ed7e5ba7 (diff) | |
parent | 3f7e35f89ff349d531542d73ba854de9695568f4 (diff) |
Merge ODP v1.37.1.0v1.37.1.0_DPDK_19.11
Merge ODP linux-generic v1.37.1.0 into linux-dpdk
Diffstat (limited to 'platform')
61 files changed, 1505 insertions, 846 deletions
diff --git a/platform/linux-dpdk/Makefile.am b/platform/linux-dpdk/Makefile.am index b3365be71..620aeddb4 100644 --- a/platform/linux-dpdk/Makefile.am +++ b/platform/linux-dpdk/Makefile.am @@ -28,11 +28,11 @@ if !ODP_ABI_COMPAT odpapiplatincludedir= $(includedir)/odp/api/plat odpapiplatinclude_HEADERS = \ include/odp/api/plat/atomic_inlines.h \ - include/odp/api/plat/buffer_inline_types.h \ include/odp/api/plat/buffer_inlines.h \ include/odp/api/plat/byteorder_inlines.h \ include/odp/api/plat/cpu_inlines.h \ include/odp/api/plat/event_inlines.h \ + include/odp/api/plat/event_inline_types.h \ include/odp/api/plat/event_vector_inline_types.h \ include/odp/api/plat/hash_inlines.h \ include/odp/api/plat/packet_flag_inlines.h \ @@ -43,6 +43,8 @@ odpapiplatinclude_HEADERS = \ include/odp/api/plat/pool_inline_types.h \ include/odp/api/plat/queue_inlines.h \ include/odp/api/plat/queue_inline_types.h \ + include/odp/api/plat/spinlock_inlines.h \ + include/odp/api/plat/spinlock_recursive_inlines.h \ include/odp/api/plat/std_inlines.h \ include/odp/api/plat/strong_types.h \ include/odp/api/plat/sync_inlines.h \ @@ -182,7 +184,7 @@ __LIB__libodp_dpdk_la_SOURCES = \ ../linux-generic/odp_dma.c \ odp_crypto.c \ odp_errno.c \ - ../linux-generic/odp_event.c \ + odp_event.c \ ../linux-generic/odp_hash_crc_gen.c \ odp_init.c \ ../linux-generic/odp_impl.c \ @@ -218,8 +220,6 @@ __LIB__libodp_dpdk_la_SOURCES = \ ../linux-generic/odp_schedule_sp.c \ odp_shared_memory.c \ ../linux-generic/odp_sorted_list.c \ - ../linux-generic/odp_spinlock.c \ - ../linux-generic/odp_spinlock_recursive.c \ ../linux-generic/odp_stash.c \ ../linux-generic/odp_std.c \ odp_system_info.c \ @@ -245,6 +245,8 @@ __LIB__libodp_dpdk_la_SOURCES += \ ../linux-generic/odp_packet_flags_api.c \ ../linux-generic/odp_packet_io_api.c \ ../linux-generic/odp_queue_api.c \ + ../linux-generic/odp_spinlock_api.c \ + ../linux-generic/odp_spinlock_recursive_api.c \ odp_std_api.c \ ../linux-generic/odp_sync_api.c \ ../linux-generic/odp_thread_api.c \ diff --git a/platform/linux-dpdk/include/odp/api/plat/buffer_inline_types.h b/platform/linux-dpdk/include/odp/api/plat/buffer_inline_types.h deleted file mode 100644 index f59df6705..000000000 --- a/platform/linux-dpdk/include/odp/api/plat/buffer_inline_types.h +++ /dev/null @@ -1,36 +0,0 @@ -/* Copyright (c) 2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -#ifndef ODP_PLAT_BUFFER_INLINE_TYPES_H_ -#define ODP_PLAT_BUFFER_INLINE_TYPES_H_ - -#ifdef __cplusplus -extern "C" { -#endif - -#include <stdint.h> - -/** @cond _ODP_HIDE_FROM_DOXYGEN_ */ - -/* Buffer header field accessor */ -#define _odp_buf_hdr_field(buf_hdr, cast, field) \ - (*(cast *)(uintptr_t)((uint8_t *)buf_hdr + \ - _odp_buffer_inline_offset.field)) - -/* Buffer header field offsets for inline functions */ -typedef struct _odp_buffer_inline_offset_t { - uint16_t event_type; - uint16_t base_data; - -} _odp_buffer_inline_offset_t; - -/** @endcond */ - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/platform/linux-dpdk/include/odp/api/plat/buffer_inlines.h b/platform/linux-dpdk/include/odp/api/plat/buffer_inlines.h index 9e0ae3f50..f536f728f 100644 --- a/platform/linux-dpdk/include/odp/api/plat/buffer_inlines.h +++ b/platform/linux-dpdk/include/odp/api/plat/buffer_inlines.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2019, Nokia +/* Copyright (c) 2019-2022, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -10,11 +10,11 @@ #include <odp/api/abi/buffer.h> #include <odp/api/abi/event_types.h> -#include <odp/api/plat/buffer_inline_types.h> +#include <odp/api/plat/event_inline_types.h> /** @cond _ODP_HIDE_FROM_DOXYGEN_ */ -extern const _odp_buffer_inline_offset_t _odp_buffer_inline_offset; +extern const _odp_event_inline_offset_t _odp_event_inline_offset; #ifndef _ODP_NO_INLINE /* Inline functions by default */ @@ -38,7 +38,7 @@ _ODP_INLINE odp_event_t odp_buffer_to_event(odp_buffer_t buf) _ODP_INLINE void *odp_buffer_addr(odp_buffer_t buf) { - return _odp_buf_hdr_field(buf, void *, base_data); + return _odp_event_hdr_field(buf, void *, base_data); } /** @endcond */ diff --git a/platform/linux-dpdk/include/odp/api/plat/event_inline_types.h b/platform/linux-dpdk/include/odp/api/plat/event_inline_types.h new file mode 120000 index 000000000..8899b80a6 --- /dev/null +++ b/platform/linux-dpdk/include/odp/api/plat/event_inline_types.h @@ -0,0 +1 @@ +../../../../../linux-generic/include/odp/api/plat/event_inline_types.h
\ No newline at end of file diff --git a/platform/linux-dpdk/include/odp/api/plat/spinlock_inlines.h b/platform/linux-dpdk/include/odp/api/plat/spinlock_inlines.h new file mode 120000 index 000000000..bbec045a5 --- /dev/null +++ b/platform/linux-dpdk/include/odp/api/plat/spinlock_inlines.h @@ -0,0 +1 @@ +../../../../../linux-generic/include/odp/api/plat/spinlock_inlines.h
\ No newline at end of file diff --git a/platform/linux-dpdk/include/odp/api/plat/spinlock_recursive_inlines.h b/platform/linux-dpdk/include/odp/api/plat/spinlock_recursive_inlines.h new file mode 120000 index 000000000..580627ade --- /dev/null +++ b/platform/linux-dpdk/include/odp/api/plat/spinlock_recursive_inlines.h @@ -0,0 +1 @@ +../../../../../linux-generic/include/odp/api/plat/spinlock_recursive_inlines.h
\ No newline at end of file diff --git a/platform/linux-dpdk/include/odp_buffer_internal.h b/platform/linux-dpdk/include/odp_buffer_internal.h index 3b99961c1..e7b1d215d 100644 --- a/platform/linux-dpdk/include/odp_buffer_internal.h +++ b/platform/linux-dpdk/include/odp_buffer_internal.h @@ -1,5 +1,5 @@ /* Copyright (c) 2013-2018, Linaro Limited - * Copyright (c) 2021, Nokia + * Copyright (c) 2021-2022, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -82,20 +82,6 @@ static inline odp_buffer_hdr_t *_odp_buf_hdr(odp_buffer_t buf) return (odp_buffer_hdr_t *)(uintptr_t)buf; } -static inline uint32_t event_flow_id(odp_event_t ev) -{ - odp_buffer_hdr_t *buf_hdr = (odp_buffer_hdr_t *)(uintptr_t)ev; - - return buf_hdr->event_hdr.flow_id; -} - -static inline void event_flow_id_set(odp_event_t ev, uint32_t flow_id) -{ - odp_buffer_hdr_t *buf_hdr = (odp_buffer_hdr_t *)(uintptr_t)ev; - - buf_hdr->event_hdr.flow_id = flow_id; -} - #ifdef __cplusplus } #endif diff --git a/platform/linux-dpdk/include/odp_event_internal.h b/platform/linux-dpdk/include/odp_event_internal.h index b3ce6c795..0614bbeae 100644 --- a/platform/linux-dpdk/include/odp_event_internal.h +++ b/platform/linux-dpdk/include/odp_event_internal.h @@ -79,11 +79,6 @@ static inline struct rte_mbuf *_odp_event_to_mbuf(odp_event_t event) return (struct rte_mbuf *)(uintptr_t)event; } -static inline odp_event_type_t _odp_event_type(odp_event_t event) -{ - return _odp_event_hdr(event)->event_type; -} - static inline void _odp_event_type_set(odp_event_t event, int ev) { _odp_event_hdr(event)->event_type = ev; diff --git a/platform/linux-dpdk/include/odp_packet_internal.h b/platform/linux-dpdk/include/odp_packet_internal.h index 02d7ff7b0..f1ae8dab0 100644 --- a/platform/linux-dpdk/include/odp_packet_internal.h +++ b/platform/linux-dpdk/include/odp_packet_internal.h @@ -410,7 +410,7 @@ int _odp_packet_udp_chksum_insert(odp_packet_t pkt); int _odp_packet_sctp_chksum_insert(odp_packet_t pkt); int _odp_packet_l4_chksum(odp_packet_hdr_t *pkt_hdr, - odp_proto_chksums_t chksums, uint64_t l4_part_sum); + odp_pktin_config_opt_t opt, uint64_t l4_part_sum); #ifdef __cplusplus } diff --git a/platform/linux-dpdk/include/odp_packet_io_internal.h b/platform/linux-dpdk/include/odp_packet_io_internal.h index 59410eef6..91f8350a8 100644 --- a/platform/linux-dpdk/include/odp_packet_io_internal.h +++ b/platform/linux-dpdk/include/odp_packet_io_internal.h @@ -111,14 +111,13 @@ struct pktio_entry { classifier_t cls; /**< classifier linked with this pktio*/ /* Driver level statistics counters */ odp_pktio_stats_t stats; - /* Statistics counters used outside drivers */ + /* Statistics counters used also outside drivers */ struct { odp_atomic_u64_t in_discards; odp_atomic_u64_t out_discards; } stats_extra; /* Latest Tx timestamp */ odp_atomic_u64_t tx_ts; - odp_proto_chksums_t in_chksums; /**< Checksums validation settings */ char name[PKTIO_NAME_LEN]; /**< name of pktio provided to internal pktio_open() calls */ char full_name[PKTIO_NAME_LEN]; /**< original pktio name passed to @@ -333,6 +332,27 @@ int _odp_lso_create_packets(odp_packet_t packet, const odp_packet_lso_opt_t *lso void _odp_pktio_allocate_and_send_tx_compl_events(const pktio_entry_t *entry, const odp_packet_t packets[], int num); +static inline int _odp_pktio_packet_to_pool(odp_packet_t *pkt, + odp_packet_hdr_t **pkt_hdr, + odp_pool_t new_pool) +{ + odp_packet_t new_pkt; + + if (odp_likely(new_pool == odp_packet_pool(*pkt))) + return 0; + + new_pkt = odp_packet_copy(*pkt, new_pool); + + if (odp_unlikely(new_pkt == ODP_PACKET_INVALID)) + return 1; + + odp_packet_free(*pkt); + *pkt = new_pkt; + *pkt_hdr = packet_hdr(new_pkt); + + return 0; +} + #ifdef __cplusplus } #endif diff --git a/platform/linux-dpdk/odp_buffer.c b/platform/linux-dpdk/odp_buffer.c index 21956be2f..58d6eaf83 100644 --- a/platform/linux-dpdk/odp_buffer.c +++ b/platform/linux-dpdk/odp_buffer.c @@ -1,30 +1,20 @@ /* Copyright (c) 2013-2018, Linaro Limited - * Copyright (c) 2021, Nokia + * Copyright (c) 2021-2022, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause */ #include <odp/api/buffer.h> + #include <odp_buffer_internal.h> #include <odp_debug_internal.h> #include <odp_pool_internal.h> -#include <odp/api/plat/buffer_inline_types.h> #include <string.h> #include <stdio.h> #include <inttypes.h> -#include <odp/visibility_begin.h> - -/* Fill in buffer header field offsets for inline functions */ -const _odp_buffer_inline_offset_t _odp_buffer_inline_offset ODP_ALIGNED_CACHE = { - .event_type = offsetof(odp_buffer_hdr_t, event_hdr.event_type), - .base_data = offsetof(odp_buffer_hdr_t, event_hdr.mb.buf_addr) -}; - -#include <odp/visibility_end.h> - uint32_t odp_buffer_size(odp_buffer_t buf) { struct rte_mbuf *mbuf = _odp_buf_to_mbuf(buf); diff --git a/platform/linux-dpdk/odp_event.c b/platform/linux-dpdk/odp_event.c new file mode 100644 index 000000000..f5b502205 --- /dev/null +++ b/platform/linux-dpdk/odp_event.c @@ -0,0 +1,127 @@ +/* Copyright (c) 2015-2018, Linaro Limited + * Copyright (c) 2020-2022, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <odp/api/event.h> +#include <odp/api/buffer.h> +#include <odp/api/crypto.h> +#include <odp/api/dma.h> +#include <odp/api/packet.h> +#include <odp/api/timer.h> +#include <odp/api/pool.h> + +#include <odp_buffer_internal.h> +#include <odp_ipsec_internal.h> +#include <odp_debug_internal.h> +#include <odp_packet_internal.h> +#include <odp_event_internal.h> +#include <odp_event_vector_internal.h> + +/* Inlined API functions */ +#include <odp/api/plat/event_inlines.h> +#include <odp/api/plat/packet_inlines.h> +#include <odp/api/plat/packet_vector_inlines.h> +#include <odp/api/plat/timer_inlines.h> + +#include <odp/api/plat/event_inline_types.h> + +#include <odp/visibility_begin.h> + +/* Fill in event header field offsets for inline functions */ +const _odp_event_inline_offset_t +_odp_event_inline_offset ODP_ALIGNED_CACHE = { + .event_type = offsetof(_odp_event_hdr_t, event_type), + .base_data = offsetof(_odp_event_hdr_t, mb.buf_addr), + .flow_id = offsetof(_odp_event_hdr_t, flow_id) +}; + +#include <odp/visibility_end.h> + +void odp_event_free(odp_event_t event) +{ + switch (odp_event_type(event)) { + case ODP_EVENT_BUFFER: + odp_buffer_free(odp_buffer_from_event(event)); + break; + case ODP_EVENT_PACKET: + odp_packet_free(odp_packet_from_event(event)); + break; + case ODP_EVENT_PACKET_VECTOR: + _odp_packet_vector_free_full(odp_packet_vector_from_event(event)); + break; + case ODP_EVENT_TIMEOUT: + odp_timeout_free(odp_timeout_from_event(event)); + break; +#if ODP_DEPRECATED_API + case ODP_EVENT_CRYPTO_COMPL: + odp_crypto_compl_free(odp_crypto_compl_from_event(event)); + break; +#endif + case ODP_EVENT_IPSEC_STATUS: + _odp_ipsec_status_free(_odp_ipsec_status_from_event(event)); + break; + case ODP_EVENT_PACKET_TX_COMPL: + odp_packet_tx_compl_free(odp_packet_tx_compl_from_event(event)); + break; + case ODP_EVENT_DMA_COMPL: + odp_dma_compl_free(odp_dma_compl_from_event(event)); + break; + default: + ODP_ABORT("Invalid event type: %d\n", odp_event_type(event)); + } +} + +void odp_event_free_multi(const odp_event_t event[], int num) +{ + int i; + + for (i = 0; i < num; i++) + odp_event_free(event[i]); +} + +void odp_event_free_sp(const odp_event_t event[], int num) +{ + odp_event_free_multi(event, num); +} + +uint64_t odp_event_to_u64(odp_event_t hdl) +{ + return _odp_pri(hdl); +} + +int odp_event_is_valid(odp_event_t event) +{ + if (event == ODP_EVENT_INVALID) + return 0; + + if (_odp_event_is_valid(event) == 0) + return 0; + + switch (odp_event_type(event)) { + case ODP_EVENT_BUFFER: + /* Fall through */ + case ODP_EVENT_PACKET: + /* Fall through */ + case ODP_EVENT_TIMEOUT: + /* Fall through */ +#if ODP_DEPRECATED_API + case ODP_EVENT_CRYPTO_COMPL: + /* Fall through */ +#endif + case ODP_EVENT_IPSEC_STATUS: + /* Fall through */ + case ODP_EVENT_PACKET_VECTOR: + /* Fall through */ + case ODP_EVENT_DMA_COMPL: + /* Fall through */ + case ODP_EVENT_PACKET_TX_COMPL: + break; + default: + return 0; + } + + return 1; +} diff --git a/platform/linux-dpdk/odp_packet.c b/platform/linux-dpdk/odp_packet.c index daef605d4..c7f166f7e 100644 --- a/platform/linux-dpdk/odp_packet.c +++ b/platform/linux-dpdk/odp_packet.c @@ -1477,13 +1477,14 @@ int _odp_packet_sctp_chksum_insert(odp_packet_t pkt) return odp_packet_copy_from_mem(pkt, pkt_hdr->p.l4_offset + 8, 4, &sum); } -int _odp_packet_l4_chksum(odp_packet_hdr_t *pkt_hdr, odp_proto_chksums_t chksums, - uint64_t l4_part_sum) +int _odp_packet_l4_chksum(odp_packet_hdr_t *pkt_hdr, + odp_pktin_config_opt_t opt, uint64_t l4_part_sum) + { uint32_t frame_len = odp_packet_len(packet_handle(pkt_hdr)); /* UDP chksum == 0 case is covered in parse_udp() */ - if (chksums.chksum.udp && + if (opt.bit.udp_chksum && pkt_hdr->p.input_flags.udp && !pkt_hdr->p.input_flags.ipfrag && !pkt_hdr->p.input_flags.udp_chksum_zero) { @@ -1502,7 +1503,7 @@ int _odp_packet_l4_chksum(odp_packet_hdr_t *pkt_hdr, odp_proto_chksums_t chksums } } - if (chksums.chksum.tcp && + if (opt.bit.tcp_chksum && pkt_hdr->p.input_flags.tcp && !pkt_hdr->p.input_flags.ipfrag) { uint16_t sum = ~packet_sum(pkt_hdr, @@ -1520,7 +1521,7 @@ int _odp_packet_l4_chksum(odp_packet_hdr_t *pkt_hdr, odp_proto_chksums_t chksums } } - if (chksums.chksum.sctp && + if (opt.bit.sctp_chksum && pkt_hdr->p.input_flags.sctp && !pkt_hdr->p.input_flags.ipfrag) { uint32_t seg_len = 0; @@ -1609,17 +1610,20 @@ int odp_packet_parse(odp_packet_t pkt, uint32_t offset, } opt.all_bits = 0; + opt.bit.ipv4_chksum = param->chksums.chksum.ipv4; + opt.bit.udp_chksum = param->chksums.chksum.udp; + opt.bit.tcp_chksum = param->chksums.chksum.tcp; + opt.bit.sctp_chksum = param->chksums.chksum.sctp; ret = _odp_packet_parse_common_l3_l4(&pkt_hdr->p, data, offset, packet_len, seg_len, layer, - ethtype, param->chksums, - &l4_part_sum, opt); + ethtype, &l4_part_sum, opt); if (ret) return -1; if (layer >= ODP_PROTO_LAYER_L4) { - ret = _odp_packet_l4_chksum(pkt_hdr, param->chksums, l4_part_sum); + ret = _odp_packet_l4_chksum(pkt_hdr, opt, l4_part_sum); if (ret) return -1; } diff --git a/platform/linux-dpdk/odp_packet_dpdk.c b/platform/linux-dpdk/odp_packet_dpdk.c index b44e0bda8..b0f19ea4f 100644 --- a/platform/linux-dpdk/odp_packet_dpdk.c +++ b/platform/linux-dpdk/odp_packet_dpdk.c @@ -930,7 +930,7 @@ static inline void prefetch_pkt(odp_packet_t pkt) odp_prefetch(&pkt_hdr->p); } -int _odp_input_pkts(pktio_entry_t *pktio_entry, odp_packet_t pkt_table[], int num) +static inline int input_pkts(pktio_entry_t *pktio_entry, odp_packet_t pkt_table[], int num) { pkt_dpdk_t * const pkt_dpdk = pkt_priv(pktio_entry); uint16_t i; @@ -939,6 +939,7 @@ int _odp_input_pkts(pktio_entry_t *pktio_entry, odp_packet_t pkt_table[], int nu odp_pktio_t input = pktio_entry->s.handle; odp_time_t ts_val; odp_time_t *ts = NULL; + const uint32_t supported_ptypes = pkt_dpdk->supported_ptypes; uint16_t num_prefetch = RTE_MIN(num, NUM_RX_PREFETCH); const odp_proto_layer_t layer = pktio_entry->s.parse_layer; @@ -966,7 +967,7 @@ int _odp_input_pkts(pktio_entry_t *pktio_entry, odp_packet_t pkt_table[], int nu rte_pktmbuf_pkt_len(mbuf), rte_pktmbuf_data_len(mbuf), mbuf, layer, - pkt_dpdk->supported_ptypes, pktin_cfg)) { + supported_ptypes, pktin_cfg)) { odp_packet_free(pkt); continue; } @@ -1017,18 +1018,24 @@ int _odp_input_pkts(pktio_entry_t *pktio_entry, odp_packet_t pkt_table[], int nu return num_pkts; } +int _odp_input_pkts(pktio_entry_t *pktio_entry, odp_packet_t pkt_table[], int num) +{ + return input_pkts(pktio_entry, pkt_table, num); +} + static int recv_pkt_dpdk(pktio_entry_t *pktio_entry, int index, odp_packet_t pkt_table[], int num) { pkt_dpdk_t * const pkt_dpdk = pkt_priv(pktio_entry); uint16_t nb_rx; - uint8_t min = pkt_dpdk->min_rx_burst; + const uint16_t port_id = pkt_dpdk->port_id; + const uint8_t min = pkt_dpdk->min_rx_burst; if (!pkt_dpdk->flags.lockless_rx) odp_ticketlock_lock(&pkt_dpdk->rx_lock[index]); if (odp_likely(num >= min)) { - nb_rx = rte_eth_rx_burst(pkt_dpdk->port_id, (uint16_t)index, + nb_rx = rte_eth_rx_burst(port_id, (uint16_t)index, (struct rte_mbuf **)pkt_table, (uint16_t)num); } else { @@ -1037,7 +1044,7 @@ static int recv_pkt_dpdk(pktio_entry_t *pktio_entry, int index, ODP_DBG("PMD requires >%d buffers burst. Current %d, dropped " "%d\n", min, num, min - num); - nb_rx = rte_eth_rx_burst(pkt_dpdk->port_id, (uint16_t)index, + nb_rx = rte_eth_rx_burst(port_id, (uint16_t)index, (struct rte_mbuf **)min_burst, min); for (i = 0; i < nb_rx; i++) { @@ -1054,10 +1061,10 @@ static int recv_pkt_dpdk(pktio_entry_t *pktio_entry, int index, odp_ticketlock_unlock(&pkt_dpdk->rx_lock[index]); /* Packets may also me received through eventdev, so don't add any - * processing here. Instead, perform all processing in _odp_input_pkts() + * processing here. Instead, perform all processing in input_pkts() * which is also called by eventdev. */ if (nb_rx) - return _odp_input_pkts(pktio_entry, pkt_table, nb_rx); + return input_pkts(pktio_entry, pkt_table, nb_rx); return 0; } diff --git a/platform/linux-dpdk/odp_system_info.c b/platform/linux-dpdk/odp_system_info.c index c506dbc1e..dcbfe4a06 100644 --- a/platform/linux-dpdk/odp_system_info.c +++ b/platform/linux-dpdk/odp_system_info.c @@ -288,6 +288,21 @@ static int read_config_file(void) return 0; } +static void print_compiler_info(void) +{ + ODP_PRINT("Compiler defines:\n"); + ODP_PRINT(" __GCC_ATOMIC_LLONG_LOCK_FREE: %d\n", __GCC_ATOMIC_LLONG_LOCK_FREE); + ODP_PRINT(" __GCC_ATOMIC_LONG_LOCK_FREE: %d\n", __GCC_ATOMIC_LONG_LOCK_FREE); + ODP_PRINT(" __GCC_ATOMIC_INT_LOCK_FREE: %d\n", __GCC_ATOMIC_INT_LOCK_FREE); + ODP_PRINT(" __GCC_HAVE_SYNC_COMPARE_AND_SWAP_16: "); +#ifdef __GCC_HAVE_SYNC_COMPARE_AND_SWAP_16 + ODP_PRINT("1\n"); +#else + ODP_PRINT("0\n"); +#endif + ODP_PRINT("\n"); +} + /* * System info initialisation */ @@ -341,6 +356,8 @@ int _odp_system_info_init(void) system_hp(&odp_global_ro.hugepage_info); + print_compiler_info(); + return 0; } diff --git a/platform/linux-generic/Makefile.am b/platform/linux-generic/Makefile.am index 6e64df740..7cec4799e 100644 --- a/platform/linux-generic/Makefile.am +++ b/platform/linux-generic/Makefile.am @@ -30,11 +30,11 @@ if !ODP_ABI_COMPAT odpapiplatincludedir= $(includedir)/odp/api/plat odpapiplatinclude_HEADERS = \ include/odp/api/plat/atomic_inlines.h \ - include/odp/api/plat/buffer_inline_types.h \ include/odp/api/plat/buffer_inlines.h \ include/odp/api/plat/byteorder_inlines.h \ include/odp/api/plat/cpu_inlines.h \ include/odp/api/plat/event_inlines.h \ + include/odp/api/plat/event_inline_types.h \ include/odp/api/plat/event_vector_inline_types.h \ include/odp/api/plat/hash_inlines.h \ include/odp/api/plat/packet_flag_inlines.h \ @@ -45,6 +45,8 @@ odpapiplatinclude_HEADERS = \ include/odp/api/plat/pool_inline_types.h \ include/odp/api/plat/queue_inlines.h \ include/odp/api/plat/queue_inline_types.h \ + include/odp/api/plat/spinlock_inlines.h \ + include/odp/api/plat/spinlock_recursive_inlines.h \ include/odp/api/plat/std_inlines.h \ include/odp/api/plat/strong_types.h \ include/odp/api/plat/sync_inlines.h \ @@ -228,8 +230,6 @@ __LIB__libodp_linux_la_SOURCES = \ odp_schedule_sp.c \ odp_shared_memory.c \ odp_sorted_list.c \ - odp_spinlock.c \ - odp_spinlock_recursive.c \ odp_stash.c \ odp_std.c \ odp_system_info.c \ @@ -284,6 +284,8 @@ __LIB__libodp_linux_la_SOURCES += \ odp_packet_flags_api.c \ odp_packet_io_api.c \ odp_queue_api.c \ + odp_spinlock_api.c \ + odp_spinlock_recursive_api.c \ odp_std_api.c \ odp_sync_api.c \ odp_thread_api.c \ diff --git a/platform/linux-generic/include-abi/odp/api/abi/spinlock.h b/platform/linux-generic/include-abi/odp/api/abi/spinlock.h index fbfbce5cc..d1e5fa1e9 100644 --- a/platform/linux-generic/include-abi/odp/api/abi/spinlock.h +++ b/platform/linux-generic/include-abi/odp/api/abi/spinlock.h @@ -5,3 +5,6 @@ */ #include <odp/api/abi-default/spinlock.h> + +/* Include inlined versions of API functions */ +#include <odp/api/plat/spinlock_inlines.h> diff --git a/platform/linux-generic/include-abi/odp/api/abi/spinlock_recursive.h b/platform/linux-generic/include-abi/odp/api/abi/spinlock_recursive.h index cc93b6acb..cdcbae1b4 100644 --- a/platform/linux-generic/include-abi/odp/api/abi/spinlock_recursive.h +++ b/platform/linux-generic/include-abi/odp/api/abi/spinlock_recursive.h @@ -5,3 +5,6 @@ */ #include <odp/api/abi-default/spinlock_recursive.h> + +/* Include inlined versions of API functions */ +#include <odp/api/plat/spinlock_recursive_inlines.h> diff --git a/platform/linux-generic/include/odp/api/plat/buffer_inline_types.h b/platform/linux-generic/include/odp/api/plat/buffer_inline_types.h deleted file mode 100644 index f59df6705..000000000 --- a/platform/linux-generic/include/odp/api/plat/buffer_inline_types.h +++ /dev/null @@ -1,36 +0,0 @@ -/* Copyright (c) 2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -#ifndef ODP_PLAT_BUFFER_INLINE_TYPES_H_ -#define ODP_PLAT_BUFFER_INLINE_TYPES_H_ - -#ifdef __cplusplus -extern "C" { -#endif - -#include <stdint.h> - -/** @cond _ODP_HIDE_FROM_DOXYGEN_ */ - -/* Buffer header field accessor */ -#define _odp_buf_hdr_field(buf_hdr, cast, field) \ - (*(cast *)(uintptr_t)((uint8_t *)buf_hdr + \ - _odp_buffer_inline_offset.field)) - -/* Buffer header field offsets for inline functions */ -typedef struct _odp_buffer_inline_offset_t { - uint16_t event_type; - uint16_t base_data; - -} _odp_buffer_inline_offset_t; - -/** @endcond */ - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/platform/linux-generic/include/odp/api/plat/buffer_inlines.h b/platform/linux-generic/include/odp/api/plat/buffer_inlines.h index 9e0ae3f50..3da402a83 100644 --- a/platform/linux-generic/include/odp/api/plat/buffer_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/buffer_inlines.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2019, Nokia +/* Copyright (c) 2019-2022, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -10,11 +10,11 @@ #include <odp/api/abi/buffer.h> #include <odp/api/abi/event_types.h> -#include <odp/api/plat/buffer_inline_types.h> +#include <odp/api/plat/event_inline_types.h> /** @cond _ODP_HIDE_FROM_DOXYGEN_ */ -extern const _odp_buffer_inline_offset_t _odp_buffer_inline_offset; +extern const _odp_event_inline_offset_t _odp_event_inline_offset; #ifndef _ODP_NO_INLINE /* Inline functions by default */ @@ -38,7 +38,7 @@ _ODP_INLINE odp_event_t odp_buffer_to_event(odp_buffer_t buf) _ODP_INLINE void *odp_buffer_addr(odp_buffer_t buf) { - return _odp_buf_hdr_field(buf, void *, base_data); + return _odp_event_hdr_field((odp_event_t)buf, void *, base_data); } /** @endcond */ diff --git a/platform/linux-generic/include/odp/api/plat/event_inline_types.h b/platform/linux-generic/include/odp/api/plat/event_inline_types.h new file mode 100644 index 000000000..c2727ec6f --- /dev/null +++ b/platform/linux-generic/include/odp/api/plat/event_inline_types.h @@ -0,0 +1,41 @@ +/* Copyright (c) 2018, Linaro Limited + * Copyright (c) 2022, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_PLAT_EVENT_INLINE_TYPES_H_ +#define ODP_PLAT_EVENT_INLINE_TYPES_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <stdint.h> + +/** @cond _ODP_HIDE_FROM_DOXYGEN_ */ + +/* Event header field accessors */ +#define _odp_event_hdr_field(event_hdr, cast, field) \ + (*(cast *)(uintptr_t)((uint8_t *)event_hdr + \ + _odp_event_inline_offset.field)) +#define _odp_event_hdr_ptr(event_hdr, cast, field) \ + ((cast *)(uintptr_t)((uint8_t *)event_hdr + \ + _odp_event_inline_offset.field)) + +/* Event header field offsets for inline functions */ +typedef struct _odp_event_inline_offset_t { + uint16_t event_type; + uint16_t base_data; + uint16_t flow_id; + +} _odp_event_inline_offset_t; + +/** @endcond */ + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/include/odp/api/plat/event_inlines.h b/platform/linux-generic/include/odp/api/plat/event_inlines.h index 43ee5e09a..27c2a82fd 100644 --- a/platform/linux-generic/include/odp/api/plat/event_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/event_inlines.h @@ -1,4 +1,5 @@ /* Copyright (c) 2018, Linaro Limited + * Copyright (c) 2022, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -7,18 +8,26 @@ #ifndef ODP_PLAT_EVENT_INLINES_H_ #define ODP_PLAT_EVENT_INLINES_H_ -#include <odp/api/abi/buffer.h> -#include <odp/api/plat/buffer_inline_types.h> +#include <odp/api/event_types.h> +#include <odp/api/packet.h> + +#include <odp/api/plat/event_inline_types.h> /** @cond _ODP_HIDE_FROM_DOXYGEN_ */ -extern const _odp_buffer_inline_offset_t _odp_buffer_inline_offset; +extern const _odp_event_inline_offset_t _odp_event_inline_offset; #ifndef _ODP_NO_INLINE /* Inline functions by default */ #define _ODP_INLINE static inline #define odp_event_type __odp_event_type #define odp_event_type_multi __odp_event_type_multi + #define odp_event_subtype __odp_event_subtype + #define odp_event_types __odp_event_types + #define odp_event_flow_id __odp_event_flow_id + #define odp_event_flow_id_set __odp_event_flow_id_set + + #include <odp/api/plat/packet_inlines.h> #else #define _ODP_INLINE #endif @@ -26,9 +35,8 @@ extern const _odp_buffer_inline_offset_t _odp_buffer_inline_offset; static inline odp_event_type_t __odp_event_type_get(odp_event_t event) { int8_t type; - odp_buffer_t buf = (odp_buffer_t)event; - type = _odp_buf_hdr_field(buf, int8_t, event_type); + type = _odp_event_hdr_field(event, int8_t, event_type); return (odp_event_type_t)type; } @@ -54,6 +62,38 @@ _ODP_INLINE int odp_event_type_multi(const odp_event_t event[], int num, return i; } +_ODP_INLINE odp_event_subtype_t odp_event_subtype(odp_event_t event) +{ + if (__odp_event_type_get(event) != ODP_EVENT_PACKET) + return ODP_EVENT_NO_SUBTYPE; + + return odp_packet_subtype(odp_packet_from_event(event)); +} + +_ODP_INLINE odp_event_type_t odp_event_types(odp_event_t event, + odp_event_subtype_t *subtype) +{ + odp_event_type_t event_type = __odp_event_type_get(event); + + *subtype = event_type == ODP_EVENT_PACKET ? + odp_packet_subtype(odp_packet_from_event(event)) : + ODP_EVENT_NO_SUBTYPE; + + return event_type; +} + +_ODP_INLINE uint32_t odp_event_flow_id(odp_event_t event) +{ + return _odp_event_hdr_field(event, uint8_t, flow_id); +} + +_ODP_INLINE void odp_event_flow_id_set(odp_event_t event, uint32_t id) +{ + uint8_t *flow_id = _odp_event_hdr_ptr(event, uint8_t, flow_id); + + *flow_id = (uint8_t)id; +} + /** @endcond */ #endif diff --git a/platform/linux-generic/include/odp/api/plat/spinlock_inlines.h b/platform/linux-generic/include/odp/api/plat/spinlock_inlines.h new file mode 100644 index 000000000..a04c43f88 --- /dev/null +++ b/platform/linux-generic/include/odp/api/plat/spinlock_inlines.h @@ -0,0 +1,65 @@ +/* Copyright (c) 2013-2018, Linaro Limited + * Copyright (c) 2022, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_PLAT_SPINLOCK_INLINES_H_ +#define ODP_PLAT_SPINLOCK_INLINES_H_ + +#include <odp/api/cpu.h> + +#include <odp/api/abi/spinlock.h> + +/** @cond _ODP_HIDE_FROM_DOXYGEN_ */ + +#ifndef _ODP_NO_INLINE + /* Inline functions by default */ + #define _ODP_INLINE static inline + #define odp_spinlock_init __odp_spinlock_init + #define odp_spinlock_lock __odp_spinlock_lock + #define odp_spinlock_trylock __odp_spinlock_trylock + #define odp_spinlock_unlock __odp_spinlock_unlock + #define odp_spinlock_is_locked __odp_spinlock_is_locked + + #include <odp/api/plat/cpu_inlines.h> +#else + #undef _ODP_INLINE + #define _ODP_INLINE +#endif + +_ODP_INLINE void odp_spinlock_init(odp_spinlock_t *spinlock) +{ + __atomic_clear(&spinlock->lock, __ATOMIC_RELAXED); +} + +_ODP_INLINE void odp_spinlock_lock(odp_spinlock_t *spinlock) +{ + /* While the lock is already taken... */ + while (__atomic_test_and_set(&spinlock->lock, __ATOMIC_ACQUIRE)) + /* ...spin reading the flag (relaxed MM), + * the loop will exit when the lock becomes available + * and we will retry the TAS operation above */ + while (__atomic_load_n(&spinlock->lock, __ATOMIC_RELAXED)) + odp_cpu_pause(); +} + +_ODP_INLINE int odp_spinlock_trylock(odp_spinlock_t *spinlock) +{ + return (__atomic_test_and_set(&spinlock->lock, __ATOMIC_ACQUIRE) == 0); +} + +_ODP_INLINE void odp_spinlock_unlock(odp_spinlock_t *spinlock) +{ + __atomic_clear(&spinlock->lock, __ATOMIC_RELEASE); +} + +_ODP_INLINE int odp_spinlock_is_locked(odp_spinlock_t *spinlock) +{ + return __atomic_load_n(&spinlock->lock, __ATOMIC_RELAXED) != 0; +} + +/** @endcond */ + +#endif diff --git a/platform/linux-generic/include/odp/api/plat/spinlock_recursive_inlines.h b/platform/linux-generic/include/odp/api/plat/spinlock_recursive_inlines.h new file mode 100644 index 000000000..2dd846fe9 --- /dev/null +++ b/platform/linux-generic/include/odp/api/plat/spinlock_recursive_inlines.h @@ -0,0 +1,91 @@ +/* Copyright (c) 2013-2018, Linaro Limited + * Copyright (c) 2022, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_PLAT_SPINLOCK_RECURSIVE_INLINES_H_ +#define ODP_PLAT_SPINLOCK_RECURSIVE_INLINES_H_ + +#include <odp/api/spinlock.h> +#include <odp/api/thread.h> + +#include <odp/api/abi/spinlock_recursive.h> + +/** @cond _ODP_HIDE_FROM_DOXYGEN_ */ + +#ifndef _ODP_NO_INLINE + /* Inline functions by default */ + #define _ODP_INLINE static inline + #define odp_spinlock_recursive_init __odp_spinlock_recursive_init + #define odp_spinlock_recursive_lock __odp_spinlock_recursive_lock + #define odp_spinlock_recursive_trylock __odp_spinlock_recursive_trylock + #define odp_spinlock_recursive_unlock __odp_spinlock_recursive_unlock + #define odp_spinlock_recursive_is_locked __odp_spinlock_recursive_is_locked + + #include <odp/api/plat/spinlock_inlines.h> + #include <odp/api/plat/thread_inlines.h> +#else + #undef _ODP_INLINE + #define _ODP_INLINE +#endif + +_ODP_INLINE void odp_spinlock_recursive_init(odp_spinlock_recursive_t *rlock) +{ + odp_spinlock_init(&rlock->lock); + rlock->owner = -1; + rlock->cnt = 0; +} + +_ODP_INLINE void odp_spinlock_recursive_lock(odp_spinlock_recursive_t *rlock) +{ + int thr = odp_thread_id(); + + if (rlock->owner == thr) { + rlock->cnt++; + return; + } + + odp_spinlock_lock(&rlock->lock); + rlock->owner = thr; + rlock->cnt = 1; +} + +_ODP_INLINE int odp_spinlock_recursive_trylock(odp_spinlock_recursive_t *rlock) +{ + int thr = odp_thread_id(); + + if (rlock->owner == thr) { + rlock->cnt++; + return 1; + } + + if (odp_spinlock_trylock(&rlock->lock)) { + rlock->owner = thr; + rlock->cnt = 1; + return 1; + } + + return 0; +} + +_ODP_INLINE void odp_spinlock_recursive_unlock(odp_spinlock_recursive_t *rlock) +{ + rlock->cnt--; + + if (rlock->cnt > 0) + return; + + rlock->owner = -1; + odp_spinlock_unlock(&rlock->lock); +} + +_ODP_INLINE int odp_spinlock_recursive_is_locked(odp_spinlock_recursive_t *rlock) +{ + return odp_thread_id() == rlock->owner ? 1 : odp_spinlock_is_locked(&rlock->lock); +} + +/** @endcond */ + +#endif diff --git a/platform/linux-generic/include/odp_buffer_internal.h b/platform/linux-generic/include/odp_buffer_internal.h index 5841720ef..8625fc5dd 100644 --- a/platform/linux-generic/include/odp_buffer_internal.h +++ b/platform/linux-generic/include/odp_buffer_internal.h @@ -50,20 +50,6 @@ static inline odp_buffer_hdr_t *_odp_buf_hdr(odp_buffer_t buf) return (odp_buffer_hdr_t *)(uintptr_t)buf; } -static inline uint32_t event_flow_id(odp_event_t ev) -{ - odp_buffer_hdr_t *buf_hdr = (odp_buffer_hdr_t *)(uintptr_t)ev; - - return buf_hdr->event_hdr.flow_id; -} - -static inline void event_flow_id_set(odp_event_t ev, uint32_t flow_id) -{ - odp_buffer_hdr_t *buf_hdr = (odp_buffer_hdr_t *)(uintptr_t)ev; - - buf_hdr->event_hdr.flow_id = flow_id; -} - #ifdef __cplusplus } #endif diff --git a/platform/linux-generic/include/odp_config_internal.h b/platform/linux-generic/include/odp_config_internal.h index 872d6f6d5..d3d09abf4 100644 --- a/platform/linux-generic/include/odp_config_internal.h +++ b/platform/linux-generic/include/odp_config_internal.h @@ -179,6 +179,16 @@ extern "C" { */ #define CONFIG_IPSEC_MAX_NUM_SA 4000 +/* + * Use 128-bit atomics for timer implementation (if available) + * + * On some platforms 128-bit atomic operations may be available, but the + * implementation of used 128-bit GCC built-in functions (e.g. + * __atomic_compare_exchange_n) utilizes expensive locking. Set to zero to use + * ODP lock based implementation instead. + */ +#define CONFIG_TIMER_128BIT_ATOMICS 1 + #ifdef __cplusplus } #endif diff --git a/platform/linux-generic/include/odp_event_internal.h b/platform/linux-generic/include/odp_event_internal.h index 92f201b01..60788fd52 100644 --- a/platform/linux-generic/include/odp_event_internal.h +++ b/platform/linux-generic/include/odp_event_internal.h @@ -79,11 +79,6 @@ static inline _odp_event_hdr_t *_odp_event_hdr(odp_event_t event) return (_odp_event_hdr_t *)(uintptr_t)event; } -static inline odp_event_type_t _odp_event_type(odp_event_t event) -{ - return _odp_event_hdr(event)->event_type; -} - static inline void _odp_event_type_set(odp_event_t event, int ev) { _odp_event_hdr(event)->event_type = ev; diff --git a/platform/linux-generic/include/odp_packet_internal.h b/platform/linux-generic/include/odp_packet_internal.h index e54d88f6a..977b3dda0 100644 --- a/platform/linux-generic/include/odp_packet_internal.h +++ b/platform/linux-generic/include/odp_packet_internal.h @@ -478,7 +478,7 @@ int _odp_packet_udp_chksum_insert(odp_packet_t pkt); int _odp_packet_sctp_chksum_insert(odp_packet_t pkt); int _odp_packet_l4_chksum(odp_packet_hdr_t *pkt_hdr, - odp_proto_chksums_t chksums, uint64_t l4_part_sum); + odp_pktin_config_opt_t opt, uint64_t l4_part_sum); #ifdef __cplusplus } diff --git a/platform/linux-generic/include/odp_packet_io_internal.h b/platform/linux-generic/include/odp_packet_io_internal.h index ca9f083da..0505d4378 100644 --- a/platform/linux-generic/include/odp_packet_io_internal.h +++ b/platform/linux-generic/include/odp_packet_io_internal.h @@ -65,12 +65,16 @@ struct pktio_if_ops; #if defined(_ODP_PKTIO_NETMAP) #define PKTIO_PRIVATE_SIZE 74752 +#elif defined(_ODP_PKTIO_XDP) && ODP_CACHE_LINE_SIZE == 128 +#define PKTIO_PRIVATE_SIZE 33792 +#elif defined(_ODP_PKTIO_XDP) +#define PKTIO_PRIVATE_SIZE 29696 #elif defined(_ODP_PKTIO_DPDK) && ODP_CACHE_LINE_SIZE == 128 #define PKTIO_PRIVATE_SIZE 10240 #elif defined(_ODP_PKTIO_DPDK) #define PKTIO_PRIVATE_SIZE 5632 #else -#define PKTIO_PRIVATE_SIZE 512 +#define PKTIO_PRIVATE_SIZE 384 #endif struct pktio_entry { @@ -124,14 +128,13 @@ struct pktio_entry { classifier_t cls; /**< classifier linked with this pktio*/ /* Driver level statistics counters */ odp_pktio_stats_t stats; - /* Statistics counters used outside drivers */ + /* Statistics counters used also outside drivers */ struct { odp_atomic_u64_t in_discards; odp_atomic_u64_t out_discards; } stats_extra; /* Latest Tx timestamp */ odp_atomic_u64_t tx_ts; - odp_proto_chksums_t in_chksums; /**< Checksums validation settings */ pktio_stats_type_t stats_type; char name[PKTIO_NAME_LEN]; /**< name of pktio provided to internal pktio_open() calls */ @@ -357,6 +360,27 @@ int _odp_lso_create_packets(odp_packet_t packet, const odp_packet_lso_opt_t *lso void _odp_pktio_allocate_and_send_tx_compl_events(const pktio_entry_t *entry, const odp_packet_t packets[], int num); +static inline int _odp_pktio_packet_to_pool(odp_packet_t *pkt, + odp_packet_hdr_t **pkt_hdr, + odp_pool_t new_pool) +{ + odp_packet_t new_pkt; + + if (odp_likely(new_pool == odp_packet_pool(*pkt))) + return 0; + + new_pkt = odp_packet_copy(*pkt, new_pool); + + if (odp_unlikely(new_pkt == ODP_PACKET_INVALID)) + return 1; + + odp_packet_free(*pkt); + *pkt = new_pkt; + *pkt_hdr = packet_hdr(new_pkt); + + return 0; +} + #ifdef __cplusplus } #endif diff --git a/platform/linux-generic/include/odp_parse_internal.h b/platform/linux-generic/include/odp_parse_internal.h index 22d8c2cf6..c467abbcd 100644 --- a/platform/linux-generic/include/odp_parse_internal.h +++ b/platform/linux-generic/include/odp_parse_internal.h @@ -64,7 +64,6 @@ int _odp_packet_parse_common_l3_l4(packet_parser_t *prs, const uint8_t *parseptr, uint32_t offset, uint32_t frame_len, uint32_t seg_len, int layer, uint16_t ethtype, - odp_proto_chksums_t chksums, uint64_t *l4_part_sum, odp_pktin_config_opt_t opt); @@ -77,17 +76,18 @@ int _odp_packet_parse_common_l3_l4(packet_parser_t *prs, * Returns 0 on success, 1 on packet errors, and -1 if the packet should be * dropped. */ -static inline int _odp_packet_parse_common(packet_parser_t *prs, +static inline int _odp_packet_parse_common(odp_packet_hdr_t *pkt_hdr, const uint8_t *ptr, uint32_t frame_len, uint32_t seg_len, int layer, - odp_proto_chksums_t chksums, - uint64_t *l4_part_sum, odp_pktin_config_opt_t opt) { + int r; uint32_t offset; uint16_t ethtype; const uint8_t *parseptr; + packet_parser_t *prs = &pkt_hdr->p; + uint64_t l4_part_sum = 0; parseptr = ptr; offset = 0; @@ -100,9 +100,14 @@ static inline int _odp_packet_parse_common(packet_parser_t *prs, ethtype = _odp_parse_eth(prs, &parseptr, &offset, frame_len); - return _odp_packet_parse_common_l3_l4(prs, parseptr, offset, frame_len, - seg_len, layer, ethtype, chksums, - l4_part_sum, opt); + r = _odp_packet_parse_common_l3_l4(prs, parseptr, offset, frame_len, + seg_len, layer, ethtype, + &l4_part_sum, opt); + + if (!r && layer >= ODP_PROTO_LAYER_L4) + r = _odp_packet_l4_chksum(pkt_hdr, opt, l4_part_sum); + + return r; } #ifdef __cplusplus diff --git a/platform/linux-generic/m4/odp_libconfig.m4 b/platform/linux-generic/m4/odp_libconfig.m4 index ab4994d61..886cc07e8 100644 --- a/platform/linux-generic/m4/odp_libconfig.m4 +++ b/platform/linux-generic/m4/odp_libconfig.m4 @@ -3,7 +3,7 @@ ########################################################################## m4_define([_odp_config_version_generation], [0]) m4_define([_odp_config_version_major], [1]) -m4_define([_odp_config_version_minor], [20]) +m4_define([_odp_config_version_minor], [21]) m4_define([_odp_config_version], [_odp_config_version_generation._odp_config_version_major._odp_config_version_minor]) diff --git a/platform/linux-generic/m4/odp_xdp.m4 b/platform/linux-generic/m4/odp_xdp.m4 index 2c6179df9..dcfd39ed7 100644 --- a/platform/linux-generic/m4/odp_xdp.m4 +++ b/platform/linux-generic/m4/odp_xdp.m4 @@ -5,7 +5,7 @@ AC_ARG_ENABLE([xdp], AS_HELP_STRING([--enable-xdp], [enable experimental XDP support for Packet I/O [default=disabled] (linux-generic)])) AS_IF([test "x$enable_xdp" = "xyes"], [ - PKG_CHECK_MODULES([LIBXDP], [libxdp], + PKG_CHECK_MODULES([LIBXDP], [libxdp >= 1.2.3], [ AC_DEFINE(_ODP_PKTIO_XDP, [1], [Define to 1 to enable xdp packet I/O support]) ], diff --git a/platform/linux-generic/odp_buffer.c b/platform/linux-generic/odp_buffer.c index f9f67a2e4..df3f047b1 100644 --- a/platform/linux-generic/odp_buffer.c +++ b/platform/linux-generic/odp_buffer.c @@ -6,26 +6,15 @@ */ #include <odp/api/buffer.h> + #include <odp_pool_internal.h> #include <odp_buffer_internal.h> #include <odp_debug_internal.h> -#include <odp/api/plat/buffer_inline_types.h> #include <string.h> #include <stdio.h> #include <inttypes.h> -#include <odp/visibility_begin.h> - -/* Fill in buffer header field offsets for inline functions */ -const _odp_buffer_inline_offset_t -_odp_buffer_inline_offset ODP_ALIGNED_CACHE = { - .event_type = offsetof(odp_buffer_hdr_t, event_hdr.event_type), - .base_data = offsetof(odp_buffer_hdr_t, event_hdr.base_data) -}; - -#include <odp/visibility_end.h> - uint32_t odp_buffer_size(odp_buffer_t buf) { odp_buffer_hdr_t *hdr = _odp_buf_hdr(buf); diff --git a/platform/linux-generic/odp_crypto_openssl.c b/platform/linux-generic/odp_crypto_openssl.c index 9f0978b49..9402c805b 100644 --- a/platform/linux-generic/odp_crypto_openssl.c +++ b/platform/linux-generic/odp_crypto_openssl.c @@ -1957,11 +1957,19 @@ int odp_crypto_capability(odp_crypto_capability_t *capa) /* Initialize crypto capability structure */ memset(capa, 0, sizeof(odp_crypto_capability_t)); + capa->max_sessions = MAX_SESSIONS; capa->sync_mode = ODP_SUPPORT_PREFERRED; capa->async_mode = ODP_SUPPORT_YES; capa->queue_type_plain = 1; capa->queue_type_sched = 1; + /* Memory allocation in libssl is not compatible with process mode */ + if (odp_global_ro.init_param.mem_model == ODP_MEM_MODEL_PROCESS) { + capa->ciphers.bit.null = 1; + capa->auths.bit.null = 1; + return 0; + } + capa->ciphers.bit.null = 1; capa->ciphers.bit.trides_cbc = 1; capa->ciphers.bit.trides_ecb = 1; @@ -2001,8 +2009,6 @@ int odp_crypto_capability(odp_crypto_capability_t *capa) capa->auths.bit.sha384 = 1; capa->auths.bit.sha512 = 1; - capa->max_sessions = MAX_SESSIONS; - return 0; } @@ -2196,6 +2202,16 @@ odp_crypto_session_create(const odp_crypto_session_param_t *param, return -1; } + /* Process mode is not supported with libssl based algos */ + if (odp_global_ro.init_param.mem_model == ODP_MEM_MODEL_PROCESS && + (param->cipher_alg != ODP_CIPHER_ALG_NULL || + param->auth_alg != ODP_AUTH_ALG_NULL)) { + *status = param->cipher_alg != ODP_CIPHER_ALG_NULL ? + ODP_CRYPTO_SES_ERR_CIPHER : ODP_CRYPTO_SES_ERR_AUTH; + *session_out = ODP_CRYPTO_SESSION_INVALID; + return -1; + } + /* Allocate memory for this session */ session = alloc_session(); if (NULL == session) { diff --git a/platform/linux-generic/odp_event.c b/platform/linux-generic/odp_event.c index 90c3e3a3c..50cb03a19 100644 --- a/platform/linux-generic/odp_event.c +++ b/platform/linux-generic/odp_event.c @@ -26,35 +26,19 @@ #include <odp/api/plat/packet_vector_inlines.h> #include <odp/api/plat/timer_inlines.h> -odp_event_subtype_t odp_event_subtype(odp_event_t event) -{ - if (_odp_event_type(event) != ODP_EVENT_PACKET) - return ODP_EVENT_NO_SUBTYPE; - - return odp_packet_subtype(odp_packet_from_event(event)); -} +#include <odp/api/plat/event_inline_types.h> -odp_event_type_t odp_event_types(odp_event_t event, - odp_event_subtype_t *subtype) -{ - odp_event_type_t event_type = _odp_event_type(event); - - *subtype = event_type == ODP_EVENT_PACKET ? - odp_packet_subtype(odp_packet_from_event(event)) : - ODP_EVENT_NO_SUBTYPE; +#include <odp/visibility_begin.h> - return event_type; -} +/* Fill in event header field offsets for inline functions */ +const _odp_event_inline_offset_t +_odp_event_inline_offset ODP_ALIGNED_CACHE = { + .event_type = offsetof(_odp_event_hdr_t, event_type), + .base_data = offsetof(_odp_event_hdr_t, base_data), + .flow_id = offsetof(_odp_event_hdr_t, flow_id) +}; -uint32_t odp_event_flow_id(odp_event_t event) -{ - return event_flow_id(event); -} - -void odp_event_flow_id_set(odp_event_t event, uint32_t flow_id) -{ - event_flow_id_set(event, flow_id); -} +#include <odp/visibility_end.h> void odp_event_free(odp_event_t event) { diff --git a/platform/linux-generic/odp_packet.c b/platform/linux-generic/odp_packet.c index 07e9c2d4d..89de5130d 100644 --- a/platform/linux-generic/odp_packet.c +++ b/platform/linux-generic/odp_packet.c @@ -1945,10 +1945,10 @@ int _odp_packet_sctp_chksum_insert(odp_packet_t pkt) } int _odp_packet_l4_chksum(odp_packet_hdr_t *pkt_hdr, - odp_proto_chksums_t chksums, uint64_t l4_part_sum) + odp_pktin_config_opt_t opt, uint64_t l4_part_sum) { /* UDP chksum == 0 case is covered in parse_udp() */ - if (chksums.chksum.udp && + if (opt.bit.udp_chksum && pkt_hdr->p.input_flags.udp && !pkt_hdr->p.input_flags.ipfrag && !pkt_hdr->p.input_flags.udp_chksum_zero) { @@ -1967,7 +1967,7 @@ int _odp_packet_l4_chksum(odp_packet_hdr_t *pkt_hdr, } } - if (chksums.chksum.tcp && + if (opt.bit.tcp_chksum && pkt_hdr->p.input_flags.tcp && !pkt_hdr->p.input_flags.ipfrag) { uint16_t sum = ~packet_sum(pkt_hdr, @@ -1985,7 +1985,7 @@ int _odp_packet_l4_chksum(odp_packet_hdr_t *pkt_hdr, } } - if (chksums.chksum.sctp && + if (opt.bit.sctp_chksum && pkt_hdr->p.input_flags.sctp && !pkt_hdr->p.input_flags.ipfrag) { uint32_t seg_len = 0; @@ -2074,17 +2074,20 @@ int odp_packet_parse(odp_packet_t pkt, uint32_t offset, } opt.all_bits = 0; + opt.bit.ipv4_chksum = param->chksums.chksum.ipv4; + opt.bit.udp_chksum = param->chksums.chksum.udp; + opt.bit.tcp_chksum = param->chksums.chksum.tcp; + opt.bit.sctp_chksum = param->chksums.chksum.sctp; ret = _odp_packet_parse_common_l3_l4(&pkt_hdr->p, data, offset, packet_len, seg_len, layer, - ethtype, param->chksums, - &l4_part_sum, opt); + ethtype, &l4_part_sum, opt); if (ret) return -1; if (layer >= ODP_PROTO_LAYER_L4) { - ret = _odp_packet_l4_chksum(pkt_hdr, param->chksums, l4_part_sum); + ret = _odp_packet_l4_chksum(pkt_hdr, opt, l4_part_sum); if (ret) return -1; } diff --git a/platform/linux-generic/odp_packet_io.c b/platform/linux-generic/odp_packet_io.c index 7c68b10c5..02d1a827c 100644 --- a/platform/linux-generic/odp_packet_io.c +++ b/platform/linux-generic/odp_packet_io.c @@ -612,12 +612,6 @@ int odp_pktio_config(odp_pktio_t hdl, const odp_pktio_config_t *config) entry->s.config = *config; - entry->s.in_chksums.all_chksum = 0; - entry->s.in_chksums.chksum.ipv4 = config->pktin.bit.ipv4_chksum; - entry->s.in_chksums.chksum.tcp = config->pktin.bit.tcp_chksum; - entry->s.in_chksums.chksum.udp = config->pktin.bit.udp_chksum; - entry->s.in_chksums.chksum.sctp = config->pktin.bit.sctp_chksum; - entry->s.enabled.tx_ts = config->pktout.bit.ts_ena; entry->s.enabled.tx_compl = config->pktout.bit.tx_compl_ena; @@ -3102,8 +3096,8 @@ int odp_packet_lso_request(odp_packet_t pkt, const odp_packet_lso_opt_t *lso_opt return -1; } - if (odp_unlikely((payload_offset + lso_opt->max_payload_len) > packet_len(pkt_hdr))) { - ODP_ERR("LSO options larger than packet data length\n"); + if (odp_unlikely(payload_offset > packet_len(pkt_hdr))) { + ODP_ERR("LSO payload offset larger than packet data length\n"); return -1; } @@ -3212,11 +3206,19 @@ int _odp_lso_num_packets(odp_packet_t packet, const odp_packet_lso_opt_t *lso_op return -1; } - if (odp_unlikely((hdr_len + payload_len) > pkt_len)) { - ODP_ERR("LSO options larger than packet data length\n"); + if (odp_unlikely(hdr_len > pkt_len)) { + ODP_ERR("LSO payload offset larger than packet data length\n"); return -1; } + if (odp_unlikely(hdr_len + payload_len > odp_packet_len(packet))) { + /* Packet does not need segmentation */ + *len_out = payload_len; + *left_over_out = 0; + + return 1; + } + if (lso_prof->param.lso_proto == ODP_LSO_PROTO_IPV4) { l3_offset = odp_packet_l3_offset(packet); iphdr_len = hdr_len - l3_offset; @@ -3364,6 +3366,14 @@ static int pktout_send_lso(odp_pktout_queue_t queue, odp_packet_t packet, if (odp_unlikely(num_pkt <= 0)) return -1; + if (odp_unlikely(num_pkt == 1)) { + /* Segmentation not needed */ + if (odp_pktout_send(queue, &packet, 1) != 1) + return -1; + + return 0; + } + /* Create packets */ odp_packet_t pkt_out[num_pkt]; diff --git a/platform/linux-generic/odp_parse.c b/platform/linux-generic/odp_parse.c index 2342a7e49..13935be56 100644 --- a/platform/linux-generic/odp_parse.c +++ b/platform/linux-generic/odp_parse.c @@ -119,7 +119,7 @@ error: */ static inline uint8_t parse_ipv4(packet_parser_t *prs, const uint8_t **parseptr, uint32_t *offset, uint32_t frame_len, - odp_proto_chksums_t chksums, + odp_pktin_config_opt_t opt, uint64_t *l4_part_sum) { const _odp_ipv4hdr_t *ipv4 = (const _odp_ipv4hdr_t *)*parseptr; @@ -137,7 +137,7 @@ static inline uint8_t parse_ipv4(packet_parser_t *prs, const uint8_t **parseptr, return 0; } - if (chksums.chksum.ipv4) { + if (opt.bit.ipv4_chksum) { prs->input_flags.l3_chksum_done = 1; if (chksum_finalize(chksum_partial(ipv4, ihl * 4, 0)) != 0xffff) { prs->flags.ip_err = 1; @@ -149,7 +149,7 @@ static inline uint8_t parse_ipv4(packet_parser_t *prs, const uint8_t **parseptr, *offset += ihl * 4; *parseptr += ihl * 4; - if (chksums.chksum.udp || chksums.chksum.tcp) + if (opt.bit.udp_chksum || opt.bit.tcp_chksum) *l4_part_sum = chksum_partial((const uint8_t *)&ipv4->src_addr, 2 * _ODP_IPV4ADDR_LEN, 0); @@ -182,7 +182,7 @@ static inline uint8_t parse_ipv4(packet_parser_t *prs, const uint8_t **parseptr, static inline uint8_t parse_ipv6(packet_parser_t *prs, const uint8_t **parseptr, uint32_t *offset, uint32_t frame_len, uint32_t seg_len, - odp_proto_chksums_t chksums, + odp_pktin_config_opt_t opt, uint64_t *l4_part_sum) { const _odp_ipv6hdr_t *ipv6 = (const _odp_ipv6hdr_t *)*parseptr; @@ -207,7 +207,7 @@ static inline uint8_t parse_ipv6(packet_parser_t *prs, const uint8_t **parseptr, *offset += sizeof(_odp_ipv6hdr_t); *parseptr += sizeof(_odp_ipv6hdr_t); - if (chksums.chksum.udp || chksums.chksum.tcp) + if (opt.bit.udp_chksum || opt.bit.tcp_chksum) *l4_part_sum = chksum_partial((const uint8_t *)&ipv6->src_addr, 2 * _ODP_IPV6ADDR_LEN, 0); @@ -253,7 +253,7 @@ static inline uint8_t parse_ipv6(packet_parser_t *prs, const uint8_t **parseptr, */ static inline void parse_tcp(packet_parser_t *prs, const uint8_t **parseptr, uint16_t tcp_len, - odp_proto_chksums_t chksums, + odp_pktin_config_opt_t opt, uint64_t *l4_part_sum) { const _odp_tcphdr_t *tcp = (const _odp_tcphdr_t *)*parseptr; @@ -262,7 +262,7 @@ static inline void parse_tcp(packet_parser_t *prs, const uint8_t **parseptr, if (odp_unlikely(tcp->hl < sizeof(_odp_tcphdr_t) / sizeof(uint32_t))) prs->flags.tcp_err = 1; - if (chksums.chksum.tcp && + if (opt.bit.tcp_chksum && !prs->input_flags.ipfrag) { *l4_part_sum += odp_cpu_to_be_16(tcp_len); #if ODP_BYTE_ORDER == ODP_BIG_ENDIAN @@ -281,7 +281,7 @@ static inline void parse_tcp(packet_parser_t *prs, const uint8_t **parseptr, * Requires PARSE_UDP_BYTES bytes of contiguous packet data. */ static inline void parse_udp(packet_parser_t *prs, const uint8_t **parseptr, - odp_proto_chksums_t chksums, + odp_pktin_config_opt_t opt, uint64_t *l4_part_sum) { const _odp_udphdr_t *udp = (const _odp_udphdr_t *)*parseptr; @@ -293,7 +293,7 @@ static inline void parse_udp(packet_parser_t *prs, const uint8_t **parseptr, return; } - if (chksums.chksum.udp && + if (opt.bit.udp_chksum && !prs->input_flags.ipfrag) { if (udp->chksum == 0) { prs->input_flags.l4_chksum_done = 1; @@ -330,7 +330,7 @@ static inline void parse_udp(packet_parser_t *prs, const uint8_t **parseptr, */ static inline void parse_sctp(packet_parser_t *prs, const uint8_t **parseptr, uint16_t sctp_len, - odp_proto_chksums_t chksums, + odp_pktin_config_opt_t opt, uint64_t *l4_part_sum) { if (odp_unlikely(sctp_len < sizeof(_odp_sctphdr_t))) { @@ -338,7 +338,7 @@ static inline void parse_sctp(packet_parser_t *prs, const uint8_t **parseptr, return; } - if (chksums.chksum.sctp && + if (opt.bit.sctp_chksum && !prs->input_flags.ipfrag) { const _odp_sctphdr_t *sctp = (const _odp_sctphdr_t *)*parseptr; @@ -358,7 +358,6 @@ int _odp_packet_parse_common_l3_l4(packet_parser_t *prs, const uint8_t *parseptr, uint32_t offset, uint32_t frame_len, uint32_t seg_len, int layer, uint16_t ethtype, - odp_proto_chksums_t chksums, uint64_t *l4_part_sum, odp_pktin_config_opt_t opt) { @@ -377,7 +376,7 @@ int _odp_packet_parse_common_l3_l4(packet_parser_t *prs, case _ODP_ETHTYPE_IPV4: prs->input_flags.ipv4 = 1; ip_proto = parse_ipv4(prs, &parseptr, &offset, frame_len, - chksums, l4_part_sum); + opt, l4_part_sum); prs->l4_offset = offset; if (prs->flags.ip_err && opt.bit.drop_ipv4_err) return -1; /* drop */ @@ -386,7 +385,7 @@ int _odp_packet_parse_common_l3_l4(packet_parser_t *prs, case _ODP_ETHTYPE_IPV6: prs->input_flags.ipv6 = 1; ip_proto = parse_ipv6(prs, &parseptr, &offset, frame_len, - seg_len, chksums, l4_part_sum); + seg_len, opt, l4_part_sum); prs->l4_offset = offset; if (prs->flags.ip_err && opt.bit.drop_ipv6_err) return -1; /* drop */ @@ -425,7 +424,7 @@ int _odp_packet_parse_common_l3_l4(packet_parser_t *prs, if (odp_unlikely(offset + _ODP_TCPHDR_LEN > seg_len)) return -1; prs->input_flags.tcp = 1; - parse_tcp(prs, &parseptr, frame_len - prs->l4_offset, chksums, + parse_tcp(prs, &parseptr, frame_len - prs->l4_offset, opt, l4_part_sum); if (prs->flags.tcp_err && opt.bit.drop_tcp_err) return -1; /* drop */ @@ -435,7 +434,7 @@ int _odp_packet_parse_common_l3_l4(packet_parser_t *prs, if (odp_unlikely(offset + _ODP_UDPHDR_LEN > seg_len)) return -1; prs->input_flags.udp = 1; - parse_udp(prs, &parseptr, chksums, l4_part_sum); + parse_udp(prs, &parseptr, opt, l4_part_sum); if (prs->flags.udp_err && opt.bit.drop_udp_err) return -1; /* drop */ break; @@ -452,7 +451,7 @@ int _odp_packet_parse_common_l3_l4(packet_parser_t *prs, case _ODP_IPPROTO_SCTP: prs->input_flags.sctp = 1; - parse_sctp(prs, &parseptr, frame_len - prs->l4_offset, chksums, + parse_sctp(prs, &parseptr, frame_len - prs->l4_offset, opt, l4_part_sum); if (prs->flags.sctp_err && opt.bit.drop_sctp_err) return -1; /* drop */ diff --git a/platform/linux-generic/odp_queue_scalable.c b/platform/linux-generic/odp_queue_scalable.c index 881dbb985..26203fa5a 100644 --- a/platform/linux-generic/odp_queue_scalable.c +++ b/platform/linux-generic/odp_queue_scalable.c @@ -118,6 +118,10 @@ static int queue_init(queue_entry_t *queue, const char *name, ring[ring_idx] = NULL; queue->s.type = queue->s.param.type; + + if (queue->s.type == ODP_QUEUE_TYPE_SCHED) + queue->s.param.deq_mode = ODP_QUEUE_OP_DISABLED; + odp_atomic_init_u64(&queue->s.num_timers, 0); queue->s.enqueue = _queue_enq; diff --git a/platform/linux-generic/odp_spinlock.c b/platform/linux-generic/odp_spinlock.c deleted file mode 100644 index b38cc6a3a..000000000 --- a/platform/linux-generic/odp_spinlock.c +++ /dev/null @@ -1,42 +0,0 @@ -/* Copyright (c) 2013-2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -#include <odp/api/spinlock.h> -#include <odp/api/cpu.h> -#include <odp_atomic_internal.h> - -#include <odp/api/plat/cpu_inlines.h> - -void odp_spinlock_init(odp_spinlock_t *spinlock) -{ - _odp_atomic_flag_init(&spinlock->lock, 0); -} - -void odp_spinlock_lock(odp_spinlock_t *spinlock) -{ - /* While the lock is already taken... */ - while (_odp_atomic_flag_tas(&spinlock->lock)) - /* ...spin reading the flag (relaxed MM), - * the loop will exit when the lock becomes available - * and we will retry the TAS operation above */ - while (_odp_atomic_flag_load(&spinlock->lock)) - odp_cpu_pause(); -} - -int odp_spinlock_trylock(odp_spinlock_t *spinlock) -{ - return (_odp_atomic_flag_tas(&spinlock->lock) == 0); -} - -void odp_spinlock_unlock(odp_spinlock_t *spinlock) -{ - _odp_atomic_flag_clear(&spinlock->lock); -} - -int odp_spinlock_is_locked(odp_spinlock_t *spinlock) -{ - return _odp_atomic_flag_load(&spinlock->lock) != 0; -} diff --git a/platform/linux-generic/odp_spinlock_api.c b/platform/linux-generic/odp_spinlock_api.c new file mode 100644 index 000000000..06925e9a5 --- /dev/null +++ b/platform/linux-generic/odp_spinlock_api.c @@ -0,0 +1,10 @@ +/* Copyright (c) 2022, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <odp/api/spinlock.h> + +#define _ODP_NO_INLINE +#include <odp/api/plat/spinlock_inlines.h> diff --git a/platform/linux-generic/odp_spinlock_recursive.c b/platform/linux-generic/odp_spinlock_recursive.c deleted file mode 100644 index 6363a3838..000000000 --- a/platform/linux-generic/odp_spinlock_recursive.c +++ /dev/null @@ -1,71 +0,0 @@ -/* Copyright (c) 2013-2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -#include <odp/api/spinlock_recursive.h> -#include <odp/api/thread.h> -#include <odp/api/plat/thread_inlines.h> - -#define NO_OWNER (-1) - -void odp_spinlock_recursive_init(odp_spinlock_recursive_t *rlock) -{ - odp_spinlock_init(&rlock->lock); - rlock->owner = NO_OWNER; - rlock->cnt = 0; -} - -void odp_spinlock_recursive_lock(odp_spinlock_recursive_t *rlock) -{ - int thr = odp_thread_id(); - - if (rlock->owner == thr) { - rlock->cnt++; - return; - } - - odp_spinlock_lock(&rlock->lock); - rlock->owner = thr; - rlock->cnt = 1; -} - -int odp_spinlock_recursive_trylock(odp_spinlock_recursive_t *rlock) -{ - int thr = odp_thread_id(); - - if (rlock->owner == thr) { - rlock->cnt++; - return 1; - } - - if (odp_spinlock_trylock(&rlock->lock)) { - rlock->owner = thr; - rlock->cnt = 1; - return 1; - } else { - return 0; - } -} - -void odp_spinlock_recursive_unlock(odp_spinlock_recursive_t *rlock) -{ - rlock->cnt--; - - if (rlock->cnt > 0) - return; - - rlock->owner = NO_OWNER; - odp_spinlock_unlock(&rlock->lock); -} - -int odp_spinlock_recursive_is_locked(odp_spinlock_recursive_t *rlock) -{ - int thr = odp_thread_id(); - - if (rlock->owner == thr) - return 1; - - return odp_spinlock_is_locked(&rlock->lock); -} diff --git a/platform/linux-generic/odp_spinlock_recursive_api.c b/platform/linux-generic/odp_spinlock_recursive_api.c new file mode 100644 index 000000000..2b1e8b200 --- /dev/null +++ b/platform/linux-generic/odp_spinlock_recursive_api.c @@ -0,0 +1,10 @@ +/* Copyright (c) 2022, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <odp/api/spinlock_recursive.h> + +#define _ODP_NO_INLINE +#include <odp/api/plat/spinlock_recursive_inlines.h> diff --git a/platform/linux-generic/odp_stash.c b/platform/linux-generic/odp_stash.c index e12f1aed3..15d995eaf 100644 --- a/platform/linux-generic/odp_stash.c +++ b/platform/linux-generic/odp_stash.c @@ -166,6 +166,7 @@ odp_stash_t odp_stash_create(const char *name, const odp_stash_param_t *param) uint64_t i, ring_size, shm_size; int ring_u64, index; char shm_name[ODP_STASH_NAME_LEN + 8]; + uint32_t shm_flags = 0; if (odp_global_ro.disable.stash) { ODP_ERR("Stash is disabled\n"); @@ -214,7 +215,10 @@ odp_stash_t odp_stash_create(const char *name, const odp_stash_param_t *param) else shm_size = sizeof(stash_t) + (ring_size * sizeof(uint32_t)); - shm = odp_shm_reserve(shm_name, shm_size, ODP_CACHE_LINE_SIZE, 0); + if (odp_global_ro.shm_single_va) + shm_flags |= ODP_SHM_SINGLE_VA; + + shm = odp_shm_reserve(shm_name, shm_size, ODP_CACHE_LINE_SIZE, shm_flags); if (shm == ODP_SHM_INVALID) { ODP_ERR("SHM reserve failed.\n"); @@ -361,7 +365,7 @@ int32_t odp_stash_put(odp_stash_t st, const void *obj, int32_t num) return -1; } -int32_t odp_stash_put_u32(odp_stash_t st, const uint32_t u32[], int32_t num) +int32_t odp_stash_put_u32(odp_stash_t st, const uint32_t val[], int32_t num) { stash_t *stash = (stash_t *)(uintptr_t)st; @@ -371,11 +375,11 @@ int32_t odp_stash_put_u32(odp_stash_t st, const uint32_t u32[], int32_t num) ODP_ASSERT(stash->obj_size == sizeof(uint32_t)); ring_u32_enq_multi(&stash->ring_u32.hdr, stash->ring_mask, - (uint32_t *)(uintptr_t)u32, num); + (uint32_t *)(uintptr_t)val, num); return num; } -int32_t odp_stash_put_u64(odp_stash_t st, const uint64_t u64[], int32_t num) +int32_t odp_stash_put_u64(odp_stash_t st, const uint64_t val[], int32_t num) { stash_t *stash = (stash_t *)(uintptr_t)st; @@ -385,7 +389,7 @@ int32_t odp_stash_put_u64(odp_stash_t st, const uint64_t u64[], int32_t num) ODP_ASSERT(stash->obj_size == sizeof(uint64_t)); ring_u64_enq_multi(&stash->ring_u64.hdr, stash->ring_mask, - (uint64_t *)(uintptr_t)u64, num); + (uint64_t *)(uintptr_t)val, num); return num; } @@ -466,7 +470,7 @@ int32_t odp_stash_get(odp_stash_t st, void *obj, int32_t num) return -1; } -int32_t odp_stash_get_u32(odp_stash_t st, uint32_t u32[], int32_t num) +int32_t odp_stash_get_u32(odp_stash_t st, uint32_t val[], int32_t num) { stash_t *stash = (stash_t *)(uintptr_t)st; @@ -475,11 +479,11 @@ int32_t odp_stash_get_u32(odp_stash_t st, uint32_t u32[], int32_t num) ODP_ASSERT(stash->obj_size == sizeof(uint32_t)); - return ring_u32_deq_multi(&stash->ring_u32.hdr, stash->ring_mask, u32, + return ring_u32_deq_multi(&stash->ring_u32.hdr, stash->ring_mask, val, num); } -int32_t odp_stash_get_u64(odp_stash_t st, uint64_t u64[], int32_t num) +int32_t odp_stash_get_u64(odp_stash_t st, uint64_t val[], int32_t num) { stash_t *stash = (stash_t *)(uintptr_t)st; @@ -488,7 +492,7 @@ int32_t odp_stash_get_u64(odp_stash_t st, uint64_t u64[], int32_t num) ODP_ASSERT(stash->obj_size == sizeof(uint64_t)); - return ring_u64_deq_multi(&stash->ring_u64.hdr, stash->ring_mask, u64, + return ring_u64_deq_multi(&stash->ring_u64.hdr, stash->ring_mask, val, num); } diff --git a/platform/linux-generic/odp_system_info.c b/platform/linux-generic/odp_system_info.c index 5665a3ece..69a088032 100644 --- a/platform/linux-generic/odp_system_info.c +++ b/platform/linux-generic/odp_system_info.c @@ -362,6 +362,21 @@ static int read_config_file(void) return 0; } +static void print_compiler_info(void) +{ + ODP_PRINT("Compiler defines:\n"); + ODP_PRINT(" __GCC_ATOMIC_LLONG_LOCK_FREE: %d\n", __GCC_ATOMIC_LLONG_LOCK_FREE); + ODP_PRINT(" __GCC_ATOMIC_LONG_LOCK_FREE: %d\n", __GCC_ATOMIC_LONG_LOCK_FREE); + ODP_PRINT(" __GCC_ATOMIC_INT_LOCK_FREE: %d\n", __GCC_ATOMIC_INT_LOCK_FREE); + ODP_PRINT(" __GCC_HAVE_SYNC_COMPARE_AND_SWAP_16: "); +#ifdef __GCC_HAVE_SYNC_COMPARE_AND_SWAP_16 + ODP_PRINT("1\n"); +#else + ODP_PRINT("0\n"); +#endif + ODP_PRINT("\n"); +} + /* * System info initialisation */ @@ -415,6 +430,8 @@ int _odp_system_info_init(void) system_hp(&odp_global_ro.hugepage_info); + print_compiler_info(); + return 0; } @@ -612,5 +629,6 @@ void odp_sys_config_print(void) ODP_PRINT("CONFIG_BURST_SIZE: %i\n", CONFIG_BURST_SIZE); ODP_PRINT("CONFIG_POOL_MAX_NUM: %i\n", CONFIG_POOL_MAX_NUM); ODP_PRINT("CONFIG_POOL_CACHE_MAX_SIZE: %i\n", CONFIG_POOL_CACHE_MAX_SIZE); + ODP_PRINT("CONFIG_TIMER_128BIT_ATOMICS: %i\n", CONFIG_TIMER_128BIT_ATOMICS); ODP_PRINT("\n"); } diff --git a/platform/linux-generic/odp_timer.c b/platform/linux-generic/odp_timer.c index 3af1ae737..9457dd4c5 100644 --- a/platform/linux-generic/odp_timer.c +++ b/platform/linux-generic/odp_timer.c @@ -38,6 +38,7 @@ #include <odp/api/plat/timer_inline_types.h> #include <odp_atomic_internal.h> +#include <odp_config_internal.h> #include <odp_debug_internal.h> #include <odp_errno_define.h> #include <odp_event_internal.h> @@ -60,6 +61,13 @@ #include <time.h> #include <unistd.h> +/* Check whether 128-bit atomics should be used */ +#if defined(ODP_ATOMIC_U128) && CONFIG_TIMER_128BIT_ATOMICS +#define USE_128BIT_ATOMICS 1 +#else +#define USE_128BIT_ATOMICS 0 +#endif + /* One divided by one nanosecond in Hz */ #define GIGA_HZ 1000000000 @@ -92,9 +100,9 @@ #define MAX_PERIODIC_TIMERS 100 /* Mutual exclusion in the absence of CAS16 */ -#ifndef ODP_ATOMIC_U128 -#define NUM_LOCKS 1024 -#define IDX2LOCK(idx) (&timer_global->locks[(idx) % NUM_LOCKS]) +#if !USE_128BIT_ATOMICS +#define NUM_LOCKS 256 +#define IDX2LOCK(tp, idx) (&(tp)->locks[(idx) % NUM_LOCKS]) #endif #include <odp/visibility_begin.h> @@ -110,20 +118,12 @@ _odp_timeout_inline_offset ODP_ALIGNED_CACHE = { #include <odp/visibility_end.h> typedef struct -#ifdef ODP_ATOMIC_U128 +#if USE_128BIT_ATOMICS ODP_ALIGNED(16) /* 16-byte atomic operations need properly aligned addresses */ #endif tick_buf_s { -#if __GCC_ATOMIC_LLONG_LOCK_FREE < 2 - /* No atomics support for 64-bit variables, will use separate lock */ - /* Use the same layout as odp_atomic_u64_t but without lock variable */ - struct { - uint64_t v; - } exp_tck;/* Expiration tick or TMO_xxx */ -#else - odp_atomic_u64_t exp_tck;/* Expiration tick or TMO_xxx */ -#endif - + /* Expiration tick or TMO_xxx */ + odp_atomic_u64_t exp_tck; union { /* ODP_EVENT_INVALID if timer not active */ odp_event_t tmo_event; @@ -134,8 +134,7 @@ tick_buf_s { } tick_buf_t; -#if __GCC_ATOMIC_LLONG_LOCK_FREE >= 2 -/* Only assert this when we perform atomic operations on tick_buf_t */ +#ifndef ODP_ATOMIC_U64_LOCK ODP_STATIC_ASSERT(sizeof(tick_buf_t) == 16, "sizeof(tick_buf_t) == 16"); #endif @@ -176,6 +175,10 @@ typedef struct timer_pool_s { double base_freq; uint64_t max_multiplier; uint8_t periodic; +#if !USE_128BIT_ATOMICS + /* Multiple locks per cache line! */ + _odp_atomic_flag_t locks[NUM_LOCKS] ODP_ALIGNED_CACHE; +#endif } timer_pool_t; @@ -199,10 +202,7 @@ typedef struct timer_global_t { odp_time_t destroy_time[MAX_TIMER_POOLS]; odp_shm_t tp_shm[MAX_TIMER_POOLS]; timer_pool_t *timer_pool[MAX_TIMER_POOLS]; -#ifndef ODP_ATOMIC_U128 - /* Multiple locks per cache line! */ - _odp_atomic_flag_t locks[NUM_LOCKS] ODP_ALIGNED_CACHE; -#endif + /* These are read frequently from inline timer */ odp_time_t poll_interval_time; odp_bool_t use_inline_timers; @@ -237,11 +237,7 @@ static void timer_init(_odp_timer_t *tim, tick_buf_t *tb, odp_queue_t _q, const tb->tmo_event = ODP_EVENT_INVALID; /* Release the timer by setting timer state to inactive */ -#if __GCC_ATOMIC_LLONG_LOCK_FREE < 2 - tb->exp_tck.v = TMO_INACTIVE; -#else odp_atomic_store_rel_u64(&tb->exp_tck, TMO_INACTIVE); -#endif } /* Teardown when timer is freed */ @@ -279,14 +275,15 @@ static inline odp_timer_pool_t timer_pool_to_hdl(timer_pool_t *tp) static inline timer_pool_t *handle_to_tp(odp_timer_t hdl) { uint32_t tp_idx = _odp_typeval(hdl) >> INDEX_BITS; + timer_pool_t *tp; - if (odp_likely(tp_idx < MAX_TIMER_POOLS)) { - timer_pool_t *tp = timer_global->timer_pool[tp_idx]; + ODP_ASSERT(tp_idx < MAX_TIMER_POOLS); - if (odp_likely(tp != NULL)) - return timer_global->timer_pool[tp_idx]; - } - ODP_ABORT("Invalid timer handle %p\n", (void *)hdl); + tp = timer_global->timer_pool[tp_idx]; + + ODP_ASSERT(tp != NULL); + + return tp; } static inline uint32_t handle_to_idx(odp_timer_t hdl, @@ -294,10 +291,11 @@ static inline uint32_t handle_to_idx(odp_timer_t hdl, { uint32_t idx = (_odp_typeval(hdl) & ((1U << INDEX_BITS) - 1U)) - 1; + ODP_ASSERT(idx < odp_atomic_load_u32(&tp->high_wm)); + __builtin_prefetch(&tp->tick_buf[idx], 0, 0); - if (odp_likely(idx < odp_atomic_load_u32(&tp->high_wm))) - return idx; - ODP_ABORT("Invalid timer handle %p\n", (void *)hdl); + + return idx; } static inline odp_timer_t tp_idx_to_handle(timer_pool_t *tp, @@ -486,16 +484,17 @@ static odp_timer_pool_t timer_pool_new(const char *name, tp->tick_buf = (void *)((char *)odp_shm_addr(shm) + sz0); tp->timers = (void *)((char *)odp_shm_addr(shm) + sz0 + sz1); +#if !USE_128BIT_ATOMICS + for (i = 0; i < NUM_LOCKS; i++) + _odp_atomic_flag_clear(&tp->locks[i]); +#endif + /* Initialize all odp_timer entries */ for (i = 0; i < tp->param.num_timers; i++) { tp->timers[i].queue = ODP_QUEUE_INVALID; set_next_free(&tp->timers[i], i + 1); tp->timers[i].user_ptr = NULL; -#if __GCC_ATOMIC_LLONG_LOCK_FREE < 2 - tp->tick_buf[i].exp_tck.v = TMO_UNUSED; -#else odp_atomic_init_u64(&tp->tick_buf[i].exp_tck, TMO_UNUSED); -#endif tp->tick_buf[i].tmo_event = ODP_EVENT_INVALID; } tp->tp_idx = tp_idx; @@ -677,7 +676,7 @@ static bool timer_reset(uint32_t idx, uint64_t abs_tck, odp_event_t *tmo_event, tick_buf_t *tb = &tp->tick_buf[idx]; if (tmo_event == NULL || *tmo_event == ODP_EVENT_INVALID) { -#ifdef ODP_ATOMIC_U128 /* Target supports 128-bit atomic operations */ +#if USE_128BIT_ATOMICS /* Target supports 128-bit atomic operations */ tick_buf_t new, old; /* Init all bits, also when tmo_event is less than 64 bits */ @@ -706,35 +705,11 @@ static bool timer_reset(uint32_t idx, uint64_t abs_tck, odp_event_t *tmo_event, } while (!_odp_atomic_u128_cmp_xchg_mm((_odp_atomic_u128_t *)tb, (_odp_u128_t *)&old, (_odp_u128_t *)&new, _ODP_MEMMODEL_RLS, _ODP_MEMMODEL_RLX)); -#elif __GCC_ATOMIC_LLONG_LOCK_FREE >= 2 && \ - defined __GCC_HAVE_SYNC_COMPARE_AND_SWAP_8 - /* Target supports lock-free 64-bit CAS (and probably exchange) */ - /* Since locks/barriers are not good for C-A15, we take an - * alternative approach using relaxed memory model */ - uint64_t old; - /* Swap in new expiration tick, get back old tick which - * will indicate active/inactive timer state */ - old = odp_atomic_xchg_u64(&tb->exp_tck, abs_tck); - - if ((old & TMO_INACTIVE) != 0) { - /* Timer was inactive (cancelled or expired), - * we can't reset a timer without a timeout event. - * Attempt to restore inactive state, we don't - * want this timer to continue as active without - * timeout as this will trigger unnecessary and - * aborted expiration attempts. - * We don't care if we fail, then some other thread - * reset or cancelled the timer. Without any - * synchronization between the threads, we have a - * data race and the behavior is undefined */ - (void)odp_atomic_cas_u64(&tb->exp_tck, &abs_tck, old); - success = false; - } -#else /* Target supports neither 128-bit nor 64-bit CAS => use lock */ +#else /* Take a related lock */ - while (_odp_atomic_flag_tas(IDX2LOCK(idx))) + while (_odp_atomic_flag_tas(IDX2LOCK(tp, idx))) /* While lock is taken, spin using relaxed loads */ - while (_odp_atomic_flag_load(IDX2LOCK(idx))) + while (_odp_atomic_flag_load(IDX2LOCK(tp, idx))) odp_cpu_pause(); /* Only if there is a timeout event can the timer be reset */ @@ -748,7 +723,7 @@ static bool timer_reset(uint32_t idx, uint64_t abs_tck, odp_event_t *tmo_event, } /* Release the lock */ - _odp_atomic_flag_clear(IDX2LOCK(idx)); + _odp_atomic_flag_clear(IDX2LOCK(tp, idx)); #endif } else { /* We have a new timeout event which replaces any old one */ @@ -763,7 +738,7 @@ static bool timer_reset(uint32_t idx, uint64_t abs_tck, odp_event_t *tmo_event, } /* Else ignore events of other types */ odp_event_t old_event = ODP_EVENT_INVALID; -#ifdef ODP_ATOMIC_U128 +#if USE_128BIT_ATOMICS tick_buf_t new, old; /* Init all bits, also when tmo_event is less than 64 bits */ @@ -781,9 +756,9 @@ static bool timer_reset(uint32_t idx, uint64_t abs_tck, odp_event_t *tmo_event, old_event = old.tmo_event; #else /* Take a related lock */ - while (_odp_atomic_flag_tas(IDX2LOCK(idx))) + while (_odp_atomic_flag_tas(IDX2LOCK(tp, idx))) /* While lock is taken, spin using relaxed loads */ - while (_odp_atomic_flag_load(IDX2LOCK(idx))) + while (_odp_atomic_flag_load(IDX2LOCK(tp, idx))) odp_cpu_pause(); /* Swap in new event, save any old event */ @@ -794,7 +769,7 @@ static bool timer_reset(uint32_t idx, uint64_t abs_tck, odp_event_t *tmo_event, tb->exp_tck.v = abs_tck; /* Release the lock */ - _odp_atomic_flag_clear(IDX2LOCK(idx)); + _odp_atomic_flag_clear(IDX2LOCK(tp, idx)); #endif /* Return old timeout event */ *tmo_event = old_event; @@ -807,7 +782,7 @@ static odp_event_t timer_set_unused(timer_pool_t *tp, uint32_t idx) tick_buf_t *tb = &tp->tick_buf[idx]; odp_event_t old_event; -#ifdef ODP_ATOMIC_U128 +#if USE_128BIT_ATOMICS tick_buf_t new, old; /* Init all bits, also when tmo_event is less than 64 bits */ @@ -824,9 +799,9 @@ static odp_event_t timer_set_unused(timer_pool_t *tp, uint32_t idx) old_event = old.tmo_event; #else /* Take a related lock */ - while (_odp_atomic_flag_tas(IDX2LOCK(idx))) + while (_odp_atomic_flag_tas(IDX2LOCK(tp, idx))) /* While lock is taken, spin using relaxed loads */ - while (_odp_atomic_flag_load(IDX2LOCK(idx))) + while (_odp_atomic_flag_load(IDX2LOCK(tp, idx))) odp_cpu_pause(); /* Update the timer state (e.g. cancel the current timeout) */ @@ -837,7 +812,7 @@ static odp_event_t timer_set_unused(timer_pool_t *tp, uint32_t idx) tb->tmo_event = ODP_EVENT_INVALID; /* Release the lock */ - _odp_atomic_flag_clear(IDX2LOCK(idx)); + _odp_atomic_flag_clear(IDX2LOCK(tp, idx)); #endif /* Return the old event */ return old_event; @@ -848,7 +823,7 @@ static odp_event_t timer_cancel(timer_pool_t *tp, uint32_t idx) tick_buf_t *tb = &tp->tick_buf[idx]; odp_event_t old_event; -#ifdef ODP_ATOMIC_U128 +#if USE_128BIT_ATOMICS tick_buf_t new, old; /* Init all bits, also when tmo_event is less than 64 bits */ @@ -880,9 +855,9 @@ static odp_event_t timer_cancel(timer_pool_t *tp, uint32_t idx) old_event = old.tmo_event; #else /* Take a related lock */ - while (_odp_atomic_flag_tas(IDX2LOCK(idx))) + while (_odp_atomic_flag_tas(IDX2LOCK(tp, idx))) /* While lock is taken, spin using relaxed loads */ - while (_odp_atomic_flag_load(IDX2LOCK(idx))) + while (_odp_atomic_flag_load(IDX2LOCK(tp, idx))) odp_cpu_pause(); /* Swap in new event, save any old event */ @@ -896,7 +871,7 @@ static odp_event_t timer_cancel(timer_pool_t *tp, uint32_t idx) tb->exp_tck.v = TMO_INACTIVE; /* Release the lock */ - _odp_atomic_flag_clear(IDX2LOCK(idx)); + _odp_atomic_flag_clear(IDX2LOCK(tp, idx)); #endif /* Return the old event */ return old_event; @@ -908,7 +883,7 @@ static inline void timer_expire(timer_pool_t *tp, uint32_t idx, uint64_t tick) tick_buf_t *tb = &tp->tick_buf[idx]; odp_event_t tmo_event = ODP_EVENT_INVALID; uint64_t exp_tck; -#ifdef ODP_ATOMIC_U128 +#if USE_128BIT_ATOMICS /* Atomic re-read for correctness */ exp_tck = odp_atomic_load_u64(&tb->exp_tck); /* Re-check exp_tck */ @@ -940,11 +915,10 @@ static inline void timer_expire(timer_pool_t *tp, uint32_t idx, uint64_t tick) } /* Else false positive, ignore */ #else - /* Take a related lock */ - while (_odp_atomic_flag_tas(IDX2LOCK(idx))) - /* While lock is taken, spin using relaxed loads */ - while (_odp_atomic_flag_load(IDX2LOCK(idx))) - odp_cpu_pause(); + /* Try to take a related lock */ + if (_odp_atomic_flag_tas(IDX2LOCK(tp, idx))) + return; + /* Proper check for timer expired */ exp_tck = tb->exp_tck.v; if (odp_likely(exp_tck <= tick)) { @@ -963,7 +937,7 @@ static inline void timer_expire(timer_pool_t *tp, uint32_t idx, uint64_t tick) } /* Else false positive, ignore */ /* Release the lock */ - _odp_atomic_flag_clear(IDX2LOCK(idx)); + _odp_atomic_flag_clear(IDX2LOCK(tp, idx)); #endif if (odp_likely(tmo_event != ODP_EVENT_INVALID)) { /* Fill in expiration tick for timeout events */ @@ -1823,11 +1797,8 @@ int odp_timeout_fresh(odp_timeout_t tmo) timer_pool_t *tp = handle_to_tp(hdl); uint32_t idx = handle_to_idx(hdl, tp); tick_buf_t *tb = &tp->tick_buf[idx]; -#if __GCC_ATOMIC_LLONG_LOCK_FREE < 2 - uint64_t exp_tck = tb->exp_tck.v; -#else uint64_t exp_tck = odp_atomic_load_u64(&tb->exp_tck); -#endif + /* Return true if the timer still has the same expiration tick * (ignoring the inactive/expired bit) as the timeout */ return hdr->expiration == (exp_tck & ~TMO_INACTIVE); @@ -1970,18 +1941,21 @@ int _odp_timer_init_global(const odp_init_t *params) timer_global->tp_shm[i] = ODP_SHM_INVALID; } -#ifndef ODP_ATOMIC_U128 - for (i = 0; i < NUM_LOCKS; i++) - _odp_atomic_flag_clear(&timer_global->locks[i]); +#if USE_128BIT_ATOMICS + ODP_PRINT("Timer using lock-less implementation\n"); #else - ODP_DBG("Using lock-less timer implementation\n"); + ODP_PRINT("Timer using lock-based implementation\n"); #endif + + ODP_PRINT("Timer config:\n"); + conf_str = "timer.inline"; if (!_odp_libconfig_lookup_int(conf_str, &val)) { ODP_ERR("Config option '%s' not found.\n", conf_str); goto error; } timer_global->use_inline_timers = val; + ODP_PRINT(" %s: %i\n", conf_str, val); conf_str = "timer.inline_poll_interval"; if (!_odp_libconfig_lookup_int(conf_str, &val)) { @@ -1989,6 +1963,7 @@ int _odp_timer_init_global(const odp_init_t *params) goto error; } timer_global->poll_interval = val; + ODP_PRINT(" %s: %i\n", conf_str, val); conf_str = "timer.inline_poll_interval_nsec"; if (!_odp_libconfig_lookup_int(conf_str, &val)) { @@ -1998,6 +1973,7 @@ int _odp_timer_init_global(const odp_init_t *params) timer_global->poll_interval_nsec = val; timer_global->poll_interval_time = odp_time_global_from_ns(timer_global->poll_interval_nsec); + ODP_PRINT(" %s: %i\n", conf_str, val); conf_str = "timer.inline_thread_type"; if (!_odp_libconfig_lookup_int(conf_str, &val)) { @@ -2005,6 +1981,8 @@ int _odp_timer_init_global(const odp_init_t *params) goto error; } timer_global->thread_type = val; + ODP_PRINT(" %s: %i\n", conf_str, val); + ODP_PRINT("\n"); if (!timer_global->use_inline_timers) { timer_res_init(); diff --git a/platform/linux-generic/odp_traffic_mngr.c b/platform/linux-generic/odp_traffic_mngr.c index 9bea659e9..9a64a0aab 100644 --- a/platform/linux-generic/odp_traffic_mngr.c +++ b/platform/linux-generic/odp_traffic_mngr.c @@ -2603,6 +2603,9 @@ static int tm_capabilities(odp_tm_capabilities_t capabilities[], cap_ptr = &capabilities[0]; memset(cap_ptr, 0, sizeof(odp_tm_capabilities_t)); + if (odp_global_ro.init_param.mem_model == ODP_MEM_MODEL_PROCESS) + return 1; + cap_ptr->max_tm_queues = ODP_TM_MAX_TM_QUEUES; cap_ptr->max_levels = ODP_TM_MAX_LEVELS; cap_ptr->tm_queue_shaper_supported = true; @@ -3060,6 +3063,11 @@ odp_tm_t odp_tm_create(const char *name, return ODP_TM_INVALID; } + if (odp_global_ro.init_param.mem_model == ODP_MEM_MODEL_PROCESS) { + ODP_ERR("TM is not supported in process mode\n"); + return ODP_TM_INVALID; + } + /* We only support global pkt priority mode */ if (requirements->pkt_prio_mode != ODP_TM_PKT_PRIO_MODE_PRESERVE) { ODP_ERR("Unsupported Packet priority mode\n"); @@ -4640,6 +4648,17 @@ int odp_tm_enq_multi_lso(odp_tm_queue_t tm_queue, const odp_packet_t packets[], goto error; } + if (odp_unlikely(num_pkt == 1)) { + /* Segmentation not needed */ + if (odp_tm_enq_multi(tm_queue, &pkt, 1) != 1) { + ODP_DBG("TM enqueue failed on packet %i\n", i); + + goto error; + } + + continue; + } + /* Create packets */ odp_packet_t pkt_out[num_pkt]; diff --git a/platform/linux-generic/pktio/dpdk.c b/platform/linux-generic/pktio/dpdk.c index 006344b48..80578c9b0 100644 --- a/platform/linux-generic/pktio/dpdk.c +++ b/platform/linux-generic/pktio/dpdk.c @@ -612,6 +612,7 @@ static inline int mbuf_to_pkt(pktio_entry_t *pktio_entry, odp_pktio_t input = pktio_entry->s.handle; uint16_t frame_offset = pktio_entry->s.pktin_frame_offset; const odp_proto_layer_t layer = pktio_entry->s.parse_layer; + const uint32_t supported_ptypes = pkt_dpdk->supported_ptypes; /* Allocate maximum sized packets */ max_len = pkt_dpdk->data_room; @@ -640,23 +641,30 @@ static inline int mbuf_to_pkt(pktio_entry_t *pktio_entry, pkt_hdr = packet_hdr(pkt); if (layer) { - uint32_t supported_ptypes = pkt_dpdk->supported_ptypes; - packet_parse_reset(pkt_hdr, 1); if (_odp_dpdk_packet_parse_common(&pkt_hdr->p, data, pkt_len, pkt_len, mbuf, layer, supported_ptypes, pktin_cfg)) { - odp_packet_free(pkt_table[i]); + odp_packet_free(pkt); rte_pktmbuf_free(mbuf); continue; } if (pktio_cls_enabled(pktio_entry)) { + odp_pool_t new_pool; + if (_odp_cls_classify_packet(pktio_entry, (const uint8_t *)data, - &pool, pkt_hdr)) { - odp_packet_free(pkt_table[i]); + &new_pool, pkt_hdr)) { + odp_packet_free(pkt); + rte_pktmbuf_free(mbuf); + continue; + } + + if (odp_unlikely(_odp_pktio_packet_to_pool( + &pkt, &pkt_hdr, new_pool))) { + odp_packet_free(pkt); rte_pktmbuf_free(mbuf); continue; } @@ -881,17 +889,15 @@ static inline int mbuf_to_pkt_zero(pktio_entry_t *pktio_entry, struct rte_mbuf *mbuf; void *data; int i, nb_pkts; - odp_pool_t pool; odp_pktin_config_opt_t pktin_cfg; odp_pktio_t input; - pkt_dpdk_t *pkt_dpdk; + pkt_dpdk_t *pkt_dpdk = pkt_priv(pktio_entry); const odp_proto_layer_t layer = pktio_entry->s.parse_layer; + const uint32_t supported_ptypes = pkt_dpdk->supported_ptypes; prefetch_pkt(mbuf_table[0]); - pkt_dpdk = pkt_priv(pktio_entry); nb_pkts = 0; - pool = pkt_dpdk->pool; set_flow_hash = pkt_dpdk->opt.set_flow_hash; pktin_cfg = pktio_entry->s.config.pktin; input = pktio_entry->s.handle; @@ -900,6 +906,8 @@ static inline int mbuf_to_pkt_zero(pktio_entry_t *pktio_entry, prefetch_pkt(mbuf_table[1]); for (i = 0; i < mbuf_num; i++) { + odp_packet_t pkt; + if (odp_likely((i + 2) < mbuf_num)) prefetch_pkt(mbuf_table[i + 2]); @@ -913,11 +921,14 @@ static inline int mbuf_to_pkt_zero(pktio_entry_t *pktio_entry, data = rte_pktmbuf_mtod(mbuf, char *); pkt_len = rte_pktmbuf_pkt_len(mbuf); pkt_hdr = pkt_hdr_from_mbuf(mbuf); + pkt = packet_handle(pkt_hdr); packet_init(pkt_hdr, pkt_len); - if (layer) { - uint32_t supported_ptypes = pkt_dpdk->supported_ptypes; + /* Init buffer segments. Currently, only single segment packets + * are supported. */ + pkt_hdr->seg_data = data; + if (layer) { if (_odp_dpdk_packet_parse_common(&pkt_hdr->p, data, pkt_len, pkt_len, mbuf, layer, supported_ptypes, @@ -927,19 +938,23 @@ static inline int mbuf_to_pkt_zero(pktio_entry_t *pktio_entry, } if (pktio_cls_enabled(pktio_entry)) { + odp_pool_t new_pool; + if (_odp_cls_classify_packet(pktio_entry, (const uint8_t *)data, - &pool, pkt_hdr)) { + &new_pool, pkt_hdr)) { + rte_pktmbuf_free(mbuf); + continue; + } + + if (odp_unlikely(_odp_pktio_packet_to_pool( + &pkt, &pkt_hdr, new_pool))) { rte_pktmbuf_free(mbuf); continue; } } } - /* Init buffer segments. Currently, only single segment packets - * are supported. */ - pkt_hdr->seg_data = data; - pkt_hdr->input = input; if (set_flow_hash && (mbuf->ol_flags & RTE_MBUF_F_RX_RSS_HASH)) @@ -947,7 +962,7 @@ static inline int mbuf_to_pkt_zero(pktio_entry_t *pktio_entry, packet_set_ts(pkt_hdr, ts); - pkt_table[nb_pkts++] = packet_handle(pkt_hdr); + pkt_table[nb_pkts++] = pkt; } return nb_pkts; diff --git a/platform/linux-generic/pktio/loop.c b/platform/linux-generic/pktio/loop.c index c702f9ded..68e1431ab 100644 --- a/platform/linux-generic/pktio/loop.c +++ b/platform/linux-generic/pktio/loop.c @@ -162,7 +162,6 @@ static int loopback_recv(pktio_entry_t *pktio_entry, int index ODP_UNUSED, int num_rx = 0; int packets = 0, errors = 0; uint32_t octets = 0; - const odp_proto_chksums_t chksums = pktio_entry->s.in_chksums; const odp_proto_layer_t layer = pktio_entry->s.parse_layer; const odp_pktin_config_opt_t opt = pktio_entry->s.config.pktin; @@ -191,7 +190,6 @@ static int loopback_recv(pktio_entry_t *pktio_entry, int index ODP_UNUSED, uint8_t buf[PARSE_BYTES]; int ret; uint32_t seg_len = odp_packet_seg_len(pkt); - uint64_t l4_part_sum = 0; /* Make sure there is enough data for the packet * parser in the case of a segmented packet. */ @@ -205,9 +203,8 @@ static int loopback_recv(pktio_entry_t *pktio_entry, int index ODP_UNUSED, } packet_parse_reset(pkt_hdr, 1); - ret = _odp_packet_parse_common(&pkt_hdr->p, pkt_addr, pkt_len, - seg_len, layer, chksums, - &l4_part_sum, opt); + ret = _odp_packet_parse_common(pkt_hdr, pkt_addr, pkt_len, + seg_len, layer, opt); if (ret) errors++; @@ -217,7 +214,6 @@ static int loopback_recv(pktio_entry_t *pktio_entry, int index ODP_UNUSED, } if (pktio_cls_enabled(pktio_entry)) { - odp_packet_t new_pkt; odp_pool_t new_pool; ret = _odp_cls_classify_packet(pktio_entry, pkt_addr, @@ -227,23 +223,15 @@ static int loopback_recv(pktio_entry_t *pktio_entry, int index ODP_UNUSED, continue; } - if (new_pool != odp_packet_pool(pkt)) { - new_pkt = odp_packet_copy(pkt, new_pool); - + if (odp_unlikely(_odp_pktio_packet_to_pool( + &pkt, &pkt_hdr, new_pool))) { odp_packet_free(pkt); - - if (new_pkt == ODP_PACKET_INVALID) { - pktio_entry->s.stats.in_discards++; - continue; - } - - pkt = new_pkt; - pkt_hdr = packet_hdr(new_pkt); + odp_atomic_inc_u64( + &pktio_entry->s.stats_extra + .in_discards); + continue; } } - - if (layer >= ODP_PROTO_LAYER_L4) - _odp_packet_l4_chksum(pkt_hdr, chksums, l4_part_sum); } packet_set_ts(pkt_hdr, ts); diff --git a/platform/linux-generic/pktio/netmap.c b/platform/linux-generic/pktio/netmap.c index 342f38431..c843d5367 100644 --- a/platform/linux-generic/pktio/netmap.c +++ b/platform/linux-generic/pktio/netmap.c @@ -828,7 +828,6 @@ static inline int netmap_pkt_to_odp(pktio_entry_t *pktio_entry, uint32_t max_len; uint16_t frame_offset = pktio_entry->s.pktin_frame_offset; int num_rx = 0; - const odp_proto_chksums_t chksums = pktio_entry->s.in_chksums; const odp_proto_layer_t layer = pktio_entry->s.parse_layer; const odp_pktin_config_opt_t opt = pktio_entry->s.config.pktin; @@ -842,7 +841,6 @@ static inline int netmap_pkt_to_odp(pktio_entry_t *pktio_entry, netmap_slot_t slot; uint16_t len; const uint8_t *buf; - uint64_t l4_part_sum = 0; slot = slot_tbl[i]; len = slot.len; @@ -850,39 +848,50 @@ static inline int netmap_pkt_to_odp(pktio_entry_t *pktio_entry, odp_prefetch(slot.buf); - pkt = pkt_tbl[num_rx]; + pkt = pkt_tbl[i]; pkt_hdr = packet_hdr(pkt); + pull_tail(pkt_hdr, max_len - len); + if (frame_offset) + pull_head(pkt_hdr, frame_offset); + + if (odp_packet_copy_from_mem(pkt, 0, len, slot.buf) != 0) { + odp_packet_free(pkt); + continue; + } + if (layer) { - if (_odp_packet_parse_common(&pkt_hdr->p, buf, len, len, - layer, chksums, &l4_part_sum, opt) < 0) + if (_odp_packet_parse_common(pkt_hdr, buf, len, len, + layer, opt) < 0) { + odp_packet_free(pkt); continue; + } if (pktio_cls_enabled(pktio_entry)) { - if (_odp_cls_classify_packet(pktio_entry, buf, &pool, - pkt_hdr)) + odp_pool_t new_pool; + + if (_odp_cls_classify_packet(pktio_entry, buf, &new_pool, + pkt_hdr)) { + odp_packet_free(pkt); continue; + } + + if (odp_unlikely(_odp_pktio_packet_to_pool( + &pkt, &pkt_hdr, new_pool))) { + odp_packet_free(pkt); + odp_atomic_inc_u64( + &pktio_entry->s.stats_extra + .in_discards); + continue; + } } } - pull_tail(pkt_hdr, max_len - len); - if (frame_offset) - pull_head(pkt_hdr, frame_offset); - - if (odp_packet_copy_from_mem(pkt, 0, len, slot.buf) != 0) - break; - pkt_hdr->input = pktio_entry->s.handle; - - if (layer >= ODP_PROTO_LAYER_L4) - _odp_packet_l4_chksum(pkt_hdr, chksums, l4_part_sum); - packet_set_ts(pkt_hdr, ts); - num_rx++; - } - if (num_rx < num) - odp_packet_free_multi(&pkt_tbl[num_rx], num - num_rx); + pkt_tbl[num_rx++] = pkt; + } return num_rx; } diff --git a/platform/linux-generic/pktio/pcap.c b/platform/linux-generic/pktio/pcap.c index af94ffa72..2de70dd1f 100644 --- a/platform/linux-generic/pktio/pcap.c +++ b/platform/linux-generic/pktio/pcap.c @@ -241,7 +241,6 @@ static int pcapif_recv_pkt(pktio_entry_t *pktio_entry, int index ODP_UNUSED, { int i; struct pcap_pkthdr *hdr; - odp_pool_t new_pool; const u_char *data; odp_packet_t pkt; odp_packet_hdr_t *pkt_hdr; @@ -252,7 +251,6 @@ static int pcapif_recv_pkt(pktio_entry_t *pktio_entry, int index ODP_UNUSED, int packets = 0, errors = 0; uint32_t octets = 0; uint16_t frame_offset = pktio_entry->s.pktin_frame_offset; - const odp_proto_chksums_t chksums = pktio_entry->s.in_chksums; const odp_proto_layer_t layer = pktio_entry->s.parse_layer; const odp_pktin_config_opt_t opt = pktio_entry->s.config.pktin; @@ -297,11 +295,8 @@ static int pcapif_recv_pkt(pktio_entry_t *pktio_entry, int index ODP_UNUSED, } if (layer) { - uint64_t l4_part_sum = 0; - - ret = _odp_packet_parse_common(&pkt_hdr->p, data, pkt_len, - pkt_len, layer, chksums, - &l4_part_sum, opt); + ret = _odp_packet_parse_common(pkt_hdr, data, pkt_len, + pkt_len, layer, opt); if (ret) errors++; @@ -311,7 +306,7 @@ static int pcapif_recv_pkt(pktio_entry_t *pktio_entry, int index ODP_UNUSED, } if (pktio_cls_enabled(pktio_entry)) { - odp_packet_t new_pkt; + odp_pool_t new_pool; ret = _odp_cls_classify_packet(pktio_entry, data, &new_pool, pkt_hdr); @@ -319,23 +314,16 @@ static int pcapif_recv_pkt(pktio_entry_t *pktio_entry, int index ODP_UNUSED, odp_packet_free(pkt); continue; } - if (new_pool != pcap->pool) { - new_pkt = odp_packet_copy(pkt, new_pool); + if (odp_unlikely(_odp_pktio_packet_to_pool( + &pkt, &pkt_hdr, new_pool))) { odp_packet_free(pkt); - - if (odp_unlikely(new_pkt == ODP_PACKET_INVALID)) { - pktio_entry->s.stats.in_discards++; - continue; - } - - pkt = new_pkt; - pkt_hdr = packet_hdr(new_pkt); + odp_atomic_inc_u64( + &pktio_entry->s.stats_extra + .in_discards); + continue; } } - - if (layer >= ODP_PROTO_LAYER_L4) - _odp_packet_l4_chksum(pkt_hdr, chksums, l4_part_sum); } packet_set_ts(pkt_hdr, ts); diff --git a/platform/linux-generic/pktio/socket.c b/platform/linux-generic/pktio/socket.c index 0d756c4e1..3ec458c0b 100644 --- a/platform/linux-generic/pktio/socket.c +++ b/platform/linux-generic/pktio/socket.c @@ -235,7 +235,6 @@ static int sock_mmsg_recv(pktio_entry_t *pktio_entry, int index ODP_UNUSED, int i; uint16_t frame_offset = pktio_entry->s.pktin_frame_offset; uint32_t alloc_len = pkt_sock->mtu + frame_offset; - const odp_proto_chksums_t chksums = pktio_entry->s.in_chksums; const odp_proto_layer_t layer = pktio_entry->s.parse_layer; const odp_pktin_config_opt_t opt = pktio_entry->s.config.pktin; @@ -266,7 +265,6 @@ static int sock_mmsg_recv(pktio_entry_t *pktio_entry, int index ODP_UNUSED, odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); uint16_t pkt_len = msgvec[i].msg_len; int ret; - uint64_t l4_part_sum = 0; if (odp_unlikely(msgvec[i].msg_hdr.msg_flags & MSG_TRUNC)) { odp_packet_free(pkt); @@ -294,19 +292,29 @@ static int sock_mmsg_recv(pktio_entry_t *pktio_entry, int index ODP_UNUSED, base = buf; } - if (_odp_packet_parse_common(&pkt_hdr->p, base, pkt_len, - seg_len, layer, chksums, - &l4_part_sum, opt) < 0) { + if (_odp_packet_parse_common(pkt_hdr, base, pkt_len, + seg_len, layer, opt) < 0) { odp_packet_free(pkt); continue; } if (pktio_cls_enabled(pktio_entry)) { - if (_odp_cls_classify_packet(pktio_entry, base, &pool, + odp_pool_t new_pool; + + if (_odp_cls_classify_packet(pktio_entry, base, &new_pool, pkt_hdr)) { odp_packet_free(pkt); continue; } + + if (odp_unlikely(_odp_pktio_packet_to_pool( + &pkt, &pkt_hdr, new_pool))) { + odp_packet_free(pkt); + odp_atomic_inc_u64( + &pktio_entry->s.stats_extra + .in_discards); + continue; + } } } @@ -318,10 +326,6 @@ static int sock_mmsg_recv(pktio_entry_t *pktio_entry, int index ODP_UNUSED, } pkt_hdr->input = pktio_entry->s.handle; - - if (layer >= ODP_PROTO_LAYER_L4) - _odp_packet_l4_chksum(pkt_hdr, chksums, l4_part_sum); - packet_set_ts(pkt_hdr, ts); pkt_table[nb_rx++] = pkt; diff --git a/platform/linux-generic/pktio/socket_mmap.c b/platform/linux-generic/pktio/socket_mmap.c index 4845b5dab..d509b75f6 100644 --- a/platform/linux-generic/pktio/socket_mmap.c +++ b/platform/linux-generic/pktio/socket_mmap.c @@ -152,7 +152,6 @@ static inline unsigned pkt_mmap_v2_rx(pktio_entry_t *pktio_entry, odp_pool_t pool = pkt_sock->pool; uint16_t frame_offset = pktio_entry->s.pktin_frame_offset; uint16_t vlan_len = 0; - const odp_proto_chksums_t chksums = pktio_entry->s.in_chksums; const odp_proto_layer_t layer = pktio_entry->s.parse_layer; const odp_pktin_config_opt_t opt = pktio_entry->s.config.pktin; @@ -168,7 +167,6 @@ static inline unsigned pkt_mmap_v2_rx(pktio_entry_t *pktio_entry, odp_packet_t pkt; odp_packet_hdr_t *hdr; int ret; - uint64_t l4_part_sum = 0; tp_hdr = (void *)next_ptr; @@ -218,27 +216,6 @@ static inline unsigned pkt_mmap_v2_rx(pktio_entry_t *pktio_entry, hdr = packet_hdr(pkt); - if (layer) { - if (_odp_packet_parse_common(&hdr->p, pkt_buf, pkt_len, - pkt_len, layer, chksums, - &l4_part_sum, opt) < 0) { - odp_packet_free(pkt); - tp_hdr->tp_status = TP_STATUS_KERNEL; - frame_num = next_frame_num; - continue; - } - - if (pktio_cls_enabled(pktio_entry)) { - if (_odp_cls_classify_packet(pktio_entry, pkt_buf, - &pool, hdr)) { - odp_packet_free(pkt); - tp_hdr->tp_status = TP_STATUS_KERNEL; - frame_num = next_frame_num; - continue; - } - } - } - if (frame_offset) pull_head(hdr, frame_offset); @@ -289,11 +266,40 @@ static inline unsigned pkt_mmap_v2_rx(pktio_entry_t *pktio_entry, *tci = odp_cpu_to_be_16(tp_hdr->tp_vlan_tci); } - hdr->input = pktio_entry->s.handle; + if (layer) { + if (_odp_packet_parse_common(hdr, pkt_buf, pkt_len, + pkt_len, layer, opt) < 0) { + odp_packet_free(pkt); + tp_hdr->tp_status = TP_STATUS_KERNEL; + frame_num = next_frame_num; + continue; + } + + if (pktio_cls_enabled(pktio_entry)) { + odp_pool_t new_pool; + + if (_odp_cls_classify_packet(pktio_entry, pkt_buf, + &new_pool, hdr)) { + odp_packet_free(pkt); + tp_hdr->tp_status = TP_STATUS_KERNEL; + frame_num = next_frame_num; + continue; + } - if (layer >= ODP_PROTO_LAYER_L4) - _odp_packet_l4_chksum(hdr, chksums, l4_part_sum); + if (odp_unlikely(_odp_pktio_packet_to_pool( + &pkt, &hdr, new_pool))) { + odp_packet_free(pkt); + tp_hdr->tp_status = TP_STATUS_KERNEL; + frame_num = next_frame_num; + odp_atomic_inc_u64( + &pktio_entry->s.stats_extra + .in_discards); + continue; + } + } + } + hdr->input = pktio_entry->s.handle; packet_set_ts(hdr, ts); tp_hdr->tp_status = TP_STATUS_KERNEL; diff --git a/platform/linux-generic/pktio/socket_xdp.c b/platform/linux-generic/pktio/socket_xdp.c index e43e4bf89..d00cbb236 100644 --- a/platform/linux-generic/pktio/socket_xdp.c +++ b/platform/linux-generic/pktio/socket_xdp.c @@ -13,6 +13,7 @@ #include <odp/api/hints.h> #include <odp/api/system_info.h> #include <odp/api/ticketlock.h> +#include <odp/api/packet_io_stats.h> #include <odp_debug_internal.h> #include <odp_macros_internal.h> @@ -21,54 +22,97 @@ #include <odp_parse_internal.h> #include <odp_classification_internal.h> #include <odp_socket_common.h> +#include <odp_libconfig_internal.h> #include <string.h> #include <errno.h> #include <sys/socket.h> #include <unistd.h> #include <poll.h> +#include <sys/ioctl.h> +#include <linux/ethtool.h> +#include <linux/sockios.h> +#include <net/if.h> +#include <linux/if_xdp.h> #include <xdp/xsk.h> -#define NUM_XDP_DESCS 1024U +#define NUM_DESCS_DEFAULT 1024U #define MIN_FRAME_SIZE 2048U + #define IF_DELIM " " #define Q_DELIM ':' +#define CONF_BASE_STR "pktio_xdp" +#define RX_DESCS_STR "num_rx_desc" +#define TX_DESCS_STR "num_tx_desc" + +enum { + RX_PKT_ALLOC_ERR, + RX_DESC_RSV_ERR, + TX_PKT_ALLOC_ERR, + TX_DESC_RSV_ERR +}; + +static const char * const internal_stats_strs[] = { + "rx_packet_allocation_errors", + "rx_umem_descriptor_reservation_errors", + "tx_packet_allocation_errors", + "tx_umem_descriptor_reservation_errors" +}; + +#define MAX_INTERNAL_STATS _ODP_ARRAY_SIZE(internal_stats_strs) + +typedef struct { + uint64_t rx_dropped; + uint64_t rx_inv_descs; + uint64_t tx_inv_descs; +} xdp_sock_stats_t; + +typedef struct { + odp_ticketlock_t rx_lock ODP_ALIGNED_CACHE; + odp_ticketlock_t tx_lock ODP_ALIGNED_CACHE; + struct xsk_ring_cons rx; + struct xsk_ring_cons compl_q; + struct xsk_ring_prod tx; + struct xsk_ring_prod fill_q; + odp_pktin_queue_stats_t qi_stats; + odp_pktout_queue_stats_t qo_stats; + xdp_sock_stats_t xdp_stats; + struct xsk_socket *xsk; + uint64_t i_stats[MAX_INTERNAL_STATS]; +} xdp_sock_t; typedef struct { struct xsk_ring_prod fill_q; struct xsk_ring_cons compl_q; struct xsk_umem *umem; pool_t *pool; + int num_rx_desc; + int num_tx_desc; + uint32_t ref_cnt; } xdp_umem_info_t; typedef struct { - struct xsk_ring_cons rx; - struct xsk_ring_cons compl_q; - struct xsk_ring_prod tx; - struct xsk_ring_prod fill_q; + xdp_sock_t qs[PKTIO_MAX_QUEUES]; xdp_umem_info_t *umem_info; - struct xsk_socket *xsk; + uint32_t num_q; int pktio_idx; int helper_sock; uint32_t mtu; uint32_t max_mtu; + uint32_t bind_q; + odp_bool_t lockless_rx; + odp_bool_t lockless_tx; } xdp_sock_info_t; typedef struct { - odp_ticketlock_t rx_lock ODP_ALIGNED_CACHE; - odp_ticketlock_t tx_lock ODP_ALIGNED_CACHE; - xdp_sock_info_t sock_info; -} pkt_xdp_t; - -typedef struct { odp_packet_hdr_t *pkt_hdr; odp_packet_t pkt; uint8_t *data; uint32_t len; } pkt_data_t; -ODP_STATIC_ASSERT(PKTIO_PRIVATE_SIZE >= sizeof(pkt_xdp_t), +ODP_STATIC_ASSERT(PKTIO_PRIVATE_SIZE >= sizeof(xdp_sock_info_t), "PKTIO_PRIVATE_SIZE too small"); static odp_bool_t disable_pktio; @@ -87,18 +131,65 @@ static int sock_xdp_init_global(void) return 0; } -static inline pkt_xdp_t *pkt_priv(pktio_entry_t *pktio_entry) +static inline xdp_sock_info_t *pkt_priv(pktio_entry_t *pktio_entry) { - return (pkt_xdp_t *)(uintptr_t)(pktio_entry->s.pkt_priv); + return (xdp_sock_info_t *)(uintptr_t)(pktio_entry->s.pkt_priv); } -static void fill_socket_config(struct xsk_socket_config *config) +static void parse_options(xdp_umem_info_t *umem_info) { - config->rx_size = NUM_XDP_DESCS; - config->tx_size = NUM_XDP_DESCS; - config->libxdp_flags = 0U; - config->xdp_flags = 0U; - config->bind_flags = XDP_ZEROCOPY; /* TODO: XDP_COPY */ + if (!_odp_libconfig_lookup_ext_int(CONF_BASE_STR, NULL, RX_DESCS_STR, + &umem_info->num_rx_desc) || + !_odp_libconfig_lookup_ext_int(CONF_BASE_STR, NULL, TX_DESCS_STR, + &umem_info->num_tx_desc)) { + ODP_ERR("Unable to parse xdp descriptor configuration, using defaults (%d).\n", + NUM_DESCS_DEFAULT); + goto defaults; + } + + if (umem_info->num_rx_desc <= 0 || umem_info->num_tx_desc <= 0 || + !_ODP_CHECK_IS_POWER2(umem_info->num_rx_desc) || + !_ODP_CHECK_IS_POWER2(umem_info->num_tx_desc)) { + ODP_ERR("Invalid xdp descriptor configuration, using defaults (%d).\n", + NUM_DESCS_DEFAULT); + goto defaults; + } + + return; + +defaults: + umem_info->num_rx_desc = NUM_DESCS_DEFAULT; + umem_info->num_tx_desc = NUM_DESCS_DEFAULT; +} + +static int umem_create(xdp_umem_info_t *umem_info, pool_t *pool) +{ + struct xsk_umem_config cfg; + + if (umem_info->ref_cnt++ > 0U) + return 0; + + parse_options(umem_info); + umem_info->pool = pool; + /* Fill queue size is recommended to be >= HW RX ring size + AF_XDP RX + * ring size, so use size twice the size of AF_XDP RX ring. */ + cfg.fill_size = umem_info->num_rx_desc * 2U; + cfg.comp_size = umem_info->num_tx_desc; + cfg.frame_size = pool->block_size; + cfg.frame_headroom = sizeof(odp_packet_hdr_t) + pool->headroom; + cfg.flags = XDP_UMEM_UNALIGNED_CHUNK_FLAG; + + return xsk_umem__create(&umem_info->umem, pool->base_addr, pool->shm_size, + &umem_info->fill_q, &umem_info->compl_q, &cfg); +} + +static void umem_delete(xdp_umem_info_t *umem_info) +{ + if (umem_info->ref_cnt-- != 1U) + return; + + while (xsk_umem__delete(umem_info->umem) == -EBUSY) + continue; } static uint32_t get_bind_queue_index(const char *devname) @@ -142,89 +233,32 @@ out: return idx; } -static odp_bool_t reserve_fill_queue_elements(xdp_sock_info_t *sock_info, int num) -{ - pool_t *pool; - odp_packet_t packets[num]; - int count; - struct xsk_ring_prod *fill_q; - uint32_t start_idx; - int pktio_idx; - uint32_t block_size; - odp_packet_hdr_t *pkt_hdr; - - pool = sock_info->umem_info->pool; - count = odp_packet_alloc_multi(pool->pool_hdl, sock_info->mtu, packets, num); - - if (count <= 0) - return false; - - fill_q = &sock_info->fill_q; - - if (xsk_ring_prod__reserve(fill_q, count, &start_idx) == 0U) { - odp_packet_free_multi(packets, count); - return false; - } - - pktio_idx = sock_info->pktio_idx; - block_size = pool->block_size; - - for (int i = 0; i < count; ++i) { - pkt_hdr = packet_hdr(packets[i]); - pkt_hdr->ms_pktio_idx = pktio_idx; - *xsk_ring_prod__fill_addr(fill_q, start_idx++) = - pkt_hdr->event_hdr.index.event * block_size; - } - - xsk_ring_prod__submit(&sock_info->fill_q, count); - - return true; -} - static int sock_xdp_open(odp_pktio_t pktio, pktio_entry_t *pktio_entry, const char *devname, odp_pool_t pool_hdl) { - pkt_xdp_t *priv; + xdp_sock_info_t *priv; pool_t *pool; - struct xsk_socket_config config; - uint32_t bind_q; int ret; if (disable_pktio) return -1; priv = pkt_priv(pktio_entry); - memset(priv, 0, sizeof(pkt_xdp_t)); + memset(priv, 0, sizeof(xdp_sock_info_t)); pool = pool_entry_from_hdl(pool_hdl); - priv->sock_info.umem_info = (xdp_umem_info_t *)pool->mem_src_data; - priv->sock_info.xsk = NULL; - /* Mark transitory kernel-owned packets with the pktio index, so that they can be freed on - * close. */ - priv->sock_info.pktio_idx = 1 + odp_pktio_index(pktio); - fill_socket_config(&config); - bind_q = get_bind_queue_index(devname); - /* With xsk_socket__create_shared(), as only one bind queue index can - * be passed, NIC in use needs to be configured accordingly to have - * only a single combined TX-RX queue, otherwise traffic may not end up - * on the socket. For now, always bind to the first queue (overridable - * with environment variable). */ - ret = xsk_socket__create_shared(&priv->sock_info.xsk, devname, bind_q, - priv->sock_info.umem_info->umem, &priv->sock_info.rx, - &priv->sock_info.tx, &priv->sock_info.fill_q, - &priv->sock_info.compl_q, &config); + priv->umem_info = (xdp_umem_info_t *)pool->mem_src_data; + ret = umem_create(priv->umem_info, pool); if (ret) { - ODP_ERR("Error creating xdp socket for bind queue %u: %d\n", bind_q, ret); - goto xsk_err; + ODP_ERR("Error creating UMEM pool for xdp: %d\n", ret); + return -1; } - /* Ring setup/clean up routines seem to be asynchronous with some drivers and might not be - * ready yet after xsk_socket__create_shared(). */ - sleep(1U); - + /* Mark transitory kernel-owned packets with the pktio index, so that they can be freed on + * close. */ + priv->pktio_idx = 1 + odp_pktio_index(pktio); /* Querying with ioctl() via AF_XDP socket doesn't seem to work, so * create a helper socket for this. */ - priv->sock_info.helper_sock = -1; ret = socket(AF_INET, SOCK_DGRAM, 0); if (ret == -1) { @@ -232,48 +266,53 @@ static int sock_xdp_open(odp_pktio_t pktio, pktio_entry_t *pktio_entry, const ch goto sock_err; } - priv->sock_info.helper_sock = ret; - priv->sock_info.mtu = _odp_mtu_get_fd(priv->sock_info.helper_sock, devname); + priv->helper_sock = ret; + priv->mtu = _odp_mtu_get_fd(priv->helper_sock, devname); - if (priv->sock_info.mtu == 0U) - goto res_err; + if (priv->mtu == 0U) + goto mtu_err; - priv->sock_info.max_mtu = pool->seg_len; + priv->max_mtu = pool->seg_len; - if (!reserve_fill_queue_elements(&priv->sock_info, config.rx_size)) { - ODP_ERR("Unable to reserve fill queue descriptors.\n"); - goto res_err; + for (int i = 0; i < PKTIO_MAX_QUEUES; ++i) { + odp_ticketlock_init(&priv->qs[i].rx_lock); + odp_ticketlock_init(&priv->qs[i].tx_lock); } - odp_ticketlock_init(&priv->rx_lock); - odp_ticketlock_init(&priv->tx_lock); + priv->bind_q = get_bind_queue_index(pktio_entry->s.name); + + ODP_DBG("Socket xdp interface (%s):\n", pktio_entry->s.name); + ODP_DBG(" num_rx_desc: %d\n", priv->umem_info->num_rx_desc); + ODP_DBG(" num_tx_desc: %d\n", priv->umem_info->num_tx_desc); + ODP_DBG(" starting bind queue: %u\n", priv->bind_q); return 0; -res_err: - close(priv->sock_info.helper_sock); - priv->sock_info.helper_sock = -1; +mtu_err: + close(priv->helper_sock); sock_err: - xsk_socket__delete(priv->sock_info.xsk); - priv->sock_info.xsk = NULL; + umem_delete(priv->umem_info); -xsk_err: return -1; } static int sock_xdp_close(pktio_entry_t *pktio_entry) { - pkt_xdp_t *priv = pkt_priv(pktio_entry); - pool_t *pool = priv->sock_info.umem_info->pool; + xdp_sock_info_t *priv = pkt_priv(pktio_entry); + pool_t *pool = priv->umem_info->pool; odp_packet_hdr_t *pkt_hdr; - if (priv->sock_info.helper_sock != -1) - close(priv->sock_info.helper_sock); + close(priv->helper_sock); - if (priv->sock_info.xsk != NULL) - xsk_socket__delete(priv->sock_info.xsk); + for (uint32_t i = 0U; i < priv->num_q; ++i) { + if (priv->qs[i].xsk != NULL) { + xsk_socket__delete(priv->qs[i].xsk); + priv->qs[i].xsk = NULL; + } + } + umem_delete(priv->umem_info); /* Ring setup/clean up routines seem to be asynchronous with some drivers and might not be * ready yet after xsk_socket__delete(). */ sleep(1U); @@ -282,7 +321,7 @@ static int sock_xdp_close(pktio_entry_t *pktio_entry) for (uint32_t i = 0U; i < pool->num + pool->skipped_blocks; ++i) { pkt_hdr = packet_hdr(packet_from_event_hdr(event_hdr_from_index(pool, i))); - if (pkt_hdr->ms_pktio_idx == priv->sock_info.pktio_idx) { + if (pkt_hdr->ms_pktio_idx == priv->pktio_idx) { pkt_hdr->ms_pktio_idx = 0U; odp_packet_free(packet_handle(pkt_hdr)); } @@ -291,6 +330,149 @@ static int sock_xdp_close(pktio_entry_t *pktio_entry) return 0; } +static int sock_xdp_stats(pktio_entry_t *pktio_entry, odp_pktio_stats_t *stats) +{ + xdp_sock_info_t *priv = pkt_priv(pktio_entry); + xdp_sock_t *sock; + odp_pktin_queue_stats_t qi_stats; + odp_pktout_queue_stats_t qo_stats; + struct xdp_statistics xdp_stats; + socklen_t optlen = sizeof(struct xdp_statistics); + + memset(stats, 0, sizeof(odp_pktio_stats_t)); + + for (uint32_t i = 0U; i < priv->num_q; ++i) { + sock = &priv->qs[i]; + qi_stats = sock->qi_stats; + qo_stats = sock->qo_stats; + stats->in_octets += qi_stats.octets; + stats->in_packets += qi_stats.packets; + stats->in_errors += qi_stats.errors; + stats->out_octets += qo_stats.octets; + stats->out_packets += qo_stats.packets; + + if (!getsockopt(xsk_socket__fd(sock->xsk), SOL_XDP, XDP_STATISTICS, &xdp_stats, + &optlen)) { + stats->in_errors += (xdp_stats.rx_dropped - sock->xdp_stats.rx_dropped); + stats->in_discards += + (xdp_stats.rx_invalid_descs - sock->xdp_stats.rx_inv_descs); + stats->out_discards += + (xdp_stats.tx_invalid_descs - sock->xdp_stats.tx_inv_descs); + } + } + + return 0; +} + +static int sock_xdp_stats_reset(pktio_entry_t *pktio_entry) +{ + xdp_sock_info_t *priv = pkt_priv(pktio_entry); + xdp_sock_t *sock; + struct xdp_statistics xdp_stats; + socklen_t optlen = sizeof(struct xdp_statistics); + + for (uint32_t i = 0U; i < priv->num_q; ++i) { + sock = &priv->qs[i]; + memset(&sock->qi_stats, 0, sizeof(odp_pktin_queue_stats_t)); + memset(&sock->qo_stats, 0, sizeof(odp_pktout_queue_stats_t)); + memset(sock->i_stats, 0, sizeof(sock->i_stats)); + + if (!getsockopt(xsk_socket__fd(sock->xsk), SOL_XDP, XDP_STATISTICS, &xdp_stats, + &optlen)) { + sock->xdp_stats.rx_dropped = xdp_stats.rx_dropped; + sock->xdp_stats.rx_inv_descs = xdp_stats.rx_invalid_descs; + sock->xdp_stats.tx_inv_descs = xdp_stats.tx_invalid_descs; + } + } + + return 0; +} + +static int sock_xdp_pktin_queue_stats(pktio_entry_t *pktio_entry, uint32_t index, + odp_pktin_queue_stats_t *pktin_stats) +{ + xdp_sock_info_t *priv = pkt_priv(pktio_entry); + xdp_sock_t *sock; + struct xdp_statistics xdp_stats; + socklen_t optlen = sizeof(struct xdp_statistics); + + sock = &priv->qs[index]; + *pktin_stats = sock->qi_stats; + + if (!getsockopt(xsk_socket__fd(sock->xsk), SOL_XDP, XDP_STATISTICS, &xdp_stats, &optlen)) { + pktin_stats->errors += (xdp_stats.rx_dropped - sock->xdp_stats.rx_dropped); + pktin_stats->discards += + (xdp_stats.rx_invalid_descs - sock->xdp_stats.rx_inv_descs); + } + + return 0; +} + +static int sock_xdp_pktout_queue_stats(pktio_entry_t *pktio_entry, uint32_t index, + odp_pktout_queue_stats_t *pktout_stats) +{ + xdp_sock_info_t *priv = pkt_priv(pktio_entry); + xdp_sock_t *sock; + struct xdp_statistics xdp_stats; + socklen_t optlen = sizeof(struct xdp_statistics); + + sock = &priv->qs[index]; + *pktout_stats = sock->qo_stats; + + if (!getsockopt(xsk_socket__fd(sock->xsk), SOL_XDP, XDP_STATISTICS, &xdp_stats, &optlen)) + pktout_stats->discards += + (xdp_stats.tx_invalid_descs - sock->xdp_stats.tx_inv_descs); + + return 0; +} + +static int sock_xdp_extra_stat_info(pktio_entry_t *pktio_entry, odp_pktio_extra_stat_info_t info[], + int num) +{ + xdp_sock_info_t *priv = pkt_priv(pktio_entry); + const int total_stats = MAX_INTERNAL_STATS * priv->num_q; + + if (info != NULL && num > 0) { + for (int i = 0; i < _ODP_MIN(num, total_stats); ++i) + snprintf(info[i].name, ODP_PKTIO_STATS_EXTRA_NAME_LEN - 1, + "q%" PRIu64 "_%s", i / MAX_INTERNAL_STATS, + internal_stats_strs[i % MAX_INTERNAL_STATS]); + } + + return total_stats; +} + +static int sock_xdp_extra_stats(pktio_entry_t *pktio_entry, uint64_t stats[], int num) +{ + xdp_sock_info_t *priv = pkt_priv(pktio_entry); + const int total_stats = MAX_INTERNAL_STATS * priv->num_q; + uint64_t *i_stats; + + if (stats != NULL && num > 0) { + for (int i = 0; i < _ODP_MIN(num, total_stats); ++i) { + i_stats = priv->qs[i / MAX_INTERNAL_STATS].i_stats; + stats[i] = i_stats[i % MAX_INTERNAL_STATS]; + } + } + + return total_stats; +} + +static int sock_xdp_extra_stat_counter(pktio_entry_t *pktio_entry, uint32_t id, uint64_t *stat) +{ + xdp_sock_info_t *priv = pkt_priv(pktio_entry); + const uint32_t total_stats = MAX_INTERNAL_STATS * priv->num_q; + + if (id >= total_stats) { + ODP_ERR("Invalid counter id: %u (allowed range: 0-%u)\n", id, total_stats - 1U); + return -1; + } + + *stat = priv->qs[id / MAX_INTERNAL_STATS].i_stats[id % MAX_INTERNAL_STATS]; + + return 0; +} + static inline void extract_data(const struct xdp_desc *rx_desc, uint8_t *pool_base_addr, pkt_data_t *pkt_data) { @@ -311,17 +493,16 @@ static inline void extract_data(const struct xdp_desc *rx_desc, uint8_t *pool_ba pkt_data->len = rx_desc->len; } -static uint32_t process_received(pktio_entry_t *pktio_entry, xdp_sock_info_t *sock_info, +static uint32_t process_received(pktio_entry_t *pktio_entry, xdp_sock_t *sock, pool_t *pool, uint32_t start_idx, odp_packet_t packets[], int num) { + struct xsk_ring_cons *rx = &sock->rx; + uint8_t *base_addr = pool->base_addr; pkt_data_t pkt_data; - struct xsk_ring_cons *rx = &sock_info->rx; - uint8_t *base_addr = sock_info->umem_info->pool->base_addr; const odp_proto_layer_t layer = pktio_entry->s.parse_layer; - const odp_proto_chksums_t in_chksums = pktio_entry->s.in_chksums; + int ret; const odp_pktin_config_opt_t opt = pktio_entry->s.config.pktin; - uint64_t l4_part_sum = 0U; - odp_pool_t *pool_hdl = &sock_info->umem_info->pool->pool_hdl; + uint64_t errors = 0U, octets = 0U; odp_pktio_t pktio_hdl = pktio_entry->s.handle; uint32_t num_rx = 0U; @@ -329,92 +510,146 @@ static uint32_t process_received(pktio_entry_t *pktio_entry, xdp_sock_info_t *so extract_data(xsk_ring_cons__rx_desc(rx, start_idx++), base_addr, &pkt_data); pkt_data.pkt_hdr->ms_pktio_idx = 0U; packet_init(pkt_data.pkt_hdr, pkt_data.len); + pkt_data.pkt_hdr->seg_data = pkt_data.data; + pkt_data.pkt_hdr->event_hdr.base_data = pkt_data.data; if (layer) { - if (_odp_packet_parse_common(&pkt_data.pkt_hdr->p, pkt_data.data, - pkt_data.len, pkt_data.len, - layer, in_chksums, &l4_part_sum, opt) < 0) { + ret = _odp_packet_parse_common(pkt_data.pkt_hdr, pkt_data.data, + pkt_data.len, pkt_data.len, + layer, opt); + + if (ret) + ++errors; + + if (ret < 0) { odp_packet_free(pkt_data.pkt); continue; } - if (pktio_cls_enabled(pktio_entry) && - _odp_cls_classify_packet(pktio_entry, pkt_data.data, pool_hdl, - pkt_data.pkt_hdr)) { - odp_packet_free(pkt_data.pkt); - continue; + if (pktio_cls_enabled(pktio_entry)) { + odp_pool_t new_pool; + + ret = _odp_cls_classify_packet(pktio_entry, pkt_data.data, + &new_pool, pkt_data.pkt_hdr); + if (ret) { + odp_packet_free(pkt_data.pkt); + continue; + } + + if (odp_unlikely(_odp_pktio_packet_to_pool( + &pkt_data.pkt, &pkt_data.pkt_hdr, new_pool))) { + odp_packet_free(pkt_data.pkt); + continue; + } } } - pkt_data.pkt_hdr->seg_data = pkt_data.data; - pkt_data.pkt_hdr->event_hdr.base_data = pkt_data.data; pkt_data.pkt_hdr->input = pktio_hdl; packets[num_rx++] = pkt_data.pkt; + octets += pkt_data.len; } + sock->qi_stats.octets += octets; + sock->qi_stats.packets += num_rx; + sock->qi_stats.errors += errors; + return num_rx; } -static int sock_xdp_recv(pktio_entry_t *pktio_entry, int index ODP_UNUSED, odp_packet_t packets[], - int num) +static odp_bool_t reserve_fill_queue_elements(xdp_sock_info_t *sock_info, xdp_sock_t *sock, + int num) { - pkt_xdp_t *priv; + pool_t *pool; + odp_packet_t packets[num]; + int count; + struct xsk_ring_prod *fill_q; + uint32_t start_idx; + int pktio_idx; + uint32_t block_size; + odp_packet_hdr_t *pkt_hdr; + + pool = sock_info->umem_info->pool; + count = odp_packet_alloc_multi(pool->pool_hdl, sock_info->mtu, packets, num); + + if (count <= 0) { + ++sock->i_stats[RX_PKT_ALLOC_ERR]; + return false; + } + + fill_q = &sock->fill_q; + + if (xsk_ring_prod__reserve(fill_q, count, &start_idx) == 0U) { + odp_packet_free_multi(packets, count); + ++sock->i_stats[RX_DESC_RSV_ERR]; + return false; + } + + pktio_idx = sock_info->pktio_idx; + block_size = pool->block_size; + + for (int i = 0; i < count; ++i) { + pkt_hdr = packet_hdr(packets[i]); + pkt_hdr->ms_pktio_idx = pktio_idx; + *xsk_ring_prod__fill_addr(fill_q, start_idx++) = + pkt_hdr->event_hdr.index.event * block_size; + } + + xsk_ring_prod__submit(&sock->fill_q, count); + + return true; +} + +static int sock_xdp_recv(pktio_entry_t *pktio_entry, int index, odp_packet_t packets[], int num) +{ + xdp_sock_info_t *priv; + xdp_sock_t *sock; struct pollfd fd; uint32_t start_idx = 0U, recvd, procd; priv = pkt_priv(pktio_entry); - odp_ticketlock_lock(&priv->rx_lock); + sock = &priv->qs[index]; + + if (!priv->lockless_rx) + odp_ticketlock_lock(&sock->rx_lock); - if (odp_unlikely(xsk_ring_prod__needs_wakeup(&priv->sock_info.fill_q))) { - fd.fd = xsk_socket__fd(priv->sock_info.xsk); + if (odp_unlikely(xsk_ring_prod__needs_wakeup(&sock->fill_q))) { + fd.fd = xsk_socket__fd(sock->xsk); fd.events = POLLIN; (void)poll(&fd, 1U, 0); } - recvd = xsk_ring_cons__peek(&priv->sock_info.rx, num, &start_idx); + recvd = xsk_ring_cons__peek(&sock->rx, num, &start_idx); if (recvd == 0U) { - odp_ticketlock_unlock(&priv->rx_lock); + if (!priv->lockless_rx) + odp_ticketlock_unlock(&sock->rx_lock); return 0; } - procd = process_received(pktio_entry, &priv->sock_info, start_idx, packets, recvd); - xsk_ring_cons__release(&priv->sock_info.rx, recvd); - (void)reserve_fill_queue_elements(&priv->sock_info, recvd); - odp_ticketlock_unlock(&priv->rx_lock); - - return procd; -} + procd = process_received(pktio_entry, sock, priv->umem_info->pool, start_idx, packets, + recvd); + xsk_ring_cons__release(&sock->rx, recvd); + (void)reserve_fill_queue_elements(priv, sock, recvd); -static inline void populate_tx_desc(pool_t *pool, odp_packet_hdr_t *pkt_hdr, - struct xdp_desc *tx_desc) -{ - uint64_t frame_off; - uint64_t pkt_off; + if (!priv->lockless_rx) + odp_ticketlock_unlock(&sock->rx_lock); - frame_off = pkt_hdr->event_hdr.index.event * pool->block_size; - pkt_off = (uint64_t)(uintptr_t)pkt_hdr->event_hdr.base_data - - (uint64_t)(uintptr_t)pool->base_addr - frame_off; - pkt_off <<= XSK_UNALIGNED_BUF_OFFSET_SHIFT; - tx_desc->addr = frame_off | pkt_off; - tx_desc->len = pkt_hdr->frame_len; + return procd; } -static void handle_pending_tx(xdp_sock_info_t *sock_info, int num) +static void handle_pending_tx(xdp_sock_t *sock, uint8_t *base_addr, int num) { struct xsk_ring_cons *compl_q; uint32_t sent; - uint8_t *base_addr; uint32_t start_idx; uint64_t frame_off; odp_packet_t pkt; - if (odp_unlikely(xsk_ring_prod__needs_wakeup(&sock_info->tx))) - (void)sendto(xsk_socket__fd(sock_info->xsk), NULL, 0U, MSG_DONTWAIT, NULL, 0U); + if (odp_unlikely(xsk_ring_prod__needs_wakeup(&sock->tx))) + (void)sendto(xsk_socket__fd(sock->xsk), NULL, 0U, MSG_DONTWAIT, NULL, 0U); - compl_q = &sock_info->compl_q; + compl_q = &sock->compl_q; sent = xsk_ring_cons__peek(compl_q, num, &start_idx); - base_addr = sock_info->umem_info->pool->base_addr; odp_packet_t packets[sent]; @@ -432,57 +667,95 @@ static void handle_pending_tx(xdp_sock_info_t *sock_info, int num) } } -static int sock_xdp_send(pktio_entry_t *pktio_entry, int index ODP_UNUSED, - const odp_packet_t packets[], int num) +static inline void populate_tx_desc(odp_packet_hdr_t *pkt_hdr, pool_t *pool, + struct xdp_desc *tx_desc, uint32_t len) { - pkt_xdp_t *priv; - xdp_sock_info_t *sock_info; + uint64_t frame_off; + uint64_t pkt_off; + + frame_off = pkt_hdr->event_hdr.index.event * pool->block_size; + pkt_off = (uint64_t)(uintptr_t)pkt_hdr->event_hdr.base_data + - (uint64_t)(uintptr_t)pool->base_addr - frame_off; + pkt_off <<= XSK_UNALIGNED_BUF_OFFSET_SHIFT; + tx_desc->addr = frame_off | pkt_off; + tx_desc->len = len; +} + +static inline void populate_tx_descs(odp_packet_hdr_t *pkt_hdr, pool_t *pool, + struct xsk_ring_prod *tx, int seg_cnt, uint32_t start_idx, + int pktio_idx) +{ + if (odp_likely(seg_cnt == 1)) { + populate_tx_desc(pkt_hdr, pool, xsk_ring_prod__tx_desc(tx, start_idx), + pkt_hdr->frame_len); + pkt_hdr->ms_pktio_idx = pktio_idx; + } else { + for (int i = 0; i < seg_cnt; ++i) { + populate_tx_desc(pkt_hdr, pool, xsk_ring_prod__tx_desc(tx, start_idx++), + pkt_hdr->seg_len); + pkt_hdr->ms_pktio_idx = pktio_idx; + pkt_hdr = pkt_hdr->seg_next; + } + } +} + +static int sock_xdp_send(pktio_entry_t *pktio_entry, int index, const odp_packet_t packets[], + int num) +{ + xdp_sock_info_t *priv; + xdp_sock_t *sock; pool_t *pool; odp_pool_t pool_hdl; - int pktio_idx, i; + int pktio_idx, i, seg_cnt; struct xsk_ring_prod *tx; + uint8_t *base_addr; odp_packet_t pkt; odp_packet_hdr_t *pkt_hdr; - uint32_t start_idx; + uint32_t tx_descs, start_idx, sent = 0U; + uint64_t octets = 0U; if (odp_unlikely(num == 0)) return 0; priv = pkt_priv(pktio_entry); - odp_ticketlock_lock(&priv->tx_lock); - sock_info = &priv->sock_info; - pool = sock_info->umem_info->pool; + sock = &priv->qs[index]; + + if (!priv->lockless_tx) + odp_ticketlock_lock(&sock->tx_lock); + + pool = priv->umem_info->pool; pool_hdl = pool->pool_hdl; - pktio_idx = sock_info->pktio_idx; - tx = &sock_info->tx; + pktio_idx = priv->pktio_idx; + tx = &sock->tx; + base_addr = priv->umem_info->pool->base_addr; + tx_descs = priv->umem_info->num_tx_desc; for (i = 0; i < num; ++i) { pkt = ODP_PACKET_INVALID; - - if (odp_unlikely(odp_packet_num_segs(packets[i])) > 1) { - /* TODO: handle segmented packets */ - ODP_ERR("Only single-segment packets supported\n"); - break; - } - pkt_hdr = packet_hdr(packets[i]); + seg_cnt = pkt_hdr->seg_count; if (pkt_hdr->event_hdr.pool_ptr != pool) { pkt = odp_packet_copy(packets[i], pool_hdl); - if (odp_unlikely(pkt == ODP_PACKET_INVALID)) + if (odp_unlikely(pkt == ODP_PACKET_INVALID)) { + ++sock->i_stats[TX_PKT_ALLOC_ERR]; break; + } pkt_hdr = packet_hdr(pkt); + seg_cnt = pkt_hdr->seg_count; } - if (xsk_ring_prod__reserve(tx, 1U, &start_idx) == 0U) { - handle_pending_tx(sock_info, NUM_XDP_DESCS); + if (xsk_ring_prod__reserve(tx, seg_cnt, &start_idx) == 0U) { + handle_pending_tx(sock, base_addr, tx_descs); - if (xsk_ring_prod__reserve(tx, 1U, &start_idx) == 0U) { + if (xsk_ring_prod__reserve(tx, seg_cnt, &start_idx) == 0U) { if (pkt != ODP_PACKET_INVALID) odp_packet_free(pkt); + ++sock->i_stats[TX_DESC_RSV_ERR]; + break; } } @@ -490,94 +763,212 @@ static int sock_xdp_send(pktio_entry_t *pktio_entry, int index ODP_UNUSED, if (pkt != ODP_PACKET_INVALID) odp_packet_free(packets[i]); - pkt_hdr->ms_pktio_idx = pktio_idx; - populate_tx_desc(pool, pkt_hdr, xsk_ring_prod__tx_desc(tx, start_idx)); + populate_tx_descs(pkt_hdr, pool, tx, seg_cnt, start_idx, pktio_idx); + sent += seg_cnt; + octets += pkt_hdr->frame_len; } - xsk_ring_prod__submit(tx, i); - handle_pending_tx(sock_info, NUM_XDP_DESCS); - odp_ticketlock_unlock(&priv->tx_lock); + xsk_ring_prod__submit(tx, sent); + handle_pending_tx(sock, base_addr, tx_descs); + sock->qo_stats.octets += octets; + sock->qo_stats.packets += i; + + if (!priv->lockless_tx) + odp_ticketlock_unlock(&sock->tx_lock); return i; } static uint32_t sock_xdp_mtu_get(pktio_entry_t *pktio_entry) { - return pkt_priv(pktio_entry)->sock_info.mtu; + return pkt_priv(pktio_entry)->mtu; } static int sock_xdp_mtu_set(pktio_entry_t *pktio_entry, uint32_t maxlen_input, uint32_t maxlen_output ODP_UNUSED) { - pkt_xdp_t *priv = pkt_priv(pktio_entry); + xdp_sock_info_t *priv = pkt_priv(pktio_entry); int ret; - ret = _odp_mtu_set_fd(priv->sock_info.helper_sock, pktio_entry->s.name, maxlen_input); + ret = _odp_mtu_set_fd(priv->helper_sock, pktio_entry->s.name, maxlen_input); if (ret) return ret; - priv->sock_info.mtu = maxlen_input; + priv->mtu = maxlen_input; return 0; } static int sock_xdp_promisc_mode_set(pktio_entry_t *pktio_entry, int enable) { - return _odp_promisc_mode_set_fd(pkt_priv(pktio_entry)->sock_info.helper_sock, + return _odp_promisc_mode_set_fd(pkt_priv(pktio_entry)->helper_sock, pktio_entry->s.name, enable); } static int sock_xdp_promisc_mode_get(pktio_entry_t *pktio_entry) { - return _odp_promisc_mode_get_fd(pkt_priv(pktio_entry)->sock_info.helper_sock, + return _odp_promisc_mode_get_fd(pkt_priv(pktio_entry)->helper_sock, pktio_entry->s.name); } static int sock_xdp_mac_addr_get(pktio_entry_t *pktio_entry ODP_UNUSED, void *mac_addr) { - return _odp_mac_addr_get_fd(pkt_priv(pktio_entry)->sock_info.helper_sock, + return _odp_mac_addr_get_fd(pkt_priv(pktio_entry)->helper_sock, pktio_entry->s.name, mac_addr) ? -1 : ETH_ALEN; } static int sock_xdp_link_status(pktio_entry_t *pktio_entry) { - return _odp_link_status_fd(pkt_priv(pktio_entry)->sock_info.helper_sock, + return _odp_link_status_fd(pkt_priv(pktio_entry)->helper_sock, pktio_entry->s.name); } static int sock_xdp_link_info(pktio_entry_t *pktio_entry, odp_pktio_link_info_t *info) { - return _odp_link_info_fd(pkt_priv(pktio_entry)->sock_info.helper_sock, + return _odp_link_info_fd(pkt_priv(pktio_entry)->helper_sock, pktio_entry->s.name, info); } +static int set_queue_capability(int fd, const char *devname, odp_pktio_capability_t *capa) +{ + struct ifreq ifr; + struct ethtool_channels channels; + uint32_t max_channels; + int ret; + + memset(&channels, 0, sizeof(struct ethtool_channels)); + channels.cmd = ETHTOOL_GCHANNELS; + snprintf(ifr.ifr_name, IF_NAMESIZE, "%s", devname); + ifr.ifr_data = (char *)&channels; + ret = ioctl(fd, SIOCETHTOOL, &ifr); + + if (ret == -1 || channels.max_combined == 0U) { + if (ret == -1 && errno != EOPNOTSUPP) { + ODP_ERR("Unable to query NIC channel capabilities: %s\n", strerror(errno)); + return -1; + } + + channels.max_combined = 1U; + } + + max_channels = _ODP_MIN((uint32_t)PKTIO_MAX_QUEUES, channels.max_combined); + capa->max_input_queues = max_channels; + capa->max_output_queues = max_channels; + + return 0; +} + static int sock_xdp_capability(pktio_entry_t *pktio_entry, odp_pktio_capability_t *capa) { - pkt_xdp_t *priv = pkt_priv(pktio_entry); + xdp_sock_info_t *priv = pkt_priv(pktio_entry); memset(capa, 0, sizeof(odp_pktio_capability_t)); - capa->max_input_queues = 1U; - capa->max_output_queues = 1U; + + if (set_queue_capability(priv->helper_sock, pktio_entry->s.name, capa)) + return -1; + capa->set_op.op.promisc_mode = 1U; capa->set_op.op.maxlen = 1U; capa->maxlen.equal = true; capa->maxlen.min_input = _ODP_SOCKET_MTU_MIN; - capa->maxlen.max_input = priv->sock_info.max_mtu; + capa->maxlen.max_input = priv->max_mtu; capa->maxlen.min_output = _ODP_SOCKET_MTU_MIN; - capa->maxlen.max_output = priv->sock_info.max_mtu; + capa->maxlen.max_output = priv->max_mtu; capa->config.parser.layer = ODP_PROTO_LAYER_ALL; - capa->stats.pktio.all_counters = 0U; - capa->stats.pktin_queue.all_counters = 0U; - capa->stats.pktout_queue.all_counters = 0U; + capa->stats.pktio.counter.in_octets = 1U; + capa->stats.pktio.counter.in_packets = 1U; + capa->stats.pktio.counter.in_errors = 1U; + capa->stats.pktio.counter.in_discards = 1U; + capa->stats.pktio.counter.out_octets = 1U; + capa->stats.pktio.counter.out_packets = 1U; + capa->stats.pktio.counter.out_discards = 1U; + + capa->stats.pktin_queue.counter.octets = 1U; + capa->stats.pktin_queue.counter.packets = 1U; + capa->stats.pktin_queue.counter.errors = 1U; + capa->stats.pktin_queue.counter.discards = 1U; + capa->stats.pktout_queue.counter.octets = 1U; + capa->stats.pktout_queue.counter.packets = 1U; + capa->stats.pktout_queue.counter.discards = 1U; + + return 0; +} + +static int sock_xdp_input_queues_config(pktio_entry_t *pktio_entry, + const odp_pktin_queue_param_t *param) +{ + xdp_sock_info_t *priv = pkt_priv(pktio_entry); + + priv->lockless_rx = pktio_entry->s.param.in_mode == ODP_PKTIN_MODE_SCHED || + param->op_mode == ODP_PKTIO_OP_MT_UNSAFE; + + return 0; +} + +static void fill_socket_config(struct xsk_socket_config *config, xdp_umem_info_t *umem_info) +{ + config->rx_size = umem_info->num_rx_desc; + config->tx_size = umem_info->num_tx_desc; + config->libxdp_flags = 0U; + config->xdp_flags = 0U; + config->bind_flags = XDP_ZEROCOPY; /* TODO: XDP_COPY */ +} + +static int sock_xdp_output_queues_config(pktio_entry_t *pktio_entry, + const odp_pktout_queue_param_t *param) +{ + xdp_sock_info_t *priv = pkt_priv(pktio_entry); + struct xsk_socket_config config; + const char *devname = pktio_entry->s.name; + uint32_t bind_q, i; + struct xsk_umem *umem; + xdp_sock_t *sock; + int ret; + + priv->lockless_tx = param->op_mode == ODP_PKTIO_OP_MT_UNSAFE; + fill_socket_config(&config, priv->umem_info); + bind_q = priv->bind_q; + umem = priv->umem_info->umem; + + for (i = 0U; i < param->num_queues; ++i) { + sock = &priv->qs[i]; + ret = xsk_socket__create_shared(&sock->xsk, devname, bind_q, umem, &sock->rx, + &sock->tx, &sock->fill_q, &sock->compl_q, &config); + + if (ret) { + ODP_ERR("Error creating xdp socket for bind queue %u: %d\n", bind_q, ret); + goto err; + } + + if (!reserve_fill_queue_elements(priv, sock, config.rx_size)) { + ODP_ERR("Unable to reserve fill queue descriptors for queue: %u.\n", + bind_q); + goto err; + } + + ++bind_q; + } + + priv->num_q = i; + /* Ring setup/clean up routines seem to be asynchronous with some drivers and might not be + * ready yet after xsk_socket__create_shared(). */ + sleep(1U); return 0; + +err: + for (uint32_t j = 0U; j < i; ++j) { + xsk_socket__delete(priv->qs[j].xsk); + priv->qs[j].xsk = NULL; + } + + return -1; } const pktio_if_ops_t _odp_sock_xdp_pktio_ops = { - /* TODO: at least stats */ .name = "socket_xdp", .print = NULL, .init_global = sock_xdp_init_global, @@ -587,13 +978,13 @@ const pktio_if_ops_t _odp_sock_xdp_pktio_ops = { .close = sock_xdp_close, .start = NULL, .stop = NULL, - .stats = NULL, - .stats_reset = NULL, - .pktin_queue_stats = NULL, - .pktout_queue_stats = NULL, - .extra_stat_info = NULL, - .extra_stats = NULL, - .extra_stat_counter = NULL, + .stats = sock_xdp_stats, + .stats_reset = sock_xdp_stats_reset, + .pktin_queue_stats = sock_xdp_pktin_queue_stats, + .pktout_queue_stats = sock_xdp_pktout_queue_stats, + .extra_stat_info = sock_xdp_extra_stat_info, + .extra_stats = sock_xdp_extra_stats, + .extra_stat_counter = sock_xdp_extra_stat_counter, .pktio_ts_res = NULL, .pktio_ts_from_ns = NULL, .pktio_time = NULL, @@ -612,8 +1003,8 @@ const pktio_if_ops_t _odp_sock_xdp_pktio_ops = { .link_info = sock_xdp_link_info, .capability = sock_xdp_capability, .config = NULL, - .input_queues_config = NULL, - .output_queues_config = NULL + .input_queues_config = sock_xdp_input_queues_config, + .output_queues_config = sock_xdp_output_queues_config }; static odp_bool_t sock_xdp_is_mem_src_active(void) @@ -647,39 +1038,13 @@ static void sock_xdp_adjust_block_size(uint8_t *data ODP_UNUSED, uint32_t *block *block_size = _ODP_MAX(size, MIN_FRAME_SIZE); } -static int sock_xdp_umem_create(uint8_t *data, pool_t *pool) -{ - struct xsk_umem_config cfg; - xdp_umem_info_t *umem_info = (xdp_umem_info_t *)data; - - umem_info->pool = pool; - /* Fill queue size is recommended to be >= HW RX ring size + AF_XDP RX - * ring size, so use size twice the size of AF_XDP RX ring. */ - cfg.fill_size = NUM_XDP_DESCS * 2U; /* TODO: num descs vs pool size */ - cfg.comp_size = NUM_XDP_DESCS; - cfg.frame_size = pool->block_size; - cfg.frame_headroom = sizeof(odp_packet_hdr_t) + pool->headroom; - cfg.flags = XDP_UMEM_UNALIGNED_CHUNK_FLAG; - - return xsk_umem__create(&umem_info->umem, pool->base_addr, pool->shm_size, - &umem_info->fill_q, &umem_info->compl_q, &cfg); -} - -static void sock_xdp_umem_delete(uint8_t *data) -{ - xdp_umem_info_t *umem_info = (xdp_umem_info_t *)data; - - while (xsk_umem__delete(umem_info->umem) == -EBUSY) - continue; -} - const _odp_pool_mem_src_ops_t _odp_pool_sock_xdp_mem_src_ops = { .name = "xdp_zc", .is_active = sock_xdp_is_mem_src_active, .force_disable = sock_xdp_force_mem_src_disable, .adjust_size = sock_xdp_adjust_block_size, - .bind = sock_xdp_umem_create, - .unbind = sock_xdp_umem_delete + .bind = NULL, + .unbind = NULL }; #else diff --git a/platform/linux-generic/pktio/tap.c b/platform/linux-generic/pktio/tap.c index 40c46b43c..c61e05575 100644 --- a/platform/linux-generic/pktio/tap.c +++ b/platform/linux-generic/pktio/tap.c @@ -285,31 +285,11 @@ static odp_packet_t pack_odp_pkt(pktio_entry_t *pktio_entry, const void *data, { odp_packet_t pkt; odp_packet_hdr_t *pkt_hdr; - odp_packet_hdr_t parsed_hdr; int num; - uint64_t l4_part_sum = 0; uint16_t frame_offset = pktio_entry->s.pktin_frame_offset; - const odp_proto_chksums_t chksums = pktio_entry->s.in_chksums; const odp_proto_layer_t layer = pktio_entry->s.parse_layer; const odp_pktin_config_opt_t opt = pktio_entry->s.config.pktin; - if (layer) { - packet_parse_reset(&parsed_hdr, 1); - packet_set_len(&parsed_hdr, len); - if (_odp_packet_parse_common(&parsed_hdr.p, data, len, len, layer, - chksums, &l4_part_sum, opt) < 0) { - return ODP_PACKET_INVALID; - } - - if (pktio_cls_enabled(pktio_entry)) { - if (_odp_cls_classify_packet(pktio_entry, data, - &pkt_priv(pktio_entry)->pool, - &parsed_hdr)) { - return ODP_PACKET_INVALID; - } - } - } - num = _odp_packet_alloc_multi(pkt_priv(pktio_entry)->pool, len + frame_offset, &pkt, 1); if (num != 1) @@ -327,10 +307,29 @@ static odp_packet_t pack_odp_pkt(pktio_entry_t *pktio_entry, const void *data, } if (layer) { - _odp_packet_copy_cls_md(pkt_hdr, &parsed_hdr); + if (_odp_packet_parse_common(pkt_hdr, data, len, len, layer, + opt) < 0) { + odp_packet_free(pkt); + return ODP_PACKET_INVALID; + } + + if (pktio_cls_enabled(pktio_entry)) { + odp_pool_t new_pool; - if (layer >= ODP_PROTO_LAYER_L4) - _odp_packet_l4_chksum(pkt_hdr, chksums, l4_part_sum); + if (_odp_cls_classify_packet(pktio_entry, data, + &new_pool, pkt_hdr)) { + odp_packet_free(pkt); + return ODP_PACKET_INVALID; + } + + if (odp_unlikely(_odp_pktio_packet_to_pool( + &pkt, &pkt_hdr, new_pool))) { + odp_packet_free(pkt); + odp_atomic_inc_u64( + &pktio_entry->s.stats_extra.in_discards); + return ODP_PACKET_INVALID; + } + } } packet_set_ts(pkt_hdr, ts); diff --git a/platform/linux-generic/test/example/ipsec_api/pktio_env b/platform/linux-generic/test/example/ipsec_api/pktio_env index 5140b1e8f..f8cadb5cb 100644 --- a/platform/linux-generic/test/example/ipsec_api/pktio_env +++ b/platform/linux-generic/test/example/ipsec_api/pktio_env @@ -35,6 +35,11 @@ if [ -n "$WITH_OPENSSL_CRYPTO" ] && [ ${WITH_OPENSSL_CRYPTO} -eq 0 ]; then exit 77 fi +if [ ${ODPH_PROC_MODE} -eq 1 ]; then + echo "Process mode not supported. Skipping." + exit 77 +fi + # Skip live and router mode tests. if [ ${IPSEC_APP_MODE} -eq 1 ] || [ ${IPSEC_APP_MODE} -eq 2 ]; then echo "IPsec Live / Router mode test. Skipping." diff --git a/platform/linux-generic/test/example/ipsec_crypto/pktio_env b/platform/linux-generic/test/example/ipsec_crypto/pktio_env index c287b08c2..d2550063e 100644 --- a/platform/linux-generic/test/example/ipsec_crypto/pktio_env +++ b/platform/linux-generic/test/example/ipsec_crypto/pktio_env @@ -35,6 +35,11 @@ if [ -n "$WITH_OPENSSL_CRYPTO" ] && [ ${WITH_OPENSSL_CRYPTO} -eq 0 ]; then exit 77 fi +if [ ${ODPH_PROC_MODE} -eq 1 ]; then + echo "Process mode not supported. Skipping." + exit 77 +fi + # Skip live and router mode tests. if [ ${IPSEC_APP_MODE} -eq 1 ] || [ ${IPSEC_APP_MODE} -eq 2 ]; then echo "Live / Router mode test. Skipping." diff --git a/platform/linux-generic/test/inline-timer.conf b/platform/linux-generic/test/inline-timer.conf index 2ade68c32..c7379e38a 100644 --- a/platform/linux-generic/test/inline-timer.conf +++ b/platform/linux-generic/test/inline-timer.conf @@ -1,6 +1,6 @@ # Mandatory fields odp_implementation = "linux-generic" -config_file_version = "0.1.20" +config_file_version = "0.1.21" timer: { # Enable inline timer implementation diff --git a/platform/linux-generic/test/packet_align.conf b/platform/linux-generic/test/packet_align.conf index f0370c7a2..433899017 100644 --- a/platform/linux-generic/test/packet_align.conf +++ b/platform/linux-generic/test/packet_align.conf @@ -1,6 +1,6 @@ # Mandatory fields odp_implementation = "linux-generic" -config_file_version = "0.1.20" +config_file_version = "0.1.21" pool: { pkt: { diff --git a/platform/linux-generic/test/process-mode.conf b/platform/linux-generic/test/process-mode.conf index 3ec28dce9..a36c9fc3d 100644 --- a/platform/linux-generic/test/process-mode.conf +++ b/platform/linux-generic/test/process-mode.conf @@ -1,6 +1,6 @@ # Mandatory fields odp_implementation = "linux-generic" -config_file_version = "0.1.20" +config_file_version = "0.1.21" # Shared memory options shm: { diff --git a/platform/linux-generic/test/sched-basic.conf b/platform/linux-generic/test/sched-basic.conf index 25e8b33e0..16654595b 100644 --- a/platform/linux-generic/test/sched-basic.conf +++ b/platform/linux-generic/test/sched-basic.conf @@ -1,6 +1,6 @@ # Mandatory fields odp_implementation = "linux-generic" -config_file_version = "0.1.20" +config_file_version = "0.1.21" # Test scheduler with an odd spread value and without dynamic load balance sched_basic: { |