diff options
Diffstat (limited to 'platform')
65 files changed, 1129 insertions, 410 deletions
diff --git a/platform/linux-dpdk/Makefile.am b/platform/linux-dpdk/Makefile.am index 61b61f6d6..279aed0c1 100644 --- a/platform/linux-dpdk/Makefile.am +++ b/platform/linux-dpdk/Makefile.am @@ -242,12 +242,13 @@ __LIB__libodp_dpdk_la_SOURCES += arch/default/odp_atomic.c \ arch/default/odp_cpu_cycles.c \ arch/default/odp_hash_crc32.c \ arch/arm/odp_sysinfo_parse.c -odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_inlines.h \ - arch/default/odp/api/abi/cpu_time.h \ +odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_time.h \ arch/default/odp/api/abi/hash_crc32.h if !ODP_ABI_COMPAT odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ arch/default/odp/api/abi/atomic_inlines.h \ + arch/default/odp/api/abi/cpu_generic.h \ + arch/arm/odp/api/abi/cpu_inlines.h \ arch/arm/odp/api/abi/cpu.h endif noinst_HEADERS += arch/arm/odp_atomic.h \ @@ -264,14 +265,15 @@ __LIB__libodp_dpdk_la_SOURCES += arch/aarch64/odp_atomic.c \ arch/aarch64/odp_global_time.c \ arch/default/odp_hash_crc32.c \ arch/aarch64/odp_sysinfo_parse.c -odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_inlines.h \ - arch/aarch64/odp/api/abi/cpu_time.h \ +odpapiabiarchinclude_HEADERS += arch/aarch64/odp/api/abi/cpu_time.h \ arch/aarch64/odp/api/abi/hash_crc32.h if !ODP_ABI_COMPAT odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ arch/aarch64/odp/api/abi/atomic_inlines.h \ arch/aarch64/odp/api/abi/atomic.h \ - arch/aarch64/odp/api/abi/cpu.h + arch/default/odp/api/abi/cpu_generic.h \ + arch/aarch64/odp/api/abi/cpu_inlines.h \ + arch/aarch64/odp/api/abi/cpu.h endif noinst_HEADERS += arch/aarch64/odp_atomic.h \ arch/aarch64/odp_cpu.h \ @@ -284,12 +286,13 @@ __LIB__libodp_dpdk_la_SOURCES += arch/default/odp_atomic.c \ arch/default/odp_global_time.c \ arch/default/odp_hash_crc32.c \ arch/default/odp_sysinfo_parse.c -odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_inlines.h \ - arch/default/odp/api/abi/cpu_time.h \ +odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_time.h \ arch/default/odp/api/abi/hash_crc32.h if !ODP_ABI_COMPAT odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ arch/default/odp/api/abi/atomic_inlines.h \ + arch/default/odp/api/abi/cpu_generic.h \ + arch/default/odp/api/abi/cpu_inlines.h \ arch/default/odp/api/abi/cpu.h endif noinst_HEADERS += arch/default/odp_atomic.h \ @@ -298,16 +301,16 @@ noinst_HEADERS += arch/default/odp_atomic.h \ endif if ARCH_IS_MIPS64 __LIB__libodp_dpdk_la_SOURCES += arch/default/odp_atomic.c \ - arch/mips64/odp_cpu_cycles.c \ arch/default/odp_global_time.c \ arch/default/odp_hash_crc32.c \ arch/mips64/odp_sysinfo_parse.c -odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_inlines.h \ - arch/default/odp/api/abi/cpu_time.h \ +odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_time.h \ arch/default/odp/api/abi/hash_crc32.h if !ODP_ABI_COMPAT odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ arch/default/odp/api/abi/atomic_inlines.h \ + arch/default/odp/api/abi/cpu_generic.h \ + arch/mips64/odp/api/abi/cpu_inlines.h \ arch/mips64/odp/api/abi/cpu.h endif noinst_HEADERS += arch/default/odp_atomic.h \ @@ -320,12 +323,13 @@ __LIB__libodp_dpdk_la_SOURCES += arch/default/odp_atomic.c \ arch/default/odp_global_time.c \ arch/default/odp_hash_crc32.c \ arch/powerpc/odp_sysinfo_parse.c -odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_inlines.h \ - arch/default/odp/api/abi/cpu_time.h \ +odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_time.h \ arch/default/odp/api/abi/hash_crc32.h if !ODP_ABI_COMPAT odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ arch/default/odp/api/abi/atomic_inlines.h \ + arch/default/odp/api/abi/cpu_generic.h \ + arch/default/odp/api/abi/cpu_inlines.h \ arch/powerpc/odp/api/abi/cpu.h endif noinst_HEADERS += arch/default/odp_atomic.h \ @@ -339,13 +343,13 @@ __LIB__libodp_dpdk_la_SOURCES += arch/default/odp_atomic.c \ arch/x86/odp_global_time.c \ arch/default/odp_hash_crc32.c \ arch/x86/odp_sysinfo_parse.c -odpapiabiarchinclude_HEADERS += arch/x86/odp/api/abi/cpu_inlines.h \ - arch/x86/odp/api/abi/cpu_rdtsc.h \ +odpapiabiarchinclude_HEADERS += arch/x86/odp/api/abi/cpu_rdtsc.h \ arch/x86/odp/api/abi/cpu_time.h \ arch/x86/odp/api/abi/hash_crc32.h if !ODP_ABI_COMPAT odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ arch/default/odp/api/abi/atomic_inlines.h \ + arch/x86/odp/api/abi/cpu_inlines.h \ arch/x86/odp/api/abi/cpu.h endif noinst_HEADERS += arch/x86/cpu_flags.h \ diff --git a/platform/linux-dpdk/include/odp_buffer_internal.h b/platform/linux-dpdk/include/odp_buffer_internal.h index 37e5d1b4f..e47030ea7 100644 --- a/platform/linux-dpdk/include/odp_buffer_internal.h +++ b/platform/linux-dpdk/include/odp_buffer_internal.h @@ -29,7 +29,6 @@ extern "C" { #include <sys/types.h> #include <odp/api/event.h> #include <odp_forward_typedefs_internal.h> -#include <odp_schedule_if.h> #include <stddef.h> /* DPDK */ diff --git a/platform/linux-dpdk/include/odp_config_internal.h b/platform/linux-dpdk/include/odp_config_internal.h index 73d2304c9..6618d413d 100644 --- a/platform/linux-dpdk/include/odp_config_internal.h +++ b/platform/linux-dpdk/include/odp_config_internal.h @@ -124,11 +124,18 @@ extern "C" { CONFIG_PACKET_HEADROOM - \ CONFIG_PACKET_TAILROOM) -/* Maximum number of shared memory blocks. +/* + * Number of shared memory blocks reserved for implementation internal use. + */ +#define CONFIG_INTERNAL_SHM_BLOCKS 20 + +/* + * Maximum number of shared memory blocks. * - * This the number of separate SHM areas that can be reserved concurrently + * This is the number of separate SHM blocks that an application can reserve + * concurrently. */ -#define ODP_CONFIG_SHM_BLOCKS (ODP_CONFIG_POOLS + 48) +#define CONFIG_SHM_BLOCKS 64 /* * Maximum event burst size diff --git a/platform/linux-dpdk/include/odp_eventdev_internal.h b/platform/linux-dpdk/include/odp_eventdev_internal.h index 496a2238f..9e1083fd5 100644 --- a/platform/linux-dpdk/include/odp_eventdev_internal.h +++ b/platform/linux-dpdk/include/odp_eventdev_internal.h @@ -36,6 +36,8 @@ extern "C" { #include <stdint.h> +#define _ODP_SCHED_ID_EVENTDEV (_ODP_SCHED_ID_SCALABLE + 1) + #define RX_ADAPTER_INIT 0 #define RX_ADAPTER_STOPPED 1 #define RX_ADAPTER_RUNNING 2 diff --git a/platform/linux-dpdk/include/odp_queue_basic_internal.h b/platform/linux-dpdk/include/odp_queue_basic_internal.h index c3ddaf334..1e542d973 100644 --- a/platform/linux-dpdk/include/odp_queue_basic_internal.h +++ b/platform/linux-dpdk/include/odp_queue_basic_internal.h @@ -116,6 +116,9 @@ int _odp_sched_queue_deq(uint32_t queue_index, odp_event_t ev[], int num, int update_status); int _odp_sched_queue_empty(uint32_t queue_index); +/* Functions by schedulers */ +int _odp_sched_basic_get_spread(uint32_t queue_index); + #ifdef __cplusplus } #endif diff --git a/platform/linux-dpdk/odp_crypto.c b/platform/linux-dpdk/odp_crypto.c index 8ed561534..13e646230 100644 --- a/platform/linux-dpdk/odp_crypto.c +++ b/platform/linux-dpdk/odp_crypto.c @@ -64,8 +64,11 @@ typedef struct crypto_session_entry_s { struct rte_cryptodev_sym_session *rte_session; struct rte_crypto_sym_xform cipher_xform; struct rte_crypto_sym_xform auth_xform; + struct { + unsigned int cdev_qpairs_shared:1; + unsigned int chained_bufs_ok:1; + } flags; uint16_t cdev_nb_qpairs; - odp_bool_t cdev_qpairs_shared; uint8_t cdev_id; uint8_t cipher_iv_data[MAX_IV_LENGTH]; uint8_t auth_iv_data[MAX_IV_LENGTH]; @@ -312,7 +315,7 @@ int _odp_crypto_init_global(void) mem_size += (MAX_SESSIONS * sizeof(crypto_session_entry_t)); /* Allocate our globally shared memory */ - shm = odp_shm_reserve("_odp_crypto_glb", mem_size, + shm = odp_shm_reserve("_odp_crypto_global", mem_size, ODP_CACHE_LINE_SIZE, 0); if (shm != ODP_SHM_INVALID) { global = odp_shm_addr(shm); @@ -1260,6 +1263,34 @@ check_finish: return -1; } +static int chained_bufs_ok(const odp_crypto_session_param_t *param, + uint8_t cdev_id) +{ + struct rte_cryptodev_info dev_info; + int chained_bufs_ok; + + rte_cryptodev_info_get(cdev_id, &dev_info); + chained_bufs_ok = !!(dev_info.feature_flags & RTE_CRYPTODEV_FF_IN_PLACE_SGL); + + /* + * Some crypto devices do not support chained buffers with all + * algorithms despite advertizing SG support in feature flags. + */ + + if (dev_info.driver_name && + !strcmp(dev_info.driver_name, "crypto_aesni_gcm") && + param->auth_alg == ODP_AUTH_ALG_AES_GMAC) + chained_bufs_ok = 0; + + if (dev_info.driver_name && + !strcmp(dev_info.driver_name, "crypto_openssl") && + (param->cipher_alg == ODP_CIPHER_ALG_AES_GCM || + param->auth_alg == ODP_AUTH_ALG_AES_GMAC)) + chained_bufs_ok = 0; + + return chained_bufs_ok; +} + #if RTE_VERSION < RTE_VERSION_NUM(19, 8, 0, 0) static int crypto_init_key(uint8_t **data, uint16_t *length, odp_crypto_key_t *key, const char *type) @@ -1482,6 +1513,7 @@ int odp_crypto_session_create(const odp_crypto_session_param_t *param, param->auth_alg == ODP_AUTH_ALG_NULL) { rte_session = NULL; cdev_id = ~0; + session->flags.chained_bufs_ok = 1; session->cdev_nb_qpairs = 0; goto out_null; } else if (param->cipher_alg == ODP_CIPHER_ALG_NULL) { @@ -1524,8 +1556,12 @@ int odp_crypto_session_create(const odp_crypto_session_param_t *param, goto err; } + session->flags.chained_bufs_ok = chained_bufs_ok(param, cdev_id); session->cdev_nb_qpairs = global->enabled_crypto_dev_qpairs[cdev_id]; - session->cdev_qpairs_shared = global->enabled_crypto_dev_qpairs_shared[cdev_id]; + if (global->enabled_crypto_dev_qpairs_shared[cdev_id]) + session->flags.cdev_qpairs_shared = 1; + else + session->flags.cdev_qpairs_shared = 0; out_null: session->rte_session = rte_session; session->cdev_id = cdev_id; @@ -1840,6 +1876,50 @@ static void crypto_fill_sym_param(crypto_session_entry_t *session, op->sym->auth.data.length = param->auth_range.length; } +/* + * Attempt to change a multi segment packet to a single segment packet by + * reducing the headroom. Shift packet data toward the start of the first + * segment and trim the tail, hopefully getting rid of the tail segment. + * + * This fails if the packet data does not fit in the first segment with + * the new headroom. A temporary copy to a bigger buffer would be needed + * in that case. + * + * Do nothing for single segment packets. + * + * We assume that odp_crypto_operation() makes no promise to not shift + * packet data within the packet. If that is not the case, the shifting + * done here needs to be undone after the crypto operation. + * + */ +static int linearize_pkt(const crypto_session_entry_t *session, odp_packet_t pkt) +{ + const uint32_t new_headroom = CONFIG_PACKET_HEADROOM; + uint32_t headroom; + uint32_t len; + uint32_t shift; + int rc; + + if (odp_likely(odp_packet_num_segs(pkt) == 1)) + return 0; + if (session->flags.chained_bufs_ok) + return 0; + + headroom = odp_packet_headroom(pkt); + if (odp_unlikely(new_headroom >= headroom)) + return -1; + + len = odp_packet_len(pkt); + shift = headroom - new_headroom; + odp_packet_push_head(pkt, shift); + odp_packet_move_data(pkt, 0, shift, len); + /* We rely on our trunc implementation to not change the handle */ + rc = odp_packet_trunc_tail(&pkt, shift, NULL, NULL); + ODP_ASSERT(rc == 0); + + return odp_packet_num_segs(pkt) != 1; +} + static int odp_crypto_int(odp_packet_t pkt_in, odp_packet_t *pkt_out, @@ -1896,6 +1976,9 @@ int odp_crypto_int(odp_packet_t pkt_in, pkt_in = ODP_PACKET_INVALID; } + if (linearize_pkt(session, out_pkt)) + goto err; + rte_session = session->rte_session; /* NULL rte_session means that it is a NULL-NULL operation. * Just return new packet. */ @@ -1914,7 +1997,7 @@ int odp_crypto_int(odp_packet_t pkt_in, int retry_count = 0; int queue_pair; int rc; - odp_bool_t queue_pairs_shared = session->cdev_qpairs_shared; + odp_bool_t queue_pairs_shared = session->flags.cdev_qpairs_shared; if (odp_unlikely(queue_pairs_shared)) queue_pair = odp_thread_id() % session->cdev_nb_qpairs; diff --git a/platform/linux-dpdk/odp_init.c b/platform/linux-dpdk/odp_init.c index 914a48fad..45f00f8ac 100644 --- a/platform/linux-dpdk/odp_init.c +++ b/platform/linux-dpdk/odp_init.c @@ -74,6 +74,7 @@ static void disable_features(odp_global_data_ro_t *global_ro, if (disable_ipsec && disable_crypto) global_ro->disable.crypto = 1; + global_ro->disable.stash = init_param->not_used.feat.stash; global_ro->disable.traffic_mngr = init_param->not_used.feat.tm; global_ro->disable.compress = init_param->not_used.feat.compress; } diff --git a/platform/linux-dpdk/odp_packet.c b/platform/linux-dpdk/odp_packet.c index f5af3ce72..065a182a8 100644 --- a/platform/linux-dpdk/odp_packet.c +++ b/platform/linux-dpdk/odp_packet.c @@ -9,6 +9,7 @@ #include <odp/api/plat/packet_inlines.h> #include <odp_packet_internal.h> #include <odp_debug_internal.h> +#include <odp_macros_internal.h> #include <odp_chksum_internal.h> #include <odp/api/hints.h> #include <odp/api/byteorder.h> @@ -365,8 +366,8 @@ int odp_packet_extend_head(odp_packet_t *pkt, uint32_t len, void **data_ptr, } /* Expand the original head segment*/ newhead->pkt_len += rte_pktmbuf_headroom(mb); + mb->data_len += rte_pktmbuf_headroom(mb); mb->data_off = 0; - mb->data_len = mb->buf_len; _copy_head_metadata(newhead, mb); mb = newhead; *pkt = (odp_packet_t)newhead; @@ -522,11 +523,18 @@ int odp_packet_trunc_tail(odp_packet_t *pkt, uint32_t len, void **tail_ptr, uint32_t *tailroom) { struct rte_mbuf *mb = pkt_to_mbuf(*pkt); + struct rte_mbuf *last_mb = rte_pktmbuf_lastseg(mb); if (odp_unlikely(len >= odp_packet_len(*pkt))) return -1; - if (rte_pktmbuf_trim(mb, len)) { + /* + * Trim only if the last segment does not become zero length. + */ + if (odp_likely(len < last_mb->data_len)) { + if (odp_unlikely(rte_pktmbuf_trim(mb, len))) + return -1; + } else { struct rte_mbuf *reverse[mb->nb_segs]; struct rte_mbuf *t = mb; int i; @@ -1311,7 +1319,15 @@ static uint32_t packet_sum_crc32c(odp_packet_hdr_t *pkt_hdr, return sum; } -/** Parser helper function for Ethernet packets */ +/* + * In the worst case we look at the Ethernet header, 8 bytes of LLC/SNAP + * header and two VLAN tags in the same packet. + */ +#define PARSE_ETH_BYTES (sizeof(_odp_ethhdr_t) + 8 + 2 * sizeof(_odp_vlanhdr_t)) +/** Parser helper function for Ethernet packets + * + * Requires up to PARSE_ETH_BYTES bytes of contiguous packet data. + */ static inline uint16_t parse_eth(packet_parser_t *prs, const uint8_t **parseptr, uint32_t *offset, uint32_t frame_len) { @@ -1328,7 +1344,7 @@ static inline uint16_t parse_eth(packet_parser_t *prs, const uint8_t **parseptr, eth = (const _odp_ethhdr_t *)*parseptr; /* Detect jumbo frames */ - if (odp_unlikely(frame_len > _ODP_ETH_LEN_MAX)) + if (odp_unlikely(frame_len - *offset > _ODP_ETH_LEN_MAX)) input_flags.jumbo = 1; /* Handle Ethernet broadcast/multicast addresses */ @@ -1386,14 +1402,27 @@ static inline uint16_t parse_eth(packet_parser_t *prs, const uint8_t **parseptr, *parseptr += sizeof(_odp_vlanhdr_t); } + /* + * The packet was too short for what we parsed. We just give up + * entirely without trying to parse what fits in the packet. + */ + if (odp_unlikely(*offset > frame_len)) { + input_flags.all = 0; + input_flags.l2 = 1; + ethtype = 0; + } + error: prs->input_flags.all |= input_flags.all; return ethtype; } +#define PARSE_IPV4_BYTES (0xfU * 4) /* max IPv4 header length with options */ /** * Parser helper function for IPv4 + * + * Requires up to PARSE_IPV4_BYTES bytes of contiguous packet data. */ static inline uint8_t parse_ipv4(packet_parser_t *prs, const uint8_t **parseptr, uint32_t *offset, uint32_t frame_len, @@ -1409,6 +1438,7 @@ static inline uint8_t parse_ipv4(packet_parser_t *prs, const uint8_t **parseptr, if (odp_unlikely(ihl < _ODP_IPV4HDR_IHL_MIN || ver != 4 || + sizeof(*ipv4) > frame_len - *offset || (l3_len > frame_len - *offset))) { prs->flags.ip_err = 1; return 0; @@ -1451,8 +1481,15 @@ static inline uint8_t parse_ipv4(packet_parser_t *prs, const uint8_t **parseptr, return ipv4->proto; } +/* + * Peeks 2 bytes beyond IPv6 base header without length check if there + * are extension headers. + */ +#define PARSE_IPV6_BYTES (sizeof(_odp_ipv6hdr_t) + 2) /** * Parser helper function for IPv6 + * + * Requires at least PARSE_IPV6_BYTES bytes of contiguous packet data. */ static inline uint8_t parse_ipv6(packet_parser_t *prs, const uint8_t **parseptr, uint32_t *offset, uint32_t frame_len, @@ -1467,8 +1504,9 @@ static inline uint8_t parse_ipv6(packet_parser_t *prs, const uint8_t **parseptr, _ODP_IPV6HDR_LEN; /* Basic sanity checks on IPv6 header */ - if ((odp_be_to_cpu_32(ipv6->ver_tc_flow) >> 28) != 6 || - l3_len > frame_len - *offset) { + if (odp_unlikely((odp_be_to_cpu_32(ipv6->ver_tc_flow) >> 28) != 6 || + sizeof(*ipv6) > frame_len - *offset || + l3_len > frame_len - *offset)) { prs->flags.ip_err = 1; return 0; } @@ -1520,8 +1558,11 @@ static inline uint8_t parse_ipv6(packet_parser_t *prs, const uint8_t **parseptr, return ipv6->next_hdr; } +#define PARSE_TCP_BYTES (sizeof(_odp_tcphdr_t)) /** * Parser helper function for TCP + * + * Requires PARSE_TCP_BYTES bytes of contiguous packet data. */ static inline void parse_tcp(packet_parser_t *prs, const uint8_t **parseptr, uint16_t tcp_len, @@ -1547,8 +1588,15 @@ static inline void parse_tcp(packet_parser_t *prs, const uint8_t **parseptr, *parseptr += len; } +/* + * In the worst case we look at the UDP header and 4 bytes of the UDP + * payload (the non-ESP marker to distinguish IKE packets from ESP packets). + */ +#define PARSE_UDP_BYTES (sizeof(_odp_udphdr_t) + 4) /** * Parser helper function for UDP + * + * Requires PARSE_UDP_BYTES bytes of contiguous packet data. */ static inline void parse_udp(packet_parser_t *prs, const uint8_t **parseptr, odp_proto_chksums_t chksums, @@ -1593,8 +1641,11 @@ static inline void parse_udp(packet_parser_t *prs, const uint8_t **parseptr, *parseptr += sizeof(_odp_udphdr_t); } +#define PARSE_SCTP_BYTES (sizeof(_odp_sctphdr_t)) /** * Parser helper function for SCTP + * + * Requires PARSE_SCTP_BYTES bytes of contiguous packet data. */ static inline void parse_sctp(packet_parser_t *prs, const uint8_t **parseptr, uint16_t sctp_len, @@ -1621,6 +1672,10 @@ static inline void parse_sctp(packet_parser_t *prs, const uint8_t **parseptr, *parseptr += sizeof(_odp_sctphdr_t); } +#define MAX3(a, b, c) (MAX(MAX((a), (b)), (c))) +#define PARSE_L3_L4_BYTES (MAX(PARSE_IPV4_BYTES, PARSE_IPV6_BYTES) + \ + MAX3(PARSE_TCP_BYTES, PARSE_UDP_BYTES, PARSE_SCTP_BYTES)) +/* Requires up to PARSE_L3_L4_BYTES bytes of contiguous packet data. */ static inline int packet_parse_common_l3_l4(packet_parser_t *prs, const uint8_t *parseptr, uint32_t offset, @@ -1984,6 +2039,8 @@ static int packet_l4_chksum(odp_packet_hdr_t *pkt_hdr, if (chksums.chksum.sctp && pkt_hdr->p.input_flags.sctp && !pkt_hdr->p.input_flags.ipfrag) { + uint32_t seg_len = 0; + _odp_sctphdr_t hdr_copy; uint32_t sum = ~packet_sum_crc32c(pkt_hdr, pkt_hdr->p.l4_offset + _ODP_SCTPHDR_LEN, @@ -1993,8 +2050,14 @@ static int packet_l4_chksum(odp_packet_hdr_t *pkt_hdr, l4_part_sum); _odp_sctphdr_t *sctp = odp_packet_offset(packet_handle(pkt_hdr), pkt_hdr->p.l4_offset, - NULL, NULL); + &seg_len, NULL); + if (odp_unlikely(seg_len < sizeof(*sctp))) { + odp_packet_t pkt = packet_handle(pkt_hdr); + sctp = &hdr_copy; + odp_packet_copy_to_mem(pkt, pkt_hdr->p.l4_offset, + sizeof(*sctp), sctp); + } pkt_hdr->p.input_flags.l4_chksum_done = 1; if (sum != sctp->chksum) { pkt_hdr->p.flags.l4_chksum_err = 1; @@ -2051,12 +2114,14 @@ int odp_packet_parse(odp_packet_t pkt, uint32_t offset, odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); const uint8_t *data; uint32_t seg_len; - uint32_t len = odp_packet_len(pkt); + uint32_t packet_len = odp_packet_len(pkt); odp_proto_t proto = param->proto; odp_proto_layer_t layer = param->last_layer; int ret; uint16_t ethtype; uint64_t l4_part_sum = 0; + const uint32_t min_seglen = PARSE_ETH_BYTES + PARSE_L3_L4_BYTES; + uint8_t buf[min_seglen]; if (proto == ODP_PROTO_NONE || layer == ODP_PROTO_LAYER_NONE) return -1; @@ -2066,6 +2131,20 @@ int odp_packet_parse(odp_packet_t pkt, uint32_t offset, if (data == NULL) return -1; + /* + * We must not have a packet segment boundary within the parsed + * packet data range. Copy enough data to a temporary buffer for + * parsing if necessary. + */ + if (odp_unlikely(pkt_hdr->buf_hdr.mb.nb_segs > 1) && + odp_unlikely(seg_len < min_seglen)) { + seg_len = min_seglen; + if (seg_len > packet_len - offset) + seg_len = packet_len - offset; + odp_packet_copy_to_mem(pkt, offset, seg_len, buf); + data = buf; + } + /* Reset parser flags, keep other flags */ packet_parse_reset(pkt_hdr, 0); @@ -2073,7 +2152,7 @@ int odp_packet_parse(odp_packet_t pkt, uint32_t offset, /* Assume valid L2 header, no CRC/FCS check in SW */ pkt_hdr->p.l2_offset = offset; - ethtype = parse_eth(&pkt_hdr->p, &data, &offset, len); + ethtype = parse_eth(&pkt_hdr->p, &data, &offset, packet_len); } else if (proto == ODP_PROTO_IPV4) { ethtype = _ODP_ETHTYPE_IPV4; } else if (proto == ODP_PROTO_IPV6) { @@ -2083,7 +2162,7 @@ int odp_packet_parse(odp_packet_t pkt, uint32_t offset, } ret = packet_parse_common_l3_l4(&pkt_hdr->p, data, offset, - len, seg_len, + packet_len, seg_len, layer, ethtype, param->chksums, &l4_part_sum); @@ -2404,6 +2483,13 @@ odp_packet_reass_status_t odp_packet_reass_status(odp_packet_t pkt) return ODP_PACKET_REASS_NONE; } +int odp_packet_reass_info(odp_packet_t pkt, odp_packet_reass_info_t *info) +{ + (void)pkt; + (void)info; + return -1; +} + int odp_packet_reass_partial_state(odp_packet_t pkt, odp_packet_t frags[], odp_packet_reass_partial_state_t *res) { diff --git a/platform/linux-dpdk/odp_pool.c b/platform/linux-dpdk/odp_pool.c index 4b5eb3206..4417ca4ee 100644 --- a/platform/linux-dpdk/odp_pool.c +++ b/platform/linux-dpdk/odp_pool.c @@ -153,7 +153,7 @@ int _odp_pool_init_global(void) uint32_t i; odp_shm_t shm; - shm = odp_shm_reserve("_odp_pool_glb", sizeof(pool_global_t), + shm = odp_shm_reserve("_odp_pool_global", sizeof(pool_global_t), ODP_CACHE_LINE_SIZE, 0); _odp_pool_glb = odp_shm_addr(shm); diff --git a/platform/linux-dpdk/odp_queue_basic.c b/platform/linux-dpdk/odp_queue_basic.c index 7145fb2ce..33e0ba6bc 100644 --- a/platform/linux-dpdk/odp_queue_basic.c +++ b/platform/linux-dpdk/odp_queue_basic.c @@ -130,7 +130,7 @@ static int queue_init_global(void) _odp_queue_inline_offset.context = offsetof(queue_entry_t, s.param.context); - shm = odp_shm_reserve("_odp_queue_gbl", + shm = odp_shm_reserve("_odp_queue_basic_global", sizeof(queue_global_t), sizeof(queue_entry_t), 0); @@ -634,7 +634,8 @@ static void queue_print(odp_queue_t handle) odp_pktio_info_t pktio_info; queue_entry_t *queue; uint32_t queue_id; - int status; + int status, prio; + int max_prio = odp_schedule_max_prio(); queue_id = queue_to_index(handle); @@ -658,7 +659,7 @@ static void queue_print(odp_queue_t handle) ODP_PRINT("\nQueue info\n"); ODP_PRINT("----------\n"); ODP_PRINT(" handle %p\n", queue->s.handle); - ODP_PRINT(" index %" PRIu32 "\n", queue->s.index); + ODP_PRINT(" index %" PRIu32 "\n", queue_id); ODP_PRINT(" name %s\n", queue->s.name); ODP_PRINT(" enq mode %s\n", queue->s.param.enq_mode == ODP_QUEUE_OP_MT ? "ODP_QUEUE_OP_MT" : @@ -686,8 +687,11 @@ static void queue_print(odp_queue_t handle) "ODP_SCHED_SYNC_ATOMIC" : (queue->s.param.sched.sync == ODP_SCHED_SYNC_ORDERED ? "ODP_SCHED_SYNC_ORDERED" : "unknown"))); - ODP_PRINT(" priority %d\n", queue->s.param.sched.prio); - ODP_PRINT(" group %d\n", queue->s.param.sched.group); + prio = queue->s.param.sched.prio; + ODP_PRINT(" priority %i (%i in API)\n", max_prio - prio, prio); + ODP_PRINT(" group %i\n", queue->s.param.sched.group); + if (_odp_sched_id == _ODP_SCHED_ID_BASIC) + ODP_PRINT(" spread %i\n", _odp_sched_basic_get_spread(queue_id)); } if (queue->s.pktin.pktio != ODP_PKTIO_INVALID) { if (!odp_pktio_info(queue->s.pktin.pktio, &pktio_info)) @@ -744,11 +748,18 @@ static void queue_print_all(void) char type_c, enq_c, deq_c, order_c, sync_c; const int col_width = 24; int prio = 0; + int spr = 0; odp_schedule_sync_t sync = ODP_SCHED_SYNC_PARALLEL; + odp_schedule_group_t grp = ODP_SCHED_GROUP_INVALID; ODP_PRINT("\nList of all queues\n"); ODP_PRINT("------------------\n"); - ODP_PRINT(" idx %-*s type stat blk enq deq ord len max_len sync prio\n", col_width, "name"); + ODP_PRINT(" idx %-*s type stat blk enq deq ord len max_len sync prio grp", + col_width, "name"); + if (_odp_sched_id == _ODP_SCHED_ID_BASIC) + ODP_PRINT(" spr\n"); + else + ODP_PRINT("\n"); for (i = 0; i < CONFIG_MAX_QUEUES; i++) { queue_entry_t *queue = qentry_from_index(i); @@ -777,7 +788,10 @@ static void queue_print_all(void) len = ring_st_length(queue->s.ring_st); max_len = ring_st_max_length(queue->s.ring_st); prio = queue->s.param.sched.prio; + grp = queue->s.param.sched.group; sync = queue->s.param.sched.sync; + if (_odp_sched_id == _ODP_SCHED_ID_BASIC) + spr = _odp_sched_basic_get_spread(index); } else { len = ring_mpmc_length(queue->s.ring_mpmc); max_len = ring_mpmc_max_length(queue->s.ring_mpmc); @@ -811,7 +825,13 @@ static void queue_print_all(void) if (type == ODP_QUEUE_TYPE_SCHED) { sync_c = (sync == ODP_SCHED_SYNC_PARALLEL) ? 'P' : ((sync == ODP_SCHED_SYNC_ATOMIC) ? 'A' : 'O'); - ODP_PRINT(" %c %4i", sync_c, prio); + /* Print prio level matching odp_schedule_print() output */ + prio = odp_schedule_max_prio() - prio; + + ODP_PRINT(" %c %4i %3i", sync_c, prio, grp); + + if (_odp_sched_id == _ODP_SCHED_ID_BASIC) + ODP_PRINT(" %3i", spr); } ODP_PRINT("\n"); diff --git a/platform/linux-dpdk/odp_queue_eventdev.c b/platform/linux-dpdk/odp_queue_eventdev.c index 96baffa6f..0960c456e 100644 --- a/platform/linux-dpdk/odp_queue_eventdev.c +++ b/platform/linux-dpdk/odp_queue_eventdev.c @@ -528,7 +528,7 @@ static int queue_init_global(void) _odp_queue_inline_offset.context = offsetof(queue_entry_t, s.param.context); - shm = odp_shm_reserve("_odp_eventdev_gbl", + shm = odp_shm_reserve("_odp_queue_eventdev_global", sizeof(eventdev_global_t), ODP_CACHE_LINE_SIZE, 0); diff --git a/platform/linux-dpdk/odp_schedule_if.c b/platform/linux-dpdk/odp_schedule_if.c index 29d38b1fd..21f5fc85b 100644 --- a/platform/linux-dpdk/odp_schedule_if.c +++ b/platform/linux-dpdk/odp_schedule_if.c @@ -1,4 +1,5 @@ /* Copyright (c) 2018, Linaro Limited + * Copyright (c) 2021, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -11,6 +12,9 @@ #include <odp_debug_internal.h> #include <odp_global_data.h> +/* Required for _ODP_SCHED_ID_EVENTDEV */ +#include <odp_eventdev_internal.h> + #include <stdlib.h> #include <string.h> @@ -25,6 +29,7 @@ extern const schedule_api_t _odp_schedule_eventdev_api; const schedule_fn_t *_odp_sched_fn; const schedule_api_t *_odp_sched_api; +int _odp_sched_id; uint64_t odp_schedule_wait_time(uint64_t ns) { @@ -216,12 +221,15 @@ int _odp_schedule_init_global(void) ODP_PRINT("Using scheduler '%s'\n", sched); if (!strcmp(sched, "basic")) { + _odp_sched_id = _ODP_SCHED_ID_BASIC; _odp_sched_fn = &_odp_schedule_basic_fn; _odp_sched_api = &_odp_schedule_basic_api; } else if (!strcmp(sched, "sp")) { + _odp_sched_id = _ODP_SCHED_ID_SP; _odp_sched_fn = &_odp_schedule_sp_fn; _odp_sched_api = &_odp_schedule_sp_api; } else if (!strcmp(sched, "eventdev")) { + _odp_sched_id = _ODP_SCHED_ID_EVENTDEV; _odp_sched_fn = &_odp_schedule_eventdev_fn; _odp_sched_api = &_odp_schedule_eventdev_api; } else { diff --git a/platform/linux-dpdk/odp_shared_memory.c b/platform/linux-dpdk/odp_shared_memory.c index 4b1432a2f..645bb8847 100644 --- a/platform/linux-dpdk/odp_shared_memory.c +++ b/platform/linux-dpdk/odp_shared_memory.c @@ -1,4 +1,5 @@ /* Copyright (c) 2017-2018, Linaro Limited + * Copyright (c) 2021, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -25,9 +26,7 @@ #define SHM_MAX_ALIGN (0x80000000) #define SHM_BLOCK_NAME "%" PRIu64 "-%d-%s" - -ODP_STATIC_ASSERT(ODP_CONFIG_SHM_BLOCKS >= ODP_CONFIG_POOLS, - "ODP_CONFIG_SHM_BLOCKS < ODP_CONFIG_POOLS"); +#define SHM_MAX_NB_BLOCKS (CONFIG_INTERNAL_SHM_BLOCKS + CONFIG_SHM_BLOCKS) ODP_STATIC_ASSERT(ODP_SHM_NAME_LEN >= RTE_MEMZONE_NAMESIZE, "ODP_SHM_NAME_LEN < RTE_MEMZONE_NAMESIZE"); @@ -68,7 +67,7 @@ typedef struct { */ typedef struct { odp_spinlock_t lock; - shm_block_t block[ODP_CONFIG_SHM_BLOCKS]; + shm_block_t block[SHM_MAX_NB_BLOCKS]; } shm_table_t; static shm_table_t *shm_tbl; @@ -80,7 +79,7 @@ static odp_bool_t mz_name_used(const char *name) { int idx; - for (idx = 0; idx < ODP_CONFIG_SHM_BLOCKS; idx++) { + for (idx = 0; idx < SHM_MAX_NB_BLOCKS; idx++) { if (shm_tbl->block[idx].mz && strncmp(name, shm_tbl->block[idx].mz->name, RTE_MEMZONE_NAMESIZE) == 0) @@ -124,7 +123,7 @@ static int find_free_block(void) { int idx; - for (idx = 0; idx < ODP_CONFIG_SHM_BLOCKS; idx++) { + for (idx = 0; idx < SHM_MAX_NB_BLOCKS; idx++) { if (shm_tbl->block[idx].mz == NULL) return idx; } @@ -145,7 +144,7 @@ static inline odp_bool_t handle_is_valid(odp_shm_t shm) { int idx = handle_to_idx(shm); - if (idx < 0 || idx >= ODP_CONFIG_SHM_BLOCKS || + if (idx < 0 || idx >= SHM_MAX_NB_BLOCKS || shm_tbl->block[idx].mz == NULL) { ODP_ERR("Invalid odp_shm_t handle: %" PRIu64 "\n", odp_shm_to_u64(shm)); @@ -199,7 +198,7 @@ int _odp_shm_term_global(void) } /* Cleanup possibly non freed memory (and complain a bit) */ - for (idx = 0; idx < ODP_CONFIG_SHM_BLOCKS; idx++) { + for (idx = 0; idx < SHM_MAX_NB_BLOCKS; idx++) { block = &shm_tbl->block[idx]; if (block->mz) { ODP_ERR("block '%s' was never freed (cleaning up...)\n", @@ -224,7 +223,7 @@ int odp_shm_capability(odp_shm_capability_t *capa) { memset(capa, 0, sizeof(odp_shm_capability_t)); - capa->max_blocks = ODP_CONFIG_SHM_BLOCKS; + capa->max_blocks = CONFIG_SHM_BLOCKS; capa->max_size = 0; capa->max_align = SHM_MAX_ALIGN; @@ -357,7 +356,7 @@ odp_shm_t odp_shm_lookup(const char *name) odp_spinlock_lock(&shm_tbl->lock); - for (idx = 0; idx < ODP_CONFIG_SHM_BLOCKS; idx++) { + for (idx = 0; idx < SHM_MAX_NB_BLOCKS; idx++) { if (shm_tbl->block[idx].mz && strncmp(name, shm_tbl->block[idx].name, ODP_SHM_NAME_LEN) == 0) { @@ -425,7 +424,7 @@ void odp_shm_print_all(void) ODP_PRINT("\nShared memory blocks\n--------------------\n"); - for (idx = 0; idx < ODP_CONFIG_SHM_BLOCKS; idx++) { + for (idx = 0; idx < SHM_MAX_NB_BLOCKS; idx++) { block = &shm_tbl->block[idx]; if (block->mz == NULL) continue; diff --git a/platform/linux-dpdk/odp_system_info.c b/platform/linux-dpdk/odp_system_info.c index af07f387a..30fcc140b 100644 --- a/platform/linux-dpdk/odp_system_info.c +++ b/platform/linux-dpdk/odp_system_info.c @@ -515,7 +515,7 @@ void odp_sys_config_print(void) ODP_PRINT("CONFIG_PACKET_SEG_SIZE: %i\n", CONFIG_PACKET_SEG_SIZE); ODP_PRINT("CONFIG_PACKET_SEG_LEN_MIN: %i\n", CONFIG_PACKET_SEG_LEN_MIN); ODP_PRINT("CONFIG_PACKET_MAX_SEG_LEN: %i\n", CONFIG_PACKET_MAX_SEG_LEN); - ODP_PRINT("ODP_CONFIG_SHM_BLOCKS: %i\n", ODP_CONFIG_SHM_BLOCKS); + ODP_PRINT("CONFIG_SHM_BLOCKS: %i\n", CONFIG_SHM_BLOCKS); ODP_PRINT("CONFIG_BURST_SIZE: %i\n", CONFIG_BURST_SIZE); ODP_PRINT("CONFIG_POOL_MAX_NUM: %i\n", CONFIG_POOL_MAX_NUM); ODP_PRINT("\n"); diff --git a/platform/linux-dpdk/odp_thread.c b/platform/linux-dpdk/odp_thread.c index 59394b3e4..7ab41cf72 100644 --- a/platform/linux-dpdk/odp_thread.c +++ b/platform/linux-dpdk/odp_thread.c @@ -76,7 +76,7 @@ int _odp_thread_init_global(void) if (num_max > ODP_THREAD_COUNT_MAX) num_max = ODP_THREAD_COUNT_MAX; - shm = odp_shm_reserve("_odp_thread_globals", + shm = odp_shm_reserve("_odp_thread_global", sizeof(thread_globals_t), ODP_CACHE_LINE_SIZE, 0); diff --git a/platform/linux-generic/Makefile.am b/platform/linux-generic/Makefile.am index 8c75e5ec0..e763c0abc 100644 --- a/platform/linux-generic/Makefile.am +++ b/platform/linux-generic/Makefile.am @@ -275,12 +275,13 @@ __LIB__libodp_linux_la_SOURCES += arch/default/odp_atomic.c \ arch/default/odp_global_time.c \ arch/default/odp_hash_crc32.c \ arch/arm/odp_sysinfo_parse.c -odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_inlines.h \ - arch/default/odp/api/abi/cpu_time.h \ +odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_time.h \ arch/default/odp/api/abi/hash_crc32.h if !ODP_ABI_COMPAT odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ arch/default/odp/api/abi/atomic_inlines.h \ + arch/default/odp/api/abi/cpu_generic.h \ + arch/arm/odp/api/abi/cpu_inlines.h \ arch/arm/odp/api/abi/cpu.h endif noinst_HEADERS += arch/arm/odp_atomic.h \ @@ -297,14 +298,15 @@ __LIB__libodp_linux_la_SOURCES += arch/aarch64/odp_atomic.c \ arch/aarch64/odp_global_time.c \ arch/default/odp_hash_crc32.c \ arch/aarch64/odp_sysinfo_parse.c -odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_inlines.h \ - arch/aarch64/odp/api/abi/cpu_time.h \ +odpapiabiarchinclude_HEADERS += arch/aarch64/odp/api/abi/cpu_time.h \ arch/aarch64/odp/api/abi/hash_crc32.h if !ODP_ABI_COMPAT odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ arch/aarch64/odp/api/abi/atomic_inlines.h \ arch/aarch64/odp/api/abi/atomic.h \ - arch/aarch64/odp/api/abi/cpu.h + arch/default/odp/api/abi/cpu_generic.h \ + arch/aarch64/odp/api/abi/cpu_inlines.h \ + arch/aarch64/odp/api/abi/cpu.h endif noinst_HEADERS += arch/aarch64/odp_atomic.h \ arch/aarch64/odp_cpu.h \ @@ -317,12 +319,13 @@ __LIB__libodp_linux_la_SOURCES += arch/default/odp_atomic.c \ arch/default/odp_global_time.c \ arch/default/odp_hash_crc32.c \ arch/default/odp_sysinfo_parse.c -odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_inlines.h \ - arch/default/odp/api/abi/cpu_time.h \ +odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_time.h \ arch/default/odp/api/abi/hash_crc32.h if !ODP_ABI_COMPAT odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ arch/default/odp/api/abi/atomic_inlines.h \ + arch/default/odp/api/abi/cpu_generic.h \ + arch/default/odp/api/abi/cpu_inlines.h \ arch/default/odp/api/abi/cpu.h endif noinst_HEADERS += arch/default/odp_atomic.h \ @@ -331,16 +334,16 @@ noinst_HEADERS += arch/default/odp_atomic.h \ endif if ARCH_IS_MIPS64 __LIB__libodp_linux_la_SOURCES += arch/default/odp_atomic.c \ - arch/mips64/odp_cpu_cycles.c \ arch/default/odp_global_time.c \ arch/default/odp_hash_crc32.c \ arch/mips64/odp_sysinfo_parse.c -odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_inlines.h \ - arch/default/odp/api/abi/cpu_time.h \ +odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_time.h \ arch/default/odp/api/abi/hash_crc32.h if !ODP_ABI_COMPAT odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ arch/default/odp/api/abi/atomic_inlines.h \ + arch/default/odp/api/abi/cpu_generic.h \ + arch/mips64/odp/api/abi/cpu_inlines.h \ arch/mips64/odp/api/abi/cpu.h endif noinst_HEADERS += arch/default/odp_atomic.h \ @@ -353,12 +356,13 @@ __LIB__libodp_linux_la_SOURCES += arch/default/odp_atomic.c \ arch/default/odp_global_time.c \ arch/default/odp_hash_crc32.c \ arch/powerpc/odp_sysinfo_parse.c -odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_inlines.h \ - arch/default/odp/api/abi/cpu_time.h \ +odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_time.h \ arch/default/odp/api/abi/hash_crc32.h if !ODP_ABI_COMPAT odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ arch/default/odp/api/abi/atomic_inlines.h \ + arch/default/odp/api/abi/cpu_generic.h \ + arch/default/odp/api/abi/cpu_inlines.h \ arch/powerpc/odp/api/abi/cpu.h endif noinst_HEADERS += arch/default/odp_atomic.h \ @@ -372,13 +376,13 @@ __LIB__libodp_linux_la_SOURCES += arch/default/odp_atomic.c \ arch/x86/odp_global_time.c \ arch/default/odp_hash_crc32.c \ arch/x86/odp_sysinfo_parse.c -odpapiabiarchinclude_HEADERS += arch/x86/odp/api/abi/cpu_inlines.h \ - arch/x86/odp/api/abi/cpu_rdtsc.h \ +odpapiabiarchinclude_HEADERS += arch/x86/odp/api/abi/cpu_rdtsc.h \ arch/x86/odp/api/abi/cpu_time.h \ arch/x86/odp/api/abi/hash_crc32.h if !ODP_ABI_COMPAT odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ arch/default/odp/api/abi/atomic_inlines.h \ + arch/x86/odp/api/abi/cpu_inlines.h \ arch/x86/odp/api/abi/cpu.h endif noinst_HEADERS += arch/x86/cpu_flags.h \ diff --git a/platform/linux-generic/arch/aarch64/odp/api/abi/atomic.h b/platform/linux-generic/arch/aarch64/odp/api/abi/atomic.h index d1dbf36b8..14cca3ca0 100644 --- a/platform/linux-generic/arch/aarch64/odp/api/abi/atomic.h +++ b/platform/linux-generic/arch/aarch64/odp/api/abi/atomic.h @@ -9,4 +9,4 @@ #endif #include <odp/api/abi-default/atomic.h> - +#include <odp/api/plat/atomic_inlines.h> diff --git a/platform/linux-generic/arch/aarch64/odp/api/abi/cpu.h b/platform/linux-generic/arch/aarch64/odp/api/abi/cpu.h index 97a2861c5..825ff19d4 100644 --- a/platform/linux-generic/arch/aarch64/odp/api/abi/cpu.h +++ b/platform/linux-generic/arch/aarch64/odp/api/abi/cpu.h @@ -17,15 +17,8 @@ extern "C" { #define ODP_CACHE_LINE_SIZE _ODP_CACHE_LINE_SIZE #endif -static inline void odp_cpu_pause(void) -{ - /* YIELD hints the CPU to switch to another thread if possible - * and executes as a NOP otherwise. - * ISB flushes the pipeline, then restarts. This is guaranteed to - * stall the CPU a number of cycles. - */ - __asm volatile("isb" ::: "memory"); -} +/* Inlined functions for non-ABI compat mode */ +#include <odp/api/plat/cpu_inlines.h> #ifdef __cplusplus } diff --git a/platform/linux-generic/arch/aarch64/odp/api/abi/cpu_inlines.h b/platform/linux-generic/arch/aarch64/odp/api/abi/cpu_inlines.h new file mode 100644 index 000000000..bf44806a0 --- /dev/null +++ b/platform/linux-generic/arch/aarch64/odp/api/abi/cpu_inlines.h @@ -0,0 +1,32 @@ +/* Copyright (c) 2016-2018, Linaro Limited + * Copyright (c) 2021, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_ARCH_CPU_INLINES_H_ +#define ODP_ARCH_CPU_INLINES_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +static inline void _odp_cpu_pause(void) +{ + /* YIELD hints the CPU to switch to another thread if possible + * and executes as a NOP otherwise. + * ISB flushes the pipeline, then restarts. This is guaranteed to + * stall the CPU a number of cycles. + */ + __asm volatile("isb" ::: "memory"); +} + +/* Use generic implementations for the rest of the functions */ +#include <odp/api/abi/cpu_generic.h> + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/arch/arm/odp/api/abi/cpu.h b/platform/linux-generic/arch/arm/odp/api/abi/cpu.h index 0ab5b8e14..9224af9a0 100644 --- a/platform/linux-generic/arch/arm/odp/api/abi/cpu.h +++ b/platform/linux-generic/arch/arm/odp/api/abi/cpu.h @@ -13,15 +13,8 @@ extern "C" { #define ODP_CACHE_LINE_SIZE 64 -static inline void odp_cpu_pause(void) -{ - /* YIELD hints the CPU to switch to another thread if possible - * and executes as a NOP otherwise. - * ISB flushes the pipeline, then restarts. This is guaranteed to - * stall the CPU a number of cycles. - */ - __asm volatile("isb" ::: "memory"); -} +/* Inlined functions for non-ABI compat mode */ +#include <odp/api/plat/cpu_inlines.h> #ifdef __cplusplus } diff --git a/platform/linux-generic/arch/arm/odp/api/abi/cpu_inlines.h b/platform/linux-generic/arch/arm/odp/api/abi/cpu_inlines.h new file mode 100644 index 000000000..bf44806a0 --- /dev/null +++ b/platform/linux-generic/arch/arm/odp/api/abi/cpu_inlines.h @@ -0,0 +1,32 @@ +/* Copyright (c) 2016-2018, Linaro Limited + * Copyright (c) 2021, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_ARCH_CPU_INLINES_H_ +#define ODP_ARCH_CPU_INLINES_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +static inline void _odp_cpu_pause(void) +{ + /* YIELD hints the CPU to switch to another thread if possible + * and executes as a NOP otherwise. + * ISB flushes the pipeline, then restarts. This is guaranteed to + * stall the CPU a number of cycles. + */ + __asm volatile("isb" ::: "memory"); +} + +/* Use generic implementations for the rest of the functions */ +#include <odp/api/abi/cpu_generic.h> + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/arch/default/odp/api/abi/cpu.h b/platform/linux-generic/arch/default/odp/api/abi/cpu.h index 8f64790c3..e09efdfcf 100644 --- a/platform/linux-generic/arch/default/odp/api/abi/cpu.h +++ b/platform/linux-generic/arch/default/odp/api/abi/cpu.h @@ -13,9 +13,8 @@ extern "C" { #define ODP_CACHE_LINE_SIZE 64 -static inline void odp_cpu_pause(void) -{ -} +/* Inlined functions for non-ABI compat mode */ +#include <odp/api/plat/cpu_inlines.h> #ifdef __cplusplus } diff --git a/platform/linux-generic/arch/default/odp/api/abi/cpu_generic.h b/platform/linux-generic/arch/default/odp/api/abi/cpu_generic.h new file mode 100644 index 000000000..b75e65717 --- /dev/null +++ b/platform/linux-generic/arch/default/odp/api/abi/cpu_generic.h @@ -0,0 +1,34 @@ +/* Copyright (c) 2015-2018, Linaro Limited + * Copyright (c) 2021, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_API_ABI_CPU_GENERIC_H_ +#define ODP_API_ABI_CPU_GENERIC_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <stdint.h> + +uint64_t _odp_cpu_cycles(void); +int _odp_cpu_cycles_init_global(void); + +static inline uint64_t _odp_cpu_cycles_max(void) +{ + return UINT64_MAX; +} + +static inline uint64_t _odp_cpu_cycles_resolution(void) +{ + return 1; +} + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/arch/default/odp/api/abi/cpu_inlines.h b/platform/linux-generic/arch/default/odp/api/abi/cpu_inlines.h index f9c2bae9d..54aeae946 100644 --- a/platform/linux-generic/arch/default/odp/api/abi/cpu_inlines.h +++ b/platform/linux-generic/arch/default/odp/api/abi/cpu_inlines.h @@ -7,9 +7,18 @@ #ifndef ODP_ARCH_CPU_INLINES_H_ #define ODP_ARCH_CPU_INLINES_H_ -#undef odp_cpu_pause -#undef odp_cpu_cycles -#undef odp_cpu_cycles_max -#undef odp_cpu_cycles_resolution +#ifdef __cplusplus +extern "C" { +#endif + +static inline void _odp_cpu_pause(void) +{ +} + +#include <odp/api/abi/cpu_generic.h> + +#ifdef __cplusplus +} +#endif #endif diff --git a/platform/linux-generic/arch/default/odp_cpu_cycles.c b/platform/linux-generic/arch/default/odp_cpu_cycles.c index 542a68dbe..5d0d5db1d 100644 --- a/platform/linux-generic/arch/default/odp_cpu_cycles.c +++ b/platform/linux-generic/arch/default/odp_cpu_cycles.c @@ -1,4 +1,5 @@ /* Copyright (c) 2015-2018, Linaro Limited + * Copyright (c) 2021, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -6,16 +7,19 @@ #include <odp_posix_extensions.h> +#include <stdint.h> #include <stdlib.h> #include <time.h> -#include <odp/api/cpu.h> #include <odp_debug_internal.h> +#include <odp_global_data.h> #include <odp_init_internal.h> #define GIGA 1000000000 -uint64_t odp_cpu_cycles(void) +#include <odp/api/abi/cpu_generic.h> + +uint64_t _odp_cpu_cycles(void) { struct timespec time; uint64_t sec, ns, hz, cycles; @@ -26,7 +30,8 @@ uint64_t odp_cpu_cycles(void) if (ret != 0) ODP_ABORT("clock_gettime failed\n"); - hz = odp_cpu_hz_max(); + hz = odp_global_ro.system_info.cpu_hz_max[0]; + sec = (uint64_t)time.tv_sec; ns = (uint64_t)time.tv_nsec; @@ -36,16 +41,6 @@ uint64_t odp_cpu_cycles(void) return cycles; } -uint64_t odp_cpu_cycles_max(void) -{ - return UINT64_MAX; -} - -uint64_t odp_cpu_cycles_resolution(void) -{ - return 1; -} - int _odp_cpu_cycles_init_global(void) { return 0; diff --git a/platform/linux-generic/arch/mips64/odp/api/abi/cpu.h b/platform/linux-generic/arch/mips64/odp/api/abi/cpu.h index 45d95e9e3..a6ce523d0 100644 --- a/platform/linux-generic/arch/mips64/odp/api/abi/cpu.h +++ b/platform/linux-generic/arch/mips64/odp/api/abi/cpu.h @@ -17,13 +17,8 @@ extern "C" { #error Please add support for your arch in cpu_arch.h #endif -static inline void odp_cpu_pause(void) -{ - __asm__ __volatile__ ("nop"); - __asm__ __volatile__ ("nop"); - __asm__ __volatile__ ("nop"); - __asm__ __volatile__ ("nop"); -} +/* Inlined functions for non-ABI compat mode */ +#include <odp/api/plat/cpu_inlines.h> #ifdef __cplusplus } diff --git a/platform/linux-generic/arch/mips64/odp/api/abi/cpu_inlines.h b/platform/linux-generic/arch/mips64/odp/api/abi/cpu_inlines.h new file mode 100644 index 000000000..d3a424432 --- /dev/null +++ b/platform/linux-generic/arch/mips64/odp/api/abi/cpu_inlines.h @@ -0,0 +1,56 @@ +/* Copyright (c) 2016-2018, Linaro Limited + * Copyright (c) 2021, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_ARCH_CPU_INLINES_H_ +#define ODP_ARCH_CPU_INLINES_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <stdint.h> + +static inline void _odp_cpu_pause(void) +{ + __asm__ __volatile__ ("nop"); + __asm__ __volatile__ ("nop"); + __asm__ __volatile__ ("nop"); + __asm__ __volatile__ ("nop"); +} + +uint64_t _odp_cpu_cycles(void) +{ + #define CVMX_TMP_STR(x) CVMX_TMP_STR2(x) + #define CVMX_TMP_STR2(x) #x + uint64_t cycle; + + __asm__ __volatile__ ("rdhwr %[rt],$" CVMX_TMP_STR(31) : + [rt] "=d" (cycle) : : "memory"); + + return cycle; +} + +uint64_t _odp_cpu_cycles_max(void) +{ + return UINT64_MAX; +} + +uint64_t _odp_cpu_cycles_resolution(void) +{ + return 1; +} + +int _odp_cpu_cycles_init_global(void) +{ + return 0; +} + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/arch/mips64/odp_cpu_cycles.c b/platform/linux-generic/arch/mips64/odp_cpu_cycles.c deleted file mode 100644 index 6423af76d..000000000 --- a/platform/linux-generic/arch/mips64/odp_cpu_cycles.c +++ /dev/null @@ -1,37 +0,0 @@ -/* Copyright (c) 2015-2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -#include <odp/api/cpu.h> -#include <odp/api/hints.h> -#include <odp/api/system_info.h> -#include <odp_init_internal.h> - -uint64_t odp_cpu_cycles(void) -{ - #define CVMX_TMP_STR(x) CVMX_TMP_STR2(x) - #define CVMX_TMP_STR2(x) #x - uint64_t cycle; - - __asm__ __volatile__ ("rdhwr %[rt],$" CVMX_TMP_STR(31) : - [rt] "=d" (cycle) : : "memory"); - - return cycle; -} - -uint64_t odp_cpu_cycles_max(void) -{ - return UINT64_MAX; -} - -uint64_t odp_cpu_cycles_resolution(void) -{ - return 1; -} - -int _odp_cpu_cycles_init_global(void) -{ - return 0; -} diff --git a/platform/linux-generic/arch/powerpc/odp/api/abi/cpu.h b/platform/linux-generic/arch/powerpc/odp/api/abi/cpu.h index 9e3338d60..ecf56e82e 100644 --- a/platform/linux-generic/arch/powerpc/odp/api/abi/cpu.h +++ b/platform/linux-generic/arch/powerpc/odp/api/abi/cpu.h @@ -1,9 +1,25 @@ /* Copyright (c) 2017-2018, Linaro Limited + * Copyright (c) 2021, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause */ -#define _ODP_NEED_GENERIC_CPU_PAUSE +#ifndef ODP_API_ABI_CPU_H_ +#define ODP_API_ABI_CPU_H_ + +#ifdef __cplusplus +extern "C" { +#endif + #define ODP_CACHE_LINE_SIZE 128 -#include <odp/api/abi-default/cpu.h> + +/* Inlined functions for non-ABI compat mode */ +#include <odp/api/plat/cpu_inlines.h> + +#ifdef __cplusplus +} +#endif + +#endif + diff --git a/platform/linux-generic/arch/x86/odp/api/abi/cpu_inlines.h b/platform/linux-generic/arch/x86/odp/api/abi/cpu_inlines.h index 1eb8a9561..4b542a577 100644 --- a/platform/linux-generic/arch/x86/odp/api/abi/cpu_inlines.h +++ b/platform/linux-generic/arch/x86/odp/api/abi/cpu_inlines.h @@ -1,4 +1,5 @@ /* Copyright (c) 2018, Linaro Limited + * Copyright (c) 2021, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -7,10 +8,14 @@ #ifndef ODP_ARCH_CPU_INLINES_H_ #define ODP_ARCH_CPU_INLINES_H_ +#ifdef __cplusplus +extern "C" { +#endif + #include <stdint.h> #include <odp/api/abi/cpu_rdtsc.h> -_ODP_INLINE void odp_cpu_pause(void) +static inline void _odp_cpu_pause(void) { #ifdef __SSE2__ __asm__ __volatile__ ("pause"); @@ -19,19 +24,23 @@ _ODP_INLINE void odp_cpu_pause(void) #endif } -_ODP_INLINE uint64_t odp_cpu_cycles(void) +static inline uint64_t _odp_cpu_cycles(void) { return _odp_cpu_rdtsc(); } -_ODP_INLINE uint64_t odp_cpu_cycles_max(void) +static inline uint64_t _odp_cpu_cycles_max(void) { return UINT64_MAX; } -_ODP_INLINE uint64_t odp_cpu_cycles_resolution(void) +static inline uint64_t _odp_cpu_cycles_resolution(void) { return 1; } +#ifdef __cplusplus +} +#endif + #endif diff --git a/platform/linux-generic/include-abi/odp/api/abi/atomic.h b/platform/linux-generic/include-abi/odp/api/abi/atomic.h index 7c11b0ab2..9c87f9cb8 100644 --- a/platform/linux-generic/include-abi/odp/api/abi/atomic.h +++ b/platform/linux-generic/include-abi/odp/api/abi/atomic.h @@ -84,7 +84,6 @@ typedef struct ODP_ALIGNED(sizeof(odp_u128_t)) odp_atomic_u128_s { * @{ */ -#define _ODP_INLINE static inline #include <odp/api/plat/atomic_inlines.h> /** diff --git a/platform/linux-generic/include-abi/odp/api/abi/std.h b/platform/linux-generic/include-abi/odp/api/abi/std.h index 175b606c5..201fca18e 100644 --- a/platform/linux-generic/include-abi/odp/api/abi/std.h +++ b/platform/linux-generic/include-abi/odp/api/abi/std.h @@ -17,7 +17,6 @@ extern "C" { #endif -#define _ODP_INLINE static inline #include <odp/api/plat/std_inlines.h> #ifdef __cplusplus diff --git a/platform/linux-generic/include-abi/odp/api/abi/sync.h b/platform/linux-generic/include-abi/odp/api/abi/sync.h index cbb6f753e..236e92c8c 100644 --- a/platform/linux-generic/include-abi/odp/api/abi/sync.h +++ b/platform/linux-generic/include-abi/odp/api/abi/sync.h @@ -21,7 +21,6 @@ extern "C" { * @{ */ -#define _ODP_INLINE static inline #include <odp/api/plat/sync_inlines.h> /** diff --git a/platform/linux-generic/include/odp/api/plat/atomic_inlines.h b/platform/linux-generic/include/odp/api/plat/atomic_inlines.h index 4ab8bb411..a9da70890 100644 --- a/platform/linux-generic/include/odp/api/plat/atomic_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/atomic_inlines.h @@ -34,6 +34,13 @@ #define odp_atomic_dec_u32 __odp_atomic_dec_u32 #define odp_atomic_cas_u32 __odp_atomic_cas_u32 #define odp_atomic_xchg_u32 __odp_atomic_xchg_u32 + #define odp_atomic_load_acq_u32 __odp_atomic_load_acq_u32 + #define odp_atomic_store_rel_u32 __odp_atomic_store_rel_u32 + #define odp_atomic_add_rel_u32 __odp_atomic_add_rel_u32 + #define odp_atomic_sub_rel_u32 __odp_atomic_sub_rel_u32 + #define odp_atomic_cas_acq_u32 __odp_atomic_cas_acq_u32 + #define odp_atomic_cas_rel_u32 __odp_atomic_cas_rel_u32 + #define odp_atomic_cas_acq_rel_u32 __odp_atomic_cas_acq_rel_u32 #define odp_atomic_max_u32 __odp_atomic_max_u32 #define odp_atomic_min_u32 __odp_atomic_min_u32 #define odp_atomic_init_u64 __odp_atomic_init_u64 @@ -56,35 +63,8 @@ #define odp_atomic_cas_acq_u64 __odp_atomic_cas_acq_u64 #define odp_atomic_cas_rel_u64 __odp_atomic_cas_rel_u64 #define odp_atomic_cas_acq_rel_u64 __odp_atomic_cas_acq_rel_u64 - #define odp_atomic_init_u64 __odp_atomic_init_u64 - #define odp_atomic_load_u64 __odp_atomic_load_u64 - #define odp_atomic_store_u64 __odp_atomic_store_u64 - #define odp_atomic_fetch_add_u64 __odp_atomic_fetch_add_u64 - #define odp_atomic_add_u64 __odp_atomic_add_u64 - #define odp_atomic_fetch_sub_u64 __odp_atomic_fetch_sub_u64 - #define odp_atomic_sub_u64 __odp_atomic_sub_u64 - #define odp_atomic_fetch_inc_u64 __odp_atomic_fetch_inc_u64 - #define odp_atomic_inc_u64 __odp_atomic_inc_u64 - #define odp_atomic_fetch_dec_u64 __odp_atomic_fetch_dec_u64 - #define odp_atomic_dec_u64 __odp_atomic_dec_u64 - #define odp_atomic_cas_u64 __odp_atomic_cas_u64 - #define odp_atomic_xchg_u64 __odp_atomic_xchg_u64 - #define odp_atomic_load_acq_u64 __odp_atomic_load_acq_u64 - #define odp_atomic_store_rel_u64 __odp_atomic_store_rel_u64 - #define odp_atomic_add_rel_u64 __odp_atomic_add_rel_u64 - #define odp_atomic_sub_rel_u64 __odp_atomic_sub_rel_u64 - #define odp_atomic_cas_acq_u64 __odp_atomic_cas_acq_u64 - #define odp_atomic_cas_rel_u64 __odp_atomic_cas_rel_u64 - #define odp_atomic_cas_acq_rel_u64 __odp_atomic_cas_acq_rel_u64 #define odp_atomic_max_u64 __odp_atomic_max_u64 #define odp_atomic_min_u64 __odp_atomic_min_u64 - #define odp_atomic_load_acq_u32 __odp_atomic_load_acq_u32 - #define odp_atomic_store_rel_u32 __odp_atomic_store_rel_u32 - #define odp_atomic_add_rel_u32 __odp_atomic_add_rel_u32 - #define odp_atomic_sub_rel_u32 __odp_atomic_sub_rel_u32 - #define odp_atomic_cas_acq_u32 __odp_atomic_cas_acq_u32 - #define odp_atomic_cas_rel_u32 __odp_atomic_cas_rel_u32 - #define odp_atomic_cas_acq_rel_u32 __odp_atomic_cas_acq_rel_u32 #define odp_atomic_init_u128 __odp_atomic_init_u128 #define odp_atomic_load_u128 __odp_atomic_load_u128 #define odp_atomic_store_u128 __odp_atomic_store_u128 diff --git a/platform/linux-generic/include/odp/api/plat/cpu_inlines.h b/platform/linux-generic/include/odp/api/plat/cpu_inlines.h index 054801e80..60c4bb920 100644 --- a/platform/linux-generic/include/odp/api/plat/cpu_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/cpu_inlines.h @@ -1,4 +1,5 @@ /* Copyright (c) 2018, Linaro Limited + * Copyright (c) 2021, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -9,6 +10,10 @@ #include <odp/api/hints.h> +#include <odp/api/abi/cpu_inlines.h> + +#include <stdint.h> + /** @cond _ODP_HIDE_FROM_DOXYGEN_ */ #ifndef _ODP_NO_INLINE @@ -23,7 +28,25 @@ #define _ODP_INLINE #endif -#include <odp/api/abi/cpu_inlines.h> +_ODP_INLINE void odp_cpu_pause(void) +{ + _odp_cpu_pause(); +} + +_ODP_INLINE uint64_t odp_cpu_cycles_max(void) +{ + return _odp_cpu_cycles_max(); +} + +_ODP_INLINE uint64_t odp_cpu_cycles_resolution(void) +{ + return _odp_cpu_cycles_resolution(); +} + +_ODP_INLINE uint64_t odp_cpu_cycles(void) +{ + return _odp_cpu_cycles(); +} _ODP_INLINE uint64_t odp_cpu_cycles_diff(uint64_t c2, uint64_t c1) { diff --git a/platform/linux-generic/include/odp_buffer_internal.h b/platform/linux-generic/include/odp_buffer_internal.h index d2e3c808f..e7bc78d6e 100644 --- a/platform/linux-generic/include/odp_buffer_internal.h +++ b/platform/linux-generic/include/odp_buffer_internal.h @@ -30,7 +30,6 @@ extern "C" { #include <odp/api/thread.h> #include <odp/api/event.h> #include <odp_forward_typedefs_internal.h> -#include <odp_schedule_if.h> #include <stddef.h> typedef union buffer_index_t { diff --git a/platform/linux-generic/include/odp_config_internal.h b/platform/linux-generic/include/odp_config_internal.h index 1e8b390dd..40d2639f1 100644 --- a/platform/linux-generic/include/odp_config_internal.h +++ b/platform/linux-generic/include/odp_config_internal.h @@ -20,9 +20,9 @@ extern "C" { #define CONFIG_NUM_CPU_IDS 256 /* - * Maximum number of pools. Limited by ISHM_MAX_NB_BLOCKS. + * Maximum number of pools. */ -#define ODP_CONFIG_POOLS 55 +#define ODP_CONFIG_POOLS 32 /* * Queues reserved for ODP internal use @@ -116,11 +116,21 @@ extern "C" { CONFIG_PACKET_HEADROOM + \ CONFIG_PACKET_TAILROOM) -/* Maximum number of shared memory blocks. +/* + * Number of shared memory blocks reserved for implementation internal use. + * + * Each pool requires three SHM blocks (buffers, ring, user area). 20 blocks are + * reserved for per ODP module global data. + */ +#define CONFIG_INTERNAL_SHM_BLOCKS ((ODP_CONFIG_POOLS * 3) + 20) + +/* + * Maximum number of shared memory blocks. * - * This the the number of separate SHM areas that can be reserved concurrently + * This is the number of separate SHM blocks that an application can reserve + * concurrently. */ -#define ODP_CONFIG_SHM_BLOCKS (ODP_CONFIG_POOLS + 48) +#define CONFIG_SHM_BLOCKS 64 /* * Maximum event burst size diff --git a/platform/linux-generic/include/odp_global_data.h b/platform/linux-generic/include/odp_global_data.h index a907f87be..e4fd583a4 100644 --- a/platform/linux-generic/include/odp_global_data.h +++ b/platform/linux-generic/include/odp_global_data.h @@ -74,6 +74,7 @@ typedef struct odp_global_data_ro_t { uint8_t compress; uint8_t crypto; uint8_t ipsec; + uint8_t stash; uint8_t traffic_mngr; } disable; diff --git a/platform/linux-generic/include/odp_pool_internal.h b/platform/linux-generic/include/odp_pool_internal.h index dc4754710..dc9f0d207 100644 --- a/platform/linux-generic/include/odp_pool_internal.h +++ b/platform/linux-generic/include/odp_pool_internal.h @@ -20,6 +20,7 @@ extern "C" { #include <odp/api/shared_memory.h> #include <odp/api/ticketlock.h> +#include <odp/api/align.h> #include <odp_buffer_internal.h> #include <odp_config_internal.h> @@ -94,7 +95,7 @@ typedef struct pool_t { pool_destroy_cb_fn ext_destroy; void *ext_desc; - struct ODP_CACHE_ALIGNED { + struct ODP_ALIGNED_CACHE { odp_atomic_u64_t alloc_ops; odp_atomic_u64_t alloc_fails; odp_atomic_u64_t free_ops; diff --git a/platform/linux-generic/include/odp_queue_basic_internal.h b/platform/linux-generic/include/odp_queue_basic_internal.h index 25e35b22c..60817fc75 100644 --- a/platform/linux-generic/include/odp_queue_basic_internal.h +++ b/platform/linux-generic/include/odp_queue_basic_internal.h @@ -119,6 +119,9 @@ int _odp_sched_queue_deq(uint32_t queue_index, odp_event_t ev[], int num, int update_status); int _odp_sched_queue_empty(uint32_t queue_index); +/* Functions by schedulers */ +int _odp_sched_basic_get_spread(uint32_t queue_index); + #ifdef __cplusplus } #endif diff --git a/platform/linux-generic/include/odp_schedule_if.h b/platform/linux-generic/include/odp_schedule_if.h index d3202543d..a804f8c95 100644 --- a/platform/linux-generic/include/odp_schedule_if.h +++ b/platform/linux-generic/include/odp_schedule_if.h @@ -1,4 +1,5 @@ /* Copyright (c) 2013-2018, Linaro Limited + * Copyright (c) 2021, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -16,8 +17,12 @@ extern "C" { #include <odp/api/schedule.h> #include <odp_forward_typedefs_internal.h> -/* Number of ordered locks per queue */ -#define SCHEDULE_ORDERED_LOCKS_PER_QUEUE 2 +#define _ODP_SCHED_ID_BASIC 0 +#define _ODP_SCHED_ID_SP 1 +#define _ODP_SCHED_ID_SCALABLE 2 + +/* Scheduler identifier */ +extern int _odp_sched_id; typedef struct schedule_config_t { struct { diff --git a/platform/linux-generic/odp_comp.c b/platform/linux-generic/odp_comp.c index 685c9098a..579bdc03b 100644 --- a/platform/linux-generic/odp_comp.c +++ b/platform/linux-generic/odp_comp.c @@ -620,7 +620,7 @@ int _odp_comp_init_global(void) mem_size = sizeof(*global); /* Allocate our globally shared memory */ - shm = odp_shm_reserve("_odp_comp_pool", mem_size, + shm = odp_shm_reserve("_odp_comp_global", mem_size, ODP_CACHE_LINE_SIZE, 0); global = odp_shm_addr(shm); diff --git a/platform/linux-generic/odp_crypto_null.c b/platform/linux-generic/odp_crypto_null.c index cf9abc99f..aa8e8a2b3 100644 --- a/platform/linux-generic/odp_crypto_null.c +++ b/platform/linux-generic/odp_crypto_null.c @@ -1,4 +1,5 @@ /* Copyright (c) 2014-2018, Linaro Limited + * Copyright (c) 2021, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -13,6 +14,7 @@ #include <odp/api/align.h> #include <odp/api/shared_memory.h> #include <odp_debug_internal.h> +#include <odp_global_data.h> #include <odp/api/hints.h> #include <odp/api/random.h> #include <odp/api/plat/packet_inlines.h> @@ -113,6 +115,11 @@ void free_session(odp_crypto_generic_session_t *session) int odp_crypto_capability(odp_crypto_capability_t *capa) { + if (odp_global_ro.disable.crypto) { + ODP_ERR("Crypto is disabled\n"); + return -1; + } + if (NULL == capa) return -1; @@ -190,6 +197,15 @@ odp_crypto_session_create(const odp_crypto_session_param_t *param, int rc; odp_crypto_generic_session_t *session; + if (odp_global_ro.disable.crypto) { + ODP_ERR("Crypto is disabled\n"); + /* Dummy output to avoid compiler warning about uninitialized + * variables */ + *status = ODP_CRYPTO_SES_CREATE_ERR_ENOMEM; + *session_out = ODP_CRYPTO_SESSION_INVALID; + return -1; + } + /* Allocate memory for this session */ session = alloc_session(); if (NULL == session) { @@ -276,12 +292,18 @@ odp_crypto_operation(odp_crypto_op_param_t *param, packet_param.auth_range = param->auth_range; rc = odp_crypto_op(¶m->pkt, &out_pkt, &packet_param, 1); - if (rc < 0) - return rc; + if (rc <= 0) + return -1; rc = odp_crypto_result(&packet_result, out_pkt); - if (rc < 0) - return rc; + if (rc < 0) { + /* + * We cannot fail since odp_crypto_op() has already processed + * the packet. Let's indicate error in the result instead. + */ + packet_hdr(out_pkt)->p.flags.crypto_err = 1; + packet_result.ok = false; + } /* Indicate to caller operation was sync */ *posted = 0; @@ -312,11 +334,16 @@ _odp_crypto_init_global(void) odp_shm_t shm; int idx; + if (odp_global_ro.disable.crypto) { + ODP_PRINT("\nODP crypto is DISABLED\n"); + return 0; + } + /* Calculate the memory size we need */ mem_size = sizeof(odp_crypto_global_t); /* Allocate our globally shared memory */ - shm = odp_shm_reserve("_odp_crypto_pool_null", mem_size, + shm = odp_shm_reserve("_odp_crypto_null_global", mem_size, ODP_CACHE_LINE_SIZE, 0); if (ODP_SHM_INVALID == shm) { @@ -346,6 +373,9 @@ int _odp_crypto_term_global(void) int count = 0; odp_crypto_generic_session_t *session; + if (odp_global_ro.disable.crypto) + return 0; + for (session = global->free; session != NULL; session = session->next) count++; if (count != MAX_SESSIONS) { @@ -353,7 +383,7 @@ int _odp_crypto_term_global(void) rc = -1; } - ret = odp_shm_free(odp_shm_lookup("_odp_crypto_pool_null")); + ret = odp_shm_free(odp_shm_lookup("_odp_crypto_null_global")); if (ret < 0) { ODP_ERR("shm free failed for _odp_crypto_pool_null\n"); rc = -1; diff --git a/platform/linux-generic/odp_crypto_openssl.c b/platform/linux-generic/odp_crypto_openssl.c index 07a91cc46..d8276b4b4 100644 --- a/platform/linux-generic/odp_crypto_openssl.c +++ b/platform/linux-generic/odp_crypto_openssl.c @@ -1,4 +1,5 @@ /* Copyright (c) 2014-2018, Linaro Limited + * Copyright (c) 2021, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -438,8 +439,12 @@ void packet_aes_xcbc_mac(odp_packet_t pkt, len -= datalen; if (eoff != 0) { if (eoff + datalen > AES_BLOCK_SIZE) { - memxor(e + eoff, data, AES_BLOCK_SIZE - eoff); - datalen -= (AES_BLOCK_SIZE - eoff); + /* bytes needed to fill the partial block */ + uint32_t remaining_len = AES_BLOCK_SIZE - eoff; + + memxor(e + eoff, data, remaining_len); + datalen -= remaining_len; + data += remaining_len; eoff = 0; EVP_EncryptUpdate(ctx, e, &dummy_len, e, sizeof(e)); @@ -695,7 +700,7 @@ int packet_cmac_eia2(odp_packet_t pkt, { CMAC_CTX *ctx = local.cmac_ctx[session->idx]; void *iv_ptr; - uint32_t offset = param->auth_range.offset; + uint32_t offset = param->auth_range.offset / 8; uint32_t len = (param->auth_range.length + 7) / 8; size_t outlen; @@ -894,120 +899,146 @@ int internal_aad(EVP_CIPHER_CTX *ctx, return ret; } -static -int internal_encrypt(EVP_CIPHER_CTX *ctx, - odp_packet_t pkt, - const odp_crypto_packet_op_param_t *param) -{ - unsigned in_pos = param->cipher_range.offset; - unsigned out_pos = param->cipher_range.offset; - unsigned in_len = param->cipher_range.length; - uint8_t block[2 * EVP_MAX_BLOCK_LENGTH]; - unsigned block_len = EVP_CIPHER_block_size(EVP_CIPHER_CTX_cipher(ctx)); - int cipher_len; - int ret; - - ODP_ASSERT(in_pos + in_len <= odp_packet_len(pkt)); - - while (in_len > 0) { - uint32_t seglen = 0; /* GCC */ - uint8_t *insegaddr = odp_packet_offset(pkt, in_pos, - &seglen, NULL); - unsigned inseglen = in_len < seglen ? in_len : seglen; - - /* There should be at least 1 additional block in out buffer */ - if (inseglen > block_len) { - unsigned part = inseglen - block_len; - - EVP_EncryptUpdate(ctx, insegaddr, &cipher_len, - insegaddr, part); - in_pos += part; - in_len -= part; - insegaddr += part; - inseglen -= part; - - out_pos += cipher_len; - } - - /* Use temporal storage */ - if (inseglen > 0) { - unsigned part = inseglen; - - EVP_EncryptUpdate(ctx, block, &cipher_len, - insegaddr, part); - in_pos += part; - in_len -= part; - insegaddr += part; - inseglen -= part; - - odp_packet_copy_from_mem(pkt, out_pos, - cipher_len, block); - out_pos += cipher_len; - } - } - - ret = EVP_EncryptFinal_ex(ctx, block, &cipher_len); - odp_packet_copy_from_mem(pkt, out_pos, cipher_len, block); +typedef int (*evp_update_t)(EVP_CIPHER_CTX *, unsigned char *, + int *, const unsigned char *, int); - return ret; -} +typedef int (*evp_final_t)(EVP_CIPHER_CTX *, unsigned char *, int *); -static -int internal_decrypt(EVP_CIPHER_CTX *ctx, - odp_packet_t pkt, - const odp_crypto_packet_op_param_t *param) -{ - unsigned in_pos = param->cipher_range.offset; - unsigned out_pos = param->cipher_range.offset; - unsigned in_len = param->cipher_range.length; - uint8_t block[2 * EVP_MAX_BLOCK_LENGTH]; - unsigned block_len = EVP_CIPHER_block_size(EVP_CIPHER_CTX_cipher(ctx)); - int cipher_len; - int ret; +static inline int internal_crypt(EVP_CIPHER_CTX *ctx, + odp_packet_t pkt, + const odp_crypto_packet_op_param_t *param, + evp_update_t EVP_update, + evp_final_t EVP_final) +{ + uint32_t in_pos = param->cipher_range.offset; + uint32_t out_pos = in_pos; + uint32_t in_len = param->cipher_range.length; + uint8_t block[EVP_MAX_BLOCK_LENGTH]; + uint32_t buffered = 0; + uint32_t block_len = EVP_CIPHER_block_size(EVP_CIPHER_CTX_cipher(ctx)); + int out_len; + int rc; ODP_ASSERT(in_pos + in_len <= odp_packet_len(pkt)); + /* + * In the following loop we process one packet segment per iteration. + * We rely on the following properties of the encrypt/decrypt update + * function with the algorithms that we use: + * + * - The function processes (and writes to output) only whole blocks. + * - Input data beyond the last full block is buffered inside OpenSSL. + * - The amount of buffered data is always less than one block. + * - Total amount of output data does not exceed the total amount + * of input data at any point. + */ while (in_len > 0) { - uint32_t seglen = 0; /* GCC */ - uint8_t *insegaddr = odp_packet_offset(pkt, in_pos, - &seglen, NULL); - unsigned inseglen = in_len < seglen ? in_len : seglen; - - /* There should be at least 1 additional block in out buffer */ - if (inseglen > block_len) { - unsigned part = inseglen - block_len; - - EVP_DecryptUpdate(ctx, insegaddr, &cipher_len, - insegaddr, part); - in_pos += part; - in_len -= part; - insegaddr += part; - inseglen -= part; - - out_pos += cipher_len; + uint32_t seglen = 0; + uint8_t *in_addr = odp_packet_offset(pkt, in_pos, + &seglen, NULL); + uint32_t len = in_len < seglen ? in_len : seglen; + + if (odp_unlikely(buffered > 0)) { + /* + * Leftover data from the previous segment is + * in the buffer inside OpenSSL. + */ + uint32_t remaining_len = block_len - buffered; + + if (odp_likely(len >= remaining_len)) { + /* + * Let's fill the buffered input data to a + * full block and get the output block to + * a memory buffer. The buffer is then copied + * to the packet, crossing segment boundary. + */ + rc = EVP_update(ctx, block, &out_len, + in_addr, remaining_len); + if (odp_unlikely(rc != 1)) + goto err; + if (odp_unlikely(out_len != (int)block_len)) + goto err; + in_addr += remaining_len; + in_pos += remaining_len; + len -= remaining_len; + in_len -= remaining_len; + buffered = 0; + rc = odp_packet_copy_from_mem(pkt, out_pos, + block_len, block); + if (odp_unlikely(rc)) + goto err; + out_pos += block_len; + } else { + /* + * Not enough data in this segment to fill + * the buffer to a full block. Fill the buffer + * a bit more and go to the next segment. + */ + rc = EVP_update(ctx, block, &out_len, + in_addr, len); + if (odp_unlikely(rc != 1)) + goto err; + if (odp_unlikely(out_len > 0)) + goto err; + in_pos += len; + in_len -= len; + buffered += len; + continue; + } } - - /* Use temporal storage */ - if (inseglen > 0) { - unsigned part = inseglen; - - EVP_DecryptUpdate(ctx, block, &cipher_len, - insegaddr, part); - in_pos += part; - in_len -= part; - insegaddr += part; - inseglen -= part; - - odp_packet_copy_from_mem(pkt, out_pos, - cipher_len, block); - out_pos += cipher_len; + ODP_ASSERT(buffered == 0); + + if (in_len > 0) { + /* + * No input is buffered inside OpenSSL. We pass the + * whole remaining segment to OpenSSL and expect to + * get a multiple of block size of data processed, + * with the rest left in the buffer. + */ + rc = EVP_update(ctx, in_addr, &out_len, in_addr, len); + if (odp_unlikely(rc != 1)) + goto err; + ODP_ASSERT(CHECK_IS_POWER2(block_len)); + buffered = len & (block_len - 1); + if (odp_unlikely(out_len + buffered != len)) + goto err; + in_pos += len; + in_len -= len; + out_pos += len - buffered; } } + if (odp_unlikely(buffered > 0)) + goto err; + /* + * We do not expect any more data out since the cipher range is + * supposed to be a multiple of the block size. + */ + rc = EVP_final(ctx, block, &out_len); + if (odp_unlikely(out_len != 0)) + return 0; + return rc; +err: + ODP_ERR("internal error\n"); + (void)EVP_final(ctx, block, &out_len); + return 0; +} - ret = EVP_DecryptFinal_ex(ctx, block, &cipher_len); - odp_packet_copy_from_mem(pkt, out_pos, cipher_len, block); +static int internal_encrypt(EVP_CIPHER_CTX *ctx, + odp_packet_t pkt, + const odp_crypto_packet_op_param_t *param) +{ + return internal_crypt(ctx, pkt, param, + EVP_EncryptUpdate, + EVP_EncryptFinal_ex); +} - return ret; +static int internal_decrypt(EVP_CIPHER_CTX *ctx, + odp_packet_t pkt, + const odp_crypto_packet_op_param_t *param) +{ + return internal_crypt(ctx, pkt, param, + EVP_DecryptUpdate, + EVP_DecryptFinal_ex); } static void @@ -1121,6 +1152,10 @@ odp_crypto_alg_err_t cipher_encrypt_bits(odp_packet_t pkt, uint32_t in_len = (param->cipher_range.length + 7) / 8; uint8_t data[in_len]; int ret; + uint32_t offset; + + /* Range offset is in bits in bit mode but must be divisible by 8. */ + offset = param->cipher_range.offset / 8; if (param->cipher_iv_ptr) iv_ptr = param->cipher_iv_ptr; @@ -1131,16 +1166,14 @@ odp_crypto_alg_err_t cipher_encrypt_bits(odp_packet_t pkt, EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv_ptr); - odp_packet_copy_to_mem(pkt, param->cipher_range.offset, in_len, - data); + odp_packet_copy_to_mem(pkt, offset, in_len, data); EVP_EncryptUpdate(ctx, data, &cipher_len, data, in_len); ret = EVP_EncryptFinal_ex(ctx, data + cipher_len, &dummy_len); cipher_len += dummy_len; - odp_packet_copy_from_mem(pkt, param->cipher_range.offset, in_len, - data); + odp_packet_copy_from_mem(pkt, offset, in_len, data); return ret <= 0 ? ODP_CRYPTO_ALG_ERR_DATA_SIZE : ODP_CRYPTO_ALG_ERR_NONE; @@ -1159,6 +1192,10 @@ odp_crypto_alg_err_t cipher_decrypt_bits(odp_packet_t pkt, uint32_t in_len = (param->cipher_range.length + 7) / 8; uint8_t data[in_len]; int ret; + uint32_t offset; + + /* Range offset is in bits in bit mode but must be divisible by 8. */ + offset = param->cipher_range.offset / 8; if (param->cipher_iv_ptr) iv_ptr = param->cipher_iv_ptr; @@ -1169,16 +1206,14 @@ odp_crypto_alg_err_t cipher_decrypt_bits(odp_packet_t pkt, EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv_ptr); - odp_packet_copy_to_mem(pkt, param->cipher_range.offset, in_len, - data); + odp_packet_copy_to_mem(pkt, offset, in_len, data); EVP_DecryptUpdate(ctx, data, &cipher_len, data, in_len); ret = EVP_DecryptFinal_ex(ctx, data + cipher_len, &dummy_len); cipher_len += dummy_len; - odp_packet_copy_from_mem(pkt, param->cipher_range.offset, in_len, - data); + odp_packet_copy_from_mem(pkt, offset, in_len, data); return ret <= 0 ? ODP_CRYPTO_ALG_ERR_DATA_SIZE : ODP_CRYPTO_ALG_ERR_NONE; @@ -2461,12 +2496,18 @@ odp_crypto_operation(odp_crypto_op_param_t *param, packet_param.auth_range = param->auth_range; rc = odp_crypto_op(¶m->pkt, &out_pkt, &packet_param, 1); - if (rc < 0) - return rc; + if (rc <= 0) + return -1; rc = odp_crypto_result(&packet_result, out_pkt); - if (rc < 0) - return rc; + if (rc < 0) { + /* + * We cannot fail since odp_crypto_op() has already processed + * the packet. Let's indicate error in the result instead. + */ + packet_hdr(out_pkt)->p.flags.crypto_err = 1; + packet_result.ok = false; + } /* Indicate to caller operation was sync */ *posted = 0; @@ -2524,7 +2565,7 @@ int _odp_crypto_init_global(void) mem_size += nlocks * sizeof(odp_ticketlock_t); /* Allocate our globally shared memory */ - shm = odp_shm_reserve("_odp_crypto_pool_ssl", mem_size, + shm = odp_shm_reserve("_odp_crypto_ssl_global", mem_size, ODP_CACHE_LINE_SIZE, 0); if (ODP_SHM_INVALID == shm) { @@ -2579,7 +2620,7 @@ int _odp_crypto_term_global(void) CRYPTO_set_id_callback(NULL); #endif - ret = odp_shm_free(odp_shm_lookup("_odp_crypto_pool_ssl")); + ret = odp_shm_free(odp_shm_lookup("_odp_crypto_ssl_global")); if (ret < 0) { ODP_ERR("shm free failed for crypto_pool\n"); rc = -1; diff --git a/platform/linux-generic/odp_init.c b/platform/linux-generic/odp_init.c index 0cb60cdfa..d5530c1f1 100644 --- a/platform/linux-generic/odp_init.c +++ b/platform/linux-generic/odp_init.c @@ -66,6 +66,7 @@ static void disable_features(odp_global_data_ro_t *global_ro, if (disable_ipsec && disable_crypto) global_ro->disable.crypto = 1; + global_ro->disable.stash = init_param->not_used.feat.stash; global_ro->disable.traffic_mngr = init_param->not_used.feat.tm; global_ro->disable.compress = init_param->not_used.feat.compress; } diff --git a/platform/linux-generic/odp_ipsec.c b/platform/linux-generic/odp_ipsec.c index 137e7b435..09596b502 100644 --- a/platform/linux-generic/odp_ipsec.c +++ b/platform/linux-generic/odp_ipsec.c @@ -20,6 +20,7 @@ #include <odp/api/plat/queue_inlines.h> #include <odp_classification_internal.h> #include <odp_libconfig_internal.h> +#include <odp_schedule_if.h> #include <protocols/eth.h> #include <protocols/ip.h> @@ -164,6 +165,9 @@ int odp_ipsec_capability(odp_ipsec_capability_t *capa) if (rc < 0) return rc; + capa->queue_type_plain = true; + capa->queue_type_sched = true; + rc = odp_queue_capability(&queue_capa); if (rc < 0) return rc; @@ -756,7 +760,8 @@ static ipsec_sa_t *ipsec_in_single(odp_packet_t pkt, odp_ipsec_sa_t sa, odp_packet_t *pkt_out, odp_bool_t enqueue_op, - odp_ipsec_op_status_t *status) + odp_ipsec_op_status_t *status, + uint32_t *orig_ip_len) { ipsec_state_t state; ipsec_sa_t *ipsec_sa = NULL; @@ -792,6 +797,7 @@ static ipsec_sa_t *ipsec_in_single(odp_packet_t pkt, status->error.alg = 1; goto exit; } + *orig_ip_len = state.ip_tot_len; /* Check IP header for IPSec protocols and look it up */ if (_ODP_IPPROTO_ESP == state.ip_next_hdr || @@ -1014,13 +1020,14 @@ static int ipsec_out_tunnel_parse_ipv6(ipsec_state_t *state, ipsec_sa_t *ipsec_sa) { _odp_ipv6hdr_t *ipv6hdr = state->ip; + uint32_t ver_tc_flow = odp_be_to_cpu_32(ipv6hdr->ver_tc_flow); ipv6hdr->hop_limit -= ipsec_sa->dec_ttl; - state->out_tunnel.ip_tos = (ipv6hdr->ver_tc_flow & + state->out_tunnel.ip_tos = (ver_tc_flow & _ODP_IPV6HDR_TC_MASK) >> _ODP_IPV6HDR_TC_SHIFT; state->out_tunnel.ip_df = 0; - state->out_tunnel.ip_flabel = (ipv6hdr->ver_tc_flow & + state->out_tunnel.ip_flabel = (ver_tc_flow & _ODP_IPV6HDR_FLOW_LABEL_MASK) >> _ODP_IPV6HDR_FLOW_LABEL_SHIFT; state->ip_next_hdr = ipv6hdr->next_hdr; @@ -1740,6 +1747,7 @@ int odp_ipsec_in(const odp_packet_t pkt_in[], int num_in, odp_ipsec_op_status_t status; odp_ipsec_sa_t sa; ipsec_sa_t *ipsec_sa; + uint32_t dummy; /* orig_ip_len not valid in sync operations */ odp_ipsec_packet_result_t *result; memset(&status, 0, sizeof(status)); @@ -1751,7 +1759,7 @@ int odp_ipsec_in(const odp_packet_t pkt_in[], int num_in, ODP_ASSERT(ODP_IPSEC_SA_INVALID != sa); } - ipsec_sa = ipsec_in_single(pkt, sa, &pkt, false, &status); + ipsec_sa = ipsec_in_single(pkt, sa, &pkt, false, &status, &dummy); packet_subtype_set(pkt, ODP_EVENT_PACKET_IPSEC); result = ipsec_pkt_result(pkt); @@ -1851,8 +1859,10 @@ int odp_ipsec_in_enq(const odp_packet_t pkt_in[], int num_in, odp_ipsec_op_status_t status; odp_ipsec_sa_t sa; ipsec_sa_t *ipsec_sa; + uint32_t orig_ip_len = 0; odp_ipsec_packet_result_t *result; odp_queue_t queue; + int rc; memset(&status, 0, sizeof(status)); @@ -1863,12 +1873,13 @@ int odp_ipsec_in_enq(const odp_packet_t pkt_in[], int num_in, ODP_ASSERT(ODP_IPSEC_SA_INVALID != sa); } - ipsec_sa = ipsec_in_single(pkt, sa, &pkt, true, &status); + ipsec_sa = ipsec_in_single(pkt, sa, &pkt, true, &status, &orig_ip_len); packet_subtype_set(pkt, ODP_EVENT_PACKET_IPSEC); result = ipsec_pkt_result(pkt); memset(result, 0, sizeof(*result)); result->status = status; + result->orig_ip_len = orig_ip_len; if (NULL != ipsec_sa) { result->sa = ipsec_sa->ipsec_sa_hdl; queue = ipsec_sa->queue; @@ -1877,10 +1888,6 @@ int odp_ipsec_in_enq(const odp_packet_t pkt_in[], int num_in, queue = ipsec_config->inbound.default_queue; } - if (odp_queue_enq(queue, odp_ipsec_packet_to_event(pkt))) { - odp_packet_free(pkt); - break; - } in_pkt++; sa_idx += sa_inc; @@ -1891,6 +1898,12 @@ int odp_ipsec_in_enq(const odp_packet_t pkt_in[], int num_in, */ if (sa == ODP_IPSEC_SA_INVALID && ipsec_sa) _odp_ipsec_sa_unuse(ipsec_sa); + + rc = odp_queue_enq(queue, odp_ipsec_packet_to_event(pkt)); + if (odp_unlikely(rc)) { + odp_packet_free(pkt); + break; + } } return in_pkt; @@ -1915,6 +1928,7 @@ int odp_ipsec_out_enq(const odp_packet_t pkt_in[], int num_in, odp_ipsec_packet_result_t *result; const odp_ipsec_out_opt_t *opt; odp_queue_t queue; + int rc; memset(&status, 0, sizeof(status)); @@ -1939,13 +1953,15 @@ int odp_ipsec_out_enq(const odp_packet_t pkt_in[], int num_in, if (ipsec_config->stats_en) ipsec_sa_err_stats_update(ipsec_sa, &status); - if (odp_queue_enq(queue, odp_ipsec_packet_to_event(pkt))) { - odp_packet_free(pkt); - break; - } in_pkt++; sa_idx += sa_inc; opt_idx += opt_inc; + + rc = odp_queue_enq(queue, odp_ipsec_packet_to_event(pkt)); + if (odp_unlikely(rc)) { + odp_packet_free(pkt); + break; + } } return in_pkt; @@ -1955,6 +1971,7 @@ int _odp_ipsec_try_inline(odp_packet_t *pkt) { odp_ipsec_op_status_t status; ipsec_sa_t *ipsec_sa; + uint32_t orig_ip_len = 0; odp_ipsec_packet_result_t *result; odp_packet_hdr_t *pkt_hdr; @@ -1964,7 +1981,7 @@ int _odp_ipsec_try_inline(odp_packet_t *pkt) memset(&status, 0, sizeof(status)); ipsec_sa = ipsec_in_single(*pkt, ODP_IPSEC_SA_INVALID, pkt, false, - &status); + &status, &orig_ip_len); /* * Route packet back in case of lookup failure or early error before * lookup @@ -1976,6 +1993,7 @@ int _odp_ipsec_try_inline(odp_packet_t *pkt) result = ipsec_pkt_result(*pkt); memset(result, 0, sizeof(*result)); result->status = status; + result->orig_ip_len = orig_ip_len; result->sa = ipsec_sa->ipsec_sa_hdl; result->flag.inline_mode = 1; @@ -2096,6 +2114,7 @@ int odp_ipsec_out_inline(const odp_packet_t pkt_in[], int num_in, } } else { odp_queue_t queue; + int rc; err: if (ipsec_config->stats_en) ipsec_sa_err_stats_update(ipsec_sa, &status); @@ -2107,8 +2126,9 @@ err: result->status = status; queue = ipsec_sa->queue; - if (odp_queue_enq(queue, - odp_ipsec_packet_to_event(pkt))) { + rc = odp_queue_enq(queue, odp_ipsec_packet_to_event(pkt)); + if (odp_unlikely(rc)) { + in_pkt++; odp_packet_free(pkt); break; } @@ -2196,6 +2216,21 @@ int odp_ipsec_stats(odp_ipsec_sa_t sa, odp_ipsec_stats_t *stats) return 0; } +int odp_ipsec_stats_multi(odp_ipsec_sa_t sa[], odp_ipsec_stats_t stats[], int num) +{ + int ret, i; + + ODP_ASSERT(NULL != stats); + + for (i = 0; i < num; i++) { + ret = odp_ipsec_stats(sa[i], &stats[i]); + if (ret) + return ret; + } + + return 0; +} + static int read_config_file(ipsec_global_t *global) { const char *str_i = "ipsec.ordering.async_inbound"; @@ -2224,7 +2259,7 @@ int _odp_ipsec_init_global(void) if (odp_global_ro.disable.ipsec) return 0; - shm = odp_shm_reserve("_odp_ipsec", sizeof(*ipsec_global), + shm = odp_shm_reserve("_odp_ipsec_global", sizeof(*ipsec_global), ODP_CACHE_LINE_SIZE, 0); if (shm == ODP_SHM_INVALID) { ODP_ERR("Shm reserve failed for odp_ipsec\n"); @@ -2260,7 +2295,7 @@ int _odp_ipsec_term_global(void) if (odp_global_ro.disable.ipsec) return 0; - shm = odp_shm_lookup("_odp_ipsec"); + shm = odp_shm_lookup("_odp_ipsec_global"); if (shm == ODP_SHM_INVALID || odp_shm_free(shm)) { ODP_ERR("Shm free failed for odp_ipsec"); diff --git a/platform/linux-generic/odp_ishm.c b/platform/linux-generic/odp_ishm.c index 7041cd2b5..ebc931658 100644 --- a/platform/linux-generic/odp_ishm.c +++ b/platform/linux-generic/odp_ishm.c @@ -75,7 +75,7 @@ * if some of the block owners never procsync() after free). This number * should take that into account) */ -#define ISHM_MAX_NB_BLOCKS 128 +#define ISHM_MAX_NB_BLOCKS (CONFIG_INTERNAL_SHM_BLOCKS + CONFIG_SHM_BLOCKS) /* * Maximum internal shared memory block name length in chars diff --git a/platform/linux-generic/odp_packet.c b/platform/linux-generic/odp_packet.c index 0986056e6..c6a50bf84 100644 --- a/platform/linux-generic/odp_packet.c +++ b/platform/linux-generic/odp_packet.c @@ -11,6 +11,7 @@ #include <odp/api/plat/packet_inlines.h> #include <odp_packet_internal.h> #include <odp_debug_internal.h> +#include <odp_macros_internal.h> #include <odp_chksum_internal.h> #include <odp_errno_define.h> #include <odp/api/hints.h> @@ -1847,7 +1848,15 @@ static uint32_t packet_sum_crc32c(odp_packet_hdr_t *pkt_hdr, return sum; } -/** Parser helper function for Ethernet packets */ +/* + * In the worst case we look at the Ethernet header, 8 bytes of LLC/SNAP + * header and two VLAN tags in the same packet. + */ +#define PARSE_ETH_BYTES (sizeof(_odp_ethhdr_t) + 8 + 2 * sizeof(_odp_vlanhdr_t)) +/** Parser helper function for Ethernet packets + * + * Requires up to PARSE_ETH_BYTES bytes of contiguous packet data. + */ static inline uint16_t parse_eth(packet_parser_t *prs, const uint8_t **parseptr, uint32_t *offset, uint32_t frame_len) { @@ -1864,7 +1873,7 @@ static inline uint16_t parse_eth(packet_parser_t *prs, const uint8_t **parseptr, eth = (const _odp_ethhdr_t *)*parseptr; /* Detect jumbo frames */ - if (odp_unlikely(frame_len > _ODP_ETH_LEN_MAX)) + if (odp_unlikely(frame_len - *offset > _ODP_ETH_LEN_MAX)) input_flags.jumbo = 1; /* Handle Ethernet broadcast/multicast addresses */ @@ -1922,14 +1931,27 @@ static inline uint16_t parse_eth(packet_parser_t *prs, const uint8_t **parseptr, *parseptr += sizeof(_odp_vlanhdr_t); } + /* + * The packet was too short for what we parsed. We just give up + * entirely without trying to parse what fits in the packet. + */ + if (odp_unlikely(*offset > frame_len)) { + input_flags.all = 0; + input_flags.l2 = 1; + ethtype = 0; + } + error: prs->input_flags.all |= input_flags.all; return ethtype; } +#define PARSE_IPV4_BYTES (0xfU * 4) /* max IPv4 header length with options */ /** * Parser helper function for IPv4 + * + * Requires up to PARSE_IPV4_BYTES bytes of contiguous packet data. */ static inline uint8_t parse_ipv4(packet_parser_t *prs, const uint8_t **parseptr, uint32_t *offset, uint32_t frame_len, @@ -1945,6 +1967,7 @@ static inline uint8_t parse_ipv4(packet_parser_t *prs, const uint8_t **parseptr, if (odp_unlikely(ihl < _ODP_IPV4HDR_IHL_MIN || ver != 4 || + sizeof(*ipv4) > frame_len - *offset || (l3_len > frame_len - *offset))) { prs->flags.ip_err = 1; return 0; @@ -1987,8 +2010,15 @@ static inline uint8_t parse_ipv4(packet_parser_t *prs, const uint8_t **parseptr, return ipv4->proto; } +/* + * Peeks 2 bytes beyond IPv6 base header without length check if there + * are extension headers. + */ +#define PARSE_IPV6_BYTES (sizeof(_odp_ipv6hdr_t) + 2) /** * Parser helper function for IPv6 + * + * Requires at least PARSE_IPV6_BYTES bytes of contiguous packet data. */ static inline uint8_t parse_ipv6(packet_parser_t *prs, const uint8_t **parseptr, uint32_t *offset, uint32_t frame_len, @@ -2003,8 +2033,9 @@ static inline uint8_t parse_ipv6(packet_parser_t *prs, const uint8_t **parseptr, _ODP_IPV6HDR_LEN; /* Basic sanity checks on IPv6 header */ - if ((odp_be_to_cpu_32(ipv6->ver_tc_flow) >> 28) != 6 || - l3_len > frame_len - *offset) { + if (odp_unlikely((odp_be_to_cpu_32(ipv6->ver_tc_flow) >> 28) != 6 || + sizeof(*ipv6) > frame_len - *offset || + l3_len > frame_len - *offset)) { prs->flags.ip_err = 1; return 0; } @@ -2056,8 +2087,11 @@ static inline uint8_t parse_ipv6(packet_parser_t *prs, const uint8_t **parseptr, return ipv6->next_hdr; } +#define PARSE_TCP_BYTES (sizeof(_odp_tcphdr_t)) /** * Parser helper function for TCP + * + * Requires PARSE_TCP_BYTES bytes of contiguous packet data. */ static inline void parse_tcp(packet_parser_t *prs, const uint8_t **parseptr, uint16_t tcp_len, @@ -2083,8 +2117,15 @@ static inline void parse_tcp(packet_parser_t *prs, const uint8_t **parseptr, *parseptr += len; } +/* + * In the worst case we look at the UDP header and 4 bytes of the UDP + * payload (the non-ESP marker to distinguish IKE packets from ESP packets). + */ +#define PARSE_UDP_BYTES (sizeof(_odp_udphdr_t) + 4) /** * Parser helper function for UDP + * + * Requires PARSE_UDP_BYTES bytes of contiguous packet data. */ static inline void parse_udp(packet_parser_t *prs, const uint8_t **parseptr, odp_proto_chksums_t chksums, @@ -2129,8 +2170,11 @@ static inline void parse_udp(packet_parser_t *prs, const uint8_t **parseptr, *parseptr += sizeof(_odp_udphdr_t); } +#define PARSE_SCTP_BYTES (sizeof(_odp_sctphdr_t)) /** * Parser helper function for SCTP + * + * Requires PARSE_SCTP_BYTES bytes of contiguous packet data. */ static inline void parse_sctp(packet_parser_t *prs, const uint8_t **parseptr, uint16_t sctp_len, @@ -2157,6 +2201,10 @@ static inline void parse_sctp(packet_parser_t *prs, const uint8_t **parseptr, *parseptr += sizeof(_odp_sctphdr_t); } +#define MAX3(a, b, c) (MAX(MAX((a), (b)), (c))) +#define PARSE_L3_L4_BYTES (MAX(PARSE_IPV4_BYTES, PARSE_IPV6_BYTES) + \ + MAX3(PARSE_TCP_BYTES, PARSE_UDP_BYTES, PARSE_SCTP_BYTES)) +/* Requires up to PARSE_L3_L4_BYTES bytes of contiguous packet data. */ static inline int packet_parse_common_l3_l4(packet_parser_t *prs, const uint8_t *parseptr, uint32_t offset, @@ -2516,6 +2564,8 @@ static int packet_l4_chksum(odp_packet_hdr_t *pkt_hdr, if (chksums.chksum.sctp && pkt_hdr->p.input_flags.sctp && !pkt_hdr->p.input_flags.ipfrag) { + uint32_t seg_len = 0; + _odp_sctphdr_t hdr_copy; uint32_t sum = ~packet_sum_crc32c(pkt_hdr, pkt_hdr->p.l4_offset + _ODP_SCTPHDR_LEN, @@ -2525,8 +2575,14 @@ static int packet_l4_chksum(odp_packet_hdr_t *pkt_hdr, l4_part_sum); _odp_sctphdr_t *sctp = packet_map(pkt_hdr, pkt_hdr->p.l4_offset, - NULL, NULL); + &seg_len, NULL); + if (odp_unlikely(seg_len < sizeof(*sctp))) { + odp_packet_t pkt = packet_handle(pkt_hdr); + sctp = &hdr_copy; + odp_packet_copy_to_mem(pkt, pkt_hdr->p.l4_offset, + sizeof(*sctp), sctp); + } pkt_hdr->p.input_flags.l4_chksum_done = 1; if (sum != sctp->chksum) { pkt_hdr->p.flags.l4_chksum_err = 1; @@ -2587,6 +2643,8 @@ int odp_packet_parse(odp_packet_t pkt, uint32_t offset, int ret; uint16_t ethtype; uint64_t l4_part_sum = 0; + const uint32_t min_seglen = PARSE_ETH_BYTES + PARSE_L3_L4_BYTES; + uint8_t buf[min_seglen]; if (proto == ODP_PROTO_NONE || layer == ODP_PROTO_LAYER_NONE) return -1; @@ -2596,6 +2654,20 @@ int odp_packet_parse(odp_packet_t pkt, uint32_t offset, if (data == NULL) return -1; + /* + * We must not have a packet segment boundary within the parsed + * packet data range. Copy enough data to a temporary buffer for + * parsing if necessary. + */ + if (odp_unlikely(pkt_hdr->seg_count > 1) && + odp_unlikely(seg_len < min_seglen)) { + seg_len = min_seglen; + if (seg_len > packet_len - offset) + seg_len = packet_len - offset; + odp_packet_copy_to_mem(pkt, offset, seg_len, buf); + data = buf; + } + /* Reset parser flags, keep other flags */ packet_parse_reset(pkt_hdr, 0); @@ -2942,6 +3014,13 @@ odp_packet_reass_status(odp_packet_t pkt) return ODP_PACKET_REASS_NONE; } +int odp_packet_reass_info(odp_packet_t pkt, odp_packet_reass_info_t *info) +{ + (void)pkt; + (void)info; + return -1; +} + int odp_packet_reass_partial_state(odp_packet_t pkt, odp_packet_t frags[], odp_packet_reass_partial_state_t *res) diff --git a/platform/linux-generic/odp_pcapng.c b/platform/linux-generic/odp_pcapng.c index 68a18b305..ce950760e 100644 --- a/platform/linux-generic/odp_pcapng.c +++ b/platform/linux-generic/odp_pcapng.c @@ -90,7 +90,7 @@ int _odp_pcapng_init_global(void) { odp_shm_t shm; - shm = odp_shm_reserve("_odp_pcapng_gbl", sizeof(pcapng_global_t), + shm = odp_shm_reserve("_odp_pcapng_global", sizeof(pcapng_global_t), ODP_PAGE_SIZE, 0); if (shm == ODP_SHM_INVALID) return -1; diff --git a/platform/linux-generic/odp_pool.c b/platform/linux-generic/odp_pool.c index d1fc94369..9c60a9458 100644 --- a/platform/linux-generic/odp_pool.c +++ b/platform/linux-generic/odp_pool.c @@ -271,7 +271,7 @@ int _odp_pool_init_global(void) uint32_t i; odp_shm_t shm; - shm = odp_shm_reserve("_odp_pool_table", + shm = odp_shm_reserve("_odp_pool_global", sizeof(pool_global_t), ODP_CACHE_LINE_SIZE, 0); diff --git a/platform/linux-generic/odp_queue_basic.c b/platform/linux-generic/odp_queue_basic.c index 428022ec8..e35f01f7d 100644 --- a/platform/linux-generic/odp_queue_basic.c +++ b/platform/linux-generic/odp_queue_basic.c @@ -132,7 +132,7 @@ static int queue_init_global(void) _odp_queue_inline_offset.context = offsetof(queue_entry_t, s.param.context); - shm = odp_shm_reserve("_odp_queue_gbl", + shm = odp_shm_reserve("_odp_queue_basic_global", sizeof(queue_global_t), sizeof(queue_entry_t), 0); @@ -161,7 +161,7 @@ static int queue_init_global(void) mem_size = sizeof(uint32_t) * CONFIG_MAX_QUEUES * (uint64_t)_odp_queue_glb->config.max_queue_size; - shm = odp_shm_reserve("_odp_queue_rings", mem_size, + shm = odp_shm_reserve("_odp_queue_basic_rings", mem_size, ODP_CACHE_LINE_SIZE, 0); @@ -677,7 +677,8 @@ static void queue_print(odp_queue_t handle) odp_pktio_info_t pktio_info; queue_entry_t *queue; uint32_t queue_id; - int status; + int status, prio; + int max_prio = odp_schedule_max_prio(); queue_id = queue_to_index(handle); @@ -701,7 +702,7 @@ static void queue_print(odp_queue_t handle) ODP_PRINT("\nQueue info\n"); ODP_PRINT("----------\n"); ODP_PRINT(" handle %p\n", queue->s.handle); - ODP_PRINT(" index %" PRIu32 "\n", queue->s.index); + ODP_PRINT(" index %" PRIu32 "\n", queue_id); ODP_PRINT(" name %s\n", queue->s.name); ODP_PRINT(" enq mode %s\n", queue->s.param.enq_mode == ODP_QUEUE_OP_MT ? "ODP_QUEUE_OP_MT" : @@ -729,8 +730,11 @@ static void queue_print(odp_queue_t handle) "ODP_SCHED_SYNC_ATOMIC" : (queue->s.param.sched.sync == ODP_SCHED_SYNC_ORDERED ? "ODP_SCHED_SYNC_ORDERED" : "unknown"))); - ODP_PRINT(" priority %d\n", queue->s.param.sched.prio); - ODP_PRINT(" group %d\n", queue->s.param.sched.group); + prio = queue->s.param.sched.prio; + ODP_PRINT(" priority %i (%i in API)\n", max_prio - prio, prio); + ODP_PRINT(" group %i\n", queue->s.param.sched.group); + if (_odp_sched_id == _ODP_SCHED_ID_BASIC) + ODP_PRINT(" spread %i\n", _odp_sched_basic_get_spread(queue_id)); } if (queue->s.pktin.pktio != ODP_PKTIO_INVALID) { if (!odp_pktio_info(queue->s.pktin.pktio, &pktio_info)) @@ -784,11 +788,17 @@ static void queue_print_all(void) char type_c, enq_c, deq_c, order_c, sync_c; const int col_width = 24; int prio = 0; + int spr = 0; odp_schedule_sync_t sync = ODP_SCHED_SYNC_PARALLEL; + odp_schedule_group_t grp = ODP_SCHED_GROUP_INVALID; ODP_PRINT("\nList of all queues\n"); ODP_PRINT("------------------\n"); - ODP_PRINT(" idx %-*s type stat blk enq deq ord len max_len sync prio\n", col_width, "name"); + ODP_PRINT(" idx %-*s type stat blk enq deq ord len max_len sync prio grp", col_width, "name"); + if (_odp_sched_id == _ODP_SCHED_ID_BASIC) + ODP_PRINT(" spr\n"); + else + ODP_PRINT("\n"); for (i = 0; i < CONFIG_MAX_QUEUES; i++) { queue_entry_t *queue = qentry_from_index(i); @@ -817,7 +827,10 @@ static void queue_print_all(void) len = ring_st_length(&queue->s.ring_st); max_len = queue->s.ring_mask + 1; prio = queue->s.param.sched.prio; + grp = queue->s.param.sched.group; sync = queue->s.param.sched.sync; + if (_odp_sched_id == _ODP_SCHED_ID_BASIC) + spr = _odp_sched_basic_get_spread(index); } else { len = ring_mpmc_length(&queue->s.ring_mpmc); max_len = queue->s.ring_mask + 1; @@ -851,7 +864,13 @@ static void queue_print_all(void) if (type == ODP_QUEUE_TYPE_SCHED) { sync_c = (sync == ODP_SCHED_SYNC_PARALLEL) ? 'P' : ((sync == ODP_SCHED_SYNC_ATOMIC) ? 'A' : 'O'); - ODP_PRINT(" %c %4i", sync_c, prio); + /* Print prio level matching odp_schedule_print() output */ + prio = odp_schedule_max_prio() - prio; + + ODP_PRINT(" %c %4i %3i", sync_c, prio, grp); + + if (_odp_sched_id == _ODP_SCHED_ID_BASIC) + ODP_PRINT(" %3i", spr); } ODP_PRINT("\n"); diff --git a/platform/linux-generic/odp_queue_lf.c b/platform/linux-generic/odp_queue_lf.c index 70d555ab5..206172bb8 100644 --- a/platform/linux-generic/odp_queue_lf.c +++ b/platform/linux-generic/odp_queue_lf.c @@ -268,7 +268,7 @@ uint32_t _odp_queue_lf_init_global(uint32_t *queue_lf_size, if (!lockfree) return 0; - shm = odp_shm_reserve("_odp_queues_lf", sizeof(queue_lf_global_t), + shm = odp_shm_reserve("_odp_queues_lf_global", sizeof(queue_lf_global_t), ODP_CACHE_LINE_SIZE, 0); if (shm == ODP_SHM_INVALID) diff --git a/platform/linux-generic/odp_schedule_basic.c b/platform/linux-generic/odp_schedule_basic.c index 479d6f956..eb17e95eb 100644 --- a/platform/linux-generic/odp_schedule_basic.c +++ b/platform/linux-generic/odp_schedule_basic.c @@ -78,6 +78,9 @@ ODP_STATIC_ASSERT((QUEUE_LOAD * CONFIG_MAX_SCHED_QUEUES) < UINT32_MAX, "Load_val /* Spread weight table */ #define SPREAD_TBL_SIZE ((MAX_SPREAD - 1) * MAX_PREFER_RATIO) +/* Random data table size */ +#define RANDOM_TBL_SIZE 128 + /* Maximum number of packet IO interfaces */ #define NUM_PKTIO ODP_CONFIG_PKTIO_ENTRIES @@ -133,6 +136,27 @@ typedef union { ODP_STATIC_ASSERT(sizeof(lock_called_t) == sizeof(uint32_t), "Lock_called_values_do_not_fit_in_uint32"); +static uint8_t sched_random_u8[] = { + 0x64, 0xe3, 0x64, 0x0a, 0x0a, 0x5b, 0x7e, 0xd7, + 0x43, 0xb7, 0x90, 0x71, 0x76, 0x17, 0x8e, 0x3f, + 0x17, 0x60, 0x7e, 0xfd, 0x99, 0xe3, 0xab, 0x06, + 0x77, 0xf9, 0x45, 0x17, 0x2f, 0x81, 0x9e, 0x7b, + 0x20, 0x1b, 0x36, 0x75, 0x69, 0xc5, 0x69, 0x27, + 0x7a, 0xf6, 0x3f, 0x63, 0x2c, 0x3f, 0x1b, 0xeb, + 0x12, 0xe1, 0x6f, 0xd4, 0xd9, 0x14, 0x97, 0xa6, + 0x2a, 0xe5, 0xb0, 0x45, 0x27, 0xa6, 0x48, 0xbc, + 0x2b, 0xec, 0xd8, 0xda, 0x55, 0xef, 0x15, 0xce, + 0xf8, 0xc2, 0x1e, 0xc8, 0x16, 0x6c, 0xf0, 0x4f, + 0x1a, 0xc7, 0x50, 0x9e, 0x0b, 0xa5, 0xe9, 0xf3, + 0x28, 0x79, 0x2e, 0x18, 0xb0, 0xb4, 0xac, 0xce, + 0x67, 0x04, 0x52, 0x98, 0xce, 0x8c, 0x05, 0x87, + 0xab, 0xc8, 0x94, 0x7e, 0x46, 0x63, 0x60, 0x8d, + 0x3d, 0x8f, 0x14, 0x85, 0x1e, 0x92, 0xd2, 0x40, + 0x2d, 0x42, 0xfe, 0xf1, 0xc2, 0xb6, 0x03, 0x43 +}; + +ODP_STATIC_ASSERT(sizeof(sched_random_u8) == RANDOM_TBL_SIZE, "Bad_random_table_size"); + /* Scheduler local data */ typedef struct ODP_ALIGNED_CACHE { uint32_t sched_round; @@ -247,6 +271,7 @@ typedef struct { /* Scheduler interface config options (not used in fast path) */ schedule_config_t config_if; uint32_t max_queues; + odp_atomic_u32_t next_rand; } sched_global_t; @@ -438,7 +463,7 @@ static int schedule_init_global(void) ODP_DBG("Schedule init ... "); - shm = odp_shm_reserve("_odp_scheduler", + shm = odp_shm_reserve("_odp_sched_basic_global", sizeof(sched_global_t), ODP_CACHE_LINE_SIZE, 0); @@ -500,6 +525,7 @@ static int schedule_init_global(void) odp_ticketlock_init(&sched->grp_lock); odp_atomic_init_u32(&sched->grp_epoch, 0); + odp_atomic_init_u32(&sched->next_rand, 0); for (i = 0; i < NUM_SCHED_GRPS; i++) { memset(sched->sched_grp[i].name, 0, ODP_SCHED_GROUP_NAME_LEN); @@ -662,25 +688,42 @@ static inline void update_queue_count(int grp, int prio, int old_spr, int new_sp odp_ticketlock_unlock(&sched->mask_lock[grp]); } -/* Select the spread that has least queues */ static uint8_t allocate_spread(int grp, int prio) { - uint8_t i; + uint8_t i, num_min, spr; uint32_t num; uint32_t min = UINT32_MAX; uint8_t num_spread = sched->config.num_spread; - uint8_t spr = 0; + uint8_t min_spr[num_spread]; + + num_min = 1; + min_spr[0] = 0; odp_ticketlock_lock(&sched->mask_lock[grp]); + /* Find spread(s) with the minimum number of queues */ for (i = 0; i < num_spread; i++) { num = sched->prio_q_count[grp][prio][i]; if (num < min) { - spr = i; min = num; + min_spr[0] = i; + num_min = 1; + } else if (num == min) { + min_spr[num_min] = i; + num_min++; } } + spr = min_spr[0]; + + /* When there are multiple minimum spreads, select one randomly */ + if (num_min > 1) { + uint32_t next_rand = odp_atomic_fetch_inc_u32(&sched->next_rand); + uint8_t rand = sched_random_u8[next_rand % RANDOM_TBL_SIZE]; + + spr = min_spr[rand % num_min]; + } + sched->prio_q_mask[grp][prio] |= 1 << spr; sched->prio_q_count[grp][prio][spr]++; @@ -719,6 +762,16 @@ static int schedule_create_queue(uint32_t queue_index, return -1; } + odp_ticketlock_lock(&sched->grp_lock); + + if (sched->sched_grp[grp].allocated == 0) { + odp_ticketlock_unlock(&sched->grp_lock); + ODP_ERR("Group not created: %i\n", grp); + return -1; + } + + odp_ticketlock_unlock(&sched->grp_lock); + spread = allocate_spread(grp, prio); sched->queue[queue_index].grp = grp; @@ -1573,11 +1626,64 @@ static uint64_t schedule_wait_time(uint64_t ns) return ns; } +static inline void spread_thrs_inc(odp_schedule_group_t group, int thr_tbl[], int count) +{ + int thr, i; + uint8_t spread; + + for (i = 0; i < count; i++) { + thr = thr_tbl[i]; + spread = spread_from_index(thr); + sched->sched_grp[group].spread_thrs[spread]++; + } +} + +static inline void spread_thrs_dec(odp_schedule_group_t group, int thr_tbl[], int count) +{ + int thr, i; + uint8_t spread; + + for (i = 0; i < count; i++) { + thr = thr_tbl[i]; + spread = spread_from_index(thr); + sched->sched_grp[group].spread_thrs[spread]--; + } +} + +static inline int threads_from_mask(int thr_tbl[], int count, const odp_thrmask_t *mask) +{ + int i; + int thr = odp_thrmask_first(mask); + + for (i = 0; i < count; i++) { + if (thr < 0) { + ODP_ERR("No more threads in the mask\n"); + return -1; + } + + thr_tbl[i] = thr; + thr = odp_thrmask_next(mask, thr); + } + + return 0; +} + static odp_schedule_group_t schedule_group_create(const char *name, const odp_thrmask_t *mask) { odp_schedule_group_t group = ODP_SCHED_GROUP_INVALID; - int i; + int count, i; + + count = odp_thrmask_count(mask); + if (count < 0) { + ODP_ERR("Bad thread count\n"); + return ODP_SCHED_GROUP_INVALID; + } + + int thr_tbl[count]; + + if (count && threads_from_mask(thr_tbl, count, mask)) + return ODP_SCHED_GROUP_INVALID; odp_ticketlock_lock(&sched->grp_lock); @@ -1595,6 +1701,7 @@ static odp_schedule_group_t schedule_group_create(const char *name, grp_update_mask(i, mask); group = (odp_schedule_group_t)i; + spread_thrs_inc(group, thr_tbl, count); sched->sched_grp[i].allocated = 1; break; } @@ -1607,25 +1714,33 @@ static odp_schedule_group_t schedule_group_create(const char *name, static int schedule_group_destroy(odp_schedule_group_t group) { odp_thrmask_t zero; - int ret; + int i; + + if (group >= NUM_SCHED_GRPS || group < SCHED_GROUP_NAMED) { + ODP_ERR("Bad group %i\n", group); + return -1; + } odp_thrmask_zero(&zero); odp_ticketlock_lock(&sched->grp_lock); - if (group < NUM_SCHED_GRPS && group >= SCHED_GROUP_NAMED && - sched->sched_grp[group].allocated) { - grp_update_mask(group, &zero); - memset(sched->sched_grp[group].name, 0, - ODP_SCHED_GROUP_NAME_LEN); - sched->sched_grp[group].allocated = 0; - ret = 0; - } else { - ret = -1; + if (sched->sched_grp[group].allocated == 0) { + odp_ticketlock_unlock(&sched->grp_lock); + ODP_ERR("Group not created: %i\n", group); + return -1; } + grp_update_mask(group, &zero); + + for (i = 0; i < MAX_SPREAD; i++) + sched->sched_grp[group].spread_thrs[i] = 0; + + memset(sched->sched_grp[group].name, 0, ODP_SCHED_GROUP_NAME_LEN); + sched->sched_grp[group].allocated = 0; + odp_ticketlock_unlock(&sched->grp_lock); - return ret; + return 0; } static odp_schedule_group_t schedule_group_lookup(const char *name) @@ -1649,7 +1764,6 @@ static odp_schedule_group_t schedule_group_lookup(const char *name) static int schedule_group_join(odp_schedule_group_t group, const odp_thrmask_t *mask) { int i, count, thr; - uint8_t spread; odp_thrmask_t new_mask; if (group >= NUM_SCHED_GRPS || group < SCHED_GROUP_NAMED) { @@ -1684,10 +1798,7 @@ static int schedule_group_join(odp_schedule_group_t group, const odp_thrmask_t * return -1; } - for (i = 0; i < count; i++) { - spread = spread_from_index(thr_tbl[i]); - sched->sched_grp[group].spread_thrs[spread]++; - } + spread_thrs_inc(group, thr_tbl, count); odp_thrmask_or(&new_mask, &sched->sched_grp[group].mask, mask); grp_update_mask(group, &new_mask); @@ -1699,7 +1810,6 @@ static int schedule_group_join(odp_schedule_group_t group, const odp_thrmask_t * static int schedule_group_leave(odp_schedule_group_t group, const odp_thrmask_t *mask) { int i, count, thr; - uint8_t spread; odp_thrmask_t new_mask; if (group >= NUM_SCHED_GRPS || group < SCHED_GROUP_NAMED) { @@ -1736,10 +1846,7 @@ static int schedule_group_leave(odp_schedule_group_t group, const odp_thrmask_t return -1; } - for (i = 0; i < count; i++) { - spread = spread_from_index(thr_tbl[i]); - sched->sched_grp[group].spread_thrs[spread]--; - } + spread_thrs_dec(group, thr_tbl, count); odp_thrmask_and(&new_mask, &sched->sched_grp[group].mask, &new_mask); grp_update_mask(group, &new_mask); @@ -1789,7 +1896,6 @@ static int schedule_thr_add(odp_schedule_group_t group, int thr) { odp_thrmask_t mask; odp_thrmask_t new_mask; - uint8_t spread = spread_from_index(thr); if (group < 0 || group >= SCHED_GROUP_NAMED) return -1; @@ -1805,7 +1911,7 @@ static int schedule_thr_add(odp_schedule_group_t group, int thr) } odp_thrmask_or(&new_mask, &sched->sched_grp[group].mask, &mask); - sched->sched_grp[group].spread_thrs[spread]++; + spread_thrs_inc(group, &thr, 1); grp_update_mask(group, &new_mask); odp_ticketlock_unlock(&sched->grp_lock); @@ -1817,7 +1923,6 @@ static int schedule_thr_rem(odp_schedule_group_t group, int thr) { odp_thrmask_t mask; odp_thrmask_t new_mask; - uint8_t spread = spread_from_index(thr); if (group < 0 || group >= SCHED_GROUP_NAMED) return -1; @@ -1834,7 +1939,7 @@ static int schedule_thr_rem(odp_schedule_group_t group, int thr) } odp_thrmask_and(&new_mask, &sched->sched_grp[group].mask, &new_mask); - sched->sched_grp[group].spread_thrs[spread]--; + spread_thrs_dec(group, &thr, 1); grp_update_mask(group, &new_mask); odp_ticketlock_unlock(&sched->grp_lock); @@ -1947,6 +2052,12 @@ static void schedule_print(void) ODP_PRINT("\n"); } +/* Returns spread for queue debug prints */ +int _odp_sched_basic_get_spread(uint32_t queue_index) +{ + return sched->queue[queue_index].spread; +} + /* Fill in scheduler interface */ const schedule_fn_t _odp_schedule_basic_fn = { .pktio_start = schedule_pktio_start, diff --git a/platform/linux-generic/odp_schedule_if.c b/platform/linux-generic/odp_schedule_if.c index 24307ab90..528c4065e 100644 --- a/platform/linux-generic/odp_schedule_if.c +++ b/platform/linux-generic/odp_schedule_if.c @@ -1,4 +1,5 @@ /* Copyright (c) 2016-2018, Linaro Limited + * Copyright (c) 2021, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -25,6 +26,7 @@ extern const schedule_api_t _odp_schedule_scalable_api; const schedule_fn_t *_odp_sched_fn; const schedule_api_t *_odp_sched_api; +int _odp_sched_id; uint64_t odp_schedule_wait_time(uint64_t ns) { @@ -216,12 +218,15 @@ int _odp_schedule_init_global(void) ODP_PRINT("Using scheduler '%s'\n", sched); if (!strcmp(sched, "basic")) { + _odp_sched_id = _ODP_SCHED_ID_BASIC; _odp_sched_fn = &_odp_schedule_basic_fn; _odp_sched_api = &_odp_schedule_basic_api; } else if (!strcmp(sched, "sp")) { + _odp_sched_id = _ODP_SCHED_ID_SP; _odp_sched_fn = &_odp_schedule_sp_fn; _odp_sched_api = &_odp_schedule_sp_api; } else if (!strcmp(sched, "scalable")) { + _odp_sched_id = _ODP_SCHED_ID_SCALABLE; _odp_sched_fn = &_odp_schedule_scalable_fn; _odp_sched_api = &_odp_schedule_scalable_api; } else { diff --git a/platform/linux-generic/odp_schedule_scalable.c b/platform/linux-generic/odp_schedule_scalable.c index 18e738330..10f456eed 100644 --- a/platform/linux-generic/odp_schedule_scalable.c +++ b/platform/linux-generic/odp_schedule_scalable.c @@ -1792,7 +1792,7 @@ static int schedule_init_global(void) uint64_t min_alloc; uint64_t max_alloc; - shm = odp_shm_reserve("_odp_sched_scalable", + shm = odp_shm_reserve("_odp_sched_scalable_global", sizeof(sched_global_t), ODP_CACHE_LINE_SIZE, 0); diff --git a/platform/linux-generic/odp_schedule_sp.c b/platform/linux-generic/odp_schedule_sp.c index c4f8344f1..4f6ce736d 100644 --- a/platform/linux-generic/odp_schedule_sp.c +++ b/platform/linux-generic/odp_schedule_sp.c @@ -172,7 +172,7 @@ static int init_global(void) ODP_DBG("Using SP scheduler\n"); - shm = odp_shm_reserve("sp_scheduler", + shm = odp_shm_reserve("_odp_sched_sp_global", sizeof(sched_global_t), ODP_CACHE_LINE_SIZE, 0); diff --git a/platform/linux-generic/odp_shared_memory.c b/platform/linux-generic/odp_shared_memory.c index ee47b7e96..966850a07 100644 --- a/platform/linux-generic/odp_shared_memory.c +++ b/platform/linux-generic/odp_shared_memory.c @@ -15,9 +15,6 @@ #include <odp_global_data.h> #include <string.h> -ODP_STATIC_ASSERT(ODP_CONFIG_SHM_BLOCKS >= ODP_CONFIG_POOLS, - "ODP_CONFIG_SHM_BLOCKS < ODP_CONFIG_POOLS"); - static inline uint32_t from_handle(odp_shm_t shm) { return _odp_typeval(shm) - 1; @@ -47,7 +44,7 @@ int odp_shm_capability(odp_shm_capability_t *capa) { memset(capa, 0, sizeof(odp_shm_capability_t)); - capa->max_blocks = ODP_CONFIG_SHM_BLOCKS; + capa->max_blocks = CONFIG_SHM_BLOCKS; capa->max_size = odp_global_ro.shm_max_size; capa->max_align = 0; diff --git a/platform/linux-generic/odp_stash.c b/platform/linux-generic/odp_stash.c index d2062379b..a1bf5b63d 100644 --- a/platform/linux-generic/odp_stash.c +++ b/platform/linux-generic/odp_stash.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2020, Nokia +/* Copyright (c) 2020-2021, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -13,6 +13,7 @@ #include <odp/api/plat/strong_types.h> #include <odp_debug_internal.h> +#include <odp_global_data.h> #include <odp_init_internal.h> #include <odp_ring_ptr_internal.h> #include <odp_ring_u32_internal.h> @@ -57,7 +58,12 @@ int _odp_stash_init_global(void) { odp_shm_t shm; - shm = odp_shm_reserve("_odp_stash_table", sizeof(stash_global_t), + if (odp_global_ro.disable.stash) { + ODP_PRINT("Stash is DISABLED\n"); + return 0; + } + + shm = odp_shm_reserve("_odp_stash_global", sizeof(stash_global_t), ODP_CACHE_LINE_SIZE, 0); stash_global = odp_shm_addr(shm); @@ -76,6 +82,9 @@ int _odp_stash_init_global(void) int _odp_stash_term_global(void) { + if (odp_global_ro.disable.stash) + return 0; + if (stash_global == NULL) return 0; @@ -89,6 +98,11 @@ int _odp_stash_term_global(void) int odp_stash_capability(odp_stash_capability_t *capa, odp_stash_type_t type) { + if (odp_global_ro.disable.stash) { + ODP_ERR("Stash is disabled\n"); + return -1; + } + (void)type; memset(capa, 0, sizeof(odp_stash_capability_t)); @@ -146,6 +160,11 @@ odp_stash_t odp_stash_create(const char *name, const odp_stash_param_t *param) int ring_ptr, index; char shm_name[ODP_STASH_NAME_LEN + 8]; + if (odp_global_ro.disable.stash) { + ODP_ERR("Stash is disabled\n"); + return ODP_STASH_INVALID; + } + if (param->obj_size > sizeof(uintptr_t)) { ODP_ERR("Too large object handle.\n"); return ODP_STASH_INVALID; diff --git a/platform/linux-generic/odp_system_info.c b/platform/linux-generic/odp_system_info.c index 27c591a9c..778ea08cb 100644 --- a/platform/linux-generic/odp_system_info.c +++ b/platform/linux-generic/odp_system_info.c @@ -579,7 +579,7 @@ void odp_sys_config_print(void) ODP_PRINT("ODP_CONFIG_PKTIO_ENTRIES: %i\n", ODP_CONFIG_PKTIO_ENTRIES); ODP_PRINT("CONFIG_PACKET_HEADROOM: %i\n", CONFIG_PACKET_HEADROOM); ODP_PRINT("CONFIG_PACKET_TAILROOM: %i\n", CONFIG_PACKET_TAILROOM); - ODP_PRINT("ODP_CONFIG_SHM_BLOCKS: %i\n", ODP_CONFIG_SHM_BLOCKS); + ODP_PRINT("CONFIG_SHM_BLOCKS: %i\n", CONFIG_SHM_BLOCKS); ODP_PRINT("CONFIG_BURST_SIZE: %i\n", CONFIG_BURST_SIZE); ODP_PRINT("CONFIG_POOL_MAX_NUM: %i\n", CONFIG_POOL_MAX_NUM); ODP_PRINT("CONFIG_POOL_CACHE_MAX_SIZE: %i\n", CONFIG_POOL_CACHE_MAX_SIZE); diff --git a/platform/linux-generic/odp_thread.c b/platform/linux-generic/odp_thread.c index af891bce8..ffb116974 100644 --- a/platform/linux-generic/odp_thread.c +++ b/platform/linux-generic/odp_thread.c @@ -68,7 +68,7 @@ int _odp_thread_init_global(void) if (num_max > ODP_THREAD_COUNT_MAX) num_max = ODP_THREAD_COUNT_MAX; - shm = odp_shm_reserve("_odp_thread_globals", + shm = odp_shm_reserve("_odp_thread_global", sizeof(thread_globals_t), ODP_CACHE_LINE_SIZE, 0); @@ -97,7 +97,7 @@ int _odp_thread_term_global(void) if (num) ODP_ERR("%u threads have not called odp_term_local().\n", num); - ret = odp_shm_free(odp_shm_lookup("_odp_thread_globals")); + ret = odp_shm_free(odp_shm_lookup("_odp_thread_global")); if (ret < 0) ODP_ERR("shm free failed for _odp_thread_globals"); diff --git a/platform/linux-generic/odp_timer.c b/platform/linux-generic/odp_timer.c index e5b42a83b..f1ca28346 100644 --- a/platform/linux-generic/odp_timer.c +++ b/platform/linux-generic/odp_timer.c @@ -1599,7 +1599,7 @@ int _odp_timer_init_global(const odp_init_t *params) return 0; } - shm = odp_shm_reserve("_odp_timer", sizeof(timer_global_t), + shm = odp_shm_reserve("_odp_timer_global", sizeof(timer_global_t), ODP_CACHE_LINE_SIZE, 0); timer_global = odp_shm_addr(shm); diff --git a/platform/linux-generic/odp_traffic_mngr.c b/platform/linux-generic/odp_traffic_mngr.c index de01af96c..e741fd80c 100644 --- a/platform/linux-generic/odp_traffic_mngr.c +++ b/platform/linux-generic/odp_traffic_mngr.c @@ -32,6 +32,7 @@ #include <odp_init_internal.h> #include <odp_errno_define.h> #include <odp_global_data.h> +#include <odp_schedule_if.h> /* Local vars */ static const @@ -4935,7 +4936,7 @@ int _odp_tm_init_global(void) return 0; } - shm = odp_shm_reserve("_odp_traffic_mng", sizeof(tm_global_t), 0, 0); + shm = odp_shm_reserve("_odp_traffic_mng_global", sizeof(tm_global_t), 0, 0); if (shm == ODP_SHM_INVALID) return -1; diff --git a/platform/linux-generic/pktio/dpdk.c b/platform/linux-generic/pktio/dpdk.c index a98324ee4..9abba6292 100644 --- a/platform/linux-generic/pktio/dpdk.c +++ b/platform/linux-generic/pktio/dpdk.c @@ -288,7 +288,7 @@ static inline void mbuf_update(struct rte_mbuf *mbuf, odp_packet_hdr_t *pkt_hdr, mbuf->refcnt = 1; mbuf->ol_flags = 0; - if (odp_unlikely(pkt_hdr->buf_hdr.base_data != pkt_hdr->seg_data)) + if (odp_unlikely(((uint8_t *)mbuf->buf_addr + mbuf->data_off) != pkt_hdr->seg_data)) mbuf->data_off = mbuf_data_off(mbuf, pkt_hdr); } diff --git a/platform/linux-generic/pktio/dpdk_parse.c b/platform/linux-generic/pktio/dpdk_parse.c index f593f4b11..c1b75f126 100644 --- a/platform/linux-generic/pktio/dpdk_parse.c +++ b/platform/linux-generic/pktio/dpdk_parse.c @@ -1,4 +1,5 @@ /* Copyright (c) 2018, Linaro Limited + * Copyright (c) 2021, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -53,7 +54,7 @@ static inline uint16_t dpdk_parse_eth(packet_parser_t *prs, eth = (const _odp_ethhdr_t *)*parseptr; /* Detect jumbo frames */ - if (odp_unlikely(frame_len > _ODP_ETH_LEN_MAX)) + if (odp_unlikely(frame_len - *offset > _ODP_ETH_LEN_MAX)) input_flags.jumbo = 1; /* Handle Ethernet broadcast/multicast addresses */ @@ -124,6 +125,16 @@ static inline uint16_t dpdk_parse_eth(packet_parser_t *prs, *parseptr += sizeof(_odp_vlanhdr_t); } + /* + * The packet was too short for what we parsed. We just give up + * entirely without trying to parse what fits in the packet. + */ + if (odp_unlikely(*offset > frame_len)) { + input_flags.all = 0; + input_flags.l2 = 1; + ethtype = 0; + } + error: prs->input_flags.all |= input_flags.all; @@ -151,6 +162,7 @@ static inline uint8_t dpdk_parse_ipv4(packet_parser_t *prs, if (odp_unlikely(ihl < _ODP_IPV4HDR_IHL_MIN || ver != 4 || + sizeof(*ipv4) > frame_len - *offset || (l3_len > frame_len - *offset))) { prs->flags.ip_err = 1; return 0; @@ -218,8 +230,9 @@ static inline uint8_t dpdk_parse_ipv6(packet_parser_t *prs, uint32_t l4_packet_type = mbuf_packet_type & RTE_PTYPE_L4_MASK; /* Basic sanity checks on IPv6 header */ - if ((odp_be_to_cpu_32(ipv6->ver_tc_flow) >> 28) != 6 || - l3_len > frame_len - *offset) { + if (odp_unlikely((odp_be_to_cpu_32(ipv6->ver_tc_flow) >> 28) != 6 || + sizeof(*ipv6) > frame_len - *offset || + l3_len > frame_len - *offset)) { prs->flags.ip_err = 1; return 0; } diff --git a/platform/linux-generic/pktio/loop.c b/platform/linux-generic/pktio/loop.c index 437977771..889a270ea 100644 --- a/platform/linux-generic/pktio/loop.c +++ b/platform/linux-generic/pktio/loop.c @@ -172,9 +172,16 @@ static int loopback_recv(pktio_entry_t *pktio_entry, int index ODP_UNUSED, pkt_hdr = packet_hdr(new_pkt); } } else { - _odp_packet_parse_layer(pkt_hdr, - pktio_entry->s.config.parser.layer, - pktio_entry->s.in_chksums); + odp_packet_parse_param_t param; + + /* + * Use odp_packet_parse() which can handle segmented + * packets. + */ + param.proto = ODP_PROTO_ETH; + param.last_layer = pktio_entry->s.config.parser.layer; + param.chksums = pktio_entry->s.in_chksums; + odp_packet_parse(packet_handle(pkt_hdr), 0, ¶m); } packet_set_ts(pkt_hdr, ts); |