diff options
Diffstat (limited to 'platform')
94 files changed, 2923 insertions, 1117 deletions
diff --git a/platform/linux-dpdk/Makefile.am b/platform/linux-dpdk/Makefile.am index da1775993..2f257044e 100644 --- a/platform/linux-dpdk/Makefile.am +++ b/platform/linux-dpdk/Makefile.am @@ -61,6 +61,7 @@ odpapiabiarchinclude_HEADERS += \ include-abi/odp/api/abi/cpumask.h \ include-abi/odp/api/abi/crypto.h \ include-abi/odp/api/abi/debug.h \ + include-abi/odp/api/abi/dma_types.h \ include-abi/odp/api/abi/errno.h \ include-abi/odp/api/abi/event.h \ include-abi/odp/api/abi/hash.h \ @@ -90,7 +91,7 @@ odpapiabiarchinclude_HEADERS += \ include-abi/odp/api/abi/thrmask.h \ include-abi/odp/api/abi/ticketlock.h \ include-abi/odp/api/abi/time.h \ - include-abi/odp/api/abi/timer.h \ + include-abi/odp/api/abi/timer_types.h \ include-abi/odp/api/abi/traffic_mngr.h \ include-abi/odp/api/abi/version.h endif @@ -118,6 +119,7 @@ noinst_HEADERS = \ ${top_srcdir}/platform/linux-generic/include/odp_name_table_internal.h \ include/odp_packet_io_internal.h \ include/odp_errno_define.h \ + include/odp_event_internal.h \ include/odp_packet_dpdk.h \ ${top_srcdir}/platform/linux-generic/include/odp_pcapng.h \ ${top_srcdir}/platform/linux-generic/include/odp_pkt_queue_internal.h \ @@ -145,7 +147,7 @@ noinst_HEADERS = \ ${top_srcdir}/platform/linux-generic/include/odp_timer_internal.h \ ${top_srcdir}/platform/linux-generic/include/odp_timer_wheel_internal.h \ ${top_srcdir}/platform/linux-generic/include/odp_traffic_mngr_internal.h \ - ${top_srcdir}/platform/linux-generic/include/odp_event_vector_internal.h \ + include/odp_event_vector_internal.h \ include/protocols/eth.h \ include/protocols/ip.h \ include/protocols/ipsec.h \ @@ -169,6 +171,7 @@ __LIB__libodp_dpdk_la_SOURCES = \ ../linux-generic/miniz/miniz_tinfl.c ../linux-generic/miniz/miniz_tinfl.h \ ../linux-generic/odp_cpumask.c \ ../linux-generic/odp_cpumask_task.c \ + ../linux-generic/odp_dma.c \ odp_crypto.c \ odp_errno.c \ ../linux-generic/odp_event.c \ @@ -245,6 +248,7 @@ __LIB__libodp_dpdk_la_SOURCES += arch/default/odp_atomic.c \ arch/default/odp_cpu_cycles.c \ arch/default/odp_global_time.c \ arch/default/odp_hash_crc32.c \ + arch/default/odp_random.c \ arch/arm/odp_sysinfo_parse.c odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_time.h \ arch/default/odp/api/abi/hash_crc32.h @@ -261,7 +265,9 @@ noinst_HEADERS += arch/arm/odp_atomic.h \ arch/arm/odp_llsc.h \ arch/default/odp_atomic.h \ arch/default/odp_cpu.h \ - arch/default/odp_cpu_idling.h + arch/default/odp_cpu_idling.h \ + arch/default/odp_random.h + endif if ARCH_IS_AARCH64 __LIB__libodp_dpdk_la_SOURCES += arch/aarch64/odp_atomic.c \ @@ -269,6 +275,7 @@ __LIB__libodp_dpdk_la_SOURCES += arch/aarch64/odp_atomic.c \ arch/aarch64/cpu_flags.c \ arch/aarch64/odp_global_time.c \ arch/default/odp_hash_crc32.c \ + arch/default/odp_random.c \ arch/aarch64/odp_sysinfo_parse.c odpapiabiarchinclude_HEADERS += arch/aarch64/odp/api/abi/cpu_time.h \ arch/aarch64/odp/api/abi/hash_crc32.h @@ -284,13 +291,15 @@ noinst_HEADERS += arch/aarch64/odp_atomic.h \ arch/aarch64/odp_cpu.h \ arch/aarch64/cpu_flags.h \ arch/aarch64/odp_cpu_idling.h \ - arch/aarch64/odp_llsc.h + arch/aarch64/odp_llsc.h \ + arch/default/odp_random.h endif if ARCH_IS_DEFAULT __LIB__libodp_dpdk_la_SOURCES += arch/default/odp_atomic.c \ arch/default/odp_cpu_cycles.c \ arch/default/odp_global_time.c \ arch/default/odp_hash_crc32.c \ + arch/default/odp_random.c \ arch/default/odp_sysinfo_parse.c odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_time.h \ arch/default/odp/api/abi/hash_crc32.h @@ -303,12 +312,14 @@ odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ endif noinst_HEADERS += arch/default/odp_atomic.h \ arch/default/odp_cpu.h \ - arch/default/odp_cpu_idling.h + arch/default/odp_cpu_idling.h \ + arch/default/odp_random.h endif if ARCH_IS_MIPS64 __LIB__libodp_dpdk_la_SOURCES += arch/default/odp_atomic.c \ arch/default/odp_global_time.c \ arch/default/odp_hash_crc32.c \ + arch/default/odp_random.c \ arch/mips64/odp_sysinfo_parse.c odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_time.h \ arch/default/odp/api/abi/hash_crc32.h @@ -321,13 +332,15 @@ odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ endif noinst_HEADERS += arch/default/odp_atomic.h \ arch/default/odp_cpu.h \ - arch/default/odp_cpu_idling.h + arch/default/odp_cpu_idling.h \ + arch/default/odp_random.h endif if ARCH_IS_POWERPC __LIB__libodp_dpdk_la_SOURCES += arch/default/odp_atomic.c \ arch/default/odp_cpu_cycles.c \ arch/default/odp_global_time.c \ arch/default/odp_hash_crc32.c \ + arch/default/odp_random.c \ arch/powerpc/odp_sysinfo_parse.c odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_time.h \ arch/default/odp/api/abi/hash_crc32.h @@ -340,7 +353,8 @@ odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ endif noinst_HEADERS += arch/default/odp_atomic.h \ arch/default/odp_cpu.h \ - arch/default/odp_cpu_idling.h + arch/default/odp_cpu_idling.h \ + arch/default/odp_random.h endif if ARCH_IS_X86 __LIB__libodp_dpdk_la_SOURCES += arch/default/odp_atomic.c \ @@ -348,6 +362,7 @@ __LIB__libodp_dpdk_la_SOURCES += arch/default/odp_atomic.c \ arch/x86/odp_cpu_cycles.c \ arch/x86/odp_global_time.c \ arch/default/odp_hash_crc32.c \ + arch/default/odp_random.c \ arch/x86/odp_sysinfo_parse.c odpapiabiarchinclude_HEADERS += arch/x86/odp/api/abi/cpu_rdtsc.h \ arch/x86/odp/api/abi/cpu_time.h \ @@ -360,6 +375,7 @@ odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ endif noinst_HEADERS += arch/x86/cpu_flags.h \ arch/x86/odp_cpu.h \ + arch/x86/odp_random.h \ arch/default/odp_atomic.h \ arch/default/odp_cpu.h \ arch/default/odp_cpu_idling.h diff --git a/platform/linux-dpdk/arch/default/odp_random.c b/platform/linux-dpdk/arch/default/odp_random.c new file mode 120000 index 000000000..a1889b546 --- /dev/null +++ b/platform/linux-dpdk/arch/default/odp_random.c @@ -0,0 +1 @@ +../../../linux-generic/arch/default/odp_random.c
\ No newline at end of file diff --git a/platform/linux-dpdk/arch/default/odp_random.h b/platform/linux-dpdk/arch/default/odp_random.h new file mode 120000 index 000000000..232858671 --- /dev/null +++ b/platform/linux-dpdk/arch/default/odp_random.h @@ -0,0 +1 @@ +../../../linux-generic/arch/default/odp_random.h
\ No newline at end of file diff --git a/platform/linux-dpdk/arch/x86/odp_random.h b/platform/linux-dpdk/arch/x86/odp_random.h new file mode 120000 index 000000000..50a20427d --- /dev/null +++ b/platform/linux-dpdk/arch/x86/odp_random.h @@ -0,0 +1 @@ +../../../linux-generic/arch/x86/odp_random.h
\ No newline at end of file diff --git a/platform/linux-dpdk/include-abi/odp/api/abi/dma_types.h b/platform/linux-dpdk/include-abi/odp/api/abi/dma_types.h new file mode 120000 index 000000000..40558269d --- /dev/null +++ b/platform/linux-dpdk/include-abi/odp/api/abi/dma_types.h @@ -0,0 +1 @@ +../../../../../linux-generic/include-abi/odp/api/abi/dma_types.h
\ No newline at end of file diff --git a/platform/linux-dpdk/include-abi/odp/api/abi/timer.h b/platform/linux-dpdk/include-abi/odp/api/abi/timer.h deleted file mode 120000 index e65d4faf5..000000000 --- a/platform/linux-dpdk/include-abi/odp/api/abi/timer.h +++ /dev/null @@ -1 +0,0 @@ -../../../../../linux-generic/include-abi/odp/api/abi/timer.h
\ No newline at end of file diff --git a/platform/linux-dpdk/include-abi/odp/api/abi/timer_types.h b/platform/linux-dpdk/include-abi/odp/api/abi/timer_types.h new file mode 120000 index 000000000..4b815a27b --- /dev/null +++ b/platform/linux-dpdk/include-abi/odp/api/abi/timer_types.h @@ -0,0 +1 @@ +../../../../../linux-generic/include-abi/odp/api/abi/timer_types.h
\ No newline at end of file diff --git a/platform/linux-dpdk/include/odp_buffer_internal.h b/platform/linux-dpdk/include/odp_buffer_internal.h index 674c6d716..42b686a2a 100644 --- a/platform/linux-dpdk/include/odp_buffer_internal.h +++ b/platform/linux-dpdk/include/odp_buffer_internal.h @@ -1,4 +1,5 @@ /* Copyright (c) 2013-2018, Linaro Limited + * Copyright (c) 2021, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -17,18 +18,20 @@ extern "C" { #endif -#include <odp/api/std_types.h> -#include <odp/api/pool.h> +#include <odp/api/align.h> #include <odp/api/buffer.h> +#include <odp/api/byteorder.h> #include <odp/api/debug.h> -#include <odp/api/align.h> +#include <odp/api/event.h> +#include <odp/api/pool.h> +#include <odp/api/std_types.h> +#include <odp/api/thread.h> + #include <odp_align_internal.h> #include <odp_config_internal.h> -#include <odp/api/byteorder.h> -#include <odp/api/thread.h> +#include <odp_event_internal.h> + #include <sys/types.h> -#include <odp/api/event.h> -#include <odp_forward_typedefs_internal.h> #include <stddef.h> /* DPDK */ @@ -45,36 +48,12 @@ extern "C" { /* Type size limits number of flow IDs supported */ #define BUF_HDR_MAX_FLOW_ID 255 -struct odp_buffer_hdr_t { - /* Underlying DPDK rte_mbuf */ - struct rte_mbuf mb; - - /* Buffer index in the pool */ - uint32_t index; - - /* Total size of all allocated segs */ - uint32_t totsize; - - /* Pool type */ - int8_t type; +/* Internal buffer header */ +typedef struct ODP_ALIGNED_CACHE odp_buffer_hdr_t { + /* Common event header */ + _odp_event_hdr_t event_hdr; - /* Event type. Maybe different than pool type (crypto compl event) */ - int8_t event_type; - - /* Event flow id */ - uint8_t flow_id; - - /* --- Mostly read only data --- */ - - /* User pointer */ - const void *user_ptr; - - /* Pool pointer */ - void *pool_ptr; - - /* User area pointer */ - void *uarea_addr; -}; +} odp_buffer_hdr_t; /* * Buffer type @@ -94,48 +73,28 @@ int _odp_buffer_type(odp_buffer_t buf); */ void _odp_buffer_type_set(odp_buffer_t buf, int type); -static inline struct rte_mbuf *buf_to_mbuf(odp_buffer_t buf) +static inline struct rte_mbuf *_odp_buf_to_mbuf(odp_buffer_t buf) { return (struct rte_mbuf *)(uintptr_t)buf; } -static inline odp_buffer_hdr_t *mbuf_to_buf_hdr(struct rte_mbuf *mbuf) -{ - return (odp_buffer_hdr_t *)(uintptr_t)mbuf; -} - -static inline odp_buffer_t buf_from_buf_hdr(odp_buffer_hdr_t *hdr) -{ - return (odp_buffer_t)hdr; -} - -static inline odp_buffer_hdr_t *buf_hdl_to_hdr(odp_buffer_t buf) +static inline odp_buffer_hdr_t *_odp_buf_hdr(odp_buffer_t buf) { return (odp_buffer_hdr_t *)(uintptr_t)buf; } -static inline odp_event_type_t _odp_buffer_event_type(odp_buffer_t buf) -{ - return buf_hdl_to_hdr(buf)->event_type; -} - -static inline void _odp_buffer_event_type_set(odp_buffer_t buf, int ev) -{ - buf_hdl_to_hdr(buf)->event_type = ev; -} - static inline uint32_t event_flow_id(odp_event_t ev) { odp_buffer_hdr_t *buf_hdr = (odp_buffer_hdr_t *)(uintptr_t)ev; - return buf_hdr->flow_id; + return buf_hdr->event_hdr.flow_id; } static inline void event_flow_id_set(odp_event_t ev, uint32_t flow_id) { odp_buffer_hdr_t *buf_hdr = (odp_buffer_hdr_t *)(uintptr_t)ev; - buf_hdr->flow_id = flow_id; + buf_hdr->event_hdr.flow_id = flow_id; } #ifdef __cplusplus diff --git a/platform/linux-dpdk/include/odp_config_internal.h b/platform/linux-dpdk/include/odp_config_internal.h index 0bbda1a4e..18754184e 100644 --- a/platform/linux-dpdk/include/odp_config_internal.h +++ b/platform/linux-dpdk/include/odp_config_internal.h @@ -58,6 +58,16 @@ extern "C" { #define CONFIG_QUEUE_MAX_ORD_LOCKS 2 /* + * Maximum number of DMA sessions + */ +#define CONFIG_MAX_DMA_SESSIONS 32 + +/* + * Stashes reserved for internal usage + */ +#define CONFIG_INTERNAL_STASHES CONFIG_MAX_DMA_SESSIONS + +/* * Maximum number of stashes */ #define CONFIG_MAX_STASHES 128 diff --git a/platform/linux-dpdk/include/odp_event_internal.h b/platform/linux-dpdk/include/odp_event_internal.h new file mode 100644 index 000000000..9221def98 --- /dev/null +++ b/platform/linux-dpdk/include/odp_event_internal.h @@ -0,0 +1,102 @@ +/* Copyright (c) 2021, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +/** + * @file + * + * ODP event descriptor - implementation internal + */ + +#ifndef ODP_EVENT_INTERNAL_H_ +#define ODP_EVENT_INTERNAL_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <odp/api/event.h> + +#include <stdint.h> + +/* DPDK */ +#include <rte_config.h> +#if defined(__clang__) +#undef RTE_TOOLCHAIN_GCC +#endif +#include <rte_mbuf.h> +/* ppc64 rte_memcpy.h (included through rte_mbuf.h) may define vector */ +#if defined(__PPC64__) && defined(vector) + #undef vector +#endif + +/* Common header for all event types. */ +typedef struct _odp_event_hdr_t { + /* Underlying DPDK rte_mbuf */ + struct rte_mbuf mb; + + /* Buffer index in the pool */ + uint32_t index; + + /* Total size of all allocated segs */ + uint32_t totsize; + + /* Pool type */ + int8_t type; + + /* Event type. Maybe different than pool type (crypto compl event) */ + int8_t event_type; + + /* Event flow id */ + uint8_t flow_id; + + /* --- Mostly read only data --- */ + + /* User pointer */ + const void *user_ptr; + + /* Pool pointer */ + void *pool_ptr; + + /* User area pointer */ + void *uarea_addr; + +} _odp_event_hdr_t; + +static inline odp_event_t _odp_event_from_hdr(_odp_event_hdr_t *hdr) +{ + return (odp_event_t)hdr; +} + +static inline _odp_event_hdr_t *_odp_event_hdr(odp_event_t event) +{ + return (_odp_event_hdr_t *)(uintptr_t)event; +} + +static inline odp_event_t _odp_event_from_mbuf(struct rte_mbuf *mbuf) +{ + return (odp_event_t)(uintptr_t)mbuf; +} + +static inline struct rte_mbuf *_odp_event_to_mbuf(odp_event_t event) +{ + return (struct rte_mbuf *)(uintptr_t)event; +} + +static inline odp_event_type_t _odp_event_type(odp_event_t event) +{ + return _odp_event_hdr(event)->event_type; +} + +static inline void _odp_event_type_set(odp_event_t event, int ev) +{ + _odp_event_hdr(event)->event_type = ev; +} + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-dpdk/include/odp_event_vector_internal.h b/platform/linux-dpdk/include/odp_event_vector_internal.h new file mode 100644 index 000000000..c866d9036 --- /dev/null +++ b/platform/linux-dpdk/include/odp_event_vector_internal.h @@ -0,0 +1,60 @@ +/* Copyright (c) 2020-2021, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +/** + * @file + * + * ODP event vector descriptor - implementation internal + */ + +#ifndef ODP_EVENT_VECTOR_INTERNAL_H_ +#define ODP_EVENT_VECTOR_INTERNAL_H_ + +#include <odp/api/align.h> +#include <odp/api/debug.h> +#include <odp/api/packet.h> + +#include <odp_event_internal.h> + +#include <stdint.h> + +/** + * Internal event vector header + */ +typedef struct ODP_ALIGNED_CACHE odp_event_vector_hdr_t { + /* Common event header */ + _odp_event_hdr_t event_hdr; + + /* Event vector size */ + uint32_t size; + + /* Vector of packet handles */ + odp_packet_t packet[]; + +} odp_event_vector_hdr_t; + +/** + * Return the vector header + */ +static inline odp_event_vector_hdr_t *_odp_packet_vector_hdr(odp_packet_vector_t pktv) +{ + return (odp_event_vector_hdr_t *)(uintptr_t)pktv; +} + +/** + * Free packet vector and contained packets + */ +static inline void _odp_packet_vector_free_full(odp_packet_vector_t pktv) +{ + odp_event_vector_hdr_t *pktv_hdr = _odp_packet_vector_hdr(pktv); + + if (pktv_hdr->size) + odp_packet_free_multi(pktv_hdr->packet, pktv_hdr->size); + + odp_packet_vector_free(pktv); +} + +#endif /* ODP_EVENT_VECTOR_INTERNAL_H_ */ diff --git a/platform/linux-dpdk/include/odp_packet_internal.h b/platform/linux-dpdk/include/odp_packet_internal.h index ca789a3a1..a3e806f54 100644 --- a/platform/linux-dpdk/include/odp_packet_internal.h +++ b/platform/linux-dpdk/include/odp_packet_internal.h @@ -1,4 +1,5 @@ /* Copyright (c) 2014-2018, Linaro Limited + * Copyright (c) 2021, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -18,20 +19,19 @@ extern "C" { #endif #include <odp/api/align.h> -#include <odp_debug_internal.h> #include <odp/api/debug.h> -#include <odp_buffer_internal.h> -#include <odp_pool_internal.h> #include <odp/api/packet.h> #include <odp/api/plat/packet_inline_types.h> #include <odp/api/packet_io.h> #include <odp/api/crypto.h> #include <odp/api/comp.h> -#include <odp_ipsec_internal.h> #include <odp/api/abi/packet.h> -#include <protocols/eth.h> -#include <odp_queue_if.h> + #include <odp_config_internal.h> +#include <odp_event_internal.h> +#include <odp_pool_internal.h> + +#include <protocols/eth.h> #include <rte_config.h> #if defined(__clang__) @@ -112,9 +112,9 @@ ODP_STATIC_ASSERT(CONFIG_PACKET_MAX_SEG_LEN <= UINT16_MAX, * packet_init(). Because of this any new fields added must be reviewed for * initialization requirements. */ -typedef struct { - /* common buffer header */ - odp_buffer_hdr_t buf_hdr; +typedef struct odp_packet_hdr_t { + /* Common event header */ + _odp_event_hdr_t event_hdr; packet_parser_t p; @@ -179,19 +179,19 @@ static inline odp_packet_t packet_handle(odp_packet_hdr_t *pkt_hdr) return (odp_packet_t)pkt_hdr; } -static inline struct rte_mbuf *pkt_to_mbuf(odp_packet_t pkt) +static inline _odp_event_hdr_t *packet_to_event_hdr(odp_packet_t pkt) { - return (struct rte_mbuf *)(uintptr_t)pkt; + return (_odp_event_hdr_t *)(uintptr_t)&packet_hdr(pkt)->event_hdr; } -static inline odp_buffer_hdr_t *packet_to_buf_hdr(odp_packet_t pkt) +static inline odp_packet_t packet_from_event_hdr(_odp_event_hdr_t *event_hdr) { - return &packet_hdr(pkt)->buf_hdr; + return (odp_packet_t)(uintptr_t)event_hdr; } -static inline odp_packet_t packet_from_buf_hdr(odp_buffer_hdr_t *buf_hdr) +static inline struct rte_mbuf *pkt_to_mbuf(odp_packet_t pkt) { - return (odp_packet_t)(odp_packet_hdr_t *)buf_hdr; + return (struct rte_mbuf *)(uintptr_t)pkt; } static inline void packet_subtype_set(odp_packet_t pkt, int ev) @@ -236,12 +236,12 @@ static inline void copy_packet_cls_metadata(odp_packet_hdr_t *src_hdr, static inline uint32_t packet_len(odp_packet_hdr_t *pkt_hdr) { - return rte_pktmbuf_pkt_len(&pkt_hdr->buf_hdr.mb); + return rte_pktmbuf_pkt_len(&pkt_hdr->event_hdr.mb); } static inline void packet_set_len(odp_packet_hdr_t *pkt_hdr, uint32_t len) { - rte_pktmbuf_pkt_len(&pkt_hdr->buf_hdr.mb) = len; + rte_pktmbuf_pkt_len(&pkt_hdr->event_hdr.mb) = len; } /* Forward declarations */ diff --git a/platform/linux-dpdk/include/odp_packet_io_internal.h b/platform/linux-dpdk/include/odp_packet_io_internal.h index 898709008..85900e791 100644 --- a/platform/linux-dpdk/include/odp_packet_io_internal.h +++ b/platform/linux-dpdk/include/odp_packet_io_internal.h @@ -18,18 +18,18 @@ extern "C" { #endif +#include <odp/api/hints.h> #include <odp/api/packet_io.h> #include <odp/api/plat/pktio_inlines.h> #include <odp/api/spinlock.h> #include <odp/api/ticketlock.h> -#include <odp_classification_datamodel.h> + #include <odp_align_internal.h> +#include <odp_classification_datamodel.h> +#include <odp_config_internal.h> #include <odp_debug_internal.h> #include <odp_queue_if.h> -#include <odp_config_internal.h> -#include <odp/api/hints.h> - #include <linux/if_ether.h> #include <sys/select.h> #include <inttypes.h> diff --git a/platform/linux-dpdk/include/odp_pool_internal.h b/platform/linux-dpdk/include/odp_pool_internal.h index 39ca0f002..107ba26db 100644 --- a/platform/linux-dpdk/include/odp_pool_internal.h +++ b/platform/linux-dpdk/include/odp_pool_internal.h @@ -20,7 +20,7 @@ extern "C" { #include <odp/api/std_types.h> #include <odp/api/pool.h> -#include <odp_buffer_internal.h> +#include <odp_event_internal.h> #include <odp/api/packet_io.h> #include <odp/api/align.h> #include <odp/api/hints.h> @@ -72,6 +72,7 @@ typedef struct ODP_ALIGNED_CACHE { uint32_t hdr_size; uint32_t num; uint32_t num_populated; + odp_pool_type_t type_2; uint8_t type; uint8_t pool_ext; odp_pool_param_t params; @@ -106,8 +107,7 @@ static inline pool_t *pool_entry_from_hdl(odp_pool_t pool_hdl) return &_odp_pool_glb->pool[_odp_typeval(pool_hdl) - 1]; } -static inline int _odp_buffer_alloc_multi(pool_t *pool, - odp_buffer_hdr_t *buf_hdr[], int num) +static inline int _odp_event_alloc_multi(pool_t *pool, _odp_event_hdr_t *event_hdr[], int num) { int i; struct rte_mempool *mp = pool->rte_mempool; @@ -119,21 +119,41 @@ static inline int _odp_buffer_alloc_multi(pool_t *pool, if (odp_unlikely(mbuf == NULL)) return i; - buf_hdr[i] = mbuf_to_buf_hdr(mbuf); + event_hdr[i] = _odp_event_hdr(_odp_event_from_mbuf(mbuf)); } return i; } -static inline void _odp_buffer_free_multi(odp_buffer_hdr_t *buf_hdr[], int num) +static inline odp_event_t _odp_event_alloc(pool_t *pool) +{ + struct rte_mbuf *mbuf; + struct rte_mempool *mp = pool->rte_mempool; + + mbuf = rte_mbuf_raw_alloc(mp); + if (odp_unlikely(mbuf == NULL)) + return ODP_EVENT_INVALID; + + return _odp_event_from_mbuf(mbuf); +} + +static inline void _odp_event_free_multi(_odp_event_hdr_t *event_hdr[], int num_free) { int i; - for (i = 0; i < num; i++) - rte_mbuf_raw_free((struct rte_mbuf *)(uintptr_t)buf_hdr[i]); + for (i = 0; i < num_free; i++) + rte_mbuf_raw_free(_odp_event_to_mbuf(_odp_event_from_hdr(event_hdr[i]))); } -int _odp_buffer_is_valid(odp_buffer_t buf); +static inline void _odp_event_free(odp_event_t event) +{ + rte_mbuf_raw_free(_odp_event_to_mbuf(event)); +} + +int _odp_event_is_valid(odp_event_t event); + +odp_pool_t _odp_pool_create(const char *name, const odp_pool_param_t *params, + odp_pool_type_t type_2); #ifdef __cplusplus } diff --git a/platform/linux-dpdk/m4/configure.m4 b/platform/linux-dpdk/m4/configure.m4 index aa369415c..90d55b6f6 100644 --- a/platform/linux-dpdk/m4/configure.m4 +++ b/platform/linux-dpdk/m4/configure.m4 @@ -6,6 +6,7 @@ ODP_ATOMIC m4_include([platform/linux-dpdk/m4/odp_cpu.m4]) m4_include([platform/linux-dpdk/m4/odp_libconfig.m4]) +m4_include([platform/linux-dpdk/m4/odp_openssl.m4]) m4_include([platform/linux-dpdk/m4/odp_pcapng.m4]) m4_include([platform/linux-dpdk/m4/odp_scheduler.m4]) @@ -67,6 +68,8 @@ AS_VAR_APPEND([PLAT_DEP_LIBS], ["${ATOMIC_LIBS} ${LIBCONFIG_LIBS} ${OPENSSL_LIBS # Add text to the end of configure with platform specific settings. # Make sure it's aligned same as other lines in configure.ac. AS_VAR_APPEND([PLAT_CFG_TEXT], [" + openssl: ${with_openssl} + openssl_rand: ${openssl_rand} pcap: ${have_pmd_pcap} pcapng: ${have_pcapng} default_config_path: ${default_config_path}"]) diff --git a/platform/linux-dpdk/m4/odp_pthread.m4 b/platform/linux-dpdk/m4/odp_pthread.m4 deleted file mode 120000 index e24304ae3..000000000 --- a/platform/linux-dpdk/m4/odp_pthread.m4 +++ /dev/null @@ -1 +0,0 @@ -../../linux-generic/m4/odp_pthread.m4
\ No newline at end of file diff --git a/platform/linux-dpdk/m4/odp_timer.m4 b/platform/linux-dpdk/m4/odp_timer.m4 deleted file mode 120000 index 2a909824b..000000000 --- a/platform/linux-dpdk/m4/odp_timer.m4 +++ /dev/null @@ -1 +0,0 @@ -../../linux-generic/m4/odp_timer.m4
\ No newline at end of file diff --git a/platform/linux-dpdk/odp_buffer.c b/platform/linux-dpdk/odp_buffer.c index b8bccd3ba..21956be2f 100644 --- a/platform/linux-dpdk/odp_buffer.c +++ b/platform/linux-dpdk/odp_buffer.c @@ -1,4 +1,5 @@ /* Copyright (c) 2013-2018, Linaro Limited + * Copyright (c) 2021, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -18,36 +19,36 @@ /* Fill in buffer header field offsets for inline functions */ const _odp_buffer_inline_offset_t _odp_buffer_inline_offset ODP_ALIGNED_CACHE = { - .event_type = offsetof(odp_buffer_hdr_t, event_type), - .base_data = offsetof(odp_buffer_hdr_t, mb.buf_addr) + .event_type = offsetof(odp_buffer_hdr_t, event_hdr.event_type), + .base_data = offsetof(odp_buffer_hdr_t, event_hdr.mb.buf_addr) }; #include <odp/visibility_end.h> uint32_t odp_buffer_size(odp_buffer_t buf) { - struct rte_mbuf *mbuf = buf_to_mbuf(buf); + struct rte_mbuf *mbuf = _odp_buf_to_mbuf(buf); return mbuf->buf_len; } int _odp_buffer_type(odp_buffer_t buf) { - odp_buffer_hdr_t *hdr = buf_hdl_to_hdr(buf); + odp_buffer_hdr_t *hdr = _odp_buf_hdr(buf); - return hdr->type; + return hdr->event_hdr.type; } void _odp_buffer_type_set(odp_buffer_t buf, int type) { - odp_buffer_hdr_t *hdr = buf_hdl_to_hdr(buf); + odp_buffer_hdr_t *hdr = _odp_buf_hdr(buf); - hdr->type = type; + hdr->event_hdr.type = type; } int odp_buffer_is_valid(odp_buffer_t buf) { - if (_odp_buffer_is_valid(buf) == 0) + if (odp_event_is_valid(odp_buffer_to_event(buf)) == 0) return 0; if (odp_event_type(odp_buffer_to_event(buf)) != ODP_EVENT_BUFFER) @@ -70,12 +71,12 @@ void odp_buffer_print(odp_buffer_t buf) return; } - hdr = buf_hdl_to_hdr(buf); - pool = hdr->pool_ptr; + hdr = _odp_buf_hdr(buf); + pool = hdr->event_hdr.pool_ptr; len += snprintf(&str[len], n - len, "Buffer\n------\n"); len += snprintf(&str[len], n - len, " pool index %u\n", pool->pool_idx); - len += snprintf(&str[len], n - len, " buffer index %u\n", hdr->index); + len += snprintf(&str[len], n - len, " buffer index %u\n", hdr->event_hdr.index); len += snprintf(&str[len], n - len, " addr %p\n", odp_buffer_addr(buf)); len += snprintf(&str[len], n - len, " size %u\n", odp_buffer_size(buf)); str[len] = 0; diff --git a/platform/linux-dpdk/odp_crypto.c b/platform/linux-dpdk/odp_crypto.c index c15e75f22..02c06be0e 100644 --- a/platform/linux-dpdk/odp_crypto.c +++ b/platform/linux-dpdk/odp_crypto.c @@ -1745,7 +1745,7 @@ static uint8_t *crypto_prepare_digest(crypto_session_entry_t *session, _odp_packet_set_data(pkt, param->hash_result_offset, 0, session->p.auth_digest_len); data = pkt_hdr->crypto_digest_buf; - mb = &pkt_hdr->buf_hdr.mb; + mb = &pkt_hdr->event_hdr.mb; *phys_addr = rte_pktmbuf_iova_offset(mb, data - rte_pktmbuf_mtod(mb, uint8_t *)); @@ -1782,9 +1782,9 @@ static void crypto_fill_aead_param(crypto_session_entry_t *session, aead_xform->aead.aad_length); op->sym->aead.aad.data = pkt_hdr->crypto_aad_buf; op->sym->aead.aad.phys_addr = - rte_pktmbuf_iova_offset(&pkt_hdr->buf_hdr.mb, + rte_pktmbuf_iova_offset(&pkt_hdr->event_hdr.mb, op->sym->aead.aad.data - - rte_pktmbuf_mtod(&pkt_hdr->buf_hdr.mb, + rte_pktmbuf_mtod(&pkt_hdr->event_hdr.mb, uint8_t *)); iv_ptr = rte_crypto_op_ctod_offset(op, uint8_t *, IV_OFFSET); if (aead_xform->aead.algo == RTE_CRYPTO_AEAD_AES_CCM) { diff --git a/platform/linux-dpdk/odp_init.c b/platform/linux-dpdk/odp_init.c index 546203a5d..a69aabff2 100644 --- a/platform/linux-dpdk/odp_init.c +++ b/platform/linux-dpdk/odp_init.c @@ -52,16 +52,21 @@ enum init_stage { IPSEC_EVENTS_INIT, IPSEC_SAD_INIT, IPSEC_INIT, + DMA_INIT, ALL_INIT /* All init stages completed */ }; odp_global_data_ro_t odp_global_ro; odp_global_data_rw_t *odp_global_rw; +/* odp_init_local() call status */ +static __thread uint8_t init_local_called; + static void disable_features(odp_global_data_ro_t *global_ro, const odp_init_t *init_param) { int disable_ipsec, disable_crypto; + int disable_dma; if (init_param == NULL) return; @@ -74,7 +79,13 @@ static void disable_features(odp_global_data_ro_t *global_ro, if (disable_ipsec && disable_crypto) global_ro->disable.crypto = 1; - global_ro->disable.stash = init_param->not_used.feat.stash; + disable_dma = init_param->not_used.feat.dma; + global_ro->disable.dma = disable_dma; + + /* DMA uses stash. Disable stash only when both are disabled. */ + if (disable_dma && init_param->not_used.feat.stash) + global_ro->disable.stash = 1; + global_ro->disable.traffic_mngr = init_param->not_used.feat.tm; global_ro->disable.compress = init_param->not_used.feat.compress; } @@ -305,6 +316,13 @@ static int term_global(enum init_stage stage) switch (stage) { case ALL_INIT: + case DMA_INIT: + if (_odp_dma_term_global()) { + ODP_ERR("ODP DMA term failed.\n"); + rc = -1; + } + /* Fall through */ + case IPSEC_INIT: if (_odp_ipsec_term_global()) { ODP_ERR("ODP IPsec term failed.\n"); @@ -639,6 +657,12 @@ int odp_init_global(odp_instance_t *instance, } stage = IPSEC_INIT; + if (_odp_dma_init_global()) { + ODP_ERR("ODP DMA init failed.\n"); + goto init_failed; + } + stage = DMA_INIT; + /* Dummy support for single instance */ *instance = (odp_instance_t)odp_global_ro.main_pid; @@ -742,6 +766,13 @@ int odp_init_local(odp_instance_t instance, odp_thread_type_t thr_type) goto init_fail; } + /* Detect if odp_init_local() has been already called from this thread */ + if (getpid() == odp_global_ro.main_pid && init_local_called) { + ODP_ERR("%s() called multiple times by the same thread\n", __func__); + goto init_fail; + } + init_local_called = 1; + if (_odp_shm_init_local()) { ODP_ERR("ODP shm local init failed.\n"); goto init_fail; @@ -805,6 +836,13 @@ init_fail: int odp_term_local(void) { + /* Check that odp_init_local() has been called by this thread */ + if (!init_local_called) { + ODP_ERR("%s() called by a non-initialized thread\n", __func__); + return -1; + } + init_local_called = 0; + return term_local(ALL_INIT); } diff --git a/platform/linux-dpdk/odp_packet.c b/platform/linux-dpdk/odp_packet.c index 5b16428db..fa84e3b4c 100644 --- a/platform/linux-dpdk/odp_packet.c +++ b/platform/linux-dpdk/odp_packet.c @@ -7,18 +7,21 @@ #include <odp/api/packet.h> #include <odp/api/plat/packet_inlines.h> -#include <odp_packet_internal.h> -#include <odp_debug_internal.h> -#include <odp_macros_internal.h> -#include <odp_chksum_internal.h> #include <odp/api/hints.h> #include <odp/api/byteorder.h> #include <odp/api/plat/byteorder_inlines.h> #include <odp/api/packet_io.h> #include <odp/api/plat/pktio_inlines.h> -#include <odp_errno_define.h> #include <odp/api/proto_stats.h> +#include <odp_align_internal.h> +#include <odp_chksum_internal.h> +#include <odp_debug_internal.h> +#include <odp_errno_define.h> +#include <odp_event_internal.h> +#include <odp_packet_internal.h> +#include <odp_macros_internal.h> + /* Inlined API functions */ #include <odp/api/plat/event_inlines.h> @@ -38,10 +41,10 @@ /* Fill in packet header field offsets for inline functions */ const _odp_packet_inline_offset_t _odp_packet_inline ODP_ALIGNED_CACHE = { - .mb = offsetof(odp_packet_hdr_t, buf_hdr.mb), - .pool = offsetof(odp_packet_hdr_t, buf_hdr.pool_ptr), + .mb = offsetof(odp_packet_hdr_t, event_hdr.mb), + .pool = offsetof(odp_packet_hdr_t, event_hdr.pool_ptr), .input = offsetof(odp_packet_hdr_t, input), - .user_ptr = offsetof(odp_packet_hdr_t, buf_hdr.user_ptr), + .user_ptr = offsetof(odp_packet_hdr_t, event_hdr.user_ptr), .l2_offset = offsetof(odp_packet_hdr_t, p.l2_offset), .l3_offset = offsetof(odp_packet_hdr_t, p.l3_offset), .l4_offset = offsetof(odp_packet_hdr_t, p.l4_offset), @@ -49,14 +52,14 @@ const _odp_packet_inline_offset_t _odp_packet_inline ODP_ALIGNED_CACHE = { .input_flags = offsetof(odp_packet_hdr_t, p.input_flags), .flags = offsetof(odp_packet_hdr_t, p.flags), .subtype = offsetof(odp_packet_hdr_t, subtype), - .buf_addr = offsetof(odp_packet_hdr_t, buf_hdr.mb.buf_addr), - .data = offsetof(odp_packet_hdr_t, buf_hdr.mb.data_off), - .pkt_len = offsetof(odp_packet_hdr_t, buf_hdr.mb.pkt_len), - .seg_len = offsetof(odp_packet_hdr_t, buf_hdr.mb.data_len), - .nb_segs = offsetof(odp_packet_hdr_t, buf_hdr.mb.nb_segs), - .user_area = offsetof(odp_packet_hdr_t, buf_hdr.uarea_addr), - .rss = offsetof(odp_packet_hdr_t, buf_hdr.mb.hash.rss), - .ol_flags = offsetof(odp_packet_hdr_t, buf_hdr.mb.ol_flags), + .buf_addr = offsetof(odp_packet_hdr_t, event_hdr.mb.buf_addr), + .data = offsetof(odp_packet_hdr_t, event_hdr.mb.data_off), + .pkt_len = offsetof(odp_packet_hdr_t, event_hdr.mb.pkt_len), + .seg_len = offsetof(odp_packet_hdr_t, event_hdr.mb.data_len), + .nb_segs = offsetof(odp_packet_hdr_t, event_hdr.mb.nb_segs), + .user_area = offsetof(odp_packet_hdr_t, event_hdr.uarea_addr), + .rss = offsetof(odp_packet_hdr_t, event_hdr.mb.hash.rss), + .ol_flags = offsetof(odp_packet_hdr_t, event_hdr.mb.ol_flags), .rss_flag = PKT_RX_RSS_HASH }; @@ -95,11 +98,6 @@ ODP_STATIC_ASSERT(ODP_TIMEOUT_INVALID == 0, "Timeout invalid not 0"); #pragma GCC diagnostic pop #endif -static inline odp_buffer_t packet_to_buffer(odp_packet_t pkt) -{ - return (odp_buffer_t)pkt; -} - /* Calculate the number of segments */ static inline int num_segments(uint32_t len, uint32_t seg_len) { @@ -190,7 +188,7 @@ static odp_packet_t packet_alloc(pool_t *pool, uint32_t len) } } - pkt_hdr->buf_hdr.totsize = seg_len * num_seg; + pkt_hdr->event_hdr.totsize = seg_len * num_seg; pkt = packet_handle(pkt_hdr); odp_packet_reset(pkt, len); @@ -238,7 +236,7 @@ int odp_packet_alloc_multi(odp_pool_t pool_hdl, uint32_t len, int odp_packet_reset(odp_packet_t pkt, uint32_t len) { odp_packet_hdr_t *const pkt_hdr = packet_hdr(pkt); - struct rte_mbuf *ms, *mb = &pkt_hdr->buf_hdr.mb; + struct rte_mbuf *ms, *mb = &pkt_hdr->event_hdr.mb; uint8_t nb_segs = 0; int32_t lenleft = len; @@ -310,12 +308,12 @@ int odp_event_filter_packet(const odp_event_t event[], uint32_t odp_packet_buf_len(odp_packet_t pkt) { - return packet_hdr(pkt)->buf_hdr.totsize; + return packet_hdr(pkt)->event_hdr.totsize; } void *odp_packet_tail(odp_packet_t pkt) { - struct rte_mbuf *mb = &(packet_hdr(pkt)->buf_hdr.mb); + struct rte_mbuf *mb = &(packet_hdr(pkt)->event_hdr.mb); mb = rte_pktmbuf_lastseg(mb); return (void *)(rte_pktmbuf_mtod(mb, char *) + mb->data_len); @@ -323,7 +321,7 @@ void *odp_packet_tail(odp_packet_t pkt) void *odp_packet_push_head(odp_packet_t pkt, uint32_t len) { - struct rte_mbuf *mb = &(packet_hdr(pkt)->buf_hdr.mb); + struct rte_mbuf *mb = &(packet_hdr(pkt)->event_hdr.mb); return (void *)rte_pktmbuf_prepend(mb, len); } @@ -339,7 +337,7 @@ static void _copy_head_metadata(struct rte_mbuf *newhead, int odp_packet_extend_head(odp_packet_t *pkt, uint32_t len, void **data_ptr, uint32_t *seg_len) { - struct rte_mbuf *mb = &(packet_hdr(*pkt)->buf_hdr.mb); + struct rte_mbuf *mb = &(packet_hdr(*pkt)->event_hdr.mb); int addheadsize = len - rte_pktmbuf_headroom(mb); if (addheadsize > 0) { @@ -380,7 +378,7 @@ int odp_packet_extend_head(odp_packet_t *pkt, uint32_t len, void **data_ptr, _copy_head_metadata(newhead, mb); mb = newhead; *pkt = (odp_packet_t)newhead; - packet_hdr(*pkt)->buf_hdr.totsize += totsize_change; + packet_hdr(*pkt)->event_hdr.totsize += totsize_change; } else { rte_pktmbuf_prepend(mb, len); } @@ -434,7 +432,7 @@ int odp_packet_trunc_head(odp_packet_t *pkt, uint32_t len, void **data_ptr, rte_pktmbuf_free(mb); mb = newhead; *pkt = (odp_packet_t)newhead; - packet_hdr(*pkt)->buf_hdr.totsize -= totsize_change; + packet_hdr(*pkt)->event_hdr.totsize -= totsize_change; } else { rte_pktmbuf_adj(mb, len); } @@ -449,7 +447,7 @@ int odp_packet_trunc_head(odp_packet_t *pkt, uint32_t len, void **data_ptr, void *odp_packet_push_tail(odp_packet_t pkt, uint32_t len) { - struct rte_mbuf *mb = &(packet_hdr(pkt)->buf_hdr.mb); + struct rte_mbuf *mb = &(packet_hdr(pkt)->event_hdr.mb); return (void *)rte_pktmbuf_append(mb, len); } @@ -457,7 +455,7 @@ void *odp_packet_push_tail(odp_packet_t pkt, uint32_t len) int odp_packet_extend_tail(odp_packet_t *pkt, uint32_t len, void **data_ptr, uint32_t *seg_len) { - struct rte_mbuf *mb = &(packet_hdr(*pkt)->buf_hdr.mb); + struct rte_mbuf *mb = &(packet_hdr(*pkt)->event_hdr.mb); int newtailsize = len - odp_packet_tailroom(*pkt); uint32_t old_pkt_len = odp_packet_len(*pkt); @@ -502,7 +500,7 @@ int odp_packet_extend_tail(odp_packet_t *pkt, uint32_t len, void **data_ptr, /* Expand the original tail */ m_last->data_len = m_last->buf_len - m_last->data_off; mb->pkt_len += len - newtailsize; - packet_hdr(*pkt)->buf_hdr.totsize += + packet_hdr(*pkt)->event_hdr.totsize += newtail->nb_segs * newtail->buf_len; } else { rte_pktmbuf_append(mb, len); @@ -595,7 +593,7 @@ void odp_packet_user_ptr_set(odp_packet_t pkt, const void *ptr) return; } - pkt_hdr->buf_hdr.user_ptr = ptr; + pkt_hdr->event_hdr.user_ptr = ptr; pkt_hdr->p.flags.user_ptr_set = 1; } @@ -709,7 +707,7 @@ int odp_packet_add_data(odp_packet_t *pkt_ptr, uint32_t offset, uint32_t len) odp_packet_t pkt = *pkt_ptr; odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); uint32_t pktlen = odp_packet_len(pkt); - pool_t *pool = pkt_hdr->buf_hdr.pool_ptr; + pool_t *pool = pkt_hdr->event_hdr.pool_ptr; odp_packet_t newpkt; if (offset > pktlen) @@ -739,7 +737,7 @@ int odp_packet_rem_data(odp_packet_t *pkt_ptr, uint32_t offset, uint32_t len) odp_packet_t pkt = *pkt_ptr; odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); uint32_t pktlen = odp_packet_len(pkt); - pool_t *pool = pkt_hdr->buf_hdr.pool_ptr; + pool_t *pool = pkt_hdr->event_hdr.pool_ptr; odp_packet_t newpkt; if (odp_unlikely(offset + len >= pktlen)) @@ -817,7 +815,7 @@ int odp_packet_concat(odp_packet_t *dst, odp_packet_t src) uint32_t src_len; if (odp_likely(!rte_pktmbuf_chain(mb_dst, mb_src))) { - dst_hdr->buf_hdr.totsize += src_hdr->buf_hdr.totsize; + dst_hdr->event_hdr.totsize += src_hdr->event_hdr.totsize; return 0; } @@ -1063,11 +1061,11 @@ void odp_packet_print(odp_packet_t pkt) int len = 0; int n = max_len - 1; odp_packet_hdr_t *hdr = packet_hdr(pkt); - pool_t *pool = hdr->buf_hdr.pool_ptr; + pool_t *pool = hdr->event_hdr.pool_ptr; len += snprintf(&str[len], n - len, "Packet\n------\n"); len += snprintf(&str[len], n - len, " pool index %u\n", pool->pool_idx); - len += snprintf(&str[len], n - len, " buf index %u\n", hdr->buf_hdr.index); + len += snprintf(&str[len], n - len, " buf index %u\n", hdr->event_hdr.index); len += snprintf(&str[len], n - len, " ev subtype %i\n", hdr->subtype); len += snprintf(&str[len], n - len, " input_flags 0x%" PRIx64 "\n", hdr->p.input_flags.all); @@ -1088,7 +1086,7 @@ void odp_packet_print(odp_packet_t pkt) " l4_offset %" PRIu32 "\n", hdr->p.l4_offset); len += snprintf(&str[len], n - len, " frame_len %" PRIu32 "\n", - hdr->buf_hdr.mb.pkt_len); + hdr->event_hdr.mb.pkt_len); len += snprintf(&str[len], n - len, " input %" PRIu64 "\n", odp_pktio_to_u64(hdr->input)); @@ -1128,16 +1126,16 @@ void odp_packet_print_data(odp_packet_t pkt, uint32_t offset, int len = 0; int n = max_len - 1; uint32_t data_len = odp_packet_len(pkt); - pool_t *pool = hdr->buf_hdr.pool_ptr; + pool_t *pool = hdr->event_hdr.pool_ptr; len += snprintf(&str[len], n - len, "Packet\n------\n"); len += snprintf(&str[len], n - len, " pool name %s\n", pool->name); len += snprintf(&str[len], n - len, - " buf index %" PRIu32 "\n", hdr->buf_hdr.index); + " buf index %" PRIu32 "\n", hdr->event_hdr.index); len += snprintf(&str[len], n - len, " segcount %" PRIu8 "\n", - hdr->buf_hdr.mb.nb_segs); + hdr->event_hdr.mb.nb_segs); len += snprintf(&str[len], n - len, " data len %" PRIu32 "\n", data_len); len += snprintf(&str[len], n - len, @@ -1186,11 +1184,11 @@ int odp_packet_is_valid(odp_packet_t pkt) if (pkt == ODP_PACKET_INVALID) return 0; - if (_odp_buffer_is_valid(packet_to_buffer(pkt)) == 0) - return 0; - ev = odp_packet_to_event(pkt); + if (_odp_event_is_valid(ev) == 0) + return 0; + if (odp_event_type(ev) != ODP_EVENT_PACKET) return 0; @@ -1230,15 +1228,15 @@ int _odp_packet_copy_md_to_packet(odp_packet_t srcpkt, odp_packet_t dstpkt) dsthdr->cos = srchdr->cos; dsthdr->cls_mark = srchdr->cls_mark; - dsthdr->buf_hdr.user_ptr = srchdr->buf_hdr.user_ptr; - dsthdr->buf_hdr.mb.port = srchdr->buf_hdr.mb.port; - dsthdr->buf_hdr.mb.ol_flags = srchdr->buf_hdr.mb.ol_flags; - dsthdr->buf_hdr.mb.packet_type = srchdr->buf_hdr.mb.packet_type; - dsthdr->buf_hdr.mb.vlan_tci = srchdr->buf_hdr.mb.vlan_tci; - dsthdr->buf_hdr.mb.hash.rss = srchdr->buf_hdr.mb.hash.rss; - dsthdr->buf_hdr.mb.hash = srchdr->buf_hdr.mb.hash; - dsthdr->buf_hdr.mb.vlan_tci_outer = srchdr->buf_hdr.mb.vlan_tci_outer; - dsthdr->buf_hdr.mb.tx_offload = srchdr->buf_hdr.mb.tx_offload; + dsthdr->event_hdr.user_ptr = srchdr->event_hdr.user_ptr; + dsthdr->event_hdr.mb.port = srchdr->event_hdr.mb.port; + dsthdr->event_hdr.mb.ol_flags = srchdr->event_hdr.mb.ol_flags; + dsthdr->event_hdr.mb.packet_type = srchdr->event_hdr.mb.packet_type; + dsthdr->event_hdr.mb.vlan_tci = srchdr->event_hdr.mb.vlan_tci; + dsthdr->event_hdr.mb.hash.rss = srchdr->event_hdr.mb.hash.rss; + dsthdr->event_hdr.mb.hash = srchdr->event_hdr.mb.hash; + dsthdr->event_hdr.mb.vlan_tci_outer = srchdr->event_hdr.mb.vlan_tci_outer; + dsthdr->event_hdr.mb.tx_offload = srchdr->event_hdr.mb.tx_offload; if (dst_size != 0) memcpy(odp_packet_user_area(dstpkt), @@ -2145,7 +2143,7 @@ int odp_packet_parse(odp_packet_t pkt, uint32_t offset, * packet data range. Copy enough data to a temporary buffer for * parsing if necessary. */ - if (odp_unlikely(pkt_hdr->buf_hdr.mb.nb_segs > 1) && + if (odp_unlikely(pkt_hdr->event_hdr.mb.nb_segs > 1) && odp_unlikely(seg_len < min_seglen)) { seg_len = min_seglen; if (seg_len > packet_len - offset) @@ -2516,7 +2514,7 @@ static inline odp_packet_hdr_t *packet_buf_to_hdr(odp_packet_buf_t pkt_buf) void *odp_packet_buf_head(odp_packet_buf_t pkt_buf) { odp_packet_hdr_t *pkt_hdr = packet_buf_to_hdr(pkt_buf); - pool_t *pool = pkt_hdr->buf_hdr.pool_ptr; + pool_t *pool = pkt_hdr->event_hdr.pool_ptr; if (odp_unlikely(pool->pool_ext == 0)) { ODP_ERR("Not an external memory pool\n"); @@ -2529,7 +2527,7 @@ void *odp_packet_buf_head(odp_packet_buf_t pkt_buf) uint32_t odp_packet_buf_size(odp_packet_buf_t pkt_buf) { odp_packet_hdr_t *pkt_hdr = packet_buf_to_hdr(pkt_buf); - pool_t *pool = pkt_hdr->buf_hdr.pool_ptr; + pool_t *pool = pkt_hdr->event_hdr.pool_ptr; return pool->seg_len; } @@ -2554,8 +2552,8 @@ void odp_packet_buf_data_set(odp_packet_buf_t pkt_buf, uint32_t data_offset, { odp_packet_hdr_t *pkt_hdr = packet_buf_to_hdr(pkt_buf); - pkt_hdr->buf_hdr.mb.data_off = data_offset; - pkt_hdr->buf_hdr.mb.data_len = data_len; + pkt_hdr->event_hdr.mb.data_off = data_offset; + pkt_hdr->event_hdr.mb.data_len = data_len; } odp_packet_buf_t odp_packet_buf_from_head(odp_pool_t pool_hdl, void *head) @@ -2581,7 +2579,7 @@ uint32_t odp_packet_disassemble(odp_packet_t pkt, odp_packet_buf_t pkt_buf[], uint32_t i; odp_packet_seg_t seg; odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - pool_t *pool = pkt_hdr->buf_hdr.pool_ptr; + pool_t *pool = pkt_hdr->event_hdr.pool_ptr; uint32_t num_segs = odp_packet_num_segs(pkt); if (odp_unlikely(pool->type != ODP_POOL_PACKET)) { @@ -2645,15 +2643,15 @@ odp_packet_t odp_packet_reassemble(odp_pool_t pool_hdl, if (i < num - 1) next_seg = (odp_packet_hdr_t *)(uintptr_t)pkt_buf[i + 1]; - data_len += cur_seg->buf_hdr.mb.data_len; + data_len += cur_seg->event_hdr.mb.data_len; mb = (struct rte_mbuf *)(uintptr_t)cur_seg; mb->next = (struct rte_mbuf *)next_seg; cur_seg = next_seg; } - pkt_hdr->buf_hdr.mb.nb_segs = num; - pkt_hdr->buf_hdr.mb.pkt_len = data_len; - pkt_hdr->buf_hdr.mb.data_off = headroom; + pkt_hdr->event_hdr.mb.nb_segs = num; + pkt_hdr->event_hdr.mb.pkt_len = data_len; + pkt_hdr->event_hdr.mb.data_off = headroom; /* Reset metadata */ pkt_hdr->subtype = ODP_EVENT_PACKET_BASIC; diff --git a/platform/linux-dpdk/odp_pool.c b/platform/linux-dpdk/odp_pool.c index 219f8378b..318e0071a 100644 --- a/platform/linux-dpdk/odp_pool.c +++ b/platform/linux-dpdk/odp_pool.c @@ -100,7 +100,7 @@ static void ptr_from_mempool(struct rte_mempool *mp ODP_UNUSED, void *opaque, args->match = true; } -static pool_t *find_pool(odp_buffer_hdr_t *buf_hdr) +static pool_t *find_pool(_odp_event_hdr_t *event_hdr) { int i; @@ -111,7 +111,7 @@ static pool_t *find_pool(odp_buffer_hdr_t *buf_hdr) if (pool->rte_mempool == NULL) continue; - args.addr = (uint8_t *)buf_hdr; + args.addr = (uint8_t *)event_hdr; args.match = false; rte_mempool_mem_iter(pool->rte_mempool, ptr_from_mempool, &args); @@ -179,6 +179,7 @@ int _odp_pool_init_global(void) } ODP_DBG("\nPool init global\n"); + ODP_DBG(" event_hdr_t size: %zu\n", sizeof(_odp_event_hdr_t)); ODP_DBG(" odp_buffer_hdr_t size: %zu\n", sizeof(odp_buffer_hdr_t)); ODP_DBG(" odp_packet_hdr_t size: %zu\n", sizeof(odp_packet_hdr_t)); ODP_DBG(" odp_timeout_hdr_t size: %zu\n", sizeof(odp_timeout_hdr_t)); @@ -213,23 +214,23 @@ int _odp_pool_term_local(void) return 0; } -int _odp_buffer_is_valid(odp_buffer_t buf) +int _odp_event_is_valid(odp_event_t event) { pool_t *pool; - odp_buffer_hdr_t *buf_hdr = buf_hdl_to_hdr(buf); + _odp_event_hdr_t *event_hdr = _odp_event_hdr(event); - if (buf == ODP_BUFFER_INVALID) + if (event == ODP_EVENT_INVALID) return 0; /* Check that buffer header is from a known pool */ - pool = find_pool(buf_hdr); + pool = find_pool(event_hdr); if (pool == NULL) return 0; - if (pool != buf_hdr->pool_ptr) + if (pool != event_hdr->pool_ptr) return 0; - if (buf_hdr->index >= pool->rte_mempool->size) + if (event_hdr->index >= pool->rte_mempool->size) return 0; return 1; @@ -311,7 +312,7 @@ odp_dpdk_mbuf_ctor(struct rte_mempool *mp, { struct mbuf_ctor_arg *mb_ctor_arg; struct rte_mbuf *mb = raw_mbuf; - struct odp_buffer_hdr_t *buf_hdr; + _odp_event_hdr_t *event_hdr; /* The rte_mbuf is at the begninning in all cases */ mb_ctor_arg = (struct mbuf_ctor_arg *)opaque_arg; @@ -356,12 +357,12 @@ odp_dpdk_mbuf_ctor(struct rte_mempool *mp, mb->next = NULL; /* Save index, might be useful for debugging purposes */ - buf_hdr = (struct odp_buffer_hdr_t *)raw_mbuf; - buf_hdr->index = i; - buf_hdr->pool_ptr = mb_ctor_arg->pool; - buf_hdr->type = mb_ctor_arg->type; - buf_hdr->event_type = mb_ctor_arg->event_type; - buf_hdr->uarea_addr = mb_ctor_arg->pool->uarea_base_addr + + event_hdr = (_odp_event_hdr_t *)raw_mbuf; + event_hdr->index = i; + event_hdr->pool_ptr = mb_ctor_arg->pool; + event_hdr->type = mb_ctor_arg->type; + event_hdr->event_type = mb_ctor_arg->event_type; + event_hdr->uarea_addr = mb_ctor_arg->pool->uarea_base_addr + i * mb_ctor_arg->pool->uarea_size; /* Initialize event vector metadata */ @@ -598,7 +599,6 @@ static int reserve_uarea(pool_t *pool, uint32_t uarea_size, uint32_t num_pkt) { odp_shm_t shm; char uarea_name[ODP_SHM_NAME_LEN]; - uint32_t shm_flags = 0; pool->uarea_shm = ODP_SHM_INVALID; @@ -615,11 +615,7 @@ static int reserve_uarea(pool_t *pool, uint32_t uarea_size, uint32_t num_pkt) pool->uarea_size = ROUNDUP_CACHE_LINE(uarea_size); pool->uarea_shm_size = num_pkt * (uint64_t)pool->uarea_size; - if (odp_global_ro.shm_single_va) - shm_flags |= ODP_SHM_SINGLE_VA; - - shm = odp_shm_reserve(uarea_name, pool->uarea_shm_size, ODP_PAGE_SIZE, - shm_flags); + shm = odp_shm_reserve(uarea_name, pool->uarea_shm_size, ODP_PAGE_SIZE, 0); if (shm == ODP_SHM_INVALID) return -1; @@ -629,7 +625,10 @@ static int reserve_uarea(pool_t *pool, uint32_t uarea_size, uint32_t num_pkt) return 0; } -odp_pool_t odp_pool_create(const char *name, const odp_pool_param_t *params) +/* Create pool according to params. Actual type of the pool is type_2, which is recorded for pool + * info calls. */ +odp_pool_t _odp_pool_create(const char *name, const odp_pool_param_t *params, + odp_pool_type_t type_2) { struct rte_pktmbuf_pool_private mbp_ctor_arg; struct mbuf_ctor_arg mb_ctor_arg; @@ -645,9 +644,6 @@ odp_pool_t odp_pool_create(const char *name, const odp_pool_param_t *params) char pool_name[ODP_POOL_NAME_LEN]; char rte_name[RTE_MEMPOOL_NAMESIZE]; - if (check_params(params)) - return ODP_POOL_INVALID; - if (name == NULL) { pool_name[0] = 0; } else { @@ -820,6 +816,7 @@ odp_pool_t odp_pool_create(const char *name, const odp_pool_param_t *params) } pool->type = type; + pool->type_2 = type_2; pool->params = *params; if (reserve_uarea(pool, uarea_size, num)) { @@ -846,6 +843,14 @@ odp_pool_t odp_pool_create(const char *name, const odp_pool_param_t *params) return pool_hdl; } +odp_pool_t odp_pool_create(const char *name, const odp_pool_param_t *params) +{ + if (check_params(params)) + return ODP_POOL_INVALID; + + return _odp_pool_create(name, params, params->type); +} + odp_pool_t odp_pool_lookup(const char *name) { uint32_t i; @@ -878,7 +883,7 @@ odp_buffer_t odp_buffer_alloc(odp_pool_t pool_hdl) ODP_ASSERT(pool->type == ODP_POOL_BUFFER); - ret = _odp_buffer_alloc_multi(pool, (odp_buffer_hdr_t **)&buf, 1); + ret = _odp_event_alloc_multi(pool, (_odp_event_hdr_t **)&buf, 1); if (odp_likely(ret == 1)) return buf; @@ -896,17 +901,17 @@ int odp_buffer_alloc_multi(odp_pool_t pool_hdl, odp_buffer_t buf[], int num) ODP_ASSERT(pool->type == ODP_POOL_BUFFER); - return _odp_buffer_alloc_multi(pool, (odp_buffer_hdr_t **)buf, num); + return _odp_event_alloc_multi(pool, (_odp_event_hdr_t **)buf, num); } void odp_buffer_free(odp_buffer_t buf) { - rte_mbuf_raw_free(buf_to_mbuf(buf)); + _odp_event_free(odp_buffer_to_event(buf)); } void odp_buffer_free_multi(const odp_buffer_t buf[], int num) { - _odp_buffer_free_multi((odp_buffer_hdr_t **)(uintptr_t)buf, num); + _odp_event_free_multi((_odp_event_hdr_t **)(uintptr_t)buf, num); } void odp_pool_print(odp_pool_t pool_hdl) @@ -990,11 +995,17 @@ int odp_pool_info(odp_pool_t pool_hdl, odp_pool_info_t *info) memset(info, 0, sizeof(odp_pool_info_t)); + info->type = pool->type_2; info->name = pool->name; if (pool->pool_ext) { info->pool_ext = 1; info->pool_ext_param = pool->ext_param; + + } else if (pool->type_2 == ODP_POOL_DMA_COMPL) { + info->dma_pool_param.num = pool->params.buf.num; + info->dma_pool_param.cache_size = pool->params.buf.cache_size; + } else { info->params = pool->params; } @@ -1036,7 +1047,7 @@ int odp_pool_destroy(odp_pool_t pool_hdl) odp_pool_t odp_buffer_pool(odp_buffer_t buf) { - pool_t *pool = buf_hdl_to_hdr(buf)->pool_ptr; + pool_t *pool = _odp_buf_hdr(buf)->event_hdr.pool_ptr; return pool->pool_hdl; } diff --git a/platform/linux-dpdk/odp_queue_basic.c b/platform/linux-dpdk/odp_queue_basic.c index e3c3241c4..bb57475a3 100644 --- a/platform/linux-dpdk/odp_queue_basic.c +++ b/platform/linux-dpdk/odp_queue_basic.c @@ -10,8 +10,6 @@ #include <odp_queue_if.h> #include <odp/api/std_types.h> #include <odp/api/align.h> -#include <odp/api/buffer.h> -#include <odp_buffer_internal.h> #include <odp_pool_internal.h> #include <odp_init_internal.h> #include <odp/api/shared_memory.h> @@ -28,6 +26,8 @@ #include <odp_timer_internal.h> #include <odp/api/plat/queue_inline_types.h> #include <odp_global_data.h> +#include <odp_queue_basic_internal.h> +#include <odp_event_internal.h> #include <odp/api/plat/ticketlock_inlines.h> #define LOCK(queue_ptr) odp_ticketlock_lock(&((queue_ptr)->s.lock)) @@ -465,7 +465,7 @@ static odp_queue_t queue_lookup(const char *name) } static inline int _plain_queue_enq_multi(odp_queue_t handle, - odp_buffer_hdr_t *buf_hdr[], int num) + _odp_event_hdr_t *event_hdr[], int num) { queue_entry_t *queue; int ret, num_enq; @@ -474,16 +474,16 @@ static inline int _plain_queue_enq_multi(odp_queue_t handle, queue = qentry_from_handle(handle); ring_mpmc = queue->s.ring_mpmc; - if (_odp_sched_fn->ord_enq_multi(handle, (void **)buf_hdr, num, &ret)) + if (_odp_sched_fn->ord_enq_multi(handle, (void **)event_hdr, num, &ret)) return ret; - num_enq = ring_mpmc_enq_multi(ring_mpmc, (void **)buf_hdr, num); + num_enq = ring_mpmc_enq_multi(ring_mpmc, (void **)event_hdr, num); return num_enq; } static inline int _plain_queue_deq_multi(odp_queue_t handle, - odp_buffer_hdr_t *buf_hdr[], int num) + _odp_event_hdr_t *event_hdr[], int num) { int num_deq; queue_entry_t *queue; @@ -492,22 +492,22 @@ static inline int _plain_queue_deq_multi(odp_queue_t handle, queue = qentry_from_handle(handle); ring_mpmc = queue->s.ring_mpmc; - num_deq = ring_mpmc_deq_multi(ring_mpmc, (void **)buf_hdr, num); + num_deq = ring_mpmc_deq_multi(ring_mpmc, (void **)event_hdr, num); return num_deq; } static int plain_queue_enq_multi(odp_queue_t handle, - odp_buffer_hdr_t *buf_hdr[], int num) + _odp_event_hdr_t *event_hdr[], int num) { - return _plain_queue_enq_multi(handle, buf_hdr, num); + return _plain_queue_enq_multi(handle, event_hdr, num); } -static int plain_queue_enq(odp_queue_t handle, odp_buffer_hdr_t *buf_hdr) +static int plain_queue_enq(odp_queue_t handle, _odp_event_hdr_t *event_hdr) { int ret; - ret = _plain_queue_enq_multi(handle, &buf_hdr, 1); + ret = _plain_queue_enq_multi(handle, &event_hdr, 1); if (ret == 1) return 0; @@ -516,27 +516,27 @@ static int plain_queue_enq(odp_queue_t handle, odp_buffer_hdr_t *buf_hdr) } static int plain_queue_deq_multi(odp_queue_t handle, - odp_buffer_hdr_t *buf_hdr[], int num) + _odp_event_hdr_t *event_hdr[], int num) { - return _plain_queue_deq_multi(handle, buf_hdr, num); + return _plain_queue_deq_multi(handle, event_hdr, num); } -static odp_buffer_hdr_t *plain_queue_deq(odp_queue_t handle) +static _odp_event_hdr_t *plain_queue_deq(odp_queue_t handle) { - odp_buffer_hdr_t *buf_hdr = NULL; + _odp_event_hdr_t *event_hdr = NULL; int ret; - ret = _plain_queue_deq_multi(handle, &buf_hdr, 1); + ret = _plain_queue_deq_multi(handle, &event_hdr, 1); if (ret == 1) - return buf_hdr; + return event_hdr; else return NULL; } -static int error_enqueue(odp_queue_t handle, odp_buffer_hdr_t *buf_hdr) +static int error_enqueue(odp_queue_t handle, _odp_event_hdr_t *event_hdr) { - (void)buf_hdr; + (void)event_hdr; ODP_ERR("Enqueue not supported (0x%" PRIx64 ")\n", odp_queue_to_u64(handle)); @@ -545,10 +545,10 @@ static int error_enqueue(odp_queue_t handle, odp_buffer_hdr_t *buf_hdr) } static int error_enqueue_multi(odp_queue_t handle, - odp_buffer_hdr_t *buf_hdr[], int num) + _odp_event_hdr_t *event_hdr[], int num) { - (void)buf_hdr; + (void)event_hdr; (void)num; ODP_ERR("Enqueue multi not supported (0x%" PRIx64 ")\n", @@ -557,7 +557,7 @@ static int error_enqueue_multi(odp_queue_t handle, return -1; } -static odp_buffer_hdr_t *error_dequeue(odp_queue_t handle) +static _odp_event_hdr_t *error_dequeue(odp_queue_t handle) { ODP_ERR("Dequeue not supported (0x%" PRIx64 ")\n", odp_queue_to_u64(handle)); @@ -566,9 +566,9 @@ static odp_buffer_hdr_t *error_dequeue(odp_queue_t handle) } static int error_dequeue_multi(odp_queue_t handle, - odp_buffer_hdr_t *buf_hdr[], int num) + _odp_event_hdr_t *event_hdr[], int num) { - (void)buf_hdr; + (void)event_hdr; (void)num; ODP_ERR("Dequeue multi not supported (0x%" PRIx64 ")\n", @@ -841,7 +841,7 @@ static void queue_print_all(void) } static inline int _sched_queue_enq_multi(odp_queue_t handle, - odp_buffer_hdr_t *buf_hdr[], int num) + _odp_event_hdr_t *event_hdr[], int num) { int sched = 0; int ret; @@ -852,12 +852,12 @@ static inline int _sched_queue_enq_multi(odp_queue_t handle, queue = qentry_from_handle(handle); ring_st = queue->s.ring_st; - if (_odp_sched_fn->ord_enq_multi(handle, (void **)buf_hdr, num, &ret)) + if (_odp_sched_fn->ord_enq_multi(handle, (void **)event_hdr, num, &ret)) return ret; LOCK(queue); - num_enq = ring_st_enq_multi(ring_st, (void **)buf_hdr, num); + num_enq = ring_st_enq_multi(ring_st, (void **)event_hdr, num); if (odp_unlikely(num_enq == 0)) { UNLOCK(queue); @@ -921,16 +921,16 @@ int _odp_sched_queue_deq(uint32_t queue_index, odp_event_t ev[], int max_num, } static int sched_queue_enq_multi(odp_queue_t handle, - odp_buffer_hdr_t *buf_hdr[], int num) + _odp_event_hdr_t *event_hdr[], int num) { - return _sched_queue_enq_multi(handle, buf_hdr, num); + return _sched_queue_enq_multi(handle, event_hdr, num); } -static int sched_queue_enq(odp_queue_t handle, odp_buffer_hdr_t *buf_hdr) +static int sched_queue_enq(odp_queue_t handle, _odp_event_hdr_t *event_hdr) { int ret; - ret = _sched_queue_enq_multi(handle, &buf_hdr, 1); + ret = _sched_queue_enq_multi(handle, &event_hdr, 1); if (ret == 1) return 0; @@ -1116,11 +1116,11 @@ static void queue_set_enq_deq_func(odp_queue_t handle, } static int queue_orig_multi(odp_queue_t handle, - odp_buffer_hdr_t **buf_hdr, int num) + _odp_event_hdr_t **event_hdr, int num) { queue_entry_t *queue = qentry_from_handle(handle); - return queue->s.orig_dequeue_multi(handle, buf_hdr, num); + return queue->s.orig_dequeue_multi(handle, event_hdr, num); } static int queue_api_enq_multi(odp_queue_t handle, @@ -1135,7 +1135,7 @@ static int queue_api_enq_multi(odp_queue_t handle, num = QUEUE_MULTI_MAX; return queue->s.enqueue_multi(handle, - (odp_buffer_hdr_t **)(uintptr_t)ev, num); + (_odp_event_hdr_t **)(uintptr_t)ev, num); } static void queue_timer_add(odp_queue_t handle) @@ -1157,7 +1157,7 @@ static int queue_api_enq(odp_queue_t handle, odp_event_t ev) queue_entry_t *queue = qentry_from_handle(handle); return queue->s.enqueue(handle, - (odp_buffer_hdr_t *)(uintptr_t)ev); + (_odp_event_hdr_t *)(uintptr_t)ev); } static int queue_api_deq_multi(odp_queue_t handle, odp_event_t ev[], int num) @@ -1168,7 +1168,7 @@ static int queue_api_deq_multi(odp_queue_t handle, odp_event_t ev[], int num) if (num > QUEUE_MULTI_MAX) num = QUEUE_MULTI_MAX; - ret = queue->s.dequeue_multi(handle, (odp_buffer_hdr_t **)ev, num); + ret = queue->s.dequeue_multi(handle, (_odp_event_hdr_t **)ev, num); if (odp_global_rw->inline_timers && odp_atomic_load_u64(&queue->s.num_timers)) diff --git a/platform/linux-dpdk/odp_queue_eventdev.c b/platform/linux-dpdk/odp_queue_eventdev.c index b650874ec..b474a515c 100644 --- a/platform/linux-dpdk/odp_queue_eventdev.c +++ b/platform/linux-dpdk/odp_queue_eventdev.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2019, Nokia +/* Copyright (c) 2019-2021, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -13,7 +13,7 @@ #include <odp/api/plat/queue_inline_types.h> #include <odp/api/plat/ticketlock_inlines.h> #include <odp_config_internal.h> -#include <odp_buffer_internal.h> +#include <odp_event_internal.h> #include <odp_debug_internal.h> #include <odp_libconfig_internal.h> #include <odp_queue_if.h> @@ -822,7 +822,7 @@ static odp_queue_t queue_lookup(const char *name) } static inline int _plain_queue_enq_multi(odp_queue_t handle, - odp_buffer_hdr_t *buf_hdr[], int num) + _odp_event_hdr_t *event_hdr[], int num) { queue_entry_t *queue; int num_enq; @@ -831,13 +831,13 @@ static inline int _plain_queue_enq_multi(odp_queue_t handle, queue = qentry_from_handle(handle); ring_mpmc = queue->s.ring_mpmc; - num_enq = ring_mpmc_enq_multi(ring_mpmc, (void **)buf_hdr, num); + num_enq = ring_mpmc_enq_multi(ring_mpmc, (void **)event_hdr, num); return num_enq; } static inline int _plain_queue_deq_multi(odp_queue_t handle, - odp_buffer_hdr_t *buf_hdr[], int num) + _odp_event_hdr_t *event_hdr[], int num) { int num_deq; queue_entry_t *queue; @@ -846,22 +846,22 @@ static inline int _plain_queue_deq_multi(odp_queue_t handle, queue = qentry_from_handle(handle); ring_mpmc = queue->s.ring_mpmc; - num_deq = ring_mpmc_deq_multi(ring_mpmc, (void **)buf_hdr, num); + num_deq = ring_mpmc_deq_multi(ring_mpmc, (void **)event_hdr, num); return num_deq; } static int plain_queue_enq_multi(odp_queue_t handle, - odp_buffer_hdr_t *buf_hdr[], int num) + _odp_event_hdr_t *event_hdr[], int num) { - return _plain_queue_enq_multi(handle, buf_hdr, num); + return _plain_queue_enq_multi(handle, event_hdr, num); } -static int plain_queue_enq(odp_queue_t handle, odp_buffer_hdr_t *buf_hdr) +static int plain_queue_enq(odp_queue_t handle, _odp_event_hdr_t *event_hdr) { int ret; - ret = _plain_queue_enq_multi(handle, &buf_hdr, 1); + ret = _plain_queue_enq_multi(handle, &event_hdr, 1); if (ret == 1) return 0; @@ -870,27 +870,27 @@ static int plain_queue_enq(odp_queue_t handle, odp_buffer_hdr_t *buf_hdr) } static int plain_queue_deq_multi(odp_queue_t handle, - odp_buffer_hdr_t *buf_hdr[], int num) + _odp_event_hdr_t *event_hdr[], int num) { - return _plain_queue_deq_multi(handle, buf_hdr, num); + return _plain_queue_deq_multi(handle, event_hdr, num); } -static odp_buffer_hdr_t *plain_queue_deq(odp_queue_t handle) +static _odp_event_hdr_t *plain_queue_deq(odp_queue_t handle) { - odp_buffer_hdr_t *buf_hdr = NULL; + _odp_event_hdr_t *event_hdr = NULL; int ret; - ret = _plain_queue_deq_multi(handle, &buf_hdr, 1); + ret = _plain_queue_deq_multi(handle, &event_hdr, 1); if (ret == 1) - return buf_hdr; + return event_hdr; else return NULL; } -static int error_enqueue(odp_queue_t handle, odp_buffer_hdr_t *buf_hdr) +static int error_enqueue(odp_queue_t handle, _odp_event_hdr_t *event_hdr) { - (void)buf_hdr; + (void)event_hdr; ODP_ERR("Enqueue not supported (0x%" PRIx64 ")\n", odp_queue_to_u64(handle)); @@ -899,10 +899,10 @@ static int error_enqueue(odp_queue_t handle, odp_buffer_hdr_t *buf_hdr) } static int error_enqueue_multi(odp_queue_t handle, - odp_buffer_hdr_t *buf_hdr[], int num) + _odp_event_hdr_t *event_hdr[], int num) { - (void)buf_hdr; + (void)event_hdr; (void)num; ODP_ERR("Enqueue multi not supported (0x%" PRIx64 ")\n", @@ -911,7 +911,7 @@ static int error_enqueue_multi(odp_queue_t handle, return -1; } -static odp_buffer_hdr_t *error_dequeue(odp_queue_t handle) +static _odp_event_hdr_t *error_dequeue(odp_queue_t handle) { ODP_ERR("Dequeue not supported (0x%" PRIx64 ")\n", odp_queue_to_u64(handle)); @@ -920,9 +920,9 @@ static odp_buffer_hdr_t *error_dequeue(odp_queue_t handle) } static int error_dequeue_multi(odp_queue_t handle, - odp_buffer_hdr_t *buf_hdr[], int num) + _odp_event_hdr_t *event_hdr[], int num) { - (void)buf_hdr; + (void)event_hdr; (void)num; ODP_ERR("Dequeue multi not supported (0x%" PRIx64 ")\n", @@ -1068,7 +1068,7 @@ static void queue_print(odp_queue_t handle) } static inline int _sched_queue_enq_multi(odp_queue_t handle, - odp_buffer_hdr_t *buf_hdr[], int num) + _odp_event_hdr_t *event_hdr[], int num) { queue_entry_t *queue; struct rte_event ev[CONFIG_BURST_SIZE]; @@ -1110,7 +1110,7 @@ static inline int _sched_queue_enq_multi(odp_queue_t handle, ev[i].event_type = RTE_EVENT_TYPE_CPU; ev[i].sub_event_type = 0; ev[i].priority = priority; - ev[i].mbuf = &buf_hdr[i]->mb; + ev[i].mbuf = &event_hdr[i]->mb; } num_enq = rte_event_enqueue_new_burst(dev_id, port_id, ev, num); @@ -1119,16 +1119,16 @@ static inline int _sched_queue_enq_multi(odp_queue_t handle, } static int sched_queue_enq_multi(odp_queue_t handle, - odp_buffer_hdr_t *buf_hdr[], int num) + _odp_event_hdr_t *event_hdr[], int num) { - return _sched_queue_enq_multi(handle, buf_hdr, num); + return _sched_queue_enq_multi(handle, event_hdr, num); } -static int sched_queue_enq(odp_queue_t handle, odp_buffer_hdr_t *buf_hdr) +static int sched_queue_enq(odp_queue_t handle, _odp_event_hdr_t *event_hdr) { int ret; - ret = _sched_queue_enq_multi(handle, &buf_hdr, 1); + ret = _sched_queue_enq_multi(handle, &event_hdr, 1); if (ret == 1) return 0; @@ -1271,11 +1271,11 @@ static void queue_set_enq_deq_func(odp_queue_t handle, } static int queue_orig_multi(odp_queue_t handle, - odp_buffer_hdr_t **buf_hdr, int num) + _odp_event_hdr_t **event_hdr, int num) { queue_entry_t *queue = qentry_from_handle(handle); - return queue->s.orig_dequeue_multi(handle, buf_hdr, num); + return queue->s.orig_dequeue_multi(handle, event_hdr, num); } static int queue_api_enq_multi(odp_queue_t handle, @@ -1290,7 +1290,7 @@ static int queue_api_enq_multi(odp_queue_t handle, num = QUEUE_MULTI_MAX; return queue->s.enqueue_multi(handle, - (odp_buffer_hdr_t **)(uintptr_t)ev, num); + (_odp_event_hdr_t **)(uintptr_t)ev, num); } static void queue_timer_add(odp_queue_t handle) @@ -1312,7 +1312,7 @@ static int queue_api_enq(odp_queue_t handle, odp_event_t ev) queue_entry_t *queue = qentry_from_handle(handle); return queue->s.enqueue(handle, - (odp_buffer_hdr_t *)(uintptr_t)ev); + (_odp_event_hdr_t *)(uintptr_t)ev); } static int queue_api_deq_multi(odp_queue_t handle, odp_event_t ev[], int num) @@ -1323,7 +1323,7 @@ static int queue_api_deq_multi(odp_queue_t handle, odp_event_t ev[], int num) if (num > QUEUE_MULTI_MAX) num = QUEUE_MULTI_MAX; - ret = queue->s.dequeue_multi(handle, (odp_buffer_hdr_t **)ev, num); + ret = queue->s.dequeue_multi(handle, (_odp_event_hdr_t **)ev, num); if (odp_global_rw->inline_timers && odp_atomic_load_u64(&queue->s.num_timers)) diff --git a/platform/linux-dpdk/odp_queue_spsc.c b/platform/linux-dpdk/odp_queue_spsc.c index d07451042..dc74c9595 100644 --- a/platform/linux-dpdk/odp_queue_spsc.c +++ b/platform/linux-dpdk/odp_queue_spsc.c @@ -1,15 +1,17 @@ /* Copyright (c) 2018, Linaro Limited + * Copyright (c) 2021, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause */ #include <odp/api/hints.h> -#include <odp_queue_basic_internal.h> #include <odp_debug_internal.h> +#include <odp_event_internal.h> +#include <odp_queue_basic_internal.h> static inline int spsc_enq_multi(odp_queue_t handle, - odp_buffer_hdr_t *buf_hdr[], int num) + _odp_event_hdr_t *event_hdr[], int num) { queue_entry_t *queue; ring_spsc_t ring_spsc; @@ -22,11 +24,11 @@ static inline int spsc_enq_multi(odp_queue_t handle, return -1; } - return ring_spsc_enq_multi(ring_spsc, (void **)buf_hdr, num); + return ring_spsc_enq_multi(ring_spsc, (void **)event_hdr, num); } static inline int spsc_deq_multi(odp_queue_t handle, - odp_buffer_hdr_t *buf_hdr[], int num) + _odp_event_hdr_t *event_hdr[], int num) { queue_entry_t *queue; ring_spsc_t ring_spsc; @@ -39,20 +41,20 @@ static inline int spsc_deq_multi(odp_queue_t handle, return -1; } - return ring_spsc_deq_multi(ring_spsc, (void **)buf_hdr, num); + return ring_spsc_deq_multi(ring_spsc, (void **)event_hdr, num); } -static int queue_spsc_enq_multi(odp_queue_t handle, odp_buffer_hdr_t *buf_hdr[], +static int queue_spsc_enq_multi(odp_queue_t handle, _odp_event_hdr_t *event_hdr[], int num) { - return spsc_enq_multi(handle, buf_hdr, num); + return spsc_enq_multi(handle, event_hdr, num); } -static int queue_spsc_enq(odp_queue_t handle, odp_buffer_hdr_t *buf_hdr) +static int queue_spsc_enq(odp_queue_t handle, _odp_event_hdr_t *event_hdr) { int ret; - ret = spsc_enq_multi(handle, &buf_hdr, 1); + ret = spsc_enq_multi(handle, &event_hdr, 1); if (ret == 1) return 0; @@ -60,21 +62,21 @@ static int queue_spsc_enq(odp_queue_t handle, odp_buffer_hdr_t *buf_hdr) return -1; } -static int queue_spsc_deq_multi(odp_queue_t handle, odp_buffer_hdr_t *buf_hdr[], +static int queue_spsc_deq_multi(odp_queue_t handle, _odp_event_hdr_t *event_hdr[], int num) { - return spsc_deq_multi(handle, buf_hdr, num); + return spsc_deq_multi(handle, event_hdr, num); } -static odp_buffer_hdr_t *queue_spsc_deq(odp_queue_t handle) +static _odp_event_hdr_t *queue_spsc_deq(odp_queue_t handle) { - odp_buffer_hdr_t *buf_hdr = NULL; + _odp_event_hdr_t *event_hdr = NULL; int ret; - ret = spsc_deq_multi(handle, &buf_hdr, 1); + ret = spsc_deq_multi(handle, &event_hdr, 1); if (ret == 1) - return buf_hdr; + return event_hdr; else return NULL; } diff --git a/platform/linux-dpdk/odp_shared_memory.c b/platform/linux-dpdk/odp_shared_memory.c index 645bb8847..c18c9acca 100644 --- a/platform/linux-dpdk/odp_shared_memory.c +++ b/platform/linux-dpdk/odp_shared_memory.c @@ -6,14 +6,17 @@ */ #include <odp_posix_extensions.h> -#include <odp_align_internal.h> -#include <odp_config_internal.h> + #include <odp/api/debug.h> -#include <odp_debug_internal.h> +#include <odp/api/plat/strong_types.h> #include <odp/api/shared_memory.h> #include <odp/api/spinlock.h> -#include <odp/api/plat/strong_types.h> + +#include <odp_align_internal.h> +#include <odp_config_internal.h> +#include <odp_debug_internal.h> #include <odp_shm_internal.h> + #include <string.h> #include <sys/mman.h> #include <sys/syscall.h> @@ -24,6 +27,9 @@ #include <rte_lcore.h> #include <rte_memzone.h> +/* Supported ODP_SHM_* flags */ +#define SUPPORTED_SHM_FLAGS (ODP_SHM_SW_ONLY | ODP_SHM_EXPORT | ODP_SHM_HP) + #define SHM_MAX_ALIGN (0x80000000) #define SHM_BLOCK_NAME "%" PRIu64 "-%d-%s" #define SHM_MAX_NB_BLOCKS (CONFIG_INTERNAL_SHM_BLOCKS + CONFIG_SHM_BLOCKS) @@ -226,6 +232,7 @@ int odp_shm_capability(odp_shm_capability_t *capa) capa->max_blocks = CONFIG_SHM_BLOCKS; capa->max_size = 0; capa->max_align = SHM_MAX_ALIGN; + capa->flags = SUPPORTED_SHM_FLAGS; return 0; } @@ -238,6 +245,12 @@ odp_shm_t odp_shm_reserve(const char *name, uint64_t size, uint64_t align, char mz_name[RTE_MEMZONE_NAMESIZE]; uint32_t mz_flags = RTE_MEMZONE_1GB | RTE_MEMZONE_SIZE_HINT_ONLY; int idx; + uint32_t supported_flgs = SUPPORTED_SHM_FLAGS; + + if (flags & ~supported_flgs) { + ODP_ERR("Unsupported SHM flag\n"); + return ODP_SHM_INVALID; + } if (align > SHM_MAX_ALIGN) { ODP_ERR("Align too large: %" PRIu64 "\n", align); diff --git a/platform/linux-dpdk/odp_timer.c b/platform/linux-dpdk/odp_timer.c index 96df1bf79..d0ebd0763 100644 --- a/platform/linux-dpdk/odp_timer.c +++ b/platform/linux-dpdk/odp_timer.c @@ -462,6 +462,12 @@ int odp_timer_res_capability(odp_timer_clk_src_t clk_src, return 0; } +void odp_timer_pool_param_init(odp_timer_pool_param_t *param) +{ + memset(param, 0, sizeof(odp_timer_pool_param_t)); + param->clk_src = ODP_CLOCK_DEFAULT; +} + odp_timer_pool_t odp_timer_pool_create(const char *name, const odp_timer_pool_param_t *param) { @@ -959,9 +965,8 @@ void *odp_timeout_user_ptr(odp_timeout_t tmo) odp_timeout_t odp_timeout_alloc(odp_pool_t pool_hdl) { - odp_timeout_t tmo; + odp_event_t event; pool_t *pool; - int ret; ODP_ASSERT(pool_hdl != ODP_POOL_INVALID); @@ -969,17 +974,16 @@ odp_timeout_t odp_timeout_alloc(odp_pool_t pool_hdl) ODP_ASSERT(pool->type == ODP_POOL_TIMEOUT); - ret = _odp_buffer_alloc_multi(pool, (odp_buffer_hdr_t **)&tmo, 1); - - if (odp_likely(ret == 1)) - return tmo; + event = _odp_event_alloc(pool); + if (odp_unlikely(event == ODP_EVENT_INVALID)) + return ODP_TIMEOUT_INVALID; - return ODP_TIMEOUT_INVALID; + return odp_timeout_from_event(event); } void odp_timeout_free(odp_timeout_t tmo) { - _odp_buffer_free_multi((odp_buffer_hdr_t **)&tmo, 1); + _odp_event_free(odp_timeout_to_event(tmo)); } void odp_timer_pool_print(odp_timer_pool_t timer_pool) diff --git a/platform/linux-generic/Makefile.am b/platform/linux-generic/Makefile.am index 8763606ad..64245801e 100644 --- a/platform/linux-generic/Makefile.am +++ b/platform/linux-generic/Makefile.am @@ -61,6 +61,7 @@ odpapiabiarchinclude_HEADERS += \ include-abi/odp/api/abi/cpumask.h \ include-abi/odp/api/abi/crypto.h \ include-abi/odp/api/abi/debug.h \ + include-abi/odp/api/abi/dma_types.h \ include-abi/odp/api/abi/errno.h \ include-abi/odp/api/abi/event.h \ include-abi/odp/api/abi/hash.h \ @@ -90,7 +91,7 @@ odpapiabiarchinclude_HEADERS += \ include-abi/odp/api/abi/thrmask.h \ include-abi/odp/api/abi/ticketlock.h \ include-abi/odp/api/abi/time.h \ - include-abi/odp/api/abi/timer.h \ + include-abi/odp/api/abi/timer_types.h \ include-abi/odp/api/abi/traffic_mngr.h \ include-abi/odp/api/abi/version.h endif @@ -106,6 +107,7 @@ noinst_HEADERS = \ include/odp_config_internal.h \ include/odp_debug_internal.h \ include/odp_errno_define.h \ + include/odp_event_internal.h \ include/odp_fdserver_internal.h \ include/odp_forward_typedefs_internal.h \ include/odp_global_data.h \ @@ -176,6 +178,7 @@ __LIB__libodp_linux_la_SOURCES = \ miniz/miniz_tinfl.c miniz/miniz_tinfl.h \ odp_cpumask.c \ odp_cpumask_task.c \ + odp_dma.c \ odp_errno.c \ odp_event.c \ odp_fdserver.c \ @@ -275,6 +278,7 @@ __LIB__libodp_linux_la_SOURCES += arch/default/odp_atomic.c \ arch/default/odp_cpu_cycles.c \ arch/default/odp_global_time.c \ arch/default/odp_hash_crc32.c \ + arch/default/odp_random.c \ arch/arm/odp_sysinfo_parse.c odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_time.h \ arch/default/odp/api/abi/hash_crc32.h @@ -291,7 +295,8 @@ noinst_HEADERS += arch/arm/odp_atomic.h \ arch/arm/odp_llsc.h \ arch/default/odp_atomic.h \ arch/default/odp_cpu.h \ - arch/default/odp_cpu_idling.h + arch/default/odp_cpu_idling.h \ + arch/default/odp_random.h endif if ARCH_IS_AARCH64 __LIB__libodp_linux_la_SOURCES += arch/aarch64/odp_atomic.c \ @@ -299,6 +304,7 @@ __LIB__libodp_linux_la_SOURCES += arch/aarch64/odp_atomic.c \ arch/aarch64/cpu_flags.c \ arch/aarch64/odp_global_time.c \ arch/default/odp_hash_crc32.c \ + arch/default/odp_random.c \ arch/aarch64/odp_sysinfo_parse.c odpapiabiarchinclude_HEADERS += arch/aarch64/odp/api/abi/cpu_time.h \ arch/aarch64/odp/api/abi/hash_crc32.h @@ -314,13 +320,15 @@ noinst_HEADERS += arch/aarch64/odp_atomic.h \ arch/aarch64/odp_cpu.h \ arch/aarch64/cpu_flags.h \ arch/aarch64/odp_cpu_idling.h \ - arch/aarch64/odp_llsc.h + arch/aarch64/odp_llsc.h \ + arch/default/odp_random.h endif if ARCH_IS_DEFAULT __LIB__libodp_linux_la_SOURCES += arch/default/odp_atomic.c \ arch/default/odp_cpu_cycles.c \ arch/default/odp_global_time.c \ arch/default/odp_hash_crc32.c \ + arch/default/odp_random.c \ arch/default/odp_sysinfo_parse.c odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_time.h \ arch/default/odp/api/abi/hash_crc32.h @@ -333,12 +341,14 @@ odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ endif noinst_HEADERS += arch/default/odp_atomic.h \ arch/default/odp_cpu.h \ - arch/default/odp_cpu_idling.h + arch/default/odp_cpu_idling.h \ + arch/default/odp_random.h endif if ARCH_IS_MIPS64 __LIB__libodp_linux_la_SOURCES += arch/default/odp_atomic.c \ arch/default/odp_global_time.c \ arch/default/odp_hash_crc32.c \ + arch/default/odp_random.c \ arch/mips64/odp_sysinfo_parse.c odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_time.h \ arch/default/odp/api/abi/hash_crc32.h @@ -351,13 +361,15 @@ odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ endif noinst_HEADERS += arch/default/odp_atomic.h \ arch/default/odp_cpu.h \ - arch/default/odp_cpu_idling.h + arch/default/odp_cpu_idling.h \ + arch/default/odp_random.h endif if ARCH_IS_POWERPC __LIB__libodp_linux_la_SOURCES += arch/default/odp_atomic.c \ arch/default/odp_cpu_cycles.c \ arch/default/odp_global_time.c \ arch/default/odp_hash_crc32.c \ + arch/default/odp_random.c \ arch/powerpc/odp_sysinfo_parse.c odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_time.h \ arch/default/odp/api/abi/hash_crc32.h @@ -370,7 +382,8 @@ odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ endif noinst_HEADERS += arch/default/odp_atomic.h \ arch/default/odp_cpu.h \ - arch/default/odp_cpu_idling.h + arch/default/odp_cpu_idling.h \ + arch/default/odp_random.h endif if ARCH_IS_X86 __LIB__libodp_linux_la_SOURCES += arch/default/odp_atomic.c \ @@ -378,6 +391,7 @@ __LIB__libodp_linux_la_SOURCES += arch/default/odp_atomic.c \ arch/x86/odp_cpu_cycles.c \ arch/x86/odp_global_time.c \ arch/default/odp_hash_crc32.c \ + arch/default/odp_random.c \ arch/x86/odp_sysinfo_parse.c odpapiabiarchinclude_HEADERS += arch/x86/odp/api/abi/cpu_rdtsc.h \ arch/x86/odp/api/abi/cpu_time.h \ @@ -390,6 +404,7 @@ odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ endif noinst_HEADERS += arch/x86/cpu_flags.h \ arch/x86/odp_cpu.h \ + arch/x86/odp_random.h \ arch/default/odp_atomic.h \ arch/default/odp_cpu.h \ arch/default/odp_cpu_idling.h diff --git a/platform/linux-generic/README b/platform/linux-generic/README index 04267909b..8f41d1d45 100644 --- a/platform/linux-generic/README +++ b/platform/linux-generic/README @@ -41,3 +41,21 @@ SPDX-License-Identifier: BSD-3-Clause socket socket_mmap tap + +5. Random data + On x86 ODP_RANDOM_TRUE type random data is generated using rdseed [1] via + compiler builtin functions. If OpenSSL is not available or its use for + generating random data is disabled with the --disable-openssl-rand + configure option, ODP_RANDOM_CRYPTO type random data is generated using + rdrand [1]. + + Note that there may be issues with the quality or security of rdrand and + rdseed. [2] + +6. References + [1] Intel Digital Random Number Generator (DRNG) Software Implementation + Guide. John P Mechalas, 17 October 2018. + https://www.intel.com/content/www/us/en/developer/articles/guide/intel-digital-random-number-generator-drng-software-implementation-guide.html + + [2] RDRAND. Wikipedia, 29 September 2021. + https://en.wikipedia.org/wiki/RDRAND#Reception diff --git a/platform/linux-generic/arch/aarch64/odp_sysinfo_parse.c b/platform/linux-generic/arch/aarch64/odp_sysinfo_parse.c index dbb6d43af..921bafaf0 100644 --- a/platform/linux-generic/arch/aarch64/odp_sysinfo_parse.c +++ b/platform/linux-generic/arch/aarch64/odp_sysinfo_parse.c @@ -120,6 +120,10 @@ static void aarch64_part_info(char *str, int maxlen, odp_cpu_arch_arm_t *cpu_isa snprintf(str, maxlen, "Cortex-A77"); *cpu_isa = ODP_CPU_ARCH_ARMV8_2; return; + case 0xd40: + snprintf(str, maxlen, "Neoverse V1"); + *cpu_isa = ODP_CPU_ARCH_ARMV8_4; + return; case 0xd41: snprintf(str, maxlen, "Cortex-A78"); *cpu_isa = ODP_CPU_ARCH_ARMV8_2; @@ -128,6 +132,18 @@ static void aarch64_part_info(char *str, int maxlen, odp_cpu_arch_arm_t *cpu_isa snprintf(str, maxlen, "Cortex-A78AE"); *cpu_isa = ODP_CPU_ARCH_ARMV8_2; return; + case 0xd46: + snprintf(str, maxlen, "Cortex-A510"); + *cpu_isa = ODP_CPU_ARCH_ARMV9_0; + return; + case 0xd47: + snprintf(str, maxlen, "Cortex-A710"); + *cpu_isa = ODP_CPU_ARCH_ARMV9_0; + return; + case 0xd49: + snprintf(str, maxlen, "Neoverse N2"); + *cpu_isa = ODP_CPU_ARCH_ARMV9_0; + return; case 0xd4a: snprintf(str, maxlen, "Neoverse E1"); *cpu_isa = ODP_CPU_ARCH_ARMV8_2; diff --git a/platform/linux-generic/arch/default/odp_random.c b/platform/linux-generic/arch/default/odp_random.c new file mode 100644 index 000000000..18d2a45d2 --- /dev/null +++ b/platform/linux-generic/arch/default/odp_random.c @@ -0,0 +1,33 @@ +/* Copyright (c) 2021, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <odp_random.h> +#include <odp/api/spec/random.h> + +#include <odp/visibility_begin.h> + +odp_random_kind_t _odp_random_max_kind_generic(void) +{ + return ODP_RANDOM_BASIC; +} + +int32_t _odp_random_true_data_generic(uint8_t *buf, uint32_t len) +{ + (void)buf; + (void)len; + + return -1; +} + +int32_t _odp_random_crypto_data_generic(uint8_t *buf, uint32_t len) +{ + (void)buf; + (void)len; + + return -1; +} + +#include <odp/visibility_end.h> diff --git a/platform/linux-generic/arch/default/odp_random.h b/platform/linux-generic/arch/default/odp_random.h new file mode 100644 index 000000000..215eb6d93 --- /dev/null +++ b/platform/linux-generic/arch/default/odp_random.h @@ -0,0 +1,41 @@ +/* Copyright (c) 2021, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_DEFAULT_RANDOM_H_ +#define ODP_DEFAULT_RANDOM_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <odp/api/spec/random.h> + +#include <stdint.h> + +odp_random_kind_t _odp_random_max_kind_generic(void); +int32_t _odp_random_true_data_generic(uint8_t *buf, uint32_t len); +int32_t _odp_random_crypto_data_generic(uint8_t *buf, uint32_t len); + +static inline odp_random_kind_t _odp_random_max_kind(void) +{ + return _odp_random_max_kind_generic(); +} + +static inline int32_t _odp_random_true_data(uint8_t *buf, uint32_t len) +{ + return _odp_random_true_data_generic(buf, len); +} + +static inline int32_t _odp_random_crypto_data(uint8_t *buf, uint32_t len) +{ + return _odp_random_crypto_data_generic(buf, len); +} + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/arch/x86/odp_random.h b/platform/linux-generic/arch/x86/odp_random.h new file mode 100644 index 000000000..54628038e --- /dev/null +++ b/platform/linux-generic/arch/x86/odp_random.h @@ -0,0 +1,160 @@ +/* Copyright (c) 2021, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +/* + * These functions implement ODP_RANDOM_CRYPTO random data using rdrand [1], + * and ODP_RANDOM_TRUE random data using rdseed [1], via compiler builtin + * functions. + * + * Note that there may be issues with the quality or security of rdrand and + * rdseed. [2] + * + * [1] Intel Digital Random Number Generator (DRNG) Software Implementation + * Guide. John P Mechalas, 17 October 2018. + * https://www.intel.com/content/www/us/en/developer/articles/guide/intel-digital-random-number-generator-drng-software-implementation-guide.html + * + * [2] RDRAND. Wikipedia, 29 September 2021. + * https://en.wikipedia.org/wiki/RDRAND#Reception + */ + +#ifndef ODP_X86_RANDOM_H_ +#define ODP_X86_RANDOM_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <odp/api/spec/random.h> + +#include <stdint.h> + +odp_random_kind_t _odp_random_max_kind_generic(void); +int32_t _odp_random_true_data_generic(uint8_t *buf, uint32_t len); +int32_t _odp_random_crypto_data_generic(uint8_t *buf, uint32_t len); + +#ifdef __RDRND__ + +static inline int _odp_random_max_kind(void) +{ +#ifdef __RDSEED__ + return ODP_RANDOM_TRUE; +#else + return ODP_RANDOM_CRYPTO; +#endif +} + +#else + +static inline int _odp_random_max_kind(void) +{ + return _odp_random_max_kind_generic(); +} + +#endif + +#ifdef __RDSEED__ + +static inline int32_t _odp_random_true_data(uint8_t *buf, uint32_t len) +{ +#ifdef __x86_64__ + for (uint32_t i = 0; i < len / 8; i++) { + while (!__builtin_ia32_rdseed_di_step((unsigned long long *)buf)) + ; + buf += 8; + } + + if (len & 4) { + while (!__builtin_ia32_rdseed_si_step((unsigned int *)buf)) + ; + buf += 4; + } +#else + for (uint32_t i = 0; i < len / 4; i++) { + while (!__builtin_ia32_rdseed_si_step((unsigned int *)buf)) + ; + buf += 4; + } +#endif + if (len & 2) { + while (!__builtin_ia32_rdseed_hi_step((unsigned short int *)buf)) + ; + buf += 2; + } + + if (len & 1) { + uint16_t w; + + while (!__builtin_ia32_rdseed_hi_step(&w)) + ; + *((uint8_t *)buf) = w & 0xff; + } + + return len; +} + +#else + +static inline int32_t _odp_random_true_data(uint8_t *buf, uint32_t len) +{ + return _odp_random_true_data_generic(buf, len); +} + +#endif + +#ifdef __RDRND__ + +static inline int32_t _odp_random_crypto_data(uint8_t *buf, uint32_t len) +{ +#ifdef __x86_64__ + for (uint32_t i = 0; i < len / 8; i++) { + while (!__builtin_ia32_rdrand64_step((unsigned long long *)buf)) + ; + buf += 8; + } + + if (len & 4) { + while (!__builtin_ia32_rdrand32_step((unsigned int *)buf)) + ; + buf += 4; + } +#else + for (uint32_t i = 0; i < len / 4; i++) { + while (!__builtin_ia32_rdrand32_step((unsigned int *)buf)) + ; + buf += 4; + } +#endif + if (len & 2) { + while (!__builtin_ia32_rdrand16_step((unsigned short int *)buf)) + ; + buf += 2; + } + + if (len & 1) { + uint16_t w; + + while (!__builtin_ia32_rdrand16_step(&w)) + ; + *((uint8_t *)buf) = w & 0xff; + } + + return len; +} + +#else + +static inline int32_t _odp_random_crypto_data(uint8_t *buf, uint32_t len) +{ + return _odp_random_crypto_data_generic(buf, len); +} + +#endif + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/include-abi/odp/api/abi/dma_types.h b/platform/linux-generic/include-abi/odp/api/abi/dma_types.h new file mode 100644 index 000000000..768591b10 --- /dev/null +++ b/platform/linux-generic/include-abi/odp/api/abi/dma_types.h @@ -0,0 +1,42 @@ +/* Copyright (c) 2021, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_API_ABI_DMA_TYPES_H_ +#define ODP_API_ABI_DMA_TYPES_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <odp/api/plat/strong_types.h> + +/** @ingroup odp_dma + * @{ + */ + +typedef ODP_HANDLE_T(odp_dma_t); + +#define ODP_DMA_INVALID _odp_cast_scalar(odp_dma_t, 0) + +typedef uint32_t odp_dma_transfer_id_t; + +#define ODP_DMA_TRANSFER_ID_INVALID ((odp_dma_transfer_id_t)0) + +typedef ODP_HANDLE_T(odp_dma_compl_t); + +#define ODP_DMA_COMPL_INVALID _odp_cast_scalar(odp_dma_compl_t, 0) + +#define ODP_DMA_NAME_LEN 32 + +/** + * @} + */ + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/include-abi/odp/api/abi/event.h b/platform/linux-generic/include-abi/odp/api/abi/event.h index 1cbb81afe..6530ac2e4 100644 --- a/platform/linux-generic/include-abi/odp/api/abi/event.h +++ b/platform/linux-generic/include-abi/odp/api/abi/event.h @@ -35,7 +35,8 @@ typedef enum odp_event_type_t { ODP_EVENT_CRYPTO_COMPL = 4, ODP_EVENT_IPSEC_STATUS = 5, ODP_EVENT_PACKET_VECTOR = 6, - ODP_EVENT_PACKET_TX_COMPL = 7 + ODP_EVENT_PACKET_TX_COMPL = 7, + ODP_EVENT_DMA_COMPL = 8, } odp_event_type_t; typedef enum odp_event_subtype_t { diff --git a/platform/linux-generic/include-abi/odp/api/abi/timer.h b/platform/linux-generic/include-abi/odp/api/abi/timer_types.h index c08da1ce3..6cfa37a36 100644 --- a/platform/linux-generic/include-abi/odp/api/abi/timer.h +++ b/platform/linux-generic/include-abi/odp/api/abi/timer_types.h @@ -11,8 +11,8 @@ * ODP timer service */ -#ifndef ODP_API_ABI_TIMER_H_ -#define ODP_API_ABI_TIMER_H_ +#ifndef ODP_API_ABI_TIMER_TYPES_H_ +#define ODP_API_ABI_TIMER_TYPES_H_ #ifdef __cplusplus extern "C" { diff --git a/platform/linux-generic/include/odp_buffer_internal.h b/platform/linux-generic/include/odp_buffer_internal.h index dec85f9d3..e0be593d0 100644 --- a/platform/linux-generic/include/odp_buffer_internal.h +++ b/platform/linux-generic/include/odp_buffer_internal.h @@ -1,5 +1,5 @@ /* Copyright (c) 2013-2018, Linaro Limited - * Copyright (c) 2019, Nokia + * Copyright (c) 2019-2021, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -29,88 +29,40 @@ extern "C" { #include <odp/api/byteorder.h> #include <odp/api/thread.h> #include <odp/api/event.h> -#include <odp_forward_typedefs_internal.h> +#include <odp_event_internal.h> #include <stddef.h> -typedef union buffer_index_t { - uint32_t u32; +/* Internal buffer header */ +typedef struct ODP_ALIGNED_CACHE odp_buffer_hdr_t { + /* Common event header */ + _odp_event_hdr_t event_hdr; - struct { - uint32_t pool :8; - uint32_t buffer :24; - }; -} buffer_index_t; - -/* Check that pool index fit into bit field */ -ODP_STATIC_ASSERT(ODP_CONFIG_POOLS <= (0xFF + 1), "TOO_MANY_POOLS"); - -/* Check that buffer index fit into bit field */ -ODP_STATIC_ASSERT(CONFIG_POOL_MAX_NUM <= (0xFFFFFF + 1), "TOO_LARGE_POOL"); - -/* Type size limits number of flow IDs supported */ -#define BUF_HDR_MAX_FLOW_ID 255 - -/* Common buffer header */ -struct ODP_ALIGNED_CACHE odp_buffer_hdr_t { - /* Initial buffer data pointer */ - uint8_t *base_data; - - /* Pool pointer */ - void *pool_ptr; - - /* --- Mostly read only data --- */ - const void *user_ptr; - - /* Initial buffer tail pointer */ - uint8_t *buf_end; - - /* User area pointer */ - void *uarea_addr; - - /* Combined pool and buffer index */ - buffer_index_t index; - - /* Reference count */ - odp_atomic_u32_t ref_cnt; - - /* Pool type */ - int8_t type; - - /* Event type. Maybe different than pool type (crypto compl event) */ - int8_t event_type; - - /* Event flow id */ - uint8_t flow_id; - - /* Data or next header */ + /* Data */ uint8_t data[]; -}; +} odp_buffer_hdr_t; /* Buffer header size is critical for performance. Ensure that it does not accidentally * grow over cache line size. Note that ODP_ALIGNED_CACHE rounds up struct size to a multiple of * ODP_CACHE_LINE_SIZE. */ ODP_STATIC_ASSERT(sizeof(odp_buffer_hdr_t) <= ODP_CACHE_LINE_SIZE, "BUFFER_HDR_SIZE_ERROR"); -odp_event_type_t _odp_buffer_event_type(odp_buffer_t buf); -void _odp_buffer_event_type_set(odp_buffer_t buf, int ev); - -static inline odp_buffer_t buf_from_buf_hdr(odp_buffer_hdr_t *hdr) +static inline odp_buffer_hdr_t *_odp_buf_hdr(odp_buffer_t buf) { - return (odp_buffer_t)hdr; + return (odp_buffer_hdr_t *)(uintptr_t)buf; } static inline uint32_t event_flow_id(odp_event_t ev) { odp_buffer_hdr_t *buf_hdr = (odp_buffer_hdr_t *)(uintptr_t)ev; - return buf_hdr->flow_id; + return buf_hdr->event_hdr.flow_id; } static inline void event_flow_id_set(odp_event_t ev, uint32_t flow_id) { odp_buffer_hdr_t *buf_hdr = (odp_buffer_hdr_t *)(uintptr_t)ev; - buf_hdr->flow_id = flow_id; + buf_hdr->event_hdr.flow_id = flow_id; } #ifdef __cplusplus diff --git a/platform/linux-generic/include/odp_config_internal.h b/platform/linux-generic/include/odp_config_internal.h index 899b261bd..872d6f6d5 100644 --- a/platform/linux-generic/include/odp_config_internal.h +++ b/platform/linux-generic/include/odp_config_internal.h @@ -54,6 +54,16 @@ extern "C" { #define CONFIG_QUEUE_MAX_ORD_LOCKS 2 /* + * Maximum number of DMA sessions + */ +#define CONFIG_MAX_DMA_SESSIONS 32 + +/* + * Stashes reserved for internal usage + */ +#define CONFIG_INTERNAL_STASHES CONFIG_MAX_DMA_SESSIONS + +/* * Maximum number of stashes */ #define CONFIG_MAX_STASHES 128 diff --git a/platform/linux-generic/include/odp_event_internal.h b/platform/linux-generic/include/odp_event_internal.h new file mode 100644 index 000000000..fa7e5f354 --- /dev/null +++ b/platform/linux-generic/include/odp_event_internal.h @@ -0,0 +1,103 @@ +/* Copyright (c) 2021, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +/** + * @file + * + * ODP event descriptor - implementation internal + */ + +#ifndef ODP_EVENT_INTERNAL_H_ +#define ODP_EVENT_INTERNAL_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <odp/api/atomic.h> +#include <odp/api/debug.h> +#include <odp/api/event.h> +#include <odp/api/std_types.h> + +#include <odp_config_internal.h> + +typedef union buffer_index_t { + uint32_t u32; + + struct { + uint32_t pool :8; + uint32_t buffer :24; + }; +} buffer_index_t; + +/* Check that pool index fit into bit field */ +ODP_STATIC_ASSERT(ODP_CONFIG_POOLS <= (0xFF + 1), "TOO_MANY_POOLS"); + +/* Check that buffer index fit into bit field */ +ODP_STATIC_ASSERT(CONFIG_POOL_MAX_NUM <= (0xFFFFFF + 1), "TOO_LARGE_POOL"); + +/* Type size limits number of flow IDs supported */ +#define BUF_HDR_MAX_FLOW_ID 255 + +/* Common header for all event types without alignment constraints. */ +typedef struct _odp_event_hdr_t { + /* Initial buffer data pointer */ + uint8_t *base_data; + + /* Pool pointer */ + void *pool_ptr; + + /* --- Mostly read only data --- */ + const void *user_ptr; + + /* Initial buffer tail pointer */ + uint8_t *buf_end; + + /* User area pointer */ + void *uarea_addr; + + /* Combined pool and buffer index */ + buffer_index_t index; + + /* Reference count */ + odp_atomic_u32_t ref_cnt; + + /* Pool type */ + int8_t type; + + /* Event type. Maybe different than pool type (crypto compl event) */ + int8_t event_type; + + /* Event flow id */ + uint8_t flow_id; + +} _odp_event_hdr_t; + +static inline odp_event_t _odp_event_from_hdr(_odp_event_hdr_t *hdr) +{ + return (odp_event_t)hdr; +} + +static inline _odp_event_hdr_t *_odp_event_hdr(odp_event_t event) +{ + return (_odp_event_hdr_t *)(uintptr_t)event; +} + +static inline odp_event_type_t _odp_event_type(odp_event_t event) +{ + return _odp_event_hdr(event)->event_type; +} + +static inline void _odp_event_type_set(odp_event_t event, int ev) +{ + _odp_event_hdr(event)->event_type = ev; +} + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/include/odp_event_vector_internal.h b/platform/linux-generic/include/odp_event_vector_internal.h index 2d51801df..33b26d711 100644 --- a/platform/linux-generic/include/odp_event_vector_internal.h +++ b/platform/linux-generic/include/odp_event_vector_internal.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2020, Nokia +/* Copyright (c) 2020-2021, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -13,18 +13,20 @@ #ifndef ODP_EVENT_VECTOR_INTERNAL_H_ #define ODP_EVENT_VECTOR_INTERNAL_H_ -#include <stdint.h> +#include <odp/api/align.h> +#include <odp/api/debug.h> #include <odp/api/packet.h> -#include <odp_buffer_internal.h> -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wpedantic" +#include <odp_event_internal.h> + +#include <stdint.h> + /** * Internal event vector header */ -typedef struct { - /* Common buffer header */ - odp_buffer_hdr_t buf_hdr; +typedef struct ODP_ALIGNED_CACHE odp_event_vector_hdr_t { + /* Common event header */ + _odp_event_hdr_t event_hdr; /* Event vector size */ uint32_t size; @@ -33,7 +35,11 @@ typedef struct { odp_packet_t packet[]; } odp_event_vector_hdr_t; -#pragma GCC diagnostic pop + +/* Vector header size is critical for performance. Ensure that it does not accidentally + * grow over cache line size. */ +ODP_STATIC_ASSERT(sizeof(odp_event_vector_hdr_t) <= ODP_CACHE_LINE_SIZE, + "EVENT_VECTOR_HDR_SIZE_ERROR"); /** * Return the vector header diff --git a/platform/linux-generic/include/odp_forward_typedefs_internal.h b/platform/linux-generic/include/odp_forward_typedefs_internal.h index fc2d74857..d7e14b953 100644 --- a/platform/linux-generic/include/odp_forward_typedefs_internal.h +++ b/platform/linux-generic/include/odp_forward_typedefs_internal.h @@ -10,7 +10,7 @@ * ODP forward typedefs - implementation internal * * This needs to be a separate file because it is needed by both - * odp_queue_internal.h and odp_buffer_internal.h and clang prohibits forward + * odp_queue_internal.h and odp_queue_lf.h and clang prohibits forward * "redefining" typedefs. Note that this file can be extended with additional * forward typedefs as needed. */ @@ -22,7 +22,6 @@ extern "C" { #endif -typedef struct odp_buffer_hdr_t odp_buffer_hdr_t; typedef union queue_entry_u queue_entry_t; #ifdef __cplusplus diff --git a/platform/linux-generic/include/odp_global_data.h b/platform/linux-generic/include/odp_global_data.h index 75978bd6c..c94369e9f 100644 --- a/platform/linux-generic/include/odp_global_data.h +++ b/platform/linux-generic/include/odp_global_data.h @@ -72,12 +72,12 @@ typedef struct odp_global_data_ro_t { uint8_t has_config_rt; config_t libconfig_default; config_t libconfig_runtime; - odp_random_kind_t ipsec_rand_kind; /* Disabled features during global init */ struct { uint8_t compress; uint8_t crypto; + uint8_t dma; uint8_t ipsec; uint8_t stash; uint8_t traffic_mngr; diff --git a/platform/linux-generic/include/odp_init_internal.h b/platform/linux-generic/include/odp_init_internal.h index 9754dfa31..2a1039854 100644 --- a/platform/linux-generic/include/odp_init_internal.h +++ b/platform/linux-generic/include/odp_init_internal.h @@ -99,6 +99,9 @@ int _odp_hash_term_global(void); int _odp_stash_init_global(void); int _odp_stash_term_global(void); +int _odp_dma_init_global(void); +int _odp_dma_term_global(void); + #ifdef __cplusplus } #endif diff --git a/platform/linux-generic/include/odp_ipsec_internal.h b/platform/linux-generic/include/odp_ipsec_internal.h index cc224e4cc..b7fb2cbf7 100644 --- a/platform/linux-generic/include/odp_ipsec_internal.h +++ b/platform/linux-generic/include/odp_ipsec_internal.h @@ -171,6 +171,7 @@ struct ipsec_sa_s { union { unsigned flags; struct { + unsigned inbound : 1; unsigned dec_ttl : 1; unsigned copy_dscp : 1; unsigned copy_df : 1; @@ -246,7 +247,28 @@ struct ipsec_sa_s { } stats; uint32_t next_sa; - odp_ipsec_sa_param_t param; + + /* Data stored solely for odp_ipsec_sa_info() */ + struct { + odp_cipher_alg_t cipher_alg; + uint32_t cipher_key_len; + uint32_t cipher_key_extra_len; + + odp_auth_alg_t auth_alg; + uint32_t auth_key_len; + uint32_t auth_key_extra_len; + + uint32_t icv_len; + uint32_t context_len; + union { + struct { + uint32_t antireplay_ws; + } in; + struct{ + uint32_t mtu; + } out; + }; + } sa_info; }; /** diff --git a/platform/linux-generic/include/odp_packet_internal.h b/platform/linux-generic/include/odp_packet_internal.h index 62f8aea25..7c9b7735e 100644 --- a/platform/linux-generic/include/odp_packet_internal.h +++ b/platform/linux-generic/include/odp_packet_internal.h @@ -1,5 +1,5 @@ /* Copyright (c) 2013-2018, Linaro Limited - * Copyright (c) 2019, Nokia + * Copyright (c) 2019-2021, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -20,15 +20,16 @@ extern "C" { #include <odp/api/align.h> #include <odp/api/debug.h> -#include <odp_buffer_internal.h> -#include <odp_pool_internal.h> #include <odp/api/packet.h> #include <odp/api/plat/packet_inline_types.h> #include <odp/api/packet_io.h> #include <odp/api/crypto.h> #include <odp/api/comp.h> -#include <odp_ipsec_internal.h> #include <odp/api/abi/packet.h> + +#include <odp_event_internal.h> +#include <odp_ipsec_internal.h> +#include <odp_pool_internal.h> #include <odp_queue_if.h> #include <stdint.h> @@ -73,8 +74,6 @@ typedef struct { ODP_STATIC_ASSERT(PKT_MAX_SEGS < UINT16_MAX, "PACKET_MAX_SEGS_ERROR"); -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wpedantic" /** * Internal Packet header * @@ -82,9 +81,9 @@ ODP_STATIC_ASSERT(PKT_MAX_SEGS < UINT16_MAX, "PACKET_MAX_SEGS_ERROR"); * packet_init(). Because of this any new fields added must be reviewed for * initialization requirements. */ -typedef struct odp_packet_hdr_t { - /* Common buffer header (cache line aligned) */ - odp_buffer_hdr_t buf_hdr; +typedef struct ODP_ALIGNED_CACHE odp_packet_hdr_t { + /* Common event header */ + _odp_event_hdr_t event_hdr; /* Segment data start */ uint8_t *seg_data; @@ -153,13 +152,10 @@ typedef struct odp_packet_hdr_t { uint8_t data[]; } odp_packet_hdr_t; -#pragma GCC diagnostic pop /* Packet header size is critical for performance. Ensure that it does not accidentally - * grow over 256 bytes when cache line size is 64 bytes (or less). With larger cache line sizes, - * the struct size is larger due to the odp_buffer_hdr_t alignment requirement. */ -ODP_STATIC_ASSERT(sizeof(odp_packet_hdr_t) <= 256 || ODP_CACHE_LINE_SIZE > 64, - "PACKET_HDR_SIZE_ERROR"); + * grow over 256 bytes. */ +ODP_STATIC_ASSERT(sizeof(odp_packet_hdr_t) <= 256, "PACKET_HDR_SIZE_ERROR"); /** * Return the packet header @@ -174,14 +170,14 @@ static inline odp_packet_t packet_handle(odp_packet_hdr_t *pkt_hdr) return (odp_packet_t)pkt_hdr; } -static inline odp_buffer_hdr_t *packet_to_buf_hdr(odp_packet_t pkt) +static inline _odp_event_hdr_t *packet_to_event_hdr(odp_packet_t pkt) { - return &packet_hdr(pkt)->buf_hdr; + return (_odp_event_hdr_t *)(uintptr_t)&packet_hdr(pkt)->event_hdr; } -static inline odp_packet_t packet_from_buf_hdr(odp_buffer_hdr_t *buf_hdr) +static inline odp_packet_t packet_from_event_hdr(_odp_event_hdr_t *event_hdr) { - return (odp_packet_t)(odp_packet_hdr_t *)buf_hdr; + return (odp_packet_t)(uintptr_t)event_hdr; } static inline odp_packet_hdr_t *packet_last_seg(odp_packet_hdr_t *hdr) @@ -202,7 +198,7 @@ static inline void packet_subtype_set(odp_packet_t pkt, int ev) */ static inline void packet_init(odp_packet_hdr_t *pkt_hdr, uint32_t len) { - pool_t *pool = pkt_hdr->buf_hdr.pool_ptr; + pool_t *pool = pkt_hdr->event_hdr.pool_ptr; uint32_t seg_len; int num = pkt_hdr->seg_count; diff --git a/platform/linux-generic/include/odp_pool_internal.h b/platform/linux-generic/include/odp_pool_internal.h index 001bdfc37..c9bae7142 100644 --- a/platform/linux-generic/include/odp_pool_internal.h +++ b/platform/linux-generic/include/odp_pool_internal.h @@ -23,6 +23,7 @@ extern "C" { #include <odp/api/align.h> #include <odp_buffer_internal.h> +#include <odp_event_internal.h> #include <odp_config_internal.h> #include <odp_ring_ptr_internal.h> #include <odp/api/plat/strong_types.h> @@ -31,22 +32,22 @@ typedef struct ODP_ALIGNED_CACHE pool_cache_t { /* Number of buffers in cache */ uint32_t cache_num; /* Cached buffers */ - odp_buffer_hdr_t *buf_hdr[CONFIG_POOL_CACHE_MAX_SIZE]; + _odp_event_hdr_t *event_hdr[CONFIG_POOL_CACHE_MAX_SIZE]; } pool_cache_t; #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wpedantic" -/* Buffer header ring */ +/* Event header ring */ typedef struct ODP_ALIGNED_CACHE { /* Ring header */ ring_ptr_t hdr; /* Ring data: buffer handles */ - odp_buffer_hdr_t *buf_hdr[CONFIG_POOL_MAX_NUM + 1]; + _odp_event_hdr_t *event_hdr[CONFIG_POOL_MAX_NUM + 1]; /* Index to pointer look-up table for external memory pool */ - odp_buffer_hdr_t *buf_hdr_by_index[]; + _odp_event_hdr_t *event_hdr_by_index[]; } pool_ring_t; #pragma GCC diagnostic pop @@ -88,6 +89,7 @@ typedef struct pool_t { uint8_t *base_addr; uint8_t *max_addr; uint8_t *uarea_base_addr; + odp_pool_type_t type_2; odp_pool_ext_param_t ext_param; /* Used by DPDK zero-copy pktio */ @@ -140,27 +142,22 @@ static inline pool_t *pool_entry_from_hdl(odp_pool_t pool_hdl) return &_odp_pool_glb->pool[_odp_typeval(pool_hdl) - 1]; } -static inline odp_buffer_hdr_t *buf_hdl_to_hdr(odp_buffer_t buf) -{ - return (odp_buffer_hdr_t *)(uintptr_t)buf; -} - -static inline odp_buffer_hdr_t *buf_hdr_from_index(pool_t *pool, - uint32_t buffer_idx) +static inline _odp_event_hdr_t *event_hdr_from_index(pool_t *pool, + uint32_t event_idx) { uint64_t block_offset; - odp_buffer_hdr_t *buf_hdr; + _odp_event_hdr_t *event_hdr; - block_offset = (buffer_idx * (uint64_t)pool->block_size) + + block_offset = (event_idx * (uint64_t)pool->block_size) + pool->block_offset; /* clang requires cast to uintptr_t */ - buf_hdr = (odp_buffer_hdr_t *)(uintptr_t)&pool->base_addr[block_offset]; + event_hdr = (_odp_event_hdr_t *)(uintptr_t)&pool->base_addr[block_offset]; - return buf_hdr; + return event_hdr; } -static inline odp_buffer_hdr_t *buf_hdr_from_index_u32(uint32_t u32) +static inline _odp_event_hdr_t *_odp_event_hdr_from_index_u32(uint32_t u32) { buffer_index_t index; uint32_t pool_idx, buffer_idx; @@ -171,12 +168,21 @@ static inline odp_buffer_hdr_t *buf_hdr_from_index_u32(uint32_t u32) buffer_idx = index.buffer; pool = pool_entry(pool_idx); - return buf_hdr_from_index(pool, buffer_idx); + return event_hdr_from_index(pool, buffer_idx); +} + +odp_event_t _odp_event_alloc(pool_t *pool); +int _odp_event_alloc_multi(pool_t *pool, _odp_event_hdr_t *event_hdr[], int num); +void _odp_event_free_multi(_odp_event_hdr_t *event_hdr[], int num_free); +int _odp_event_is_valid(odp_event_t event); + +static inline void _odp_event_free(odp_event_t event) +{ + _odp_event_free_multi((_odp_event_hdr_t **)&event, 1); } -int _odp_buffer_alloc_multi(pool_t *pool, odp_buffer_hdr_t *buf_hdr[], int num); -void _odp_buffer_free_multi(odp_buffer_hdr_t *buf_hdr[], int num_free); -int _odp_buffer_is_valid(odp_buffer_t buf); +odp_pool_t _odp_pool_create(const char *name, const odp_pool_param_t *params, + odp_pool_type_t type_2); #ifdef __cplusplus } diff --git a/platform/linux-generic/include/odp_queue_if.h b/platform/linux-generic/include/odp_queue_if.h index fa92a4171..ed4ec4e61 100644 --- a/platform/linux-generic/include/odp_queue_if.h +++ b/platform/linux-generic/include/odp_queue_if.h @@ -1,4 +1,5 @@ /* Copyright (c) 2017, ARM Limited + * Copyright (c) 2021, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -14,6 +15,8 @@ extern "C" { #include <odp/api/queue.h> #include <odp/api/schedule.h> #include <odp/api/packet_io.h> + +#include <odp_event_internal.h> #include <odp_forward_typedefs_internal.h> #define QUEUE_MULTI_MAX CONFIG_BURST_SIZE @@ -22,12 +25,12 @@ typedef int (*queue_init_global_fn_t)(void); typedef int (*queue_term_global_fn_t)(void); typedef int (*queue_init_local_fn_t)(void); typedef int (*queue_term_local_fn_t)(void); -typedef int (*queue_enq_fn_t)(odp_queue_t queue, odp_buffer_hdr_t *buf_hdr); +typedef int (*queue_enq_fn_t)(odp_queue_t queue, _odp_event_hdr_t *event_hdr); typedef int (*queue_enq_multi_fn_t)(odp_queue_t queue, - odp_buffer_hdr_t **buf_hdr, int num); -typedef odp_buffer_hdr_t *(*queue_deq_fn_t)(odp_queue_t queue); + _odp_event_hdr_t **event_hdr, int num); +typedef _odp_event_hdr_t *(*queue_deq_fn_t)(odp_queue_t queue); typedef int (*queue_deq_multi_fn_t)(odp_queue_t queue, - odp_buffer_hdr_t **buf_hdr, int num); + _odp_event_hdr_t **event_hdr, int num); typedef odp_pktout_queue_t (*queue_get_pktout_fn_t)(odp_queue_t queue); typedef void (*queue_set_pktout_fn_t)(odp_queue_t queue, odp_pktio_t pktio, int index); diff --git a/platform/linux-generic/include/odp_queue_scalable_internal.h b/platform/linux-generic/include/odp_queue_scalable_internal.h index 9f326a9ee..6f9b85c85 100644 --- a/platform/linux-generic/include/odp_queue_scalable_internal.h +++ b/platform/linux-generic/include/odp_queue_scalable_internal.h @@ -17,7 +17,7 @@ extern "C" { #include <odp/api/queue.h> #include <odp_forward_typedefs_internal.h> #include <odp_queue_if.h> -#include <odp_buffer_internal.h> +#include <odp_event_internal.h> #include <odp_align_internal.h> #include <odp/api/packet_io.h> #include <odp/api/align.h> @@ -58,10 +58,10 @@ union queue_entry_u { uint8_t pad[ROUNDUP_CACHE_LINE(sizeof(struct queue_entry_s))]; }; -int _odp_queue_deq(sched_elem_t *q, odp_buffer_hdr_t *buf_hdr[], int num); +int _odp_queue_deq(sched_elem_t *q, _odp_event_hdr_t *event_hdr[], int num); int _odp_queue_deq_sc(sched_elem_t *q, odp_event_t *evp, int num); int _odp_queue_deq_mc(sched_elem_t *q, odp_event_t *evp, int num); -int _odp_queue_enq_sp(sched_elem_t *q, odp_buffer_hdr_t *buf_hdr[], int num); +int _odp_queue_enq_sp(sched_elem_t *q, _odp_event_hdr_t *event_hdr[], int num); queue_entry_t *_odp_qentry_from_ext(odp_queue_t handle); /* Round up memory size to next cache line size to diff --git a/platform/linux-generic/include/odp_random_openssl_internal.h b/platform/linux-generic/include/odp_random_openssl_internal.h index 3205a2c32..5cb9006d1 100644 --- a/platform/linux-generic/include/odp_random_openssl_internal.h +++ b/platform/linux-generic/include/odp_random_openssl_internal.h @@ -13,10 +13,7 @@ extern "C" { #include <stdint.h> -#include <odp/api/random.h> - -odp_random_kind_t _odp_random_openssl_max_kind(void); -int32_t _odp_random_openssl_data(uint8_t *buf, uint32_t len, odp_random_kind_t kind); +int32_t _odp_random_openssl_data(uint8_t *buf, uint32_t len); int _odp_random_openssl_init_local(void); int _odp_random_openssl_term_local(void); diff --git a/platform/linux-generic/include/odp_random_std_internal.h b/platform/linux-generic/include/odp_random_std_internal.h index 69f8b6d85..fb350fd22 100644 --- a/platform/linux-generic/include/odp_random_std_internal.h +++ b/platform/linux-generic/include/odp_random_std_internal.h @@ -13,11 +13,8 @@ extern "C" { #include <stdint.h> -#include <odp/api/random.h> - -odp_random_kind_t _odp_random_std_max_kind(void); int32_t _odp_random_std_test_data(uint8_t *buf, uint32_t len, uint64_t *seed); -int32_t _odp_random_std_data(uint8_t *buf, uint32_t len, odp_random_kind_t kind); +int32_t _odp_random_std_data(uint8_t *buf, uint32_t len); int _odp_random_std_init_local(void); int _odp_random_std_term_local(void); diff --git a/platform/linux-generic/include/odp_schedule_if.h b/platform/linux-generic/include/odp_schedule_if.h index a804f8c95..dddd2182d 100644 --- a/platform/linux-generic/include/odp_schedule_if.h +++ b/platform/linux-generic/include/odp_schedule_if.h @@ -13,9 +13,10 @@ extern "C" { #endif #include <odp/api/queue.h> -#include <odp_queue_if.h> #include <odp/api/schedule.h> -#include <odp_forward_typedefs_internal.h> + +#include <odp_event_internal.h> +#include <odp_queue_if.h> #define _ODP_SCHED_ID_BASIC 0 #define _ODP_SCHED_ID_SP 1 @@ -45,8 +46,8 @@ typedef int (*schedule_create_queue_fn_t)(uint32_t queue_index, typedef void (*schedule_destroy_queue_fn_t)(uint32_t queue_index); typedef int (*schedule_sched_queue_fn_t)(uint32_t queue_index); typedef int (*schedule_unsched_queue_fn_t)(uint32_t queue_index); -typedef int (*schedule_ord_enq_multi_fn_t)(odp_queue_t queue, - void *buf_hdr[], int num, int *ret); +typedef int (*schedule_ord_enq_multi_fn_t)(odp_queue_t queue, void *event_hdr[], + int num, int *ret); typedef int (*schedule_init_global_fn_t)(void); typedef int (*schedule_term_global_fn_t)(void); typedef int (*schedule_init_local_fn_t)(void); @@ -87,7 +88,7 @@ extern const schedule_fn_t *_odp_sched_fn; /* Interface for the scheduler */ int _odp_sched_cb_pktin_poll(int pktio_index, int pktin_index, - odp_buffer_hdr_t *hdr_tbl[], int num); + _odp_event_hdr_t *hdr_tbl[], int num); int _odp_sched_cb_pktin_poll_one(int pktio_index, int rx_queue, odp_event_t evts[]); void _odp_sched_cb_pktio_stop_finalize(int pktio_index); diff --git a/platform/linux-generic/include/odp_schedule_scalable.h b/platform/linux-generic/include/odp_schedule_scalable.h index c5e6a2880..207573f4c 100644 --- a/platform/linux-generic/include/odp_schedule_scalable.h +++ b/platform/linux-generic/include/odp_schedule_scalable.h @@ -13,6 +13,7 @@ #include <odp/api/schedule.h> #include <odp/api/ticketlock.h> +#include <odp_event_internal.h> #include <odp_schedule_scalable_config.h> #include <odp_schedule_scalable_ordered.h> #include <odp_llqueue.h> @@ -74,13 +75,13 @@ typedef struct ODP_ALIGNED_CACHE { ringidx_t prod_read SPLIT_PC; ringidx_t prod_write; ringidx_t prod_mask; - odp_buffer_hdr_t **prod_ring; + _odp_event_hdr_t **prod_ring; ringidx_t cons_write SPLIT_PC; ringidx_t cons_read; reorder_window_t *rwin; void *user_ctx; #ifdef CONFIG_SPLIT_PRODCONS - odp_buffer_hdr_t **cons_ring; + _odp_event_hdr_t **cons_ring; ringidx_t cons_mask; uint16_t cons_type; #else diff --git a/platform/linux-generic/include/odp_schedule_scalable_ordered.h b/platform/linux-generic/include/odp_schedule_scalable_ordered.h index 3fa81f750..21c89bed2 100644 --- a/platform/linux-generic/include/odp_schedule_scalable_ordered.h +++ b/platform/linux-generic/include/odp_schedule_scalable_ordered.h @@ -13,6 +13,7 @@ #include <odp_align_internal.h> #include <odp_bitset.h> +#include <odp_event_internal.h> #include <odp_ishmpool_internal.h> /* High level functioning of reordering @@ -106,7 +107,7 @@ struct ODP_ALIGNED_CACHE reorder_context { /* Number of events stored in this reorder context */ uint8_t numevts; /* Events stored in this context */ - odp_buffer_hdr_t *events[RC_EVT_SIZE]; + _odp_event_hdr_t *events[RC_EVT_SIZE]; queue_entry_t *destq[RC_EVT_SIZE]; }; @@ -119,6 +120,6 @@ void _odp_rwin_unreserve_sc(reorder_window_t *rwin, uint32_t sn); void _odp_rctx_init(reorder_context_t *rctx, uint16_t idx, reorder_window_t *rwin, uint32_t sn); void _odp_rctx_release(reorder_context_t *rctx); -int _odp_rctx_save(queue_entry_t *queue, odp_buffer_hdr_t *buf_hdr[], int num); +int _odp_rctx_save(queue_entry_t *queue, _odp_event_hdr_t *event_hdr[], int num); #endif /* ODP_SCHEDULE_SCALABLE_ORDERED_H */ diff --git a/platform/linux-generic/include/odp_sysinfo_internal.h b/platform/linux-generic/include/odp_sysinfo_internal.h index 81bfd045f..16e4ced84 100644 --- a/platform/linux-generic/include/odp_sysinfo_internal.h +++ b/platform/linux-generic/include/odp_sysinfo_internal.h @@ -17,7 +17,6 @@ extern "C" { #include <string.h> int _odp_cpuinfo_parser(FILE *file, system_info_t *sysinfo); -uint64_t odp_cpu_hz_current(int id); uint64_t odp_cpu_arch_hz_current(int id); void _odp_sys_info_print_arch(void); diff --git a/platform/linux-generic/include/odp_timer_internal.h b/platform/linux-generic/include/odp_timer_internal.h index 435fa8b70..2a7173d29 100644 --- a/platform/linux-generic/include/odp_timer_internal.h +++ b/platform/linux-generic/include/odp_timer_internal.h @@ -1,4 +1,5 @@ /* Copyright (c) 2014-2018, Linaro Limited + * Copyright (c) 2021, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -15,19 +16,18 @@ #include <odp/api/align.h> #include <odp/api/debug.h> -#include <odp_buffer_internal.h> -#include <odp_pool_internal.h> #include <odp/api/timer.h> + +#include <odp_event_internal.h> #include <odp_global_data.h> +#include <odp_pool_internal.h> -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wpedantic" /** * Internal Timeout header */ -typedef struct { - /* common buffer header */ - odp_buffer_hdr_t buf_hdr; +typedef struct ODP_ALIGNED_CACHE odp_timeout_hdr_t { + /* Common event header */ + _odp_event_hdr_t event_hdr; /* Requested expiration time */ uint64_t expiration; @@ -39,7 +39,6 @@ typedef struct { odp_timer_t timer; } odp_timeout_hdr_t; -#pragma GCC diagnostic pop /* A larger decrement value should be used after receiving events compared to * an 'empty' call. */ diff --git a/platform/linux-generic/m4/configure.m4 b/platform/linux-generic/m4/configure.m4 index ffac70414..9481deca1 100644 --- a/platform/linux-generic/m4/configure.m4 +++ b/platform/linux-generic/m4/configure.m4 @@ -21,6 +21,7 @@ AS_IF([test "x$with_pcap" != xno], AM_CONDITIONAL([ODP_PKTIO_PCAP], [test x$have_pcap = xyes]) m4_include([platform/linux-generic/m4/odp_libconfig.m4]) +m4_include([platform/linux-generic/m4/odp_openssl.m4]) m4_include([platform/linux-generic/m4/odp_pcapng.m4]) m4_include([platform/linux-generic/m4/odp_netmap.m4]) m4_include([platform/linux-generic/m4/odp_dpdk.m4]) @@ -31,6 +32,8 @@ AS_VAR_APPEND([PLAT_DEP_LIBS], ["${ATOMIC_LIBS} ${LIBCONFIG_LIBS} ${OPENSSL_LIBS # Add text to the end of configure with platform specific settings. # Make sure it's aligned same as other lines in configure.ac. AS_VAR_APPEND([PLAT_CFG_TEXT], [" + openssl: ${with_openssl} + openssl_rand: ${openssl_rand} pcap: ${have_pcap} pcapng: ${have_pcapng} default_config_path: ${default_config_path}"]) diff --git a/platform/linux-generic/m4/odp_libconfig.m4 b/platform/linux-generic/m4/odp_libconfig.m4 index ccbf1d6f5..90b49d155 100644 --- a/platform/linux-generic/m4/odp_libconfig.m4 +++ b/platform/linux-generic/m4/odp_libconfig.m4 @@ -3,7 +3,7 @@ ########################################################################## m4_define([_odp_config_version_generation], [0]) m4_define([_odp_config_version_major], [1]) -m4_define([_odp_config_version_minor], [18]) +m4_define([_odp_config_version_minor], [19]) m4_define([_odp_config_version], [_odp_config_version_generation._odp_config_version_major._odp_config_version_minor]) diff --git a/platform/linux-generic/m4/odp_openssl.m4 b/platform/linux-generic/m4/odp_openssl.m4 new file mode 100644 index 000000000..c9d584c64 --- /dev/null +++ b/platform/linux-generic/m4/odp_openssl.m4 @@ -0,0 +1,36 @@ +########################################################################## +# Enable/disable usage of OpenSSL library +########################################################################## +AC_ARG_WITH([openssl], + [AS_HELP_STRING([--without-openssl], + [compile without OpenSSL (may result in disabled crypto and random support)] + [[default=with] (linux-generic)])], + [], + [with_openssl=yes]) +AS_IF([test "$with_openssl" != "no"], + [ODP_OPENSSL + have_openssl=1], [have_openssl=0]) +AM_CONDITIONAL([WITH_OPENSSL], [test x$with_openssl != xno]) +AC_DEFINE_UNQUOTED([_ODP_OPENSSL], [$have_openssl], + [Define to 1 to enable OpenSSL support]) + +AS_IF([test "${with_openssl}" = "no"], + [AC_MSG_WARN([Strong cryptography is not available without OpenSSL])]) + +########################################################################## +# Enable/disable usage of OpenSSL for random data +########################################################################## +have_openssl_rand=1 +AC_ARG_ENABLE([openssl-rand], + [AS_HELP_STRING([--disable-openssl-rand], + [disable OpenSSL random data (use arch-specific instead)] + [[default=enabled] (linux-generic)])], + [if test "x$enableval" = "xno"; then + have_openssl_rand=0 + fi]) + +AS_IF([test "$have_openssl" != "1"], [have_openssl_rand=0]) +AS_IF([test "$have_openssl_rand" = "1"], [openssl_rand=yes], [openssl_rand=no]) + +AC_DEFINE_UNQUOTED([_ODP_OPENSSL_RAND], [$have_openssl_rand], + [Define to 1 to enable OpenSSL support]) diff --git a/platform/linux-generic/odp_buffer.c b/platform/linux-generic/odp_buffer.c index fed113923..0e606017f 100644 --- a/platform/linux-generic/odp_buffer.c +++ b/platform/linux-generic/odp_buffer.c @@ -20,16 +20,16 @@ /* Fill in buffer header field offsets for inline functions */ const _odp_buffer_inline_offset_t _odp_buffer_inline_offset ODP_ALIGNED_CACHE = { - .event_type = offsetof(odp_buffer_hdr_t, event_type), - .base_data = offsetof(odp_buffer_hdr_t, base_data) + .event_type = offsetof(odp_buffer_hdr_t, event_hdr.event_type), + .base_data = offsetof(odp_buffer_hdr_t, event_hdr.base_data) }; #include <odp/visibility_end.h> uint32_t odp_buffer_size(odp_buffer_t buf) { - odp_buffer_hdr_t *hdr = buf_hdl_to_hdr(buf); - pool_t *pool = hdr->pool_ptr; + odp_buffer_hdr_t *hdr = _odp_buf_hdr(buf); + pool_t *pool = hdr->event_hdr.pool_ptr; return pool->seg_len; } @@ -47,12 +47,13 @@ void odp_buffer_print(odp_buffer_t buf) return; } - hdr = buf_hdl_to_hdr(buf); + hdr = _odp_buf_hdr(buf); len += snprintf(&str[len], n - len, "Buffer\n------\n"); - len += snprintf(&str[len], n - len, " pool index %u\n", hdr->index.pool); - len += snprintf(&str[len], n - len, " buffer index %u\n", hdr->index.buffer); - len += snprintf(&str[len], n - len, " addr %p\n", (void *)hdr->base_data); + len += snprintf(&str[len], n - len, " pool index %u\n", hdr->event_hdr.index.pool); + len += snprintf(&str[len], n - len, " buffer index %u\n", hdr->event_hdr.index.buffer); + len += snprintf(&str[len], n - len, " addr %p\n", + (void *)hdr->event_hdr.base_data); len += snprintf(&str[len], n - len, " size %u\n", odp_buffer_size(buf)); str[len] = 0; diff --git a/platform/linux-generic/odp_cpumask_task.c b/platform/linux-generic/odp_cpumask_task.c index 70b85f1cb..ba77522c3 100644 --- a/platform/linux-generic/odp_cpumask_task.c +++ b/platform/linux-generic/odp_cpumask_task.c @@ -1,4 +1,5 @@ /* Copyright (c) 2015-2018, Linaro Limited + * Copyright (c) 2021, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -47,20 +48,11 @@ int odp_cpumask_default_control(odp_cpumask_t *mask, int num) odp_cpumask_t overlap; int cpu, i; - /* - * If no user supplied number then default to one control CPU. - */ - if (0 == num) { - num = 1; - } else { - /* - * If user supplied number is too large, then attempt - * to use all installed control CPUs - */ - cpu = odp_cpumask_count(&odp_global_ro.control_cpus); - if (cpu < num) - num = cpu; - } + /* If no user supplied number or it's too large, attempt to use all + * control CPUs. */ + cpu = odp_cpumask_count(&odp_global_ro.control_cpus); + if (num == 0 || cpu < num) + num = cpu; /* build the mask, allocating upwards from lowest numbered CPU */ odp_cpumask_zero(mask); diff --git a/platform/linux-generic/odp_dma.c b/platform/linux-generic/odp_dma.c new file mode 100644 index 000000000..412c33bd5 --- /dev/null +++ b/platform/linux-generic/odp_dma.c @@ -0,0 +1,926 @@ +/* Copyright (c) 2021, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <odp/api/dma.h> +#include <odp/api/shared_memory.h> +#include <odp/api/ticketlock.h> +#include <odp/api/align.h> +#include <odp/api/buffer.h> +#include <odp/api/stash.h> +#include <odp/api/pool.h> +#include <odp/api/plat/std_inlines.h> +#include <odp_global_data.h> +#include <odp_debug_internal.h> +#include <odp_init_internal.h> +#include <odp_event_internal.h> +#include <odp_pool_internal.h> + +#include <string.h> +#include <inttypes.h> + +#define MAX_SESSIONS CONFIG_MAX_DMA_SESSIONS +#define MAX_TRANSFERS 256 +#define MAX_SEGS 16 +#define MAX_SEG_LEN (128 * 1024) + +typedef struct segment_t { + void *addr; + uint32_t len; + +} segment_t; + +typedef struct transfer_t { + void *dst; + void *src; + uint32_t len; + +} transfer_t; + +typedef struct result_t { + void *user_ptr; + +} result_t; + +typedef struct ODP_ALIGNED_CACHE dma_session_t { + odp_ticketlock_t lock; + odp_dma_param_t dma_param; + uint8_t active; + char name[ODP_DMA_NAME_LEN]; + odp_stash_t stash; + result_t result[MAX_TRANSFERS]; + +} dma_session_t; + +typedef struct dma_global_t { + odp_shm_t shm; + + /* Buffer pool capability and default parameters */ + odp_pool_capability_t pool_capa; + odp_pool_param_t pool_param; + + dma_session_t session[MAX_SESSIONS]; + +} dma_global_t; + +static dma_global_t *_odp_dma_glb; + +static inline dma_session_t *dma_session_from_handle(odp_dma_t dma) +{ + return (dma_session_t *)(uintptr_t)dma; +} + +int odp_dma_capability(odp_dma_capability_t *capa) +{ + if (odp_global_ro.disable.dma) { + ODP_ERR("DMA is disabled\n"); + return -1; + } + + memset(capa, 0, sizeof(odp_dma_capability_t)); + + capa->max_sessions = MAX_SESSIONS; + capa->max_transfers = MAX_TRANSFERS; + capa->max_src_segs = MAX_SEGS; + capa->max_dst_segs = MAX_SEGS; + capa->max_segs = 2 * MAX_SEGS; + capa->max_seg_len = MAX_SEG_LEN; + + capa->compl_mode_mask = ODP_DMA_COMPL_SYNC | ODP_DMA_COMPL_NONE | + ODP_DMA_COMPL_EVENT | ODP_DMA_COMPL_POLL; + + capa->queue_type_sched = 1; + capa->queue_type_plain = 1; + + capa->pool.max_pools = _odp_dma_glb->pool_capa.buf.max_pools; + capa->pool.max_num = _odp_dma_glb->pool_capa.buf.max_num; + capa->pool.min_cache_size = _odp_dma_glb->pool_capa.buf.min_cache_size; + capa->pool.max_cache_size = _odp_dma_glb->pool_capa.buf.max_cache_size; + + return 0; +} + +void odp_dma_param_init(odp_dma_param_t *param) +{ + memset(param, 0, sizeof(odp_dma_param_t)); + + param->direction = ODP_DMA_MAIN_TO_MAIN; + param->type = ODP_DMA_TYPE_COPY; + param->mt_mode = ODP_DMA_MT_SAFE; + param->order = ODP_DMA_ORDER_NONE; +} + +static odp_stash_t create_stash(void) +{ + odp_stash_param_t stash_param; + odp_stash_t stash; + uint32_t id, tmp, i; + int32_t ret; + + odp_stash_param_init(&stash_param); + stash_param.num_obj = MAX_TRANSFERS; + stash_param.obj_size = sizeof(uint32_t); + stash_param.cache_size = 0; + + stash = odp_stash_create("_odp_dma_transfer_id", &stash_param); + + if (stash == ODP_STASH_INVALID) { + ODP_ERR("Stash create failed\n"); + return ODP_STASH_INVALID; + } + + /* Zero is invalid ID */ + for (id = 1; id < MAX_TRANSFERS + 1; id++) { + ret = odp_stash_put_u32(stash, &id, 1); + if (ret != 1) { + ODP_ERR("Stash put failed: %i, %u\n", ret, id); + break; + } + } + + if (ret != 1) { + for (i = 0; i < id; i++) { + if (odp_stash_get_u32(stash, &tmp, 1) != 1) { + ODP_ERR("Stash get failed: %u\n", i); + break; + } + } + + if (odp_stash_destroy(stash)) + ODP_ERR("Stash destroy failed\n"); + + return ODP_STASH_INVALID; + } + + return stash; +} + +static int destroy_stash(odp_stash_t stash) +{ + uint32_t tmp; + int32_t num; + int ret = 0; + + while (1) { + num = odp_stash_get_u32(stash, &tmp, 1); + + if (num == 1) + continue; + + if (num == 0) + break; + + ODP_ERR("Stash get failed: %i\n", num); + ret = -1; + break; + } + + if (odp_stash_destroy(stash)) { + ODP_ERR("Stash destroy failed\n"); + ret = -1; + } + + return ret; +} + +odp_dma_t odp_dma_create(const char *name, const odp_dma_param_t *param) +{ + odp_dma_capability_t dma_capa; + int i; + dma_session_t *session = NULL; + + if (odp_global_ro.disable.dma) { + ODP_ERR("DMA is disabled\n"); + return ODP_DMA_INVALID; + } + + if ((param->direction != ODP_DMA_MAIN_TO_MAIN) || + (param->type != ODP_DMA_TYPE_COPY)) { + ODP_ERR("Bad DMA parameter\n"); + return ODP_DMA_INVALID; + } + + if (param->compl_mode_mask == 0) { + ODP_ERR("Empty compl mode mask\n"); + return ODP_DMA_INVALID; + } + + if (odp_dma_capability(&dma_capa)) { + ODP_ERR("DMA capa failed\n"); + return ODP_DMA_INVALID; + } + + if (param->compl_mode_mask & ~dma_capa.compl_mode_mask) { + ODP_ERR("Compl mode not supported\n"); + return ODP_DMA_INVALID; + } + + for (i = 0; i < MAX_SESSIONS; i++) { + if (_odp_dma_glb->session[i].active) + continue; + + odp_ticketlock_lock(&_odp_dma_glb->session[i].lock); + + if (_odp_dma_glb->session[i].active) { + odp_ticketlock_unlock(&_odp_dma_glb->session[i].lock); + continue; + } + + session = &_odp_dma_glb->session[i]; + session->active = 1; + odp_ticketlock_unlock(&_odp_dma_glb->session[i].lock); + break; + } + + if (session == NULL) { + ODP_DBG("Out of DMA sessions\n"); + return ODP_DMA_INVALID; + } + + session->stash = ODP_STASH_INVALID; + + /* Create stash for transfer IDs */ + if (param->compl_mode_mask & ODP_DMA_COMPL_POLL) { + session->stash = create_stash(); + + if (session->stash == ODP_STASH_INVALID) + return ODP_DMA_INVALID; + } + + session->name[0] = 0; + + if (name) { + strncpy(session->name, name, ODP_DMA_NAME_LEN - 1); + session->name[ODP_DMA_NAME_LEN - 1] = 0; + } + + session->dma_param = *param; + + return (odp_dma_t)session; +} + +int odp_dma_destroy(odp_dma_t dma) +{ + dma_session_t *session = dma_session_from_handle(dma); + int ret = 0; + + if (dma == ODP_DMA_INVALID) { + ODP_ERR("Bad DMA handle\n"); + return -1; + } + + if (session->stash != ODP_STASH_INVALID) + if (destroy_stash(session->stash)) + ret = -1; + + odp_ticketlock_lock(&session->lock); + + if (session->active == 0) { + ODP_ERR("Session not created\n"); + odp_ticketlock_unlock(&session->lock); + return -1; + } + + session->active = 0; + odp_ticketlock_unlock(&session->lock); + + return ret; +} + +odp_dma_t odp_dma_lookup(const char *name) +{ + dma_session_t *session; + int i; + + for (i = 0; i < MAX_SESSIONS; i++) { + session = &_odp_dma_glb->session[i]; + + odp_ticketlock_lock(&session->lock); + + if (session->active == 0) { + odp_ticketlock_unlock(&session->lock); + continue; + } + + if (strcmp(session->name, name) == 0) { + /* found it */ + odp_ticketlock_unlock(&session->lock); + return (odp_dma_t)session; + } + odp_ticketlock_unlock(&session->lock); + } + + return ODP_DMA_INVALID; +} + +void odp_dma_transfer_param_init(odp_dma_transfer_param_t *trs_param) +{ + memset(trs_param, 0, sizeof(odp_dma_transfer_param_t)); + + trs_param->src_format = ODP_DMA_FORMAT_ADDR; + trs_param->dst_format = ODP_DMA_FORMAT_ADDR; + trs_param->num_src = 1; + trs_param->num_dst = 1; +} + +static uint32_t transfer_len(const odp_dma_transfer_param_t *trs_param) +{ + uint32_t i; + uint32_t src_len = 0; + uint32_t dst_len = 0; + + for (i = 0; i < trs_param->num_src; i++) + src_len += trs_param->src_seg[i].len; + + for (i = 0; i < trs_param->num_dst; i++) + dst_len += trs_param->dst_seg[i].len; + + if (src_len != dst_len) + return 0; + + return src_len; +} + +static inline void segment_raw(segment_t seg[], int num, const odp_dma_seg_t *dma_seg) +{ + int i; + + for (i = 0; i < num; i++) { + seg[i].addr = dma_seg[i].addr; + seg[i].len = dma_seg[i].len; + } +} + +static inline int segment_pkt(segment_t seg[], int num_seg, const odp_dma_seg_t *dma_seg) +{ + odp_packet_t pkt; + uint32_t offset; + void *addr; + uint32_t seg_len, tot_len, len; + int i; + int num = 0; + + for (i = 0; i < num_seg; i++) { + pkt = dma_seg[i].packet; + offset = dma_seg[i].offset; + tot_len = dma_seg[i].len; + + if (odp_unlikely(offset + tot_len > odp_packet_len(pkt))) { + ODP_ERR("Bad packet segment len/offset (%u/%u)\n", tot_len, offset); + return 0; + } + + while (tot_len) { + addr = odp_packet_offset(pkt, offset, &seg_len, NULL); + + if (odp_unlikely(addr == NULL)) { + ODP_ERR("Bad packet offset %u\n", offset); + return 0; + } + + seg[num].addr = addr; + len = tot_len; + if (tot_len > seg_len) + len = seg_len; + + seg[num].len = len; + + tot_len -= len; + offset += len; + num++; + + if (odp_unlikely(num >= MAX_SEGS)) { + ODP_ERR("Too many packet segments\n"); + return 0; + } + } + } + + return num; +} + +static int transfer_table(transfer_t *trs, const segment_t src_seg[], const segment_t dst_seg[], + int max_num, uint32_t tot_len) +{ + uint32_t len, src_len, dst_len; + uint8_t *src_ptr, *dst_ptr; + int i; + int src = 0; + int dst = 0; + + src_ptr = src_seg[0].addr; + dst_ptr = dst_seg[0].addr; + src_len = src_seg[0].len; + dst_len = dst_seg[0].len; + + len = src_len; + if (dst_len < src_len) + len = dst_len; + + for (i = 0; i < max_num; i++) { + trs[i].src = src_ptr; + trs[i].dst = dst_ptr; + trs[i].len = len; + tot_len -= len; + + if (tot_len == 0) + break; + + if (dst_len < src_len) { + dst++; + dst_ptr = dst_seg[dst].addr; + dst_len = dst_seg[dst].len; + src_ptr += len; + src_len -= len; + } else if (src_len < dst_len) { + src++; + src_ptr = src_seg[src].addr; + src_len = src_seg[src].len; + dst_ptr += len; + dst_len -= len; + } else { /* equal lengths */ + dst++; + src++; + dst_ptr = dst_seg[dst].addr; + dst_len = dst_seg[dst].len; + src_ptr = src_seg[src].addr; + src_len = src_seg[src].len; + } + + len = src_len; + if (dst_len < src_len) + len = dst_len; + } + + return i + 1; +} + +int odp_dma_transfer(odp_dma_t dma, const odp_dma_transfer_param_t *transfer, + odp_dma_result_t *result) +{ + int num, i; + uint32_t tot_len; + dma_session_t *session = dma_session_from_handle(dma); + int num_src, num_dst; + const int max_num = 2 * MAX_SEGS; + transfer_t trs[max_num]; + segment_t src[MAX_SEGS]; + segment_t dst[MAX_SEGS]; + + if (odp_unlikely(dma == ODP_DMA_INVALID)) { + ODP_ERR("Bad DMA handle\n"); + return -1; + } + + if (odp_unlikely(session->active == 0)) { + ODP_ERR("Session not created\n"); + return -1; + } + + if (odp_unlikely(transfer->num_src == 0 || transfer->num_src > MAX_SEGS)) { + ODP_ERR("Bad number of src segments\n"); + return -1; + } + + if (odp_unlikely(transfer->num_dst == 0 || transfer->num_dst > MAX_SEGS)) { + ODP_ERR("Bad number of dst segments\n"); + return -1; + } + + tot_len = transfer_len(transfer); + + if (odp_unlikely(tot_len == 0)) { + ODP_ERR("Bad transfer length\n"); + return -1; + } + + if (transfer->src_format == ODP_DMA_FORMAT_ADDR) { + num_src = transfer->num_src; + segment_raw(src, num_src, transfer->src_seg); + } else { + num_src = segment_pkt(src, transfer->num_src, transfer->src_seg); + + if (odp_unlikely(num_src == 0)) + return -1; + } + + if (transfer->dst_format == ODP_DMA_FORMAT_ADDR) { + num_dst = transfer->num_dst; + segment_raw(dst, num_dst, transfer->dst_seg); + } else { + num_dst = segment_pkt(dst, transfer->num_dst, transfer->dst_seg); + + if (odp_unlikely(num_dst == 0)) + return -1; + } + + num = transfer_table(trs, src, dst, max_num, tot_len); + + if (odp_unlikely(num > max_num)) { + ODP_ERR("Segment table error\n"); + return -1; + } + + for (i = 0; i < num; i++) + memcpy(trs[i].dst, trs[i].src, trs[i].len); + + if (result) { + memset(result, 0, sizeof(odp_dma_result_t)); + result->success = 1; + } + + return 1; +} + +int odp_dma_transfer_multi(odp_dma_t dma, const odp_dma_transfer_param_t *trs_param[], + odp_dma_result_t *result[], int num) +{ + int i; + odp_dma_result_t *res = NULL; + int ret = 0; + + if (odp_unlikely(num < 1)) { + ODP_ERR("Bad number of transfers\n"); + return -1; + } + + for (i = 0; i < num; i++) { + if (result) + res = result[i]; + + ret = odp_dma_transfer(dma, trs_param[i], res); + + if (odp_unlikely(ret != 1)) + break; + } + + if (odp_unlikely(i == 0)) + return ret; + + return i; +} + +void odp_dma_compl_param_init(odp_dma_compl_param_t *compl_param) +{ + memset(compl_param, 0, sizeof(odp_dma_compl_param_t)); + compl_param->queue = ODP_QUEUE_INVALID; + compl_param->event = ODP_EVENT_INVALID; + compl_param->transfer_id = ODP_DMA_TRANSFER_ID_INVALID; +} + +odp_dma_transfer_id_t odp_dma_transfer_id_alloc(odp_dma_t dma) +{ + int32_t num; + uint32_t id; + dma_session_t *session = dma_session_from_handle(dma); + + num = odp_stash_get_u32(session->stash, &id, 1); + + if (odp_unlikely(num != 1)) + return ODP_DMA_TRANSFER_ID_INVALID; + + return id; +} + +void odp_dma_transfer_id_free(odp_dma_t dma, odp_dma_transfer_id_t transfer_id) +{ + int32_t num; + dma_session_t *session = dma_session_from_handle(dma); + uint32_t id = transfer_id; + + num = odp_stash_put_u32(session->stash, &id, 1); + + if (odp_unlikely(num != 1)) + ODP_ERR("Stash put failed\n"); +} + +static inline uint32_t index_from_transfer_id(odp_dma_transfer_id_t transfer_id) +{ + return transfer_id - 1; +} + +int odp_dma_transfer_start(odp_dma_t dma, const odp_dma_transfer_param_t *transfer, + const odp_dma_compl_param_t *compl) +{ + int ret; + dma_session_t *session = dma_session_from_handle(dma); + + if (odp_unlikely(dma == ODP_DMA_INVALID)) { + ODP_ERR("Bad DMA handle\n"); + return -1; + } + + /* Check completion mode */ + switch (compl->compl_mode) { + case ODP_DMA_COMPL_NONE: + break; + case ODP_DMA_COMPL_POLL: + if (compl->transfer_id == ODP_DMA_TRANSFER_ID_INVALID || + compl->transfer_id > MAX_TRANSFERS) { + ODP_ERR("Bad transfer ID: %u\n", compl->transfer_id); + return -1; + } + break; + case ODP_DMA_COMPL_EVENT: + if (compl->event == ODP_EVENT_INVALID || + compl->queue == ODP_QUEUE_INVALID) { + ODP_ERR("Bad event or queue\n"); + return -1; + } + break; + default: + ODP_ERR("Bad completion mode %u\n", compl->compl_mode); + return -1; + } + + ret = odp_dma_transfer(dma, transfer, NULL); + + if (odp_unlikely(ret < 1)) + return ret; + + if (compl->compl_mode == ODP_DMA_COMPL_POLL) { + uint32_t index = index_from_transfer_id(compl->transfer_id); + + session->result[index].user_ptr = compl->user_ptr; + + } else if (compl->compl_mode == ODP_DMA_COMPL_EVENT) { + odp_dma_result_t *result; + odp_buffer_t buf = (odp_buffer_t)(uintptr_t)compl->event; + + if (odp_unlikely(odp_event_type(compl->event) != ODP_EVENT_DMA_COMPL)) { + ODP_ERR("Bad completion event type\n"); + return -1; + } + + result = odp_buffer_addr(buf); + result->success = 1; + result->user_ptr = compl->user_ptr; + + if (odp_unlikely(odp_queue_enq(compl->queue, compl->event))) { + ODP_ERR("Completion event enqueue failed %" PRIu64 "\n", + odp_queue_to_u64(compl->queue)); + return -1; + } + } + + return 1; +} + +int odp_dma_transfer_start_multi(odp_dma_t dma, const odp_dma_transfer_param_t *trs_param[], + const odp_dma_compl_param_t *compl_param[], int num) +{ + int i; + int ret = 0; + + if (odp_unlikely(num < 1)) { + ODP_ERR("Bad number of transfers\n"); + return -1; + } + + for (i = 0; i < num; i++) { + ret = odp_dma_transfer_start(dma, trs_param[i], compl_param[i]); + + if (odp_unlikely(ret != 1)) + break; + } + + if (odp_unlikely(i == 0)) + return ret; + + return i; +} + +int odp_dma_transfer_done(odp_dma_t dma, odp_dma_transfer_id_t transfer_id, + odp_dma_result_t *result) +{ + dma_session_t *session = dma_session_from_handle(dma); + + if (odp_unlikely(dma == ODP_DMA_INVALID)) { + ODP_ERR("Bad DMA handle\n"); + return -1; + } + + if (odp_unlikely(transfer_id == ODP_DMA_TRANSFER_ID_INVALID || + transfer_id > MAX_TRANSFERS)) { + ODP_ERR("Bad transfer ID: %u\n", transfer_id); + return -1; + } + + if (result) { + uint32_t index = index_from_transfer_id(transfer_id); + + result->success = 1; + result->user_ptr = session->result[index].user_ptr; + } + + return 1; +} + +void odp_dma_pool_param_init(odp_dma_pool_param_t *pool_param) +{ + memset(pool_param, 0, sizeof(odp_dma_pool_param_t)); + + pool_param->cache_size = _odp_dma_glb->pool_param.buf.cache_size; +} + +odp_pool_t odp_dma_pool_create(const char *name, const odp_dma_pool_param_t *dma_pool_param) +{ + odp_pool_t pool; + odp_pool_param_t pool_param; + uint32_t num = dma_pool_param->num; + uint32_t cache_size = dma_pool_param->cache_size; + + if (num > _odp_dma_glb->pool_capa.buf.max_num) { + ODP_ERR("Too many DMA completion events: %u\n", num); + return ODP_POOL_INVALID; + } + + if (cache_size < _odp_dma_glb->pool_capa.buf.min_cache_size || + cache_size > _odp_dma_glb->pool_capa.buf.max_cache_size) { + ODP_ERR("Bad cache size: %u\n", cache_size); + return ODP_POOL_INVALID; + } + + odp_pool_param_init(&pool_param); + pool_param.type = ODP_POOL_BUFFER; + pool_param.buf.num = num; + pool_param.buf.cache_size = cache_size; + pool_param.buf.size = sizeof(odp_dma_result_t); + + pool = _odp_pool_create(name, &pool_param, ODP_POOL_DMA_COMPL); + + return pool; +} + +odp_dma_compl_t odp_dma_compl_alloc(odp_pool_t pool) +{ + odp_buffer_t buf; + odp_event_t ev; + odp_dma_result_t *result; + + buf = odp_buffer_alloc(pool); + + if (odp_unlikely(buf == ODP_BUFFER_INVALID)) + return ODP_DMA_COMPL_INVALID; + + result = odp_buffer_addr(buf); + memset(result, 0, sizeof(odp_dma_result_t)); + + ev = odp_buffer_to_event(buf); + _odp_event_type_set(ev, ODP_EVENT_DMA_COMPL); + + return (odp_dma_compl_t)(uintptr_t)buf; +} + +void odp_dma_compl_free(odp_dma_compl_t dma_compl) +{ + odp_event_t ev; + odp_buffer_t buf = (odp_buffer_t)(uintptr_t)dma_compl; + + if (odp_unlikely(dma_compl == ODP_DMA_COMPL_INVALID)) { + ODP_ERR("Bad DMA compl handle\n"); + return; + } + + ev = odp_buffer_to_event(buf); + _odp_event_type_set(ev, ODP_EVENT_BUFFER); + + odp_buffer_free(buf); +} + +odp_dma_compl_t odp_dma_compl_from_event(odp_event_t ev) +{ + return (odp_dma_compl_t)(uintptr_t)ev; +} + +odp_event_t odp_dma_compl_to_event(odp_dma_compl_t dma_compl) +{ + return (odp_event_t)(uintptr_t)dma_compl; +} + +int odp_dma_compl_result(odp_dma_compl_t dma_compl, odp_dma_result_t *result_out) +{ + odp_dma_result_t *result; + odp_buffer_t buf = (odp_buffer_t)(uintptr_t)dma_compl; + + if (odp_unlikely(dma_compl == ODP_DMA_COMPL_INVALID)) { + ODP_ERR("Bad DMA compl handle\n"); + return -1; + } + + result = odp_buffer_addr(buf); + + if (result_out) + *result_out = *result; + + return result->success ? 0 : -1; +} + +uint64_t odp_dma_to_u64(odp_dma_t dma) +{ + return (uint64_t)(uintptr_t)dma; +} + +uint64_t odp_dma_compl_to_u64(odp_dma_compl_t dma_compl) +{ + return (uint64_t)(uintptr_t)dma_compl; +} + +void odp_dma_print(odp_dma_t dma) +{ + dma_session_t *session = dma_session_from_handle(dma); + + if (dma == ODP_DMA_INVALID) { + ODP_ERR("Bad DMA handle\n"); + return; + } + + ODP_PRINT("\nDMA info\n"); + ODP_PRINT("--------\n"); + ODP_PRINT(" DMA handle 0x%" PRIx64 "\n", odp_dma_to_u64(dma)); + ODP_PRINT(" name %s\n", session->name); + ODP_PRINT("\n"); +} + +void odp_dma_compl_print(odp_dma_compl_t dma_compl) +{ + odp_dma_result_t result; + int ret; + + if (dma_compl == ODP_DMA_COMPL_INVALID) { + ODP_ERR("Bad DMA compl handle\n"); + return; + } + + ret = odp_dma_compl_result(dma_compl, &result); + + ODP_PRINT("\nDMA completion\n"); + ODP_PRINT("--------------\n"); + ODP_PRINT(" Compl event handle: 0x%" PRIx64 "\n", (uint64_t)(uintptr_t)dma_compl); + + if (ret == 0) { + ODP_PRINT(" Result: %s\n", result.success ? "success" : "fail"); + ODP_PRINT(" User pointer: 0x%" PRIx64 "\n", + (uint64_t)(uintptr_t)result.user_ptr); + } else { + ODP_PRINT(" No result metadata\n"); + } + + ODP_PRINT("\n"); +} + +int _odp_dma_init_global(void) +{ + odp_shm_t shm; + int i; + + if (odp_global_ro.disable.dma) { + ODP_PRINT("DMA is DISABLED\n"); + return 0; + } + + shm = odp_shm_reserve("_odp_dma_global", sizeof(dma_global_t), ODP_CACHE_LINE_SIZE, 0); + _odp_dma_glb = odp_shm_addr(shm); + + if (_odp_dma_glb == NULL) { + ODP_ERR("SHM reserve failed\n"); + return -1; + } + + memset(_odp_dma_glb, 0, sizeof(dma_global_t)); + _odp_dma_glb->shm = shm; + + odp_pool_param_init(&_odp_dma_glb->pool_param); + + if (odp_pool_capability(&_odp_dma_glb->pool_capa)) { + ODP_ERR("Pool capability failed\n"); + return -1; + } + + for (i = 0; i < MAX_SESSIONS; i++) + odp_ticketlock_init(&_odp_dma_glb->session[i].lock); + + return 0; +} + +int _odp_dma_term_global(void) +{ + odp_shm_t shm; + + if (odp_global_ro.disable.dma) + return 0; + + if (_odp_dma_glb == NULL) + return 0; + + shm = _odp_dma_glb->shm; + + if (odp_shm_free(shm)) { + ODP_ERR("SHM free failed\n"); + return -1; + } + + return 0; +} diff --git a/platform/linux-generic/odp_event.c b/platform/linux-generic/odp_event.c index 5398442d6..b78881cb4 100644 --- a/platform/linux-generic/odp_event.c +++ b/platform/linux-generic/odp_event.c @@ -1,5 +1,5 @@ /* Copyright (c) 2015-2018, Linaro Limited - * Copyright (c) 2020, Nokia + * Copyright (c) 2020-2021, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -15,6 +15,7 @@ #include <odp_ipsec_internal.h> #include <odp_debug_internal.h> #include <odp_packet_internal.h> +#include <odp_event_internal.h> #include <odp_event_vector_internal.h> /* Inlined API functions */ @@ -24,8 +25,7 @@ odp_event_subtype_t odp_event_subtype(odp_event_t event) { - if (_odp_buffer_event_type(odp_buffer_from_event(event)) != - ODP_EVENT_PACKET) + if (_odp_event_type(event) != ODP_EVENT_PACKET) return ODP_EVENT_NO_SUBTYPE; return odp_packet_subtype(odp_packet_from_event(event)); @@ -34,8 +34,7 @@ odp_event_subtype_t odp_event_subtype(odp_event_t event) odp_event_type_t odp_event_types(odp_event_t event, odp_event_subtype_t *subtype) { - odp_buffer_t buf = odp_buffer_from_event(event); - odp_event_type_t event_type = _odp_buffer_event_type(buf); + odp_event_type_t event_type = _odp_event_type(event); *subtype = event_type == ODP_EVENT_PACKET ? odp_packet_subtype(odp_packet_from_event(event)) : @@ -75,6 +74,9 @@ void odp_event_free(odp_event_t event) case ODP_EVENT_IPSEC_STATUS: _odp_ipsec_status_free(_odp_ipsec_status_from_event(event)); break; + case ODP_EVENT_DMA_COMPL: + odp_dma_compl_free(odp_dma_compl_from_event(event)); + break; default: ODP_ABORT("Invalid event type: %d\n", odp_event_type(event)); } @@ -100,13 +102,10 @@ uint64_t odp_event_to_u64(odp_event_t hdl) int odp_event_is_valid(odp_event_t event) { - odp_buffer_t buf; - if (event == ODP_EVENT_INVALID) return 0; - buf = odp_buffer_from_event(event); - if (_odp_buffer_is_valid(buf) == 0) + if (_odp_event_is_valid(event) == 0) return 0; switch (odp_event_type(event)) { @@ -121,6 +120,8 @@ int odp_event_is_valid(odp_event_t event) case ODP_EVENT_IPSEC_STATUS: /* Fall through */ case ODP_EVENT_PACKET_VECTOR: + /* Fall through */ + case ODP_EVENT_DMA_COMPL: break; default: return 0; diff --git a/platform/linux-generic/odp_init.c b/platform/linux-generic/odp_init.c index 18646dc08..0c49946b0 100644 --- a/platform/linux-generic/odp_init.c +++ b/platform/linux-generic/odp_init.c @@ -1,4 +1,5 @@ /* Copyright (c) 2013-2018, Linaro Limited + * Copyright (c) 2021, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -44,16 +45,21 @@ enum init_stage { IPSEC_EVENTS_INIT, IPSEC_SAD_INIT, IPSEC_INIT, + DMA_INIT, ALL_INIT /* All init stages completed */ }; odp_global_data_ro_t odp_global_ro; odp_global_data_rw_t *odp_global_rw; +/* odp_init_local() call status */ +static __thread uint8_t init_local_called; + static void disable_features(odp_global_data_ro_t *global_ro, const odp_init_t *init_param) { int disable_ipsec, disable_crypto; + int disable_dma; if (init_param == NULL) return; @@ -66,7 +72,13 @@ static void disable_features(odp_global_data_ro_t *global_ro, if (disable_ipsec && disable_crypto) global_ro->disable.crypto = 1; - global_ro->disable.stash = init_param->not_used.feat.stash; + disable_dma = init_param->not_used.feat.dma; + global_ro->disable.dma = disable_dma; + + /* DMA uses stash. Disable stash only when both are disabled. */ + if (disable_dma && init_param->not_used.feat.stash) + global_ro->disable.stash = 1; + global_ro->disable.traffic_mngr = init_param->not_used.feat.tm; global_ro->disable.compress = init_param->not_used.feat.compress; } @@ -119,6 +131,13 @@ static int term_global(enum init_stage stage) switch (stage) { case ALL_INIT: + case DMA_INIT: + if (_odp_dma_term_global()) { + ODP_ERR("ODP DMA term failed.\n"); + rc = -1; + } + /* Fall through */ + case IPSEC_INIT: if (_odp_ipsec_term_global()) { ODP_ERR("ODP IPsec term failed.\n"); @@ -458,6 +477,12 @@ int odp_init_global(odp_instance_t *instance, } stage = IPSEC_INIT; + if (_odp_dma_init_global()) { + ODP_ERR("ODP DMA init failed.\n"); + goto init_failed; + } + stage = DMA_INIT; + *instance = (odp_instance_t)odp_global_ro.main_pid; return 0; @@ -560,6 +585,13 @@ int odp_init_local(odp_instance_t instance, odp_thread_type_t thr_type) goto init_fail; } + /* Detect if odp_init_local() has been already called from this thread */ + if (getpid() == odp_global_ro.main_pid && init_local_called) { + ODP_ERR("%s() called multiple times by the same thread\n", __func__); + goto init_fail; + } + init_local_called = 1; + if (_odp_ishm_init_local()) { ODP_ERR("ODP ishm local init failed.\n"); goto init_fail; @@ -623,6 +655,13 @@ init_fail: int odp_term_local(void) { + /* Check that odp_init_local() has been called by this thread */ + if (!init_local_called) { + ODP_ERR("%s() called by a non-initialized thread\n", __func__); + return -1; + } + init_local_called = 0; + return term_local(ALL_INIT); } diff --git a/platform/linux-generic/odp_ipsec.c b/platform/linux-generic/odp_ipsec.c index 3bd524c3c..e28611849 100644 --- a/platform/linux-generic/odp_ipsec.c +++ b/platform/linux-generic/odp_ipsec.c @@ -186,12 +186,38 @@ int odp_ipsec_capability(odp_ipsec_capability_t *capa) return 0; } +static int cipher_requires_randomness(odp_cipher_alg_t cipher) +{ + int ret; + + switch (cipher) { + case ODP_CIPHER_ALG_NULL: + case ODP_CIPHER_ALG_AES_CTR: +#if ODP_DEPRECATED_API + case ODP_CIPHER_ALG_AES128_GCM: +#endif + case ODP_CIPHER_ALG_AES_GCM: + case ODP_CIPHER_ALG_AES_CCM: + case ODP_CIPHER_ALG_CHACHA20_POLY1305: + ret = 0; + break; + default: + ret = 1; + break; + } + return ret; +} + int odp_ipsec_cipher_capability(odp_cipher_alg_t cipher, odp_ipsec_cipher_capability_t capa[], int num) { uint32_t req_iv_len; int rc, i, out, max_capa; + if (odp_random_max_kind() < ODP_RANDOM_CRYPTO && + cipher_requires_randomness(cipher)) + return 0; + max_capa = odp_crypto_cipher_capability(cipher, NULL, 0); if (max_capa <= 0) return max_capa; @@ -1254,7 +1280,7 @@ static int ipsec_random_data(uint8_t *data, uint32_t len) uint32_t rnd_len; rnd_len = odp_random_data(buffer, IPSEC_RANDOM_BUF_SIZE, - odp_global_ro.ipsec_rand_kind); + ODP_RANDOM_CRYPTO); if (odp_unlikely(rnd_len != IPSEC_RANDOM_BUF_SIZE)) return -1; memcpy(data, &buffer[0], len); @@ -2435,10 +2461,6 @@ int _odp_ipsec_init_global(void) memset(&default_out_opt, 0, sizeof(default_out_opt)); - odp_global_ro.ipsec_rand_kind = ODP_RANDOM_CRYPTO; - if (odp_global_ro.ipsec_rand_kind > odp_random_max_kind()) - odp_global_ro.ipsec_rand_kind = odp_random_max_kind(); - return 0; } diff --git a/platform/linux-generic/odp_ipsec_events.c b/platform/linux-generic/odp_ipsec_events.c index a199ffdf3..f229a5cf2 100644 --- a/platform/linux-generic/odp_ipsec_events.c +++ b/platform/linux-generic/odp_ipsec_events.c @@ -93,7 +93,7 @@ static odp_event_t ipsec_status_to_event(ipsec_status_t status) static ipsec_status_hdr_t *ipsec_status_hdr_from_buf(odp_buffer_t buf) { - return (ipsec_status_hdr_t *)(void *)buf_hdl_to_hdr(buf); + return (ipsec_status_hdr_t *)(void *)_odp_buf_hdr(buf); } static ipsec_status_hdr_t *ipsec_status_hdr(ipsec_status_t status) @@ -110,7 +110,7 @@ static ipsec_status_t odp_ipsec_status_alloc(void) if (odp_unlikely(buf == ODP_BUFFER_INVALID)) return ODP_IPSEC_STATUS_INVALID; - _odp_buffer_event_type_set(buf, ODP_EVENT_IPSEC_STATUS); + _odp_event_type_set(odp_buffer_to_event(buf), ODP_EVENT_IPSEC_STATUS); return _odp_ipsec_status_from_event(odp_buffer_to_event(buf)); } diff --git a/platform/linux-generic/odp_ipsec_sad.c b/platform/linux-generic/odp_ipsec_sad.c index 756370516..50f2bb360 100644 --- a/platform/linux-generic/odp_ipsec_sad.c +++ b/platform/linux-generic/odp_ipsec_sad.c @@ -480,6 +480,24 @@ static int ipsec_antireplay_init(ipsec_sa_t *ipsec_sa, return 0; } +static void store_sa_info(ipsec_sa_t *ipsec_sa, const odp_ipsec_sa_param_t *p) +{ + ipsec_sa->sa_info.cipher_alg = p->crypto.cipher_alg; + ipsec_sa->sa_info.cipher_key_len = p->crypto.cipher_key.length; + ipsec_sa->sa_info.cipher_key_extra_len = p->crypto.cipher_key.length; + ipsec_sa->sa_info.auth_alg = p->crypto.auth_alg; + ipsec_sa->sa_info.auth_key_len = p->crypto.auth_key.length; + ipsec_sa->sa_info.auth_key_extra_len = p->crypto.auth_key_extra.length; + + ipsec_sa->sa_info.icv_len = p->crypto.icv_len; + ipsec_sa->sa_info.context_len = p->context_len; + + if (p->dir == ODP_IPSEC_DIR_INBOUND) + ipsec_sa->sa_info.in.antireplay_ws = p->inbound.antireplay_ws; + else + ipsec_sa->sa_info.out.mtu = p->outbound.mtu; +} + odp_ipsec_sa_t odp_ipsec_sa_create(const odp_ipsec_sa_param_t *param) { ipsec_sa_t *ipsec_sa; @@ -487,12 +505,18 @@ odp_ipsec_sa_t odp_ipsec_sa_create(const odp_ipsec_sa_param_t *param) odp_crypto_ses_create_err_t ses_create_rc; const odp_crypto_key_t *salt_param = NULL; + if (!odp_ipsec_cipher_capability(param->crypto.cipher_alg, NULL, 0) || + !odp_ipsec_auth_capability(param->crypto.auth_alg, NULL, 0)) + return ODP_IPSEC_SA_INVALID; + ipsec_sa = ipsec_sa_reserve(); if (NULL == ipsec_sa) { ODP_ERR("No more free SA\n"); return ODP_IPSEC_SA_INVALID; } + store_sa_info(ipsec_sa, param); + ipsec_sa->proto = param->proto; ipsec_sa->spi = param->spi; ipsec_sa->context = param->context; @@ -506,6 +530,7 @@ odp_ipsec_sa_t odp_ipsec_sa_create(const odp_ipsec_sa_param_t *param) ipsec_sa->esn = param->opt.esn; if (ODP_IPSEC_DIR_INBOUND == param->dir) { + ipsec_sa->inbound = 1; ipsec_sa->lookup_mode = param->inbound.lookup_mode; if (ODP_IPSEC_LOOKUP_DSTADDR_SPI == ipsec_sa->lookup_mode) { ipsec_sa->in.lookup_ver = @@ -551,20 +576,6 @@ odp_ipsec_sa_t odp_ipsec_sa_create(const odp_ipsec_sa_param_t *param) odp_atomic_init_u64(&ipsec_sa->stats.post_lifetime_err_pkts, 0); odp_atomic_init_u64(&ipsec_sa->stats.post_lifetime_err_bytes, 0); - /* Copy application provided parameter values. */ - ipsec_sa->param = *param; - - /* Set all the key related pointers and ip address pointers to null. */ - ipsec_sa->param.crypto.cipher_key.data = NULL; - ipsec_sa->param.crypto.cipher_key_extra.data = NULL; - ipsec_sa->param.crypto.auth_key.data = NULL; - ipsec_sa->param.crypto.auth_key_extra.data = NULL; - ipsec_sa->param.inbound.lookup_param.dst_addr = NULL; - ipsec_sa->param.outbound.tunnel.ipv4.src_addr = NULL; - ipsec_sa->param.outbound.tunnel.ipv4.dst_addr = NULL; - ipsec_sa->param.outbound.tunnel.ipv6.src_addr = NULL; - ipsec_sa->param.outbound.tunnel.ipv6.dst_addr = NULL; - if (ODP_IPSEC_MODE_TUNNEL == ipsec_sa->mode && ODP_IPSEC_DIR_OUTBOUND == param->dir) { if (ODP_IPSEC_TUNNEL_IPV4 == param->outbound.tunnel.type) { @@ -1161,22 +1172,35 @@ void _odp_ipsec_sa_stats_pkts(ipsec_sa_t *sa, odp_ipsec_stats_t *stats) static void ipsec_out_sa_info(ipsec_sa_t *ipsec_sa, odp_ipsec_sa_info_t *sa_info) { + odp_ipsec_tunnel_param_t *tun_param = &sa_info->param.outbound.tunnel; + + tun_param->type = ipsec_sa->tun_ipv4 ? ODP_IPSEC_TUNNEL_IPV4 : + ODP_IPSEC_TUNNEL_IPV6; + tun_param->ipv4.dscp = ipsec_sa->out.tun_ipv4.param.dscp; + tun_param->ipv4.df = ipsec_sa->out.tun_ipv4.param.df; + tun_param->ipv4.ttl = ipsec_sa->out.tun_ipv4.param.ttl; + tun_param->ipv6.flabel = ipsec_sa->out.tun_ipv6.param.flabel; + tun_param->ipv6.dscp = ipsec_sa->out.tun_ipv6.param.dscp; + tun_param->ipv6.hlimit = ipsec_sa->out.tun_ipv6.param.hlimit; + + sa_info->param.outbound.frag_mode = ipsec_sa->out.frag_mode; + sa_info->param.outbound.mtu = ipsec_sa->sa_info.out.mtu; + sa_info->outbound.seq_num = (uint64_t)odp_atomic_load_u64(&ipsec_sa->hot.out.seq) - 1; - if (ipsec_sa->param.mode == ODP_IPSEC_MODE_TUNNEL) { + if (ipsec_sa->mode == ODP_IPSEC_MODE_TUNNEL) { uint8_t *src, *dst; - if (ipsec_sa->param.outbound.tunnel.type == - ODP_IPSEC_TUNNEL_IPV4) { + if (ipsec_sa->tun_ipv4) { src = sa_info->outbound.tunnel.ipv4.src_addr; dst = sa_info->outbound.tunnel.ipv4.dst_addr; memcpy(src, &ipsec_sa->out.tun_ipv4.src_ip, ODP_IPV4_ADDR_SIZE); memcpy(dst, &ipsec_sa->out.tun_ipv4.dst_ip, ODP_IPV4_ADDR_SIZE); - sa_info->param.outbound.tunnel.ipv4.src_addr = src; - sa_info->param.outbound.tunnel.ipv4.dst_addr = dst; + tun_param->ipv4.src_addr = src; + tun_param->ipv4.dst_addr = dst; } else { src = sa_info->outbound.tunnel.ipv6.src_addr; dst = sa_info->outbound.tunnel.ipv6.dst_addr; @@ -1184,8 +1208,8 @@ static void ipsec_out_sa_info(ipsec_sa_t *ipsec_sa, odp_ipsec_sa_info_t *sa_info ODP_IPV6_ADDR_SIZE); memcpy(dst, &ipsec_sa->out.tun_ipv6.dst_ip, ODP_IPV6_ADDR_SIZE); - sa_info->param.outbound.tunnel.ipv6.src_addr = src; - sa_info->param.outbound.tunnel.ipv6.dst_addr = dst; + tun_param->ipv6.src_addr = src; + tun_param->ipv6.dst_addr = dst; } } } @@ -1194,9 +1218,16 @@ static void ipsec_in_sa_info(ipsec_sa_t *ipsec_sa, odp_ipsec_sa_info_t *sa_info) { uint8_t *dst = sa_info->inbound.lookup_param.dst_addr; + sa_info->param.inbound.lookup_mode = ipsec_sa->lookup_mode; + sa_info->param.inbound.lookup_param.ip_version = ipsec_sa->in.lookup_ver; + sa_info->param.inbound.lookup_param.dst_addr = dst; + sa_info->param.inbound.antireplay_ws = ipsec_sa->sa_info.in.antireplay_ws; + sa_info->param.inbound.pipeline = ODP_IPSEC_PIPELINE_NONE; + sa_info->param.inbound.dest_cos = ODP_COS_INVALID; + sa_info->param.inbound.reassembly_en = false; + if (ipsec_sa->lookup_mode == ODP_IPSEC_LOOKUP_DSTADDR_SPI) { - if (ipsec_sa->param.inbound.lookup_param.ip_version == - ODP_IPSEC_IPV4) + if (ipsec_sa->in.lookup_ver == ODP_IPSEC_IPV4) memcpy(dst, &ipsec_sa->in.lookup_dst_ipv4, ODP_IPV4_ADDR_SIZE); else @@ -1216,6 +1247,7 @@ static void ipsec_in_sa_info(ipsec_sa_t *ipsec_sa, odp_ipsec_sa_info_t *sa_info) int odp_ipsec_sa_info(odp_ipsec_sa_t sa, odp_ipsec_sa_info_t *sa_info) { ipsec_sa_t *ipsec_sa; + odp_ipsec_sa_param_t *param; ipsec_sa = _odp_ipsec_sa_entry_from_hdl(sa); @@ -1223,13 +1255,46 @@ int odp_ipsec_sa_info(odp_ipsec_sa_t sa, odp_ipsec_sa_info_t *sa_info) ODP_ASSERT(sa_info != NULL); memset(sa_info, 0, sizeof(*sa_info)); - - sa_info->param = ipsec_sa->param; - - if (ipsec_sa->param.dir == ODP_IPSEC_DIR_OUTBOUND) - ipsec_out_sa_info(ipsec_sa, sa_info); - else + param = &sa_info->param; + + param->dir = ipsec_sa->inbound ? ODP_IPSEC_DIR_INBOUND : + ODP_IPSEC_DIR_OUTBOUND; + param->proto = ipsec_sa->proto; + param->mode = ipsec_sa->mode; + + param->crypto.cipher_alg = ipsec_sa->sa_info.cipher_alg; + param->crypto.cipher_key.data = NULL; + param->crypto.cipher_key.length = ipsec_sa->sa_info.cipher_key_len; + param->crypto.cipher_key_extra.data = NULL; + param->crypto.cipher_key_extra.length = ipsec_sa->sa_info.cipher_key_extra_len; + param->crypto.auth_alg = ipsec_sa->sa_info.auth_alg; + param->crypto.auth_key.data = NULL; + param->crypto.auth_key.length = ipsec_sa->sa_info.auth_key_len; + param->crypto.auth_key_extra.data = NULL; + param->crypto.auth_key_extra.length = ipsec_sa->sa_info.auth_key_extra_len; + param->crypto.icv_len = ipsec_sa->sa_info.icv_len; + + param->opt.esn = ipsec_sa->esn; + param->opt.udp_encap = ipsec_sa->udp_encap; + param->opt.copy_dscp = ipsec_sa->copy_dscp; + param->opt.copy_flabel = ipsec_sa->copy_flabel; + param->opt.copy_df = ipsec_sa->copy_df; + param->opt.dec_ttl = ipsec_sa->dec_ttl; + + param->lifetime.soft_limit.bytes = ipsec_sa->soft_limit_bytes; + param->lifetime.soft_limit.packets = ipsec_sa->soft_limit_packets; + param->lifetime.hard_limit.bytes = ipsec_sa->hard_limit_bytes; + param->lifetime.hard_limit.packets = ipsec_sa->hard_limit_packets; + + param->spi = ipsec_sa->spi; + param->dest_queue = ipsec_sa->queue; + param->context = ipsec_sa->context; + param->context_len = ipsec_sa->sa_info.context_len; + + if (ipsec_sa->inbound) ipsec_in_sa_info(ipsec_sa, sa_info); + else + ipsec_out_sa_info(ipsec_sa, sa_info); return 0; } diff --git a/platform/linux-generic/odp_ishm.c b/platform/linux-generic/odp_ishm.c index a10b9d5df..dea4d56f0 100644 --- a/platform/linux-generic/odp_ishm.c +++ b/platform/linux-generic/odp_ishm.c @@ -1071,7 +1071,8 @@ int _odp_ishm_reserve(const char *name, uint64_t size, int fd, /* Get system page sizes: page_hp_size is 0 if no huge page available*/ page_sz = odp_sys_page_size(); - page_hp_size = odp_sys_huge_page_size(); + /* Use normal pages if ODP_SHM_NO_HP was used */ + page_hp_size = (user_flags & ODP_SHM_NO_HP) ? 0 : odp_sys_huge_page_size(); /* grab a new entry: */ for (new_index = 0; new_index < ISHM_MAX_NB_BLOCKS; new_index++) { diff --git a/platform/linux-generic/odp_packet.c b/platform/linux-generic/odp_packet.c index ed5d81952..a15508ca2 100644 --- a/platform/linux-generic/odp_packet.c +++ b/platform/linux-generic/odp_packet.c @@ -1,5 +1,5 @@ /* Copyright (c) 2013-2018, Linaro Limited - * Copyright (c) 2019, Nokia + * Copyright (c) 2019-2021, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -21,6 +21,8 @@ #include <odp/api/plat/pktio_inlines.h> #include <odp/api/proto_stats.h> +#include <odp_event_internal.h> + /* Inlined API functions */ #include <odp/api/plat/event_inlines.h> @@ -45,11 +47,11 @@ const _odp_packet_inline_offset_t _odp_packet_inline ODP_ALIGNED_CACHE = { .frame_len = offsetof(odp_packet_hdr_t, frame_len), .headroom = offsetof(odp_packet_hdr_t, headroom), .tailroom = offsetof(odp_packet_hdr_t, tailroom), - .pool = offsetof(odp_packet_hdr_t, buf_hdr.pool_ptr), + .pool = offsetof(odp_packet_hdr_t, event_hdr.pool_ptr), .input = offsetof(odp_packet_hdr_t, input), .seg_count = offsetof(odp_packet_hdr_t, seg_count), - .user_ptr = offsetof(odp_packet_hdr_t, buf_hdr.user_ptr), - .user_area = offsetof(odp_packet_hdr_t, buf_hdr.uarea_addr), + .user_ptr = offsetof(odp_packet_hdr_t, event_hdr.user_ptr), + .user_area = offsetof(odp_packet_hdr_t, event_hdr.uarea_addr), .l2_offset = offsetof(odp_packet_hdr_t, p.l2_offset), .l3_offset = offsetof(odp_packet_hdr_t, p.l3_offset), .l4_offset = offsetof(odp_packet_hdr_t, p.l4_offset), @@ -81,11 +83,6 @@ ODP_STATIC_ASSERT(ODP_TIMEOUT_INVALID == 0, "Timeout invalid not 0"); #pragma GCC diagnostic pop #endif -static inline odp_buffer_t packet_to_buffer(odp_packet_t pkt) -{ - return (odp_buffer_t)pkt; -} - static inline odp_packet_hdr_t *packet_seg_to_hdr(odp_packet_seg_t seg) { return (odp_packet_hdr_t *)(uintptr_t)seg; @@ -144,7 +141,7 @@ static inline void *packet_tail(odp_packet_hdr_t *pkt_hdr) static inline uint32_t seg_headroom(odp_packet_hdr_t *pkt_seg) { - odp_buffer_hdr_t *hdr = &pkt_seg->buf_hdr; + _odp_event_hdr_t *hdr = &pkt_seg->event_hdr; pool_t *pool = hdr->pool_ptr; uint8_t *base = hdr->base_data; uint8_t *head = pkt_seg->seg_data; @@ -154,7 +151,7 @@ static inline uint32_t seg_headroom(odp_packet_hdr_t *pkt_seg) static inline uint32_t seg_tailroom(odp_packet_hdr_t *pkt_seg) { - odp_buffer_hdr_t *hdr = &pkt_seg->buf_hdr; + _odp_event_hdr_t *hdr = &pkt_seg->event_hdr; uint8_t *tail = pkt_seg->seg_data + pkt_seg->seg_len; return hdr->buf_end - tail; @@ -196,9 +193,9 @@ static inline void packet_seg_copy_md(odp_packet_hdr_t *dst, if (src->p.flags.payload_off) dst->payload_offset = src->payload_offset; - /* buffer header side packet metadata */ - dst->buf_hdr.user_ptr = src->buf_hdr.user_ptr; - dst->buf_hdr.uarea_addr = src->buf_hdr.uarea_addr; + /* event header side packet metadata */ + dst->event_hdr.user_ptr = src->event_hdr.user_ptr; + dst->event_hdr.uarea_addr = src->event_hdr.uarea_addr; /* segmentation data is not copied: * seg_next @@ -314,13 +311,13 @@ static inline void link_segments(odp_packet_hdr_t *pkt_hdr[], int num) int cur = 0; odp_packet_hdr_t *hdr; odp_packet_hdr_t *head = pkt_hdr[0]; - uint32_t seg_len = ((pool_t *)(head->buf_hdr.pool_ptr))->seg_len; + uint32_t seg_len = ((pool_t *)(head->event_hdr.pool_ptr))->seg_len; while (1) { - odp_buffer_hdr_t *buf_hdr = &pkt_hdr[cur]->buf_hdr; + _odp_event_hdr_t *event_hdr = &pkt_hdr[cur]->event_hdr; hdr = pkt_hdr[cur]; - hdr->seg_data = buf_hdr->base_data; + hdr->seg_data = event_hdr->base_data; hdr->seg_len = seg_len; /* init_segments() handles first seg ref_cnt init */ @@ -328,7 +325,7 @@ static inline void link_segments(odp_packet_hdr_t *pkt_hdr[], int num) uint32_t prev_ref; odp_atomic_u32_t *ref_cnt; - ref_cnt = &pkt_hdr[cur]->buf_hdr.ref_cnt; + ref_cnt = &pkt_hdr[cur]->event_hdr.ref_cnt; prev_ref = odp_atomic_fetch_inc_u32(ref_cnt); ODP_ASSERT(prev_ref == 0); @@ -353,10 +350,10 @@ static inline void init_segments(odp_packet_hdr_t *pkt_hdr[], int num) /* First segment is the packet descriptor */ hdr = pkt_hdr[0]; - seg_len = ((pool_t *)(hdr->buf_hdr.pool_ptr))->seg_len; + seg_len = ((pool_t *)(hdr->event_hdr.pool_ptr))->seg_len; /* Defaults for single segment packet */ - hdr->seg_data = hdr->buf_hdr.base_data; + hdr->seg_data = hdr->event_hdr.base_data; hdr->seg_len = seg_len; hdr->seg_next = NULL; @@ -364,7 +361,7 @@ static inline void init_segments(odp_packet_hdr_t *pkt_hdr[], int num) if (ODP_DEBUG == 1) { uint32_t prev_ref = - odp_atomic_fetch_inc_u32(&hdr->buf_hdr.ref_cnt); + odp_atomic_fetch_inc_u32(&hdr->event_hdr.ref_cnt); ODP_ASSERT(prev_ref == 0); } @@ -377,10 +374,10 @@ static inline void init_segments(odp_packet_hdr_t *pkt_hdr[], int num) static inline void reset_segments(odp_packet_hdr_t *pkt_hdr) { void *base; - uint32_t seg_len = ((pool_t *)(pkt_hdr->buf_hdr.pool_ptr))->seg_len; + uint32_t seg_len = ((pool_t *)(pkt_hdr->event_hdr.pool_ptr))->seg_len; while (pkt_hdr != NULL) { - base = pkt_hdr->buf_hdr.base_data; + base = pkt_hdr->event_hdr.base_data; pkt_hdr->seg_len = seg_len; pkt_hdr->seg_data = base; @@ -417,11 +414,11 @@ static inline odp_packet_hdr_t *alloc_segments(pool_t *pool, int num) odp_packet_hdr_t *pkt_hdr[num]; int ret; - ret = _odp_buffer_alloc_multi(pool, (odp_buffer_hdr_t **)pkt_hdr, num); + ret = _odp_event_alloc_multi(pool, (_odp_event_hdr_t **)pkt_hdr, num); if (odp_unlikely(ret != num)) { if (ret > 0) - _odp_buffer_free_multi((odp_buffer_hdr_t **)pkt_hdr, ret); + _odp_event_free_multi((_odp_event_hdr_t **)pkt_hdr, ret); return NULL; } @@ -477,25 +474,25 @@ static inline odp_packet_hdr_t *add_segments(odp_packet_hdr_t *pkt_hdr, return pkt_hdr; } -static inline void buffer_ref_inc(odp_buffer_hdr_t *buf_hdr) +static inline void segment_ref_inc(_odp_event_hdr_t *event_hdr) { - uint32_t ref_cnt = odp_atomic_load_u32(&buf_hdr->ref_cnt); + uint32_t ref_cnt = odp_atomic_load_u32(&event_hdr->ref_cnt); /* First count increment after alloc */ if (odp_likely(ref_cnt == 0)) - odp_atomic_store_u32(&buf_hdr->ref_cnt, 2); + odp_atomic_store_u32(&event_hdr->ref_cnt, 2); else - odp_atomic_inc_u32(&buf_hdr->ref_cnt); + odp_atomic_inc_u32(&event_hdr->ref_cnt); } -static inline uint32_t buffer_ref_dec(odp_buffer_hdr_t *buf_hdr) +static inline uint32_t segment_ref_dec(_odp_event_hdr_t *event_hdr) { - return odp_atomic_fetch_dec_u32(&buf_hdr->ref_cnt); + return odp_atomic_fetch_dec_u32(&event_hdr->ref_cnt); } -static inline uint32_t buffer_ref(odp_buffer_hdr_t *buf_hdr) +static inline uint32_t segment_ref(_odp_event_hdr_t *event_hdr) { - return odp_atomic_load_u32(&buf_hdr->ref_cnt); + return odp_atomic_load_u32(&event_hdr->ref_cnt); } static inline int is_multi_ref(uint32_t ref_cnt) @@ -503,17 +500,7 @@ static inline int is_multi_ref(uint32_t ref_cnt) return (ref_cnt > 1); } -static inline void packet_ref_inc(odp_packet_hdr_t *pkt_hdr) -{ - odp_packet_hdr_t *hdr = pkt_hdr; - - while (hdr != NULL) { - buffer_ref_inc(&hdr->buf_hdr); - hdr = hdr->seg_next; - } -} - -static inline void packet_free_multi(odp_buffer_hdr_t *hdr[], int num) +static inline void packet_free_multi(_odp_event_hdr_t *hdr[], int num) { int i; uint32_t ref_cnt; @@ -521,10 +508,10 @@ static inline void packet_free_multi(odp_buffer_hdr_t *hdr[], int num) for (i = 0; i < num; i++) { /* Zero when reference API has not been used */ - ref_cnt = buffer_ref(hdr[i]); + ref_cnt = segment_ref(hdr[i]); if (odp_unlikely(ref_cnt)) { - ref_cnt = buffer_ref_dec(hdr[i]); + ref_cnt = segment_ref_dec(hdr[i]); if (is_multi_ref(ref_cnt)) { num_ref++; @@ -540,21 +527,21 @@ static inline void packet_free_multi(odp_buffer_hdr_t *hdr[], int num) num -= num_ref; if (odp_likely(num)) - _odp_buffer_free_multi(hdr, num); + _odp_event_free_multi(hdr, num); } static inline void free_all_segments(odp_packet_hdr_t *pkt_hdr, int num) { int i; - odp_buffer_hdr_t *buf_hdr[num]; + _odp_event_hdr_t *event_hdr[num]; odp_packet_hdr_t *seg_hdr = pkt_hdr; for (i = 0; i < num; i++) { - buf_hdr[i] = &seg_hdr->buf_hdr; + event_hdr[i] = &seg_hdr->event_hdr; seg_hdr = seg_hdr->seg_next; } - packet_free_multi(buf_hdr, num); + packet_free_multi(event_hdr, num); } static inline odp_packet_hdr_t *free_segments(odp_packet_hdr_t *pkt_hdr, @@ -566,14 +553,14 @@ static inline odp_packet_hdr_t *free_segments(odp_packet_hdr_t *pkt_hdr, int num_remain = pkt_hdr->seg_count - num; odp_packet_hdr_t *hdr = pkt_hdr; odp_packet_hdr_t *last_hdr = packet_last_seg(pkt_hdr); - odp_buffer_hdr_t *buf_hdr[num]; + _odp_event_hdr_t *event_hdr[num]; if (head) { odp_packet_hdr_t *new_hdr; for (i = 0; i < num; i++) { seg_hdr = packet_seg_step(&hdr); - buf_hdr[i] = &seg_hdr->buf_hdr; + event_hdr[i] = &seg_hdr->event_hdr; } /* The first remaining header is the new packet descriptor. @@ -597,7 +584,7 @@ static inline odp_packet_hdr_t *free_segments(odp_packet_hdr_t *pkt_hdr, pkt_hdr = new_hdr; - packet_free_multi(buf_hdr, num); + packet_free_multi(event_hdr, num); } else { /* Free last 'num' bufs. * First, find the last remaining header. */ @@ -608,10 +595,10 @@ static inline odp_packet_hdr_t *free_segments(odp_packet_hdr_t *pkt_hdr, for (i = 0; i < num; i++) { seg_hdr = packet_seg_step(&hdr); - buf_hdr[i] = &seg_hdr->buf_hdr; + event_hdr[i] = &seg_hdr->event_hdr; } - packet_free_multi(buf_hdr, num); + packet_free_multi(event_hdr, num); /* Head segment remains, no need to copy or update majority * of the metadata. */ @@ -637,8 +624,8 @@ static inline int packet_alloc(pool_t *pool, uint32_t len, int max_pkt, odp_packet_hdr_t *hdr_next; odp_packet_hdr_t *hdr; - num_buf = _odp_buffer_alloc_multi(pool, (odp_buffer_hdr_t **)pkt_hdr, - max_buf); + num_buf = _odp_event_alloc_multi(pool, (_odp_event_hdr_t **)pkt_hdr, + max_buf); /* Failed to allocate all segments */ if (odp_unlikely(num_buf != max_buf)) { @@ -648,10 +635,10 @@ static inline int packet_alloc(pool_t *pool, uint32_t len, int max_pkt, num_free = num_buf - (num * num_seg); if (num_free > 0) { - odp_buffer_hdr_t **p; + _odp_event_hdr_t **p; - p = (odp_buffer_hdr_t **)&pkt_hdr[num_buf - num_free]; - _odp_buffer_free_multi(p, num_free); + p = (_odp_event_hdr_t **)&pkt_hdr[num_buf - num_free]; + _odp_event_free_multi(p, num_free); } if (num == 0) @@ -744,12 +731,12 @@ void odp_packet_free(odp_packet_t pkt) odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); int num_seg = pkt_hdr->seg_count; - ODP_ASSERT(buffer_ref(&pkt_hdr->buf_hdr) > 0); + ODP_ASSERT(segment_ref(&pkt_hdr->event_hdr) > 0); if (odp_likely(num_seg == 1)) { - odp_buffer_hdr_t *buf_hdr = &pkt_hdr->buf_hdr; + _odp_event_hdr_t *event_hdr = &pkt_hdr->event_hdr; - packet_free_multi(&buf_hdr, 1); + packet_free_multi(&event_hdr, 1); } else { free_all_segments(pkt_hdr, num_seg); } @@ -757,7 +744,7 @@ void odp_packet_free(odp_packet_t pkt) void odp_packet_free_multi(const odp_packet_t pkt[], int num) { - odp_buffer_hdr_t *buf_hdr[num]; + _odp_event_hdr_t *event_hdr[num]; int i; int num_freed = 0; @@ -765,7 +752,7 @@ void odp_packet_free_multi(const odp_packet_t pkt[], int num) odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt[i]); int num_seg = pkt_hdr->seg_count; - ODP_ASSERT(buffer_ref(&pkt_hdr->buf_hdr) > 0); + ODP_ASSERT(segment_ref(&pkt_hdr->event_hdr) > 0); if (odp_unlikely(num_seg > 1)) { free_all_segments(pkt_hdr, num_seg); @@ -773,11 +760,11 @@ void odp_packet_free_multi(const odp_packet_t pkt[], int num) continue; } - buf_hdr[i - num_freed] = &pkt_hdr->buf_hdr; + event_hdr[i - num_freed] = &pkt_hdr->event_hdr; } if (odp_likely(num - num_freed)) - packet_free_multi(buf_hdr, num - num_freed); + packet_free_multi(event_hdr, num - num_freed); } void odp_packet_free_sp(const odp_packet_t pkt[], int num) @@ -788,7 +775,7 @@ void odp_packet_free_sp(const odp_packet_t pkt[], int num) int odp_packet_reset(odp_packet_t pkt, uint32_t len) { odp_packet_hdr_t *const pkt_hdr = packet_hdr(pkt); - pool_t *pool = pkt_hdr->buf_hdr.pool_ptr; + pool_t *pool = pkt_hdr->event_hdr.pool_ptr; int num = pkt_hdr->seg_count; int num_req; @@ -837,7 +824,7 @@ int odp_event_filter_packet(const odp_event_t event[], uint32_t odp_packet_buf_len(odp_packet_t pkt) { odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - pool_t *pool = pkt_hdr->buf_hdr.pool_ptr; + pool_t *pool = pkt_hdr->event_hdr.pool_ptr; return pool->max_seg_len * pkt_hdr->seg_count; } @@ -869,7 +856,7 @@ int odp_packet_extend_head(odp_packet_t *pkt, uint32_t len, int ret = 0; if (len > headroom) { - pool_t *pool = pkt_hdr->buf_hdr.pool_ptr; + pool_t *pool = pkt_hdr->event_hdr.pool_ptr; int num; void *ptr; @@ -978,7 +965,7 @@ int odp_packet_extend_tail(odp_packet_t *pkt, uint32_t len, ODP_ASSERT(odp_packet_has_ref(*pkt) == 0); if (len > tailroom) { - pool_t *pool = pkt_hdr->buf_hdr.pool_ptr; + pool_t *pool = pkt_hdr->event_hdr.pool_ptr; int num; void *ptr; @@ -1089,7 +1076,7 @@ void odp_packet_user_ptr_set(odp_packet_t pkt, const void *ptr) return; } - pkt_hdr->buf_hdr.user_ptr = ptr; + pkt_hdr->event_hdr.user_ptr = ptr; pkt_hdr->p.flags.user_ptr_set = 1; } @@ -1222,7 +1209,7 @@ int odp_packet_add_data(odp_packet_t *pkt_ptr, uint32_t offset, uint32_t len) odp_packet_t pkt = *pkt_ptr; odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); uint32_t pktlen = pkt_hdr->frame_len; - pool_t *pool = pkt_hdr->buf_hdr.pool_ptr; + pool_t *pool = pkt_hdr->event_hdr.pool_ptr; odp_packet_t newpkt; if (offset > pktlen) @@ -1252,7 +1239,7 @@ int odp_packet_rem_data(odp_packet_t *pkt_ptr, uint32_t offset, uint32_t len) odp_packet_t pkt = *pkt_ptr; odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); uint32_t pktlen = pkt_hdr->frame_len; - pool_t *pool = pkt_hdr->buf_hdr.pool_ptr; + pool_t *pool = pkt_hdr->event_hdr.pool_ptr; odp_packet_t newpkt; if (offset + len >= pktlen) @@ -1284,7 +1271,7 @@ int odp_packet_align(odp_packet_t *pkt, uint32_t offset, uint32_t len, uint32_t shift; uint32_t seglen = 0; /* GCC */ odp_packet_hdr_t *pkt_hdr = packet_hdr(*pkt); - pool_t *pool = pkt_hdr->buf_hdr.pool_ptr; + pool_t *pool = pkt_hdr->event_hdr.pool_ptr; void *addr = packet_map(pkt_hdr, offset, &seglen, NULL); uint64_t uaddr = (uint64_t)(uintptr_t)addr; uint64_t misalign; @@ -1326,8 +1313,8 @@ int odp_packet_concat(odp_packet_t *dst, odp_packet_t src) { odp_packet_hdr_t *dst_hdr = packet_hdr(*dst); odp_packet_hdr_t *src_hdr = packet_hdr(src); - pool_t *dst_pool = dst_hdr->buf_hdr.pool_ptr; - pool_t *src_pool = src_hdr->buf_hdr.pool_ptr; + pool_t *dst_pool = dst_hdr->event_hdr.pool_ptr; + pool_t *src_pool = src_hdr->event_hdr.pool_ptr; uint32_t dst_len = dst_hdr->frame_len; uint32_t src_len = src_hdr->frame_len; @@ -1589,8 +1576,8 @@ void odp_packet_print(odp_packet_t pkt) odp_packet_hdr_t *hdr = packet_hdr(pkt); len += snprintf(&str[len], n - len, "Packet\n------\n"); - len += snprintf(&str[len], n - len, " pool index %u\n", hdr->buf_hdr.index.pool); - len += snprintf(&str[len], n - len, " buf index %u\n", hdr->buf_hdr.index.buffer); + len += snprintf(&str[len], n - len, " pool index %u\n", hdr->event_hdr.index.pool); + len += snprintf(&str[len], n - len, " buf index %u\n", hdr->event_hdr.index.buffer); len += snprintf(&str[len], n - len, " ev subtype %i\n", hdr->subtype); len += snprintf(&str[len], n - len, " input_flags 0x%" PRIx64 "\n", hdr->p.input_flags.all); @@ -1627,7 +1614,7 @@ void odp_packet_print(odp_packet_t pkt) for (int seg_idx = 0; seg != ODP_PACKET_SEG_INVALID; seg_idx++) { odp_packet_hdr_t *seg_hdr = packet_seg_to_hdr(seg); - odp_buffer_hdr_t *buf_hdr = &seg_hdr->buf_hdr; + _odp_event_hdr_t *event_hdr = &seg_hdr->event_hdr; char seg_str[max_len]; int str_len; @@ -1637,7 +1624,7 @@ void odp_packet_print(odp_packet_t pkt) seg_idx, odp_packet_seg_data_len(pkt, seg), odp_packet_seg_data(pkt, seg), - buffer_ref(buf_hdr)); + segment_ref(event_hdr)); /* Prevent print buffer overflow */ if (n - len - str_len < 10) { @@ -1663,14 +1650,14 @@ void odp_packet_print_data(odp_packet_t pkt, uint32_t offset, int len = 0; int n = max_len - 1; uint32_t data_len = odp_packet_len(pkt); - pool_t *pool = hdr->buf_hdr.pool_ptr; + pool_t *pool = hdr->event_hdr.pool_ptr; len += snprintf(&str[len], n - len, "Packet\n------\n"); len += snprintf(&str[len], n - len, " pool index %" PRIu32 "\n", pool->pool_idx); len += snprintf(&str[len], n - len, " buf index %" PRIu32 "\n", - hdr->buf_hdr.index.buffer); + hdr->event_hdr.index.buffer); len += snprintf(&str[len], n - len, " seg_count %" PRIu16 "\n", hdr->seg_count); len += snprintf(&str[len], n - len, @@ -1721,11 +1708,11 @@ int odp_packet_is_valid(odp_packet_t pkt) if (pkt == ODP_PACKET_INVALID) return 0; - if (_odp_buffer_is_valid(packet_to_buffer(pkt)) == 0) - return 0; - ev = odp_packet_to_event(pkt); + if (_odp_event_is_valid(ev) == 0) + return 0; + if (odp_event_type(ev) != ODP_EVENT_PACKET) return 0; @@ -1757,8 +1744,8 @@ int _odp_packet_copy_md_to_packet(odp_packet_t srcpkt, odp_packet_t dstpkt) { odp_packet_hdr_t *srchdr = packet_hdr(srcpkt); odp_packet_hdr_t *dsthdr = packet_hdr(dstpkt); - pool_t *src_pool = srchdr->buf_hdr.pool_ptr; - pool_t *dst_pool = dsthdr->buf_hdr.pool_ptr; + pool_t *src_pool = srchdr->event_hdr.pool_ptr; + pool_t *dst_pool = dsthdr->event_hdr.pool_ptr; uint32_t src_uarea_size = src_pool->param_uarea_size; uint32_t dst_uarea_size = dst_pool->param_uarea_size; @@ -1766,10 +1753,10 @@ int _odp_packet_copy_md_to_packet(odp_packet_t srcpkt, odp_packet_t dstpkt) dsthdr->dst_queue = srchdr->dst_queue; dsthdr->cos = srchdr->cos; dsthdr->cls_mark = srchdr->cls_mark; - dsthdr->buf_hdr.user_ptr = srchdr->buf_hdr.user_ptr; - if (dsthdr->buf_hdr.uarea_addr != NULL && - srchdr->buf_hdr.uarea_addr != NULL) { - memcpy(dsthdr->buf_hdr.uarea_addr, srchdr->buf_hdr.uarea_addr, + dsthdr->event_hdr.user_ptr = srchdr->event_hdr.user_ptr; + if (dsthdr->event_hdr.uarea_addr != NULL && + srchdr->event_hdr.uarea_addr != NULL) { + memcpy(dsthdr->event_hdr.uarea_addr, srchdr->event_hdr.uarea_addr, dst_uarea_size <= src_uarea_size ? dst_uarea_size : src_uarea_size); } @@ -2790,7 +2777,10 @@ odp_packet_t odp_packet_ref_static(odp_packet_t pkt) { odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - packet_ref_inc(pkt_hdr); + while (pkt_hdr != NULL) { + segment_ref_inc(&pkt_hdr->event_hdr); + pkt_hdr = pkt_hdr->seg_next; + } return pkt; } @@ -2844,14 +2834,14 @@ odp_packet_t odp_packet_ref_pkt(odp_packet_t pkt, uint32_t offset, int odp_packet_has_ref(odp_packet_t pkt) { - odp_buffer_hdr_t *buf_hdr; + _odp_event_hdr_t *event_hdr; odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); uint32_t ref_cnt; while (pkt_hdr != NULL) { - buf_hdr = &pkt_hdr->buf_hdr; + event_hdr = &pkt_hdr->event_hdr; - ref_cnt = buffer_ref(buf_hdr); + ref_cnt = segment_ref(event_hdr); if (is_multi_ref(ref_cnt)) return 1; @@ -3048,7 +3038,7 @@ static inline odp_packet_hdr_t *packet_buf_to_hdr(odp_packet_buf_t pkt_buf) void *odp_packet_buf_head(odp_packet_buf_t pkt_buf) { odp_packet_hdr_t *pkt_hdr = packet_buf_to_hdr(pkt_buf); - pool_t *pool = pkt_hdr->buf_hdr.pool_ptr; + pool_t *pool = pkt_hdr->event_hdr.pool_ptr; uint32_t head_offset = sizeof(odp_packet_hdr_t) + pool->ext_param.pkt.app_header_size; if (odp_unlikely(pool->pool_ext == 0)) { @@ -3062,7 +3052,7 @@ void *odp_packet_buf_head(odp_packet_buf_t pkt_buf) uint32_t odp_packet_buf_size(odp_packet_buf_t pkt_buf) { odp_packet_hdr_t *pkt_hdr = packet_buf_to_hdr(pkt_buf); - pool_t *pool = pkt_hdr->buf_hdr.pool_ptr; + pool_t *pool = pkt_hdr->event_hdr.pool_ptr; uint32_t head_offset = sizeof(odp_packet_hdr_t) + pool->ext_param.pkt.app_header_size; return pool->ext_param.pkt.buf_size - head_offset; @@ -3114,7 +3104,7 @@ uint32_t odp_packet_disassemble(odp_packet_t pkt, odp_packet_buf_t pkt_buf[], ui uint32_t i; odp_packet_seg_t seg; odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - pool_t *pool = pkt_hdr->buf_hdr.pool_ptr; + pool_t *pool = pkt_hdr->event_hdr.pool_ptr; uint32_t num_segs = odp_packet_num_segs(pkt); if (odp_unlikely(pool->type != ODP_POOL_PACKET)) { diff --git a/platform/linux-generic/odp_packet_io.c b/platform/linux-generic/odp_packet_io.c index bd8bb58e8..a3a3f1c19 100644 --- a/platform/linux-generic/odp_packet_io.c +++ b/platform/linux-generic/odp_packet_io.c @@ -61,9 +61,9 @@ static inline pktio_entry_t *pktio_entry_by_index(int index) return _odp_pktio_entry_ptr[index]; } -static inline odp_buffer_hdr_t *packet_vector_to_buf_hdr(odp_packet_vector_t pktv) +static inline _odp_event_hdr_t *packet_vector_to_event_hdr(odp_packet_vector_t pktv) { - return &_odp_packet_vector_hdr(pktv)->buf_hdr; + return (_odp_event_hdr_t *)(uintptr_t)&_odp_packet_vector_hdr(pktv)->event_hdr; } static int read_config_file(pktio_global_t *pktio_glb) @@ -825,13 +825,13 @@ static inline odp_packet_vector_t packet_vector_create(odp_packet_t packets[], u } static inline int pktin_recv_buf(pktio_entry_t *entry, int pktin_index, - odp_buffer_hdr_t *buffer_hdrs[], int num) + _odp_event_hdr_t *event_hdrs[], int num) { odp_packet_t pkt; odp_packet_t packets[num]; odp_packet_hdr_t *pkt_hdr; odp_pool_t pool = ODP_POOL_INVALID; - odp_buffer_hdr_t *buf_hdr; + _odp_event_hdr_t *event_hdr; int i, pkts, num_rx, num_ev, num_dst; odp_queue_t cur_queue; odp_event_t ev[num]; @@ -860,7 +860,7 @@ static inline int pktin_recv_buf(pktio_entry_t *entry, int pktin_index, for (i = 0; i < pkts; i++) { pkt = packets[i]; pkt_hdr = packet_hdr(pkt); - buf_hdr = packet_to_buf_hdr(pkt); + event_hdr = packet_to_event_hdr(pkt); if (odp_unlikely(pkt_hdr->p.input_flags.dst_queue)) { /* Sort events for enqueue multi operation(s) based on CoS @@ -888,7 +888,7 @@ static inline int pktin_recv_buf(pktio_entry_t *entry, int pktin_index, num_ev++; continue; } - buffer_hdrs[num_rx++] = buf_hdr; + event_hdrs[num_rx++] = event_hdr; } /* Optimization for the common case */ @@ -897,13 +897,13 @@ static inline int pktin_recv_buf(pktio_entry_t *entry, int pktin_index, return num_rx; /* Create packet vector */ - odp_packet_vector_t pktv = packet_vector_create((odp_packet_t *)buffer_hdrs, + odp_packet_vector_t pktv = packet_vector_create((odp_packet_t *)event_hdrs, num_rx, pool); if (odp_unlikely(pktv == ODP_PACKET_VECTOR_INVALID)) return 0; - buffer_hdrs[0] = packet_vector_to_buf_hdr(pktv); + event_hdrs[0] = packet_vector_to_event_hdr(pktv); return 1; } @@ -975,15 +975,15 @@ static inline int packet_vector_send(odp_pktout_queue_t pktout_queue, odp_event_ return 0; } -static int pktout_enqueue(odp_queue_t queue, odp_buffer_hdr_t *buf_hdr) +static int pktout_enqueue(odp_queue_t queue, _odp_event_hdr_t *event_hdr) { - odp_event_t event = odp_buffer_to_event(buf_from_buf_hdr(buf_hdr)); - odp_packet_t pkt = packet_from_buf_hdr(buf_hdr); + odp_event_t event = _odp_event_from_hdr(event_hdr); + odp_packet_t pkt = packet_from_event_hdr(event_hdr); odp_pktout_queue_t pktout_queue; int len = 1; int nbr; - if (_odp_sched_fn->ord_enq_multi(queue, (void **)buf_hdr, len, &nbr)) + if (_odp_sched_fn->ord_enq_multi(queue, (void **)event_hdr, len, &nbr)) return (nbr == len ? 0 : -1); pktout_queue = _odp_queue_fn->get_pktout(queue); @@ -995,7 +995,7 @@ static int pktout_enqueue(odp_queue_t queue, odp_buffer_hdr_t *buf_hdr) return (nbr == len ? 0 : -1); } -static int pktout_enq_multi(odp_queue_t queue, odp_buffer_hdr_t *buf_hdr[], +static int pktout_enq_multi(odp_queue_t queue, _odp_event_hdr_t *event_hdr[], int num) { odp_event_t event; @@ -1005,18 +1005,18 @@ static int pktout_enq_multi(odp_queue_t queue, odp_buffer_hdr_t *buf_hdr[], int nbr; int i; - if (_odp_sched_fn->ord_enq_multi(queue, (void **)buf_hdr, num, &nbr)) + if (_odp_sched_fn->ord_enq_multi(queue, (void **)event_hdr, num, &nbr)) return nbr; for (i = 0; i < num; ++i) { - event = odp_buffer_to_event(buf_from_buf_hdr(buf_hdr[i])); + event = _odp_event_from_hdr(event_hdr[i]); if (odp_event_type(event) == ODP_EVENT_PACKET_VECTOR) { have_pktv = 1; break; } - pkt_tbl[i] = packet_from_buf_hdr(buf_hdr[i]); + pkt_tbl[i] = packet_from_event_hdr(event_hdr[i]); } pktout_queue = _odp_queue_fn->get_pktout(queue); @@ -1025,13 +1025,13 @@ static int pktout_enq_multi(odp_queue_t queue, odp_buffer_hdr_t *buf_hdr[], return odp_pktout_send(pktout_queue, pkt_tbl, num); for (i = 0; i < num; ++i) { - event = odp_buffer_to_event(buf_from_buf_hdr(buf_hdr[i])); + event = _odp_event_from_hdr(event_hdr[i]); if (odp_event_type(event) == ODP_EVENT_PACKET_VECTOR) { if (odp_unlikely(packet_vector_send(pktout_queue, event))) break; } else { - odp_packet_t pkt = packet_from_buf_hdr(buf_hdr[i]); + odp_packet_t pkt = packet_from_event_hdr(event_hdr[i]); nbr = odp_pktout_send(pktout_queue, &pkt, 1); if (odp_unlikely(nbr != 1)) @@ -1041,10 +1041,10 @@ static int pktout_enq_multi(odp_queue_t queue, odp_buffer_hdr_t *buf_hdr[], return i; } -static odp_buffer_hdr_t *pktin_dequeue(odp_queue_t queue) +static _odp_event_hdr_t *pktin_dequeue(odp_queue_t queue) { - odp_buffer_hdr_t *buf_hdr; - odp_buffer_hdr_t *hdr_tbl[QUEUE_MULTI_MAX]; + _odp_event_hdr_t *event_hdr; + _odp_event_hdr_t *hdr_tbl[QUEUE_MULTI_MAX]; int pkts; odp_pktin_queue_t pktin_queue = _odp_queue_fn->get_pktin(queue); odp_pktio_t pktio = pktin_queue.pktio; @@ -1053,8 +1053,8 @@ static odp_buffer_hdr_t *pktin_dequeue(odp_queue_t queue) ODP_ASSERT(entry != NULL); - if (_odp_queue_fn->orig_deq_multi(queue, &buf_hdr, 1) == 1) - return buf_hdr; + if (_odp_queue_fn->orig_deq_multi(queue, &event_hdr, 1) == 1) + return event_hdr; pkts = pktin_recv_buf(entry, pktin_index, hdr_tbl, QUEUE_MULTI_MAX); @@ -1074,19 +1074,19 @@ static odp_buffer_hdr_t *pktin_dequeue(odp_queue_t queue) ODP_DBG("Interface %s dropped %i packets\n", entry->s.name, num - num_enq); - _odp_buffer_free_multi(&hdr_tbl[num_enq + 1], num - num_enq); + _odp_event_free_multi(&hdr_tbl[num_enq + 1], num - num_enq); } } - buf_hdr = hdr_tbl[0]; - return buf_hdr; + event_hdr = hdr_tbl[0]; + return event_hdr; } -static int pktin_deq_multi(odp_queue_t queue, odp_buffer_hdr_t *buf_hdr[], +static int pktin_deq_multi(odp_queue_t queue, _odp_event_hdr_t *event_hdr[], int num) { int nbr; - odp_buffer_hdr_t *hdr_tbl[QUEUE_MULTI_MAX]; + _odp_event_hdr_t *hdr_tbl[QUEUE_MULTI_MAX]; int pkts, i, j; odp_pktin_queue_t pktin_queue = _odp_queue_fn->get_pktin(queue); odp_pktio_t pktio = pktin_queue.pktio; @@ -1095,7 +1095,7 @@ static int pktin_deq_multi(odp_queue_t queue, odp_buffer_hdr_t *buf_hdr[], ODP_ASSERT(entry != NULL); - nbr = _odp_queue_fn->orig_deq_multi(queue, buf_hdr, num); + nbr = _odp_queue_fn->orig_deq_multi(queue, event_hdr, num); if (odp_unlikely(nbr > num)) ODP_ABORT("queue_deq_multi req: %d, returned %d\n", num, nbr); @@ -1111,7 +1111,7 @@ static int pktin_deq_multi(odp_queue_t queue, odp_buffer_hdr_t *buf_hdr[], return nbr; for (i = 0; i < pkts && nbr < num; i++, nbr++) - buf_hdr[nbr] = hdr_tbl[i]; + event_hdr[nbr] = hdr_tbl[i]; /* Queue the rest for later */ for (j = 0; i < pkts; i++, j++) @@ -1128,7 +1128,7 @@ static int pktin_deq_multi(odp_queue_t queue, odp_buffer_hdr_t *buf_hdr[], ODP_DBG("Interface %s dropped %i packets\n", entry->s.name, j - num_enq); - _odp_buffer_free_multi(&buf_hdr[num_enq], j - num_enq); + _odp_event_free_multi(&event_hdr[num_enq], j - num_enq); } } @@ -1225,7 +1225,7 @@ int _odp_sched_cb_pktin_poll_one(int pktio_index, } int _odp_sched_cb_pktin_poll(int pktio_index, int pktin_index, - odp_buffer_hdr_t *hdr_tbl[], int num) + _odp_event_hdr_t *hdr_tbl[], int num) { pktio_entry_t *entry = pktio_entry_by_index(pktio_index); int state = entry->s.state; diff --git a/platform/linux-generic/odp_packet_vector.c b/platform/linux-generic/odp_packet_vector.c index 6f0ee201a..66570739b 100644 --- a/platform/linux-generic/odp_packet_vector.c +++ b/platform/linux-generic/odp_packet_vector.c @@ -5,7 +5,6 @@ */ #include <odp/api/align.h> -#include <odp/api/buffer.h> #include <odp/api/hints.h> #include <odp/api/packet.h> #include <odp/api/pool.h> @@ -24,17 +23,21 @@ /* Packet vector header field offsets for inline functions */ const _odp_event_vector_inline_offset_t _odp_event_vector_inline ODP_ALIGNED_CACHE = { .packet = offsetof(odp_event_vector_hdr_t, packet), - .pool = offsetof(odp_event_vector_hdr_t, buf_hdr.pool_ptr), + .pool = offsetof(odp_event_vector_hdr_t, event_hdr.pool_ptr), .size = offsetof(odp_event_vector_hdr_t, size) }; #include <odp/visibility_end.h> +static inline odp_event_vector_hdr_t *event_vector_hdr_from_event(odp_event_t event) +{ + return (odp_event_vector_hdr_t *)(uintptr_t)event; +} + odp_packet_vector_t odp_packet_vector_alloc(odp_pool_t pool_hdl) { - odp_packet_vector_t pktv; + odp_event_t event; pool_t *pool; - int ret; ODP_ASSERT(pool_hdl != ODP_POOL_INVALID); @@ -42,12 +45,13 @@ odp_packet_vector_t odp_packet_vector_alloc(odp_pool_t pool_hdl) ODP_ASSERT(pool->type == ODP_POOL_VECTOR); - ret = _odp_buffer_alloc_multi(pool, (odp_buffer_hdr_t **)&pktv, 1); + event = _odp_event_alloc(pool); + if (odp_unlikely(event == ODP_EVENT_INVALID)) + return ODP_PACKET_VECTOR_INVALID; - if (odp_likely(ret == 1)) - return pktv; + ODP_ASSERT(event_vector_hdr_from_event(event)->size == 0); - return ODP_PACKET_VECTOR_INVALID; + return odp_packet_vector_from_event(event); } void odp_packet_vector_free(odp_packet_vector_t pktv) @@ -56,7 +60,7 @@ void odp_packet_vector_free(odp_packet_vector_t pktv) pktv_hdr->size = 0; - _odp_buffer_free_multi((odp_buffer_hdr_t **)&pktv_hdr, 1); + _odp_event_free(odp_packet_vector_to_event(pktv)); } int odp_packet_vector_valid(odp_packet_vector_t pktv) @@ -69,16 +73,16 @@ int odp_packet_vector_valid(odp_packet_vector_t pktv) if (odp_unlikely(pktv == ODP_PACKET_VECTOR_INVALID)) return 0; - if (_odp_buffer_is_valid((odp_buffer_t)pktv) == 0) - return 0; - ev = odp_packet_vector_to_event(pktv); + if (_odp_event_is_valid(ev) == 0) + return 0; + if (odp_event_type(ev) != ODP_EVENT_PACKET_VECTOR) return 0; pktv_hdr = _odp_packet_vector_hdr(pktv); - pool = pktv_hdr->buf_hdr.pool_ptr; + pool = pktv_hdr->event_hdr.pool_ptr; if (odp_unlikely(pktv_hdr->size > pool->params.vector.max_size)) return 0; diff --git a/platform/linux-generic/odp_pool.c b/platform/linux-generic/odp_pool.c index 49d2e74f5..ca696fc96 100644 --- a/platform/linux-generic/odp_pool.c +++ b/platform/linux-generic/odp_pool.c @@ -24,6 +24,7 @@ #include <odp_shm_internal.h> #include <odp_timer_internal.h> #include <odp_event_vector_internal.h> +#include <odp_buffer_internal.h> #include <string.h> #include <stdio.h> @@ -79,9 +80,9 @@ static inline odp_pool_t pool_index_to_handle(uint32_t pool_idx) static inline pool_t *pool_from_buf(odp_buffer_t buf) { - odp_buffer_hdr_t *buf_hdr = buf_hdl_to_hdr(buf); + odp_buffer_hdr_t *buf_hdr = _odp_buf_hdr(buf); - return buf_hdr->pool_ptr; + return buf_hdr->event_hdr.pool_ptr; } static inline void cache_init(pool_cache_t *cache) @@ -90,7 +91,7 @@ static inline void cache_init(pool_cache_t *cache) } static inline uint32_t cache_pop(pool_cache_t *cache, - odp_buffer_hdr_t *buf_hdr[], int max_num) + _odp_event_hdr_t *event_hdr[], int max_num) { uint32_t cache_num = cache->cache_num; uint32_t num_ch = max_num; @@ -104,36 +105,36 @@ static inline uint32_t cache_pop(pool_cache_t *cache, /* Get buffers from the cache */ cache_begin = cache_num - num_ch; for (i = 0; i < num_ch; i++) - buf_hdr[i] = cache->buf_hdr[cache_begin + i]; + event_hdr[i] = cache->event_hdr[cache_begin + i]; cache->cache_num = cache_num - num_ch; return num_ch; } -static inline void cache_push(pool_cache_t *cache, odp_buffer_hdr_t *buf_hdr[], +static inline void cache_push(pool_cache_t *cache, _odp_event_hdr_t *event_hdr[], uint32_t num) { uint32_t cache_num = cache->cache_num; uint32_t i; for (i = 0; i < num; i++) - cache->buf_hdr[cache_num + i] = buf_hdr[i]; + cache->event_hdr[cache_num + i] = event_hdr[i]; cache->cache_num = cache_num + num; } static void cache_flush(pool_cache_t *cache, pool_t *pool) { - odp_buffer_hdr_t *buf_hdr; + _odp_event_hdr_t *event_hdr; ring_ptr_t *ring; uint32_t mask; ring = &pool->ring->hdr; mask = pool->ring_mask; - while (cache_pop(cache, &buf_hdr, 1)) - ring_ptr_enq(ring, mask, buf_hdr); + while (cache_pop(cache, &event_hdr, 1)) + ring_ptr_enq(ring, mask, event_hdr); } static inline uint64_t cache_total_available(pool_t *pool) @@ -299,6 +300,7 @@ int _odp_pool_init_global(void) } ODP_DBG("\nPool init global\n"); + ODP_DBG(" event_hdr_t size %zu\n", sizeof(_odp_event_hdr_t)); ODP_DBG(" buffer_hdr_t size %zu\n", sizeof(odp_buffer_hdr_t)); ODP_DBG(" packet_hdr_t size %zu\n", sizeof(odp_packet_hdr_t)); ODP_DBG(" timeout_hdr_t size %zu\n", sizeof(odp_timeout_hdr_t)); @@ -393,7 +395,7 @@ static pool_t *reserve_pool(uint32_t shmflags, uint8_t pool_ext, uint32_t num) /* Reserve memory for the ring, and for lookup table in case of pool ext */ mem_size = sizeof(pool_ring_t); if (pool_ext) - mem_size += num * sizeof(odp_buffer_hdr_t *); + mem_size += num * sizeof(_odp_event_hdr_t *); shm = odp_shm_reserve(ring_name, mem_size, ODP_CACHE_LINE_SIZE, shmflags); @@ -417,30 +419,32 @@ static pool_t *reserve_pool(uint32_t shmflags, uint8_t pool_ext, uint32_t num) return NULL; } -static void init_buffer_hdr(pool_t *pool, odp_buffer_hdr_t *buf_hdr, uint32_t buf_index, - uint32_t hdr_len, uint8_t *data_ptr, void *uarea) +static void init_event_hdr(pool_t *pool, _odp_event_hdr_t *event_hdr, uint32_t buf_index, + uint32_t hdr_len, uint8_t *data_ptr, void *uarea) { odp_pool_type_t type = pool->type; - memset(buf_hdr, 0, hdr_len); + memset(event_hdr, 0, hdr_len); - /* Initialize buffer metadata */ - buf_hdr->index.u32 = 0; - buf_hdr->index.pool = pool->pool_idx; - buf_hdr->index.buffer = buf_index; - buf_hdr->type = type; - buf_hdr->event_type = type; - buf_hdr->pool_ptr = pool; - buf_hdr->uarea_addr = uarea; - odp_atomic_init_u32(&buf_hdr->ref_cnt, 0); + /* Initialize common event metadata */ + event_hdr->index.u32 = 0; + event_hdr->index.pool = pool->pool_idx; + event_hdr->index.buffer = buf_index; + event_hdr->type = type; + event_hdr->event_type = type; + event_hdr->pool_ptr = pool; + event_hdr->uarea_addr = uarea; + odp_atomic_init_u32(&event_hdr->ref_cnt, 0); /* Store base values for fast init */ - buf_hdr->base_data = data_ptr; - buf_hdr->buf_end = data_ptr + pool->seg_len + pool->tailroom; + if (type == ODP_POOL_BUFFER || type == ODP_POOL_PACKET) { + event_hdr->base_data = data_ptr; + event_hdr->buf_end = data_ptr + pool->seg_len + pool->tailroom; + } /* Initialize segmentation metadata */ if (type == ODP_POOL_PACKET) { - odp_packet_hdr_t *pkt_hdr = (void *)buf_hdr; + odp_packet_hdr_t *pkt_hdr = (void *)event_hdr; pkt_hdr->seg_data = data_ptr; pkt_hdr->seg_len = pool->seg_len; @@ -450,22 +454,24 @@ static void init_buffer_hdr(pool_t *pool, odp_buffer_hdr_t *buf_hdr, uint32_t bu /* Initialize event vector metadata */ if (type == ODP_POOL_VECTOR) { - odp_event_vector_hdr_t *vect_hdr = (void *)buf_hdr; + odp_event_vector_hdr_t *vect_hdr = (void *)event_hdr; - vect_hdr->size = 0; - buf_hdr->event_type = ODP_EVENT_PACKET_VECTOR; + vect_hdr->size = 0; + event_hdr->event_type = ODP_EVENT_PACKET_VECTOR; } } static void init_buffers(pool_t *pool) { uint64_t i; + _odp_event_hdr_t *event_hdr; odp_buffer_hdr_t *buf_hdr; odp_packet_hdr_t *pkt_hdr; odp_shm_info_t shm_info; void *addr; void *uarea = NULL; - uint8_t *data; + uint8_t *data = NULL; + uint8_t *data_ptr = NULL; uint32_t offset, hdr_len; ring_ptr_t *ring; uint32_t mask; @@ -486,6 +492,7 @@ static void init_buffers(pool_t *pool) addr = &pool->base_addr[(i * pool->block_size) + pool->block_offset]; + event_hdr = addr; buf_hdr = addr; pkt_hdr = addr; @@ -508,23 +515,34 @@ static void init_buffers(pool_t *pool) if (pool->uarea_size) uarea = &pool->uarea_base_addr[(i - skipped_blocks) * pool->uarea_size]; - data = buf_hdr->data; - if (type == ODP_POOL_PACKET) - data = pkt_hdr->data; + /* Only buffers and packets have data pointer */ + if (type == ODP_POOL_BUFFER || type == ODP_POOL_PACKET) { + if (type == ODP_POOL_BUFFER) + data = buf_hdr->data; + else + data = pkt_hdr->data; - offset = pool->headroom; + offset = pool->headroom; - /* move to correct align */ - while (((uintptr_t)&data[offset]) % pool->align != 0) - offset++; + /* Move to correct align */ + while (((uintptr_t)&data[offset]) % pool->align != 0) + offset++; + + hdr_len = (uintptr_t)data - (uintptr_t)event_hdr; + data_ptr = &data[offset]; + } else { + if (type == ODP_POOL_TIMEOUT) + hdr_len = sizeof(odp_timeout_hdr_t); + else + hdr_len = sizeof(odp_event_vector_hdr_t); + } - hdr_len = (uintptr_t)data - (uintptr_t)buf_hdr; - init_buffer_hdr(pool, buf_hdr, i, hdr_len, &data[offset], uarea); + init_event_hdr(pool, event_hdr, i, hdr_len, data_ptr, uarea); /* Store buffer into the global pool */ if (!skip) - ring_ptr_enq(ring, mask, buf_hdr); + ring_ptr_enq(ring, mask, event_hdr); } pool->skipped_blocks = skipped_blocks; } @@ -606,8 +624,10 @@ static int reserve_uarea(pool_t *pool, uint32_t uarea_size, uint32_t num_pkt, ui return 0; } -static odp_pool_t pool_create(const char *name, const odp_pool_param_t *params, - uint32_t shmflags) +/* Create pool according to params. Actual type of the pool is type_2, which is recorded for pool + * info calls. */ +odp_pool_t _odp_pool_create(const char *name, const odp_pool_param_t *params, + odp_pool_type_t type_2) { pool_t *pool; uint32_t uarea_size, headroom, tailroom; @@ -616,11 +636,17 @@ static odp_pool_t pool_create(const char *name, const odp_pool_param_t *params, uint32_t max_len, cache_size; uint32_t ring_size; odp_pool_type_t type = params->type; + uint32_t shmflags = 0; uint32_t num_extra = 0; const char *max_prefix = "pool_000_"; int max_prefix_len = strlen(max_prefix); char shm_name[ODP_POOL_NAME_LEN + max_prefix_len]; + if (type == ODP_POOL_PACKET) + shmflags = ODP_SHM_PROC; + if (odp_global_ro.shm_single_va) + shmflags |= ODP_SHM_SINGLE_VA; + align = 0; if (type == ODP_POOL_PACKET) { @@ -738,6 +764,7 @@ static odp_pool_t pool_create(const char *name, const odp_pool_param_t *params, sprintf(shm_name, "pool_%03i_%s", pool->pool_idx, pool->name); pool->type = type; + pool->type_2 = type_2; pool->params = *params; pool->block_offset = 0; @@ -1019,17 +1046,10 @@ static int check_params(const odp_pool_param_t *params) odp_pool_t odp_pool_create(const char *name, const odp_pool_param_t *params) { - uint32_t shm_flags = 0; - if (check_params(params)) return ODP_POOL_INVALID; - if (params->type == ODP_POOL_PACKET) - shm_flags = ODP_SHM_PROC; - if (odp_global_ro.shm_single_va) - shm_flags |= ODP_SHM_SINGLE_VA; - - return pool_create(name, params, shm_flags); + return _odp_pool_create(name, params, params->type); } int odp_pool_destroy(odp_pool_t pool_hdl) @@ -1073,16 +1093,6 @@ int odp_pool_destroy(odp_pool_t pool_hdl) return 0; } -odp_event_type_t _odp_buffer_event_type(odp_buffer_t buf) -{ - return buf_hdl_to_hdr(buf)->event_type; -} - -void _odp_buffer_event_type_set(odp_buffer_t buf, int ev) -{ - buf_hdl_to_hdr(buf)->event_type = ev; -} - odp_pool_t odp_pool_lookup(const char *name) { uint32_t i; @@ -1112,11 +1122,17 @@ int odp_pool_info(odp_pool_t pool_hdl, odp_pool_info_t *info) memset(info, 0, sizeof(odp_pool_info_t)); + info->type = pool->type_2; info->name = pool->name; if (pool->pool_ext) { info->pool_ext = 1; info->pool_ext_param = pool->ext_param; + + } else if (pool->type_2 == ODP_POOL_DMA_COMPL) { + info->dma_pool_param.num = pool->params.buf.num; + info->dma_pool_param.cache_size = pool->params.buf.cache_size; + } else { info->params = pool->params; } @@ -1130,18 +1146,18 @@ int odp_pool_info(odp_pool_t pool_hdl, odp_pool_info_t *info) return 0; } -int _odp_buffer_alloc_multi(pool_t *pool, odp_buffer_hdr_t *buf_hdr[], int max_num) +int _odp_event_alloc_multi(pool_t *pool, _odp_event_hdr_t *event_hdr[], int max_num) { uint32_t pool_idx = pool->pool_idx; pool_cache_t *cache = local.cache[pool_idx]; ring_ptr_t *ring; - odp_buffer_hdr_t *hdr; + _odp_event_hdr_t *hdr; uint32_t mask, num_ch, num_alloc, i; uint32_t num_deq = 0; uint32_t burst_size = pool->burst_size; /* First pull packets from local cache */ - num_ch = cache_pop(cache, buf_hdr, max_num); + num_ch = cache_pop(cache, event_hdr, max_num); if (CONFIG_POOL_STATISTICS && pool->params.stats.bit.cache_alloc_ops && num_ch) odp_atomic_inc_u64(&pool->stats.cache_alloc_ops); @@ -1155,7 +1171,7 @@ int _odp_buffer_alloc_multi(pool_t *pool, odp_buffer_hdr_t *buf_hdr[], int max_n if (odp_unlikely(num_deq > burst_size)) burst = num_deq; - odp_buffer_hdr_t *hdr_tmp[burst]; + _odp_event_hdr_t *hdr_tmp[burst]; ring = &pool->ring->hdr; mask = pool->ring_mask; @@ -1180,7 +1196,7 @@ int _odp_buffer_alloc_multi(pool_t *pool, odp_buffer_hdr_t *buf_hdr[], int max_n hdr = hdr_tmp[i]; odp_prefetch(hdr); - buf_hdr[idx] = hdr; + event_hdr[idx] = hdr; } /* Cache possible extra buffers. Cache is currently empty. */ @@ -1193,8 +1209,8 @@ int _odp_buffer_alloc_multi(pool_t *pool, odp_buffer_hdr_t *buf_hdr[], int max_n return num_alloc; } -static inline void buffer_free_to_pool(pool_t *pool, - odp_buffer_hdr_t *buf_hdr[], int num) +static inline void event_free_to_pool(pool_t *pool, + _odp_event_hdr_t *event_hdr[], int num) { uint32_t pool_idx = pool->pool_idx; pool_cache_t *cache = local.cache[pool_idx]; @@ -1208,7 +1224,7 @@ static inline void buffer_free_to_pool(pool_t *pool, ring = &pool->ring->hdr; mask = pool->ring_mask; - ring_ptr_enq_multi(ring, mask, (void **)buf_hdr, num); + ring_ptr_enq_multi(ring, mask, (void **)event_hdr, num); if (CONFIG_POOL_STATISTICS && pool->params.stats.bit.free_ops) odp_atomic_inc_u64(&pool->stats.free_ops); @@ -1231,21 +1247,21 @@ static inline void buffer_free_to_pool(pool_t *pool, if (odp_unlikely((uint32_t)num > cache_num)) burst = cache_num; - odp_buffer_hdr_t *buf_hdr[burst]; + _odp_event_hdr_t *event_hdr[burst]; - cache_pop(cache, buf_hdr, burst); + cache_pop(cache, event_hdr, burst); - ring_ptr_enq_multi(ring, mask, (void **)buf_hdr, burst); + ring_ptr_enq_multi(ring, mask, (void **)event_hdr, burst); if (CONFIG_POOL_STATISTICS && pool->params.stats.bit.free_ops) odp_atomic_inc_u64(&pool->stats.free_ops); } - cache_push(cache, buf_hdr, num); + cache_push(cache, event_hdr, num); if (CONFIG_POOL_STATISTICS && pool->params.stats.bit.cache_free_ops) odp_atomic_inc_u64(&pool->stats.cache_free_ops); } -void _odp_buffer_free_multi(odp_buffer_hdr_t *buf_hdr[], int num_total) +void _odp_event_free_multi(_odp_event_hdr_t *event_hdr[], int num_total) { pool_t *pool; int num; @@ -1255,18 +1271,18 @@ void _odp_buffer_free_multi(odp_buffer_hdr_t *buf_hdr[], int num_total) while (1) { num = 1; i = 1; - pool = buf_hdr[first]->pool_ptr; + pool = event_hdr[first]->pool_ptr; /* 'num' buffers are from the same pool */ if (num_total > 1) { for (i = first; i < num_total; i++) - if (pool != buf_hdr[i]->pool_ptr) + if (pool != event_hdr[i]->pool_ptr) break; num = i - first; } - buffer_free_to_pool(pool, &buf_hdr[first], num); + event_free_to_pool(pool, &event_hdr[first], num); if (i == num_total) return; @@ -1287,7 +1303,7 @@ odp_buffer_t odp_buffer_alloc(odp_pool_t pool_hdl) ODP_ASSERT(pool->type == ODP_POOL_BUFFER); - ret = _odp_buffer_alloc_multi(pool, (odp_buffer_hdr_t **)&buf, 1); + ret = _odp_event_alloc_multi(pool, (_odp_event_hdr_t **)&buf, 1); if (odp_likely(ret == 1)) return buf; @@ -1295,6 +1311,19 @@ odp_buffer_t odp_buffer_alloc(odp_pool_t pool_hdl) return ODP_BUFFER_INVALID; } +odp_event_t _odp_event_alloc(pool_t *pool) +{ + odp_event_t event; + int ret; + + ret = _odp_event_alloc_multi(pool, (_odp_event_hdr_t **)&event, 1); + + if (odp_likely(ret == 1)) + return event; + + return ODP_EVENT_INVALID; +} + int odp_buffer_alloc_multi(odp_pool_t pool_hdl, odp_buffer_t buf[], int num) { pool_t *pool; @@ -1305,17 +1334,17 @@ int odp_buffer_alloc_multi(odp_pool_t pool_hdl, odp_buffer_t buf[], int num) ODP_ASSERT(pool->type == ODP_POOL_BUFFER); - return _odp_buffer_alloc_multi(pool, (odp_buffer_hdr_t **)buf, num); + return _odp_event_alloc_multi(pool, (_odp_event_hdr_t **)buf, num); } void odp_buffer_free(odp_buffer_t buf) { - _odp_buffer_free_multi((odp_buffer_hdr_t **)&buf, 1); + _odp_event_free_multi((_odp_event_hdr_t **)&buf, 1); } void odp_buffer_free_multi(const odp_buffer_t buf[], int num) { - _odp_buffer_free_multi((odp_buffer_hdr_t **)(uintptr_t)buf, num); + _odp_event_free_multi((_odp_event_hdr_t **)(uintptr_t)buf, num); } int odp_pool_capability(odp_pool_capability_t *capa) @@ -1571,10 +1600,10 @@ int odp_pool_stats_reset(odp_pool_t pool_hdl) return 0; } -static pool_t *find_pool(odp_buffer_hdr_t *buf_hdr) +static pool_t *find_pool(_odp_event_hdr_t *event_hdr) { int i; - uint8_t *ptr = (uint8_t *)buf_hdr; + uint8_t *ptr = (uint8_t *)event_hdr; for (i = 0; i < ODP_CONFIG_POOLS; i++) { pool_t *pool = pool_entry(i); @@ -1589,23 +1618,23 @@ static pool_t *find_pool(odp_buffer_hdr_t *buf_hdr) return NULL; } -int _odp_buffer_is_valid(odp_buffer_t buf) +int _odp_event_is_valid(odp_event_t event) { pool_t *pool; - odp_buffer_hdr_t *buf_hdr = buf_hdl_to_hdr(buf); + _odp_event_hdr_t *event_hdr = _odp_event_hdr(event); - if (buf == ODP_BUFFER_INVALID) + if (event == ODP_EVENT_INVALID) return 0; /* Check that buffer header is from a known pool */ - pool = find_pool(buf_hdr); + pool = find_pool(event_hdr); if (pool == NULL) return 0; - if (pool != buf_hdr->pool_ptr) + if (pool != event_hdr->pool_ptr) return 0; - if (buf_hdr->index.buffer >= (pool->num + pool->skipped_blocks)) + if (event_hdr->index.buffer >= (pool->num + pool->skipped_blocks)) return 0; return 1; @@ -1613,7 +1642,7 @@ int _odp_buffer_is_valid(odp_buffer_t buf) int odp_buffer_is_valid(odp_buffer_t buf) { - if (_odp_buffer_is_valid(buf) == 0) + if (_odp_event_is_valid(odp_buffer_to_event(buf)) == 0) return 0; if (odp_event_type(odp_buffer_to_event(buf)) != ODP_EVENT_BUFFER) @@ -1793,7 +1822,7 @@ int odp_pool_ext_populate(odp_pool_t pool_hdl, void *buf[], uint32_t buf_size, u uint32_t flags) { pool_t *pool; - odp_buffer_hdr_t *buf_hdr; + _odp_event_hdr_t *event_hdr; ring_ptr_t *ring; uint32_t i, ring_mask, buf_index, head_offset; uint32_t num_populated; @@ -1841,14 +1870,14 @@ int odp_pool_ext_populate(odp_pool_t pool_hdl, void *buf[], uint32_t buf_size, u head_offset = sizeof(odp_packet_hdr_t) + pool->ext_param.pkt.app_header_size; for (i = 0; i < num; i++) { - buf_hdr = buf[i]; + event_hdr = buf[i]; - if ((uintptr_t)buf_hdr & (ODP_CACHE_LINE_SIZE - 1)) { + if ((uintptr_t)event_hdr & (ODP_CACHE_LINE_SIZE - 1)) { ODP_ERR("Bad packet buffer align: buf[%u]\n", i); return -1; } - if (((uintptr_t)buf_hdr + head_offset) & (MIN_HEAD_ALIGN - 1)) { + if (((uintptr_t)event_hdr + head_offset) & (MIN_HEAD_ALIGN - 1)) { ODP_ERR("Bad head pointer align: buf[%u]\n", i); return -1; } @@ -1856,12 +1885,12 @@ int odp_pool_ext_populate(odp_pool_t pool_hdl, void *buf[], uint32_t buf_size, u if (pool->uarea_size) uarea = &pool->uarea_base_addr[buf_index * pool->uarea_size]; - data_ptr = (uint8_t *)buf_hdr + head_offset + pool->headroom; - init_buffer_hdr(pool, buf_hdr, buf_index, hdr_size, data_ptr, uarea); - pool->ring->buf_hdr_by_index[buf_index] = buf_hdr; + data_ptr = (uint8_t *)event_hdr + head_offset + pool->headroom; + init_event_hdr(pool, event_hdr, buf_index, hdr_size, data_ptr, uarea); + pool->ring->event_hdr_by_index[buf_index] = event_hdr; buf_index++; - ring_ptr_enq(ring, ring_mask, buf_hdr); + ring_ptr_enq(ring, ring_mask, event_hdr); } pool->num_populated += num; diff --git a/platform/linux-generic/odp_queue_basic.c b/platform/linux-generic/odp_queue_basic.c index fe4d90930..fe523cc62 100644 --- a/platform/linux-generic/odp_queue_basic.c +++ b/platform/linux-generic/odp_queue_basic.c @@ -1,4 +1,5 @@ /* Copyright (c) 2013-2018, Linaro Limited + * Copyright (c) 2021, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -9,8 +10,6 @@ #include <odp_queue_if.h> #include <odp/api/std_types.h> #include <odp/api/align.h> -#include <odp/api/buffer.h> -#include <odp_buffer_internal.h> #include <odp_pool_internal.h> #include <odp_init_internal.h> #include <odp_timer_internal.h> @@ -27,6 +26,8 @@ #include <odp_libconfig_internal.h> #include <odp/api/plat/queue_inline_types.h> #include <odp_global_data.h> +#include <odp_queue_basic_internal.h> +#include <odp_event_internal.h> #include <odp/api/plat/ticketlock_inlines.h> #define LOCK(queue_ptr) odp_ticketlock_lock(&((queue_ptr)->s.lock)) @@ -477,81 +478,81 @@ static odp_queue_t queue_lookup(const char *name) return ODP_QUEUE_INVALID; } -static inline void buffer_index_from_buf(uint32_t buffer_index[], - odp_buffer_hdr_t *buf_hdr[], int num) +static inline void event_index_from_hdr(uint32_t event_index[], + _odp_event_hdr_t *event_hdr[], int num) { int i; for (i = 0; i < num; i++) - buffer_index[i] = buf_hdr[i]->index.u32; + event_index[i] = event_hdr[i]->index.u32; } -static inline void buffer_index_to_buf(odp_buffer_hdr_t *buf_hdr[], - uint32_t buffer_index[], int num) +static inline void event_index_to_hdr(_odp_event_hdr_t *event_hdr[], + uint32_t event_index[], int num) { int i; for (i = 0; i < num; i++) { - buf_hdr[i] = buf_hdr_from_index_u32(buffer_index[i]); - odp_prefetch(buf_hdr[i]); + event_hdr[i] = _odp_event_hdr_from_index_u32(event_index[i]); + odp_prefetch(event_hdr[i]); } } static inline int _plain_queue_enq_multi(odp_queue_t handle, - odp_buffer_hdr_t *buf_hdr[], int num) + _odp_event_hdr_t *event_hdr[], int num) { queue_entry_t *queue; int ret, num_enq; ring_mpmc_t *ring_mpmc; - uint32_t buf_idx[num]; + uint32_t event_idx[num]; queue = qentry_from_handle(handle); ring_mpmc = &queue->s.ring_mpmc; - if (_odp_sched_fn->ord_enq_multi(handle, (void **)buf_hdr, num, &ret)) + if (_odp_sched_fn->ord_enq_multi(handle, (void **)event_hdr, num, &ret)) return ret; - buffer_index_from_buf(buf_idx, buf_hdr, num); + event_index_from_hdr(event_idx, event_hdr, num); num_enq = ring_mpmc_enq_multi(ring_mpmc, queue->s.ring_data, - queue->s.ring_mask, buf_idx, num); + queue->s.ring_mask, event_idx, num); return num_enq; } static inline int _plain_queue_deq_multi(odp_queue_t handle, - odp_buffer_hdr_t *buf_hdr[], int num) + _odp_event_hdr_t *event_hdr[], int num) { int num_deq; queue_entry_t *queue; ring_mpmc_t *ring_mpmc; - uint32_t buf_idx[num]; + uint32_t event_idx[num]; queue = qentry_from_handle(handle); ring_mpmc = &queue->s.ring_mpmc; num_deq = ring_mpmc_deq_multi(ring_mpmc, queue->s.ring_data, - queue->s.ring_mask, buf_idx, num); + queue->s.ring_mask, event_idx, num); if (num_deq == 0) return 0; - buffer_index_to_buf(buf_hdr, buf_idx, num_deq); + event_index_to_hdr(event_hdr, event_idx, num_deq); return num_deq; } static int plain_queue_enq_multi(odp_queue_t handle, - odp_buffer_hdr_t *buf_hdr[], int num) + _odp_event_hdr_t *event_hdr[], int num) { - return _plain_queue_enq_multi(handle, buf_hdr, num); + return _plain_queue_enq_multi(handle, event_hdr, num); } -static int plain_queue_enq(odp_queue_t handle, odp_buffer_hdr_t *buf_hdr) +static int plain_queue_enq(odp_queue_t handle, _odp_event_hdr_t *event_hdr) { int ret; - ret = _plain_queue_enq_multi(handle, &buf_hdr, 1); + ret = _plain_queue_enq_multi(handle, &event_hdr, 1); if (ret == 1) return 0; @@ -560,27 +561,27 @@ static int plain_queue_enq(odp_queue_t handle, odp_buffer_hdr_t *buf_hdr) } static int plain_queue_deq_multi(odp_queue_t handle, - odp_buffer_hdr_t *buf_hdr[], int num) + _odp_event_hdr_t *event_hdr[], int num) { - return _plain_queue_deq_multi(handle, buf_hdr, num); + return _plain_queue_deq_multi(handle, event_hdr, num); } -static odp_buffer_hdr_t *plain_queue_deq(odp_queue_t handle) +static _odp_event_hdr_t *plain_queue_deq(odp_queue_t handle) { - odp_buffer_hdr_t *buf_hdr = NULL; + _odp_event_hdr_t *event_hdr = NULL; int ret; - ret = _plain_queue_deq_multi(handle, &buf_hdr, 1); + ret = _plain_queue_deq_multi(handle, &event_hdr, 1); if (ret == 1) - return buf_hdr; + return event_hdr; else return NULL; } -static int error_enqueue(odp_queue_t handle, odp_buffer_hdr_t *buf_hdr) +static int error_enqueue(odp_queue_t handle, _odp_event_hdr_t *event_hdr) { - (void)buf_hdr; + (void)event_hdr; ODP_ERR("Enqueue not supported (0x%" PRIx64 ")\n", odp_queue_to_u64(handle)); @@ -589,9 +590,9 @@ static int error_enqueue(odp_queue_t handle, odp_buffer_hdr_t *buf_hdr) } static int error_enqueue_multi(odp_queue_t handle, - odp_buffer_hdr_t *buf_hdr[], int num) + _odp_event_hdr_t *event_hdr[], int num) { - (void)buf_hdr; + (void)event_hdr; (void)num; ODP_ERR("Enqueue multi not supported (0x%" PRIx64 ")\n", @@ -600,7 +601,7 @@ static int error_enqueue_multi(odp_queue_t handle, return -1; } -static odp_buffer_hdr_t *error_dequeue(odp_queue_t handle) +static _odp_event_hdr_t *error_dequeue(odp_queue_t handle) { ODP_ERR("Dequeue not supported (0x%" PRIx64 ")\n", odp_queue_to_u64(handle)); @@ -609,9 +610,9 @@ static odp_buffer_hdr_t *error_dequeue(odp_queue_t handle) } static int error_dequeue_multi(odp_queue_t handle, - odp_buffer_hdr_t *buf_hdr[], int num) + _odp_event_hdr_t *event_hdr[], int num) { - (void)buf_hdr; + (void)event_hdr; (void)num; ODP_ERR("Dequeue multi not supported (0x%" PRIx64 ")\n", @@ -880,27 +881,27 @@ static void queue_print_all(void) } static inline int _sched_queue_enq_multi(odp_queue_t handle, - odp_buffer_hdr_t *buf_hdr[], int num) + _odp_event_hdr_t *event_hdr[], int num) { int sched = 0; int ret; queue_entry_t *queue; int num_enq; ring_st_t *ring_st; - uint32_t buf_idx[num]; + uint32_t event_idx[num]; queue = qentry_from_handle(handle); ring_st = &queue->s.ring_st; - if (_odp_sched_fn->ord_enq_multi(handle, (void **)buf_hdr, num, &ret)) + if (_odp_sched_fn->ord_enq_multi(handle, (void **)event_hdr, num, &ret)) return ret; - buffer_index_from_buf(buf_idx, buf_hdr, num); + event_index_from_hdr(event_idx, event_hdr, num); LOCK(queue); num_enq = ring_st_enq_multi(ring_st, queue->s.ring_data, - queue->s.ring_mask, buf_idx, num); + queue->s.ring_mask, event_idx, num); if (odp_unlikely(num_enq == 0)) { UNLOCK(queue); @@ -927,7 +928,7 @@ int _odp_sched_queue_deq(uint32_t queue_index, odp_event_t ev[], int max_num, int num_deq, status; ring_st_t *ring_st; queue_entry_t *queue = qentry_from_index(queue_index); - uint32_t buf_idx[max_num]; + uint32_t event_idx[max_num]; ring_st = &queue->s.ring_st; @@ -948,7 +949,7 @@ int _odp_sched_queue_deq(uint32_t queue_index, odp_event_t ev[], int max_num, } num_deq = ring_st_deq_multi(ring_st, queue->s.ring_data, - queue->s.ring_mask, buf_idx, max_num); + queue->s.ring_mask, event_idx, max_num); if (num_deq == 0) { /* Already empty queue */ @@ -962,22 +963,22 @@ int _odp_sched_queue_deq(uint32_t queue_index, odp_event_t ev[], int max_num, UNLOCK(queue); - buffer_index_to_buf((odp_buffer_hdr_t **)ev, buf_idx, num_deq); + event_index_to_hdr((_odp_event_hdr_t **)ev, event_idx, num_deq); return num_deq; } static int sched_queue_enq_multi(odp_queue_t handle, - odp_buffer_hdr_t *buf_hdr[], int num) + _odp_event_hdr_t *event_hdr[], int num) { - return _sched_queue_enq_multi(handle, buf_hdr, num); + return _sched_queue_enq_multi(handle, event_hdr, num); } -static int sched_queue_enq(odp_queue_t handle, odp_buffer_hdr_t *buf_hdr) +static int sched_queue_enq(odp_queue_t handle, _odp_event_hdr_t *event_hdr) { int ret; - ret = _sched_queue_enq_multi(handle, &buf_hdr, 1); + ret = _sched_queue_enq_multi(handle, &event_hdr, 1); if (ret == 1) return 0; @@ -1157,11 +1158,11 @@ static void queue_set_enq_deq_func(odp_queue_t handle, } static int queue_orig_multi(odp_queue_t handle, - odp_buffer_hdr_t **buf_hdr, int num) + _odp_event_hdr_t **event_hdr, int num) { queue_entry_t *queue = qentry_from_handle(handle); - return queue->s.orig_dequeue_multi(handle, buf_hdr, num); + return queue->s.orig_dequeue_multi(handle, event_hdr, num); } static int queue_api_enq_multi(odp_queue_t handle, @@ -1176,7 +1177,7 @@ static int queue_api_enq_multi(odp_queue_t handle, num = QUEUE_MULTI_MAX; return queue->s.enqueue_multi(handle, - (odp_buffer_hdr_t **)(uintptr_t)ev, num); + (_odp_event_hdr_t **)(uintptr_t)ev, num); } static void queue_timer_add(odp_queue_t handle) @@ -1198,7 +1199,7 @@ static int queue_api_enq(odp_queue_t handle, odp_event_t ev) queue_entry_t *queue = qentry_from_handle(handle); return queue->s.enqueue(handle, - (odp_buffer_hdr_t *)(uintptr_t)ev); + (_odp_event_hdr_t *)(uintptr_t)ev); } static int queue_api_deq_multi(odp_queue_t handle, odp_event_t ev[], int num) @@ -1209,7 +1210,7 @@ static int queue_api_deq_multi(odp_queue_t handle, odp_event_t ev[], int num) if (num > QUEUE_MULTI_MAX) num = QUEUE_MULTI_MAX; - ret = queue->s.dequeue_multi(handle, (odp_buffer_hdr_t **)ev, num); + ret = queue->s.dequeue_multi(handle, (_odp_event_hdr_t **)ev, num); if (odp_global_rw->inline_timers && odp_atomic_load_u64(&queue->s.num_timers)) diff --git a/platform/linux-generic/odp_queue_lf.c b/platform/linux-generic/odp_queue_lf.c index f1b265970..f5c3830e8 100644 --- a/platform/linux-generic/odp_queue_lf.c +++ b/platform/linux-generic/odp_queue_lf.c @@ -1,4 +1,5 @@ /* Copyright (c) 2018-2018, Linaro Limited + * Copyright (c) 2021, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -8,12 +9,14 @@ #include <odp/api/atomic.h> #include <odp/api/plat/atomic_inlines.h> #include <odp/api/shared_memory.h> + +#include <odp_debug_internal.h> +#include <odp_event_internal.h> #include <odp_queue_basic_internal.h> + #include <string.h> #include <stdio.h> -#include <odp_debug_internal.h> - #define RING_LF_SIZE 32 #define QUEUE_LF_NUM 128 #define ENQ_RETRIES (RING_LF_SIZE / 4) @@ -112,7 +115,7 @@ static inline int next_idx(int idx) return next; } -static int queue_lf_enq(odp_queue_t handle, odp_buffer_hdr_t *buf_hdr) +static int queue_lf_enq(odp_queue_t handle, _odp_event_hdr_t *event_hdr) { queue_entry_t *queue; queue_lf_t *queue_lf; @@ -125,7 +128,7 @@ static int queue_lf_enq(odp_queue_t handle, odp_buffer_hdr_t *buf_hdr) queue = qentry_from_handle(handle); queue_lf = queue->s.queue_lf; - new_val.s.ptr = (uintptr_t)buf_hdr; + new_val.s.ptr = (uintptr_t)event_hdr; new_val.s.counter = odp_atomic_fetch_inc_u64(&queue_lf->enq_counter); idx = 0; @@ -159,18 +162,18 @@ static int queue_lf_enq(odp_queue_t handle, odp_buffer_hdr_t *buf_hdr) return -1; } -static int queue_lf_enq_multi(odp_queue_t handle, odp_buffer_hdr_t **buf_hdr, +static int queue_lf_enq_multi(odp_queue_t handle, _odp_event_hdr_t **event_hdr, int num) { (void)num; - if (queue_lf_enq(handle, buf_hdr[0]) == 0) + if (queue_lf_enq(handle, event_hdr[0]) == 0) return 1; return 0; } -static odp_buffer_hdr_t *queue_lf_deq(odp_queue_t handle) +static _odp_event_hdr_t *queue_lf_deq(odp_queue_t handle) { queue_entry_t *queue; queue_lf_t *queue_lf; @@ -179,7 +182,7 @@ static odp_buffer_hdr_t *queue_lf_deq(odp_queue_t handle) ring_lf_node_t node_val, old_val, new_val; ring_lf_node_t *node, *old; uint64_t lowest, counter; - odp_buffer_hdr_t *buf_hdr; + _odp_event_hdr_t *event_hdr; queue = qentry_from_handle(handle); queue_lf = queue->s.queue_lf; @@ -226,21 +229,21 @@ static odp_buffer_hdr_t *queue_lf_deq(odp_queue_t handle) } } - buf_hdr = (void *)(uintptr_t)old_val.s.ptr; + event_hdr = (void *)(uintptr_t)old_val.s.ptr; /* Try to remove data */ if (lockfree_cas_acq_rel_u128(&old->u128, old_val.u128, new_val.u128)) - return buf_hdr; + return event_hdr; } return NULL; } -static int queue_lf_deq_multi(odp_queue_t handle, odp_buffer_hdr_t **buf_hdr, +static int queue_lf_deq_multi(odp_queue_t handle, _odp_event_hdr_t **event_hdr, int num) { - odp_buffer_hdr_t *buf; + _odp_event_hdr_t *buf; (void)num; @@ -249,7 +252,7 @@ static int queue_lf_deq_multi(odp_queue_t handle, odp_buffer_hdr_t **buf_hdr, if (buf == NULL) return 0; - buf_hdr[0] = buf; + event_hdr[0] = buf; return 1; } diff --git a/platform/linux-generic/odp_queue_scalable.c b/platform/linux-generic/odp_queue_scalable.c index 916cb6739..bf5489516 100644 --- a/platform/linux-generic/odp_queue_scalable.c +++ b/platform/linux-generic/odp_queue_scalable.c @@ -20,6 +20,7 @@ #include <odp_config_internal.h> #include <odp_debug_internal.h> +#include <odp_event_internal.h> #include <odp_packet_io_internal.h> #include <odp_pool_internal.h> #include <odp_queue_scalable_internal.h> @@ -48,11 +49,11 @@ typedef struct queue_table_t { static queue_table_t *queue_tbl; static _odp_ishm_pool_t *queue_shm_pool; -static int _queue_enq(odp_queue_t handle, odp_buffer_hdr_t *buf_hdr); -static odp_buffer_hdr_t *_queue_deq(odp_queue_t handle); -static int _queue_enq_multi(odp_queue_t handle, odp_buffer_hdr_t *buf_hdr[], +static int _queue_enq(odp_queue_t handle, _odp_event_hdr_t *event_hdr); +static _odp_event_hdr_t *_queue_deq(odp_queue_t handle); +static int _queue_enq_multi(odp_queue_t handle, _odp_event_hdr_t *event_hdr[], int num); -static int _queue_deq_multi(odp_queue_t handle, odp_buffer_hdr_t *buf_hdr[], +static int _queue_deq_multi(odp_queue_t handle, _odp_event_hdr_t *event_hdr[], int num); static queue_entry_t *get_qentry(uint32_t queue_id) @@ -99,7 +100,7 @@ static int queue_init(queue_entry_t *queue, const char *name, ringidx_t ring_idx; sched_elem_t *sched_elem; uint32_t ring_size; - odp_buffer_hdr_t **ring; + _odp_event_hdr_t **ring; uint32_t size; sched_elem = &queue->s.sched_elem; @@ -109,8 +110,8 @@ static int queue_init(queue_entry_t *queue, const char *name, queue->s.name[ODP_QUEUE_NAME_LEN - 1] = 0; memcpy(&queue->s.param, param, sizeof(odp_queue_param_t)); - size = ring_size * sizeof(odp_buffer_hdr_t *); - ring = (odp_buffer_hdr_t **)shm_pool_alloc_align(queue_shm_pool, size); + size = ring_size * sizeof(_odp_event_hdr_t *); + ring = (_odp_event_hdr_t **)shm_pool_alloc_align(queue_shm_pool, size); if (NULL == ring) return -1; @@ -207,13 +208,13 @@ static int queue_init_global(void) pool_size = sizeof(queue_table_t); /* Add storage required for queues */ pool_size += (CONFIG_SCAL_QUEUE_SIZE * - sizeof(odp_buffer_hdr_t *)) * CONFIG_MAX_QUEUES; + sizeof(_odp_event_hdr_t *)) * CONFIG_MAX_QUEUES; /* Add the reorder window size */ pool_size += sizeof(reorder_window_t) * CONFIG_MAX_QUEUES; /* Choose min_alloc and max_alloc such that buddy allocator is selected. */ min_alloc = 0; - max_alloc = CONFIG_SCAL_QUEUE_SIZE * sizeof(odp_buffer_hdr_t *); + max_alloc = CONFIG_SCAL_QUEUE_SIZE * sizeof(_odp_event_hdr_t *); queue_shm_pool = _odp_ishm_pool_create("queue_shm_pool", pool_size, min_alloc, max_alloc, 0); @@ -514,7 +515,7 @@ static odp_queue_t queue_lookup(const char *name) #ifndef CONFIG_QSCHST_LOCK static inline int _odp_queue_enq(sched_elem_t *q, - odp_buffer_hdr_t *buf_hdr[], + _odp_event_hdr_t *event_hdr[], int num) { ringidx_t old_read; @@ -522,7 +523,7 @@ static inline int _odp_queue_enq(sched_elem_t *q, ringidx_t new_write; int actual; uint32_t mask; - odp_buffer_hdr_t **ring; + _odp_event_hdr_t **ring; mask = q->prod_mask; ring = q->prod_ring; @@ -552,7 +553,7 @@ static inline int _odp_queue_enq(sched_elem_t *q, #endif /* Store our event(s) in the ring */ do { - ring[old_write & mask] = *buf_hdr++; + ring[old_write & mask] = *event_hdr++; } while (++old_write != new_write); old_write -= actual; @@ -580,7 +581,7 @@ static inline int _odp_queue_enq(sched_elem_t *q, #endif int _odp_queue_enq_sp(sched_elem_t *q, - odp_buffer_hdr_t *buf_hdr[], + _odp_event_hdr_t *event_hdr[], int num) { ringidx_t old_read; @@ -588,7 +589,7 @@ int _odp_queue_enq_sp(sched_elem_t *q, ringidx_t new_write; int actual; uint32_t mask; - odp_buffer_hdr_t **ring; + _odp_event_hdr_t **ring; mask = q->prod_mask; ring = q->prod_ring; @@ -606,7 +607,7 @@ int _odp_queue_enq_sp(sched_elem_t *q, /* Store our event(s) in the ring */ do { - ring[old_write & mask] = *buf_hdr++; + ring[old_write & mask] = *event_hdr++; } while (++old_write != new_write); old_write -= actual; @@ -626,7 +627,7 @@ int _odp_queue_enq_sp(sched_elem_t *q, return actual; } -static int _queue_enq_multi(odp_queue_t handle, odp_buffer_hdr_t *buf_hdr[], +static int _queue_enq_multi(odp_queue_t handle, _odp_event_hdr_t *event_hdr[], int num) { int actual; @@ -637,15 +638,15 @@ static int _queue_enq_multi(odp_queue_t handle, odp_buffer_hdr_t *buf_hdr[], ts = _odp_sched_ts; if (ts && odp_unlikely(ts->out_of_order) && (queue->s.param.order == ODP_QUEUE_ORDER_KEEP)) { - actual = _odp_rctx_save(queue, buf_hdr, num); + actual = _odp_rctx_save(queue, event_hdr, num); return actual; } #ifdef CONFIG_QSCHST_LOCK LOCK(&queue->s.sched_elem.qschlock); - actual = _odp_queue_enq_sp(&queue->s.sched_elem, buf_hdr, num); + actual = _odp_queue_enq_sp(&queue->s.sched_elem, event_hdr, num); #else - actual = _odp_queue_enq(&queue->s.sched_elem, buf_hdr, num); + actual = _odp_queue_enq(&queue->s.sched_elem, event_hdr, num); #endif if (odp_likely(queue->s.sched_elem.schedq != NULL && actual != 0)) { @@ -663,14 +664,14 @@ static int _queue_enq_multi(odp_queue_t handle, odp_buffer_hdr_t *buf_hdr[], return actual; } -static int _queue_enq(odp_queue_t handle, odp_buffer_hdr_t *buf_hdr) +static int _queue_enq(odp_queue_t handle, _odp_event_hdr_t *event_hdr) { - return odp_likely(_queue_enq_multi(handle, &buf_hdr, 1) == 1) ? 0 : -1; + return odp_likely(_queue_enq_multi(handle, &event_hdr, 1) == 1) ? 0 : -1; } static int queue_enq_multi(odp_queue_t handle, const odp_event_t ev[], int num) { - odp_buffer_hdr_t *buf_hdr[QUEUE_MULTI_MAX]; + _odp_event_hdr_t *event_hdr[QUEUE_MULTI_MAX]; queue_entry_t *queue; int i; @@ -680,20 +681,20 @@ static int queue_enq_multi(odp_queue_t handle, const odp_event_t ev[], int num) queue = _odp_qentry_from_ext(handle); for (i = 0; i < num; i++) - buf_hdr[i] = buf_hdl_to_hdr(odp_buffer_from_event(ev[i])); + event_hdr[i] = _odp_event_hdr(ev[i]); - return queue->s.enqueue_multi(handle, buf_hdr, num); + return queue->s.enqueue_multi(handle, event_hdr, num); } static int queue_enq(odp_queue_t handle, odp_event_t ev) { - odp_buffer_hdr_t *buf_hdr; + _odp_event_hdr_t *event_hdr; queue_entry_t *queue; queue = _odp_qentry_from_ext(handle); - buf_hdr = buf_hdl_to_hdr(odp_buffer_from_event(ev)); + event_hdr = _odp_event_hdr(ev); - return queue->s.enqueue(handle, buf_hdr); + return queue->s.enqueue(handle, event_hdr); } /* Single-consumer dequeue. */ @@ -704,7 +705,7 @@ int _odp_queue_deq_sc(sched_elem_t *q, odp_event_t *evp, int num) ringidx_t old_write; ringidx_t new_read; uint32_t mask; - odp_buffer_hdr_t **ring; + _odp_event_hdr_t **ring; /* Load consumer ring state (read & write index). */ old_read = q->cons_read; @@ -724,7 +725,7 @@ int _odp_queue_deq_sc(sched_elem_t *q, odp_event_t *evp, int num) mask = q->cons_mask; ring = q->cons_ring; do { - *evp++ = odp_buffer_to_event(buf_from_buf_hdr(ring[old_read & mask])); + *evp++ = _odp_event_from_hdr(ring[old_read & mask]); } while (++old_read != new_read); /* Signal producers that empty slots are available @@ -739,15 +740,15 @@ int _odp_queue_deq_sc(sched_elem_t *q, odp_event_t *evp, int num) return actual; } -int _odp_queue_deq(sched_elem_t *q, odp_buffer_hdr_t *buf_hdr[], int num) +int _odp_queue_deq(sched_elem_t *q, _odp_event_hdr_t *event_hdr[], int num) { int actual; ringidx_t old_read; ringidx_t old_write; ringidx_t new_read; uint32_t mask; - odp_buffer_hdr_t **ring; - odp_buffer_hdr_t **p_buf_hdr; + _odp_event_hdr_t **ring; + _odp_event_hdr_t **p_event_hdr; mask = q->cons_mask; ring = q->cons_ring; @@ -778,9 +779,9 @@ int _odp_queue_deq(sched_elem_t *q, odp_buffer_hdr_t *buf_hdr[], int num) #ifdef CONFIG_SPLIT_PRODCONS __builtin_prefetch(&q->prod_read, 0, 0); #endif - p_buf_hdr = buf_hdr; + p_event_hdr = event_hdr; do { - *p_buf_hdr++ = ring[old_read & mask]; + *p_event_hdr++ = ring[old_read & mask]; } while (++old_read != new_read); old_read -= actual; @@ -809,7 +810,7 @@ int _odp_queue_deq(sched_elem_t *q, odp_buffer_hdr_t *buf_hdr[], int num) int _odp_queue_deq_mc(sched_elem_t *q, odp_event_t *evp, int num) { int ret, evt_idx; - odp_buffer_hdr_t *hdr_tbl[QUEUE_MULTI_MAX]; + _odp_event_hdr_t *hdr_tbl[QUEUE_MULTI_MAX]; if (num > QUEUE_MULTI_MAX) num = QUEUE_MULTI_MAX; @@ -817,13 +818,13 @@ int _odp_queue_deq_mc(sched_elem_t *q, odp_event_t *evp, int num) ret = _odp_queue_deq(q, hdr_tbl, num); if (odp_likely(ret != 0)) { for (evt_idx = 0; evt_idx < num; evt_idx++) - evp[evt_idx] = odp_buffer_to_event(buf_from_buf_hdr(hdr_tbl[evt_idx])); + evp[evt_idx] = _odp_event_from_hdr(hdr_tbl[evt_idx]); } return ret; } -static int _queue_deq_multi(odp_queue_t handle, odp_buffer_hdr_t *buf_hdr[], +static int _queue_deq_multi(odp_queue_t handle, _odp_event_hdr_t *event_hdr[], int num) { sched_elem_t *q; @@ -831,19 +832,19 @@ static int _queue_deq_multi(odp_queue_t handle, odp_buffer_hdr_t *buf_hdr[], queue = qentry_from_int(handle); q = &queue->s.sched_elem; - return _odp_queue_deq(q, buf_hdr, num); + return _odp_queue_deq(q, event_hdr, num); } -static odp_buffer_hdr_t *_queue_deq(odp_queue_t handle) +static _odp_event_hdr_t *_queue_deq(odp_queue_t handle) { sched_elem_t *q; - odp_buffer_hdr_t *buf_hdr; + _odp_event_hdr_t *event_hdr; queue_entry_t *queue; queue = qentry_from_int(handle); q = &queue->s.sched_elem; - if (_odp_queue_deq(q, &buf_hdr, 1) == 1) - return buf_hdr; + if (_odp_queue_deq(q, &event_hdr, 1) == 1) + return event_hdr; else return NULL; } @@ -858,7 +859,7 @@ static int queue_deq_multi(odp_queue_t handle, odp_event_t ev[], int num) queue = _odp_qentry_from_ext(handle); - ret = queue->s.dequeue_multi(handle, (odp_buffer_hdr_t **)ev, num); + ret = queue->s.dequeue_multi(handle, (_odp_event_hdr_t **)ev, num); if (odp_global_rw->inline_timers && odp_atomic_load_u64(&queue->s.num_timers)) @@ -1121,10 +1122,10 @@ static void queue_set_enq_deq_func(odp_queue_t handle, } static int queue_orig_multi(odp_queue_t handle, - odp_buffer_hdr_t **buf_hdr, int num) + _odp_event_hdr_t **event_hdr, int num) { return qentry_from_int(handle)->s.orig_dequeue_multi(handle, - buf_hdr, num); + event_hdr, num); } static void queue_timer_add(odp_queue_t handle) diff --git a/platform/linux-generic/odp_queue_spsc.c b/platform/linux-generic/odp_queue_spsc.c index 92f16e657..c1b02d457 100644 --- a/platform/linux-generic/odp_queue_spsc.c +++ b/platform/linux-generic/odp_queue_spsc.c @@ -1,38 +1,40 @@ /* Copyright (c) 2018, Linaro Limited + * Copyright (c) 2021, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause */ -#include <string.h> -#include <stdio.h> -#include <odp_queue_basic_internal.h> +#include <odp_debug_internal.h> +#include <odp_event_internal.h> #include <odp_pool_internal.h> +#include <odp_queue_basic_internal.h> -#include <odp_debug_internal.h> +#include <string.h> +#include <stdio.h> -static inline void buffer_index_from_buf(uint32_t buffer_index[], - odp_buffer_hdr_t *buf_hdr[], int num) +static inline void event_index_from_hdr(uint32_t event_index[], + _odp_event_hdr_t *event_hdr[], int num) { int i; for (i = 0; i < num; i++) - buffer_index[i] = buf_hdr[i]->index.u32; + event_index[i] = event_hdr[i]->index.u32; } -static inline void buffer_index_to_buf(odp_buffer_hdr_t *buf_hdr[], - uint32_t buffer_index[], int num) +static inline void event_index_to_hdr(_odp_event_hdr_t *event_hdr[], + uint32_t event_index[], int num) { int i; for (i = 0; i < num; i++) { - buf_hdr[i] = buf_hdr_from_index_u32(buffer_index[i]); - odp_prefetch(buf_hdr[i]); + event_hdr[i] = _odp_event_hdr_from_index_u32(event_index[i]); + odp_prefetch(event_hdr[i]); } } static inline int spsc_enq_multi(odp_queue_t handle, - odp_buffer_hdr_t *buf_hdr[], int num) + _odp_event_hdr_t *event_hdr[], int num) { queue_entry_t *queue; ring_spsc_t *ring_spsc; @@ -41,7 +43,7 @@ static inline int spsc_enq_multi(odp_queue_t handle, queue = qentry_from_handle(handle); ring_spsc = &queue->s.ring_spsc; - buffer_index_from_buf(buf_idx, buf_hdr, num); + event_index_from_hdr(buf_idx, event_hdr, num); if (odp_unlikely(queue->s.status < QUEUE_STATUS_READY)) { ODP_ERR("Bad queue status\n"); @@ -53,7 +55,7 @@ static inline int spsc_enq_multi(odp_queue_t handle, } static inline int spsc_deq_multi(odp_queue_t handle, - odp_buffer_hdr_t *buf_hdr[], int num) + _odp_event_hdr_t *event_hdr[], int num) { queue_entry_t *queue; int num_deq; @@ -74,22 +76,22 @@ static inline int spsc_deq_multi(odp_queue_t handle, if (num_deq == 0) return 0; - buffer_index_to_buf(buf_hdr, buf_idx, num_deq); + event_index_to_hdr(event_hdr, buf_idx, num_deq); return num_deq; } -static int queue_spsc_enq_multi(odp_queue_t handle, odp_buffer_hdr_t *buf_hdr[], +static int queue_spsc_enq_multi(odp_queue_t handle, _odp_event_hdr_t *event_hdr[], int num) { - return spsc_enq_multi(handle, buf_hdr, num); + return spsc_enq_multi(handle, event_hdr, num); } -static int queue_spsc_enq(odp_queue_t handle, odp_buffer_hdr_t *buf_hdr) +static int queue_spsc_enq(odp_queue_t handle, _odp_event_hdr_t *event_hdr) { int ret; - ret = spsc_enq_multi(handle, &buf_hdr, 1); + ret = spsc_enq_multi(handle, &event_hdr, 1); if (ret == 1) return 0; @@ -97,21 +99,21 @@ static int queue_spsc_enq(odp_queue_t handle, odp_buffer_hdr_t *buf_hdr) return -1; } -static int queue_spsc_deq_multi(odp_queue_t handle, odp_buffer_hdr_t *buf_hdr[], +static int queue_spsc_deq_multi(odp_queue_t handle, _odp_event_hdr_t *event_hdr[], int num) { - return spsc_deq_multi(handle, buf_hdr, num); + return spsc_deq_multi(handle, event_hdr, num); } -static odp_buffer_hdr_t *queue_spsc_deq(odp_queue_t handle) +static _odp_event_hdr_t *queue_spsc_deq(odp_queue_t handle) { - odp_buffer_hdr_t *buf_hdr = NULL; + _odp_event_hdr_t *event_hdr = NULL; int ret; - ret = spsc_deq_multi(handle, &buf_hdr, 1); + ret = spsc_deq_multi(handle, &event_hdr, 1); if (ret == 1) - return buf_hdr; + return event_hdr; else return NULL; } diff --git a/platform/linux-generic/odp_random.c b/platform/linux-generic/odp_random.c index acae9663d..3060e8ed9 100644 --- a/platform/linux-generic/odp_random.c +++ b/platform/linux-generic/odp_random.c @@ -12,19 +12,38 @@ #include <odp_init_internal.h> #include <odp_random_std_internal.h> #include <odp_random_openssl_internal.h> +#include <odp_random.h> odp_random_kind_t odp_random_max_kind(void) { - if (_ODP_OPENSSL) - return _odp_random_openssl_max_kind(); - return _odp_random_std_max_kind(); + odp_random_kind_t kind, max_kind = ODP_RANDOM_BASIC; + + if (_ODP_OPENSSL_RAND) + max_kind = ODP_RANDOM_CRYPTO; + + kind = _odp_random_max_kind(); + if (kind > max_kind) + max_kind = kind; + + return max_kind; } int32_t odp_random_data(uint8_t *buf, uint32_t len, odp_random_kind_t kind) { - if (_ODP_OPENSSL) - return _odp_random_openssl_data(buf, len, kind); - return _odp_random_std_data(buf, len, kind); + switch (kind) { + case ODP_RANDOM_BASIC: + if (_ODP_OPENSSL_RAND) + return _odp_random_openssl_data(buf, len); + return _odp_random_std_data(buf, len); + case ODP_RANDOM_CRYPTO: + if (_ODP_OPENSSL_RAND) + return _odp_random_openssl_data(buf, len); + return _odp_random_crypto_data(buf, len); + case ODP_RANDOM_TRUE: + return _odp_random_true_data(buf, len); + } + + return -1; } int32_t odp_random_test_data(uint8_t *buf, uint32_t len, uint64_t *seed) @@ -34,14 +53,14 @@ int32_t odp_random_test_data(uint8_t *buf, uint32_t len, uint64_t *seed) int _odp_random_init_local(void) { - if (_ODP_OPENSSL) + if (_ODP_OPENSSL_RAND) return _odp_random_openssl_init_local(); return _odp_random_std_init_local(); } int _odp_random_term_local(void) { - if (_ODP_OPENSSL) + if (_ODP_OPENSSL_RAND) return _odp_random_openssl_term_local(); return _odp_random_std_term_local(); } diff --git a/platform/linux-generic/odp_random_openssl.c b/platform/linux-generic/odp_random_openssl.c index a74f99bd4..fdc40871b 100644 --- a/platform/linux-generic/odp_random_openssl.c +++ b/platform/linux-generic/odp_random_openssl.c @@ -7,49 +7,28 @@ #include <odp_posix_extensions.h> #include <stdint.h> -#include <odp/api/random.h> #include <odp/autoheader_internal.h> #include <odp_init_internal.h> #include <odp_random_openssl_internal.h> -#if _ODP_OPENSSL +#if _ODP_OPENSSL_RAND #include <openssl/rand.h> -odp_random_kind_t _odp_random_openssl_max_kind(void) -{ - return ODP_RANDOM_CRYPTO; -} - -int32_t _odp_random_openssl_data(uint8_t *buf, uint32_t len, - odp_random_kind_t kind) +int32_t _odp_random_openssl_data(uint8_t *buf, uint32_t len) { int rc; - switch (kind) { - case ODP_RANDOM_BASIC: - case ODP_RANDOM_CRYPTO: - rc = RAND_bytes(buf, len); - return (1 == rc) ? (int)len /*success*/: -1 /*failure*/; - - case ODP_RANDOM_TRUE: - default: - return -1; - } + rc = RAND_bytes(buf, len); + return (1 == rc) ? (int)len /*success*/: -1 /*failure*/; } #else /* Dummy functions for building without OpenSSL support */ -odp_random_kind_t _odp_random_openssl_max_kind(void) -{ - return ODP_RANDOM_BASIC; -} - int32_t _odp_random_openssl_data(uint8_t *buf ODP_UNUSED, - uint32_t len ODP_UNUSED, - odp_random_kind_t kind ODP_UNUSED) + uint32_t len ODP_UNUSED) { return -1; } -#endif /* _ODP_OPENSSL */ +#endif /* _ODP_OPENSSL_RAND */ int _odp_random_openssl_init_local(void) { diff --git a/platform/linux-generic/odp_random_std.c b/platform/linux-generic/odp_random_std.c index 3b7187af1..3afd049f4 100644 --- a/platform/linux-generic/odp_random_std.c +++ b/platform/linux-generic/odp_random_std.c @@ -7,7 +7,6 @@ #include <odp_posix_extensions.h> #include <stdint.h> #include <stdlib.h> -#include <odp/api/random.h> #include <odp/api/byteorder.h> #include <odp/api/cpu.h> #include <odp/api/debug.h> @@ -20,11 +19,6 @@ ODP_STATIC_ASSERT(RAND_MAX >= UINT16_MAX, "RAND_MAX too small"); ODP_STATIC_ASSERT((RAND_MAX & (RAND_MAX + 1ULL)) == 0, "RAND_MAX not power of two - 1"); -odp_random_kind_t _odp_random_std_max_kind(void) -{ - return ODP_RANDOM_BASIC; -} - static int32_t _random_data(uint8_t *buf, uint32_t len, uint32_t *seed) { union { @@ -57,11 +51,8 @@ int32_t _odp_random_std_test_data(uint8_t *buf, uint32_t len, uint64_t *seed) static __thread uint32_t this_seed; -int32_t _odp_random_std_data(uint8_t *buf, uint32_t len, odp_random_kind_t kind) +int32_t _odp_random_std_data(uint8_t *buf, uint32_t len) { - if (kind != ODP_RANDOM_BASIC) - return -1; - return _random_data(buf, len, &this_seed); } diff --git a/platform/linux-generic/odp_schedule_basic.c b/platform/linux-generic/odp_schedule_basic.c index f86914722..0761def57 100644 --- a/platform/linux-generic/odp_schedule_basic.c +++ b/platform/linux-generic/odp_schedule_basic.c @@ -36,6 +36,7 @@ #include <odp_libconfig_internal.h> #include <odp/api/plat/queue_inlines.h> #include <odp_global_data.h> +#include <odp_event_internal.h> #include <string.h> @@ -122,7 +123,7 @@ ODP_STATIC_ASSERT((8 * sizeof(prio_q_mask_t)) >= MAX_SPREAD, /* Storage for stashed enqueue operation arguments */ typedef struct { - odp_buffer_hdr_t *buf_hdr[QUEUE_MULTI_MAX]; + _odp_event_hdr_t *event_hdr[QUEUE_MULTI_MAX]; odp_queue_t queue; int num; } ordered_stash_t; @@ -902,15 +903,15 @@ static inline void ordered_stash_release(void) for (i = 0; i < sched_local.ordered.stash_num; i++) { odp_queue_t queue; - odp_buffer_hdr_t **buf_hdr; + _odp_event_hdr_t **event_hdr; int num, num_enq; queue = sched_local.ordered.stash[i].queue; - buf_hdr = sched_local.ordered.stash[i].buf_hdr; + event_hdr = sched_local.ordered.stash[i].event_hdr; num = sched_local.ordered.stash[i].num; num_enq = odp_queue_enq_multi(queue, - (odp_event_t *)buf_hdr, num); + (odp_event_t *)event_hdr, num); /* Drop packets that were not enqueued */ if (odp_unlikely(num_enq < num)) { @@ -918,7 +919,7 @@ static inline void ordered_stash_release(void) num_enq = 0; ODP_DBG("Dropped %i packets\n", num - num_enq); - _odp_buffer_free_multi(&buf_hdr[num_enq], num - num_enq); + _odp_event_free_multi(&event_hdr[num_enq], num - num_enq); } } sched_local.ordered.stash_num = 0; @@ -1076,7 +1077,7 @@ static inline int copy_from_stash(odp_event_t out_ev[], unsigned int max) return i; } -static int schedule_ord_enq_multi(odp_queue_t dst_queue, void *buf_hdr[], +static int schedule_ord_enq_multi(odp_queue_t dst_queue, void *event_hdr[], int num, int *ret) { int i; @@ -1123,7 +1124,7 @@ static int schedule_ord_enq_multi(odp_queue_t dst_queue, void *buf_hdr[], sched_local.ordered.stash[stash_num].queue = dst_queue; sched_local.ordered.stash[stash_num].num = num; for (i = 0; i < num; i++) - sched_local.ordered.stash[stash_num].buf_hdr[i] = buf_hdr[i]; + sched_local.ordered.stash[stash_num].event_hdr[i] = event_hdr[i]; sched_local.ordered.stash_num++; @@ -1140,12 +1141,12 @@ static inline int poll_pktin(uint32_t qi, int direct_recv, odp_event_t ev_tbl[], int max_num) { int pktio_index, pktin_index, num, num_pktin; - odp_buffer_hdr_t **hdr_tbl; + _odp_event_hdr_t **hdr_tbl; int ret; void *q_int; - odp_buffer_hdr_t *b_hdr[CONFIG_BURST_SIZE]; + _odp_event_hdr_t *b_hdr[CONFIG_BURST_SIZE]; - hdr_tbl = (odp_buffer_hdr_t **)ev_tbl; + hdr_tbl = (_odp_event_hdr_t **)ev_tbl; if (!direct_recv) { hdr_tbl = b_hdr; @@ -1194,7 +1195,7 @@ static inline int poll_pktin(uint32_t qi, int direct_recv, num_enq = 0; ODP_DBG("Dropped %i packets\n", num - num_enq); - _odp_buffer_free_multi(&b_hdr[num_enq], num - num_enq); + _odp_event_free_multi(&b_hdr[num_enq], num - num_enq); } return ret; @@ -1988,6 +1989,7 @@ static void schedule_print(void) ring_u32_t *ring; odp_schedule_capability_t capa; int num_spread = sched->config.num_spread; + const int col_width = 24; (void)schedule_capability(&capa); @@ -2039,14 +2041,14 @@ static void schedule_print(void) } } - ODP_PRINT("\n Number of threads:\n"); - ODP_PRINT(" spread\n"); + ODP_PRINT("\n Number of threads per schedule group:\n"); + ODP_PRINT(" name spread\n"); for (grp = 0; grp < NUM_SCHED_GRPS; grp++) { if (sched->sched_grp[grp].allocated == 0) continue; - ODP_PRINT(" group %i:", grp); + ODP_PRINT(" group %i: %-*s", grp, col_width, sched->sched_grp[grp].name); for (spr = 0; spr < num_spread; spr++) ODP_PRINT(" %u", sched->sched_grp[grp].spread_thrs[spr]); diff --git a/platform/linux-generic/odp_schedule_scalable.c b/platform/linux-generic/odp_schedule_scalable.c index 10f456eed..5343c2834 100644 --- a/platform/linux-generic/odp_schedule_scalable.c +++ b/platform/linux-generic/odp_schedule_scalable.c @@ -30,6 +30,7 @@ #include <odp_queue_scalable_internal.h> #include <odp_schedule_if.h> #include <odp_bitset.h> +#include <odp_event_internal.h> #include <odp_packet_io_internal.h> #include <odp_timer_internal.h> @@ -828,12 +829,11 @@ events_dequeued: } if (num_rx > num) { /* Events remain, enqueue them */ - odp_buffer_hdr_t *bufs[QUEUE_MULTI_MAX]; + _odp_event_hdr_t *events[QUEUE_MULTI_MAX]; for (i = num; i < num_rx; i++) - bufs[i] = - (odp_buffer_hdr_t *)(void *)rx_evts[i]; - i = _odp_queue_enq_sp(elem, &bufs[num], num_rx - num); + events[i] = _odp_event_hdr(rx_evts[i]); + i = _odp_queue_enq_sp(elem, &events[num], num_rx - num); /* Enqueue must succeed as the queue was empty */ ODP_ASSERT(i == num_rx - num); } @@ -2095,7 +2095,7 @@ static int sched_queue(uint32_t queue_index) return 0; } -static int ord_enq_multi(odp_queue_t handle, void *buf_hdr[], int num, +static int ord_enq_multi(odp_queue_t handle, void *event_hdr[], int num, int *ret) { @@ -2107,7 +2107,7 @@ static int ord_enq_multi(odp_queue_t handle, void *buf_hdr[], int num, queue = qentry_from_int(handle); if (ts && odp_unlikely(ts->out_of_order) && (queue->s.param.order == ODP_QUEUE_ORDER_KEEP)) { - actual = _odp_rctx_save(queue, (odp_buffer_hdr_t **)buf_hdr, num); + actual = _odp_rctx_save(queue, (_odp_event_hdr_t **)event_hdr, num); *ret = actual; return 1; } diff --git a/platform/linux-generic/odp_schedule_scalable_ordered.c b/platform/linux-generic/odp_schedule_scalable_ordered.c index 991be658e..5f1f8a405 100644 --- a/platform/linux-generic/odp_schedule_scalable_ordered.c +++ b/platform/linux-generic/odp_schedule_scalable_ordered.c @@ -9,9 +9,11 @@ #include <odp/api/shared_memory.h> #include <odp/api/cpu.h> #include <odp/api/plat/cpu_inlines.h> + +#include <odp_bitset.h> +#include <odp_event_internal.h> #include <odp_queue_scalable_internal.h> #include <odp_schedule_if.h> -#include <odp_bitset.h> #include <string.h> @@ -255,7 +257,7 @@ static void olock_release(const reorder_context_t *rctx) olock_unlock(rctx, rwin, i); } -static void blocking_enqueue(queue_entry_t *q, odp_buffer_hdr_t **evts, int num) +static void blocking_enqueue(queue_entry_t *q, _odp_event_hdr_t **evts, int num) { int actual; @@ -315,7 +317,7 @@ void _odp_rctx_release(reorder_context_t *rctx) /* Save destination queue and events in the reorder context for deferred * enqueue. */ -int _odp_rctx_save(queue_entry_t *queue, odp_buffer_hdr_t *buf_hdr[], int num) +int _odp_rctx_save(queue_entry_t *queue, _odp_event_hdr_t *event_hdr[], int num) { int i; sched_scalable_thread_state_t *ts; @@ -361,7 +363,7 @@ int _odp_rctx_save(queue_entry_t *queue, odp_buffer_hdr_t *buf_hdr[], int num) /* The last rctx (so far) */ cur->next_idx = first->idx; } - cur->events[cur->numevts] = buf_hdr[i]; + cur->events[cur->numevts] = event_hdr[i]; cur->destq[cur->numevts] = queue; cur->numevts++; } diff --git a/platform/linux-generic/odp_schedule_sp.c b/platform/linux-generic/odp_schedule_sp.c index 470075cea..f5aba1a88 100644 --- a/platform/linux-generic/odp_schedule_sp.c +++ b/platform/linux-generic/odp_schedule_sp.c @@ -20,10 +20,12 @@ #include <odp/api/plat/time_inlines.h> #include <odp/api/schedule.h> #include <odp/api/shared_memory.h> + #include <odp_schedule_if.h> #include <odp_debug_internal.h> #include <odp_align_internal.h> #include <odp_config_internal.h> +#include <odp_event_internal.h> #include <odp_ring_u32_internal.h> #include <odp_timer_internal.h> #include <odp_queue_basic_internal.h> @@ -600,7 +602,7 @@ static uint64_t schedule_wait_time(uint64_t ns) } static inline void enqueue_packets(odp_queue_t queue, - odp_buffer_hdr_t *hdr_tbl[], int num_pkt) + _odp_event_hdr_t *hdr_tbl[], int num_pkt) { int num_enq, num_drop; @@ -644,7 +646,7 @@ static int schedule_multi(odp_queue_t *from, uint64_t wait, cmd = sched_cmd(); if (cmd && cmd->s.type == CMD_PKTIO) { - odp_buffer_hdr_t *hdr_tbl[CONFIG_BURST_SIZE]; + _odp_event_hdr_t *hdr_tbl[CONFIG_BURST_SIZE]; int i; int num_pkt = 0; int max_num = CONFIG_BURST_SIZE; diff --git a/platform/linux-generic/odp_shared_memory.c b/platform/linux-generic/odp_shared_memory.c index 966850a07..424d32ccf 100644 --- a/platform/linux-generic/odp_shared_memory.c +++ b/platform/linux-generic/odp_shared_memory.c @@ -1,11 +1,12 @@ -/* Copyright (c) 2019, Nokia - * Copyright (c) 2013-2018, Linaro Limited +/* Copyright (c) 2013-2018, Linaro Limited + * Copyright (c) 2019-2021, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause */ #include <odp_config_internal.h> +#include <odp_debug_internal.h> #include <odp/api/debug.h> #include <odp/api/std_types.h> #include <odp/api/shared_memory.h> @@ -15,6 +16,10 @@ #include <odp_global_data.h> #include <string.h> +/* Supported ODP_SHM_* flags */ +#define SUPPORTED_SHM_FLAGS (ODP_SHM_SW_ONLY | ODP_SHM_PROC | ODP_SHM_SINGLE_VA | ODP_SHM_EXPORT | \ + ODP_SHM_HP | ODP_SHM_NO_HP) + static inline uint32_t from_handle(odp_shm_t shm) { return _odp_typeval(shm) - 1; @@ -47,6 +52,7 @@ int odp_shm_capability(odp_shm_capability_t *capa) capa->max_blocks = CONFIG_SHM_BLOCKS; capa->max_size = odp_global_ro.shm_max_size; capa->max_align = 0; + capa->flags = SUPPORTED_SHM_FLAGS; return 0; } @@ -56,6 +62,12 @@ odp_shm_t odp_shm_reserve(const char *name, uint64_t size, uint64_t align, { int block_index; uint32_t flgs = 0; /* internal ishm flags */ + uint32_t supported_flgs = SUPPORTED_SHM_FLAGS; + + if (flags & ~supported_flgs) { + ODP_ERR("Unsupported SHM flag\n"); + return ODP_SHM_INVALID; + } flgs = get_ishm_flags(flags); diff --git a/platform/linux-generic/odp_stash.c b/platform/linux-generic/odp_stash.c index 57cf747f1..9dbc8cc26 100644 --- a/platform/linux-generic/odp_stash.c +++ b/platform/linux-generic/odp_stash.c @@ -20,6 +20,8 @@ #include <odp_ring_u32_internal.h> #include <odp_ring_u64_internal.h> +ODP_STATIC_ASSERT(CONFIG_INTERNAL_STASHES < CONFIG_MAX_STASHES, "TOO_MANY_INTERNAL_STASHES"); + #define MAX_RING_SIZE (1024 * 1024) #define MIN_RING_SIZE 64 @@ -110,8 +112,8 @@ int odp_stash_capability(odp_stash_capability_t *capa, odp_stash_type_t type) (void)type; memset(capa, 0, sizeof(odp_stash_capability_t)); - capa->max_stashes_any_type = CONFIG_MAX_STASHES; - capa->max_stashes = CONFIG_MAX_STASHES; + capa->max_stashes_any_type = CONFIG_MAX_STASHES - CONFIG_INTERNAL_STASHES; + capa->max_stashes = CONFIG_MAX_STASHES - CONFIG_INTERNAL_STASHES; capa->max_num_obj = MAX_RING_SIZE; capa->max_obj_size = sizeof(uint64_t); diff --git a/platform/linux-generic/odp_system_info.c b/platform/linux-generic/odp_system_info.c index 778ea08cb..1e4f775e1 100644 --- a/platform/linux-generic/odp_system_info.c +++ b/platform/linux-generic/odp_system_info.c @@ -24,9 +24,9 @@ #include <odp_config_internal.h> #include <odp/api/align.h> #include <odp/api/cpu.h> + #include <errno.h> #include <pthread.h> -#include <sched.h> #include <string.h> #include <stdio.h> #include <inttypes.h> @@ -276,6 +276,21 @@ static uint64_t read_cpufreq(const char *filename, int id) return ret; } +static inline uint64_t cpu_hz_current(int id) +{ + uint64_t cur_hz = read_cpufreq("cpuinfo_cur_freq", id); + + if (!cur_hz) + cur_hz = odp_cpu_arch_hz_current(id); + + return cur_hz; +} + +static inline uint64_t cpu_hz_static(int id) +{ + return odp_global_ro.system_info.cpu_hz[id]; +} + /* * Analysis of /sys/devices/system/cpu/ files */ @@ -337,6 +352,13 @@ static int read_config_file(void) } odp_global_ro.system_info.default_cpu_hz_max = (uint64_t)val * 1000000; + str = "system.cpu_hz_static"; + if (!_odp_libconfig_lookup_int(str, &val)) { + ODP_ERR("Config option '%s' not found.\n", str); + return -1; + } + odp_global_ro.system_info.cpu_hz_static = !!val; + return 0; } @@ -364,6 +386,10 @@ int _odp_system_info_init(void) "CPU IDs. Increase CONFIG_NUM_CPU_IDS value.\n", num_cpus); + /* Read and save all CPU frequencies for static mode */ + for (i = 0; i < CONFIG_NUM_CPU_IDS; i++) + odp_global_ro.system_info.cpu_hz[i] = cpu_hz_current(i); + /* By default, read max frequency from a cpufreq file */ for (i = 0; i < CONFIG_NUM_CPU_IDS; i++) { uint64_t cpu_hz_max = read_cpufreq("cpuinfo_max_freq", i); @@ -407,26 +433,22 @@ int _odp_system_info_term(void) * Public access functions ************************* */ -uint64_t odp_cpu_hz_current(int id) -{ - uint64_t cur_hz = read_cpufreq("cpuinfo_cur_freq", id); - - if (!cur_hz) - cur_hz = odp_cpu_arch_hz_current(id); - - return cur_hz; -} - uint64_t odp_cpu_hz(void) { - int id = sched_getcpu(); + int id = odp_cpu_id(); - return odp_cpu_hz_current(id); + if (odp_global_ro.system_info.cpu_hz_static) + return cpu_hz_static(id); + return cpu_hz_current(id); } uint64_t odp_cpu_hz_id(int id) { - return odp_cpu_hz_current(id); + ODP_ASSERT(id >= 0 && id < CONFIG_NUM_CPU_IDS); + + if (odp_global_ro.system_info.cpu_hz_static) + return cpu_hz_static(id); + return cpu_hz_current(id); } uint64_t odp_cpu_hz_max(void) diff --git a/platform/linux-generic/odp_timer.c b/platform/linux-generic/odp_timer.c index a28d31245..cd98fcc7d 100644 --- a/platform/linux-generic/odp_timer.c +++ b/platform/linux-generic/odp_timer.c @@ -51,6 +51,7 @@ #include <odp_timer_internal.h> #include <odp/api/plat/queue_inlines.h> #include <odp_global_data.h> +#include <odp_event_internal.h> /* Inlined API functions */ #include <odp/api/plat/event_inlines.h> @@ -92,8 +93,8 @@ tick_buf_s { #endif union { - /* ODP_BUFFER_INVALID if timer not active */ - odp_buffer_t tmo_buf; + /* ODP_EVENT_INVALID if timer not active */ + odp_event_t tmo_event; /* Ensures that tick_buf_t is 128 bits */ uint64_t tmo_u64; @@ -192,7 +193,7 @@ static void timer_init(_odp_timer_t *tim, tick_buf_t *tb, odp_queue_t _q, const tim->queue = _q; tim->user_ptr = _up; tb->tmo_u64 = 0; - tb->tmo_buf = ODP_BUFFER_INVALID; + tb->tmo_event = ODP_EVENT_INVALID; /* Release the timer by setting timer state to inactive */ #if __GCC_ATOMIC_LLONG_LOCK_FREE < 2 @@ -206,7 +207,7 @@ static void timer_init(_odp_timer_t *tim, tick_buf_t *tb, odp_queue_t _q, const static void timer_fini(_odp_timer_t *tim, tick_buf_t *tb) { ODP_ASSERT(tb->exp_tck.v == TMO_UNUSED); - ODP_ASSERT(tb->tmo_buf == ODP_BUFFER_INVALID); + ODP_ASSERT(tb->tmo_event == ODP_EVENT_INVALID); tim->queue = ODP_QUEUE_INVALID; tim->user_ptr = NULL; } @@ -266,16 +267,14 @@ static inline odp_timer_t tp_idx_to_handle(timer_pool_t *tp, (idx + 1)); } -static odp_timeout_hdr_t *timeout_hdr_from_buf(odp_buffer_t buf) +static inline odp_timeout_hdr_t *timeout_hdr_from_event(odp_event_t event) { - return (odp_timeout_hdr_t *)(void *)buf_hdl_to_hdr(buf); + return (odp_timeout_hdr_t *)(uintptr_t)event; } -static odp_timeout_hdr_t *timeout_hdr(odp_timeout_t tmo) +static inline odp_timeout_hdr_t *timeout_hdr(odp_timeout_t tmo) { - odp_buffer_t buf = odp_buffer_from_event(odp_timeout_to_event(tmo)); - - return timeout_hdr_from_buf(buf); + return (odp_timeout_hdr_t *)(uintptr_t)tmo; } static odp_timer_pool_t timer_pool_new(const char *name, @@ -286,7 +285,7 @@ static odp_timer_pool_t timer_pool_new(const char *name, size_t sz0, sz1, sz2; uint64_t tp_size; uint64_t res_ns, nsec_per_scan; - uint32_t flags = ODP_SHM_SW_ONLY; + uint32_t flags = 0; if (odp_global_ro.shm_single_va) flags |= ODP_SHM_SINGLE_VA; @@ -376,7 +375,7 @@ static odp_timer_pool_t timer_pool_new(const char *name, #else odp_atomic_init_u64(&tp->tick_buf[i].exp_tck, TMO_UNUSED); #endif - tp->tick_buf[i].tmo_buf = ODP_BUFFER_INVALID; + tp->tick_buf[i].tmo_event = ODP_EVENT_INVALID; } tp->tp_idx = tp_idx; odp_spinlock_init(&tp->lock); @@ -517,16 +516,15 @@ static inline odp_timer_t timer_alloc(timer_pool_t *tp, odp_queue_t queue, const return hdl; } -static odp_buffer_t timer_set_unused(timer_pool_t *tp, - uint32_t idx); +static odp_event_t timer_set_unused(timer_pool_t *tp, uint32_t idx); -static inline odp_buffer_t timer_free(timer_pool_t *tp, uint32_t idx) +static inline odp_event_t timer_free(timer_pool_t *tp, uint32_t idx) { _odp_timer_t *tim = &tp->timers[idx]; /* Free the timer by setting timer state to unused and - * grab any timeout buffer */ - odp_buffer_t old_buf = timer_set_unused(tp, idx); + * grab any timeout event */ + odp_event_t old_event = timer_set_unused(tp, idx); /* Remove timer from queue */ _odp_queue_fn->timer_rem(tim->queue); @@ -542,7 +540,7 @@ static inline odp_buffer_t timer_free(timer_pool_t *tp, uint32_t idx) tp->num_alloc--; odp_spinlock_unlock(&tp->lock); - return old_buf; + return old_event; } /****************************************************************************** @@ -550,36 +548,36 @@ static inline odp_buffer_t timer_free(timer_pool_t *tp, uint32_t idx) * expire/reset/cancel timer *****************************************************************************/ -static bool timer_reset(uint32_t idx, uint64_t abs_tck, odp_buffer_t *tmo_buf, +static bool timer_reset(uint32_t idx, uint64_t abs_tck, odp_event_t *tmo_event, timer_pool_t *tp) { bool success = true; tick_buf_t *tb = &tp->tick_buf[idx]; - if (tmo_buf == NULL || *tmo_buf == ODP_BUFFER_INVALID) { + if (tmo_event == NULL || *tmo_event == ODP_EVENT_INVALID) { #ifdef ODP_ATOMIC_U128 /* Target supports 128-bit atomic operations */ tick_buf_t new, old; - /* Init all bits, also when tmo_buf is less than 64 bits */ + /* Init all bits, also when tmo_event is less than 64 bits */ new.tmo_u64 = 0; old.tmo_u64 = 0; do { /* Relaxed and non-atomic read of current values */ old.exp_tck.v = tb->exp_tck.v; - old.tmo_buf = tb->tmo_buf; + old.tmo_event = tb->tmo_event; - /* Check if there actually is a timeout buffer + /* Check if there actually is a timeout event * present */ - if (old.tmo_buf == ODP_BUFFER_INVALID) { + if (old.tmo_event == ODP_EVENT_INVALID) { /* Cannot reset a timer with neither old nor - * new timeout buffer */ + * new timeout event */ success = false; break; } /* Set up new values */ new.exp_tck.v = abs_tck; - new.tmo_buf = old.tmo_buf; + new.tmo_event = old.tmo_event; /* Atomic CAS will fail if we experienced torn reads, * retry update sequence until CAS succeeds */ @@ -598,7 +596,7 @@ static bool timer_reset(uint32_t idx, uint64_t abs_tck, odp_buffer_t *tmo_buf, if ((old & TMO_INACTIVE) != 0) { /* Timer was inactive (cancelled or expired), - * we can't reset a timer without a timeout buffer. + * we can't reset a timer without a timeout event. * Attempt to restore inactive state, we don't * want this timer to continue as active without * timeout as this will trigger unnecessary and @@ -617,13 +615,13 @@ static bool timer_reset(uint32_t idx, uint64_t abs_tck, odp_buffer_t *tmo_buf, while (_odp_atomic_flag_load(IDX2LOCK(idx))) odp_cpu_pause(); - /* Only if there is a timeout buffer can be reset the timer */ - if (odp_likely(tb->tmo_buf != ODP_BUFFER_INVALID)) { + /* Only if there is a timeout event can the timer be reset */ + if (odp_likely(tb->tmo_event != ODP_EVENT_INVALID)) { /* Write the new expiration tick */ tb->exp_tck.v = abs_tck; } else { /* Cannot reset a timer with neither old nor new - * timeout buffer */ + * timeout event */ success = false; } @@ -631,35 +629,34 @@ static bool timer_reset(uint32_t idx, uint64_t abs_tck, odp_buffer_t *tmo_buf, _odp_atomic_flag_clear(IDX2LOCK(idx)); #endif } else { - /* We have a new timeout buffer which replaces any old one */ + /* We have a new timeout event which replaces any old one */ /* Fill in some (constant) header fields for timeout events */ - if (odp_event_type(odp_buffer_to_event(*tmo_buf)) == - ODP_EVENT_TIMEOUT) { - /* Convert from buffer to timeout hdr */ + if (odp_event_type(*tmo_event) == ODP_EVENT_TIMEOUT) { + /* Convert from event to timeout hdr */ odp_timeout_hdr_t *tmo_hdr = - timeout_hdr_from_buf(*tmo_buf); + timeout_hdr_from_event(*tmo_event); tmo_hdr->timer = tp_idx_to_handle(tp, idx); tmo_hdr->user_ptr = tp->timers[idx].user_ptr; /* expiration field filled in when timer expires */ } - /* Else ignore buffers of other types */ - odp_buffer_t old_buf = ODP_BUFFER_INVALID; + /* Else ignore events of other types */ + odp_event_t old_event = ODP_EVENT_INVALID; #ifdef ODP_ATOMIC_U128 tick_buf_t new, old; - /* Init all bits, also when tmo_buf is less than 64 bits */ + /* Init all bits, also when tmo_event is less than 64 bits */ new.tmo_u64 = 0; new.exp_tck.v = abs_tck; - new.tmo_buf = *tmo_buf; + new.tmo_event = *tmo_event; - /* We are releasing the new timeout buffer to some other + /* We are releasing the new timeout event to some other * thread */ _odp_atomic_u128_xchg_mm((_odp_atomic_u128_t *)tb, (_uint128_t *)&new, (_uint128_t *)&old, _ODP_MEMMODEL_ACQ_RLS); - old_buf = old.tmo_buf; + old_event = old.tmo_event; #else /* Take a related lock */ while (_odp_atomic_flag_tas(IDX2LOCK(idx))) @@ -667,9 +664,9 @@ static bool timer_reset(uint32_t idx, uint64_t abs_tck, odp_buffer_t *tmo_buf, while (_odp_atomic_flag_load(IDX2LOCK(idx))) odp_cpu_pause(); - /* Swap in new buffer, save any old buffer */ - old_buf = tb->tmo_buf; - tb->tmo_buf = *tmo_buf; + /* Swap in new event, save any old event */ + old_event = tb->tmo_event; + tb->tmo_event = *tmo_event; /* Write the new expiration tick */ tb->exp_tck.v = abs_tck; @@ -677,33 +674,32 @@ static bool timer_reset(uint32_t idx, uint64_t abs_tck, odp_buffer_t *tmo_buf, /* Release the lock */ _odp_atomic_flag_clear(IDX2LOCK(idx)); #endif - /* Return old timeout buffer */ - *tmo_buf = old_buf; + /* Return old timeout event */ + *tmo_event = old_event; } return success; } -static odp_buffer_t timer_set_unused(timer_pool_t *tp, - uint32_t idx) +static odp_event_t timer_set_unused(timer_pool_t *tp, uint32_t idx) { tick_buf_t *tb = &tp->tick_buf[idx]; - odp_buffer_t old_buf; + odp_event_t old_event; #ifdef ODP_ATOMIC_U128 tick_buf_t new, old; - /* Init all bits, also when tmo_buf is less than 64 bits */ + /* Init all bits, also when tmo_event is less than 64 bits */ new.tmo_u64 = 0; /* Update the timer state (e.g. cancel the current timeout) */ new.exp_tck.v = TMO_UNUSED; - /* Swap out the old buffer */ - new.tmo_buf = ODP_BUFFER_INVALID; + /* Swap out the old event */ + new.tmo_event = ODP_EVENT_INVALID; _odp_atomic_u128_xchg_mm((_odp_atomic_u128_t *)tb, (_uint128_t *)&new, (_uint128_t *)&old, _ODP_MEMMODEL_RLX); - old_buf = old.tmo_buf; + old_event = old.tmo_event; #else /* Take a related lock */ while (_odp_atomic_flag_tas(IDX2LOCK(idx))) @@ -714,44 +710,43 @@ static odp_buffer_t timer_set_unused(timer_pool_t *tp, /* Update the timer state (e.g. cancel the current timeout) */ tb->exp_tck.v = TMO_UNUSED; - /* Swap out the old buffer */ - old_buf = tb->tmo_buf; - tb->tmo_buf = ODP_BUFFER_INVALID; + /* Swap out the old event */ + old_event = tb->tmo_event; + tb->tmo_event = ODP_EVENT_INVALID; /* Release the lock */ _odp_atomic_flag_clear(IDX2LOCK(idx)); #endif - /* Return the old buffer */ - return old_buf; + /* Return the old event */ + return old_event; } -static odp_buffer_t timer_cancel(timer_pool_t *tp, - uint32_t idx) +static odp_event_t timer_cancel(timer_pool_t *tp, uint32_t idx) { tick_buf_t *tb = &tp->tick_buf[idx]; - odp_buffer_t old_buf; + odp_event_t old_event; #ifdef ODP_ATOMIC_U128 tick_buf_t new, old; - /* Init all bits, also when tmo_buf is less than 64 bits */ + /* Init all bits, also when tmo_event is less than 64 bits */ new.tmo_u64 = 0; old.tmo_u64 = 0; do { /* Relaxed and non-atomic read of current values */ old.exp_tck.v = tb->exp_tck.v; - old.tmo_buf = tb->tmo_buf; + old.tmo_event = tb->tmo_event; /* Check if it is not expired already */ if (old.exp_tck.v & TMO_INACTIVE) { - old.tmo_buf = ODP_BUFFER_INVALID; + old.tmo_event = ODP_EVENT_INVALID; break; } /* Set up new values */ new.exp_tck.v = TMO_INACTIVE; - new.tmo_buf = ODP_BUFFER_INVALID; + new.tmo_event = ODP_EVENT_INVALID; /* Atomic CAS will fail if we experienced torn reads, * retry update sequence until CAS succeeds */ @@ -760,7 +755,7 @@ static odp_buffer_t timer_cancel(timer_pool_t *tp, (_uint128_t *)&new, _ODP_MEMMODEL_RLS, _ODP_MEMMODEL_RLX)); - old_buf = old.tmo_buf; + old_event = old.tmo_event; #else /* Take a related lock */ while (_odp_atomic_flag_tas(IDX2LOCK(idx))) @@ -768,56 +763,56 @@ static odp_buffer_t timer_cancel(timer_pool_t *tp, while (_odp_atomic_flag_load(IDX2LOCK(idx))) odp_cpu_pause(); - /* Swap in new buffer, save any old buffer */ - old_buf = tb->tmo_buf; - tb->tmo_buf = ODP_BUFFER_INVALID; + /* Swap in new event, save any old event */ + old_event = tb->tmo_event; + tb->tmo_event = ODP_EVENT_INVALID; /* Write the new expiration tick if it not cancelled */ if (tb->exp_tck.v & TMO_INACTIVE) - old_buf = ODP_BUFFER_INVALID; + old_event = ODP_EVENT_INVALID; else tb->exp_tck.v = TMO_INACTIVE; /* Release the lock */ _odp_atomic_flag_clear(IDX2LOCK(idx)); #endif - /* Return the old buffer */ - return old_buf; + /* Return the old event */ + return old_event; } static inline void timer_expire(timer_pool_t *tp, uint32_t idx, uint64_t tick) { _odp_timer_t *tim = &tp->timers[idx]; tick_buf_t *tb = &tp->tick_buf[idx]; - odp_buffer_t tmo_buf = ODP_BUFFER_INVALID; + odp_event_t tmo_event = ODP_EVENT_INVALID; uint64_t exp_tck; #ifdef ODP_ATOMIC_U128 /* Atomic re-read for correctness */ exp_tck = odp_atomic_load_u64(&tb->exp_tck); /* Re-check exp_tck */ if (odp_likely(exp_tck <= tick)) { - /* Attempt to grab timeout buffer, replace with inactive timer - * and invalid buffer */ + /* Attempt to grab timeout event, replace with inactive timer + * and invalid event. */ tick_buf_t new, old; - /* Init all bits, also when tmo_buf is less than 64 bits */ + /* Init all bits, also when tmo_event is less than 64 bits. */ new.tmo_u64 = 0; old.tmo_u64 = 0; old.exp_tck.v = exp_tck; - old.tmo_buf = tb->tmo_buf; + old.tmo_event = tb->tmo_event; /* Set the inactive/expired bit keeping the expiration tick so * that we can check against the expiration tick of the timeout * when it is received */ new.exp_tck.v = exp_tck | TMO_INACTIVE; - new.tmo_buf = ODP_BUFFER_INVALID; + new.tmo_event = ODP_EVENT_INVALID; int succ = _odp_atomic_u128_cmp_xchg_mm((_odp_atomic_u128_t *)tb, (_uint128_t *)&old, (_uint128_t *)&new, _ODP_MEMMODEL_RLS, _ODP_MEMMODEL_RLX); if (succ) - tmo_buf = old.tmo_buf; + tmo_event = old.tmo_event; /* Else CAS failed, something changed => skip timer * this tick, it will be checked again next tick */ } @@ -831,41 +826,40 @@ static inline void timer_expire(timer_pool_t *tp, uint32_t idx, uint64_t tick) /* Proper check for timer expired */ exp_tck = tb->exp_tck.v; if (odp_likely(exp_tck <= tick)) { - /* Verify that there is a timeout buffer */ - if (odp_likely(tb->tmo_buf != ODP_BUFFER_INVALID)) { - /* Grab timeout buffer, replace with inactive timer - * and invalid buffer */ - tmo_buf = tb->tmo_buf; - tb->tmo_buf = ODP_BUFFER_INVALID; + /* Verify that there is a timeout event */ + if (odp_likely(tb->tmo_event != ODP_EVENT_INVALID)) { + /* Grab timeout event, replace with inactive timer + * and invalid event. */ + tmo_event = tb->tmo_event; + tb->tmo_event = ODP_EVENT_INVALID; /* Set the inactive/expired bit keeping the expiration * tick so that we can check against the expiration * tick of the timeout when it is received */ tb->exp_tck.v |= TMO_INACTIVE; } - /* Else somehow active timer without user buffer */ + /* Else somehow active timer without user event */ } /* Else false positive, ignore */ /* Release the lock */ _odp_atomic_flag_clear(IDX2LOCK(idx)); #endif - if (odp_likely(tmo_buf != ODP_BUFFER_INVALID)) { + if (odp_likely(tmo_event != ODP_EVENT_INVALID)) { /* Fill in expiration tick for timeout events */ - if (odp_event_type(odp_buffer_to_event(tmo_buf)) == - ODP_EVENT_TIMEOUT) { - /* Convert from buffer to timeout hdr */ + if (odp_event_type(tmo_event) == ODP_EVENT_TIMEOUT) { + /* Convert from event to timeout hdr */ odp_timeout_hdr_t *tmo_hdr = - timeout_hdr_from_buf(tmo_buf); + timeout_hdr_from_event(tmo_event); tmo_hdr->expiration = exp_tck; /* timer and user_ptr fields filled in when timer * was set */ } /* Else ignore events of other types */ /* Post the timeout to the destination queue */ - int rc = odp_queue_enq(tim->queue, - odp_buffer_to_event(tmo_buf)); + int rc = odp_queue_enq(tim->queue, tmo_event); + if (odp_unlikely(rc != 0)) { - odp_buffer_free(tmo_buf); - ODP_ABORT("Failed to enqueue timeout buffer (%d)\n", + _odp_event_free(tmo_event); + ODP_ABORT("Failed to enqueue timeout event (%d)\n", rc); } } @@ -1270,6 +1264,12 @@ int odp_timer_res_capability(odp_timer_clk_src_t clk_src, return 0; } +void odp_timer_pool_param_init(odp_timer_pool_param_t *param) +{ + memset(param, 0, sizeof(odp_timer_pool_param_t)); + param->clk_src = ODP_CLOCK_DEFAULT; +} + odp_timer_pool_t odp_timer_pool_create(const char *name, const odp_timer_pool_param_t *param) { @@ -1384,9 +1384,8 @@ odp_event_t odp_timer_free(odp_timer_t hdl) { timer_pool_t *tp = handle_to_tp(hdl); uint32_t idx = handle_to_idx(hdl, tp); - odp_buffer_t old_buf = timer_free(tp, idx); - return odp_buffer_to_event(old_buf); + return timer_free(tp, idx); } int odp_timer_set_abs(odp_timer_t hdl, @@ -1401,7 +1400,7 @@ int odp_timer_set_abs(odp_timer_t hdl, return ODP_TIMER_TOO_NEAR; if (odp_unlikely(abs_tck > cur_tick + tp->max_rel_tck)) return ODP_TIMER_TOO_FAR; - if (timer_reset(idx, abs_tck, (odp_buffer_t *)tmo_ev, tp)) + if (timer_reset(idx, abs_tck, tmo_ev, tp)) return ODP_TIMER_SUCCESS; else return ODP_TIMER_FAIL; @@ -1420,7 +1419,7 @@ int odp_timer_set_rel(odp_timer_t hdl, return ODP_TIMER_TOO_NEAR; if (odp_unlikely(rel_tck > tp->max_rel_tck)) return ODP_TIMER_TOO_FAR; - if (timer_reset(idx, abs_tck, (odp_buffer_t *)tmo_ev, tp)) + if (timer_reset(idx, abs_tck, tmo_ev, tp)) return ODP_TIMER_SUCCESS; else return ODP_TIMER_FAIL; @@ -1431,10 +1430,10 @@ int odp_timer_cancel(odp_timer_t hdl, odp_event_t *tmo_ev) timer_pool_t *tp = handle_to_tp(hdl); uint32_t idx = handle_to_idx(hdl, tp); /* Set the expiration tick of the timer to TMO_INACTIVE */ - odp_buffer_t old_buf = timer_cancel(tp, idx); + odp_event_t old_event = timer_cancel(tp, idx); - if (old_buf != ODP_BUFFER_INVALID) { - *tmo_ev = odp_buffer_to_event(old_buf); + if (old_event != ODP_EVENT_INVALID) { + *tmo_ev = old_event; return 0; /* Active timer cancelled, timeout returned */ } else { return -1; /* Timer already expired, no timeout returned */ @@ -1498,9 +1497,8 @@ void *odp_timeout_user_ptr(odp_timeout_t tmo) odp_timeout_t odp_timeout_alloc(odp_pool_t pool_hdl) { - odp_timeout_t tmo; + odp_event_t event; pool_t *pool; - int ret; ODP_ASSERT(pool_hdl != ODP_POOL_INVALID); @@ -1508,17 +1506,16 @@ odp_timeout_t odp_timeout_alloc(odp_pool_t pool_hdl) ODP_ASSERT(pool->type == ODP_POOL_TIMEOUT); - ret = _odp_buffer_alloc_multi(pool, (odp_buffer_hdr_t **)&tmo, 1); - - if (odp_likely(ret == 1)) - return tmo; + event = _odp_event_alloc(pool); + if (odp_unlikely(event == ODP_EVENT_INVALID)) + return ODP_TIMEOUT_INVALID; - return ODP_TIMEOUT_INVALID; + return odp_timeout_from_event(event); } void odp_timeout_free(odp_timeout_t tmo) { - _odp_buffer_free_multi((odp_buffer_hdr_t **)&tmo, 1); + _odp_event_free(odp_timeout_to_event(tmo)); } void odp_timer_pool_print(odp_timer_pool_t timer_pool) diff --git a/platform/linux-generic/odp_traffic_mngr.c b/platform/linux-generic/odp_traffic_mngr.c index e741fd80c..e9e6822a1 100644 --- a/platform/linux-generic/odp_traffic_mngr.c +++ b/platform/linux-generic/odp_traffic_mngr.c @@ -33,6 +33,7 @@ #include <odp_errno_define.h> #include <odp_global_data.h> #include <odp_schedule_if.h> +#include <odp_event_internal.h> /* Local vars */ static const @@ -166,19 +167,19 @@ static inline tm_node_obj_t *tm_nobj_from_index(uint32_t node_id) return &tm_glb->node_obj.obj[node_id]; } -static int queue_tm_reenq(odp_queue_t queue, odp_buffer_hdr_t *buf_hdr) +static int queue_tm_reenq(odp_queue_t queue, _odp_event_hdr_t *event_hdr) { odp_tm_queue_t tm_queue = MAKE_ODP_TM_QUEUE(odp_queue_context(queue)); - odp_packet_t pkt = packet_from_buf_hdr(buf_hdr); + odp_packet_t pkt = packet_from_event_hdr(event_hdr); return odp_tm_enq(tm_queue, pkt); } -static int queue_tm_reenq_multi(odp_queue_t queue, odp_buffer_hdr_t *buf[], +static int queue_tm_reenq_multi(odp_queue_t queue, _odp_event_hdr_t *event[], int num) { (void)queue; - (void)buf; + (void)event; (void)num; ODP_ABORT("Invalid call to queue_tm_reenq_multi()\n"); return 0; diff --git a/platform/linux-generic/pktio/dpdk.c b/platform/linux-generic/pktio/dpdk.c index 36c1ec06e..a0407ca62 100644 --- a/platform/linux-generic/pktio/dpdk.c +++ b/platform/linux-generic/pktio/dpdk.c @@ -305,7 +305,7 @@ static void pktmbuf_init(struct rte_mempool *mp, void *opaque_arg ODP_UNUSED, void *buf_addr; pkt_hdr = pkt_hdr_from_mbuf(m); - buf_addr = pkt_hdr->buf_hdr.base_data - RTE_PKTMBUF_HEADROOM; + buf_addr = pkt_hdr->event_hdr.base_data - RTE_PKTMBUF_HEADROOM; priv_size = rte_pktmbuf_priv_size(mp); mbuf_size = sizeof(struct rte_mbuf); diff --git a/platform/linux-generic/pktio/ipc.c b/platform/linux-generic/pktio/ipc.c index b89252303..6a78415c5 100644 --- a/platform/linux-generic/pktio/ipc.c +++ b/platform/linux-generic/pktio/ipc.c @@ -787,7 +787,7 @@ static int ipc_pktio_send_lockless(pktio_entry_t *pktio_entry, pool_t *pool; pkt_hdr = packet_hdr(pkt); - pool = pkt_hdr->buf_hdr.pool_ptr; + pool = pkt_hdr->event_hdr.pool_ptr; if (pool->pool_idx != ipc_pool->pool_idx || odp_packet_has_ref(pkt)) { diff --git a/platform/linux-generic/pktio/loop.c b/platform/linux-generic/pktio/loop.c index 666e7ce90..ae6403fe6 100644 --- a/platform/linux-generic/pktio/loop.c +++ b/platform/linux-generic/pktio/loop.c @@ -18,6 +18,7 @@ #include <odp_queue_if.h> #include <odp/api/plat/queue_inlines.h> #include <odp_global_data.h> +#include <odp_event_internal.h> #include <protocols/eth.h> #include <protocols/ip.h> @@ -97,7 +98,7 @@ static int loopback_recv(pktio_entry_t *pktio_entry, int index ODP_UNUSED, odp_packet_t pkts[], int num) { int nbr, i; - odp_buffer_hdr_t *hdr_tbl[QUEUE_MULTI_MAX]; + _odp_event_hdr_t *hdr_tbl[QUEUE_MULTI_MAX]; odp_queue_t queue; odp_packet_hdr_t *pkt_hdr; odp_packet_t pkt; @@ -123,7 +124,7 @@ static int loopback_recv(pktio_entry_t *pktio_entry, int index ODP_UNUSED, for (i = 0; i < nbr; i++) { uint32_t pkt_len; - pkt = packet_from_buf_hdr(hdr_tbl[i]); + pkt = packet_from_event_hdr(hdr_tbl[i]); pkt_len = odp_packet_len(pkt); pkt_hdr = packet_hdr(pkt); @@ -297,7 +298,7 @@ static int loopback_send(pktio_entry_t *pktio_entry, int index ODP_UNUSED, const odp_packet_t pkt_tbl[], int num) { pkt_loop_t *pkt_loop = pkt_priv(pktio_entry); - odp_buffer_hdr_t *hdr_tbl[QUEUE_MULTI_MAX]; + _odp_event_hdr_t *hdr_tbl[QUEUE_MULTI_MAX]; odp_queue_t queue; int i; int ret; @@ -323,7 +324,7 @@ static int loopback_send(pktio_entry_t *pktio_entry, int index ODP_UNUSED, } break; } - hdr_tbl[i] = packet_to_buf_hdr(pkt_tbl[i]); + hdr_tbl[i] = packet_to_event_hdr(pkt_tbl[i]); bytes += pkt_len; /* Store cumulative byte counts to update 'stats.out_octets' * correctly in case enq_multi() fails to enqueue all packets. diff --git a/platform/linux-generic/test/inline-timer.conf b/platform/linux-generic/test/inline-timer.conf index 90709d86d..38da87617 100644 --- a/platform/linux-generic/test/inline-timer.conf +++ b/platform/linux-generic/test/inline-timer.conf @@ -1,6 +1,6 @@ # Mandatory fields odp_implementation = "linux-generic" -config_file_version = "0.1.18" +config_file_version = "0.1.19" timer: { # Enable inline timer implementation diff --git a/platform/linux-generic/test/packet_align.conf b/platform/linux-generic/test/packet_align.conf index f9b39abf6..e3e4251cd 100644 --- a/platform/linux-generic/test/packet_align.conf +++ b/platform/linux-generic/test/packet_align.conf @@ -1,6 +1,6 @@ # Mandatory fields odp_implementation = "linux-generic" -config_file_version = "0.1.18" +config_file_version = "0.1.19" pool: { pkt: { diff --git a/platform/linux-generic/test/process-mode.conf b/platform/linux-generic/test/process-mode.conf index a4b5d3f39..7453ffa36 100644 --- a/platform/linux-generic/test/process-mode.conf +++ b/platform/linux-generic/test/process-mode.conf @@ -1,6 +1,6 @@ # Mandatory fields odp_implementation = "linux-generic" -config_file_version = "0.1.18" +config_file_version = "0.1.19" # Shared memory options shm: { diff --git a/platform/linux-generic/test/sched-basic.conf b/platform/linux-generic/test/sched-basic.conf index 4ef0ab044..32abdb5b3 100644 --- a/platform/linux-generic/test/sched-basic.conf +++ b/platform/linux-generic/test/sched-basic.conf @@ -1,6 +1,6 @@ # Mandatory fields odp_implementation = "linux-generic" -config_file_version = "0.1.18" +config_file_version = "0.1.19" # Test scheduler with an odd spread value and without dynamic load balance sched_basic: { |