diff options
Diffstat (limited to 'platform/linux-generic/include')
17 files changed, 127 insertions, 138 deletions
diff --git a/platform/linux-generic/include/odp/api/plat/packet_inline_types.h b/platform/linux-generic/include/odp/api/plat/packet_inline_types.h index 126404f2d..c5293fc86 100644 --- a/platform/linux-generic/include/odp/api/plat/packet_inline_types.h +++ b/platform/linux-generic/include/odp/api/plat/packet_inline_types.h @@ -115,7 +115,7 @@ typedef union { uint32_t all_flags; struct { - uint32_t reserved1: 6; + uint32_t reserved1: 7; /* * Init flags @@ -146,14 +146,13 @@ typedef union { uint32_t udp_err: 1; /* UDP error */ uint32_t sctp_err: 1; /* SCTP error */ uint32_t l4_chksum_err: 1; /* L4 checksum error */ - uint32_t crypto_err: 1; /* Crypto packet operation error */ }; /* Flag groups */ struct { - uint32_t reserved2: 6; + uint32_t reserved2: 7; uint32_t other: 18; /* All other flags */ - uint32_t error: 8; /* All error flags */ + uint32_t error: 7; /* All error flags */ } all; } _odp_packet_flags_t; diff --git a/platform/linux-generic/include/odp_align_internal.h b/platform/linux-generic/include/odp_align_internal.h deleted file mode 100644 index df2182df2..000000000 --- a/platform/linux-generic/include/odp_align_internal.h +++ /dev/null @@ -1,67 +0,0 @@ -/* Copyright (c) 2014-2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -/** - * @file - * - * ODP internal alignments - */ - -#ifndef ODP_ALIGN_INTERNAL_H_ -#define ODP_ALIGN_INTERNAL_H_ - -#ifdef __cplusplus -extern "C" { -#endif - -#include <odp/api/align.h> -#include <stdint.h> - -/* Macros to calculate ODP_ROUNDUP_POWER2_U32() in five rounds of shift - * and OR operations. */ -#define _RSHIFT_U32(x, y) (((uint32_t)(x)) >> (y)) -#define _POW2_U32_R1(x) (((uint32_t)(x)) | _RSHIFT_U32(x, 1)) -#define _POW2_U32_R2(x) (_POW2_U32_R1(x) | _RSHIFT_U32(_POW2_U32_R1(x), 2)) -#define _POW2_U32_R3(x) (_POW2_U32_R2(x) | _RSHIFT_U32(_POW2_U32_R2(x), 4)) -#define _POW2_U32_R4(x) (_POW2_U32_R3(x) | _RSHIFT_U32(_POW2_U32_R3(x), 8)) -#define _POW2_U32_R5(x) (_POW2_U32_R4(x) | _RSHIFT_U32(_POW2_U32_R4(x), 16)) - -/* Round up a uint32_t value 'x' to the next power of two. - * - * The value is not round up, if it's already a power of two (including 1). - * The value must be larger than 0 and not exceed 0x80000000. - */ -#define ROUNDUP_POWER2_U32(x) \ - ((((uint32_t)(x)) > 0x80000000) ? 0 : (_POW2_U32_R5(x - 1) + 1)) - -/* - * Round up 'x' to alignment 'align' - */ -#define ROUNDUP_ALIGN(x, align)\ - ((align) * (((x) + (align) - 1) / (align))) - -/* - * Round up 'x' to cache line size alignment - */ -#define ROUNDUP_CACHE_LINE(x)\ - ROUNDUP_ALIGN(x, ODP_CACHE_LINE_SIZE) - -/* - * Round down 'x' to 'align' alignment, which is a power of two - */ -#define ROUNDDOWN_POWER2(x, align)\ - ((x) & (~((align) - 1))) - -/* - * Check if value is a power of two - */ -#define CHECK_IS_POWER2(x) ((((x) - 1) & (x)) == 0) - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/platform/linux-generic/include/odp_buffer_internal.h b/platform/linux-generic/include/odp_buffer_internal.h index e0be593d0..5841720ef 100644 --- a/platform/linux-generic/include/odp_buffer_internal.h +++ b/platform/linux-generic/include/odp_buffer_internal.h @@ -24,7 +24,6 @@ extern "C" { #include <odp/api/buffer.h> #include <odp/api/debug.h> #include <odp/api/align.h> -#include <odp_align_internal.h> #include <odp_config_internal.h> #include <odp/api/byteorder.h> #include <odp/api/thread.h> diff --git a/platform/linux-generic/include/odp_classification_datamodel.h b/platform/linux-generic/include/odp_classification_datamodel.h index 6b50fef68..6e89a9947 100644 --- a/platform/linux-generic/include/odp_classification_datamodel.h +++ b/platform/linux-generic/include/odp_classification_datamodel.h @@ -21,10 +21,13 @@ extern "C" { #include <odp/api/spinlock.h> #include <odp/api/classification.h> #include <odp/api/debug.h> + +#include <odp_macros_internal.h> #include <odp_pool_internal.h> #include <odp_packet_internal.h> #include <odp_packet_io_internal.h> #include <odp_queue_if.h> + #include <protocols/ip.h> /* Maximum Class Of Service Entry */ @@ -155,7 +158,7 @@ struct cos_s { typedef union cos_u { struct cos_s s; - uint8_t pad[ROUNDUP_CACHE_LINE(sizeof(struct cos_s))]; + uint8_t pad[_ODP_ROUNDUP_CACHE_LINE(sizeof(struct cos_s))]; } cos_t; /* Pattern Matching Rule */ @@ -172,7 +175,7 @@ struct pmr_s { typedef union pmr_u { struct pmr_s s; - uint8_t pad[ROUNDUP_CACHE_LINE(sizeof(struct pmr_s))]; + uint8_t pad[_ODP_ROUNDUP_CACHE_LINE(sizeof(struct pmr_s))]; } pmr_t; typedef struct _cls_queue_grp_tbl_s { @@ -181,7 +184,7 @@ typedef struct _cls_queue_grp_tbl_s { typedef union _cls_queue_grp_tbl_t { _cls_queue_grp_tbl_s s; - uint8_t pad[ROUNDUP_CACHE_LINE(sizeof(_cls_queue_grp_tbl_s))]; + uint8_t pad[_ODP_ROUNDUP_CACHE_LINE(sizeof(_cls_queue_grp_tbl_s))]; } _cls_queue_grp_tbl_t; /** diff --git a/platform/linux-generic/include/odp_macros_internal.h b/platform/linux-generic/include/odp_macros_internal.h index b8be7f938..abf017aec 100644 --- a/platform/linux-generic/include/odp_macros_internal.h +++ b/platform/linux-generic/include/odp_macros_internal.h @@ -1,4 +1,5 @@ -/* Copyright (c) 2018-2018, Linaro Limited +/* Copyright (c) 2014-2018, Linaro Limited + * Copyright (c) 2022, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -17,37 +18,67 @@ extern "C" { #endif -#include <odp/api/debug.h> +#include <odp/api/align.h> -#define ARRAY_SIZE(x) (sizeof(x) / sizeof((x)[0])) +#include <stdint.h> -#define MIN(a, b) \ +#define _ODP_ARRAY_SIZE(x) (sizeof(x) / sizeof((x)[0])) + +#define _ODP_MIN(a, b) \ __extension__ ({ \ __typeof__(a) tmp_a = (a); \ __typeof__(b) tmp_b = (b); \ tmp_a < tmp_b ? tmp_a : tmp_b; \ }) -#define MAX(a, b) \ +#define _ODP_MAX(a, b) \ __extension__ ({ \ __typeof__(a) tmp_a = (a); \ __typeof__(b) tmp_b = (b); \ tmp_a > tmp_b ? tmp_a : tmp_b; \ }) -#define MAX3(a, b, c) (MAX(MAX((a), (b)), (c))) +#define _ODP_MAX3(a, b, c) (_ODP_MAX(_ODP_MAX((a), (b)), (c))) -#define odp_container_of(pointer, type, member) \ - ((type *)(void *)(((char *)pointer) - offsetof(type, member))) +/* Macros to calculate ODP_ROUNDUP_POWER2_U32() in five rounds of shift + * and OR operations. */ +#define __ODP_RSHIFT_U32(x, y) (((uint32_t)(x)) >> (y)) +#define __ODP_POW2_U32_R1(x) (((uint32_t)(x)) | __ODP_RSHIFT_U32(x, 1)) +#define __ODP_POW2_U32_R2(x) (__ODP_POW2_U32_R1(x) | __ODP_RSHIFT_U32(__ODP_POW2_U32_R1(x), 2)) +#define __ODP_POW2_U32_R3(x) (__ODP_POW2_U32_R2(x) | __ODP_RSHIFT_U32(__ODP_POW2_U32_R2(x), 4)) +#define __ODP_POW2_U32_R4(x) (__ODP_POW2_U32_R3(x) | __ODP_RSHIFT_U32(__ODP_POW2_U32_R3(x), 8)) +#define __ODP_POW2_U32_R5(x) (__ODP_POW2_U32_R4(x) | __ODP_RSHIFT_U32(__ODP_POW2_U32_R4(x), 16)) -#define DIV_ROUND_UP(a, b) \ - __extension__ ({ \ - __typeof__(a) tmp_a = (a); \ - __typeof__(b) tmp_b = (b); \ - ODP_STATIC_ASSERT(__builtin_constant_p(b), ""); \ - ODP_STATIC_ASSERT((((b) - 1) & (b)) == 0, ""); \ - (tmp_a + tmp_b - 1) >> __builtin_ctz(tmp_b); \ - }) +/* Round up a uint32_t value 'x' to the next power of two. + * + * The value is not round up, if it's already a power of two (including 1). + * The value must be larger than 0 and not exceed 0x80000000. + */ +#define _ODP_ROUNDUP_POWER2_U32(x) \ + ((((uint32_t)(x)) > 0x80000000) ? 0 : (__ODP_POW2_U32_R5(x - 1) + 1)) + +/* + * Round up 'x' to alignment 'align' + */ +#define _ODP_ROUNDUP_ALIGN(x, align)\ + ((align) * (((x) + (align) - 1) / (align))) + +/* + * Round up 'x' to cache line size alignment + */ +#define _ODP_ROUNDUP_CACHE_LINE(x)\ + _ODP_ROUNDUP_ALIGN(x, ODP_CACHE_LINE_SIZE) + +/* + * Round down 'x' to 'align' alignment, which is a power of two + */ +#define _ODP_ROUNDDOWN_POWER2(x, align)\ + ((x) & (~((align) - 1))) + +/* + * Check if value is a power of two + */ +#define _ODP_CHECK_IS_POWER2(x) ((((x) - 1) & (x)) == 0) #ifdef __cplusplus } diff --git a/platform/linux-generic/include/odp_packet_dpdk.h b/platform/linux-generic/include/odp_packet_dpdk.h index b326000e6..7d42971be 100644 --- a/platform/linux-generic/include/odp_packet_dpdk.h +++ b/platform/linux-generic/include/odp_packet_dpdk.h @@ -26,16 +26,6 @@ struct rte_mbuf; #define PTYPE_UDP 0x20 #define PTYPE_TCP 0x40 -/** - * Calculate size of zero-copy DPDK packet pool object - */ -uint32_t _odp_dpdk_pool_obj_size(pool_t *pool, uint32_t block_size); - -/** - * Create zero-copy DPDK packet pool - */ -int _odp_dpdk_pool_create(pool_t *pool); - /** Packet parser using DPDK interface */ int _odp_dpdk_packet_parse_common(packet_parser_t *pkt_hdr, const uint8_t *ptr, diff --git a/platform/linux-generic/include/odp_packet_internal.h b/platform/linux-generic/include/odp_packet_internal.h index a8f58146a..e54d88f6a 100644 --- a/platform/linux-generic/include/odp_packet_internal.h +++ b/platform/linux-generic/include/odp_packet_internal.h @@ -21,19 +21,21 @@ extern "C" { #include <odp/api/align.h> #include <odp/api/atomic.h> #include <odp/api/debug.h> +#include <odp/api/hints.h> #include <odp/api/packet.h> -#include <odp/api/plat/packet_inline_types.h> #include <odp/api/packet_io.h> #include <odp/api/crypto.h> #include <odp/api/comp.h> #include <odp/api/std.h> -#include <odp/api/abi/packet.h> + +#include <odp/api/plat/packet_inline_types.h> #include <odp_debug_internal.h> #include <odp_event_internal.h> #include <odp_ipsec_internal.h> #include <odp_pool_internal.h> #include <odp_queue_if.h> +#include <odp_config_internal.h> #include <stdint.h> #include <string.h> @@ -44,12 +46,6 @@ ODP_STATIC_ASSERT(sizeof(_odp_packet_input_flags_t) == sizeof(uint64_t), ODP_STATIC_ASSERT(sizeof(_odp_packet_flags_t) == sizeof(uint32_t), "PACKET_FLAGS_SIZE_ERROR"); -/* Packet extra data length */ -#define PKT_EXTRA_LEN 128 - -/* Packet extra data types */ -#define PKT_EXTRA_TYPE_DPDK 1 - /* Maximum number of segments per packet */ #define PKT_MAX_SEGS 255 @@ -150,14 +146,15 @@ typedef struct ODP_ALIGNED_CACHE odp_packet_hdr_t { /* LSO profile index */ uint8_t lso_profile_idx; + /* Pktio where packet is used as a memory source */ + uint8_t ms_pktio_idx; + union { - struct { - /* Result for crypto packet op */ - odp_crypto_packet_result_t crypto_op_result; + /* Result for crypto packet op */ + odp_crypto_packet_result_t crypto_op_result; - /* Context for IPsec */ - odp_ipsec_packet_result_t ipsec_ctx; - }; + /* Context for IPsec */ + odp_ipsec_packet_result_t ipsec_ctx; /* Result for comp packet op */ odp_comp_packet_result_t comp_op_result; @@ -172,6 +169,8 @@ typedef struct ODP_ALIGNED_CACHE odp_packet_hdr_t { * grow over 256 bytes. */ ODP_STATIC_ASSERT(sizeof(odp_packet_hdr_t) <= 256, "PACKET_HDR_SIZE_ERROR"); +ODP_STATIC_ASSERT(ODP_CONFIG_PKTIO_ENTRIES < UINT8_MAX, "MS_PKTIO_IDX_SIZE_ERROR"); + /** * Return the packet header */ @@ -298,6 +297,8 @@ static inline void _odp_packet_copy_md(odp_packet_hdr_t *dst_hdr, odp_packet_hdr_t *src_hdr, odp_bool_t uarea_copy) { + int8_t subtype = src_hdr->subtype; + /* Lengths and segmentation data are not copied: * .frame_len * .headroom @@ -308,6 +309,7 @@ static inline void _odp_packet_copy_md(odp_packet_hdr_t *dst_hdr, * .seg_count */ dst_hdr->input = src_hdr->input; + dst_hdr->subtype = subtype; dst_hdr->dst_queue = src_hdr->dst_queue; dst_hdr->cos = src_hdr->cos; dst_hdr->cls_mark = src_hdr->cls_mark; @@ -351,6 +353,15 @@ static inline void _odp_packet_copy_md(odp_packet_hdr_t *dst_hdr, dst_hdr->uarea_addr = src_uarea; } } + + if (odp_unlikely(subtype != ODP_EVENT_PACKET_BASIC)) { + if (subtype == ODP_EVENT_PACKET_IPSEC) + dst_hdr->ipsec_ctx = src_hdr->ipsec_ctx; + else if (subtype == ODP_EVENT_PACKET_CRYPTO) + dst_hdr->crypto_op_result = src_hdr->crypto_op_result; + else if (subtype == ODP_EVENT_PACKET_COMP) + dst_hdr->comp_op_result = src_hdr->comp_op_result; + } } static inline void _odp_packet_copy_cls_md(odp_packet_hdr_t *dst_hdr, diff --git a/platform/linux-generic/include/odp_packet_io_internal.h b/platform/linux-generic/include/odp_packet_io_internal.h index aed6de412..ca9f083da 100644 --- a/platform/linux-generic/include/odp_packet_io_internal.h +++ b/platform/linux-generic/include/odp_packet_io_internal.h @@ -26,10 +26,10 @@ extern "C" { #include <odp/api/plat/packet_io_inlines.h> #include <odp/autoheader_internal.h> -#include <odp_align_internal.h> #include <odp_classification_datamodel.h> #include <odp_config_internal.h> #include <odp_debug_internal.h> +#include <odp_macros_internal.h> #include <odp_packet_io_stats_common.h> #include <odp_queue_if.h> @@ -70,7 +70,7 @@ struct pktio_if_ops; #elif defined(_ODP_PKTIO_DPDK) #define PKTIO_PRIVATE_SIZE 5632 #else -#define PKTIO_PRIVATE_SIZE 384 +#define PKTIO_PRIVATE_SIZE 512 #endif struct pktio_entry { @@ -173,7 +173,7 @@ struct pktio_entry { typedef union { struct pktio_entry s; - uint8_t pad[ROUNDUP_CACHE_LINE(sizeof(struct pktio_entry))]; + uint8_t pad[_ODP_ROUNDUP_CACHE_LINE(sizeof(struct pktio_entry))]; } pktio_entry_t; typedef struct { @@ -308,6 +308,7 @@ static inline void _odp_pktio_tx_ts_set(pktio_entry_t *entry) extern const pktio_if_ops_t _odp_netmap_pktio_ops; extern const pktio_if_ops_t _odp_dpdk_pktio_ops; +extern const pktio_if_ops_t _odp_sock_xdp_pktio_ops; extern const pktio_if_ops_t _odp_sock_mmsg_pktio_ops; extern const pktio_if_ops_t _odp_sock_mmap_pktio_ops; extern const pktio_if_ops_t _odp_loopback_pktio_ops; diff --git a/platform/linux-generic/include/odp_parse_internal.h b/platform/linux-generic/include/odp_parse_internal.h index 8aa5e118b..22d8c2cf6 100644 --- a/platform/linux-generic/include/odp_parse_internal.h +++ b/platform/linux-generic/include/odp_parse_internal.h @@ -45,8 +45,8 @@ extern "C" { #define PARSE_SCTP_BYTES (sizeof(_odp_sctphdr_t)) /* _odp_packet_parse_common_l3_l4() requires up to this many bytes. */ -#define PARSE_L3_L4_BYTES (MAX(PARSE_IPV4_BYTES, PARSE_IPV6_BYTES) + \ - MAX3(PARSE_TCP_BYTES, PARSE_UDP_BYTES, PARSE_SCTP_BYTES)) +#define PARSE_L3_L4_BYTES (_ODP_MAX(PARSE_IPV4_BYTES, PARSE_IPV6_BYTES) + \ + _ODP_MAX3(PARSE_TCP_BYTES, PARSE_UDP_BYTES, PARSE_SCTP_BYTES)) /* _odp_packet_parse_common() requires up to this many bytes. */ #define PARSE_BYTES (PARSE_ETH_BYTES + PARSE_L3_L4_BYTES) diff --git a/platform/linux-generic/include/odp_pool_internal.h b/platform/linux-generic/include/odp_pool_internal.h index 4c9f9a9ce..824aa9292 100644 --- a/platform/linux-generic/include/odp_pool_internal.h +++ b/platform/linux-generic/include/odp_pool_internal.h @@ -28,6 +28,8 @@ extern "C" { #include <odp_ring_ptr_internal.h> #include <odp/api/plat/strong_types.h> +#define _ODP_POOL_MEM_SRC_DATA_SIZE 128 + typedef struct ODP_ALIGNED_CACHE pool_cache_t { /* Number of buffers in cache */ uint32_t cache_num; @@ -52,8 +54,7 @@ typedef struct ODP_ALIGNED_CACHE { } pool_ring_t; #pragma GCC diagnostic pop -/* Callback function for pool destroy */ -typedef void (*pool_destroy_cb_fn)(void *pool); +struct _odp_pool_mem_src_ops_t; typedef struct pool_t { odp_ticketlock_t lock ODP_ALIGNED_CACHE; @@ -91,14 +92,11 @@ typedef struct pool_t { uint8_t *uarea_base_addr; odp_pool_type_t type_2; odp_pool_ext_param_t ext_param; - - /* Used by DPDK zero-copy pktio */ - uint32_t dpdk_elt_size; uint32_t skipped_blocks; - uint8_t pool_in_use; uint8_t mem_from_huge_pages; - pool_destroy_cb_fn ext_destroy; - void *ext_desc; + const struct _odp_pool_mem_src_ops_t *mem_src_ops; + /* Private area for memory source operations */ + uint8_t mem_src_data[_ODP_POOL_MEM_SRC_DATA_SIZE] ODP_ALIGNED_CACHE; struct ODP_ALIGNED_CACHE { odp_atomic_u64_t alloc_ops; @@ -130,6 +128,25 @@ typedef struct pool_global_t { } pool_global_t; +/* Operations for when ODP packet pool is used as a memory source for e.g. zero-copy packet IO + * purposes */ +typedef struct _odp_pool_mem_src_ops_t { + /* Name of the ops provider */ + const char *name; + /* Signal if ops provider is an active user for the pool as a memory source */ + odp_bool_t (*is_active)(void); + /* Force disable for the ops provider (for now, if one active memory source user is found, + * others are disabled) */ + void (*force_disable)(void); + /* Adjust pool block sizes as required by memory consumer */ + void (*adjust_size)(uint8_t *data, uint32_t *block_size, uint32_t *block_offset, + uint32_t *flags); + /* Bind the pool as a memory source */ + int (*bind)(uint8_t *data, pool_t *pool); + /* Unbind the pool as a memory source */ + void (*unbind)(uint8_t *data); +} _odp_pool_mem_src_ops_t; + extern pool_global_t *_odp_pool_glb; static inline pool_t *pool_entry(uint32_t pool_idx) diff --git a/platform/linux-generic/include/odp_queue_basic_internal.h b/platform/linux-generic/include/odp_queue_basic_internal.h index d8a3226cb..9babb62e9 100644 --- a/platform/linux-generic/include/odp_queue_basic_internal.h +++ b/platform/linux-generic/include/odp_queue_basic_internal.h @@ -17,12 +17,12 @@ extern "C" { #include <odp_forward_typedefs_internal.h> #include <odp_queue_if.h> #include <odp_buffer_internal.h> -#include <odp_align_internal.h> #include <odp/api/packet_io.h> #include <odp/api/align.h> #include <odp/api/hints.h> #include <odp/api/ticketlock.h> #include <odp_config_internal.h> +#include <odp_macros_internal.h> #include <odp_ring_mpmc_internal.h> #include <odp_ring_st_internal.h> #include <odp_ring_spsc_internal.h> @@ -69,7 +69,7 @@ struct queue_entry_s { union queue_entry_u { struct queue_entry_s s; - uint8_t pad[ROUNDUP_CACHE_LINE(sizeof(struct queue_entry_s))]; + uint8_t pad[_ODP_ROUNDUP_CACHE_LINE(sizeof(struct queue_entry_s))]; }; typedef struct queue_global_t { diff --git a/platform/linux-generic/include/odp_queue_scalable_internal.h b/platform/linux-generic/include/odp_queue_scalable_internal.h index 6f9b85c85..3c74d1699 100644 --- a/platform/linux-generic/include/odp_queue_scalable_internal.h +++ b/platform/linux-generic/include/odp_queue_scalable_internal.h @@ -18,12 +18,12 @@ extern "C" { #include <odp_forward_typedefs_internal.h> #include <odp_queue_if.h> #include <odp_event_internal.h> -#include <odp_align_internal.h> #include <odp/api/packet_io.h> #include <odp/api/align.h> #include <odp/api/hints.h> #include <odp/api/ticketlock.h> #include <odp_config_internal.h> +#include <odp_macros_internal.h> #include <odp_schedule_scalable.h> #include <odp_schedule_scalable_ordered.h> @@ -55,7 +55,7 @@ struct queue_entry_s { union queue_entry_u { struct queue_entry_s s; - uint8_t pad[ROUNDUP_CACHE_LINE(sizeof(struct queue_entry_s))]; + uint8_t pad[_ODP_ROUNDUP_CACHE_LINE(sizeof(struct queue_entry_s))]; }; int _odp_queue_deq(sched_elem_t *q, _odp_event_hdr_t *event_hdr[], int num); @@ -71,7 +71,7 @@ static inline void *shm_pool_alloc_align(_odp_ishm_pool_t *pool, uint32_t size) { void *addr; - addr = _odp_ishm_pool_alloc(pool, ROUNDUP_CACHE_LINE(size)); + addr = _odp_ishm_pool_alloc(pool, _ODP_ROUNDUP_CACHE_LINE(size)); ODP_ASSERT(((uintptr_t)addr & (ODP_CACHE_LINE_SIZE - 1)) == 0); return addr; diff --git a/platform/linux-generic/include/odp_ring_internal.h b/platform/linux-generic/include/odp_ring_internal.h index d11e81bf2..961e83448 100644 --- a/platform/linux-generic/include/odp_ring_internal.h +++ b/platform/linux-generic/include/odp_ring_internal.h @@ -15,12 +15,14 @@ extern "C" { #endif +#include <odp/api/align.h> #include <odp/api/atomic.h> #include <odp/api/cpu.h> #include <odp/api/hints.h> -#include <odp_align_internal.h> + #include <odp/api/plat/atomic_inlines.h> #include <odp/api/plat/cpu_inlines.h> + #include <odp_ring_common.h> /* Generic ring implementation diff --git a/platform/linux-generic/include/odp_ring_mpmc_internal.h b/platform/linux-generic/include/odp_ring_mpmc_internal.h index 473e69e90..6ed4dd4d1 100644 --- a/platform/linux-generic/include/odp_ring_mpmc_internal.h +++ b/platform/linux-generic/include/odp_ring_mpmc_internal.h @@ -11,10 +11,11 @@ extern "C" { #endif +#include <odp/api/align.h> #include <odp/api/atomic.h> #include <odp/api/cpu.h> #include <odp/api/hints.h> -#include <odp_align_internal.h> + #include <odp/api/plat/atomic_inlines.h> #include <odp/api/plat/cpu_inlines.h> diff --git a/platform/linux-generic/include/odp_ring_st_internal.h b/platform/linux-generic/include/odp_ring_st_internal.h index 23b012d96..406d043b5 100644 --- a/platform/linux-generic/include/odp_ring_st_internal.h +++ b/platform/linux-generic/include/odp_ring_st_internal.h @@ -11,8 +11,8 @@ extern "C" { #endif +#include <odp/api/align.h> #include <odp/api/hints.h> -#include <odp_align_internal.h> /* Basic ring for single thread usage. Operations must be synchronized by using * locks (or other means), when multiple threads use the same ring. */ diff --git a/platform/linux-generic/include/odp_schedule_if.h b/platform/linux-generic/include/odp_schedule_if.h index cec9c8bb8..d9ba165bc 100644 --- a/platform/linux-generic/include/odp_schedule_if.h +++ b/platform/linux-generic/include/odp_schedule_if.h @@ -131,6 +131,7 @@ typedef struct { uint32_t lock_index); void (*schedule_order_lock_start)(uint32_t lock_index); void (*schedule_order_lock_wait)(uint32_t lock_index); + void (*schedule_order_wait)(void); void (*schedule_print)(void); } schedule_api_t; diff --git a/platform/linux-generic/include/odp_schedule_scalable_ordered.h b/platform/linux-generic/include/odp_schedule_scalable_ordered.h index 21c89bed2..be4894f73 100644 --- a/platform/linux-generic/include/odp_schedule_scalable_ordered.h +++ b/platform/linux-generic/include/odp_schedule_scalable_ordered.h @@ -9,11 +9,12 @@ #ifndef ODP_SCHEDULE_SCALABLE_ORDERED_H #define ODP_SCHEDULE_SCALABLE_ORDERED_H +#include <odp/api/align.h> #include <odp/api/shared_memory.h> -#include <odp_align_internal.h> #include <odp_bitset.h> #include <odp_event_internal.h> +#include <odp_macros_internal.h> #include <odp_ishmpool_internal.h> /* High level functioning of reordering @@ -68,7 +69,7 @@ typedef struct ODP_ALIGNED(sizeof(uint64_t)) hc { * Should be at least one per CPU. */ #define RWIN_SIZE 32 -ODP_STATIC_ASSERT(CHECK_IS_POWER2(RWIN_SIZE), "RWIN_SIZE is not a power of 2"); +ODP_STATIC_ASSERT(_ODP_CHECK_IS_POWER2(RWIN_SIZE), "RWIN_SIZE is not a power of 2"); typedef struct reorder_context reorder_context_t; |