diff options
author | Bill Fischofer <bill.fischofer@linaro.org> | 2014-12-15 19:09:49 +0200 |
---|---|---|
committer | Maxim Uvarov <maxim.uvarov@linaro.org> | 2014-12-16 01:26:23 +0300 |
commit | 0d934707499ff6770829d6f305bf3679247d8d0f (patch) | |
tree | 83c361e8dbc6214613d647e957589a1defad26b9 /platform/linux-generic/include | |
parent | 2fadaf4fef2b309da5c7477a9848737ec43e462e (diff) |
api: buffer: change pool create
Modify pool create API to be able to specify number of buffers directly.
Signed-off-by: Bill Fischofer <bill.fischofer@linaro.org>
Signed-off-by: Taras Kondratiuk <taras.kondratiuk@linaro.org>
Reviewed-and-tested-by: Bill Fischofer <bill.fischofer@linaro.org>
Signed-off-by: Maxim Uvarov <maxim.uvarov@linaro.org>
Diffstat (limited to 'platform/linux-generic/include')
-rw-r--r-- | platform/linux-generic/include/api/odp_buffer_pool.h | 47 | ||||
-rw-r--r-- | platform/linux-generic/include/api/odp_platform_types.h | 3 | ||||
-rw-r--r-- | platform/linux-generic/include/odp_buffer_inlines.h | 150 | ||||
-rw-r--r-- | platform/linux-generic/include/odp_buffer_internal.h | 152 | ||||
-rw-r--r-- | platform/linux-generic/include/odp_buffer_pool_internal.h | 346 | ||||
-rw-r--r-- | platform/linux-generic/include/odp_internal.h | 2 | ||||
-rw-r--r-- | platform/linux-generic/include/odp_packet_internal.h | 50 | ||||
-rw-r--r-- | platform/linux-generic/include/odp_timer_internal.h | 11 |
8 files changed, 629 insertions, 132 deletions
diff --git a/platform/linux-generic/include/api/odp_buffer_pool.h b/platform/linux-generic/include/api/odp_buffer_pool.h index b287f79d8..4ccdfe613 100644 --- a/platform/linux-generic/include/api/odp_buffer_pool.h +++ b/platform/linux-generic/include/api/odp_buffer_pool.h @@ -33,21 +33,48 @@ extern "C" { #define ODP_BUFFER_POOL_NAME_LEN 32 /** + * Buffer pool parameters + * Used to communicate buffer pool creation options. + */ +typedef struct odp_buffer_pool_param_t { + uint32_t buf_size; /**< Buffer size in bytes. The maximum + number of bytes application will + store in each buffer. For packets, this + is the maximum packet data length, and + configured headroom and tailroom will be + added to this number */ + uint32_t buf_align; /**< Minimum buffer alignment in bytes. + Valid values are powers of two. Use 0 + for default alignment. Default will + always be a multiple of 8. */ + uint32_t num_bufs; /**< Number of buffers in the pool */ + int buf_type; /**< Buffer type */ +} odp_buffer_pool_param_t; + +/** * Create a buffer pool + * This routine is used to create a buffer pool. It take three + * arguments: the optional name of the pool to be created, an optional shared + * memory handle, and a parameter struct that describes the pool to be + * created. If a name is not specified the result is an anonymous pool that + * cannot be referenced by odp_buffer_pool_lookup(). * - * @param name Name of the pool (max ODP_BUFFER_POOL_NAME_LEN - 1 chars) - * @param base_addr Pool base address - * @param size Pool size in bytes - * @param buf_size Buffer size in bytes - * @param buf_align Minimum buffer alignment - * @param buf_type Buffer type + * @param name Name of the pool, max ODP_BUFFER_POOL_NAME_LEN-1 chars. + * May be specified as NULL for anonymous pools. * - * @return Buffer pool handle + * @param shm The shared memory object in which to create the pool. + * Use ODP_SHM_NULL to reserve default memory type + * for the buffer type. + * + * @param params Buffer pool parameters. + * + * @return Handle of the created buffer pool + * @retval ODP_BUFFER_POOL_INVALID Buffer pool could not be created */ + odp_buffer_pool_t odp_buffer_pool_create(const char *name, - void *base_addr, uint64_t size, - size_t buf_size, size_t buf_align, - int buf_type); + odp_shm_t shm, + odp_buffer_pool_param_t *params); /** diff --git a/platform/linux-generic/include/api/odp_platform_types.h b/platform/linux-generic/include/api/odp_platform_types.h index 68d0c55d0..2181eb695 100644 --- a/platform/linux-generic/include/api/odp_platform_types.h +++ b/platform/linux-generic/include/api/odp_platform_types.h @@ -27,7 +27,7 @@ typedef uint32_t odp_buffer_pool_t; /** Invalid buffer pool */ -#define ODP_BUFFER_POOL_INVALID (0) +#define ODP_BUFFER_POOL_INVALID (0xffffffff) /** ODP buffer */ typedef uint32_t odp_buffer_t; @@ -74,6 +74,7 @@ typedef uint32_t odp_shm_t; /** Invalid shared memory block */ #define ODP_SHM_INVALID 0 +#define ODP_SHM_NULL ODP_SHM_INVALID /**< Synonym for buffer pool use */ /** * @} diff --git a/platform/linux-generic/include/odp_buffer_inlines.h b/platform/linux-generic/include/odp_buffer_inlines.h new file mode 100644 index 000000000..6227482f6 --- /dev/null +++ b/platform/linux-generic/include/odp_buffer_inlines.h @@ -0,0 +1,150 @@ +/* Copyright (c) 2014, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +/** + * @file + * + * Inline functions for ODP buffer mgmt routines - implementation internal + */ + +#ifndef ODP_BUFFER_INLINES_H_ +#define ODP_BUFFER_INLINES_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +static inline odp_buffer_t odp_buffer_encode_handle(odp_buffer_hdr_t *hdr) +{ + odp_buffer_bits_t handle; + uint32_t pool_id = pool_handle_to_index(hdr->pool_hdl); + struct pool_entry_s *pool = get_pool_entry(pool_id); + + handle.pool_id = pool_id; + handle.index = ((uint8_t *)hdr - pool->pool_mdata_addr) / + ODP_CACHE_LINE_SIZE; + handle.seg = 0; + + return handle.u32; +} + +static inline odp_buffer_t odp_hdr_to_buf(odp_buffer_hdr_t *hdr) +{ + return hdr->handle.handle; +} + +static inline odp_buffer_hdr_t *odp_buf_to_hdr(odp_buffer_t buf) +{ + odp_buffer_bits_t handle; + uint32_t pool_id; + uint32_t index; + struct pool_entry_s *pool; + + handle.u32 = buf; + pool_id = handle.pool_id; + index = handle.index; + +#ifdef POOL_ERROR_CHECK + if (odp_unlikely(pool_id > ODP_CONFIG_BUFFER_POOLS)) { + ODP_ERR("odp_buf_to_hdr: Bad pool id\n"); + return NULL; + } +#endif + + pool = get_pool_entry(pool_id); + +#ifdef POOL_ERROR_CHECK + if (odp_unlikely(index > pool->params.num_bufs - 1)) { + ODP_ERR("odp_buf_to_hdr: Bad buffer index\n"); + return NULL; + } +#endif + + return (odp_buffer_hdr_t *)(void *) + (pool->pool_mdata_addr + (index * ODP_CACHE_LINE_SIZE)); +} + +static inline uint32_t odp_buffer_refcount(odp_buffer_hdr_t *buf) +{ + return odp_atomic_load_u32(&buf->ref_count); +} + +static inline uint32_t odp_buffer_incr_refcount(odp_buffer_hdr_t *buf, + uint32_t val) +{ + return odp_atomic_fetch_add_u32(&buf->ref_count, val) + val; +} + +static inline uint32_t odp_buffer_decr_refcount(odp_buffer_hdr_t *buf, + uint32_t val) +{ + uint32_t tmp; + + tmp = odp_atomic_fetch_sub_u32(&buf->ref_count, val); + + if (tmp < val) { + odp_atomic_fetch_add_u32(&buf->ref_count, val - tmp); + return 0; + } else { + return tmp - val; + } +} + +static inline odp_buffer_hdr_t *validate_buf(odp_buffer_t buf) +{ + odp_buffer_bits_t handle; + odp_buffer_hdr_t *buf_hdr; + handle.u32 = buf; + + /* For buffer handles, segment index must be 0 and pool id in range */ + if (handle.seg != 0 || handle.pool_id >= ODP_CONFIG_BUFFER_POOLS) + return NULL; + + pool_entry_t *pool = odp_pool_to_entry(handle.pool_id); + + /* If pool not created, handle is invalid */ + if (pool->s.pool_shm == ODP_SHM_INVALID) + return NULL; + + uint32_t buf_stride = pool->s.buf_stride / ODP_CACHE_LINE_SIZE; + + /* A valid buffer index must be on stride, and must be in range */ + if ((handle.index % buf_stride != 0) || + ((uint32_t)(handle.index / buf_stride) >= pool->s.params.num_bufs)) + return NULL; + + buf_hdr = (odp_buffer_hdr_t *)(void *) + (pool->s.pool_mdata_addr + + (handle.index * ODP_CACHE_LINE_SIZE)); + + /* Handle is valid, so buffer is valid if it is allocated */ + return buf_hdr->allocator == ODP_FREEBUF ? NULL : buf_hdr; +} + +int odp_buffer_snprint(char *str, uint32_t n, odp_buffer_t buf); + +static inline void *buffer_map(odp_buffer_hdr_t *buf, + uint32_t offset, + uint32_t *seglen, + uint32_t limit) +{ + int seg_index = offset / buf->segsize; + int seg_offset = offset % buf->segsize; + + if (seglen != NULL) { + uint32_t buf_left = limit - offset; + *seglen = buf_left < buf->segsize ? + buf_left : buf->segsize - seg_offset; + } + + return (void *)(seg_offset + (uint8_t *)buf->addr[seg_index]); +} + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/include/odp_buffer_internal.h b/platform/linux-generic/include/odp_buffer_internal.h index 0027bfcf3..859633e3c 100644 --- a/platform/linux-generic/include/odp_buffer_internal.h +++ b/platform/linux-generic/include/odp_buffer_internal.h @@ -24,99 +24,133 @@ extern "C" { #include <odp_buffer.h> #include <odp_debug.h> #include <odp_align.h> - -/* TODO: move these to correct files */ - -typedef uint64_t odp_phys_addr_t; +#include <odp_align_internal.h> +#include <odp_config.h> +#include <odp_byteorder.h> +#include <odp_thread.h> + + +#define ODP_BITSIZE(x) \ + ((x) <= 2 ? 1 : \ + ((x) <= 4 ? 2 : \ + ((x) <= 8 ? 3 : \ + ((x) <= 16 ? 4 : \ + ((x) <= 32 ? 5 : \ + ((x) <= 64 ? 6 : \ + ((x) <= 128 ? 7 : \ + ((x) <= 256 ? 8 : \ + ((x) <= 512 ? 9 : \ + ((x) <= 1024 ? 10 : \ + ((x) <= 2048 ? 11 : \ + ((x) <= 4096 ? 12 : \ + ((x) <= 8196 ? 13 : \ + ((x) <= 16384 ? 14 : \ + ((x) <= 32768 ? 15 : \ + ((x) <= 65536 ? 16 : \ + (0/0))))))))))))))))) + +ODP_STATIC_ASSERT(ODP_CONFIG_PACKET_BUF_LEN_MIN >= 256, + "ODP Segment size must be a minimum of 256 bytes"); + +ODP_STATIC_ASSERT((ODP_CONFIG_PACKET_BUF_LEN_MIN % ODP_CACHE_LINE_SIZE) == 0, + "ODP Segment size must be a multiple of cache line size"); + +ODP_STATIC_ASSERT((ODP_CONFIG_PACKET_BUF_LEN_MAX % + ODP_CONFIG_PACKET_BUF_LEN_MIN) == 0, + "Packet max size must be a multiple of segment size"); + +#define ODP_BUFFER_MAX_SEG \ + (ODP_CONFIG_PACKET_BUF_LEN_MAX / ODP_CONFIG_PACKET_BUF_LEN_MIN) + +/* We can optimize storage of small raw buffers within metadata area */ +#define ODP_MAX_INLINE_BUF ((sizeof(void *)) * (ODP_BUFFER_MAX_SEG - 1)) + +#define ODP_BUFFER_POOL_BITS ODP_BITSIZE(ODP_CONFIG_BUFFER_POOLS) +#define ODP_BUFFER_SEG_BITS ODP_BITSIZE(ODP_BUFFER_MAX_SEG) +#define ODP_BUFFER_INDEX_BITS (32 - ODP_BUFFER_POOL_BITS - ODP_BUFFER_SEG_BITS) +#define ODP_BUFFER_PREFIX_BITS (ODP_BUFFER_POOL_BITS + ODP_BUFFER_INDEX_BITS) +#define ODP_BUFFER_MAX_POOLS (1 << ODP_BUFFER_POOL_BITS) +#define ODP_BUFFER_MAX_BUFFERS (1 << ODP_BUFFER_INDEX_BITS) #define ODP_BUFFER_MAX_INDEX (ODP_BUFFER_MAX_BUFFERS - 2) #define ODP_BUFFER_INVALID_INDEX (ODP_BUFFER_MAX_BUFFERS - 1) -#define ODP_BUFS_PER_CHUNK 16 -#define ODP_BUFS_PER_SCATTER 4 - -#define ODP_BUFFER_TYPE_CHUNK 0xffff - - -#define ODP_BUFFER_POOL_BITS 4 -#define ODP_BUFFER_INDEX_BITS (32 - ODP_BUFFER_POOL_BITS) -#define ODP_BUFFER_MAX_POOLS (1 << ODP_BUFFER_POOL_BITS) -#define ODP_BUFFER_MAX_BUFFERS (1 << ODP_BUFFER_INDEX_BITS) - typedef union odp_buffer_bits_t { uint32_t u32; odp_buffer_t handle; struct { +#if ODP_BYTE_ORDER == ODP_BIG_ENDIAN uint32_t pool_id:ODP_BUFFER_POOL_BITS; uint32_t index:ODP_BUFFER_INDEX_BITS; + uint32_t seg:ODP_BUFFER_SEG_BITS; +#else + uint32_t seg:ODP_BUFFER_SEG_BITS; + uint32_t index:ODP_BUFFER_INDEX_BITS; + uint32_t pool_id:ODP_BUFFER_POOL_BITS; +#endif }; -} odp_buffer_bits_t; + struct { +#if ODP_BYTE_ORDER == ODP_BIG_ENDIAN + uint32_t prefix:ODP_BUFFER_PREFIX_BITS; + uint32_t pfxseg:ODP_BUFFER_SEG_BITS; +#else + uint32_t pfxseg:ODP_BUFFER_SEG_BITS; + uint32_t prefix:ODP_BUFFER_PREFIX_BITS; +#endif + }; +} odp_buffer_bits_t; /* forward declaration */ struct odp_buffer_hdr_t; - -/* - * Scatter/gather list of buffers - */ -typedef struct odp_buffer_scatter_t { - /* buffer pointers */ - struct odp_buffer_hdr_t *buf[ODP_BUFS_PER_SCATTER]; - int num_bufs; /* num buffers */ - int pos; /* position on the list */ - size_t total_len; /* Total length */ -} odp_buffer_scatter_t; - - -/* - * Chunk of buffers (in single pool) - */ -typedef struct odp_buffer_chunk_t { - uint32_t num_bufs; /* num buffers */ - uint32_t buf_index[ODP_BUFS_PER_CHUNK]; /* buffers */ -} odp_buffer_chunk_t; - - /* Common buffer header */ typedef struct odp_buffer_hdr_t { struct odp_buffer_hdr_t *next; /* next buf in a list */ + int allocator; /* allocating thread id */ odp_buffer_bits_t handle; /* handle */ - odp_phys_addr_t phys_addr; /* physical data start address */ - void *addr; /* virtual data start address */ - uint32_t index; /* buf index in the pool */ + union { + uint32_t all; + struct { + uint32_t zeroized:1; /* Zeroize buf data on free */ + uint32_t hdrdata:1; /* Data is in buffer hdr */ + }; + } flags; + int type; /* buffer type */ size_t size; /* max data size */ - size_t cur_offset; /* current offset */ odp_atomic_u32_t ref_count; /* reference count */ - odp_buffer_scatter_t scatter; /* Scatter/gather list */ - int type; /* type of next header */ odp_buffer_pool_t pool_hdl; /* buffer pool handle */ - + union { + uint64_t buf_u64; /* user u64 */ + void *buf_ctx; /* user context */ + void *udata_addr; /* user metadata addr */ + }; + size_t udata_size; /* size of user metadata */ + uint32_t segcount; /* segment count */ + uint32_t segsize; /* segment size */ + void *addr[ODP_BUFFER_MAX_SEG]; /* block addrs */ } odp_buffer_hdr_t; -/* Ensure next header starts from 8 byte align */ -ODP_STATIC_ASSERT((sizeof(odp_buffer_hdr_t) % 8) == 0, "ODP_BUFFER_HDR_T__SIZE_ERROR"); +typedef struct odp_buffer_hdr_stride { + uint8_t pad[ODP_CACHE_LINE_SIZE_ROUNDUP(sizeof(odp_buffer_hdr_t))]; +} odp_buffer_hdr_stride; +typedef struct odp_buf_blk_t { + struct odp_buf_blk_t *next; + struct odp_buf_blk_t *prev; +} odp_buf_blk_t; /* Raw buffer header */ typedef struct { odp_buffer_hdr_t buf_hdr; /* common buffer header */ - uint8_t buf_data[]; /* start of buffer data area */ } odp_raw_buffer_hdr_t; +/* Free buffer marker */ +#define ODP_FREEBUF -1 -/* Chunk header */ -typedef struct odp_buffer_chunk_hdr_t { - odp_buffer_hdr_t buf_hdr; - odp_buffer_chunk_t chunk; -} odp_buffer_chunk_hdr_t; - - -int odp_buffer_snprint(char *str, size_t n, odp_buffer_t buf); - -void odp_buffer_copy_scatter(odp_buffer_t buf_dst, odp_buffer_t buf_src); - +/* Forward declarations */ +odp_buffer_t buffer_alloc(odp_buffer_pool_t pool, size_t size); #ifdef __cplusplus } diff --git a/platform/linux-generic/include/odp_buffer_pool_internal.h b/platform/linux-generic/include/odp_buffer_pool_internal.h index 07602feb9..2e48ac3f7 100644 --- a/platform/linux-generic/include/odp_buffer_pool_internal.h +++ b/platform/linux-generic/include/odp_buffer_pool_internal.h @@ -19,13 +19,44 @@ extern "C" { #endif #include <odp_std_types.h> -#include <odp_buffer_pool.h> -#include <odp_buffer_internal.h> #include <odp_align.h> #include <odp_align_internal.h> +#include <odp_buffer_pool.h> +#include <odp_buffer_internal.h> #include <odp_hints.h> #include <odp_config.h> #include <odp_debug.h> +#include <odp_shared_memory.h> +#include <odp_atomic.h> +#include <odp_atomic_internal.h> +#include <string.h> + +/** + * Buffer initialization routine prototype + * + * @note Routines of this type MAY be passed as part of the + * _odp_buffer_pool_init_t structure to be called whenever a + * buffer is allocated to initialize the user metadata + * associated with that buffer. + */ +typedef void (_odp_buf_init_t)(odp_buffer_t buf, void *buf_init_arg); + +/** + * Buffer pool initialization parameters + * Used to communicate buffer pool initialization options. Internal for now. + */ +typedef struct _odp_buffer_pool_init_t { + size_t udata_size; /**< Size of user metadata for each buffer */ + _odp_buf_init_t *buf_init; /**< Buffer initialization routine to use */ + void *buf_init_arg; /**< Argument to be passed to buf_init() */ +} _odp_buffer_pool_init_t; /**< Type of buffer initialization struct */ + +/* Local cache for buffer alloc/free acceleration */ +typedef struct local_cache_t { + odp_buffer_hdr_t *buf_freelist; /* The local cache */ + uint64_t bufallocs; /* Local buffer alloc count */ + uint64_t buffrees; /* Local buffer free count */ +} local_cache_t; /* Use ticketlock instead of spinlock */ #define POOL_USE_TICKETLOCK @@ -36,11 +67,16 @@ extern "C" { #ifdef POOL_USE_TICKETLOCK #include <odp_ticketlock.h> +#define POOL_LOCK(a) odp_ticketlock_lock(a) +#define POOL_UNLOCK(a) odp_ticketlock_unlock(a) +#define POOL_LOCK_INIT(a) odp_ticketlock_init(a) #else #include <odp_spinlock.h> +#define POOL_LOCK(a) odp_spinlock_lock(a) +#define POOL_UNLOCK(a) odp_spinlock_unlock(a) +#define POOL_LOCK_INIT(a) odp_spinlock_init(a) #endif - struct pool_entry_s { #ifdef POOL_USE_TICKETLOCK odp_ticketlock_t lock ODP_ALIGNED_CACHE; @@ -48,74 +84,294 @@ struct pool_entry_s { odp_spinlock_t lock ODP_ALIGNED_CACHE; #endif - odp_buffer_chunk_hdr_t *head; - uint64_t free_bufs; char name[ODP_BUFFER_POOL_NAME_LEN]; - - odp_buffer_pool_t pool_hdl ODP_ALIGNED_CACHE; - uintptr_t buf_base; - size_t buf_size; - size_t buf_offset; - uint64_t num_bufs; - void *pool_base_addr; - uint64_t pool_size; - size_t user_size; - size_t user_align; - int buf_type; - size_t hdr_size; + odp_buffer_pool_param_t params; + _odp_buffer_pool_init_t init_params; + odp_buffer_pool_t pool_hdl; + uint32_t pool_id; + odp_shm_t pool_shm; + union { + uint32_t all; + struct { + uint32_t has_name:1; + uint32_t user_supplied_shm:1; + uint32_t unsegmented:1; + uint32_t zeroized:1; + uint32_t predefined:1; + }; + } flags; + uint32_t quiesced; + uint32_t low_wm_assert; + uint8_t *pool_base_addr; + uint8_t *pool_mdata_addr; + size_t pool_size; + uint32_t buf_align; + uint32_t buf_stride; + _odp_atomic_ptr_t buf_freelist; + _odp_atomic_ptr_t blk_freelist; + odp_atomic_u32_t bufcount; + odp_atomic_u32_t blkcount; + odp_atomic_u64_t bufallocs; + odp_atomic_u64_t buffrees; + odp_atomic_u64_t blkallocs; + odp_atomic_u64_t blkfrees; + odp_atomic_u64_t bufempty; + odp_atomic_u64_t blkempty; + odp_atomic_u64_t high_wm_count; + odp_atomic_u64_t low_wm_count; + uint32_t seg_size; + uint32_t high_wm; + uint32_t low_wm; + uint32_t headroom; + uint32_t tailroom; }; typedef union pool_entry_u { struct pool_entry_s s; + uint8_t pad[ODP_CACHE_LINE_SIZE_ROUNDUP(sizeof(struct pool_entry_s))]; } pool_entry_t; extern void *pool_entry_ptr[]; +#if defined(ODP_CONFIG_SECURE_POOLS) && (ODP_CONFIG_SECURE_POOLS == 1) +#define buffer_is_secure(buf) (buf->flags.zeroized) +#define pool_is_secure(pool) (pool->flags.zeroized) +#else +#define buffer_is_secure(buf) 0 +#define pool_is_secure(pool) 0 +#endif -static inline void *get_pool_entry(uint32_t pool_id) +#define TAG_ALIGN ((size_t)16) + +#define odp_cs(ptr, old, new) \ + _odp_atomic_ptr_cmp_xchg_strong(&ptr, (void **)&old, (void *)new, \ + _ODP_MEMMODEL_SC, \ + _ODP_MEMMODEL_SC) + +/* Helper functions for pointer tagging to avoid ABA race conditions */ +#define odp_tag(ptr) \ + (((size_t)ptr) & (TAG_ALIGN - 1)) + +#define odp_detag(ptr) \ + ((void *)(((size_t)ptr) & -TAG_ALIGN)) + +#define odp_retag(ptr, tag) \ + ((void *)(((size_t)ptr) | odp_tag(tag))) + + +static inline void *get_blk(struct pool_entry_s *pool) { - return pool_entry_ptr[pool_id]; + void *oldhead, *myhead, *newhead; + + oldhead = _odp_atomic_ptr_load(&pool->blk_freelist, _ODP_MEMMODEL_ACQ); + + do { + size_t tag = odp_tag(oldhead); + myhead = odp_detag(oldhead); + if (odp_unlikely(myhead == NULL)) + break; + newhead = odp_retag(((odp_buf_blk_t *)myhead)->next, tag + 1); + } while (odp_cs(pool->blk_freelist, oldhead, newhead) == 0); + + if (odp_unlikely(myhead == NULL)) + odp_atomic_inc_u64(&pool->blkempty); + else + odp_atomic_dec_u32(&pool->blkcount); + + return (void *)myhead; } -static inline uint32_t pool_handle_to_index(odp_buffer_pool_t pool_hdl) +static inline void ret_blk(struct pool_entry_s *pool, void *block) { - return pool_hdl - 1; + void *oldhead, *myhead, *myblock; + + oldhead = _odp_atomic_ptr_load(&pool->blk_freelist, _ODP_MEMMODEL_ACQ); + + do { + size_t tag = odp_tag(oldhead); + myhead = odp_detag(oldhead); + ((odp_buf_blk_t *)block)->next = myhead; + myblock = odp_retag(block, tag + 1); + } while (odp_cs(pool->blk_freelist, oldhead, myblock) == 0); + + odp_atomic_inc_u32(&pool->blkcount); + odp_atomic_inc_u64(&pool->blkfrees); } -static inline odp_buffer_hdr_t *odp_buf_to_hdr(odp_buffer_t buf) +static inline odp_buffer_hdr_t *get_buf(struct pool_entry_s *pool) { - odp_buffer_bits_t handle; - uint32_t pool_id; - uint32_t index; - struct pool_entry_s *pool; - odp_buffer_hdr_t *hdr; - - handle.u32 = buf; - pool_id = handle.pool_id; - index = handle.index; - -#ifdef POOL_ERROR_CHECK - if (odp_unlikely(pool_id > ODP_CONFIG_BUFFER_POOLS)) { - ODP_ERR("odp_buf_to_hdr: Bad pool id\n"); - return NULL; + odp_buffer_hdr_t *oldhead, *myhead, *newhead; + + oldhead = _odp_atomic_ptr_load(&pool->buf_freelist, _ODP_MEMMODEL_ACQ); + + do { + size_t tag = odp_tag(oldhead); + myhead = odp_detag(oldhead); + if (odp_unlikely(myhead == NULL)) + break; + newhead = odp_retag(myhead->next, tag + 1); + } while (odp_cs(pool->buf_freelist, oldhead, newhead) == 0); + + if (odp_unlikely(myhead == NULL)) { + odp_atomic_inc_u64(&pool->bufempty); + } else { + uint64_t bufcount = + odp_atomic_fetch_sub_u32(&pool->bufcount, 1) - 1; + + /* Check for low watermark condition */ + if (bufcount == pool->low_wm && !pool->low_wm_assert) { + pool->low_wm_assert = 1; + odp_atomic_inc_u64(&pool->low_wm_count); + } + + odp_atomic_inc_u64(&pool->bufallocs); + myhead->next = myhead; /* Mark buffer allocated */ + myhead->allocator = odp_thread_id(); } -#endif - pool = get_pool_entry(pool_id); + return (void *)myhead; +} + +static inline void ret_buf(struct pool_entry_s *pool, odp_buffer_hdr_t *buf) +{ + odp_buffer_hdr_t *oldhead, *myhead, *mybuf; + + buf->allocator = ODP_FREEBUF; /* Mark buffer free */ -#ifdef POOL_ERROR_CHECK - if (odp_unlikely(index > pool->num_bufs - 1)) { - ODP_ERR("odp_buf_to_hdr: Bad buffer index\n"); - return NULL; + if (!buf->flags.hdrdata && buf->type != ODP_BUFFER_TYPE_RAW) { + while (buf->segcount > 0) { + if (buffer_is_secure(buf) || pool_is_secure(pool)) + memset(buf->addr[buf->segcount - 1], + 0, buf->segsize); + ret_blk(pool, buf->addr[--buf->segcount]); + } + buf->size = 0; } -#endif - hdr = (odp_buffer_hdr_t *)(pool->buf_base + index * pool->buf_size); + oldhead = _odp_atomic_ptr_load(&pool->buf_freelist, _ODP_MEMMODEL_ACQ); + + do { + size_t tag = odp_tag(oldhead); + myhead = odp_detag(oldhead); + buf->next = myhead; + mybuf = odp_retag(buf, tag + 1); + } while (odp_cs(pool->buf_freelist, oldhead, mybuf) == 0); + + uint64_t bufcount = odp_atomic_fetch_add_u32(&pool->bufcount, 1) + 1; + + /* Check if low watermark condition should be deasserted */ + if (bufcount == pool->high_wm && pool->low_wm_assert) { + pool->low_wm_assert = 0; + odp_atomic_inc_u64(&pool->high_wm_count); + } - return hdr; + odp_atomic_inc_u64(&pool->buffrees); } +static inline void *get_local_buf(local_cache_t *buf_cache, + struct pool_entry_s *pool, + size_t totsize) +{ + odp_buffer_hdr_t *buf = buf_cache->buf_freelist; + + if (odp_likely(buf != NULL)) { + buf_cache->buf_freelist = buf->next; + + if (odp_unlikely(buf->size < totsize)) { + intmax_t needed = totsize - buf->size; + + do { + void *blk = get_blk(pool); + if (odp_unlikely(blk == NULL)) { + ret_buf(pool, buf); + buf_cache->buffrees--; + return NULL; + } + buf->addr[buf->segcount++] = blk; + needed -= pool->seg_size; + } while (needed > 0); + + buf->size = buf->segcount * pool->seg_size; + } + + buf_cache->bufallocs++; + buf->allocator = odp_thread_id(); /* Mark buffer allocated */ + } + + return buf; +} + +static inline void ret_local_buf(local_cache_t *buf_cache, + odp_buffer_hdr_t *buf) +{ + buf->allocator = ODP_FREEBUF; + buf->next = buf_cache->buf_freelist; + buf_cache->buf_freelist = buf; + + buf_cache->buffrees++; +} + +static inline void flush_cache(local_cache_t *buf_cache, + struct pool_entry_s *pool) +{ + odp_buffer_hdr_t *buf = buf_cache->buf_freelist; + uint32_t flush_count = 0; + + while (buf != NULL) { + odp_buffer_hdr_t *next = buf->next; + ret_buf(pool, buf); + buf = next; + flush_count++; + } + + odp_atomic_add_u64(&pool->bufallocs, buf_cache->bufallocs); + odp_atomic_add_u64(&pool->buffrees, buf_cache->buffrees - flush_count); + + buf_cache->buf_freelist = NULL; + buf_cache->bufallocs = 0; + buf_cache->buffrees = 0; +} + +static inline odp_buffer_pool_t pool_index_to_handle(uint32_t pool_id) +{ + return pool_id; +} + +static inline uint32_t pool_handle_to_index(odp_buffer_pool_t pool_hdl) +{ + return pool_hdl; +} + +static inline void *get_pool_entry(uint32_t pool_id) +{ + return pool_entry_ptr[pool_id]; +} + +static inline pool_entry_t *odp_pool_to_entry(odp_buffer_pool_t pool) +{ + return (pool_entry_t *)get_pool_entry(pool_handle_to_index(pool)); +} + +static inline pool_entry_t *odp_buf_to_pool(odp_buffer_hdr_t *buf) +{ + return odp_pool_to_entry(buf->pool_hdl); +} + +static inline uint32_t odp_buffer_pool_segment_size(odp_buffer_pool_t pool) +{ + return odp_pool_to_entry(pool)->s.seg_size; +} + +static inline uint32_t odp_buffer_pool_headroom(odp_buffer_pool_t pool) +{ + return odp_pool_to_entry(pool)->s.headroom; +} + +static inline uint32_t odp_buffer_pool_tailroom(odp_buffer_pool_t pool) +{ + return odp_pool_to_entry(pool)->s.tailroom; +} #ifdef __cplusplus } diff --git a/platform/linux-generic/include/odp_internal.h b/platform/linux-generic/include/odp_internal.h index 04c1030b3..ee2ab1a0d 100644 --- a/platform/linux-generic/include/odp_internal.h +++ b/platform/linux-generic/include/odp_internal.h @@ -44,6 +44,8 @@ int odp_schedule_init_local(void); int odp_timer_init_global(void); int odp_timer_disarm_all(void); +void _odp_flush_caches(void); + #ifdef __cplusplus } #endif diff --git a/platform/linux-generic/include/odp_packet_internal.h b/platform/linux-generic/include/odp_packet_internal.h index 49c59b28d..f34a83dda 100644 --- a/platform/linux-generic/include/odp_packet_internal.h +++ b/platform/linux-generic/include/odp_packet_internal.h @@ -22,6 +22,7 @@ extern "C" { #include <odp_debug.h> #include <odp_buffer_internal.h> #include <odp_buffer_pool_internal.h> +#include <odp_buffer_inlines.h> #include <odp_packet.h> #include <odp_packet_io.h> @@ -92,7 +93,8 @@ typedef union { }; } output_flags_t; -ODP_STATIC_ASSERT(sizeof(output_flags_t) == sizeof(uint32_t), "OUTPUT_FLAGS_SIZE_ERROR"); +ODP_STATIC_ASSERT(sizeof(output_flags_t) == sizeof(uint32_t), + "OUTPUT_FLAGS_SIZE_ERROR"); /** * Internal Packet header @@ -105,25 +107,23 @@ typedef struct { error_flags_t error_flags; output_flags_t output_flags; - uint32_t frame_offset; /**< offset to start of frame, even on error */ uint32_t l2_offset; /**< offset to L2 hdr, e.g. Eth */ uint32_t l3_offset; /**< offset to L3 hdr, e.g. IPv4, IPv6 */ uint32_t l4_offset; /**< offset to L4 hdr (TCP, UDP, SCTP, also ICMP) */ uint32_t frame_len; + uint32_t headroom; + uint32_t tailroom; uint64_t user_ctx; /* user context */ odp_pktio_t input; - - uint32_t pad; - uint8_t buf_data[]; /* start of buffer data area */ } odp_packet_hdr_t; -ODP_STATIC_ASSERT(sizeof(odp_packet_hdr_t) == ODP_OFFSETOF(odp_packet_hdr_t, buf_data), - "ODP_PACKET_HDR_T__SIZE_ERR"); -ODP_STATIC_ASSERT(sizeof(odp_packet_hdr_t) % sizeof(uint64_t) == 0, - "ODP_PACKET_HDR_T__SIZE_ERR2"); +typedef struct odp_packet_hdr_stride { + uint8_t pad[ODP_CACHE_LINE_SIZE_ROUNDUP(sizeof(odp_packet_hdr_t))]; +} odp_packet_hdr_stride; + /** * Return the packet header @@ -138,6 +138,38 @@ static inline odp_packet_hdr_t *odp_packet_hdr(odp_packet_t pkt) */ void odp_packet_parse(odp_packet_t pkt, size_t len, size_t l2_offset); +/** + * Initialize packet buffer + */ +static inline void packet_init(pool_entry_t *pool, + odp_packet_hdr_t *pkt_hdr, + size_t size) +{ + /* + * Reset parser metadata. Note that we clear via memset to make + * this routine indepenent of any additional adds to packet metadata. + */ + const size_t start_offset = ODP_FIELD_SIZEOF(odp_packet_hdr_t, buf_hdr); + uint8_t *start; + size_t len; + + start = (uint8_t *)pkt_hdr + start_offset; + len = sizeof(odp_packet_hdr_t) - start_offset; + memset(start, 0, len); + + /* + * Packet headroom is set from the pool's headroom + * Packet tailroom is rounded up to fill the last + * segment occupied by the allocated length. + */ + pkt_hdr->frame_len = size; + pkt_hdr->headroom = pool->s.headroom; + pkt_hdr->tailroom = + (pool->s.seg_size * pkt_hdr->buf_hdr.segcount) - + (pool->s.headroom + size); +} + + #ifdef __cplusplus } #endif diff --git a/platform/linux-generic/include/odp_timer_internal.h b/platform/linux-generic/include/odp_timer_internal.h index ad28f534e..2ff36ce0d 100644 --- a/platform/linux-generic/include/odp_timer_internal.h +++ b/platform/linux-generic/include/odp_timer_internal.h @@ -51,14 +51,9 @@ typedef struct odp_timeout_hdr_t { uint8_t buf_data[]; } odp_timeout_hdr_t; - - -ODP_STATIC_ASSERT(sizeof(odp_timeout_hdr_t) == - ODP_OFFSETOF(odp_timeout_hdr_t, buf_data), - "ODP_TIMEOUT_HDR_T__SIZE_ERR"); - -ODP_STATIC_ASSERT(sizeof(odp_timeout_hdr_t) % sizeof(uint64_t) == 0, - "ODP_TIMEOUT_HDR_T__SIZE_ERR2"); +typedef struct odp_timeout_hdr_stride { + uint8_t pad[ODP_CACHE_LINE_SIZE_ROUNDUP(sizeof(odp_timeout_hdr_t))]; +} odp_timeout_hdr_stride; /** |