api: buffer: change pool create

Modify pool create API to be able to specify number of buffers directly.

Signed-off-by: Bill Fischofer <bill.fischofer@linaro.org>
Signed-off-by: Taras Kondratiuk <taras.kondratiuk@linaro.org>
Reviewed-and-tested-by: Bill Fischofer <bill.fischofer@linaro.org>
Signed-off-by: Maxim Uvarov <maxim.uvarov@linaro.org>

8 files changed, 629 insertions, 132 deletions

diff --git a/platform/linux-generic/include/api/odp_buffer_pool.h b/platform/linux-generic/include/api/odp_buffer_pool.h
index b287f79d8..4ccdfe613 100644
--- a/platform/linux-generic/include/api/odp_buffer_pool.h
+++ b/platform/linux-generic/include/api/odp_buffer_pool.h
@@ -33,21 +33,48 @@ extern "C" {
 #define ODP_BUFFER_POOL_NAME_LEN  32
 
 /**
+ * Buffer pool parameters
+ * Used to communicate buffer pool creation options.
+ */
+typedef struct odp_buffer_pool_param_t {
+	uint32_t buf_size;  /**< Buffer size in bytes.  The maximum
+			       number of bytes application will
+			       store in each buffer. For packets, this
+			       is the maximum packet data length, and
+			       configured headroom and tailroom will be
+			       added to this number */
+	uint32_t buf_align; /**< Minimum buffer alignment in bytes.
+			       Valid values are powers of two.  Use 0
+			       for default alignment.  Default will
+			       always be a multiple of 8. */
+	uint32_t num_bufs;  /**< Number of buffers in the pool */
+	int      buf_type;  /**< Buffer type */
+} odp_buffer_pool_param_t;
+
+/**
  * Create a buffer pool
+ * This routine is used to create a buffer pool. It take three
+ * arguments: the optional name of the pool to be created, an optional shared
+ * memory handle, and a parameter struct that describes the pool to be
+ * created. If a name is not specified the result is an anonymous pool that
+ * cannot be referenced by odp_buffer_pool_lookup().
  *
- * @param name      Name of the pool (max ODP_BUFFER_POOL_NAME_LEN - 1 chars)
- * @param base_addr Pool base address
- * @param size      Pool size in bytes
- * @param buf_size  Buffer size in bytes
- * @param buf_align Minimum buffer alignment
- * @param buf_type  Buffer type
+ * @param name     Name of the pool, max ODP_BUFFER_POOL_NAME_LEN-1 chars.
+ *                 May be specified as NULL for anonymous pools.
  *
- * @return Buffer pool handle
+ * @param shm      The shared memory object in which to create the pool.
+ *                 Use ODP_SHM_NULL to reserve default memory type
+ *                 for the buffer type.
+ *
+ * @param params   Buffer pool parameters.
+ *
+ * @return Handle of the created buffer pool
+ * @retval ODP_BUFFER_POOL_INVALID  Buffer pool could not be created
  */
+
 odp_buffer_pool_t odp_buffer_pool_create(const char *name,
-					 void *base_addr, uint64_t size,
-					 size_t buf_size, size_t buf_align,
-					 int buf_type);
+					 odp_shm_t shm,
+					 odp_buffer_pool_param_t *params);
 
 
 /**
diff --git a/platform/linux-generic/include/api/odp_platform_types.h b/platform/linux-generic/include/api/odp_platform_types.h
index 68d0c55d0..2181eb695 100644
--- a/platform/linux-generic/include/api/odp_platform_types.h
+++ b/platform/linux-generic/include/api/odp_platform_types.h
@@ -27,7 +27,7 @@
 typedef uint32_t odp_buffer_pool_t;
 
 /** Invalid buffer pool */
-#define ODP_BUFFER_POOL_INVALID (0)
+#define ODP_BUFFER_POOL_INVALID (0xffffffff)
 
 /** ODP buffer */
 typedef uint32_t odp_buffer_t;
@@ -74,6 +74,7 @@ typedef uint32_t odp_shm_t;
 
 /** Invalid shared memory block */
 #define ODP_SHM_INVALID 0
+#define ODP_SHM_NULL ODP_SHM_INVALID /**< Synonym for buffer pool use */
 
 /**
  * @}
diff --git a/platform/linux-generic/include/odp_buffer_inlines.h b/platform/linux-generic/include/odp_buffer_inlines.h
new file mode 100644
index 000000000..6227482f6
--- /dev/null
+++ b/platform/linux-generic/include/odp_buffer_inlines.h
@@ -0,0 +1,150 @@
+/* Copyright (c) 2014, Linaro Limited
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier:     BSD-3-Clause
+ */
+
+/**
+ * @file
+ *
+ * Inline functions for ODP buffer mgmt routines - implementation internal
+ */
+
+#ifndef ODP_BUFFER_INLINES_H_
+#define ODP_BUFFER_INLINES_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+static inline odp_buffer_t odp_buffer_encode_handle(odp_buffer_hdr_t *hdr)
+{
+	odp_buffer_bits_t handle;
+	uint32_t pool_id = pool_handle_to_index(hdr->pool_hdl);
+	struct pool_entry_s *pool = get_pool_entry(pool_id);
+
+	handle.pool_id = pool_id;
+	handle.index = ((uint8_t *)hdr - pool->pool_mdata_addr) /
+		ODP_CACHE_LINE_SIZE;
+	handle.seg = 0;
+
+	return handle.u32;
+}
+
+static inline odp_buffer_t odp_hdr_to_buf(odp_buffer_hdr_t *hdr)
+{
+	return hdr->handle.handle;
+}
+
+static inline odp_buffer_hdr_t *odp_buf_to_hdr(odp_buffer_t buf)
+{
+	odp_buffer_bits_t handle;
+	uint32_t pool_id;
+	uint32_t index;
+	struct pool_entry_s *pool;
+
+	handle.u32 = buf;
+	pool_id    = handle.pool_id;
+	index      = handle.index;
+
+#ifdef POOL_ERROR_CHECK
+	if (odp_unlikely(pool_id > ODP_CONFIG_BUFFER_POOLS)) {
+		ODP_ERR("odp_buf_to_hdr: Bad pool id\n");
+		return NULL;
+	}
+#endif
+
+	pool = get_pool_entry(pool_id);
+
+#ifdef POOL_ERROR_CHECK
+	if (odp_unlikely(index > pool->params.num_bufs - 1)) {
+		ODP_ERR("odp_buf_to_hdr: Bad buffer index\n");
+		return NULL;
+	}
+#endif
+
+	return (odp_buffer_hdr_t *)(void *)
+		(pool->pool_mdata_addr + (index * ODP_CACHE_LINE_SIZE));
+}
+
+static inline uint32_t odp_buffer_refcount(odp_buffer_hdr_t *buf)
+{
+	return odp_atomic_load_u32(&buf->ref_count);
+}
+
+static inline uint32_t odp_buffer_incr_refcount(odp_buffer_hdr_t *buf,
+						uint32_t val)
+{
+	return odp_atomic_fetch_add_u32(&buf->ref_count, val) + val;
+}
+
+static inline uint32_t odp_buffer_decr_refcount(odp_buffer_hdr_t *buf,
+						uint32_t val)
+{
+	uint32_t tmp;
+
+	tmp = odp_atomic_fetch_sub_u32(&buf->ref_count, val);
+
+	if (tmp < val) {
+		odp_atomic_fetch_add_u32(&buf->ref_count, val - tmp);
+		return 0;
+	} else {
+		return tmp - val;
+	}
+}
+
+static inline odp_buffer_hdr_t *validate_buf(odp_buffer_t buf)
+{
+	odp_buffer_bits_t handle;
+	odp_buffer_hdr_t *buf_hdr;
+	handle.u32 = buf;
+
+	/* For buffer handles, segment index must be 0 and pool id in range */
+	if (handle.seg != 0 || handle.pool_id >= ODP_CONFIG_BUFFER_POOLS)
+		return NULL;
+
+	pool_entry_t *pool = odp_pool_to_entry(handle.pool_id);
+
+	/* If pool not created, handle is invalid */
+	if (pool->s.pool_shm == ODP_SHM_INVALID)
+		return NULL;
+
+	uint32_t buf_stride = pool->s.buf_stride / ODP_CACHE_LINE_SIZE;
+
+	/* A valid buffer index must be on stride, and must be in range */
+	if ((handle.index % buf_stride != 0) ||
+	    ((uint32_t)(handle.index / buf_stride) >= pool->s.params.num_bufs))
+		return NULL;
+
+	buf_hdr = (odp_buffer_hdr_t *)(void *)
+		(pool->s.pool_mdata_addr +
+		 (handle.index * ODP_CACHE_LINE_SIZE));
+
+	/* Handle is valid, so buffer is valid if it is allocated */
+	return buf_hdr->allocator == ODP_FREEBUF ? NULL : buf_hdr;
+}
+
+int odp_buffer_snprint(char *str, uint32_t n, odp_buffer_t buf);
+
+static inline void *buffer_map(odp_buffer_hdr_t *buf,
+			       uint32_t offset,
+			       uint32_t *seglen,
+			       uint32_t limit)
+{
+	int seg_index  = offset / buf->segsize;
+	int seg_offset = offset % buf->segsize;
+
+	if (seglen != NULL) {
+		uint32_t buf_left = limit - offset;
+		*seglen = buf_left < buf->segsize ?
+			buf_left : buf->segsize - seg_offset;
+	}
+
+	return (void *)(seg_offset + (uint8_t *)buf->addr[seg_index]);
+}
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/platform/linux-generic/include/odp_buffer_internal.h b/platform/linux-generic/include/odp_buffer_internal.h
index 0027bfcf3..859633e3c 100644
--- a/platform/linux-generic/include/odp_buffer_internal.h
+++ b/platform/linux-generic/include/odp_buffer_internal.h
@@ -24,99 +24,133 @@ extern "C" {
 #include <odp_buffer.h>
 #include <odp_debug.h>
 #include <odp_align.h>
-
-/* TODO: move these to correct files */
-
-typedef uint64_t odp_phys_addr_t;
+#include <odp_align_internal.h>
+#include <odp_config.h>
+#include <odp_byteorder.h>
+#include <odp_thread.h>
+
+
+#define ODP_BITSIZE(x) \
+	((x) <=     2 ?  1 : \
+	((x) <=     4 ?  2 : \
+	((x) <=     8 ?  3 : \
+	((x) <=    16 ?  4 : \
+	((x) <=    32 ?  5 : \
+	((x) <=    64 ?  6 : \
+	((x) <=   128 ?  7 : \
+	((x) <=   256 ?  8 : \
+	((x) <=   512 ?  9 : \
+	((x) <=  1024 ? 10 : \
+	((x) <=  2048 ? 11 : \
+	((x) <=  4096 ? 12 : \
+	((x) <=  8196 ? 13 : \
+	((x) <= 16384 ? 14 : \
+	((x) <= 32768 ? 15 : \
+	((x) <= 65536 ? 16 : \
+	 (0/0)))))))))))))))))
+
+ODP_STATIC_ASSERT(ODP_CONFIG_PACKET_BUF_LEN_MIN >= 256,
+		  "ODP Segment size must be a minimum of 256 bytes");
+
+ODP_STATIC_ASSERT((ODP_CONFIG_PACKET_BUF_LEN_MIN % ODP_CACHE_LINE_SIZE) == 0,
+		  "ODP Segment size must be a multiple of cache line size");
+
+ODP_STATIC_ASSERT((ODP_CONFIG_PACKET_BUF_LEN_MAX %
+		   ODP_CONFIG_PACKET_BUF_LEN_MIN) == 0,
+		  "Packet max size must be a multiple of segment size");
+
+#define ODP_BUFFER_MAX_SEG \
+	(ODP_CONFIG_PACKET_BUF_LEN_MAX / ODP_CONFIG_PACKET_BUF_LEN_MIN)
+
+/* We can optimize storage of small raw buffers within metadata area */
+#define ODP_MAX_INLINE_BUF     ((sizeof(void *)) * (ODP_BUFFER_MAX_SEG - 1))
+
+#define ODP_BUFFER_POOL_BITS   ODP_BITSIZE(ODP_CONFIG_BUFFER_POOLS)
+#define ODP_BUFFER_SEG_BITS    ODP_BITSIZE(ODP_BUFFER_MAX_SEG)
+#define ODP_BUFFER_INDEX_BITS  (32 - ODP_BUFFER_POOL_BITS - ODP_BUFFER_SEG_BITS)
+#define ODP_BUFFER_PREFIX_BITS (ODP_BUFFER_POOL_BITS + ODP_BUFFER_INDEX_BITS)
+#define ODP_BUFFER_MAX_POOLS   (1 << ODP_BUFFER_POOL_BITS)
+#define ODP_BUFFER_MAX_BUFFERS (1 << ODP_BUFFER_INDEX_BITS)
 
 #define ODP_BUFFER_MAX_INDEX     (ODP_BUFFER_MAX_BUFFERS - 2)
 #define ODP_BUFFER_INVALID_INDEX (ODP_BUFFER_MAX_BUFFERS - 1)
 
-#define ODP_BUFS_PER_CHUNK       16
-#define ODP_BUFS_PER_SCATTER      4
-
-#define ODP_BUFFER_TYPE_CHUNK    0xffff
-
-
-#define ODP_BUFFER_POOL_BITS   4
-#define ODP_BUFFER_INDEX_BITS  (32 - ODP_BUFFER_POOL_BITS)
-#define ODP_BUFFER_MAX_POOLS   (1 << ODP_BUFFER_POOL_BITS)
-#define ODP_BUFFER_MAX_BUFFERS (1 << ODP_BUFFER_INDEX_BITS)
-
 typedef union odp_buffer_bits_t {
 	uint32_t     u32;
 	odp_buffer_t handle;
 
 	struct {
+#if ODP_BYTE_ORDER == ODP_BIG_ENDIAN
 		uint32_t pool_id:ODP_BUFFER_POOL_BITS;
 		uint32_t index:ODP_BUFFER_INDEX_BITS;
+		uint32_t seg:ODP_BUFFER_SEG_BITS;
+#else
+		uint32_t seg:ODP_BUFFER_SEG_BITS;
+		uint32_t index:ODP_BUFFER_INDEX_BITS;
+		uint32_t pool_id:ODP_BUFFER_POOL_BITS;
+#endif
 	};
-} odp_buffer_bits_t;
 
+	struct {
+#if ODP_BYTE_ORDER == ODP_BIG_ENDIAN
+		uint32_t prefix:ODP_BUFFER_PREFIX_BITS;
+		uint32_t pfxseg:ODP_BUFFER_SEG_BITS;
+#else
+		uint32_t pfxseg:ODP_BUFFER_SEG_BITS;
+		uint32_t prefix:ODP_BUFFER_PREFIX_BITS;
+#endif
+	};
+} odp_buffer_bits_t;
 
 /* forward declaration */
 struct odp_buffer_hdr_t;
 
-
-/*
- * Scatter/gather list of buffers
- */
-typedef struct odp_buffer_scatter_t {
-	/* buffer pointers */
-	struct odp_buffer_hdr_t *buf[ODP_BUFS_PER_SCATTER];
-	int                      num_bufs;   /* num buffers */
-	int                      pos;        /* position on the list */
-	size_t                   total_len;  /* Total length */
-} odp_buffer_scatter_t;
-
-
-/*
- * Chunk of buffers (in single pool)
- */
-typedef struct odp_buffer_chunk_t {
-	uint32_t num_bufs;                      /* num buffers */
-	uint32_t buf_index[ODP_BUFS_PER_CHUNK]; /* buffers */
-} odp_buffer_chunk_t;
-
-
 /* Common buffer header */
 typedef struct odp_buffer_hdr_t {
 	struct odp_buffer_hdr_t *next;       /* next buf in a list */
+	int                      allocator;  /* allocating thread id */
 	odp_buffer_bits_t        handle;     /* handle */
-	odp_phys_addr_t          phys_addr;  /* physical data start address */
-	void                    *addr;       /* virtual data start address */
-	uint32_t                 index;	     /* buf index in the pool */
+	union {
+		uint32_t all;
+		struct {
+			uint32_t zeroized:1; /* Zeroize buf data on free */
+			uint32_t hdrdata:1;  /* Data is in buffer hdr */
+		};
+	} flags;
+	int                      type;       /* buffer type */
 	size_t                   size;       /* max data size */
-	size_t                   cur_offset; /* current offset */
 	odp_atomic_u32_t         ref_count;  /* reference count */
-	odp_buffer_scatter_t     scatter;    /* Scatter/gather list */
-	int                      type;       /* type of next header */
 	odp_buffer_pool_t        pool_hdl;   /* buffer pool handle */
-
+	union {
+		uint64_t         buf_u64;    /* user u64 */
+		void            *buf_ctx;    /* user context */
+		void            *udata_addr; /* user metadata addr */
+	};
+	size_t                   udata_size; /* size of user metadata */
+	uint32_t                 segcount;   /* segment count */
+	uint32_t                 segsize;    /* segment size */
+	void                    *addr[ODP_BUFFER_MAX_SEG]; /* block addrs */
 } odp_buffer_hdr_t;
 
-/* Ensure next header starts from 8 byte align */
-ODP_STATIC_ASSERT((sizeof(odp_buffer_hdr_t) % 8) == 0, "ODP_BUFFER_HDR_T__SIZE_ERROR");
+typedef struct odp_buffer_hdr_stride {
+	uint8_t pad[ODP_CACHE_LINE_SIZE_ROUNDUP(sizeof(odp_buffer_hdr_t))];
+} odp_buffer_hdr_stride;
 
+typedef struct odp_buf_blk_t {
+	struct odp_buf_blk_t *next;
+	struct odp_buf_blk_t *prev;
+} odp_buf_blk_t;
 
 /* Raw buffer header */
 typedef struct {
 	odp_buffer_hdr_t buf_hdr;    /* common buffer header */
-	uint8_t          buf_data[]; /* start of buffer data area */
 } odp_raw_buffer_hdr_t;
 
+/* Free buffer marker */
+#define ODP_FREEBUF -1
 
-/* Chunk header */
-typedef struct odp_buffer_chunk_hdr_t {
-	odp_buffer_hdr_t   buf_hdr;
-	odp_buffer_chunk_t chunk;
-} odp_buffer_chunk_hdr_t;
-
-
-int odp_buffer_snprint(char *str, size_t n, odp_buffer_t buf);
-
-void odp_buffer_copy_scatter(odp_buffer_t buf_dst, odp_buffer_t buf_src);
-
+/* Forward declarations */
+odp_buffer_t buffer_alloc(odp_buffer_pool_t pool, size_t size);
 
 #ifdef __cplusplus
 }
diff --git a/platform/linux-generic/include/odp_buffer_pool_internal.h b/platform/linux-generic/include/odp_buffer_pool_internal.h
index 07602feb9..2e48ac3f7 100644
--- a/platform/linux-generic/include/odp_buffer_pool_internal.h
+++ b/platform/linux-generic/include/odp_buffer_pool_internal.h
@@ -19,13 +19,44 @@ extern "C" {
 #endif
 
 #include <odp_std_types.h>
-#include <odp_buffer_pool.h>
-#include <odp_buffer_internal.h>
 #include <odp_align.h>
 #include <odp_align_internal.h>
+#include <odp_buffer_pool.h>
+#include <odp_buffer_internal.h>
 #include <odp_hints.h>
 #include <odp_config.h>
 #include <odp_debug.h>
+#include <odp_shared_memory.h>
+#include <odp_atomic.h>
+#include <odp_atomic_internal.h>
+#include <string.h>
+
+/**
+ * Buffer initialization routine prototype
+ *
+ * @note Routines of this type MAY be passed as part of the
+ * _odp_buffer_pool_init_t structure to be called whenever a
+ * buffer is allocated to initialize the user metadata
+ * associated with that buffer.
+ */
+typedef void (_odp_buf_init_t)(odp_buffer_t buf, void *buf_init_arg);
+
+/**
+ * Buffer pool initialization parameters
+ * Used to communicate buffer pool initialization options. Internal for now.
+ */
+typedef struct _odp_buffer_pool_init_t {
+	size_t udata_size;         /**< Size of user metadata for each buffer */
+	_odp_buf_init_t *buf_init; /**< Buffer initialization routine to use */
+	void *buf_init_arg;        /**< Argument to be passed to buf_init() */
+} _odp_buffer_pool_init_t;         /**< Type of buffer initialization struct */
+
+/* Local cache for buffer alloc/free acceleration */
+typedef struct local_cache_t {
+	odp_buffer_hdr_t *buf_freelist;  /* The local cache */
+	uint64_t bufallocs;              /* Local buffer alloc count */
+	uint64_t buffrees;               /* Local buffer free count */
+} local_cache_t;
 
 /* Use ticketlock instead of spinlock */
 #define POOL_USE_TICKETLOCK
@@ -36,11 +67,16 @@ extern "C" {
 
 #ifdef POOL_USE_TICKETLOCK
 #include <odp_ticketlock.h>
+#define POOL_LOCK(a)      odp_ticketlock_lock(a)
+#define POOL_UNLOCK(a)    odp_ticketlock_unlock(a)
+#define POOL_LOCK_INIT(a) odp_ticketlock_init(a)
 #else
 #include <odp_spinlock.h>
+#define POOL_LOCK(a)      odp_spinlock_lock(a)
+#define POOL_UNLOCK(a)    odp_spinlock_unlock(a)
+#define POOL_LOCK_INIT(a) odp_spinlock_init(a)
 #endif
 
-
 struct pool_entry_s {
 #ifdef POOL_USE_TICKETLOCK
 	odp_ticketlock_t        lock ODP_ALIGNED_CACHE;
@@ -48,74 +84,294 @@ struct pool_entry_s {
 	odp_spinlock_t          lock ODP_ALIGNED_CACHE;
 #endif
 
-	odp_buffer_chunk_hdr_t *head;
-	uint64_t                free_bufs;
 	char                    name[ODP_BUFFER_POOL_NAME_LEN];
-
-	odp_buffer_pool_t       pool_hdl ODP_ALIGNED_CACHE;
-	uintptr_t               buf_base;
-	size_t                  buf_size;
-	size_t                  buf_offset;
-	uint64_t                num_bufs;
-	void                   *pool_base_addr;
-	uint64_t                pool_size;
-	size_t                  user_size;
-	size_t                  user_align;
-	int                     buf_type;
-	size_t                  hdr_size;
+	odp_buffer_pool_param_t params;
+	_odp_buffer_pool_init_t init_params;
+	odp_buffer_pool_t       pool_hdl;
+	uint32_t                pool_id;
+	odp_shm_t               pool_shm;
+	union {
+		uint32_t all;
+		struct {
+			uint32_t has_name:1;
+			uint32_t user_supplied_shm:1;
+			uint32_t unsegmented:1;
+			uint32_t zeroized:1;
+			uint32_t predefined:1;
+		};
+	} flags;
+	uint32_t                quiesced;
+	uint32_t                low_wm_assert;
+	uint8_t                *pool_base_addr;
+	uint8_t                *pool_mdata_addr;
+	size_t                  pool_size;
+	uint32_t                buf_align;
+	uint32_t                buf_stride;
+	_odp_atomic_ptr_t       buf_freelist;
+	_odp_atomic_ptr_t       blk_freelist;
+	odp_atomic_u32_t        bufcount;
+	odp_atomic_u32_t        blkcount;
+	odp_atomic_u64_t        bufallocs;
+	odp_atomic_u64_t        buffrees;
+	odp_atomic_u64_t        blkallocs;
+	odp_atomic_u64_t        blkfrees;
+	odp_atomic_u64_t        bufempty;
+	odp_atomic_u64_t        blkempty;
+	odp_atomic_u64_t        high_wm_count;
+	odp_atomic_u64_t        low_wm_count;
+	uint32_t                seg_size;
+	uint32_t                high_wm;
+	uint32_t                low_wm;
+	uint32_t                headroom;
+	uint32_t                tailroom;
 };
 
 typedef union pool_entry_u {
 	struct pool_entry_s s;
+
 	uint8_t pad[ODP_CACHE_LINE_SIZE_ROUNDUP(sizeof(struct pool_entry_s))];
 } pool_entry_t;
 
 extern void *pool_entry_ptr[];
 
+#if defined(ODP_CONFIG_SECURE_POOLS) && (ODP_CONFIG_SECURE_POOLS == 1)
+#define buffer_is_secure(buf) (buf->flags.zeroized)
+#define pool_is_secure(pool) (pool->flags.zeroized)
+#else
+#define buffer_is_secure(buf) 0
+#define pool_is_secure(pool) 0
+#endif
 
-static inline void *get_pool_entry(uint32_t pool_id)
+#define TAG_ALIGN ((size_t)16)
+
+#define odp_cs(ptr, old, new) \
+	_odp_atomic_ptr_cmp_xchg_strong(&ptr, (void **)&old, (void *)new, \
+					_ODP_MEMMODEL_SC, \
+					_ODP_MEMMODEL_SC)
+
+/* Helper functions for pointer tagging to avoid ABA race conditions */
+#define odp_tag(ptr) \
+	(((size_t)ptr) & (TAG_ALIGN - 1))
+
+#define odp_detag(ptr) \
+	((void *)(((size_t)ptr) & -TAG_ALIGN))
+
+#define odp_retag(ptr, tag) \
+	((void *)(((size_t)ptr) | odp_tag(tag)))
+
+
+static inline void *get_blk(struct pool_entry_s *pool)
 {
-	return pool_entry_ptr[pool_id];
+	void *oldhead, *myhead, *newhead;
+
+	oldhead = _odp_atomic_ptr_load(&pool->blk_freelist, _ODP_MEMMODEL_ACQ);
+
+	do {
+		size_t tag = odp_tag(oldhead);
+		myhead = odp_detag(oldhead);
+		if (odp_unlikely(myhead == NULL))
+			break;
+		newhead = odp_retag(((odp_buf_blk_t *)myhead)->next, tag + 1);
+	} while (odp_cs(pool->blk_freelist, oldhead, newhead) == 0);
+
+	if (odp_unlikely(myhead == NULL))
+		odp_atomic_inc_u64(&pool->blkempty);
+	else
+		odp_atomic_dec_u32(&pool->blkcount);
+
+	return (void *)myhead;
 }
 
-static inline uint32_t pool_handle_to_index(odp_buffer_pool_t pool_hdl)
+static inline void ret_blk(struct pool_entry_s *pool, void *block)
 {
-	return pool_hdl - 1;
+	void *oldhead, *myhead, *myblock;
+
+	oldhead = _odp_atomic_ptr_load(&pool->blk_freelist, _ODP_MEMMODEL_ACQ);
+
+	do {
+		size_t tag = odp_tag(oldhead);
+		myhead = odp_detag(oldhead);
+		((odp_buf_blk_t *)block)->next = myhead;
+		myblock = odp_retag(block, tag + 1);
+	} while (odp_cs(pool->blk_freelist, oldhead, myblock) == 0);
+
+	odp_atomic_inc_u32(&pool->blkcount);
+	odp_atomic_inc_u64(&pool->blkfrees);
 }
 
-static inline odp_buffer_hdr_t *odp_buf_to_hdr(odp_buffer_t buf)
+static inline odp_buffer_hdr_t *get_buf(struct pool_entry_s *pool)
 {
-	odp_buffer_bits_t handle;
-	uint32_t pool_id;
-	uint32_t index;
-	struct pool_entry_s *pool;
-	odp_buffer_hdr_t *hdr;
-
-	handle.u32 = buf;
-	pool_id    = handle.pool_id;
-	index      = handle.index;
-
-#ifdef POOL_ERROR_CHECK
-	if (odp_unlikely(pool_id > ODP_CONFIG_BUFFER_POOLS)) {
-		ODP_ERR("odp_buf_to_hdr: Bad pool id\n");
-		return NULL;
+	odp_buffer_hdr_t *oldhead, *myhead, *newhead;
+
+	oldhead = _odp_atomic_ptr_load(&pool->buf_freelist, _ODP_MEMMODEL_ACQ);
+
+	do {
+		size_t tag = odp_tag(oldhead);
+		myhead = odp_detag(oldhead);
+		if (odp_unlikely(myhead == NULL))
+			break;
+		newhead = odp_retag(myhead->next, tag + 1);
+	} while (odp_cs(pool->buf_freelist, oldhead, newhead) == 0);
+
+	if (odp_unlikely(myhead == NULL)) {
+		odp_atomic_inc_u64(&pool->bufempty);
+	} else {
+		uint64_t bufcount =
+			odp_atomic_fetch_sub_u32(&pool->bufcount, 1) - 1;
+
+		/* Check for low watermark condition */
+		if (bufcount == pool->low_wm && !pool->low_wm_assert) {
+			pool->low_wm_assert = 1;
+			odp_atomic_inc_u64(&pool->low_wm_count);
+		}
+
+		odp_atomic_inc_u64(&pool->bufallocs);
+		myhead->next = myhead;  /* Mark buffer allocated */
+		myhead->allocator = odp_thread_id();
 	}
-#endif
 
-	pool = get_pool_entry(pool_id);
+	return (void *)myhead;
+}
+
+static inline void ret_buf(struct pool_entry_s *pool, odp_buffer_hdr_t *buf)
+{
+	odp_buffer_hdr_t *oldhead, *myhead, *mybuf;
+
+	buf->allocator = ODP_FREEBUF;  /* Mark buffer free */
 
-#ifdef POOL_ERROR_CHECK
-	if (odp_unlikely(index > pool->num_bufs - 1)) {
-		ODP_ERR("odp_buf_to_hdr: Bad buffer index\n");
-		return NULL;
+	if (!buf->flags.hdrdata && buf->type != ODP_BUFFER_TYPE_RAW) {
+		while (buf->segcount > 0) {
+			if (buffer_is_secure(buf) || pool_is_secure(pool))
+				memset(buf->addr[buf->segcount - 1],
+				       0, buf->segsize);
+			ret_blk(pool, buf->addr[--buf->segcount]);
+		}
+		buf->size = 0;
 	}
-#endif
 
-	hdr = (odp_buffer_hdr_t *)(pool->buf_base + index * pool->buf_size);
+	oldhead = _odp_atomic_ptr_load(&pool->buf_freelist, _ODP_MEMMODEL_ACQ);
+
+	do {
+		size_t tag = odp_tag(oldhead);
+		myhead = odp_detag(oldhead);
+		buf->next = myhead;
+		mybuf = odp_retag(buf, tag + 1);
+	} while (odp_cs(pool->buf_freelist, oldhead, mybuf) == 0);
+
+	uint64_t bufcount = odp_atomic_fetch_add_u32(&pool->bufcount, 1) + 1;
+
+	/* Check if low watermark condition should be deasserted */
+	if (bufcount == pool->high_wm && pool->low_wm_assert) {
+		pool->low_wm_assert = 0;
+		odp_atomic_inc_u64(&pool->high_wm_count);
+	}
 
-	return hdr;
+	odp_atomic_inc_u64(&pool->buffrees);
 }
 
+static inline void *get_local_buf(local_cache_t *buf_cache,
+				  struct pool_entry_s *pool,
+				  size_t totsize)
+{
+	odp_buffer_hdr_t *buf = buf_cache->buf_freelist;
+
+	if (odp_likely(buf != NULL)) {
+		buf_cache->buf_freelist = buf->next;
+
+		if (odp_unlikely(buf->size < totsize)) {
+			intmax_t needed = totsize - buf->size;
+
+			do {
+				void *blk = get_blk(pool);
+				if (odp_unlikely(blk == NULL)) {
+					ret_buf(pool, buf);
+					buf_cache->buffrees--;
+					return NULL;
+				}
+				buf->addr[buf->segcount++] = blk;
+				needed -= pool->seg_size;
+			} while (needed > 0);
+
+			buf->size = buf->segcount * pool->seg_size;
+		}
+
+		buf_cache->bufallocs++;
+		buf->allocator = odp_thread_id();  /* Mark buffer allocated */
+	}
+
+	return buf;
+}
+
+static inline void ret_local_buf(local_cache_t *buf_cache,
+				odp_buffer_hdr_t *buf)
+{
+	buf->allocator = ODP_FREEBUF;
+	buf->next = buf_cache->buf_freelist;
+	buf_cache->buf_freelist = buf;
+
+	buf_cache->buffrees++;
+}
+
+static inline void flush_cache(local_cache_t *buf_cache,
+			       struct pool_entry_s *pool)
+{
+	odp_buffer_hdr_t *buf = buf_cache->buf_freelist;
+	uint32_t flush_count = 0;
+
+	while (buf != NULL) {
+		odp_buffer_hdr_t *next = buf->next;
+		ret_buf(pool, buf);
+		buf = next;
+		flush_count++;
+	}
+
+	odp_atomic_add_u64(&pool->bufallocs, buf_cache->bufallocs);
+	odp_atomic_add_u64(&pool->buffrees, buf_cache->buffrees - flush_count);
+
+	buf_cache->buf_freelist = NULL;
+	buf_cache->bufallocs = 0;
+	buf_cache->buffrees = 0;
+}
+
+static inline odp_buffer_pool_t pool_index_to_handle(uint32_t pool_id)
+{
+	return pool_id;
+}
+
+static inline uint32_t pool_handle_to_index(odp_buffer_pool_t pool_hdl)
+{
+	return pool_hdl;
+}
+
+static inline void *get_pool_entry(uint32_t pool_id)
+{
+	return pool_entry_ptr[pool_id];
+}
+
+static inline pool_entry_t *odp_pool_to_entry(odp_buffer_pool_t pool)
+{
+	return (pool_entry_t *)get_pool_entry(pool_handle_to_index(pool));
+}
+
+static inline pool_entry_t *odp_buf_to_pool(odp_buffer_hdr_t *buf)
+{
+	return odp_pool_to_entry(buf->pool_hdl);
+}
+
+static inline uint32_t odp_buffer_pool_segment_size(odp_buffer_pool_t pool)
+{
+	return odp_pool_to_entry(pool)->s.seg_size;
+}
+
+static inline uint32_t odp_buffer_pool_headroom(odp_buffer_pool_t pool)
+{
+	return odp_pool_to_entry(pool)->s.headroom;
+}
+
+static inline uint32_t odp_buffer_pool_tailroom(odp_buffer_pool_t pool)
+{
+	return odp_pool_to_entry(pool)->s.tailroom;
+}
 
 #ifdef __cplusplus
 }
diff --git a/platform/linux-generic/include/odp_internal.h b/platform/linux-generic/include/odp_internal.h
index 04c1030b3..ee2ab1a0d 100644
--- a/platform/linux-generic/include/odp_internal.h
+++ b/platform/linux-generic/include/odp_internal.h
@@ -44,6 +44,8 @@ int odp_schedule_init_local(void);
 int odp_timer_init_global(void);
 int odp_timer_disarm_all(void);
 
+void _odp_flush_caches(void);
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/platform/linux-generic/include/odp_packet_internal.h b/platform/linux-generic/include/odp_packet_internal.h
index 49c59b28d..f34a83dda 100644
--- a/platform/linux-generic/include/odp_packet_internal.h
+++ b/platform/linux-generic/include/odp_packet_internal.h
@@ -22,6 +22,7 @@ extern "C" {
 #include <odp_debug.h>
 #include <odp_buffer_internal.h>
 #include <odp_buffer_pool_internal.h>
+#include <odp_buffer_inlines.h>
 #include <odp_packet.h>
 #include <odp_packet_io.h>
 
@@ -92,7 +93,8 @@ typedef union {
 	};
 } output_flags_t;
 
-ODP_STATIC_ASSERT(sizeof(output_flags_t) == sizeof(uint32_t), "OUTPUT_FLAGS_SIZE_ERROR");
+ODP_STATIC_ASSERT(sizeof(output_flags_t) == sizeof(uint32_t),
+		  "OUTPUT_FLAGS_SIZE_ERROR");
 
 /**
  * Internal Packet header
@@ -105,25 +107,23 @@ typedef struct {
 	error_flags_t  error_flags;
 	output_flags_t output_flags;
 
-	uint32_t frame_offset; /**< offset to start of frame, even on error */
 	uint32_t l2_offset; /**< offset to L2 hdr, e.g. Eth */
 	uint32_t l3_offset; /**< offset to L3 hdr, e.g. IPv4, IPv6 */
 	uint32_t l4_offset; /**< offset to L4 hdr (TCP, UDP, SCTP, also ICMP) */
 
 	uint32_t frame_len;
+	uint32_t headroom;
+	uint32_t tailroom;
 
 	uint64_t user_ctx;        /* user context */
 
 	odp_pktio_t input;
-
-	uint32_t pad;
-	uint8_t  buf_data[]; /* start of buffer data area */
 } odp_packet_hdr_t;
 
-ODP_STATIC_ASSERT(sizeof(odp_packet_hdr_t) == ODP_OFFSETOF(odp_packet_hdr_t, buf_data),
-	   "ODP_PACKET_HDR_T__SIZE_ERR");
-ODP_STATIC_ASSERT(sizeof(odp_packet_hdr_t) % sizeof(uint64_t) == 0,
-	   "ODP_PACKET_HDR_T__SIZE_ERR2");
+typedef struct odp_packet_hdr_stride {
+	uint8_t pad[ODP_CACHE_LINE_SIZE_ROUNDUP(sizeof(odp_packet_hdr_t))];
+} odp_packet_hdr_stride;
+
 
 /**
  * Return the packet header
@@ -138,6 +138,38 @@ static inline odp_packet_hdr_t *odp_packet_hdr(odp_packet_t pkt)
  */
 void odp_packet_parse(odp_packet_t pkt, size_t len, size_t l2_offset);
 
+/**
+ * Initialize packet buffer
+ */
+static inline void packet_init(pool_entry_t *pool,
+			       odp_packet_hdr_t *pkt_hdr,
+			       size_t size)
+{
+       /*
+	* Reset parser metadata.  Note that we clear via memset to make
+	* this routine indepenent of any additional adds to packet metadata.
+	*/
+	const size_t start_offset = ODP_FIELD_SIZEOF(odp_packet_hdr_t, buf_hdr);
+	uint8_t *start;
+	size_t len;
+
+	start = (uint8_t *)pkt_hdr + start_offset;
+	len = sizeof(odp_packet_hdr_t) - start_offset;
+	memset(start, 0, len);
+
+       /*
+	* Packet headroom is set from the pool's headroom
+	* Packet tailroom is rounded up to fill the last
+	* segment occupied by the allocated length.
+	*/
+	pkt_hdr->frame_len = size;
+	pkt_hdr->headroom  = pool->s.headroom;
+	pkt_hdr->tailroom  =
+		(pool->s.seg_size * pkt_hdr->buf_hdr.segcount) -
+		(pool->s.headroom + size);
+}
+
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/platform/linux-generic/include/odp_timer_internal.h b/platform/linux-generic/include/odp_timer_internal.h
index ad28f534e..2ff36ce0d 100644
--- a/platform/linux-generic/include/odp_timer_internal.h
+++ b/platform/linux-generic/include/odp_timer_internal.h
@@ -51,14 +51,9 @@ typedef struct odp_timeout_hdr_t {
 	uint8_t buf_data[];
 } odp_timeout_hdr_t;
 
-
-
-ODP_STATIC_ASSERT(sizeof(odp_timeout_hdr_t) ==
-	   ODP_OFFSETOF(odp_timeout_hdr_t, buf_data),
-	   "ODP_TIMEOUT_HDR_T__SIZE_ERR");
-
-ODP_STATIC_ASSERT(sizeof(odp_timeout_hdr_t) % sizeof(uint64_t) == 0,
-	   "ODP_TIMEOUT_HDR_T__SIZE_ERR2");
+typedef struct odp_timeout_hdr_stride {
+	uint8_t pad[ODP_CACHE_LINE_SIZE_ROUNDUP(sizeof(odp_timeout_hdr_t))];
+} odp_timeout_hdr_stride;
 
 
 /**