Merge ODP v1.40.0.0v1.40.0.0_DPDK_19.11

Merge ODP linux-generic v1.40.0.0 into linux-dpdk.

17 files changed, 487 insertions, 55 deletions

diff --git a/platform/linux-generic/include/odp/api/plat/buffer_inlines.h b/platform/linux-generic/include/odp/api/plat/buffer_inlines.h
index 34d4b5675..75ef36cf3 100644
--- a/platform/linux-generic/include/odp/api/plat/buffer_inlines.h
+++ b/platform/linux-generic/include/odp/api/plat/buffer_inlines.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2019-2022, Nokia
+/* Copyright (c) 2019-2023, Nokia
  * All rights reserved.
  *
  * SPDX-License-Identifier:     BSD-3-Clause
@@ -7,11 +7,10 @@
 #ifndef ODP_PLAT_BUFFER_INLINES_H_
 #define ODP_PLAT_BUFFER_INLINES_H_
 
+#include <odp/api/buffer_types.h>
 #include <odp/api/event.h>
 #include <odp/api/pool_types.h>
 
-#include <odp/api/abi/buffer.h>
-
 #include <odp/api/plat/buffer_inline_types.h>
 #include <odp/api/plat/debug_inlines.h>
 #include <odp/api/plat/event_inline_types.h>
diff --git a/platform/linux-generic/include/odp/api/plat/crypto_inlines.h b/platform/linux-generic/include/odp/api/plat/crypto_inlines.h
new file mode 100644
index 000000000..8e98d8580
--- /dev/null
+++ b/platform/linux-generic/include/odp/api/plat/crypto_inlines.h
@@ -0,0 +1,65 @@
+/* Copyright (c) 2022, Nokia
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier:     BSD-3-Clause
+ */
+
+#ifndef ODP_PLAT_CRYPTO_INLINES_H_
+#define ODP_PLAT_CRYPTO_INLINES_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <odp/api/crypto_types.h>
+#include <odp/api/event.h>
+#include <odp/api/packet.h>
+
+#include <odp/api/plat/debug_inlines.h>
+#include <odp/api/plat/packet_inline_types.h>
+
+/** @cond _ODP_HIDE_FROM_DOXYGEN_ */
+
+#ifndef _ODP_NO_INLINE
+	/* Inline functions by default */
+	#define _ODP_INLINE static inline
+	#define odp_crypto_packet_from_event __odp_crypto_packet_from_event
+	#define odp_crypto_packet_to_event __odp_crypto_packet_to_event
+	#define odp_crypto_result __odp_crypto_result
+#else
+	#define _ODP_INLINE
+#endif
+
+_ODP_INLINE odp_packet_t odp_crypto_packet_from_event(odp_event_t ev)
+{
+	_ODP_ASSERT(odp_event_type(ev) == ODP_EVENT_PACKET);
+	_ODP_ASSERT(odp_event_subtype(ev) == ODP_EVENT_PACKET_CRYPTO);
+
+	return odp_packet_from_event(ev);
+}
+
+_ODP_INLINE odp_event_t odp_crypto_packet_to_event(odp_packet_t pkt)
+{
+	return odp_packet_to_event(pkt);
+}
+
+_ODP_INLINE int odp_crypto_result(odp_crypto_packet_result_t *result, odp_packet_t pkt)
+{
+	odp_crypto_packet_result_t *op_result;
+
+	_ODP_ASSERT(odp_packet_subtype(pkt) == ODP_EVENT_PACKET_CRYPTO);
+
+	op_result = _odp_pkt_get_ptr(pkt, odp_crypto_packet_result_t, crypto_op);
+
+	*result = *op_result;
+
+	return 0;
+}
+
+/** @endcond */
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/platform/linux-generic/include/odp/api/plat/event_validation_external.h b/platform/linux-generic/include/odp/api/plat/event_validation_external.h
new file mode 100644
index 000000000..7f5c0364f
--- /dev/null
+++ b/platform/linux-generic/include/odp/api/plat/event_validation_external.h
@@ -0,0 +1,111 @@
+/* Copyright (c) 2023, Nokia
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier:     BSD-3-Clause
+ */
+
+/**
+ * @file
+ *
+ * ODP event validation
+ *
+ * @warning These definitions are not part of ODP API, they are for
+ * implementation internal use only.
+ */
+
+#ifndef ODP_EVENT_VALIDATION_EXTERNAL_H_
+#define ODP_EVENT_VALIDATION_EXTERNAL_H_
+
+#include <odp/autoheader_external.h>
+
+#include <odp/api/buffer_types.h>
+#include <odp/api/event_types.h>
+#include <odp/api/hints.h>
+#include <odp/api/packet_types.h>
+
+/** @cond _ODP_HIDE_FROM_DOXYGEN_ */
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/** Enumerations for identifying ODP API functions */
+typedef enum {
+	_ODP_EV_BUFFER_FREE = 0,
+	_ODP_EV_BUFFER_FREE_MULTI,
+	_ODP_EV_BUFFER_IS_VALID,
+	_ODP_EV_EVENT_FREE,
+	_ODP_EV_EVENT_FREE_MULTI,
+	_ODP_EV_EVENT_FREE_SP,
+	_ODP_EV_EVENT_IS_VALID,
+	_ODP_EV_PACKET_FREE,
+	_ODP_EV_PACKET_FREE_MULTI,
+	_ODP_EV_PACKET_FREE_SP,
+	_ODP_EV_PACKET_IS_VALID,
+	_ODP_EV_QUEUE_ENQ,
+	_ODP_EV_QUEUE_ENQ_MULTI,
+	_ODP_EV_MAX
+} _odp_ev_id_t;
+
+/* Implementation internal event validation functions */
+#if _ODP_EVENT_VALIDATION
+
+int _odp_event_validate(odp_event_t event, _odp_ev_id_t id);
+
+int _odp_event_validate_multi(const odp_event_t event[], int num, _odp_ev_id_t id);
+
+int _odp_buffer_validate(odp_buffer_t buf, _odp_ev_id_t ev_id);
+
+int _odp_buffer_validate_multi(const odp_buffer_t buf[], int num, _odp_ev_id_t ev_id);
+
+int _odp_packet_validate(odp_packet_t pkt, _odp_ev_id_t ev_id);
+
+int _odp_packet_validate_multi(const odp_packet_t pkt[], int num, _odp_ev_id_t ev_id);
+
+#else
+
+static inline int _odp_event_validate(odp_event_t event ODP_UNUSED, _odp_ev_id_t ev_id ODP_UNUSED)
+{
+	return 0;
+}
+
+static inline int _odp_event_validate_multi(const odp_event_t event[] ODP_UNUSED,
+					    int num ODP_UNUSED,
+					    _odp_ev_id_t ev_id ODP_UNUSED)
+{
+	return 0;
+}
+
+static inline int _odp_buffer_validate(odp_buffer_t buf ODP_UNUSED, _odp_ev_id_t ev_id ODP_UNUSED)
+{
+	return 0;
+}
+
+static inline int _odp_buffer_validate_multi(const odp_buffer_t buf[] ODP_UNUSED,
+					     int num ODP_UNUSED,
+					     _odp_ev_id_t ev_id ODP_UNUSED)
+{
+	return 0;
+}
+
+static inline int _odp_packet_validate(odp_packet_t pkt ODP_UNUSED, _odp_ev_id_t ev_id ODP_UNUSED)
+{
+	return 0;
+}
+
+static inline int _odp_packet_validate_multi(const odp_packet_t pkt[] ODP_UNUSED,
+					     int num ODP_UNUSED,
+					     _odp_ev_id_t ev_id ODP_UNUSED)
+{
+	return 0;
+}
+
+#endif /* _ODP_EVENT_VALIDATION */
+
+#ifdef __cplusplus
+}
+#endif
+
+/** @endcond */
+
+#endif
diff --git a/platform/linux-generic/include/odp/api/plat/packet_inline_types.h b/platform/linux-generic/include/odp/api/plat/packet_inline_types.h
index ae03457f9..6773b73ad 100644
--- a/platform/linux-generic/include/odp/api/plat/packet_inline_types.h
+++ b/platform/linux-generic/include/odp/api/plat/packet_inline_types.h
@@ -53,6 +53,7 @@ typedef struct _odp_packet_inline_offset_t {
 	uint16_t subtype;
 	uint16_t cls_mark;
 	uint16_t ipsec_ctx;
+	uint16_t crypto_op;
 
 } _odp_packet_inline_offset_t;
 
diff --git a/platform/linux-generic/include/odp/api/plat/packet_inlines.h b/platform/linux-generic/include/odp/api/plat/packet_inlines.h
index 01d47d837..93e95e21c 100644
--- a/platform/linux-generic/include/odp/api/plat/packet_inlines.h
+++ b/platform/linux-generic/include/odp/api/plat/packet_inlines.h
@@ -632,7 +632,8 @@ _ODP_INLINE uint32_t odp_packet_buf_size(odp_packet_buf_t pkt_buf)
 	odp_pool_t pool = _odp_pkt_get(pkt_buf, odp_pool_t, pool);
 
 	return _odp_pool_get(pool, uint32_t, ext_pkt_buf_size) -
-			_odp_pool_get(pool, uint32_t, ext_head_offset);
+			_odp_pool_get(pool, uint32_t, ext_head_offset) -
+			_odp_pool_get(pool, uint32_t, trailer_size);
 }
 
 _ODP_INLINE void *odp_packet_buf_head(odp_packet_buf_t pkt_buf)
diff --git a/platform/linux-generic/include/odp/api/plat/pool_inline_types.h b/platform/linux-generic/include/odp/api/plat/pool_inline_types.h
index 02f59f982..fbff7eda7 100644
--- a/platform/linux-generic/include/odp/api/plat/pool_inline_types.h
+++ b/platform/linux-generic/include/odp/api/plat/pool_inline_types.h
@@ -30,6 +30,7 @@ typedef struct _odp_pool_inline_offset_t {
 	uint16_t index;
 	uint16_t seg_len;
 	uint16_t uarea_size;
+	uint16_t trailer_size;
 	uint16_t ext_head_offset;
 	uint16_t ext_pkt_buf_size;
 
diff --git a/platform/linux-generic/include/odp/api/plat/queue_inlines.h b/platform/linux-generic/include/odp/api/plat/queue_inlines.h
index 22673a887..609c0c9e4 100644
--- a/platform/linux-generic/include/odp/api/plat/queue_inlines.h
+++ b/platform/linux-generic/include/odp/api/plat/queue_inlines.h
@@ -1,4 +1,5 @@
 /* Copyright (c) 2018, Linaro Limited
+ * Copyright (c) 2023, Nokia
  * All rights reserved.
  *
  * SPDX-License-Identifier:     BSD-3-Clause
@@ -7,6 +8,9 @@
 #ifndef ODP_PLAT_QUEUE_INLINES_H_
 #define ODP_PLAT_QUEUE_INLINES_H_
 
+#include <odp/api/hints.h>
+
+#include <odp/api/plat/event_validation_external.h>
 #include <odp/api/plat/queue_inline_types.h>
 
 /** @cond _ODP_HIDE_FROM_DOXYGEN_ */
@@ -37,12 +41,18 @@ _ODP_INLINE void *odp_queue_context(odp_queue_t handle)
 
 _ODP_INLINE int odp_queue_enq(odp_queue_t queue, odp_event_t ev)
 {
+	if (odp_unlikely(_odp_event_validate(ev, _ODP_EV_QUEUE_ENQ)))
+		return -1;
+
 	return _odp_queue_api->queue_enq(queue, ev);
 }
 
 _ODP_INLINE int odp_queue_enq_multi(odp_queue_t queue,
 				    const odp_event_t events[], int num)
 {
+	if (odp_unlikely(_odp_event_validate_multi(events, num, _ODP_EV_QUEUE_ENQ_MULTI)))
+		return -1;
+
 	return _odp_queue_api->queue_enq_multi(queue, events, num);
 }
 
diff --git a/platform/linux-generic/include/odp_config_internal.h b/platform/linux-generic/include/odp_config_internal.h
index d3d09abf4..e4f8d6d6d 100644
--- a/platform/linux-generic/include/odp_config_internal.h
+++ b/platform/linux-generic/include/odp_config_internal.h
@@ -1,5 +1,5 @@
 /* Copyright (c) 2016-2018, Linaro Limited
- * Copyright (c) 2019-2021, Nokia
+ * Copyright (c) 2019-2023, Nokia
  * All rights reserved.
  *
  * SPDX-License-Identifier:     BSD-3-Clause
@@ -66,7 +66,7 @@ extern "C" {
 /*
  * Maximum number of stashes
  */
-#define CONFIG_MAX_STASHES 128
+#define CONFIG_MAX_STASHES 2048
 
 /*
  * Maximum number of packet IO resources
@@ -134,10 +134,10 @@ extern "C" {
 /*
  * Number of shared memory blocks reserved for implementation internal use.
  *
- * Each stash requires one SHM block, each pool requires three blocks (buffers,
- * ring, user area), and 20 blocks are reserved for per ODP module global data.
+ * Each pool requires three blocks (buffers, ring, user area), and 20 blocks
+ * are reserved for per ODP module global data.
  */
-#define CONFIG_INTERNAL_SHM_BLOCKS (CONFIG_MAX_STASHES + (ODP_CONFIG_POOLS * 3)  + 20)
+#define CONFIG_INTERNAL_SHM_BLOCKS ((ODP_CONFIG_POOLS * 3) + 20)
 
 /*
  * Maximum number of shared memory blocks.
diff --git a/platform/linux-generic/include/odp_event_internal.h b/platform/linux-generic/include/odp_event_internal.h
index 5a29e926e..4bc28d708 100644
--- a/platform/linux-generic/include/odp_event_internal.h
+++ b/platform/linux-generic/include/odp_event_internal.h
@@ -53,7 +53,7 @@ typedef struct _odp_event_hdr_t {
 
 	/* --- Mostly read only data --- */
 
-	/* Initial buffer tail pointer */
+	/* Initial buffer tail pointer and endmark location (if enabled) */
 	uint8_t  *buf_end;
 
 	/* Combined pool and event index */
@@ -85,6 +85,11 @@ static inline void _odp_event_type_set(odp_event_t event, int ev)
 	_odp_event_hdr(event)->event_type = ev;
 }
 
+static inline uint64_t *_odp_event_endmark_get_ptr(odp_event_t event)
+{
+	return (uint64_t *)(uintptr_t)_odp_event_hdr(event)->buf_end;
+}
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/platform/linux-generic/include/odp_event_validation_internal.h b/platform/linux-generic/include/odp_event_validation_internal.h
new file mode 100644
index 000000000..f4ac16f31
--- /dev/null
+++ b/platform/linux-generic/include/odp_event_validation_internal.h
@@ -0,0 +1,52 @@
+/* Copyright (c) 2023, Nokia
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier:     BSD-3-Clause
+ */
+
+#ifndef ODP_EVENT_VALIDATION_INTERNAL_H_
+#define ODP_EVENT_VALIDATION_INTERNAL_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <odp/autoheader_external.h>
+
+#include <odp/api/event.h>
+#include <odp/api/hints.h>
+
+#include <odp/api/plat/event_validation_external.h>
+
+#include <odp_event_internal.h>
+
+#include <stdint.h>
+
+#if _ODP_EVENT_VALIDATION
+
+#define _ODP_EV_ENDMARK_VAL  0xDEADBEEFDEADBEEF
+#define _ODP_EV_ENDMARK_SIZE (sizeof(uint64_t))
+
+static inline void _odp_event_endmark_set(odp_event_t event)
+{
+	uint64_t *endmark_ptr;
+
+	endmark_ptr = _odp_event_endmark_get_ptr(event);
+	*endmark_ptr = _ODP_EV_ENDMARK_VAL;
+}
+
+#else
+
+#define _ODP_EV_ENDMARK_VAL  0
+#define _ODP_EV_ENDMARK_SIZE 0
+
+static inline void _odp_event_endmark_set(odp_event_t event ODP_UNUSED)
+{
+}
+
+#endif
+
+#ifdef __cplusplus
+}
+#endif
+#endif
diff --git a/platform/linux-generic/include/odp_init_internal.h b/platform/linux-generic/include/odp_init_internal.h
index 2a1039854..24e8346ad 100644
--- a/platform/linux-generic/include/odp_init_internal.h
+++ b/platform/linux-generic/include/odp_init_internal.h
@@ -33,6 +33,9 @@ int _odp_pool_init_local(void);
 int _odp_pool_term_global(void);
 int _odp_pool_term_local(void);
 
+int _odp_event_validation_init_global(void);
+int _odp_event_validation_term_global(void);
+
 int _odp_queue_init_global(void);
 int _odp_queue_term_global(void);
 
diff --git a/platform/linux-generic/include/odp_packet_io_internal.h b/platform/linux-generic/include/odp_packet_io_internal.h
index 954602959..187a3a76f 100644
--- a/platform/linux-generic/include/odp_packet_io_internal.h
+++ b/platform/linux-generic/include/odp_packet_io_internal.h
@@ -1,5 +1,5 @@
 /* Copyright (c) 2013-2018, Linaro Limited
- * Copyright (c) 2019-2022, Nokia
+ * Copyright (c) 2019-2023, Nokia
  * All rights reserved.
  *
  * SPDX-License-Identifier:     BSD-3-Clause
@@ -68,12 +68,8 @@ struct pktio_if_ops;
 #define PKTIO_PRIVATE_SIZE 33792
 #elif defined(_ODP_PKTIO_XDP)
 #define PKTIO_PRIVATE_SIZE 29696
-#elif defined(_ODP_PKTIO_DPDK) && ODP_CACHE_LINE_SIZE == 128
-#define PKTIO_PRIVATE_SIZE 4160
-#elif defined(_ODP_PKTIO_DPDK)
-#define PKTIO_PRIVATE_SIZE 3968
 #else
-#define PKTIO_PRIVATE_SIZE 384
+#define PKTIO_PRIVATE_SIZE 9216
 #endif
 
 typedef struct ODP_ALIGNED_CACHE {
diff --git a/platform/linux-generic/include/odp_pool_internal.h b/platform/linux-generic/include/odp_pool_internal.h
index 1c5b51c3d..c8d2168f3 100644
--- a/platform/linux-generic/include/odp_pool_internal.h
+++ b/platform/linux-generic/include/odp_pool_internal.h
@@ -87,6 +87,7 @@ typedef struct pool_t {
 	uint32_t         block_size;
 	uint32_t         block_offset;
 	uint32_t         num_populated;
+	uint32_t         trailer_size;
 	uint8_t         *base_addr;
 	uint8_t         *max_addr;
 	uint8_t         *uarea_base_addr;
diff --git a/platform/linux-generic/include/odp_queue_basic_internal.h b/platform/linux-generic/include/odp_queue_basic_internal.h
index 830f50a9d..3cdcf8600 100644
--- a/platform/linux-generic/include/odp_queue_basic_internal.h
+++ b/platform/linux-generic/include/odp_queue_basic_internal.h
@@ -1,4 +1,5 @@
 /* Copyright (c) 2013-2018, Linaro Limited
+ * Copyright (c) 2023, Nokia
  * All rights reserved.
  *
  * SPDX-License-Identifier:     BSD-3-Clause
@@ -23,7 +24,7 @@ extern "C" {
 #include <odp/api/ticketlock.h>
 #include <odp_config_internal.h>
 #include <odp_macros_internal.h>
-#include <odp_ring_mpmc_internal.h>
+#include <odp_ring_mpmc_u32_internal.h>
 #include <odp_ring_st_internal.h>
 #include <odp_ring_spsc_internal.h>
 #include <odp_queue_lf.h>
@@ -47,7 +48,7 @@ typedef struct ODP_ALIGNED_CACHE queue_entry_s {
 	odp_queue_type_t     type;
 
 	/* MPMC ring (2 cache lines). */
-	ring_mpmc_t          ring_mpmc;
+	ring_mpmc_u32_t      ring_mpmc;
 
 	odp_ticketlock_t     lock;
 	union {
diff --git a/platform/linux-generic/include/odp_ring_mpmc_internal.h b/platform/linux-generic/include/odp_ring_mpmc_internal.h
index 6ed4dd4d1..e35179267 100644
--- a/platform/linux-generic/include/odp_ring_mpmc_internal.h
+++ b/platform/linux-generic/include/odp_ring_mpmc_internal.h
@@ -1,4 +1,5 @@
 /* Copyright (c) 2018, Linaro Limited
+ * Copyright (c) 2023, Nokia
  * All rights reserved.
  *
  * SPDX-License-Identifier:     BSD-3-Clause
@@ -19,6 +20,8 @@ extern "C" {
 #include <odp/api/plat/atomic_inlines.h>
 #include <odp/api/plat/cpu_inlines.h>
 
+#include <odp_ring_common.h>
+
 /* Ring of uint32_t data
  *
  * Ring stores head and tail counters. Ring indexes are formed from these
@@ -34,14 +37,22 @@ extern "C" {
  *     r_tail  r_head          w_tail      w_head
  *
  */
-typedef struct {
+
+struct ring_mpmc_common {
 	odp_atomic_u32_t r_head ODP_ALIGNED_CACHE;
 	odp_atomic_u32_t r_tail;
 
 	odp_atomic_u32_t w_head ODP_ALIGNED_CACHE;
 	odp_atomic_u32_t w_tail;
+};
+
+typedef struct ODP_ALIGNED_CACHE {
+	struct ring_mpmc_common r;
+} ring_mpmc_u32_t;
 
-} ring_mpmc_t;
+typedef struct ODP_ALIGNED_CACHE {
+	struct ring_mpmc_common r;
+} ring_mpmc_u64_t;
 
 static inline int ring_mpmc_cas_u32(odp_atomic_u32_t *atom,
 				    uint32_t *old_val, uint32_t new_val)
@@ -52,21 +63,63 @@ static inline int ring_mpmc_cas_u32(odp_atomic_u32_t *atom,
 					   __ATOMIC_RELAXED);
 }
 
+#endif /* End of include guards */
+
+#undef _ring_mpmc_gen_t
+#undef _ring_mpmc_data_t
+#undef _RING_MPMC_INIT
+#undef _RING_MPMC_DEQ_MULTI
+#undef _RING_MPMC_ENQ_MULTI
+#undef _RING_MPMC_DEQ_BATCH
+#undef _RING_MPMC_ENQ_BATCH
+#undef _RING_MPMC_IS_EMPTY
+#undef _RING_MPMC_LEN
+
+/* This header should NOT be included directly. There are no include guards for
+ * the following types and function definitions! */
+#ifndef _ODP_RING_TYPE
+#error Include type specific (u32/u64) ring header instead of this common file.
+#endif
+
+#if _ODP_RING_TYPE == _ODP_RING_TYPE_U32
+	#define _ring_mpmc_gen_t ring_mpmc_u32_t
+	#define _ring_mpmc_data_t uint32_t
+
+	#define _RING_MPMC_INIT ring_mpmc_u32_init
+	#define _RING_MPMC_DEQ_MULTI ring_mpmc_u32_deq_multi
+	#define _RING_MPMC_ENQ_MULTI ring_mpmc_u32_enq_multi
+	#define _RING_MPMC_DEQ_BATCH ring_mpmc_u32_deq_batch
+	#define _RING_MPMC_ENQ_BATCH ring_mpmc_u32_enq_batch
+	#define _RING_MPMC_IS_EMPTY ring_mpmc_u32_is_empty
+	#define _RING_MPMC_LEN ring_mpmc_u32_len
+#elif _ODP_RING_TYPE == _ODP_RING_TYPE_U64
+	#define _ring_mpmc_gen_t ring_mpmc_u64_t
+	#define _ring_mpmc_data_t uint64_t
+
+	#define _RING_MPMC_INIT ring_mpmc_u64_init
+	#define _RING_MPMC_DEQ_MULTI ring_mpmc_u64_deq_multi
+	#define _RING_MPMC_ENQ_MULTI ring_mpmc_u64_enq_multi
+	#define _RING_MPMC_DEQ_BATCH ring_mpmc_u64_deq_batch
+	#define _RING_MPMC_ENQ_BATCH ring_mpmc_u64_enq_batch
+	#define _RING_MPMC_IS_EMPTY ring_mpmc_u64_is_empty
+	#define _RING_MPMC_LEN ring_mpmc_u64_len
+#endif
+
 /* Initialize ring */
-static inline void ring_mpmc_init(ring_mpmc_t *ring)
+static inline void _RING_MPMC_INIT(_ring_mpmc_gen_t *ring)
 {
-	odp_atomic_init_u32(&ring->w_head, 0);
-	odp_atomic_init_u32(&ring->w_tail, 0);
-	odp_atomic_init_u32(&ring->r_head, 0);
-	odp_atomic_init_u32(&ring->r_tail, 0);
+	odp_atomic_init_u32(&ring->r.w_head, 0);
+	odp_atomic_init_u32(&ring->r.w_tail, 0);
+	odp_atomic_init_u32(&ring->r.r_head, 0);
+	odp_atomic_init_u32(&ring->r.r_tail, 0);
 }
 
-/* Dequeue data from the ring head. Num is smaller than ring size. */
-static inline uint32_t ring_mpmc_deq_multi(ring_mpmc_t *ring,
-					   uint32_t *ring_data,
-					   uint32_t ring_mask,
-					   uint32_t data[],
-					   uint32_t num)
+/* Dequeue data from the ring head */
+static inline uint32_t _RING_MPMC_DEQ_MULTI(_ring_mpmc_gen_t *ring,
+					    _ring_mpmc_data_t *ring_data,
+					    uint32_t ring_mask,
+					    _ring_mpmc_data_t data[],
+					    uint32_t num)
 {
 	uint32_t old_head, new_head, w_tail, num_data, i;
 
@@ -75,9 +128,9 @@ static inline uint32_t ring_mpmc_deq_multi(ring_mpmc_t *ring,
 	 * When CAS operation succeeds, this thread owns data between old
 	 * and new r_head. */
 	do {
-		old_head = odp_atomic_load_acq_u32(&ring->r_head);
+		old_head = odp_atomic_load_acq_u32(&ring->r.r_head);
 		odp_prefetch(&ring_data[(old_head + 1) & ring_mask]);
-		w_tail   = odp_atomic_load_acq_u32(&ring->w_tail);
+		w_tail   = odp_atomic_load_acq_u32(&ring->r.w_tail);
 		num_data = w_tail - old_head;
 
 		/* Ring is empty */
@@ -90,7 +143,49 @@ static inline uint32_t ring_mpmc_deq_multi(ring_mpmc_t *ring,
 
 		new_head = old_head + num;
 
-	} while (odp_unlikely(ring_mpmc_cas_u32(&ring->r_head, &old_head,
+	} while (odp_unlikely(ring_mpmc_cas_u32(&ring->r.r_head, &old_head,
+						new_head) == 0));
+
+	/* Read data. This will not move above load acquire of r_head. */
+	for (i = 0; i < num; i++)
+		data[i] = ring_data[(old_head + 1 + i) & ring_mask];
+
+	/* Wait until other readers have updated the tail */
+	while (odp_unlikely(odp_atomic_load_u32(&ring->r.r_tail) != old_head))
+		odp_cpu_pause();
+
+	/* Release the new reader tail, writers acquire it. */
+	odp_atomic_store_rel_u32(&ring->r.r_tail, new_head);
+
+	return num;
+}
+
+/* Dequeue num or 0 data from the ring head */
+static inline uint32_t _RING_MPMC_DEQ_BATCH(_ring_mpmc_gen_t *ring,
+					    _ring_mpmc_data_t *ring_data,
+					    uint32_t ring_mask,
+					    _ring_mpmc_data_t data[],
+					    uint32_t num)
+{
+	uint32_t old_head, new_head, w_tail, num_data, i;
+
+	/* Load acquires ensure that w_tail load happens after r_head load,
+	 * and thus r_head value is always behind or equal to w_tail value.
+	 * When CAS operation succeeds, this thread owns data between old
+	 * and new r_head. */
+	do {
+		old_head = odp_atomic_load_acq_u32(&ring->r.r_head);
+		odp_prefetch(&ring_data[(old_head + 1) & ring_mask]);
+		w_tail   = odp_atomic_load_acq_u32(&ring->r.w_tail);
+		num_data = w_tail - old_head;
+
+		/* Not enough data available */
+		if (num_data < num)
+			return 0;
+
+		new_head = old_head + num;
+
+	} while (odp_unlikely(ring_mpmc_cas_u32(&ring->r.r_head, &old_head,
 						new_head) == 0));
 
 	/* Read data. This will not move above load acquire of r_head. */
@@ -98,21 +193,21 @@ static inline uint32_t ring_mpmc_deq_multi(ring_mpmc_t *ring,
 		data[i] = ring_data[(old_head + 1 + i) & ring_mask];
 
 	/* Wait until other readers have updated the tail */
-	while (odp_unlikely(odp_atomic_load_u32(&ring->r_tail) != old_head))
+	while (odp_unlikely(odp_atomic_load_u32(&ring->r.r_tail) != old_head))
 		odp_cpu_pause();
 
 	/* Release the new reader tail, writers acquire it. */
-	odp_atomic_store_rel_u32(&ring->r_tail, new_head);
+	odp_atomic_store_rel_u32(&ring->r.r_tail, new_head);
 
 	return num;
 }
 
-/* Enqueue multiple data into the ring tail. Num is smaller than ring size. */
-static inline uint32_t ring_mpmc_enq_multi(ring_mpmc_t *ring,
-					   uint32_t *ring_data,
-					   uint32_t ring_mask,
-					   const uint32_t data[],
-					   uint32_t num)
+/* Enqueue multiple data into the ring tail */
+static inline uint32_t _RING_MPMC_ENQ_MULTI(_ring_mpmc_gen_t *ring,
+					    _ring_mpmc_data_t *ring_data,
+					    uint32_t ring_mask,
+					    const _ring_mpmc_data_t data[],
+					    uint32_t num)
 {
 	uint32_t old_head, new_head, r_tail, num_free, i;
 	uint32_t size = ring_mask + 1;
@@ -122,8 +217,8 @@ static inline uint32_t ring_mpmc_enq_multi(ring_mpmc_t *ring,
 	 * When CAS operation succeeds, this thread owns data between old
 	 * and new w_head. */
 	do {
-		r_tail   = odp_atomic_load_acq_u32(&ring->r_tail);
-		old_head = odp_atomic_load_acq_u32(&ring->w_head);
+		r_tail   = odp_atomic_load_acq_u32(&ring->r.r_tail);
+		old_head = odp_atomic_load_acq_u32(&ring->r.w_head);
 
 		num_free = size - (old_head - r_tail);
 
@@ -137,7 +232,50 @@ static inline uint32_t ring_mpmc_enq_multi(ring_mpmc_t *ring,
 
 		new_head = old_head + num;
 
-	} while (odp_unlikely(ring_mpmc_cas_u32(&ring->w_head, &old_head,
+	} while (odp_unlikely(ring_mpmc_cas_u32(&ring->r.w_head, &old_head,
+						new_head) == 0));
+
+	/* Write data. This will not move above load acquire of w_head. */
+	for (i = 0; i < num; i++)
+		ring_data[(old_head + 1 + i) & ring_mask] = data[i];
+
+	/* Wait until other writers have updated the tail */
+	while (odp_unlikely(odp_atomic_load_u32(&ring->r.w_tail) != old_head))
+		odp_cpu_pause();
+
+	/* Release the new writer tail, readers acquire it. */
+	odp_atomic_store_rel_u32(&ring->r.w_tail, new_head);
+
+	return num;
+}
+
+/* Enqueue num or 0 data into the ring tail */
+static inline uint32_t _RING_MPMC_ENQ_BATCH(_ring_mpmc_gen_t *ring,
+					    _ring_mpmc_data_t *ring_data,
+					    uint32_t ring_mask,
+					    const _ring_mpmc_data_t data[],
+					    uint32_t num)
+{
+	uint32_t old_head, new_head, r_tail, num_free, i;
+	uint32_t size = ring_mask + 1;
+
+	/* Load acquires ensure that w_head load happens after r_tail load,
+	 * and thus r_tail value is always behind or equal to w_head value.
+	 * When CAS operation succeeds, this thread owns data between old
+	 * and new w_head. */
+	do {
+		r_tail   = odp_atomic_load_acq_u32(&ring->r.r_tail);
+		old_head = odp_atomic_load_acq_u32(&ring->r.w_head);
+
+		num_free = size - (old_head - r_tail);
+
+		/* Not enough free space available */
+		if (num_free < num)
+			return 0;
+
+		new_head = old_head + num;
+
+	} while (odp_unlikely(ring_mpmc_cas_u32(&ring->r.w_head, &old_head,
 						new_head) == 0));
 
 	/* Write data. This will not move above load acquire of w_head. */
@@ -145,29 +283,29 @@ static inline uint32_t ring_mpmc_enq_multi(ring_mpmc_t *ring,
 		ring_data[(old_head + 1 + i) & ring_mask] = data[i];
 
 	/* Wait until other writers have updated the tail */
-	while (odp_unlikely(odp_atomic_load_u32(&ring->w_tail) != old_head))
+	while (odp_unlikely(odp_atomic_load_u32(&ring->r.w_tail) != old_head))
 		odp_cpu_pause();
 
 	/* Release the new writer tail, readers acquire it. */
-	odp_atomic_store_rel_u32(&ring->w_tail, new_head);
+	odp_atomic_store_rel_u32(&ring->r.w_tail, new_head);
 
 	return num;
 }
 
 /* Check if ring is empty */
-static inline int ring_mpmc_is_empty(ring_mpmc_t *ring)
+static inline int _RING_MPMC_IS_EMPTY(_ring_mpmc_gen_t *ring)
 {
-	uint32_t head = odp_atomic_load_u32(&ring->r_head);
-	uint32_t tail = odp_atomic_load_u32(&ring->w_tail);
+	uint32_t head = odp_atomic_load_u32(&ring->r.r_head);
+	uint32_t tail = odp_atomic_load_u32(&ring->r.w_tail);
 
 	return head == tail;
 }
 
 /* Return current ring length */
-static inline uint32_t ring_mpmc_length(ring_mpmc_t *ring)
+static inline uint32_t _RING_MPMC_LEN(_ring_mpmc_gen_t *ring)
 {
-	uint32_t head = odp_atomic_load_u32(&ring->r_head);
-	uint32_t tail = odp_atomic_load_u32(&ring->w_tail);
+	uint32_t head = odp_atomic_load_u32(&ring->r.r_head);
+	uint32_t tail = odp_atomic_load_u32(&ring->r.w_tail);
 
 	return tail - head;
 }
@@ -175,5 +313,3 @@ static inline uint32_t ring_mpmc_length(ring_mpmc_t *ring)
 #ifdef __cplusplus
 }
 #endif
-
-#endif
diff --git a/platform/linux-generic/include/odp_ring_mpmc_u32_internal.h b/platform/linux-generic/include/odp_ring_mpmc_u32_internal.h
new file mode 100644
index 000000000..4699b5b47
--- /dev/null
+++ b/platform/linux-generic/include/odp_ring_mpmc_u32_internal.h
@@ -0,0 +1,25 @@
+/* Copyright (c) 2023, Nokia
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier:     BSD-3-Clause
+ */
+
+#ifndef ODP_RING_MPMC_U32_INTERNAL_H_
+#define ODP_RING_MPMC_U32_INTERNAL_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <odp_ring_common.h>
+
+#undef _ODP_RING_TYPE
+#define _ODP_RING_TYPE _ODP_RING_TYPE_U32
+
+#include <odp_ring_mpmc_internal.h>
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/platform/linux-generic/include/odp_ring_mpmc_u64_internal.h b/platform/linux-generic/include/odp_ring_mpmc_u64_internal.h
new file mode 100644
index 000000000..e7bf31a94
--- /dev/null
+++ b/platform/linux-generic/include/odp_ring_mpmc_u64_internal.h
@@ -0,0 +1,25 @@
+/* Copyright (c) 2023, Nokia
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier:     BSD-3-Clause
+ */
+
+#ifndef ODP_RING_MPMC_U64_INTERNAL_H_
+#define ODP_RING_MPMC_U64_INTERNAL_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <odp_ring_common.h>
+
+#undef _ODP_RING_TYPE
+#define _ODP_RING_TYPE _ODP_RING_TYPE_U64
+
+#include <odp_ring_mpmc_internal.h>
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif