Multi-enq and deq operations

- Added multi-buffer enqueue and dequeue operations for queues
- And scheduler/packet IO support for those
- Added/renamed schedule functions
  - _poll removed from functions names (default operation)
  - former odp_schedule() is now odp_schedule_once()
- Enabled -O3 optimization (again)

Signed-off-by: Petri Savolainen <petri.savolainen@linaro.org>

7 files changed, 149 insertions, 7 deletions

diff --git a/platform/linux-generic/include/odp_buffer_internal.h b/platform/linux-generic/include/odp_buffer_internal.h
index f14738409..c4e5b3322 100644
--- a/platform/linux-generic/include/odp_buffer_internal.h
+++ b/platform/linux-generic/include/odp_buffer_internal.h
@@ -105,9 +105,6 @@ typedef struct odp_buffer_chunk_hdr_t {
 } odp_buffer_chunk_hdr_t;
 
 
-
-odp_buffer_hdr_t *odp_buf_to_hdr(odp_buffer_t buf);
-
 int odp_buffer_snprint(char *str, size_t n, odp_buffer_t buf);
 
 
diff --git a/platform/linux-generic/include/odp_buffer_pool_internal.h b/platform/linux-generic/include/odp_buffer_pool_internal.h
new file mode 100644
index 000000000..380de3692
--- /dev/null
+++ b/platform/linux-generic/include/odp_buffer_pool_internal.h
@@ -0,0 +1,116 @@
+/* Copyright (c) 2013, Linaro Limited
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier:     BSD-3-Clause
+ */
+
+
+/**
+ * @file
+ *
+ * ODP buffer pool - internal header
+ */
+
+#ifndef ODP_BUFFER_POOL_INTERNAL_H_
+#define ODP_BUFFER_POOL_INTERNAL_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <odp_std_types.h>
+#include <odp_buffer_pool.h>
+#include <odp_buffer_internal.h>
+#include <odp_align.h>
+#include <odp_hints.h>
+#include <odp_config.h>
+#include <odp_debug.h>
+
+/* Use ticketlock instead of spinlock */
+#define POOL_USE_TICKETLOCK
+
+/* Extra error checks */
+/* #define POOL_ERROR_CHECK */
+
+
+#ifdef POOL_USE_TICKETLOCK
+#include <odp_ticketlock.h>
+#else
+#include <odp_spinlock.h>
+#endif
+
+
+struct pool_entry_s {
+#ifdef POOL_USE_TICKETLOCK
+	odp_ticketlock_t        lock ODP_ALIGNED_CACHE;
+#else
+	odp_spinlock_t          lock ODP_ALIGNED_CACHE;
+#endif
+
+	odp_buffer_chunk_hdr_t *head;
+	uint64_t                free_bufs;
+	char                    name[ODP_BUFFER_POOL_NAME_LEN];
+
+
+	odp_buffer_pool_t       pool ODP_ALIGNED_CACHE;
+	uintptr_t               buf_base;
+	size_t                  buf_size;
+	size_t                  buf_offset;
+	uint64_t                num_bufs;
+	void                   *pool_base_addr;
+	uint64_t                pool_size;
+	size_t                  payload_size;
+	size_t                  payload_align;
+	int                     buf_type;
+	size_t                  hdr_size;
+};
+
+
+extern void *pool_entry_ptr[];
+
+
+static inline void *get_pool_entry(odp_buffer_pool_t pool_id)
+{
+	return pool_entry_ptr[pool_id];
+}
+
+
+static inline odp_buffer_hdr_t *odp_buf_to_hdr(odp_buffer_t buf)
+{
+	odp_buffer_bits_t handle;
+	uint32_t pool_id;
+	uint32_t index;
+	struct pool_entry_s *pool;
+	odp_buffer_hdr_t *hdr;
+
+	handle.u32 = buf;
+	pool_id    = handle.pool;
+	index      = handle.index;
+
+#ifdef POOL_ERROR_CHECK
+	if (odp_unlikely(pool_id > ODP_CONFIG_BUFFER_POOLS)) {
+		ODP_ERR("odp_buf_to_hdr: Bad pool id\n");
+		return NULL;
+	}
+#endif
+
+	pool = get_pool_entry(pool_id);
+
+#ifdef POOL_ERROR_CHECK
+	if (odp_unlikely(index > pool->num_bufs - 1)) {
+		ODP_ERR("odp_buf_to_hdr: Bad buffer index\n");
+		return NULL;
+	}
+#endif
+
+	hdr = (odp_buffer_hdr_t *)(pool->buf_base + index * pool->buf_size);
+
+	return hdr;
+}
+
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/platform/linux-generic/include/odp_internal.h b/platform/linux-generic/include/odp_internal.h
index cfd19c948..1f42d2abb 100644
--- a/platform/linux-generic/include/odp_internal.h
+++ b/platform/linux-generic/include/odp_internal.h
@@ -40,6 +40,7 @@ int odp_queue_init_global(void);
 
 
 int odp_schedule_init_global(void);
+int odp_schedule_init_local(void);
 
 
 #ifdef __cplusplus
diff --git a/platform/linux-generic/include/odp_packet_internal.h b/platform/linux-generic/include/odp_packet_internal.h
index 034162fb2..afef97630 100644
--- a/platform/linux-generic/include/odp_packet_internal.h
+++ b/platform/linux-generic/include/odp_packet_internal.h
@@ -21,6 +21,7 @@ extern "C" {
 #include <odp_align.h>
 #include <odp_debug.h>
 #include <odp_buffer_internal.h>
+#include <odp_buffer_pool_internal.h>
 #include <odp_packet.h>
 #include <odp_packet_io.h>
 
diff --git a/platform/linux-generic/include/odp_packet_io_queue.h b/platform/linux-generic/include/odp_packet_io_queue.h
index 58d69611e..18e55f604 100644
--- a/platform/linux-generic/include/odp_packet_io_queue.h
+++ b/platform/linux-generic/include/odp_packet_io_queue.h
@@ -21,14 +21,27 @@ extern "C" {
 #include <odp_queue_internal.h>
 #include <odp_buffer_internal.h>
 
+/** Max nbr of pkts to receive in one burst (keep same as QUEUE_MULTI_MAX) */
 #define ODP_PKTIN_QUEUE_MAX_BURST 16
+/* pktin_deq_multi() depends on the condition: */
+ODP_ASSERT(ODP_PKTIN_QUEUE_MAX_BURST >= QUEUE_MULTI_MAX,
+	   ODP_PKTIN_DEQ_MULTI_MAX_ERROR);
 
 int pktin_enqueue(queue_entry_t *queue, odp_buffer_hdr_t *buf_hdr);
 odp_buffer_hdr_t *pktin_dequeue(queue_entry_t *queue);
 
+int pktin_enq_multi(queue_entry_t *queue, odp_buffer_hdr_t *buf_hdr[], int num);
+int pktin_deq_multi(queue_entry_t *queue, odp_buffer_hdr_t *buf_hdr[], int num);
+
+
 int pktout_enqueue(queue_entry_t *queue, odp_buffer_hdr_t *buf_hdr);
 odp_buffer_hdr_t *pktout_dequeue(queue_entry_t *queue);
 
+int pktout_enq_multi(queue_entry_t *queue, odp_buffer_hdr_t *buf_hdr[],
+		     int num);
+int pktout_deq_multi(queue_entry_t *queue, odp_buffer_hdr_t *buf_hdr[],
+		     int num);
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/platform/linux-generic/include/odp_queue_internal.h b/platform/linux-generic/include/odp_queue_internal.h
index 6d6700a60..fede56a0c 100644
--- a/platform/linux-generic/include/odp_queue_internal.h
+++ b/platform/linux-generic/include/odp_queue_internal.h
@@ -32,6 +32,8 @@ extern "C" {
 #include <odp_spinlock.h>
 #endif
 
+#define QUEUE_MULTI_MAX 8
+
 #define QUEUE_STATUS_FREE     0
 #define QUEUE_STATUS_READY    1
 #define QUEUE_STATUS_NOTSCHED 2
@@ -40,8 +42,13 @@ extern "C" {
 /* forward declaration */
 union queue_entry_u;
 
-typedef int (*enqueue_func_t)(union queue_entry_u *, odp_buffer_hdr_t *);
-typedef	odp_buffer_hdr_t *(*dequeue_func_t)(union queue_entry_u *);
+typedef int (*enq_func_t)(union queue_entry_u *, odp_buffer_hdr_t *);
+typedef	odp_buffer_hdr_t *(*deq_func_t)(union queue_entry_u *);
+
+typedef int (*enq_multi_func_t)(union queue_entry_u *,
+				odp_buffer_hdr_t **, int);
+typedef	int (*deq_multi_func_t)(union queue_entry_u *,
+				odp_buffer_hdr_t **, int);
 
 struct queue_entry_s {
 #ifdef USE_TICKETLOCK
@@ -54,8 +61,11 @@ struct queue_entry_s {
 	odp_buffer_hdr_t *tail;
 	int               status;
 
-	enqueue_func_t    enqueue ODP_ALIGNED_CACHE;
-	dequeue_func_t    dequeue;
+	enq_func_t       enqueue ODP_ALIGNED_CACHE;
+	deq_func_t       dequeue;
+	enq_multi_func_t enqueue_multi;
+	deq_multi_func_t dequeue_multi;
+
 	odp_queue_t       handle;
 	odp_buffer_t      sched_buf;
 	odp_queue_type_t  type;
@@ -76,6 +86,9 @@ queue_entry_t *get_qentry(uint32_t queue_id);
 int queue_enq(queue_entry_t *queue, odp_buffer_hdr_t *buf_hdr);
 odp_buffer_hdr_t *queue_deq(queue_entry_t *queue);
 
+int queue_enq_multi(queue_entry_t *queue, odp_buffer_hdr_t *buf_hdr[], int num);
+int queue_deq_multi(queue_entry_t *queue, odp_buffer_hdr_t *buf_hdr[], int num);
+
 void queue_lock(queue_entry_t *queue);
 void queue_unlock(queue_entry_t *queue);
 
diff --git a/platform/linux-generic/include/odp_schedule_internal.h b/platform/linux-generic/include/odp_schedule_internal.h
index bd0ec27bb..b0135c58b 100644
--- a/platform/linux-generic/include/odp_schedule_internal.h
+++ b/platform/linux-generic/include/odp_schedule_internal.h
@@ -17,6 +17,7 @@ extern "C" {
 #include <odp_buffer.h>
 #include <odp_queue.h>
 
+void odp_schedule_mask_set(odp_queue_t queue, int prio);
 
 odp_buffer_t odp_schedule_buffer_alloc(odp_queue_t queue);