diff options
author | Jere Leppänen <jere.leppanen@nokia.com> | 2022-06-08 19:02:40 +0300 |
---|---|---|
committer | Matias Elo <matias.elo@nokia.com> | 2022-06-21 16:02:01 +0300 |
commit | 1f683c665f8644143538a50aaf24530599eefada (patch) | |
tree | 60d4afd1f8f489aa1a04b103516cf2077bff243b | |
parent | 37bf0f0d27b7322b8ed602001330f3fb32c53e34 (diff) |
linux-gen: queue: remove an extra level of hierarchy from queue_entry_t
Remove an extra level of hierarchy from queue_entry_t, and make the
type cache aligned.
Signed-off-by: Jere Leppänen <jere.leppanen@nokia.com>
Reviewed-by: Tuomas Taipale <tuomas.taipale@nokia.com>
-rw-r--r-- | platform/linux-generic/include/odp_forward_typedefs_internal.h | 2 | ||||
-rw-r--r-- | platform/linux-generic/include/odp_queue_basic_internal.h | 11 | ||||
-rw-r--r-- | platform/linux-generic/include/odp_queue_scalable_internal.h | 13 | ||||
-rw-r--r-- | platform/linux-generic/odp_queue_basic.c | 372 | ||||
-rw-r--r-- | platform/linux-generic/odp_queue_lf.c | 6 | ||||
-rw-r--r-- | platform/linux-generic/odp_queue_scalable.c | 264 | ||||
-rw-r--r-- | platform/linux-generic/odp_queue_spsc.c | 34 | ||||
-rw-r--r-- | platform/linux-generic/odp_schedule_basic.c | 4 | ||||
-rw-r--r-- | platform/linux-generic/odp_schedule_scalable.c | 4 | ||||
-rw-r--r-- | platform/linux-generic/odp_schedule_scalable_ordered.c | 2 |
10 files changed, 351 insertions, 361 deletions
diff --git a/platform/linux-generic/include/odp_forward_typedefs_internal.h b/platform/linux-generic/include/odp_forward_typedefs_internal.h index d7e14b953..350ad6a36 100644 --- a/platform/linux-generic/include/odp_forward_typedefs_internal.h +++ b/platform/linux-generic/include/odp_forward_typedefs_internal.h @@ -22,7 +22,7 @@ extern "C" { #endif -typedef union queue_entry_u queue_entry_t; +typedef struct queue_entry_s queue_entry_t; #ifdef __cplusplus } diff --git a/platform/linux-generic/include/odp_queue_basic_internal.h b/platform/linux-generic/include/odp_queue_basic_internal.h index 9babb62e9..830f50a9d 100644 --- a/platform/linux-generic/include/odp_queue_basic_internal.h +++ b/platform/linux-generic/include/odp_queue_basic_internal.h @@ -34,7 +34,7 @@ extern "C" { #define QUEUE_STATUS_NOTSCHED 3 #define QUEUE_STATUS_SCHED 4 -struct queue_entry_s { +typedef struct ODP_ALIGNED_CACHE queue_entry_s { /* The first cache line is read only */ queue_enq_fn_t enqueue ODP_ALIGNED_CACHE; queue_deq_fn_t dequeue; @@ -65,12 +65,7 @@ struct queue_entry_s { void *queue_lf; int spsc; char name[ODP_QUEUE_NAME_LEN]; -}; - -union queue_entry_u { - struct queue_entry_s s; - uint8_t pad[_ODP_ROUNDUP_CACHE_LINE(sizeof(struct queue_entry_s))]; -}; +} queue_entry_t; typedef struct queue_global_t { queue_entry_t queue[CONFIG_MAX_QUEUES]; @@ -94,7 +89,7 @@ static inline uint32_t queue_to_index(odp_queue_t handle) { queue_entry_t *qentry = (queue_entry_t *)(uintptr_t)handle; - return qentry->s.index; + return qentry->index; } static inline queue_entry_t *qentry_from_index(uint32_t queue_id) diff --git a/platform/linux-generic/include/odp_queue_scalable_internal.h b/platform/linux-generic/include/odp_queue_scalable_internal.h index 3c74d1699..f7fc94e71 100644 --- a/platform/linux-generic/include/odp_queue_scalable_internal.h +++ b/platform/linux-generic/include/odp_queue_scalable_internal.h @@ -31,7 +31,7 @@ extern "C" { #define QUEUE_STATUS_DESTROYED 1 #define QUEUE_STATUS_READY 2 -struct queue_entry_s { +struct ODP_ALIGNED_CACHE queue_entry_s { sched_elem_t sched_elem; odp_ticketlock_t lock ODP_ALIGNED_CACHE; @@ -53,11 +53,6 @@ struct queue_entry_s { char name[ODP_QUEUE_NAME_LEN]; }; -union queue_entry_u { - struct queue_entry_s s; - uint8_t pad[_ODP_ROUNDUP_CACHE_LINE(sizeof(struct queue_entry_s))]; -}; - int _odp_queue_deq(sched_elem_t *q, _odp_event_hdr_t *event_hdr[], int num); int _odp_queue_deq_sc(sched_elem_t *q, odp_event_t *evp, int num); int _odp_queue_deq_mc(sched_elem_t *q, odp_event_t *evp, int num); @@ -79,7 +74,7 @@ static inline void *shm_pool_alloc_align(_odp_ishm_pool_t *pool, uint32_t size) static inline uint32_t queue_to_id(odp_queue_t handle) { - return _odp_qentry_from_ext(handle)->s.index; + return _odp_qentry_from_ext(handle)->index; } static inline queue_entry_t *qentry_from_int(odp_queue_t handle) @@ -94,12 +89,12 @@ static inline odp_queue_t qentry_to_int(queue_entry_t *qentry) static inline odp_queue_t queue_get_handle(queue_entry_t *queue) { - return queue->s.handle; + return queue->handle; } static inline reorder_window_t *queue_get_rwin(queue_entry_t *queue) { - return queue->s.sched_elem.rwin; + return queue->sched_elem.rwin; } #ifdef __cplusplus diff --git a/platform/linux-generic/odp_queue_basic.c b/platform/linux-generic/odp_queue_basic.c index 9a1abfd3b..f485b7819 100644 --- a/platform/linux-generic/odp_queue_basic.c +++ b/platform/linux-generic/odp_queue_basic.c @@ -31,9 +31,9 @@ #include <odp_macros_internal.h> #include <odp/api/plat/ticketlock_inlines.h> -#define LOCK(queue_ptr) odp_ticketlock_lock(&((queue_ptr)->s.lock)) -#define UNLOCK(queue_ptr) odp_ticketlock_unlock(&((queue_ptr)->s.lock)) -#define LOCK_INIT(queue_ptr) odp_ticketlock_init(&((queue_ptr)->s.lock)) +#define LOCK(queue_ptr) odp_ticketlock_lock(&((queue_ptr)->lock)) +#define UNLOCK(queue_ptr) odp_ticketlock_unlock(&((queue_ptr)->lock)) +#define LOCK_INIT(queue_ptr) odp_ticketlock_init(&((queue_ptr)->lock)) #include <string.h> #include <inttypes.h> @@ -122,7 +122,7 @@ static int queue_init_global(void) memset(&_odp_queue_inline_offset, 0, sizeof(_odp_queue_inline_offset_t)); _odp_queue_inline_offset.context = offsetof(queue_entry_t, - s.param.context); + param.context); shm = odp_shm_reserve("_odp_queue_basic_global", sizeof(queue_global_t), @@ -140,8 +140,8 @@ static int queue_init_global(void) queue_entry_t *queue = qentry_from_index(i); LOCK_INIT(queue); - queue->s.index = i; - queue->s.handle = (odp_queue_t)queue; + queue->index = i; + queue->handle = (odp_queue_t)queue; } if (read_config_file(_odp_queue_glb)) { @@ -200,8 +200,8 @@ static int queue_term_global(void) for (i = 0; i < CONFIG_MAX_QUEUES; i++) { queue = qentry_from_index(i); LOCK(queue); - if (queue->s.status != QUEUE_STATUS_FREE) { - ODP_ERR("Not destroyed queue: %s\n", queue->s.name); + if (queue->status != QUEUE_STATUS_FREE) { + ODP_ERR("Not destroyed queue: %s\n", queue->name); ret = -1; } UNLOCK(queue); @@ -229,30 +229,30 @@ static int queue_capability(odp_queue_capability_t *capa) static odp_queue_type_t queue_type(odp_queue_t handle) { - return qentry_from_handle(handle)->s.type; + return qentry_from_handle(handle)->type; } static odp_schedule_sync_t queue_sched_type(odp_queue_t handle) { - return qentry_from_handle(handle)->s.param.sched.sync; + return qentry_from_handle(handle)->param.sched.sync; } static odp_schedule_prio_t queue_sched_prio(odp_queue_t handle) { - return qentry_from_handle(handle)->s.param.sched.prio; + return qentry_from_handle(handle)->param.sched.prio; } static odp_schedule_group_t queue_sched_group(odp_queue_t handle) { - return qentry_from_handle(handle)->s.param.sched.group; + return qentry_from_handle(handle)->param.sched.group; } static uint32_t queue_lock_count(odp_queue_t handle) { queue_entry_t *queue = qentry_from_handle(handle); - return queue->s.param.sched.sync == ODP_SCHED_SYNC_ORDERED ? - queue->s.param.sched.lock_count : 0; + return queue->param.sched.sync == ODP_SCHED_SYNC_ORDERED ? + queue->param.sched.lock_count : 0; } static odp_queue_t queue_create(const char *name, @@ -309,17 +309,17 @@ static odp_queue_t queue_create(const char *name, for (; i < max_idx; i++) { queue = qentry_from_index(i); - if (queue->s.status != QUEUE_STATUS_FREE) + if (queue->status != QUEUE_STATUS_FREE) continue; LOCK(queue); - if (queue->s.status == QUEUE_STATUS_FREE) { + if (queue->status == QUEUE_STATUS_FREE) { if (queue_init(queue, name, param)) { UNLOCK(queue); return ODP_QUEUE_INVALID; } - if (!queue->s.spsc && + if (!queue->spsc && param->nonblocking == ODP_NONBLOCKING_LF) { queue_lf_func_t *lf_fn; @@ -331,21 +331,21 @@ static odp_queue_t queue_create(const char *name, UNLOCK(queue); return ODP_QUEUE_INVALID; } - queue->s.queue_lf = queue_lf; + queue->queue_lf = queue_lf; - queue->s.enqueue = lf_fn->enq; - queue->s.enqueue_multi = lf_fn->enq_multi; - queue->s.dequeue = lf_fn->deq; - queue->s.dequeue_multi = lf_fn->deq_multi; - queue->s.orig_dequeue_multi = lf_fn->deq_multi; + queue->enqueue = lf_fn->enq; + queue->enqueue_multi = lf_fn->enq_multi; + queue->dequeue = lf_fn->deq; + queue->dequeue_multi = lf_fn->deq_multi; + queue->orig_dequeue_multi = lf_fn->deq_multi; } if (type == ODP_QUEUE_TYPE_SCHED) - queue->s.status = QUEUE_STATUS_NOTSCHED; + queue->status = QUEUE_STATUS_NOTSCHED; else - queue->s.status = QUEUE_STATUS_READY; + queue->status = QUEUE_STATUS_READY; - handle = queue->s.handle; + handle = queue->handle; UNLOCK(queue); break; } @@ -356,9 +356,9 @@ static odp_queue_t queue_create(const char *name, return ODP_QUEUE_INVALID; if (type == ODP_QUEUE_TYPE_SCHED) { - if (_odp_sched_fn->create_queue(queue->s.index, - &queue->s.param.sched)) { - queue->s.status = QUEUE_STATUS_FREE; + if (_odp_sched_fn->create_queue(queue->index, + &queue->param.sched)) { + queue->status = QUEUE_STATUS_FREE; ODP_ERR("schedule queue init failed\n"); return ODP_QUEUE_INVALID; } @@ -373,7 +373,7 @@ void _odp_sched_queue_set_status(uint32_t queue_index, int status) LOCK(queue); - queue->s.status = status; + queue->status = status; UNLOCK(queue); } @@ -389,48 +389,48 @@ static int queue_destroy(odp_queue_t handle) return -1; LOCK(queue); - if (queue->s.status == QUEUE_STATUS_FREE) { + if (queue->status == QUEUE_STATUS_FREE) { UNLOCK(queue); - ODP_ERR("queue \"%s\" already free\n", queue->s.name); + ODP_ERR("queue \"%s\" already free\n", queue->name); return -1; } - if (queue->s.status == QUEUE_STATUS_DESTROYED) { + if (queue->status == QUEUE_STATUS_DESTROYED) { UNLOCK(queue); - ODP_ERR("queue \"%s\" already destroyed\n", queue->s.name); + ODP_ERR("queue \"%s\" already destroyed\n", queue->name); return -1; } - if (queue->s.spsc) - empty = ring_spsc_is_empty(&queue->s.ring_spsc); - else if (queue->s.type == ODP_QUEUE_TYPE_SCHED) - empty = ring_st_is_empty(&queue->s.ring_st); + if (queue->spsc) + empty = ring_spsc_is_empty(&queue->ring_spsc); + else if (queue->type == ODP_QUEUE_TYPE_SCHED) + empty = ring_st_is_empty(&queue->ring_st); else - empty = ring_mpmc_is_empty(&queue->s.ring_mpmc); + empty = ring_mpmc_is_empty(&queue->ring_mpmc); if (!empty) { UNLOCK(queue); - ODP_ERR("queue \"%s\" not empty\n", queue->s.name); + ODP_ERR("queue \"%s\" not empty\n", queue->name); return -1; } - switch (queue->s.status) { + switch (queue->status) { case QUEUE_STATUS_READY: - queue->s.status = QUEUE_STATUS_FREE; + queue->status = QUEUE_STATUS_FREE; break; case QUEUE_STATUS_NOTSCHED: - queue->s.status = QUEUE_STATUS_FREE; - _odp_sched_fn->destroy_queue(queue->s.index); + queue->status = QUEUE_STATUS_FREE; + _odp_sched_fn->destroy_queue(queue->index); break; case QUEUE_STATUS_SCHED: /* Queue is still in scheduling */ - queue->s.status = QUEUE_STATUS_DESTROYED; + queue->status = QUEUE_STATUS_DESTROYED; break; default: ODP_ABORT("Unexpected queue status\n"); } - if (queue->s.queue_lf) - _odp_queue_lf_destroy(queue->s.queue_lf); + if (queue->queue_lf) + _odp_queue_lf_destroy(queue->queue_lf); UNLOCK(queue); @@ -441,7 +441,7 @@ static int queue_context_set(odp_queue_t handle, void *context, uint32_t len ODP_UNUSED) { odp_mb_full(); - qentry_from_handle(handle)->s.param.context = context; + qentry_from_handle(handle)->param.context = context; odp_mb_full(); return 0; } @@ -453,15 +453,15 @@ static odp_queue_t queue_lookup(const char *name) for (i = 0; i < CONFIG_MAX_QUEUES; i++) { queue_entry_t *queue = qentry_from_index(i); - if (queue->s.status == QUEUE_STATUS_FREE || - queue->s.status == QUEUE_STATUS_DESTROYED) + if (queue->status == QUEUE_STATUS_FREE || + queue->status == QUEUE_STATUS_DESTROYED) continue; LOCK(queue); - if (strcmp(name, queue->s.name) == 0) { + if (strcmp(name, queue->name) == 0) { /* found it */ UNLOCK(queue); - return queue->s.handle; + return queue->handle; } UNLOCK(queue); } @@ -498,15 +498,15 @@ static inline int _plain_queue_enq_multi(odp_queue_t handle, uint32_t event_idx[num]; queue = qentry_from_handle(handle); - ring_mpmc = &queue->s.ring_mpmc; + ring_mpmc = &queue->ring_mpmc; if (_odp_sched_fn->ord_enq_multi(handle, (void **)event_hdr, num, &ret)) return ret; event_index_from_hdr(event_idx, event_hdr, num); - num_enq = ring_mpmc_enq_multi(ring_mpmc, queue->s.ring_data, - queue->s.ring_mask, event_idx, num); + num_enq = ring_mpmc_enq_multi(ring_mpmc, queue->ring_data, + queue->ring_mask, event_idx, num); return num_enq; } @@ -520,10 +520,10 @@ static inline int _plain_queue_deq_multi(odp_queue_t handle, uint32_t event_idx[num]; queue = qentry_from_handle(handle); - ring_mpmc = &queue->s.ring_mpmc; + ring_mpmc = &queue->ring_mpmc; - num_deq = ring_mpmc_deq_multi(ring_mpmc, queue->s.ring_data, - queue->s.ring_mask, event_idx, num); + num_deq = ring_mpmc_deq_multi(ring_mpmc, queue->ring_data, + queue->ring_mask, event_idx, num); if (num_deq == 0) return 0; @@ -647,7 +647,7 @@ static int queue_info(odp_queue_t handle, odp_queue_info_t *info) queue = qentry_from_index(queue_id); LOCK(queue); - status = queue->s.status; + status = queue->status; if (odp_unlikely(status == QUEUE_STATUS_FREE || status == QUEUE_STATUS_DESTROYED)) { @@ -656,8 +656,8 @@ static int queue_info(odp_queue_t handle, odp_queue_info_t *info) return -1; } - info->name = queue->s.name; - info->param = queue->s.param; + info->name = queue->name; + info->param = queue->param; UNLOCK(queue); @@ -683,7 +683,7 @@ static void queue_print(odp_queue_t handle) queue = qentry_from_index(queue_id); LOCK(queue); - status = queue->s.status; + status = queue->status; if (odp_unlikely(status == QUEUE_STATUS_FREE || status == QUEUE_STATUS_DESTROYED)) { @@ -693,72 +693,72 @@ static void queue_print(odp_queue_t handle) } ODP_PRINT("\nQueue info\n"); ODP_PRINT("----------\n"); - ODP_PRINT(" handle %p\n", (void *)queue->s.handle); + ODP_PRINT(" handle %p\n", (void *)queue->handle); ODP_PRINT(" index %" PRIu32 "\n", queue_id); - ODP_PRINT(" name %s\n", queue->s.name); + ODP_PRINT(" name %s\n", queue->name); ODP_PRINT(" enq mode %s\n", - queue->s.param.enq_mode == ODP_QUEUE_OP_MT ? "ODP_QUEUE_OP_MT" : - (queue->s.param.enq_mode == ODP_QUEUE_OP_MT_UNSAFE ? "ODP_QUEUE_OP_MT_UNSAFE" : - (queue->s.param.enq_mode == ODP_QUEUE_OP_DISABLED ? "ODP_QUEUE_OP_DISABLED" : + queue->param.enq_mode == ODP_QUEUE_OP_MT ? "ODP_QUEUE_OP_MT" : + (queue->param.enq_mode == ODP_QUEUE_OP_MT_UNSAFE ? "ODP_QUEUE_OP_MT_UNSAFE" : + (queue->param.enq_mode == ODP_QUEUE_OP_DISABLED ? "ODP_QUEUE_OP_DISABLED" : "unknown"))); ODP_PRINT(" deq mode %s\n", - queue->s.param.deq_mode == ODP_QUEUE_OP_MT ? "ODP_QUEUE_OP_MT" : - (queue->s.param.deq_mode == ODP_QUEUE_OP_MT_UNSAFE ? "ODP_QUEUE_OP_MT_UNSAFE" : - (queue->s.param.deq_mode == ODP_QUEUE_OP_DISABLED ? "ODP_QUEUE_OP_DISABLED" : + queue->param.deq_mode == ODP_QUEUE_OP_MT ? "ODP_QUEUE_OP_MT" : + (queue->param.deq_mode == ODP_QUEUE_OP_MT_UNSAFE ? "ODP_QUEUE_OP_MT_UNSAFE" : + (queue->param.deq_mode == ODP_QUEUE_OP_DISABLED ? "ODP_QUEUE_OP_DISABLED" : "unknown"))); ODP_PRINT(" non-blocking %s\n", - queue->s.param.nonblocking == ODP_BLOCKING ? "ODP_BLOCKING" : - (queue->s.param.nonblocking == ODP_NONBLOCKING_LF ? "ODP_NONBLOCKING_LF" : - (queue->s.param.nonblocking == ODP_NONBLOCKING_WF ? "ODP_NONBLOCKING_WF" : + queue->param.nonblocking == ODP_BLOCKING ? "ODP_BLOCKING" : + (queue->param.nonblocking == ODP_NONBLOCKING_LF ? "ODP_NONBLOCKING_LF" : + (queue->param.nonblocking == ODP_NONBLOCKING_WF ? "ODP_NONBLOCKING_WF" : "unknown"))); ODP_PRINT(" type %s\n", - queue->s.type == ODP_QUEUE_TYPE_PLAIN ? "ODP_QUEUE_TYPE_PLAIN" : - (queue->s.type == ODP_QUEUE_TYPE_SCHED ? "ODP_QUEUE_TYPE_SCHED" : "unknown")); - if (queue->s.type == ODP_QUEUE_TYPE_SCHED) { + queue->type == ODP_QUEUE_TYPE_PLAIN ? "ODP_QUEUE_TYPE_PLAIN" : + (queue->type == ODP_QUEUE_TYPE_SCHED ? "ODP_QUEUE_TYPE_SCHED" : "unknown")); + if (queue->type == ODP_QUEUE_TYPE_SCHED) { ODP_PRINT(" sync %s\n", - queue->s.param.sched.sync == ODP_SCHED_SYNC_PARALLEL ? + queue->param.sched.sync == ODP_SCHED_SYNC_PARALLEL ? "ODP_SCHED_SYNC_PARALLEL" : - (queue->s.param.sched.sync == ODP_SCHED_SYNC_ATOMIC ? + (queue->param.sched.sync == ODP_SCHED_SYNC_ATOMIC ? "ODP_SCHED_SYNC_ATOMIC" : - (queue->s.param.sched.sync == ODP_SCHED_SYNC_ORDERED ? + (queue->param.sched.sync == ODP_SCHED_SYNC_ORDERED ? "ODP_SCHED_SYNC_ORDERED" : "unknown"))); - prio = queue->s.param.sched.prio; + prio = queue->param.sched.prio; ODP_PRINT(" priority %i (%i in API)\n", max_prio - prio, prio); - ODP_PRINT(" group %i\n", queue->s.param.sched.group); + ODP_PRINT(" group %i\n", queue->param.sched.group); if (_odp_sched_id == _ODP_SCHED_ID_BASIC) ODP_PRINT(" spread %i\n", _odp_sched_basic_get_spread(queue_id)); } - if (queue->s.pktin.pktio != ODP_PKTIO_INVALID) { - if (!odp_pktio_info(queue->s.pktin.pktio, &pktio_info)) + if (queue->pktin.pktio != ODP_PKTIO_INVALID) { + if (!odp_pktio_info(queue->pktin.pktio, &pktio_info)) ODP_PRINT(" pktin %s\n", pktio_info.name); } - if (queue->s.pktout.pktio != ODP_PKTIO_INVALID) { - if (!odp_pktio_info(queue->s.pktout.pktio, &pktio_info)) + if (queue->pktout.pktio != ODP_PKTIO_INVALID) { + if (!odp_pktio_info(queue->pktout.pktio, &pktio_info)) ODP_PRINT(" pktout %s\n", pktio_info.name); } ODP_PRINT(" timers %" PRIu64 "\n", - odp_atomic_load_u64(&queue->s.num_timers)); + odp_atomic_load_u64(&queue->num_timers)); ODP_PRINT(" status %s\n", - queue->s.status == QUEUE_STATUS_READY ? "ready" : - (queue->s.status == QUEUE_STATUS_NOTSCHED ? "not scheduled" : - (queue->s.status == QUEUE_STATUS_SCHED ? "scheduled" : "unknown"))); - ODP_PRINT(" param.size %" PRIu32 "\n", queue->s.param.size); - if (queue->s.queue_lf) { + queue->status == QUEUE_STATUS_READY ? "ready" : + (queue->status == QUEUE_STATUS_NOTSCHED ? "not scheduled" : + (queue->status == QUEUE_STATUS_SCHED ? "scheduled" : "unknown"))); + ODP_PRINT(" param.size %" PRIu32 "\n", queue->param.size); + if (queue->queue_lf) { ODP_PRINT(" implementation queue_lf\n"); ODP_PRINT(" length %" PRIu32 "/%" PRIu32 "\n", - _odp_queue_lf_length(queue->s.queue_lf), _odp_queue_lf_max_length()); - } else if (queue->s.spsc) { + _odp_queue_lf_length(queue->queue_lf), _odp_queue_lf_max_length()); + } else if (queue->spsc) { ODP_PRINT(" implementation ring_spsc\n"); ODP_PRINT(" length %" PRIu32 "/%" PRIu32 "\n", - ring_spsc_length(&queue->s.ring_spsc), queue->s.ring_mask + 1); - } else if (queue->s.type == ODP_QUEUE_TYPE_SCHED) { + ring_spsc_length(&queue->ring_spsc), queue->ring_mask + 1); + } else if (queue->type == ODP_QUEUE_TYPE_SCHED) { ODP_PRINT(" implementation ring_st\n"); ODP_PRINT(" length %" PRIu32 "/%" PRIu32 "\n", - ring_st_length(&queue->s.ring_st), queue->s.ring_mask + 1); + ring_st_length(&queue->ring_st), queue->ring_mask + 1); } else { ODP_PRINT(" implementation ring_mpmc\n"); ODP_PRINT(" length %" PRIu32 "/%" PRIu32 "\n", - ring_mpmc_length(&queue->s.ring_mpmc), queue->s.ring_mask + 1); + ring_mpmc_length(&queue->ring_mpmc), queue->ring_mask + 1); } ODP_PRINT("\n"); @@ -795,37 +795,37 @@ static void queue_print_all(void) for (i = 0; i < CONFIG_MAX_QUEUES; i++) { queue_entry_t *queue = qentry_from_index(i); - if (queue->s.status < QUEUE_STATUS_READY) + if (queue->status < QUEUE_STATUS_READY) continue; LOCK(queue); - status = queue->s.status; - index = queue->s.index; - name = queue->s.name; - type = queue->s.type; - blocking = queue->s.param.nonblocking; - enq_mode = queue->s.param.enq_mode; - deq_mode = queue->s.param.deq_mode; - order = queue->s.param.order; - - if (queue->s.queue_lf) { - len = _odp_queue_lf_length(queue->s.queue_lf); + status = queue->status; + index = queue->index; + name = queue->name; + type = queue->type; + blocking = queue->param.nonblocking; + enq_mode = queue->param.enq_mode; + deq_mode = queue->param.deq_mode; + order = queue->param.order; + + if (queue->queue_lf) { + len = _odp_queue_lf_length(queue->queue_lf); max_len = _odp_queue_lf_max_length(); - } else if (queue->s.spsc) { - len = ring_spsc_length(&queue->s.ring_spsc); - max_len = queue->s.ring_mask + 1; + } else if (queue->spsc) { + len = ring_spsc_length(&queue->ring_spsc); + max_len = queue->ring_mask + 1; } else if (type == ODP_QUEUE_TYPE_SCHED) { - len = ring_st_length(&queue->s.ring_st); - max_len = queue->s.ring_mask + 1; - prio = queue->s.param.sched.prio; - grp = queue->s.param.sched.group; - sync = queue->s.param.sched.sync; + len = ring_st_length(&queue->ring_st); + max_len = queue->ring_mask + 1; + prio = queue->param.sched.prio; + grp = queue->param.sched.group; + sync = queue->param.sched.sync; if (_odp_sched_id == _ODP_SCHED_ID_BASIC) spr = _odp_sched_basic_get_spread(index); } else { - len = ring_mpmc_length(&queue->s.ring_mpmc); - max_len = queue->s.ring_mask + 1; + len = ring_mpmc_length(&queue->ring_mpmc); + max_len = queue->ring_mask + 1; } UNLOCK(queue); @@ -882,7 +882,7 @@ static inline int _sched_queue_enq_multi(odp_queue_t handle, uint32_t event_idx[num]; queue = qentry_from_handle(handle); - ring_st = &queue->s.ring_st; + ring_st = &queue->ring_st; if (_odp_sched_fn->ord_enq_multi(handle, (void **)event_hdr, num, &ret)) return ret; @@ -891,23 +891,23 @@ static inline int _sched_queue_enq_multi(odp_queue_t handle, LOCK(queue); - num_enq = ring_st_enq_multi(ring_st, queue->s.ring_data, - queue->s.ring_mask, event_idx, num); + num_enq = ring_st_enq_multi(ring_st, queue->ring_data, + queue->ring_mask, event_idx, num); if (odp_unlikely(num_enq == 0)) { UNLOCK(queue); return 0; } - if (queue->s.status == QUEUE_STATUS_NOTSCHED) { - queue->s.status = QUEUE_STATUS_SCHED; + if (queue->status == QUEUE_STATUS_NOTSCHED) { + queue->status = QUEUE_STATUS_SCHED; sched = 1; } UNLOCK(queue); /* Add queue to scheduling */ - if (sched && _odp_sched_fn->sched_queue(queue->s.index)) + if (sched && _odp_sched_fn->sched_queue(queue->index)) ODP_ABORT("schedule_queue failed\n"); return num_enq; @@ -921,17 +921,17 @@ int _odp_sched_queue_deq(uint32_t queue_index, odp_event_t ev[], int max_num, queue_entry_t *queue = qentry_from_index(queue_index); uint32_t event_idx[max_num]; - ring_st = &queue->s.ring_st; + ring_st = &queue->ring_st; LOCK(queue); - status = queue->s.status; + status = queue->status; if (odp_unlikely(status < QUEUE_STATUS_READY)) { /* Bad queue, or queue has been destroyed. * Inform scheduler about a destroyed queue. */ - if (queue->s.status == QUEUE_STATUS_DESTROYED) { - queue->s.status = QUEUE_STATUS_FREE; + if (queue->status == QUEUE_STATUS_DESTROYED) { + queue->status = QUEUE_STATUS_FREE; _odp_sched_fn->destroy_queue(queue_index); } @@ -939,13 +939,13 @@ int _odp_sched_queue_deq(uint32_t queue_index, odp_event_t ev[], int max_num, return -1; } - num_deq = ring_st_deq_multi(ring_st, queue->s.ring_data, - queue->s.ring_mask, event_idx, max_num); + num_deq = ring_st_deq_multi(ring_st, queue->ring_data, + queue->ring_mask, event_idx, max_num); if (num_deq == 0) { /* Already empty queue */ if (update_status && status == QUEUE_STATUS_SCHED) - queue->s.status = QUEUE_STATUS_NOTSCHED; + queue->status = QUEUE_STATUS_NOTSCHED; UNLOCK(queue); @@ -984,16 +984,16 @@ int _odp_sched_queue_empty(uint32_t queue_index) LOCK(queue); - if (odp_unlikely(queue->s.status < QUEUE_STATUS_READY)) { + if (odp_unlikely(queue->status < QUEUE_STATUS_READY)) { /* Bad queue, or queue has been destroyed. */ UNLOCK(queue); return -1; } - if (ring_st_is_empty(&queue->s.ring_st)) { + if (ring_st_is_empty(&queue->ring_st)) { /* Already empty queue. Update status. */ - if (queue->s.status == QUEUE_STATUS_SCHED) - queue->s.status = QUEUE_STATUS_NOTSCHED; + if (queue->status == QUEUE_STATUS_SCHED) + queue->status = QUEUE_STATUS_NOTSCHED; ret = 1; } @@ -1014,23 +1014,23 @@ static int queue_init(queue_entry_t *queue, const char *name, queue_type = param->type; if (name == NULL) { - queue->s.name[0] = 0; + queue->name[0] = 0; } else { - strncpy(queue->s.name, name, ODP_QUEUE_NAME_LEN - 1); - queue->s.name[ODP_QUEUE_NAME_LEN - 1] = 0; + strncpy(queue->name, name, ODP_QUEUE_NAME_LEN - 1); + queue->name[ODP_QUEUE_NAME_LEN - 1] = 0; } - memcpy(&queue->s.param, param, sizeof(odp_queue_param_t)); - if (queue->s.param.sched.lock_count > _odp_sched_fn->max_ordered_locks()) + memcpy(&queue->param, param, sizeof(odp_queue_param_t)); + if (queue->param.sched.lock_count > _odp_sched_fn->max_ordered_locks()) return -1; if (queue_type == ODP_QUEUE_TYPE_SCHED) - queue->s.param.deq_mode = ODP_QUEUE_OP_DISABLED; + queue->param.deq_mode = ODP_QUEUE_OP_DISABLED; - queue->s.type = queue_type; - odp_atomic_init_u64(&queue->s.num_timers, 0); + queue->type = queue_type; + odp_atomic_init_u64(&queue->num_timers, 0); - queue->s.pktin = PKTIN_INVALID; - queue->s.pktout = PKTOUT_INVALID; + queue->pktin = PKTIN_INVALID; + queue->pktout = PKTOUT_INVALID; queue_size = param->size; if (queue_size == 0) @@ -1047,7 +1047,7 @@ static int queue_init(queue_entry_t *queue, const char *name, return -1; } - offset = queue->s.index * (uint64_t)_odp_queue_glb->config.max_queue_size; + offset = queue->index * (uint64_t)_odp_queue_glb->config.max_queue_size; /* Single-producer / single-consumer plain queue has simple and * lock-free implementation */ @@ -1055,37 +1055,37 @@ static int queue_init(queue_entry_t *queue, const char *name, (param->enq_mode == ODP_QUEUE_OP_MT_UNSAFE) && (param->deq_mode == ODP_QUEUE_OP_MT_UNSAFE); - queue->s.spsc = spsc; - queue->s.queue_lf = NULL; + queue->spsc = spsc; + queue->queue_lf = NULL; /* Default to error functions */ - queue->s.enqueue = error_enqueue; - queue->s.enqueue_multi = error_enqueue_multi; - queue->s.dequeue = error_dequeue; - queue->s.dequeue_multi = error_dequeue_multi; - queue->s.orig_dequeue_multi = error_dequeue_multi; + queue->enqueue = error_enqueue; + queue->enqueue_multi = error_enqueue_multi; + queue->dequeue = error_dequeue; + queue->dequeue_multi = error_dequeue_multi; + queue->orig_dequeue_multi = error_dequeue_multi; if (spsc) { _odp_queue_spsc_init(queue, queue_size); } else { if (queue_type == ODP_QUEUE_TYPE_PLAIN) { - queue->s.enqueue = plain_queue_enq; - queue->s.enqueue_multi = plain_queue_enq_multi; - queue->s.dequeue = plain_queue_deq; - queue->s.dequeue_multi = plain_queue_deq_multi; - queue->s.orig_dequeue_multi = plain_queue_deq_multi; + queue->enqueue = plain_queue_enq; + queue->enqueue_multi = plain_queue_enq_multi; + queue->dequeue = plain_queue_deq; + queue->dequeue_multi = plain_queue_deq_multi; + queue->orig_dequeue_multi = plain_queue_deq_multi; - queue->s.ring_data = &_odp_queue_glb->ring_data[offset]; - queue->s.ring_mask = queue_size - 1; - ring_mpmc_init(&queue->s.ring_mpmc); + queue->ring_data = &_odp_queue_glb->ring_data[offset]; + queue->ring_mask = queue_size - 1; + ring_mpmc_init(&queue->ring_mpmc); } else { - queue->s.enqueue = sched_queue_enq; - queue->s.enqueue_multi = sched_queue_enq_multi; + queue->enqueue = sched_queue_enq; + queue->enqueue_multi = sched_queue_enq_multi; - queue->s.ring_data = &_odp_queue_glb->ring_data[offset]; - queue->s.ring_mask = queue_size - 1; - ring_st_init(&queue->s.ring_st); + queue->ring_data = &_odp_queue_glb->ring_data[offset]; + queue->ring_mask = queue_size - 1; + ring_st_init(&queue->ring_st); } } @@ -1101,30 +1101,30 @@ static odp_pktout_queue_t queue_get_pktout(odp_queue_t handle) { queue_entry_t *qentry = qentry_from_handle(handle); - return qentry->s.pktout; + return qentry->pktout; } static void queue_set_pktout(odp_queue_t handle, odp_pktio_t pktio, int index) { queue_entry_t *qentry = qentry_from_handle(handle); - qentry->s.pktout.pktio = pktio; - qentry->s.pktout.index = index; + qentry->pktout.pktio = pktio; + qentry->pktout.index = index; } static odp_pktin_queue_t queue_get_pktin(odp_queue_t handle) { queue_entry_t *qentry = qentry_from_handle(handle); - return qentry->s.pktin; + return qentry->pktin; } static void queue_set_pktin(odp_queue_t handle, odp_pktio_t pktio, int index) { queue_entry_t *qentry = qentry_from_handle(handle); - qentry->s.pktin.pktio = pktio; - qentry->s.pktin.index = index; + qentry->pktin.pktio = pktio; + qentry->pktin.index = index; } static void queue_set_enq_deq_func(odp_queue_t handle, @@ -1136,16 +1136,16 @@ static void queue_set_enq_deq_func(odp_queue_t handle, queue_entry_t *qentry = qentry_from_handle(handle); if (enq) - qentry->s.enqueue = enq; + qentry->enqueue = enq; if (enq_multi) - qentry->s.enqueue_multi = enq_multi; + qentry->enqueue_multi = enq_multi; if (deq) - qentry->s.dequeue = deq; + qentry->dequeue = deq; if (deq_multi) - qentry->s.dequeue_multi = deq_multi; + qentry->dequeue_multi = deq_multi; } static int queue_orig_multi(odp_queue_t handle, @@ -1153,7 +1153,7 @@ static int queue_orig_multi(odp_queue_t handle, { queue_entry_t *queue = qentry_from_handle(handle); - return queue->s.orig_dequeue_multi(handle, event_hdr, num); + return queue->orig_dequeue_multi(handle, event_hdr, num); } static int queue_api_enq_multi(odp_queue_t handle, @@ -1167,7 +1167,7 @@ static int queue_api_enq_multi(odp_queue_t handle, if (num > QUEUE_MULTI_MAX) num = QUEUE_MULTI_MAX; - return queue->s.enqueue_multi(handle, + return queue->enqueue_multi(handle, (_odp_event_hdr_t **)(uintptr_t)ev, num); } @@ -1175,21 +1175,21 @@ static void queue_timer_add(odp_queue_t handle) { queue_entry_t *queue = qentry_from_handle(handle); - odp_atomic_inc_u64(&queue->s.num_timers); + odp_atomic_inc_u64(&queue->num_timers); } static void queue_timer_rem(odp_queue_t handle) { queue_entry_t *queue = qentry_from_handle(handle); - odp_atomic_dec_u64(&queue->s.num_timers); + odp_atomic_dec_u64(&queue->num_timers); } static int queue_api_enq(odp_queue_t handle, odp_event_t ev) { queue_entry_t *queue = qentry_from_handle(handle); - return queue->s.enqueue(handle, + return queue->enqueue(handle, (_odp_event_hdr_t *)(uintptr_t)ev); } @@ -1201,10 +1201,10 @@ static int queue_api_deq_multi(odp_queue_t handle, odp_event_t ev[], int num) if (num > QUEUE_MULTI_MAX) num = QUEUE_MULTI_MAX; - ret = queue->s.dequeue_multi(handle, (_odp_event_hdr_t **)ev, num); + ret = queue->dequeue_multi(handle, (_odp_event_hdr_t **)ev, num); if (odp_global_rw->inline_timers && - odp_atomic_load_u64(&queue->s.num_timers)) + odp_atomic_load_u64(&queue->num_timers)) timer_run(ret ? 2 : 1); return ret; @@ -1213,10 +1213,10 @@ static int queue_api_deq_multi(odp_queue_t handle, odp_event_t ev[], int num) static odp_event_t queue_api_deq(odp_queue_t handle) { queue_entry_t *queue = qentry_from_handle(handle); - odp_event_t ev = (odp_event_t)queue->s.dequeue(handle); + odp_event_t ev = (odp_event_t)queue->dequeue(handle); if (odp_global_rw->inline_timers && - odp_atomic_load_u64(&queue->s.num_timers)) + odp_atomic_load_u64(&queue->num_timers)) timer_run(ev != ODP_EVENT_INVALID ? 2 : 1); return ev; diff --git a/platform/linux-generic/odp_queue_lf.c b/platform/linux-generic/odp_queue_lf.c index 58959df31..f511bcec5 100644 --- a/platform/linux-generic/odp_queue_lf.c +++ b/platform/linux-generic/odp_queue_lf.c @@ -125,7 +125,7 @@ static int queue_lf_enq(odp_queue_t handle, _odp_event_hdr_t *event_hdr) ring_lf_node_t *node; queue = qentry_from_handle(handle); - queue_lf = queue->s.queue_lf; + queue_lf = queue->queue_lf; new_val.s.ptr = (uintptr_t)event_hdr; new_val.s.counter = odp_atomic_fetch_inc_u64(&queue_lf->enq_counter); @@ -184,7 +184,7 @@ static _odp_event_hdr_t *queue_lf_deq(odp_queue_t handle) _odp_event_hdr_t *event_hdr; queue = qentry_from_handle(handle); - queue_lf = queue->s.queue_lf; + queue_lf = queue->queue_lf; new_val.s.counter = 0; new_val.s.ptr = 0; old = NULL; @@ -325,7 +325,7 @@ void *_odp_queue_lf_create(queue_entry_t *queue) return NULL; } - if (queue->s.type != ODP_QUEUE_TYPE_PLAIN) + if (queue->type != ODP_QUEUE_TYPE_PLAIN) return NULL; for (i = 0; i < QUEUE_LF_NUM; i++) { diff --git a/platform/linux-generic/odp_queue_scalable.c b/platform/linux-generic/odp_queue_scalable.c index 26203fa5a..049e7dae8 100644 --- a/platform/linux-generic/odp_queue_scalable.c +++ b/platform/linux-generic/odp_queue_scalable.c @@ -102,12 +102,12 @@ static int queue_init(queue_entry_t *queue, const char *name, _odp_event_hdr_t **ring; uint32_t size; - sched_elem = &queue->s.sched_elem; + sched_elem = &queue->sched_elem; ring_size = param->size > 0 ? _ODP_ROUNDUP_POWER2_U32(param->size) : CONFIG_SCAL_QUEUE_SIZE; - strncpy(queue->s.name, name ? name : "", ODP_QUEUE_NAME_LEN - 1); - queue->s.name[ODP_QUEUE_NAME_LEN - 1] = 0; - memcpy(&queue->s.param, param, sizeof(odp_queue_param_t)); + strncpy(queue->name, name ? name : "", ODP_QUEUE_NAME_LEN - 1); + queue->name[ODP_QUEUE_NAME_LEN - 1] = 0; + memcpy(&queue->param, param, sizeof(odp_queue_param_t)); size = ring_size * sizeof(_odp_event_hdr_t *); ring = (_odp_event_hdr_t **)shm_pool_alloc_align(queue_shm_pool, size); @@ -117,19 +117,19 @@ static int queue_init(queue_entry_t *queue, const char *name, for (ring_idx = 0; ring_idx < ring_size; ring_idx++) ring[ring_idx] = NULL; - queue->s.type = queue->s.param.type; + queue->type = queue->param.type; - if (queue->s.type == ODP_QUEUE_TYPE_SCHED) - queue->s.param.deq_mode = ODP_QUEUE_OP_DISABLED; + if (queue->type == ODP_QUEUE_TYPE_SCHED) + queue->param.deq_mode = ODP_QUEUE_OP_DISABLED; - odp_atomic_init_u64(&queue->s.num_timers, 0); + odp_atomic_init_u64(&queue->num_timers, 0); - queue->s.enqueue = _queue_enq; - queue->s.dequeue = _queue_deq; - queue->s.enqueue_multi = _queue_enq_multi; - queue->s.dequeue_multi = _queue_deq_multi; - queue->s.orig_dequeue_multi = _queue_deq_multi; - queue->s.pktin = PKTIN_INVALID; + queue->enqueue = _queue_enq; + queue->dequeue = _queue_deq; + queue->enqueue_multi = _queue_enq_multi; + queue->dequeue_multi = _queue_deq_multi; + queue->orig_dequeue_multi = _queue_deq_multi; + queue->pktin = PKTIN_INVALID; sched_elem->node.next = NULL; #ifdef CONFIG_QSCHST_LOCK @@ -140,8 +140,8 @@ static int queue_init(queue_entry_t *queue, const char *name, sched_elem->qschst.cur_ticket = 0; sched_elem->qschst.nxt_ticket = 0; sched_elem->pop_deficit = 0; - if (queue->s.type == ODP_QUEUE_TYPE_SCHED) - sched_elem->qschst_type = queue->s.param.sched.sync; + if (queue->type == ODP_QUEUE_TYPE_SCHED) + sched_elem->qschst_type = queue->param.sched.sync; else sched_elem->qschst_type = ODP_NO_SCHED_QUEUE; /* 2nd cache line - enqueue */ @@ -154,7 +154,7 @@ static int queue_init(queue_entry_t *queue, const char *name, sched_elem->cons_write = 0; sched_elem->rwin = NULL; sched_elem->schedq = NULL; - sched_elem->user_ctx = queue->s.param.context; + sched_elem->user_ctx = queue->param.context; #ifdef CONFIG_SPLIT_PRODCONS sched_elem->cons_ring = ring; sched_elem->cons_mask = ring_size - 1; @@ -162,13 +162,13 @@ static int queue_init(queue_entry_t *queue, const char *name, #endif /* Queue initialized successfully, add it to the sched group */ - if (queue->s.type == ODP_QUEUE_TYPE_SCHED) { + if (queue->type == ODP_QUEUE_TYPE_SCHED) { int prio = odp_schedule_max_prio() - param->sched.prio; - if (queue->s.param.sched.sync == ODP_SCHED_SYNC_ORDERED) { + if (queue->param.sched.sync == ODP_SCHED_SYNC_ORDERED) { sched_elem->rwin = _odp_rwin_alloc(queue_shm_pool, - queue->s.param.sched.lock_count); + queue->param.sched.lock_count); if (sched_elem->rwin == NULL) { ODP_ERR("Reorder window not created\n"); goto rwin_create_failed; @@ -202,7 +202,7 @@ static int queue_init_global(void) memset(&_odp_queue_inline_offset, 0, sizeof(_odp_queue_inline_offset_t)); _odp_queue_inline_offset.context = offsetof(queue_entry_t, - s.param.context); + param.context); /* Create shared memory pool to allocate shared memory for the * queues. Use the default queue size. @@ -242,9 +242,9 @@ static int queue_init_global(void) queue_entry_t *queue; queue = get_qentry(i); - LOCK_INIT(&queue->s.lock); - queue->s.index = i; - queue->s.handle = (odp_queue_t)queue; + LOCK_INIT(&queue->lock); + queue->index = i; + queue->handle = (odp_queue_t)queue; } ODP_DBG("done\n"); @@ -274,9 +274,9 @@ static int queue_term_global(void) for (i = 0; i < CONFIG_MAX_QUEUES; i++) { queue = &queue_tbl->queue[i]; - if (__atomic_load_n(&queue->s.status, + if (__atomic_load_n(&queue->status, __ATOMIC_RELAXED) != QUEUE_STATUS_FREE) { - ODP_ERR("Not destroyed queue: %s\n", queue->s.name); + ODP_ERR("Not destroyed queue: %s\n", queue->name); rc = -1; } } @@ -317,30 +317,30 @@ static int queue_capability(odp_queue_capability_t *capa) static odp_queue_type_t queue_type(odp_queue_t handle) { - return _odp_qentry_from_ext(handle)->s.type; + return _odp_qentry_from_ext(handle)->type; } static odp_schedule_sync_t queue_sched_type(odp_queue_t handle) { - return _odp_qentry_from_ext(handle)->s.param.sched.sync; + return _odp_qentry_from_ext(handle)->param.sched.sync; } static odp_schedule_prio_t queue_sched_prio(odp_queue_t handle) { - return _odp_qentry_from_ext(handle)->s.param.sched.prio; + return _odp_qentry_from_ext(handle)->param.sched.prio; } static odp_schedule_group_t queue_sched_group(odp_queue_t handle) { - return _odp_qentry_from_ext(handle)->s.param.sched.group; + return _odp_qentry_from_ext(handle)->param.sched.group; } static uint32_t queue_lock_count(odp_queue_t handle) { queue_entry_t *queue = _odp_qentry_from_ext(handle); - return queue->s.param.sched.sync == ODP_SCHED_SYNC_ORDERED ? - queue->s.param.sched.lock_count : 0; + return queue->param.sched.sync == ODP_SCHED_SYNC_ORDERED ? + queue->param.sched.lock_count : 0; } static odp_queue_t queue_create(const char *name, @@ -382,21 +382,21 @@ static odp_queue_t queue_create(const char *name, for (; queue_idx < max_idx; queue_idx++) { queue = &queue_tbl->queue[queue_idx]; - if (queue->s.status != QUEUE_STATUS_FREE) + if (queue->status != QUEUE_STATUS_FREE) continue; - LOCK(&queue->s.lock); - if (queue->s.status == QUEUE_STATUS_FREE) { + LOCK(&queue->lock); + if (queue->status == QUEUE_STATUS_FREE) { if (queue_init(queue, name, param)) { - UNLOCK(&queue->s.lock); + UNLOCK(&queue->lock); return handle; } - queue->s.status = QUEUE_STATUS_READY; - handle = queue->s.handle; - UNLOCK(&queue->s.lock); + queue->status = QUEUE_STATUS_READY; + handle = queue->handle; + UNLOCK(&queue->lock); break; } - UNLOCK(&queue->s.lock); + UNLOCK(&queue->lock); } return handle; } @@ -410,12 +410,12 @@ static int queue_destroy(odp_queue_t handle) return -1; queue = _odp_qentry_from_ext(handle); - LOCK(&queue->s.lock); - if (queue->s.status != QUEUE_STATUS_READY) { - UNLOCK(&queue->s.lock); + LOCK(&queue->lock); + if (queue->status != QUEUE_STATUS_READY) { + UNLOCK(&queue->lock); return -1; } - q = &queue->s.sched_elem; + q = &queue->sched_elem; #ifdef CONFIG_QSCHST_LOCK LOCK(&q->qschlock); @@ -425,7 +425,7 @@ static int queue_destroy(odp_queue_t handle) #ifdef CONFIG_QSCHST_LOCK UNLOCK(&q->qschlock); #endif - UNLOCK(&queue->s.lock); + UNLOCK(&queue->lock); return -1; } /* Enqueue is now disabled */ @@ -440,7 +440,7 @@ static int queue_destroy(odp_queue_t handle) #ifdef CONFIG_QSCHST_LOCK UNLOCK(&q->qschlock); #endif - UNLOCK(&queue->s.lock); + UNLOCK(&queue->lock); return -1; } #ifdef CONFIG_QSCHST_LOCK @@ -468,13 +468,13 @@ static int queue_destroy(odp_queue_t handle) if (q->rwin != NULL) { if (_odp_rwin_free(queue_shm_pool, q->rwin) < 0) { ODP_ERR("Failed to free reorder window\n"); - UNLOCK(&queue->s.lock); + UNLOCK(&queue->lock); return -1; } q->rwin = NULL; } - queue->s.status = QUEUE_STATUS_FREE; - UNLOCK(&queue->s.lock); + queue->status = QUEUE_STATUS_FREE; + UNLOCK(&queue->lock); return 0; } @@ -482,7 +482,7 @@ static int queue_context_set(odp_queue_t handle, void *context, uint32_t len ODP_UNUSED) { odp_mb_full(); - _odp_qentry_from_ext(handle)->s.param.context = context; + _odp_qentry_from_ext(handle)->param.context = context; odp_mb_full(); return 0; } @@ -494,17 +494,17 @@ static odp_queue_t queue_lookup(const char *name) for (i = 0; i < CONFIG_MAX_QUEUES; i++) { queue_entry_t *queue = &queue_tbl->queue[i]; - if (queue->s.status == QUEUE_STATUS_FREE || - queue->s.status == QUEUE_STATUS_DESTROYED) + if (queue->status == QUEUE_STATUS_FREE || + queue->status == QUEUE_STATUS_DESTROYED) continue; - LOCK(&queue->s.lock); - if (strcmp(name, queue->s.name) == 0) { + LOCK(&queue->lock); + if (strcmp(name, queue->name) == 0) { /* found it */ - UNLOCK(&queue->s.lock); - return queue->s.handle; + UNLOCK(&queue->lock); + return queue->handle; } - UNLOCK(&queue->s.lock); + UNLOCK(&queue->lock); } return ODP_QUEUE_INVALID; @@ -634,29 +634,29 @@ static int _queue_enq_multi(odp_queue_t handle, _odp_event_hdr_t *event_hdr[], queue = qentry_from_int(handle); ts = _odp_sched_ts; if (ts && odp_unlikely(ts->out_of_order) && - (queue->s.param.order == ODP_QUEUE_ORDER_KEEP)) { + (queue->param.order == ODP_QUEUE_ORDER_KEEP)) { actual = _odp_rctx_save(queue, event_hdr, num); return actual; } #ifdef CONFIG_QSCHST_LOCK - LOCK(&queue->s.sched_elem.qschlock); - actual = _odp_queue_enq_sp(&queue->s.sched_elem, event_hdr, num); + LOCK(&queue->sched_elem.qschlock); + actual = _odp_queue_enq_sp(&queue->sched_elem, event_hdr, num); #else - actual = _odp_queue_enq(&queue->s.sched_elem, event_hdr, num); + actual = _odp_queue_enq(&queue->sched_elem, event_hdr, num); #endif - if (odp_likely(queue->s.sched_elem.schedq != NULL && actual != 0)) { + if (odp_likely(queue->sched_elem.schedq != NULL && actual != 0)) { /* Perform scheduler related updates. */ #ifdef CONFIG_QSCHST_LOCK - _odp_sched_update_enq_sp(&queue->s.sched_elem, actual); + _odp_sched_update_enq_sp(&queue->sched_elem, actual); #else - _odp_sched_update_enq(&queue->s.sched_elem, actual); + _odp_sched_update_enq(&queue->sched_elem, actual); #endif } #ifdef CONFIG_QSCHST_LOCK - UNLOCK(&queue->s.sched_elem.qschlock); + UNLOCK(&queue->sched_elem.qschlock); #endif return actual; } @@ -680,7 +680,7 @@ static int queue_enq_multi(odp_queue_t handle, const odp_event_t ev[], int num) for (i = 0; i < num; i++) event_hdr[i] = _odp_event_hdr(ev[i]); - return queue->s.enqueue_multi(handle, event_hdr, num); + return queue->enqueue_multi(handle, event_hdr, num); } static int queue_enq(odp_queue_t handle, odp_event_t ev) @@ -691,7 +691,7 @@ static int queue_enq(odp_queue_t handle, odp_event_t ev) queue = _odp_qentry_from_ext(handle); event_hdr = _odp_event_hdr(ev); - return queue->s.enqueue(handle, event_hdr); + return queue->enqueue(handle, event_hdr); } /* Single-consumer dequeue. */ @@ -828,7 +828,7 @@ static int _queue_deq_multi(odp_queue_t handle, _odp_event_hdr_t *event_hdr[], queue_entry_t *queue; queue = qentry_from_int(handle); - q = &queue->s.sched_elem; + q = &queue->sched_elem; return _odp_queue_deq(q, event_hdr, num); } @@ -839,7 +839,7 @@ static _odp_event_hdr_t *_queue_deq(odp_queue_t handle) queue_entry_t *queue; queue = qentry_from_int(handle); - q = &queue->s.sched_elem; + q = &queue->sched_elem; if (_odp_queue_deq(q, &event_hdr, 1) == 1) return event_hdr; else @@ -856,10 +856,10 @@ static int queue_deq_multi(odp_queue_t handle, odp_event_t ev[], int num) queue = _odp_qentry_from_ext(handle); - ret = queue->s.dequeue_multi(handle, (_odp_event_hdr_t **)ev, num); + ret = queue->dequeue_multi(handle, (_odp_event_hdr_t **)ev, num); if (odp_global_rw->inline_timers && - odp_atomic_load_u64(&queue->s.num_timers)) + odp_atomic_load_u64(&queue->num_timers)) timer_run(ret ? 2 : 1); return ret; @@ -868,10 +868,10 @@ static int queue_deq_multi(odp_queue_t handle, odp_event_t ev[], int num) static odp_event_t queue_deq(odp_queue_t handle) { queue_entry_t *queue = _odp_qentry_from_ext(handle); - odp_event_t ev = (odp_event_t)queue->s.dequeue(handle); + odp_event_t ev = (odp_event_t)queue->dequeue(handle); if (odp_global_rw->inline_timers && - odp_atomic_load_u64(&queue->s.num_timers)) + odp_atomic_load_u64(&queue->num_timers)) timer_run(ev != ODP_EVENT_INVALID ? 2 : 1); return ev; @@ -911,20 +911,20 @@ static int queue_info(odp_queue_t handle, odp_queue_info_t *info) queue = get_qentry(queue_id); - LOCK(&queue->s.lock); - status = queue->s.status; + LOCK(&queue->lock); + status = queue->status; if (odp_unlikely(status == QUEUE_STATUS_FREE || status == QUEUE_STATUS_DESTROYED)) { - UNLOCK(&queue->s.lock); + UNLOCK(&queue->lock); ODP_ERR("Invalid queue status:%d\n", status); return -1; } - info->name = queue->s.name; - info->param = queue->s.param; + info->name = queue->name; + info->param = queue->param; - UNLOCK(&queue->s.lock); + UNLOCK(&queue->lock); return 0; } @@ -946,58 +946,58 @@ static void queue_print(odp_queue_t handle) queue = get_qentry(queue_id); - LOCK(&queue->s.lock); - status = queue->s.status; + LOCK(&queue->lock); + status = queue->status; if (odp_unlikely(status == QUEUE_STATUS_FREE || status == QUEUE_STATUS_DESTROYED)) { - UNLOCK(&queue->s.lock); + UNLOCK(&queue->lock); ODP_ERR("Invalid queue status:%d\n", status); return; } ODP_PRINT("\nQueue info\n"); ODP_PRINT("----------\n"); - ODP_PRINT(" handle %p\n", (void *)queue->s.handle); - ODP_PRINT(" index %" PRIu32 "\n", queue->s.index); - ODP_PRINT(" name %s\n", queue->s.name); + ODP_PRINT(" handle %p\n", (void *)queue->handle); + ODP_PRINT(" index %" PRIu32 "\n", queue->index); + ODP_PRINT(" name %s\n", queue->name); ODP_PRINT(" enq mode %s\n", - queue->s.param.enq_mode == ODP_QUEUE_OP_MT ? "ODP_QUEUE_OP_MT" : - (queue->s.param.enq_mode == ODP_QUEUE_OP_MT_UNSAFE ? "ODP_QUEUE_OP_MT_UNSAFE" : - (queue->s.param.enq_mode == ODP_QUEUE_OP_DISABLED ? "ODP_QUEUE_OP_DISABLED" : + queue->param.enq_mode == ODP_QUEUE_OP_MT ? "ODP_QUEUE_OP_MT" : + (queue->param.enq_mode == ODP_QUEUE_OP_MT_UNSAFE ? "ODP_QUEUE_OP_MT_UNSAFE" : + (queue->param.enq_mode == ODP_QUEUE_OP_DISABLED ? "ODP_QUEUE_OP_DISABLED" : "unknown"))); ODP_PRINT(" deq mode %s\n", - queue->s.param.deq_mode == ODP_QUEUE_OP_MT ? "ODP_QUEUE_OP_MT" : - (queue->s.param.deq_mode == ODP_QUEUE_OP_MT_UNSAFE ? "ODP_QUEUE_OP_MT_UNSAFE" : - (queue->s.param.deq_mode == ODP_QUEUE_OP_DISABLED ? "ODP_QUEUE_OP_DISABLED" : + queue->param.deq_mode == ODP_QUEUE_OP_MT ? "ODP_QUEUE_OP_MT" : + (queue->param.deq_mode == ODP_QUEUE_OP_MT_UNSAFE ? "ODP_QUEUE_OP_MT_UNSAFE" : + (queue->param.deq_mode == ODP_QUEUE_OP_DISABLED ? "ODP_QUEUE_OP_DISABLED" : "unknown"))); ODP_PRINT(" type %s\n", - queue->s.type == ODP_QUEUE_TYPE_PLAIN ? "ODP_QUEUE_TYPE_PLAIN" : - (queue->s.type == ODP_QUEUE_TYPE_SCHED ? "ODP_QUEUE_TYPE_SCHED" : "unknown")); - if (queue->s.type == ODP_QUEUE_TYPE_SCHED) { + queue->type == ODP_QUEUE_TYPE_PLAIN ? "ODP_QUEUE_TYPE_PLAIN" : + (queue->type == ODP_QUEUE_TYPE_SCHED ? "ODP_QUEUE_TYPE_SCHED" : "unknown")); + if (queue->type == ODP_QUEUE_TYPE_SCHED) { ODP_PRINT(" sync %s\n", - queue->s.param.sched.sync == ODP_SCHED_SYNC_PARALLEL ? + queue->param.sched.sync == ODP_SCHED_SYNC_PARALLEL ? "ODP_SCHED_SYNC_PARALLEL" : - (queue->s.param.sched.sync == ODP_SCHED_SYNC_ATOMIC ? + (queue->param.sched.sync == ODP_SCHED_SYNC_ATOMIC ? "ODP_SCHED_SYNC_ATOMIC" : - (queue->s.param.sched.sync == ODP_SCHED_SYNC_ORDERED ? + (queue->param.sched.sync == ODP_SCHED_SYNC_ORDERED ? "ODP_SCHED_SYNC_ORDERED" : "unknown"))); - ODP_PRINT(" priority %d\n", queue->s.param.sched.prio); - ODP_PRINT(" group %d\n", queue->s.param.sched.group); + ODP_PRINT(" priority %d\n", queue->param.sched.prio); + ODP_PRINT(" group %d\n", queue->param.sched.group); } - if (queue->s.pktin.pktio != ODP_PKTIO_INVALID) { - if (!odp_pktio_info(queue->s.pktin.pktio, &pktio_info)) + if (queue->pktin.pktio != ODP_PKTIO_INVALID) { + if (!odp_pktio_info(queue->pktin.pktio, &pktio_info)) ODP_PRINT(" pktin %s\n", pktio_info.name); } - if (queue->s.pktout.pktio != ODP_PKTIO_INVALID) { - if (!odp_pktio_info(queue->s.pktout.pktio, &pktio_info)) + if (queue->pktout.pktio != ODP_PKTIO_INVALID) { + if (!odp_pktio_info(queue->pktout.pktio, &pktio_info)) ODP_PRINT(" pktout %s\n", pktio_info.name); } ODP_PRINT(" timers %" PRIu64 "\n", - odp_atomic_load_u64(&queue->s.num_timers)); - ODP_PRINT(" param.size %" PRIu32 "\n", queue->s.param.size); + odp_atomic_load_u64(&queue->num_timers)); + ODP_PRINT(" param.size %" PRIu32 "\n", queue->param.size); ODP_PRINT("\n"); - UNLOCK(&queue->s.lock); + UNLOCK(&queue->lock); } static void queue_print_all(void) @@ -1023,23 +1023,23 @@ static void queue_print_all(void) for (i = 0; i < CONFIG_MAX_QUEUES; i++) { queue_entry_t *queue = &queue_tbl->queue[i]; - if (queue->s.status != QUEUE_STATUS_READY) + if (queue->status != QUEUE_STATUS_READY) continue; - LOCK(&queue->s.lock); + LOCK(&queue->lock); - status = queue->s.status; - index = queue->s.index; - name = queue->s.name; - type = queue->s.type; - blocking = queue->s.param.nonblocking; - enq_mode = queue->s.param.enq_mode; - deq_mode = queue->s.param.deq_mode; - order = queue->s.param.order; - prio = queue->s.param.sched.prio; - sync = queue->s.param.sched.sync; + status = queue->status; + index = queue->index; + name = queue->name; + type = queue->type; + blocking = queue->param.nonblocking; + enq_mode = queue->param.enq_mode; + deq_mode = queue->param.deq_mode; + order = queue->param.order; + prio = queue->param.sched.prio; + sync = queue->param.sched.sync; - UNLOCK(&queue->s.lock); + UNLOCK(&queue->lock); if (status != QUEUE_STATUS_READY) continue; @@ -1079,24 +1079,24 @@ static uint64_t queue_to_u64(odp_queue_t hdl) static odp_pktout_queue_t queue_get_pktout(odp_queue_t handle) { - return qentry_from_int(handle)->s.pktout; + return qentry_from_int(handle)->pktout; } static void queue_set_pktout(odp_queue_t handle, odp_pktio_t pktio, int index) { - qentry_from_int(handle)->s.pktout.pktio = pktio; - qentry_from_int(handle)->s.pktout.index = index; + qentry_from_int(handle)->pktout.pktio = pktio; + qentry_from_int(handle)->pktout.index = index; } static odp_pktin_queue_t queue_get_pktin(odp_queue_t handle) { - return qentry_from_int(handle)->s.pktin; + return qentry_from_int(handle)->pktin; } static void queue_set_pktin(odp_queue_t handle, odp_pktio_t pktio, int index) { - qentry_from_int(handle)->s.pktin.pktio = pktio; - qentry_from_int(handle)->s.pktin.index = index; + qentry_from_int(handle)->pktin.pktio = pktio; + qentry_from_int(handle)->pktin.index = index; } static void queue_set_enq_deq_func(odp_queue_t handle, @@ -1106,22 +1106,22 @@ static void queue_set_enq_deq_func(odp_queue_t handle, queue_deq_multi_fn_t deq_multi) { if (enq) - qentry_from_int(handle)->s.enqueue = enq; + qentry_from_int(handle)->enqueue = enq; if (enq_multi) - qentry_from_int(handle)->s.enqueue_multi = enq_multi; + qentry_from_int(handle)->enqueue_multi = enq_multi; if (deq) - qentry_from_int(handle)->s.dequeue = deq; + qentry_from_int(handle)->dequeue = deq; if (deq_multi) - qentry_from_int(handle)->s.dequeue_multi = deq_multi; + qentry_from_int(handle)->dequeue_multi = deq_multi; } static int queue_orig_multi(odp_queue_t handle, _odp_event_hdr_t **event_hdr, int num) { - return qentry_from_int(handle)->s.orig_dequeue_multi(handle, + return qentry_from_int(handle)->orig_dequeue_multi(handle, event_hdr, num); } @@ -1129,14 +1129,14 @@ static void queue_timer_add(odp_queue_t handle) { queue_entry_t *queue = _odp_qentry_from_ext(handle); - odp_atomic_inc_u64(&queue->s.num_timers); + odp_atomic_inc_u64(&queue->num_timers); } static void queue_timer_rem(odp_queue_t handle) { queue_entry_t *queue = _odp_qentry_from_ext(handle); - odp_atomic_dec_u64(&queue->s.num_timers); + odp_atomic_dec_u64(&queue->num_timers); } /* API functions */ diff --git a/platform/linux-generic/odp_queue_spsc.c b/platform/linux-generic/odp_queue_spsc.c index c1b02d457..b2fed7d0d 100644 --- a/platform/linux-generic/odp_queue_spsc.c +++ b/platform/linux-generic/odp_queue_spsc.c @@ -41,17 +41,17 @@ static inline int spsc_enq_multi(odp_queue_t handle, uint32_t buf_idx[num]; queue = qentry_from_handle(handle); - ring_spsc = &queue->s.ring_spsc; + ring_spsc = &queue->ring_spsc; event_index_from_hdr(buf_idx, event_hdr, num); - if (odp_unlikely(queue->s.status < QUEUE_STATUS_READY)) { + if (odp_unlikely(queue->status < QUEUE_STATUS_READY)) { ODP_ERR("Bad queue status\n"); return -1; } - return ring_spsc_enq_multi(ring_spsc, queue->s.ring_data, - queue->s.ring_mask, buf_idx, num); + return ring_spsc_enq_multi(ring_spsc, queue->ring_data, + queue->ring_mask, buf_idx, num); } static inline int spsc_deq_multi(odp_queue_t handle, @@ -63,15 +63,15 @@ static inline int spsc_deq_multi(odp_queue_t handle, uint32_t buf_idx[num]; queue = qentry_from_handle(handle); - ring_spsc = &queue->s.ring_spsc; + ring_spsc = &queue->ring_spsc; - if (odp_unlikely(queue->s.status < QUEUE_STATUS_READY)) { + if (odp_unlikely(queue->status < QUEUE_STATUS_READY)) { /* Bad queue, or queue has been destroyed. */ return -1; } - num_deq = ring_spsc_deq_multi(ring_spsc, queue->s.ring_data, - queue->s.ring_mask, buf_idx, num); + num_deq = ring_spsc_deq_multi(ring_spsc, queue->ring_data, + queue->ring_mask, buf_idx, num); if (num_deq == 0) return 0; @@ -122,15 +122,15 @@ void _odp_queue_spsc_init(queue_entry_t *queue, uint32_t queue_size) { uint64_t offset; - queue->s.enqueue = queue_spsc_enq; - queue->s.dequeue = queue_spsc_deq; - queue->s.enqueue_multi = queue_spsc_enq_multi; - queue->s.dequeue_multi = queue_spsc_deq_multi; - queue->s.orig_dequeue_multi = queue_spsc_deq_multi; + queue->enqueue = queue_spsc_enq; + queue->dequeue = queue_spsc_deq; + queue->enqueue_multi = queue_spsc_enq_multi; + queue->dequeue_multi = queue_spsc_deq_multi; + queue->orig_dequeue_multi = queue_spsc_deq_multi; - offset = queue->s.index * (uint64_t)_odp_queue_glb->config.max_queue_size; + offset = queue->index * (uint64_t)_odp_queue_glb->config.max_queue_size; - queue->s.ring_data = &_odp_queue_glb->ring_data[offset]; - queue->s.ring_mask = queue_size - 1; - ring_spsc_init(&queue->s.ring_spsc); + queue->ring_data = &_odp_queue_glb->ring_data[offset]; + queue->ring_mask = queue_size - 1; + ring_spsc_init(&queue->ring_spsc); } diff --git a/platform/linux-generic/odp_schedule_basic.c b/platform/linux-generic/odp_schedule_basic.c index 4b711ec93..81065a8d5 100644 --- a/platform/linux-generic/odp_schedule_basic.c +++ b/platform/linux-generic/odp_schedule_basic.c @@ -1132,7 +1132,7 @@ static int schedule_ord_enq_multi(odp_queue_t dst_queue, void *event_hdr[], dst_qentry = qentry_from_handle(dst_queue); - if (dst_qentry->s.param.order == ODP_QUEUE_ORDER_IGNORE) + if (dst_qentry->param.order == ODP_QUEUE_ORDER_IGNORE) return 0; src_queue = sched_local.ordered.src_queue; @@ -1146,7 +1146,7 @@ static int schedule_ord_enq_multi(odp_queue_t dst_queue, void *event_hdr[], } /* Pktout may drop packets, so the operation cannot be stashed. */ - if (dst_qentry->s.pktout.pktio != ODP_PKTIO_INVALID || + if (dst_qentry->pktout.pktio != ODP_PKTIO_INVALID || odp_unlikely(stash_num >= MAX_ORDERED_STASH)) { /* If the local stash is full, wait until it is our turn and * then release the stash and do enqueue directly. */ diff --git a/platform/linux-generic/odp_schedule_scalable.c b/platform/linux-generic/odp_schedule_scalable.c index 66cb66315..08f19c4e5 100644 --- a/platform/linux-generic/odp_schedule_scalable.c +++ b/platform/linux-generic/odp_schedule_scalable.c @@ -718,7 +718,7 @@ static void pktio_start(int pktio_idx, __atomic_fetch_add(&global->poll_count[pktio_idx], 1, __ATOMIC_RELAXED); qentry = _odp_qentry_from_ext(odpq[i]); - elem = &qentry->s.sched_elem; + elem = &qentry->sched_elem; elem->cons_type |= FLAG_PKTIN; /* Set pktin queue flag */ elem->pktio_idx = pktio_idx; elem->rx_queue = rxq; @@ -2102,7 +2102,7 @@ static int ord_enq_multi(odp_queue_t handle, void *event_hdr[], int num, ts = _odp_sched_ts; queue = qentry_from_int(handle); if (ts && odp_unlikely(ts->out_of_order) && - (queue->s.param.order == ODP_QUEUE_ORDER_KEEP)) { + (queue->param.order == ODP_QUEUE_ORDER_KEEP)) { actual = _odp_rctx_save(queue, (_odp_event_hdr_t **)event_hdr, num); *ret = actual; return 1; diff --git a/platform/linux-generic/odp_schedule_scalable_ordered.c b/platform/linux-generic/odp_schedule_scalable_ordered.c index 5f1f8a405..54181f10f 100644 --- a/platform/linux-generic/odp_schedule_scalable_ordered.c +++ b/platform/linux-generic/odp_schedule_scalable_ordered.c @@ -264,7 +264,7 @@ static void blocking_enqueue(queue_entry_t *q, _odp_event_hdr_t **evts, int num) /* Iterate until all events have been successfully enqueued */ for (;;) { /* Attempt to enqueue remaining events */ - actual = q->s.enqueue_multi(qentry_to_int(q), evts, num); + actual = q->enqueue_multi(qentry_to_int(q), evts, num); if (odp_unlikely(actual < 0)) ODP_ERR("Failed to enqueue deferred events\n"); /* Update for potential partial success */ |