diff options
Diffstat (limited to 'platform/linux-generic/include')
6 files changed, 193 insertions, 8 deletions
diff --git a/platform/linux-generic/include/odp/api/plat/event_inlines.h b/platform/linux-generic/include/odp/api/plat/event_inlines.h index 37c015b21..4e3368ff0 100644 --- a/platform/linux-generic/include/odp/api/plat/event_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/event_inlines.h @@ -1,5 +1,5 @@ /* Copyright (c) 2018, Linaro Limited - * Copyright (c) 2022, Nokia + * Copyright (c) 2022-2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -8,11 +8,17 @@ #ifndef ODP_PLAT_EVENT_INLINES_H_ #define ODP_PLAT_EVENT_INLINES_H_ +#include <odp/api/buffer_types.h> +#include <odp/api/dma.h> #include <odp/api/event_types.h> #include <odp/api/packet_types.h> +#include <odp/api/timer_types.h> +#include <odp/api/plat/buffer_inline_types.h> #include <odp/api/plat/event_inline_types.h> +#include <odp/api/plat/event_vector_inline_types.h> #include <odp/api/plat/packet_inline_types.h> +#include <odp/api/plat/timer_inline_types.h> /** @cond _ODP_HIDE_FROM_DOXYGEN_ */ @@ -21,6 +27,7 @@ #define _ODP_INLINE static inline #define odp_event_type __odp_event_type #define odp_event_type_multi __odp_event_type_multi + #define odp_event_user_area __odp_event_user_area #define odp_event_subtype __odp_event_subtype #define odp_event_types __odp_event_types #define odp_event_flow_id __odp_event_flow_id @@ -59,6 +66,26 @@ _ODP_INLINE int odp_event_type_multi(const odp_event_t event[], int num, return i; } +_ODP_INLINE void *odp_event_user_area(odp_event_t event) +{ + const odp_event_type_t type = __odp_event_type_get(event); + + switch (type) { + case ODP_EVENT_BUFFER: + return _odp_buffer_get((odp_buffer_t)event, void *, uarea_addr); + case ODP_EVENT_PACKET: + return _odp_pkt_get((odp_packet_t)event, void *, user_area); + case ODP_EVENT_PACKET_VECTOR: + return _odp_event_vect_get((odp_packet_vector_t)event, void *, uarea_addr); + case ODP_EVENT_TIMEOUT: + return _odp_timeout_hdr_field((odp_timeout_t)event, void *, uarea_addr); + case ODP_EVENT_DMA_COMPL: + return odp_dma_compl_user_area((odp_dma_compl_t)event); + default: + return NULL; + } +} + _ODP_INLINE odp_event_subtype_t odp_event_subtype(odp_event_t event) { if (__odp_event_type_get(event) != ODP_EVENT_PACKET) diff --git a/platform/linux-generic/include/odp/api/plat/packet_inline_types.h b/platform/linux-generic/include/odp/api/plat/packet_inline_types.h index 6773b73ad..5186efed0 100644 --- a/platform/linux-generic/include/odp/api/plat/packet_inline_types.h +++ b/platform/linux-generic/include/odp/api/plat/packet_inline_types.h @@ -123,7 +123,7 @@ typedef union { uint32_t all_flags; struct { - uint32_t reserved1: 6; + uint32_t reserved1: 5; /* * Init flags @@ -142,6 +142,7 @@ typedef union { uint32_t l4_chksum: 1; /* L4 chksum override */ uint32_t ts_set: 1; /* Set Tx timestamp */ uint32_t tx_compl: 1; /* Tx completion event requested */ + uint32_t free_ctrl: 1; /* Don't free option */ uint32_t tx_aging: 1; /* Packet aging at Tx requested */ uint32_t shaper_len_adj: 8; /* Adjustment for traffic mgr */ @@ -159,8 +160,8 @@ typedef union { /* Flag groups */ struct { - uint32_t reserved2: 6; - uint32_t other: 19; /* All other flags */ + uint32_t reserved2: 5; + uint32_t other: 20; /* All other flags */ uint32_t error: 7; /* All error flags */ } all; diff --git a/platform/linux-generic/include/odp/api/plat/packet_inlines.h b/platform/linux-generic/include/odp/api/plat/packet_inlines.h index f03c88e10..960dbc5fc 100644 --- a/platform/linux-generic/include/odp/api/plat/packet_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/packet_inlines.h @@ -18,7 +18,7 @@ #include <odp/api/hints.h> #include <odp/api/packet_types.h> #include <odp/api/pool_types.h> -#include <odp/api/time.h> +#include <odp/api/time_types.h> #include <odp/api/plat/debug_inlines.h> #include <odp/api/plat/packet_io_inlines.h> diff --git a/platform/linux-generic/include/odp/api/plat/time_inlines.h b/platform/linux-generic/include/odp/api/plat/time_inlines.h index 2ffb94c66..f8f4bee89 100644 --- a/platform/linux-generic/include/odp/api/plat/time_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/time_inlines.h @@ -8,12 +8,14 @@ #ifndef ODP_PLAT_TIME_INLINES_H_ #define ODP_PLAT_TIME_INLINES_H_ -#include <stdint.h> #include <odp/api/align.h> #include <odp/api/hints.h> +#include <odp/api/time_types.h> #include <odp/api/abi/cpu_time.h> +#include <stdint.h> + /** @cond _ODP_HIDE_FROM_DOXYGEN_ */ #define _ODP_TIMESPEC_SIZE 16 diff --git a/platform/linux-generic/include/odp_classification_internal.h b/platform/linux-generic/include/odp_classification_internal.h index c3ecf4079..7841e64fb 100644 --- a/platform/linux-generic/include/odp_classification_internal.h +++ b/platform/linux-generic/include/odp_classification_internal.h @@ -1,5 +1,5 @@ /* Copyright (c) 2014-2018, Linaro Limited - * Copyright (c) 2021, Nokia + * Copyright (c) 2021-2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -21,13 +21,20 @@ extern "C" { #include <odp/api/atomic.h> #include <odp/api/classification.h> +#include <odp/api/event.h> #include <odp/api/hints.h> +#include <odp/api/packet.h> +#include <odp/api/pool.h> #include <odp/api/queue.h> +#include <odp/api/std_types.h> + +#include <odp_debug_internal.h> #include <odp_packet_internal.h> -#include <odp/api/packet_io.h> #include <odp_packet_io_internal.h> #include <odp_classification_datamodel.h> +#include <stdint.h> + extern cls_global_t *_odp_cls_global; static inline cos_t *_odp_cos_entry_from_idx(uint32_t ndx) @@ -72,6 +79,153 @@ static inline void _odp_cos_queue_stats_add(cos_t *cos, odp_queue_t queue, odp_atomic_add_u64(&cos->queue_stats[queue_idx].discards, discards); } +static inline void _odp_cos_vector_enq(odp_queue_t queue, odp_event_t events[], uint32_t num, + cos_t *cos) +{ + odp_packet_vector_t pktv; + const odp_pool_t pool = cos->vector.pool; + const uint32_t max_size = cos->vector.max_size; + uint32_t num_enq; + int num_pktv = (num + max_size - 1) / max_size; + int ret; + int i; + odp_packet_vector_t pktv_tbl[num_pktv]; + odp_event_t event_tbl[num_pktv]; + + for (i = 0; i < num_pktv; i++) { + pktv = odp_packet_vector_alloc(pool); + if (odp_unlikely(pktv == ODP_PACKET_VECTOR_INVALID)) + break; + pktv_tbl[i] = pktv; + event_tbl[i] = odp_packet_vector_to_event(pktv); + } + if (odp_unlikely(i == 0)) { + odp_event_free_multi(events, num); + _odp_cos_queue_stats_add(cos, queue, 0, num); + return; + } + num_pktv = i; + num_enq = 0; + for (i = 0; i < num_pktv; i++) { + odp_packet_t *pkt_tbl; + int pktv_size = max_size; + + pktv = pktv_tbl[i]; + + if (num_enq + max_size > num) + pktv_size = num - num_enq; + + odp_packet_vector_tbl(pktv, &pkt_tbl); + odp_packet_from_event_multi(pkt_tbl, &events[num_enq], pktv_size); + odp_packet_vector_size_set(pktv, pktv_size); + num_enq += pktv_size; + } + + ret = odp_queue_enq_multi(queue, event_tbl, num_pktv); + if (odp_likely(ret == num_pktv)) { + _odp_cos_queue_stats_add(cos, queue, num_enq, num - num_enq); + } else { + uint32_t enqueued; + + if (ret < 0) + ret = 0; + enqueued = max_size * ret; + _odp_cos_queue_stats_add(cos, queue, enqueued, num - enqueued); + odp_event_free_multi(&event_tbl[ret], num_pktv - ret); + } +} + +/** + * Enqueue packets into destination CoS + */ +static inline void _odp_cos_enq(uint16_t cos_id, odp_queue_t dst, odp_packet_t packets[], int num) +{ + _ODP_ASSERT(cos_id != CLS_COS_IDX_NONE); + _ODP_ASSERT(dst != ODP_QUEUE_INVALID); + + cos_t *cos = _odp_cos_entry_from_idx(cos_id); + + if (num < 2 || !cos->vector.enable) { + int ret = odp_queue_enq_multi(dst, (odp_event_t *)packets, num); + + if (odp_unlikely(ret != num)) { + if (ret < 0) + ret = 0; + + odp_packet_free_multi(&packets[ret], num - ret); + } + _odp_cos_queue_stats_add(cos, dst, ret, num - ret); + } else { + _odp_cos_vector_enq(dst, (odp_event_t *)packets, num, cos); + } +} + +/** + * Enqueue all remaining packets in 'packets' array + */ +static inline void _odp_cls_enq_all(odp_packet_t packets[], int num) +{ + odp_packet_hdr_t *prev_hdr; + odp_packet_hdr_t *latest_hdr = packet_hdr(packets[num - 1]); + + if (num < 2) { + _odp_cos_enq(latest_hdr->cos, latest_hdr->dst_queue, packets, 1); + return; + } + + prev_hdr = packet_hdr(packets[num - 2]); + + if (prev_hdr->dst_queue == latest_hdr->dst_queue && prev_hdr->cos == latest_hdr->cos) { + _odp_cos_enq(prev_hdr->cos, prev_hdr->dst_queue, packets, num); + } else { + _odp_cos_enq(prev_hdr->cos, prev_hdr->dst_queue, packets, num - 1); + _odp_cos_enq(latest_hdr->cos, latest_hdr->dst_queue, &packets[num - 1], 1); + } +} + +/** + * Enqueue packets into classifier destination queue + * + * Called by pktio devices for each received packet when classifier has been enabled. Postpones the + * actual enqueue operation and stores packets in 'packets' array until destination queue or CoS + * change, or 'last' flag is set. + * + * @param[out] packets Packet array to be enqueued + * @param num Number of handles in 'packets' array + * @param last Enqueue all packets + * + * @return Number of packets remaining in 'packets' array + */ +static inline int _odp_cls_enq(odp_packet_t packets[], int num, odp_bool_t last) +{ + odp_packet_hdr_t *prev_hdr, *latest_hdr; + + _ODP_ASSERT(num > 0); + + if (last) { + _odp_cls_enq_all(packets, num); + return 0; + } + + /* Only one packet, so postpone enqueue */ + if (num < 2) + return num; + + prev_hdr = packet_hdr(packets[num - 2]); + latest_hdr = packet_hdr(packets[num - 1]); + + /* Postpone enqueue if destination queue and CoS have not changed */ + if (prev_hdr->dst_queue == latest_hdr->dst_queue && prev_hdr->cos == latest_hdr->cos) + return num; + + /* Perform previously postponed enqueue operation and move the last packet (different + * destination) in 'packets' array to be the first entry */ + _odp_cos_enq(prev_hdr->cos, prev_hdr->dst_queue, packets, num - 1); + packets[0] = packets[num - 1]; + + return 1; +} + /** Classification Internal function **/ /** diff --git a/platform/linux-generic/include/odp_packet_io_internal.h b/platform/linux-generic/include/odp_packet_io_internal.h index 9caf3c2e9..05dda9897 100644 --- a/platform/linux-generic/include/odp_packet_io_internal.h +++ b/platform/linux-generic/include/odp_packet_io_internal.h @@ -22,6 +22,7 @@ extern "C" { #include <odp/api/packet_io.h> #include <odp/api/spinlock.h> #include <odp/api/ticketlock.h> +#include <odp/api/time.h> #include <odp/api/plat/packet_io_inlines.h> |