From d6461dfb25798aeb91bea58ebcd6ca6b128a835a Mon Sep 17 00:00:00 2001 From: Tuomas Taipale Date: Tue, 20 Jun 2023 06:15:32 +0000 Subject: linux-dpdk: remove rte_mbuf from common event header Remove `struct rte_mbuf` from the common event header. Concrete pool element types now have an explicit `struct rte_mbuf` field followed by a new `_odp_event_hdr_int_t` type field which is the existing `_odp_event_hdr_t` with `struct rte_mbuf` field removed. `_odp_event_hdr_t` remains but only for casting purposes. The removal results in a saving of full cache line for each type. Signed-off-by: Tuomas Taipale Reviewed-by: Matias Elo --- .../linux-dpdk/include/event_vector_internal.h | 1 - platform/linux-dpdk/include/odp_buffer_internal.h | 9 +++-- platform/linux-dpdk/include/odp_event_internal.h | 19 ++++++---- .../linux-dpdk/include/odp_event_vector_internal.h | 17 +++++++-- platform/linux-dpdk/include/odp_packet_internal.h | 39 ++++++++++---------- platform/linux-dpdk/include/odp_timer_internal.h | 9 +++-- platform/linux-dpdk/odp_crypto.c | 11 +++--- platform/linux-dpdk/odp_event.c | 6 ++-- platform/linux-dpdk/odp_packet.c | 42 +++++++++++----------- platform/linux-dpdk/odp_pool.c | 20 +++++------ 10 files changed, 98 insertions(+), 75 deletions(-) delete mode 120000 platform/linux-dpdk/include/event_vector_internal.h (limited to 'platform/linux-dpdk') diff --git a/platform/linux-dpdk/include/event_vector_internal.h b/platform/linux-dpdk/include/event_vector_internal.h deleted file mode 120000 index 8487d0256..000000000 --- a/platform/linux-dpdk/include/event_vector_internal.h +++ /dev/null @@ -1 +0,0 @@ -../../linux-generic/include/odp_event_vector_internal.h \ No newline at end of file diff --git a/platform/linux-dpdk/include/odp_buffer_internal.h b/platform/linux-dpdk/include/odp_buffer_internal.h index b82f6af55..215f010cd 100644 --- a/platform/linux-dpdk/include/odp_buffer_internal.h +++ b/platform/linux-dpdk/include/odp_buffer_internal.h @@ -1,5 +1,5 @@ /* Copyright (c) 2013-2018, Linaro Limited - * Copyright (c) 2021-2022, Nokia + * Copyright (c) 2021-2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -49,8 +49,11 @@ extern "C" { /* Internal buffer header */ typedef struct ODP_ALIGNED_CACHE odp_buffer_hdr_t { - /* Common event header */ - _odp_event_hdr_t event_hdr; + /* Underlying DPDK rte_mbuf */ + struct rte_mbuf mb; + + /* Common internal header */ + _odp_event_hdr_int_t event_hdr; /* User area pointer */ void *uarea_addr; diff --git a/platform/linux-dpdk/include/odp_event_internal.h b/platform/linux-dpdk/include/odp_event_internal.h index 5ccf5a3ed..6a707606a 100644 --- a/platform/linux-dpdk/include/odp_event_internal.h +++ b/platform/linux-dpdk/include/odp_event_internal.h @@ -33,11 +33,7 @@ extern "C" { #undef vector #endif -/* Common header for all event types. */ -typedef struct _odp_event_hdr_t { - /* Underlying DPDK rte_mbuf */ - struct rte_mbuf mb; - +typedef struct _odp_event_hdr_int_t { /* Pool handle */ odp_pool_t pool; @@ -53,6 +49,17 @@ typedef struct _odp_event_hdr_t { /* Event flow id */ uint8_t flow_id; +} _odp_event_hdr_int_t; + +/* Common header for all event types. Helper for casting, actual pool element types should begin + * with explicit struct rte_mbuf and _odp_event_hdr_int_t fields. */ +typedef struct ODP_ALIGNED_CACHE _odp_event_hdr_t { + /* Underlying DPDK rte_mbuf */ + struct rte_mbuf mb; + + /* Common internal header */ + _odp_event_hdr_int_t hdr; + } _odp_event_hdr_t; static inline odp_event_t _odp_event_from_hdr(_odp_event_hdr_t *hdr) @@ -77,7 +84,7 @@ static inline struct rte_mbuf *_odp_event_to_mbuf(odp_event_t event) static inline void _odp_event_type_set(odp_event_t event, int ev) { - _odp_event_hdr(event)->event_type = ev; + _odp_event_hdr(event)->hdr.event_type = ev; } static inline uint64_t *_odp_event_endmark_get_ptr(odp_event_t event) diff --git a/platform/linux-dpdk/include/odp_event_vector_internal.h b/platform/linux-dpdk/include/odp_event_vector_internal.h index 5fa8c31c6..3bab5e462 100644 --- a/platform/linux-dpdk/include/odp_event_vector_internal.h +++ b/platform/linux-dpdk/include/odp_event_vector_internal.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2020-2022, Nokia +/* Copyright (c) 2020-2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -27,8 +27,11 @@ * Internal event vector header */ typedef struct ODP_ALIGNED_CACHE odp_event_vector_hdr_t { - /* Common event header */ - _odp_event_hdr_t event_hdr; + /* Underlying DPDK rte_mbuf */ + struct rte_mbuf mb; + + /* Common internal header */ + _odp_event_hdr_int_t event_hdr; /* User area pointer */ void *uarea_addr; @@ -52,6 +55,14 @@ static inline odp_event_vector_hdr_t *_odp_packet_vector_hdr(odp_packet_vector_t return (odp_event_vector_hdr_t *)(uintptr_t)pktv; } +/** + * Return the event header + */ +static inline _odp_event_hdr_t *_odp_packet_vector_to_event_hdr(odp_packet_vector_t pktv) +{ + return (_odp_event_hdr_t *)(uintptr_t)_odp_packet_vector_hdr(pktv); +} + /** * Free packet vector and contained packets */ diff --git a/platform/linux-dpdk/include/odp_packet_internal.h b/platform/linux-dpdk/include/odp_packet_internal.h index 141ee0af0..df4f2318e 100644 --- a/platform/linux-dpdk/include/odp_packet_internal.h +++ b/platform/linux-dpdk/include/odp_packet_internal.h @@ -1,5 +1,5 @@ /* Copyright (c) 2014-2018, Linaro Limited - * Copyright (c) 2021-2022, Nokia + * Copyright (c) 2021-2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -117,9 +117,12 @@ typedef struct { * packet_init(). Because of this any new fields added must be reviewed for * initialization requirements. */ -typedef struct odp_packet_hdr_t { - /* Common event header */ - _odp_event_hdr_t event_hdr; +typedef struct ODP_ALIGNED_CACHE odp_packet_hdr_t { + /* Underlying DPDK rte_mbuf */ + struct rte_mbuf mb; + + /* Common internal header */ + _odp_event_hdr_int_t event_hdr; packet_parser_t p; @@ -132,14 +135,14 @@ typedef struct odp_packet_hdr_t { * processing and as Tx completion event queue. */ odp_queue_t dst_queue; + /* --- 64-byte cache line boundary --- */ + /* User area pointer */ void *uarea_addr; /* User context pointer */ const void *user_ptr; - /* --- 64-byte cache line boundary --- */ - /* Classifier mark */ uint16_t cls_mark; @@ -180,7 +183,7 @@ typedef struct odp_packet_hdr_t { /* Temp storage for AAD */ #define PACKET_AAD_MAX 32 uint8_t crypto_aad_buf[PACKET_AAD_MAX]; -} odp_packet_hdr_t __rte_cache_aligned; +} odp_packet_hdr_t; /** * Return the packet header @@ -197,7 +200,7 @@ static inline odp_packet_t packet_handle(odp_packet_hdr_t *pkt_hdr) static inline _odp_event_hdr_t *packet_to_event_hdr(odp_packet_t pkt) { - return (_odp_event_hdr_t *)(uintptr_t)&packet_hdr(pkt)->event_hdr; + return (_odp_event_hdr_t *)(uintptr_t)packet_hdr(pkt); } static inline odp_packet_t packet_from_event_hdr(_odp_event_hdr_t *event_hdr) @@ -283,14 +286,14 @@ static inline void _odp_packet_copy_md(odp_packet_hdr_t *dst_hdr, dst_hdr->cls_mark = src_hdr->cls_mark; dst_hdr->user_ptr = src_hdr->user_ptr; - dst_hdr->event_hdr.mb.port = src_hdr->event_hdr.mb.port; - dst_hdr->event_hdr.mb.ol_flags = src_hdr->event_hdr.mb.ol_flags; - dst_hdr->event_hdr.mb.packet_type = src_hdr->event_hdr.mb.packet_type; - dst_hdr->event_hdr.mb.vlan_tci = src_hdr->event_hdr.mb.vlan_tci; - dst_hdr->event_hdr.mb.hash.rss = src_hdr->event_hdr.mb.hash.rss; - dst_hdr->event_hdr.mb.hash = src_hdr->event_hdr.mb.hash; - dst_hdr->event_hdr.mb.vlan_tci_outer = src_hdr->event_hdr.mb.vlan_tci_outer; - dst_hdr->event_hdr.mb.tx_offload = src_hdr->event_hdr.mb.tx_offload; + dst_hdr->mb.port = src_hdr->mb.port; + dst_hdr->mb.ol_flags = src_hdr->mb.ol_flags; + dst_hdr->mb.packet_type = src_hdr->mb.packet_type; + dst_hdr->mb.vlan_tci = src_hdr->mb.vlan_tci; + dst_hdr->mb.hash.rss = src_hdr->mb.hash.rss; + dst_hdr->mb.hash = src_hdr->mb.hash; + dst_hdr->mb.vlan_tci_outer = src_hdr->mb.vlan_tci_outer; + dst_hdr->mb.tx_offload = src_hdr->mb.tx_offload; if (src_hdr->p.input_flags.timestamp) dst_hdr->timestamp = src_hdr->timestamp; @@ -349,12 +352,12 @@ static inline void _odp_packet_copy_cls_md(odp_packet_hdr_t *dst_hdr, static inline uint32_t packet_len(odp_packet_hdr_t *pkt_hdr) { - return rte_pktmbuf_pkt_len(&pkt_hdr->event_hdr.mb); + return rte_pktmbuf_pkt_len(&pkt_hdr->mb); } static inline void packet_set_len(odp_packet_hdr_t *pkt_hdr, uint32_t len) { - rte_pktmbuf_pkt_len(&pkt_hdr->event_hdr.mb) = len; + rte_pktmbuf_pkt_len(&pkt_hdr->mb) = len; } /* Reset parser metadata for a new parse */ diff --git a/platform/linux-dpdk/include/odp_timer_internal.h b/platform/linux-dpdk/include/odp_timer_internal.h index f504a20aa..d77898e9a 100644 --- a/platform/linux-dpdk/include/odp_timer_internal.h +++ b/platform/linux-dpdk/include/odp_timer_internal.h @@ -1,5 +1,5 @@ /* Copyright (c) 2014-2018, Linaro Limited - * Copyright (c) 2021-2022, Nokia + * Copyright (c) 2021-2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -26,8 +26,11 @@ * Internal Timeout header */ typedef struct ODP_ALIGNED_CACHE odp_timeout_hdr_t { - /* Common event header */ - _odp_event_hdr_t event_hdr; + /* Underlying DPDK rte_mbuf */ + struct rte_mbuf mb; + + /* Common internal header */ + _odp_event_hdr_int_t event_hdr; /* Requested expiration time */ uint64_t expiration; diff --git a/platform/linux-dpdk/odp_crypto.c b/platform/linux-dpdk/odp_crypto.c index 9111f85c1..5761787f8 100644 --- a/platform/linux-dpdk/odp_crypto.c +++ b/platform/linux-dpdk/odp_crypto.c @@ -1662,10 +1662,8 @@ static uint8_t *crypto_prepare_digest(const crypto_session_entry_t *session, session->p.auth_digest_len); } data = pkt_hdr->crypto_digest_buf; - mb = &pkt_hdr->event_hdr.mb; - *phys_addr = - rte_pktmbuf_iova_offset(mb, data - - rte_pktmbuf_mtod(mb, uint8_t *)); + mb = &pkt_hdr->mb; + *phys_addr = rte_pktmbuf_iova_offset(mb, data - rte_pktmbuf_mtod(mb, uint8_t *)); return data; } @@ -1693,10 +1691,9 @@ static void crypto_fill_aead_param(const crypto_session_entry_t *session, session->p.auth_aad_len); op->sym->aead.aad.data = pkt_hdr->crypto_aad_buf; op->sym->aead.aad.phys_addr = - rte_pktmbuf_iova_offset(&pkt_hdr->event_hdr.mb, + rte_pktmbuf_iova_offset(&pkt_hdr->mb, op->sym->aead.aad.data - - rte_pktmbuf_mtod(&pkt_hdr->event_hdr.mb, - uint8_t *)); + rte_pktmbuf_mtod(&pkt_hdr->mb, uint8_t *)); iv_ptr = rte_crypto_op_ctod_offset(op, uint8_t *, IV_OFFSET); if (session->p.cipher_alg == ODP_CIPHER_ALG_AES_CCM) { *iv_ptr = iv_len; diff --git a/platform/linux-dpdk/odp_event.c b/platform/linux-dpdk/odp_event.c index 589defc5c..dff3e2ed2 100644 --- a/platform/linux-dpdk/odp_event.c +++ b/platform/linux-dpdk/odp_event.c @@ -34,10 +34,10 @@ /* Fill in event header field offsets for inline functions */ const _odp_event_inline_offset_t _odp_event_inline_offset ODP_ALIGNED_CACHE = { - .event_type = offsetof(_odp_event_hdr_t, event_type), + .event_type = offsetof(_odp_event_hdr_t, hdr.event_type), .base_data = offsetof(_odp_event_hdr_t, mb.buf_addr), - .flow_id = offsetof(_odp_event_hdr_t, flow_id), - .pool = offsetof(_odp_event_hdr_t, pool), + .flow_id = offsetof(_odp_event_hdr_t, hdr.flow_id), + .pool = offsetof(_odp_event_hdr_t, hdr.pool), .buf_len = offsetof(_odp_event_hdr_t, mb.buf_len) }; diff --git a/platform/linux-dpdk/odp_packet.c b/platform/linux-dpdk/odp_packet.c index 9b8755ce9..06c6c36ad 100644 --- a/platform/linux-dpdk/odp_packet.c +++ b/platform/linux-dpdk/odp_packet.c @@ -54,7 +54,7 @@ /* Fill in packet header field offsets for inline functions */ const _odp_packet_inline_offset_t _odp_packet_inline ODP_ALIGNED_CACHE = { - .mb = offsetof(odp_packet_hdr_t, event_hdr.mb), + .mb = offsetof(odp_packet_hdr_t, mb), .pool = offsetof(odp_packet_hdr_t, event_hdr.pool), .input = offsetof(odp_packet_hdr_t, input), .user_ptr = offsetof(odp_packet_hdr_t, user_ptr), @@ -68,14 +68,14 @@ const _odp_packet_inline_offset_t _odp_packet_inline ODP_ALIGNED_CACHE = { .cls_mark = offsetof(odp_packet_hdr_t, cls_mark), .ipsec_ctx = offsetof(odp_packet_hdr_t, ipsec_ctx), .crypto_op = offsetof(odp_packet_hdr_t, crypto_op_result), - .buf_addr = offsetof(odp_packet_hdr_t, event_hdr.mb.buf_addr), - .data = offsetof(odp_packet_hdr_t, event_hdr.mb.data_off), - .pkt_len = offsetof(odp_packet_hdr_t, event_hdr.mb.pkt_len), - .seg_len = offsetof(odp_packet_hdr_t, event_hdr.mb.data_len), - .nb_segs = offsetof(odp_packet_hdr_t, event_hdr.mb.nb_segs), + .buf_addr = offsetof(odp_packet_hdr_t, mb.buf_addr), + .data = offsetof(odp_packet_hdr_t, mb.data_off), + .pkt_len = offsetof(odp_packet_hdr_t, mb.pkt_len), + .seg_len = offsetof(odp_packet_hdr_t, mb.data_len), + .nb_segs = offsetof(odp_packet_hdr_t, mb.nb_segs), .user_area = offsetof(odp_packet_hdr_t, uarea_addr), - .rss = offsetof(odp_packet_hdr_t, event_hdr.mb.hash.rss), - .ol_flags = offsetof(odp_packet_hdr_t, event_hdr.mb.ol_flags), + .rss = offsetof(odp_packet_hdr_t, mb.hash.rss), + .ol_flags = offsetof(odp_packet_hdr_t, mb.ol_flags), .rss_flag = RTE_MBUF_F_RX_RSS_HASH }; @@ -132,7 +132,7 @@ static inline int num_segments(uint32_t len, uint32_t seg_len) static inline int packet_reset(odp_packet_t pkt, uint32_t len) { odp_packet_hdr_t *const pkt_hdr = packet_hdr(pkt); - struct rte_mbuf *ms, *mb = &pkt_hdr->event_hdr.mb; + struct rte_mbuf *ms, *mb = &pkt_hdr->mb; uint8_t nb_segs = 0; int32_t lenleft = len; @@ -354,7 +354,7 @@ int odp_event_filter_packet(const odp_event_t event[], void *odp_packet_tail(odp_packet_t pkt) { - struct rte_mbuf *mb = &(packet_hdr(pkt)->event_hdr.mb); + struct rte_mbuf *mb = &(packet_hdr(pkt)->mb); mb = rte_pktmbuf_lastseg(mb); return (void *)(rte_pktmbuf_mtod(mb, char *) + mb->data_len); @@ -362,7 +362,7 @@ void *odp_packet_tail(odp_packet_t pkt) void *odp_packet_push_head(odp_packet_t pkt, uint32_t len) { - struct rte_mbuf *mb = &(packet_hdr(pkt)->event_hdr.mb); + struct rte_mbuf *mb = &(packet_hdr(pkt)->mb); return (void *)rte_pktmbuf_prepend(mb, len); } @@ -378,7 +378,7 @@ static void _copy_head_metadata(struct rte_mbuf *newhead, int odp_packet_extend_head(odp_packet_t *pkt, uint32_t len, void **data_ptr, uint32_t *seg_len) { - struct rte_mbuf *mb = &(packet_hdr(*pkt)->event_hdr.mb); + struct rte_mbuf *mb = &(packet_hdr(*pkt)->mb); int addheadsize = len - rte_pktmbuf_headroom(mb); if (addheadsize > 0) { @@ -482,7 +482,7 @@ int odp_packet_trunc_head(odp_packet_t *pkt, uint32_t len, void **data_ptr, void *odp_packet_push_tail(odp_packet_t pkt, uint32_t len) { - struct rte_mbuf *mb = &(packet_hdr(pkt)->event_hdr.mb); + struct rte_mbuf *mb = &(packet_hdr(pkt)->mb); return (void *)rte_pktmbuf_append(mb, len); } @@ -490,7 +490,7 @@ void *odp_packet_push_tail(odp_packet_t pkt, uint32_t len) int odp_packet_extend_tail(odp_packet_t *pkt, uint32_t len, void **data_ptr, uint32_t *seg_len) { - struct rte_mbuf *mb = &(packet_hdr(*pkt)->event_hdr.mb); + struct rte_mbuf *mb = &(packet_hdr(*pkt)->mb); int newtailsize = len - odp_packet_tailroom(*pkt); uint32_t old_pkt_len = odp_packet_len(*pkt); @@ -1041,7 +1041,7 @@ void odp_packet_print(odp_packet_t pkt) len += _odp_snprint(&str[len], n - len, " l4_offset %" PRIu32 "\n", hdr->p.l4_offset); len += _odp_snprint(&str[len], n - len, - " frame_len %" PRIu32 "\n", hdr->event_hdr.mb.pkt_len); + " frame_len %" PRIu32 "\n", hdr->mb.pkt_len); len += _odp_snprint(&str[len], n - len, " input %" PRIu64 "\n", odp_pktio_to_u64(hdr->input)); len += _odp_snprint(&str[len], n - len, @@ -1089,7 +1089,7 @@ void odp_packet_print_data(odp_packet_t pkt, uint32_t offset, len += _odp_snprint(&str[len], n - len, " buf index %" PRIu32 "\n", hdr->event_hdr.index); len += _odp_snprint(&str[len], n - len, - " segcount %" PRIu8 "\n", hdr->event_hdr.mb.nb_segs); + " segcount %" PRIu8 "\n", hdr->mb.nb_segs); len += _odp_snprint(&str[len], n - len, " data len %" PRIu32 "\n", data_len); len += _odp_snprint(&str[len], n - len, @@ -1528,7 +1528,7 @@ int odp_packet_parse(odp_packet_t pkt, uint32_t offset, * packet data range. Copy enough data to a temporary buffer for * parsing if necessary. */ - if (odp_unlikely(pkt_hdr->event_hdr.mb.nb_segs > 1) && + if (odp_unlikely(pkt_hdr->mb.nb_segs > 1) && odp_unlikely(seg_len < min_seglen)) { seg_len = min_seglen; if (seg_len > packet_len - offset) @@ -1892,15 +1892,15 @@ odp_packet_t odp_packet_reassemble(odp_pool_t pool_hdl, if (i < num - 1) next_seg = (odp_packet_hdr_t *)(uintptr_t)pkt_buf[i + 1]; - data_len += cur_seg->event_hdr.mb.data_len; + data_len += cur_seg->mb.data_len; mb = (struct rte_mbuf *)(uintptr_t)cur_seg; mb->next = (struct rte_mbuf *)next_seg; cur_seg = next_seg; } - pkt_hdr->event_hdr.mb.nb_segs = num; - pkt_hdr->event_hdr.mb.pkt_len = data_len; - pkt_hdr->event_hdr.mb.data_off = headroom; + pkt_hdr->mb.nb_segs = num; + pkt_hdr->mb.pkt_len = data_len; + pkt_hdr->mb.data_off = headroom; /* Reset metadata */ pkt_hdr->subtype = ODP_EVENT_PACKET_BASIC; diff --git a/platform/linux-dpdk/odp_pool.c b/platform/linux-dpdk/odp_pool.c index a4a3f398c..862019333 100644 --- a/platform/linux-dpdk/odp_pool.c +++ b/platform/linux-dpdk/odp_pool.c @@ -235,10 +235,10 @@ int _odp_event_is_valid(odp_event_t event) if (pool == NULL) return 0; - if (pool != _odp_pool_entry(event_hdr->pool)) + if (pool != _odp_pool_entry(event_hdr->hdr.pool)) return 0; - if (event_hdr->index >= pool->rte_mempool->size) + if (event_hdr->hdr.index >= pool->rte_mempool->size) return 0; return 1; @@ -612,10 +612,10 @@ static void init_obj_priv_data(struct rte_mempool *mp ODP_UNUSED, void *arg, voi /* No need for headroom in non-packet objects */ mb->data_off = 0; - event_hdr->index = i; - event_hdr->pool = _odp_pool_handle(priv_data->pool); - event_hdr->type = priv_data->type; - event_hdr->event_type = priv_data->event_type; + event_hdr->hdr.index = i; + event_hdr->hdr.pool = _odp_pool_handle(priv_data->pool); + event_hdr->hdr.type = priv_data->type; + event_hdr->hdr.event_type = priv_data->event_type; switch (priv_data->type) { case ODP_POOL_BUFFER: @@ -1286,10 +1286,10 @@ static void init_ext_obj(struct rte_mempool *mp, void *arg, void *mbuf, unsigned rte_mbuf_refcnt_set(mb, 1); mb->next = NULL; /* Save index, might be useful for debugging purposes */ - event_hdr->index = i; - event_hdr->pool = _odp_pool_handle(mb_ctor_arg->pool); - event_hdr->type = mb_ctor_arg->type; - event_hdr->event_type = mb_ctor_arg->event_type; + event_hdr->hdr.index = i; + event_hdr->hdr.pool = _odp_pool_handle(mb_ctor_arg->pool); + event_hdr->hdr.type = mb_ctor_arg->type; + event_hdr->hdr.event_type = mb_ctor_arg->event_type; switch (mb_ctor_arg->type) { case ODP_POOL_BUFFER: -- cgit v1.2.3